Commit f72d6207 authored by Frederik Wegner's avatar Frederik Wegner
Browse files

mongo-migrate.py is now versioned in deployment repo.

parent 48372a9a
from pymongo import MongoClient
import pymysql.cursors
import json
import time
import argparse
import random
import string
class FakeConnection:
def __init__(self, path):
self.f = open(path, 'w')
self.c = FileWriteCursor(self.f)
def cursor(self):
return self.c
def commit(self):
pass
def close(self):
self.f.close()
class FileWriteCursor:
def __init__(self, f):
self.f = f
def __enter__(self):
self.execute("SET AUTOCOMMIT=0")
return self
def __exit__(self, ab, b, c):
self.execute("COMMIT")
pass
def execute(self, sql, params=()):
self.f.write(sql % params)
if not sql[-1] == ";":
self.f.write(";")
self.f.write("\n")
def filterRepresentsInteger(s):
try:
int(s)
return s
except ValueError:
return 0
def dictGetStringSafe(d, k):
if k in d.keys():
return d[k].replace("'", "\\'")
else:
return ""
def dictGetBoolStringSafe(d, k):
if k in d.keys():
return d[k]
else:
return "false"
def cleanTables(cursor):
cursor.execute("SET FOREIGN_KEY_CHECKS=0", ())
tables = [
'round',
'title',
'avatar',
'subject',
'pool',
'maintainer',
'user',
'question',
'duel_player',
'duel',
'roundquestion',
'optionanswer',
'course',
'university',
'flashcard',
'friend',
'subject_course',
'traininglist',
'trainingsubscription',
'cardsubscription',
'useranswer',
'optionuseranswer',
'textuseranswer',
'pool_question']
for t in tables:
cursor.execute("TRUNCATE TABLE `%s`", (t))
cursor.execute("SET FOREIGN_KEY_CHECKS=1", ())
def insertUniversityDefault(cursor):
sql = """INSERT INTO `university` (`iduniversity`, `name`, `shortform`, `city`, `country`)
VALUES (1, "TU Darmstadt", "TUDa", "Darmstadt", "DE")
"""
cursor.execute(sql, ())
def insertSubject(cursor, s):
sql = """INSERT INTO `subject` (`name`, `shortform`, `department`, `idmongo`)
VALUES ('%s','%s','%s','%s')"""
cursor.execute(sql, (
s['subjectName'].replace("'", "\\'"),
s['subjectCode'],
s['subjectDepartment'],
str(s['_id'])
))
def insertTitle(cursor, title):
sql = """INSERT INTO `title` (
`name`,
`subject`,
`unlock_score`,
`unlock_win`,
`idmongo`)
SELECT '%s',`idsubject`,%s,%s,'%s' FROM `subject`
WHERE `shortform`='%s'"""
cursor.execute(sql, (
title['titleName'].replace("'", "\\'"),
title['unlockScore'],
title['unlockWin'],
str(title['_id']),
#where
title['subjectCode']
))
def insertUser(cursor, user):
newpw = ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))
newpw = bytes(newpw, 'ascii')
newpwhash = bcrypt.hashpw(newpw, bcrypt.gensalt(rounds=10, prefix=b"2a"))
sql = """INSERT IGNORE INTO `user` (
`username`,
`password`,
`email`,
`semester`,
`experience`,
`verified`,
`university`,
`selected_title`,
`idmongo`)
SELECT '%s','%s','%s',%s,%s,%s,%s,`idtitle`,'%s' FROM `title`
WHERE `name`='%s';"""
sql += " -- PW:" + newpw
print(sql)
cursor.execute(sql, (
user['userName'].replace("'", "\\'"),
newpwhash,
user['userEmail'],
filterRepresentsInteger(user['semester']),
user['score'],
user['isVerified'],
user['university'],
str(user['_id']),
#WHERE
user['currentTitle']
))
def accessTypeToLevel(accT):
if accT == "sa":
return 0
elif accT == "a":
return 1
else:
return 2
def insertMaintainer(cursor, m):
sql = """INSERT INTO `maintainer` (
`login`,
`password`,
`level`,
`name`,
`university`,
`email`,
`idmongo`)
SELECT '%s','%s','%s','%s',1,'%s', '%s'"""
cursor.execute(sql, (
m['userName'].replace("'", "\\'")[:45],
m['userPassword'],
accessTypeToLevel(m['accessType']),
m['userFullName'].replace("'", "\\'")[:45],
m['email'][:45],
str(m['_id'])
))
def insertPool(cursor, p):
# pool names are restricted to 45 characters
name = p['parentSubjectName'].replace("'", "\\'")
nameCut = name[:45]
nameRest = name[45:]
description = "Description missing!"
if len(nameRest) > 0:
description = "pool names can only be 45 characters long. Original name " + name + "was cut during migration"
sql = """INSERT INTO `pool` (
`name`,
`shortform`,
`description`,
`image`,
`idmongo`
) VALUES ('%s','%s','%s',1,'%s')"""
cursor.execute(sql, (
nameCut,
p['parentSubjectCode'],
description,
str(p['_id'])
))
def insertQuestion(cursor, q):
sql = """INSERT INTO `question` (
`text`,
`idmongo`,
`verified`,
`reviewed`,
`published`,
`answertype`,
`subject`
) SELECT '%s','%s',%s,%s, %s, '%s',`idsubject` FROM `subject`
WHERE `shortform`='%s'"""
cursor.execute(sql, (
q['questionText'].replace("'", "\\'"),
str(q['_id']),
dictGetBoolStringSafe(q,'verified'),
dictGetBoolStringSafe(q,'reviewed'),
dictGetBoolStringSafe(q,'published'),
"option",
#WHERE
q['subjectCode']
))
insertOptionAnswer(cursor, q)
insertExplanation(cursor, q)
def insertExplanation(cursor, q):
questionMongoID = str(q['_id'])
sql = """INSERT INTO `explanation` (
`text`,
`question`
) SELECT '%s',`idquestion` FROM `question`
WHERE `idmongo`='%s'"""
cursor.execute(sql, (
dictGetStringSafe(q,'explanation'),
questionMongoID
))
def insertOptionAnswer(cursor, q):
questionMongoID = str(q['_id'])
for option in q['answers']:
sql = """INSERT INTO `optionanswer` (
`text`,
`correct`,
`question`
) SELECT '%s', %s, `idquestion` FROM `question`
WHERE `idmongo`='%s'"""
cursor.execute(sql, (
option['answerText'].replace("'", "\\'"),
option['isCorrect'],
questionMongoID
))
def connect():
return pymysql.connect(host='localhost',
user='akamuuser',
password='alalalal',
db='akamu_stage',
charset='utf8',
cursorclass=pymysql.cursors.DictCursor)
parser = argparse.ArgumentParser(description='Migrate akamudb from mongodb to mysql.')
parser.add_argument("-d", "--database", type=str, default="akamudb", dest="database",
help="name fo the database")
parser.add_argument("-a", "--address", type=str, default="ds163060.mlab.com:63060", dest="address",
help="address the database is hosted at")
parser.add_argument("-u", "--user", type=str, default="akamu", dest="username",
help="username for database authentication")
parser.add_argument("-p", "--password", type=str, required=False, dest="password",
help="password for database authentication. If not specified commandline prompt will ask to enter it.")
args = parser.parse_args()
if not args.password:
args.password = input("Enter mongodb password: ")
# sqlConnection = connect()
sqlConnection = FakeConnection("migratedump.sql")
uri = 'mongodb://{username}:{password}@{address}/{database}'.format(
username=args.username,
password=args.password,
address=args.address,
database=args.database)
client = MongoClient(uri)
db = client['akamudb']
try:
with sqlConnection.cursor() as cursor:
cleanTables(cursor)
insertUniversityDefault(cursor)
for maintainer in db['webUser'].find():
insertMaintainer(cursor, maintainer)
for subject in db['subject'].find():
insertSubject(cursor, subject)
for title in db['title'].find():
insertTitle(cursor, title)
# for user in db['appUser'].find():
# insertUser(cursor, user)
for pool in db['parentSubject'].find():
insertPool(cursor, pool)
for question in db['question'].find():
insertQuestion(cursor, question)
sqlConnection.commit()
finally:
sqlConnection.close()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment