klmp200/Sith
1
0
Fork 0
mirror of https://github.com/ae-utbm/sith.git synced 2024-11-26 02:54:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update SAS migration script

Browse Source
This commit is contained in:
Skia 2016-11-28 12:08:10 +01:00
parent 47cb5e60ce
commit 7969e9ba5f
1 changed files with 70 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
migrate.py
View File

@ -1029,8 +1029,9 @@ def migrate_sas():
album_link = {} album_link = {}
picture_link = {} picture_link = {}
FILE_ROOT = "/data/sas/" FILE_ROOT = "/data/sas/"
SithFile.objects.filter(id__gt=18886).delete() SithFile.objects.filter(id__gte=18892).delete()
print("Album/Pictures deleted") print("Album/Pictures deleted")
reset_index('core', 'sas')
cur = db.cursor(MySQLdb.cursors.SSDictCursor) cur = db.cursor(MySQLdb.cursors.SSDictCursor)
cur.execute(""" cur.execute("""
SELECT * SELECT *
@ -1058,10 +1059,20 @@ def migrate_sas():
except: pass except: pass
print("Album migrated at %s" % datetime.datetime.now()) print("Album migrated at %s" % datetime.datetime.now())
print("Running time: %s" % (datetime.datetime.now()-start)) print("Running time: %s" % (datetime.datetime.now()-start))
with open("albums.link", "w") as f:
f.write(str(album_link))
cur.close()
finished = False
chunk = 0
while not finished:
cur = db.cursor(MySQLdb.cursors.SSDictCursor)
cur.execute(""" cur.execute("""
SELECT * SELECT *
FROM sas_photos FROM sas_photos
""") ORDER BY 'id_photo'
LIMIT %s, 1000
""", (chunk*1000, ))
has_result = False
for r in cur: for r in cur:
try: try:
user = User.objects.filter(id=r['id_utilisateur']).first() or root user = User.objects.filter(id=r['id_utilisateur']).first() or root
@ -1075,10 +1086,10 @@ def migrate_sas():
file_name += "/" + str(r['id_photo']) + ".jpg" file_name += "/" + str(r['id_photo']) + ".jpg"
file = File(open(file_name, "rb")) file = File(open(file_name, "rb"))
file.name = str(r['id_photo']) + ".jpg" file.name = (str(r['id_photo']) + ".jpg").encode('utf-8')
p = Picture( p = Picture(
name=to_unicode(str(r['id_photo'])) + ".jpg", name=(to_unicode(str(r['id_photo'])) + ".jpg").encode('utf-8'),
owner=user, owner=user,
is_moderated=True, is_moderated=True,
is_folder=False, is_folder=False,
@ -1094,6 +1105,7 @@ def migrate_sas():
if f.name == "date": if f.name == "date":
f.auto_now = False f.auto_now = False
p.generate_thumbnails() p.generate_thumbnails()
p.save()
db2 = MySQLdb.connect(**settings.OLD_MYSQL_INFOS) db2 = MySQLdb.connect(**settings.OLD_MYSQL_INFOS)
cur2 = db2.cursor(MySQLdb.cursors.SSDictCursor) cur2 = db2.cursor(MySQLdb.cursors.SSDictCursor)
cur2.execute(""" cur2.execute("""
@ -1108,11 +1120,14 @@ def migrate_sas():
PeoplePictureRelation(user=u, picture=p).save() PeoplePictureRelation(user=u, picture=p).save()
except: except:
print("Fail to associate user %d to picture %d" % (r2['id_utilisateur'], p.id)) print("Fail to associate user %d to picture %d" % (r2['id_utilisateur'], p.id))
picture_link[str(r['id_photo'])] = p.id has_result = True
except Exception as e: except Exception as e:
pass
print("FAIL to migrate Picture: %s" % (repr(e))) print("FAIL to migrate Picture: %s" % (repr(e)))
cur.close() cur.close()
print("Chunk %d migrated at %s" % (chunk, str(datetime.datetime.now())))
print("Running time: %s" % (datetime.datetime.now()-start))
chunk += 1
finished = not has_result
print("SAS migrated at %s" % datetime.datetime.now()) print("SAS migrated at %s" % datetime.datetime.now())
print("Running time: %s" % (datetime.datetime.now()-start)) print("Running time: %s" % (datetime.datetime.now()-start))