From 4ec328556e531bb8777920048d5e5afe9de4f565 Mon Sep 17 00:00:00 2001 From: Skia Date: Sat, 13 Aug 2016 16:08:02 +0200 Subject: [PATCH] Migrate clubs --- club/migrations/0003_auto_20160813_1448.py | 19 ++ club/migrations/0004_auto_20160813_1551.py | 19 ++ club/models.py | 8 +- club/templates/club/club_detail.jinja | 2 +- club/templates/club/club_tools.jinja | 3 + core/management/commands/populate.py | 6 +- migrate.py | 346 +++++++++++++-------- 7 files changed, 265 insertions(+), 138 deletions(-) create mode 100644 club/migrations/0003_auto_20160813_1448.py create mode 100644 club/migrations/0004_auto_20160813_1551.py diff --git a/club/migrations/0003_auto_20160813_1448.py b/club/migrations/0003_auto_20160813_1448.py new file mode 100644 index 00000000..801c7a25 --- /dev/null +++ b/club/migrations/0003_auto_20160813_1448.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('club', '0002_club_home'), + ] + + operations = [ + migrations.AlterField( + model_name='club', + name='name', + field=models.CharField(verbose_name='name', max_length=64), + ), + ] diff --git a/club/migrations/0004_auto_20160813_1551.py b/club/migrations/0004_auto_20160813_1551.py new file mode 100644 index 00000000..8004b164 --- /dev/null +++ b/club/migrations/0004_auto_20160813_1551.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('club', '0003_auto_20160813_1448'), + ] + + operations = [ + migrations.AlterField( + model_name='membership', + name='description', + field=models.CharField(verbose_name='description', blank=True, max_length=128), + ), + ] diff --git a/club/models.py b/club/models.py index a96612c1..44aeb373 100644 --- a/club/models.py +++ b/club/models.py @@ -15,7 +15,7 @@ class Club(models.Model): """ The Club class, made as a tree to allow nice tidy organization """ - name = models.CharField(_('name'), max_length=30) + name = models.CharField(_('name'), max_length=64) parent = models.ForeignKey('Club', related_name='children', null=True, blank=True) unix_name = models.CharField(_('unix name'), max_length=30, unique=True, validators=[ @@ -72,10 +72,10 @@ class Club(models.Model): def save(self, *args, **kwargs): with transaction.atomic(): creation = False - if self.id is None: + old = Club.objects.filter(id=self.id).first() + if not old: creation = True else: - old = Club.objects.filter(id=self.id).first() if old.unix_name != self.unix_name: self._change_unixname(self.unix_name) super(Club, self).save(*args, **kwargs) @@ -148,7 +148,7 @@ class Membership(models.Model): end_date = models.DateField(_('end date'), null=True, blank=True) role = models.IntegerField(_('role'), choices=sorted(settings.SITH_CLUB_ROLES.items()), default=sorted(settings.SITH_CLUB_ROLES.items())[0][0]) - description = models.CharField(_('description'), max_length=30, null=False, blank=True) + description = models.CharField(_('description'), max_length=128, null=False, blank=True) def clean(self): sub = Subscriber.objects.filter(pk=self.user.pk).first() diff --git a/club/templates/club/club_detail.jinja b/club/templates/club/club_detail.jinja index c5814da5..5e9e6384 100644 --- a/club/templates/club/club_detail.jinja +++ b/club/templates/club/club_detail.jinja @@ -16,7 +16,7 @@

{{ club.name }}

{{ club.address }}

diff --git a/club/templates/club/club_tools.jinja b/club/templates/club/club_tools.jinja index 514c91b1..c8d49ee3 100644 --- a/club/templates/club/club_tools.jinja +++ b/club/templates/club/club_tools.jinja @@ -19,6 +19,9 @@ {% if object.club_account %}
  • Accouting: {{ object }}
    • {% endif %} + {% if object.unix_name == settings.SITH_LAUNDERETTE_MANAGER['unix_name'] %} +
  • {% trans %}Launderette{% endtrans %}
    • + {% endif %} {% endblock %} diff --git a/core/management/commands/populate.py b/core/management/commands/populate.py index cb5b30b5..7ec26c92 100644 --- a/core/management/commands/populate.py +++ b/core/management/commands/populate.py @@ -43,13 +43,13 @@ class Command(BaseCommand): home_root.save() club_root = SithFile(parent=None, name="clubs", is_folder=True, owner=root) club_root.save() - main_club = Club(name=settings.SITH_MAIN_CLUB['name'], unix_name=settings.SITH_MAIN_CLUB['unix_name'], + main_club = Club(id=1, name=settings.SITH_MAIN_CLUB['name'], unix_name=settings.SITH_MAIN_CLUB['unix_name'], address=settings.SITH_MAIN_CLUB['address']) main_club.save() - bar_club = Club(name=settings.SITH_BAR_MANAGER['name'], unix_name=settings.SITH_BAR_MANAGER['unix_name'], + bar_club = Club(id=2, name=settings.SITH_BAR_MANAGER['name'], unix_name=settings.SITH_BAR_MANAGER['unix_name'], address=settings.SITH_BAR_MANAGER['address']) bar_club.save() - launderette_club = Club(name=settings.SITH_LAUNDERETTE_MANAGER['name'], + launderette_club = Club(id=84, name=settings.SITH_LAUNDERETTE_MANAGER['name'], unix_name=settings.SITH_LAUNDERETTE_MANAGER['unix_name'], address=settings.SITH_LAUNDERETTE_MANAGER['address']) launderette_club.save() diff --git a/migrate.py b/migrate.py index b3fc2a3a..d110ef9e 100644 --- a/migrate.py +++ b/migrate.py @@ -2,15 +2,20 @@ import MySQLdb import os import django import random +from io import StringIO os.environ["DJANGO_SETTINGS_MODULE"] = "sith.settings" +os.environ['DJANGO_COLORS'] = 'nocolor' django.setup() -from core.models import User, SithFile from django.db import IntegrityError from django.conf import settings +from django.core.management import call_command +from django.db import connection -PROFILE_ROOT = "/data/matmatronch/" + +from core.models import User, SithFile +from club.models import Club, Membership db = MySQLdb.connect( host="ae-db", @@ -19,144 +24,225 @@ db = MySQLdb.connect( db="ae2-taiste", charset='utf8', use_unicode=True) -c = db.cursor(MySQLdb.cursors.DictCursor) -c.execute(""" -SELECT * -FROM utilisateurs utl -JOIN utl_etu ue -ON ue.id_utilisateur = utl.id_utilisateur -JOIN utl_etu_utbm ueu -ON ueu.id_utilisateur = utl.id_utilisateur -JOIN utl_extra uxtra -ON uxtra.id_utilisateur = utl.id_utilisateur -JOIN loc_ville ville -ON utl.id_ville = ville.id_ville -WHERE utl.id_utilisateur > 9000 -""") -User.objects.filter(id__gt=0).delete() -print("Users deleted") -# guy = c.fetchone() -# for k,v in sorted(guy.items()): -# print("%40s | %40s" % (k, v)) -SEX = {'1': 'MAN', '2': 'WOMAN', None: 'MAN'} -TSHIRT = { - None: '-', - '': '-', - 'NULL': '-', - 'XS': 'XS', - 'S': 'S', - 'M': 'M', - 'L': 'L', - 'XL': 'XL', - 'XXL': 'XXL', - 'XXXL': 'XXXL', - } -ROLE = { - 'doc': 'DOCTOR', - 'etu': 'STUDENT', - 'anc': 'FORMER STUDENT', - 'ens': 'TEACHER', - 'adm': 'ADMINISTRATIVE', - 'srv': 'SERVICE', - 'per': 'AGENT', - None: '', - } -DEPARTMENTS = { - 'tc': 'TC', - 'gi': 'GI', - 'gesc': 'GESC', - 'na': 'NA', - 'mc': 'MC', - 'imap': 'IMAP', - 'huma': 'HUMA', - 'edim': 'EDIM', - 'ee': 'EE', - 'imsi': 'IMSI', - 'truc': 'NA', - None: 'NA', - } +def reset_index(*args): + sqlcmd = StringIO() + call_command("sqlsequencereset", *args, stdout=sqlcmd) + cursor = connection.cursor() + cursor.execute(sqlcmd.getvalue()) def to_unicode(s): if s: return bytes(s, 'cp1252', errors="replace").decode('utf-8', errors='replace') return "" -def get_random_free_email(): - id = random.randrange(4000) - email = "no_email_%s@git.an" % random.randrange(4000, 40000) - while User.objects.filter(email=email).exists(): + +def migrate_users(): + SEX = {'1': 'MAN', '2': 'WOMAN', None: 'MAN'} + TSHIRT = { + None: '-', + '': '-', + 'NULL': '-', + 'XS': 'XS', + 'S': 'S', + 'M': 'M', + 'L': 'L', + 'XL': 'XL', + 'XXL': 'XXL', + 'XXXL': 'XXXL', + } + ROLE = { + 'doc': 'DOCTOR', + 'etu': 'STUDENT', + 'anc': 'FORMER STUDENT', + 'ens': 'TEACHER', + 'adm': 'ADMINISTRATIVE', + 'srv': 'SERVICE', + 'per': 'AGENT', + None: '', + } + DEPARTMENTS = { + 'tc': 'TC', + 'gi': 'GI', + 'gesc': 'GESC', + 'na': 'NA', + 'mc': 'MC', + 'imap': 'IMAP', + 'huma': 'HUMA', + 'edim': 'EDIM', + 'ee': 'EE', + 'imsi': 'IMSI', + 'truc': 'NA', + None: 'NA', + } + + def get_random_free_email(): + id = random.randrange(4000) email = "no_email_%s@git.an" % random.randrange(4000, 40000) - return email + while User.objects.filter(email=email).exists(): + email = "no_email_%s@git.an" % random.randrange(4000, 40000) + return email -for u in c.fetchall(): - try: - new = User( - id=u['id_utilisateur'], - last_name=to_unicode(u['nom_utl']) or "Bou", - first_name=to_unicode(u['prenom_utl']) or "Bi", - email=u['email_utl'], - second_email=u['email_utbm'] or "", - date_of_birth=u['date_naissance_utl'], - last_update=u['date_maj_utl'], - nick_name=to_unicode(u['surnom_utbm']), - sex=SEX[u['sexe_utl']], - tshirt_size=TSHIRT[u['taille_tshirt_utl']], - role=ROLE[u['role_utbm']], - department=DEPARTMENTS[u['departement_utbm']], - dpt_option=to_unicode(u['filiere_utbm']), - semester=u['semestre_utbm'] or 0, - quote=to_unicode(u['citation']), - school=to_unicode(u['nom_ecole_etudiant']), - promo=u['promo_utbm'] or 0, - forum_signature=to_unicode(u['signature_utl']), - address=(to_unicode(u['addresse_utl']) + ", " + to_unicode(u['cpostal_ville']) + " " + to_unicode(u['nom_ville'])), - parent_address=(to_unicode(u['adresse_parents']) + ", " + to_unicode(u['cpostal_parents']) + " " + to_unicode(u['ville_parents'])), - phone=u['tel_portable_utl'] or "", - parent_phone=u['tel_parents'] or "", - ) - new.generate_username() - new.save() - except IntegrityError as e: - if "Key (email)" in repr(e): - new.email = get_random_free_email() - new.save() - print("New email generated") - else: - print("FAIL for user %s: %s" % (u['id_utilisateur'], repr(e))) - except Exception as e: - print("FAIL for user %s: %s" % (u['id_utilisateur'], repr(e))) -c.close() + c = db.cursor(MySQLdb.cursors.DictCursor) + c.execute(""" + SELECT * + FROM utilisateurs utl + JOIN utl_etu ue + ON ue.id_utilisateur = utl.id_utilisateur + JOIN utl_etu_utbm ueu + ON ueu.id_utilisateur = utl.id_utilisateur + JOIN utl_extra uxtra + ON uxtra.id_utilisateur = utl.id_utilisateur + JOIN loc_ville ville + ON utl.id_ville = ville.id_ville + -- WHERE utl.id_utilisateur > 9000 + """) + User.objects.filter(id__gt=0).delete() + print("Users deleted") -from os import listdir -from django.core.files import File - -profile = SithFile.objects.filter(parent=None, name="profiles").first() -for filename in listdir(PROFILE_ROOT): - if filename.split('.')[-2] != "mini": + for u in c.fetchall(): try: - uid = filename.split('.')[0].split('-')[0] - user = User.objects.filter(id=int(uid)).first() - if user: - f = File(open(PROFILE_ROOT + '/' + filename, 'rb')) - t = filename.split('.')[1] - new_file = SithFile(parent=profile, name=filename, - file=f, owner=user, is_folder=False, mime_type="image/jpeg", size=f.size) - if t == "identity": - new_file.save() - user.profile_pict = new_file - user.save() - elif t == "blouse": - new_file.save() - user.scrub_pict = new_file - user.save() - else: - new_file.save() - user.avatar_pict = new_file - user.save() + new = User( + id=u['id_utilisateur'], + last_name=to_unicode(u['nom_utl']) or "Bou", + first_name=to_unicode(u['prenom_utl']) or "Bi", + email=u['email_utl'], + second_email=u['email_utbm'] or "", + date_of_birth=u['date_naissance_utl'], + last_update=u['date_maj_utl'], + nick_name=to_unicode(u['surnom_utbm']), + sex=SEX[u['sexe_utl']], + tshirt_size=TSHIRT[u['taille_tshirt_utl']], + role=ROLE[u['role_utbm']], + department=DEPARTMENTS[u['departement_utbm']], + dpt_option=to_unicode(u['filiere_utbm']), + semester=u['semestre_utbm'] or 0, + quote=to_unicode(u['citation']), + school=to_unicode(u['nom_ecole_etudiant']), + promo=u['promo_utbm'] or 0, + forum_signature=to_unicode(u['signature_utl']), + address=(to_unicode(u['addresse_utl']) + ", " + to_unicode(u['cpostal_ville']) + " " + to_unicode(u['nom_ville'])), + parent_address=(to_unicode(u['adresse_parents']) + ", " + to_unicode(u['cpostal_parents']) + " " + to_unicode(u['ville_parents'])), + phone=u['tel_portable_utl'] or "", + parent_phone=u['tel_parents'] or "", + ) + new.generate_username() + new.save() + except IntegrityError as e: + if "Key (email)" in repr(e): + new.email = get_random_free_email() + new.save() + print("New email generated") + else: + print("FAIL for user %s: %s" % (u['id_utilisateur'], repr(e))) except Exception as e: - print(repr(e)) + print("FAIL for user %s: %s" % (u['id_utilisateur'], repr(e))) + c.close() + reset_index('core') -# profile_pict = models.OneToOneField('SithFile', related_name='profile_of', verbose_name=_("profile"), null=True, blank=True) -# avatar_pict = models.OneToOneField('SithFile', related_name='avatar_of', verbose_name=_("avatar"), null=True, blank=True) -# scrub_pict = models.OneToOneField('SithFile', related_name='scrub_of', verbose_name=_("scrub"), null=True, blank=True) +def migrate_profile_pict(): + PROFILE_ROOT = "/data/matmatronch/" + + from os import listdir + from django.core.files import File + + profile = SithFile.objects.filter(parent=None, name="profiles").first() + profile.children.all().delete() + print("Profiles pictures deleted") + for filename in listdir(PROFILE_ROOT): + if filename.split('.')[-2] != "mini": + try: + uid = filename.split('.')[0].split('-')[0] + user = User.objects.filter(id=int(uid)).first() + if user: + f = File(open(PROFILE_ROOT + '/' + filename, 'rb')) + f.name = f.name.split('/')[-1] + t = filename.split('.')[1] + new_file = SithFile(parent=profile, name=filename, + file=f, owner=user, is_folder=False, mime_type="image/jpeg", size=f.size) + if t == "identity": + new_file.save() + user.profile_pict = new_file + user.save() + elif t == "blouse": + new_file.save() + user.scrub_pict = new_file + user.save() + else: + new_file.save() + user.avatar_pict = new_file + user.save() + except Exception as e: + print(repr(e)) + +def migrate_clubs(): + cur = db.cursor(MySQLdb.cursors.DictCursor) + cur.execute(""" + SELECT * + FROM asso asso + WHERE nom_unix_asso <> "ae" + AND nom_unix_asso <> "bdf" + AND nom_unix_asso <> "laverie" + """) + # club = cur.fetchone() + # for k,v in club.items(): + # print("%40s | %40s" % (k, v)) + + for c in cur.fetchall(): + try: + new = Club( + id=c['id_asso'], + name=to_unicode(c['nom_asso']), + unix_name=to_unicode(c['nom_unix_asso']), + address=to_unicode(c['adresse_postale']), + ) + new.save() + except Exception as e: + print("FAIL for club %s: %s" % (c['nom_unix_asso'], repr(e))) + cur.execute(""" + SELECT * + FROM asso + """) + for c in cur.fetchall(): + club = Club.objects.filter(id=c['id_asso']).first() + parent = Club.objects.filter(id=c['id_asso_parent']).first() + club.parent = parent + club.save() + cur.close() + +def migrate_club_memberships(): + cur = db.cursor(MySQLdb.cursors.DictCursor) + cur.execute(""" + SELECT * + FROM asso_membre + """) + + Membership.objects.all().delete() + print("Memberships deleted") + for m in cur.fetchall(): + try: + club = Club.objects.filter(id=m['id_asso']).first() + user = User.objects.filter(id=m['id_utilisateur']).first() + if club and user: + new = Membership( + club=club, + user=user, + start_date=m['date_debut'], + end_date=m['date_fin'], + role=m['role'], + description=to_unicode(m['desc_role']), + ) + new.save() + except Exception as e: + print("FAIL for club membership %s: %s" % (m['id_asso'], repr(e))) + cur.close() + + +def main(): + # migrate_users() + # migrate_profile_pict() + # migrate_clubs() + migrate_club_memberships() + +if __name__ == "__main__": + main()