Sith/migrate.py

#!/usr/bin/env python3
# -*- coding:utf-8 -*
#
# Copyright 2016,2017
# - Skia <skia@libskia.so>
#
# Ce fichier fait partie du site de l'Association des Étudiants de l'UTBM,
# http://ae.utbm.fr.
#
# This program is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License a published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Sofware Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
#
#

import MySQLdb
import os
import django
import random
import datetime
from io import StringIO
from pytz import timezone
from os import listdir

os.environ["DJANGO_SETTINGS_MODULE"] = "sith.settings"
os.environ["DJANGO_COLORS"] = "nocolor"
django.setup()

from django.db import IntegrityError
from django.conf import settings
from django.core.management import call_command
from django.db import connection
from django.forms import ValidationError
from django.core.files import File


from core.models import User, SithFile
from core.utils import doku_to_markdown, bbcode_to_markdown
from club.models import Club, Membership, Mailing, MailingSubscription
from counter.models import (
    Customer,
    Counter,
    Selling,
    Refilling,
    Product,
    ProductType,
    Permanency,
    Eticket,
)
from subscription.models import Subscription
from eboutic.models import Invoice, InvoiceItem
from accounting.models import (
    BankAccount,
    ClubAccount,
    GeneralJournal,
    Operation,
    AccountingType,
    Company,
    SimplifiedAccountingType,
    Label,
)
from sas.models import Album, Picture, PeoplePictureRelation
from forum.models import (
    Forum,
    ForumTopic,
    ForumMessage,
    ForumMessageMeta,
    ForumUserInfo,
)
from pedagogy.models import UV, UVComment, UVResult

db = MySQLdb.connect(**settings.OLD_MYSQL_INFOS)
start = datetime.datetime.now()


def reset_index(*args):
    sqlcmd = StringIO()
    call_command("sqlsequencereset", *args, stdout=sqlcmd)
    cursor = connection.cursor()
    cursor.execute(sqlcmd.getvalue())


def to_unicode(s):
    if s:
        return bytes(s, "cp1252", errors="replace").decode("utf-8", errors="replace")
    return ""


def migrate_core():
    def migrate_users():
        SEX = {"1": "MAN", "2": "WOMAN", None: "MAN"}
        TSHIRT = {
            None: "-",
            "": "-",
            "NULL": "-",
            "XS": "XS",
            "S": "S",
            "M": "M",
            "L": "L",
            "XL": "XL",
            "XXL": "XXL",
            "XXXL": "XXXL",
        }
        ROLE = {
            "doc": "DOCTOR",
            "etu": "STUDENT",
            "anc": "FORMER STUDENT",
            "ens": "TEACHER",
            "adm": "ADMINISTRATIVE",
            "srv": "SERVICE",
            "per": "AGENT",
            None: "",
        }
        DEPARTMENTS = {
            "tc": "TC",
            "gi": "GI",
            "gesc": "GESC",
            "na": "NA",
            "mc": "MC",
            "imap": "IMAP",
            "huma": "HUMA",
            "edim": "EDIM",
            "ee": "EE",
            "imsi": "IMSI",
            "truc": "NA",
            None: "NA",
        }

        def get_random_free_email():
            email = "no_email_%s@git.an" % random.randrange(4000, 40000)
            while User.objects.filter(email=email).exists():
                email = "no_email_%s@git.an" % random.randrange(4000, 40000)
            return email

        c = db.cursor(MySQLdb.cursors.SSDictCursor)
        c.execute(
            """
        SELECT *
        FROM utilisateurs utl
        LEFT JOIN utl_etu ue
        ON ue.id_utilisateur = utl.id_utilisateur
        LEFT JOIN utl_etu_utbm ueu
        ON ueu.id_utilisateur = utl.id_utilisateur
        LEFT JOIN utl_extra uxtra
        ON uxtra.id_utilisateur = utl.id_utilisateur
        LEFT JOIN loc_ville ville
        ON utl.id_ville = ville.id_ville
        -- WHERE utl.id_utilisateur = 9360
        """
        )
        User.objects.filter(id__gt=0).delete()
        print("Users deleted")

        for u in c:
            try:
                new = User(
                    id=u["id_utilisateur"],
                    last_name=to_unicode(u["nom_utl"]) or "Bou",
                    first_name=to_unicode(u["prenom_utl"]) or "Bi",
                    email=u["email_utl"],
                    second_email=u["email_utbm"] or "",
                    date_of_birth=u["date_naissance_utl"],
                    last_update=u["date_maj_utl"],
                    nick_name=to_unicode(u["surnom_utbm"]),
                    sex=SEX[u["sexe_utl"]],
                    tshirt_size=TSHIRT[u["taille_tshirt_utl"]],
                    role=ROLE[u["role_utbm"]],
                    department=DEPARTMENTS[u["departement_utbm"]],
                    dpt_option=to_unicode(u["filiere_utbm"]),
                    semester=u["semestre_utbm"] or 0,
                    quote=to_unicode(u["citation"]),
                    school=to_unicode(u["nom_ecole_etudiant"]),
                    promo=u["promo_utbm"] or 0,
                    forum_signature=to_unicode(u["signature_utl"]),
                    address=(
                        to_unicode(u["addresse_utl"])
                        + ", "
                        + to_unicode(u["cpostal_ville"])
                        + " "
                        + to_unicode(u["nom_ville"])
                    ),
                    parent_address=(
                        to_unicode(u["adresse_parents"])
                        + ", "
                        + to_unicode(u["cpostal_parents"])
                        + " "
                        + to_unicode(u["ville_parents"])
                    ),
                    phone=u["tel_portable_utl"] or "",
                    parent_phone=u["tel_parents"] or "",
                    is_subscriber_viewable=bool(u["publique_utl"]),
                )
                new.generate_username()
                new.set_password(str(random.randrange(1000000, 10000000)))
                new.save()
            except IntegrityError as e:
                if "Key (email)" in repr(e):
                    new.email = get_random_free_email()
                    new.save()
                    print("New email generated")
                else:
                    print("FAIL for user %s: %s" % (u["id_utilisateur"], repr(e)))
            except Exception as e:
                print("FAIL for user %s: %s" % (u["id_utilisateur"], repr(e)))
        c.close()
        print("Users migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_profile_pict():
        PROFILE_ROOT = "/data/matmatronch/"

        profile = SithFile.objects.filter(parent=None, name="profiles").first()
        profile.children.all().delete()
        print("Profiles pictures deleted")
        for filename in listdir(PROFILE_ROOT):
            if filename.split(".")[-2] != "mini":
                try:
                    uid = filename.split(".")[0].split("-")[0]
                    user = User.objects.filter(id=int(uid)).first()
                    if user:
                        f = File(open(PROFILE_ROOT + "/" + filename, "rb"))
                        f.name = f.name.split("/")[-1]
                        t = filename.split(".")[1]
                        new_file = SithFile(
                            parent=profile,
                            name=filename,
                            file=f,
                            owner=user,
                            is_folder=False,
                            mime_type="image/jpeg",
                            size=f.size,
                        )
                        if t == "identity":
                            new_file.save()
                            user.profile_pict = new_file
                            user.save()
                        elif t == "blouse":
                            new_file.save()
                            user.scrub_pict = new_file
                            user.save()
                        else:
                            new_file.save()
                            user.avatar_pict = new_file
                            user.save()
                except Exception as e:
                    print(repr(e))
        print("Profile pictures migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    migrate_users()
    migrate_profile_pict()


def migrate_club():
    def migrate_clubs():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM asso asso
        WHERE nom_unix_asso <> "ae"
            AND nom_unix_asso <> "bdf"
            AND nom_unix_asso <> "laverie"
        """
        )
        # club = cur.fetchone()
        # for k,v in club.items():
        #     print("%40s | %40s" % (k, v))

        for c in cur:
            try:
                new = Club(
                    id=c["id_asso"],
                    name=to_unicode(c["nom_asso"]),
                    unix_name=to_unicode(c["nom_unix_asso"]),
                    address=to_unicode(c["adresse_postale"]),
                )
                new.save()
            except Exception as e:
                print("FAIL for club %s: %s" % (c["nom_unix_asso"], repr(e)))
        cur.execute(
            """
        SELECT *
        FROM asso
        """
        )
        for c in cur:
            club = Club.objects.filter(id=c["id_asso"]).first()
            parent = Club.objects.filter(id=c["id_asso_parent"]).first()
            club.parent = parent
            club.save()
        cur.close()
        print("Clubs migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_club_memberships():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM asso_membre
        """
        )

        Membership.objects.all().delete()
        print("Memberships deleted")
        for m in cur:
            try:
                club = Club.objects.filter(id=m["id_asso"]).first()
                user = User.objects.filter(id=m["id_utilisateur"]).first()
                if club and user:
                    new = Membership(
                        id=Membership.objects.count() + 1,
                        club=club,
                        user=user,
                        start_date=m["date_debut"],
                        end_date=m["date_fin"],
                        role=m["role"],
                        description=to_unicode(m["desc_role"]),
                    )
                    new.save()
            except Exception as e:
                print("FAIL for club membership %s: %s" % (m["id_asso"], repr(e)))
        cur.close()
        print("Clubs memberships migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    # migrate_clubs()
    migrate_club_memberships()


def migrate_subscriptions():
    LOCATION = {5: "SEVENANS", 6: "BELFORT", 9: "MONTBELIARD", None: "SEVENANS"}
    TYPE = {
        0: "un-semestre",
        1: "deux-semestres",
        2: "cursus-tronc-commun",
        3: "cursus-branche",
        4: "membre-honoraire",
        5: "assidu",
        6: "amicale/doceo",
        7: "reseau-ut",
        8: "crous",
        9: "sbarro/esta",
        10: "cursus-alternant",
        None: "un-semestre",
    }
    PAYMENT = {1: "CHECK", 2: "CARD", 3: "CASH", 4: "OTHER", 5: "EBOUTIC", 0: "OTHER"}
    cur = db.cursor(MySQLdb.cursors.SSDictCursor)
    cur.execute(
        """
    SELECT *
    FROM ae_cotisations
    """
    )

    Subscription.objects.all().delete()
    print("Subscriptions deleted")
    Customer.objects.all().delete()
    print("Customers deleted")
    for r in cur:
        try:
            user = User.objects.filter(id=r["id_utilisateur"]).first()
            if user:
                new = Subscription(
                    id=r["id_cotisation"],
                    member=user,
                    subscription_start=r["date_cotis"],
                    subscription_end=r["date_fin_cotis"],
                    subscription_type=TYPE[r["type_cotis"]],
                    payment_method=PAYMENT[r["mode_paiement_cotis"]],
                    location=LOCATION[r["id_comptoir"]],
                )
                new.save()
        except Exception as e:
            print("FAIL for subscription %s: %s" % (r["id_cotisation"], repr(e)))
    cur.close()
    print("Subscriptions migrated at %s" % datetime.datetime.now())
    print("Running time: %s" % (datetime.datetime.now() - start))


def migrate_counter():
    def update_customer_account():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM ae_carte carte
        JOIN ae_cotisations cotis
        ON carte.id_cotisation = cotis.id_cotisation
        """
        )
        for r in cur:
            try:
                user = Customer.objects.filter(user_id=r["id_utilisateur"]).first()
                if user:
                    user.account_id = str(r["id_carte_ae"]) + r["cle_carteae"].lower()
                    user.save()
            except Exception as e:
                print(
                    "FAIL to update customer account for %s: %s"
                    % (r["id_cotisation"], repr(e))
                )
        cur.close()
        print("Customer accounts migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_counters():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpt_comptoir
        """
        )
        Counter.objects.all().delete()
        for r in cur:
            try:
                club = Club.objects.filter(id=r["id_assocpt"]).first()
                new = Counter(
                    id=r["id_comptoir"],
                    name=to_unicode(r["nom_cpt"]),
                    club=club,
                    type="OFFICE",
                )
                new.save()
            except Exception as e:
                print("FAIL to migrate counter %s: %s" % (r["id_comptoir"], repr(e)))
        cur.close()
        eboutic = Counter.objects.filter(id=3).first()
        eboutic.type = "EBOUTIC"
        eboutic.save()
        print("Counters migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def reset_customer_amount():
        Refilling.objects.all().delete()
        Selling.objects.all().delete()
        Invoice.objects.all().delete()
        for c in Customer.objects.all():
            c.amount = 0
            c.save()
        print("Customer amount reset")

    def migrate_refillings():
        BANK = {
            0: "OTHER",
            1: "SOCIETE-GENERALE",
            2: "BANQUE-POPULAIRE",
            3: "BNP",
            4: "CAISSE-EPARGNE",
            5: "CIC",
            6: "CREDIT-AGRICOLE",
            7: "CREDIT-MUTUEL",
            8: "CREDIT-LYONNAIS",
            9: "LA-POSTE",
            100: "OTHER",
            None: "OTHER",
        }
        PAYMENT = {2: "CARD", 1: "CASH", 0: "CHECK"}
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpt_rechargements
        """
        )
        root_cust = Customer.objects.filter(user__id=0).first()
        mde = Counter.objects.filter(id=1).first()
        Refilling.objects.all().delete()
        print("Refillings deleted")
        fail = 100
        for r in cur:
            try:
                cust = Customer.objects.filter(user__id=r["id_utilisateur"]).first()
                user = User.objects.filter(id=r["id_utilisateur"]).first()
                if not cust:
                    if not user:
                        cust = root_cust
                    else:
                        cust = Customer(
                            user=user,
                            amount=0,
                            account_id=Customer.generate_account_id(fail),
                        )
                        cust.save()
                        fail += 1
                op = User.objects.filter(id=r["id_utilisateur_operateur"]).first()
                counter = Counter.objects.filter(id=r["id_comptoir"]).first()
                new = Refilling(
                    id=r["id_rechargement"],
                    counter=counter or mde,
                    customer=cust or root_cust,
                    operator=op or root_cust.user,
                    amount=r["montant_rech"] / 100,
                    payment_method=PAYMENT[r["type_paiement_rech"]],
                    bank=BANK[r["banque_rech"]],
                    date=r["date_rech"].replace(tzinfo=timezone("Europe/Paris")),
                )
                new.save()
            except Exception as e:
                print(
                    "FAIL to migrate refilling %s for %s: %s"
                    % (r["id_rechargement"], r["id_utilisateur"], repr(e))
                )
        cur.close()
        print("Refillings migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_typeproducts():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpt_type_produit
        """
        )
        ProductType.objects.all().delete()
        print("Product types deleted")
        for r in cur:
            try:
                new = ProductType(
                    id=r["id_typeprod"],
                    name=to_unicode(r["nom_typeprod"]),
                    description=to_unicode(r["description_typeprod"]),
                )
                new.save()
            except Exception as e:
                print(
                    "FAIL to migrate product type %s: %s" % (r["nom_typeprod"], repr(e))
                )
        cur.close()
        print("Product types migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_products():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpt_produits
        """
        )
        Product.objects.all().delete()
        print("Product deleted")
        for r in cur:
            try:
                type = ProductType.objects.filter(id=r["id_typeprod"]).first()
                club = Club.objects.filter(id=r["id_assocpt"]).first()
                new = Product(
                    id=r["id_produit"],
                    product_type=type,
                    name=to_unicode(r["nom_prod"]),
                    description=to_unicode(r["description_prod"]),
                    code=to_unicode(r["cbarre_prod"]),
                    purchase_price=r["prix_achat_prod"] / 100,
                    selling_price=r["prix_vente_prod"] / 100,
                    special_selling_price=r["prix_vente_barman_prod"] / 100,
                    club=club,
                    limit_age=r["mineur"] or 0,
                    tray=bool(r["plateau"]),
                )
                new.save()
            except Exception as e:
                print("FAIL to migrate product %s: %s" % (r["nom_prod"], repr(e)))
        cur.close()
        print("Product migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_product_pict():
        FILE_ROOT = "/data/files/"

        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpt_produits
        WHERE id_file IS NOT NULL
        """
        )
        for r in cur:
            try:
                prod = Product.objects.filter(id=r["id_produit"]).first()
                if prod:
                    f = File(open(FILE_ROOT + "/" + str(r["id_file"]) + ".1", "rb"))
                    f.name = prod.name
                    prod.icon = f
                    prod.save()
            except Exception as e:
                print(repr(e))
        print("Product pictures migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_products_to_counter():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpt_mise_en_vente
        """
        )
        for r in cur:
            try:
                product = Product.objects.filter(id=r["id_produit"]).first()
                counter = Counter.objects.filter(id=r["id_comptoir"]).first()
                counter.products.add(product)
                counter.save()
            except Exception as e:
                print(
                    "FAIL to set product %s in counter %s: %s"
                    % (product, counter, repr(e))
                )
        cur.close()
        print("Product in counters migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_invoices():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpt_vendu ven
        LEFT JOIN cpt_debitfacture fac
        ON ven.id_facture = fac.id_facture
        WHERE fac.mode_paiement = 'SG'
        """
        )
        Invoice.objects.all().delete()
        print("Invoices deleted")
        Refilling.objects.filter(payment_method="CARD").delete()
        print("Card refillings deleted")
        Selling.objects.filter(payment_method="CARD").delete()
        print("Card sellings deleted")
        root = User.objects.filter(id=0).first()
        for r in cur:
            try:
                product = Product.objects.filter(id=r["id_produit"]).first()
                user = User.objects.filter(id=r["id_utilisateur_client"]).first()
                i = Invoice.objects.filter(id=r["id_facture"]).first() or Invoice(
                    id=r["id_facture"]
                )
                i.user = user or root
                for f in i._meta.local_fields:
                    if f.name == "date":
                        f.auto_now = False
                i.date = r["date_facture"].replace(tzinfo=timezone("Europe/Paris"))
                i.save()
                InvoiceItem(
                    invoice=i,
                    product_id=product.id,
                    product_name=product.name,
                    type_id=product.product_type.id,
                    product_unit_price=r["prix_unit"] / 100,
                    quantity=r["quantite"],
                ).save()
            except ValidationError as e:
                print(repr(e) + " for %s (%s)" % (customer, customer.user.id))
            except Exception as e:
                print("FAIL to migrate invoice %s: %s" % (r["id_facture"], repr(e)))
        cur.close()
        for i in Invoice.objects.all():
            for f in i._meta.local_fields:
                if f.name == "date":
                    f.auto_now = False
            i.validate()
        print("Invoices migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_sellings():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpt_vendu ven
        LEFT JOIN cpt_debitfacture fac
        ON ven.id_facture = fac.id_facture
        WHERE fac.mode_paiement = 'AE'
        """
        )
        Selling.objects.filter(payment_method="SITH_ACCOUNT").delete()
        print("Sith account selling deleted")
        for c in Customer.objects.all():
            c.amount = sum([r.amount for r in c.refillings.all()])
            c.save()
        print("Customer amount reset to sum of refillings")
        ae = Club.objects.filter(unix_name="ae").first()
        mde = Counter.objects.filter(id=1).first()
        root = User.objects.filter(id=0).first()
        beer = Product.objects.filter(id=1).first()
        for r in cur:
            try:
                product = Product.objects.filter(id=r["id_produit"]).first() or beer
                club = Club.objects.filter(id=r["id_assocpt"]).first() or ae
                counter = Counter.objects.filter(id=r["id_comptoir"]).first() or mde
                op = User.objects.filter(id=r["id_utilisateur"]).first() or root
                customer = (
                    Customer.objects.filter(user__id=r["id_utilisateur_client"]).first()
                    or root.customer
                )
                new = Selling(
                    label=product.name or "Produit inexistant",
                    counter=counter,
                    club=club,
                    product=product,
                    seller=op,
                    customer=customer,
                    unit_price=r["prix_unit"] / 100,
                    quantity=r["quantite"],
                    payment_method="SITH_ACCOUNT",
                    date=r["date_facture"].replace(tzinfo=timezone("Europe/Paris")),
                )
                new.save()
            except ValidationError as e:
                print(repr(e) + " for %s (%s)" % (customer, customer.user.id))
            except Exception as e:
                print("FAIL to migrate selling %s: %s" % (r["id_facture"], repr(e)))
        cur.close()
        print("Sellings migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_permanencies():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpt_tracking
        """
        )
        Permanency.objects.all().delete()
        print("Permanencies deleted")
        for r in cur:
            try:
                counter = Counter.objects.filter(id=r["id_comptoir"]).first()
                user = User.objects.filter(id=r["id_utilisateur"]).first()
                new = Permanency(
                    user=user,
                    counter=counter,
                    start=r["logged_time"].replace(tzinfo=timezone("Europe/Paris")),
                    activity=r["logged_time"].replace(tzinfo=timezone("Europe/Paris")),
                    end=r["closed_time"].replace(tzinfo=timezone("Europe/Paris")),
                )
                new.save()
            except Exception as e:
                print("FAIL to migrate permanency: %s" % (repr(e)))
        cur.close()
        print("Permanencies migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    update_customer_account()
    migrate_counters()
    migrate_permanencies()
    migrate_typeproducts()
    migrate_products()
    migrate_product_pict()
    migrate_products_to_counter()
    reset_customer_amount()
    migrate_invoices()
    migrate_refillings()
    migrate_sellings()


def check_accounts():
    cur = db.cursor(MySQLdb.cursors.SSDictCursor)
    cur.execute(
        """
    SELECT *
    FROM utilisateurs
    """
    )
    mde = Counter.objects.filter(id=1).first()
    ae = Club.objects.filter(unix_name="ae").first()
    root = User.objects.filter(id=0).first()
    for r in cur:
        if r["montant_compte"] and r["montant_compte"] > 0:
            try:
                cust = Customer.objects.filter(user__id=r["id_utilisateur"]).first()
                if int(cust.amount * 100) != r["montant_compte"]:
                    print(
                        "Adding %s to %s's account"
                        % (float(cust.amount) - (r["montant_compte"] / 100), cust.user)
                    )
                    new = Selling(
                        label="Ajustement migration base de donnée",
                        counter=mde,
                        club=ae,
                        product=None,
                        seller=root,
                        customer=cust,
                        unit_price=float(cust.amount) - (r["montant_compte"] / 100.0),
                        quantity=1,
                        payment_method="SITH_ACCOUNT",
                    )
                    new.save()
            except Exception as e:
                print("FAIL to adjust user account: %s" % (repr(e)))


### Accounting


def migrate_accounting():
    def migrate_companies():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM entreprise
        """
        )
        Company.objects.all().delete()
        print("Company deleted")
        for r in cur:
            try:
                new = Company(
                    id=r["id_ent"],
                    name=to_unicode(r["nom_entreprise"]),
                    street=to_unicode(r["rue_entreprise"]),
                    city=to_unicode(r["ville_entreprise"]),
                    postcode=to_unicode(r["cpostal_entreprise"]),
                    country=to_unicode(r["pays_entreprise"]),
                    phone=to_unicode(r["telephone_entreprise"]),
                    email=to_unicode(r["email_entreprise"]),
                    website=to_unicode(r["siteweb_entreprise"]),
                )
                new.save()
            except Exception as e:
                print("FAIL to migrate company: %s" % (repr(e)))
        cur.close()
        print("Companies migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_bank_accounts():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpta_cpbancaire
        """
        )
        BankAccount.objects.all().delete()
        print("Bank accounts deleted")
        ae = Club.objects.filter(unix_name="ae").first()
        for r in cur:
            try:
                new = BankAccount(
                    id=r["id_cptbc"], club=ae, name=to_unicode(r["nom_cptbc"])
                )
                new.save()
            except Exception as e:
                print("FAIL to migrate bank account: %s" % (repr(e)))
        cur.close()
        print("Bank accounts migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_club_accounts():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpta_cpasso
        """
        )
        ClubAccount.objects.all().delete()
        print("Club accounts deleted")
        ae = Club.objects.filter(id=1).first()
        for r in cur:
            try:
                club = Club.objects.filter(id=r["id_asso"]).first() or ae
                bank_acc = BankAccount.objects.filter(id=r["id_cptbc"]).first()
                new = ClubAccount(
                    id=r["id_cptasso"],
                    club=club,
                    name=club.name[:30],
                    bank_account=bank_acc,
                )
                new.save()
            except Exception as e:
                print("FAIL to migrate club account: %s" % (repr(e)))
        cur.close()
        print("Club accounts migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_journals():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpta_classeur
        """
        )
        GeneralJournal.objects.all().delete()
        print("General journals deleted")
        for r in cur:
            try:
                club_acc = ClubAccount.objects.filter(id=r["id_cptasso"]).first()
                new = GeneralJournal(
                    id=r["id_classeur"],
                    club_account=club_acc,
                    name=to_unicode(r["nom_classeur"]),
                    start_date=r["date_debut_classeur"],
                    end_date=r["date_fin_classeur"],
                    closed=bool(r["ferme"]),
                )
                new.save()
            except Exception as e:
                print("FAIL to migrate general journal: %s" % (repr(e)))
        cur.close()
        print("General journals migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_accounting_types():
        MOVEMENT = {-1: "DEBIT", 0: "NEUTRAL", 1: "CREDIT"}
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpta_op_plcptl
        """
        )
        AccountingType.objects.all().delete()
        print("Accounting types deleted")
        for r in cur:
            try:
                new = AccountingType(
                    id=r["id_opstd"],
                    code=str(r["code_plan"]),
                    label=to_unicode(r["libelle_plan"]).capitalize(),
                    movement_type=MOVEMENT[r["type_mouvement"]],
                )
                new.save()
            except Exception as e:
                print("FAIL to migrate accounting type: %s" % (repr(e)))
        cur.close()
        print("Accounting types migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_simpleaccounting_types():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpta_op_clb
        WHERE id_asso IS NULL
        """
        )
        SimplifiedAccountingType.objects.all().delete()
        print("Simple accounting types deleted")
        for r in cur:
            try:
                at = AccountingType.objects.filter(id=r["id_opstd"]).first()
                new = SimplifiedAccountingType(
                    id=r["id_opclb"],
                    label=to_unicode(r["libelle_opclb"]).capitalize(),
                    accounting_type=at,
                )
                new.save()
            except Exception as e:
                print("FAIL to migrate simple type: %s" % (repr(e)))
        cur.close()
        print("Simple accounting types migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_labels():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpta_libelle
        WHERE id_asso IS NOT NULL
        """
        )
        Label.objects.all().delete()
        print("Labels deleted")
        for r in cur:
            try:
                club_accounts = ClubAccount.objects.filter(club__id=r["id_asso"]).all()
                for ca in club_accounts:
                    new = Label(club_account=ca, name=to_unicode(r["nom_libelle"]))
                    new.save()
            except Exception as e:
                print("FAIL to migrate label: %s" % (repr(e)))
        cur.close()
        print("Labels migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def migrate_operations():
        MODE = {
            1: "CHECK",
            2: "CASH",
            3: "TRANSFERT",
            4: "CARD",
            0: "CASH",
            None: "CASH",
        }
        MOVEMENT_TYPE = {-1: "DEBIT", 0: "NEUTRAL", 1: "CREDIT", None: "NEUTRAL"}
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpta_operation op
        LEFT JOIN cpta_op_clb clb
        ON op.id_opclb = clb.id_opclb
        LEFT JOIN cpta_libelle lab
        ON op.id_libelle = lab.id_libelle
        """
        )
        Operation.objects.all().delete()
        print("Operation deleted")
        for r in cur:
            try:
                simple_type = None
                accounting_type = None
                label = None
                if r["id_opclb"]:
                    simple_type = SimplifiedAccountingType.objects.filter(
                        id=r["id_opclb"]
                    ).first()
                if r["id_opstd"]:
                    accounting_type = AccountingType.objects.filter(
                        id=r["id_opstd"]
                    ).first()
                if not accounting_type and simple_type:
                    accounting_type = simple_type.accounting_type
                if not accounting_type:
                    accounting_type = AccountingType.objects.filter(
                        movement_type=MOVEMENT_TYPE[r["type_mouvement"]]
                    ).first()
                journal = GeneralJournal.objects.filter(id=r["id_classeur"]).first()
                if r["id_libelle"]:
                    label = journal.club_account.labels.filter(
                        name=to_unicode(r["nom_libelle"])
                    ).first()

                def get_target_type():
                    if r["id_utilisateur"]:
                        return "USER"
                    if r["id_asso"]:
                        return "CLUB"
                    if r["id_ent"]:
                        return "COMPANY"
                    if r["id_classeur"]:
                        return "ACCOUNT"

                def get_target_id():
                    return (
                        int(
                            r["id_utilisateur"]
                            or r["id_asso"]
                            or r["id_ent"]
                            or r["id_classeur"]
                        )
                        or None
                    )

                new = Operation(
                    id=r["id_op"],
                    journal=journal,
                    amount=r["montant_op"] / 100,
                    date=r["date_op"] or journal.end_date,
                    remark=to_unicode(r["commentaire_op"]),
                    mode=MODE[r["mode_op"]],
                    cheque_number=str(r["num_cheque_op"]),
                    done=bool(r["op_effctue"]),
                    simpleaccounting_type=simple_type,
                    accounting_type=accounting_type,
                    target_type=get_target_type(),
                    target_id=get_target_id(),
                    target_label="-",
                    label=label,
                )
                try:
                    new.clean()
                except:
                    new.target_id = get_target_id()
                    new.target_type = "OTHER"
                new.save()
            except Exception as e:
                print("FAIL to migrate operation: %s" % (repr(e)))
        cur.close()
        print("Operations migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    def make_operation_links():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM cpta_operation
        """
        )
        for r in cur:
            if r["id_op_liee"]:
                try:
                    op1 = Operation.objects.filter(id=r["id_op"]).first()
                    op2 = Operation.objects.filter(id=r["id_op_liee"]).first()
                    op1.linked_operation = op2
                    op1.save()
                    op2.linked_operation = op1
                    op2.save()
                except Exception as e:
                    print("FAIL to link operations: %s" % (repr(e)))
        cur.close()
        print("Operations links migrated at %s" % datetime.datetime.now())
        print("Running time: %s" % (datetime.datetime.now() - start))

    migrate_companies()
    migrate_accounting_types()
    migrate_simpleaccounting_types()
    migrate_bank_accounts()
    migrate_club_accounts()
    migrate_labels()
    migrate_journals()
    migrate_operations()
    make_operation_links()


def migrate_godfathers():
    cur = db.cursor(MySQLdb.cursors.SSDictCursor)
    cur.execute(
        """
    SELECT *
    FROM parrains
    """
    )
    for r in cur:
        try:
            father = User.objects.filter(id=r["id_utilisateur"]).first()
            child = User.objects.filter(id=r["id_utilisateur_fillot"]).first()
            father.godchildren.add(child)
            father.save()
        except Exception as e:
            print("FAIL to migrate godfathering: %s" % (repr(e)))
    cur.close()
    print("Godfathers migrated at %s" % datetime.datetime.now())
    print("Running time: %s" % (datetime.datetime.now() - start))


def migrate_etickets():
    FILE_ROOT = "/data/files/"
    cur = db.cursor(MySQLdb.cursors.SSDictCursor)
    cur.execute(
        """
    SELECT *
    FROM cpt_etickets
    """
    )
    Eticket.objects.all().delete()
    print("Etickets deleted")
    for r in cur:
        try:
            p = Product.objects.filter(id=r["id_produit"]).first()
            try:
                f = File(open(FILE_ROOT + "/" + str(r["banner"]) + ".1", "rb"))
            except:
                f = None
            e = Eticket(
                product=p, secret=to_unicode(r["secret"]), banner=f, event_title=p.name
            )
            e.save()
            e.secret = to_unicode(r["secret"])
            e.save()
        except Exception as e:
            print("FAIL to migrate eticket: %s" % (repr(e)))
    cur.close()
    print("Etickets migrated at %s" % datetime.datetime.now())
    print("Running time: %s" % (datetime.datetime.now() - start))


def migrate_sas():
    album_link = {}
    picture_link = {}
    FILE_ROOT = "/data/sas/"
    SithFile.objects.filter(id__gte=18892).delete()
    print("Album/Pictures deleted")
    reset_index("core", "sas")
    cur = db.cursor(MySQLdb.cursors.SSDictCursor)
    cur.execute(
        """
    SELECT *
    FROM sas_cat_photos
    """
    )
    root = User.objects.filter(username="root").first()
    for r in cur:
        try:
            a = Album(
                name=to_unicode(r["nom_catph"]),
                owner=root,
                is_moderated=True,
                parent=None,
            )
            a.save()
            album_link[str(r["id_catph"])] = a.id
        except Exception as e:
            print("FAIL to migrate Album: %s" % (repr(e)))
    print("Album moved, need to make the tree")
    cur.execute(
        """
    SELECT *
    FROM sas_cat_photos
    """
    )
    for r in cur:
        try:
            p = Album.objects.filter(id=album_link[str(r["id_catph_parent"])]).first()
            a = Album.objects.filter(id=album_link[str(r["id_catph"])]).first()
            a.parent = p
            a.save()
        except:
            pass
    print("Album migrated at %s" % datetime.datetime.now())
    print("Running time: %s" % (datetime.datetime.now() - start))
    with open("albums.link", "w") as f:
        f.write(str(album_link))
    cur.close()
    finished = False
    chunk = 0
    while not finished:
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        cur.execute(
            """
        SELECT *
        FROM sas_photos
        ORDER BY 'id_photo'
        LIMIT %s, 1000
        """,
            (chunk * 1000,),
        )
        has_result = False
        for r in cur:
            try:
                user = User.objects.filter(id=r["id_utilisateur"]).first() or root
                parent = Album.objects.filter(id=album_link[str(r["id_catph"])]).first()

                file_name = FILE_ROOT
                if r["date_prise_vue"]:
                    file_name += r["date_prise_vue"].strftime("%Y/%m/%d")
                else:
                    file_name += "/".join(["1970", "01", "01"])
                file_name += "/" + str(r["id_photo"]) + ".jpg"

                file = File(open(file_name, "rb"))
                file.name = str(r["id_photo"]) + ".jpg"

                p = Picture(
                    name=str(r["id_photo"]) + ".jpg",
                    owner=user,
                    is_moderated=True,
                    is_folder=False,
                    mime_type="image/jpeg",
                    parent=parent,
                    file=file,
                )
                if r["date_prise_vue"]:
                    p.date = r["date_prise_vue"].replace(
                        tzinfo=timezone("Europe/Paris")
                    )
                else:
                    p.date = r["date_ajout_ph"].replace(tzinfo=timezone("Europe/Paris"))
                for f in p._meta.local_fields:
                    if f.name == "date":
                        f.auto_now = False
                p.generate_thumbnails()
                p.save()
                db2 = MySQLdb.connect(**settings.OLD_MYSQL_INFOS)
                cur2 = db2.cursor(MySQLdb.cursors.SSDictCursor)
                cur2.execute(
                    """
                SELECT *
                FROM sas_personnes_photos
                WHERE id_photo = %s
                """,
                    (r["id_photo"],),
                )
                for r2 in cur2:
                    try:
                        u = User.objects.filter(id=r2["id_utilisateur"]).first()
                        if u:
                            PeoplePictureRelation(user=u, picture=p).save()
                    except:
                        print(
                            "Fail to associate user %d to picture %d"
                            % (r2["id_utilisateur"], p.id)
                        )
                has_result = True
            except Exception as e:
                print("FAIL to migrate Picture: %s" % (repr(e)))
        cur.close()
        print("Chunk %d migrated at %s" % (chunk, str(datetime.datetime.now())))
        print("Running time: %s" % (datetime.datetime.now() - start))
        chunk += 1
        finished = not has_result
    print("SAS migrated at %s" % datetime.datetime.now())
    print("Running time: %s" % (datetime.datetime.now() - start))

    # try:
    #     f = File(open(FILE_ROOT + '/' + str(r['banner']) + ".1", 'rb'))
    # except:
    #     f = None


def reset_sas_moderators():
    cur = db.cursor(MySQLdb.cursors.SSDictCursor)
    cur.execute(
        """
    SELECT *
    FROM sas_photos
    WHERE id_utilisateur_moderateur IS NOT NULL
    """
    )
    for r in cur:
        try:
            name = str(r["id_photo"]) + ".jpg"
            pict = SithFile.objects.filter(name__icontains=name, is_in_sas=True).first()
            user = User.objects.filter(id=r["id_utilisateur_moderateur"]).first()
            if pict and user:
                pict.moderator = user
                pict.save()
            else:
                print(
                    "No pict %s (%s) or user %s (%s)"
                    % (pict, name, user, r["id_utilisateur_moderateur"])
                )
        except Exception as e:
            print(repr(e))


def migrate_forum():
    print("Migrating forum")

    def migrate_forums():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        print("  Cleaning up forums")
        Forum.objects.all().delete()
        cur.execute(
            """
        SELECT *
        FROM frm_forum
        WHERE id_forum <> 1
        """
        )
        print("  Migrating forums")
        for r in cur:
            try:
                # parent = Forum.objects.filter(id=r['id_forum_parent']).first()
                club = Club.objects.filter(id=r["id_asso"]).first()
                ae = Club.objects.filter(id=settings.SITH_MAIN_CLUB_ID).first()
                forum = Forum(
                    id=r["id_forum"],
                    name=to_unicode(r["titre_forum"]),
                    description=to_unicode(r["description_forum"])[:511],
                    is_category=bool(r["categorie_forum"]),
                    # parent=parent,
                    owner_club=club or ae,
                    number=r["ordre_forum"],
                )
                forum.save()
            except Exception as e:
                print("  FAIL to migrate forum: %s" % (repr(e)))
        cur.execute(
            """
        SELECT *
        FROM frm_forum
        WHERE id_forum_parent <> 1
        """
        )
        for r in cur:
            parent = Forum.objects.filter(id=r["id_forum_parent"]).first()
            forum = Forum.objects.filter(id=r["id_forum"]).first()
            forum.parent = parent
            forum.save()
        cur.close()
        print("  Forums migrated at %s" % datetime.datetime.now())
        print("  Running time: %s" % (datetime.datetime.now() - start))

    def migrate_topics():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        print("  Cleaning up topics")
        ForumTopic.objects.all().delete()
        cur.execute(
            """
        SELECT *
        FROM frm_sujet
        """
        )
        print("  Migrating topics")
        for r in cur:
            try:
                parent = Forum.objects.filter(id=r["id_forum"]).first()
                saloon = Forum.objects.filter(id=3).first()
                author = User.objects.filter(id=r["id_utilisateur"]).first()
                root = User.objects.filter(id=0).first()
                topic = ForumTopic(
                    id=r["id_sujet"],
                    author=author or root,
                    forum=parent or saloon,
                    _title=to_unicode(r["titre_sujet"])[:64],
                    description=to_unicode(r["soustitre_sujet"]),
                )
                topic.save()
            except Exception as e:
                print("  FAIL to migrate topic: %s" % (repr(e)))
        cur.close()
        print("  Topics migrated at %s" % datetime.datetime.now())
        print("  Running time: %s" % (datetime.datetime.now() - start))

    def migrate_messages():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        print("  Cleaning up messages")
        ForumMessage.objects.all().delete()
        cur.execute(
            """
        SELECT *
        FROM frm_message
        """
        )
        print("  Migrating messages")
        for r in cur:
            try:
                topic = ForumTopic.objects.filter(id=r["id_sujet"]).first()
                author = User.objects.filter(id=r["id_utilisateur"]).first()
                root = User.objects.filter(id=0).first()
                msg = ForumMessage(
                    id=r["id_message"],
                    topic=topic,
                    author=author or root,
                    title=to_unicode(r["titre_message"])[:63],
                    date=r["date_message"].replace(tzinfo=timezone("Europe/Paris")),
                )
                try:
                    if r["syntaxengine_message"] == "doku":
                        msg.message = doku_to_markdown(to_unicode(r["contenu_message"]))
                    else:
                        msg.message = bbcode_to_markdown(
                            to_unicode(r["contenu_message"])
                        )
                except:
                    msg.message = to_unicode(r["contenu_message"])
                msg.save()
            except Exception as e:
                print("  FAIL to migrate message: %s" % (repr(e)))
        cur.close()
        print("  Messages migrated at %s" % datetime.datetime.now())
        print("  Running time: %s" % (datetime.datetime.now() - start))

    def migrate_message_infos():
        cur = db.cursor(MySQLdb.cursors.SSDictCursor)
        print("  Cleaning up message meta")
        ForumMessageMeta.objects.all().delete()
        cur.execute(
            """
        SELECT *
        FROM frm_modere_info
        """
        )
        print("  Migrating message meta")
        ACTIONS = {
            "EDIT": "EDIT",
            "AUTOEDIT": "EDIT",
            "UNDELETE": "UNDELETE",
            "DELETE": "DELETE",
            "DELETEFIRST": "DELETE",
            "AUTODELETE": "DELETE",
        }
        for r in cur:
            try:
                msg = ForumMessage.objects.filter(id=r["id_message"]).first()
                author = User.objects.filter(id=r["id_utilisateur"]).first()
                root = User.objects.filter(id=0).first()
                meta = ForumMessageMeta(
                    message=msg,
                    user=author or root,
                    date=r["modere_date"].replace(tzinfo=timezone("Europe/Paris")),
                    action=ACTIONS[r["modere_action"]],
                )
                meta.save()
            except Exception as e:
                print("  FAIL to migrate message meta: %s" % (repr(e)))
        cur.close()
        print("  Messages meta migrated at %s" % datetime.datetime.now())
        print("  Running time: %s" % (datetime.datetime.now() - start))

    migrate_forums()
    migrate_topics()
    migrate_messages()
    migrate_message_infos()
    print("Forum migrated at %s" % datetime.datetime.now())
    print("Running time: %s" % (datetime.datetime.now() - start))


def migrate_mailings():
    cur = db.cursor(MySQLdb.cursors.SSDictCursor)

    print("Delete all mailings")

    Mailing.objects.all().delete()

    print("Migrating old mailing database")

    cur.execute(
        """
        SELECT * FROM mailing
    """
    )

    moderator = User.objects.get(id=0)

    for mailing in cur:
        club = Club.objects.filter(id=mailing["id_asso_parent"])
        if club.exists():
            print(mailing)
            club = club.first()
            if mailing["nom"]:
                mailing["nom"] = "." + mailing["nom"]
            Mailing(
                id=mailing["id_mailing"],
                club=club,
                email=to_unicode(club.unix_name + mailing["nom"]),
                moderator=moderator,
                is_moderated=(mailing["is_valid"] > 0),
            ).save()
    print("-------------------")

    cur.execute(
        """
        SELECT * FROM mailing_membres
    """
    )

    for mailing_sub in cur:
        mailing = Mailing.objects.filter(id=mailing_sub["id_mailing"])
        if mailing.exists():
            print(mailing_sub)
            mailing = mailing.first()
            if (
                mailing_sub["id_user"]
                and User.objects.filter(id=mailing_sub["id_user"]).exists()
            ):
                user = User.objects.get(id=mailing_sub["id_user"])
                MailingSubscription(mailing=mailing, user=user, email=user.email).save()
            elif mailing_sub["email"]:
                MailingSubscription(
                    mailing=mailing, email=to_unicode(mailing_sub["email"])
                ).save()


def migrate_club_again():
    cur = db.cursor(MySQLdb.cursors.SSDictCursor)
    cur.execute("SELECT * FROM asso")

    print("Migrating club is_active")

    for club in cur:
        try:
            c = Club.objects.get(unix_name=club["nom_unix_asso"])
            c.is_active = club["hidden"] == 0
            c.save()
        except:
            pass


def migrate_pedagogy():
    cur = db.cursor(MySQLdb.cursors.SSDictCursor)

    print("Migrating UVs")
    root = User.objects.get(id=0)
    semester_conversion = {
        "closed": "CLOSED",
        "A": "AUTUMN",
        "P": "SPRING",
        "AP": "AUTUMN_AND_SPRING",
    }

    def department_conversion(department):
        # Default of this enum is HUMA
        if not department or department == "Humas":
            return "HUMA"
        return department

    def convert_number(num, default=0):
        if not num:
            return default
        return num

    def convert_text(text):
        if not text:
            return ""
        return doku_to_markdown(to_unicode(text))

    cur.execute(
        """
    SELECT * FROM pedag_uv
    LEFT JOIN pedag_uv_dept dept
    ON dept.id_uv = pedag_uv.id_uv
    """
    )
    for uv in cur:
        UV(
            id=uv["id_uv"],
            code=uv["code"],
            author=root,
            credit_type=uv["type"],
            semester=semester_conversion[uv["semestre"]],
            language="FR",  # No infos in previous guide about that
            credits=convert_number(uv["guide_credits"]),
            department=department_conversion(uv["departement"]),
            title=convert_text(uv["intitule"]),
            manager=convert_text(uv["responsable"]),
            objectives=convert_text(uv["guide_objectifs"]),
            program=convert_text(uv["guide_programme"]),
            skills="",  # No info in previous guide about that
            key_concepts="",  # No info either
            hours_CM=convert_number(uv["guide_c"]),
            hours_TD=convert_number(uv["guide_td"]),
            hours_TP=convert_number(uv["guide_tp"]),
            hours_THE=convert_number(uv["guide_the"]),
            hours_TE=0,  # No info either
        ).save()

    print("Migrating UV Comments")

    cur.execute("SELECT * FROM pedag_uv_commentaire")

    for comment in cur:
        author = User.objects.filter(id=comment["id_utilisateur"]).first()
        uv = UV.objects.filter(id=comment["id_uv"]).first()
        if not author or not uv:
            continue
        UVComment(
            id=comment["id_commentaire"],
            author=author,
            uv=uv,
            comment=convert_text(comment["content"]),
            grade_global=convert_number(comment["note_generale"], -1),
            grade_utility=convert_number(comment["note_utilite"], -1),
            grade_interest=convert_number(comment["note_interet"], -1),
            grade_teaching=convert_number(comment["note_enseignement"], -1),
            grade_work_load=convert_number(comment["note_travail"], -1),
            publish_date=comment["date"].replace(tzinfo=timezone("Europe/Paris")),
        ).save()

    print("Migrating UV Results")
    cur.execute("SELECT * FROM pedag_resultat")

    for result in cur:
        author = User.objects.filter(id=comment["id_utilisateur"]).first()
        uv = UV.objects.filter(id=comment["id_uv"]).first()
        if not author or not uv:
            continue
        UVResult(
            id=result["id_resultat"],
            uv=uv,
            user=author,
            grade=result["note"],
            semester=result["semestre"],
        ).save()


def main():
    print("Start at %s" % start)
    # Core
    # migrate_core()
    # Club
    # migrate_club()
    # Subscriptions
    # migrate_subscriptions()
    # Counters
    # migrate_counter()
    # check_accounts()
    # Accounting
    # migrate_accounting()
    # migrate_godfathers()
    # migrate_etickets()
    # reset_index('core', 'club', 'subscription', 'accounting', 'eboutic', 'launderette', 'counter')
    # migrate_sas()
    # reset_index('core', 'sas')
    # reset_sas_moderators()
    # migrate_forum()
    # reset_index('forum')
    # migrate_mailings()
    # migrate_club_again()
    migrate_pedagogy()
    reset_index("pedagogy")
    end = datetime.datetime.now()
    print("End at %s" % end)
    print("Running time: %s" % (end - start))


if __name__ == "__main__":
    main()