From 997641d5141d8c1ba226f5d76417f4397ac298d3 Mon Sep 17 00:00:00 2001 From: imperosol Date: Tue, 21 Jan 2025 15:40:12 +0100 Subject: [PATCH] management command to update the whole uv guide --- pedagogy/management/__init__.py | 0 pedagogy/management/commands/__init__.py | 0 .../management/commands/update_uv_guide.py | 37 ++++++++++++++ pedagogy/schemas.py | 10 ++-- pedagogy/utbm_api.py | 51 +++++++++++++++---- 5 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 pedagogy/management/__init__.py create mode 100644 pedagogy/management/commands/__init__.py create mode 100644 pedagogy/management/commands/update_uv_guide.py diff --git a/pedagogy/management/__init__.py b/pedagogy/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pedagogy/management/commands/__init__.py b/pedagogy/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pedagogy/management/commands/update_uv_guide.py b/pedagogy/management/commands/update_uv_guide.py new file mode 100644 index 00000000..cf525a1f --- /dev/null +++ b/pedagogy/management/commands/update_uv_guide.py @@ -0,0 +1,37 @@ +from django.conf import settings +from django.core.management import BaseCommand + +from core.models import User +from pedagogy.models import UV +from pedagogy.schemas import UvSchema +from pedagogy.utbm_api import UtbmApiClient + + +class Command(BaseCommand): + help = "Update the UV guide" + + def handle(self, *args, **options): + seen_uvs: set[int] = set() + root_user = User.objects.get(pk=settings.SITH_ROOT_USER_ID) + with UtbmApiClient() as client: + self.stdout.write( + "Fetching UVs from the UTBM API.\n" + "This may take a few minutes to complete." + ) + for uv in client.fetch_uvs(): + db_uv = UV.objects.filter(code=uv.code).first() + if db_uv is None: + db_uv = UV(code=uv.code, author=root_user) + fields = list(UvSchema.model_fields.keys()) + fields.remove("id") + fields.remove("code") + for field in fields: + setattr(db_uv, field, getattr(uv, field)) + db_uv.save() + # if it's a creation, django will set the id when saving, + # so at this point, a db_uv will always have an id + seen_uvs.add(db_uv.id) + # UVs that are in database but have not been returned by the API + # are considered as closed UEs + UV.objects.exclude(id__in=seen_uvs).update(semester="CLOSED") + self.stdout.write(self.style.SUCCESS("UV guide updated successfully")) diff --git a/pedagogy/schemas.py b/pedagogy/schemas.py index 716e9a3c..cbcd9157 100644 --- a/pedagogy/schemas.py +++ b/pedagogy/schemas.py @@ -54,11 +54,11 @@ class UtbmFullUvSchema(Schema): code: str departement: str = "NA" - libelle: str - objectifs: str - programme: str - acquisition_competences: str - acquisition_notions: str + libelle: str | None + objectifs: str | None + programme: str | None + acquisition_competences: str | None + acquisition_notions: str | None langue: str code_langue: str credits_ects: int diff --git a/pedagogy/utbm_api.py b/pedagogy/utbm_api.py index f0d1377d..dd7ec933 100644 --- a/pedagogy/utbm_api.py +++ b/pedagogy/utbm_api.py @@ -1,5 +1,7 @@ """Set of functions to interact with the UTBM UV api.""" +from typing import Iterator + import requests from django.conf import settings from django.utils.functional import cached_property @@ -11,7 +13,7 @@ class UtbmApiClient(requests.Session): """A wrapper around `requests.Session` to perform requests to the UTBM UV API.""" BASE_URL = settings.SITH_PEDAGOGY_UTBM_API - _cache = {} + _cache = {"short_uvs": {}} @cached_property def current_year(self) -> int: @@ -26,8 +28,6 @@ class UtbmApiClient(requests.Session): """Get the list of UVs in their short format from the UTBM API""" if year is None: year = self.current_year - if "short_uvs" not in self._cache: - self._cache["short_uvs"] = {} if lang not in self._cache["short_uvs"]: self._cache["short_uvs"][lang] = {} if year not in self._cache["short_uvs"][lang]: @@ -37,6 +37,39 @@ class UtbmApiClient(requests.Session): self._cache["short_uvs"][lang][year] = uvs return self._cache["short_uvs"][lang][year] + def fetch_uvs( + self, lang: str = "fr", year: int | None = None + ) -> Iterator[UvSchema]: + """Fetch all UVs from the UTBM API, parsed in a format that we can use. + + Warning: + We need infos from the full uv schema, and the UTBM UV API + has no route to get all of them at once. + We must do one request per UV (for a total of around 730 UVs), + which takes a lot of time. + Hopefully, there seems to be no rate-limit, so an error + in the middle of the process isn't likely to occur. + """ + if year is None: + year = self.current_year + shorts_uvs = self.fetch_short_uvs(lang, year) + # When UVs are common to multiple branches (like most HUMA) + # the UTBM API duplicates them for every branch. + # We have no way in our db to link a UV to multiple formations, + # so we just create a single UV, which formation is the one + # of the first UV found in the list. + # For example, if we have CC01 (TC), CC01 (IMSI) and CC01 (EDIM), + # we will only keep CC01 (TC). + unique_short_uvs = {} + for uv in shorts_uvs: + if uv.code not in unique_short_uvs: + unique_short_uvs[uv.code] = uv + for uv in unique_short_uvs.values(): + uv_url = f"{self.BASE_URL}/uv/{lang}/{year}/{uv.code}/{uv.code_formation}" + response = requests.get(uv_url) + full_uv = UtbmFullUvSchema.model_validate_json(response.content) + yield make_clean_uv(uv, full_uv) + def find_uv(self, lang: str, code: str, year: int | None = None) -> UvSchema | None: """Find an UV from the UTBM API.""" # query the UV list @@ -92,9 +125,9 @@ def make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> UvS semester = "CLOSED" return UvSchema( - title=full_uv.libelle, + title=full_uv.libelle or "", code=full_uv.code, - credit_type=short_uv.code_categorie, + credit_type=short_uv.code_categorie or "FREE", semester=semester, language=short_uv.code_langue.upper(), credits=full_uv.credits_ects, @@ -105,8 +138,8 @@ def make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> UvS hours_TE=next((i.nbh for i in full_uv.activites if i.code == "TE"), 0) // 60, hours_CM=next((i.nbh for i in full_uv.activites if i.code == "CM"), 0) // 60, manager=full_uv.respo_automne or full_uv.respo_printemps or "", - objectives=full_uv.objectifs, - program=full_uv.programme, - skills=full_uv.acquisition_competences, - key_concepts=full_uv.acquisition_notions, + objectives=full_uv.objectifs or "", + program=full_uv.programme or "", + skills=full_uv.acquisition_competences or "", + key_concepts=full_uv.acquisition_notions or "", )