adapt pedagogy to new UTBM UE API

pedagogy/management/commands/update_ue_guide.py

@@ -18,7 +18,12 @@ class Command(BaseCommand):
                 "Fetching UEs from the UTBM API.\n"
                 "This may take a few minutes to complete."
             )
+            i = 0
             for ue in client.fetch_ues():
+                if (i := (i + 1)) % 10 == 0:
+                    self.stdout.write(f"{i} UEs processed")
+                # this is a N+1 query, but it isn't a problem,
+                # as fetching a UE from the UTBM API is taking most of the time anyway
                 db_ue = UE.objects.filter(code=ue.code).first()
                 if db_ue is None:
                     db_ue = UE(code=ue.code, author=root_user)

pedagogy/schemas.py

@@ -5,12 +5,22 @@ from django.urls import reverse
 from django.utils import html
 from haystack.query import SearchQuerySet
 from ninja import FilterLookup, FilterSchema, ModelSchema, Schema
-from pydantic import AliasPath, ConfigDict, Field, TypeAdapter
+from pydantic import AliasPath, BeforeValidator, ConfigDict, Field, TypeAdapter
 from pydantic.alias_generators import to_camel
 
 from pedagogy.models import UE
 
 
+class FormationSchema(Schema):
+    model_config = ConfigDict(validate_by_name=True, validate_by_alias=True)
+
+    code: str
+    label: str = Field(alias="libelle")
+
+
+FormationListSchema = TypeAdapter(list[FormationSchema])
+
+
 class UtbmShortUeSchema(Schema):
     """Short representation of an UE in the UTBM API.
 
@@ -19,19 +29,17 @@ class UtbmShortUeSchema(Schema):
         The UTBM API returns more data than that.
     """
 
-    model_config = ConfigDict(alias_generator=to_camel)
+    model_config = ConfigDict(validate_by_name=True)
 
     code: str
-    code_formation: str
+    category: str = Field(alias="codeCategorie")
-    code_categorie: str | None
+    formation: FormationSchema
-    code_langue: str
+    lang: str = Field(alias="codeLangue")
-    ouvert_automne: bool
+    open_autumn: bool = Field(alias="ouvertAutomne")
-    ouvert_printemps: bool
+    open_spring: bool = Field(alias="ouvertPrintemps")
 
 
 class WorkloadSchema(Schema):
-    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
-
     code: Literal["TD", "TP", "CM", "THE", "TE"]
     nbh: int
 
@@ -48,22 +56,30 @@ class SemesterUeState(Schema):
 ShortUeList = TypeAdapter(list[UtbmShortUeSchema])
 
 
+class SyllabusItemSchema(Schema):
+    model_config = ConfigDict(validate_by_name=True)
+
+    label: str = Field(alias="libelle")
+    # We want only strings, but the UTBM API can return null,
+    # so convert null values to empty strings
+    value: Annotated[str, BeforeValidator(lambda x: x or "")] = Field(alias="valeur")
+
+
 class UtbmFullUeSchema(Schema):
     """Long representation of an UE in the UTBM API."""
 
-    model_config = ConfigDict(alias_generator=to_camel)
+    model_config = ConfigDict(validate_by_name=True, alias_generator=to_camel)
 
     code: str
     departement: str = "NA"
-    libelle: str | None
+    label: str = Field(alias="libelle")
-    objectifs: str | None
+    syllabus: list[SyllabusItemSchema] = Field(
-    programme: str | None
+        default=[],
-    acquisition_competences: str | None
+        validation_alias=AliasPath("listeElementConstitutif", 0, "listeSaisieSyllabus"),
-    acquisition_notions: str | None
+    )
-    langue: str
+    lang: str = Field(None, validation_alias=AliasPath("langueEnseignement", "code"))
-    code_langue: str
     credits_ects: int
-    activites: list[WorkloadSchema]
+    activites: list[WorkloadSchema] = Field(alias="listeActivite")
     respo_automne: str | None = Field(
         None, validation_alias=AliasPath("automne", "responsable")
     )

pedagogy/utbm_api.py

@@ -5,8 +5,16 @@ from typing import Iterator
 import requests
 from django.conf import settings
 from django.utils.functional import cached_property
+from pydantic import TypeAdapter
 
-from pedagogy.schemas import ShortUeList, UeSchema, UtbmFullUeSchema, UtbmShortUeSchema
+from pedagogy.schemas import (
+    FormationSchema,
+    UeSchema,
+    UtbmFullUeSchema,
+    UtbmShortUeSchema,
+)
+
+FormationListSchema = TypeAdapter(list[FormationSchema])
 
 
 class UtbmApiClient(requests.Session):
@@ -18,28 +26,44 @@ class UtbmApiClient(requests.Session):
     @cached_property
     def current_year(self) -> int:
         """Fetch from the API the latest existing year"""
-        url = f"{self.BASE_URL}/guides/fr"
+        url = f"{self.BASE_URL}/guide/"
-        response = self.get(url)
+        response = self.get(url, {"langue": "fr"})
-        return response.json()[-1]["annee"]
+        return max(i["annee"] for i in response.json())
 
-    def fetch_short_ues(
+    @cached_property
-        self, lang: str = "fr", year: int | None = None
+    def formations(self) -> list[FormationSchema]:
-    ) -> list[UtbmShortUeSchema]:
+        response = self.get(
+            f"{self.BASE_URL}/formation/",
+            {"langue": "fr", "annee_univ": self.current_year, "typeFormation": "ING"},
+        )
+        return FormationListSchema.validate_json(response.text, by_alias=True)
+
+    def fetch_short_ues(self, year: int | None = None) -> list[UtbmShortUeSchema]:
         """Get the list of UEs in their short format from the UTBM API"""
         if year is None:
             year = self.current_year
-        if lang not in self._cache["short_ues"]:
+        if year not in self._cache["short_ues"]:
-            self._cache["short_ues"][lang] = {}
+            ues = []
-        if year not in self._cache["short_ues"][lang]:
+            for formation in self.formations:
-            url = f"{self.BASE_URL}/uvs/{lang}/{year}"
+                response = self.get(
-            response = self.get(url)
+                    f"{self.BASE_URL}/ue/",
-            ues = ShortUeList.validate_json(response.content)
+                    {
-            self._cache["short_ues"][lang][year] = ues
+                        "langue": "fr",
-        return self._cache["short_ues"][lang][year]
+                        "annee_univ": year,
+                        "codeFormation": formation.code,
+                    },
+                )
+                ues.extend(
+                    [
+                        UtbmShortUeSchema.model_validate({**ue, "formation": formation})
+                        for ue in response.json()
+                        if ue["codeCategorie"] is not None
+                    ]
+                )
+            self._cache["short_ues"][year] = ues
+        return self._cache["short_ues"][year]
 
-    def fetch_ues(
+    def fetch_ues(self, year: int | None = None) -> Iterator[UeSchema]:
-        self, lang: str = "fr", year: int | None = None
-    ) -> Iterator[UeSchema]:
         """Fetch all UEs from the UTBM API, parsed in a format that we can use.
 
         Warning:
@@ -52,7 +76,7 @@ class UtbmApiClient(requests.Session):
         """
         if year is None:
             year = self.current_year
-        shorts_ues = self.fetch_short_ues(lang, year)
+        shorts_ues = self.fetch_short_ues(year)
         # When UEs are common to multiple branches (like most HUMA)
         # the UTBM API duplicates them for every branch.
         # We have no way in our db to link a UE to multiple formations,
@@ -65,27 +89,40 @@ class UtbmApiClient(requests.Session):
             if ue.code not in unique_short_ues:
                 unique_short_ues[ue.code] = ue
         for ue in unique_short_ues.values():
-            ue_url = f"{self.BASE_URL}/uv/{lang}/{year}/{ue.code}/{ue.code_formation}"
+            if ue.lang.upper() != "FR":
-            response = requests.get(ue_url)
+                continue
+            response = requests.get(
+                f"{self.BASE_URL}/ue/{ue.code}",
+                {
+                    "langue": ue.lang,
+                    "annee_univ": year,
+                    "codeFormation": ue.formation.code,
+                },
+            )
             full_ue = UtbmFullUeSchema.model_validate_json(response.content)
             yield make_clean_ue(ue, full_ue)
 
-    def find_uu(self, lang: str, code: str, year: int | None = None) -> UeSchema | None:
+    def find_ue(self, code: str, year: int | None = None) -> UeSchema | None:
-        """Find an UE from the UTBM API."""
+        """Find a UE from the UTBM API."""
-        # query the UE list
         if not year:
             year = self.current_year
         # the UTBM API has no way to fetch a single short ue,
         # and short ues contain infos that we need and are not
         # in the full ue schema, so we must fetch everything.
-        short_ues = self.fetch_short_ues(lang, year)
+        short_ues = self.fetch_short_ues(year)
         short_ue = next((ue for ue in short_ues if ue.code == code), None)
         if short_ue is None:
             return None
 
         # get detailed information about the UE
-        ue_url = f"{self.BASE_URL}/uv/{lang}/{year}/{code}/{short_ue.code_formation}"
+        response = requests.get(
-        response = requests.get(ue_url)
+            f"{self.BASE_URL}/ue/{code}",
+            {
+                "langue": "fr",
+                "annee_univ": year,
+                "codeFormation": short_ue.formation.code,
+            },
+        )
         full_ue = UtbmFullUeSchema.model_validate_json(response.content)
         return make_clean_ue(short_ue, full_ue)
 
@@ -112,9 +149,9 @@ def make_clean_ue(short_ue: UtbmShortUeSchema, full_ue: UtbmFullUeSchema) -> UeS
             "ED": "EDIM",
             "AI": "GI",
             "AM": "MC",
-        }.get(short_ue.code_formation, "NA")
+        }.get(short_ue.formation.code, "NA")
 
-    match short_ue.ouvert_printemps, short_ue.ouvert_automne:
+    match short_ue.open_spring, short_ue.open_autumn:
         case True, True:
             semester = "AUTUMN_AND_SPRING"
         case True, False:
@@ -125,11 +162,11 @@ def make_clean_ue(short_ue: UtbmShortUeSchema, full_ue: UtbmFullUeSchema) -> UeS
             semester = "CLOSED"
 
     return UeSchema(
-        title=full_ue.libelle or "",
+        title=full_ue.label or "",
         code=full_ue.code,
-        credit_type=short_ue.code_categorie or "FREE",
+        credit_type=short_ue.category or "FREE",
         semester=semester,
-        language=short_ue.code_langue.upper(),
+        language=short_ue.lang.upper(),
         credits=full_ue.credits_ects,
         department=department,
         hours_THE=next((i.nbh for i in full_ue.activites if i.code == "THE"), 0) // 60,
@@ -138,8 +175,16 @@ def make_clean_ue(short_ue: UtbmShortUeSchema, full_ue: UtbmFullUeSchema) -> UeS
         hours_TE=next((i.nbh for i in full_ue.activites if i.code == "TE"), 0) // 60,
         hours_CM=next((i.nbh for i in full_ue.activites if i.code == "CM"), 0) // 60,
         manager=full_ue.respo_automne or full_ue.respo_printemps or "",
-        objectives=full_ue.objectifs or "",
+        objectives=next(
-        program=full_ue.programme or "",
+            (i.value for i in full_ue.syllabus if i.label == "Objectifs"), ""
-        skills=full_ue.acquisition_competences or "",
+        ),
-        key_concepts=full_ue.acquisition_notions or "",
+        program=next((i.value for i in full_ue.syllabus if i.label == "Programme"), ""),
+        skills=next(
+            (i.value for i in full_ue.syllabus if i.label == "Acquis d'apprentissage"),
+            "",
+        ),
+        key_concepts=next(
+            (i.value for i in full_ue.syllabus if i.label == "Notions-clés"),
+            "",
+        ),
     )

sith/settings.py

@@ -493,7 +493,7 @@ SITH_LOG_OPERATION_TYPE = [
     ("REFILLING_DELETION", _("Refilling deletion")),
 ]
 
-SITH_PEDAGOGY_UTBM_API = "https://extranet1.utbm.fr/gpedago/api/guide"
+SITH_PEDAGOGY_UTBM_API = "https://api.utbm.fr/offre-formation/public"
 
 # Defines pagination for cash summary
 SITH_COUNTER_CASH_SUMMARY_LENGTH = 50