Merge pull request #1010 from ae-utbm/populate-all-uvs

Management command to populate all uvs

com/calendar.py

@@ -2,7 +2,7 @@ from datetime import datetime, timedelta
 from pathlib import Path
 from typing import final
 
-import urllib3
+import requests
 from dateutil.relativedelta import relativedelta
 from django.conf import settings
 from django.urls import reverse
@@ -35,16 +35,15 @@ class IcsCalendar:
 
     @classmethod
     def make_external(cls) -> Path | None:
-        calendar = urllib3.request(
-            "GET",
-            "https://calendar.google.com/calendar/ical/ae.utbm%40gmail.com/public/basic.ics",
+        calendar = requests.get(
+            "https://calendar.google.com/calendar/ical/ae.utbm%40gmail.com/public/basic.ics"
         )
-        if calendar.status != 200:
+        if not calendar.ok:
             return None
 
         cls._CACHE_FOLDER.mkdir(parents=True, exist_ok=True)
         with open(cls._EXTERNAL_CALENDAR, "wb") as f:
-            _ = f.write(calendar.data)
+            _ = f.write(calendar.content)
         return cls._EXTERNAL_CALENDAR
 
     @classmethod

com/tests/test_api.py

@@ -16,11 +16,11 @@ from com.calendar import IcsCalendar
 
 @dataclass
 class MockResponse:
-    status: int
+    ok: bool
     value: str
 
     @property
-    def data(self):
+    def content(self):
         return self.value.encode("utf8")
 
 
@@ -38,7 +38,7 @@ class TestExternalCalendar:
     @pytest.fixture
     def mock_request(self):
         mock = MagicMock()
-        with patch("urllib3.request", mock):
+        with patch("requests.get", mock):
             yield mock
 
     @pytest.fixture
@@ -52,15 +52,12 @@ class TestExternalCalendar:
     def clear_cache(self):
         IcsCalendar._EXTERNAL_CALENDAR.unlink(missing_ok=True)
 
-    @pytest.mark.parametrize("error_code", [403, 404, 500])
-    def test_fetch_error(
-        self, client: Client, mock_request: MagicMock, error_code: int
-    ):
-        mock_request.return_value = MockResponse(error_code, "not allowed")
+    def test_fetch_error(self, client: Client, mock_request: MagicMock):
+        mock_request.return_value = MockResponse(ok=False, value="not allowed")
         assert client.get(reverse("api:calendar_external")).status_code == 404
 
     def test_fetch_success(self, client: Client, mock_request: MagicMock):
-        external_response = MockResponse(200, "Definitely an ICS")
+        external_response = MockResponse(ok=True, value="Definitely an ICS")
         mock_request.return_value = external_response
         response = client.get(reverse("api:calendar_external"))
         assert response.status_code == 200

locale/fr/LC_MESSAGES/django.po

@@ -935,10 +935,6 @@ msgstr "rôle"
 msgid "description"
 msgstr "description"
 
-#: club/models.py
-msgid "past member"
-msgstr "ancien membre"
-
 #: club/models.py
 msgid "Email address"
 msgstr "Adresse email"
@@ -3308,8 +3304,8 @@ msgstr "Nom d'utilisateur, email, ou numéro de compte AE"
 
 #: core/views/forms.py
 msgid ""
-"Profile: you need to be visible on the picture, in order to be recognized "
-"(e.g. by the barmen)"
+"Profile: you need to be visible on the picture, in order to be recognized (e."
+"g. by the barmen)"
 msgstr ""
 "Photo de profil: vous devez être visible sur la photo afin d'être reconnu "
 "(par exemple par les barmen)"
@@ -3919,8 +3915,8 @@ msgstr ""
 #: counter/templates/counter/mails/account_dump.jinja
 msgid "If you think this was a mistake, please mail us at ae@utbm.fr."
 msgstr ""
-"Si vous pensez qu'il s'agit d'une erreur, veuillez envoyer un mail à "
-"ae@utbm.fr."
+"Si vous pensez qu'il s'agit d'une erreur, veuillez envoyer un mail à ae@utbm."
+"fr."
 
 #: counter/templates/counter/mails/account_dump.jinja
 msgid ""
@@ -4948,6 +4944,10 @@ msgstr "Département"
 msgid "Credit type"
 msgstr "Type de crédit"
 
+#: pedagogy/templates/pedagogy/guide.jinja
+msgid "closed uv"
+msgstr "uv fermée"
+
 #: pedagogy/templates/pedagogy/macros.jinja
 msgid " not rated "
 msgstr "non noté"
@@ -5990,3 +5990,6 @@ msgstr "Vous ne pouvez plus écrire de commentaires, la date est passée."
 #, python-format
 msgid "Maximum characters: %(max_length)s"
 msgstr "Nombre de caractères max: %(max_length)s"
+
+#~ msgid "past member"
+#~ msgstr "ancien membre"

pedagogy/api.py

@@ -10,13 +10,13 @@ from ninja_extra.pagination import PageNumberPaginationExtra, PaginatedResponseS
 from core.auth.api_permissions import HasPerm
 from pedagogy.models import UV
 from pedagogy.schemas import SimpleUvSchema, UvFilterSchema, UvSchema
-from pedagogy.utbm_api import find_uv
+from pedagogy.utbm_api import UtbmApiClient
 
 
 @api_controller("/uv")
 class UvController(ControllerBase):
     @route.get(
-        "/{year}/{code}",
+        "/{code}",
         permissions=[
             # this route will almost always be called in the context
             # of a UV creation/edition
@@ -26,10 +26,14 @@ class UvController(ControllerBase):
         response=UvSchema,
     )
     def fetch_from_utbm_api(
-        self, year: Annotated[int, Ge(2010)], code: str, lang: Query[str] = "fr"
+        self,
+        code: str,
+        lang: Query[str] = "fr",
+        year: Query[Annotated[int, Ge(2010)] | None] = None,
     ):
         """Fetch UV data from the UTBM API and returns it after some parsing."""
-        res = find_uv(lang, year, code)
+        with UtbmApiClient() as client:
+            res = client.find_uv(lang, code, year)
         if res is None:
             raise NotFound
         return res
@@ -42,4 +46,4 @@ class UvController(ControllerBase):
     )
     @paginate(PageNumberPaginationExtra, page_size=100)
     def fetch_uv_list(self, search: Query[UvFilterSchema]):
-        return search.filter(UV.objects.values())
+        return search.filter(UV.objects.order_by("code").values())

pedagogy/management/commands/update_uv_guide.py

@@ -0,0 +1,37 @@
+from django.conf import settings
+from django.core.management import BaseCommand
+
+from core.models import User
+from pedagogy.models import UV
+from pedagogy.schemas import UvSchema
+from pedagogy.utbm_api import UtbmApiClient
+
+
+class Command(BaseCommand):
+    help = "Update the UV guide"
+
+    def handle(self, *args, **options):
+        seen_uvs: set[int] = set()
+        root_user = User.objects.get(pk=settings.SITH_ROOT_USER_ID)
+        with UtbmApiClient() as client:
+            self.stdout.write(
+                "Fetching UVs from the UTBM API.\n"
+                "This may take a few minutes to complete."
+            )
+            for uv in client.fetch_uvs():
+                db_uv = UV.objects.filter(code=uv.code).first()
+                if db_uv is None:
+                    db_uv = UV(code=uv.code, author=root_user)
+                fields = list(UvSchema.model_fields.keys())
+                fields.remove("id")
+                fields.remove("code")
+                for field in fields:
+                    setattr(db_uv, field, getattr(uv, field))
+                db_uv.save()
+                # if it's a creation, django will set the id when saving,
+                # so at this point, a db_uv will always have an id
+                seen_uvs.add(db_uv.id)
+        # UVs that are in database but have not been returned by the API
+        # are considered as closed UEs
+        UV.objects.exclude(id__in=seen_uvs).update(semester="CLOSED")
+        self.stdout.write(self.style.SUCCESS("UV guide updated successfully"))

pedagogy/schemas.py

@@ -54,11 +54,11 @@ class UtbmFullUvSchema(Schema):
 
     code: str
     departement: str = "NA"
-    libelle: str
-    objectifs: str
-    programme: str
-    acquisition_competences: str
-    acquisition_notions: str
+    libelle: str | None
+    objectifs: str | None
+    programme: str | None
+    acquisition_competences: str | None
+    acquisition_notions: str | None
     langue: str
     code_langue: str
     credits_ects: int

pedagogy/static/pedagogy/css/pedagogy.scss

@@ -47,11 +47,14 @@ $large-devices: 992px;
     }
   }
 
-  #dynamic_view {
+  #uv-list {
     font-size: 1.1em;
 
     overflow-wrap: break-word;
-
+    .closed td.title {
+      color: lighten($black-color, 10%);
+      font-style: italic;
+    }
     td {
       text-align: center;
       border: none;

pedagogy/templates/pedagogy/guide.jinja

@@ -85,7 +85,7 @@
         </div>
       </div>
     </form>
-    <table id="dynamic_view">
+    <table id="uv-list">
       <thead>
         <tr>
           <td>{% trans %}UV{% endtrans %}</td>
@@ -102,11 +102,17 @@
           {% endif %}
         </tr>
       </thead>
-      <tbody id="dynamic_view_content" :aria-busy="loading">
+      <tbody :aria-busy="loading">
         <template x-for="uv in uvs.results" :key="uv.id">
-          <tr @click="window.location.href = `/pedagogy/uv/${uv.id}`" class="clickable">
+          <tr
+            @click="window.location.href = `/pedagogy/uv/${uv.id}`"
+            class="clickable"
+            :class="{closed: uv.semester === 'CLOSED'}"
+          >
             <td><a :href="`/pedagogy/uv/${uv.id}`" x-text="uv.code"></a></td>
-            <td x-text="uv.title"></td>
+            <td class="title"
+                x-text="uv.title + (uv.semester === 'CLOSED' ? ' ({% trans %}closed uv{% endtrans %})' : '')"
+            ></td>
             <td x-text="uv.department"></td>
             <td x-text="uv.credit_type"></td>
             <td><i :class="uv.semester.includes('AUTUMN') && 'fa fa-leaf'"></i></td>

pedagogy/templates/pedagogy/uv_edit.jinja

@@ -46,12 +46,7 @@
       const codeInput = document.querySelector('input[name="code"]')
 
       autofillBtn.addEventListener('click', () => {
-        const today = new Date()
-        let year = today.getFullYear()
-        if (today.getMonth() < 7) {  // student year starts in september
-          year--
-        }
-        const url = `/api/uv/${year}/${codeInput.value}`;
+        const url = `/api/uv/${codeInput.value}`;
         deleteQuickNotifs()
 
         $.ajax({
@@ -70,7 +65,7 @@
               .filter(([elem, _]) => !!elem)  // skip non-existing DOM elements
               .forEach(([elem, val]) => {  // write the value in the form field
                 if (elem.tagName === 'TEXTAREA') {
-                                // MD editor text input
+                  // MD editor text input
                   elem.parentNode.querySelector('.CodeMirror').CodeMirror.setValue(val);
                 } else {
                   elem.value = val;

pedagogy/utbm_api.py

@@ -1,32 +1,96 @@
 """Set of functions to interact with the UTBM UV api."""
 
-import urllib
+from typing import Iterator
 
+import requests
 from django.conf import settings
+from django.utils.functional import cached_property
 
 from pedagogy.schemas import ShortUvList, UtbmFullUvSchema, UtbmShortUvSchema, UvSchema
 
 
-def find_uv(lang, year, code) -> UvSchema | None:
-    """Find an UV from the UTBM API."""
-    # query the UV list
-    base_url = settings.SITH_PEDAGOGY_UTBM_API
-    uvs_url = f"{base_url}/uvs/{lang}/{year}"
-    response = urllib.request.urlopen(uvs_url)
-    uvs: list[UtbmShortUvSchema] = ShortUvList.validate_json(response.read())
+class UtbmApiClient(requests.Session):
+    """A wrapper around `requests.Session` to perform requests to the UTBM UV API."""
 
-    short_uv = next((uv for uv in uvs if uv.code == code), None)
-    if short_uv is None:
-        return None
+    BASE_URL = settings.SITH_PEDAGOGY_UTBM_API
+    _cache = {"short_uvs": {}}
 
-    # get detailed information about the UV
-    uv_url = f"{base_url}/uv/{lang}/{year}/{code}/{short_uv.code_formation}"
-    response = urllib.request.urlopen(uv_url)
-    full_uv = UtbmFullUvSchema.model_validate_json(response.read())
-    return _make_clean_uv(short_uv, full_uv)
+    @cached_property
+    def current_year(self) -> int:
+        """Fetch from the API the latest existing year"""
+        url = f"{self.BASE_URL}/guides/fr"
+        response = self.get(url)
+        return response.json()[-1]["annee"]
+
+    def fetch_short_uvs(
+        self, lang: str = "fr", year: int | None = None
+    ) -> list[UtbmShortUvSchema]:
+        """Get the list of UVs in their short format from the UTBM API"""
+        if year is None:
+            year = self.current_year
+        if lang not in self._cache["short_uvs"]:
+            self._cache["short_uvs"][lang] = {}
+        if year not in self._cache["short_uvs"][lang]:
+            url = f"{self.BASE_URL}/uvs/{lang}/{year}"
+            response = self.get(url)
+            uvs = ShortUvList.validate_json(response.content)
+            self._cache["short_uvs"][lang][year] = uvs
+        return self._cache["short_uvs"][lang][year]
+
+    def fetch_uvs(
+        self, lang: str = "fr", year: int | None = None
+    ) -> Iterator[UvSchema]:
+        """Fetch all UVs from the UTBM API, parsed in a format that we can use.
+
+        Warning:
+            We need infos from the full uv schema, and the UTBM UV API
+            has no route to get all of them at once.
+            We must do one request per UV (for a total of around 730 UVs),
+            which takes a lot of time.
+            Hopefully, there seems to be no rate-limit, so an error
+            in the middle of the process isn't likely to occur.
+        """
+        if year is None:
+            year = self.current_year
+        shorts_uvs = self.fetch_short_uvs(lang, year)
+        # When UVs are common to multiple branches (like most HUMA)
+        # the UTBM API duplicates them for every branch.
+        # We have no way in our db to link a UV to multiple formations,
+        # so we just create a single UV, which formation is the one
+        # of the first UV found in the list.
+        # For example, if we have CC01 (TC), CC01 (IMSI) and CC01 (EDIM),
+        # we will only keep CC01 (TC).
+        unique_short_uvs = {}
+        for uv in shorts_uvs:
+            if uv.code not in unique_short_uvs:
+                unique_short_uvs[uv.code] = uv
+        for uv in unique_short_uvs.values():
+            uv_url = f"{self.BASE_URL}/uv/{lang}/{year}/{uv.code}/{uv.code_formation}"
+            response = requests.get(uv_url)
+            full_uv = UtbmFullUvSchema.model_validate_json(response.content)
+            yield make_clean_uv(uv, full_uv)
+
+    def find_uv(self, lang: str, code: str, year: int | None = None) -> UvSchema | None:
+        """Find an UV from the UTBM API."""
+        # query the UV list
+        if not year:
+            year = self.current_year
+        # the UTBM API has no way to fetch a single short uv,
+        # and short uvs contain infos that we need and are not
+        # in the full uv schema, so we must fetch everything.
+        short_uvs = self.fetch_short_uvs(lang, year)
+        short_uv = next((uv for uv in short_uvs if uv.code == code), None)
+        if short_uv is None:
+            return None
+
+        # get detailed information about the UV
+        uv_url = f"{self.BASE_URL}/uv/{lang}/{year}/{code}/{short_uv.code_formation}"
+        response = requests.get(uv_url)
+        full_uv = UtbmFullUvSchema.model_validate_json(response.content)
+        return make_clean_uv(short_uv, full_uv)
 
 
-def _make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> UvSchema:
+def make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> UvSchema:
     """Cleans the data up so that it corresponds to our data representation.
 
     Some of the needed information are in the short uv schema, some
@@ -61,9 +125,9 @@ def _make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> Uv
             semester = "CLOSED"
 
     return UvSchema(
-        title=full_uv.libelle,
+        title=full_uv.libelle or "",
         code=full_uv.code,
-        credit_type=short_uv.code_categorie,
+        credit_type=short_uv.code_categorie or "FREE",
         semester=semester,
         language=short_uv.code_langue.upper(),
         credits=full_uv.credits_ects,
@@ -74,8 +138,8 @@ def _make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> Uv
         hours_TE=next((i.nbh for i in full_uv.activites if i.code == "TE"), 0) // 60,
         hours_CM=next((i.nbh for i in full_uv.activites if i.code == "CM"), 0) // 60,
         manager=full_uv.respo_automne or full_uv.respo_printemps or "",
-        objectives=full_uv.objectifs,
-        program=full_uv.programme,
-        skills=full_uv.acquisition_competences,
-        key_concepts=full_uv.acquisition_notions,
+        objectives=full_uv.objectifs or "",
+        program=full_uv.programme or "",
+        skills=full_uv.acquisition_competences or "",
+        key_concepts=full_uv.acquisition_notions or "",
     )

pyproject.toml

@@ -44,6 +44,7 @@ dependencies = [
     "django-honeypot<2.0.0,>=1.2.1",
     "pydantic-extra-types<3.0.0,>=2.10.1",
     "ical<9.0.0,>=8.3.0",
+    "requests>=2.32.3",
 ]
 
 [project.urls]

uv.lock

@@ -155,7 +155,7 @@ name = "click"
 version = "8.1.8"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "colorama", marker = "platform_system == 'Windows'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 }
 wheels = [
@@ -744,7 +744,7 @@ version = "1.6.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "click" },
-    { name = "colorama", marker = "sys_platform == 'win32'" },
+    { name = "colorama", marker = "platform_system == 'Windows'" },
     { name = "ghp-import" },
     { name = "jinja2" },
     { name = "markdown" },
@@ -1437,6 +1437,7 @@ dependencies = [
     { name = "pydantic-extra-types" },
     { name = "python-dateutil" },
     { name = "reportlab" },
+    { name = "requests" },
     { name = "sentry-sdk" },
     { name = "sphinx" },
     { name = "tomli" },
@@ -1495,6 +1496,7 @@ requires-dist = [
     { name = "pydantic-extra-types", specifier = ">=2.10.1,<3.0.0" },
     { name = "python-dateutil", specifier = ">=2.9.0.post0,<3.0.0.0" },
     { name = "reportlab", specifier = ">=4.2.5,<5.0.0" },
+    { name = "requests", specifier = ">=2.32.3" },
     { name = "sentry-sdk", specifier = ">=2.19.2,<3.0.0" },
     { name = "sphinx", specifier = ">=5,<6" },
     { name = "tomli", specifier = ">=2.2.1,<3.0.0" },