From 85c8b7d11c9713252e27500bb2b61c21c1ced53a Mon Sep 17 00:00:00 2001 From: imperosol Date: Mon, 20 Jan 2025 19:20:13 +0100 Subject: [PATCH 01/48] Use requests for external requests L'API de requests est beaucoup plus claire que celle d'urllib et urllib3. --- com/calendar.py | 11 +++++------ com/tests/test_api.py | 15 ++++++--------- pedagogy/utbm_api.py | 15 +++++++-------- pyproject.toml | 1 + uv.lock | 6 ++++-- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/com/calendar.py b/com/calendar.py index 9003d6de..f3c612e1 100644 --- a/com/calendar.py +++ b/com/calendar.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from pathlib import Path from typing import final -import urllib3 +import requests from dateutil.relativedelta import relativedelta from django.conf import settings from django.urls import reverse @@ -35,16 +35,15 @@ class IcsCalendar: @classmethod def make_external(cls) -> Path | None: - calendar = urllib3.request( - "GET", - "https://calendar.google.com/calendar/ical/ae.utbm%40gmail.com/public/basic.ics", + calendar = requests.get( + "https://calendar.google.com/calendar/ical/ae.utbm%40gmail.com/public/basic.ics" ) - if calendar.status != 200: + if not calendar.ok: return None cls._CACHE_FOLDER.mkdir(parents=True, exist_ok=True) with open(cls._EXTERNAL_CALENDAR, "wb") as f: - _ = f.write(calendar.data) + _ = f.write(calendar.content) return cls._EXTERNAL_CALENDAR @classmethod diff --git a/com/tests/test_api.py b/com/tests/test_api.py index f131052e..c84d8448 100644 --- a/com/tests/test_api.py +++ b/com/tests/test_api.py @@ -16,11 +16,11 @@ from com.calendar import IcsCalendar @dataclass class MockResponse: - status: int + ok: bool value: str @property - def data(self): + def content(self): return self.value.encode("utf8") @@ -38,7 +38,7 @@ class TestExternalCalendar: @pytest.fixture def mock_request(self): mock = MagicMock() - with patch("urllib3.request", mock): + with patch("requests.get", mock): yield mock @pytest.fixture @@ -52,15 +52,12 @@ class TestExternalCalendar: def clear_cache(self): IcsCalendar._EXTERNAL_CALENDAR.unlink(missing_ok=True) - @pytest.mark.parametrize("error_code", [403, 404, 500]) - def test_fetch_error( - self, client: Client, mock_request: MagicMock, error_code: int - ): - mock_request.return_value = MockResponse(error_code, "not allowed") + def test_fetch_error(self, client: Client, mock_request: MagicMock): + mock_request.return_value = MockResponse(ok=False, value="not allowed") assert client.get(reverse("api:calendar_external")).status_code == 404 def test_fetch_success(self, client: Client, mock_request: MagicMock): - external_response = MockResponse(200, "Definitely an ICS") + external_response = MockResponse(ok=True, value="Definitely an ICS") mock_request.return_value = external_response response = client.get(reverse("api:calendar_external")) assert response.status_code == 200 diff --git a/pedagogy/utbm_api.py b/pedagogy/utbm_api.py index fdd6d1fb..700c22f3 100644 --- a/pedagogy/utbm_api.py +++ b/pedagogy/utbm_api.py @@ -1,7 +1,6 @@ """Set of functions to interact with the UTBM UV api.""" -import urllib - +import requests from django.conf import settings from pedagogy.schemas import ShortUvList, UtbmFullUvSchema, UtbmShortUvSchema, UvSchema @@ -12,8 +11,8 @@ def find_uv(lang, year, code) -> UvSchema | None: # query the UV list base_url = settings.SITH_PEDAGOGY_UTBM_API uvs_url = f"{base_url}/uvs/{lang}/{year}" - response = urllib.request.urlopen(uvs_url) - uvs: list[UtbmShortUvSchema] = ShortUvList.validate_json(response.read()) + response = requests.get(uvs_url) + uvs: list[UtbmShortUvSchema] = ShortUvList.validate_json(response.content) short_uv = next((uv for uv in uvs if uv.code == code), None) if short_uv is None: @@ -21,12 +20,12 @@ def find_uv(lang, year, code) -> UvSchema | None: # get detailed information about the UV uv_url = f"{base_url}/uv/{lang}/{year}/{code}/{short_uv.code_formation}" - response = urllib.request.urlopen(uv_url) - full_uv = UtbmFullUvSchema.model_validate_json(response.read()) - return _make_clean_uv(short_uv, full_uv) + response = requests.get(uv_url) + full_uv = UtbmFullUvSchema.model_validate_json(response.content) + return make_clean_uv(short_uv, full_uv) -def _make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> UvSchema: +def make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> UvSchema: """Cleans the data up so that it corresponds to our data representation. Some of the needed information are in the short uv schema, some diff --git a/pyproject.toml b/pyproject.toml index 3e2cdf0f..b3db97cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "django-honeypot<2.0.0,>=1.2.1", "pydantic-extra-types<3.0.0,>=2.10.1", "ical<9.0.0,>=8.3.0", + "requests>=2.32.3", ] [project.urls] diff --git a/uv.lock b/uv.lock index 2ea5e370..abf0fb98 100644 --- a/uv.lock +++ b/uv.lock @@ -155,7 +155,7 @@ name = "click" version = "8.1.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } wheels = [ @@ -744,7 +744,7 @@ version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, { name = "ghp-import" }, { name = "jinja2" }, { name = "markdown" }, @@ -1437,6 +1437,7 @@ dependencies = [ { name = "pydantic-extra-types" }, { name = "python-dateutil" }, { name = "reportlab" }, + { name = "requests" }, { name = "sentry-sdk" }, { name = "sphinx" }, { name = "tomli" }, @@ -1495,6 +1496,7 @@ requires-dist = [ { name = "pydantic-extra-types", specifier = ">=2.10.1,<3.0.0" }, { name = "python-dateutil", specifier = ">=2.9.0.post0,<3.0.0.0" }, { name = "reportlab", specifier = ">=4.2.5,<5.0.0" }, + { name = "requests", specifier = ">=2.32.3" }, { name = "sentry-sdk", specifier = ">=2.19.2,<3.0.0" }, { name = "sphinx", specifier = ">=5,<6" }, { name = "tomli", specifier = ">=2.2.1,<3.0.0" }, From 6d519e3a07de2aa7371c36ed88d30de81680481d Mon Sep 17 00:00:00 2001 From: imperosol Date: Tue, 21 Jan 2025 13:57:18 +0100 Subject: [PATCH 02/48] Custom client for UTBM UV API calls --- pedagogy/api.py | 12 +++-- pedagogy/templates/pedagogy/uv_edit.jinja | 9 +--- pedagogy/utbm_api.py | 62 +++++++++++++++++------ 3 files changed, 57 insertions(+), 26 deletions(-) diff --git a/pedagogy/api.py b/pedagogy/api.py index e8d34351..ce1fdde0 100644 --- a/pedagogy/api.py +++ b/pedagogy/api.py @@ -10,13 +10,13 @@ from ninja_extra.pagination import PageNumberPaginationExtra, PaginatedResponseS from core.auth.api_permissions import HasPerm from pedagogy.models import UV from pedagogy.schemas import SimpleUvSchema, UvFilterSchema, UvSchema -from pedagogy.utbm_api import find_uv +from pedagogy.utbm_api import UtbmApiClient @api_controller("/uv") class UvController(ControllerBase): @route.get( - "/{year}/{code}", + "/{code}", permissions=[ # this route will almost always be called in the context # of a UV creation/edition @@ -26,10 +26,14 @@ class UvController(ControllerBase): response=UvSchema, ) def fetch_from_utbm_api( - self, year: Annotated[int, Ge(2010)], code: str, lang: Query[str] = "fr" + self, + code: str, + lang: Query[str] = "fr", + year: Query[Annotated[int, Ge(2010)] | None] = None, ): """Fetch UV data from the UTBM API and returns it after some parsing.""" - res = find_uv(lang, year, code) + with UtbmApiClient() as client: + res = client.find_uv(lang, code, year) if res is None: raise NotFound return res diff --git a/pedagogy/templates/pedagogy/uv_edit.jinja b/pedagogy/templates/pedagogy/uv_edit.jinja index 7ab54105..b4869e14 100644 --- a/pedagogy/templates/pedagogy/uv_edit.jinja +++ b/pedagogy/templates/pedagogy/uv_edit.jinja @@ -46,12 +46,7 @@ const codeInput = document.querySelector('input[name="code"]') autofillBtn.addEventListener('click', () => { - const today = new Date() - let year = today.getFullYear() - if (today.getMonth() < 7) { // student year starts in september - year-- - } - const url = `/api/uv/${year}/${codeInput.value}`; + const url = `/api/uv/${codeInput.value}`; deleteQuickNotifs() $.ajax({ @@ -70,7 +65,7 @@ .filter(([elem, _]) => !!elem) // skip non-existing DOM elements .forEach(([elem, val]) => { // write the value in the form field if (elem.tagName === 'TEXTAREA') { - // MD editor text input + // MD editor text input elem.parentNode.querySelector('.CodeMirror').CodeMirror.setValue(val); } else { elem.value = val; diff --git a/pedagogy/utbm_api.py b/pedagogy/utbm_api.py index 700c22f3..f0d1377d 100644 --- a/pedagogy/utbm_api.py +++ b/pedagogy/utbm_api.py @@ -2,27 +2,59 @@ import requests from django.conf import settings +from django.utils.functional import cached_property from pedagogy.schemas import ShortUvList, UtbmFullUvSchema, UtbmShortUvSchema, UvSchema -def find_uv(lang, year, code) -> UvSchema | None: - """Find an UV from the UTBM API.""" - # query the UV list - base_url = settings.SITH_PEDAGOGY_UTBM_API - uvs_url = f"{base_url}/uvs/{lang}/{year}" - response = requests.get(uvs_url) - uvs: list[UtbmShortUvSchema] = ShortUvList.validate_json(response.content) +class UtbmApiClient(requests.Session): + """A wrapper around `requests.Session` to perform requests to the UTBM UV API.""" - short_uv = next((uv for uv in uvs if uv.code == code), None) - if short_uv is None: - return None + BASE_URL = settings.SITH_PEDAGOGY_UTBM_API + _cache = {} - # get detailed information about the UV - uv_url = f"{base_url}/uv/{lang}/{year}/{code}/{short_uv.code_formation}" - response = requests.get(uv_url) - full_uv = UtbmFullUvSchema.model_validate_json(response.content) - return make_clean_uv(short_uv, full_uv) + @cached_property + def current_year(self) -> int: + """Fetch from the API the latest existing year""" + url = f"{self.BASE_URL}/guides/fr" + response = self.get(url) + return response.json()[-1]["annee"] + + def fetch_short_uvs( + self, lang: str = "fr", year: int | None = None + ) -> list[UtbmShortUvSchema]: + """Get the list of UVs in their short format from the UTBM API""" + if year is None: + year = self.current_year + if "short_uvs" not in self._cache: + self._cache["short_uvs"] = {} + if lang not in self._cache["short_uvs"]: + self._cache["short_uvs"][lang] = {} + if year not in self._cache["short_uvs"][lang]: + url = f"{self.BASE_URL}/uvs/{lang}/{year}" + response = self.get(url) + uvs = ShortUvList.validate_json(response.content) + self._cache["short_uvs"][lang][year] = uvs + return self._cache["short_uvs"][lang][year] + + def find_uv(self, lang: str, code: str, year: int | None = None) -> UvSchema | None: + """Find an UV from the UTBM API.""" + # query the UV list + if not year: + year = self.current_year + # the UTBM API has no way to fetch a single short uv, + # and short uvs contain infos that we need and are not + # in the full uv schema, so we must fetch everything. + short_uvs = self.fetch_short_uvs(lang, year) + short_uv = next((uv for uv in short_uvs if uv.code == code), None) + if short_uv is None: + return None + + # get detailed information about the UV + uv_url = f"{self.BASE_URL}/uv/{lang}/{year}/{code}/{short_uv.code_formation}" + response = requests.get(uv_url) + full_uv = UtbmFullUvSchema.model_validate_json(response.content) + return make_clean_uv(short_uv, full_uv) def make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> UvSchema: From 78f3caa455e7144c69dd2dc15368cd05d6eab86f Mon Sep 17 00:00:00 2001 From: imperosol Date: Tue, 21 Jan 2025 15:40:12 +0100 Subject: [PATCH 03/48] management command to update the whole uv guide --- pedagogy/management/__init__.py | 0 pedagogy/management/commands/__init__.py | 0 .../management/commands/update_uv_guide.py | 37 ++++++++++++++ pedagogy/schemas.py | 10 ++-- pedagogy/utbm_api.py | 51 +++++++++++++++---- 5 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 pedagogy/management/__init__.py create mode 100644 pedagogy/management/commands/__init__.py create mode 100644 pedagogy/management/commands/update_uv_guide.py diff --git a/pedagogy/management/__init__.py b/pedagogy/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pedagogy/management/commands/__init__.py b/pedagogy/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pedagogy/management/commands/update_uv_guide.py b/pedagogy/management/commands/update_uv_guide.py new file mode 100644 index 00000000..cf525a1f --- /dev/null +++ b/pedagogy/management/commands/update_uv_guide.py @@ -0,0 +1,37 @@ +from django.conf import settings +from django.core.management import BaseCommand + +from core.models import User +from pedagogy.models import UV +from pedagogy.schemas import UvSchema +from pedagogy.utbm_api import UtbmApiClient + + +class Command(BaseCommand): + help = "Update the UV guide" + + def handle(self, *args, **options): + seen_uvs: set[int] = set() + root_user = User.objects.get(pk=settings.SITH_ROOT_USER_ID) + with UtbmApiClient() as client: + self.stdout.write( + "Fetching UVs from the UTBM API.\n" + "This may take a few minutes to complete." + ) + for uv in client.fetch_uvs(): + db_uv = UV.objects.filter(code=uv.code).first() + if db_uv is None: + db_uv = UV(code=uv.code, author=root_user) + fields = list(UvSchema.model_fields.keys()) + fields.remove("id") + fields.remove("code") + for field in fields: + setattr(db_uv, field, getattr(uv, field)) + db_uv.save() + # if it's a creation, django will set the id when saving, + # so at this point, a db_uv will always have an id + seen_uvs.add(db_uv.id) + # UVs that are in database but have not been returned by the API + # are considered as closed UEs + UV.objects.exclude(id__in=seen_uvs).update(semester="CLOSED") + self.stdout.write(self.style.SUCCESS("UV guide updated successfully")) diff --git a/pedagogy/schemas.py b/pedagogy/schemas.py index 716e9a3c..cbcd9157 100644 --- a/pedagogy/schemas.py +++ b/pedagogy/schemas.py @@ -54,11 +54,11 @@ class UtbmFullUvSchema(Schema): code: str departement: str = "NA" - libelle: str - objectifs: str - programme: str - acquisition_competences: str - acquisition_notions: str + libelle: str | None + objectifs: str | None + programme: str | None + acquisition_competences: str | None + acquisition_notions: str | None langue: str code_langue: str credits_ects: int diff --git a/pedagogy/utbm_api.py b/pedagogy/utbm_api.py index f0d1377d..dd7ec933 100644 --- a/pedagogy/utbm_api.py +++ b/pedagogy/utbm_api.py @@ -1,5 +1,7 @@ """Set of functions to interact with the UTBM UV api.""" +from typing import Iterator + import requests from django.conf import settings from django.utils.functional import cached_property @@ -11,7 +13,7 @@ class UtbmApiClient(requests.Session): """A wrapper around `requests.Session` to perform requests to the UTBM UV API.""" BASE_URL = settings.SITH_PEDAGOGY_UTBM_API - _cache = {} + _cache = {"short_uvs": {}} @cached_property def current_year(self) -> int: @@ -26,8 +28,6 @@ class UtbmApiClient(requests.Session): """Get the list of UVs in their short format from the UTBM API""" if year is None: year = self.current_year - if "short_uvs" not in self._cache: - self._cache["short_uvs"] = {} if lang not in self._cache["short_uvs"]: self._cache["short_uvs"][lang] = {} if year not in self._cache["short_uvs"][lang]: @@ -37,6 +37,39 @@ class UtbmApiClient(requests.Session): self._cache["short_uvs"][lang][year] = uvs return self._cache["short_uvs"][lang][year] + def fetch_uvs( + self, lang: str = "fr", year: int | None = None + ) -> Iterator[UvSchema]: + """Fetch all UVs from the UTBM API, parsed in a format that we can use. + + Warning: + We need infos from the full uv schema, and the UTBM UV API + has no route to get all of them at once. + We must do one request per UV (for a total of around 730 UVs), + which takes a lot of time. + Hopefully, there seems to be no rate-limit, so an error + in the middle of the process isn't likely to occur. + """ + if year is None: + year = self.current_year + shorts_uvs = self.fetch_short_uvs(lang, year) + # When UVs are common to multiple branches (like most HUMA) + # the UTBM API duplicates them for every branch. + # We have no way in our db to link a UV to multiple formations, + # so we just create a single UV, which formation is the one + # of the first UV found in the list. + # For example, if we have CC01 (TC), CC01 (IMSI) and CC01 (EDIM), + # we will only keep CC01 (TC). + unique_short_uvs = {} + for uv in shorts_uvs: + if uv.code not in unique_short_uvs: + unique_short_uvs[uv.code] = uv + for uv in unique_short_uvs.values(): + uv_url = f"{self.BASE_URL}/uv/{lang}/{year}/{uv.code}/{uv.code_formation}" + response = requests.get(uv_url) + full_uv = UtbmFullUvSchema.model_validate_json(response.content) + yield make_clean_uv(uv, full_uv) + def find_uv(self, lang: str, code: str, year: int | None = None) -> UvSchema | None: """Find an UV from the UTBM API.""" # query the UV list @@ -92,9 +125,9 @@ def make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> UvS semester = "CLOSED" return UvSchema( - title=full_uv.libelle, + title=full_uv.libelle or "", code=full_uv.code, - credit_type=short_uv.code_categorie, + credit_type=short_uv.code_categorie or "FREE", semester=semester, language=short_uv.code_langue.upper(), credits=full_uv.credits_ects, @@ -105,8 +138,8 @@ def make_clean_uv(short_uv: UtbmShortUvSchema, full_uv: UtbmFullUvSchema) -> UvS hours_TE=next((i.nbh for i in full_uv.activites if i.code == "TE"), 0) // 60, hours_CM=next((i.nbh for i in full_uv.activites if i.code == "CM"), 0) // 60, manager=full_uv.respo_automne or full_uv.respo_printemps or "", - objectives=full_uv.objectifs, - program=full_uv.programme, - skills=full_uv.acquisition_competences, - key_concepts=full_uv.acquisition_notions, + objectives=full_uv.objectifs or "", + program=full_uv.programme or "", + skills=full_uv.acquisition_competences or "", + key_concepts=full_uv.acquisition_notions or "", ) From 5fa431e29ba50280e2f374cb08fd6dc1fdae3ea6 Mon Sep 17 00:00:00 2001 From: imperosol Date: Tue, 21 Jan 2025 16:01:50 +0100 Subject: [PATCH 04/48] Visually differentiate closed UVs from the others --- locale/fr/LC_MESSAGES/django.po | 19 +++++++++++-------- pedagogy/api.py | 2 +- pedagogy/static/pedagogy/css/pedagogy.scss | 7 +++++-- pedagogy/templates/pedagogy/guide.jinja | 14 ++++++++++---- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/locale/fr/LC_MESSAGES/django.po b/locale/fr/LC_MESSAGES/django.po index 0cfef902..10fe885d 100644 --- a/locale/fr/LC_MESSAGES/django.po +++ b/locale/fr/LC_MESSAGES/django.po @@ -935,10 +935,6 @@ msgstr "rôle" msgid "description" msgstr "description" -#: club/models.py -msgid "past member" -msgstr "ancien membre" - #: club/models.py msgid "Email address" msgstr "Adresse email" @@ -3308,8 +3304,8 @@ msgstr "Nom d'utilisateur, email, ou numéro de compte AE" #: core/views/forms.py msgid "" -"Profile: you need to be visible on the picture, in order to be recognized " -"(e.g. by the barmen)" +"Profile: you need to be visible on the picture, in order to be recognized (e." +"g. by the barmen)" msgstr "" "Photo de profil: vous devez être visible sur la photo afin d'être reconnu " "(par exemple par les barmen)" @@ -3919,8 +3915,8 @@ msgstr "" #: counter/templates/counter/mails/account_dump.jinja msgid "If you think this was a mistake, please mail us at ae@utbm.fr." msgstr "" -"Si vous pensez qu'il s'agit d'une erreur, veuillez envoyer un mail à " -"ae@utbm.fr." +"Si vous pensez qu'il s'agit d'une erreur, veuillez envoyer un mail à ae@utbm." +"fr." #: counter/templates/counter/mails/account_dump.jinja msgid "" @@ -4948,6 +4944,10 @@ msgstr "Département" msgid "Credit type" msgstr "Type de crédit" +#: pedagogy/templates/pedagogy/guide.jinja +msgid "closed uv" +msgstr "uv fermée" + #: pedagogy/templates/pedagogy/macros.jinja msgid " not rated " msgstr "non noté" @@ -5990,3 +5990,6 @@ msgstr "Vous ne pouvez plus écrire de commentaires, la date est passée." #, python-format msgid "Maximum characters: %(max_length)s" msgstr "Nombre de caractères max: %(max_length)s" + +#~ msgid "past member" +#~ msgstr "ancien membre" diff --git a/pedagogy/api.py b/pedagogy/api.py index ce1fdde0..9ad0c3f6 100644 --- a/pedagogy/api.py +++ b/pedagogy/api.py @@ -46,4 +46,4 @@ class UvController(ControllerBase): ) @paginate(PageNumberPaginationExtra, page_size=100) def fetch_uv_list(self, search: Query[UvFilterSchema]): - return search.filter(UV.objects.values()) + return search.filter(UV.objects.order_by("code").values()) diff --git a/pedagogy/static/pedagogy/css/pedagogy.scss b/pedagogy/static/pedagogy/css/pedagogy.scss index a4ebb370..51656615 100644 --- a/pedagogy/static/pedagogy/css/pedagogy.scss +++ b/pedagogy/static/pedagogy/css/pedagogy.scss @@ -47,11 +47,14 @@ $large-devices: 992px; } } - #dynamic_view { + #uv-list { font-size: 1.1em; overflow-wrap: break-word; - + .closed td.title { + color: lighten($black-color, 10%); + font-style: italic; + } td { text-align: center; border: none; diff --git a/pedagogy/templates/pedagogy/guide.jinja b/pedagogy/templates/pedagogy/guide.jinja index 79b66c24..460fdcc5 100644 --- a/pedagogy/templates/pedagogy/guide.jinja +++ b/pedagogy/templates/pedagogy/guide.jinja @@ -85,7 +85,7 @@ - +
@@ -102,11 +102,17 @@ {% endif %} - +
{% trans %}UV{% endtrans %}