Optimize galaxy generation

En réorganisant les requêtes à la db, on diminue par 100 le temps d'exécution de la commande `rule_galaxy` (~6h => ~2min)

.github/actions/setup_project/action.yml

@@ -1,15 +1,24 @@
 name: "Setup project"
 description: "Setup Python and Poetry"
+inputs:
+  full:
+    description: > 
+      If true, do a full setup, else install
+      only python, uv and non-xapian python deps
+    required: false
+    default: "false"
 runs:
   using: composite
   steps:
     - name: Install apt packages
+      if: ${{ inputs.full == 'true' }}
       uses: awalsh128/cache-apt-pkgs-action@v1.4.3
       with:
         packages: gettext
         version: 1.0  # increment to reset cache
 
     - name: Install Redis
+      if: ${{ inputs.full == 'true' }}
       uses: shogo82148/actions-setup-redis@v1
       with:
         redis-version: "7.x"
@@ -37,15 +46,20 @@ runs:
       shell: bash
 
     - name: Install Xapian
+      if: ${{ inputs.full == 'true' }}
       run: uv run ./manage.py install_xapian
       shell: bash
 
+    # compiling xapian accounts for almost the entirety of the virtualenv setup,
+    # so we save the virtual environment only on workflows where it has been installed
     - name: Save cached virtualenv
+      if: ${{ inputs.full == 'true' }}
       uses: actions/cache/save@v4
       with:
         key: venv-${{ runner.os }}-${{ hashFiles('.python-version') }}-${{ hashFiles('pyproject.toml') }}-${{ env.CACHE_SUFFIX }}
         path: .venv
 
     - name: Compile gettext messages
+      if: ${{ inputs.full == 'true' }}
       run: uv run ./manage.py compilemessages
       shell: bash

.github/workflows/ci.yml

@@ -37,6 +37,8 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
       - uses: ./.github/actions/setup_project
+        with:
+          full: true
         env:
           # To avoid race conditions on environment cache
           CACHE_SUFFIX: ${{ matrix.pytest-mark }}

.github/workflows/deploy_docs.yml

@@ -2,11 +2,7 @@ name: deploy_docs
 on:
   push:
     branches:
-      - master
+      - taiste
-env:
-  SECRET_KEY: notTheRealOne
-  DATABASE_URL: sqlite:///db.sqlite3
-  CACHE_URL: redis://127.0.0.1:6379/0
 permissions:
   contents: write
 jobs:

api/urls.py

@@ -2,7 +2,7 @@ from ninja_extra import NinjaExtraAPI
 
 api = NinjaExtraAPI(
     title="PICON",
-    description="Portail Interaction de Communication avec les Services Étudiants",
+    description="Portail Interactif de Communication avec les Outils Numériques",
     version="0.2.0",
     urls_namespace="api",
     csrf=True,

club/api.py

@@ -1,6 +1,7 @@
 from typing import Annotated
 
 from annotated_types import MinLen
+from django.db.models import Prefetch
 from ninja.security import SessionAuth
 from ninja_extra import ControllerBase, api_controller, paginate, route
 from ninja_extra.pagination import PageNumberPaginationExtra
@@ -8,7 +9,7 @@ from ninja_extra.schemas import PaginatedResponseSchema
 
 from api.auth import ApiKeyAuth
 from api.permissions import CanAccessLookup, HasPerm
-from club.models import Club
+from club.models import Club, Membership
 from club.schemas import ClubSchema, SimpleClubSchema
 
 
@@ -33,6 +34,9 @@ class ClubController(ControllerBase):
         url_name="fetch_club",
     )
     def fetch_club(self, club_id: int):
-        return self.get_object_or_exception(
+        prefetch = Prefetch(
-            Club.objects.prefetch_related("members", "members__user"), id=club_id
+            "members", queryset=Membership.objects.ongoing().select_related("user")
+        )
+        return self.get_object_or_exception(
+            Club.objects.prefetch_related(prefetch), id=club_id
         )

club/tests/test_club_controller.py

@@ -1,7 +1,10 @@
+from datetime import date, timedelta
+
 import pytest
 from django.test import Client
 from django.urls import reverse
 from model_bakery import baker
+from model_bakery.recipe import Recipe
 from pytest_django.asserts import assertNumQueries
 
 from club.models import Club, Membership
@@ -9,13 +12,32 @@ from core.baker_recipes import subscriber_user
 
 
 @pytest.mark.django_db
-def test_fetch_club(client: Client):
+class TestFetchClub:
-    club = baker.make(Club)
+    @pytest.fixture()
-    baker.make(Membership, club=club, _quantity=10, _bulk_create=True)
+    def club(self):
-    user = subscriber_user.make()
+        club = baker.make(Club)
-    client.force_login(user)
+        last_month = date.today() - timedelta(days=30)
-    with assertNumQueries(7):
+        yesterday = date.today() - timedelta(days=1)
-        # - 4 queries for authentication
+        membership_recipe = Recipe(Membership, club=club, start_date=last_month)
-        # - 3 queries for the actual data
+        membership_recipe.make(end_date=None, _quantity=10, _bulk_create=True)
+        membership_recipe.make(end_date=yesterday, _quantity=10, _bulk_create=True)
+        return club
+
+    def test_fetch_club_members(self, client: Client, club: Club):
+        user = subscriber_user.make()
+        client.force_login(user)
         res = client.get(reverse("api:fetch_club", kwargs={"club_id": club.id}))
         assert res.status_code == 200
+        member_ids = {member["user"]["id"] for member in res.json()["members"]}
+        assert member_ids == set(
+            club.members.ongoing().values_list("user_id", flat=True)
+        )
+
+    def test_fetch_club_nb_queries(self, client: Client, club: Club):
+        user = subscriber_user.make()
+        client.force_login(user)
+        with assertNumQueries(6):
+            # - 4 queries for authentication
+            # - 2 queries for the actual data
+            res = client.get(reverse("api:fetch_club", kwargs={"club_id": club.id}))
+            assert res.status_code == 200

com/models.py

@@ -170,7 +170,6 @@ def news_notification_callback(notif: Notification):
     if count:
         notif.viewed = False
         notif.param = str(count)
-        notif.date = timezone.now()
     else:
         notif.viewed = True
 

core/migrations/0047_alter_notification_date_alter_notification_type.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.2.1 on 2025-06-11 16:10
+
+from django.db import migrations, models
+
+import core.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [("core", "0046_permissionrights")]
+
+    operations = [
+        migrations.AlterField(
+            model_name="notification",
+            name="date",
+            field=models.DateTimeField(auto_now=True, verbose_name="date"),
+        ),
+        migrations.AlterField(
+            model_name="notification",
+            name="type",
+            field=models.CharField(
+                choices=core.models.get_notification_types,
+                default="GENERIC",
+                max_length=32,
+                verbose_name="type",
+            ),
+        ),
+    ]

core/models.py

@@ -1451,6 +1451,10 @@ class PageRev(models.Model):
         return self.page.can_be_edited_by(user)
 
 
+def get_notification_types():
+    return settings.SITH_NOTIFICATIONS
+
+
 class Notification(models.Model):
     user = models.ForeignKey(
         User, related_name="notifications", on_delete=models.CASCADE
@@ -1458,9 +1462,9 @@ class Notification(models.Model):
     url = models.CharField(_("url"), max_length=255)
     param = models.CharField(_("param"), max_length=128, default="")
     type = models.CharField(
-        _("type"), max_length=32, choices=settings.SITH_NOTIFICATIONS, default="GENERIC"
+        _("type"), max_length=32, choices=get_notification_types, default="GENERIC"
     )
-    date = models.DateTimeField(_("date"), default=timezone.now)
+    date = models.DateTimeField(_("date"), auto_now=True)
     viewed = models.BooleanField(_("viewed"), default=False, db_index=True)
 
     def __str__(self):

core/static/bundled/core/navbar-index.ts

@@ -0,0 +1,36 @@
+import { exportToHtml } from "#core:utils/globals";
+
+exportToHtml("showMenu", () => {
+  const navbar = document.getElementById("navbar-content");
+  const current = navbar.getAttribute("mobile-display");
+  navbar.setAttribute("mobile-display", current === "hidden" ? "revealed" : "hidden");
+});
+
+document.addEventListener("alpine:init", () => {
+  const menuItems = document.querySelectorAll(".navbar details[name='navbar'].menu");
+  const isDesktop = () => {
+    return window.innerWidth >= 500;
+  };
+  for (const item of menuItems) {
+    item.addEventListener("mouseover", () => {
+      if (isDesktop()) {
+        item.setAttribute("open", "");
+      }
+    });
+    item.addEventListener("mouseout", () => {
+      if (isDesktop()) {
+        item.removeAttribute("open");
+      }
+    });
+    item.addEventListener("click", (event: MouseEvent) => {
+      // Don't close when clicking on desktop mode
+      if ((event.target as HTMLElement).nodeName !== "SUMMARY" || event.detail === 0) {
+        return;
+      }
+
+      if (isDesktop()) {
+        event.preventDefault();
+      }
+    });
+  }
+});

core/templates/core/base.jinja

@@ -18,6 +18,7 @@
       <noscript><link rel="stylesheet" href="{{ static('bundled/fontawesome-index.css') }}"></noscript>
 
       <script src="{{ url('javascript-catalog') }}"></script>
+      <script type="module" src={{ static("bundled/core/navbar-index.ts") }}></script>
       <script type="module" src={{ static("bundled/core/components/include-index.ts") }}></script>
       <script type="module" src="{{ static('bundled/alpine-index.js') }}"></script>
       <script type="module" src="{{ static('bundled/htmx-index.js') }}"></script>
@@ -107,39 +108,6 @@
 
     {% block script %}
       <script>
-        const menuItems = document.querySelectorAll(".navbar details[name='navbar'].menu");
-        const isDesktop = () => {
-          return window.innerWidth >= 500;
-        }
-        for (const item of menuItems){
-          item.addEventListener("mouseover", () => {
-            if (isDesktop()){
-              item.setAttribute("open", "");
-            }
-          })
-          item.addEventListener("mouseout", () => {
-            if (isDesktop()){
-              item.removeAttribute("open");
-            }
-          })
-          item.addEventListener("click", (event) => {
-            // Ignore keyboard clicks
-            if (event.detail === 0){
-              return;
-            }
-
-            if (isDesktop()){
-              event.preventDefault();
-            }
-          })
-        }
-
-        function showMenu() {
-          let navbar = document.getElementById("navbar-content");
-          const current = navbar.getAttribute("mobile-display");
-          navbar.setAttribute("mobile-display", current === "hidden" ? "revealed" : "hidden")
-        }
-
         document.addEventListener("keydown", (e) => {
           // Looking at the `s` key when not typing in a form
           if (e.keyCode !== 83 || ["INPUT", "TEXTAREA", "SELECT"].includes(e.target.nodeName)) {

galaxy/models.py

@@ -23,20 +23,21 @@
 
 from __future__ import annotations
 
+import itertools
 import logging
 import math
 import time
+from collections import defaultdict
 from typing import NamedTuple, TypedDict
 
 from django.db import models
-from django.db.models import Case, Count, F, Q, Value, When
+from django.db.models import Count, F, Q, QuerySet
-from django.db.models.functions import Concat
 from django.utils.timezone import localdate
 from django.utils.translation import gettext_lazy as _
 
-from club.models import Club
+from club.models import Membership
 from core.models import User
-from sas.models import Picture
+from sas.models import PeoplePictureRelation, Picture
 
 
 class GalaxyStar(models.Model):
@@ -114,18 +115,9 @@ class GalaxyLane(models.Model):
         default=0,
         help_text=_("Distance separating star1 and star2"),
     )
-    family = models.PositiveIntegerField(
+    family = models.PositiveIntegerField(_("family score"), default=0)
-        _("family score"),
+    pictures = models.PositiveIntegerField(_("pictures score"), default=0)
-        default=0,
+    clubs = models.PositiveIntegerField(_("clubs score"), default=0)
-    )
-    pictures = models.PositiveIntegerField(
-        _("pictures score"),
-        default=0,
-    )
-    clubs = models.PositiveIntegerField(
-        _("clubs score"),
-        default=0,
-    )
 
     def __str__(self):
         return f"{self.star1} -> {self.star2} ({self.distance})"
@@ -174,6 +166,7 @@ class Galaxy(models.Model):
     logger = logging.getLogger("main")
 
     GALAXY_SCALE_FACTOR = 2_000
+    DEFAULT_PICTURE_COUNT_THRESHOLD = 10
     FAMILY_LINK_POINTS = 366  # Equivalent to a leap year together in a club, because.
     PICTURE_POINTS = 2  # Equivalent to two days as random members of a club.
     CLUBS_POINTS = 1  # One day together as random members in a club is one point.
@@ -187,15 +180,13 @@ class Galaxy(models.Model):
         stars_count = self.stars.count()
         s = f"GLX-ID{self.pk}-SC{stars_count}-"
         if self.state is None:
-            s += "CHS"  # CHAOS
+            s += "CHAOS"
         else:
-            s += "RLD"  # RULED
+            s += "RULED"
         return s
 
     @classmethod
-    def get_current_galaxy(
+    def get_current_galaxy(cls) -> Galaxy:
-        cls,
-    ) -> Galaxy:  # __future__.annotations is required for this
         return Galaxy.objects.filter(state__isnull=False).last()
 
     ###################
@@ -203,7 +194,18 @@ class Galaxy(models.Model):
     ###################
 
     @classmethod
-    def compute_user_score(cls, user: User) -> int:
+    def get_rulable_users(
+        cls, picture_count_threshold: int = DEFAULT_PICTURE_COUNT_THRESHOLD
+    ) -> QuerySet[User]:
+        return (
+            User.objects.exclude(subscriptions=None)
+            .annotate(pictures_count=Count("pictures"))
+            .filter(pictures_count__gt=picture_count_threshold)
+            .distinct()
+        )
+
+    @classmethod
+    def compute_individual_scores(cls) -> dict[int, int]:
         """Compute an individual score for each citizen.
 
         It will later be used by the graph algorithm to push
@@ -211,87 +213,50 @@ class Galaxy(models.Model):
 
         Idea: This could be added to the computation:
 
-        - Forum posts
         - Picture count
         - Counter consumption
         - Barman time
         - ...
         """
-        user_score = 1
+        users = (
-        user_score += cls.query_user_score(user)
+            User.objects.annotate(
-
+                score=(
+                    Count("godchildren", distinct=True) * cls.FAMILY_LINK_POINTS
+                    + Count("godfathers", distinct=True) * cls.FAMILY_LINK_POINTS
+                    + Count("pictures", distinct=True) * cls.PICTURE_POINTS
+                    + Count("memberships", distinct=True) * cls.CLUBS_POINTS
+                )
+            )
+            .filter(score__gt=0)
+            .values("id", "score")
+        )
         # TODO:
         # Scale that value with some magic number to accommodate to typical data
         # Really active galaxy citizen after 5 years typically have a score of about XXX
         # Citizen that were seen regularly without taking much part in organizations typically have a score of about XXX
         # Citizen that only went to a few events typically score about XXX
-        user_score = int(math.log2(user_score))
+        res = {u["id"]: int(math.log2(u["score"] + 1)) for u in users}
-
+        return res
-        return user_score
-
-    @classmethod
-    def query_user_score(cls, user: User) -> int:
-        """Get the individual score of the given user in the galaxy."""
-        score_query = (
-            User.objects.filter(id=user.id)
-            .annotate(
-                godchildren_count=Count("godchildren", distinct=True)
-                * cls.FAMILY_LINK_POINTS,
-                godfathers_count=Count("godfathers", distinct=True)
-                * cls.FAMILY_LINK_POINTS,
-                pictures_score=Count("pictures", distinct=True) * cls.PICTURE_POINTS,
-                clubs_score=Count("memberships", distinct=True) * cls.CLUBS_POINTS,
-            )
-            .aggregate(
-                score=models.Sum(
-                    F("godchildren_count")
-                    + F("godfathers_count")
-                    + F("pictures_score")
-                    + F("clubs_score")
-                )
-            )
-        )
-        return score_query.get("score")
 
     ####################
     # Inter-user score #
     ####################
 
     @classmethod
-    def compute_users_score(cls, user1: User, user2: User) -> RelationScore:
+    def compute_user_family_score(cls, user: User) -> defaultdict[int, int]:
-        """Compute the relationship scores of the two given users.
-
-        The computation is done with the following fields :
-
-        - family: if they have some godfather/godchild relation
-        - pictures: in how many pictures are both tagged
-        - clubs: during how many days they were members of the same clubs
-        """
-        family = cls.compute_users_family_score(user1, user2)
-        pictures = cls.compute_users_pictures_score(user1, user2)
-        clubs = cls.compute_users_clubs_score(user1, user2)
-        return RelationScore(family=family, pictures=pictures, clubs=clubs)
-
-    @classmethod
-    def compute_users_family_score(cls, user1: User, user2: User) -> int:
         """Compute the family score of the relation between the given users.
 
         This takes into account mutual godfathers.
-
-        Returns:
-             366 if user1 is the godfather of user2 (or vice versa) else 0
         """
-        link_count = User.objects.filter(
+        godchildren = User.objects.filter(godchildren=user).values_list("id", flat=True)
-            Q(id=user1.id, godfathers=user2) | Q(id=user2.id, godfathers=user1)
+        godfathers = User.objects.filter(godfathers=user).values_list("id", flat=True)
-        ).count()
+        result = defaultdict(int)
-        if link_count > 0:
+        for parent in itertools.chain(godchildren, godfathers):
-            cls.logger.debug(
+            result[parent] += cls.FAMILY_LINK_POINTS
-                f"\t\t- '{user1}' and '{user2}' have {link_count} direct family link"
+        return result
-            )
-        return link_count * cls.FAMILY_LINK_POINTS
 
     @classmethod
-    def compute_users_pictures_score(cls, user1: User, user2: User) -> int:
+    def compute_user_pictures_score(cls, user: User) -> defaultdict[int, int]:
         """Compute the pictures score of the relation between the given users.
 
         The pictures score is obtained by counting the number
@@ -301,19 +266,19 @@ class Galaxy(models.Model):
         Returns:
              The number of pictures both users have in common, times 2
         """
-        picture_count = (
+        common_photos = (
-            Picture.objects.filter(people__user__in=(user1,))
+            PeoplePictureRelation.objects.filter(
-            .filter(people__user__in=(user2,))
+                picture__in=Picture.objects.filter(people__user=user)
-            .count()
-        )
-        if picture_count:
-            cls.logger.debug(
-                f"\t\t- '{user1}' was pictured with '{user2}' {picture_count} times"
             )
-        return picture_count * cls.PICTURE_POINTS
+            .values("user")
+            .annotate(count=Count("user"))
+        )
+        return defaultdict(
+            int, {p["user"]: p["count"] * cls.PICTURE_POINTS for p in common_photos}
+        )
 
     @classmethod
-    def compute_users_clubs_score(cls, user1: User, user2: User) -> int:
+    def compute_user_clubs_score(cls, user: User) -> defaultdict[int, int]:
         """Compute the clubs score of the relation between the given users.
 
         The club score is obtained by counting the number of days
@@ -324,54 +289,36 @@ class Galaxy(models.Model):
         (two years) and user2 was a member of the same club from 01/01/2021 to
         31/12/2022 (also two years, but with an offset of one year), then their
         club score is 365.
-
-        Returns:
-            the number of days during which both users were in the same club
         """
-        common_clubs = Club.objects.filter(members__in=user1.memberships.all()).filter(
+        memberships = user.memberships.only("start_date", "end_date", "club_id")
-            members__in=user2.memberships.all()
+        result = defaultdict(int)
-        )
+        now = localdate()
-        user1_memberships = user1.memberships.filter(club__in=common_clubs)
+        for membership in memberships:
-        user2_memberships = user2.memberships.filter(club__in=common_clubs)
+            # This is a N+1 query, but 92% of galaxy users have less than 10 memberships.
-
+            # Only 5 users have more than 30 memberships.
-        score = 0
+            common_memberships = (
-        for user1_membership in user1_memberships:
+                Membership.objects.exclude(user=user)
-            if user1_membership.end_date is None:
+                .filter(
-                # user1_membership.save() is not called in this function, hence this is safe
+                    Q(  # start2 <= start1 <= end2
-                user1_membership.end_date = localdate()
+                        start_date__lte=membership.start_date,
-            query = Q(  # start2 <= start1 <= end2
+                        end_date__gte=membership.start_date,
-                start_date__lte=user1_membership.start_date,
-                end_date__gte=user1_membership.start_date,
-            )
-            query |= Q(  # start2 <= start1 <= now
-                start_date__lte=user1_membership.start_date, end_date=None
-            )
-            query |= Q(  # start1 <= start2 <= end2
-                start_date__gte=user1_membership.start_date,
-                start_date__lte=user1_membership.end_date,
-            )
-            for user2_membership in user2_memberships.filter(
-                query, club=user1_membership.club
-            ):
-                if user2_membership.end_date is None:
-                    user2_membership.end_date = localdate()
-                latest_start = max(
-                    user1_membership.start_date, user2_membership.start_date
-                )
-                earliest_end = min(user1_membership.end_date, user2_membership.end_date)
-                cls.logger.debug(
-                    "\t\t- '%s' was with '%s' in %s starting on %s until %s (%s days)"
-                    % (
-                        user1,
-                        user2,
-                        user2_membership.club,
-                        latest_start,
-                        earliest_end,
-                        (earliest_end - latest_start).days,
                     )
+                    | Q(  # start2 <= start1 <= now
+                        start_date__lte=membership.start_date, end_date=None
+                    )
+                    | Q(  # start1 <= start2 <= end2
+                        start_date__gte=membership.start_date,
+                        start_date__lte=membership.end_date or now,
+                    ),
+                    club_id=membership.club_id,
                 )
-                score += cls.CLUBS_POINTS * (earliest_end - latest_start).days
+                .only("start_date", "end_date", "user_id")
-        return score
+            )
+            for other in common_memberships:
+                start = max(membership.start_date, other.start_date)
+                end = min(membership.end_date or now, other.end_date or now)
+                result[other.user_id] += (end - start).days * cls.CLUBS_POINTS
+        return result
 
     ###################
     # Rule the galaxy #
@@ -406,7 +353,9 @@ class Galaxy(models.Model):
         cls.logger.debug(f"\t\t> Scaled distance: {value}")
         return int(value)
 
-    def rule(self, picture_count_threshold=10) -> None:
+    def rule(
+        self, picture_count_threshold: int = DEFAULT_PICTURE_COUNT_THRESHOLD
+    ) -> None:
         """Main function of the Galaxy.
 
         Iterate over all the rulable users to promote them to citizens.
@@ -427,41 +376,30 @@ class Galaxy(models.Model):
         """
         total_time = time.time()
         self.logger.info("Listing rulable citizen.")
-        rulable_users = (
-            User.objects.filter(subscriptions__isnull=False)
-            .annotate(pictures_count=Count("pictures"))
-            .filter(pictures_count__gt=picture_count_threshold)
-            .distinct()
-        )
 
         # force fetch of the whole query to make sure there won't
         # be any more db hits
         # this is memory expensive but prevents a lot of db hits, therefore
         # is far more time efficient
 
-        rulable_users = list(rulable_users)
+        rulable_users = list(self.get_rulable_users(picture_count_threshold))
         rulable_users_count = len(rulable_users)
         user1_count = 0
         self.logger.info(
             f"{rulable_users_count} citizen have been listed. Starting to rule."
         )
 
-        stars = []
         self.logger.info("Creating stars for all citizen")
-        for user in rulable_users:
+        individual_scores = self.compute_individual_scores()
-            star = GalaxyStar(
+        GalaxyStar.objects.bulk_create(
-                owner=user, galaxy=self, mass=self.compute_user_score(user)
+            [
-            )
+                GalaxyStar(owner=user, galaxy=self, mass=individual_scores[user.id])
-            stars.append(star)
+                for user in rulable_users
-        GalaxyStar.objects.bulk_create(stars)
+            ]
-
+        )
-        stars = {}
+        stars = {star.owner_id: star for star in self.stars.all()}
-        for star in GalaxyStar.objects.filter(galaxy=self):
-            stars[star.owner.id] = star
 
         self.logger.info("Creating lanes between stars")
-        # Display current speed every $speed_count_frequency users
-        speed_count_frequency = max(rulable_users_count // 10, 1)  # ten time at most
         global_avg_speed_accumulator = 0
         global_avg_speed_count = 0
         t_global_start = time.time()
@@ -472,20 +410,19 @@ class Galaxy(models.Model):
 
             star1 = stars[user1.id]
 
-            user_avg_speed = 0
-            user_avg_speed_count = 0
-
-            tstart = time.time()
             lanes = []
-            for user2_count, user2 in enumerate(rulable_users, start=1):
+            family_scores = self.compute_user_family_score(user1)
-                self.logger.debug("")
+            picture_scores = self.compute_user_pictures_score(user1)
-                self.logger.debug(
+            club_scores = self.compute_user_clubs_score(user1)
-                    f"\t> Examining '{user1}' ({user1_count}/{rulable_users_count}) with '{user2}' ({user2_count}/{rulable_users_count2})"
-                )
 
+            for user2 in rulable_users:
                 star2 = stars[user2.id]
 
-                score = Galaxy.compute_users_score(user1, user2)
+                score = RelationScore(
+                    family=family_scores.get(user2.id, 0),
+                    pictures=picture_scores.get(user2.id, 0),
+                    clubs=club_scores.get(user2.id, 0),
+                )
                 distance = self.scale_distance(sum(score))
                 if distance < 30:  # TODO: this needs tuning with real-world data
                     lanes.append(
@@ -498,22 +435,8 @@ class Galaxy(models.Model):
                             clubs=score.clubs,
                         )
                     )
-
-                if user2_count % speed_count_frequency == 0:
-                    tend = time.time()
-                    delta = tend - tstart
-                    speed = float(speed_count_frequency) / delta
-                    user_avg_speed += speed
-                    user_avg_speed_count += 1
-                    self.logger.debug(
-                        f"\tSpeed: {speed:.2f} users per second (time for last {speed_count_frequency} citizens: {delta:.2f} second)"
-                    )
-                    tstart = time.time()
-
             GalaxyLane.objects.bulk_create(lanes)
 
-            self.logger.info("")
-
             t_global_end = time.time()
             global_delta = t_global_end - t_global_start
             speed = 1.0 / global_delta
@@ -521,21 +444,19 @@ class Galaxy(models.Model):
             global_avg_speed_count += 1
             global_avg_speed = global_avg_speed_accumulator / global_avg_speed_count
 
-            self.logger.info(f" Ruling of {self} ".center(60, "#"))
+            if user1_count % 50 == 0:
-            self.logger.info(
+                self.logger.info("")
-                f"Progression: {user1_count}/{rulable_users_count} citizen -- {rulable_users_count - user1_count} remaining"
+                self.logger.info(f" Ruling of {self} ".center(60, "#"))
-            )
+                self.logger.info(
-            self.logger.info(f"Speed: {60.0 * global_avg_speed:.2f} citizen per minute")
+                    f"Progression: {user1_count}/{rulable_users_count} "
-
+                    f"citizen -- {rulable_users_count - user1_count} remaining"
-            # We can divide the computed ETA by 2 because each loop, there is one citizen less to check, and maths tell
+                )
-            # us that this averages to a division by two
+                self.logger.info(f"Speed: {global_avg_speed:.2f} citizen per second")
-            eta = rulable_users_count2 / global_avg_speed / 2
+                eta = rulable_users_count2 // global_avg_speed
-            eta_hours = int(eta // 3600)
+                self.logger.info(
-            eta_minutes = int(eta // 60 % 60)
+                    f"ETA: {int(eta // 60 % 60)} minutes {int(eta % 60)} seconds"
-            self.logger.info(
+                )
-                f"ETA: {eta_hours} hours {eta_minutes} minutes ({eta / 3600 / 24:.2f} days)"
+                self.logger.info("#" * 60)
-            )
-            self.logger.info("#" * 60)
             t_global_start = time.time()
 
         # Here, we get the IDs of the old galaxies that we'll need to delete. In normal operation, only one galaxy
@@ -556,11 +477,10 @@ class Galaxy(models.Model):
             Galaxy.objects.filter(pk__in=old_galaxies_pks).delete()
 
         total_time = time.time() - total_time
-        total_time_hours = int(total_time // 3600)
         total_time_minutes = int(total_time // 60 % 60)
         total_time_seconds = int(total_time % 60)
         self.logger.info(
-            f"{self} ruled in {total_time:.2f} seconds ({total_time_hours} hours, {total_time_minutes} minutes, {total_time_seconds} seconds)"
+            f"{self} ruled in {total_time_minutes} minutes, {total_time_seconds} seconds"
         )
 
     def make_state(self) -> None:
@@ -568,59 +488,34 @@ class Galaxy(models.Model):
         self.logger.info(
             "Caching current Galaxy state for a quicker display of the Empire's power."
         )
-
-        without_nickname = Concat(
-            F("owner__first_name"), Value(" "), F("owner__last_name")
-        )
-        with_nickname = Concat(
-            F("owner__first_name"),
-            Value(" "),
-            F("owner__last_name"),
-            Value(" ("),
-            F("owner__nick_name"),
-            Value(")"),
-        )
         stars = (
             GalaxyStar.objects.filter(galaxy=self)
-            .order_by(
+            .order_by("owner_id")
-                "owner"
+            .select_related("owner")
-            )  # This helps determinism for the tests and doesn't cost much
-            .annotate(
-                owner_name=Case(
-                    When(owner__nick_name=None, then=without_nickname),
-                    default=with_nickname,
-                )
-            )
         )
         lanes = (
             GalaxyLane.objects.filter(star1__galaxy=self)
-            .order_by(
+            .order_by("star1")
-                "star1"
-            )  # This helps determinism for the tests and doesn't cost much
             .annotate(
-                star1_owner=F("star1__owner__id"),
+                star1_owner=F("star1__owner_id"), star2_owner=F("star2__owner_id")
-                star2_owner=F("star2__owner__id"),
             )
         )
         json = GalaxyDict(
             nodes=[
                 StarDict(
-                    id=star.owner_id,
+                    id=star.owner_id, name=star.owner.get_display_name(), mass=star.mass
-                    name=star.owner_name,
-                    mass=star.mass,
                 )
                 for star in stars
             ],
-            links=[],
+            links=[
-        )
-        for path in lanes:
-            json["links"].append(
                 {
                     "source": path.star1_owner,
                     "target": path.star2_owner,
                     "value": path.distance,
                 }
-            )
+                for path in lanes
+            ],
+        )
         self.state = json
         self.save()
         self.logger.info(f"{self} is now ready!")

galaxy/tests.py

@@ -33,7 +33,7 @@ from core.models import User
 from galaxy.models import Galaxy
 
 
-@pytest.mark.skip(reason="Galaxy is disabled for now")
+# @pytest.mark.skip(reason="Galaxy is disabled for now")
 class TestGalaxyModel(TestCase):
     @classmethod
     def setUpTestData(cls):
@@ -48,15 +48,19 @@ class TestGalaxyModel(TestCase):
 
     def test_user_self_score(self):
         """Test that individual user scores are correct."""
-        with self.assertNumQueries(8):
+        with self.assertNumQueries(1):
-            assert Galaxy.compute_user_score(self.root) == 9
+            scores = Galaxy.compute_individual_scores()
-            assert Galaxy.compute_user_score(self.skia) == 10
+            expected = {
-            assert Galaxy.compute_user_score(self.sli) == 8
+                self.root.id: 9,
-            assert Galaxy.compute_user_score(self.krophil) == 2
+                self.skia.id: 10,
-            assert Galaxy.compute_user_score(self.richard) == 10
+                self.sli.id: 8,
-            assert Galaxy.compute_user_score(self.subscriber) == 8
+                self.krophil.id: 2,
-            assert Galaxy.compute_user_score(self.public) == 8
+                self.richard.id: 10,
-            assert Galaxy.compute_user_score(self.com) == 1
+                self.subscriber.id: 8,
+                self.public.id: 8,
+                self.com.id: 1,
+            }
+            assert scores.items() >= expected.items()
 
     def test_users_score(self):
         """Test on the default dataset generated by the `populate` command
@@ -118,17 +122,23 @@ class TestGalaxyModel(TestCase):
             self.com,
         ]
 
-        with self.assertNumQueries(100):
+        with self.assertNumQueries(44):
             while len(users) > 0:
                 user1 = users.pop(0)
+                family_scores = Galaxy.compute_user_family_score(user1)
+                picture_scores = Galaxy.compute_user_pictures_score(user1)
+                club_scores = Galaxy.compute_user_clubs_score(user1)
                 for user2 in users:
-                    score = Galaxy.compute_users_score(user1, user2)
                     u1 = computed_scores.get(user1.username, {})
                     u1[user2.username] = {
-                        "score": sum(score),
+                        "score": (
-                        "family": score.family,
+                            family_scores[user2.id]
-                        "pictures": score.pictures,
+                            + picture_scores[user2.id]
-                        "clubs": score.clubs,
+                            + club_scores[user2.id]
+                        ),
+                        "family": family_scores[user2.id],
+                        "pictures": picture_scores[user2.id],
+                        "clubs": club_scores[user2.id],
                     }
                     computed_scores[user1.username] = u1
 
@@ -140,12 +150,12 @@ class TestGalaxyModel(TestCase):
         that the number of queries to rule the galaxy is stable.
         """
         galaxy = Galaxy.objects.create()
-        with self.assertNumQueries(58):
+        with self.assertNumQueries(39):
             galaxy.rule(0)  # We want everybody here
 
 
 @pytest.mark.slow
-@pytest.mark.skip(reason="Galaxy is disabled for now")
+# @pytest.mark.skip(reason="Galaxy is disabled for now")
 class TestGalaxyView(TestCase):
     @classmethod
     def setUpTestData(cls):

locale/fr/LC_MESSAGES/django.po

@@ -5103,8 +5103,9 @@ msgid "There are %s pictures to be moderated in the SAS"
 msgstr "Il y a %s photos à modérer dans le SAS"
 
 #: sith/settings.py
-msgid "You've been identified on some pictures"
+#, python-format
-msgstr "Vous avez été identifié sur des photos"
+msgid "You've been identified in album %s"
+msgstr "Vous avez été identifié dans l'album %s"
 
 #: sith/settings.py
 #, python-format

pyproject.toml

@@ -44,7 +44,7 @@ dependencies = [
     "django-honeypot>=1.3.0,<2",
     "pydantic-extra-types<3.0.0,>=2.10.3",
     "ical>=10.0.3,<11",
-    "redis[hiredis]>=5.3.0,<7.0.0",
+    "redis[hiredis]<6.0.0,>=5.3.0",
     "environs[django]<15.0.0,>=14.1.1",
     "requests>=2.32.3",
     "honcho>=2.0.0",

sas/api.py

@@ -2,7 +2,6 @@ from typing import Any, Literal
 
 from django.conf import settings
 from django.core.exceptions import ValidationError
-from django.db.models import F
 from django.urls import reverse
 from ninja import Body, File, Query
 from ninja.security import SessionAuth
@@ -105,8 +104,7 @@ class PicturesController(ControllerBase):
             filters.filter(Picture.objects.viewable_by(user))
             .distinct()
             .order_by("-parent__date", "date")
-            .select_related("owner")
+            .select_related("owner", "parent")
-            .annotate(album=F("parent__name"))
         )
 
     @route.post(
@@ -153,7 +151,9 @@ class PicturesController(ControllerBase):
 
     @route.put("/{picture_id}/identified", permissions=[IsAuthenticated, CanView])
     def identify_users(self, picture_id: NonNegativeInt, users: set[NonNegativeInt]):
-        picture = self.get_object_or_exception(Picture, pk=picture_id)
+        picture = self.get_object_or_exception(
+            Picture.objects.select_related("parent"), pk=picture_id
+        )
         db_users = list(User.objects.filter(id__in=users))
         if len(users) != len(db_users):
             raise NotFound
@@ -166,13 +166,15 @@ class PicturesController(ControllerBase):
         ]
         PeoplePictureRelation.objects.bulk_create(relations)
         for u in identified:
+            html_id = f"album-{picture.parent_id}"
+            url = reverse(
+                "sas:user_pictures", kwargs={"user_id": u.id}, fragment=html_id
+            )
             Notification.objects.get_or_create(
                 user=u,
                 viewed=False,
                 type="NEW_PICTURES",
-                defaults={
+                defaults={"url": url, "param": picture.parent.name},
-                    "url": reverse("sas:user_pictures", kwargs={"user_id": u.id})
-                },
             )
 
     @route.delete("/{picture_id}", permissions=[IsSasAdmin])

sas/migrations/0005_alter_sasfile_options.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.2.3 on 2025-06-17 18:53
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [("sas", "0004_picturemoderationrequest_and_more")]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="sasfile",
+            options={
+                "permissions": [
+                    ("moderate_sasfile", "Can moderate SAS files"),
+                    ("view_unmoderated_sasfile", "Can view not moderated SAS files"),
+                ]
+            },
+        ),
+    ]

sas/models.py

@@ -25,11 +25,10 @@ from django.core.cache import cache
 from django.db import models
 from django.db.models import Exists, OuterRef, Q
 from django.urls import reverse
-from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from PIL import Image
 
-from core.models import SithFile, User
+from core.models import Notification, SithFile, User
 from core.utils import exif_auto_rotate, resize_image
 
 
@@ -42,6 +41,10 @@ class SasFile(SithFile):
 
     class Meta:
         proxy = True
+        permissions = [
+            ("moderate_sasfile", "Can moderate SAS files"),
+            ("view_unmoderated_sasfile", "Can view not moderated SAS files"),
+        ]
 
     def can_be_viewed_by(self, user):
         if user.is_anonymous:
@@ -60,7 +63,7 @@ class SasFile(SithFile):
         return self.id in viewable
 
     def can_be_edited_by(self, user):
-        return user.is_root or user.is_in_group(pk=settings.SITH_GROUP_SAS_ADMIN_ID)
+        return user.has_perm("sas.change_sasfile")
 
 
 class PictureQuerySet(models.QuerySet):
@@ -70,7 +73,7 @@ class PictureQuerySet(models.QuerySet):
         Warning:
             Calling this queryset method may add several additional requests.
         """
-        if user.is_root or user.is_in_group(pk=settings.SITH_GROUP_SAS_ADMIN_ID):
+        if user.has_perm("sas.moderate_sasfile"):
             return self.all()
         if user.was_subscribed:
             return self.filter(Q(is_moderated=True) | Q(owner=user))
@@ -183,7 +186,7 @@ class AlbumQuerySet(models.QuerySet):
         Warning:
             Calling this queryset method may add several additional requests.
         """
-        if user.is_root or user.is_in_group(pk=settings.SITH_GROUP_SAS_ADMIN_ID):
+        if user.has_perm("sas.moderate_sasfile"):
             return self.all()
         if user.was_subscribed:
             return self.filter(Q(is_moderated=True) | Q(owner=user))
@@ -256,14 +259,10 @@ class Album(SasFile):
             self.save()
 
 
-def sas_notification_callback(notif):
+def sas_notification_callback(notif: Notification):
     count = Picture.objects.filter(is_moderated=False).count()
-    if count:
+    notif.viewed = not bool(count)
-        notif.viewed = False
+    notif.param = str(count)
-    else:
-        notif.viewed = True
-    notif.param = "%s" % count
-    notif.date = timezone.now()
 
 
 class PeoplePictureRelation(models.Model):

sas/schemas.py

@@ -18,6 +18,12 @@ class AlbumFilterSchema(FilterSchema):
     parent_id: int | None = Field(None, q="parent_id")
 
 
+class SimpleAlbumSchema(ModelSchema):
+    class Meta:
+        model = Album
+        fields = ["id", "name"]
+
+
 class AlbumSchema(ModelSchema):
     class Meta:
         model = Album
@@ -70,7 +76,7 @@ class PictureSchema(ModelSchema):
     full_size_url: str
     compressed_url: str
     thumb_url: str
-    album: str
+    album: SimpleAlbumSchema = Field(alias="parent")
     report_url: str
     edit_url: str
 

sas/static/bundled/sas/user/pictures-index.ts

@@ -22,11 +22,11 @@ document.addEventListener("alpine:init", () => {
       } as PicturesFetchPicturesData);
 
       this.albums = this.pictures.reduce(
-        (acc: Record<string, PictureSchema[]>, picture: PictureSchema) => {
+        (acc: Record<number, PictureSchema[]>, picture: PictureSchema) => {
-          if (!acc[picture.album]) {
+          if (!acc[picture.album.id]) {
-            acc[picture.album] = [];
+            acc[picture.album.id] = [];
           }
-          acc[picture.album].push(picture);
+          acc[picture.album.id].push(picture);
           return acc;
         },
         {},

sas/templates/sas/user_pictures.jinja

@@ -20,11 +20,11 @@
       {{ download_button(_("Download all my pictures")) }}
     {% endif %}
 
-    <template x-for="[album, pictures] in Object.entries(albums)" x-cloak>
+    <template x-for="[album_id, pictures] in Object.entries(albums)" x-cloak>
       <section>
         <br />
         <div class="row">
-          <h4 x-text="album"></h4>
+          <h4 x-text="pictures[0].album.name" :id="`album-${album_id}`"></h4>
           {% if user.id == object.id %}
             &nbsp;{{ download_button("") }}
           {% endif %}

sith/settings.py

@@ -677,7 +677,7 @@ SITH_NOTIFICATIONS = [
     ("NEWS_MODERATION", _("There are %s fresh news to be moderated")),
     ("FILE_MODERATION", _("New files to be moderated")),
     ("SAS_MODERATION", _("There are %s pictures to be moderated in the SAS")),
-    ("NEW_PICTURES", _("You've been identified on some pictures")),
+    ("NEW_PICTURES", _("You've been identified in album %s")),
     ("REFILLING", _("You just refilled of %s €")),
     ("SELLING", _("You just bought %s")),
     ("GENERIC", _("You have a notification")),