diff --git a/core/admin.py b/core/admin.py index 5de89ada..051cab37 100644 --- a/core/admin.py +++ b/core/admin.py @@ -79,9 +79,9 @@ class PageAdmin(admin.ModelAdmin): @admin.register(SithFile) class SithFileAdmin(admin.ModelAdmin): - list_display = ("name", "owner", "size", "date", "is_in_sas") + list_display = ("name", "owner", "size", "date") autocomplete_fields = ("parent", "owner", "moderator") - search_fields = ("name", "parent__name") + search_fields = ("name",) @admin.register(OperationLog) diff --git a/core/api.py b/core/api.py index e1b3bbbd..4c8c1d5d 100644 --- a/core/api.py +++ b/core/api.py @@ -73,7 +73,7 @@ class SithFileController(ControllerBase): ) @paginate(PageNumberPaginationExtra, page_size=50) def search_files(self, search: Annotated[str, annotated_types.MinLen(1)]): - return SithFile.objects.filter(is_in_sas=False).filter(name__icontains=search) + return SithFile.objects.filter(name__icontains=search) @api_controller("/group") diff --git a/core/management/commands/populate.py b/core/management/commands/populate.py index 26bc6074..05eca0b6 100644 --- a/core/management/commands/populate.py +++ b/core/management/commands/populate.py @@ -118,7 +118,6 @@ class Command(BaseCommand): p.save(force_lock=True) club_root = SithFile.objects.create(name="clubs", owner=root) - sas = SithFile.objects.create(name="SAS", owner=root) main_club = Club.objects.create( id=1, name=settings.SITH_MAIN_CLUB["name"], @@ -805,14 +804,7 @@ Welcome to the wiki page! # SAS for f in self.SAS_FIXTURE_PATH.glob("*"): if f.is_dir(): - album = Album( - parent=sas, - name=f.name, - owner=root, - is_folder=True, - is_in_sas=True, - is_moderated=True, - ) + album = Album(name=f.name) album.clean() album.save() for p in f.iterdir(): @@ -820,17 +812,13 @@ Welcome to the wiki page! pict = Picture( parent=album, name=p.name, - file=file, + original=file, owner=root, - is_folder=False, - is_in_sas=True, is_moderated=True, - mime_type="image/webp", - size=file.size, ) - pict.file.name = p.name - pict.clean() + pict.original.name = pict.name pict.generate_thumbnails() + pict.save() img_skia = Picture.objects.get(name="skia.jpg") img_sli = Picture.objects.get(name="sli.jpg") diff --git a/core/migrations/0044_alter_userban_options_remove_sithfile_is_in_sas.py b/core/migrations/0044_alter_userban_options_remove_sithfile_is_in_sas.py new file mode 100644 index 00000000..ade779ee --- /dev/null +++ b/core/migrations/0044_alter_userban_options_remove_sithfile_is_in_sas.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.17 on 2025-01-26 15:01 + +from typing import TYPE_CHECKING + +from django.db import migrations +from django.db.migrations.state import StateApps + +if TYPE_CHECKING: + import core.models + + +def remove_sas_sithfiles(apps: StateApps, schema_editor): + SithFile: type[core.models.SithFile] = apps.get_model("core", "SithFile") + SithFile.objects.all().delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0043_bangroup_alter_group_description_alter_user_groups_and_more"), + ("sas", "0006_alter_peoplepicturerelation_picture_and_more"), + ] + + operations = [ + migrations.RunPython( + remove_sas_sithfiles, reverse_code=migrations.RunPython.noop, elidable=True + ), + migrations.AlterModelOptions( + name="userban", + options={"verbose_name": "user ban", "verbose_name_plural": "user bans"}, + ), + migrations.RemoveField( + model_name="sithfile", + name="is_in_sas", + ), + ] diff --git a/core/models.py b/core/models.py index b1caa912..7f482c85 100644 --- a/core/models.py +++ b/core/models.py @@ -898,9 +898,6 @@ class SithFile(models.Model): on_delete=models.CASCADE, ) asked_for_removal = models.BooleanField(_("asked for removal"), default=False) - is_in_sas = models.BooleanField( - _("is in the SAS"), default=False, db_index=True - ) # Allows to query this flag, updated at each call to save() class Meta: verbose_name = _("file") @@ -909,24 +906,12 @@ class SithFile(models.Model): return self.get_parent_path() + "/" + self.name def save(self, *args, **kwargs): - sas = SithFile.objects.filter(id=settings.SITH_SAS_ROOT_DIR_ID).first() - self.is_in_sas = sas in self.get_parent_list() or self == sas copy_rights = False if self.id is None: copy_rights = True super().save(*args, **kwargs) if copy_rights: self.copy_rights() - if self.is_in_sas: - for user in User.objects.filter( - groups__id__in=[settings.SITH_GROUP_SAS_ADMIN_ID] - ): - Notification( - user=user, - url=reverse("sas:moderation"), - type="SAS_MODERATION", - param="1", - ).save() def is_owned_by(self, user: User) -> bool: if user.is_anonymous: @@ -939,8 +924,6 @@ class SithFile(models.Model): return user.is_board_member if user.is_com_admin: return True - if self.is_in_sas and user.is_in_group(pk=settings.SITH_GROUP_SAS_ADMIN_ID): - return True return user.id == self.owner_id def can_be_viewed_by(self, user: User) -> bool: @@ -1106,18 +1089,6 @@ class SithFile(models.Model): def is_file(self): return not self.is_folder - @cached_property - def as_picture(self): - from sas.models import Picture - - return Picture.objects.filter(id=self.id).first() - - @cached_property - def as_album(self): - from sas.models import Album - - return Album.objects.filter(id=self.id).first() - def get_parent_list(self): parents = [] current = self.parent diff --git a/core/views/files.py b/core/views/files.py index 04498d5c..15993e51 100644 --- a/core/views/files.py +++ b/core/views/files.py @@ -402,7 +402,7 @@ class FileDeleteView(AllowFragment, CanEditPropMixin, DeleteView): class FileModerationView(AllowFragment, ListView): model = SithFile template_name = "core/file_moderation.jinja" - queryset = SithFile.objects.filter(is_moderated=False, is_in_sas=False) + queryset = SithFile.objects.filter(is_moderated=False) paginate_by = 100 def dispatch(self, request: HttpRequest, *args, **kwargs): diff --git a/galaxy/management/commands/generate_galaxy_test_data.py b/galaxy/management/commands/generate_galaxy_test_data.py index d0dea4a5..0f34dd6e 100644 --- a/galaxy/management/commands/generate_galaxy_test_data.py +++ b/galaxy/management/commands/generate_galaxy_test_data.py @@ -25,13 +25,12 @@ import warnings from datetime import timedelta from typing import Final, Optional -from django.conf import settings from django.core.files.base import ContentFile from django.core.management.base import BaseCommand from django.utils import timezone from club.models import Club, Membership -from core.models import Group, Page, SithFile, User +from core.models import Group, Page, User from sas.models import Album, PeoplePictureRelation, Picture from subscription.models import Subscription @@ -98,13 +97,8 @@ class Command(BaseCommand): self.NB_CLUBS = options["club_count"] root = User.objects.filter(username="root").first() - sas = SithFile.objects.get(id=settings.SITH_SAS_ROOT_DIR_ID) self.galaxy_album = Album.objects.create( - name="galaxy-register-file", - owner=root, - is_moderated=True, - is_in_sas=True, - parent=sas, + name="galaxy-register-file", owner=root, is_moderated=True ) self.make_clubs() @@ -288,14 +282,10 @@ class Command(BaseCommand): owner=u, name=f"galaxy-picture {u} {i // self.NB_USERS}", is_moderated=True, - is_folder=False, parent=self.galaxy_album, - is_in_sas=True, - file=ContentFile(RED_PIXEL_PNG), + original=ContentFile(RED_PIXEL_PNG), compressed=ContentFile(RED_PIXEL_PNG), thumbnail=ContentFile(RED_PIXEL_PNG), - mime_type="image/png", - size=len(RED_PIXEL_PNG), ) ) self.picts[i].file.name = self.picts[i].name diff --git a/locale/fr/LC_MESSAGES/django.po b/locale/fr/LC_MESSAGES/django.po index dc6b5ee6..25bc0a96 100644 --- a/locale/fr/LC_MESSAGES/django.po +++ b/locale/fr/LC_MESSAGES/django.po @@ -6,7 +6,7 @@ msgid "" msgstr "" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2025-01-19 18:12+0100\n" +"POT-Creation-Date: 2025-01-26 12:47+0100\n" "PO-Revision-Date: 2016-07-18\n" "Last-Translator: Maréchal \n" @@ -17,7 +17,7 @@ msgstr "" "Plural-Forms: nplurals=2; plural=(n > 1);\n" #: accounting/models.py club/models.py com/models.py counter/models.py -#: forum/models.py launderette/models.py +#: forum/models.py launderette/models.py sas/models.py msgid "name" msgstr "nom" @@ -935,10 +935,6 @@ msgstr "rôle" msgid "description" msgstr "description" -#: club/models.py -msgid "past member" -msgstr "ancien membre" - #: club/models.py msgid "Email address" msgstr "Adresse email" @@ -948,7 +944,7 @@ msgid "Enter a valid address. Only the root of the address is needed." msgstr "" "Entrez une adresse valide. Seule la racine de l'adresse est nécessaire." -#: club/models.py com/models.py core/models.py +#: club/models.py com/models.py core/models.py sas/models.py msgid "is moderated" msgstr "est modéré" @@ -2052,11 +2048,11 @@ msgstr "avoir une notification pour chaque click" msgid "get a notification for every refilling" msgstr "avoir une notification pour chaque rechargement" -#: core/models.py sas/forms.py +#: core/models.py sas/models.py msgid "file name" msgstr "nom du fichier" -#: core/models.py +#: core/models.py sas/models.py msgid "parent" msgstr "parent" @@ -2064,11 +2060,11 @@ msgstr "parent" msgid "compressed file" msgstr "version allégée" -#: core/models.py +#: core/models.py sas/models.py msgid "thumbnail" msgstr "miniature" -#: core/models.py +#: core/models.py sas/models.py msgid "owner" msgstr "propriétaire" @@ -2092,14 +2088,10 @@ msgstr "type mime" msgid "size" msgstr "taille" -#: core/models.py +#: core/models.py sas/models.py msgid "asked for removal" msgstr "retrait demandé" -#: core/models.py -msgid "is in the SAS" -msgstr "est dans le SAS" - #: core/models.py msgid "Character '/' not authorized in name" msgstr "Le caractère '/' n'est pas autorisé dans les noms de fichier" @@ -3299,8 +3291,8 @@ msgstr "Nom d'utilisateur, email, ou numéro de compte AE" #: core/views/forms.py msgid "" -"Profile: you need to be visible on the picture, in order to be recognized " -"(e.g. by the barmen)" +"Profile: you need to be visible on the picture, in order to be recognized (e." +"g. by the barmen)" msgstr "" "Photo de profil: vous devez être visible sur la photo afin d'être reconnu " "(par exemple par les barmen)" @@ -3642,7 +3634,7 @@ msgstr "élément de relevé de caisse" msgid "banner" msgstr "bannière" -#: counter/models.py +#: counter/models.py sas/models.py msgid "event date" msgstr "date de l'événement" @@ -3906,8 +3898,8 @@ msgstr "" #: counter/templates/counter/mails/account_dump.jinja msgid "If you think this was a mistake, please mail us at ae@utbm.fr." msgstr "" -"Si vous pensez qu'il s'agit d'une erreur, veuillez envoyer un mail à " -"ae@utbm.fr." +"Si vous pensez qu'il s'agit d'une erreur, veuillez envoyer un mail à ae@utbm." +"fr." #: counter/templates/counter/mails/account_dump.jinja msgid "" @@ -4324,11 +4316,11 @@ msgstr "début des candidatures" msgid "end candidature" msgstr "fin des candidatures" -#: election/models.py +#: election/models.py sas/models.py msgid "edit groups" msgstr "groupe d'édition" -#: election/models.py +#: election/models.py sas/models.py msgid "view groups" msgstr "groupe de vue" @@ -5143,6 +5135,22 @@ msgstr "Erreur de création de l'album %(album)s : %(msg)s" msgid "You already requested moderation for this picture." msgstr "Vous avez déjà déposé une demande de retrait pour cette photo." +#: sas/models.py +msgid "The date on which the photos in this album were taken" +msgstr "La date à laquelle les photos de cet album ont été prises" + +#: sas/models.py +msgid "album" +msgstr "album" + +#: sas/models.py +msgid "original image" +msgstr "image originale" + +#: sas/models.py +msgid "compressed image" +msgstr "version compressée" + #: sas/models.py msgid "picture" msgstr "photo" diff --git a/sas/admin.py b/sas/admin.py index ac980341..b06f364a 100644 --- a/sas/admin.py +++ b/sas/admin.py @@ -20,9 +20,9 @@ from sas.models import Album, PeoplePictureRelation, Picture, PictureModerationR @admin.register(Picture) class PictureAdmin(admin.ModelAdmin): - list_display = ("name", "parent", "date", "size", "is_moderated") + list_display = ("name", "parent", "is_moderated") search_fields = ("name",) - autocomplete_fields = ("owner", "parent", "edit_groups", "view_groups", "moderator") + autocomplete_fields = ("owner", "parent", "moderator") @admin.register(PeoplePictureRelation) @@ -33,9 +33,9 @@ class PeoplePictureRelationAdmin(admin.ModelAdmin): @admin.register(Album) class AlbumAdmin(admin.ModelAdmin): - list_display = ("name", "parent", "date", "owner", "is_moderated") + list_display = ("name", "parent") search_fields = ("name",) - autocomplete_fields = ("owner", "parent", "edit_groups", "view_groups") + autocomplete_fields = ("parent", "edit_groups", "view_groups") @admin.register(PictureModerationRequest) diff --git a/sas/api.py b/sas/api.py index 96bafb87..f2408194 100644 --- a/sas/api.py +++ b/sas/api.py @@ -69,7 +69,7 @@ class PicturesController(ControllerBase): return ( filters.filter(Picture.objects.viewable_by(user)) .distinct() - .order_by("-parent__date", "date") + .order_by("-parent__event_date", "created_at") .select_related("owner") .annotate(album=F("parent__name")) ) diff --git a/sas/baker_recipes.py b/sas/baker_recipes.py index 1f7a7667..974b0aec 100644 --- a/sas/baker_recipes.py +++ b/sas/baker_recipes.py @@ -3,13 +3,7 @@ from model_bakery.recipe import Recipe from sas.models import Picture -picture_recipe = Recipe( - Picture, - is_in_sas=True, - is_folder=False, - is_moderated=True, - name=seq("Picture "), -) +picture_recipe = Recipe(Picture, is_moderated=True, name=seq("Picture ")) """A SAS Picture fixture. Warnings: diff --git a/sas/forms.py b/sas/forms.py index 926fe6ca..885005dd 100644 --- a/sas/forms.py +++ b/sas/forms.py @@ -23,12 +23,7 @@ class SASForm(forms.Form): def process(self, parent, owner, files, *, automodere=False): try: if self.cleaned_data["album_name"] != "": - album = Album( - parent=parent, - name=self.cleaned_data["album_name"], - owner=owner, - is_moderated=automodere, - ) + album = Album(parent=parent, name=self.cleaned_data["album_name"]) album.clean() album.save() except Exception as e: @@ -41,11 +36,8 @@ class SASForm(forms.Form): new_file = Picture( parent=parent, name=f.name, - file=f, + original=f, owner=owner, - mime_type=f.content_type, - size=f.size, - is_folder=False, is_moderated=automodere, ) if automodere: @@ -72,13 +64,12 @@ class PictureEditForm(forms.ModelForm): class AlbumEditForm(forms.ModelForm): class Meta: model = Album - fields = ["name", "date", "file", "parent", "edit_groups"] + fields = ["name", "date", "thumbnail", "parent", "edit_groups"] widgets = { "parent": AutoCompleteSelectAlbum, "edit_groups": AutoCompleteSelectMultipleGroup, } - name = forms.CharField(max_length=Album.NAME_MAX_LENGTH, label=_("file name")) date = forms.DateField(label=_("Date"), widget=SelectDate, required=True) recursive = forms.BooleanField(label=_("Apply rights recursively"), required=False) diff --git a/sas/migrations/0005_delete_album_delete_picture_delete_sasfile_and_more.py b/sas/migrations/0005_delete_album_delete_picture_delete_sasfile_and_more.py new file mode 100644 index 00000000..0ea75c5c --- /dev/null +++ b/sas/migrations/0005_delete_album_delete_picture_delete_sasfile_and_more.py @@ -0,0 +1,341 @@ +# Generated by Django 4.2.17 on 2025-01-22 21:53 +import collections +import itertools +import logging +from typing import TYPE_CHECKING + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models +from django.db.migrations.state import StateApps + +import sas.models + +if TYPE_CHECKING: + import core.models + + +def copy_albums_and_pictures(apps: StateApps, schema_editor): + SithFile: type[core.models.SithFile] = apps.get_model("core", "SithFile") + Album: type[sas.models.Album] = apps.get_model("sas", "Album") + Picture: type[sas.models.Picture] = apps.get_model("sas", "Picture") + logger = logging.getLogger("django") + + # Il y a environ 1800 albums, 257k photos et 488k identifications + # d'utilisateurs dans la db de prod. + # En supposant qu'une insertion prenne 10ms (ce qui est très optimiste), + # migrer tous les enregistrements de la db prendrait plus de 2h. + # C'est trop long. + # Mais d'un autre côté, j'ai pas assez confiance dans les capacités de nos + # machines pour charger presque un million d'objets en mémoire. + # Pour faire un compromis, les albums sont migrés individuellement un à un, + # mais tous les objets liés à ces albums + # (photos, groupes de vue, groupe d'édition, identification d'utilisateurs) + # sont migrés en tas. + # + # Ordre des opérations : + # 1. On migre les albums 1 à 1 (il y en a 1800, donc c'est relativement court) + # 2. On migre les photos par paquet de 2500 (soit ~une centaine d'opérations) + # 3. On migre tous les groupes de vue et tous les groupes d'édition des albums + # + # Au total, la migration devrait demander aux alentours de 2000 insertions, + # ce qui est un compromis acceptable entre une migration + # pas trop longue et une RAM pas trop surchargée. + # + # Pour ce qui est de la répartition des tables, quatre nouvelles tables + # sont créées : sas_album, sas_picture, + # sas_pictureviewgroups et sas_picture_editgroups. + # Tous les albums et toutes les photos qui sont dans core_sithfile + # vont être copiés dans ces tables. + # Comme les albums sont migrés un à un, ils recevront une nouvelle + # clef primaire. + # Pour les photos, en revanche, c'est beaucoup plus sûr de leur donner + # le même id que celui qu'il y avait dans core_sithfile. + # + # Les identifications des photos ne sont pas migrées pour l'instant. + # Ce qu'on va faire, c'est qu'on va changer la contrainte de clef étrangère + # sur la colonne des photos pour pointer vers sas_picture + # au lieu de core_sithfile. + # Cependant, pour que ça marche, + # il faut qu'au moment où ce changement est effectué, + # toutes les clefs primaires référencées existent à la fois dans + # les deux tables, sinon les contraintes d'intégrité ne sont pas respectées. + # La migration de ce fichier va donc s'occuper de créer les nouvelles tables + # et d'y copier les données nécessaires. + # Puis une deuxième migration s'occupera de changer les contraintes. + # Et enfin une troisième migration supprimera les anciennes données. + # + # Pavé César + + albums = SithFile.objects.filter(is_in_sas=True, is_folder=True).prefetch_related( + "view_groups", "edit_groups" + ) + old_albums = collections.deque( + albums.filter(parent_id=settings.SITH_SAS_ROOT_DIR_ID) + ) + + # Changement de représentation en DB. + # Dans l'ancien système, un fichier était dans le SAS si + # un fichier spécial (le SAS_ROOT) était parmi ses ancêtres. + # Comme maintenant les fichiers du SAS sont dans des tables à part, + # il ne peut plus y avoir de confusion. + # Les photos ont donc obligatoirement un parent (qui est un album) + # et les albums peuvent avoir un parent null. + # Un album sans parent est considéré comme se trouvant à la racine + # de l'arborescence. + # En quelque sorte, None est le nouveau SITH_SAS_ROOT_DIR_ID + album_id_old_to_new = {settings.SITH_SAS_ROOT_DIR_ID: None} + + logger.info(f"migrating {albums.count()} albums") + while len(old_albums) > 0: + # Comme les albums référencent leur parent, les albums doivent être migrés + # par ordre croissant de profondeur dans l'arborescence. + # Chaque album est donc pris par la gauche de la file + # et ses enfants ajoutés sur la droite. + old_album = old_albums.popleft() + old_albums.extend(list(albums.filter(parent=old_album))) + new_album = Album.objects.create( + parent_id=album_id_old_to_new[old_album.parent_id], + event_date=old_album.date.date(), + name=old_album.name, + thumbnail=(old_album.file or None), + ) + # on garde un dictionnaire qui associe les id des albums dans l'ancienne table + # à leur id dans la nouvelle table, pour pouvoir recréer + # les liens de parenté entre albums + album_id_old_to_new[old_album.id] = new_album.id + + pictures = SithFile.objects.filter(is_in_sas=True, is_folder=False) + nb_pictures = pictures.count() + logger.info(f"migrating {nb_pictures} pictures") + for i, pictures_batch in enumerate(itertools.batched(pictures, 2500), start=1): + Picture.objects.bulk_create( + [ + Picture( + id=p.id, + name=p.name, + parent_id=album_id_old_to_new[p.parent_id], + thumbnail=p.thumbnail, + compressed=p.compressed, + original=p.file, + owner_id=p.owner_id, + created_at=p.date, + is_moderated=p.is_moderated, + asked_for_removal=p.asked_for_removal, + moderator_id=p.moderator_id, + ) + for p in pictures_batch + ] + ) + logger.info(f"Migrated {min(i * 2500, nb_pictures)} / {nb_pictures} pictures") + + logger.info("Migrating album groups") + albums = SithFile.objects.filter(is_in_sas=True, is_folder=True).exclude( + id=settings.SITH_SAS_ROOT_DIR_ID + ) + Album.edit_groups.through.objects.bulk_create( + [ + Album.view_groups.through( + album=album_id_old_to_new[g.sithfile_id], group_id=g.group_id + ) + for g in SithFile.view_groups.through.objects.filter(sithfile__in=albums) + ] + ) + Album.edit_groups.through.objects.bulk_create( + [ + Album.view_groups.through( + album=album_id_old_to_new[g.sithfile_id], group_id=g.group_id + ) + for g in SithFile.view_groups.through.objects.filter(sithfile__in=albums) + ] + ) + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("core", "0043_bangroup_alter_group_description_alter_user_groups_and_more"), + ("sas", "0004_picturemoderationrequest_and_more"), + ] + + operations = [ + # les relations et les demandes de modération étaient liées à SithFile, + # via le model proxy Picture. + # Pour que la migration marche malgré la disparition du modèle Proxy, + # on change la relation pour qu'elle pointe directement vers SithFile + migrations.AlterField( + model_name="peoplepicturerelation", + name="picture", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="people", + to="core.sithfile", + verbose_name="picture", + ), + ), + migrations.AlterField( + model_name="picturemoderationrequest", + name="picture", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="moderation_requests", + to="core.sithfile", + verbose_name="Picture", + ), + ), + migrations.DeleteModel(name="Album"), + migrations.DeleteModel(name="Picture"), + migrations.DeleteModel(name="SasFile"), + migrations.CreateModel( + name="Album", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "thumbnail", + models.FileField( + max_length=256, + upload_to=sas.models.get_thumbnail_directory, + verbose_name="thumbnail", + ), + ), + ("name", models.CharField(max_length=100, verbose_name="name")), + ( + "event_date", + models.DateField( + default=django.utils.timezone.localdate, + help_text="The date on which the photos in this album were taken", + verbose_name="event date", + ), + ), + ( + "edit_groups", + models.ManyToManyField( + related_name="editable_albums", + to="core.group", + verbose_name="edit groups", + ), + ), + ( + "parent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="sas.album", + verbose_name="parent", + ), + ), + ( + "view_groups", + models.ManyToManyField( + related_name="viewable_albums", + to="core.group", + verbose_name="view groups", + ), + ), + ], + options={"verbose_name": "album"}, + ), + migrations.CreateModel( + name="Picture", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "thumbnail", + models.FileField( + unique=True, + upload_to=sas.models.get_thumbnail_directory, + verbose_name="thumbnail", + max_length=256, + ), + ), + ("name", models.CharField(max_length=256, verbose_name="file name")), + ( + "original", + models.FileField( + unique=True, + upload_to=sas.models.get_directory, + verbose_name="original image", + max_length=256, + ), + ), + ( + "compressed", + models.FileField( + unique=True, + upload_to=sas.models.get_compressed_directory, + verbose_name="compressed image", + max_length=256, + ), + ), + ("created_at", models.DateTimeField(default=django.utils.timezone.now)), + ( + "is_moderated", + models.BooleanField(default=False, verbose_name="is moderated"), + ), + ( + "asked_for_removal", + models.BooleanField( + default=False, verbose_name="asked for removal" + ), + ), + ( + "moderator", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="moderated_pictures", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="owned_pictures", + to=settings.AUTH_USER_MODEL, + verbose_name="owner", + ), + ), + ( + "parent", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="pictures", + to="sas.album", + verbose_name="album", + ), + ), + ], + options={"abstract": False}, + ), + migrations.AddConstraint( + model_name="album", + constraint=models.UniqueConstraint( + fields=("name", "parent"), name="unique_album_name_if_same_parent" + ), + ), + migrations.RunPython( + copy_albums_and_pictures, + reverse_code=migrations.RunPython.noop, + elidable=True, + ), + ] diff --git a/sas/migrations/0006_alter_peoplepicturerelation_picture_and_more.py b/sas/migrations/0006_alter_peoplepicturerelation_picture_and_more.py new file mode 100644 index 00000000..4239dbd9 --- /dev/null +++ b/sas/migrations/0006_alter_peoplepicturerelation_picture_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.17 on 2025-01-25 23:50 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [("sas", "0005_delete_album_delete_picture_delete_sasfile_and_more")] + + operations = [ + migrations.AlterField( + model_name="peoplepicturerelation", + name="picture", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="people", + to="sas.picture", + verbose_name="picture", + ), + ), + migrations.AlterField( + model_name="picturemoderationrequest", + name="picture", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="moderation_requests", + to="sas.picture", + verbose_name="Picture", + ), + ), + ] diff --git a/sas/models.py b/sas/models.py index bf87786d..61703e47 100644 --- a/sas/models.py +++ b/sas/models.py @@ -17,7 +17,6 @@ from __future__ import annotations import contextlib from io import BytesIO -from pathlib import Path from typing import ClassVar, Self from django.conf import settings @@ -26,22 +25,38 @@ from django.db import models from django.db.models import Exists, OuterRef from django.urls import reverse from django.utils import timezone +from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ from PIL import Image -from core.models import SithFile, User +from core.models import Group, Notification, User from core.utils import exif_auto_rotate, resize_image -class SasFile(SithFile): - """Proxy model for any file in the SAS. +def get_directory(instance: SasFile, filename: str): + return f"./{instance.parent_path}/{filename}" + + +def get_compressed_directory(instance: SasFile, filename: str): + return f"./.compressed/{instance.parent_path}/{filename}" + + +def get_thumbnail_directory(instance: SasFile, filename: str): + if isinstance(instance, Album): + name, extension = filename.rsplit(".", 1) + filename = f"{name}/thumb.{extension}" + return f"./.thumbnails/{instance.parent_path}/{filename}" + + +class SasFile(models.Model): + """Abstract model for SAS files May be used to have logic that should be shared by both [Picture][sas.models.Picture] and [Album][sas.models.Album]. """ class Meta: - proxy = True + abstract = True def can_be_viewed_by(self, user): if user.is_anonymous: @@ -62,6 +77,122 @@ class SasFile(SithFile): def can_be_edited_by(self, user): return user.is_root or user.is_in_group(pk=settings.SITH_GROUP_SAS_ADMIN_ID) + @cached_property + def parent_path(self) -> str: + return "/".join(["SAS", *[p.name for p in self.parent_list]]) + + @cached_property + def parent_list(self) -> list[Self]: + parents = [] + current = self.parent + while current is not None: + parents.append(current) + current = current.parent + return parents + + +class AlbumQuerySet(models.QuerySet): + def viewable_by(self, user: User) -> Self: + """Filter the albums that this user can view. + + Warning: + Calling this queryset method may add several additional requests. + """ + if user.is_root or user.is_in_group(pk=settings.SITH_GROUP_SAS_ADMIN_ID): + return self.all() + if user.was_subscribed: + return self.filter( + Exists(Picture.objects.filter(parent=OuterRef("pk"), is_moderated=True)) + ) + # known bug : if all children of an album are also albums + # then this album is excluded, even if one of the sub-albums should be visible. + # The fs-like navigation is likely to be half-broken for non-subscribers, + # but that's ok, since non-subscribers are expected to see only the albums + # containing pictures on which they have been identified (hence, very few). + # Most, if not all, of their albums will be displayed on the + # `latest albums` section of the SAS. + # Moreover, they will still see all of their picture in their profile. + return self.filter( + Exists(Picture.objects.filter(parent_id=OuterRef("pk")).viewable_by(user)) + ) + + def annotate_is_moderated(self) -> Self: + # an album is moderated if it has at least one moderated photo + # if there is no photo at all, the album isn't considered as non-moderated + # (it's just empty) + return self.annotate( + is_moderated=Exists( + Picture.objects.filter(parent=OuterRef("pk"), is_moderated=True) + ) + | ~Exists(Picture.objects.filter(parent=OuterRef("pk"))) + ) + + +class Album(SasFile): + NAME_MAX_LENGTH: ClassVar[int] = 50 + + name = models.CharField(_("name"), max_length=100) + parent = models.ForeignKey( + "self", + related_name="children", + verbose_name=_("parent"), + null=True, + blank=True, + on_delete=models.CASCADE, + ) + thumbnail = models.FileField( + upload_to=get_thumbnail_directory, verbose_name=_("thumbnail"), max_length=256 + ) + view_groups = models.ManyToManyField( + Group, related_name="viewable_albums", verbose_name=_("view groups") + ) + edit_groups = models.ManyToManyField( + Group, related_name="editable_albums", verbose_name=_("edit groups") + ) + event_date = models.DateField( + _("event date"), + help_text=_("The date on which the photos in this album were taken"), + default=timezone.localdate, + ) + + objects = AlbumQuerySet.as_manager() + + class Meta: + verbose_name = _("album") + constraints = [ + models.UniqueConstraint( + fields=["name", "parent"], name="unique_album_name_if_same_parent" + ) + ] + + def __str__(self): + return f"Album {self.name}" + + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + for user in User.objects.filter( + groups__id__in=[settings.SITH_GROUP_SAS_ADMIN_ID] + ): + Notification( + user=user, + url=reverse("sas:moderation"), + type="SAS_MODERATION", + param="1", + ).save() + + def get_absolute_url(self): + return reverse("sas:album", kwargs={"album_id": self.id}) + + def get_download_url(self): + return reverse("sas:album_preview", kwargs={"album_id": self.id}) + + def generate_thumbnail(self): + p = self.pictures.order_by("?").first() or self.children.order_by("?").first() + if p and p.thumbnail: + self.thumbnail = p.thumbnail + self.thumbnail.name = f"{self.name}/thumb.webp" + self.save() + class PictureQuerySet(models.QuerySet): def viewable_by(self, user: User) -> Self: @@ -77,23 +208,62 @@ class PictureQuerySet(models.QuerySet): return self.filter(people__user_id=user.id, is_moderated=True) -class SASPictureManager(models.Manager): - def get_queryset(self): - return super().get_queryset().filter(is_in_sas=True, is_folder=False) - - class Picture(SasFile): - class Meta: - proxy = True + name = models.CharField(_("file name"), max_length=256) + parent = models.ForeignKey( + Album, + related_name="pictures", + verbose_name=_("album"), + on_delete=models.CASCADE, + ) + thumbnail = models.FileField( + upload_to=get_thumbnail_directory, + verbose_name=_("thumbnail"), + unique=True, + max_length=256, + ) + original = models.FileField( + upload_to=get_directory, + verbose_name=_("original image"), + max_length=256, + unique=True, + ) + compressed = models.FileField( + upload_to=get_compressed_directory, + verbose_name=_("compressed image"), + max_length=256, + unique=True, + ) + created_at = models.DateTimeField(default=timezone.now) + owner = models.ForeignKey( + User, + related_name="owned_pictures", + verbose_name=_("owner"), + on_delete=models.PROTECT, + ) - objects = SASPictureManager.from_queryset(PictureQuerySet)() + is_moderated = models.BooleanField(_("is moderated"), default=False) + asked_for_removal = models.BooleanField(_("asked for removal"), default=False) + moderator = models.ForeignKey( + User, + related_name="moderated_pictures", + null=True, + blank=True, + on_delete=models.SET_NULL, + ) - @property - def is_vertical(self): - with open(settings.MEDIA_ROOT / self.file.name, "rb") as f: - im = Image.open(BytesIO(f.read())) - (w, h) = im.size - return (w / h) < 1 + objects = PictureQuerySet.as_manager() + + def __str__(self): + return self.name + + def save(self, *args, **kwargs): + if self._state.adding: + self.generate_thumbnails() + super().save(*args, **kwargs) + + def get_absolute_url(self): + return reverse("sas:picture", kwargs={"picture_id": self.id}) def get_download_url(self): return reverse("sas:download", kwargs={"picture_id": self.id}) @@ -104,42 +274,33 @@ class Picture(SasFile): def get_download_thumb_url(self): return reverse("sas:download_thumb", kwargs={"picture_id": self.id}) - def get_absolute_url(self): - return reverse("sas:picture", kwargs={"picture_id": self.id}) + @property + def is_vertical(self): + # original, compressed and thumbnail image have all three the same ratio, + # so the smallest one is used to tell if the image is vertical + im = Image.open(BytesIO(self.thumbnail.read())) + (w, h) = im.size + return w < h - def generate_thumbnails(self, *, overwrite=False): - im = Image.open(BytesIO(self.file.read())) + def generate_thumbnails(self): + im = Image.open(self.original) with contextlib.suppress(Exception): im = exif_auto_rotate(im) # convert the compressed image and the thumbnail into webp - # The original image keeps its original type, because it's not - # meant to be shown on the website, but rather to keep the real image - # for less frequent cases (like downloading the pictures of an user) - extension = self.mime_type.split("/")[-1] # the HD version of the image doesn't need to be optimized, because : # - it isn't frequently queried - # - optimizing large images takes a lot time, which greatly hinders the UX + # - optimizing large images takes a lot of time, which greatly hinders the UX # - photographers usually already optimize their images - file = resize_image(im, max(im.size), extension, optimize=False) thumb = resize_image(im, 200, "webp") compressed = resize_image(im, 1200, "webp") - if overwrite: - self.file.delete() - self.thumbnail.delete() - self.compressed.delete() - new_extension_name = str(Path(self.name).with_suffix(".webp")) - self.file = file - self.file.name = self.name self.thumbnail = thumb - self.thumbnail.name = new_extension_name + self.thumbnail.name = self.name self.compressed = compressed - self.compressed.name = new_extension_name - self.save() + self.compressed.name = self.name def rotate(self, degree): - for attr in ["file", "compressed", "thumbnail"]: - name = self.__getattribute__(attr).name - with open(settings.MEDIA_ROOT / name, "r+b") as file: + for field in self.original, self.compressed, self.thumbnail: + with open(field.file, "r+b") as file: if file: im = Image.open(BytesIO(file.read())) file.seek(0) @@ -152,108 +313,6 @@ class Picture(SasFile): progressive=True, ) - def get_next(self): - if self.is_moderated: - pictures_qs = self.parent.children.filter( - is_moderated=True, - asked_for_removal=False, - is_folder=False, - id__gt=self.id, - ) - else: - pictures_qs = Picture.objects.filter(id__gt=self.id, is_moderated=False) - return pictures_qs.order_by("id").first() - - def get_previous(self): - if self.is_moderated: - pictures_qs = self.parent.children.filter( - is_moderated=True, - asked_for_removal=False, - is_folder=False, - id__lt=self.id, - ) - else: - pictures_qs = Picture.objects.filter(id__lt=self.id, is_moderated=False) - return pictures_qs.order_by("-id").first() - - -class AlbumQuerySet(models.QuerySet): - def viewable_by(self, user: User) -> Self: - """Filter the albums that this user can view. - - Warning: - Calling this queryset method may add several additional requests. - """ - if user.is_root or user.is_in_group(pk=settings.SITH_GROUP_SAS_ADMIN_ID): - return self.all() - if user.was_subscribed: - return self.filter(is_moderated=True) - # known bug : if all children of an album are also albums - # then this album is excluded, even if one of the sub-albums should be visible. - # The fs-like navigation is likely to be half-broken for non-subscribers, - # but that's ok, since non-subscribers are expected to see only the albums - # containing pictures on which they have been identified (hence, very few). - # Most, if not all, of their albums will be displayed on the - # `latest albums` section of the SAS. - # Moreover, they will still see all of their picture in their profile. - return self.filter( - Exists(Picture.objects.filter(parent_id=OuterRef("pk")).viewable_by(user)) - ) - - -class SASAlbumManager(models.Manager): - def get_queryset(self): - return super().get_queryset().filter(is_in_sas=True, is_folder=True) - - -class Album(SasFile): - NAME_MAX_LENGTH: ClassVar[int] = 50 - """Maximum length of an album's name. - - [SithFile][core.models.SithFile] have a maximum length - of 256 characters. - However, this limit is too high for albums. - Names longer than 50 characters are harder to read - and harder to display on the SAS page. - - It is to be noted, though, that this does not - add or modify any db behaviour. - It's just a constant to be used in views and forms. - """ - - class Meta: - proxy = True - - objects = SASAlbumManager.from_queryset(AlbumQuerySet)() - - @property - def children_pictures(self): - return Picture.objects.filter(parent=self) - - @property - def children_albums(self): - return Album.objects.filter(parent=self) - - def get_absolute_url(self): - return reverse("sas:album", kwargs={"album_id": self.id}) - - def get_download_url(self): - return reverse("sas:album_preview", kwargs={"album_id": self.id}) - - def generate_thumbnail(self): - p = ( - self.children_pictures.order_by("?").first() - or self.children_albums.exclude(file=None) - .exclude(file="") - .order_by("?") - .first() - ) - if p and p.file: - image = resize_image(Image.open(BytesIO(p.file.read())), 200, "webp") - self.file = image - self.file.name = f"{self.name}/thumb.webp" - self.save() - def sas_notification_callback(notif): count = Picture.objects.filter(is_moderated=False).count() diff --git a/sas/schemas.py b/sas/schemas.py index 5e049858..e6e76568 100644 --- a/sas/schemas.py +++ b/sas/schemas.py @@ -31,7 +31,7 @@ class PictureFilterSchema(FilterSchema): class PictureSchema(ModelSchema): class Meta: model = Picture - fields = ["id", "name", "date", "size", "is_moderated", "asked_for_removal"] + fields = ["id", "name", "created_at", "is_moderated", "asked_for_removal"] owner: UserProfileSchema sas_url: str diff --git a/sas/static/bundled/sas/viewer-index.ts b/sas/static/bundled/sas/viewer-index.ts index faa9505a..cbc62103 100644 --- a/sas/static/bundled/sas/viewer-index.ts +++ b/sas/static/bundled/sas/viewer-index.ts @@ -141,7 +141,8 @@ exportToHtml("loadViewer", (config: ViewerConfig) => { // biome-ignore lint/style/useNamingConvention: api is in snake_case full_size_url: "", owner: "", - date: new Date(), + // biome-ignore lint/style/useNamingConvention: api is in snake_case + created_at: new Date(), identifications: [], }, /** diff --git a/sas/templates/sas/album.jinja b/sas/templates/sas/album.jinja index 70aa79df..4707cf2b 100644 --- a/sas/templates/sas/album.jinja +++ b/sas/templates/sas/album.jinja @@ -18,7 +18,7 @@ {% block content %} - SAS / {{ print_path(album.parent) }} {{ album.get_display_name() }} + SAS / {{ print_path(album.parent) }} {{ album.name }} {% set is_sas_admin = user.can_edit(album) %} @@ -29,7 +29,7 @@ {% csrf_token %}
-

{{ album.get_display_name() }}

+

{{ album.name }}

{% trans %}Edit{% endtrans %} diff --git a/sas/templates/sas/macros.jinja b/sas/templates/sas/macros.jinja index a7f48618..3c0dfbce 100644 --- a/sas/templates/sas/macros.jinja +++ b/sas/templates/sas/macros.jinja @@ -1,9 +1,7 @@ {% macro display_album(a, edit_mode) %} - {% if a.file %} + {% if a.thumbnail %} {% set img = a.get_download_url() %} - {% elif a.children.filter(is_folder=False, is_moderated=True).exists() %} - {% set img = a.children.filter(is_folder=False).first().as_picture.get_download_thumb_url() %} {% else %} {% set img = static('core/img/sas.jpg') %} {% endif %} @@ -27,6 +25,6 @@ {% macro print_path(file) %} {% if file and file.parent %} {{ print_path(file.parent) }} - {{ file.get_display_name() }} / + {{ file.name }} / {% endif %} {% endmacro %} \ No newline at end of file diff --git a/sas/templates/sas/picture.jinja b/sas/templates/sas/picture.jinja index c0fe4dd4..6737216a 100644 --- a/sas/templates/sas/picture.jinja +++ b/sas/templates/sas/picture.jinja @@ -1,9 +1,9 @@ {% extends "core/base.jinja" %} {%- block additional_css -%} - - - + + + {%- endblock -%} {%- block additional_js -%} @@ -95,7 +95,7 @@
diff --git a/sas/tests/test_api.py b/sas/tests/test_api.py index 9b24688b..c4717b6a 100644 --- a/sas/tests/test_api.py +++ b/sas/tests/test_api.py @@ -24,8 +24,8 @@ class TestSas(TestCase): cls.user_b, cls.user_c = subscriber_user.make(_quantity=2) picture = picture_recipe.extend(owner=owner) - cls.album_a = baker.make(Album, is_in_sas=True, parent=sas) - cls.album_b = baker.make(Album, is_in_sas=True, parent=sas) + cls.album_a = baker.make(Album) + cls.album_b = baker.make(Album) relation_recipe = Recipe(PeoplePictureRelation) relations = [] for album in cls.album_a, cls.album_b: diff --git a/sas/views.py b/sas/views.py index 7c8c8ea2..9c04c611 100644 --- a/sas/views.py +++ b/sas/views.py @@ -12,6 +12,7 @@ # OR WITHIN THE LOCAL FILE "LICENSE" # # +from pathlib import Path from typing import Any from django.conf import settings @@ -25,7 +26,7 @@ from django.views.generic.edit import FormMixin, FormView, UpdateView from core.auth.mixins import CanEditMixin, CanViewMixin from core.models import SithFile, User -from core.views.files import FileView, send_file +from core.views.files import FileView, send_raw_file from sas.forms import ( AlbumEditForm, PictureEditForm, @@ -42,16 +43,12 @@ class SASMainView(FormView): def post(self, request, *args, **kwargs): self.form = self.get_form() - parent = SithFile.objects.filter(id=settings.SITH_SAS_ROOT_DIR_ID).first() - files = request.FILES.getlist("images") root = User.objects.filter(username="root").first() if request.user.is_authenticated and request.user.is_in_group( pk=settings.SITH_GROUP_SAS_ADMIN_ID ): if self.form.is_valid(): - self.form.process( - parent=parent, owner=root, files=files, automodere=True - ) + self.form.process(parent=None, owner=root, files=[], automodere=True) if self.form.is_valid(): return super().form_valid(self.form) else: @@ -60,10 +57,8 @@ class SASMainView(FormView): def get_context_data(self, **kwargs): kwargs = super().get_context_data(**kwargs) - albums_qs = Album.objects.viewable_by(self.request.user) - kwargs["categories"] = list( - albums_qs.filter(parent_id=settings.SITH_SAS_ROOT_DIR_ID).order_by("id") - ) + albums_qs = Album.objects.annotate_is_moderated().viewable_by(self.request.user) + kwargs["categories"] = list(albums_qs.filter(parent=None).order_by("id")) kwargs["latest"] = list(albums_qs.order_by("-id")[:5]) return kwargs @@ -73,6 +68,9 @@ class PictureView(CanViewMixin, DetailView): pk_url_kwarg = "picture_id" template_name = "sas/picture.jinja" + def get_queryset(self): + return super().get_queryset().select_related("parent") + def get(self, request, *args, **kwargs): self.object = self.get_object() if "rotate_right" in request.GET: @@ -82,25 +80,35 @@ class PictureView(CanViewMixin, DetailView): return super().get(request, *args, **kwargs) def get_context_data(self, **kwargs): - return super().get_context_data(**kwargs) | { - "album": Album.objects.get(children=self.object) - } + return super().get_context_data(**kwargs) | {"album": self.object.parent} def send_album(request, album_id): - return send_file(request, album_id, Album) + album = get_object_or_404(Album, id=album_id) + if not album.can_be_viewed_by(request.user): + raise PermissionDenied + return send_raw_file(Path(album.thumbnail.path)) def send_pict(request, picture_id): - return send_file(request, picture_id, Picture) + picture = get_object_or_404(Picture, id=picture_id) + if not picture.can_be_viewed_by(request.user): + raise PermissionDenied + return send_raw_file(Path(picture.original.path)) def send_compressed(request, picture_id): - return send_file(request, picture_id, Picture, "compressed") + picture = get_object_or_404(Picture, id=picture_id) + if not picture.can_be_viewed_by(request.user): + raise PermissionDenied + return send_raw_file(Path(picture.compressed.path)) def send_thumb(request, picture_id): - return send_file(request, picture_id, Picture, "thumbnail") + picture = get_object_or_404(Picture, id=picture_id) + if not picture.can_be_viewed_by(request.user): + raise PermissionDenied + return send_raw_file(Path(picture.thumbnail.path)) class AlbumUploadView(CanViewMixin, DetailView, FormMixin): @@ -113,11 +121,10 @@ class AlbumUploadView(CanViewMixin, DetailView, FormMixin): if not self.object.file: self.object.generate_thumbnail() self.form = self.get_form() - parent = SithFile.objects.filter(id=self.object.id).first() files = request.FILES.getlist("images") if request.user.is_subscribed and self.form.is_valid(): self.form.process( - parent=parent, + parent=self.object, owner=request.user, files=files, automodere=( @@ -186,9 +193,7 @@ class AlbumView(CanViewMixin, DetailView, FormMixin): id__in=self.request.session["clipboard"] ) kwargs["children_albums"] = list( - Album.objects.viewable_by(self.request.user) - .filter(parent_id=self.object.id) - .order_by("-date") + self.object.children.viewable_by(self.request.user).order_by("-event_date") ) return kwargs @@ -220,7 +225,7 @@ class ModerationView(TemplateView): def get_context_data(self, **kwargs): kwargs = super().get_context_data(**kwargs) kwargs["albums_to_moderate"] = Album.objects.filter( - is_moderated=False, is_in_sas=True, is_folder=True + is_moderated=False ).order_by("id") pictures = Picture.objects.filter(is_moderated=False).select_related("parent") kwargs["pictures"] = pictures