Merge pull request #679 from ae-utbm/xapian-from-sources

Xapian from sources and fix CVE

.github/actions/setup_project/action.yml

@@ -6,13 +6,13 @@ runs:
     - name: Install apt packages
       uses: awalsh128/cache-apt-pkgs-action@latest
       with:
-        packages: gettext libxapian-dev libgraphviz-dev
+        packages: gettext libgraphviz-dev
         version: 1.0  # increment to reset cache
 
     - name: Install dependencies
       run: |
         sudo apt update
-        sudo apt install gettext libxapian-dev libgraphviz-dev
+        sudo apt install gettext libgraphviz-dev
       shell: bash
 
     - name: Set up python
@@ -48,6 +48,10 @@ runs:
       run: poetry install -E testing -E docs
       shell: bash
 
+    - name: Install xapian
+      run: poetry run ./manage.py install_xapian
+      shell: bash
+
     - name: Compile gettext messages
       run: poetry run ./manage.py compilemessages
       shell: bash

core/management/commands/install_xapian.py

@@ -0,0 +1,67 @@
+# -*- coding:utf-8 -*
+#
+# Copyright 2024 © AE UTBM
+# ae@utbm.fr / ae.info@utbm.fr
+#
+# This file is part of the website of the UTBM Student Association (AE UTBM),
+# https://ae.utbm.fr.
+#
+# You can find the source code of the website at https://github.com/ae-utbm/sith3
+#
+# LICENSED UNDER THE GNU GENERAL PUBLIC LICENSE VERSION 3 (GPLv3)
+# SEE : https://raw.githubusercontent.com/ae-utbm/sith3/master/LICENSE
+# OR WITHIN THE LOCAL FILE "LICENSE"
+#
+#
+
+import os
+import tomli
+import subprocess
+from django.core.management.base import BaseCommand, CommandParser
+from pathlib import Path
+
+
+class Command(BaseCommand):
+    help = "Install xapian"
+
+    def add_arguments(self, parser: CommandParser):
+        parser.add_argument(
+            "-f",
+            "--force",
+            action="store_true",
+            help="Force installation even if already installed",
+        )
+
+    def _current_version(self) -> str | None:
+        try:
+            import xapian
+        except ImportError:
+            return None
+        return xapian.version_string()
+
+    def _desired_version(self) -> str:
+        with open(
+            Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb"
+        ) as f:
+            pyproject = tomli.load(f)
+            return pyproject["tool"]["xapian"]["version"]
+
+    def handle(self, force: bool, *args, **options):
+        if not os.environ.get("VIRTUAL_ENV", None):
+            print("No virtual environment detected, this command can't be used")
+            return
+
+        desired = self._desired_version()
+        if desired == self._current_version():
+            if not force:
+                print(
+                    f"Version {desired} is already installed, use --force to re-install"
+                )
+                return
+            print(f"Version {desired} is already installed, re-installing")
+        print(f"Installing xapian version {desired} at {os.environ['VIRTUAL_ENV']}")
+        subprocess.run(
+            [str(Path(__file__).parent / "install_xapian.sh"), desired],
+            env=dict(os.environ),
+        ).check_returncode()
+        print("Installation success")

core/management/commands/install_xapian.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Originates from https://gist.github.com/jorgecarleitao/ab6246c86c936b9c55fd
+# first argument of the script is Xapian version (e.g. 1.2.19)
+VERSION=$1
+
+# Cleanup env vars for auto discovery mechanism
+export CPATH=
+export LIBRARY_PATH=
+export CFLAGS=
+export LDFLAGS=
+export CCFLAGS=
+export CXXFLAGS=
+export CPPFLAGS=
+
+# prepare
+rm -rf "$VIRTUAL_ENV/packages"
+mkdir -p "$VIRTUAL_ENV/packages" && cd "$VIRTUAL_ENV/packages" || exit 1
+
+CORE=xapian-core-$VERSION
+BINDINGS=xapian-bindings-$VERSION
+
+# download
+echo "Downloading source..."
+curl -O "https://oligarchy.co.uk/xapian/$VERSION/${CORE}.tar.xz"
+curl -O "https://oligarchy.co.uk/xapian/$VERSION/${BINDINGS}.tar.xz"
+
+# extract
+echo "Extracting source..."
+tar xf "${CORE}.tar.xz"
+tar xf "${BINDINGS}.tar.xz"
+
+# install
+echo "Installing Xapian-core..."
+cd "$VIRTUAL_ENV/packages/${CORE}" || exit 1
+./configure --prefix="$VIRTUAL_ENV" && make && make install
+
+PYTHON_FLAG=--with-python3
+
+echo "Installing Xapian-bindings..."
+cd "$VIRTUAL_ENV/packages/${BINDINGS}" || exit 1
+./configure --prefix="$VIRTUAL_ENV" $PYTHON_FLAG XAPIAN_CONFIG="$VIRTUAL_ENV/bin/xapian-config" && make && make install
+
+# clean
+rm -rf "$VIRTUAL_ENV/packages"
+
+# test
+python -c "import xapian"

core/views/files.py

@@ -79,12 +79,37 @@ def send_file(request, file_id, file_class=SithFile, file_attr="file"):
         return response
 
 
+class MultipleFileInput(forms.ClearableFileInput):
+    allow_multiple_selected = True
+
+
+class _MultipleFieldMixin:
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault("widget", MultipleFileInput())
+        super().__init__(*args, **kwargs)
+
+    def clean(self, data, initial=None):
+        single_file_clean = super().clean
+        if isinstance(data, (list, tuple)):
+            result = [single_file_clean(d, initial) for d in data]
+        else:
+            result = [single_file_clean(data, initial)]
+        return result
+
+
+class MultipleFileField(_MultipleFieldMixin, forms.FileField):
+    ...
+
+
+class MultipleImageField(_MultipleFieldMixin, forms.ImageField):
+    ...
+
+
 class AddFilesForm(forms.Form):
     folder_name = forms.CharField(
         label=_("Add a new folder"), max_length=30, required=False
     )
-    file_field = forms.FileField(
+    file_field = MultipleFileField(
-        widget=forms.ClearableFileInput(attrs={"multiple": True}),
         label=_("Files"),
         required=False,
     )

doc/start/install.rst

@@ -9,7 +9,6 @@ Certaines dépendances sont nécessaires niveau système :
 * poetry
 * libssl
 * libjpeg
-* libxapian-dev
 * zlib1g-dev
 * python
 * gettext
@@ -76,13 +75,13 @@ Sur Ubuntu
     # Sait-on jamais
     sudo apt update
 
-    sudo apt install python-is-python3 # Permet d'utiliser python au lieu de python3, c'est optionel
+    sudo apt install python-is-python3 # Permet d'utiliser python au lieu de python3, c'est optionnel
 
     sudo apt install build-essentials libssl-dev libjpeg-dev zlib1g-dev python-dev \
-                     libffi-dev python-dev-is-python3 libgraphviz-dev pkg-config libxapian-dev \
+                     libffi-dev python-dev-is-python3 libgraphviz-dev pkg-config \
-                     gettext git
+                     gettext git pipx
 
-    curl -sSL https://install.python-poetry.org | python -
+    pipx install poetry
 
 .. note::
 
@@ -93,21 +92,20 @@ Sur MacOS
 ~~~~~~~~~
 
 Pour installer les dépendances, il est fortement recommandé d'installer le gestionnaire de paquets `homebrew <https://brew.sh/index_fr>`_.  
+Il est également nécessaire d'avoir installé xcode
 
 .. sourcecode:: bash
 
-    brew install git python xapian graphviz poetry
+    echo 'export PATH="$(brew --prefix graphviz)/bin:$PATH"' >> ~/.zshrc
-
+    echo 'export CFLAGS="-isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk -I $(brew --prefix graphviz)/include"' >> ~/.zshrc
-    # Si vous aviez une version de python ne venant pas de homebrew
+    echo 'export LDFLAGS="-L /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib -L $(brew --prefix graphviz)/lib"' >> ~/.zshrc
-    brew link --overwrite python
 
+    brew install git python graphviz pipx
+    pipx install poetry
 
     # Pour bien configurer gettext
     brew link gettext # (suivez bien les instructions supplémentaires affichées)
 
-    # Pour installer poetry
-    pip3 install poetry
-
 .. note::
 
     Si vous rencontrez des erreurs lors de votre configuration, n'hésitez pas à vérifier l'état de votre installation homebrew avec :code:`brew doctor`
@@ -134,6 +132,9 @@ Finaliser l'installation
     # Activation de l'environnement virtuel
     poetry shell
 
+    # Installe xapian
+    python manage.py install_xapian
+
     # Prépare la base de données
     python manage.py setup
 

pyproject.toml

@@ -6,6 +6,7 @@ authors = [
 	"Skia <skia@hya.sk>",
 	"klmp200 <antoine@bartuccio.fr>",
 	"Krophil <pierre.brunet@krophil.fr>",
+	"Maréchal <thgirod@hotmail.com>",
 	"Och <francescowitz68@gmail.com>",
 	"tleb <tleb@openmailbox.org>",
 	"Soldat <ryan-68@live.fr>",
@@ -19,7 +20,7 @@ homepage = "https://ae.utbm.fr/"
 license = "GPL-3.0-only"
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.10,<3.12" # Version is held back by mistune
 Django = "^3.2"
 Pillow = "^9.2"
 mistune = "^0.8.4"
@@ -31,29 +32,32 @@ djangorestframework = "^3.13"
 django-phonenumber-field = "^6.3"
 phonenumbers = "^8.12"
 django-ajax-selects = "^2.1.0"
-reportlab = "^3.6"
+reportlab = "^4.2"
 django-haystack = "^3.2.1"
 xapian-haystack = "^3.0.1"
-xapian-bindings = "^0.1.0"
 libsass = "^0.22"
 django-ordered-model = "^3.7"
 django-simple-captcha = "^0.5.17"
 python-dateutil = "^2.8.2"
-psycopg2-binary = "2.9.3"
+psycopg2-binary = "^2.9"
 sentry-sdk = "^1.21.0"
-pygraphviz = "^1.9"
+pygraphviz = "^1.1"
 Jinja2 = "^3.1"
 django-countries = "^7.5.1"
 dict2xml = "^1.7.3"
+Sphinx = "^5" # Needed for building xapian
+tomli = "^2.0.1"
 
 # Extra optional dependencies
 coverage = {version = "^5.5", optional = true}
 
 # Docs extra dependencies
-Sphinx = {version = "^4.2.0", optional = true}
 sphinx-rtd-theme = {version = "^1.0.0", optional = true}
 sphinx-copybutton = {version = "^0.4.0", optional = true}
 
+[tool.xapian]
+version = "1.4.25"
+
 [tool.poetry.extras]
 testing = ["coverage"]
 docs = ["Sphinx", "sphinx-rtd-theme", "sphinx-copybutton"]

sas/views.py

@@ -30,7 +30,7 @@ from ajax_select import make_ajax_field
 from ajax_select.fields import AutoCompleteSelectMultipleField
 
 from core.views import CanViewMixin, CanEditMixin
-from core.views.files import send_file, FileView
+from core.views.files import send_file, FileView, MultipleImageField
 from core.models import SithFile, User, Notification, RealGroup
 
 from sas.models import Picture, Album, PeoplePictureRelation
@@ -40,8 +40,7 @@ class SASForm(forms.Form):
     album_name = forms.CharField(
         label=_("Add a new album"), max_length=30, required=False
     )
-    images = forms.ImageField(
+    images = MultipleImageField(
-        widget=forms.ClearableFileInput(attrs={"multiple": True}),
         label=_("Upload images"),
         required=False,
     )