Merge pull request #679 from ae-utbm/xapian-from-sources

Xapian from sources and fix CVE
This commit is contained in:
Bartuccio Antoine 2024-06-26 11:48:31 +02:00 committed by GitHub
commit a8b9f38000
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 885 additions and 673 deletions

View File

@ -6,13 +6,13 @@ runs:
- name: Install apt packages - name: Install apt packages
uses: awalsh128/cache-apt-pkgs-action@latest uses: awalsh128/cache-apt-pkgs-action@latest
with: with:
packages: gettext libxapian-dev libgraphviz-dev packages: gettext libgraphviz-dev
version: 1.0 # increment to reset cache version: 1.0 # increment to reset cache
- name: Install dependencies - name: Install dependencies
run: | run: |
sudo apt update sudo apt update
sudo apt install gettext libxapian-dev libgraphviz-dev sudo apt install gettext libgraphviz-dev
shell: bash shell: bash
- name: Set up python - name: Set up python
@ -48,6 +48,10 @@ runs:
run: poetry install -E testing -E docs run: poetry install -E testing -E docs
shell: bash shell: bash
- name: Install xapian
run: poetry run ./manage.py install_xapian
shell: bash
- name: Compile gettext messages - name: Compile gettext messages
run: poetry run ./manage.py compilemessages run: poetry run ./manage.py compilemessages
shell: bash shell: bash

View File

@ -0,0 +1,67 @@
# -*- coding:utf-8 -*
#
# Copyright 2024 © AE UTBM
# ae@utbm.fr / ae.info@utbm.fr
#
# This file is part of the website of the UTBM Student Association (AE UTBM),
# https://ae.utbm.fr.
#
# You can find the source code of the website at https://github.com/ae-utbm/sith3
#
# LICENSED UNDER THE GNU GENERAL PUBLIC LICENSE VERSION 3 (GPLv3)
# SEE : https://raw.githubusercontent.com/ae-utbm/sith3/master/LICENSE
# OR WITHIN THE LOCAL FILE "LICENSE"
#
#
import os
import tomli
import subprocess
from django.core.management.base import BaseCommand, CommandParser
from pathlib import Path
class Command(BaseCommand):
help = "Install xapian"
def add_arguments(self, parser: CommandParser):
parser.add_argument(
"-f",
"--force",
action="store_true",
help="Force installation even if already installed",
)
def _current_version(self) -> str | None:
try:
import xapian
except ImportError:
return None
return xapian.version_string()
def _desired_version(self) -> str:
with open(
Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb"
) as f:
pyproject = tomli.load(f)
return pyproject["tool"]["xapian"]["version"]
def handle(self, force: bool, *args, **options):
if not os.environ.get("VIRTUAL_ENV", None):
print("No virtual environment detected, this command can't be used")
return
desired = self._desired_version()
if desired == self._current_version():
if not force:
print(
f"Version {desired} is already installed, use --force to re-install"
)
return
print(f"Version {desired} is already installed, re-installing")
print(f"Installing xapian version {desired} at {os.environ['VIRTUAL_ENV']}")
subprocess.run(
[str(Path(__file__).parent / "install_xapian.sh"), desired],
env=dict(os.environ),
).check_returncode()
print("Installation success")

View File

@ -0,0 +1,47 @@
#!/usr/bin/env bash
# Originates from https://gist.github.com/jorgecarleitao/ab6246c86c936b9c55fd
# first argument of the script is Xapian version (e.g. 1.2.19)
VERSION=$1
# Cleanup env vars for auto discovery mechanism
export CPATH=
export LIBRARY_PATH=
export CFLAGS=
export LDFLAGS=
export CCFLAGS=
export CXXFLAGS=
export CPPFLAGS=
# prepare
rm -rf "$VIRTUAL_ENV/packages"
mkdir -p "$VIRTUAL_ENV/packages" && cd "$VIRTUAL_ENV/packages" || exit 1
CORE=xapian-core-$VERSION
BINDINGS=xapian-bindings-$VERSION
# download
echo "Downloading source..."
curl -O "https://oligarchy.co.uk/xapian/$VERSION/${CORE}.tar.xz"
curl -O "https://oligarchy.co.uk/xapian/$VERSION/${BINDINGS}.tar.xz"
# extract
echo "Extracting source..."
tar xf "${CORE}.tar.xz"
tar xf "${BINDINGS}.tar.xz"
# install
echo "Installing Xapian-core..."
cd "$VIRTUAL_ENV/packages/${CORE}" || exit 1
./configure --prefix="$VIRTUAL_ENV" && make && make install
PYTHON_FLAG=--with-python3
echo "Installing Xapian-bindings..."
cd "$VIRTUAL_ENV/packages/${BINDINGS}" || exit 1
./configure --prefix="$VIRTUAL_ENV" $PYTHON_FLAG XAPIAN_CONFIG="$VIRTUAL_ENV/bin/xapian-config" && make && make install
# clean
rm -rf "$VIRTUAL_ENV/packages"
# test
python -c "import xapian"

View File

@ -79,12 +79,37 @@ def send_file(request, file_id, file_class=SithFile, file_attr="file"):
return response return response
class MultipleFileInput(forms.ClearableFileInput):
allow_multiple_selected = True
class _MultipleFieldMixin:
def __init__(self, *args, **kwargs):
kwargs.setdefault("widget", MultipleFileInput())
super().__init__(*args, **kwargs)
def clean(self, data, initial=None):
single_file_clean = super().clean
if isinstance(data, (list, tuple)):
result = [single_file_clean(d, initial) for d in data]
else:
result = [single_file_clean(data, initial)]
return result
class MultipleFileField(_MultipleFieldMixin, forms.FileField):
...
class MultipleImageField(_MultipleFieldMixin, forms.ImageField):
...
class AddFilesForm(forms.Form): class AddFilesForm(forms.Form):
folder_name = forms.CharField( folder_name = forms.CharField(
label=_("Add a new folder"), max_length=30, required=False label=_("Add a new folder"), max_length=30, required=False
) )
file_field = forms.FileField( file_field = MultipleFileField(
widget=forms.ClearableFileInput(attrs={"multiple": True}),
label=_("Files"), label=_("Files"),
required=False, required=False,
) )

View File

@ -9,7 +9,6 @@ Certaines dépendances sont nécessaires niveau système :
* poetry * poetry
* libssl * libssl
* libjpeg * libjpeg
* libxapian-dev
* zlib1g-dev * zlib1g-dev
* python * python
* gettext * gettext
@ -76,13 +75,13 @@ Sur Ubuntu
# Sait-on jamais # Sait-on jamais
sudo apt update sudo apt update
sudo apt install python-is-python3 # Permet d'utiliser python au lieu de python3, c'est optionel sudo apt install python-is-python3 # Permet d'utiliser python au lieu de python3, c'est optionnel
sudo apt install build-essentials libssl-dev libjpeg-dev zlib1g-dev python-dev \ sudo apt install build-essentials libssl-dev libjpeg-dev zlib1g-dev python-dev \
libffi-dev python-dev-is-python3 libgraphviz-dev pkg-config libxapian-dev \ libffi-dev python-dev-is-python3 libgraphviz-dev pkg-config \
gettext git gettext git pipx
curl -sSL https://install.python-poetry.org | python - pipx install poetry
.. note:: .. note::
@ -93,21 +92,20 @@ Sur MacOS
~~~~~~~~~ ~~~~~~~~~
Pour installer les dépendances, il est fortement recommandé d'installer le gestionnaire de paquets `homebrew <https://brew.sh/index_fr>`_. Pour installer les dépendances, il est fortement recommandé d'installer le gestionnaire de paquets `homebrew <https://brew.sh/index_fr>`_.
Il est également nécessaire d'avoir installé xcode
.. sourcecode:: bash .. sourcecode:: bash
brew install git python xapian graphviz poetry echo 'export PATH="$(brew --prefix graphviz)/bin:$PATH"' >> ~/.zshrc
echo 'export CFLAGS="-isysroot /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk -I $(brew --prefix graphviz)/include"' >> ~/.zshrc
# Si vous aviez une version de python ne venant pas de homebrew echo 'export LDFLAGS="-L /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib -L $(brew --prefix graphviz)/lib"' >> ~/.zshrc
brew link --overwrite python
brew install git python graphviz pipx
pipx install poetry
# Pour bien configurer gettext # Pour bien configurer gettext
brew link gettext # (suivez bien les instructions supplémentaires affichées) brew link gettext # (suivez bien les instructions supplémentaires affichées)
# Pour installer poetry
pip3 install poetry
.. note:: .. note::
Si vous rencontrez des erreurs lors de votre configuration, n'hésitez pas à vérifier l'état de votre installation homebrew avec :code:`brew doctor` Si vous rencontrez des erreurs lors de votre configuration, n'hésitez pas à vérifier l'état de votre installation homebrew avec :code:`brew doctor`
@ -134,6 +132,9 @@ Finaliser l'installation
# Activation de l'environnement virtuel # Activation de l'environnement virtuel
poetry shell poetry shell
# Installe xapian
python manage.py install_xapian
# Prépare la base de données # Prépare la base de données
python manage.py setup python manage.py setup

1359
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -6,6 +6,7 @@ authors = [
"Skia <skia@hya.sk>", "Skia <skia@hya.sk>",
"klmp200 <antoine@bartuccio.fr>", "klmp200 <antoine@bartuccio.fr>",
"Krophil <pierre.brunet@krophil.fr>", "Krophil <pierre.brunet@krophil.fr>",
"Maréchal <thgirod@hotmail.com>",
"Och <francescowitz68@gmail.com>", "Och <francescowitz68@gmail.com>",
"tleb <tleb@openmailbox.org>", "tleb <tleb@openmailbox.org>",
"Soldat <ryan-68@live.fr>", "Soldat <ryan-68@live.fr>",
@ -19,7 +20,7 @@ homepage = "https://ae.utbm.fr/"
license = "GPL-3.0-only" license = "GPL-3.0-only"
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.8" python = "^3.10,<3.12" # Version is held back by mistune
Django = "^3.2" Django = "^3.2"
Pillow = "^9.2" Pillow = "^9.2"
mistune = "^0.8.4" mistune = "^0.8.4"
@ -31,29 +32,32 @@ djangorestframework = "^3.13"
django-phonenumber-field = "^6.3" django-phonenumber-field = "^6.3"
phonenumbers = "^8.12" phonenumbers = "^8.12"
django-ajax-selects = "^2.1.0" django-ajax-selects = "^2.1.0"
reportlab = "^3.6" reportlab = "^4.2"
django-haystack = "^3.2.1" django-haystack = "^3.2.1"
xapian-haystack = "^3.0.1" xapian-haystack = "^3.0.1"
xapian-bindings = "^0.1.0"
libsass = "^0.22" libsass = "^0.22"
django-ordered-model = "^3.7" django-ordered-model = "^3.7"
django-simple-captcha = "^0.5.17" django-simple-captcha = "^0.5.17"
python-dateutil = "^2.8.2" python-dateutil = "^2.8.2"
psycopg2-binary = "2.9.3" psycopg2-binary = "^2.9"
sentry-sdk = "^1.21.0" sentry-sdk = "^1.21.0"
pygraphviz = "^1.9" pygraphviz = "^1.1"
Jinja2 = "^3.1" Jinja2 = "^3.1"
django-countries = "^7.5.1" django-countries = "^7.5.1"
dict2xml = "^1.7.3" dict2xml = "^1.7.3"
Sphinx = "^5" # Needed for building xapian
tomli = "^2.0.1"
# Extra optional dependencies # Extra optional dependencies
coverage = {version = "^5.5", optional = true} coverage = {version = "^5.5", optional = true}
# Docs extra dependencies # Docs extra dependencies
Sphinx = {version = "^4.2.0", optional = true}
sphinx-rtd-theme = {version = "^1.0.0", optional = true} sphinx-rtd-theme = {version = "^1.0.0", optional = true}
sphinx-copybutton = {version = "^0.4.0", optional = true} sphinx-copybutton = {version = "^0.4.0", optional = true}
[tool.xapian]
version = "1.4.25"
[tool.poetry.extras] [tool.poetry.extras]
testing = ["coverage"] testing = ["coverage"]
docs = ["Sphinx", "sphinx-rtd-theme", "sphinx-copybutton"] docs = ["Sphinx", "sphinx-rtd-theme", "sphinx-copybutton"]

View File

@ -30,7 +30,7 @@ from ajax_select import make_ajax_field
from ajax_select.fields import AutoCompleteSelectMultipleField from ajax_select.fields import AutoCompleteSelectMultipleField
from core.views import CanViewMixin, CanEditMixin from core.views import CanViewMixin, CanEditMixin
from core.views.files import send_file, FileView from core.views.files import send_file, FileView, MultipleImageField
from core.models import SithFile, User, Notification, RealGroup from core.models import SithFile, User, Notification, RealGroup
from sas.models import Picture, Album, PeoplePictureRelation from sas.models import Picture, Album, PeoplePictureRelation
@ -40,8 +40,7 @@ class SASForm(forms.Form):
album_name = forms.CharField( album_name = forms.CharField(
label=_("Add a new album"), max_length=30, required=False label=_("Add a new album"), max_length=30, required=False
) )
images = forms.ImageField( images = MultipleImageField(
widget=forms.ClearableFileInput(attrs={"multiple": True}),
label=_("Upload images"), label=_("Upload images"),
required=False, required=False,
) )