diff --git a/core/management/commands/install_xapian.py b/core/management/commands/install_xapian.py index 4be1d907..7cdf6244 100644 --- a/core/management/commands/install_xapian.py +++ b/core/management/commands/install_xapian.py @@ -13,12 +13,41 @@ # # +import hashlib +import multiprocessing import os +import shutil import subprocess +import sys +import tarfile +from dataclasses import dataclass from pathlib import Path +from typing import Self import tomli -from django.core.management.base import BaseCommand, CommandParser +import urllib3 +from django.core.management.base import BaseCommand, CommandParser, OutputWrapper +from urllib3.response import HTTPException + + +@dataclass +class XapianSpec: + version: str + core_sha1: str + bindings_sha1: str + + @classmethod + def from_pyproject(cls) -> Self: + with open( + Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb" + ) as f: + pyproject = tomli.load(f) + spec = pyproject["tool"]["xapian"] + return cls( + version=spec["version"], + core_sha1=spec["core-sha1"], + bindings_sha1=spec["bindings-sha1"], + ) class Command(BaseCommand): @@ -39,13 +68,6 @@ class Command(BaseCommand): return None return xapian.version_string() - def _desired_version(self) -> str: - with open( - Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb" - ) as f: - pyproject = tomli.load(f) - return pyproject["tool"]["xapian"]["version"] - def handle(self, *args, force: bool, **options): if not os.environ.get("VIRTUAL_ENV", None): self.stdout.write( @@ -53,20 +75,152 @@ class Command(BaseCommand): ) return - desired = self._desired_version() - if desired == self._current_version(): + desired = XapianSpec.from_pyproject() + if desired.version == self._current_version(): if not force: self.stdout.write( - f"Version {desired} is already installed, use --force to re-install" + f"Version {desired.version} is already installed, use --force to re-install" ) return - self.stdout.write(f"Version {desired} is already installed, re-installing") - self.stdout.write( - f"Installing xapian version {desired} at {os.environ['VIRTUAL_ENV']}" - ) - subprocess.run( - [str(Path(__file__).parent / "install_xapian.sh"), desired], - env=dict(os.environ), - check=True, - ) + self.stdout.write( + f"Version {desired.version} is already installed, re-installing" + ) + XapianInstaller(desired, self.stdout, self.stderr).run() self.stdout.write("Installation success") + + +class XapianInstaller: + def __init__( + self, + spec: XapianSpec, + stdout: OutputWrapper, + stderr: OutputWrapper, + ): + self._version = spec.version + self._core_sha1 = spec.core_sha1 + self._bindings_sha1 = spec.bindings_sha1 + + self._stdout = stdout + self._stderr = stderr + self._virtual_env = os.environ.get("VIRTUAL_ENV", None) + + if not self._virtual_env: + raise RuntimeError("You are not inside a virtual environment") + self._virtual_env = Path(self._virtual_env) + + self._dest_dir = Path(self._virtual_env) / "packages" + self._core = f"xapian-core-{self._version}" + self._bindings = f"xapian-bindings-{self._version}" + + def _setup_env(self): + os.environ.update( + { + "CPATH": "", + "LIBRARY_PATH": "", + "CFLAGS": "", + "LDFLAGS": "", + "CCFLAGS": "", + "CXXFLAGS": "", + "CPPFLAGS": "", + } + ) + + def _prepare_dest_folder(self): + shutil.rmtree(self._dest_dir, ignore_errors=True) + self._dest_dir.mkdir(parents=True) + + def _download(self): + def download(url: str, dest: Path, sha1_hash: str): + resp = urllib3.request("GET", url) + if resp.status != 200: + raise HTTPException(f"Could not download {url}") + if hashlib.sha1(resp.data).hexdigest() != sha1_hash: + raise ValueError(f"File downloaded from {url} is compromised") + with open(dest, "wb") as f: + f.write(resp.data) + + self._stdout.write("Downloading source…") + + core = self._dest_dir / f"{self._core}.tar.xz" + bindings = self._dest_dir / f"{self._bindings}.tar.xz" + download( + f"https://oligarchy.co.uk/xapian/{self._version}/{self._core}.tar.xz", + core, + "e2b4b4cf6076873ec9402cab7b9a3b71dcf95e20", + ) + download( + f"https://oligarchy.co.uk/xapian/{self._version}/{self._bindings}.tar.xz", + bindings, + "782f568d2ea3ca751c519a2814a35c7dc86df3a4", + ) + self._stdout.write("Extracting source …") + with tarfile.open(core) as tar: + tar.extractall(self._dest_dir) + with tarfile.open(bindings) as tar: + tar.extractall(self._dest_dir) + + os.remove(core) + os.remove(bindings) + + def _install(self): + self._stdout.write("Installing Xapian-core…") + subprocess.run( + ["./configure", "--prefix", str(self._virtual_env)], + env=dict(os.environ), + cwd=self._dest_dir / self._core, + ).check_returncode() + subprocess.run( + [ + "make", + "-j", + str(multiprocessing.cpu_count()), + ], + env=dict(os.environ), + cwd=self._dest_dir / self._core, + ).check_returncode() + subprocess.run( + ["make", "install"], + env=dict(os.environ), + cwd=self._dest_dir / self._core, + ).check_returncode() + + self._stdout.write("Installing Xapian-bindings") + subprocess.run( + [ + "./configure", + "--prefix", + str(self._virtual_env), + "--with-python3", + f"XAPIAN_CONFIG={self._virtual_env / 'bin'/'xapian-config'}", + ], + env=dict(os.environ), + cwd=self._dest_dir / self._bindings, + ).check_returncode() + subprocess.run( + [ + "make", + "-j", + str(multiprocessing.cpu_count()), + ], + env=dict(os.environ), + cwd=self._dest_dir / self._bindings, + ).check_returncode() + subprocess.run( + ["make", "install"], + env=dict(os.environ), + cwd=self._dest_dir / self._bindings, + ).check_returncode() + + def _post_clean(self): + shutil.rmtree(self._dest_dir, ignore_errors=True) + + def _test(self): + subprocess.run([sys.executable, "-c", "import xapian"]).check_returncode() + + def run(self): + self._setup_env() + self._prepare_dest_folder() + self._download() + self._install() + self._post_clean() + self._test() diff --git a/core/management/commands/install_xapian.sh b/core/management/commands/install_xapian.sh deleted file mode 100755 index 3ca2ac17..00000000 --- a/core/management/commands/install_xapian.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Originates from https://gist.github.com/jorgecarleitao/ab6246c86c936b9c55fd -# first argument of the script is Xapian version (e.g. 1.2.19) -VERSION="$1" - -# Cleanup env vars for auto discovery mechanism -export CPATH= -export LIBRARY_PATH= -export CFLAGS= -export LDFLAGS= -export CCFLAGS= -export CXXFLAGS= -export CPPFLAGS= - -# prepare -rm -rf "$VIRTUAL_ENV/packages" -mkdir -p "$VIRTUAL_ENV/packages" && cd "$VIRTUAL_ENV/packages" || exit 1 - -CORE=xapian-core-$VERSION -BINDINGS=xapian-bindings-$VERSION - -# download -echo "Downloading source..." -curl -O "https://oligarchy.co.uk/xapian/$VERSION/${CORE}.tar.xz" -curl -O "https://oligarchy.co.uk/xapian/$VERSION/${BINDINGS}.tar.xz" - -# extract -echo "Extracting source..." -tar xf "${CORE}.tar.xz" -tar xf "${BINDINGS}.tar.xz" - -# install -echo "Installing Xapian-core..." -cd "$VIRTUAL_ENV/packages/${CORE}" || exit 1 -./configure --prefix="$VIRTUAL_ENV" && make -j"$(nproc)" && make install - -PYTHON_FLAG=--with-python3 - -echo "Installing Xapian-bindings..." -cd "$VIRTUAL_ENV/packages/${BINDINGS}" || exit 1 -./configure --prefix="$VIRTUAL_ENV" $PYTHON_FLAG XAPIAN_CONFIG="$VIRTUAL_ENV/bin/xapian-config" && make -j"$(nproc)" && make install - -# clean -rm -rf "$VIRTUAL_ENV/packages" - -# test -python -c "import xapian" diff --git a/pyproject.toml b/pyproject.toml index 7dc2e8dd..9a5e9135 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,8 @@ optional = true [tool.xapian] version = "1.4.25" +core-sha1 = "e2b4b4cf6076873ec9402cab7b9a3b71dcf95e20" +bindings-sha1 = "782f568d2ea3ca751c519a2814a35c7dc86df3a4" [tool.ruff] output-format = "concise" # makes ruff error logs easier to read