Migrate xapian install script to python

core/management/commands/install_xapian.py

@@ -13,12 +13,41 @@
 #
 #
 
+import hashlib
+import multiprocessing
 import os
+import shutil
 import subprocess
+import sys
+import tarfile
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Self
 
 import tomli
-from django.core.management.base import BaseCommand, CommandParser
+import urllib3
+from django.core.management.base import BaseCommand, CommandParser, OutputWrapper
+from urllib3.response import HTTPException
+
+
+@dataclass
+class XapianSpec:
+    version: str
+    core_sha1: str
+    bindings_sha1: str
+
+    @classmethod
+    def from_pyproject(cls) -> Self:
+        with open(
+            Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb"
+        ) as f:
+            pyproject = tomli.load(f)
+            spec = pyproject["tool"]["xapian"]
+            return cls(
+                version=spec["version"],
+                core_sha1=spec["core-sha1"],
+                bindings_sha1=spec["bindings-sha1"],
+            )
 
 
 class Command(BaseCommand):
@@ -39,13 +68,6 @@ class Command(BaseCommand):
             return None
         return xapian.version_string()
 
-    def _desired_version(self) -> str:
-        with open(
-            Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb"
-        ) as f:
-            pyproject = tomli.load(f)
-            return pyproject["tool"]["xapian"]["version"]
-
     def handle(self, *args, force: bool, **options):
         if not os.environ.get("VIRTUAL_ENV", None):
             self.stdout.write(
@@ -53,20 +75,152 @@ class Command(BaseCommand):
             )
             return
 
-        desired = self._desired_version()
+        desired = XapianSpec.from_pyproject()
-        if desired == self._current_version():
+        if desired.version == self._current_version():
             if not force:
                 self.stdout.write(
-                    f"Version {desired} is already installed, use --force to re-install"
+                    f"Version {desired.version} is already installed, use --force to re-install"
                 )
                 return
-            self.stdout.write(f"Version {desired} is already installed, re-installing")
+            self.stdout.write(
-        self.stdout.write(
+                f"Version {desired.version} is already installed, re-installing"
-            f"Installing xapian version {desired} at {os.environ['VIRTUAL_ENV']}"
+            )
-        )
+        XapianInstaller(desired, self.stdout, self.stderr).run()
-        subprocess.run(
-            [str(Path(__file__).parent / "install_xapian.sh"), desired],
-            env=dict(os.environ),
-            check=True,
-        )
         self.stdout.write("Installation success")
+
+
+class XapianInstaller:
+    def __init__(
+        self,
+        spec: XapianSpec,
+        stdout: OutputWrapper,
+        stderr: OutputWrapper,
+    ):
+        self._version = spec.version
+        self._core_sha1 = spec.core_sha1
+        self._bindings_sha1 = spec.bindings_sha1
+
+        self._stdout = stdout
+        self._stderr = stderr
+        self._virtual_env = os.environ.get("VIRTUAL_ENV", None)
+
+        if not self._virtual_env:
+            raise RuntimeError("You are not inside a virtual environment")
+        self._virtual_env = Path(self._virtual_env)
+
+        self._dest_dir = Path(self._virtual_env) / "packages"
+        self._core = f"xapian-core-{self._version}"
+        self._bindings = f"xapian-bindings-{self._version}"
+
+    def _setup_env(self):
+        os.environ.update(
+            {
+                "CPATH": "",
+                "LIBRARY_PATH": "",
+                "CFLAGS": "",
+                "LDFLAGS": "",
+                "CCFLAGS": "",
+                "CXXFLAGS": "",
+                "CPPFLAGS": "",
+            }
+        )
+
+    def _prepare_dest_folder(self):
+        shutil.rmtree(self._dest_dir, ignore_errors=True)
+        self._dest_dir.mkdir(parents=True)
+
+    def _download(self):
+        def download(url: str, dest: Path, sha1_hash: str):
+            resp = urllib3.request("GET", url)
+            if resp.status != 200:
+                raise HTTPException(f"Could not download {url}")
+            if hashlib.sha1(resp.data).hexdigest() != sha1_hash:
+                raise ValueError(f"File downloaded from {url} is compromised")
+            with open(dest, "wb") as f:
+                f.write(resp.data)
+
+        self._stdout.write("Downloading source…")
+
+        core = self._dest_dir / f"{self._core}.tar.xz"
+        bindings = self._dest_dir / f"{self._bindings}.tar.xz"
+        download(
+            f"https://oligarchy.co.uk/xapian/{self._version}/{self._core}.tar.xz",
+            core,
+            "e2b4b4cf6076873ec9402cab7b9a3b71dcf95e20",
+        )
+        download(
+            f"https://oligarchy.co.uk/xapian/{self._version}/{self._bindings}.tar.xz",
+            bindings,
+            "782f568d2ea3ca751c519a2814a35c7dc86df3a4",
+        )
+        self._stdout.write("Extracting source …")
+        with tarfile.open(core) as tar:
+            tar.extractall(self._dest_dir)
+        with tarfile.open(bindings) as tar:
+            tar.extractall(self._dest_dir)
+
+        os.remove(core)
+        os.remove(bindings)
+
+    def _install(self):
+        self._stdout.write("Installing Xapian-core…")
+        subprocess.run(
+            ["./configure", "--prefix", str(self._virtual_env)],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._core,
+        ).check_returncode()
+        subprocess.run(
+            [
+                "make",
+                "-j",
+                str(multiprocessing.cpu_count()),
+            ],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._core,
+        ).check_returncode()
+        subprocess.run(
+            ["make", "install"],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._core,
+        ).check_returncode()
+
+        self._stdout.write("Installing Xapian-bindings")
+        subprocess.run(
+            [
+                "./configure",
+                "--prefix",
+                str(self._virtual_env),
+                "--with-python3",
+                f"XAPIAN_CONFIG={self._virtual_env / 'bin'/'xapian-config'}",
+            ],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._bindings,
+        ).check_returncode()
+        subprocess.run(
+            [
+                "make",
+                "-j",
+                str(multiprocessing.cpu_count()),
+            ],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._bindings,
+        ).check_returncode()
+        subprocess.run(
+            ["make", "install"],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._bindings,
+        ).check_returncode()
+
+    def _post_clean(self):
+        shutil.rmtree(self._dest_dir, ignore_errors=True)
+
+    def _test(self):
+        subprocess.run([sys.executable, "-c", "import xapian"]).check_returncode()
+
+    def run(self):
+        self._setup_env()
+        self._prepare_dest_folder()
+        self._download()
+        self._install()
+        self._post_clean()
+        self._test()

core/management/commands/install_xapian.sh

@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-# Originates from https://gist.github.com/jorgecarleitao/ab6246c86c936b9c55fd
-# first argument of the script is Xapian version (e.g. 1.2.19)
-VERSION="$1"
-
-# Cleanup env vars for auto discovery mechanism
-export CPATH=
-export LIBRARY_PATH=
-export CFLAGS=
-export LDFLAGS=
-export CCFLAGS=
-export CXXFLAGS=
-export CPPFLAGS=
-
-# prepare
-rm -rf "$VIRTUAL_ENV/packages"
-mkdir -p "$VIRTUAL_ENV/packages" && cd "$VIRTUAL_ENV/packages" || exit 1
-
-CORE=xapian-core-$VERSION
-BINDINGS=xapian-bindings-$VERSION
-
-# download
-echo "Downloading source..."
-curl -O "https://oligarchy.co.uk/xapian/$VERSION/${CORE}.tar.xz"
-curl -O "https://oligarchy.co.uk/xapian/$VERSION/${BINDINGS}.tar.xz"
-
-# extract
-echo "Extracting source..."
-tar xf "${CORE}.tar.xz"
-tar xf "${BINDINGS}.tar.xz"
-
-# install
-echo "Installing Xapian-core..."
-cd "$VIRTUAL_ENV/packages/${CORE}" || exit 1
-./configure --prefix="$VIRTUAL_ENV" && make -j"$(nproc)" && make install
-
-PYTHON_FLAG=--with-python3
-
-echo "Installing Xapian-bindings..."
-cd "$VIRTUAL_ENV/packages/${BINDINGS}" || exit 1
-./configure --prefix="$VIRTUAL_ENV" $PYTHON_FLAG XAPIAN_CONFIG="$VIRTUAL_ENV/bin/xapian-config" && make -j"$(nproc)" && make install
-
-# clean
-rm -rf "$VIRTUAL_ENV/packages"
-
-# test
-python -c "import xapian"

pyproject.toml

@@ -92,6 +92,8 @@ optional = true
 
 [tool.xapian]
 version = "1.4.25"
+core-sha1 = "e2b4b4cf6076873ec9402cab7b9a3b71dcf95e20"
+bindings-sha1 = "782f568d2ea3ca751c519a2814a35c7dc86df3a4"
 
 [tool.ruff]
 output-format = "concise" # makes ruff error logs easier to read