Fix windows path and shell calls issues

core/management/commands/install_xapian.py

@@ -13,12 +13,42 @@
 #
 #
 
+import hashlib
+import multiprocessing
 import os
+import platform
+import shutil
 import subprocess
+import sys
+import tarfile
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Self
 
 import tomli
-from django.core.management.base import BaseCommand, CommandParser
+import urllib3
+from django.core.management.base import BaseCommand, CommandParser, OutputWrapper
+from urllib3.response import HTTPException
+
+
+@dataclass
+class XapianSpec:
+    version: str
+    core_sha1: str
+    bindings_sha1: str
+
+    @classmethod
+    def from_pyproject(cls) -> Self:
+        with open(
+            Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb"
+        ) as f:
+            pyproject = tomli.load(f)
+            spec = pyproject["tool"]["xapian"]
+            return cls(
+                version=spec["version"],
+                core_sha1=spec["core-sha1"],
+                bindings_sha1=spec["bindings-sha1"],
+            )
 
 
 class Command(BaseCommand):
@@ -39,13 +69,6 @@ class Command(BaseCommand):
             return None
         return xapian.version_string()
 
-    def _desired_version(self) -> str:
-        with open(
-            Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb"
-        ) as f:
-            pyproject = tomli.load(f)
-            return pyproject["tool"]["xapian"]["version"]
-
     def handle(self, *args, force: bool, **options):
         if not os.environ.get("VIRTUAL_ENV", None):
             self.stdout.write(
@@ -53,20 +76,185 @@ class Command(BaseCommand):
             )
             return
 
-        desired = self._desired_version()
-        if desired == self._current_version():
+        desired = XapianSpec.from_pyproject()
+        if desired.version == self._current_version():
             if not force:
                 self.stdout.write(
-                    f"Version {desired} is already installed, use --force to re-install"
+                    f"Version {desired.version} is already installed, use --force to re-install"
                 )
                 return
-            self.stdout.write(f"Version {desired} is already installed, re-installing")
-        self.stdout.write(
-            f"Installing xapian version {desired} at {os.environ['VIRTUAL_ENV']}"
-        )
-        subprocess.run(
-            [str(Path(__file__).parent / "install_xapian.sh"), desired],
-            env=dict(os.environ),
-            check=True,
-        )
+            self.stdout.write(
+                f"Version {desired.version} is already installed, re-installing"
+            )
+        XapianInstaller(desired, self.stdout, self.stderr).run()
         self.stdout.write("Installation success")
+
+
+class XapianInstaller:
+    def __init__(
+        self,
+        spec: XapianSpec,
+        stdout: OutputWrapper,
+        stderr: OutputWrapper,
+    ):
+        self._version = spec.version
+        self._core_sha1 = spec.core_sha1
+        self._bindings_sha1 = spec.bindings_sha1
+
+        self._stdout = stdout
+        self._stderr = stderr
+        self._virtual_env = os.environ.get("VIRTUAL_ENV", None)
+
+        if not self._virtual_env:
+            raise RuntimeError("You are not inside a virtual environment")
+        self._virtual_env = Path(self._virtual_env)
+
+        self._dest_dir = Path(self._virtual_env) / "packages"
+        self._core = f"xapian-core-{self._version}"
+        self._bindings = f"xapian-bindings-{self._version}"
+
+    @property
+    def _is_windows(self) -> bool:
+        return platform.system() == "Windows"
+
+    def _util_download(self, url: str, dest: Path, sha1_hash: str) -> None:
+        resp = urllib3.request("GET", url)
+        if resp.status != 200:
+            raise HTTPException(f"Could not download {url}")
+        if hashlib.sha1(resp.data).hexdigest() != sha1_hash:
+            raise ValueError(f"File downloaded from {url} is compromised")
+        with open(dest, "wb") as f:
+            f.write(resp.data)
+
+    def _setup_env(self):
+        os.environ.update(
+            {
+                "CPATH": "",
+                "LIBRARY_PATH": "",
+                "CFLAGS": "",
+                "LDFLAGS": "",
+                "CCFLAGS": "",
+                "CXXFLAGS": "",
+                "CPPFLAGS": "",
+            }
+        )
+
+    def _prepare_dest_folder(self):
+        shutil.rmtree(self._dest_dir, ignore_errors=True)
+        self._dest_dir.mkdir(parents=True, exist_ok=True)
+
+    def _download(self):
+        self._stdout.write("Downloading source…")
+
+        core = self._dest_dir / f"{self._core}.tar.xz"
+        bindings = self._dest_dir / f"{self._bindings}.tar.xz"
+        self._util_download(
+            f"https://oligarchy.co.uk/xapian/{self._version}/{self._core}.tar.xz",
+            core,
+            self._core_sha1,
+        )
+        self._util_download(
+            f"https://oligarchy.co.uk/xapian/{self._version}/{self._bindings}.tar.xz",
+            bindings,
+            self._bindings_sha1,
+        )
+        self._stdout.write("Extracting source …")
+        with tarfile.open(core) as tar:
+            tar.extractall(self._dest_dir)
+        with tarfile.open(bindings) as tar:
+            tar.extractall(self._dest_dir)
+
+        os.remove(core)
+        os.remove(bindings)
+
+    def _install(self):
+        self._stdout.write("Installing Xapian-core…")
+        def configure() -> list[str]:
+            if self._is_windows:
+                return ["sh", "configure"]
+            return ["./configure"]
+        def enable_static() -> list[str]:
+            if self._is_windows:
+                return ["--enable-shared", "--disable-static"]
+            return []
+
+        # Make sure that xapian finds the correct executable
+        os.environ["PYTHON3"] = str(Path(sys.executable).as_posix())
+        
+        subprocess.run(
+            [*configure(), "--prefix", str(self._virtual_env.as_posix()), *enable_static(),],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._core,
+            check=False,
+            shell=self._is_windows,
+        ).check_returncode()
+        subprocess.run(
+            [
+                "make",
+                "-j",
+                str(multiprocessing.cpu_count()),
+            ],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._core,
+            check=False,
+            shell=self._is_windows,
+        ).check_returncode()
+        subprocess.run(
+            ["make", "install"],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._core,
+            check=False,
+            shell=self._is_windows,
+
+        ).check_returncode()
+        
+
+        self._stdout.write("Installing Xapian-bindings")
+        subprocess.run(
+            [
+                *configure(),
+                "--prefix",
+                str(self._virtual_env.as_posix()),
+                "--with-python3",
+                f"XAPIAN_CONFIG={(self._virtual_env / 'bin'/'xapian-config').as_posix()}",
+                *enable_static(),
+            ],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._bindings,
+            check=False,
+            shell=self._is_windows,
+        ).check_returncode()
+        subprocess.run(
+            [
+                "make",
+                "-j",
+                str(multiprocessing.cpu_count()),
+            ],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._bindings,
+            check=False,
+            shell=self._is_windows,
+        ).check_returncode()
+        subprocess.run(
+            ["make", "install"],
+            env=dict(os.environ),
+            cwd=self._dest_dir / self._bindings,
+            check=False,
+            shell=self._is_windows,
+        ).check_returncode()
+
+    def _post_clean(self):
+        shutil.rmtree(self._dest_dir, ignore_errors=True)
+
+    def _test(self):
+        subprocess.run(
+            [sys.executable, "-c", "import xapian"], check=False, shell=self._is_windows,
+        ).check_returncode()
+
+    def run(self):
+        self._setup_env()
+        self._prepare_dest_folder()
+        self._download()
+        self._install()
+        self._post_clean()
+        self._test()

core/management/commands/install_xapian.sh

@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-# Originates from https://gist.github.com/jorgecarleitao/ab6246c86c936b9c55fd
-# first argument of the script is Xapian version (e.g. 1.2.19)
-VERSION="$1"
-
-# Cleanup env vars for auto discovery mechanism
-export CPATH=
-export LIBRARY_PATH=
-export CFLAGS=
-export LDFLAGS=
-export CCFLAGS=
-export CXXFLAGS=
-export CPPFLAGS=
-
-# prepare
-rm -rf "$VIRTUAL_ENV/packages"
-mkdir -p "$VIRTUAL_ENV/packages" && cd "$VIRTUAL_ENV/packages" || exit 1
-
-CORE=xapian-core-$VERSION
-BINDINGS=xapian-bindings-$VERSION
-
-# download
-echo "Downloading source..."
-curl -O "https://oligarchy.co.uk/xapian/$VERSION/${CORE}.tar.xz"
-curl -O "https://oligarchy.co.uk/xapian/$VERSION/${BINDINGS}.tar.xz"
-
-# extract
-echo "Extracting source..."
-tar xf "${CORE}.tar.xz"
-tar xf "${BINDINGS}.tar.xz"
-
-# install
-echo "Installing Xapian-core..."
-cd "$VIRTUAL_ENV/packages/${CORE}" || exit 1
-./configure --prefix="$VIRTUAL_ENV" && make -j"$(nproc)" && make install
-
-PYTHON_FLAG=--with-python3
-
-echo "Installing Xapian-bindings..."
-cd "$VIRTUAL_ENV/packages/${BINDINGS}" || exit 1
-./configure --prefix="$VIRTUAL_ENV" $PYTHON_FLAG XAPIAN_CONFIG="$VIRTUAL_ENV/bin/xapian-config" && make -j"$(nproc)" && make install
-
-# clean
-rm -rf "$VIRTUAL_ENV/packages"
-
-# test
-python -c "import xapian"

pyproject.toml

@@ -84,6 +84,8 @@ default-groups = ["dev", "tests", "docs"]
 
 [tool.xapian]
 version = "1.4.25"
+core-sha1 = "e2b4b4cf6076873ec9402cab7b9a3b71dcf95e20"
+bindings-sha1 = "782f568d2ea3ca751c519a2814a35c7dc86df3a4"
 
 [tool.ruff]
 output-format = "concise" # makes ruff error logs easier to read

staticfiles/processors.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import subprocess
+import platform
 from dataclasses import dataclass
 from hashlib import sha1
 from itertools import chain
@@ -94,7 +95,7 @@ class JSBundler:
     @staticmethod
     def compile():
         """Bundle js files with the javascript bundler for production."""
-        process = subprocess.Popen(["npm", "run", "compile"])
+        process = subprocess.Popen(["npm", "run", "compile"], shell=platform.system() == "Windows")
         process.wait()
         if process.returncode:
             raise RuntimeError(f"Bundler failed with returncode {process.returncode}")
@@ -103,7 +104,7 @@ class JSBundler:
     def runserver() -> subprocess.Popen:
         """Bundle js files automatically in background when called in debug mode."""
         logging.getLogger("django").info("Running javascript bundling server")
-        return subprocess.Popen(["npm", "run", "serve"])
+        return subprocess.Popen(["npm", "run", "serve"], shell=platform.system() == "Windows")
 
     @staticmethod
     def get_manifest() -> JSBundlerManifest:
@@ -197,4 +198,4 @@ class OpenApi:
         with open(out, "w") as f:
             _ = f.write(schema)
 
-        subprocess.run(["npx", "openapi-ts"], check=True)
+        subprocess.run(["npx", "openapi-ts"], check=True, shell=platform.system() == "Windows")

vite.config.mts

@@ -1,5 +1,5 @@
 // biome-ignore lint/correctness/noNodejsModules: this is backend side
-import { parse, resolve } from "node:path";
+import { parse, resolve, sep } from "node:path";
 import inject from "@rollup/plugin-inject";
 import { glob } from "glob";
 import { type AliasOptions, type UserConfig, defineConfig } from "vite";
@@ -31,7 +31,7 @@ function getAliases(): AliasOptions {
 function getRelativeAssetPath(path: string): string {
   let relativePath: string[] = [];
   const fullPath = parse(path);
-  for (const dir of fullPath.dir.split("/").reverse()) {
+  for (const dir of fullPath.dir.split(sep).reverse()) {
     if (dir === "bundled") {
       break;
     }
@@ -40,7 +40,7 @@ function getRelativeAssetPath(path: string): string {
   // We collected folders in reverse order, we put them back in the original order
   relativePath = relativePath.reverse();
   relativePath.push(fullPath.name);
-  return relativePath.join("/");
+  return relativePath.join(sep);
 }
 
 // biome-ignore lint/style/noDefaultExport: this is recommended by documentation