diff --git a/core/management/commands/install_xapian.py b/core/management/commands/install_xapian.py index 4be1d907..70ecf24b 100644 --- a/core/management/commands/install_xapian.py +++ b/core/management/commands/install_xapian.py @@ -13,12 +13,42 @@ # # +import hashlib +import multiprocessing import os +import platform +import shutil import subprocess +import sys +import tarfile +from dataclasses import dataclass from pathlib import Path +from typing import Self import tomli -from django.core.management.base import BaseCommand, CommandParser +import urllib3 +from django.core.management.base import BaseCommand, CommandParser, OutputWrapper +from urllib3.response import HTTPException + + +@dataclass +class XapianSpec: + version: str + core_sha1: str + bindings_sha1: str + + @classmethod + def from_pyproject(cls) -> Self: + with open( + Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb" + ) as f: + pyproject = tomli.load(f) + spec = pyproject["tool"]["xapian"] + return cls( + version=spec["version"], + core_sha1=spec["core-sha1"], + bindings_sha1=spec["bindings-sha1"], + ) class Command(BaseCommand): @@ -39,13 +69,6 @@ class Command(BaseCommand): return None return xapian.version_string() - def _desired_version(self) -> str: - with open( - Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb" - ) as f: - pyproject = tomli.load(f) - return pyproject["tool"]["xapian"]["version"] - def handle(self, *args, force: bool, **options): if not os.environ.get("VIRTUAL_ENV", None): self.stdout.write( @@ -53,20 +76,185 @@ class Command(BaseCommand): ) return - desired = self._desired_version() - if desired == self._current_version(): + desired = XapianSpec.from_pyproject() + if desired.version == self._current_version(): if not force: self.stdout.write( - f"Version {desired} is already installed, use --force to re-install" + f"Version {desired.version} is already installed, use --force to re-install" ) return - self.stdout.write(f"Version {desired} is already installed, re-installing") - self.stdout.write( - f"Installing xapian version {desired} at {os.environ['VIRTUAL_ENV']}" - ) - subprocess.run( - [str(Path(__file__).parent / "install_xapian.sh"), desired], - env=dict(os.environ), - check=True, - ) + self.stdout.write( + f"Version {desired.version} is already installed, re-installing" + ) + XapianInstaller(desired, self.stdout, self.stderr).run() self.stdout.write("Installation success") + + +class XapianInstaller: + def __init__( + self, + spec: XapianSpec, + stdout: OutputWrapper, + stderr: OutputWrapper, + ): + self._version = spec.version + self._core_sha1 = spec.core_sha1 + self._bindings_sha1 = spec.bindings_sha1 + + self._stdout = stdout + self._stderr = stderr + self._virtual_env = os.environ.get("VIRTUAL_ENV", None) + + if not self._virtual_env: + raise RuntimeError("You are not inside a virtual environment") + self._virtual_env = Path(self._virtual_env) + + self._dest_dir = Path(self._virtual_env) / "packages" + self._core = f"xapian-core-{self._version}" + self._bindings = f"xapian-bindings-{self._version}" + + @property + def _is_windows(self) -> bool: + return platform.system() == "Windows" + + def _util_download(self, url: str, dest: Path, sha1_hash: str) -> None: + resp = urllib3.request("GET", url) + if resp.status != 200: + raise HTTPException(f"Could not download {url}") + if hashlib.sha1(resp.data).hexdigest() != sha1_hash: + raise ValueError(f"File downloaded from {url} is compromised") + with open(dest, "wb") as f: + f.write(resp.data) + + def _setup_env(self): + os.environ.update( + { + "CPATH": "", + "LIBRARY_PATH": "", + "CFLAGS": "", + "LDFLAGS": "", + "CCFLAGS": "", + "CXXFLAGS": "", + "CPPFLAGS": "", + } + ) + + def _prepare_dest_folder(self): + shutil.rmtree(self._dest_dir, ignore_errors=True) + self._dest_dir.mkdir(parents=True, exist_ok=True) + + def _download(self): + self._stdout.write("Downloading source…") + + core = self._dest_dir / f"{self._core}.tar.xz" + bindings = self._dest_dir / f"{self._bindings}.tar.xz" + self._util_download( + f"https://oligarchy.co.uk/xapian/{self._version}/{self._core}.tar.xz", + core, + self._core_sha1, + ) + self._util_download( + f"https://oligarchy.co.uk/xapian/{self._version}/{self._bindings}.tar.xz", + bindings, + self._bindings_sha1, + ) + self._stdout.write("Extracting source …") + with tarfile.open(core) as tar: + tar.extractall(self._dest_dir) + with tarfile.open(bindings) as tar: + tar.extractall(self._dest_dir) + + os.remove(core) + os.remove(bindings) + + def _install(self): + self._stdout.write("Installing Xapian-core…") + def configure() -> list[str]: + if self._is_windows: + return ["sh", "configure"] + return ["./configure"] + def enable_static() -> list[str]: + if self._is_windows: + return ["--enable-shared", "--disable-static"] + return [] + + # Make sure that xapian finds the correct executable + os.environ["PYTHON3"] = str(Path(sys.executable).as_posix()) + + subprocess.run( + [*configure(), "--prefix", str(self._virtual_env.as_posix()), *enable_static(),], + env=dict(os.environ), + cwd=self._dest_dir / self._core, + check=False, + shell=self._is_windows, + ).check_returncode() + subprocess.run( + [ + "make", + "-j", + str(multiprocessing.cpu_count()), + ], + env=dict(os.environ), + cwd=self._dest_dir / self._core, + check=False, + shell=self._is_windows, + ).check_returncode() + subprocess.run( + ["make", "install"], + env=dict(os.environ), + cwd=self._dest_dir / self._core, + check=False, + shell=self._is_windows, + + ).check_returncode() + + + self._stdout.write("Installing Xapian-bindings") + subprocess.run( + [ + *configure(), + "--prefix", + str(self._virtual_env.as_posix()), + "--with-python3", + f"XAPIAN_CONFIG={(self._virtual_env / 'bin'/'xapian-config').as_posix()}", + *enable_static(), + ], + env=dict(os.environ), + cwd=self._dest_dir / self._bindings, + check=False, + shell=self._is_windows, + ).check_returncode() + subprocess.run( + [ + "make", + "-j", + str(multiprocessing.cpu_count()), + ], + env=dict(os.environ), + cwd=self._dest_dir / self._bindings, + check=False, + shell=self._is_windows, + ).check_returncode() + subprocess.run( + ["make", "install"], + env=dict(os.environ), + cwd=self._dest_dir / self._bindings, + check=False, + shell=self._is_windows, + ).check_returncode() + + def _post_clean(self): + shutil.rmtree(self._dest_dir, ignore_errors=True) + + def _test(self): + subprocess.run( + [sys.executable, "-c", "import xapian"], check=False, shell=self._is_windows, + ).check_returncode() + + def run(self): + self._setup_env() + self._prepare_dest_folder() + self._download() + self._install() + self._post_clean() + self._test() diff --git a/core/management/commands/install_xapian.sh b/core/management/commands/install_xapian.sh deleted file mode 100755 index 3ca2ac17..00000000 --- a/core/management/commands/install_xapian.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Originates from https://gist.github.com/jorgecarleitao/ab6246c86c936b9c55fd -# first argument of the script is Xapian version (e.g. 1.2.19) -VERSION="$1" - -# Cleanup env vars for auto discovery mechanism -export CPATH= -export LIBRARY_PATH= -export CFLAGS= -export LDFLAGS= -export CCFLAGS= -export CXXFLAGS= -export CPPFLAGS= - -# prepare -rm -rf "$VIRTUAL_ENV/packages" -mkdir -p "$VIRTUAL_ENV/packages" && cd "$VIRTUAL_ENV/packages" || exit 1 - -CORE=xapian-core-$VERSION -BINDINGS=xapian-bindings-$VERSION - -# download -echo "Downloading source..." -curl -O "https://oligarchy.co.uk/xapian/$VERSION/${CORE}.tar.xz" -curl -O "https://oligarchy.co.uk/xapian/$VERSION/${BINDINGS}.tar.xz" - -# extract -echo "Extracting source..." -tar xf "${CORE}.tar.xz" -tar xf "${BINDINGS}.tar.xz" - -# install -echo "Installing Xapian-core..." -cd "$VIRTUAL_ENV/packages/${CORE}" || exit 1 -./configure --prefix="$VIRTUAL_ENV" && make -j"$(nproc)" && make install - -PYTHON_FLAG=--with-python3 - -echo "Installing Xapian-bindings..." -cd "$VIRTUAL_ENV/packages/${BINDINGS}" || exit 1 -./configure --prefix="$VIRTUAL_ENV" $PYTHON_FLAG XAPIAN_CONFIG="$VIRTUAL_ENV/bin/xapian-config" && make -j"$(nproc)" && make install - -# clean -rm -rf "$VIRTUAL_ENV/packages" - -# test -python -c "import xapian" diff --git a/pyproject.toml b/pyproject.toml index a4d16abc..051154a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,8 @@ default-groups = ["dev", "tests", "docs"] [tool.xapian] version = "1.4.25" +core-sha1 = "e2b4b4cf6076873ec9402cab7b9a3b71dcf95e20" +bindings-sha1 = "782f568d2ea3ca751c519a2814a35c7dc86df3a4" [tool.ruff] output-format = "concise" # makes ruff error logs easier to read diff --git a/staticfiles/processors.py b/staticfiles/processors.py index 3a0df243..9766601a 100644 --- a/staticfiles/processors.py +++ b/staticfiles/processors.py @@ -1,6 +1,7 @@ import json import logging import subprocess +import platform from dataclasses import dataclass from hashlib import sha1 from itertools import chain @@ -94,7 +95,7 @@ class JSBundler: @staticmethod def compile(): """Bundle js files with the javascript bundler for production.""" - process = subprocess.Popen(["npm", "run", "compile"]) + process = subprocess.Popen(["npm", "run", "compile"], shell=platform.system() == "Windows") process.wait() if process.returncode: raise RuntimeError(f"Bundler failed with returncode {process.returncode}") @@ -103,7 +104,7 @@ class JSBundler: def runserver() -> subprocess.Popen: """Bundle js files automatically in background when called in debug mode.""" logging.getLogger("django").info("Running javascript bundling server") - return subprocess.Popen(["npm", "run", "serve"]) + return subprocess.Popen(["npm", "run", "serve"], shell=platform.system() == "Windows") @staticmethod def get_manifest() -> JSBundlerManifest: @@ -197,4 +198,4 @@ class OpenApi: with open(out, "w") as f: _ = f.write(schema) - subprocess.run(["npx", "openapi-ts"], check=True) + subprocess.run(["npx", "openapi-ts"], check=True, shell=platform.system() == "Windows") diff --git a/vite.config.mts b/vite.config.mts index 015465e5..941cf164 100644 --- a/vite.config.mts +++ b/vite.config.mts @@ -1,5 +1,5 @@ // biome-ignore lint/correctness/noNodejsModules: this is backend side -import { parse, resolve } from "node:path"; +import { parse, resolve, sep } from "node:path"; import inject from "@rollup/plugin-inject"; import { glob } from "glob"; import { type AliasOptions, type UserConfig, defineConfig } from "vite"; @@ -31,7 +31,7 @@ function getAliases(): AliasOptions { function getRelativeAssetPath(path: string): string { let relativePath: string[] = []; const fullPath = parse(path); - for (const dir of fullPath.dir.split("/").reverse()) { + for (const dir of fullPath.dir.split(sep).reverse()) { if (dir === "bundled") { break; } @@ -40,7 +40,7 @@ function getRelativeAssetPath(path: string): string { // We collected folders in reverse order, we put them back in the original order relativePath = relativePath.reverse(); relativePath.push(fullPath.name); - return relativePath.join("/"); + return relativePath.join(sep); } // biome-ignore lint/style/noDefaultExport: this is recommended by documentation