mirror of
https://github.com/ae-utbm/sith.git
synced 2024-11-22 06:03:20 +00:00
Migrate xapian install script to python
This commit is contained in:
parent
b9cbba2309
commit
8747c21b12
@ -13,12 +13,41 @@
|
||||
#
|
||||
#
|
||||
|
||||
import hashlib
|
||||
import multiprocessing
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Self
|
||||
|
||||
import tomli
|
||||
from django.core.management.base import BaseCommand, CommandParser
|
||||
import urllib3
|
||||
from django.core.management.base import BaseCommand, CommandParser, OutputWrapper
|
||||
from urllib3.response import HTTPException
|
||||
|
||||
|
||||
@dataclass
|
||||
class XapianSpec:
|
||||
version: str
|
||||
core_sha1: str
|
||||
bindings_sha1: str
|
||||
|
||||
@classmethod
|
||||
def from_pyproject(cls) -> Self:
|
||||
with open(
|
||||
Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb"
|
||||
) as f:
|
||||
pyproject = tomli.load(f)
|
||||
spec = pyproject["tool"]["xapian"]
|
||||
return cls(
|
||||
version=spec["version"],
|
||||
core_sha1=spec["core-sha1"],
|
||||
bindings_sha1=spec["bindings-sha1"],
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
@ -39,13 +68,6 @@ class Command(BaseCommand):
|
||||
return None
|
||||
return xapian.version_string()
|
||||
|
||||
def _desired_version(self) -> str:
|
||||
with open(
|
||||
Path(__file__).parent.parent.parent.parent / "pyproject.toml", "rb"
|
||||
) as f:
|
||||
pyproject = tomli.load(f)
|
||||
return pyproject["tool"]["xapian"]["version"]
|
||||
|
||||
def handle(self, *args, force: bool, **options):
|
||||
if not os.environ.get("VIRTUAL_ENV", None):
|
||||
self.stdout.write(
|
||||
@ -53,20 +75,152 @@ class Command(BaseCommand):
|
||||
)
|
||||
return
|
||||
|
||||
desired = self._desired_version()
|
||||
if desired == self._current_version():
|
||||
desired = XapianSpec.from_pyproject()
|
||||
if desired.version == self._current_version():
|
||||
if not force:
|
||||
self.stdout.write(
|
||||
f"Version {desired} is already installed, use --force to re-install"
|
||||
f"Version {desired.version} is already installed, use --force to re-install"
|
||||
)
|
||||
return
|
||||
self.stdout.write(f"Version {desired} is already installed, re-installing")
|
||||
self.stdout.write(
|
||||
f"Installing xapian version {desired} at {os.environ['VIRTUAL_ENV']}"
|
||||
)
|
||||
subprocess.run(
|
||||
[str(Path(__file__).parent / "install_xapian.sh"), desired],
|
||||
env=dict(os.environ),
|
||||
check=True,
|
||||
)
|
||||
self.stdout.write(
|
||||
f"Version {desired.version} is already installed, re-installing"
|
||||
)
|
||||
XapianInstaller(desired, self.stdout, self.stderr).run()
|
||||
self.stdout.write("Installation success")
|
||||
|
||||
|
||||
class XapianInstaller:
|
||||
def __init__(
|
||||
self,
|
||||
spec: XapianSpec,
|
||||
stdout: OutputWrapper,
|
||||
stderr: OutputWrapper,
|
||||
):
|
||||
self._version = spec.version
|
||||
self._core_sha1 = spec.core_sha1
|
||||
self._bindings_sha1 = spec.bindings_sha1
|
||||
|
||||
self._stdout = stdout
|
||||
self._stderr = stderr
|
||||
self._virtual_env = os.environ.get("VIRTUAL_ENV", None)
|
||||
|
||||
if not self._virtual_env:
|
||||
raise RuntimeError("You are not inside a virtual environment")
|
||||
self._virtual_env = Path(self._virtual_env)
|
||||
|
||||
self._dest_dir = Path(self._virtual_env) / "packages"
|
||||
self._core = f"xapian-core-{self._version}"
|
||||
self._bindings = f"xapian-bindings-{self._version}"
|
||||
|
||||
def _setup_env(self):
|
||||
os.environ.update(
|
||||
{
|
||||
"CPATH": "",
|
||||
"LIBRARY_PATH": "",
|
||||
"CFLAGS": "",
|
||||
"LDFLAGS": "",
|
||||
"CCFLAGS": "",
|
||||
"CXXFLAGS": "",
|
||||
"CPPFLAGS": "",
|
||||
}
|
||||
)
|
||||
|
||||
def _prepare_dest_folder(self):
|
||||
shutil.rmtree(self._dest_dir, ignore_errors=True)
|
||||
self._dest_dir.mkdir(parents=True)
|
||||
|
||||
def _download(self):
|
||||
def download(url: str, dest: Path, sha1_hash: str):
|
||||
resp = urllib3.request("GET", url)
|
||||
if resp.status != 200:
|
||||
raise HTTPException(f"Could not download {url}")
|
||||
if hashlib.sha1(resp.data).hexdigest() != sha1_hash:
|
||||
raise ValueError(f"File downloaded from {url} is compromised")
|
||||
with open(dest, "wb") as f:
|
||||
f.write(resp.data)
|
||||
|
||||
self._stdout.write("Downloading source…")
|
||||
|
||||
core = self._dest_dir / f"{self._core}.tar.xz"
|
||||
bindings = self._dest_dir / f"{self._bindings}.tar.xz"
|
||||
download(
|
||||
f"https://oligarchy.co.uk/xapian/{self._version}/{self._core}.tar.xz",
|
||||
core,
|
||||
"e2b4b4cf6076873ec9402cab7b9a3b71dcf95e20",
|
||||
)
|
||||
download(
|
||||
f"https://oligarchy.co.uk/xapian/{self._version}/{self._bindings}.tar.xz",
|
||||
bindings,
|
||||
"782f568d2ea3ca751c519a2814a35c7dc86df3a4",
|
||||
)
|
||||
self._stdout.write("Extracting source …")
|
||||
with tarfile.open(core) as tar:
|
||||
tar.extractall(self._dest_dir)
|
||||
with tarfile.open(bindings) as tar:
|
||||
tar.extractall(self._dest_dir)
|
||||
|
||||
os.remove(core)
|
||||
os.remove(bindings)
|
||||
|
||||
def _install(self):
|
||||
self._stdout.write("Installing Xapian-core…")
|
||||
subprocess.run(
|
||||
["./configure", "--prefix", str(self._virtual_env)],
|
||||
env=dict(os.environ),
|
||||
cwd=self._dest_dir / self._core,
|
||||
).check_returncode()
|
||||
subprocess.run(
|
||||
[
|
||||
"make",
|
||||
"-j",
|
||||
str(multiprocessing.cpu_count()),
|
||||
],
|
||||
env=dict(os.environ),
|
||||
cwd=self._dest_dir / self._core,
|
||||
).check_returncode()
|
||||
subprocess.run(
|
||||
["make", "install"],
|
||||
env=dict(os.environ),
|
||||
cwd=self._dest_dir / self._core,
|
||||
).check_returncode()
|
||||
|
||||
self._stdout.write("Installing Xapian-bindings")
|
||||
subprocess.run(
|
||||
[
|
||||
"./configure",
|
||||
"--prefix",
|
||||
str(self._virtual_env),
|
||||
"--with-python3",
|
||||
f"XAPIAN_CONFIG={self._virtual_env / 'bin'/'xapian-config'}",
|
||||
],
|
||||
env=dict(os.environ),
|
||||
cwd=self._dest_dir / self._bindings,
|
||||
).check_returncode()
|
||||
subprocess.run(
|
||||
[
|
||||
"make",
|
||||
"-j",
|
||||
str(multiprocessing.cpu_count()),
|
||||
],
|
||||
env=dict(os.environ),
|
||||
cwd=self._dest_dir / self._bindings,
|
||||
).check_returncode()
|
||||
subprocess.run(
|
||||
["make", "install"],
|
||||
env=dict(os.environ),
|
||||
cwd=self._dest_dir / self._bindings,
|
||||
).check_returncode()
|
||||
|
||||
def _post_clean(self):
|
||||
shutil.rmtree(self._dest_dir, ignore_errors=True)
|
||||
|
||||
def _test(self):
|
||||
subprocess.run([sys.executable, "-c", "import xapian"]).check_returncode()
|
||||
|
||||
def run(self):
|
||||
self._setup_env()
|
||||
self._prepare_dest_folder()
|
||||
self._download()
|
||||
self._install()
|
||||
self._post_clean()
|
||||
self._test()
|
||||
|
||||
@ -1,47 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Originates from https://gist.github.com/jorgecarleitao/ab6246c86c936b9c55fd
|
||||
# first argument of the script is Xapian version (e.g. 1.2.19)
|
||||
VERSION="$1"
|
||||
|
||||
# Cleanup env vars for auto discovery mechanism
|
||||
export CPATH=
|
||||
export LIBRARY_PATH=
|
||||
export CFLAGS=
|
||||
export LDFLAGS=
|
||||
export CCFLAGS=
|
||||
export CXXFLAGS=
|
||||
export CPPFLAGS=
|
||||
|
||||
# prepare
|
||||
rm -rf "$VIRTUAL_ENV/packages"
|
||||
mkdir -p "$VIRTUAL_ENV/packages" && cd "$VIRTUAL_ENV/packages" || exit 1
|
||||
|
||||
CORE=xapian-core-$VERSION
|
||||
BINDINGS=xapian-bindings-$VERSION
|
||||
|
||||
# download
|
||||
echo "Downloading source..."
|
||||
curl -O "https://oligarchy.co.uk/xapian/$VERSION/${CORE}.tar.xz"
|
||||
curl -O "https://oligarchy.co.uk/xapian/$VERSION/${BINDINGS}.tar.xz"
|
||||
|
||||
# extract
|
||||
echo "Extracting source..."
|
||||
tar xf "${CORE}.tar.xz"
|
||||
tar xf "${BINDINGS}.tar.xz"
|
||||
|
||||
# install
|
||||
echo "Installing Xapian-core..."
|
||||
cd "$VIRTUAL_ENV/packages/${CORE}" || exit 1
|
||||
./configure --prefix="$VIRTUAL_ENV" && make -j"$(nproc)" && make install
|
||||
|
||||
PYTHON_FLAG=--with-python3
|
||||
|
||||
echo "Installing Xapian-bindings..."
|
||||
cd "$VIRTUAL_ENV/packages/${BINDINGS}" || exit 1
|
||||
./configure --prefix="$VIRTUAL_ENV" $PYTHON_FLAG XAPIAN_CONFIG="$VIRTUAL_ENV/bin/xapian-config" && make -j"$(nproc)" && make install
|
||||
|
||||
# clean
|
||||
rm -rf "$VIRTUAL_ENV/packages"
|
||||
|
||||
# test
|
||||
python -c "import xapian"
|
||||
@ -92,6 +92,8 @@ optional = true
|
||||
|
||||
[tool.xapian]
|
||||
version = "1.4.25"
|
||||
core-sha1 = "e2b4b4cf6076873ec9402cab7b9a3b71dcf95e20"
|
||||
bindings-sha1 = "782f568d2ea3ca751c519a2814a35c7dc86df3a4"
|
||||
|
||||
[tool.ruff]
|
||||
output-format = "concise" # makes ruff error logs easier to read
|
||||
|
||||
Loading…
Reference in New Issue
Block a user