Completely integrate wepack in django

* Migrate alpine
* Migrate jquery and jquery-ui
* Migrate shorten
* Add babel for javascript
* Introduce staticfiles django app
* Only bundle -index.js files in static/webpack
* Unify scss and webpack generated files
* Convert scss calls to static
* Add --clear-generated option to collectstatic
* Fix docs warnings

staticfiles/.gitignore

@@ -0,0 +1 @@
+generated/

staticfiles/apps.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+from django.contrib.staticfiles.apps import StaticFilesConfig
+
+GENERATED_ROOT = Path(__file__).parent.resolve() / "generated"
+IGNORE_PATTERNS_WEBPACK = ["webpack/*"]
+IGNORE_PATTERNS_SCSS = ["*.scss"]
+IGNORE_PATTERNS = [
+    *StaticFilesConfig.ignore_patterns,
+    *IGNORE_PATTERNS_WEBPACK,
+    *IGNORE_PATTERNS_SCSS,
+]
+
+
+# We override the original staticfiles app according to https://docs.djangoproject.com/en/4.2/ref/contrib/staticfiles/#customizing-the-ignored-pattern-list
+# However, this is buggy and requires us to have an exact naming of the class like this to be detected
+# Also, it requires to create all commands in management/commands again or they don't get detected by django
+# Workaround originates from https://stackoverflow.com/a/78724835/12640533
+class StaticFilesConfig(StaticFilesConfig):
+    """
+    Application in charge of processing statics files.
+    It replaces the original django staticfiles
+    It integrates scss files and webpack.
+    It makes sure that statics are properly collected and that they are automatically
+        when using the development server.
+    """
+
+    ignore_patterns = IGNORE_PATTERNS
+    name = "staticfiles"

staticfiles/finders.py

@@ -0,0 +1,36 @@
+import os
+
+from django.contrib.staticfiles import utils
+from django.contrib.staticfiles.finders import FileSystemFinder
+from django.core.files.storage import FileSystemStorage
+
+from staticfiles.apps import GENERATED_ROOT, IGNORE_PATTERNS_WEBPACK
+
+
+class GeneratedFilesFinder(FileSystemFinder):
+    """Find generated and regular static files"""
+
+    def __init__(self, app_names=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # Add GENERATED_ROOT after adding everything in settings.STATICFILES_DIRS
+        self.locations.append(("", GENERATED_ROOT))
+        generated_storage = FileSystemStorage(location=GENERATED_ROOT)
+        generated_storage.prefix = ""
+        self.storages[GENERATED_ROOT] = generated_storage
+
+    def list(self, ignore_patterns: list[str]):
+        # List all files availables
+        for _, root in self.locations:
+            # Skip nonexistent directories.
+            if not os.path.isdir(root):
+                continue
+
+            ignored = ignore_patterns
+            # We don't want to ignore webpack files in the generated folder
+            if root == GENERATED_ROOT:
+                ignored = list(set(ignored) - set(IGNORE_PATTERNS_WEBPACK))
+
+            storage = self.storages[root]
+            for path in utils.get_files(storage, ignored):
+                yield path, storage

staticfiles/management/commands/collectstatic.py

@@ -0,0 +1,60 @@
+import shutil
+from pathlib import Path
+
+from django.contrib.staticfiles.finders import get_finders
+from django.contrib.staticfiles.management.commands.collectstatic import (
+    Command as CollectStatic,
+)
+
+from staticfiles.apps import GENERATED_ROOT, IGNORE_PATTERNS_SCSS
+from staticfiles.processors import Scss, Webpack
+
+
+class Command(CollectStatic):
+    """Integrate webpack and css compilation to collectstatic"""
+
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        parser.add_argument(
+            "--clear-generated",
+            action="store_true",
+            help="Delete the generated folder after collecting statics.",
+        )
+
+    def set_options(self, **options):
+        super().set_options(**options)
+        self.clear_generated = options["clear_generated"]
+
+    def collect_scss(self) -> list[Scss.CompileArg]:
+        files: list[Scss.CompileArg] = []
+        for finder in get_finders():
+            for path, storage in finder.list(
+                set(self.ignore_patterns) - set(IGNORE_PATTERNS_SCSS)
+            ):
+                path = Path(path)
+                if path.suffix != ".scss":
+                    continue
+                files.append(
+                    Scss.CompileArg(absolute=storage.path(path), relative=path)
+                )
+        return files
+
+    def collect(self):
+        if self.clear:  # Clear generated folder
+            shutil.rmtree(GENERATED_ROOT, ignore_errors=True)
+
+        def to_path(location: str | tuple[str, str]) -> Path:
+            if isinstance(location, tuple):
+                # staticfiles can be in a (prefix, path) format
+                _, location = location
+            return Path(location)
+
+        Scss.compile(self.collect_scss())
+        Webpack.compile()
+
+        collected = super().collect()
+
+        if self.clear_generated:
+            shutil.rmtree(GENERATED_ROOT, ignore_errors=True)
+
+        return collected

staticfiles/management/commands/findstatic.py

@@ -0,0 +1,2 @@
+# ruff: noqa: F401
+from django.contrib.staticfiles.management.commands.findstatic import Command

staticfiles/management/commands/runserver.py

@@ -0,0 +1,22 @@
+import os
+
+from django.conf import settings
+from django.contrib.staticfiles.management.commands.runserver import (
+    Command as Runserver,
+)
+from django.utils.autoreload import DJANGO_AUTORELOAD_ENV
+
+from staticfiles.processors import Webpack
+
+
+class Command(Runserver):
+    """Light wrapper around the statics runserver that integrates webpack auto bundling"""
+
+    def run(self, **options):
+        # Only run webpack server when debug is enabled
+        # Also protects from re-launching the server if django reloads it
+        if os.environ.get(DJANGO_AUTORELOAD_ENV) is None and settings.DEBUG:
+            with Webpack.runserver():
+                super().run(**options)
+                return
+        super().run(**options)

staticfiles/processors.py

@@ -0,0 +1,73 @@
+import logging
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+import rjsmin
+import sass
+from django.conf import settings
+
+from staticfiles.apps import GENERATED_ROOT
+
+
+class Webpack:
+    @staticmethod
+    def compile():
+        """Bundle js files with webpack for production."""
+        process = subprocess.Popen(["npm", "run", "compile"])
+        process.wait()
+        if process.returncode:
+            raise RuntimeError(f"Webpack failed with returncode {process.returncode}")
+
+    @staticmethod
+    def runserver() -> subprocess.Popen:
+        """Bundle js files automatically in background when called in debug mode."""
+        logging.getLogger("django").info("Running webpack server")
+        return subprocess.Popen(["npm", "run", "serve"])
+
+
+class Scss:
+    @dataclass
+    class CompileArg:
+        absolute: Path  # Absolute path to the file
+        relative: Path  # Relative path inside the folder it has been collected
+
+    @staticmethod
+    def compile(files: CompileArg | Iterable[CompileArg]):
+        """Compile scss files to css files."""
+        # Generate files inside the generated folder
+        # .css files respects the hierarchy in the static folder it was found
+        # This converts arg.absolute -> generated/{arg.relative}.scss
+        # Example:
+        #   app/static/foo.scss          -> generated/foo.css
+        #   app/static/bar/foo.scss      -> generated/bar/foo.css
+        #   custom/location/bar/foo.scss -> generated/bar/foo.css
+        if isinstance(files, Scss.CompileArg):
+            files = [files]
+
+        base_args = {"output_style": "compressed", "precision": settings.SASS_PRECISION}
+
+        compiled_files = {
+            file.relative.with_suffix(".css"): sass.compile(
+                filename=str(file.absolute), **base_args
+            )
+            for file in files
+        }
+        for file, content in compiled_files.items():
+            dest = GENERATED_ROOT / file
+            dest.parent.mkdir(exist_ok=True, parents=True)
+            dest.write_text(content)
+
+
+class JS:
+    @staticmethod
+    def minify():
+        to_exec = [
+            p for p in settings.STATIC_ROOT.rglob("*.js") if ".min" not in p.suffixes
+        ]
+        for path in to_exec:
+            p = path.resolve()
+            minified = rjsmin.jsmin(p.read_text())
+            p.write_text(minified)
+            logging.getLogger("main").info(f"Minified {path}")

staticfiles/storage.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+from django.conf import settings
+from django.contrib.staticfiles.finders import find
+from django.contrib.staticfiles.storage import (
+    ManifestStaticFilesStorage,
+)
+from django.core.files.storage import Storage
+
+from staticfiles.processors import JS, Scss
+
+
+class ManifestPostProcessingStorage(ManifestStaticFilesStorage):
+    def url(self, name: str, *, force: bool = False) -> str:
+        """Get the URL for a file, convert .scss calls to .css ones"""
+        # This name swap has to be done here
+        # Otherwise, the manifest isn't aware of the file and can't work properly
+        path = Path(name)
+        if path.suffix == ".scss":
+            # Compile scss files automatically in debug mode
+            if settings.DEBUG:
+                Scss.compile(
+                    [
+                        Scss.CompileArg(absolute=Path(p), relative=Path(name))
+                        for p in find(name, all=True)
+                    ]
+                )
+            name = str(path.with_suffix(".css"))
+
+        return super().url(name, force=force)
+
+    def post_process(
+        self, paths: dict[str, tuple[Storage, str]], *, dry_run: bool = False
+    ):
+        # Whether we get the files that were processed by ManifestFilesMixin
+        # by calling super() or whether we get them from the manifest file
+        # makes no difference - we have to open the manifest file anyway
+        # because we need to update the paths stored inside it.
+        yield from super().post_process(paths, dry_run)
+        if not dry_run:
+            JS.minify()