Merge pull request #928 from ae-utbm/vite

Integrate vite manifests in django

staticfiles/apps.py

@@ -3,7 +3,9 @@ from pathlib import Path
 from django.contrib.staticfiles.apps import StaticFilesConfig
 
 GENERATED_ROOT = Path(__file__).parent.resolve() / "generated"
-IGNORE_PATTERNS_BUNDLED = ["bundled/*"]
+BUNDLED_FOLDER_NAME = "bundled"
+BUNDLED_ROOT = GENERATED_ROOT / BUNDLED_FOLDER_NAME
+IGNORE_PATTERNS_BUNDLED = [f"{BUNDLED_FOLDER_NAME}/*"]
 IGNORE_PATTERNS_SCSS = ["*.scss"]
 IGNORE_PATTERNS_TYPESCRIPT = ["*.ts"]
 IGNORE_PATTERNS = [

staticfiles/processors.py

@@ -1,16 +1,93 @@
+import json
 import logging
 import subprocess
 from dataclasses import dataclass
 from hashlib import sha1
+from itertools import chain
 from pathlib import Path
-from typing import Iterable
+from typing import Iterable, Self
 
 import rjsmin
 import sass
 from django.conf import settings
 
 from sith.urls import api
-from staticfiles.apps import GENERATED_ROOT
+from staticfiles.apps import BUNDLED_FOLDER_NAME, BUNDLED_ROOT, GENERATED_ROOT
+
+
+@dataclass
+class JsBundlerManifestEntry:
+    src: str
+    out: str
+
+    @classmethod
+    def from_json_entry(cls, entry: dict[str, any]) -> list[Self]:
+        # We have two parts for a manifest entry
+        # The `src` element which is what the user asks django as a static
+        # The `out` element which is it's real name in the output static folder
+
+        # For the src part:
+        # The manifest file contains the path of the file relative to the project root
+        # We want the relative path of the file inside their respective static folder
+        # because that's what the user types when importing statics and that's what django gives us
+        # This is really similar to what we are doing in the bundler, it uses a similar algorithm
+        # Example:
+        # core/static/bundled/alpine-index.js -> bundled/alpine-index.js
+        # core/static/bundled/components/include-index.ts -> core/static/bundled/components/include-index.ts
+        def get_relative_src_name(name: str) -> str:
+            original_path = Path(name)
+            relative_path: list[str] = []
+            for directory in reversed(original_path.parts):
+                relative_path.append(directory)
+                # Contrary to the bundler algorithm, we do want to keep the bundled prefix
+                if directory == BUNDLED_FOLDER_NAME:
+                    break
+            return str(Path(*reversed(relative_path)))
+
+        # For the out part:
+        # The bundler is configured to output files in generated/bundled and considers this folders as it's root
+        # Thus, the output name doesn't contain the `bundled` prefix that we need, we add it ourselves
+        ret = [
+            cls(
+                src=get_relative_src_name(entry["src"]),
+                out=str(Path(BUNDLED_FOLDER_NAME) / entry["file"]),
+            )
+        ]
+
+        def remove_hash(path: Path) -> str:
+            # Hashes are configured to be surrounded by `.`
+            # Filenames are like this path/to/file.hash.ext
+            unhashed = ".".join(path.stem.split(".")[:-1])
+            return str(path.with_stem(unhashed))
+
+        # CSS files generated by entrypoints don't have their own entry in the manifest
+        # They are however listed as an attribute of the entry point that generates them
+        # Their listed name is the one that has been generated inside the generated/bundled folder
+        # We prefix it with `bundled` and then generate an `src` name by removing the hash
+        for css in entry.get("css", []):
+            path = Path(BUNDLED_FOLDER_NAME) / css
+            ret.append(
+                cls(
+                    src=remove_hash(path),
+                    out=str(path),
+                )
+            )
+        return ret
+
+
+class JSBundlerManifest:
+    def __init__(self, manifest: Path):
+        with open(manifest, "r") as f:
+            self._manifest = json.load(f)
+
+        self._files = chain(
+            *[
+                JsBundlerManifestEntry.from_json_entry(value)
+                for value in self._manifest.values()
+                if value.get("isEntry", False)
+            ]
+        )
+        self.mapping = {file.src: file.out for file in self._files}
 
 
 class JSBundler:
@@ -28,6 +105,16 @@ class JSBundler:
         logging.getLogger("django").info("Running javascript bundling server")
         return subprocess.Popen(["npm", "run", "serve"])
 
+    @staticmethod
+    def get_manifest() -> JSBundlerManifest:
+        return JSBundlerManifest(BUNDLED_ROOT / ".vite" / "manifest.json")
+
+    @staticmethod
+    def is_in_bundle(name: str | None) -> bool:
+        if name is None:
+            return False
+        return Path(name).parts[0] == BUNDLED_FOLDER_NAME
+
 
 class Scss:
     @dataclass
@@ -69,7 +156,7 @@ class JS:
             p
             for p in settings.STATIC_ROOT.rglob("*.js")
             if ".min" not in p.suffixes
-            and (settings.STATIC_ROOT / "bundled") not in p.parents
+            and (settings.STATIC_ROOT / BUNDLED_FOLDER_NAME) not in p.parents
         ]
         for path in to_exec:
             p = path.resolve()

staticfiles/storage.py

@@ -7,14 +7,26 @@ from django.contrib.staticfiles.storage import (
 )
 from django.core.files.storage import Storage
 
-from staticfiles.processors import JS, Scss
+from staticfiles.processors import JS, JSBundler, Scss
 
 
 class ManifestPostProcessingStorage(ManifestStaticFilesStorage):
     def url(self, name: str, *, force: bool = False) -> str:
-        """Get the URL for a file, convert .scss calls to .css ones and .ts to .js"""
+        """Get the URL for a file, convert .scss calls to .css calls to bundled files to their output ones"""
         # This name swap has to be done here
         # Otherwise, the manifest isn't aware of the file and can't work properly
+        if settings.DEBUG:
+            # In production, the bundler manifest is used at compile time, we don't need to convert anything
+            try:
+                manifest = JSBundler.get_manifest()
+            except FileNotFoundError as e:
+                raise Exception(
+                    "Error loading manifest file, the bundler seems to be busy"
+                ) from e
+            converted = manifest.mapping.get(name, None)
+            if converted:
+                name = converted
+
         path = Path(name)
         if path.suffix == ".scss":
             # Compile scss files automatically in debug mode
@@ -27,11 +39,14 @@ class ManifestPostProcessingStorage(ManifestStaticFilesStorage):
                 )
             name = str(path.with_suffix(".css"))
 
-        elif path.suffix == ".ts":
-            name = str(path.with_suffix(".js"))
-
         return super().url(name, force=force)
 
+    def hashed_name(self, name, content=None, filename=None):
+        # Ignore bundled files since they will be added at post process
+        if JSBundler.is_in_bundle(name):
+            return name
+        return super().hashed_name(name, content, filename)
+
     def post_process(
         self, paths: dict[str, tuple[Storage, str]], *, dry_run: bool = False
     ):
@@ -42,3 +57,7 @@ class ManifestPostProcessingStorage(ManifestStaticFilesStorage):
         yield from super().post_process(paths, dry_run)
         if not dry_run:
             JS.minify()
+
+        manifest = JSBundler.get_manifest()
+        self.hashed_files.update(manifest.mapping)
+        self.save_manifest()

vite.config.mts

@@ -2,7 +2,7 @@
 import { parse, resolve } from "node:path";
 import inject from "@rollup/plugin-inject";
 import { glob } from "glob";
-import type { AliasOptions, UserConfig } from "vite";
+import { type AliasOptions, type UserConfig, defineConfig } from "vite";
 import type { Rollup } from "vite";
 import { viteStaticCopy } from "vite-plugin-static-copy";
 import tsconfig from "./tsconfig.json";
@@ -44,66 +44,67 @@ function getRelativeAssetPath(path: string): string {
 }
 
 // biome-ignore lint/style/noDefaultExport: this is recommended by documentation
-export default {
-  base: "/static/bundled/",
-  appType: "custom",
-  build: {
-    outDir: outDir,
-    modulePreload: false, // would require `import 'vite/modulepreload-polyfill'` to always be injected
-    emptyOutDir: true,
-    rollupOptions: {
-      input: collectedFiles,
-      output: {
-        // Mirror architecture of static folders in generated .js and .css
-        entryFileNames: (chunkInfo: Rollup.PreRenderedChunk) => {
-          if (chunkInfo.facadeModuleId !== null) {
-            return `${getRelativeAssetPath(chunkInfo.facadeModuleId)}.js`;
-          }
-          return "[name].js";
-        },
-        assetFileNames: (chunkInfo: Rollup.PreRenderedAsset) => {
-          if (
-            chunkInfo.names?.length === 1 &&
-            chunkInfo.originalFileNames?.length === 1 &&
-            collectedFiles.includes(chunkInfo.originalFileNames[0])
-          ) {
-            return (
-              getRelativeAssetPath(chunkInfo.originalFileNames[0]) +
-              parse(chunkInfo.names[0]).ext
-            );
-          }
-          return "[name].[ext]";
+export default defineConfig((config: UserConfig) => {
+  return {
+    base: "/static/bundled/",
+    appType: "custom",
+    build: {
+      outDir: outDir,
+      manifest: true, // goes into .vite/manifest.json in the build folder
+      modulePreload: false, // would require `import 'vite/modulepreload-polyfill'` to always be injected
+      emptyOutDir: config.mode === "production", // Avoid rebuilding everything in dev mode
+      rollupOptions: {
+        input: collectedFiles,
+        output: {
+          // Mirror architecture of static folders in generated .js and .css
+          entryFileNames: (chunkInfo: Rollup.PreRenderedChunk) => {
+            if (chunkInfo.facadeModuleId !== null) {
+              return `${getRelativeAssetPath(chunkInfo.facadeModuleId)}.[hash].js`;
+            }
+            return "[name].[hash].js";
+          },
+          assetFileNames: (chunkInfo: Rollup.PreRenderedAsset) => {
+            if (
+              chunkInfo.names?.length === 1 &&
+              chunkInfo.originalFileNames?.length === 1 &&
+              collectedFiles.includes(chunkInfo.originalFileNames[0])
+            ) {
+              return `${getRelativeAssetPath(chunkInfo.originalFileNames[0])}.[hash][extname]`;
+            }
+            return "[name].[hash][extname]";
+          },
+          chunkFileNames: "[name].[hash].js",
         },
       },
     },
-  },
-  resolve: {
-    alias: getAliases(),
-  },
+    resolve: {
+      alias: getAliases(),
+    },
 
-  plugins: [
-    inject({
-      // biome-ignore lint/style/useNamingConvention: that's how it's called
-      Alpine: "alpinejs",
-    }),
-    viteStaticCopy({
-      targets: [
-        {
-          src: resolve(nodeModules, "jquery/dist/jquery.min.js"),
-          dest: vendored,
-        },
-        {
-          src: resolve(nodeModules, "jquery-ui/dist/jquery-ui.min.js"),
-          dest: vendored,
-        },
-        {
-          src: resolve(nodeModules, "jquery.shorten/src/jquery.shorten.min.js"),
-          dest: vendored,
-        },
-      ],
-    }),
-  ],
-  optimizeDeps: {
-    include: ["jquery"],
-  },
-} satisfies UserConfig;
+    plugins: [
+      inject({
+        // biome-ignore lint/style/useNamingConvention: that's how it's called
+        Alpine: "alpinejs",
+      }),
+      viteStaticCopy({
+        targets: [
+          {
+            src: resolve(nodeModules, "jquery/dist/jquery.min.js"),
+            dest: vendored,
+          },
+          {
+            src: resolve(nodeModules, "jquery-ui/dist/jquery-ui.min.js"),
+            dest: vendored,
+          },
+          {
+            src: resolve(nodeModules, "jquery.shorten/src/jquery.shorten.min.js"),
+            dest: vendored,
+          },
+        ],
+      }),
+    ],
+    optimizeDeps: {
+      include: ["jquery"],
+    },
+  } satisfies UserConfig;
+});