From 8fc1a754de026cb2d455800e75d75e030f1ac8fd Mon Sep 17 00:00:00 2001 From: Sli Date: Wed, 20 Nov 2024 18:24:28 +0100 Subject: [PATCH 1/2] Integrates vite manifests to django --- staticfiles/apps.py | 1 + staticfiles/processors.py | 55 +++++++++++++++++++++++++++++++++++++-- staticfiles/storage.py | 29 +++++++++++++++++---- vite.config.mts | 13 +++++---- 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/staticfiles/apps.py b/staticfiles/apps.py index d4d27d0b..dfec2735 100644 --- a/staticfiles/apps.py +++ b/staticfiles/apps.py @@ -3,6 +3,7 @@ from pathlib import Path from django.contrib.staticfiles.apps import StaticFilesConfig GENERATED_ROOT = Path(__file__).parent.resolve() / "generated" +BUNDLED_ROOT = GENERATED_ROOT / "bundled" IGNORE_PATTERNS_BUNDLED = ["bundled/*"] IGNORE_PATTERNS_SCSS = ["*.scss"] IGNORE_PATTERNS_TYPESCRIPT = ["*.ts"] diff --git a/staticfiles/processors.py b/staticfiles/processors.py index cfbbe1fb..8d458686 100644 --- a/staticfiles/processors.py +++ b/staticfiles/processors.py @@ -1,16 +1,57 @@ +import json import logging import subprocess from dataclasses import dataclass from hashlib import sha1 +from itertools import chain from pathlib import Path -from typing import Iterable +from typing import Iterable, Self import rjsmin import sass from django.conf import settings from sith.urls import api -from staticfiles.apps import GENERATED_ROOT +from staticfiles.apps import BUNDLED_ROOT, GENERATED_ROOT + + +@dataclass +class JsBundlerManifestEntry: + out: str + src: str + + @classmethod + def from_json_entry(cls, entry: dict[str, any]) -> list[Self]: + ret = [ + cls( + out=str(Path("bundled") / entry["file"]), + src=str(Path(*Path(entry["src"]).parts[2:])), + ) + ] + for css in entry.get("css", []): + path = Path("bundled") / css + ret.append( + cls( + out=str(path), + src=str(path.with_stem(entry["name"])), + ) + ) + return ret + + +class JSBundlerManifest: + def __init__(self, manifest: Path): + with open(manifest, "r") as f: + self._manifest = json.load(f) + + self._files = chain( + *[ + JsBundlerManifestEntry.from_json_entry(value) + for value in self._manifest.values() + if value.get("isEntry", False) + ] + ) + self.mapping = {file.src: file.out for file in self._files} class JSBundler: @@ -28,6 +69,16 @@ class JSBundler: logging.getLogger("django").info("Running javascript bundling server") return subprocess.Popen(["npm", "run", "serve"]) + @staticmethod + def get_manifest() -> JSBundlerManifest: + return JSBundlerManifest(BUNDLED_ROOT / ".vite" / "manifest.json") + + @staticmethod + def is_in_bundle(name: str | None) -> bool: + if name is None: + return False + return name.startswith("bundled/") + class Scss: @dataclass diff --git a/staticfiles/storage.py b/staticfiles/storage.py index 3aaca470..0022438c 100644 --- a/staticfiles/storage.py +++ b/staticfiles/storage.py @@ -7,15 +7,27 @@ from django.contrib.staticfiles.storage import ( ) from django.core.files.storage import Storage -from staticfiles.processors import JS, Scss +from staticfiles.processors import JS, JSBundler, Scss class ManifestPostProcessingStorage(ManifestStaticFilesStorage): def url(self, name: str, *, force: bool = False) -> str: - """Get the URL for a file, convert .scss calls to .css ones and .ts to .js""" + """Get the URL for a file, convert .scss calls to .css calls to bundled files to their output ones""" # This name swap has to be done here # Otherwise, the manifest isn't aware of the file and can't work properly + if settings.DEBUG: + try: + manifest = JSBundler.get_manifest() + except Exception as e: + raise Exception( + "Error loading manifest file, the bundler seems to be busy" + ) from e + converted = manifest.mapping.get(name, None) + if converted: + name = converted + path = Path(name) + # Call bundler manifest if path.suffix == ".scss": # Compile scss files automatically in debug mode if settings.DEBUG: @@ -27,11 +39,14 @@ class ManifestPostProcessingStorage(ManifestStaticFilesStorage): ) name = str(path.with_suffix(".css")) - elif path.suffix == ".ts": - name = str(path.with_suffix(".js")) - return super().url(name, force=force) + def hashed_name(self, name, content=None, filename=None): + # Ignore bundled files since they will be added at post process + if JSBundler.is_in_bundle(name): + return name + return super().hashed_name(name, content, filename) + def post_process( self, paths: dict[str, tuple[Storage, str]], *, dry_run: bool = False ): @@ -42,3 +57,7 @@ class ManifestPostProcessingStorage(ManifestStaticFilesStorage): yield from super().post_process(paths, dry_run) if not dry_run: JS.minify() + + manifest = JSBundler.get_manifest() + self.hashed_files.update(manifest.mapping) + self.save_manifest() diff --git a/vite.config.mts b/vite.config.mts index fab81862..43ec5ad9 100644 --- a/vite.config.mts +++ b/vite.config.mts @@ -49,6 +49,7 @@ export default { appType: "custom", build: { outDir: outDir, + manifest: true, // goes into .vite/manifest.json in the build folder modulePreload: false, // would require `import 'vite/modulepreload-polyfill'` to always be injected emptyOutDir: true, rollupOptions: { @@ -57,9 +58,9 @@ export default { // Mirror architecture of static folders in generated .js and .css entryFileNames: (chunkInfo: Rollup.PreRenderedChunk) => { if (chunkInfo.facadeModuleId !== null) { - return `${getRelativeAssetPath(chunkInfo.facadeModuleId)}.js`; + return `${getRelativeAssetPath(chunkInfo.facadeModuleId)}.[hash].js`; } - return "[name].js"; + return "[name].[hash].js"; }, assetFileNames: (chunkInfo: Rollup.PreRenderedAsset) => { if ( @@ -67,13 +68,11 @@ export default { chunkInfo.originalFileNames?.length === 1 && collectedFiles.includes(chunkInfo.originalFileNames[0]) ) { - return ( - getRelativeAssetPath(chunkInfo.originalFileNames[0]) + - parse(chunkInfo.names[0]).ext - ); + return `${getRelativeAssetPath(chunkInfo.originalFileNames[0])}.[hash][extname]`; } - return "[name].[ext]"; + return "[name].[hash][extname]"; }, + chunkFileNames: "[name].[hash].js", }, }, }, From 0739ce2fb4ae9a7ee5b8edd417314d14e1720041 Mon Sep 17 00:00:00 2001 From: Sli Date: Thu, 21 Nov 2024 00:33:40 +0100 Subject: [PATCH 2/2] Improve readability and usability --- staticfiles/apps.py | 5 +- staticfiles/processors.py | 52 ++++++++++++++--- staticfiles/storage.py | 4 +- vite.config.mts | 120 +++++++++++++++++++------------------- 4 files changed, 110 insertions(+), 71 deletions(-) diff --git a/staticfiles/apps.py b/staticfiles/apps.py index dfec2735..728a7063 100644 --- a/staticfiles/apps.py +++ b/staticfiles/apps.py @@ -3,8 +3,9 @@ from pathlib import Path from django.contrib.staticfiles.apps import StaticFilesConfig GENERATED_ROOT = Path(__file__).parent.resolve() / "generated" -BUNDLED_ROOT = GENERATED_ROOT / "bundled" -IGNORE_PATTERNS_BUNDLED = ["bundled/*"] +BUNDLED_FOLDER_NAME = "bundled" +BUNDLED_ROOT = GENERATED_ROOT / BUNDLED_FOLDER_NAME +IGNORE_PATTERNS_BUNDLED = [f"{BUNDLED_FOLDER_NAME}/*"] IGNORE_PATTERNS_SCSS = ["*.scss"] IGNORE_PATTERNS_TYPESCRIPT = ["*.ts"] IGNORE_PATTERNS = [ diff --git a/staticfiles/processors.py b/staticfiles/processors.py index 8d458686..3a0df243 100644 --- a/staticfiles/processors.py +++ b/staticfiles/processors.py @@ -12,28 +12,64 @@ import sass from django.conf import settings from sith.urls import api -from staticfiles.apps import BUNDLED_ROOT, GENERATED_ROOT +from staticfiles.apps import BUNDLED_FOLDER_NAME, BUNDLED_ROOT, GENERATED_ROOT @dataclass class JsBundlerManifestEntry: - out: str src: str + out: str @classmethod def from_json_entry(cls, entry: dict[str, any]) -> list[Self]: + # We have two parts for a manifest entry + # The `src` element which is what the user asks django as a static + # The `out` element which is it's real name in the output static folder + + # For the src part: + # The manifest file contains the path of the file relative to the project root + # We want the relative path of the file inside their respective static folder + # because that's what the user types when importing statics and that's what django gives us + # This is really similar to what we are doing in the bundler, it uses a similar algorithm + # Example: + # core/static/bundled/alpine-index.js -> bundled/alpine-index.js + # core/static/bundled/components/include-index.ts -> core/static/bundled/components/include-index.ts + def get_relative_src_name(name: str) -> str: + original_path = Path(name) + relative_path: list[str] = [] + for directory in reversed(original_path.parts): + relative_path.append(directory) + # Contrary to the bundler algorithm, we do want to keep the bundled prefix + if directory == BUNDLED_FOLDER_NAME: + break + return str(Path(*reversed(relative_path))) + + # For the out part: + # The bundler is configured to output files in generated/bundled and considers this folders as it's root + # Thus, the output name doesn't contain the `bundled` prefix that we need, we add it ourselves ret = [ cls( - out=str(Path("bundled") / entry["file"]), - src=str(Path(*Path(entry["src"]).parts[2:])), + src=get_relative_src_name(entry["src"]), + out=str(Path(BUNDLED_FOLDER_NAME) / entry["file"]), ) ] + + def remove_hash(path: Path) -> str: + # Hashes are configured to be surrounded by `.` + # Filenames are like this path/to/file.hash.ext + unhashed = ".".join(path.stem.split(".")[:-1]) + return str(path.with_stem(unhashed)) + + # CSS files generated by entrypoints don't have their own entry in the manifest + # They are however listed as an attribute of the entry point that generates them + # Their listed name is the one that has been generated inside the generated/bundled folder + # We prefix it with `bundled` and then generate an `src` name by removing the hash for css in entry.get("css", []): - path = Path("bundled") / css + path = Path(BUNDLED_FOLDER_NAME) / css ret.append( cls( + src=remove_hash(path), out=str(path), - src=str(path.with_stem(entry["name"])), ) ) return ret @@ -77,7 +113,7 @@ class JSBundler: def is_in_bundle(name: str | None) -> bool: if name is None: return False - return name.startswith("bundled/") + return Path(name).parts[0] == BUNDLED_FOLDER_NAME class Scss: @@ -120,7 +156,7 @@ class JS: p for p in settings.STATIC_ROOT.rglob("*.js") if ".min" not in p.suffixes - and (settings.STATIC_ROOT / "bundled") not in p.parents + and (settings.STATIC_ROOT / BUNDLED_FOLDER_NAME) not in p.parents ] for path in to_exec: p = path.resolve() diff --git a/staticfiles/storage.py b/staticfiles/storage.py index 0022438c..442de568 100644 --- a/staticfiles/storage.py +++ b/staticfiles/storage.py @@ -16,9 +16,10 @@ class ManifestPostProcessingStorage(ManifestStaticFilesStorage): # This name swap has to be done here # Otherwise, the manifest isn't aware of the file and can't work properly if settings.DEBUG: + # In production, the bundler manifest is used at compile time, we don't need to convert anything try: manifest = JSBundler.get_manifest() - except Exception as e: + except FileNotFoundError as e: raise Exception( "Error loading manifest file, the bundler seems to be busy" ) from e @@ -27,7 +28,6 @@ class ManifestPostProcessingStorage(ManifestStaticFilesStorage): name = converted path = Path(name) - # Call bundler manifest if path.suffix == ".scss": # Compile scss files automatically in debug mode if settings.DEBUG: diff --git a/vite.config.mts b/vite.config.mts index 43ec5ad9..564781f3 100644 --- a/vite.config.mts +++ b/vite.config.mts @@ -2,7 +2,7 @@ import { parse, resolve } from "node:path"; import inject from "@rollup/plugin-inject"; import { glob } from "glob"; -import type { AliasOptions, UserConfig } from "vite"; +import { type AliasOptions, type UserConfig, defineConfig } from "vite"; import type { Rollup } from "vite"; import { viteStaticCopy } from "vite-plugin-static-copy"; import tsconfig from "./tsconfig.json"; @@ -44,65 +44,67 @@ function getRelativeAssetPath(path: string): string { } // biome-ignore lint/style/noDefaultExport: this is recommended by documentation -export default { - base: "/static/bundled/", - appType: "custom", - build: { - outDir: outDir, - manifest: true, // goes into .vite/manifest.json in the build folder - modulePreload: false, // would require `import 'vite/modulepreload-polyfill'` to always be injected - emptyOutDir: true, - rollupOptions: { - input: collectedFiles, - output: { - // Mirror architecture of static folders in generated .js and .css - entryFileNames: (chunkInfo: Rollup.PreRenderedChunk) => { - if (chunkInfo.facadeModuleId !== null) { - return `${getRelativeAssetPath(chunkInfo.facadeModuleId)}.[hash].js`; - } - return "[name].[hash].js"; +export default defineConfig((config: UserConfig) => { + return { + base: "/static/bundled/", + appType: "custom", + build: { + outDir: outDir, + manifest: true, // goes into .vite/manifest.json in the build folder + modulePreload: false, // would require `import 'vite/modulepreload-polyfill'` to always be injected + emptyOutDir: config.mode === "production", // Avoid rebuilding everything in dev mode + rollupOptions: { + input: collectedFiles, + output: { + // Mirror architecture of static folders in generated .js and .css + entryFileNames: (chunkInfo: Rollup.PreRenderedChunk) => { + if (chunkInfo.facadeModuleId !== null) { + return `${getRelativeAssetPath(chunkInfo.facadeModuleId)}.[hash].js`; + } + return "[name].[hash].js"; + }, + assetFileNames: (chunkInfo: Rollup.PreRenderedAsset) => { + if ( + chunkInfo.names?.length === 1 && + chunkInfo.originalFileNames?.length === 1 && + collectedFiles.includes(chunkInfo.originalFileNames[0]) + ) { + return `${getRelativeAssetPath(chunkInfo.originalFileNames[0])}.[hash][extname]`; + } + return "[name].[hash][extname]"; + }, + chunkFileNames: "[name].[hash].js", }, - assetFileNames: (chunkInfo: Rollup.PreRenderedAsset) => { - if ( - chunkInfo.names?.length === 1 && - chunkInfo.originalFileNames?.length === 1 && - collectedFiles.includes(chunkInfo.originalFileNames[0]) - ) { - return `${getRelativeAssetPath(chunkInfo.originalFileNames[0])}.[hash][extname]`; - } - return "[name].[hash][extname]"; - }, - chunkFileNames: "[name].[hash].js", }, }, - }, - resolve: { - alias: getAliases(), - }, + resolve: { + alias: getAliases(), + }, - plugins: [ - inject({ - // biome-ignore lint/style/useNamingConvention: that's how it's called - Alpine: "alpinejs", - }), - viteStaticCopy({ - targets: [ - { - src: resolve(nodeModules, "jquery/dist/jquery.min.js"), - dest: vendored, - }, - { - src: resolve(nodeModules, "jquery-ui/dist/jquery-ui.min.js"), - dest: vendored, - }, - { - src: resolve(nodeModules, "jquery.shorten/src/jquery.shorten.min.js"), - dest: vendored, - }, - ], - }), - ], - optimizeDeps: { - include: ["jquery"], - }, -} satisfies UserConfig; + plugins: [ + inject({ + // biome-ignore lint/style/useNamingConvention: that's how it's called + Alpine: "alpinejs", + }), + viteStaticCopy({ + targets: [ + { + src: resolve(nodeModules, "jquery/dist/jquery.min.js"), + dest: vendored, + }, + { + src: resolve(nodeModules, "jquery-ui/dist/jquery-ui.min.js"), + dest: vendored, + }, + { + src: resolve(nodeModules, "jquery.shorten/src/jquery.shorten.min.js"), + dest: vendored, + }, + ], + }), + ], + optimizeDeps: { + include: ["jquery"], + }, + } satisfies UserConfig; +});