//! Build-time web asset indexer (milestone-8 ruling 24). //! //! `gen_web_assets ` reads a built web dist directory and //! writes into ``: //! //! - `.gz` next to each asset worth compressing, unless the dist already //! ships one (a Vite plugin may pre-compress), in which case the shipped //! sibling is decompressed and byte-compared against its base file, and a //! `.gz` with no base file at all is rejected. Compression happens here, at //! build time, because `flate.Compress` needs a 64 KiB window per stream — //! a cost the server must not pay per request for immutable content. //! - `assets.zig`, the module index the server embeds: one entry per servable //! path with its bytes, content type and a strong ETag. The build system //! merges `` with a copy of the dist into one WriteFiles directory, //! so every `@embedFile` path below resolves inside the module root. //! //! The entry list is sorted so the output is byte-identical across runs; the //! build cache keys on it. const std = @import("std"); const Allocator = std.mem.Allocator; /// A single asset never legitimately exceeds this; a bigger file is a build /// mistake, not content to embed. const max_asset_bytes = 64 * 1024 * 1024; /// Below this a gzip member's own header and footer eat the savings. const min_compress_bytes = 128; const Asset = struct { /// Request path, `/`-prefixed. path: []const u8, /// Path relative to the module root, for `@embedFile`. file: []const u8, content_type: []const u8, etag: [32]u8, }; pub fn main(init: std.process.Init) !void { const arena = init.arena.allocator(); const io = init.io; const args = try init.minimal.args.toSlice(arena); if (args.len != 3) std.process.fatal("usage: gen_web_assets ", .{}); var dist = std.Io.Dir.cwd().openDir(io, args[1], .{ .iterate = true }) catch |err| { std.process.fatal("cannot open dist directory '{s}': {t}", .{ args[1], err }); }; defer dist.close(io); var out = std.Io.Dir.cwd().openDir(io, args[2], .{}) catch |err| { std.process.fatal("cannot open output directory '{s}': {t}", .{ args[2], err }); }; defer out.close(io); const names = try collectSorted(arena, io, dist); var assets: std.ArrayList(Asset) = .empty; for (names) |name| { if (std.mem.endsWith(u8, name, ".gz")) { if (!contains(names, name[0 .. name.len - 3])) { // The server only ever reaches a `.gz` through its base file's // entry, so an orphan is unservable bytes in the binary. std.process.fatal("'{s}' has no base file; nothing can serve it", .{name}); } // A pre-compressed sibling; indexed alongside its base file below. continue; } const bytes = dist.readFileAlloc(io, name, arena, .limited(max_asset_bytes)) catch |err| { std.process.fatal("cannot read '{s}': {t}", .{ name, err }); }; try assets.append(arena, .{ .path = try std.fmt.allocPrint(arena, "/{s}", .{name}), .file = name, .content_type = contentType(name), .etag = etagOf(bytes), }); const sibling = try std.fmt.allocPrint(arena, "{s}.gz", .{name}); const gz = if (contains(names, sibling)) shipped: { const shipped = dist.readFileAlloc(io, sibling, arena, .limited(max_asset_bytes)) catch |err| { std.process.fatal("cannot read '{s}': {t}", .{ sibling, err }); }; try verifyShipped(arena, sibling, shipped, bytes); break :shipped shipped; } else try compressWorthwhile(arena, io, out, sibling, bytes) orelse continue; try assets.append(arena, .{ .path = try std.fmt.allocPrint(arena, "/{s}", .{sibling}), .file = sibling, .content_type = contentType(name), .etag = etagOf(gz), }); } const index = try renderIndex(arena, assets.items); try out.writeFile(io, .{ .sub_path = "assets.zig", .data = index }); } fn collectSorted(arena: Allocator, io: std.Io, dist: std.Io.Dir) ![]const []const u8 { var names: std.ArrayList([]const u8) = .empty; var walker = try dist.walk(arena); defer walker.deinit(); while (try walker.next(io)) |entry| { if (entry.kind != .file) continue; for (entry.path) |c| { if (!std.ascii.isAlphanumeric(c) and std.mem.findScalar(u8, "._-/", c) == null) { std.process.fatal("asset name '{s}' has a character the index cannot carry", .{entry.path}); } } try names.append(arena, try arena.dupe(u8, entry.path)); } std.mem.sort([]const u8, names.items, {}, lessThan); return names.items; } fn lessThan(_: void, a: []const u8, b: []const u8) bool { return std.mem.order(u8, a, b) == .lt; } fn contains(sorted: []const []const u8, name: []const u8) bool { for (sorted) |candidate| { if (std.mem.eql(u8, candidate, name)) return true; } return false; } /// Fatal unless `gz` decompresses to exactly `base`. /// /// The server answers a compressed request with the sibling's bytes under the /// *base* file's identity, so a sibling the dist shipped from a stale or /// truncated build would hand every client the wrong content behind a correct /// ETag. Nothing downstream can catch that: gzip carries a CRC of whatever it /// was compressed from, not of what it was supposed to be. fn verifyShipped(arena: Allocator, sub_path: []const u8, gz: []const u8, base: []const u8) !void { var input: std.Io.Reader = .fixed(gz); const window = try arena.alloc(u8, std.compress.flate.max_window_len); var decompress: std.compress.flate.Decompress = .init(&input, .gzip, window); const plain = decompress.reader.allocRemaining(arena, .limited(max_asset_bytes)) catch |err| { // `Reader` collapses every decode failure into `ReadFailed` and stashes // the real one; the stashed name is what identifies the bad file. std.process.fatal("'{s}' is not readable gzip: {t}", .{ sub_path, decompress.err orelse err }); }; if (!std.mem.eql(u8, plain, base)) { std.process.fatal( "'{s}' does not decompress to its base file ({d} bytes out, {d} expected)", .{ sub_path, plain.len, base.len }, ); } } /// Gzips `bytes`; writes and returns the result only when it is smaller than /// the original, else null. Equal-or-larger output means the asset is already /// compressed (an image, a font) and the sibling would waste binary size. fn compressWorthwhile( arena: Allocator, io: std.Io, out: std.Io.Dir, sub_path: []const u8, bytes: []const u8, ) !?[]const u8 { if (bytes.len < min_compress_bytes) return null; var sink: std.Io.Writer.Allocating = try .initCapacity(arena, @max(64, bytes.len / 2)); const window = try arena.alloc(u8, std.compress.flate.max_window_len); var compress = try std.compress.flate.Compress.init(&sink.writer, window, .gzip, .best); try compress.writer.writeAll(bytes); try compress.finish(); const gz = sink.written(); if (gz.len >= bytes.len) return null; if (std.Io.Dir.path.dirname(sub_path)) |parent| try out.createDirPath(io, parent); try out.writeFile(io, .{ .sub_path = sub_path, .data = gz }); return gz; } fn etagOf(bytes: []const u8) [32]u8 { var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; std.crypto.hash.sha2.Sha256.hash(bytes, &digest, .{}); return std.fmt.bytesToHex(digest[0..16].*, .lower); } fn contentType(name: []const u8) []const u8 { const map = [_]struct { ext: []const u8, mime: []const u8 }{ .{ .ext = ".html", .mime = "text/html; charset=utf-8" }, .{ .ext = ".js", .mime = "text/javascript" }, .{ .ext = ".mjs", .mime = "text/javascript" }, .{ .ext = ".css", .mime = "text/css" }, .{ .ext = ".svg", .mime = "image/svg+xml" }, .{ .ext = ".png", .mime = "image/png" }, .{ .ext = ".ico", .mime = "image/x-icon" }, .{ .ext = ".json", .mime = "application/json" }, .{ .ext = ".map", .mime = "application/json" }, .{ .ext = ".webmanifest", .mime = "application/manifest+json" }, .{ .ext = ".txt", .mime = "text/plain; charset=utf-8" }, .{ .ext = ".woff2", .mime = "font/woff2" }, .{ .ext = ".woff", .mime = "font/woff" }, .{ .ext = ".wasm", .mime = "application/wasm" }, }; for (map) |entry| { if (std.mem.endsWith(u8, name, entry.ext)) return entry.mime; } return "application/octet-stream"; } fn renderIndex(arena: Allocator, assets: []const Asset) ![]const u8 { var sink: std.Io.Writer.Allocating = try .initCapacity(arena, 4096); const w = &sink.writer; try w.writeAll( \\//! Generated by tools/gen_web_assets.zig. Do not edit. \\ \\pub const File = struct { \\ /// Request path, `/`-prefixed. \\ path: []const u8, \\ bytes: []const u8, \\ content_type: []const u8, \\ /// Strong validator, quotes included, hashed from `bytes` at build time. \\ etag: []const u8, \\}; \\ \\pub const files: []const File = &.{ \\ ); for (assets) |asset| { try w.print( " .{{ .path = \"{s}\", .bytes = @embedFile(\"{s}\"), " ++ ".content_type = \"{s}\", .etag = \"\\\"{s}\\\"\" }},\n", .{ asset.path, asset.file, asset.content_type, asset.etag }, ); } try w.writeAll("};\n"); return sink.written(); }