mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 21 Activity

release: nix flake with tag-pinned hashes, reproducible tarballs (milestone 40)

Browse Source 
flake.nix fetches the release tarballs and carries their SRI hashes in a generated block. The cut tool builds the release locally with the toolchain gates.yml pins, in a normalized nine-variable environment, writes the hashes into flake.nix, and commits it with build.zig.zon as the single bump commit. The package job verifies the pins on the bump commit and the publish job verifies them again on the tag, before anything is uploaded.

The tarballs are written by dist_stage (std.tar.Writer, flate gzip) instead of the runner's tar and gzip, and -ffile-prefix-map keeps checkout paths out of the C objects; two checkouts at different absolute paths produce byte-identical archives. nxdns version, /api/version and the admin footer report the version only: the bump commit cannot know its own sha.
This commit is contained in:
mokhtar
2026-09-08 21:45:22 +02:00
parent 3e57f43e08
commit 22abcd9b7b
41 changed files with 1739 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.zig
+91 -46
View File
@@ -43,7 +43,6 @@ pub fn build(b: *std.Build) void {
// the default, so `zig build` and `zig build test` need no flag.
const version_option = b.option([]const u8, "version-string", "Version reported by `nxdns version` (required by `dist`)");
const version_string = version_option orelse "0.1.0-dev";
const git_commit = b.option([]const u8, "git-commit", "Git commit reported by `nxdns version`") orelse "unknown";
const admin_dist = b.option(
[]const u8,
"admin-dist",
@@ -77,7 +76,6 @@ pub fn build(b: *std.Build) void {
options.addOption(bool, "integration", integration);
options.addOption(bool, "live", live);
options.addOption([]const u8, "version_string", version_string);
options.addOption([]const u8, "git_commit", git_commit);
options.addOption([]const u8, "zig_version_string", builtin.zig_version_string);
options.addOption([]const u8, "contract_samples_out", contract_samples_out);
@@ -319,10 +317,22 @@ pub fn build(b: *std.Build) void {
cut_tests_run.setCwd(b.path("."));
test_step.dependOn(&cut_tests_run.step);
// `dist_stage` owns the release archive bytes, and its reproducibility is
// the property the flake pins depend on, so it is tested like any other
// decision this build makes.
const dist_stage_tests = b.addTest(.{
.name = "dist-stage-tool",
.root_module = b.createModule(.{
.root_source_file = b.path("tools/dist_stage.zig"),
.target = b.graph.host,
.optimize = optimize,
}),
});
test_step.dependOn(&b.addRunArtifact(dist_stage_tests).step);
addDist(b, options, admin_assets, .{
.version = version_option,
.version_string = version_string,
.git_commit = git_commit,
.admin_dist = admin_dist,
});
}
@@ -355,14 +365,18 @@ const DistOptions = struct {
/// from one that happens to equal the default.
version: ?[]const u8,
version_string: []const u8,
git_commit: []const u8,
admin_dist: []const u8,
};
/// `dist` builds everything releasable; `verify-dist` asserts the result.
/// Both run on a laptop exactly as they run on the runner, which is the point:
/// release checks that only exist in CI shell are the brittleness milestone 14
/// set out to remove.
/// `dist` builds everything releasable; `verify-dist` asserts the result;
/// `pin-flake` writes the resulting hashes into `flake.nix` and `verify-pins`
/// asserts that they still describe the bytes under `zig-out/dist`.
///
/// All four run on a laptop exactly as they run on the runner, which is the
/// point: release checks that only exist in CI shell are the brittleness
/// milestone 14 set out to remove. The pins depend on it twice over — the cut
/// writes them here and CI recomputes them there, and the two only agree
/// because it is one build graph rather than two scripts.
fn addDist(
b: *std.Build,
options: *std.Build.Step.Options,
@@ -371,11 +385,15 @@ fn addDist(
) void {
const dist_step = b.step("dist", "Build the release tarballs, checksums and staged payloads");
const verify_step = b.step("verify-dist", "Verify the release artifacts under zig-out/dist");
const pin_step = b.step("pin-flake", "Write the release hashes under zig-out/dist into flake.nix");
const verify_pins_step = b.step("verify-pins", "Check flake.nix pins the hashes of the release under zig-out/dist");
if (distPreflight(b, dist_options)) |problem| {
const fail = b.addFail(problem);
dist_step.dependOn(&fail.step);
verify_step.dependOn(&fail.step);
pin_step.dependOn(&fail.step);
verify_pins_step.dependOn(&fail.step);
return;
}
@@ -401,7 +419,6 @@ fn addDist(
verify_run.addArgs(&.{ "--dist-dir", b.getInstallPath(.prefix, "dist") });
verify_run.addArgs(&.{ "--work-dir", b.getInstallPath(.prefix, "dist-verify") });
verify_run.addArgs(&.{ "--version", dist_options.version_string });
verify_run.addArgs(&.{ "--git-commit", dist_options.git_commit });
verify_run.addArg("--zon");
verify_run.addFileArg(b.path("build.zig.zon"));
verify_run.addArgs(&.{ "--max-bytes", b.fmt("{d}", .{max_binary_bytes}) });
@@ -447,36 +464,19 @@ fn addDist(
stage_run.addArg("--licenses");
stage_run.addDirectoryArg(staged_licenses);
// Two commands, never one: `addSystemCommand` executes argv directly
// and does not interpret `|`, and a shell pipeline without `pipefail`
// would report only gzip's status while a failed tar passed silently.
const tar_run = b.addSystemCommand(&.{
"tar",
"--format=gnu",
"--sort=name",
"--mtime=@0",
"--owner=0",
"--group=0",
"--numeric-owner",
"-c",
"-f",
});
setReproducibleEnv(tar_run);
const tar_file = tar_run.addOutputFileArg(b.fmt("{s}.tar", .{name}));
tar_run.addArg("-C");
// The tarball is written by our own tool rather than by the runner's
// `tar` and `gzip`: the release hashes are pinned in `flake.nix` before
// CI rebuilds them, so the bytes may depend on the staged tree and on
// nothing else the host supplies.
const archive_run = b.addRunArtifact(stage_tool);
archive_run.addArg("archive");
archive_run.addArg("--root");
// The staged payload is the sole entry of its cache directory, so its
// parent is what `-C` needs and declaring it declares the payload.
tar_run.addDirectoryArg(staged.dirname());
tar_run.addArg(name);
// `-n` is required because `--mtime=@0` normalises the tar member times
// but not the timestamp gzip writes into its own header. `-c` is
// required because plain `gzip <file>` rewrites its input in place, and
// the input here is a content-addressed cache entry.
const gzip_run = b.addSystemCommand(&.{ "gzip", "-n", "-9", "-c" });
setReproducibleEnv(gzip_run);
gzip_run.addFileArg(tar_file);
const tarball = gzip_run.captureStdOut(.{ .basename = b.fmt("{s}.tar.gz", .{name}) });
// parent is what `--root` needs and declaring it declares the payload.
archive_run.addDirectoryArg(staged.dirname());
archive_run.addArgs(&.{ "--payload", name });
archive_run.addArg("--out");
const tarball = archive_run.addOutputFileArg(b.fmt("{s}.tar.gz", .{name}));
const install_binary = b.addInstallFile(
staged.path(b, "nxdns"),
@@ -513,6 +513,38 @@ fn addDist(
verify_run.step.dependOn(dist_step);
verify_step.dependOn(&verify_run.step);
// The pins in `flake.nix` are written before CI ever builds the release, so
// the run that rebuilds it has to prove they describe its own bytes.
//
// It is a step of its own and NOT part of `verify-dist`. An ordinary commit
// between two cuts builds the `build.zig.zon` version from a tree that
// differs from the released one, so its bytes never match the pins and
// checking them there would fail every such build. The two CI jobs that may
// not skip it call it by name: the package job on the bump commit, and the
// publish job unconditionally before any upload.
const pin_check_run = b.addRunArtifact(stage_tool);
pin_check_run.has_side_effects = true;
pin_check_run.addArg("pin-check");
pin_check_run.addArgs(&.{ "--sums", b.getInstallPath(.prefix, "dist/SHA256SUMS") });
pin_check_run.addArg("--flake");
pin_check_run.addFileArg(b.path("flake.nix"));
pin_check_run.addArgs(&.{ "--version", dist_options.version_string });
pin_check_run.step.dependOn(dist_step);
verify_pins_step.dependOn(&pin_check_run.step);
// The write half, run by the cut and by nothing else. `flake.nix` is named
// as a plain path rather than a `LazyPath`: this run edits the source file
// in place, and a file argument would declare it an input of a step that is
// in fact its author.
const pin_run = b.addRunArtifact(stage_tool);
pin_run.has_side_effects = true;
pin_run.addArg("pin");
pin_run.addArgs(&.{ "--sums", b.getInstallPath(.prefix, "dist/SHA256SUMS") });
pin_run.addArgs(&.{ "--flake", b.pathFromRoot("flake.nix") });
pin_run.addArgs(&.{ "--version", dist_options.version_string });
pin_run.step.dependOn(dist_step);
pin_step.dependOn(&pin_run.step);
}
/// The one message `dist` and `verify-dist` fail with when the release inputs
@@ -564,13 +596,6 @@ fn hostTool(b: *std.Build, name: []const u8) *std.Build.Step.Compile {
});
}
/// Locale and time zone leak into archive metadata and into tool output.
/// Pinning both is the cheap half of reproducibility (milestone-14 ruling 12).
fn setReproducibleEnv(run: *std.Build.Step.Run) void {
run.setEnvironmentVariable("LC_ALL", "C");
run.setEnvironmentVariable("TZ", "UTC");
}
/// Milestone-15 ruling 4: every `*.zig` under `src/` must appear in
/// `src/tests.zig` as a line that trims to exactly `_ = @import("<path>");`,
/// where `<path>` is relative to `src/`. Whole-line equality, not a substring
@@ -744,7 +769,10 @@ fn addExecutable(
exe.root_module.addAnonymousImport("admin_assets", .{ .root_source_file = admin_assets });
exe.root_module.linkLibrary(sqliteLibrary(b, target, optimize));
exe.root_module.linkLibrary(mbedtlsLibrary(b, target, optimize));
exe.root_module.addCSourceFile(.{ .file = b.path("src/platform/mbedtls_shim.c") });
exe.root_module.addCSourceFile(.{
.file = b.path("src/platform/mbedtls_shim.c"),
.flags = &.{filePrefixMap(b, .build_root)},
});
addMbedtlsThreadingMacros(exe.root_module);
return exe;
}
@@ -810,6 +838,8 @@ fn sqliteLibrary(
"-DSQLITE_THREADSAFE=1",
"-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1",
"-DSQLITE_OMIT_LOAD_EXTENSION",
filePrefixMap(b, .build_root),
filePrefixMap(b, .global_cache),
},
});
return lib;
@@ -846,19 +876,34 @@ fn mbedtlsLibrary(
lib.root_module.addIncludePath(dep.path(include_dir));
}
const c_flags = [_][]const u8{ filePrefixMap(b, .build_root), filePrefixMap(b, .global_cache) };
lib.root_module.addCSourceFiles(.{
.root = dep.path("library"),
.files = &mbedtls_library_sources,
.flags = &c_flags,
});
lib.root_module.addCSourceFiles(.{
.root = dep.path("3rdparty"),
.files = &mbedtls_3rdparty_sources,
.flags = &c_flags,
});
lib.installHeadersDirectory(dep.path("include/mbedtls"), "mbedtls", .{});
lib.installHeadersDirectory(dep.path("include/psa"), "psa", .{});
return lib;
}
/// `__FILE__` in the C sources (mbedTLS debug and assertion macros) would
/// otherwise embed the absolute checkout path into the release binary, and the
/// flake pins require the bytes to be the same on every machine that builds
/// the tag. Both roots a dependency can be stored under are mapped.
fn filePrefixMap(b: *std.Build, root: enum { build_root, global_cache }) []const u8 {
const path = switch (root) {
.build_root => b.build_root.path orelse ".",
.global_cache => b.graph.global_cache_root.path orelse ".",
};
return b.fmt("-ffile-prefix-map={s}=.", .{path});
}
/// Context sizes change with threading enabled, so every compilation unit that
/// includes mbedTLS headers (the library itself and `mbedtls_shim.c`) must see
/// the same macros. Concurrent handshakes share `ssl_config`, the CTR-DRBG, and