cut: build the release with the official zig tarball, skip hidden bundle files; re-pin 0.0.17
Gates / frontend (push) Successful in 2m23s
Gates / test (push) Successful in 3m15s
Gates / test-aarch64 (push) Successful in 8m38s
Gates / package (push) Successful in 5m0s
Gates / container (push) Successful in 19s
CI / gates (push) Successful in 17m17s
Release / guard (push) Successful in 37s
Gates / frontend (push) Successful in 2m15s
Gates / test (push) Successful in 2m34s
Gates / test-aarch64 (push) Successful in 7m33s
Gates / package (push) Successful in 51s
Gates / container (push) Successful in 10s
Release / gates (push) Successful in 11m14s
Release / publish (push) Successful in 8m35s

The first 0.0.17 cut (run 687) failed verify-pins in CI for two reasons. The asset generator embedded admin/dist/.src-hash, a freshness stamp that CI's artifact copy does not carry; it now skips dotfiles. And the Arch zig package emits different code than the ziglang.org tarball that CI installs, so the cut downloads the pinned tarball (ZIG_TARBALL_SHA256 in gates.yml, the full digest keys the cache) and builds the release with it. flake.nix is re-pinned to the bytes both now produce.

The saturated-primary pool test gates its holders on a semaphore instead of sleeps and releases every spawned holder on the way out, so a loaded runner cannot flake it. The package job uploads the payload before the pin check and runs the check when the version or flake.nix changed against the parent. The verify-a-release recipe clones the tag first and builds with the official zig.
This commit is contained in:
2026-09-08 23:57:07 +02:00
parent d596fd788a
commit f168247b33
10 changed files with 348 additions and 92 deletions
+207 -26
View File
@@ -135,7 +135,7 @@ const poll_interval_ns: u64 = 15 * std.time.ns_per_s;
const progress_every_polls: usize = 8;
/// The bound on a captured git command that talks to the network. The
/// interactive commands are not captured and not bounded — see `gitInherit`.
/// interactive commands are not captured and not bounded — see `runInherited`.
const git_network_timeout_s: u64 = 120;
const git_local_timeout_s: u64 = 30;
@@ -1424,16 +1424,18 @@ fn attemptBudgetNs(started_ns: i96, now_ns: i96, budget_ns: u64, ceiling_ns: u64
/// than carrying its own copy: two pins that can drift are not a pin.
const gates_workflow_path = ".gitea/workflows/gates.yml";
/// The three toolchain versions `gates.yml` declares.
/// The toolchain `gates.yml` declares: three versions and the hash of the zig
/// tarball those versions are meaningless without.
const ToolchainPins = struct {
zig: []const u8,
zig_tarball_sha256: []const u8,
node: []const u8,
npm: []const u8,
};
const ToolchainKey = enum { ZIG_VERSION, NODE_VERSION, NPM_VERSION };
const ToolchainKey = enum { ZIG_VERSION, ZIG_TARBALL_SHA256, NODE_VERSION, NPM_VERSION };
/// The three pins out of the top-level `env:` block of `gates.yml`.
/// The four pins out of the top-level `env:` block of `gates.yml`.
///
/// A line parser and not a YAML library: the one shape this has to read is
/// ` KEY: "value"` under a column-zero `env:`, and a dependency that can parse
@@ -1442,10 +1444,10 @@ const ToolchainKey = enum { ZIG_VERSION, NODE_VERSION, NPM_VERSION };
/// non-comment line that is not indented, which is how the file's own `jobs:`
/// key terminates it.
///
/// Null when any of the three is missing, because a parity check that silently
/// Null when any of the four is missing, because a parity check that silently
/// dropped one of them would report parity it never established.
fn parseToolchainPins(source: []const u8) ?ToolchainPins {
var found: [3]?[]const u8 = .{ null, null, null };
var found: [@typeInfo(ToolchainKey).@"enum".fields.len]?[]const u8 = @splat(null);
var lines = std.mem.splitScalar(u8, source, '\n');
var inside = false;
@@ -1468,13 +1470,28 @@ fn parseToolchainPins(source: []const u8) ?ToolchainPins {
found[@intFromEnum(which)] = value;
}
const digest = found[@intFromEnum(ToolchainKey.ZIG_TARBALL_SHA256)] orelse return null;
if (!isSha256Hex(digest)) return null;
return .{
.zig = found[@intFromEnum(ToolchainKey.ZIG_VERSION)] orelse return null,
.zig_tarball_sha256 = digest,
.node = found[@intFromEnum(ToolchainKey.NODE_VERSION)] orelse return null,
.npm = found[@intFromEnum(ToolchainKey.NPM_VERSION)] orelse return null,
};
}
/// A digest that is anything but 64 lowercase hex digits cannot name a cache
/// directory or be compared to a computed hash, so the parser refuses it.
fn isSha256Hex(digest: []const u8) bool {
if (digest.len != 64) return false;
for (digest) |c| switch (c) {
'0'...'9', 'a'...'f' => {},
else => return false,
};
return true;
}
/// Every variable a release build is allowed to see, and nothing else.
///
/// The bundle and the tarballs are hashed into `flake.nix` before CI rebuilds
@@ -1524,7 +1541,7 @@ fn scratchRoot(ctx: *Ctx, version: []const u8) []const u8 {
///
/// stdio is inherited: these commands take minutes and an operator watching a
/// cut needs to see npm and zig make progress. Their success is read from the
/// termination state, exactly as `gitInherit` reads it.
/// termination state, exactly as `runInherited` reads it.
///
/// The umask arrives through `sh` rather than through this process: zig 0.16.0's
/// standard library exposes `umask(2)` only as a libc extern (`std.c.umask`),
@@ -1581,7 +1598,8 @@ fn runPinned(
/// recompute. A mismatch is a refusal and not a warning: the alternative is a
/// tag whose flake pins bytes no CI run can reproduce, which is only discovered
/// after the tag is public.
fn toolchainParity(ctx: *Ctx) !void {
/// Returns the absolute path of the official zig this cut must build with.
fn toolchainParity(ctx: *Ctx) ![]const u8 {
// The admin bundle is built by Rolldown and Lightning CSS, whose native
// bindings are chosen per host; CI builds it on an x86_64 Ubuntu runner.
// A bundle built on another architecture is a different set of bytes, so
@@ -1598,7 +1616,7 @@ fn toolchainParity(ctx: *Ctx) !void {
return CheckFailed;
};
const pins = parseToolchainPins(source) orelse {
ctx.soft("toolchain", "{s} does not declare ZIG_VERSION, NODE_VERSION and NPM_VERSION in its top-level `env:` block", .{gates_workflow_path});
ctx.soft("toolchain", "{s} does not declare ZIG_VERSION, ZIG_TARBALL_SHA256, NODE_VERSION and NPM_VERSION in its top-level `env:` block", .{gates_workflow_path});
return CheckFailed;
};
@@ -1607,7 +1625,14 @@ fn toolchainParity(ctx: *Ctx) !void {
// print the bare number already.
try assertVersion(ctx, "node", &.{ "node", "--version" }, "v", pins.node);
try assertVersion(ctx, "npm", &.{ "npm", "--version" }, "", pins.npm);
try assertVersion(ctx, "zig", &.{ "zig", "version" }, "", pins.zig);
// The zig on PATH is not consulted here, and `zig version` would not settle
// it anyway: the Arch Linux 0.16.0 package is built against the system LLVM
// and emits different machine code from the ziglang.org tarball CI installs,
// while both print `0.16.0`. The first real cut pinned hashes CI could not
// reproduce for exactly that reason. PATH's zig still compiles this program
// — that is `zig build cut`, whose bytes nobody hashes.
const zig_exe = try officialZig(ctx, pins);
// The npm config paths in the release environment silence `~/.npmrc` and
// the node install's `etc/npmrc` only while nothing exists at them.
@@ -1623,7 +1648,110 @@ fn toolchainParity(ctx: *Ctx) !void {
ctx.soft("toolchain", "{s} exists, and npm would read it as {s}: the release environment relies on that path being absent", .{ entry.value, entry.key });
return CheckFailed;
}
ctx.pass("toolchain", "x86_64 Linux glibc with zig {s}, node {s} and npm {s}, as {s} pins them", .{ pins.zig, pins.node, pins.npm, gates_workflow_path });
ctx.pass("toolchain", "x86_64 Linux glibc with node {s}, npm {s}, and the official zig {s} at {s}, as {s} pins them", .{ pins.node, pins.npm, pins.zig, zig_exe, gates_workflow_path });
return zig_exe;
}
/// The official toolchain: the host is asserted x86_64 Linux above, so the
/// triple is a constant and not a lookup.
const zig_tarball_triple = "x86_64-linux";
fn zigTarballUrl(arena: Allocator, version: []const u8) []const u8 {
return std.fmt.allocPrint(
arena,
"https://ziglang.org/download/{s}/zig-" ++ zig_tarball_triple ++ "-{s}.tar.xz",
.{ version, version },
) catch @panic("OOM");
}
/// The cache is keyed by version and lives outside the repository: it is a
/// toolchain, not a build output, and a download per cut would make the network
/// an input of every release.
/// Keyed by the pinned digest as well as the version: a changed pin for the
/// same version must not find yesterday's extraction and skip the download.
fn zigCacheRoot(arena: Allocator, home: []const u8, version: []const u8, digest: []const u8) []const u8 {
return std.fmt.allocPrint(arena, "{s}/.cache/nxdns-cut/zig-{s}-{s}", .{ home, version, digest }) catch @panic("OOM");
}
/// The tarball unpacks to one directory named after itself, which is why the
/// cache root is not that directory.
fn zigBinaryPath(arena: Allocator, home: []const u8, version: []const u8, digest: []const u8) []const u8 {
return std.fmt.allocPrint(
arena,
"{s}/zig-" ++ zig_tarball_triple ++ "-{s}/zig",
.{ zigCacheRoot(arena, home, version, digest), version },
) catch @panic("OOM");
}
/// The pinned zig from ziglang.org, downloaded once per version and verified
/// every run.
///
/// There is no fall back to PATH: a cut that silently built with whatever zig
/// the operator's distribution ships is the failure this exists to prevent.
fn officialZig(ctx: *Ctx, pins: ToolchainPins) ![]const u8 {
const home = ctx.get("HOME");
if (home.len == 0) {
ctx.soft("toolchain", "HOME is not set, and the official zig {s} is cached under $HOME/.cache/nxdns-cut", .{pins.zig});
return CheckFailed;
}
const root = zigCacheRoot(ctx.arena, home, pins.zig, pins.zig_tarball_sha256);
const exe = zigBinaryPath(ctx.arena, home, pins.zig, pins.zig_tarball_sha256);
const cached = if (Io.Dir.accessAbsolute(ctx.io, exe, .{})) |_| true else |err| switch (err) {
error.FileNotFound => false,
else => {
ctx.soft("toolchain", "cannot probe {s}: {t}", .{ exe, err });
return CheckFailed;
},
};
if (!cached) {
// A partially extracted cache from an interrupted run is not a
// toolchain, so the directory is rebuilt rather than added to.
Io.Dir.cwd().deleteTree(ctx.io, root) catch |err| {
ctx.soft("toolchain", "cannot clear {s}: {t}", .{ root, err });
return CheckFailed;
};
Io.Dir.cwd().createDirPath(ctx.io, root) catch |err| {
ctx.soft("toolchain", "cannot create {s}: {t}", .{ root, err });
return CheckFailed;
};
const tarball = ctx.fmt("{s}/zig.tar.xz", .{root});
const url = zigTarballUrl(ctx.arena, pins.zig);
ctx.note("toolchain: downloading {s}", .{url});
try runInherited(ctx, "toolchain", &.{ "curl", "-fsSL", "-o", tarball, url });
try assertTarballDigest(ctx, tarball, pins.zig_tarball_sha256);
try runInherited(ctx, "toolchain", &.{ "tar", "-xJf", tarball, "-C", root });
Io.Dir.cwd().deleteFile(ctx.io, tarball) catch |err| {
ctx.soft("toolchain", "cannot remove {s}: {t}", .{ tarball, err });
return CheckFailed;
};
}
try assertVersion(ctx, "zig", &.{ exe, "version" }, "", pins.zig);
return exe;
}
/// CI installs the same tarball through setup-zig, which checks it by minisign
/// against ziglang.org's public key; this digest is the cut's half of that same
/// assertion, so both machines build with bytes they each verified.
fn assertTarballDigest(ctx: *Ctx, path: []const u8, expected: []const u8) !void {
const bytes = Io.Dir.cwd().readFileAlloc(ctx.io, path, ctx.gpa, .limited(max_input_bytes)) catch |err| {
ctx.soft("toolchain", "cannot read {s}: {t}", .{ path, err });
return CheckFailed;
};
defer ctx.gpa.free(bytes);
var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined;
std.crypto.hash.sha2.Sha256.hash(bytes, &digest, .{});
const hex = std.fmt.bytesToHex(digest, .lower);
if (!std.mem.eql(u8, &hex, expected)) {
ctx.soft("toolchain", "{s} hashes to {s}, and {s} pins ZIG_TARBALL_SHA256 {s}", .{
path, hex, gates_workflow_path, expected,
});
return CheckFailed;
}
}
fn assertVersion(
@@ -1661,7 +1789,10 @@ fn assertVersion(
/// `-Dversion-string` disagrees with the manifest, so the manifest has to be
/// bumped first; and `flake.nix` is part of the bump commit's diff, so the pins
/// have to exist before the commit is made.
fn pinStage(ctx: *Ctx, version: []const u8, environ: *const std.process.Environ.Map) !void {
///
/// `zig_exe` is the official ziglang.org toolchain the parity check resolved.
/// Every build here goes through it by absolute path, never through PATH.
fn pinStage(ctx: *Ctx, version: []const u8, environ: *const std.process.Environ.Map, zig_exe: []const u8) !void {
// A fresh local cache each run: the one input of these bytes that lives
// outside the repository and the toolchain. The global cache stays shared:
// it is content-addressed, and a fresh one would refetch every dependency
@@ -1690,12 +1821,12 @@ fn pinStage(ctx: *Ctx, version: []const u8, environ: *const std.process.Environ.
const dist_flags = [_][]const u8{ version_flag, "-Dadmin-dist=admin/dist", "-Doptimize=ReleaseSafe" };
const cache_flags = [_][]const u8{ "--cache-dir", cache_dir };
try runPinned(ctx, "dist", environ, repo_root, try zigBuild(ctx, "dist", &dist_flags, &cache_flags));
try runPinned(ctx, "dist", environ, repo_root, try zigBuild(ctx, zig_exe, "dist", &dist_flags, &cache_flags));
ctx.pass("dist", "the {s} tarballs and SHA256SUMS are under zig-out/dist", .{version});
try runPinned(ctx, "pin", environ, repo_root, try zigBuild(ctx, "pin-flake", &dist_flags, &cache_flags));
try runPinned(ctx, "verify-dist", environ, repo_root, try zigBuild(ctx, "verify-dist", &dist_flags, &cache_flags));
try runPinned(ctx, "verify-pins", environ, repo_root, try zigBuild(ctx, "verify-pins", &dist_flags, &cache_flags));
try runPinned(ctx, "pin", environ, repo_root, try zigBuild(ctx, zig_exe, "pin-flake", &dist_flags, &cache_flags));
try runPinned(ctx, "verify-dist", environ, repo_root, try zigBuild(ctx, zig_exe, "verify-dist", &dist_flags, &cache_flags));
try runPinned(ctx, "verify-pins", environ, repo_root, try zigBuild(ctx, zig_exe, "verify-pins", &dist_flags, &cache_flags));
ctx.pass("verify-pins", "flake.nix pins the hashes of the {s} tarballs this machine just built", .{version});
// Evaluation only: `--no-build` keeps this from fetching the tarballs the
@@ -1704,14 +1835,16 @@ fn pinStage(ctx: *Ctx, version: []const u8, environ: *const std.process.Environ.
ctx.pass("flake-check", "`nix flake check --no-build` accepts the rewritten flake.nix", .{});
}
/// Every release byte comes out of the official zig, addressed by absolute path.
fn zigBuild(
ctx: *Ctx,
zig_exe: []const u8,
step: []const u8,
dist_flags: []const []const u8,
cache_flags: []const []const u8,
) ![]const []const u8 {
var argv: std.ArrayList([]const u8) = .empty;
try argv.appendSlice(ctx.arena, &.{ "zig", "build", step });
try argv.appendSlice(ctx.arena, &.{ zig_exe, "build", step });
try argv.appendSlice(ctx.arena, dist_flags);
try argv.appendSlice(ctx.arena, cache_flags);
return argv.items;
@@ -1778,13 +1911,15 @@ fn capture(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8, time
};
}
/// A git command that must be able to talk to the operator: `commit -S`, `tag
/// -s` and both pushes reach gpg and ssh, either of which may need a passphrase
/// A command that must be able to talk to the operator: `commit -S`, `tag -s`
/// and both pushes reach gpg and ssh, either of which may need a passphrase
/// from a terminal. Piping their stdio would turn a pinentry prompt into a hang.
/// The toolchain download uses it too, so curl and tar report their own
/// failures where the operator can read them.
///
/// The termination state is read rather than assumed: a gpg-agent that dies
/// takes git with it by signal, and `.signal` is not `.exited 0`.
fn gitInherit(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8) !void {
fn runInherited(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8) !void {
var child = std.process.spawn(ctx.io, .{
.argv = argv,
.stdin = .inherit,
@@ -1820,7 +1955,7 @@ fn gitInherit(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8) !
/// stdout is a pipe because the porcelain report is the only account of whether
/// the ref actually moved. stdin and stderr stay on the terminal: ssh writes its
/// prompts and progress there, and piping them would turn a key passphrase into
/// a hang — the same reason `gitInherit` exists.
/// a hang — the same reason `runInherited` exists.
fn gitPushPorcelain(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8) ![]const u8 {
var child = std.process.spawn(ctx.io, .{
.argv = argv,
@@ -2116,7 +2251,7 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void {
// Both of these run before the manifest is rewritten: they can refuse, and
// a refusal after the rewrite leaves a dirty build.zig.zon that the next
// run's clean-tree gate rejects.
try toolchainParity(ctx);
const zig_exe = try toolchainParity(ctx);
var environ = normalizedEnvironment(ctx.gpa, ctx.env) catch |err| {
ctx.soft("pin", "cannot build the release environment: {t}", .{err});
return CheckFailed;
@@ -2124,7 +2259,7 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void {
defer environ.deinit();
if (bump_needed) try writeZonVersion(ctx, version, zon_source);
try pinStage(ctx, version, &environ);
try pinStage(ctx, version, &environ, zig_exe);
try commitBump(ctx, version, bump_needed);
const sha = try headSha(ctx);
@@ -2182,7 +2317,7 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void {
}
ctx.pass("tag", "the verified tag {s} ({s}) is unchanged", .{ tag, object });
} else {
try gitInherit(ctx, "tag", &.{ "git", "tag", "-s", tag, "-m", tag, sha });
try runInherited(ctx, "tag", &.{ "git", "tag", "-s", tag, "-m", tag, sha });
try verifyTag(ctx, tag, sha);
}
@@ -2927,7 +3062,7 @@ fn commitBump(ctx: *Ctx, version: []const u8, bump_needed: bool) !void {
return CheckFailed;
}
try gitInherit(ctx, "bump", &.{
try runInherited(ctx, "bump", &.{
"git", "commit",
"-S", "-m",
ctx.fmt("build: bump version to {s}", .{version}), "--",
@@ -4656,6 +4791,7 @@ test "the toolchain pins come out of the top-level env block of gates.yml" {
\\
\\env:
\\ ZIG_VERSION: "0.16.0"
\\ ZIG_TARBALL_SHA256: "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00"
\\ # A comment between two entries, which the file has.
\\ NODE_VERSION: "24.19.0"
\\ NPM_VERSION: "11.17.0"
@@ -4668,12 +4804,13 @@ test "the toolchain pins come out of the top-level env block of gates.yml" {
;
const pins = parseToolchainPins(source).?;
try testing.expectEqualStrings("0.16.0", pins.zig);
try testing.expectEqualStrings("70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", pins.zig_tarball_sha256);
try testing.expectEqualStrings("24.19.0", pins.node);
try testing.expectEqualStrings("11.17.0", pins.npm);
}
test "a gates.yml missing any one pin yields no pins at all" {
// Parity established for two of three tools is not parity, so there is no
// Parity established for some of the pins is not parity, so there is no
// partial answer to return.
try testing.expect(parseToolchainPins(
\\env:
@@ -4684,6 +4821,25 @@ test "a gates.yml missing any one pin yields no pins at all" {
try testing.expect(parseToolchainPins("jobs:\n package:\n") == null);
}
test "a zig tarball digest that is not 64 lowercase hex digits yields no pins" {
// The digest names the cache directory and is compared to a computed hash;
// a truncated or uppercase value would do neither correctly.
const shapes = [_][]const u8{
"70e49664a743",
"70E49664A74374B48B51E6F3FDFBF437F6395D42509050588BD49ABE52BA3D00",
"70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d0g",
};
for (shapes) |digest| {
const source = std.fmt.allocPrint(
testing.allocator,
"env:\n ZIG_VERSION: \"0.16.0\"\n ZIG_TARBALL_SHA256: \"{s}\"\n NODE_VERSION: \"24.19.0\"\n NPM_VERSION: \"11.17.0\"\n",
.{digest},
) catch unreachable;
defer testing.allocator.free(source);
try testing.expect(parseToolchainPins(source) == null);
}
}
test "the release environment is exactly nine variables" {
var parent: std.process.Environ.Map = .init(testing.allocator);
defer parent.deinit();
@@ -4746,6 +4902,7 @@ test "this repository's gates.yml pins the toolchain the cut asserts" {
try testing.expect(pins.zig.len != 0);
try testing.expect(pins.node.len != 0);
try testing.expect(pins.npm.len != 0);
try testing.expectEqual(@as(usize, 64), pins.zig_tarball_sha256.len);
}
test "release.yml pins the same toolchain as gates.yml" {
@@ -4764,10 +4921,34 @@ test "release.yml pins the same toolchain as gates.yml" {
const expected = parseToolchainPins(gates) orelse return error.NoToolchainPins;
const found = parseToolchainPins(release) orelse return error.NoToolchainPins;
try testing.expectEqualStrings(expected.zig, found.zig);
try testing.expectEqualStrings(expected.zig_tarball_sha256, found.zig_tarball_sha256);
try testing.expectEqualStrings(expected.node, found.node);
try testing.expectEqualStrings(expected.npm, found.npm);
}
test "the official zig is addressed by version, under HOME and never on PATH" {
// The download and the cache path are the whole of what makes the release
// bytes reproducible off this machine, so their text is asserted rather
// than trusted to a format string. Nothing here touches the network.
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
try testing.expectEqualStrings(
"https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz",
zigTarballUrl(arena, "0.16.0"),
);
const digest = "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00";
try testing.expectEqualStrings(
"/home/someone/.cache/nxdns-cut/zig-0.16.0-70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00",
zigCacheRoot(arena, "/home/someone", "0.16.0", digest),
);
try testing.expectEqualStrings(
"/home/someone/.cache/nxdns-cut/zig-0.16.0-70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00/zig-x86_64-linux-0.16.0/zig",
zigBinaryPath(arena, "/home/someone", "0.16.0", digest),
);
}
test "the sh wrapper sets the umask, keeps the real command and reports its exit code" {
// The umask is the one release input this program cannot set on itself:
// zig 0.16.0 exposes `umask(2)` only through libc, which these tools do not