From 3e57f43e085d918af0dd683439e05c255b9191dd Mon Sep 17 00:00:00 2001 From: m5r Date: Tue, 8 Sep 2026 01:47:09 +0200 Subject: [PATCH] cut: rerun a flaky release run once, resume after the tag The v0.0.16 cut pushed the tag and then reported a failure because the release.yml run lost one gate to a timing-flaky test while its guard had passed and its publish job never ran. Rerunning the run by hand published the release; the tool had exited 1 and could not be run again, since a tag on origin meant "derive the next version". The release stage now classifies a failed run from forge state: retryable when the conclusion is failure, the run is on its first attempt, every commit-status context in a failure state is a gate, and no release is published for the tag. A retryable run is rerun once through the actions API and the same run id is watched again, restarted only once the forge reports a higher run_attempt, so a concluded previous attempt is never read as the result. The rerun decision parses statuses strictly and treats anything it cannot read as terminal. A tag already on origin that peels to HEAD with no published release resumes at the release stage after the origin tag is verified (annotated, tool convention, pinned signing fingerprint). Both decisions are pure, table-tested functions. Spec bullets in specs/release-cut.md. --- specs/release-cut.md | 6 +- tools/cut.zig | 967 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 882 insertions(+), 91 deletions(-) diff --git a/specs/release-cut.md b/specs/release-cut.md index 9ae1370..47606b1 100644 --- a/specs/release-cut.md +++ b/specs/release-cut.md @@ -20,7 +20,7 @@ Recipes only — no variables, no embedded logic: Wired like the other host tools (`hostTool` + `addRunArtifact`, see build.zig ~230): `zig build cut -- {major|minor|patch}`. NOT installed to zig-out/bin. Its tests join `zig build test`. -Constants: one repo API base `https://git.mial.net/api/v1/repos/mokhtar/nxdns` (the tool can only ever target this repo — no configurability). The runs API needs a token (verified: anonymous GET is 401); read it from `~/.config/tea/config.yml` (logins entry for git.mial.net); a missing token is a clear error naming the file. +Constants: one repo API base `https://git.mial.net/api/v1/repos/mokhtar/nxdns` (the tool can only ever target this repo — no configurability); the endpoints under it are `/actions/runs`, `/commits/{sha}/status`, `/releases/tags/{tag}` and `/actions/runs/{id}/rerun`, the last being the only non-GET this program makes. The runs API needs a token (verified: anonymous GET is 401); read it from `~/.config/tea/config.yml` (logins entry for git.mial.net); a missing token is a clear error naming the file. ### Sequence @@ -38,10 +38,12 @@ Constants: one repo API base `https://git.mial.net/api/v1/repos/mokhtar/nxdns` ( A failed run must not strand the operator: - Bump pushed, then failure: rerun continues (preflight sees the version already bumped). - Local tag exists but never reached origin: verify it is an annotated tag by this tool's convention pointing at the current HEAD — adopt it; otherwise refuse with the exact `git tag -d` to run. Never delete a tag that exists on origin. +- Release run failed, retryably: one automatic rerun (`POST {api_base}/actions/runs/{id}/rerun`, 201, then the wait targets that same run id — a rerun keeps it — and treats the run as restarted only once the forge reports a HIGHER `run_attempt`, so the concluded previous attempt is never read as the result and a rerun that finishes between two polls still is). Retryable only while the run is on its first attempt (`run_attempt` is 1; an absent attempt number is not read as 1), the conclusion is `failure` (never `cancelled`), at least one commit-status context is in a failure state, every context in a failure state is a gate (`Gates / …` or `Release / gates`, event suffix dropped), and the release object for the tag is absent or a draft; a published release is terminal, as `release.yml`'s guard already states. The bound is the forge's attempt count and not a counter in this process, so a run already rerun by hand or by an earlier invocation is never rerun again. The statuses are parsed STRICTLY for this decision — an entry missing a string `context`, `status` or `target_url` refuses the whole payload — because an unreadable entry is not counted as a failure and would leave a list of nothing but gates; the failing-job report keeps the tolerant parse. Anything the tool cannot read — the statuses, the release object, the attempt number — is terminal rather than retried. +- Tag pushed, then failure: the tag on origin is no longer a reason to derive the next version. When origin's peeled tag object is the current HEAD and no release is published for it, the cut resumes at the release stage — no bump, no push, no CI wait, no tag, and no preflight, because everything the preflight guards has already happened — and reports it (`cut: resuming v0.0.16 at the release stage: the tag is on origin at HEAD and no release is published`). Peeling to HEAD is not on its own enough to resume on a tag: before the release wait, origin's tag object is verified the way the adopt path verifies a local one — annotated (a tag whose ref line and peeled line are one object is lightweight and refused), fetched when it is not here and compared against origin's object id when it is, carrying this tool's `v` message, pointing at that commit, and signed under the fingerprint `release.yml` pins — so a lightweight or unsigned tag somebody pushed at HEAD is a refusal naming what is wrong, never a deletion or a move. A published release, or a tag pointing anywhere else, derives the next version as before. `planVersion` takes the manifest, the bump kind and that one three-state fact, and stays a pure table-tested function. ### Tests (in-file, join `zig build test`) -Pure functions unit-tested: semver validation (accept/reject table incl. leading zeroes, `v` prefix), bump-kind parse, derivation table with the minor/major resets and overflow refusals, derive-vs-resume decision for all three kinds, zon `.version` parse + rewrite round-trip, changelog heading + non-empty body check, runs-JSON → decision (running / success / failure / no-run), tea-config token extraction. Process spawning and HTTP live behind thin call sites and are not mocked. +Pure functions unit-tested: semver validation (accept/reject table incl. leading zeroes, `v` prefix), bump-kind parse, derivation table with the minor/major resets and overflow refusals, derive-vs-resume decision for all three kinds, zon `.version` parse + rewrite round-trip, changelog heading + non-empty body check, runs-JSON → decision (running / success / failure / no-run, the run-id narrowing a rerun needs, and whether an observed `run_attempt` has advanced past the one a rerun was asked for), failing-contexts JSON → retryable or terminal, release-object status and JSON → absent / draft / published, `ls-remote` peeling an annotated tag to its commit, tea-config token extraction. Process spawning and HTTP live behind thin call sites and are not mocked. ## Anti-requirements diff --git a/tools/cut.zig b/tools/cut.zig index 46bb77a..76e5b81 100644 --- a/tools/cut.zig +++ b/tools/cut.zig @@ -292,28 +292,58 @@ const Plan = union(enum) { /// `build.zig.zon` declares a version that was never tagged, so an earlier /// cut got as far as the bump commit and stopped. Finish that one. resumed: Semver, + /// The tag is on origin, at this HEAD, and nothing is published under it: + /// an earlier cut got all the way past the tag push and stopped. Everything + /// up to and including the tag is already done, so this one watches the + /// release run and nothing else. + resume_release: Semver, /// The declared version is released; this is the next one. derived: Semver, fn semver(plan: Plan) Semver { return switch (plan) { - .resumed, .derived => |value| value, + .resumed, .resume_release, .derived => |value| value, }; } }; -/// The whole version decision, as a function of the manifest, the bump kind and -/// one fact about the forge. +/// Whether the forge has a release object for a tag, and what state it is in. /// -/// The resume branch is what keeps a rerun from double-incrementing. A cut that -/// committed the bump and then failed leaves `build.zig.zon` declaring a version -/// with no tag behind it; incrementing again would skip that version forever and -/// strand the commit that carries it. The absence of the tag is precisely the -/// signal that the declared version is still in flight, and it is the same -/// `ls-remote` the preflight needs anyway. -fn planVersion(declared: Semver, kind: BumpKind, declared_tag_on_origin: bool) error{Overflow}!Plan { - if (!declared_tag_on_origin) return .{ .resumed = declared }; - return .{ .derived = try nextVersion(declared, kind) }; +/// `release.yml`'s own guard states the rule this mirrors: the draft is the unit +/// of work, so a re-run clears a leftover draft and repeats, while a PUBLISHED +/// release for the tag is terminal and nothing may run against it again. +const ReleaseState = enum { absent, draft, published }; + +/// What origin knows about the tag `build.zig.zon` declares. +const DeclaredTag = union(enum) { + /// No such tag on origin. + absent, + /// On origin, and its peeled object is the commit that is HEAD here. + at_head: ReleaseState, + /// On origin at some other commit. + elsewhere, +}; + +/// The whole version decision, as a function of the manifest, the bump kind and +/// what origin knows about the declared tag. +/// +/// The resume branches are what keep a rerun from double-incrementing. A cut +/// that committed the bump and then failed leaves `build.zig.zon` declaring a +/// version with no tag behind it; incrementing again would skip that version +/// forever and strand the commit that carries it. A cut that got as far as +/// pushing the tag and then failed cannot be resumed by the absence of the tag, +/// because the tag is there — the fact that separates it from a finished release +/// is that nothing is published under it, and the fact that separates it from +/// somebody else's old tag is that it points at this HEAD. +fn planVersion(declared: Semver, kind: BumpKind, tag: DeclaredTag) error{Overflow}!Plan { + return switch (tag) { + .absent => .{ .resumed = declared }, + .at_head => |release| switch (release) { + .absent, .draft => .{ .resume_release = declared }, + .published => .{ .derived = try nextVersion(declared, kind) }, + }, + .elsewhere => .{ .derived = try nextVersion(declared, kind) }, + }; } /// The version field of `build.zig.zon`, parsed exactly as @@ -988,8 +1018,16 @@ const RunState = union(enum) { /// No run matches yet. Distinct from a run that exists and is queued: the /// two are bounded by different deadlines. absent, - running: u64, - concluded: struct { id: u64, conclusion: []const u8 }, + running: Seen, + concluded: Concluded, + + /// `attempt` is Gitea's `run_attempt`, which is 1 for a run that has never + /// been rerun and counts up from there (verified against the live listing: + /// run 683 reads 2 after its rerun). It is optional because it is the fact + /// the rerun decision turns on, and a payload that does not carry it must + /// make that decision fail closed rather than supply a number nobody read. + const Seen = struct { id: u64, attempt: ?u32 }; + const Concluded = struct { id: u64, attempt: ?u32, conclusion: []const u8 }; }; /// A `GET /actions/runs` payload reduced to a decision. @@ -997,7 +1035,10 @@ const RunState = union(enum) { /// The match is exact on all three of `path`, `head_sha` and `event`, and the /// highest matching id wins so a re-run supersedes the attempt it replaces. /// `min_id` excludes runs that predate the push this program just made; see -/// `runIdFloor` for when that is and is not the right question. +/// `runIdFloor` for when that is and is not the right question. `only_id` +/// narrows the match to one run instead: a rerun keeps the id of the attempt it +/// replaces, so the run to watch after one is the run that was just watched, and +/// nothing newer may stand in for it. /// /// A payload that is not an object carrying a `workflow_runs` array is an /// error, never "no run yet": an error body from the API or from something in @@ -1009,6 +1050,7 @@ fn decideRun( path: []const u8, sha: []const u8, min_id: ?u64, + only_id: ?u64, ) error{BadPayload}!RunState { const value = std.json.parseFromSliceLeaky(std.json.Value, arena, payload, .{}) catch { return error.BadPayload; @@ -1032,6 +1074,9 @@ fn decideRun( if (min_id) |floor| { if (id <= floor) continue; } + if (only_id) |wanted| { + if (id != wanted) continue; + } if (best == null or id > best_id) { best = run; best_id = id; @@ -1043,14 +1088,18 @@ fn decideRun( .string => |text| text, else => "", }; - if (!std.mem.eql(u8, status, "completed")) return .{ .running = best_id }; + const attempt: ?u32 = switch (run.get("run_attempt") orelse std.json.Value{ .null = {} }) { + .integer => |number| if (number > 0 and number <= std.math.maxInt(u32)) @intCast(number) else null, + else => null, + }; + if (!std.mem.eql(u8, status, "completed")) return .{ .running = .{ .id = best_id, .attempt = attempt } }; const conclusion = switch (run.get("conclusion") orelse std.json.Value{ .null = {} }) { .string => |text| text, // A completed run with no conclusion is not something this program can // read as success, and saying so beats inventing one. else => "unknown", }; - return .{ .concluded = .{ .id = best_id, .conclusion = conclusion } }; + return .{ .concluded = .{ .id = best_id, .attempt = attempt, .conclusion = conclusion } }; } fn stringField(object: std.json.ObjectMap, name: []const u8, want: []const u8) bool { @@ -1087,6 +1136,20 @@ const FailingContext = struct { description: []const u8, }; +/// How an unreadable status entry is treated. +const ContextParse = enum { + /// Skip it. The failing-job report is a diagnostic printed beside a failure + /// that is already being reported, and a partial list beats none. + tolerant, + /// Refuse the whole payload. The rerun decision turns on EVERY job that + /// failed being a gate, and an entry whose `status` cannot be read is not + /// counted as a failure at all — so one unreadable entry beside one + /// readable gate failure would classify as retryable while hiding a guard + /// or publish failure. `target_url` is held to the same bar: an entry that + /// cannot be attributed to a run might belong to this one. + strict, +}; + /// The commit statuses of one run that are not `success`. Gitea posts one status /// context per job, named ` / (push)`, with a `target_url` under /// `/actions/runs//`; the same commit carries the contexts of every run that @@ -1095,6 +1158,7 @@ fn failingContexts( arena: Allocator, payload: []const u8, run_id: u64, + mode: ContextParse, ) error{ BadPayload, OutOfMemory }![]const FailingContext { const value = std.json.parseFromSliceLeaky(std.json.Value, arena, payload, .{}) catch { return error.BadPayload; @@ -1104,11 +1168,19 @@ fn failingContexts( if (statuses != .array) return error.BadPayload; const needle = try std.fmt.allocPrint(arena, "/actions/runs/{d}/", .{run_id}); + const strict = mode == .strict; var list: std.ArrayList(FailingContext) = .empty; for (statuses.array.items) |item| { - if (item != .object) continue; + if (item != .object) { + if (strict) return error.BadPayload; + continue; + } + if (strict and !hasString(item.object, "target_url")) return error.BadPayload; const target = jsonString(item.object, "target_url"); if (std.mem.indexOf(u8, target, needle) == null) continue; + if (strict and !(hasString(item.object, "context") and hasString(item.object, "status"))) { + return error.BadPayload; + } const status = jsonString(item.object, "status"); if (std.mem.eql(u8, status, "success")) continue; try list.append(arena, .{ @@ -1120,6 +1192,95 @@ fn failingContexts( return list.items; } +/// Whether one commit-status context belongs to the gate set. +/// +/// Gitea names a context ` / ()`, so the event suffix is +/// dropped before the name is read: `Gates / test (push)` and `Release / gates +/// (push)` are the two shapes the gate set actually produces on this repository, +/// and comparing the raw string would miss both. +fn isGateContext(context: []const u8) bool { + const name = std.mem.trimEnd(u8, contextWithoutEvent(context), " "); + return std.mem.startsWith(u8, name, "Gates / ") or std.mem.eql(u8, name, "Release / gates"); +} + +/// A context without its trailing ` ()`, if it has one. +fn contextWithoutEvent(context: []const u8) []const u8 { + if (!std.mem.endsWith(u8, context, ")")) return context; + const open = std.mem.lastIndexOfScalar(u8, context, '(') orelse return context; + return context[0..open]; +} + +/// The status values that mean a job did not pass. `failingContexts` returns +/// every context that is not `success`, which includes the `skipped` publish job +/// of a run whose gates failed — that job did not fail, it never ran. +fn isFailureStatus(status: []const u8) bool { + return std.mem.eql(u8, status, "failure") or std.mem.eql(u8, status, "error"); +} + +const RerunDecision = enum { retryable, terminal }; + +/// Whether a concluded-but-unsuccessful `release.yml` run is worth one more +/// attempt. +/// +/// Retryable means the run is still on its FIRST attempt, every job that +/// actually FAILED is a gate, and the release this run exists to make is still +/// unmade. A guard failure is a statement about the tag and repeats; a publish +/// failure has already touched the release; a cancelled run was stopped by a +/// person, and rerunning it would undo that decision. A run with no failing +/// context at all is not understood, and a failure this program cannot explain +/// is not one it may retry. +/// +/// The attempt number is what makes "once" mean once. A counter in this process +/// would bound only the reruns THIS invocation made, so a rerun by hand, or by +/// an earlier invocation that resumed the same tag, would each be followed by +/// another. The forge counts the attempts, and it is the only party that sees +/// all of them. An absent attempt number is not read as 1: a payload that does +/// not say which attempt this is cannot license one. +fn classifyReleaseFailure( + conclusion: []const u8, + attempt: ?u32, + failing: []const FailingContext, + release: ReleaseState, +) RerunDecision { + if ((attempt orelse 0) != 1) return .terminal; + if (!std.mem.eql(u8, conclusion, "failure")) return .terminal; + if (release == .published) return .terminal; + + var failed: usize = 0; + for (failing) |entry| { + if (!isFailureStatus(entry.status)) continue; + failed += 1; + if (!isGateContext(entry.context)) return .terminal; + } + return if (failed == 0) .terminal else .retryable; +} + +/// `GET /releases/tags/` reduced to the one fact the resume and the rerun +/// both turn on. A 404 is the answer "there is no release", not an error; every +/// other non-200 is unreadable and must not be guessed at, because reading an +/// outage as "absent" would rerun a run against a published release. +fn decideReleaseState( + arena: Allocator, + status: u16, + payload: []const u8, +) error{BadPayload}!ReleaseState { + if (status == 404) return .absent; + if (status != 200) return error.BadPayload; + const value = std.json.parseFromSliceLeaky(std.json.Value, arena, payload, .{}) catch { + return error.BadPayload; + }; + if (value != .object) return error.BadPayload; + return switch (value.object.get("draft") orelse std.json.Value{ .null = {} }) { + .bool => |is_draft| if (is_draft) .draft else .published, + else => error.BadPayload, + }; +} + +fn hasString(object: std.json.ObjectMap, name: []const u8) bool { + const value = object.get(name) orelse return false; + return value == .string; +} + fn jsonString(object: std.json.ObjectMap, name: []const u8) []const u8 { const value = object.get(name) orelse return ""; return switch (value) { @@ -1206,6 +1367,31 @@ fn lsRemoteFind(stdout: []const u8, ref: []const u8) ?[]const u8 { return null; } +/// The COMMIT a ref resolves to in `git ls-remote` output: the `refs/…^{}` line +/// where the ref is an annotated tag, and the plain line otherwise. +/// +/// `lsRemoteFind` answers "is this ref there", and for that the tag object is a +/// fine answer. Asking whether a tag points at HEAD is a different question: +/// an annotated tag's own object id is the hash of the tag, never the commit, +/// so comparing that against HEAD would say "elsewhere" for every tag this +/// program makes. +fn lsRemotePeeled(stdout: []const u8, ref: []const u8) ?[]const u8 { + var plain: ?[]const u8 = null; + var lines = std.mem.splitScalar(u8, stdout, '\n'); + while (lines.next()) |raw| { + const line = std.mem.trimEnd(u8, raw, " \t\r"); + const tab = std.mem.indexOfScalar(u8, line, '\t') orelse continue; + const name = std.mem.trim(u8, line[tab + 1 ..], " \t"); + const object = std.mem.trim(u8, line[0..tab], " \t"); + if (std.mem.endsWith(u8, name, "^{}")) { + if (std.mem.eql(u8, name[0 .. name.len - 3], ref)) return object; + continue; + } + if (std.mem.eql(u8, name, ref)) plain = object; + } + return plain; +} + fn deadlineExpired(started_ns: i96, now_ns: i96, budget_ns: u64) bool { if (now_ns <= started_ns) return false; return @as(u128, @intCast(now_ns - started_ns)) >= budget_ns; @@ -1388,6 +1574,7 @@ const Fetched = struct { const Attempt = struct { gpa: Allocator, io: Io, + method: http.Method, url: []const u8, authorization: []const u8, }; @@ -1397,7 +1584,7 @@ const HttpOutcome = union(enum) { expiry: Io.Cancelable!void, }; -fn attemptGet(attempt: Attempt) anyerror!Fetched { +fn attemptRequest(attempt: Attempt) anyerror!Fetched { var client: http.Client = .{ .allocator = attempt.gpa, .io = attempt.io }; defer client.deinit(); @@ -1408,7 +1595,7 @@ fn attemptGet(attempt: Attempt) anyerror!Fetched { }; const result = try client.fetch(.{ .location = .{ .url = attempt.url }, - .method = .GET, + .method = attempt.method, .extra_headers = &headers, .response_writer = &body.writer, .redirect_behavior = .unhandled, @@ -1421,16 +1608,17 @@ fn sleepNs(io: Io, ns: u64) Io.Cancelable!void { return duration.sleep(io); } -/// One bounded GET. +/// One bounded request. /// /// `std.http.Client` takes no deadline in 0.16.0, so the request races a sleep /// on the monotonic clock and the loser is canceled — `src/filter/manager.zig` /// bounds blocklist downloads the same way. Everything the attempt allocates /// comes from `scratch`, which the caller drops after each poll; a wait that /// runs for two hours must not accumulate two hours of response bodies. -fn httpGet( +fn httpSend( ctx: *Ctx, scratch: Allocator, + method: http.Method, url: []const u8, authorization: []const u8, budget_ns: u64, @@ -1442,18 +1630,19 @@ fn httpGet( const attempt: Attempt = .{ .gpa = scratch, .io = ctx.io, + .method = method, .url = url, .authorization = authorization, }; - race.concurrent(.fetch, attemptGet, .{attempt}) catch |err| switch (err) { + race.concurrent(.fetch, attemptRequest, .{attempt}) catch |err| switch (err) { error.ConcurrencyUnavailable => { - ctx.soft("http", "no unit of concurrency is available to bound GET {s}", .{url}); + ctx.soft("http", "no unit of concurrency is available to bound {t} {s}", .{ method, url }); return CheckFailed; }, }; race.concurrent(.expiry, sleepNs, .{ ctx.io, budget_ns }) catch |err| switch (err) { error.ConcurrencyUnavailable => { - ctx.soft("http", "no unit of concurrency is available to bound GET {s}", .{url}); + ctx.soft("http", "no unit of concurrency is available to bound {t} {s}", .{ method, url }); return CheckFailed; }, }; @@ -1464,7 +1653,7 @@ fn httpGet( // A canceled sleep means this process is being torn down, not that // the forge is slow. try result; - ctx.soft("http", "GET {s} did not answer within {d}s", .{ url, budget_ns / std.time.ns_per_s }); + ctx.soft("http", "{t} {s} did not answer within {d}s", .{ method, url, budget_ns / std.time.ns_per_s }); return CheckFailed; }, } @@ -1480,7 +1669,7 @@ fn apiGet( authorization: []const u8, budget_ns: u64, ) ![]const u8 { - const response = try httpGet(ctx, scratch, url, authorization, budget_ns); + const response = try httpSend(ctx, scratch, .GET, url, authorization, budget_ns); if (response.status != 200) { ctx.soft(check, "GET {s} answered {d}: {s}", .{ url, response.status, std.mem.trimEnd(u8, response.body, "\n"), @@ -1520,13 +1709,35 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void { return CheckFailed; }; - // The one fact the version decision turns on. A transport failure here is a + // The facts the version decision turns on. A transport failure here is a // refusal, never "the tag is absent": read as absent it would resume a // version that is in fact already released. const declared_tag = ctx.fmt("v{s}", .{declared_text}); - const declared_on_origin = try remoteRef(ctx, ctx.fmt("refs/tags/{s}", .{declared_tag})); + const declared_tag_ref = ctx.fmt("refs/tags/{s}", .{declared_tag}); + const declared_on_origin = try remoteRefPeeled(ctx, declared_tag_ref); - const plan = planVersion(declared, kind, declared_on_origin != null) catch |err| { + // Only the branch that has to ask the forge about a release needs the token + // this early. On every other branch the preflight reads it along with the + // rest of the checks, so an operator with no token still learns about the + // tree, the branch and the changelog in the same run. + var authorization: ?[]const u8 = null; + var tagged_head: ?[]const u8 = null; + const declared_state: DeclaredTag = if (declared_on_origin) |object| state: { + const head = try headSha(ctx); + if (!std.mem.eql(u8, object, head)) break :state .elsewhere; + tagged_head = head; + + authorization = readAuthorization(ctx); + const token = authorization orelse { + ctx.soft("resume", "{s} is on origin at HEAD, and without a token the release for it cannot be read; that is the fact that separates a cut to finish from the next version to cut", .{declared_tag}); + return CheckFailed; + }; + var scratch = std.heap.ArenaAllocator.init(ctx.gpa); + defer scratch.deinit(); + break :state .{ .at_head = try releaseState(ctx, scratch.allocator(), token, declared_tag) }; + } else .absent; + + const plan = planVersion(declared, kind, declared_state) catch |err| { ctx.soft("bump-kind", "a {t} bump of {s} overflows: {t}", .{ kind, declared_text, err }); return CheckFailed; }; @@ -1540,6 +1751,29 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void { return CheckFailed; } + // Everything the preflight guards has already happened on this branch: the + // tag is public, so there is nothing left to refuse and nothing left to + // change. What remains is the release run and the release object. + switch (plan) { + .resume_release => { + ctx.note("resuming {s} at the release stage: the tag is on origin at HEAD and no release is published", .{declared_tag}); + // Both non-null on this branch: reading HEAD and then the release + // object under this tag is what put the plan here, and re-reading + // HEAD could disagree with the comparison that decided it. + const token = authorization orelse return CheckFailed; + const sha = tagged_head orelse return CheckFailed; + // Pointing at HEAD is not enough to adopt a tag. Anybody can push a + // lightweight or unsigned tag of this name at this commit, and + // resuming on it would spend a release run — and this program's + // report — on a tag the release guard will refuse. + try verifyOriginTag(ctx, declared_tag, declared_tag_ref, sha); + try awaitRelease(ctx, token, declared_tag, sha, null); + try reportRelease(ctx, token, declared_tag); + return; + }, + .resumed, .derived => {}, + } + const bump_needed = switch (plan) { .resumed => resumed: { ctx.note("resuming {s}: build.zig.zon declares it and {s} is not on origin, so an earlier cut committed the bump and stopped", .{ @@ -1547,18 +1781,20 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void { }); break :resumed false; }, + // Returned above. + .resume_release => return CheckFailed, .derived => derived: { ctx.note("cutting {s}: {s} is released and this is a {t} bump", .{ version, declared_tag, kind }); break :derived true; }, }; - const checked = try preflight(ctx, version, bump_needed, plan); + const checked = try preflight(ctx, version, bump_needed, plan, authorization); if (ctx.failures != 0) return CheckFailed; // A preflight that found nothing wrong found a token; this is a refusal // rather than an assertion because nothing else in this file may assume a // check's failure was recorded. - const authorization = checked.authorization orelse return CheckFailed; + const token = checked.authorization orelse return CheckFailed; if (bump_needed) try bump(ctx, version, zon_source); @@ -1581,13 +1817,20 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void { } ctx.pass("push-master", "origin/{s} is {s}", .{ master_ref, sha }); - try waitForRun(ctx, authorization, .{ + const ci_wait: Wait = .{ .label = "ci.yml", .path = ci_run_path, .sha = sha, .floor = ci_floor, .completion_ns = ci_completion_ns, - }); + }; + switch (try waitForRun(ctx, token, ci_wait)) { + .succeeded => {}, + // A failed CI run is never rerun: nothing has been tagged yet, so the + // operator can fix the cause and run the cut again, which is a better + // answer than a retry that hides a real failure. + .failed => |failed| return failRun(ctx, token, ci_wait, failed), + } // Between the CI wait and the tag an operator has had minutes to commit // something. The tag names `sha` explicitly, so a moved HEAD would silently @@ -1631,15 +1874,9 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void { } ctx.pass("push-tag", "{s} is on origin", .{tag}); - try waitForRun(ctx, authorization, .{ - .label = "release.yml", - .path = ctx.fmt("release.yml@refs/tags/{s}", .{tag}), - .sha = sha, - .floor = release_floor, - .completion_ns = release_completion_ns, - }); + try awaitRelease(ctx, token, tag, sha, release_floor); - try reportRelease(ctx, authorization, tag); + try reportRelease(ctx, token, tag); } /// Pushes one ref and returns the run-id floor the wait should use: `floor` when @@ -1750,9 +1987,17 @@ const Preflight = struct { /// every failure is recorded; the caller refuses if any of them failed. Only the /// version derivation refuses on the spot — an unknown bump kind, an unreadable /// manifest or an overflow leaves no version to check anything else against. -fn preflight(ctx: *Ctx, version: []const u8, bump_needed: bool, plan: Plan) !Preflight { +fn preflight( + ctx: *Ctx, + version: []const u8, + bump_needed: bool, + plan: Plan, + /// The token, when the version decision already had to read it. Reading it + /// twice would report a missing token twice. + known_authorization: ?[]const u8, +) !Preflight { var result: Preflight = .{}; - result.authorization = readAuthorization(ctx); + result.authorization = known_authorization orelse readAuthorization(ctx); const status = try gitCapture(ctx, &.{ "git", "status", "--porcelain" }, git_local_timeout_s); if (!status.ok()) { @@ -1797,7 +2042,9 @@ fn preflight(ctx: *Ctx, version: []const u8, bump_needed: bool, plan: Plan) !Pre // add a second network round that can disagree with the first. A derived // version has never been looked up at all. const on_origin = switch (plan) { - .resumed => null, + // `.resume_release` never reaches the preflight: its tag IS on origin, + // and the caller has already gone straight to the release stage. + .resumed, .resume_release => null, .derived => remoteRef(ctx, tag_ref) catch |err| switch (err) { error.CheckFailed => { // `remoteRef` already reported the transport failure. It must @@ -1834,7 +2081,7 @@ fn preflight(ctx: *Ctx, version: []const u8, bump_needed: bool, plan: Plan) !Pre return result; } switch (plan) { - .resumed => ctx.pass("tag-free", "{s} is not on origin, which is why this cut resumes it", .{tag}), + .resumed, .resume_release => ctx.pass("tag-free", "{s} is not on origin, which is why this cut resumes it", .{tag}), .derived => ctx.pass("tag-free", "{s} is not on origin", .{tag}), } @@ -2151,6 +2398,86 @@ fn localTag(ctx: *Ctx, tag: []const u8, bump_needed: bool) !?[]const u8 { return object.trimmedStdout(); } +/// Proves the tag ORIGIN publishes is one this program could have made: an +/// annotated tag carrying this tool's message convention, pointing at `commit`, +/// signed under the certificate `release.yml` pins. +/// +/// This is the adopt path's `localTag` question asked about origin's object +/// rather than a local ref, and it exists because the resume decision cannot +/// rest on "a tag of this name peels to HEAD". The local ref of that name is not +/// evidence either: it can be stale, or a different tag entirely, so it is +/// compared against origin's object id and the object is fetched when it is +/// missing. Nothing here deletes or moves a tag — the tag is on origin, and a +/// tag on origin is never this program's to change. +fn verifyOriginTag(ctx: *Ctx, tag: []const u8, tag_ref: []const u8, commit: []const u8) !void { + const object = try remoteRef(ctx, tag_ref) orelse { + ctx.soft("origin-tag", "{s} vanished from origin between two lookups", .{tag}); + return CheckFailed; + }; + // `ls-remote` prints a `^{}` line only for an annotated tag, so a tag whose + // ref line and peeled line are the same object has no tag object at all. + if (std.mem.eql(u8, object, commit)) { + ctx.soft("origin-tag", "{s} on origin is a lightweight tag at {s}, not an annotated tag this tool made; the release guard refuses it", .{ tag, commit }); + return CheckFailed; + } + + const local = try gitCapture(ctx, &.{ + "git", "rev-parse", "--verify", "--quiet", tag_ref, + }, git_local_timeout_s); + if (!local.ok()) { + const fetched = try gitCapture(ctx, &.{ + "git", "fetch", "origin", ctx.fmt("{s}:{s}", .{ tag_ref, tag_ref }), + }, git_network_timeout_s); + if (!fetched.ok()) { + ctx.soft("origin-tag", "cannot fetch {s} from origin to verify it: {s}", .{ + tag, std.mem.trimEnd(u8, fetched.combined(ctx.arena), "\n"), + }); + return CheckFailed; + } + } + const here = try gitCapture(ctx, &.{ "git", "rev-parse", tag_ref }, git_local_timeout_s); + if (!here.ok() or !std.mem.eql(u8, here.trimmedStdout(), object)) { + ctx.soft("origin-tag", "{s} here is the object {s}, but origin publishes {s}; reconcile them before resuming", .{ + tag, here.trimmedStdout(), object, + }); + return CheckFailed; + } + + const kind = try gitCapture(ctx, &.{ "git", "cat-file", "-t", object }, git_local_timeout_s); + if (!kind.ok() or !std.mem.eql(u8, kind.trimmedStdout(), "tag")) { + ctx.soft("origin-tag", "{s} on origin is a '{s}' object, not an annotated tag", .{ + tag, kind.trimmedStdout(), + }); + return CheckFailed; + } + + const subject = try gitCapture(ctx, &.{ + "git", "tag", "-l", "--format=%(contents:subject)", tag, + }, git_local_timeout_s); + if (!subject.ok() or !std.mem.eql(u8, subject.trimmedStdout(), tag)) { + ctx.soft("origin-tag", "{s} carries the message '{s}', not '{s}', so this tool did not make it", .{ + tag, subject.trimmedStdout(), tag, + }); + return CheckFailed; + } + + const target = try gitCapture(ctx, &.{ + "git", "rev-parse", ctx.fmt("{s}^{{commit}}", .{tag_ref}), + }, git_local_timeout_s); + if (!target.ok() or !std.mem.eql(u8, target.trimmedStdout(), commit)) { + ctx.soft("origin-tag", "{s} points at {s}, not at the {s} origin peels it to", .{ + tag, target.trimmedStdout(), commit, + }); + return CheckFailed; + } + + // The same check the tag push makes, and the same one release.yml's guard + // will make: a signature that is merely present is not a signature from the + // pinned certificate. + try verifyTag(ctx, tag, commit); + ctx.pass("origin-tag", "{s} ({s}) on origin is this tool's own signed tag at {s}", .{ tag, object, commit }); +} + /// Rewrites `build.zig.zon` and commits it, and nothing else. fn bump(ctx: *Ctx, version: []const u8, source: [:0]const u8) !void { const rewritten = rewriteZonVersion(ctx.arena, source, version) catch |err| { @@ -2240,6 +2567,19 @@ fn remoteRef(ctx: *Ctx, ref: []const u8) !?[]const u8 { return lsRemoteFind(run.stdout, ref); } +/// The COMMIT `origin` has for a ref, peeling an annotated tag. Same refusal +/// discipline as `remoteRef`: a failed command is never an absent ref. +fn remoteRefPeeled(ctx: *Ctx, ref: []const u8) !?[]const u8 { + const run = try gitCapture(ctx, &.{ "git", "ls-remote", "origin", ref }, git_network_timeout_s); + if (!run.ok()) { + ctx.soft("ls-remote", "`git ls-remote origin {s}` exited {d}: {s}", .{ + ref, run.code, std.mem.trimEnd(u8, run.combined(ctx.arena), "\n"), + }); + return CheckFailed; + } + return lsRemotePeeled(run.stdout, ref); +} + fn reassert(ctx: *Ctx, sha: []const u8) !void { const head = try headSha(ctx); if (!std.mem.eql(u8, head, sha)) { @@ -2282,6 +2622,28 @@ const Wait = struct { sha: []const u8, floor: ?u64, completion_ns: u64, + /// Set when this wait follows a rerun of a run that had already concluded: + /// the run's id, and the `run_attempt` it was rerun FROM. + /// + /// A rerun keeps the run id, so only that id may match. It is the attempt + /// number, not the status, that says whether the rerun has happened yet: + /// the listing answers with the finished previous attempt for as long as + /// the forge takes to requeue, and a status test would read that as the + /// result. It also has to be the attempt number rather than "has been seen + /// running", because a rerun can finish between two polls and never be + /// observed running at all. + rerun_of: ?struct { id: u64, attempt: u32 } = null, +}; + +/// A run that concluded as something other than `success`. +const ConcludedFailure = struct { id: u64, attempt: ?u32, conclusion: []const u8 }; + +/// What a concluded run was. +const WaitResult = union(enum) { + succeeded: u64, + /// Reported by the caller, which is the only place that knows whether this + /// failure is the end of the cut or the reason for one more attempt. + failed: ConcludedFailure, }; /// Waits for one workflow run to appear and then to conclude. @@ -2293,20 +2655,31 @@ const Wait = struct { /// runs from the moment the run first APPEARS — it is sized against what the /// workflow's own jobs may take, and time spent queueing before the run existed /// is not time any of those jobs had. -fn waitForRun(ctx: *Ctx, authorization: []const u8, wait: Wait) !void { +fn waitForRun(ctx: *Ctx, authorization: []const u8, wait: Wait) !WaitResult { const started = Io.Clock.awake.now(ctx.io); var appeared: ?Io.Timestamp = null; var seen_id: ?u64 = null; var polls: usize = 0; + var awaiting_restart = wait.rerun_of != null; - ctx.note("waiting for the {s} run on {s}", .{ wait.label, wait.sha }); + if (wait.rerun_of) |from| { + ctx.note("waiting for the {s} run {d} to leave attempt {d}", .{ wait.label, from.id, from.attempt }); + } else { + ctx.note("waiting for the {s} run on {s}", .{ wait.label, wait.sha }); + } while (true) { const now = Io.Clock.awake.now(ctx.io); const since = appeared orelse started; const budget = if (appeared == null) run_startup_ns else wait.completion_ns; if (deadlineExpired(since.nanoseconds, now.nanoseconds, budget)) { - if (seen_id) |id| { + if (awaiting_restart) { + ctx.soft( + "run-wait", + "the {s} run {d} was rerun but is still reported on attempt {d} {d:.0}s later", + .{ wait.label, wait.rerun_of.?.id, wait.rerun_of.?.attempt, elapsedSeconds(since.nanoseconds, now.nanoseconds) }, + ); + } else if (seen_id) |id| { ctx.soft( "run-wait", "the {s} run {d} on {s} has not concluded {d:.0}s after it appeared; the ceiling is {d} minutes", @@ -2329,31 +2702,36 @@ fn waitForRun(ctx: *Ctx, authorization: []const u8, wait: Wait) !void { http_attempt_ns, )); - switch (state) { + // Until the rerun shows a HIGHER attempt number, every reading is of + // the attempt that already concluded, whatever its status says. + if (awaiting_restart and attemptAdvanced(state, wait.rerun_of.?.attempt)) { + awaiting_restart = false; + ctx.note("{s} run {d} is on attempt {d}", .{ + wait.label, wait.rerun_of.?.id, wait.rerun_of.?.attempt + 1, + }); + } + + if (!awaiting_restart) switch (state) { .absent => {}, - .running => |id| { + .running => |run| { if (appeared == null) { appeared = Io.Clock.awake.now(ctx.io); ctx.note("{s} run {d} appeared after {d:.0}s and is running", .{ - wait.label, id, elapsedSeconds(started.nanoseconds, appeared.?.nanoseconds), + wait.label, run.id, elapsedSeconds(started.nanoseconds, appeared.?.nanoseconds), }); } - seen_id = id; + seen_id = run.id; }, .concluded => |done| { if (std.mem.eql(u8, done.conclusion, "success")) { ctx.pass("run-wait", "{s} run {d} succeeded after {d:.0}s", .{ wait.label, done.id, elapsedSeconds(started.nanoseconds, Io.Clock.awake.now(ctx.io).nanoseconds), }); - return; + return .{ .succeeded = done.id }; } - ctx.soft("run-wait", "{s} run {d} concluded '{s}' on {s}", .{ - wait.label, done.id, done.conclusion, wait.sha, - }); - try reportFailingJobs(ctx, authorization, wait.sha, done.id); - return CheckFailed; + return .{ .failed = .{ .id = done.id, .attempt = done.attempt, .conclusion = done.conclusion } }; }, - } + }; polls += 1; if (polls % progress_every_polls == 0) { @@ -2368,13 +2746,27 @@ fn waitForRun(ctx: *Ctx, authorization: []const u8, wait: Wait) !void { } } +/// Whether an observed run has moved past the attempt a rerun was asked for. +/// A run whose payload carries no attempt number has not moved past anything: +/// the wait keeps waiting and its deadline reports that, which beats adopting +/// the previous attempt's conclusion as this one's. +fn attemptAdvanced(state: RunState, from: u32) bool { + const attempt = switch (state) { + .absent => return false, + .running => |run| run.attempt, + .concluded => |done| done.attempt, + }; + return (attempt orelse 0) > from; +} + fn pollRun(ctx: *Ctx, authorization: []const u8, wait: Wait, budget_ns: u64) !RunState { var scratch = std.heap.ArenaAllocator.init(ctx.gpa); defer scratch.deinit(); const arena = scratch.allocator(); const body = try apiGet(ctx, arena, "run-wait", runs_url, authorization, budget_ns); - const state = decideRun(arena, body, wait.path, wait.sha, wait.floor) catch { + const only_id = if (wait.rerun_of) |from| from.id else null; + const state = decideRun(arena, body, wait.path, wait.sha, wait.floor, only_id) catch { ctx.soft("run-wait", "the runs listing answered 200 with a payload that is not a run list", .{}); return CheckFailed; }; @@ -2383,14 +2775,151 @@ fn pollRun(ctx: *Ctx, authorization: []const u8, wait: Wait, budget_ns: u64) !Ru // copied out. return switch (state) { .absent => .absent, - .running => |id| .{ .running = id }, + .running => |run| .{ .running = run }, .concluded => |done| .{ .concluded = .{ .id = done.id, + .attempt = done.attempt, .conclusion = try ctx.arena.dupe(u8, done.conclusion), } }, }; } +/// Reports a concluded-but-unsuccessful run and refuses. This is what +/// `waitForRun` used to do inline; it moved out so the release stage can look at +/// the same failure first and decide whether to retry it. +fn failRun(ctx: *Ctx, authorization: []const u8, wait: Wait, failed: ConcludedFailure) !void { + ctx.soft("run-wait", "{s} run {d} concluded '{s}' on {s}", .{ + wait.label, failed.id, failed.conclusion, wait.sha, + }); + try reportFailingJobs(ctx, authorization, wait.sha, failed.id); + return CheckFailed; +} + +/// Waits for the `release.yml` run, and gives a retryable failure exactly one +/// more attempt. +/// +/// v0.0.16's release run failed on a timing-flaky gate while its guard had +/// already passed and its publish job had not run at all. Rerunning that run by +/// hand published the release; the tool had reported a failure and exited. The +/// condition is narrow and the bound is the forge's own `run_attempt` rather +/// than a counter in this process — see `classifyReleaseFailure` — so a broken +/// gate still stops the cut, and so does a run somebody already reran by hand. +fn awaitRelease( + ctx: *Ctx, + authorization: []const u8, + tag: []const u8, + sha: []const u8, + floor: ?u64, +) !void { + var wait: Wait = .{ + .label = "release.yml", + .path = ctx.fmt("release.yml@refs/tags/{s}", .{tag}), + .sha = sha, + .floor = floor, + .completion_ns = release_completion_ns, + }; + + while (true) { + const failed = switch (try waitForRun(ctx, authorization, wait)) { + .succeeded => return, + .failed => |done| done, + }; + + const decision = try classifyFailedReleaseRun(ctx, authorization, tag, sha, failed); + if (decision == .terminal) return failRun(ctx, authorization, wait, failed); + // `classifyReleaseFailure` refuses every attempt but the first, so this + // is the only place the attempt to rerun FROM can come from. + const attempt = failed.attempt orelse return failRun(ctx, authorization, wait, failed); + + // The failure that is about to be retried is still named: an operator + // reading this afterwards has to be able to see what was flaky. + ctx.note("release.yml run {d} concluded '{s}'; every job that failed is a gate and nothing is published for {s}", .{ + failed.id, failed.conclusion, tag, + }); + try reportFailingJobs(ctx, authorization, sha, failed.id); + + try rerunRun(ctx, authorization, failed.id); + ctx.note("rerunning release.yml run {d} once, from attempt {d}", .{ failed.id, attempt }); + + // The rerun keeps the id, so the same run is watched again — and only + // that run, until the forge says it is on a later attempt. + wait.floor = null; + wait.rerun_of = .{ .id = failed.id, .attempt = attempt }; + } +} + +/// Reads the two facts `classifyReleaseFailure` judges and applies it. Anything +/// that cannot be read is terminal and says so: a failure this program cannot +/// explain is not one it may retry. +fn classifyFailedReleaseRun( + ctx: *Ctx, + authorization: []const u8, + tag: []const u8, + sha: []const u8, + failed: ConcludedFailure, +) !RerunDecision { + const run_id = failed.id; + var scratch = std.heap.ArenaAllocator.init(ctx.gpa); + defer scratch.deinit(); + const arena = scratch.allocator(); + + const status_url = try std.fmt.allocPrint(arena, api_base ++ "/commits/{s}/status", .{sha}); + const status_body = apiGet(ctx, arena, "rerun", status_url, authorization, http_attempt_ns) catch { + ctx.note("the commit statuses for {s} could not be read, so run {d} is not rerun", .{ sha, run_id }); + return .terminal; + }; + // Strict: an entry this parser cannot read is not a job it may ignore. A + // guard failure hidden behind a malformed status would otherwise leave a + // list of nothing but gates, and read as retryable. + const failing = failingContexts(arena, status_body, run_id, .strict) catch { + ctx.note("the commit statuses for {s} are not in the expected shape, so run {d} is not rerun", .{ sha, run_id }); + return .terminal; + }; + + const release = releaseState(ctx, arena, authorization, tag) catch { + ctx.note("the release object for {s} could not be read, so run {d} is not rerun", .{ tag, run_id }); + return .terminal; + }; + return classifyReleaseFailure(failed.conclusion, failed.attempt, failing, release); +} + +/// `GET /releases/tags/`, where a 404 is an answer rather than a failure. +/// `reportRelease` uses `apiGet` instead, because by the time it runs a missing +/// release IS the failure. +fn releaseState( + ctx: *Ctx, + scratch: Allocator, + authorization: []const u8, + tag: []const u8, +) !ReleaseState { + const url = try std.fmt.allocPrint(scratch, api_base ++ "/releases/tags/{s}", .{tag}); + const response = try httpSend(ctx, scratch, .GET, url, authorization, http_attempt_ns); + return decideReleaseState(scratch, response.status, response.body) catch { + ctx.soft("release-state", "GET {s} answered {d} with a payload this program cannot read", .{ + url, response.status, + }); + return CheckFailed; + }; +} + +/// Asks the forge to run one workflow run again. Gitea answers 201; every other +/// status is a refusal that names it, because a rerun that did not happen must +/// never be waited on as though it had. +fn rerunRun(ctx: *Ctx, authorization: []const u8, run_id: u64) !void { + var scratch = std.heap.ArenaAllocator.init(ctx.gpa); + defer scratch.deinit(); + const arena = scratch.allocator(); + + const url = try std.fmt.allocPrint(arena, api_base ++ "/actions/runs/{d}/rerun", .{run_id}); + const response = try httpSend(ctx, arena, .POST, url, authorization, http_attempt_ns); + if (response.status != 201) { + ctx.soft("rerun", "POST {s} answered {d}: {s}", .{ + url, response.status, std.mem.trimEnd(u8, response.body, "\n"), + }); + return CheckFailed; + } +} + /// Names the jobs that failed, so the operator gets a job rather than a run to /// open. Best-effort: a failure to read the statuses must not replace the /// failure that is actually being reported. @@ -2407,7 +2936,7 @@ fn reportFailingJobs(ctx: *Ctx, authorization: []const u8, sha: []const u8, run_ ctx.note("the commit statuses for {s} could not be read; open the run itself", .{sha}); return; }; - const failing = failingContexts(arena, body, run_id) catch { + const failing = failingContexts(arena, body, run_id, .tolerant) catch { ctx.note("the commit statuses for {s} are not in the expected shape; open the run itself", .{sha}); return; }; @@ -2565,18 +3094,32 @@ test "a bump kind derives the next version with the right resets" { test "an untagged declared version is resumed, a released one is incremented" { const declared = parseSemver("0.0.8").?; - // v0.0.8 is on origin, so 0.0.8 is released and the cut moves past it. - const released = try planVersion(declared, .patch, true); + // v0.0.8 is on origin at HEAD and published, so 0.0.8 is released and the + // cut moves past it. A tag on origin somewhere else is the same answer by a + // different route: it is not this HEAD's cut to finish. + const released = try planVersion(declared, .patch, .{ .at_head = .published }); try testing.expectEqual(parseSemver("0.0.9").?, released.derived); - try testing.expectEqual(parseSemver("0.1.0").?, (try planVersion(declared, .minor, true)).derived); - try testing.expectEqual(parseSemver("1.0.0").?, (try planVersion(declared, .major, true)).derived); + try testing.expectEqual(parseSemver("0.1.0").?, (try planVersion(declared, .minor, .{ .at_head = .published })).derived); + try testing.expectEqual(parseSemver("1.0.0").?, (try planVersion(declared, .major, .{ .at_head = .published })).derived); + try testing.expectEqual(parseSemver("0.0.9").?, (try planVersion(declared, .patch, .elsewhere)).derived); + + // The tag is on origin at this HEAD and nothing is published under it, so + // an earlier cut got past the tag push and stopped. Resuming means the + // release stage only — the bump, the push and the tag are already done — + // and a leftover DRAFT is the same case: release.yml's own guard clears a + // draft and repeats, and only a published release is terminal. + for ([_]ReleaseState{ .absent, .draft }) |unpublished| { + const at_release = try planVersion(declared, .patch, .{ .at_head = unpublished }); + try testing.expectEqual(declared, at_release.resume_release); + try testing.expectEqual(declared, at_release.semver()); + } // v0.0.8 is NOT on origin, so the bump commit exists and the tag never // followed: this is the rerun of that cut, and incrementing again would // skip 0.0.8 forever and strand the commit that declares it. The kind is // ignored on this branch — every kind resumes the same version. inline for (.{ BumpKind.patch, BumpKind.minor, BumpKind.major }) |kind| { - const in_flight = try planVersion(declared, kind, false); + const in_flight = try planVersion(declared, kind, .absent); try testing.expectEqual(declared, in_flight.resumed); try testing.expectEqual(declared, in_flight.semver()); } @@ -2584,8 +3127,9 @@ test "an untagged declared version is resumed, a released one is incremented" { // The overflow refusal reaches the caller through the plan, not a panic, // and only on the branch that actually increments. const ceiling: Semver = .{ .major = std.math.maxInt(u32), .minor = 0, .patch = 0 }; - try testing.expectError(error.Overflow, planVersion(ceiling, .major, true)); - try testing.expectEqual(ceiling, (try planVersion(ceiling, .major, false)).resumed); + try testing.expectError(error.Overflow, planVersion(ceiling, .major, .{ .at_head = .published })); + try testing.expectError(error.Overflow, planVersion(ceiling, .major, .elsewhere)); + try testing.expectEqual(ceiling, (try planVersion(ceiling, .major, .absent)).resumed); } test "the zon version survives a rewrite round trip" { @@ -2834,11 +3378,11 @@ test "the runs listing decides appear, run, succeed and fail" { const other = "0906d76c000000000000000000000000000000ff"; const payload = \\{"total_count":4,"workflow_runs":[ - \\ {"id":566,"path":"release.yml@refs/tags/v0.0.7","event":"push", + \\ {"id":566,"path":"release.yml@refs/tags/v0.0.7","event":"push","run_attempt":1, \\ "head_sha":"324704b53f180a3eccb23c4389af03b413b69488","status":"in_progress","conclusion":null}, - \\ {"id":565,"path":"ci.yml@refs/heads/master","event":"push", + \\ {"id":565,"path":"ci.yml@refs/heads/master","event":"push","run_attempt":1, \\ "head_sha":"324704b53f180a3eccb23c4389af03b413b69488","status":"completed","conclusion":"success"}, - \\ {"id":562,"path":"ci.yml@refs/heads/master","event":"push", + \\ {"id":562,"path":"ci.yml@refs/heads/master","event":"push","run_attempt":1, \\ "head_sha":"324704b53f180a3eccb23c4389af03b413b69488","status":"completed","conclusion":"failure"}, \\ {"id":564,"path":"ci.yml@refs/heads/master","event":"push", \\ "head_sha":"0906d76c000000000000000000000000000000ff","status":"completed","conclusion":"cancelled"} @@ -2847,38 +3391,283 @@ test "the runs listing decides appear, run, succeed and fail" { // The newest matching run wins, so a re-run supersedes the attempt it // replaces rather than the other way round. - const newest = try decideRun(arena, payload, ci_run_path, sha, null); + const newest = try decideRun(arena, payload, ci_run_path, sha, null, null); try testing.expectEqual(@as(u64, 565), newest.concluded.id); try testing.expectEqualStrings("success", newest.concluded.conclusion); // A floor above the successful run leaves only the older one, which is how // a stale run is kept from being mistaken for the run a push triggered. - const floored = try decideRun(arena, payload, ci_run_path, sha, 565); + const floored = try decideRun(arena, payload, ci_run_path, sha, 565, null); try testing.expectEqual(RunState.absent, floored); - const older = try decideRun(arena, payload, ci_run_path, sha, 561); + const older = try decideRun(arena, payload, ci_run_path, sha, 561, null); try testing.expectEqual(@as(u64, 565), older.concluded.id); // The workflow and the ref both have to match: a tag run is not a CI run. - const release = try decideRun(arena, payload, "release.yml@refs/tags/v0.0.7", sha, null); - try testing.expectEqual(@as(u64, 566), release.running); - try testing.expectEqual(RunState.absent, try decideRun(arena, payload, ci_run_path, other, 564)); - try testing.expectEqual(RunState.absent, try decideRun(arena, payload, "gates.yml@refs/heads/master", sha, null)); + const release = try decideRun(arena, payload, "release.yml@refs/tags/v0.0.7", sha, null, null); + try testing.expectEqual(@as(u64, 566), release.running.id); + try testing.expectEqual(@as(?u32, 1), release.running.attempt); + try testing.expectEqual(RunState.absent, try decideRun(arena, payload, ci_run_path, other, 564, null)); + try testing.expectEqual(RunState.absent, try decideRun(arena, payload, "gates.yml@refs/heads/master", sha, null, null)); - const failed = try decideRun(arena, payload, ci_run_path, other, null); + const failed = try decideRun(arena, payload, ci_run_path, other, null, null); try testing.expectEqualStrings("cancelled", failed.concluded.conclusion); + // Run 564 carries no `run_attempt`, and an absent attempt number reads as + // null rather than 1: it is the fact the rerun decision turns on, and it + // has to fail closed. + try testing.expectEqual(@as(?u32, null), failed.concluded.attempt); // An error body must never read as "no run yet" and time out with the wrong // diagnosis. - try testing.expectError(error.BadPayload, decideRun(arena, "{\"message\":\"token required\"}", ci_run_path, sha, null)); - try testing.expectError(error.BadPayload, decideRun(arena, "[]", ci_run_path, sha, null)); - try testing.expectError(error.BadPayload, decideRun(arena, "not json", ci_run_path, sha, null)); - try testing.expectEqual(RunState.absent, try decideRun(arena, "{\"workflow_runs\":[]}", ci_run_path, sha, null)); + try testing.expectError(error.BadPayload, decideRun(arena, "{\"message\":\"token required\"}", ci_run_path, sha, null, null)); + try testing.expectError(error.BadPayload, decideRun(arena, "[]", ci_run_path, sha, null, null)); + try testing.expectError(error.BadPayload, decideRun(arena, "not json", ci_run_path, sha, null, null)); + try testing.expectEqual(RunState.absent, try decideRun(arena, "{\"workflow_runs\":[]}", ci_run_path, sha, null, null)); try testing.expectEqual(@as(u64, 566), try highestRunId(arena, payload)); try testing.expectEqual(@as(u64, 0), try highestRunId(arena, "{\"workflow_runs\":[]}")); try testing.expectError(error.BadPayload, highestRunId(arena, "{}")); } +test "a run id narrows the match to the run a rerun replaces" { + var arena_state = std.heap.ArenaAllocator.init(testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const sha = "324704b53f180a3eccb23c4389af03b413b69488"; + const path = "release.yml@refs/tags/v0.0.16"; + const payload = + \\{"total_count":2,"workflow_runs":[ + \\ {"id":690,"path":"release.yml@refs/tags/v0.0.16","event":"push","run_attempt":1, + \\ "head_sha":"324704b53f180a3eccb23c4389af03b413b69488","status":"in_progress","conclusion":null}, + \\ {"id":683,"path":"release.yml@refs/tags/v0.0.16","event":"push","run_attempt":1, + \\ "head_sha":"324704b53f180a3eccb23c4389af03b413b69488","status":"completed","conclusion":"failure"} + \\]} + ; + + // Without the narrowing the newest run wins, which is the wrong run to + // watch after a rerun: a rerun keeps the id it replaces. + try testing.expectEqual(@as(u64, 690), (try decideRun(arena, payload, path, sha, null, null)).running.id); + const first = try decideRun(arena, payload, path, sha, null, 683); + try testing.expectEqual(@as(u64, 683), first.concluded.id); + try testing.expectEqual(RunState.absent, try decideRun(arena, payload, path, sha, null, 684)); + + // The attempt number, not the status, says whether the rerun has happened + // yet. This is the reading right after the POST: run 683 is still the + // attempt that concluded, so it is not a result. + try testing.expect(!attemptAdvanced(first, 1)); + + // A rerun that finishes between two polls is never observed running, so a + // `completed` reading on the NEW attempt has to be the result — a wait that + // required a non-completed status first would sit out its whole deadline. + const reran = + \\{"total_count":1,"workflow_runs":[ + \\ {"id":683,"path":"release.yml@refs/tags/v0.0.16","event":"push","run_attempt":2, + \\ "head_sha":"324704b53f180a3eccb23c4389af03b413b69488","status":"completed","conclusion":"success"} + \\]} + ; + const second = try decideRun(arena, reran, path, sha, null, 683); + try testing.expect(attemptAdvanced(second, 1)); + try testing.expectEqualStrings("success", second.concluded.conclusion); + + // A payload with no attempt number never advances: the wait keeps waiting + // and its deadline reports that, rather than adopting the old attempt's + // conclusion as the new one's. + const attemptless = + \\{"total_count":1,"workflow_runs":[ + \\ {"id":683,"path":"release.yml@refs/tags/v0.0.16","event":"push", + \\ "head_sha":"324704b53f180a3eccb23c4389af03b413b69488","status":"completed","conclusion":"failure"} + \\]} + ; + try testing.expect(!attemptAdvanced(try decideRun(arena, attemptless, path, sha, null, 683), 1)); + try testing.expect(!attemptAdvanced(RunState.absent, 1)); +} + +test "a release run is rerun only on its first attempt, when every failure is a gate and nothing is published" { + var arena_state = std.heap.ArenaAllocator.init(testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // Run 683, the v0.0.16 release: the guard passed, a gate failed on timing + // and publish never ran. Rerunning it by hand published the release. + const gate_failure = + \\{"state":"failure","statuses":[ + \\ {"status":"success","context":"Release / guard (push)","description":"Successful in 19s", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/900"}, + \\ {"status":"failure","context":"Gates / test (push)","description":"Failing after 3m", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/901"}, + \\ {"status":"skipped","context":"Release / publish (push)","description":"Has been skipped", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/902"} + \\]} + ; + const gates = try failingContexts(arena, gate_failure, 683, .strict); + try testing.expectEqual(RerunDecision.retryable, classifyReleaseFailure("failure", 1, gates, .absent)); + + // The forge counts the attempts, so a run somebody already reran — by hand, + // or in an earlier invocation of this tool that resumed the same tag — is + // never rerun again. A counter in this process could not see either of + // those. An unknown attempt is not read as the first one. + try testing.expectEqual(RerunDecision.terminal, classifyReleaseFailure("failure", 2, gates, .absent)); + try testing.expectEqual(RerunDecision.terminal, classifyReleaseFailure("failure", null, gates, .absent)); + // release.yml's guard clears a leftover draft and repeats, so a draft is + // still work in progress. + try testing.expectEqual(RerunDecision.retryable, classifyReleaseFailure("failure", 1, gates, .draft)); + // A published release is terminal: nothing may run against it again. + try testing.expectEqual(RerunDecision.terminal, classifyReleaseFailure("failure", 1, gates, .published)); + // A person stopped this one. Rerunning it would undo that decision. + try testing.expectEqual(RerunDecision.terminal, classifyReleaseFailure("cancelled", 1, gates, .absent)); + + // The guard is a statement about the tag, so it fails again the same way. + const guard_failure = + \\{"state":"failure","statuses":[ + \\ {"status":"failure","context":"Release / guard (push)","description":"Failing after 19s", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/900"} + \\]} + ; + try testing.expectEqual(RerunDecision.terminal, classifyReleaseFailure( + "failure", + 1, + try failingContexts(arena, guard_failure, 683, .strict), + .absent, + )); + + // Publish has already touched the release. + const publish_failure = + \\{"state":"failure","statuses":[ + \\ {"status":"success","context":"Release / guard (push)","description":"Successful in 19s", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/900"}, + \\ {"status":"failure","context":"Release / publish (push)","description":"Failing after 4m", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/902"} + \\]} + ; + try testing.expectEqual(RerunDecision.terminal, classifyReleaseFailure( + "failure", + 1, + try failingContexts(arena, publish_failure, 683, .strict), + .absent, + )); + + // A run whose statuses name no failure at all is not understood, and a + // failure this program cannot explain is not one it may retry. The second + // case is the same thing through the status list: only a skipped job. + try testing.expectEqual(RerunDecision.terminal, classifyReleaseFailure("failure", 1, &.{}, .absent)); + const nothing_failed = + \\{"state":"failure","statuses":[ + \\ {"status":"skipped","context":"Release / publish (push)","description":"Has been skipped", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/902"} + \\]} + ; + try testing.expectEqual(RerunDecision.terminal, classifyReleaseFailure( + "failure", + 1, + try failingContexts(arena, nothing_failed, 683, .strict), + .absent, + )); + + // The gate set is named two ways on this repository, and the event suffix + // belongs to neither name. + try testing.expect(isGateContext("Gates / test (push)")); + try testing.expect(isGateContext("Release / gates (push)")); + try testing.expect(isGateContext("Release / gates")); + try testing.expect(!isGateContext("Release / guard (push)")); + try testing.expect(!isGateContext("Release / publish (push)")); + try testing.expect(!isGateContext("Gatesmith / test (push)")); +} + +test "an unreadable status entry is skipped by the report and refused by the rerun" { + var arena_state = std.heap.ArenaAllocator.init(testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // One readable gate failure and one entry whose `status` is not a string. + // The tolerant parser drops the second, which leaves a list of nothing but + // gates — and that list would be rerun while a guard or publish failure sat + // unread inside the entry that was dropped. + const mixed = + \\{"state":"failure","statuses":[ + \\ {"status":"failure","context":"Gates / test (push)","description":"Failing after 3m", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/901"}, + \\ {"status":null,"context":"Release / guard (push)","description":"", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/900"} + \\]} + ; + const reported = try failingContexts(arena, mixed, 683, .tolerant); + try testing.expectEqual(@as(usize, 2), reported.len); + try testing.expectEqualStrings("Gates / test (push)", reported[0].context); + // The second entry survives with an EMPTY status, which `isFailureStatus` + // does not count as a failure — so the classification sees a list of + // nothing but gates and says retryable. That is the hole. + try testing.expectEqualStrings("", reported[1].status); + try testing.expectEqual(RerunDecision.retryable, classifyReleaseFailure("failure", 1, reported, .absent)); + + // Strictly, the same payload is not a decision anybody may make, and the + // caller turns that into a terminal failure. + try testing.expectError(error.BadPayload, failingContexts(arena, mixed, 683, .strict)); + + // A missing `context`, and an entry that cannot be attributed to any run at + // all, are the same refusal. + const no_context = + \\{"state":"failure","statuses":[ + \\ {"status":"failure","description":"Failing after 3m", + \\ "target_url":"/mokhtar/nxdns/actions/runs/683/jobs/901"} + \\]} + ; + try testing.expectError(error.BadPayload, failingContexts(arena, no_context, 683, .strict)); + const no_target = + \\{"state":"failure","statuses":[ + \\ {"status":"failure","context":"Gates / test (push)","description":"Failing after 3m"} + \\]} + ; + try testing.expectError(error.BadPayload, failingContexts(arena, no_target, 683, .strict)); + try testing.expectEqual(@as(usize, 0), (try failingContexts(arena, no_target, 683, .tolerant)).len); +} + +test "the release object says absent, draft or published" { + var arena_state = std.heap.ArenaAllocator.init(testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + try testing.expectEqual(ReleaseState.absent, try decideReleaseState( + arena, + 404, + "{\"errors\":null,\"message\":\"release does not exist\"}", + )); + try testing.expectEqual(ReleaseState.draft, try decideReleaseState( + arena, + 200, + "{\"tag_name\":\"v0.0.16\",\"draft\":true,\"prerelease\":false}", + )); + try testing.expectEqual(ReleaseState.published, try decideReleaseState( + arena, + 200, + "{\"tag_name\":\"v0.0.16\",\"draft\":false,\"prerelease\":false}", + )); + + // An outage read as "absent" would rerun a run against a published release. + try testing.expectError(error.BadPayload, decideReleaseState(arena, 500, "gateway")); + try testing.expectError(error.BadPayload, decideReleaseState(arena, 401, "{}")); + try testing.expectError(error.BadPayload, decideReleaseState(arena, 200, "{\"tag_name\":\"v0.0.16\"}")); + try testing.expectError(error.BadPayload, decideReleaseState(arena, 200, "not json")); + try testing.expectError(error.BadPayload, decideReleaseState(arena, 200, "[]")); +} + +test "an annotated tag on origin is compared by the commit it peels to" { + const listing = + "9f2b2e5f0f7b1f0d1c3a4b5c6d7e8f9012345678\trefs/tags/v0.0.16\n" ++ + "324704b53f180a3eccb23c4389af03b413b69488\trefs/tags/v0.0.16^{}\n"; + + // The tag object's own id is the hash of the tag, never the commit, so + // `lsRemoteFind` cannot answer "does this tag point at HEAD". + try testing.expectEqualStrings("9f2b2e5f0f7b1f0d1c3a4b5c6d7e8f9012345678", lsRemoteFind(listing, "refs/tags/v0.0.16").?); + try testing.expectEqualStrings("324704b53f180a3eccb23c4389af03b413b69488", lsRemotePeeled(listing, "refs/tags/v0.0.16").?); + + // A lightweight tag and a branch have no peeled line, and the plain object + // is the commit. + const light = "324704b53f180a3eccb23c4389af03b413b69488\trefs/heads/master\n"; + try testing.expectEqualStrings("324704b53f180a3eccb23c4389af03b413b69488", lsRemotePeeled(light, "refs/heads/master").?); + try testing.expect(lsRemotePeeled(listing, "refs/tags/v0.0.15") == null); + try testing.expect(lsRemotePeeled("", "refs/tags/v0.0.16") == null); +} + test "the failing jobs of a run are named from its own statuses" { var arena_state = std.heap.ArenaAllocator.init(testing.allocator); defer arena_state.deinit(); @@ -2894,16 +3683,16 @@ test "the failing jobs of a run are named from its own statuses" { \\ "target_url":"/mokhtar/nxdns/actions/runs/560/jobs/842"} \\]} ; - const failing = try failingContexts(arena, payload, 561); + const failing = try failingContexts(arena, payload, 561, .tolerant); try testing.expectEqual(@as(usize, 1), failing.len); try testing.expectEqualStrings("Release / publish (push)", failing[0].context); try testing.expectEqualStrings("failure", failing[0].status); // The same commit carries every run's contexts, so a different run's // failure is not this run's. - try testing.expectEqual(@as(usize, 1), (try failingContexts(arena, payload, 560)).len); - try testing.expectEqual(@as(usize, 0), (try failingContexts(arena, payload, 999)).len); - try testing.expectError(error.BadPayload, failingContexts(arena, "{}", 561)); + try testing.expectEqual(@as(usize, 1), (try failingContexts(arena, payload, 560, .tolerant)).len); + try testing.expectEqual(@as(usize, 0), (try failingContexts(arena, payload, 999, .tolerant)).len); + try testing.expectError(error.BadPayload, failingContexts(arena, "{}", 561, .tolerant)); } test "the tea config yields the token of the matching forge" {