upstream: one absolute per-query budget across queueing and failover

waiting for a slot now spends the query budget; truncated attempts that
expire fault the budget, not the upstream, and are never attributed.
admission sweeps in priority order before blocking. forward zones spend
read_timeout_ms once across udp, truncation and tcp. adds
nxdns_upstream_budget_exhausted_total and a 64-upstream validation limit.

src/upstream/transport.zig

@@ -1,4 +1,4 @@
-//! Shared vocabulary for every upstream client: endpoint URLs, the three
+//! Shared vocabulary for every upstream client: endpoint URLs, the four
 //! disjoint failure groups, the `Client` interface, and response validation.
 //!
 //! Everything here except the `Client` vtable is pure. `validateResponse` takes
@@ -8,8 +8,10 @@
 //!
 //! The failure classification is the reason this file exists. Health and
 //! backoff must count only what the peer did wrong: a local `OutOfMemory` says
-//! nothing about the upstream, and `error.Canceled` says nothing at all. The
-//! three error sets below are disjoint by construction and `group` switches
+//! nothing about the upstream, `error.Canceled` says nothing at all, and
+//! `error.BudgetExhausted` says the caller ran out of time before the peer was
+//! given its interval. The four error sets below are disjoint by construction
+//! and `group` switches
 //! over them exhaustively, so a new failure mode cannot silently land in the
 //! wrong bucket.
 
@@ -188,9 +190,15 @@ pub const LocalResource = error{
 
 pub const Cancellation = error{Canceled};
 
-pub const ExchangeError = PeerFault || LocalResource || Cancellation;
+/// The caller's own time ran out before any peer could be given the observation
+/// interval it was configured to get. Evidence about this process's budget, not
+/// about any endpoint, so it is never recorded against health — that is the
+/// whole reason it is not a `PeerFault`.
+pub const BudgetFault = error{BudgetExhausted};
 
-pub const Group = enum { peer_fault, local_resource, cancellation };
+pub const ExchangeError = PeerFault || LocalResource || Cancellation || BudgetFault;
+
+pub const Group = enum { peer_fault, local_resource, cancellation, budget_exhausted };
 
 /// Exhaustive switch over `ExchangeError` — no `else` arm. A new error member
 /// must break the build here, so no failure can silently land in the wrong
@@ -218,6 +226,8 @@ pub fn group(err: ExchangeError) Group {
         => .local_resource,
 
         error.Canceled => .cancellation,
+
+        error.BudgetExhausted => .budget_exhausted,
     };
 }
 
@@ -269,29 +279,29 @@ pub fn closeBlocked(io: std.Io, target: anytype) void {
     }
 }
 
-/// The payload of `f`'s return type, which `raceWithin` requires to be
+/// The payload of `f`'s return type, which the race harness requires to be
 /// `ExchangeError!T`. A raced function with any other error set would let a
 /// failure reach the pool without passing through `group`.
 fn RacedPayload(comptime f: anytype) type {
     const info = @typeInfo(@TypeOf(f));
-    if (info != .@"fn") @compileError("raceWithin needs a function, found " ++ @typeName(@TypeOf(f)));
+    if (info != .@"fn") @compileError("the race harness needs a function, found " ++ @typeName(@TypeOf(f)));
     const Return = info.@"fn".return_type orelse
-        @compileError("raceWithin needs a function with a concrete return type");
+        @compileError("the race harness needs a function with a concrete return type");
     const union_info = switch (@typeInfo(Return)) {
         .error_union => |u| u,
-        else => @compileError("raceWithin needs `ExchangeError!T`, found " ++ @typeName(Return)),
+        else => @compileError("the race harness needs `ExchangeError!T`, found " ++ @typeName(Return)),
     };
     if (union_info.error_set != ExchangeError)
-        @compileError("raceWithin needs `ExchangeError!T`, found " ++ @typeName(Return));
+        @compileError("the race harness needs `ExchangeError!T`, found " ++ @typeName(Return));
     return union_info.payload;
 }
 
 /// Runs `f(args...)` raced against `budget`, and cancels the loser.
 ///
 /// No stream read or write in 0.16.0 takes a timeout, so a deadline is a second
-/// task rather than a socket option. This is the one copy of that harness: the
-/// pool races an attempt and its whole failover loop through it, and the
-/// forward client races its TCP exchange.
+/// task rather than a socket option. `raceUntilTagged` is the one copy of that
+/// harness; this is the untagged wrapper for callers that own no deadline and
+/// only need "a bound on this one operation".
 ///
 /// `error.Timeout` means the budget won. A canceled sleep means the whole task
 /// is being torn down rather than the budget running out, so it stays
@@ -303,33 +313,69 @@ pub fn raceWithin(
     comptime f: anytype,
     args: anytype,
 ) ExchangeError!RacedPayload(f) {
-    const Outcome = union(enum) {
+    var outcome: RaceOutcome = .completed;
+    return raceUntilTagged(io, .fromNow(io, budget), &outcome, f, args);
+}
+
+/// Which side of the race ended the call.
+///
+/// `completed` means the raced operation itself returned — including when what
+/// it returned is `error.Timeout`, which is then the peer's own timeout and
+/// real evidence about that peer. `expired` means the caller's clock ran out
+/// with the operation still in flight, which is evidence about the budget only.
+pub const RaceOutcome = enum { completed, expired };
+
+/// `raceWithin` with the two timer origins told apart, and with an ABSOLUTE
+/// expiry rather than a duration.
+///
+/// The timestamp is the point of the whole function. A duration recomputed from
+/// a deadline and then slept re-anchors at "now", so every re-race drifts a
+/// little past the caller's real deadline and a truncated attempt is then
+/// indistinguishable from a full one. The caller that owns the deadline
+/// computes the instant once and passes it here.
+///
+/// `outcome` is written before this returns on both racing paths. It is left
+/// untouched when the race cannot start at all (`error.SystemResources`) or
+/// when the whole task is being canceled, since neither is an observation about
+/// this budget; callers initialize it to the value they want in those cases.
+pub fn raceUntilTagged(
+    io: std.Io,
+    expiry_at: std.Io.Clock.Timestamp,
+    outcome: *RaceOutcome,
+    comptime f: anytype,
+    args: anytype,
+) ExchangeError!RacedPayload(f) {
+    const Slot = union(enum) {
         raced: ExchangeError!RacedPayload(f),
         expiry: std.Io.Cancelable!void,
     };
 
-    var outcomes: [2]Outcome = undefined;
-    var race: std.Io.Select(Outcome) = .init(io, &outcomes);
+    var slots: [2]Slot = undefined;
+    var race: std.Io.Select(Slot) = .init(io, &slots);
     defer race.cancelDiscard();
 
     race.concurrent(.raced, f, args) catch |err| switch (err) {
         error.ConcurrencyUnavailable => return error.SystemResources,
     };
-    race.concurrent(.expiry, expire, .{ io, budget }) catch |err| switch (err) {
+    race.concurrent(.expiry, expire, .{ io, expiry_at }) catch |err| switch (err) {
         error.ConcurrencyUnavailable => return error.SystemResources,
     };
 
     switch (try race.await()) {
-        .raced => |result| return result,
+        .raced => |result| {
+            outcome.* = .completed;
+            return result;
+        },
         .expiry => |result| {
             try result;
+            outcome.* = .expired;
             return error.Timeout;
         },
     }
 }
 
-fn expire(io: std.Io, budget: std.Io.Clock.Duration) std.Io.Cancelable!void {
-    return budget.sleep(io);
+fn expire(io: std.Io, expiry_at: std.Io.Clock.Timestamp) std.Io.Cancelable!void {
+    return expiry_at.wait(io);
 }
 
 /// A thing that sends one DNS message and returns one validated DNS message.
@@ -347,13 +393,21 @@ pub const Client = struct {
     /// Returns a prefix of `response_buf`. The returned message has already
     /// passed `validateResponse` against `query`.
     ///
-    /// `selected` names the resolver the exchange used. An implementation
-    /// writes it *before* each attempt, never after, so a failed exchange still
-    /// names the last resolver it tried — a SERVFAIL row without its resolver
-    /// explains nothing. The slice must outlive the call; every implementation
-    /// borrows storage it owns for at least the query's duration. Callers
-    /// initialize it to null: a `null` after the call means no resolver was
-    /// reached at all.
+    /// `selected` names the resolver the exchange used. A single-endpoint
+    /// implementation (DoH, DoT, the forward client, test fakes) may write it
+    /// *before* each attempt: it has one resolver and records no health, so
+    /// "the one I tried" is an honest answer even for a failure, and a SERVFAIL
+    /// row without its resolver explains nothing.
+    ///
+    /// `Pool` is stricter, and documents the rule on `Pool.exchange`: it names
+    /// only endpoints whose outcome it recorded, so a query that ran out of
+    /// budget blames nobody and may leave this `null`. Both are within this
+    /// contract — the guarantee here is that a non-null value names a resolver
+    /// this exchange really used, never that a failure leaves one behind.
+    ///
+    /// The slice must outlive the call; every implementation borrows storage it
+    /// owns for at least the query's duration. Callers initialize it to null: a
+    /// `null` after the call means no resolver is being reported.
     pub fn exchange(
         self: Client,
         io: std.Io,
@@ -530,8 +584,8 @@ test "parse rejects a fragment" {
     try testing.expectError(error.BadUrl, Endpoint.parse("tls://dns.google#f"));
 }
 
-test "the three error groups are disjoint" {
-    const sets = .{ PeerFault, LocalResource, Cancellation };
+test "the four error groups are disjoint" {
+    const sets = .{ PeerFault, LocalResource, Cancellation, BudgetFault };
     inline for (sets, 0..) |a, i| {
         inline for (sets, 0..) |b, j| {
             if (i >= j) continue;
@@ -548,7 +602,8 @@ test "the three error groups are disjoint" {
     // member added to two sets at once cannot pass unnoticed.
     const total = @typeInfo(PeerFault).error_set.?.len +
         @typeInfo(LocalResource).error_set.?.len +
-        @typeInfo(Cancellation).error_set.?.len;
+        @typeInfo(Cancellation).error_set.?.len +
+        @typeInfo(BudgetFault).error_set.?.len;
     try testing.expectEqual(total, @typeInfo(ExchangeError).error_set.?.len);
 }
 
@@ -558,6 +613,7 @@ test "group classifies each member" {
     try testing.expectEqual(Group.local_resource, group(error.OutOfMemory));
     try testing.expectEqual(Group.local_resource, group(error.BufferTooSmall));
     try testing.expectEqual(Group.cancellation, group(error.Canceled));
+    try testing.expectEqual(Group.budget_exhausted, group(error.BudgetExhausted));
 }
 
 test "mapLocal folds only local and cancellation errors" {
@@ -641,6 +697,72 @@ test "raceWithin passes the raced task's own failure through" {
     );
 }
 
+test "raceUntilTagged tells a leaf Timeout apart from an expiry" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    // The leaf's own timeout: it returned, so the peer really did time out and
+    // the outcome is `completed` even though the error is the same one an
+    // expiry produces.
+    var outcome: RaceOutcome = .expired;
+    const far: std.Io.Clock.Timestamp = .fromNow(io, .{ .raw = .fromSeconds(30), .clock = .awake });
+    try testing.expectError(
+        error.Timeout,
+        raceUntilTagged(io, far, &outcome, racedReply, .{
+            io, 0, @as(ExchangeError!usize, error.Timeout),
+        }),
+    );
+    try testing.expectEqual(RaceOutcome.completed, outcome);
+
+    // The expiry side, distinguishable only through the tag.
+    outcome = .completed;
+    const soon: std.Io.Clock.Timestamp = .fromNow(io, .{ .raw = .fromMilliseconds(20), .clock = .awake });
+    try testing.expectError(
+        error.Timeout,
+        raceUntilTagged(io, soon, &outcome, racedReply, .{
+            io, 30_000, @as(ExchangeError!usize, 7),
+        }),
+    );
+    try testing.expectEqual(RaceOutcome.expired, outcome);
+}
+
+test "raceUntilTagged tags a successful completion" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var outcome: RaceOutcome = .expired;
+    const far: std.Io.Clock.Timestamp = .fromNow(io, .{ .raw = .fromSeconds(30), .clock = .awake });
+    const len = try raceUntilTagged(io, far, &outcome, racedReply, .{
+        io, 0, @as(ExchangeError!usize, 7),
+    });
+    try testing.expectEqual(@as(usize, 7), len);
+    try testing.expectEqual(RaceOutcome.completed, outcome);
+}
+
+test "raceUntilTagged honours an expiry already in the past" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    // A deadline the caller has already spent: the expiry wins immediately
+    // rather than re-anchoring a duration at "now" and granting a fresh budget.
+    const past = std.Io.Clock.Timestamp.now(io, .awake)
+        .subDuration(.{ .raw = .fromSeconds(1), .clock = .awake });
+    var outcome: RaceOutcome = .completed;
+    const started = std.Io.Clock.awake.now(io);
+    try testing.expectError(
+        error.Timeout,
+        raceUntilTagged(io, past, &outcome, racedReply, .{
+            io, 30_000, @as(ExchangeError!usize, 7),
+        }),
+    );
+    try testing.expectEqual(RaceOutcome.expired, outcome);
+    const elapsed_ns = std.Io.Clock.awake.now(io).nanoseconds - started.nanoseconds;
+    try testing.expect(elapsed_ns < @as(i96, 5) * std.time.ns_per_s);
+}
+
 test "closeBlocked closes a target of either close shape" {
     // The two shapes the transports use: a socket or a plain stream, which
     // closes through the `Io`, and a `TlsStream`, which owns the one it was