upstream: a diagnostics episode follows health, and a peer fault carries its cause

src/upstream/transport.zig

@@ -190,6 +190,11 @@ pub const LocalResource = error{
 
 pub const Cancellation = error{Canceled};
 
+/// What a leaf client may fail with. A leaf never *errors* on a peer fault: it
+/// returns one as an `Outcome.fault` value, because the pool needs the concrete
+/// cause and an error cannot carry one.
+pub const LeafError = LocalResource || Cancellation;
+
 /// The caller's own time ran out before any peer could be given the observation
 /// interval it was configured to get. Evidence about this process's budget, not
 /// about any endpoint, so it is never recorded against health — that is the
@@ -238,7 +243,7 @@ pub fn group(err: ExchangeError) Group {
 /// This is the only place a foreign error set is folded in. Everywhere else
 /// the call site names the peer fault it means, because the call site is what
 /// knows whether it was connecting, sending or receiving.
-pub fn mapLocal(err: anyerror) ?ExchangeError {
+pub fn mapLocal(err: anyerror) ?LeafError {
     return switch (err) {
         error.OutOfMemory => error.OutOfMemory,
         error.SystemResources => error.SystemResources,
@@ -279,21 +284,37 @@ pub fn closeBlocked(io: std.Io, target: anytype) void {
     }
 }
 
-/// The payload of `f`'s return type, which the race harness requires to be
-/// `ExchangeError!T`. A raced function with any other error set would let a
-/// failure reach the pool without passing through `group`.
-fn RacedPayload(comptime f: anytype) type {
+/// The error union `f` returns, which the race harness requires it to have.
+fn racedUnion(comptime f: anytype) std.builtin.Type.ErrorUnion {
     const info = @typeInfo(@TypeOf(f));
     if (info != .@"fn") @compileError("the race harness needs a function, found " ++ @typeName(@TypeOf(f)));
     const Return = info.@"fn".return_type orelse
         @compileError("the race harness needs a function with a concrete return type");
-    const union_info = switch (@typeInfo(Return)) {
+    return switch (@typeInfo(Return)) {
         .error_union => |u| u,
-        else => @compileError("the race harness needs `ExchangeError!T`, found " ++ @typeName(Return)),
+        else => @compileError("the race harness needs `E!T`, found " ++ @typeName(Return)),
     };
-    if (union_info.error_set != ExchangeError)
-        @compileError("the race harness needs `ExchangeError!T`, found " ++ @typeName(Return));
-    return union_info.payload;
+}
+
+/// The payload of `f`'s return type.
+fn RacedPayload(comptime f: anytype) type {
+    return racedUnion(f).payload;
+}
+
+/// `f`'s own error set.
+fn RacedError(comptime f: anytype) type {
+    return racedUnion(f).error_set;
+}
+
+/// What racing `f` can fail with: `f`'s own errors, plus the three the harness
+/// itself produces — the expiry, a backend that cannot start a second task, and
+/// the whole task being torn down.
+///
+/// Derived rather than fixed at `ExchangeError`, so a caller's narrow error set
+/// survives the race. That is what lets the pool prove at the type level that a
+/// leaf cannot hand it a `PeerFault`, instead of asserting it.
+pub fn RaceError(comptime f: anytype) type {
+    return RacedError(f) || error{ Timeout, SystemResources, Canceled };
 }
 
 /// Runs `f(args...)` raced against `budget`, and cancels the loser.
@@ -312,7 +333,7 @@ pub fn raceWithin(
     budget: std.Io.Clock.Duration,
     comptime f: anytype,
     args: anytype,
-) ExchangeError!RacedPayload(f) {
+) RaceError(f)!RacedPayload(f) {
     var outcome: RaceOutcome = .completed;
     return raceUntilTagged(io, .fromNow(io, budget), &outcome, f, args);
 }
@@ -344,9 +365,9 @@ pub fn raceUntilTagged(
     outcome: *RaceOutcome,
     comptime f: anytype,
     args: anytype,
-) ExchangeError!RacedPayload(f) {
+) RaceError(f)!RacedPayload(f) {
     const Slot = union(enum) {
-        raced: ExchangeError!RacedPayload(f),
+        raced: RacedError(f)!RacedPayload(f),
         expiry: std.Io.Cancelable!void,
     };
 
@@ -378,8 +399,12 @@ fn expire(io: std.Io, expiry_at: std.Io.Clock.Timestamp) std.Io.Cancelable!void
     return expiry_at.wait(io);
 }
 
-/// A thing that sends one DNS message and returns one validated DNS message.
-/// Implemented by DohClient, DotClient, Pool, and test fakes.
+/// A thing that sends one DNS message and returns one validated DNS message,
+/// with a peer fault already reduced to an error.
+///
+/// Implemented by `Pool`, the forward client, and the fakes that stand in for
+/// either. The leaf clients are on the other side of the pool and implement
+/// `Leaf` instead, which keeps the fault as a value.
 pub const Client = struct {
     ptr: *anyopaque,
     exchangeFn: *const fn (
@@ -394,7 +419,8 @@ pub const Client = struct {
     /// passed `validateResponse` against `query`.
     ///
     /// `selected` names the resolver the exchange used. A single-endpoint
-    /// implementation (DoH, DoT, the forward client, test fakes) may write it
+    /// implementation — the forward client and the test fakes; the DoH and DoT
+    /// clients are `Leaf`s and a `Pool` carries their answer here — may write it
     /// *before* each attempt: it has one resolver and records no health, so
     /// "the one I tried" is an honest answer even for a failure, and a SERVFAIL
     /// row without its resolver explains nothing.
@@ -419,6 +445,70 @@ pub const Client = struct {
     }
 };
 
+/// One peer fault as a value: the taxonomy `PeerFault` names, plus the concrete
+/// error that produced it.
+///
+/// The classification is what health and backoff count; the cause is what tells
+/// an operator which failure it was. `SendFailed` alone cannot separate a peer
+/// that reset the connection from one whose TLS record was rejected, and the
+/// leaf that unwrapped the cause is the only place that still holds it.
+///
+/// There is no phase field: the taxonomy already names the phase for every kind
+/// that has one, and `TlsFailed` cannot say where it failed.
+pub const Fault = struct {
+    kind: PeerFault,
+    cause: anyerror,
+
+    /// The one text every surface prints. `<Kind> (cause <Cause>)`.
+    pub fn format(self: Fault, w: *std.Io.Writer) std.Io.Writer.Error!void {
+        try w.print("{t} (cause {s})", .{ self.kind, @errorName(self.cause) });
+    }
+};
+
+/// What one exchange against one endpoint produced.
+pub const Outcome = union(enum) {
+    /// A prefix of the caller's `response_buf`, already validated against the
+    /// query.
+    reply: []u8,
+    fault: Fault,
+};
+
+/// A client of exactly one endpoint: DoH, DoT, and the pool's test fakes.
+///
+/// Separate from `Client` because the two answer different questions. A `Leaf`
+/// reports what the peer did, faults included, and leaves every judgement to
+/// its caller. A `Client` is the resolver the handler asks for an answer, and a
+/// fault has already become an error by the time it is reached.
+///
+/// No `selected` out-parameter: the pool discards a leaf's own identity anyway,
+/// since the entry's endpoint is the pool's naming of the same resolver.
+pub const Leaf = struct {
+    ptr: *anyopaque,
+    exchangeFn: *const fn (
+        ptr: *anyopaque,
+        io: std.Io,
+        query: []const u8,
+        response_buf: []u8,
+    ) LeafError!Outcome,
+
+    pub fn exchange(
+        self: Leaf,
+        io: std.Io,
+        query: []const u8,
+        response_buf: []u8,
+    ) LeafError!Outcome {
+        return self.exchangeFn(self.ptr, io, query, response_buf);
+    }
+};
+
+/// The fault a call site's phase means, unless `err` is one this process owns.
+/// The leaf counterpart of `mapPhase`: same rule, but the peer case comes back
+/// as a value carrying the cause instead of as a bare error.
+pub fn faultOrLocal(err: anyerror, phase: PeerFault) LeafError!Fault {
+    if (mapLocal(err)) |local| return local;
+    return .{ .kind = phase, .cause = err };
+}
+
 /// The unwrap helpers below turn the single collapsed error `std.http.Client`
 /// reports into the concrete cause it stashed. Every HTTP caller in this tree
 /// uses them: the DoH client classifies by the unwrapped cause, the blocklist
@@ -743,14 +833,16 @@ test "raceWithin passes the raced task's own failure through" {
     );
 }
 
-test "raceUntilTagged tells a leaf Timeout apart from an expiry" {
+test "raceUntilTagged tells a raced Timeout apart from an expiry" {
     var threaded: std.Io.Threaded = .init(testing.allocator, .{});
     defer threaded.deinit();
     const io = threaded.io();
 
-    // The leaf's own timeout: it returned, so the peer really did time out and
-    // the outcome is `completed` even though the error is the same one an
-    // expiry produces.
+    // A raced function that returned `error.Timeout` of its own: it completed,
+    // so the tag says `completed` even though the error is the one an expiry
+    // produces. No leaf does this — a leaf's own timeout is a `Fault` — but the
+    // harness serves callers with any error set, and the tag is what separates
+    // the two for every one of them.
     var outcome: RaceOutcome = .expired;
     const far: std.Io.Clock.Timestamp = .fromNow(io, .{ .raw = .fromSeconds(30), .clock = .awake });
     try testing.expectError(