upstream: a diagnostics episode follows health, and a peer fault carries its cause

2026-09-12 20:24:49 +02:00
parent 6e9a36903e
commit 08cdf86ecd
20 changed files with 1797 additions and 426 deletions
@@ -557,6 +557,13 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {

    var upstreams: upstream_owner.Owner = .init(upstream_generation);
    defer upstreams.deinit(io);
+    upstreams.diagnostics = event_store;
+    // The store outlived the process that wrote it, so every `upstream.exchange`
+    // episode in it describes a pool that no longer exists. This closes the ones
+    // no enabled upstream of this boot can justify, and projects the health of
+    // the ones that remain — a fresh pool is all clear, so a warning that
+    // survives this boot is one this process opened.
+    upstreams.reconcileDiagnostics(io, upstream_generation);

    // -----------------------------------------------------------------------
    // per-query state
@@ -978,14 +978,14 @@ fn probeUpstreams(r: Runner, cfg: model.Config) !usize {
        // what keeps the close off `dot` while it is still undefined.
        var dot_wired = false;
        defer if (dot_wired) dot.close(r.io);
-        const client: transport.Client = switch (endpoint.scheme) {
+        const client: transport.Leaf = switch (endpoint.scheme) {
            .doh => doh: {
                doh = doh_client.DohClient.init(&http, endpoint, &request_buf, &transfer_buf) catch {
                    try r.out.print("FAIL upstreams[{d}] {f}: not a usable DoH url\n", .{ i, safe_url.redactQuoted(server.url) });
                    failures += 1;
                    continue;
                };
-                break :doh doh.client();
+                break :doh doh.leaf();
            },
            .dot => dot: {
                dot = dot_client.DotClient.init(endpoint, server.tls_name, r.gpa, &bundle, &bundle_lock, null, .{
@@ -995,7 +995,7 @@ fn probeUpstreams(r: Runner, cfg: model.Config) !usize {
                    .stream_write = tls_buffers[3 * chunk ..],
                });
                dot_wired = true;
-                break :dot dot.client();
+                break :dot dot.leaf();
            },
        };

@@ -1792,8 +1792,8 @@ test "a failed DoT probe reaches close through the per-iteration defer without a

    try testing.expectEqual(@as(usize, 2), try probeUpstreams(r, cfg));
    try testing.expectEqualStrings(
-        "FAIL upstreams[0] 'tls://dns.example:853': ConnectFailed\n" ++
-            "FAIL upstreams[1] 'tls://other.example:853': ConnectFailed\n",
+        "FAIL upstreams[0] 'tls://dns.example:853': ConnectFailed (cause ConnectFailed)\n" ++
+            "FAIL upstreams[1] 'tls://other.example:853': ConnectFailed (cause ConnectFailed)\n",
        captured.out.written(),
    );
 }
@@ -93,15 +93,21 @@ fn queryWithId(buf: *[query_bytes.len]u8, id: u16) []const u8 {
 /// Echoes the question and appends one A record. This is the smallest thing a
 /// real upstream could return that the handler forwards unchanged, so the
 /// assertions below check bytes that travelled the whole path.
-fn answerQuery(query: []const u8, response_buf: []u8) transport.ExchangeError![]u8 {
-    const request = packet.parse(query) catch return error.BadResponse;
-    const q = packet.firstQuestion(request) orelse return error.BadResponse;
+fn answerQuery(query: []const u8, response_buf: []u8) transport.Outcome {
+    const request = packet.parse(query) catch return peerFault(error.BadResponse);
+    const q = packet.firstQuestion(request) orelse return peerFault(error.BadResponse);

    var b = packet.ResponseBuilder.init(response_buf, request.header, q) catch
-        return error.ResponseTooLarge;
+        return peerFault(error.ResponseTooLarge);
    b.addAnswer(q.name, .a, .in, answer_ttl, &answer_rdata) catch
-        return error.ResponseTooLarge;
-    return b.finish();
+        return peerFault(error.ResponseTooLarge);
+    return .{ .reply = b.finish() };
+}
+
+/// A fault whose cause is its own classification: these fixtures fail on
+/// purpose and have no concrete cause behind the classification.
+fn peerFault(kind: transport.PeerFault) transport.Outcome {
+    return .{ .fault = .{ .kind = kind, .cause = kind } };
 }

 /// The healthy upstream. `calls` is atomic because the listener tasks run on
@@ -114,16 +120,14 @@ const GoodUpstream = struct {
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
-        selected: *?[]const u8,
-    ) transport.ExchangeError![]u8 {
+    ) transport.LeafError!transport.Outcome {
        _ = io;
-        selected.* = "fake://good-upstream";
        const self: *GoodUpstream = @ptrCast(@alignCast(ptr));
        _ = self.calls.fetchAdd(1, .monotonic);
        return answerQuery(query, response_buf);
    }

-    fn client(self: *GoodUpstream) transport.Client {
+    fn leaf(self: *GoodUpstream) transport.Leaf {
        return .{ .ptr = self, .exchangeFn = exchangeFn };
    }
 };
@@ -140,17 +144,15 @@ const FaultyUpstream = struct {
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
-        selected: *?[]const u8,
-    ) transport.ExchangeError![]u8 {
+    ) transport.LeafError!transport.Outcome {
        _ = io;
-        selected.* = "fake://faulty-upstream";
        const self: *FaultyUpstream = @ptrCast(@alignCast(ptr));
        const seen = self.calls.fetchAdd(1, .monotonic);
-        if (seen < self.fail_first) return self.fault;
+        if (seen < self.fail_first) return peerFault(self.fault);
        return answerQuery(query, response_buf);
    }

-    fn client(self: *FaultyUpstream) transport.Client {
+    fn leaf(self: *FaultyUpstream) transport.Leaf {
        return .{ .ptr = self, .exchangeFn = exchangeFn };
    }
 };
@@ -166,10 +168,10 @@ const EntryStorage = struct {
    fn entry(
        self: *EntryStorage,
        url: []const u8,
-        upstream_client: transport.Client,
+        upstream_leaf: transport.Leaf,
        priority: i32,
    ) pool.Entry {
-        self.slots[0] = .{ .client = upstream_client };
+        self.slots[0] = .{ .client = upstream_leaf };
        return .{
            .endpoint = transport.Endpoint.parse(url) catch unreachable,
            .slots = &self.slots,
@@ -270,8 +272,8 @@ test "the whole resolver answers over udp and tcp and fails over to a healthy up
    var bad_storage: EntryStorage = .{};
    var good_storage: EntryStorage = .{};
    var entries = [_]pool.Entry{
-        bad_storage.entry("https://bad.example/dns-query", bad.client(), 10),
-        good_storage.entry("tls://good.example", good.client(), 20),
+        bad_storage.entry("https://bad.example/dns-query", bad.leaf(), 10),
+        good_storage.entry("tls://good.example", good.leaf(), 20),
    };
    var upstreams: pool.Pool = .init(&entries, test_cfg, pool_timeouts, 1);

@@ -322,6 +322,74 @@ pub const Store = struct {
        if (!self.active.put(code, digest, id)) self.untracked_active_count += 1;
    }

+    /// Opens the episode of `subject_key` when none is open, and restates the
+    /// severity and the detail of the one that is.
+    ///
+    /// The projection counterpart of `report`. A reconciliation asserts the
+    /// state an endpoint is in now; it is not a new observation, so it must not
+    /// raise `occurrences` or move `last_seen`. It does own the text: a retired
+    /// generation's late report can leave a stale cause on a card the live
+    /// generation still holds open, and this is what restores the true one.
+    /// Only a recorded failure calls `report`.
+    ///
+    /// The open check is a statement rather than a mirror lookup: the mirror is
+    /// a hint that can point at a row that is gone or already closed, and
+    /// `report` recovers from that through the touch it was making anyway.
+    /// There is no write here to learn it from, so this asks. That costs one
+    /// SELECT per reconciled subject, on a path that runs at boot and at a
+    /// generation retirement.
+    pub fn ensureOpen(
+        self: *Store,
+        io: std.Io,
+        now_s: i64,
+        code: Code,
+        subject_key: []const u8,
+        subject_label: []const u8,
+        severity: Severity,
+        detail: []const u8,
+    ) void {
+        var key_buf: [max_subject_key_len]u8 = undefined;
+        const key = canonicalKey(subject_key, &key_buf);
+        const label = truncate(subject_label, max_subject_label_len);
+        const text = truncate(detail, max_detail_len);
+        const digest = digestOf(key);
+
+        self.mutex.lockUncancelable(io);
+        defer self.mutex.unlock(io);
+
+        self.count();
+        const existing = events_repo.selectActiveId(self.database, wire(code), key) catch |err|
+            return self.recordFailure(err);
+
+        // The read is not what clears the write latch: a `resolveExcept` that
+        // failed a moment ago is still the last word on whether this store can
+        // write, and only a write of our own can answer that.
+        if (existing) |id| {
+            self.count();
+            _ = events_repo.restateActive(self.database, id, severity.text(), text) catch |err|
+                return self.recordFailure(err);
+            return self.recordSuccess();
+        }
+
+        // Nothing is open, so a mirror entry claiming otherwise is stale and
+        // would make the insert below look like a collision.
+        if (self.active.find(code, digest)) |entry| self.active.remove(entry);
+
+        self.count();
+        const id = events_repo.insertActive(
+            self.database,
+            now_s,
+            wire(code),
+            key,
+            label,
+            severity.text(),
+            text,
+        ) catch |err| return self.recordFailure(err);
+        self.recordSuccess();
+
+        if (!self.active.put(code, digest, id)) self.untracked_active_count += 1;
+    }
+
    /// Records that `subject_key` is working again, closing its episode if one
    /// is open.
    ///
@@ -930,6 +998,38 @@ test "resolving a subject with nothing open executes no SQL at all" {
    try testing.expectEqual(after_resolve, store.statements);
 }

+test "ensureOpen opens a missing episode and restates an open one without counting it" {
+    var fx: Fixture = .{};
+    try fx.init(1000);
+    defer fx.deinit();
+    const store = &fx.store;
+
+    // The reconciliation path: it must be able to reassert a subject that is
+    // still failing without inventing an occurrence no exchange produced.
+    store.ensureOpen(fx.io, 1000, .upstream_exchange, "https://a.example", "a.example", .warning, "Timeout (cause Timeout)");
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"));
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT occurrences FROM operational_events"));
+    try testing.expectEqual(@as(i64, 1000), try fx.count("SELECT last_seen FROM operational_events"));
+
+    // The second call is a reconciliation of a card that is already open: the
+    // text and the severity are the reconciler's to state, and the counters
+    // belong to the exchanges that actually failed.
+    store.ensureOpen(fx.io, 1500, .upstream_exchange, "https://a.example", "a.example", .@"error", "SendFailed (cause BrokenPipe)");
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT occurrences FROM operational_events"));
+    try testing.expectEqual(@as(i64, 1000), try fx.count("SELECT first_seen FROM operational_events"));
+    try testing.expectEqual(@as(i64, 1000), try fx.count("SELECT last_seen FROM operational_events"));
+    try testing.expectEqualStrings("error", try fx.text("SELECT severity FROM operational_events"));
+    try testing.expectEqualStrings("SendFailed (cause BrokenPipe)", try fx.text("SELECT detail FROM operational_events"));
+
+    // And once the episode is resolved it opens a second one, like any other
+    // entry point.
+    store.resolve(fx.io, 1600, .upstream_exchange, "https://a.example");
+    store.ensureOpen(fx.io, 1700, .upstream_exchange, "https://a.example", "a.example", .warning, "Timeout (cause Timeout)");
+    try testing.expectEqual(@as(i64, 2), try fx.count("SELECT count(*) FROM operational_events"));
+    try testing.expectEqual(@as(i64, 1700), try fx.count("SELECT first_seen FROM operational_events WHERE id = 2"));
+}
+
 test "a one-shot event inserts already resolved and never enters the mirror" {
    var fx: Fixture = .{};
    try fx.init(1000);
@@ -135,6 +135,35 @@ pub fn touchActive(
    return database.changes() != 0;
 }

+const restate_sql =
+    \\UPDATE operational_events
+    \\   SET detail = ?2,
+    \\       severity = ?3
+    \\ WHERE id = ?1 AND resolved_at IS NULL
+;
+
+/// Restates what an open episode says without claiming it happened again:
+/// `occurrences`, `first_seen` and `last_seen` keep the values the real
+/// failures wrote. The severity `CASE` of `touch_sql` is deliberately absent —
+/// a reconciliation asserts the current state of the subject, so it must be
+/// able to lower a severity a retired generation's late report raised.
+///
+/// False means the row was not there or was already resolved.
+pub fn restateActive(
+    database: *db.Db,
+    id: i64,
+    severity: []const u8,
+    detail: []const u8,
+) db.Error!bool {
+    var stmt = try database.prepare(restate_sql);
+    defer stmt.deinit();
+    try stmt.bindInt(1, id);
+    try stmt.bindText(2, detail);
+    try stmt.bindText(3, severity);
+    try stmt.exec();
+    return database.changes() != 0;
+}
+
 const resolve_by_id_sql =
    "UPDATE operational_events SET resolved_at = ?2 WHERE id = ?1 AND resolved_at IS NULL";

@@ -21,6 +21,7 @@ comptime {
    _ = @import("upstream/health.zig");
    _ = @import("upstream/doh_client.zig");
    _ = @import("upstream/doh_client_live_test.zig");
+    _ = @import("upstream/doh_client_integration_test.zig");
    _ = @import("upstream/pool.zig");
    _ = @import("upstream/owner.zig");
    _ = @import("upstream/dot_client.zig");
@@ -66,7 +66,7 @@ pub const DohClient = struct {
        };
    }

-    pub fn client(self: *DohClient) transport.Client {
+    pub fn leaf(self: *DohClient) transport.Leaf {
        return .{ .ptr = self, .exchangeFn = exchangeFn };
    }

@@ -75,12 +75,8 @@ pub const DohClient = struct {
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
-        selected: *?[]const u8,
-    ) transport.ExchangeError![]u8 {
+    ) transport.LeafError!transport.Outcome {
        const self: *DohClient = @ptrCast(@alignCast(ptr));
-        // The endpoint outlives the client, so the borrow is safe for the whole
-        // query. Set before the attempt: a failure names this resolver too.
-        selected.* = self.endpoint.url;
        return self.exchange(io, query, response_buf);
    }

@@ -96,7 +92,7 @@ pub const DohClient = struct {
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
-    ) transport.ExchangeError![]u8 {
+    ) transport.LeafError!transport.Outcome {
        // `std.http.Client` carries the `std.Io` it was constructed with and
        // takes none per request, so the interface's `io` is unused here. It
        // stays in the signature because DoT and the pool need it.
@@ -117,22 +113,22 @@ pub const DohClient = struct {
            // `Request.Headers` has no `accept` field, so this one goes in by
            // hand.
            .extra_headers = &.{.{ .name = "accept", .value = media_type }},
-        }) catch |err| return mapError(err, .connect);
+        }) catch |err| return fault(err, .connect);
        defer req.deinit();

        req.sendBodyComplete(self.request_buf[0..query.len]) catch |err|
-            return mapError(sendCause(&req, err), .send);
+            return fault(sendCause(&req, err), .send);

        // An empty redirect buffer is legal under `.not_allowed`: a redirect
        // is an error before the location is ever read.
-        var resp = req.receiveHead(&.{}) catch |err| return mapError(headCause(&req, err), .receive);
+        var resp = req.receiveHead(&.{}) catch |err| return fault(headCause(&req, err), .receive);

-        if (resp.head.status != .ok) return error.HttpStatus;
+        if (resp.head.status != .ok) return peerFault(error.HttpStatus, error.HttpStatus);
        // `head.content_type` points into memory that `resp.reader` invalidates,
        // so the check happens before the body stream starts.
-        if (!contentTypeOk(resp.head.content_type)) return error.HttpContentType;
+        if (!contentTypeOk(resp.head.content_type)) return peerFault(error.HttpContentType, error.HttpContentType);
        if (resp.head.content_length) |declared| {
-            if (declared > response_buf.len) return error.ResponseTooLarge;
+            if (declared > response_buf.len) return peerFault(error.ResponseTooLarge, error.ResponseTooLarge);
        }

        const body = resp.reader(self.transfer_buf);
@@ -140,7 +136,7 @@ pub const DohClient = struct {
        var ended = false;
        while (len < response_buf.len) {
            const n = body.readSliceShort(response_buf[len..]) catch |err|
-                return mapError(bodyCause(&resp, err), .receive);
+                return fault(bodyCause(&resp, err), .receive);
            len += n;
            if (n == 0) {
                ended = true;
@@ -152,12 +148,13 @@ pub const DohClient = struct {
            // that fits from one that was cut off.
            var probe: [1]u8 = undefined;
            const n = body.readSliceShort(&probe) catch |err|
-                return mapError(bodyCause(&resp, err), .receive);
-            if (n != 0) return error.ResponseTooLarge;
+                return fault(bodyCause(&resp, err), .receive);
+            if (n != 0) return peerFault(error.ResponseTooLarge, error.ResponseTooLarge);
        }

-        try transport.validateResponse(query, response_buf[0..len]);
-        return response_buf[0..len];
+        transport.validateResponse(query, response_buf[0..len]) catch |err|
+            return peerFault(err, err);
+        return .{ .reply = response_buf[0..len] };
    }
 };

@@ -177,8 +174,18 @@ const Phase = enum { connect, send, receive };
 ///   `Connection.getReadError` (Client.zig:392), so its record-layer members
 ///   arrive here as themselves. Without them a decode error or a bad record MAC
 ///   would be reported as a plain receive failure.
-fn mapError(err: anyerror, phase: Phase) transport.ExchangeError {
+fn fault(err: anyerror, phase: Phase) transport.LeafError!transport.Outcome {
    if (transport.mapLocal(err)) |local| return local;
+    return peerFault(kindOf(err, phase), err);
+}
+
+/// A peer fault as an `Outcome`. Written out rather than inlined at every call
+/// site so the classification and the cause cannot drift apart by a typo.
+fn peerFault(kind: transport.PeerFault, cause: anyerror) transport.Outcome {
+    return .{ .fault = .{ .kind = kind, .cause = cause } };
+}
+
+fn kindOf(err: anyerror, phase: Phase) transport.PeerFault {
    switch (err) {
        error.TlsInitializationFailed,
        error.CertificateBundleLoadFailure,
@@ -209,7 +216,7 @@ fn mapError(err: anyerror, phase: Phase) transport.ExchangeError {
 comptime {
    for (@typeInfo(std.crypto.tls.Client.ReadError).error_set.?) |member| {
        const value: anyerror = @field(std.crypto.tls.Client.ReadError, member.name);
-        if (mapError(value, .receive) != error.TlsFailed) {
+        if (kindOf(value, .receive) != error.TlsFailed) {
            @compileError("unclassified TLS read cause: " ++ member.name);
        }
    }
@@ -268,7 +275,7 @@ test "init builds a uri from the endpoint url" {
    try testing.expectEqualStrings("dns.example", doh.endpoint.host);
 }

-test "DohClient satisfies the transport.Client interface" {
+test "DohClient satisfies the transport.Leaf interface" {
    var http: std.http.Client = undefined;
    var request_buf: [min_request_buf]u8 = undefined;
    var transfer_buf: [min_transfer_buf]u8 = undefined;
@@ -278,7 +285,7 @@ test "DohClient satisfies the transport.Client interface" {
    // Instantiation is the check: the vtable is built from `exchangeFn`, so a
    // signature drift is a compile error here. The `std.http.Client` above is
    // never driven, and no exchange runs.
-    const c: transport.Client = doh.client();
+    const c: transport.Leaf = doh.leaf();
    try testing.expectEqual(@as(*anyopaque, @ptrCast(&doh)), c.ptr);
    try testing.expectEqual(
        @as(@TypeOf(c.exchangeFn), DohClient.exchangeFn),
@@ -318,34 +325,46 @@ test "contentTypeOk rejects anything else" {
    try testing.expect(!contentTypeOk("application/dns-message-extra"));
 }

-test "mapError maps local errors before phase errors" {
-    try testing.expectEqual(error.OutOfMemory, mapError(error.OutOfMemory, .connect));
-    try testing.expectEqual(error.Canceled, mapError(error.Canceled, .receive));
-    try testing.expectEqual(error.Unexpected, mapError(error.Unexpected, .send));
+test "a local cause stays an error instead of becoming a fault" {
+    try testing.expectError(error.OutOfMemory, fault(error.OutOfMemory, .connect));
+    try testing.expectError(error.Canceled, fault(error.Canceled, .receive));
+    try testing.expectError(error.Unexpected, fault(error.Unexpected, .send));
 }

-test "mapError maps the collapsed tls errors regardless of phase" {
-    try testing.expectEqual(error.TlsFailed, mapError(error.TlsInitializationFailed, .connect));
-    try testing.expectEqual(error.TlsFailed, mapError(error.TlsInitializationFailed, .receive));
-    try testing.expectEqual(error.TlsFailed, mapError(error.CertificateBundleLoadFailure, .connect));
+test "kindOf maps the collapsed tls errors regardless of phase" {
+    try testing.expectEqual(error.TlsFailed, kindOf(error.TlsInitializationFailed, .connect));
+    try testing.expectEqual(error.TlsFailed, kindOf(error.TlsInitializationFailed, .receive));
+    try testing.expectEqual(error.TlsFailed, kindOf(error.CertificateBundleLoadFailure, .connect));
 }

-test "mapError maps every unwrapped record-layer cause to TlsFailed" {
+test "kindOf maps every unwrapped record-layer cause to TlsFailed" {
    // The set is the one `Connection.getReadError` can hand back, so the loop
    // fails the day std adds a member the switch does not name.
    inline for (@typeInfo(std.crypto.tls.Client.ReadError).error_set.?) |member| {
        try testing.expectEqual(
-            transport.ExchangeError.TlsFailed,
-            mapError(@field(std.crypto.tls.Client.ReadError, member.name), .receive),
+            transport.PeerFault.TlsFailed,
+            kindOf(@field(std.crypto.tls.Client.ReadError, member.name), .receive),
        );
    }
 }

-test "mapError maps remaining errors by phase" {
-    try testing.expectEqual(error.ConnectFailed, mapError(error.ConnectionRefused, .connect));
-    try testing.expectEqual(error.SendFailed, mapError(error.WriteFailed, .send));
-    try testing.expectEqual(error.ReceiveFailed, mapError(error.ReadFailed, .receive));
-    try testing.expectEqual(error.ReceiveFailed, mapError(error.HttpHeadersInvalid, .receive));
+test "kindOf maps remaining errors by phase" {
+    try testing.expectEqual(error.ConnectFailed, kindOf(error.ConnectionRefused, .connect));
+    try testing.expectEqual(error.SendFailed, kindOf(error.WriteFailed, .send));
+    try testing.expectEqual(error.ReceiveFailed, kindOf(error.ReadFailed, .receive));
+    try testing.expectEqual(error.ReceiveFailed, kindOf(error.HttpHeadersInvalid, .receive));
+}
+
+test "a fault carries the classification and the concrete cause" {
+    const failed = try faultOf(error.ConnectionRefused, .connect);
+    try testing.expectEqual(transport.PeerFault.ConnectFailed, failed.kind);
+    try testing.expectEqual(@as(anyerror, error.ConnectionRefused), failed.cause);
+
+    var buf: [64]u8 = undefined;
+    try testing.expectEqualStrings(
+        "ConnectFailed (cause ConnectionRefused)",
+        try std.fmt.bufPrint(&buf, "{f}", .{failed}),
+    );
 }

 /// Only the fields the unwrap helpers read are set. The rest of a `Connection`
@@ -367,6 +386,15 @@ fn stubConnection(
    return connection;
 }

+/// The fault half of `fault`, for the unwrap tests. A local cause leaves this
+/// as an error, which is what those tests assert instead.
+fn faultOf(err: anyerror, phase: Phase) !transport.Fault {
+    return switch (try fault(err, phase)) {
+        .reply => error.TestExpectedFault,
+        .fault => |f| f,
+    };
+}
+
 fn stubRequest(connection: *Connection, body_err: ?std.http.Reader.BodyError) Request {
    var req: Request = undefined;
    req.connection = connection;
@@ -377,59 +405,46 @@ fn stubRequest(connection: *Connection, body_err: ?std.http.Reader.BodyError) Re
 test "the send unwrap keeps a cancelled write out of the peer fault group" {
    var connection = stubConnection(null, error.Canceled);
    var req = stubRequest(&connection, null);
-    const mapped = mapError(sendCause(&req, error.WriteFailed), .send);
-    try testing.expectEqual(transport.ExchangeError.Canceled, mapped);
-    try testing.expectEqual(transport.Group.cancellation, transport.group(mapped));
+    try testing.expectError(error.Canceled, fault(sendCause(&req, error.WriteFailed), .send));
 }

 test "the send unwrap keeps a local resource write failure out of the peer fault group" {
    var connection = stubConnection(null, error.SystemResources);
    var req = stubRequest(&connection, null);
-    const mapped = mapError(sendCause(&req, error.WriteFailed), .send);
-    try testing.expectEqual(transport.ExchangeError.SystemResources, mapped);
-    try testing.expectEqual(transport.Group.local_resource, transport.group(mapped));
+    try testing.expectError(error.SystemResources, fault(sendCause(&req, error.WriteFailed), .send));
 }

 test "the send unwrap reports a peer side cause as a send fault" {
    var connection = stubConnection(null, error.ConnectionResetByPeer);
    var req = stubRequest(&connection, null);
-    try testing.expectEqual(
-        transport.ExchangeError.SendFailed,
-        mapError(sendCause(&req, error.WriteFailed), .send),
-    );
+    const failed = try faultOf(sendCause(&req, error.WriteFailed), .send);
+    try testing.expectEqual(transport.PeerFault.SendFailed, failed.kind);
+    try testing.expectEqual(@as(anyerror, error.ConnectionResetByPeer), failed.cause);
 }

 test "the head unwrap keeps a local resource read failure out of the peer fault group" {
    var connection = stubConnection(error.SystemResources, null);
    var req = stubRequest(&connection, null);
-    const mapped = mapError(headCause(&req, error.ReadFailed), .receive);
-    try testing.expectEqual(transport.ExchangeError.SystemResources, mapped);
-    try testing.expectEqual(transport.Group.local_resource, transport.group(mapped));
+    try testing.expectError(error.SystemResources, fault(headCause(&req, error.ReadFailed), .receive));

    var canceled = stubConnection(error.Canceled, null);
    var canceled_req = stubRequest(&canceled, null);
-    try testing.expectEqual(
-        transport.ExchangeError.Canceled,
-        mapError(headCause(&canceled_req, error.ReadFailed), .receive),
-    );
+    try testing.expectError(error.Canceled, fault(headCause(&canceled_req, error.ReadFailed), .receive));
 }

 test "the head unwrap reports a peer side cause as a receive fault" {
    var connection = stubConnection(error.ConnectionResetByPeer, null);
    var req = stubRequest(&connection, null);
-    try testing.expectEqual(
-        transport.ExchangeError.ReceiveFailed,
-        mapError(headCause(&req, error.ReadFailed), .receive),
-    );
+    const failed = try faultOf(headCause(&req, error.ReadFailed), .receive);
+    try testing.expectEqual(transport.PeerFault.ReceiveFailed, failed.kind);
+    try testing.expectEqual(@as(anyerror, error.ConnectionResetByPeer), failed.cause);
 }

 test "the body unwrap keeps a cancelled read out of the peer fault group" {
    var connection = stubConnection(error.Canceled, null);
    var req = stubRequest(&connection, null);
    const resp: Response = .{ .request = &req, .head = undefined };
-    const mapped = mapError(bodyCause(&resp, error.ReadFailed), .receive);
-    try testing.expectEqual(transport.ExchangeError.Canceled, mapped);
-    try testing.expectEqual(transport.Group.cancellation, transport.group(mapped));
+    try testing.expectError(error.Canceled, fault(bodyCause(&resp, error.ReadFailed), .receive));
 }

 test "the body unwrap prefers an http framing fault over the connection" {
@@ -440,10 +455,9 @@ test "the body unwrap prefers an http framing fault over the connection" {
    var req = stubRequest(&connection, error.HttpChunkTruncated);
    const resp: Response = .{ .request = &req, .head = undefined };
    try testing.expectEqual(error.HttpChunkTruncated, bodyCause(&resp, error.ReadFailed));
-    try testing.expectEqual(
-        transport.ExchangeError.ReceiveFailed,
-        mapError(bodyCause(&resp, error.ReadFailed), .receive),
-    );
+    const failed = try faultOf(bodyCause(&resp, error.ReadFailed), .receive);
+    try testing.expectEqual(transport.PeerFault.ReceiveFailed, failed.kind);
+    try testing.expectEqual(@as(anyerror, error.HttpChunkTruncated), failed.cause);
 }

 test "the unwraps report the collapsed error when no cause was stored" {
@@ -455,14 +469,13 @@ test "the unwraps report the collapsed error when no cause was stored" {
    try testing.expectEqual(error.ReadFailed, headCause(&req, error.ReadFailed));
    try testing.expectEqual(error.ReadFailed, bodyCause(&resp, error.ReadFailed));

-    try testing.expectEqual(
-        transport.ExchangeError.SendFailed,
-        mapError(sendCause(&req, error.WriteFailed), .send),
-    );
-    try testing.expectEqual(
-        transport.ExchangeError.ReceiveFailed,
-        mapError(bodyCause(&resp, error.ReadFailed), .receive),
-    );
+    const sent = try faultOf(sendCause(&req, error.WriteFailed), .send);
+    try testing.expectEqual(transport.PeerFault.SendFailed, sent.kind);
+    try testing.expectEqual(@as(anyerror, error.WriteFailed), sent.cause);
+
+    const received = try faultOf(bodyCause(&resp, error.ReadFailed), .receive);
+    try testing.expectEqual(transport.PeerFault.ReceiveFailed, received.kind);
+    try testing.expectEqual(@as(anyerror, error.ReadFailed), received.cause);
 }

 test "the unwraps pass a non-collapsed error through untouched" {
@@ -475,8 +488,7 @@ test "the unwraps pass a non-collapsed error through untouched" {
    try testing.expectEqual(error.EndOfStream, sendCause(&req, error.EndOfStream));
    try testing.expectEqual(error.HttpHeadersInvalid, headCause(&req, error.HttpHeadersInvalid));
    try testing.expectEqual(error.EndOfStream, bodyCause(&resp, error.EndOfStream));
-    try testing.expectEqual(
-        transport.ExchangeError.ReceiveFailed,
-        mapError(headCause(&req, error.HttpHeadersInvalid), .receive),
-    );
+    const failed = try faultOf(headCause(&req, error.HttpHeadersInvalid), .receive);
+    try testing.expectEqual(transport.PeerFault.ReceiveFailed, failed.kind);
+    try testing.expectEqual(@as(anyerror, error.HttpHeadersInvalid), failed.cause);
 }
@@ -0,0 +1,78 @@
+//! Hermetic loopback test for `doh_client.zig`.
+//!
+//! It lives in its own file because it needs `@import("build_options")`, which
+//! only exists when the compilation is driven by build.zig. `-Dintegration`
+//! gates it; nothing here leaves the machine and nothing here resolves a name.
+//!
+//! What it covers that the stub-connection tests in `doh_client.zig` cannot:
+//! those call the unwrap and classification helpers directly, so they prove the
+//! helpers and not the path. This drives the whole of `DohClient.exchange`
+//! against a peer that is provably not listening, and asserts that the concrete
+//! cause survives the classification and reaches the returned `Fault`. Without
+//! it, a future `exchange` that dropped the cause on the floor would still pass
+//! every other test in this tree.
+
+const std = @import("std");
+const build_options = @import("build_options");
+const net = std.Io.net;
+
+const doh_client = @import("doh_client.zig");
+const transport = @import("transport.zig");
+
+const testing = std.testing;
+
+/// A query for example.com A: id 0x1234, RD set, one question.
+const query_bytes =
+    "\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00" ++
+    "\x07example\x03com\x00\x00\x01\x00\x01";
+
+/// A loopback TCP port that is bound and never listening, held open for the
+/// whole test.
+///
+/// Bound, because an ephemeral port the kernel hands out is the only port a
+/// test can be sure of; a hardcoded one could belong to something. Held rather
+/// than closed, because a closed port is free for another process on this
+/// machine to take between the close and the connect, and then the refusal this
+/// test asserts would be a connection instead. Never listening, because a
+/// connect to a bound TCP port with no accept queue is refused, which is the
+/// deterministic peer failure the test needs.
+fn bindDeadPort(io: std.Io) !net.Socket {
+    return (net.IpAddress{ .ip4 = .loopback(0) }).bind(io, .{ .mode = .stream });
+}
+
+test "a refused connection reaches the caller as a ConnectFailed carrying its cause" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    var threaded: std.Io.Threaded = .init(gpa, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var dead = try bindDeadPort(io);
+    defer dead.close(io);
+
+    var url_buf: [64]u8 = undefined;
+    const url = try std.fmt.bufPrint(&url_buf, "https://127.0.0.1:{d}/dns-query", .{dead.address.ip4.port});
+
+    var http: std.http.Client = .{ .allocator = gpa, .io = io };
+    defer http.deinit();
+
+    var request_buf: [doh_client.min_request_buf]u8 = undefined;
+    var transfer_buf: [doh_client.min_transfer_buf]u8 = undefined;
+    var doh = try doh_client.DohClient.init(&http, try .parse(url), &request_buf, &transfer_buf);
+
+    var response_buf: [512]u8 = undefined;
+    switch (try doh.exchange(io, query_bytes, &response_buf)) {
+        .reply => return error.TestExpectedFault,
+        .fault => |fault| {
+            try testing.expectEqual(transport.PeerFault.ConnectFailed, fault.kind);
+            try testing.expectEqual(@as(anyerror, error.ConnectionRefused), fault.cause);
+
+            var text: [64]u8 = undefined;
+            try testing.expectEqualStrings(
+                "ConnectFailed (cause ConnectionRefused)",
+                try std.fmt.bufPrint(&text, "{f}", .{fault}),
+            );
+        },
+    }
+}
@@ -44,10 +44,10 @@ fn runExchange(io: std.Io, params: Params) anyerror!usize {
    const endpoint = try transport.Endpoint.parse("https://cloudflare-dns.com/dns-query");
    var doh = try doh_client.DohClient.init(&http, endpoint, &request_buf, &transfer_buf);

-    var selected: ?[]const u8 = null;
-    const reply = try doh.client().exchange(io, query_bytes, params.response_buf, &selected);
-    std.debug.assert(std.mem.eql(u8, selected.?, endpoint.url));
-    return reply.len;
+    return switch (try doh.leaf().exchange(io, query_bytes, params.response_buf)) {
+        .reply => |reply| reply.len,
+        .fault => |failed| failed.kind,
+    };
 }

 fn expire(io: std.Io, duration: std.Io.Clock.Duration) std.Io.Cancelable!void {
@@ -186,7 +186,7 @@ pub const DotClient = struct {
        }
    }

-    pub fn client(self: *DotClient) transport.Client {
+    pub fn leaf(self: *DotClient) transport.Leaf {
        return .{ .ptr = self, .exchangeFn = exchangeFn };
    }

@@ -199,12 +199,8 @@ pub const DotClient = struct {
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
-        selected: *?[]const u8,
-    ) transport.ExchangeError![]u8 {
+    ) transport.LeafError!transport.Outcome {
        const self: *DotClient = @ptrCast(@alignCast(ptr));
-        // The endpoint outlives the client, so the borrow is safe for the whole
-        // query. Set before the attempt: a failure names this resolver too.
-        selected.* = self.endpoint.url;
        return self.exchange(io, query, response_buf);
    }

@@ -222,24 +218,24 @@ pub const DotClient = struct {
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
-    ) transport.ExchangeError![]u8 {
+    ) transport.LeafError!transport.Outcome {
        // The length prefix is 16-bit, so a longer query cannot be framed. No
        // listener in this process can produce one; a caller that does gets a
        // local error rather than a silently truncated frame.
        if (query.len > transport.max_message_len) return error.BufferTooSmall;

        const reused = self.session != null;
-        if (!reused) try self.dial(io);
+        if (!reused) if (try self.dial(io)) |dial_fault| return .{ .fault = dial_fault };

        const failure = switch (self.transact(query, response_buf)) {
-            .ok => |reply| return reply,
+            .ok => |reply| return .{ .reply = reply },
            .failed => |failure| failure,
        };

        switch (retryDecision(reused, failure.received_any, failure.cause)) {
            .final => {
                self.close(io);
-                return transport.mapPhase(failure.cause, failure.phase);
+                return .{ .fault = try transport.faultOrLocal(failure.cause, failure.phase) };
            },
            .retry => {},
        }
@@ -249,38 +245,41 @@ pub const DotClient = struct {
        // `reuse_recoveries` is what makes the churn visible.
        log.debug("{f}", .{self.diagnose(.{ .stale_session = failure.cause })});
        self.close(io);
-        try self.dial(io);
+        if (try self.dial(io)) |dial_fault| return .{ .fault = dial_fault };

        switch (self.transact(query, response_buf)) {
            .ok => |reply| {
                if (self.reuse_recoveries) |counter| _ = counter.fetchAdd(1, .monotonic);
-                return reply;
+                return .{ .reply = reply };
            },
            // The retry's outcome is the exchange's outcome: one redial, never
            // two.
            .failed => |retried| {
                self.close(io);
-                return transport.mapPhase(retried.cause, retried.phase);
+                return .{ .fault = try transport.faultOrLocal(retried.cause, retried.phase) };
            },
        }
    }

    /// Opens a session and leaves it in `self.session`, or leaves `self.session`
-    /// null and returns the classified failure. No partially initialized
-    /// session ever survives this call.
-    fn dial(self: *DotClient, io: std.Io) transport.ExchangeError!void {
+    /// null and returns the classified fault. `null` means a session is open. No
+    /// partially initialized session ever survives this call.
+    fn dial(self: *DotClient, io: std.Io) transport.LeafError!?transport.Fault {
        std.debug.assert(self.session == null);

        const address = resolveAddress(self.endpoint) catch |err| {
+            // A host that is not an IP literal is a config error, and
+            // `resolveAddress` has already folded the parse failure into the
+            // classification, so the cause it carries is the classification.
            log.warn("{f}", .{self.diagnose(.not_an_ip_literal)});
-            return err;
+            return .{ .kind = err, .cause = err };
        };

-        try self.ensureBundle(io);
+        if (try self.ensureBundle(io)) |bundle_fault| return bundle_fault;

        const stream = address.connect(io, .{ .mode = .stream }) catch |err| {
            log.debug("{f}", .{self.diagnose(.{ .connect_failed = err })});
-            return transport.mapPhase(err, error.ConnectFailed);
+            return try transport.faultOrLocal(err, error.ConnectFailed);
        };

        // Emplaced before the handshake, never built beside it and copied in:
@@ -314,8 +313,9 @@ pub const DotClient = struct {
                .verify_name = self.verify_name,
                .cause = cause,
            } })});
-            return transport.mapPhase(cause, error.TlsFailed);
+            return try transport.faultOrLocal(cause, error.TlsFailed);
        };
+        return null;
    }

    /// One query and one reply on the open session, with enough detail on
@@ -380,17 +380,17 @@ pub const DotClient = struct {
    /// cancellation into `error.CertificateBundleLoadFailure`. That name cannot
    /// tell an `error.OutOfMemory` from a corrupt PEM file, and the first is a
    /// local resource failure that must not count against the upstream's
-    /// health. Scanning here keeps the concrete error for `transport.mapPhase`.
-    fn ensureBundle(self: *DotClient, io: std.Io) transport.ExchangeError!void {
+    /// health. Scanning here keeps the concrete error for the fault.
+    fn ensureBundle(self: *DotClient, io: std.Io) transport.LeafError!?transport.Fault {
        {
            try self.bundle_lock.lockShared(io);
            defer self.bundle_lock.unlockShared(io);
-            if (self.bundle.map.count() != 0) return;
+            if (self.bundle.map.count() != 0) return null;
        }

        try self.bundle_lock.lock(io);
        defer self.bundle_lock.unlock(io);
-        if (self.bundle.map.count() != 0) return;
+        if (self.bundle.map.count() != 0) return null;

        // A partial scan leaves entries in `map`, which the check above would
        // read as "already loaded". Reset so the next exchange scans again.
@@ -398,8 +398,9 @@ pub const DotClient = struct {
            self.bundle.deinit(self.gpa);
            self.bundle.* = .empty;
            log.warn("{f}", .{self.diagnose(.{ .bundle_load_failed = err })});
-            return transport.mapPhase(err, error.TlsFailed);
+            return try transport.faultOrLocal(err, error.TlsFailed);
        };
+        return null;
    }
 };

@@ -642,40 +643,39 @@ fn stubStream(

 test "the handshake unwrap keeps a cancelled read out of the peer fault group" {
    var stream = stubStream(error.Canceled, null, null);
-    const mapped = transport.mapPhase(concreteHandshake(&stream, error.ReadFailed), error.TlsFailed);
-    try testing.expectEqual(transport.ExchangeError.Canceled, mapped);
-    try testing.expectEqual(transport.Group.cancellation, transport.group(mapped));
+    try testing.expectError(
+        error.Canceled,
+        transport.faultOrLocal(concreteHandshake(&stream, error.ReadFailed), error.TlsFailed),
+    );
 }

 test "the handshake unwrap keeps a local resource write failure out of the peer fault group" {
    var stream = stubStream(null, error.SystemResources, null);
-    const mapped = transport.mapPhase(concreteHandshake(&stream, error.WriteFailed), error.TlsFailed);
-    try testing.expectEqual(transport.ExchangeError.SystemResources, mapped);
-    try testing.expectEqual(transport.Group.local_resource, transport.group(mapped));
+    try testing.expectError(
+        error.SystemResources,
+        transport.faultOrLocal(concreteHandshake(&stream, error.WriteFailed), error.TlsFailed),
+    );
 }

 test "the handshake unwrap reports a peer side cause as a TLS fault" {
    var reset = stubStream(error.ConnectionResetByPeer, null, null);
-    try testing.expectEqual(
-        transport.ExchangeError.TlsFailed,
-        transport.mapPhase(concreteHandshake(&reset, error.ReadFailed), error.TlsFailed),
-    );
+    const read_failed = try transport.faultOrLocal(concreteHandshake(&reset, error.ReadFailed), error.TlsFailed);
+    try testing.expectEqual(transport.PeerFault.TlsFailed, read_failed.kind);
+    try testing.expectEqual(@as(anyerror, error.ConnectionResetByPeer), read_failed.cause);

    var refused = stubStream(null, error.ConnectionRefused, null);
-    try testing.expectEqual(
-        transport.ExchangeError.TlsFailed,
-        transport.mapPhase(concreteHandshake(&refused, error.WriteFailed), error.TlsFailed),
-    );
+    const write_failed = try transport.faultOrLocal(concreteHandshake(&refused, error.WriteFailed), error.TlsFailed);
+    try testing.expectEqual(transport.PeerFault.TlsFailed, write_failed.kind);
+    try testing.expectEqual(@as(anyerror, error.ConnectionRefused), write_failed.cause);
 }

 test "the handshake unwrap reports a TLS fault when no cause was stored" {
    var stream = stubStream(null, null, null);
    try testing.expectEqual(error.ReadFailed, concreteHandshake(&stream, error.ReadFailed));
    try testing.expectEqual(error.WriteFailed, concreteHandshake(&stream, error.WriteFailed));
-    try testing.expectEqual(
-        transport.ExchangeError.TlsFailed,
-        transport.mapPhase(concreteHandshake(&stream, error.ReadFailed), error.TlsFailed),
-    );
+    const failed = try transport.faultOrLocal(concreteHandshake(&stream, error.ReadFailed), error.TlsFailed);
+    try testing.expectEqual(transport.PeerFault.TlsFailed, failed.kind);
+    try testing.expectEqual(@as(anyerror, error.ReadFailed), failed.cause);
 }

 test "the handshake unwrap passes other errors through untouched" {
@@ -685,42 +685,52 @@ test "the handshake unwrap passes other errors through untouched" {
        concreteHandshake(&stream, error.CertificateExpired),
    );
    try testing.expectEqual(error.Canceled, concreteHandshake(&stream, error.Canceled));
-    try testing.expectEqual(
-        transport.ExchangeError.TlsFailed,
-        transport.mapPhase(concreteHandshake(&stream, error.CertificateExpired), error.TlsFailed),
-    );
-    try testing.expectEqual(
-        transport.ExchangeError.Canceled,
-        transport.mapPhase(concreteHandshake(&stream, error.Canceled), error.TlsFailed),
+    const expired = try transport.faultOrLocal(concreteHandshake(&stream, error.CertificateExpired), error.TlsFailed);
+    try testing.expectEqual(transport.PeerFault.TlsFailed, expired.kind);
+    try testing.expectEqual(@as(anyerror, error.CertificateExpired), expired.cause);
+    try testing.expectError(
+        error.Canceled,
+        transport.faultOrLocal(concreteHandshake(&stream, error.Canceled), error.TlsFailed),
    );
 }

-/// What `exchange` would return for a failure `transact` reported.
-fn mappedFailure(outcome: Transact) transport.ExchangeError {
-    return transport.mapPhase(outcome.failed.cause, outcome.failed.phase);
+/// The fault `exchange` returns for a failure `transact` reported. A local cause leaves this as an
+/// error, which is what the tests of those causes assert instead.
+fn mappedFailure(outcome: Transact) transport.LeafError!transport.Fault {
+    return transport.faultOrLocal(outcome.failed.cause, outcome.failed.phase);
 }

 test "the send and receive unwraps prefer the stored cause" {
    var send = stubStream(null, error.Canceled, null);
-    try testing.expectEqual(
-        transport.ExchangeError.Canceled,
-        mappedFailure(sendFailed(&send, error.WriteFailed)),
-    );
+    try testing.expectError(error.Canceled, mappedFailure(sendFailed(&send, error.WriteFailed)));

    // The TLS client's own error wins over the socket reader's.
    var receive = stubStream(error.ConnectionResetByPeer, null, error.TlsAlert);
-    try testing.expectEqual(
-        transport.ExchangeError.ReceiveFailed,
-        mappedFailure(receiveFailed(&receive, error.ReadFailed, false)),
-    );
+    const received = try mappedFailure(receiveFailed(&receive, error.ReadFailed, false));
+    try testing.expectEqual(transport.PeerFault.ReceiveFailed, received.kind);
+    try testing.expectEqual(@as(anyerror, error.TlsAlert), received.cause);

    var socket = stubStream(error.SystemResources, null, null);
-    try testing.expectEqual(
-        transport.ExchangeError.SystemResources,
+    try testing.expectError(
+        error.SystemResources,
        mappedFailure(receiveFailed(&socket, error.ReadFailed, true)),
    );
 }

+test "a transact failure becomes a fault carrying its concrete cause" {
+    var stream = stubStream(null, error.ConnectionResetByPeer, null);
+
+    const sent = try mappedFailure(sendFailed(&stream, error.WriteFailed));
+    try testing.expectEqual(transport.PeerFault.SendFailed, sent.kind);
+    try testing.expectEqual(@as(anyerror, error.ConnectionResetByPeer), sent.cause);
+
+    var text: [64]u8 = undefined;
+    try testing.expectEqualStrings(
+        "SendFailed (cause ConnectionResetByPeer)",
+        try std.fmt.bufPrint(&text, "{f}", .{sent}),
+    );
+}
+
 test "a send failure is always pre-first-byte, and a receive failure reports what it read" {
    // The retry rule reads `received_any`, so where it comes from is part of the
    // contract rather than an incidental field: nothing is read before the query
@@ -804,14 +814,13 @@ test "a validation failure is final and its phase survives the mapping" {
    };
    for (outcomes) |outcome| {
        try testing.expectEqual(RetryDecision.final, retryDecision(true, true, outcome.cause));
-        try testing.expectEqual(
-            @as(transport.ExchangeError, outcome.phase),
-            mappedFailure(.{ .failed = .{
-                .cause = outcome.cause,
-                .phase = outcome.phase,
-                .received_any = true,
-            } }),
-        );
+        const failed = try mappedFailure(.{ .failed = .{
+            .cause = outcome.cause,
+            .phase = outcome.phase,
+            .received_any = true,
+        } });
+        try testing.expectEqual(outcome.phase, failed.kind);
+        try testing.expectEqual(outcome.cause, failed.cause);
    }
 }

@@ -830,26 +839,21 @@ test "a CA bundle scan failure keeps local resource errors out of the peer fault
            transport.Group.local_resource,
            transport.group(transport.mapPhase(err, error.TlsFailed)),
        );
+        try testing.expectError(err, transport.faultOrLocal(err, error.TlsFailed));
    }

-    try testing.expectEqual(
-        transport.ExchangeError.Canceled,
-        transport.mapPhase(error.Canceled, error.TlsFailed),
-    );
+    try testing.expectError(error.Canceled, transport.faultOrLocal(error.Canceled, error.TlsFailed));

    // A missing or corrupt bundle is not this process running out of anything,
-    // so it stays a TLS fault.
-    try testing.expectEqual(
-        transport.ExchangeError.TlsFailed,
-        transport.mapPhase(error.FileNotFound, error.TlsFailed),
-    );
-    try testing.expectEqual(
-        transport.ExchangeError.TlsFailed,
-        transport.mapPhase(error.MissingEndCertificateMarker, error.TlsFailed),
-    );
+    // so it stays a TLS fault, and the cause names which of the two it was.
+    for ([_]anyerror{ error.FileNotFound, error.MissingEndCertificateMarker }) |err| {
+        const failed = try transport.faultOrLocal(err, error.TlsFailed);
+        try testing.expectEqual(transport.PeerFault.TlsFailed, failed.kind);
+        try testing.expectEqual(err, failed.cause);
+    }
 }

-test "DotClient satisfies the Client interface" {
+test "DotClient satisfies the Leaf interface" {
    const gpa = testing.allocator;

    const buffer = try gpa.alloc(u8, 4 * tls.Client.min_buffer_len);
@@ -878,7 +882,7 @@ test "DotClient satisfies the Client interface" {
    dot.close(undefined);
    try testing.expect(dot.session == null);

-    const iface: transport.Client = dot.client();
+    const iface: transport.Leaf = dot.leaf();
    try testing.expectEqual(@as(*anyopaque, @ptrCast(&dot)), iface.ptr);
 }

@@ -391,12 +391,12 @@ fn twoExchangesOverOneSession(io: std.Io, fixture: *ClientFixture) anyerror!void
    var buf: [512]u8 = undefined;

    const first = try fixture.dot.exchange(io, query_bytes, &buf);
-    try testing.expectEqualSlices(u8, response_bytes, first);
+    try testing.expectEqualSlices(u8, response_bytes, first.reply);
    // The point of the milestone: the connection outlives the exchange.
    try testing.expect(fixture.dot.session != null);

    const second = try fixture.dot.exchange(io, query_bytes, &buf);
-    try testing.expectEqualSlices(u8, response_bytes, second);
+    try testing.expectEqualSlices(u8, response_bytes, second.reply);
    try testing.expect(fixture.dot.session != null);
 }

@@ -451,7 +451,7 @@ test "a session the upstream closed is recovered by one redial and counted, not
    // Through a real pool entry, because the claim is about what the pool does
    // *not* see: an upstream reaping an idle connection must not cost it health
    // or raise an operational event.
-    var slots = [_]pool_mod.Slot{.{ .client = fixture.dot.client() }};
+    var slots = [_]pool_mod.Slot{.{ .client = fixture.dot.leaf() }};
    var entries = [_]pool_mod.Entry{.{
        .endpoint = fixture.dot.endpoint,
        .slots = &slots,
@@ -496,7 +496,7 @@ fn exchangeThenReadTruncatedReply(io: std.Io, fixture: *ClientFixture) anyerror!

    // One prefix byte arrived, so the reply had started: re-sending the query on
    // a fresh connection would be a second question, not a recovery.
-    try testing.expectError(error.ReceiveFailed, fixture.dot.exchange(io, query_bytes, &buf));
+    try expectFault(error.ReceiveFailed, error.EndOfStream, try fixture.dot.exchange(io, query_bytes, &buf));
    try testing.expect(fixture.dot.session == null);
 }

@@ -532,7 +532,7 @@ fn exchangeThenReadWrongId(io: std.Io, fixture: *ClientFixture) anyerror!void {
    // The read succeeded; only the bytes are wrong. Nothing about that says the
    // connection is stale, so it is final — but the stream position after a
    // frame this client will not trust is unknowable, so the session goes.
-    try testing.expectError(error.ResponseMismatch, fixture.dot.exchange(io, query_bytes, &buf));
+    try expectFault(error.ResponseMismatch, error.ResponseMismatch, try fixture.dot.exchange(io, query_bytes, &buf));
    try testing.expect(fixture.dot.session == null);
 }

@@ -571,7 +571,7 @@ fn exchangeThenFailTheRetry(io: std.Io, fixture: *ClientFixture) anyerror!void {
    // taken, and connected. The exchange on that fresh session then fails its
    // read, and the retry's outcome is the exchange's outcome — no third dial,
    // because a session this call dialed itself is never retried.
-    try testing.expectError(error.ReceiveFailed, fixture.dot.exchange(io, query_bytes, &buf));
+    try expectFault(error.ReceiveFailed, error.EndOfStream, try fixture.dot.exchange(io, query_bytes, &buf));
    try testing.expect(fixture.dot.session == null);

    // What the `nxdns check` probe loop relies on: closing a client whose
@@ -609,7 +609,20 @@ test "a stale session whose retry fails is final and leaves no session behind" {
    try testing.expectEqual(@as(u64, 0), fixture.recoveries.load(.acquire));
 }

-fn oneExchange(io: std.Io, fixture: *ClientFixture) transport.ExchangeError!void {
+/// A returned fault, whole: the classification the pool counts and the concrete
+/// cause it records. Both are asserted, because the cause is what an operator
+/// reads and only a real exchange proves it survives the wire.
+fn expectFault(kind: transport.PeerFault, cause: anyerror, outcome: transport.Outcome) !void {
+    switch (outcome) {
+        .reply => return error.TestExpectedFault,
+        .fault => |f| {
+            try testing.expectEqual(kind, f.kind);
+            try testing.expectEqual(cause, f.cause);
+        },
+    }
+}
+
+fn oneExchange(io: std.Io, fixture: *ClientFixture) transport.LeafError!void {
    var buf: [512]u8 = undefined;
    _ = try fixture.dot.exchange(io, query_bytes, &buf);
 }
@@ -62,10 +62,10 @@ fn runExchange(io: std.Io, params: Params) anyerror!usize {
    // so the task that built the client is what has to close it. Without this
    // the socket and its TLS state outlive the test.
    defer client.close(io);
-    var selected: ?[]const u8 = null;
-    const reply = try client.client().exchange(io, query_bytes, params.response_buf, &selected);
-    std.debug.assert(std.mem.eql(u8, selected.?, endpoint.url));
-    return reply.len;
+    return switch (try client.leaf().exchange(io, query_bytes, params.response_buf)) {
+        .reply => |reply| reply.len,
+        .fault => |failed| failed.kind,
+    };
 }

 fn expire(io: std.Io, duration: std.Io.Clock.Duration) std.Io.Cancelable!void {
@@ -20,7 +20,7 @@
 //!   * Data is data. `total_successes`, `total_failures`, the window and the
 //!     `@max` of `last_success_at` / `last_error_at` always update, however old
 //!     `at` is.
-//!   * `consecutive_failures`, `backoff_until` and `last_error_buf` describe
+//!   * `consecutive_failures`, `backoff_until` and the last fault describe
 //!     the present, so a newer recorded outcome overrules a stale call.
 //!
 //! `recordSuccess` clears `consecutive_failures` and `backoff_until` only when
@@ -33,25 +33,67 @@
 //! `State` carries no lock. The pool owns the mutex.

 const std = @import("std");
+const transport = @import("transport.zig");

 pub const Config = struct {
-    /// Consecutive peer faults before the endpoint is put in backoff.
+    /// Consecutive peer faults before the endpoint is put in backoff. At least
+    /// 1: a state tripped by zero failures has no fault to name, and the
+    /// episode the pool opens from it could not say what went wrong. Nothing
+    /// builds this from user input, so the default is checked below and
+    /// `Pool.init` asserts what a caller passes.
    failure_threshold: u8 = 2,
    base_backoff_ms: u32 = 500,
    max_backoff_ms: u32 = 60_000,
 };

+comptime {
+    const default: Config = .{};
+    if (default.failure_threshold < 1) @compileError("failure_threshold must be at least 1");
+}
+
 /// Rolling success-rate window, in samples. Equal to the bit width of
 /// `State.window`.
 pub const window_len = 32;

-/// Bytes kept of an `@errorName`, truncated to fit.
-pub const error_name_capacity = 48;
+/// Bytes kept of a rendered `transport.Fault`, truncated to fit. Sized so the
+/// widest classification and the widest concrete cause name fit whole; the
+/// comptime-driven test below recomputes that worst case from the std error
+/// sets, so a wider name std adds fails the test run rather than silently
+/// truncating an operator's only diagnostic.
+pub const error_name_capacity = 64;

 /// The shift is capped so `base_backoff_ms << shift` cannot run away; by then
 /// `max_backoff_ms` has clamped the result many doublings ago.
 const max_shift = 20;

+/// The complete desired state of this endpoint's Diagnostics episode after one
+/// recorded outcome. This file decides it, because "failing" is this file's
+/// predicate and nobody else's; the pool projects it onto the store.
+///
+/// A state, never a transition. A transition has to be applied to whatever the
+/// store holds, so two effects that reach the pool out of order can leave the
+/// wrong one standing: a "nothing changed" overtaking a required report would
+/// drop the report for good. A desired state makes last-writer-wins by
+/// revision correct: a projection of the whole state is also what repairs a
+/// card left over from a latched store write, on the next outcome. What a
+/// per-pool revision cannot see — an episode that outlived a restart, or one a
+/// retired generation opened — is repaired by `Pool.reconcile` instead, at boot
+/// and at every retirement, not by waiting for an outcome.
+pub const Effect = struct {
+    /// Per-state counter, incremented by every mutation. The pool applies
+    /// effects in this order and drops one that arrives behind a newer sibling.
+    revision: u64,
+    state: Episode,
+
+    /// A tagged union rather than an enum beside a fault field, so an episode
+    /// without the fault that explains it is unrepresentable. Named `Episode`
+    /// rather than `State` because this file's `State` is the health record.
+    pub const Episode = union(enum) {
+        clear,
+        tripped: transport.Fault,
+    };
+};
+
 pub const State = struct {
    consecutive_failures: u32,
    total_successes: u64,
@@ -62,6 +104,12 @@ pub const State = struct {
    /// Length of the `@errorName` held in `last_error_buf`, truncated to fit.
    last_error_len: u8,
    backoff_until: ?std.Io.Timestamp,
+    /// The effective last fault, as a value. `last_error_buf` is its rendering;
+    /// both are kept because the text is what every surface prints and the
+    /// value is what an `Effect` carries out of the mutex.
+    last_fault: ?transport.Fault,
+    /// Incremented by every mutation; see `Effect.revision`.
+    revision: u64,
    /// Bitset, 1 = success, LSB = most recent.
    window: u32,
    window_filled: u8,
@@ -75,11 +123,22 @@ pub const State = struct {
        .last_error_buf = @splat(0),
        .last_error_len = 0,
        .backoff_until = null,
+        .last_fault = null,
+        .revision = 0,
        .window = 0,
        .window_filled = 0,
    };

-    pub fn recordSuccess(self: *State, at: std.Io.Timestamp) void {
+    /// A success that is the newest outcome clears the count, and the effect
+    /// then says `clear`.
+    ///
+    /// `null` when the state stays tripped, which is the stale success behind a
+    /// newer failure (case 1 at `recordFailure`). The card that failure opened
+    /// is already right, and projecting `tripped` again would count a
+    /// successful exchange as an occurrence of the episode. A `clear`-shaped
+    /// no-op is not an option either: it would advance `applied_revision` past
+    /// a report still in flight and drop it.
+    pub fn recordSuccess(self: *State, at: std.Io.Timestamp, cfg: Config) ?Effect {
        if (self.isNewestOutcome(at)) {
            self.consecutive_failures = 0;
            self.backoff_until = null;
@@ -87,6 +146,32 @@ pub const State = struct {
        self.total_successes += 1;
        self.push(1);
        self.last_success_at = later(self.last_success_at, at);
+        self.revision += 1;
+        if (self.tripped(cfg)) return null;
+        return self.effect(cfg);
+    }
+
+    /// The pool's definition of "failing this endpoint": the predicate that
+    /// puts it in backoff. Not `available`, which turns true again the moment a
+    /// backoff expires, before any recovery is proven.
+    pub fn tripped(self: *const State, cfg: Config) bool {
+        return self.consecutive_failures >= cfg.failure_threshold;
+    }
+
+    /// The desired episode state this endpoint is in right now, without a
+    /// mutation and without advancing the revision. The pool projects this at
+    /// the two points a recorded outcome cannot reach: the first generation's
+    /// boot, and the retirement of a generation whose late outcomes are now
+    /// impossible.
+    pub fn currentEffect(self: *const State, cfg: Config) Effect {
+        return self.effect(cfg);
+    }
+
+    /// A tripped state has always recorded a failure, because the threshold is
+    /// at least 1, so the fault is always there to name.
+    fn effect(self: *const State, cfg: Config) Effect {
+        if (!self.tripped(cfg)) return .{ .revision = self.revision, .state = .clear };
+        return .{ .revision = self.revision, .state = .{ .tripped = self.last_fault.? } };
    }

    /// True when no recorded outcome is newer than `at`. Both timestamp fields
@@ -96,9 +181,9 @@ pub const State = struct {
        return !newerThan(self.last_success_at, at) and !newerThan(self.last_error_at, at);
    }

-    /// `err_name` is `@errorName` of a PeerFault member. `rand` supplies
-    /// jitter; the caller owns the RNG so this stays pure and the test is
-    /// deterministic.
+    /// `fault` is the peer fault as a value: `transport.group` has already
+    /// ruled that this is the peer's doing. `rand` supplies jitter; the caller
+    /// owns the RNG so this stays pure and the test is deterministic.
    ///
    /// A stale failure, that is one whose `at` is older than an outcome already
    /// recorded, is held to the mirror image of the stale-success rule. Three
@@ -119,27 +204,29 @@ pub const State = struct {
    pub fn recordFailure(
        self: *State,
        at: std.Io.Timestamp,
-        err_name: []const u8,
+        fault: transport.Fault,
        cfg: Config,
        rand: u32,
-    ) void {
+    ) Effect {
        const newer_success = newerThan(self.last_success_at, at);
        const newer_failure = newerThan(self.last_error_at, at);

        if (!newer_failure) {
-            const copied = @min(err_name.len, self.last_error_buf.len);
-            @memcpy(self.last_error_buf[0..copied], err_name[0..copied]);
-            self.last_error_len = @intCast(copied);
+            self.last_fault = fault;
+            var writer: std.Io.Writer = .fixed(&self.last_error_buf);
+            writer.print("{f}", .{fault}) catch {};
+            self.last_error_len = @intCast(writer.end);
        }

        self.total_failures += 1;
        self.push(0);
        self.last_error_at = later(self.last_error_at, at);
+        self.revision += 1;

-        if (newer_success) return;
+        if (newer_success) return self.effect(cfg);

        self.consecutive_failures +|= 1;
-        if (self.consecutive_failures < cfg.failure_threshold) return;
+        if (self.consecutive_failures < cfg.failure_threshold) return self.effect(cfg);

        const delay_ms = backoffDelayMs(self.consecutive_failures, cfg);
        const half = delay_ms / 2;
@@ -148,6 +235,7 @@ pub const State = struct {
            .nanoseconds = at.nanoseconds + @as(i96, jittered) * std.time.ns_per_ms,
        };
        self.backoff_until = later(self.backoff_until, deadline);
+        return self.effect(cfg);
    }

    pub fn available(self: *const State, now: std.Io.Timestamp) bool {
@@ -212,10 +300,16 @@ fn ms(count: i96) i96 {
    return count * std.time.ns_per_ms;
 }

+/// A fault whose cause is its own classification: the shape an expiry has, and
+/// short enough to keep the timestamp tests about timestamps.
+fn peerFault(kind: transport.PeerFault) transport.Fault {
+    return .{ .kind = kind, .cause = kind };
+}
+
 test "a failure below the threshold leaves the endpoint available" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);

    try testing.expectEqual(@as(u32, 1), state.consecutive_failures);
    try testing.expectEqual(@as(u64, 1), state.total_failures);
@@ -226,8 +320,8 @@ test "a failure below the threshold leaves the endpoint available" {
 test "reaching the threshold puts the endpoint in backoff until the deadline" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);

    // Two failures, threshold 2, shift 0: delay 500 ms, jitter 0 => 250 ms.
    const until = state.backoff_until.?;
@@ -244,7 +338,7 @@ test "consecutive failures grow the delay and saturate at max_backoff_ms" {
    var previous: i96 = -1;
    var i: usize = 0;
    while (i < 40) : (i += 1) {
-        state.recordFailure(ts(0), "Timeout", cfg, 0);
+        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
        if (state.backoff_until) |until| {
            try testing.expect(until.nanoseconds >= previous);
            previous = until.nanoseconds;
@@ -262,11 +356,11 @@ test "consecutive failures grow the delay and saturate at max_backoff_ms" {
 test "a success resets the consecutive count, the window and the backoff" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
    try testing.expect(state.backoff_until != null);

-    state.recordSuccess(ts(ms(1)));
+    _ = state.recordSuccess(ts(ms(1)), cfg);
    try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
    try testing.expectEqual(@as(u64, 1), state.total_successes);
@@ -280,8 +374,8 @@ test "jitter stays inside half the delay and the whole delay" {

    for ([_]u32{ 0, std.math.maxInt(u32), 1, 12345 }) |rand| {
        var state: State = .init;
-        state.recordFailure(ts(0), "Timeout", cfg, rand);
-        state.recordFailure(ts(0), "Timeout", cfg, rand);
+        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, rand);
+        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, rand);
        const offset = state.backoff_until.?.nanoseconds;
        try testing.expect(offset >= ms(delay / 2));
        try testing.expect(offset <= ms(delay));
@@ -289,20 +383,21 @@ test "jitter stays inside half the delay and the whole delay" {
 }

 test "an out-of-order success does not move last_success_at backwards" {
+    const cfg: Config = .{};
    var state: State = .init;
-    state.recordSuccess(ts(100));
-    state.recordSuccess(ts(50));
+    _ = state.recordSuccess(ts(100), cfg);
+    _ = state.recordSuccess(ts(50), cfg);
    try testing.expectEqual(@as(i96, 100), state.last_success_at.?.nanoseconds);
 }

 test "an out-of-order failure does not shorten the backoff" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    const until = state.backoff_until.?.nanoseconds;

-    state.recordFailure(ts(ms(50)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(50)), peerFault(error.Timeout), cfg, 0);
    try testing.expect(state.backoff_until.?.nanoseconds >= until);
    try testing.expectEqual(@as(i96, ms(100)), state.last_error_at.?.nanoseconds);
 }
@@ -310,11 +405,11 @@ test "an out-of-order failure does not shorten the backoff" {
 test "an out-of-order success does not clear the backoff of a newer failure" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    const until = state.backoff_until.?.nanoseconds;

-    state.recordSuccess(ts(ms(50)));
+    _ = state.recordSuccess(ts(ms(50)), cfg);
    try testing.expectEqual(@as(i96, until), state.backoff_until.?.nanoseconds);
    try testing.expectEqual(@as(u32, 2), state.consecutive_failures);
    try testing.expectEqual(@as(u64, 1), state.total_successes);
@@ -325,10 +420,10 @@ test "an out-of-order success does not clear the backoff of a newer failure" {
 test "a stale failure behind a newer success does not raise the consecutive count" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(ms(10)), "Timeout", cfg, 0);
-    state.recordSuccess(ts(ms(100)));
-    state.recordFailure(ts(ms(20)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(30)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(10)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordSuccess(ts(ms(100)), cfg);
+    _ = state.recordFailure(ts(ms(20)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(30)), peerFault(error.Timeout), cfg, 0);

    try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
@@ -338,21 +433,21 @@ test "a stale failure behind a newer success does not raise the consecutive coun
 test "a stale failure behind a newer success still counts into the totals" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordSuccess(ts(ms(100)));
-    state.recordFailure(ts(ms(20)), "Timeout", cfg, 0);
+    _ = state.recordSuccess(ts(ms(100)), cfg);
+    _ = state.recordFailure(ts(ms(20)), peerFault(error.Timeout), cfg, 0);

    try testing.expectEqual(@as(u64, 1), state.total_failures);
    try testing.expectEqual(@as(u8, 2), state.window_filled);
    try testing.expectEqual(@as(u32, 0), state.window & 1);
    try testing.expectEqual(@as(i96, ms(20)), state.last_error_at.?.nanoseconds);
-    try testing.expectEqualStrings("Timeout", state.lastError());
+    try testing.expectEqualStrings("Timeout (cause Timeout)", state.lastError());
 }

 test "a stale failure with no newer success still counts as consecutive" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(50)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(50)), peerFault(error.Timeout), cfg, 0);

    try testing.expectEqual(@as(u32, 2), state.consecutive_failures);
    try testing.expect(state.backoff_until != null);
@@ -362,19 +457,19 @@ test "a stale failure with no newer success still counts as consecutive" {
 test "a stale failure does not overwrite the error of a newer failure" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(50)), "ConnectFailed", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(50)), peerFault(error.ConnectFailed), cfg, 0);

-    try testing.expectEqualStrings("Timeout", state.lastError());
+    try testing.expectEqualStrings("Timeout (cause Timeout)", state.lastError());
 }

 test "the newest failure records its own error and extends the backoff" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(ms(50)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "ConnectFailed", cfg, 0);
+    _ = state.recordFailure(ts(ms(50)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.ConnectFailed), cfg, 0);

-    try testing.expectEqualStrings("ConnectFailed", state.lastError());
+    try testing.expectEqualStrings("ConnectFailed (cause ConnectFailed)", state.lastError());
    try testing.expectEqual(@as(u32, 2), state.consecutive_failures);
    try testing.expectEqual(ms(100) + ms(250), state.backoff_until.?.nanoseconds);
 }
@@ -382,11 +477,11 @@ test "the newest failure records its own error and extends the backoff" {
 test "the newest success clears the backoff of an older failure" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    try testing.expect(state.backoff_until != null);

-    state.recordSuccess(ts(ms(101)));
+    _ = state.recordSuccess(ts(ms(101)), cfg);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
    try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
 }
@@ -394,10 +489,10 @@ test "the newest success clears the backoff of an older failure" {
 test "a success at the timestamp of the newest failure clears the backoff" {
    const cfg: Config = .{};
    var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);

-    state.recordSuccess(ts(ms(100)));
+    _ = state.recordSuccess(ts(ms(100)), cfg);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
    try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
 }
@@ -409,37 +504,176 @@ test "successRate over a half-success window is 0.5" {

    var i: usize = 0;
    while (i < 4) : (i += 1) {
-        state.recordSuccess(ts(0));
-        state.recordFailure(ts(0), "Timeout", cfg, 0);
+        _ = state.recordSuccess(ts(0), cfg);
+        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
    }
    try testing.expectEqual(@as(u8, 8), state.window_filled);
    try testing.expectEqual(@as(f32, 0.5), state.successRate());
 }

 test "successRate counts only the filled part of the window" {
+    const cfg: Config = .{};
    var state: State = .init;
-    state.recordSuccess(ts(0));
+    _ = state.recordSuccess(ts(0), cfg);
    try testing.expectEqual(@as(f32, 1.0), state.successRate());

    var i: usize = 0;
-    while (i < window_len * 2) : (i += 1) state.recordSuccess(ts(0));
+    while (i < window_len * 2) : (i += 1) _ = state.recordSuccess(ts(0), cfg);
    try testing.expectEqual(@as(u8, window_len), state.window_filled);
    try testing.expectEqual(@as(f32, 1.0), state.successRate());
 }

-test "lastError returns the last recorded name, truncated not overflowed" {
+test "lastError renders the classification and the concrete cause" {
    const cfg: Config = .{};
    var state: State = .init;
    try testing.expectEqualStrings("", state.lastError());

-    state.recordFailure(ts(0), "ConnectFailed", cfg, 0);
-    try testing.expectEqualStrings("ConnectFailed", state.lastError());
+    _ = state.recordFailure(ts(0), .{ .kind = error.SendFailed, .cause = error.BrokenPipe }, cfg, 0);
+    try testing.expectEqualStrings("SendFailed (cause BrokenPipe)", state.lastError());
+    try testing.expectEqual(transport.PeerFault.SendFailed, state.last_fault.?.kind);

-    state.recordFailure(ts(0), "Timeout", cfg, 0);
-    try testing.expectEqualStrings("Timeout", state.lastError());
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
+    try testing.expectEqualStrings("Timeout (cause Timeout)", state.lastError());
+}

-    const long = "A" ** 200;
-    state.recordFailure(ts(0), long, cfg, 0);
-    try testing.expectEqual(@as(usize, 48), state.lastError().len);
-    try testing.expectEqualStrings(long[0..48], state.lastError());
+test "lastError truncates a cause too wide for the buffer instead of overflowing" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    const Wide = error{AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA};
+    _ = state.recordFailure(
+        ts(0),
+        .{ .kind = error.ReceiveFailed, .cause = Wide.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA },
+        cfg,
+        0,
+    );
+
+    const whole = "ReceiveFailed (cause " ++ "A" ** 64 ++ ")";
+    try testing.expectEqual(@as(usize, error_name_capacity), state.lastError().len);
+    try testing.expectEqualStrings(whole[0..error_name_capacity], state.lastError());
+}
+
+test "error_name_capacity holds the widest classification and cause whole" {
+    // The widest concrete cause an upstream unwrap can produce. The three sets
+    // are the ones `transport.zig`'s unwraps return a member of; a wider name
+    // std adds fails here rather than truncating an operator's diagnostic.
+    const widest_cause = comptime blk: {
+        var widest: []const u8 = "";
+        for (@typeInfo(std.crypto.tls.Client.ReadError).error_set.?) |member| {
+            if (member.name.len > widest.len) widest = member.name;
+        }
+        for (@typeInfo(std.http.Client.RequestError).error_set.?) |member| {
+            if (member.name.len > widest.len) widest = member.name;
+        }
+        for (@typeInfo(std.http.Reader.BodyError).error_set.?) |member| {
+            if (member.name.len > widest.len) widest = member.name;
+        }
+        break :blk widest;
+    };
+    const widest_kind = comptime blk: {
+        var widest: []const u8 = "";
+        for (@typeInfo(transport.PeerFault).error_set.?) |member| {
+            if (member.name.len > widest.len) widest = member.name;
+        }
+        break :blk widest;
+    };
+
+    try testing.expectEqualStrings("DetectingNetworkConfigurationFailed", widest_cause);
+    try testing.expect(widest_kind.len + " (cause ".len + widest_cause.len + ")".len <= error_name_capacity);
+}
+
+/// The fault of a `tripped` effect, or an error when the effect says `clear`.
+fn trippedFault(effect: Effect) !transport.Fault {
+    return switch (effect.state) {
+        .clear => error.TestExpectedTripped,
+        .tripped => |fault| fault,
+    };
+}
+
+test "one failure stays clear and the second trips the episode" {
+    const cfg: Config = .{};
+    var state: State = .init;
+
+    const first = state.recordFailure(ts(0), peerFault(error.ConnectFailed), cfg, 0);
+    try testing.expectEqual(Effect.Episode.clear, first.state);
+
+    const second = state.recordFailure(ts(ms(1)), .{ .kind = error.SendFailed, .cause = error.BrokenPipe }, cfg, 0);
+    const fault = try trippedFault(second);
+    try testing.expectEqual(transport.PeerFault.SendFailed, fault.kind);
+    try testing.expectEqual(@as(anyerror, error.BrokenPipe), fault.cause);
+    try testing.expect(second.revision > first.revision);
+}
+
+test "a failure while tripped stays tripped" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(1)), peerFault(error.Timeout), cfg, 0);
+
+    const third = state.recordFailure(ts(ms(2)), peerFault(error.Timeout), cfg, 0);
+    _ = try trippedFault(third);
+}
+
+test "a success on a tripped state returns clear, and so does one below the threshold" {
+    const cfg: Config = .{};
+    var state: State = .init;
+
+    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(0), cfg).?.state);
+
+    _ = state.recordFailure(ts(ms(1)), peerFault(error.Timeout), cfg, 0);
+    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(ms(2)), cfg).?.state);
+
+    _ = state.recordFailure(ts(ms(3)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(4)), peerFault(error.Timeout), cfg, 0);
+    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(ms(5)), cfg).?.state);
+    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(ms(6)), cfg).?.state);
+}
+
+test "a stale success behind a newer failure projects nothing" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    _ = state.recordFailure(ts(ms(10)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(20)), peerFault(error.Timeout), cfg, 0);
+
+    // The endpoint is still failing, so the card the failures opened is
+    // already right. Reporting it again on a success would count that success
+    // as an occurrence of the episode.
+    try testing.expectEqual(@as(?Effect, null), state.recordSuccess(ts(ms(5)), cfg));
+    try testing.expect(state.tripped(cfg));
+    try testing.expectEqual(@as(u64, 1), state.total_successes);
+}
+
+test "a stale failure carries the newer effective fault, not its own" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    _ = state.recordFailure(ts(ms(10)), peerFault(error.Timeout), cfg, 0);
+    const newest = state.recordFailure(
+        ts(ms(20)),
+        .{ .kind = error.SendFailed, .cause = error.BrokenPipe },
+        cfg,
+        0,
+    );
+    _ = try trippedFault(newest);
+
+    const stale = state.recordFailure(ts(ms(15)), .{ .kind = error.ConnectFailed, .cause = error.ConnectionRefused }, cfg, 0);
+    const fault = try trippedFault(stale);
+    try testing.expectEqual(transport.PeerFault.SendFailed, fault.kind);
+    try testing.expectEqual(@as(anyerror, error.BrokenPipe), fault.cause);
+    try testing.expectEqualStrings("SendFailed (cause BrokenPipe)", state.lastError());
+}
+
+test "every mutation advances the revision" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    var previous: u64 = 0;
+
+    // The successes here are all the newest outcome, so none of them is the
+    // one case that projects nothing.
+    for (0..8) |i| {
+        const effect = if (i % 3 == 0)
+            state.recordSuccess(ts(ms(@intCast(i))), cfg).?
+        else
+            state.recordFailure(ts(ms(@intCast(i))), peerFault(error.Timeout), cfg, 0);
+        try testing.expect(effect.revision > previous);
+        previous = effect.revision;
+    }
 }
@@ -23,6 +23,7 @@ const tls = std.crypto.tls;
 const doh_client = @import("doh_client.zig");
 const dot_client = @import("dot_client.zig");
 const events = @import("../storage/events.zig");
+const events_fixture = @import("../storage/events_fixture.zig");
 const health = @import("health.zig");
 const model = @import("../config/model.zig");
 const pool_mod = @import("pool.zig");
@@ -285,6 +286,11 @@ pub const Owner = struct {
    /// `old.refs == 0` comparison and the branch on its result, and `replace`
    /// then returns null for every input. See AGENTS.md.
    published: u64 = 0,
+    /// The Diagnostics store, for the `upstream.exchange` reconciliation at
+    /// boot and at every retirement. Defaulted rather than an `init` parameter,
+    /// for the reason `Pool.diagnostics` is: the composition root wires it
+    /// after the owner exists, and every unit test here runs without one.
+    diagnostics: ?*events.Store = null,

    pub fn init(live: *Generation) Owner {
        return .{ .live = live };
@@ -323,13 +329,84 @@ pub const Owner = struct {
        return copies;
    }

+    /// Drops one pin, and tears the generation down when it was retired and
+    /// this was its last reader.
    pub fn release(self: *Owner, io: std.Io, generation: *Generation) void {
+        if (self.drop(io, generation)) self.retireDisplaced(io, generation);
+    }
+
+    /// Drops one pin and says whether that made the generation this caller's to
+    /// tear down. The mutex is released before the caller acts on the answer,
+    /// because tearing a generation down closes sockets.
+    fn drop(self: *Owner, io: std.Io, generation: *Generation) bool {
        self.mutex.lockUncancelable(io);
+        defer self.mutex.unlock(io);
        std.debug.assert(generation.refs > 0);
        generation.refs -= 1;
-        const retire_it = generation.retired and generation.refs == 0;
-        self.mutex.unlock(io);
-        if (retire_it) generation.retire(io);
+        return generation.retired and generation.refs == 0;
+    }
+
+    /// Tears a displaced generation down and reconciles its episodes against
+    /// whatever is live now.
+    ///
+    /// The one retire-and-reconcile, and both paths a displaced generation can
+    /// leave by call it: `release` when the last reader of a retired generation
+    /// goes, and the publisher when `replace` found no reader at all and handed
+    /// the generation back. Two call sites and one rule, so the idle path
+    /// cannot skip the reconciliation the pinned path does — which is what it
+    /// did before this.
+    ///
+    /// This instant is the whole point: after it, no exchange of `generation`
+    /// can project anything onto the store, so whatever it left standing is
+    /// nobody's truth and the live generation's health is the answer.
+    ///
+    /// The loop is the pin the reconcile itself takes. Reconciling reads the
+    /// generation that is live now, which a concurrent `replace` can retire
+    /// under it, and dropping that pin lands back here. One iteration per
+    /// concurrent replace, and a replace is a configuration write.
+    pub fn retireDisplaced(self: *Owner, io: std.Io, generation: *Generation) void {
+        // Only the publisher's idle path and a reader's last release reach
+        // here, and both hold a generation the swap already displaced.
+        std.debug.assert(generation.retired);
+        std.debug.assert(generation.refs == 0);
+
+        var target = generation;
+        while (true) {
+            target.retire(io);
+            // No store means nothing to reconcile, and then no pin to take.
+            if (self.diagnostics == null) return;
+
+            const live = self.acquire(io);
+            self.reconcileDiagnostics(io, live);
+            if (!self.drop(io, live)) return;
+            target = live;
+        }
+    }
+
+    /// Closes every `upstream.exchange` episode `generation` cannot justify,
+    /// and projects the health of the endpoints it can.
+    ///
+    /// `resolveExcept` rather than the scoped walk `reconcileReport` does,
+    /// because the two codes have different owners. Every `upstream.exchange`
+    /// episode in the store was opened by a pool of this process, so an active
+    /// one outside the kept set names an upstream that was removed, disabled or
+    /// failed to build, and closing it is right. `configuration.load` is shared
+    /// with the boot collector, which is why its rule stays scoped.
+    ///
+    /// Called at the two points revision order cannot reach: the first
+    /// generation at boot, and the retirement of a displaced one.
+    pub fn reconcileDiagnostics(self: *Owner, io: std.Io, generation: *Generation) void {
+        const store = self.diagnostics orelse return;
+        const pool = generation.pool orelse return;
+
+        // The store refuses a kept list longer than it can canonicalize, and
+        // refusing is right there, so the two bounds must agree rather than one
+        // silently clipping the other.
+        comptime std.debug.assert(pool_mod.Pool.max_entries <= events.Store.max_kept_keys);
+        var kept: [pool_mod.Pool.max_entries][]const u8 = undefined;
+        const count = pool.enabledUrls(&kept);
+        store.resolveExcept(io, std.Io.Clock.real.now(io).toSeconds(), .upstream_exchange, kept[0..count]);
+        pool.reconcile(io);
    }

    /// Publishes `prepared` and retires the live generation. Infallible and
@@ -534,7 +611,7 @@ const Upstreams = struct {
                self.doh_buf[base + doh_request_buf_len ..][0..doh_transfer_buf_len],
            ) catch return false;
            self.doh_used = index + 1;
-            slot.* = .{ .client = self.doh[index].client() };
+            slot.* = .{ .client = self.doh[index].leaf() };
        }
        return true;
    }
@@ -571,7 +648,7 @@ const Upstreams = struct {
                },
            );
            self.dot_used = index + 1;
-            slot.* = .{ .client = self.dot[index].client() };
+            slot.* = .{ .client = self.dot[index].leaf() };
        }
    }

@@ -679,6 +756,21 @@ fn buildTestGeneration(
    bundle: *Certificate.Bundle,
    bundle_lock: *std.Io.RwLock,
    servers: []const model.UpstreamServer,
+) BuildError!*Generation {
+    return buildTestGenerationWith(io, gpa, http, bundle, bundle_lock, servers, null);
+}
+
+/// The generation a reconciliation test needs: its pool projects onto the same
+/// store the owner reconciles against, which is how the composition root wires
+/// the two.
+fn buildTestGenerationWith(
+    io: std.Io,
+    gpa: Allocator,
+    http: *std.http.Client,
+    bundle: *Certificate.Bundle,
+    bundle_lock: *std.Io.RwLock,
+    servers: []const model.UpstreamServer,
+    store: ?*events.Store,
 ) BuildError!*Generation {
    return build(.{
        .gpa = gpa,
@@ -689,6 +781,7 @@ fn buildTestGeneration(
        .bundle_lock = bundle_lock,
        .timeouts = test_timeouts,
        .seed = 1,
+        .diagnostics = store,
    });
 }

@@ -743,7 +836,7 @@ test "a replace with no reader holding the live generation retires it through th
    // Nobody holds G1: `replace` must hand it back, because no release will.
    const displaced = owner.replace(io, g2) orelse return error.ExpectedIdleGeneration;
    try testing.expectEqual(g1, displaced);
-    displaced.retire(io);
+    owner.retireDisplaced(io, displaced);

    owner.deinit(io);
 }
@@ -918,9 +1011,177 @@ test "a metrics scrape running against the owner survives a replace under it" {
    var future = try io.concurrent(Scrape.run, .{ &owner, io, &started, &seen });

    started.waitUncancelable(io);
-    if (owner.replace(io, g2)) |old| old.retire(io);
+    if (owner.replace(io, g2)) |old| owner.retireDisplaced(io, old);

    future.await(io);
    // Every one of the 256 scrapes read at least one intact URL.
    try testing.expect(seen >= 256);
 }
+
+test "boot reconciliation closes the episodes of upstreams this generation does not serve" {
+    var t: TestIo = .init(testing.allocator);
+    defer t.deinit();
+    const io = t.io();
+
+    var http: std.http.Client = .{ .allocator = testing.allocator, .io = io };
+    defer http.deinit();
+    var bundle: Certificate.Bundle = .empty;
+    defer bundle.deinit(testing.allocator);
+    var bundle_lock: std.Io.RwLock = .init;
+
+    // Two episodes outlived the process that opened them. One names the
+    // upstream this generation serves and one does not, and neither has a pool
+    // that could ever resolve it through an exchange: the first has recorded
+    // nothing yet, and the second no longer exists in the configuration at all.
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+    fx.store.report(io, 1000, .upstream_exchange, "https://kept.example/dns-query", "https://kept.example", .warning, "before the restart");
+    fx.store.report(io, 1000, .upstream_exchange, "https://gone.example/dns-query", "https://gone.example", .warning, "before the restart");
+
+    const generation = try buildTestGenerationWith(io, testing.allocator, &http, &bundle, &bundle_lock, &.{
+        .{ .url = "https://kept.example/dns-query" },
+    }, &fx.store);
+    var owner: Owner = .init(generation);
+    defer owner.deinit(io);
+    owner.diagnostics = &fx.store;
+
+    owner.reconcileDiagnostics(io, generation);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "an idle replace that drops an upstream closes its episode through retireDisplaced" {
+    var t: TestIo = .init(testing.allocator);
+    defer t.deinit();
+    const io = t.io();
+
+    var http: std.http.Client = .{ .allocator = testing.allocator, .io = io };
+    defer http.deinit();
+    var bundle: Certificate.Bundle = .empty;
+    defer bundle.deinit(testing.allocator);
+    var bundle_lock: std.Io.RwLock = .init;
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    const g1 = try buildTestGenerationWith(io, testing.allocator, &http, &bundle, &bundle_lock, &.{
+        .{ .url = "https://gone.example/dns-query" },
+    }, &fx.store);
+    const g2 = try buildTestGenerationWith(io, testing.allocator, &http, &bundle, &bundle_lock, &.{
+        .{ .url = "https://kept.example/dns-query" },
+    }, &fx.store);
+
+    var owner: Owner = .init(g1);
+    defer owner.deinit(io);
+    owner.diagnostics = &fx.store;
+
+    fx.store.report(io, 1100, .upstream_exchange, "https://gone.example/dns-query", "https://gone.example", .warning, "before the reload");
+
+    // No reader holds G1, so `replace` hands it back for the publisher to tear
+    // down. That teardown is the reconciliation point for the upstream the new
+    // configuration no longer serves.
+    const displaced = owner.replace(io, g2) orelse return error.TestUnexpectedResult;
+    try testing.expectEqual(
+        @as(i64, 1),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+
+    owner.retireDisplaced(io, displaced);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "an idle replace resolves an episode the live pool never opened" {
+    var t: TestIo = .init(testing.allocator);
+    defer t.deinit();
+    const io = t.io();
+
+    var http: std.http.Client = .{ .allocator = testing.allocator, .io = io };
+    defer http.deinit();
+    var bundle: Certificate.Bundle = .empty;
+    defer bundle.deinit(testing.allocator);
+    var bundle_lock: std.Io.RwLock = .init;
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    const g1 = try buildTestGenerationWith(io, testing.allocator, &http, &bundle, &bundle_lock, &.{
+        .{ .url = "https://one.example/dns-query" },
+    }, &fx.store);
+    const g2 = try buildTestGenerationWith(io, testing.allocator, &http, &bundle, &bundle_lock, &.{
+        .{ .url = "https://one.example/dns-query" },
+    }, &fx.store);
+
+    var owner: Owner = .init(g1);
+    defer owner.deinit(io);
+    owner.diagnostics = &fx.store;
+
+    // The same upstream survives the reload, so `resolveExcept` keeps the card
+    // and only the fresh pool's own projection can close it. That pool has
+    // recorded nothing, so it projects clear.
+    fx.store.report(io, 1100, .upstream_exchange, "https://one.example/dns-query", "https://one.example", .warning, "opened by the old generation");
+
+    const displaced = owner.replace(io, g2) orelse return error.TestUnexpectedResult;
+    owner.retireDisplaced(io, displaced);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "a retired generation's late report is corrected when its last reader leaves" {
+    var t: TestIo = .init(testing.allocator);
+    defer t.deinit();
+    const io = t.io();
+
+    var http: std.http.Client = .{ .allocator = testing.allocator, .io = io };
+    defer http.deinit();
+    var bundle: Certificate.Bundle = .empty;
+    defer bundle.deinit(testing.allocator);
+    var bundle_lock: std.Io.RwLock = .init;
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    const g1 = try buildTestGenerationWith(io, testing.allocator, &http, &bundle, &bundle_lock, &.{
+        .{ .url = "https://one.example/dns-query" },
+    }, &fx.store);
+    const g2 = try buildTestGenerationWith(io, testing.allocator, &http, &bundle, &bundle_lock, &.{
+        .{ .url = "https://one.example/dns-query" },
+    }, &fx.store);
+
+    var owner: Owner = .init(g1);
+    defer owner.deinit(io);
+    owner.diagnostics = &fx.store;
+
+    // A reader pins G1 across the replace, which is what lets G1 project after
+    // G2 is live. Revisions are per pool, so G2 has no way to know that report
+    // happened; the release below is the point after which G1 can project no
+    // more, and reconciling there is what corrects it.
+    const held = owner.acquire(io);
+    try testing.expectEqual(@as(?*Generation, null), owner.replace(io, g2));
+
+    fx.store.report(io, 1100, .upstream_exchange, "https://one.example/dns-query", "https://one.example", .warning, "the retired generation's last word");
+    try testing.expectEqual(
+        @as(i64, 1),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+
+    owner.release(io, held);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
@@ -190,6 +190,11 @@ pub const LocalResource = error{

 pub const Cancellation = error{Canceled};

+/// What a leaf client may fail with. A leaf never *errors* on a peer fault: it
+/// returns one as an `Outcome.fault` value, because the pool needs the concrete
+/// cause and an error cannot carry one.
+pub const LeafError = LocalResource || Cancellation;
+
 /// The caller's own time ran out before any peer could be given the observation
 /// interval it was configured to get. Evidence about this process's budget, not
 /// about any endpoint, so it is never recorded against health — that is the
@@ -238,7 +243,7 @@ pub fn group(err: ExchangeError) Group {
 /// This is the only place a foreign error set is folded in. Everywhere else
 /// the call site names the peer fault it means, because the call site is what
 /// knows whether it was connecting, sending or receiving.
-pub fn mapLocal(err: anyerror) ?ExchangeError {
+pub fn mapLocal(err: anyerror) ?LeafError {
    return switch (err) {
        error.OutOfMemory => error.OutOfMemory,
        error.SystemResources => error.SystemResources,
@@ -279,21 +284,37 @@ pub fn closeBlocked(io: std.Io, target: anytype) void {
    }
 }

-/// The payload of `f`'s return type, which the race harness requires to be
-/// `ExchangeError!T`. A raced function with any other error set would let a
-/// failure reach the pool without passing through `group`.
-fn RacedPayload(comptime f: anytype) type {
+/// The error union `f` returns, which the race harness requires it to have.
+fn racedUnion(comptime f: anytype) std.builtin.Type.ErrorUnion {
    const info = @typeInfo(@TypeOf(f));
    if (info != .@"fn") @compileError("the race harness needs a function, found " ++ @typeName(@TypeOf(f)));
    const Return = info.@"fn".return_type orelse
        @compileError("the race harness needs a function with a concrete return type");
-    const union_info = switch (@typeInfo(Return)) {
+    return switch (@typeInfo(Return)) {
        .error_union => |u| u,
-        else => @compileError("the race harness needs `ExchangeError!T`, found " ++ @typeName(Return)),
+        else => @compileError("the race harness needs `E!T`, found " ++ @typeName(Return)),
    };
-    if (union_info.error_set != ExchangeError)
-        @compileError("the race harness needs `ExchangeError!T`, found " ++ @typeName(Return));
-    return union_info.payload;
+}
+
+/// The payload of `f`'s return type.
+fn RacedPayload(comptime f: anytype) type {
+    return racedUnion(f).payload;
+}
+
+/// `f`'s own error set.
+fn RacedError(comptime f: anytype) type {
+    return racedUnion(f).error_set;
+}
+
+/// What racing `f` can fail with: `f`'s own errors, plus the three the harness
+/// itself produces — the expiry, a backend that cannot start a second task, and
+/// the whole task being torn down.
+///
+/// Derived rather than fixed at `ExchangeError`, so a caller's narrow error set
+/// survives the race. That is what lets the pool prove at the type level that a
+/// leaf cannot hand it a `PeerFault`, instead of asserting it.
+pub fn RaceError(comptime f: anytype) type {
+    return RacedError(f) || error{ Timeout, SystemResources, Canceled };
 }

 /// Runs `f(args...)` raced against `budget`, and cancels the loser.
@@ -312,7 +333,7 @@ pub fn raceWithin(
    budget: std.Io.Clock.Duration,
    comptime f: anytype,
    args: anytype,
-) ExchangeError!RacedPayload(f) {
+) RaceError(f)!RacedPayload(f) {
    var outcome: RaceOutcome = .completed;
    return raceUntilTagged(io, .fromNow(io, budget), &outcome, f, args);
 }
@@ -344,9 +365,9 @@ pub fn raceUntilTagged(
    outcome: *RaceOutcome,
    comptime f: anytype,
    args: anytype,
-) ExchangeError!RacedPayload(f) {
+) RaceError(f)!RacedPayload(f) {
    const Slot = union(enum) {
-        raced: ExchangeError!RacedPayload(f),
+        raced: RacedError(f)!RacedPayload(f),
        expiry: std.Io.Cancelable!void,
    };

@@ -378,8 +399,12 @@ fn expire(io: std.Io, expiry_at: std.Io.Clock.Timestamp) std.Io.Cancelable!void
    return expiry_at.wait(io);
 }

-/// A thing that sends one DNS message and returns one validated DNS message.
-/// Implemented by DohClient, DotClient, Pool, and test fakes.
+/// A thing that sends one DNS message and returns one validated DNS message,
+/// with a peer fault already reduced to an error.
+///
+/// Implemented by `Pool`, the forward client, and the fakes that stand in for
+/// either. The leaf clients are on the other side of the pool and implement
+/// `Leaf` instead, which keeps the fault as a value.
 pub const Client = struct {
    ptr: *anyopaque,
    exchangeFn: *const fn (
@@ -394,7 +419,8 @@ pub const Client = struct {
    /// passed `validateResponse` against `query`.
    ///
    /// `selected` names the resolver the exchange used. A single-endpoint
-    /// implementation (DoH, DoT, the forward client, test fakes) may write it
+    /// implementation — the forward client and the test fakes; the DoH and DoT
+    /// clients are `Leaf`s and a `Pool` carries their answer here — may write it
    /// *before* each attempt: it has one resolver and records no health, so
    /// "the one I tried" is an honest answer even for a failure, and a SERVFAIL
    /// row without its resolver explains nothing.
@@ -419,6 +445,70 @@ pub const Client = struct {
    }
 };

+/// One peer fault as a value: the taxonomy `PeerFault` names, plus the concrete
+/// error that produced it.
+///
+/// The classification is what health and backoff count; the cause is what tells
+/// an operator which failure it was. `SendFailed` alone cannot separate a peer
+/// that reset the connection from one whose TLS record was rejected, and the
+/// leaf that unwrapped the cause is the only place that still holds it.
+///
+/// There is no phase field: the taxonomy already names the phase for every kind
+/// that has one, and `TlsFailed` cannot say where it failed.
+pub const Fault = struct {
+    kind: PeerFault,
+    cause: anyerror,
+
+    /// The one text every surface prints. `<Kind> (cause <Cause>)`.
+    pub fn format(self: Fault, w: *std.Io.Writer) std.Io.Writer.Error!void {
+        try w.print("{t} (cause {s})", .{ self.kind, @errorName(self.cause) });
+    }
+};
+
+/// What one exchange against one endpoint produced.
+pub const Outcome = union(enum) {
+    /// A prefix of the caller's `response_buf`, already validated against the
+    /// query.
+    reply: []u8,
+    fault: Fault,
+};
+
+/// A client of exactly one endpoint: DoH, DoT, and the pool's test fakes.
+///
+/// Separate from `Client` because the two answer different questions. A `Leaf`
+/// reports what the peer did, faults included, and leaves every judgement to
+/// its caller. A `Client` is the resolver the handler asks for an answer, and a
+/// fault has already become an error by the time it is reached.
+///
+/// No `selected` out-parameter: the pool discards a leaf's own identity anyway,
+/// since the entry's endpoint is the pool's naming of the same resolver.
+pub const Leaf = struct {
+    ptr: *anyopaque,
+    exchangeFn: *const fn (
+        ptr: *anyopaque,
+        io: std.Io,
+        query: []const u8,
+        response_buf: []u8,
+    ) LeafError!Outcome,
+
+    pub fn exchange(
+        self: Leaf,
+        io: std.Io,
+        query: []const u8,
+        response_buf: []u8,
+    ) LeafError!Outcome {
+        return self.exchangeFn(self.ptr, io, query, response_buf);
+    }
+};
+
+/// The fault a call site's phase means, unless `err` is one this process owns.
+/// The leaf counterpart of `mapPhase`: same rule, but the peer case comes back
+/// as a value carrying the cause instead of as a bare error.
+pub fn faultOrLocal(err: anyerror, phase: PeerFault) LeafError!Fault {
+    if (mapLocal(err)) |local| return local;
+    return .{ .kind = phase, .cause = err };
+}
+
 /// The unwrap helpers below turn the single collapsed error `std.http.Client`
 /// reports into the concrete cause it stashed. Every HTTP caller in this tree
 /// uses them: the DoH client classifies by the unwrapped cause, the blocklist
@@ -743,14 +833,16 @@ test "raceWithin passes the raced task's own failure through" {
    );
 }

-test "raceUntilTagged tells a leaf Timeout apart from an expiry" {
+test "raceUntilTagged tells a raced Timeout apart from an expiry" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

-    // The leaf's own timeout: it returned, so the peer really did time out and
-    // the outcome is `completed` even though the error is the same one an
-    // expiry produces.
+    // A raced function that returned `error.Timeout` of its own: it completed,
+    // so the tag says `completed` even though the error is the one an expiry
+    // produces. No leaf does this — a leaf's own timeout is a `Fault` — but the
+    // harness serves callers with any error set, and the tag is what separates
+    // the two for every one of them.
    var outcome: RaceOutcome = .expired;
    const far: std.Io.Clock.Timestamp = .fromNow(io, .{ .raw = .fromSeconds(30), .clock = .awake });
    try testing.expectError(
@@ -636,9 +636,16 @@ pub const Plan = struct {
                );
            }
            // A generation no reader held at the swap has no release left to
-            // tear it down, so the publisher does — after the reconciliation,
-            // which reads only the copies taken at prepare.
-            if (self.retired_upstream) |old| old.retire(io);
+            // tear it down, so the publisher does — after the reconciliation
+            // above, which reads only the copies taken at prepare.
+            //
+            // `retireDisplaced`, not `retire`: the teardown of a displaced
+            // generation is also when its `upstream.exchange` episodes are
+            // reconciled, and this path and a reader's release are the two ways
+            // a generation reaches it. The `configuration.load` half above
+            // stays here, because that code is shared with the boot collector
+            // and its rule is scoped.
+            if (self.retired_upstream) |old| self.state.upstreams.?.retireDisplaced(io, old);
        }

        self.* = undefined;
@@ -1520,18 +1520,16 @@ const AnsweringClient = struct {
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
-        selected: *?[]const u8,
-    ) transport.ExchangeError![]u8 {
+    ) transport.LeafError!transport.Outcome {
        _ = io;
        _ = query;
        const self: *AnsweringClient = @ptrCast(@alignCast(ptr));
        _ = self.calls.fetchAdd(1, .acq_rel);
-        selected.* = "fake://leaf";
        @memcpy(response_buf[0..reply.len], reply);
-        return response_buf[0..reply.len];
+        return .{ .reply = response_buf[0..reply.len] };
    }

-    fn client(self: *AnsweringClient) transport.Client {
+    fn leaf(self: *AnsweringClient) transport.Leaf {
        return .{ .ptr = self, .exchangeFn = exchangeFn };
    }
 };
@@ -1546,10 +1544,10 @@ test "the queue families carry what a real pool recorded, through the real snaps
    defer threaded.deinit();
    const io = threaded.io();

-    var leaf: AnsweringClient = .{};
+    var answering: AnsweringClient = .{};
    var slots = [_]pool_mod.Slot{
-        .{ .client = leaf.client() },
-        .{ .client = leaf.client() },
+        .{ .client = answering.leaf() },
+        .{ .client = answering.leaf() },
    };
    var recoveries: std.atomic.Value(u64) = .init(0);
    var entries = [_]pool_mod.Entry{.{
@@ -1569,7 +1567,7 @@ test "the queue families carry what a real pool recorded, through the real snaps
    var buf: [512]u8 = undefined;
    var selected: ?[]const u8 = null;
    _ = try pool.exchange(io, "\x12\x34\x01\x00", &buf, &selected);
-    try testing.expectEqual(@as(u32, 1), leaf.calls.load(.acquire));
+    try testing.expectEqual(@as(u32, 1), answering.calls.load(.acquire));

    // The counters a burst would move, written straight into the entry: this
    // test is about what the snapshot path carries, not about reproducing a