upstream: a diagnostics episode follows health, and a peer fault carries its cause

src/upstream/health.zig

@@ -20,7 +20,7 @@
 //!   * Data is data. `total_successes`, `total_failures`, the window and the
 //!     `@max` of `last_success_at` / `last_error_at` always update, however old
 //!     `at` is.
-//!   * `consecutive_failures`, `backoff_until` and `last_error_buf` describe
+//!   * `consecutive_failures`, `backoff_until` and the last fault describe
 //!     the present, so a newer recorded outcome overrules a stale call.
 //!
 //! `recordSuccess` clears `consecutive_failures` and `backoff_until` only when
@@ -33,25 +33,67 @@
 //! `State` carries no lock. The pool owns the mutex.
 
 const std = @import("std");
+const transport = @import("transport.zig");
 
 pub const Config = struct {
-    /// Consecutive peer faults before the endpoint is put in backoff.
+    /// Consecutive peer faults before the endpoint is put in backoff. At least
+    /// 1: a state tripped by zero failures has no fault to name, and the
+    /// episode the pool opens from it could not say what went wrong. Nothing
+    /// builds this from user input, so the default is checked below and
+    /// `Pool.init` asserts what a caller passes.
     failure_threshold: u8 = 2,
     base_backoff_ms: u32 = 500,
     max_backoff_ms: u32 = 60_000,
 };
 
+comptime {
+    const default: Config = .{};
+    if (default.failure_threshold < 1) @compileError("failure_threshold must be at least 1");
+}
+
 /// Rolling success-rate window, in samples. Equal to the bit width of
 /// `State.window`.
 pub const window_len = 32;
 
-/// Bytes kept of an `@errorName`, truncated to fit.
-pub const error_name_capacity = 48;
+/// Bytes kept of a rendered `transport.Fault`, truncated to fit. Sized so the
+/// widest classification and the widest concrete cause name fit whole; the
+/// comptime-driven test below recomputes that worst case from the std error
+/// sets, so a wider name std adds fails the test run rather than silently
+/// truncating an operator's only diagnostic.
+pub const error_name_capacity = 64;
 
 /// The shift is capped so `base_backoff_ms << shift` cannot run away; by then
 /// `max_backoff_ms` has clamped the result many doublings ago.
 const max_shift = 20;
 
+/// The complete desired state of this endpoint's Diagnostics episode after one
+/// recorded outcome. This file decides it, because "failing" is this file's
+/// predicate and nobody else's; the pool projects it onto the store.
+///
+/// A state, never a transition. A transition has to be applied to whatever the
+/// store holds, so two effects that reach the pool out of order can leave the
+/// wrong one standing: a "nothing changed" overtaking a required report would
+/// drop the report for good. A desired state makes last-writer-wins by
+/// revision correct: a projection of the whole state is also what repairs a
+/// card left over from a latched store write, on the next outcome. What a
+/// per-pool revision cannot see — an episode that outlived a restart, or one a
+/// retired generation opened — is repaired by `Pool.reconcile` instead, at boot
+/// and at every retirement, not by waiting for an outcome.
+pub const Effect = struct {
+    /// Per-state counter, incremented by every mutation. The pool applies
+    /// effects in this order and drops one that arrives behind a newer sibling.
+    revision: u64,
+    state: Episode,
+
+    /// A tagged union rather than an enum beside a fault field, so an episode
+    /// without the fault that explains it is unrepresentable. Named `Episode`
+    /// rather than `State` because this file's `State` is the health record.
+    pub const Episode = union(enum) {
+        clear,
+        tripped: transport.Fault,
+    };
+};
+
 pub const State = struct {
     consecutive_failures: u32,
     total_successes: u64,
@@ -62,6 +104,12 @@ pub const State = struct {
     /// Length of the `@errorName` held in `last_error_buf`, truncated to fit.
     last_error_len: u8,
     backoff_until: ?std.Io.Timestamp,
+    /// The effective last fault, as a value. `last_error_buf` is its rendering;
+    /// both are kept because the text is what every surface prints and the
+    /// value is what an `Effect` carries out of the mutex.
+    last_fault: ?transport.Fault,
+    /// Incremented by every mutation; see `Effect.revision`.
+    revision: u64,
     /// Bitset, 1 = success, LSB = most recent.
     window: u32,
     window_filled: u8,
@@ -75,11 +123,22 @@ pub const State = struct {
         .last_error_buf = @splat(0),
         .last_error_len = 0,
         .backoff_until = null,
+        .last_fault = null,
+        .revision = 0,
         .window = 0,
         .window_filled = 0,
     };
 
-    pub fn recordSuccess(self: *State, at: std.Io.Timestamp) void {
+    /// A success that is the newest outcome clears the count, and the effect
+    /// then says `clear`.
+    ///
+    /// `null` when the state stays tripped, which is the stale success behind a
+    /// newer failure (case 1 at `recordFailure`). The card that failure opened
+    /// is already right, and projecting `tripped` again would count a
+    /// successful exchange as an occurrence of the episode. A `clear`-shaped
+    /// no-op is not an option either: it would advance `applied_revision` past
+    /// a report still in flight and drop it.
+    pub fn recordSuccess(self: *State, at: std.Io.Timestamp, cfg: Config) ?Effect {
         if (self.isNewestOutcome(at)) {
             self.consecutive_failures = 0;
             self.backoff_until = null;
@@ -87,6 +146,32 @@ pub const State = struct {
         self.total_successes += 1;
         self.push(1);
         self.last_success_at = later(self.last_success_at, at);
+        self.revision += 1;
+        if (self.tripped(cfg)) return null;
+        return self.effect(cfg);
+    }
+
+    /// The pool's definition of "failing this endpoint": the predicate that
+    /// puts it in backoff. Not `available`, which turns true again the moment a
+    /// backoff expires, before any recovery is proven.
+    pub fn tripped(self: *const State, cfg: Config) bool {
+        return self.consecutive_failures >= cfg.failure_threshold;
+    }
+
+    /// The desired episode state this endpoint is in right now, without a
+    /// mutation and without advancing the revision. The pool projects this at
+    /// the two points a recorded outcome cannot reach: the first generation's
+    /// boot, and the retirement of a generation whose late outcomes are now
+    /// impossible.
+    pub fn currentEffect(self: *const State, cfg: Config) Effect {
+        return self.effect(cfg);
+    }
+
+    /// A tripped state has always recorded a failure, because the threshold is
+    /// at least 1, so the fault is always there to name.
+    fn effect(self: *const State, cfg: Config) Effect {
+        if (!self.tripped(cfg)) return .{ .revision = self.revision, .state = .clear };
+        return .{ .revision = self.revision, .state = .{ .tripped = self.last_fault.? } };
     }
 
     /// True when no recorded outcome is newer than `at`. Both timestamp fields
@@ -96,9 +181,9 @@ pub const State = struct {
         return !newerThan(self.last_success_at, at) and !newerThan(self.last_error_at, at);
     }
 
-    /// `err_name` is `@errorName` of a PeerFault member. `rand` supplies
-    /// jitter; the caller owns the RNG so this stays pure and the test is
-    /// deterministic.
+    /// `fault` is the peer fault as a value: `transport.group` has already
+    /// ruled that this is the peer's doing. `rand` supplies jitter; the caller
+    /// owns the RNG so this stays pure and the test is deterministic.
     ///
     /// A stale failure, that is one whose `at` is older than an outcome already
     /// recorded, is held to the mirror image of the stale-success rule. Three
@@ -119,27 +204,29 @@ pub const State = struct {
     pub fn recordFailure(
         self: *State,
         at: std.Io.Timestamp,
-        err_name: []const u8,
+        fault: transport.Fault,
         cfg: Config,
         rand: u32,
-    ) void {
+    ) Effect {
         const newer_success = newerThan(self.last_success_at, at);
         const newer_failure = newerThan(self.last_error_at, at);
 
         if (!newer_failure) {
-            const copied = @min(err_name.len, self.last_error_buf.len);
-            @memcpy(self.last_error_buf[0..copied], err_name[0..copied]);
-            self.last_error_len = @intCast(copied);
+            self.last_fault = fault;
+            var writer: std.Io.Writer = .fixed(&self.last_error_buf);
+            writer.print("{f}", .{fault}) catch {};
+            self.last_error_len = @intCast(writer.end);
         }
 
         self.total_failures += 1;
         self.push(0);
         self.last_error_at = later(self.last_error_at, at);
+        self.revision += 1;
 
-        if (newer_success) return;
+        if (newer_success) return self.effect(cfg);
 
         self.consecutive_failures +|= 1;
-        if (self.consecutive_failures < cfg.failure_threshold) return;
+        if (self.consecutive_failures < cfg.failure_threshold) return self.effect(cfg);
 
         const delay_ms = backoffDelayMs(self.consecutive_failures, cfg);
         const half = delay_ms / 2;
@@ -148,6 +235,7 @@ pub const State = struct {
             .nanoseconds = at.nanoseconds + @as(i96, jittered) * std.time.ns_per_ms,
         };
         self.backoff_until = later(self.backoff_until, deadline);
+        return self.effect(cfg);
     }
 
     pub fn available(self: *const State, now: std.Io.Timestamp) bool {
@@ -212,10 +300,16 @@ fn ms(count: i96) i96 {
     return count * std.time.ns_per_ms;
 }
 
+/// A fault whose cause is its own classification: the shape an expiry has, and
+/// short enough to keep the timestamp tests about timestamps.
+fn peerFault(kind: transport.PeerFault) transport.Fault {
+    return .{ .kind = kind, .cause = kind };
+}
+
 test "a failure below the threshold leaves the endpoint available" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
 
     try testing.expectEqual(@as(u32, 1), state.consecutive_failures);
     try testing.expectEqual(@as(u64, 1), state.total_failures);
@@ -226,8 +320,8 @@ test "a failure below the threshold leaves the endpoint available" {
 test "reaching the threshold puts the endpoint in backoff until the deadline" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
 
     // Two failures, threshold 2, shift 0: delay 500 ms, jitter 0 => 250 ms.
     const until = state.backoff_until.?;
@@ -244,7 +338,7 @@ test "consecutive failures grow the delay and saturate at max_backoff_ms" {
     var previous: i96 = -1;
     var i: usize = 0;
     while (i < 40) : (i += 1) {
-        state.recordFailure(ts(0), "Timeout", cfg, 0);
+        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
         if (state.backoff_until) |until| {
             try testing.expect(until.nanoseconds >= previous);
             previous = until.nanoseconds;
@@ -262,11 +356,11 @@ test "consecutive failures grow the delay and saturate at max_backoff_ms" {
 test "a success resets the consecutive count, the window and the backoff" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
     try testing.expect(state.backoff_until != null);
 
-    state.recordSuccess(ts(ms(1)));
+    _ = state.recordSuccess(ts(ms(1)), cfg);
     try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
     try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
     try testing.expectEqual(@as(u64, 1), state.total_successes);
@@ -280,8 +374,8 @@ test "jitter stays inside half the delay and the whole delay" {
 
     for ([_]u32{ 0, std.math.maxInt(u32), 1, 12345 }) |rand| {
         var state: State = .init;
-        state.recordFailure(ts(0), "Timeout", cfg, rand);
-        state.recordFailure(ts(0), "Timeout", cfg, rand);
+        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, rand);
+        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, rand);
         const offset = state.backoff_until.?.nanoseconds;
         try testing.expect(offset >= ms(delay / 2));
         try testing.expect(offset <= ms(delay));
@@ -289,20 +383,21 @@ test "jitter stays inside half the delay and the whole delay" {
 }
 
 test "an out-of-order success does not move last_success_at backwards" {
+    const cfg: Config = .{};
     var state: State = .init;
-    state.recordSuccess(ts(100));
-    state.recordSuccess(ts(50));
+    _ = state.recordSuccess(ts(100), cfg);
+    _ = state.recordSuccess(ts(50), cfg);
     try testing.expectEqual(@as(i96, 100), state.last_success_at.?.nanoseconds);
 }
 
 test "an out-of-order failure does not shorten the backoff" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
     const until = state.backoff_until.?.nanoseconds;
 
-    state.recordFailure(ts(ms(50)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(50)), peerFault(error.Timeout), cfg, 0);
     try testing.expect(state.backoff_until.?.nanoseconds >= until);
     try testing.expectEqual(@as(i96, ms(100)), state.last_error_at.?.nanoseconds);
 }
@@ -310,11 +405,11 @@ test "an out-of-order failure does not shorten the backoff" {
 test "an out-of-order success does not clear the backoff of a newer failure" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
     const until = state.backoff_until.?.nanoseconds;
 
-    state.recordSuccess(ts(ms(50)));
+    _ = state.recordSuccess(ts(ms(50)), cfg);
     try testing.expectEqual(@as(i96, until), state.backoff_until.?.nanoseconds);
     try testing.expectEqual(@as(u32, 2), state.consecutive_failures);
     try testing.expectEqual(@as(u64, 1), state.total_successes);
@@ -325,10 +420,10 @@ test "an out-of-order success does not clear the backoff of a newer failure" {
 test "a stale failure behind a newer success does not raise the consecutive count" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(ms(10)), "Timeout", cfg, 0);
-    state.recordSuccess(ts(ms(100)));
-    state.recordFailure(ts(ms(20)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(30)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(10)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordSuccess(ts(ms(100)), cfg);
+    _ = state.recordFailure(ts(ms(20)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(30)), peerFault(error.Timeout), cfg, 0);
 
     try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
     try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
@@ -338,21 +433,21 @@ test "a stale failure behind a newer success does not raise the consecutive coun
 test "a stale failure behind a newer success still counts into the totals" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordSuccess(ts(ms(100)));
-    state.recordFailure(ts(ms(20)), "Timeout", cfg, 0);
+    _ = state.recordSuccess(ts(ms(100)), cfg);
+    _ = state.recordFailure(ts(ms(20)), peerFault(error.Timeout), cfg, 0);
 
     try testing.expectEqual(@as(u64, 1), state.total_failures);
     try testing.expectEqual(@as(u8, 2), state.window_filled);
     try testing.expectEqual(@as(u32, 0), state.window & 1);
     try testing.expectEqual(@as(i96, ms(20)), state.last_error_at.?.nanoseconds);
-    try testing.expectEqualStrings("Timeout", state.lastError());
+    try testing.expectEqualStrings("Timeout (cause Timeout)", state.lastError());
 }
 
 test "a stale failure with no newer success still counts as consecutive" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(50)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(50)), peerFault(error.Timeout), cfg, 0);
 
     try testing.expectEqual(@as(u32, 2), state.consecutive_failures);
     try testing.expect(state.backoff_until != null);
@@ -362,19 +457,19 @@ test "a stale failure with no newer success still counts as consecutive" {
 test "a stale failure does not overwrite the error of a newer failure" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(50)), "ConnectFailed", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(50)), peerFault(error.ConnectFailed), cfg, 0);
 
-    try testing.expectEqualStrings("Timeout", state.lastError());
+    try testing.expectEqualStrings("Timeout (cause Timeout)", state.lastError());
 }
 
 test "the newest failure records its own error and extends the backoff" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(ms(50)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "ConnectFailed", cfg, 0);
+    _ = state.recordFailure(ts(ms(50)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.ConnectFailed), cfg, 0);
 
-    try testing.expectEqualStrings("ConnectFailed", state.lastError());
+    try testing.expectEqualStrings("ConnectFailed (cause ConnectFailed)", state.lastError());
     try testing.expectEqual(@as(u32, 2), state.consecutive_failures);
     try testing.expectEqual(ms(100) + ms(250), state.backoff_until.?.nanoseconds);
 }
@@ -382,11 +477,11 @@ test "the newest failure records its own error and extends the backoff" {
 test "the newest success clears the backoff of an older failure" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
     try testing.expect(state.backoff_until != null);
 
-    state.recordSuccess(ts(ms(101)));
+    _ = state.recordSuccess(ts(ms(101)), cfg);
     try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
     try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
 }
@@ -394,10 +489,10 @@ test "the newest success clears the backoff of an older failure" {
 test "a success at the timestamp of the newest failure clears the backoff" {
     const cfg: Config = .{};
     var state: State = .init;
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
-    state.recordFailure(ts(ms(100)), "Timeout", cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
 
-    state.recordSuccess(ts(ms(100)));
+    _ = state.recordSuccess(ts(ms(100)), cfg);
     try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
     try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
 }
@@ -409,37 +504,176 @@ test "successRate over a half-success window is 0.5" {
 
     var i: usize = 0;
     while (i < 4) : (i += 1) {
-        state.recordSuccess(ts(0));
-        state.recordFailure(ts(0), "Timeout", cfg, 0);
+        _ = state.recordSuccess(ts(0), cfg);
+        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
     }
     try testing.expectEqual(@as(u8, 8), state.window_filled);
     try testing.expectEqual(@as(f32, 0.5), state.successRate());
 }
 
 test "successRate counts only the filled part of the window" {
+    const cfg: Config = .{};
     var state: State = .init;
-    state.recordSuccess(ts(0));
+    _ = state.recordSuccess(ts(0), cfg);
     try testing.expectEqual(@as(f32, 1.0), state.successRate());
 
     var i: usize = 0;
-    while (i < window_len * 2) : (i += 1) state.recordSuccess(ts(0));
+    while (i < window_len * 2) : (i += 1) _ = state.recordSuccess(ts(0), cfg);
     try testing.expectEqual(@as(u8, window_len), state.window_filled);
     try testing.expectEqual(@as(f32, 1.0), state.successRate());
 }
 
-test "lastError returns the last recorded name, truncated not overflowed" {
+test "lastError renders the classification and the concrete cause" {
     const cfg: Config = .{};
     var state: State = .init;
     try testing.expectEqualStrings("", state.lastError());
 
-    state.recordFailure(ts(0), "ConnectFailed", cfg, 0);
-    try testing.expectEqualStrings("ConnectFailed", state.lastError());
+    _ = state.recordFailure(ts(0), .{ .kind = error.SendFailed, .cause = error.BrokenPipe }, cfg, 0);
+    try testing.expectEqualStrings("SendFailed (cause BrokenPipe)", state.lastError());
+    try testing.expectEqual(transport.PeerFault.SendFailed, state.last_fault.?.kind);
 
-    state.recordFailure(ts(0), "Timeout", cfg, 0);
-    try testing.expectEqualStrings("Timeout", state.lastError());
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
+    try testing.expectEqualStrings("Timeout (cause Timeout)", state.lastError());
+}
 
-    const long = "A" ** 200;
-    state.recordFailure(ts(0), long, cfg, 0);
-    try testing.expectEqual(@as(usize, 48), state.lastError().len);
-    try testing.expectEqualStrings(long[0..48], state.lastError());
+test "lastError truncates a cause too wide for the buffer instead of overflowing" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    const Wide = error{AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA};
+    _ = state.recordFailure(
+        ts(0),
+        .{ .kind = error.ReceiveFailed, .cause = Wide.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA },
+        cfg,
+        0,
+    );
+
+    const whole = "ReceiveFailed (cause " ++ "A" ** 64 ++ ")";
+    try testing.expectEqual(@as(usize, error_name_capacity), state.lastError().len);
+    try testing.expectEqualStrings(whole[0..error_name_capacity], state.lastError());
+}
+
+test "error_name_capacity holds the widest classification and cause whole" {
+    // The widest concrete cause an upstream unwrap can produce. The three sets
+    // are the ones `transport.zig`'s unwraps return a member of; a wider name
+    // std adds fails here rather than truncating an operator's diagnostic.
+    const widest_cause = comptime blk: {
+        var widest: []const u8 = "";
+        for (@typeInfo(std.crypto.tls.Client.ReadError).error_set.?) |member| {
+            if (member.name.len > widest.len) widest = member.name;
+        }
+        for (@typeInfo(std.http.Client.RequestError).error_set.?) |member| {
+            if (member.name.len > widest.len) widest = member.name;
+        }
+        for (@typeInfo(std.http.Reader.BodyError).error_set.?) |member| {
+            if (member.name.len > widest.len) widest = member.name;
+        }
+        break :blk widest;
+    };
+    const widest_kind = comptime blk: {
+        var widest: []const u8 = "";
+        for (@typeInfo(transport.PeerFault).error_set.?) |member| {
+            if (member.name.len > widest.len) widest = member.name;
+        }
+        break :blk widest;
+    };
+
+    try testing.expectEqualStrings("DetectingNetworkConfigurationFailed", widest_cause);
+    try testing.expect(widest_kind.len + " (cause ".len + widest_cause.len + ")".len <= error_name_capacity);
+}
+
+/// The fault of a `tripped` effect, or an error when the effect says `clear`.
+fn trippedFault(effect: Effect) !transport.Fault {
+    return switch (effect.state) {
+        .clear => error.TestExpectedTripped,
+        .tripped => |fault| fault,
+    };
+}
+
+test "one failure stays clear and the second trips the episode" {
+    const cfg: Config = .{};
+    var state: State = .init;
+
+    const first = state.recordFailure(ts(0), peerFault(error.ConnectFailed), cfg, 0);
+    try testing.expectEqual(Effect.Episode.clear, first.state);
+
+    const second = state.recordFailure(ts(ms(1)), .{ .kind = error.SendFailed, .cause = error.BrokenPipe }, cfg, 0);
+    const fault = try trippedFault(second);
+    try testing.expectEqual(transport.PeerFault.SendFailed, fault.kind);
+    try testing.expectEqual(@as(anyerror, error.BrokenPipe), fault.cause);
+    try testing.expect(second.revision > first.revision);
+}
+
+test "a failure while tripped stays tripped" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(1)), peerFault(error.Timeout), cfg, 0);
+
+    const third = state.recordFailure(ts(ms(2)), peerFault(error.Timeout), cfg, 0);
+    _ = try trippedFault(third);
+}
+
+test "a success on a tripped state returns clear, and so does one below the threshold" {
+    const cfg: Config = .{};
+    var state: State = .init;
+
+    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(0), cfg).?.state);
+
+    _ = state.recordFailure(ts(ms(1)), peerFault(error.Timeout), cfg, 0);
+    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(ms(2)), cfg).?.state);
+
+    _ = state.recordFailure(ts(ms(3)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(4)), peerFault(error.Timeout), cfg, 0);
+    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(ms(5)), cfg).?.state);
+    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(ms(6)), cfg).?.state);
+}
+
+test "a stale success behind a newer failure projects nothing" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    _ = state.recordFailure(ts(ms(10)), peerFault(error.Timeout), cfg, 0);
+    _ = state.recordFailure(ts(ms(20)), peerFault(error.Timeout), cfg, 0);
+
+    // The endpoint is still failing, so the card the failures opened is
+    // already right. Reporting it again on a success would count that success
+    // as an occurrence of the episode.
+    try testing.expectEqual(@as(?Effect, null), state.recordSuccess(ts(ms(5)), cfg));
+    try testing.expect(state.tripped(cfg));
+    try testing.expectEqual(@as(u64, 1), state.total_successes);
+}
+
+test "a stale failure carries the newer effective fault, not its own" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    _ = state.recordFailure(ts(ms(10)), peerFault(error.Timeout), cfg, 0);
+    const newest = state.recordFailure(
+        ts(ms(20)),
+        .{ .kind = error.SendFailed, .cause = error.BrokenPipe },
+        cfg,
+        0,
+    );
+    _ = try trippedFault(newest);
+
+    const stale = state.recordFailure(ts(ms(15)), .{ .kind = error.ConnectFailed, .cause = error.ConnectionRefused }, cfg, 0);
+    const fault = try trippedFault(stale);
+    try testing.expectEqual(transport.PeerFault.SendFailed, fault.kind);
+    try testing.expectEqual(@as(anyerror, error.BrokenPipe), fault.cause);
+    try testing.expectEqualStrings("SendFailed (cause BrokenPipe)", state.lastError());
+}
+
+test "every mutation advances the revision" {
+    const cfg: Config = .{};
+    var state: State = .init;
+    var previous: u64 = 0;
+
+    // The successes here are all the newest outcome, so none of them is the
+    // one case that projects nothing.
+    for (0..8) |i| {
+        const effect = if (i % 3 == 0)
+            state.recordSuccess(ts(ms(@intCast(i))), cfg).?
+        else
+            state.recordFailure(ts(ms(@intCast(i))), peerFault(error.Timeout), cfg, 0);
+        try testing.expect(effect.revision > previous);
+        previous = effect.revision;
+    }
 }