nxdns/src/upstream/health.zig

//! Per-endpoint health and exponential backoff. Pure: timestamps and jitter
//! arrive as parameters, so the pool owns the clock and the RNG and this file
//! is testable without a backend.
//!
//! Only peer faults reach this file. `transport.group` decides that; a local
//! resource error or a cancellation must never be recorded, or a full disk
//! would take every upstream out of service.
//!
//! Out-of-order completions are the normal case, not an edge case: two
//! concurrent exchanges against the same endpoint finish in either order, so
//! `at` moves backwards between calls routinely. Every timestamp field
//! therefore updates through `@max` on `.nanoseconds`, and `backoff_until` is
//! only ever extended. A late-arriving success must not shorten a backoff that
//! a later failure already set.
//!
//! Changes to state that describe the endpoint *now* are therefore
//! conditional, while the accumulated counters are not. Both record functions
//! split the same way:
//!
//!   * Data is data. `total_successes`, `total_failures`, the window and the
//!     `@max` of `last_success_at` / `last_error_at` always update, however old
//!     `at` is.
//!   * `consecutive_failures`, `backoff_until` and the last fault describe
//!     the present, so a newer recorded outcome overrules a stale call.
//!
//! `recordSuccess` clears `consecutive_failures` and `backoff_until` only when
//! its `at` is the newest outcome the state has seen, that is `at >=
//! max(last_success_at, last_error_at)` before the update. A stale success
//! leaves the backoff of a newer failure alone.
//!
//! `recordFailure` is the mirror image. See the rule at that function.
//!
//! `State` carries no lock. The pool owns the mutex.

const std = @import("std");
const transport = @import("transport.zig");

pub const Config = struct {
    /// Consecutive peer faults before the endpoint is put in backoff. At least
    /// 1: a state tripped by zero failures has no fault to name, and the
    /// episode the pool opens from it could not say what went wrong. Nothing
    /// builds this from user input, so the default is checked below and
    /// `Pool.init` asserts what a caller passes.
    failure_threshold: u8 = 2,
    base_backoff_ms: u32 = 500,
    max_backoff_ms: u32 = 60_000,
};

comptime {
    const default: Config = .{};
    if (default.failure_threshold < 1) @compileError("failure_threshold must be at least 1");
}

/// Rolling success-rate window, in samples. Equal to the bit width of
/// `State.window`.
pub const window_len = 32;

/// Bytes kept of a rendered `transport.Fault`, truncated to fit. Sized so the
/// widest classification and the widest concrete cause name fit whole, and so
/// does the widest HTTP status with its phrase; the comptime-driven test below
/// recomputes that worst case from std, so a wider name std adds fails the test
/// run rather than silently truncating an operator's only diagnostic.
pub const error_name_capacity = 64;

/// The shift is capped so `base_backoff_ms << shift` cannot run away; by then
/// `max_backoff_ms` has clamped the result many doublings ago.
const max_shift = 20;

/// The complete desired state of this endpoint's Diagnostics episode after one
/// recorded outcome. This file decides it, because "failing" is this file's
/// predicate and nobody else's; the pool projects it onto the store.
///
/// A state, never a transition. A transition has to be applied to whatever the
/// store holds, so two effects that reach the pool out of order can leave the
/// wrong one standing: a "nothing changed" overtaking a required report would
/// drop the report for good. A desired state makes last-writer-wins by
/// revision correct: a projection of the whole state is also what repairs a
/// card left over from a latched store write, on the next outcome. What a
/// per-pool revision cannot see — an episode that outlived a restart, or one a
/// retired generation opened — is repaired by `Pool.reconcile` instead, at boot
/// and at every retirement, not by waiting for an outcome.
pub const Effect = struct {
    /// Per-state counter, incremented by every mutation. The pool applies
    /// effects in this order and drops one that arrives behind a newer sibling.
    revision: u64,
    state: Episode,

    /// A tagged union rather than an enum beside a fault field, so an episode
    /// without the fault that explains it is unrepresentable. Named `Episode`
    /// rather than `State` because this file's `State` is the health record.
    pub const Episode = union(enum) {
        clear,
        tripped: transport.Fault,
    };
};

pub const State = struct {
    consecutive_failures: u32,
    total_successes: u64,
    total_failures: u64,
    last_success_at: ?std.Io.Timestamp,
    last_error_at: ?std.Io.Timestamp,
    last_error_buf: [error_name_capacity]u8,
    /// Length of the `@errorName` held in `last_error_buf`, truncated to fit.
    last_error_len: u8,
    backoff_until: ?std.Io.Timestamp,
    /// The effective last fault, as a value. `last_error_buf` is its rendering;
    /// both are kept because the text is what every surface prints and the
    /// value is what an `Effect` carries out of the mutex.
    last_fault: ?transport.Fault,
    /// Incremented by every mutation; see `Effect.revision`.
    revision: u64,
    /// Bitset, 1 = success, LSB = most recent.
    window: u32,
    window_filled: u8,

    pub const init: State = .{
        .consecutive_failures = 0,
        .total_successes = 0,
        .total_failures = 0,
        .last_success_at = null,
        .last_error_at = null,
        .last_error_buf = @splat(0),
        .last_error_len = 0,
        .backoff_until = null,
        .last_fault = null,
        .revision = 0,
        .window = 0,
        .window_filled = 0,
    };

    /// A success that is the newest outcome clears the count, and the effect
    /// then says `clear`.
    ///
    /// `null` when the state stays tripped, which is the stale success behind a
    /// newer failure (case 1 at `recordFailure`). The card that failure opened
    /// is already right, and projecting `tripped` again would count a
    /// successful exchange as an occurrence of the episode. A `clear`-shaped
    /// no-op is not an option either: it would advance `applied_revision` past
    /// a report still in flight and drop it.
    pub fn recordSuccess(self: *State, at: std.Io.Timestamp, cfg: Config) ?Effect {
        if (self.isNewestOutcome(at)) {
            self.consecutive_failures = 0;
            self.backoff_until = null;
        }
        self.total_successes += 1;
        self.push(1);
        self.last_success_at = later(self.last_success_at, at);
        self.revision += 1;
        if (self.tripped(cfg)) return null;
        return self.effect(cfg);
    }

    /// The pool's definition of "failing this endpoint": the predicate that
    /// puts it in backoff. Not `available`, which turns true again the moment a
    /// backoff expires, before any recovery is proven.
    pub fn tripped(self: *const State, cfg: Config) bool {
        return self.consecutive_failures >= cfg.failure_threshold;
    }

    /// The desired episode state this endpoint is in right now, without a
    /// mutation and without advancing the revision. The pool projects this at
    /// the two points a recorded outcome cannot reach: the first generation's
    /// boot, and the retirement of a generation whose late outcomes are now
    /// impossible.
    pub fn currentEffect(self: *const State, cfg: Config) Effect {
        return self.effect(cfg);
    }

    /// A tripped state has always recorded a failure, because the threshold is
    /// at least 1, so the fault is always there to name.
    fn effect(self: *const State, cfg: Config) Effect {
        if (!self.tripped(cfg)) return .{ .revision = self.revision, .state = .clear };
        return .{ .revision = self.revision, .state = .{ .tripped = self.last_fault.? } };
    }

    /// True when no recorded outcome is newer than `at`. Both timestamp fields
    /// update through `@max`, so their maximum is the newest outcome the state
    /// has seen and no separate field is needed.
    fn isNewestOutcome(self: *const State, at: std.Io.Timestamp) bool {
        return !newerThan(self.last_success_at, at) and !newerThan(self.last_error_at, at);
    }

    /// `fault` is the peer fault as a value: `transport.group` has already
    /// ruled that this is the peer's doing. `rand` supplies jitter; the caller
    /// owns the RNG so this stays pure and the test is deterministic.
    ///
    /// A stale failure, that is one whose `at` is older than an outcome already
    /// recorded, is held to the mirror image of the stale-success rule. Three
    /// cases, decided on the state *before* this call updates it:
    ///
    ///   1. A newer success exists. The endpoint answered after this failure,
    ///      so this failure cannot make it "consecutively failing" now: leave
    ///      `consecutive_failures` and `backoff_until` alone. A run of failures
    ///      that a later success ended is over.
    ///   2. No newer success, but a newer failure exists. The run of failures
    ///      is unbroken and this call is part of it, so the count and the
    ///      backoff update as usual; order inside a run does not matter. The
    ///      newer failure already wrote `last_error_buf`, so its text stays.
    ///   3. `at` is the newest outcome. Everything updates.
    ///
    /// `backoff_until` only ever extends through `@max`, so case 2 can lengthen
    /// a backoff but never shorten one.
    pub fn recordFailure(
        self: *State,
        at: std.Io.Timestamp,
        fault: transport.Fault,
        cfg: Config,
        rand: u32,
    ) Effect {
        const newer_success = newerThan(self.last_success_at, at);
        const newer_failure = newerThan(self.last_error_at, at);

        if (!newer_failure) {
            self.last_fault = fault;
            var writer: std.Io.Writer = .fixed(&self.last_error_buf);
            writer.print("{f}", .{fault}) catch {};
            self.last_error_len = @intCast(writer.end);
        }

        self.total_failures += 1;
        self.push(0);
        self.last_error_at = later(self.last_error_at, at);
        self.revision += 1;

        if (newer_success) return self.effect(cfg);

        self.consecutive_failures +|= 1;
        if (self.consecutive_failures < cfg.failure_threshold) return self.effect(cfg);

        const delay_ms = backoffDelayMs(self.consecutive_failures, cfg);
        const half = delay_ms / 2;
        const jittered = half + rand % (half + 1);
        const deadline: std.Io.Timestamp = .{
            .nanoseconds = at.nanoseconds + @as(i96, jittered) * std.time.ns_per_ms,
        };
        self.backoff_until = later(self.backoff_until, deadline);
        return self.effect(cfg);
    }

    pub fn available(self: *const State, now: std.Io.Timestamp) bool {
        const until = self.backoff_until orelse return true;
        return now.nanoseconds > until.nanoseconds;
    }

    /// Over the filled part of the window; 1.0 when the window is empty, so a
    /// fresh endpoint is not reported as failing.
    pub fn successRate(self: *const State) f32 {
        if (self.window_filled == 0) return 1.0;
        const filled: u6 = @intCast(@min(self.window_filled, window_len));
        const mask: u32 = if (filled == window_len)
            std.math.maxInt(u32)
        else
            (@as(u32, 1) << @intCast(filled)) - 1;
        const successes = @popCount(self.window & mask);
        return @as(f32, @floatFromInt(successes)) / @as(f32, @floatFromInt(filled));
    }

    pub fn lastError(self: *const State) []const u8 {
        return self.last_error_buf[0..self.last_error_len];
    }

    fn push(self: *State, bit: u1) void {
        self.window = (self.window << 1) | bit;
        self.window_filled = @min(self.window_filled + 1, window_len);
    }
};

/// True when `existing` is set and strictly newer than `at`. Equal timestamps
/// are not stale: a call at the timestamp of the newest outcome still counts as
/// current.
fn newerThan(existing: ?std.Io.Timestamp, at: std.Io.Timestamp) bool {
    const previous = existing orelse return false;
    return previous.nanoseconds > at.nanoseconds;
}

fn later(existing: ?std.Io.Timestamp, candidate: std.Io.Timestamp) std.Io.Timestamp {
    const previous = existing orelse return candidate;
    return .{ .nanoseconds = @max(previous.nanoseconds, candidate.nanoseconds) };
}

/// `min(max_backoff_ms, base_backoff_ms << shift)` with
/// `shift = min(consecutive_failures - failure_threshold, max_shift)`. The
/// arithmetic runs in u64 so a large `base_backoff_ms` cannot wrap before the
/// clamp applies.
fn backoffDelayMs(consecutive_failures: u32, cfg: Config) u32 {
    const over = consecutive_failures - cfg.failure_threshold;
    const shift: u6 = @intCast(@min(over, max_shift));
    const shifted = @as(u64, cfg.base_backoff_ms) << shift;
    return @intCast(@min(@as(u64, cfg.max_backoff_ms), shifted));
}

const testing = std.testing;

fn ts(nanoseconds: i96) std.Io.Timestamp {
    return .{ .nanoseconds = nanoseconds };
}

fn ms(count: i96) i96 {
    return count * std.time.ns_per_ms;
}

/// A fault whose cause is its own classification: the shape an expiry has, and
/// short enough to keep the timestamp tests about timestamps.
fn peerFault(kind: transport.PeerFault) transport.Fault {
    return .{ .kind = kind, .cause = kind };
}

test "a failure below the threshold leaves the endpoint available" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);

    try testing.expectEqual(@as(u32, 1), state.consecutive_failures);
    try testing.expectEqual(@as(u64, 1), state.total_failures);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
    try testing.expect(state.available(ts(0)));
}

test "reaching the threshold puts the endpoint in backoff until the deadline" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);

    // Two failures, threshold 2, shift 0: delay 500 ms, jitter 0 => 250 ms.
    const until = state.backoff_until.?;
    try testing.expectEqual(ms(250), until.nanoseconds);
    try testing.expect(!state.available(ts(0)));
    try testing.expect(!state.available(until));
    try testing.expect(state.available(ts(until.nanoseconds + 1)));
}

test "consecutive failures grow the delay and saturate at max_backoff_ms" {
    const cfg: Config = .{};
    var state: State = .init;

    var previous: i96 = -1;
    var i: usize = 0;
    while (i < 40) : (i += 1) {
        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
        if (state.backoff_until) |until| {
            try testing.expect(until.nanoseconds >= previous);
            previous = until.nanoseconds;
        }
    }

    // Jitter 0 halves the delay, and the delay itself is clamped.
    try testing.expectEqual(ms(cfg.max_backoff_ms / 2), state.backoff_until.?.nanoseconds);
    try testing.expectEqual(@as(u32, 500), backoffDelayMs(2, cfg));
    try testing.expectEqual(@as(u32, 1000), backoffDelayMs(3, cfg));
    try testing.expectEqual(@as(u32, 2000), backoffDelayMs(4, cfg));
    try testing.expectEqual(cfg.max_backoff_ms, backoffDelayMs(40, cfg));
}

test "a success resets the consecutive count, the window and the backoff" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
    try testing.expect(state.backoff_until != null);

    _ = state.recordSuccess(ts(ms(1)), cfg);
    try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
    try testing.expectEqual(@as(u64, 1), state.total_successes);
    try testing.expectEqual(@as(u32, 1), state.window & 1);
    try testing.expect(state.available(ts(0)));
}

test "jitter stays inside half the delay and the whole delay" {
    const cfg: Config = .{};
    const delay = backoffDelayMs(2, cfg);

    for ([_]u32{ 0, std.math.maxInt(u32), 1, 12345 }) |rand| {
        var state: State = .init;
        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, rand);
        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, rand);
        const offset = state.backoff_until.?.nanoseconds;
        try testing.expect(offset >= ms(delay / 2));
        try testing.expect(offset <= ms(delay));
    }
}

test "an out-of-order success does not move last_success_at backwards" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordSuccess(ts(100), cfg);
    _ = state.recordSuccess(ts(50), cfg);
    try testing.expectEqual(@as(i96, 100), state.last_success_at.?.nanoseconds);
}

test "an out-of-order failure does not shorten the backoff" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    const until = state.backoff_until.?.nanoseconds;

    _ = state.recordFailure(ts(ms(50)), peerFault(error.Timeout), cfg, 0);
    try testing.expect(state.backoff_until.?.nanoseconds >= until);
    try testing.expectEqual(@as(i96, ms(100)), state.last_error_at.?.nanoseconds);
}

test "an out-of-order success does not clear the backoff of a newer failure" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    const until = state.backoff_until.?.nanoseconds;

    _ = state.recordSuccess(ts(ms(50)), cfg);
    try testing.expectEqual(@as(i96, until), state.backoff_until.?.nanoseconds);
    try testing.expectEqual(@as(u32, 2), state.consecutive_failures);
    try testing.expectEqual(@as(u64, 1), state.total_successes);
    try testing.expectEqual(@as(u32, 1), state.window & 1);
    try testing.expectEqual(@as(i96, ms(50)), state.last_success_at.?.nanoseconds);
}

test "a stale failure behind a newer success does not raise the consecutive count" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(10)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordSuccess(ts(ms(100)), cfg);
    _ = state.recordFailure(ts(ms(20)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(30)), peerFault(error.Timeout), cfg, 0);

    try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
    try testing.expect(state.available(ts(ms(31))));
}

test "a stale failure behind a newer success still counts into the totals" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordSuccess(ts(ms(100)), cfg);
    _ = state.recordFailure(ts(ms(20)), peerFault(error.Timeout), cfg, 0);

    try testing.expectEqual(@as(u64, 1), state.total_failures);
    try testing.expectEqual(@as(u8, 2), state.window_filled);
    try testing.expectEqual(@as(u32, 0), state.window & 1);
    try testing.expectEqual(@as(i96, ms(20)), state.last_error_at.?.nanoseconds);
    try testing.expectEqualStrings("Timeout (cause Timeout)", state.lastError());
}

test "a stale failure with no newer success still counts as consecutive" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(50)), peerFault(error.Timeout), cfg, 0);

    try testing.expectEqual(@as(u32, 2), state.consecutive_failures);
    try testing.expect(state.backoff_until != null);
    try testing.expectEqual(@as(i96, ms(100)), state.last_error_at.?.nanoseconds);
}

test "a stale failure does not overwrite the error of a newer failure" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(50)), peerFault(error.ConnectFailed), cfg, 0);

    try testing.expectEqualStrings("Timeout (cause Timeout)", state.lastError());
}

test "the newest failure records its own error and extends the backoff" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(50)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(100)), peerFault(error.ConnectFailed), cfg, 0);

    try testing.expectEqualStrings("ConnectFailed (cause ConnectFailed)", state.lastError());
    try testing.expectEqual(@as(u32, 2), state.consecutive_failures);
    try testing.expectEqual(ms(100) + ms(250), state.backoff_until.?.nanoseconds);
}

test "the newest success clears the backoff of an older failure" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    try testing.expect(state.backoff_until != null);

    _ = state.recordSuccess(ts(ms(101)), cfg);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
    try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
}

test "a success at the timestamp of the newest failure clears the backoff" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(100)), peerFault(error.Timeout), cfg, 0);

    _ = state.recordSuccess(ts(ms(100)), cfg);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), state.backoff_until);
    try testing.expectEqual(@as(u32, 0), state.consecutive_failures);
}

test "successRate over a half-success window is 0.5" {
    const cfg: Config = .{};
    var state: State = .init;
    try testing.expectEqual(@as(f32, 1.0), state.successRate());

    var i: usize = 0;
    while (i < 4) : (i += 1) {
        _ = state.recordSuccess(ts(0), cfg);
        _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
    }
    try testing.expectEqual(@as(u8, 8), state.window_filled);
    try testing.expectEqual(@as(f32, 0.5), state.successRate());
}

test "successRate counts only the filled part of the window" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordSuccess(ts(0), cfg);
    try testing.expectEqual(@as(f32, 1.0), state.successRate());

    var i: usize = 0;
    while (i < window_len * 2) : (i += 1) _ = state.recordSuccess(ts(0), cfg);
    try testing.expectEqual(@as(u8, window_len), state.window_filled);
    try testing.expectEqual(@as(f32, 1.0), state.successRate());
}

test "lastError renders the classification and the concrete cause" {
    const cfg: Config = .{};
    var state: State = .init;
    try testing.expectEqualStrings("", state.lastError());

    _ = state.recordFailure(ts(0), .{ .kind = error.SendFailed, .cause = error.BrokenPipe }, cfg, 0);
    try testing.expectEqualStrings("SendFailed (cause BrokenPipe)", state.lastError());
    try testing.expectEqual(transport.PeerFault.SendFailed, state.last_fault.?.kind);

    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
    try testing.expectEqualStrings("Timeout (cause Timeout)", state.lastError());
}

test "lastError truncates a cause too wide for the buffer instead of overflowing" {
    const cfg: Config = .{};
    var state: State = .init;
    const Wide = error{AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA};
    _ = state.recordFailure(
        ts(0),
        .{ .kind = error.ReceiveFailed, .cause = Wide.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA },
        cfg,
        0,
    );

    const whole = "ReceiveFailed (cause " ++ "A" ** 64 ++ ")";
    try testing.expectEqual(@as(usize, error_name_capacity), state.lastError().len);
    try testing.expectEqualStrings(whole[0..error_name_capacity], state.lastError());
}

test "error_name_capacity holds the widest classification, cause and status whole" {
    // The widest concrete cause an upstream unwrap can produce. The three sets
    // are the ones `transport.zig`'s unwraps return a member of; a wider name
    // std adds fails here rather than truncating an operator's diagnostic.
    const widest_cause = comptime blk: {
        var widest: []const u8 = "";
        for (@typeInfo(std.crypto.tls.Client.ReadError).error_set.?) |member| {
            if (member.name.len > widest.len) widest = member.name;
        }
        for (@typeInfo(std.http.Client.RequestError).error_set.?) |member| {
            if (member.name.len > widest.len) widest = member.name;
        }
        for (@typeInfo(std.http.Reader.BodyError).error_set.?) |member| {
            if (member.name.len > widest.len) widest = member.name;
        }
        break :blk widest;
    };
    const widest_kind = comptime blk: {
        var widest: []const u8 = "";
        for (@typeInfo(transport.PeerFault).error_set.?) |member| {
            if (member.name.len > widest.len) widest = member.name;
        }
        break :blk widest;
    };

    try testing.expectEqualStrings("DetectingNetworkConfigurationFailed", widest_cause);
    try testing.expect(widest_kind.len + " (cause ".len + widest_cause.len + ")".len <= error_name_capacity);

    const widest_phrase = comptime blk: {
        var widest: []const u8 = "";
        for (@typeInfo(std.http.Status).@"enum".fields) |field| {
            const phrase = @as(std.http.Status, @enumFromInt(field.value)).phrase() orelse continue;
            if (phrase.len > widest.len) widest = phrase;
        }
        break :blk widest;
    };
    // `std.http.Status` is a u10, so a code has at most four digits.
    const widest_status = "HttpStatus (status ".len + 4 + " ".len + widest_phrase.len + ")".len;
    try testing.expect(widest_status <= error_name_capacity);
}

/// The fault of a `tripped` effect, or an error when the effect says `clear`.
fn trippedFault(effect: Effect) !transport.Fault {
    return switch (effect.state) {
        .clear => error.TestExpectedTripped,
        .tripped => |fault| fault,
    };
}

test "one failure stays clear and the second trips the episode" {
    const cfg: Config = .{};
    var state: State = .init;

    const first = state.recordFailure(ts(0), peerFault(error.ConnectFailed), cfg, 0);
    try testing.expectEqual(Effect.Episode.clear, first.state);

    const second = state.recordFailure(ts(ms(1)), .{ .kind = error.SendFailed, .cause = error.BrokenPipe }, cfg, 0);
    const fault = try trippedFault(second);
    try testing.expectEqual(transport.PeerFault.SendFailed, fault.kind);
    try testing.expectEqual(@as(anyerror, error.BrokenPipe), fault.cause);
    try testing.expect(second.revision > first.revision);
}

test "a failure while tripped stays tripped" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(0), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(1)), peerFault(error.Timeout), cfg, 0);

    const third = state.recordFailure(ts(ms(2)), peerFault(error.Timeout), cfg, 0);
    _ = try trippedFault(third);
}

test "a success on a tripped state returns clear, and so does one below the threshold" {
    const cfg: Config = .{};
    var state: State = .init;

    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(0), cfg).?.state);

    _ = state.recordFailure(ts(ms(1)), peerFault(error.Timeout), cfg, 0);
    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(ms(2)), cfg).?.state);

    _ = state.recordFailure(ts(ms(3)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(4)), peerFault(error.Timeout), cfg, 0);
    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(ms(5)), cfg).?.state);
    try testing.expectEqual(Effect.Episode.clear, state.recordSuccess(ts(ms(6)), cfg).?.state);
}

test "a stale success behind a newer failure projects nothing" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(10)), peerFault(error.Timeout), cfg, 0);
    _ = state.recordFailure(ts(ms(20)), peerFault(error.Timeout), cfg, 0);

    // The endpoint is still failing, so the card the failures opened is
    // already right. Reporting it again on a success would count that success
    // as an occurrence of the episode.
    try testing.expectEqual(@as(?Effect, null), state.recordSuccess(ts(ms(5)), cfg));
    try testing.expect(state.tripped(cfg));
    try testing.expectEqual(@as(u64, 1), state.total_successes);
}

test "a stale failure carries the newer effective fault, not its own" {
    const cfg: Config = .{};
    var state: State = .init;
    _ = state.recordFailure(ts(ms(10)), peerFault(error.Timeout), cfg, 0);
    const newest = state.recordFailure(
        ts(ms(20)),
        .{ .kind = error.SendFailed, .cause = error.BrokenPipe },
        cfg,
        0,
    );
    _ = try trippedFault(newest);

    const stale = state.recordFailure(ts(ms(15)), .{ .kind = error.ConnectFailed, .cause = error.ConnectionRefused }, cfg, 0);
    const fault = try trippedFault(stale);
    try testing.expectEqual(transport.PeerFault.SendFailed, fault.kind);
    try testing.expectEqual(@as(anyerror, error.BrokenPipe), fault.cause);
    try testing.expectEqualStrings("SendFailed (cause BrokenPipe)", state.lastError());
}

test "every mutation advances the revision" {
    const cfg: Config = .{};
    var state: State = .init;
    var previous: u64 = 0;

    // The successes here are all the newest outcome, so none of them is the
    // one case that projects nothing.
    for (0..8) |i| {
        const effect = if (i % 3 == 0)
            state.recordSuccess(ts(ms(@intCast(i))), cfg).?
        else
            state.recordFailure(ts(ms(@intCast(i))), peerFault(error.Timeout), cfg, 0);
        try testing.expect(effect.revision > previous);
        previous = effect.revision;
    }
}