//! Priority-ordered sequential failover across upstream endpoints, with two
//! deadlines, health tracking and backoff (PLAN §9).
//!
//! The pool is itself a `transport.Client`, so the handler above it sees one
//! interface and knows nothing about how many upstreams exist.
//!
//! Two deadlines, nested. `timeouts.attempt` bounds one exchange against one
//! entry; `timeouts.total` bounds the whole failover loop, every attempt
//! together. The outer one is what the client asking the question actually
//! waits for: without it, N unreachable upstreams cost N × attempt, and the
//! resolver above has already given up. The outer expiry is `error.Timeout`
//! unconditionally — the fast-failure paths (no entry enabled) return long
//! before the budget, so an expiry always means an attempt was in flight.
//!
//! Two passes, not one. Pass one walks the enabled entries that health says are
//! available. Pass two runs only when pass one attempted nothing, and skips the
//! backoff check: every endpoint being in backoff must not turn into SERVFAIL
//! for every client. A probe is better than a guaranteed failure, and the probe
//! is how backoff recovers.
//!
//! Concurrency (PLAN §9, spec S4.2): several handler tasks share one `Pool`,
//! and they hold two different locks at two different scopes.
//!
//! `Pool.mutex` guards health mutation, the jitter RNG and `snapshot`. It is
//! never held across an exchange, so a slow upstream cannot block a health read
//! or an attempt on some other entry.
//!
//! `Entry.busy` guards one entry's `client`. A task holds it for a whole
//! attempt against that entry — the exchange and the timeout race around it —
//! and drops it before the failover loop moves to the next candidate. This is
//! what makes the pool safe to share: `DohClient` owns a request buffer and a
//! transfer buffer, `DotClient` owns four TLS buffers and the stream state
//! built on them, so one client may be inside `exchange` only once at a time.
//!
//! Because that wait is serializing, pass one re-reads health after it takes
//! `busy`: the attempts a task queued behind can fail the entry into backoff
//! while it waits, and pass one must not attempt an entry that is unavailable by
//! the time it gets its turn. Lock order is always `busy` then `Pool.mutex`,
//! never the reverse.
//!
//! The guarantee is therefore: at most one in-flight exchange per entry, any
//! number of entries in flight at once, and bookkeeping that never waits on a
//! peer. Two concurrent queries that resolve to the same sole upstream do
//! serialize. At household scale that is the right trade — the alternative is a
//! client instance per listener task, which multiplies TLS buffers and
//! connections for a query rate that never needed them.

const std = @import("std");

const events = @import("../storage/events.zig");
const health = @import("health.zig");
const history_mod = @import("history.zig");
const safe_url = @import("../safe_url.zig");
const transport = @import("transport.zig");

const log = std.log.scoped(.upstream);

/// The one line `exchange` writes about a failed attempt, as a value.
///
/// It is a value rather than a format string at the call site for the same
/// reason `dot_client.Diagnostic` is: a `std.log` line is not readable from a
/// unit test under the default test runner, so the test below reads this
/// instead of stderr. An upstream url is operator-supplied and a DoH one carries
/// its credential in the path — `https://dns.nextdns.io/abcd12` is a whole
/// NextDNS account identifier — so it reaches the line through
/// `safe_url.redactQuoted`. Quoted rather than bare because the error name
/// follows it: a redacted authority may still hold a space and a `:`, so
/// unquoted, a url ending `ok failed: Timeout` would report a failure that did
/// not happen.
const AttemptFailure = struct {
    endpoint: transport.Endpoint,
    err: transport.ExchangeError,

    pub fn format(self: AttemptFailure, w: *std.Io.Writer) std.Io.Writer.Error!void {
        try w.print("upstream {f} failed: {t}", .{ safe_url.redactQuoted(self.endpoint.url), self.err });
    }
};

pub const Entry = struct {
    endpoint: transport.Endpoint,
    client: transport.Client,
    /// Lower is tried first (PLAN §11.2 `upstreams.priority`).
    priority: i32,
    enabled: bool,
    health: health.State,
    /// Held for the whole of one attempt against this entry, so `client` is
    /// never re-entered while it is using its own buffers. Defaulted because
    /// `init` sorts `entries` by value, which may only copy unlocked mutexes,
    /// and it runs before the pool is reachable by any task.
    busy: std.Io.Mutex = .init,
};

/// A copy of one entry's health, taken under the mutex. Feeds
/// `GET /api/upstream/health`.
pub const Snapshot = struct {
    /// Whole, not redacted. `GET /api/upstream/health` returns this to a session
    /// that `GET /api/upstreams` already serves the same url to in full, so
    /// redacting here would hide nothing from that reader and would make two
    /// responses of one API disagree. A consumer reachable without a session has
    /// to redact it itself.
    url: []const u8,
    enabled: bool,
    available: bool,
    consecutive_failures: u32,
    total_successes: u64,
    total_failures: u64,
    success_rate: f32,
    last_success_at: ?std.Io.Timestamp,
    last_error_at: ?std.Io.Timestamp,
    /// Borrowed from the entry; valid until that entry's next failure.
    last_error: []const u8,
    backoff_until: ?std.Io.Timestamp,
};

/// The two budgets, named rather than positional: they are the same type, so
/// two parameters in a row could be swapped at a call site and still compile,
/// and the swap would be invisible until an operator watched a query take five
/// attempts of two and a half seconds each.
pub const Timeouts = struct {
    /// Bounds one exchange against one entry.
    attempt: std.Io.Clock.Duration,
    /// Bounds the whole failover loop.
    total: std.Io.Clock.Duration,
};

pub const Pool = struct {
    /// Caller-owned, sorted ascending by priority in `init`.
    entries: []Entry,
    cfg: health.Config,
    /// Both on the `.awake` clock, so a suspended Pi does not burn a budget.
    timeouts: Timeouts,
    mutex: std.Io.Mutex,
    rng: std.Random.DefaultPrng,
    /// Where recorded outcomes also go, as per-minute aggregates for the
    /// dashboard's ranged view (m26). Defaulted rather than an `init`
    /// parameter: the composition root wires it after the pool exists, and the
    /// pool is fully usable without it — `nxdns check` and every unit test here
    /// run with no history at all.
    history: ?*history_mod.Accumulator = null,
    /// The diagnostics store, wired the same way and for the same reason as
    /// `history`. Every emit here sits outside `mutex`; see `recordHistory`.
    diagnostics: ?*events.Store = null,

    pub fn init(
        entries: []Entry,
        cfg: health.Config,
        timeouts: Timeouts,
        seed: u64,
    ) Pool {
        std.debug.assert(entries.len > 0);
        // Stable, so entries sharing a priority keep their configured order.
        std.mem.sort(Entry, entries, {}, byPriority);
        return .{
            .entries = entries,
            .cfg = cfg,
            .timeouts = timeouts,
            .mutex = .init,
            .rng = .init(seed),
        };
    }

    fn byPriority(_: void, a: Entry, b: Entry) bool {
        return a.priority < b.priority;
    }

    pub fn client(self: *Pool) transport.Client {
        return .{ .ptr = self, .exchangeFn = exchangeErased };
    }

    fn exchangeErased(
        ptr: *anyopaque,
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
    ) transport.ExchangeError![]u8 {
        const self: *Pool = @ptrCast(@alignCast(ptr));
        return self.exchange(io, query, response_buf);
    }

    /// `response_buf` is handed to each attempt in turn, so a failed attempt
    /// may have written into it. The returned slice is only meaningful on
    /// success; on error the buffer's contents are undefined.
    ///
    /// The failover loop runs raced against `timeouts.total`. Losing that race
    /// cancels the loop, which unwinds whatever attempt was in flight: the
    /// `Entry.busy` lock is taken cancelably on purpose and released by defer,
    /// and the health bookkeeping around a completed exchange is uncancelable,
    /// so a canceled attempt leaves no lock held and no counter half-written.
    pub fn exchange(
        self: *Pool,
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
    ) transport.ExchangeError![]u8 {
        const len = try transport.raceWithin(io, self.timeouts.total, exchangeLoopLen, .{
            self, io, query, response_buf,
        });
        return response_buf[0..len];
    }

    /// The two-pass failover loop, as a raceable task. It returns the reply's
    /// length rather than its slice for the reason `exchangeLen` in the tests
    /// below does: `Io.concurrent` stores the future's return value, so the
    /// bytes are read back out of the caller's `response_buf` by `exchange`.
    fn exchangeLoopLen(
        self: *Pool,
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
    ) transport.ExchangeError!usize {
        const now = std.Io.Clock.awake.now(io);
        var last_fault: ?transport.ExchangeError = null;
        var attempted = false;

        var pass: u8 = 0;
        while (pass < 2) : (pass += 1) {
            for (self.entries) |*entry| {
                if (!entry.enabled) continue;
                if (pass == 0 and !self.entryAvailable(io, entry, now)) continue;

                // Cancelable, unlike the health-bookkeeping locks below: a task
                // waiting its turn on a busy upstream has done nothing that a
                // cancellation could corrupt, so it gives up here rather than
                // queueing behind an exchange it will not use. The wait itself
                // is bounded by the holder's `timeouts.attempt`; the waiter's
                // own budget only starts once it has the lock, and the whole
                // loop is bounded by `timeouts.total` regardless.
                try entry.busy.lock(io);
                defer entry.busy.unlock(io);

                // The check above ran before the wait, and the attempts this one
                // queued behind may have failed the entry into backoff while it
                // waited. Pass one must not touch an entry that is unavailable
                // now, so re-read health against a fresh `now` and move on if it
                // is. Pass two skips this on purpose: it probes regardless of
                // backoff, which is how backoff recovers.
                if (pass == 0) {
                    const recheck = std.Io.Clock.awake.now(io);
                    if (!self.entryAvailable(io, entry, recheck)) continue;
                }
                attempted = true;

                const result = self.attempt(io, entry.client, query, response_buf);
                const completed_at = std.Io.Clock.awake.now(io);

                const response = result catch |err| switch (transport.group(err)) {
                    .peer_fault => {
                        log.debug("{f}", .{AttemptFailure{ .endpoint = entry.endpoint, .err = err }});
                        self.recordFailure(io, entry, completed_at, err);
                        last_fault = err;
                        continue;
                    },
                    // The next upstream would hit the same wall, and this says
                    // nothing about any peer, so it is never recorded.
                    .local_resource => return err,
                    .cancellation => return error.Canceled,
                };

                self.recordSuccess(io, entry, completed_at);
                return response.len;
            }
            if (attempted) break;
        }

        // Guaranteed non-null whenever any entry is enabled: pass two attempts
        // every enabled entry regardless of backoff.
        if (last_fault) |err| return err;
        return error.ConnectFailed;
    }

    /// Copies health into `out` in pool order; returns the number written.
    pub fn snapshot(self: *Pool, io: std.Io, out: []Snapshot) std.Io.Cancelable!usize {
        const now = std.Io.Clock.awake.now(io);
        try self.mutex.lock(io);
        defer self.mutex.unlock(io);

        const count = @min(self.entries.len, out.len);
        for (self.entries[0..count], out[0..count]) |*entry, *slot| {
            slot.* = .{
                .url = entry.endpoint.url,
                .enabled = entry.enabled,
                .available = entry.enabled and entry.health.available(now),
                .consecutive_failures = entry.health.consecutive_failures,
                .total_successes = entry.health.total_successes,
                .total_failures = entry.health.total_failures,
                .success_rate = entry.health.successRate(),
                .last_success_at = entry.health.last_success_at,
                .last_error_at = entry.health.last_error_at,
                .last_error = entry.health.lastError(),
                .backoff_until = entry.health.backoff_until,
            };
        }
        return count;
    }

    /// One exchange raced against the per-attempt budget.
    fn attempt(
        self: *Pool,
        io: std.Io,
        entry_client: transport.Client,
        query: []const u8,
        response_buf: []u8,
    ) transport.ExchangeError![]u8 {
        return transport.raceWithin(io, self.timeouts.attempt, transport.Client.exchange, .{
            entry_client, io, query, response_buf,
        });
    }

    fn entryAvailable(
        self: *Pool,
        io: std.Io,
        entry: *const Entry,
        now: std.Io.Timestamp,
    ) bool {
        self.mutex.lockUncancelable(io);
        defer self.mutex.unlock(io);
        return entry.health.available(now);
    }

    fn recordSuccess(self: *Pool, io: std.Io, entry: *Entry, at: std.Io.Timestamp) void {
        {
            // Uncancelable: this section takes no Io and never blocks on a
            // peer. Losing the bookkeeping for a completed exchange to a
            // cancellation that arrives one instruction later would corrupt
            // health for good.
            self.mutex.lockUncancelable(io);
            defer self.mutex.unlock(io);
            entry.health.recordSuccess(at);
        }
        // The block above closes before this line, and that ordering is the
        // constraint: the accumulator takes a mutex of its own, and no task may
        // hold one of the two while it takes the other.
        self.recordHistory(io, entry, .success);
        self.recordDiagnostics(io, entry, .success);
    }

    fn recordFailure(
        self: *Pool,
        io: std.Io,
        entry: *Entry,
        at: std.Io.Timestamp,
        err: transport.ExchangeError,
    ) void {
        {
            self.mutex.lockUncancelable(io);
            defer self.mutex.unlock(io);
            entry.health.recordFailure(at, @errorName(err), self.cfg, self.rng.random().int(u32));
        }
        // After the pool mutex is released, for the reason `recordSuccess`
        // states.
        self.recordHistory(io, entry, .{ .failure = @errorName(err) });
        self.recordDiagnostics(io, entry, .{ .failure = @errorName(err) });
    }

    const Outcome = union(enum) { success, failure: []const u8 };

    /// The wall clock, not the `.awake` timestamp the health state runs on:
    /// history is aggregated into wall-clock minutes so a dashboard period
    /// means the same thing here as everywhere else on the page.
    fn recordHistory(self: *Pool, io: std.Io, entry: *Entry, outcome: Outcome) void {
        const history = self.history orelse return;
        const wall_s = std.Io.Clock.real.now(io).toSeconds();
        switch (outcome) {
            .success => history.recordSuccess(io, entry.endpoint.url, wall_s),
            .failure => |name| history.recordFailure(io, entry.endpoint.url, wall_s, name),
        }
    }

    /// The same placement discipline as `recordHistory`: the store takes a
    /// mutex of its own, so this runs after the pool's is released.
    ///
    /// A success is the steady state of the whole program, so `resolve` is
    /// built to issue no SQL when nothing is open (`storage/events.zig`).
    fn recordDiagnostics(self: *Pool, io: std.Io, entry: *Entry, outcome: Outcome) void {
        const store = self.diagnostics orelse return;
        const url = entry.endpoint.url;
        const now_s = std.Io.Clock.real.now(io).toSeconds();
        switch (outcome) {
            .success => store.resolve(io, now_s, .upstream_exchange, url),
            .failure => |name| {
                var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
                const label = std.fmt.bufPrint(&label_buf, "{f}", .{safe_url.redact(url)}) catch &label_buf;
                var detail_buf: [events.Store.max_detail_len]u8 = undefined;
                const detail = std.fmt.bufPrint(&detail_buf, "upstream {f} failed: {s}", .{
                    safe_url.redactQuoted(url),
                    name,
                }) catch &detail_buf;
                store.report(io, now_s, .upstream_exchange, url, label, .warning, detail);
            },
        }
    }
};

const db = @import("../storage/db.zig");
const events_fixture = @import("../storage/events_fixture.zig");
const querylog_schema = @import("../storage/querylog_schema.zig");
const upstream_history_repo = @import("../storage/repositories/upstream_history_repo.zig");

const testing = std.testing;

/// A query for example.com A: id 0x1234, RD set, one question.
const query_bytes =
    "\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00" ++
    "\x07example\x03com\x00\x00\x01\x00\x01";

/// The matching response: the question echoed plus one A record.
const response_bytes =
    "\x12\x34\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00" ++
    "\x07example\x03com\x00\x00\x01\x00\x01" ++
    "\xc0\x0c\x00\x01\x00\x01\x00\x00\x01\x2c\x00\x04\x5d\xb8\xd8\x22";

/// Stands in for a DoH or DoT client. Every behaviour the pool has to react to
/// is one variant, and every call is counted so a test can assert that an entry
/// in backoff was not touched.
const Fake = struct {
    behavior: Behavior,
    calls: usize = 0,
    in_flight: std.atomic.Value(u32) = .init(0),
    /// The most tasks ever inside `exchangeFn` at once. The per-entry lock is
    /// only doing its job while this stays at 1.
    peak_in_flight: std.atomic.Value(u32) = .init(0),

    const Behavior = union(enum) {
        /// Copy these bytes into the caller's buffer and return them.
        reply: []const u8,
        /// Fail with this error.
        fail: transport.ExchangeError,
        /// Sleep, then reply. Used to outrun the pool's attempt budget.
        slow: struct { duration: std.Io.Clock.Duration, reply: []const u8 },
        /// Sleep, then fail. Holds the entry's lock long enough for a second
        /// task to queue on it before the failure opens a backoff window.
        slow_fail: struct { duration: std.Io.Clock.Duration, err: transport.ExchangeError },
    };

    fn exchangeFn(
        ptr: *anyopaque,
        io: std.Io,
        query: []const u8,
        response_buf: []u8,
    ) transport.ExchangeError![]u8 {
        _ = query;
        const self: *Fake = @ptrCast(@alignCast(ptr));
        const entrants = self.in_flight.fetchAdd(1, .acq_rel) + 1;
        defer _ = self.in_flight.fetchSub(1, .acq_rel);
        _ = self.peak_in_flight.fetchMax(entrants, .acq_rel);

        self.calls += 1;
        switch (self.behavior) {
            .reply => |bytes| return copy(bytes, response_buf),
            .fail => |err| return err,
            .slow => |slow| {
                try slow.duration.sleep(io);
                return copy(slow.reply, response_buf);
            },
            .slow_fail => |slow| {
                try slow.duration.sleep(io);
                return slow.err;
            },
        }
    }

    fn copy(bytes: []const u8, response_buf: []u8) transport.ExchangeError![]u8 {
        if (bytes.len > response_buf.len) return error.ResponseTooLarge;
        @memcpy(response_buf[0..bytes.len], bytes);
        return response_buf[0..bytes.len];
    }

    fn client(self: *Fake) transport.Client {
        return .{ .ptr = self, .exchangeFn = exchangeFn };
    }
};

fn testEntry(url: []const u8, fake: *Fake, priority: i32) Entry {
    return .{
        .endpoint = Endpoint.parse(url) catch unreachable,
        .client = fake.client(),
        .priority = priority,
        .enabled = true,
        .health = .init,
    };
}

const Endpoint = transport.Endpoint;

/// Long enough that no test can outlive a backoff it just set, and short enough
/// that nothing waits on it.
const test_cfg: health.Config = .{
    .failure_threshold = 2,
    .base_backoff_ms = 60_000,
    .max_backoff_ms = 60_000,
};

/// Long enough that nothing in a test reaches either budget unless the test is
/// about a budget, and short enough that a stuck test still ends.
const test_timeouts: Timeouts = .{
    .attempt = .{ .raw = .fromSeconds(10), .clock = .awake },
    .total = .{ .raw = .fromSeconds(30), .clock = .awake },
};

fn expectFailureLine(expected: []const u8, url: []const u8, err: transport.ExchangeError) !void {
    var buf: [8 * safe_url.max_len]u8 = undefined;
    const line = try std.fmt.bufPrint(&buf, "{f}", .{AttemptFailure{
        .endpoint = try .parse(url),
        .err = err,
    }});
    try testing.expectEqualStrings(expected, line);
}

test "the failed-attempt line names an upstream by a url carrying no credential" {
    // A NextDNS DoH upstream puts the whole account identifier in the path, and
    // this line ran at `debug` on every peer fault, so a debug-level operator
    // persisted it to the journal once per failure.
    try expectFailureLine(
        "upstream 'https://dns.nextdns.io' failed: Timeout",
        "https://dns.nextdns.io/abcd12",
        error.Timeout,
    );
    try expectFailureLine(
        "upstream 'https://cdn.example:8443' failed: TlsFailed",
        "https://cdn.example:8443/d/hunter2/dns-query",
        error.TlsFailed,
    );
    // What it still says, because an operator reading a failover has to know
    // which upstream failed: the scheme, the host and the port.
    try expectFailureLine(
        "upstream 'tls://9.9.9.9:853' failed: ConnectFailed",
        "tls://9.9.9.9:853",
        error.ConnectFailed,
    );
}

test "Pool satisfies the Client interface" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var fake: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{testEntry("https://a.example/dns-query", &fake, 10)};
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var buf: [512]u8 = undefined;
    const reply = try pool.client().exchange(io, query_bytes, &buf);
    try testing.expectEqualSlices(u8, response_bytes, reply);
}

test "entries are tried in ascending priority order" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var low: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var high: Fake = .{ .behavior = .{ .reply = response_bytes } };
    // Registered out of order: priority, not position, decides.
    var entries = [_]Entry{
        testEntry("https://high.example/dns-query", &high, 100),
        testEntry("https://low.example/dns-query", &low, 10),
    };
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    try testing.expectEqual(@as(i32, 10), entries[0].priority);

    var buf: [512]u8 = undefined;
    _ = try pool.exchange(io, query_bytes, &buf);
    try testing.expectEqual(@as(usize, 1), low.calls);
    try testing.expectEqual(@as(usize, 0), high.calls);
}

test "a peer fault fails over to the next entry and is recorded" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var bad: Fake = .{ .behavior = .{ .fail = error.Timeout } };
    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://bad.example/dns-query", &bad, 10),
        testEntry("https://good.example/dns-query", &good, 20),
    };
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var buf: [512]u8 = undefined;
    const reply = try pool.exchange(io, query_bytes, &buf);
    try testing.expectEqualSlices(u8, response_bytes, reply);

    try testing.expectEqual(@as(u32, 1), entries[0].health.consecutive_failures);
    try testing.expectEqual(@as(u64, 1), entries[0].health.total_failures);
    try testing.expectEqualStrings("Timeout", entries[0].health.lastError());
    try testing.expectEqual(@as(u64, 1), entries[1].health.total_successes);
}

test "an entry in backoff is skipped while another is available" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var bad: Fake = .{ .behavior = .{ .fail = error.ConnectFailed } };
    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://bad.example/dns-query", &bad, 10),
        testEntry("https://good.example/dns-query", &good, 20),
    };
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var buf: [512]u8 = undefined;
    // Two failures reach `failure_threshold` and open a backoff window.
    _ = try pool.exchange(io, query_bytes, &buf);
    _ = try pool.exchange(io, query_bytes, &buf);
    try testing.expectEqual(@as(usize, 2), bad.calls);
    try testing.expect(entries[0].health.backoff_until != null);

    _ = try pool.exchange(io, query_bytes, &buf);
    try testing.expectEqual(@as(usize, 2), bad.calls);
    try testing.expectEqual(@as(usize, 3), good.calls);
}

test "every entry in backoff is still probed" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var first: Fake = .{ .behavior = .{ .fail = error.ConnectFailed } };
    var second: Fake = .{ .behavior = .{ .fail = error.BadResponse } };
    var entries = [_]Entry{
        testEntry("https://first.example/dns-query", &first, 10),
        testEntry("https://second.example/dns-query", &second, 20),
    };
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var buf: [512]u8 = undefined;
    try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
    try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
    try testing.expect(entries[0].health.backoff_until != null);
    try testing.expect(entries[1].health.backoff_until != null);

    // Pass one now has no candidate at all. Pass two probes both anyway.
    try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
    try testing.expectEqual(@as(usize, 3), first.calls);
    try testing.expectEqual(@as(usize, 3), second.calls);
}

test "a local resource error short-circuits and records nothing" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var broke: Fake = .{ .behavior = .{ .fail = error.OutOfMemory } };
    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://broke.example/dns-query", &broke, 10),
        testEntry("https://good.example/dns-query", &good, 20),
    };
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var buf: [512]u8 = undefined;
    try testing.expectError(error.OutOfMemory, pool.exchange(io, query_bytes, &buf));
    try testing.expectEqual(@as(usize, 0), good.calls);
    try testing.expectEqual(@as(u64, 0), entries[0].health.total_failures);
    try testing.expectEqual(@as(u32, 0), entries[0].health.consecutive_failures);
}

test "a cancellation short-circuits and records nothing" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var canceled: Fake = .{ .behavior = .{ .fail = error.Canceled } };
    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://canceled.example/dns-query", &canceled, 10),
        testEntry("https://good.example/dns-query", &good, 20),
    };
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var buf: [512]u8 = undefined;
    try testing.expectError(error.Canceled, pool.exchange(io, query_bytes, &buf));
    try testing.expectEqual(@as(usize, 0), good.calls);
    try testing.expectEqual(@as(u64, 0), entries[0].health.total_failures);
}

test "an attempt that outruns the budget is a recorded Timeout" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var slow: Fake = .{ .behavior = .{ .slow = .{
        .duration = .{ .raw = .fromSeconds(30), .clock = .awake },
        .reply = response_bytes,
    } } };
    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://slow.example/dns-query", &slow, 10),
        testEntry("https://good.example/dns-query", &good, 20),
    };
    // A tight attempt budget under a loose total one, so the attempt deadline
    // is the only one that can fire and the failover after it has room to run.
    var pool: Pool = .init(&entries, test_cfg, .{
        .attempt = .{ .raw = .fromMilliseconds(20), .clock = .awake },
        .total = .{ .raw = .fromSeconds(30), .clock = .awake },
    }, 1);

    var buf: [512]u8 = undefined;
    const reply = try pool.exchange(io, query_bytes, &buf);
    try testing.expectEqualSlices(u8, response_bytes, reply);

    try testing.expectEqual(@as(usize, 1), slow.calls);
    try testing.expectEqual(@as(u32, 1), entries[0].health.consecutive_failures);
    try testing.expectEqualStrings("Timeout", entries[0].health.lastError());
    try testing.expectEqual(@as(u64, 1), entries[1].health.total_successes);
}

test "two stalling upstreams cost the total budget, not one budget each" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    // Both entries stall past their own attempt budget, so without the outer
    // race the exchange would take two attempt budgets and then some. The
    // total is set below two attempts, which is what makes the assertion mean
    // something: only the outer deadline can end this call in time.
    var first: Fake = .{ .behavior = .{ .slow = .{
        .duration = .{ .raw = .fromSeconds(30), .clock = .awake },
        .reply = response_bytes,
    } } };
    var second: Fake = .{ .behavior = .{ .slow = .{
        .duration = .{ .raw = .fromSeconds(30), .clock = .awake },
        .reply = response_bytes,
    } } };
    var entries = [_]Entry{
        testEntry("https://first.example/dns-query", &first, 10),
        testEntry("https://second.example/dns-query", &second, 20),
    };
    var pool: Pool = .init(&entries, test_cfg, .{
        .attempt = .{ .raw = .fromMilliseconds(200), .clock = .awake },
        .total = .{ .raw = .fromMilliseconds(60), .clock = .awake },
    }, 1);

    var buf: [512]u8 = undefined;
    const started = std.Io.Clock.awake.now(io);
    try testing.expectError(error.Timeout, pool.exchange(io, query_bytes, &buf));
    const elapsed_ns = std.Io.Clock.awake.now(io).nanoseconds - started.nanoseconds;

    // Under one attempt budget, so the outer deadline is provably what fired.
    // The bound is generous against a loaded CI box; the failure it catches is
    // a whole extra attempt, not a scheduling hiccup.
    try testing.expect(elapsed_ns < @as(i96, 200) * std.time.ns_per_ms);
    // The second entry was never reached: the loop was canceled inside the
    // first attempt.
    try testing.expectEqual(@as(usize, 0), second.calls);
}

test "every entry disabled yields ConnectFailed without waiting out the total budget" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var one: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var two: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://one.example/dns-query", &one, 10),
        testEntry("https://two.example/dns-query", &two, 20),
    };
    for (&entries) |*entry| entry.enabled = false;
    // A total budget long enough that waiting it out would be unmistakable.
    // The outer expiry returns `error.Timeout` unconditionally, so this pins
    // that the fast-failure path still wins its own race.
    var pool: Pool = .init(&entries, test_cfg, .{
        .attempt = .{ .raw = .fromSeconds(10), .clock = .awake },
        .total = .{ .raw = .fromSeconds(30), .clock = .awake },
    }, 1);

    var buf: [512]u8 = undefined;
    const started = std.Io.Clock.awake.now(io);
    try testing.expectError(error.ConnectFailed, pool.exchange(io, query_bytes, &buf));
    const elapsed_ns = std.Io.Clock.awake.now(io).nanoseconds - started.nanoseconds;

    try testing.expect(elapsed_ns < @as(i96, 5) * std.time.ns_per_s);
    try testing.expectEqual(@as(usize, 0), one.calls);
    try testing.expectEqual(@as(usize, 0), two.calls);
}

test "a wired accumulator receives both outcomes the pool records" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var bad: Fake = .{ .behavior = .{ .fail = error.Timeout } };
    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://bad.example/dns-query", &bad, 10),
        testEntry("https://good.example/dns-query", &good, 20),
    };
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    const acc = try testing.allocator.create(history_mod.Accumulator);
    defer testing.allocator.destroy(acc);
    acc.* = .init;
    pool.history = acc;

    var buf: [512]u8 = undefined;
    // One exchange: the first entry fails over into the second, so this drives
    // one failure and one success.
    _ = try pool.exchange(io, query_bytes, &buf);

    // Two cells, one per url, in whatever minute the wall clock is in.
    try testing.expectEqual(@as(u32, 2), acc.snapshotStats(io).pending);

    // Read back through the flush path rather than through the accumulator's
    // private cells: the whole point of the hook is that these outcomes reach
    // storage under the right url.
    var database = try db.Db.open(":memory:", .{ .mode = .memory });
    defer database.close();
    try db.applyPragmas(&database, .{});
    try database.exec(querylog_schema.ddl);
    acc.flushOnce(io, &database, upstream_history_repo.flush);

    // A window wide enough that a minute boundary crossed mid-test changes
    // nothing about what it contains.
    const now = std.Io.Clock.real.now(io).toSeconds();
    const failing = try upstream_history_repo.windowStats(
        &database,
        "https://bad.example/dns-query",
        now - 3600,
        now + 3600,
    );
    try testing.expectEqual(@as(u64, 1), failing.failures);
    try testing.expectEqual(@as(u64, 0), failing.successes);
    try testing.expect(failing.last_failure_ts != null);
    try testing.expectEqualStrings("Timeout", failing.lastFailureError());

    const succeeding = try upstream_history_repo.windowStats(
        &database,
        "https://good.example/dns-query",
        now - 3600,
        now + 3600,
    );
    try testing.expectEqual(@as(u64, 1), succeeding.successes);
    try testing.expectEqual(@as(u64, 0), succeeding.failures);
    try testing.expectEqual(@as(?i64, null), succeeding.last_failure_ts);
}

test "snapshot reports the counters in pool order" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var bad: Fake = .{ .behavior = .{ .fail = error.Timeout } };
    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://bad.example/dns-query", &bad, 10),
        testEntry("https://good.example/dns-query", &good, 20),
    };
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var buf: [512]u8 = undefined;
    _ = try pool.exchange(io, query_bytes, &buf);
    _ = try pool.exchange(io, query_bytes, &buf);

    var out: [4]Snapshot = undefined;
    const written = try pool.snapshot(io, &out);
    try testing.expectEqual(@as(usize, 2), written);

    try testing.expectEqualStrings("https://bad.example/dns-query", out[0].url);
    try testing.expect(out[0].enabled);
    try testing.expect(!out[0].available);
    try testing.expectEqual(@as(u32, 2), out[0].consecutive_failures);
    try testing.expectEqual(@as(u64, 2), out[0].total_failures);
    try testing.expectEqual(@as(u64, 0), out[0].total_successes);
    try testing.expectEqual(@as(f32, 0.0), out[0].success_rate);
    try testing.expectEqualStrings("Timeout", out[0].last_error);
    try testing.expect(out[0].last_error_at != null);
    try testing.expect(out[0].backoff_until != null);

    try testing.expectEqualStrings("https://good.example/dns-query", out[1].url);
    try testing.expect(out[1].available);
    try testing.expectEqual(@as(u64, 2), out[1].total_successes);
    try testing.expectEqual(@as(f32, 1.0), out[1].success_rate);
    try testing.expect(out[1].last_success_at != null);
    try testing.expectEqual(@as(?std.Io.Timestamp, null), out[1].backoff_until);

    // A short `out` truncates rather than overflowing.
    var one: [1]Snapshot = undefined;
    try testing.expectEqual(@as(usize, 1), try pool.snapshot(io, &one));
}

/// `Io.concurrent` stores the return value in the future, so the reply slice is
/// reduced to its length here and the bytes are read back out of the caller's
/// buffer. Returning a `usize` also lets the test discard a result with
/// `catch 0`.
fn exchangeLen(pool: *Pool, io: std.Io, buf: []u8) transport.ExchangeError!usize {
    const reply = try pool.exchange(io, query_bytes, buf);
    return reply.len;
}

test "concurrent exchanges through one entry do not overlap" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    // Slow enough that an unserialized second task would still be inside the
    // fake when the first one is, and short enough to stay far under the
    // 10-second attempt budget even when the two run back to back.
    var fake: Fake = .{ .behavior = .{ .slow = .{
        .duration = .{ .raw = .fromMilliseconds(50), .clock = .awake },
        .reply = response_bytes,
    } } };
    var entries = [_]Entry{testEntry("https://only.example/dns-query", &fake, 10)};
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var buf_a: [512]u8 = undefined;
    var buf_b: [512]u8 = undefined;

    var first = io.concurrent(exchangeLen, .{ &pool, io, &buf_a }) catch |err| switch (err) {
        error.ConcurrencyUnavailable => return error.SkipZigTest,
    };
    defer _ = first.await(io) catch 0;
    var second = io.concurrent(exchangeLen, .{ &pool, io, &buf_b }) catch |err| switch (err) {
        error.ConcurrencyUnavailable => return error.SkipZigTest,
    };
    defer _ = second.await(io) catch 0;

    const len_a = try first.await(io);
    const len_b = try second.await(io);

    try testing.expectEqualSlices(u8, response_bytes, buf_a[0..len_a]);
    try testing.expectEqualSlices(u8, response_bytes, buf_b[0..len_b]);
    try testing.expectEqual(@as(usize, 2), fake.calls);
    try testing.expectEqual(@as(u32, 1), fake.peak_in_flight.load(.acquire));
    try testing.expectEqual(@as(u64, 2), entries[0].health.total_successes);
}

test "an entry that enters backoff while a task waits on it is not attempted" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    // One failure is enough to open the backoff window, so the first task's
    // failure makes the entry unavailable to the second task, which by then is
    // already past its own availability check and waiting on `busy`.
    const cfg: health.Config = .{
        .failure_threshold = 1,
        .base_backoff_ms = 60_000,
        .max_backoff_ms = 60_000,
    };

    var slow_bad: Fake = .{ .behavior = .{ .slow_fail = .{
        .duration = .{ .raw = .fromMilliseconds(50), .clock = .awake },
        .err = error.ConnectFailed,
    } } };
    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://slow-bad.example/dns-query", &slow_bad, 10),
        testEntry("https://good.example/dns-query", &good, 20),
    };
    var pool: Pool = .init(&entries, cfg, test_timeouts, 1);

    var buf_a: [512]u8 = undefined;
    var buf_b: [512]u8 = undefined;

    var first = io.concurrent(exchangeLen, .{ &pool, io, &buf_a }) catch |err| switch (err) {
        error.ConcurrencyUnavailable => return error.SkipZigTest,
    };
    defer _ = first.await(io) catch 0;
    var second = io.concurrent(exchangeLen, .{ &pool, io, &buf_b }) catch |err| switch (err) {
        error.ConcurrencyUnavailable => return error.SkipZigTest,
    };
    defer _ = second.await(io) catch 0;

    const len_a = try first.await(io);
    const len_b = try second.await(io);

    // Both tasks fail over to the healthy entry and get an answer.
    try testing.expectEqualSlices(u8, response_bytes, buf_a[0..len_a]);
    try testing.expectEqualSlices(u8, response_bytes, buf_b[0..len_b]);
    try testing.expectEqual(@as(usize, 2), good.calls);

    // The point of the test: the entry was attempted once, not twice. Without
    // the re-check the second task would take the lock and attempt it anyway.
    try testing.expectEqual(@as(usize, 1), slow_bad.calls);
    try testing.expectEqual(@as(u64, 1), entries[0].health.total_failures);
    try testing.expect(entries[0].health.backoff_until != null);
}

test "a successful exchange with nothing open costs the store no statement" {
    if (@FieldType(events.Store, "statements") != u64) return error.SkipZigTest;

    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{testEntry("https://good.example/dns-query", &good, 10)};
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var fx: events_fixture.Fixture = .{};
    try fx.init(io, 1000);
    defer fx.deinit();
    pool.diagnostics = &fx.store;

    var buf: [512]u8 = undefined;
    const before = fx.store.statements;
    for (0..20) |_| _ = try pool.exchange(io, query_bytes, &buf);
    try testing.expectEqual(before, fx.store.statements);
    try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
}

test "a failing then recovering upstream leaves exactly one resolved episode" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var flaky: Fake = .{ .behavior = .{ .fail = error.Timeout } };
    var standby: Fake = .{ .behavior = .{ .reply = response_bytes } };
    var entries = [_]Entry{
        testEntry("https://flaky.example/dns-query", &flaky, 10),
        testEntry("https://standby.example/dns-query", &standby, 20),
    };
    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);

    var fx: events_fixture.Fixture = .{};
    try fx.init(io, 1000);
    defer fx.deinit();
    pool.diagnostics = &fx.store;

    var buf: [512]u8 = undefined;
    _ = try pool.exchange(io, query_bytes, &buf);
    // Backoff would park the failing entry, so the second failure is driven
    // through `recordFailure` itself rather than through another exchange.
    pool.recordFailure(io, &entries[0], std.Io.Clock.awake.now(io), error.ConnectFailed);

    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
    try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
    try testing.expectEqualStrings("upstream.exchange", try fx.text("SELECT code FROM operational_events"));
    try testing.expectEqualStrings(
        "https://flaky.example/dns-query",
        try fx.text("SELECT subject_key FROM operational_events"),
    );

    flaky.behavior = .{ .reply = response_bytes };
    pool.recordSuccess(io, &entries[0], std.Io.Clock.awake.now(io));

    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
    try testing.expectEqual(
        @as(i64, 0),
        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
    );
    try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
}
