milestone 30: overview as a dashboard, explicit health contract, period aggregations

src/web/handlers/health.zig

@@ -6,6 +6,13 @@
 //!
 //! Unauthenticated and rate-limit exempt, like `/metrics`.
 //!
+//! **Nothing may degrade the rollup without appearing in the response.** The
+//! body is five condition objects and a status computed from exactly their
+//! states, so an operator reading it can always name the condition that
+//! degraded the box. Anything a counter already records — gated refreshes,
+//! disk sample failures, the snapshot generation — stays in `/metrics` and in
+//! Diagnostics rather than becoming a sixth hidden input here.
+//!
 //! `rollup` is pure so the whole degraded matrix is testable without a running
 //! server; `handle` only gathers the inputs.
 
@@ -13,16 +20,39 @@ const std = @import("std");
 
 const disk_monitor = @import("../../storage/disk_monitor.zig");
 const http_util = @import("../http_util.zig");
+const logger_mod = @import("../../storage/logger.zig");
 const metrics = @import("../metrics.zig");
+const pause_mod = @import("../../server/pause.zig");
 const pool_mod = @import("../../upstream/pool.zig");
 const server = @import("../server.zig");
 
-pub const Disk = struct {
+/// Is filtering in force. `unavailable` is not an operator's doing: it is the
+/// state in which the query path has no filter snapshot to evaluate against,
+/// which `handler.zig` records as the `snapshot_unavailable` provenance.
+pub const Protection = struct {
     state: []const u8,
-    free_bytes: u64,
-    db_bytes: u64,
-    log_bytes: u64,
-    sample_failures: u64,
+    /// The second filtering resumes at. Null for both an indefinite pause and
+    /// every non-paused state, which the state field already tells apart.
+    until: ?i64,
+};
+
+pub const Upstreams = struct {
+    state: []const u8,
+    available: u32,
+    /// Enabled upstreams: the pool is built from those alone, so this is what
+    /// `available` is out of.
+    total: u32,
+};
+
+/// Whether Activity can be trusted. `dropped_total` is cumulative and does not
+/// decide the state — a drop that happened an hour ago is not a fault now.
+pub const QueryHistory = struct {
+    state: []const u8,
+    dropped_total: u64,
+    /// The newest drop, or null while nothing has been dropped. Stamped by a
+    /// separate atomic from the count, so a reader can momentarily see a
+    /// non-zero `dropped_total` beside a null here.
+    last_drop_s: ?i64,
 };
 
 /// The diagnostics store's own state, not a summary of what it holds: `state`
@@ -34,21 +64,18 @@ pub const Diagnostics = struct {
     active_errors: u32,
 };
 
-pub const Upstreams = struct {
-    available: u32,
-    total: u32,
+pub const Disk = struct {
+    state: []const u8,
+    free_bytes: u64,
 };
 
 pub const Body = struct {
     status: []const u8,
-    disk: Disk,
+    protection: Protection,
     upstreams: Upstreams,
+    query_history: QueryHistory,
     diagnostics: Diagnostics,
-    queries_dropped: u64,
-    writer_failed: bool,
-    refreshes_gated: u64,
-    /// Null before the first filter snapshot is published.
-    snapshot_generation: ?u64,
+    disk: Disk,
 };
 
 /// What the rollup is computed from. Every field has a defined value even when
@@ -56,41 +83,79 @@ pub const Body = struct {
 /// server should report: no disk reading, no upstreams, nothing published.
 pub const Input = struct {
     disk_state: disk_monitor.State = .ok,
-    disk: disk_monitor.Gauges = .{ .free_bytes = 0, .db_bytes = 0, .log_bytes = 0 },
-    disk_sample_failures: u64 = 0,
+    disk_free_bytes: u64 = 0,
     upstreams_available: u32 = 0,
     upstreams_total: u32 = 0,
     queries_dropped: u64 = 0,
+    last_drop_s: ?i64 = null,
     writer_failed: bool = false,
-    /// Current state, not a count: the upstream-history flush is failing right
-    /// now. Cleared by the next flush that succeeds (m26 ruling 7).
-    ///
-    /// `rows_dropped` deliberately does not appear here. It is cumulative, and
-    /// a rollup that is computed statelessly cannot ask whether a counter grew
-    /// — so feeding it in would latch `/api/health` to degraded forever after
-    /// one overflow. Drops surface through the metric and through the API's
-    /// per-window `complete` instead.
-    history_flush_failing: bool = false,
+    /// The logger's disk-gating episode. `losing` is the only state that
+    /// degrades: it means the gate is holding writes back *and* has already
+    /// cost rows in the episode that is open now.
+    gate_episode: logger_mod.GateEpisode = .open,
+    /// A filter snapshot exists for the query path to evaluate against. The
+    /// benign default matches every other field here, and `collect` assigns it
+    /// explicitly for the same reason `diagnostics_present` is assigned there.
+    snapshot_available: bool = true,
+    /// `pause.Pause.until` verbatim: 0 not paused, -1 indefinite, otherwise the
+    /// second filtering resumes at. Raw rather than a decided boolean so the
+    /// expiry rule stays `pause.Pause`'s and is exercised by these tests.
+    pause_until: i64 = 0,
+    /// The clock the pause is compared against.
+    now_s: i64 = 0,
     /// The diagnostics store exists. The benign default matches every other
     /// field here — a half-wired `Input` reports a box with nothing wrong — but
     /// `collect` must assign it explicitly, because in a serving process an
     /// absent store means `Store.init` failed.
     diagnostics_present: bool = true,
     /// The last diagnostics write failed. Current state, cleared by the next
-    /// write that succeeds, like `history_flush_failing`.
+    /// write that succeeds.
     diagnostics_write_failed: bool = false,
     diagnostics_active_warnings: u32 = 0,
     diagnostics_active_errors: u32 = 0,
-    refreshes_gated: u64 = 0,
-    snapshot_generation: ?u64 = null,
 };
 
 pub const status_ok = "ok";
 pub const status_degraded = "degraded";
 
+pub const protection_active = "active";
+pub const protection_paused = "paused";
+pub const protection_unavailable = "unavailable";
+
+pub const upstreams_ok = "ok";
+pub const upstreams_unavailable = "unavailable";
+
+pub const query_history_recording = "recording";
+pub const query_history_losing = "losing";
+pub const query_history_failed = "failed";
+
 pub const diagnostics_recording = "recording";
 pub const diagnostics_unavailable = "unavailable";
 
+pub const disk_ok = "ok";
+pub const disk_low = "low";
+pub const disk_critical = "critical";
+
+/// Precedence `unavailable` → `paused` → `active`. With no snapshot the pause
+/// flag says nothing an operator can act on: filtering is off either way, and
+/// resuming would not turn it back on.
+pub fn protection(input: Input) Protection {
+    if (!input.snapshot_available) return .{ .state = protection_unavailable, .until = null };
+
+    const state: pause_mod.Pause = .{ .until = .init(input.pause_until) };
+    if (!state.isPaused(input.now_s)) return .{ .state = protection_active, .until = null };
+    return .{
+        .state = protection_paused,
+        .until = if (input.pause_until > 0) input.pause_until else null,
+    };
+}
+
+pub fn queryHistoryState(input: Input) []const u8 {
+    if (input.writer_failed) return query_history_failed;
+    if (input.gate_episode == .losing) return query_history_losing;
+    return query_history_recording;
+}
+
 /// The operational log is not recording — either the store never opened or its
 /// writes are failing. Both mean the same thing to an operator: the record of
 /// what went wrong is not being kept.
@@ -98,37 +163,51 @@ pub fn diagnosticsUnavailable(input: Input) bool {
     return !input.diagnostics_present or input.diagnostics_write_failed;
 }
 
-/// Conditions an operator must act on, and every one of them is a fact about
-/// now rather than a count of the past: a disk that is filling stops the query
-/// log, a pool with nothing available stops resolution, a failed writer means
-/// rows are being lost right now, and a failing history flush means the
-/// dashboard's upstream numbers are not being recorded. Each clears itself when
-/// the underlying condition does.
+/// `warn` reads as a log level rather than as a quantity of disk. The monitor
+/// keeps its own name; the wire says what an operator sees on the page.
+pub fn diskState(state: disk_monitor.State) []const u8 {
+    return switch (state) {
+        .ok => disk_ok,
+        .warn => disk_low,
+        .critical => disk_critical,
+    };
+}
+
+/// The degrading set, named rather than gestured at: protection `unavailable`,
+/// upstreams `unavailable`, query history `losing` or `failed`, diagnostics
+/// `unavailable`, disk `low` or `critical`.
+///
+/// A pause is deliberately not in it. It is an operator's own choice, and a
+/// monitor that pages on it would be paging on a button the operator pressed.
 pub fn degraded(input: Input) bool {
-    return input.disk_state != .ok or input.upstreams_available == 0 or
-        input.writer_failed or input.history_flush_failing or diagnosticsUnavailable(input);
+    const history = queryHistoryState(input);
+    return !input.snapshot_available or
+        input.upstreams_available == 0 or
+        !std.mem.eql(u8, history, query_history_recording) or
+        diagnosticsUnavailable(input) or
+        input.disk_state != .ok;
 }
 
 pub fn rollup(input: Input) Body {
     return .{
         .status = if (degraded(input)) status_degraded else status_ok,
-        .disk = .{
-            .state = @tagName(input.disk_state),
-            .free_bytes = input.disk.free_bytes,
-            .db_bytes = input.disk.db_bytes,
-            .log_bytes = input.disk.log_bytes,
-            .sample_failures = input.disk_sample_failures,
+        .protection = protection(input),
+        .upstreams = .{
+            .state = if (input.upstreams_available == 0) upstreams_unavailable else upstreams_ok,
+            .available = input.upstreams_available,
+            .total = input.upstreams_total,
+        },
+        .query_history = .{
+            .state = queryHistoryState(input),
+            .dropped_total = input.queries_dropped,
+            .last_drop_s = input.last_drop_s,
         },
-        .upstreams = .{ .available = input.upstreams_available, .total = input.upstreams_total },
         .diagnostics = .{
             .state = if (diagnosticsUnavailable(input)) diagnostics_unavailable else diagnostics_recording,
             .active_warnings = input.diagnostics_active_warnings,
             .active_errors = input.diagnostics_active_errors,
         },
-        .queries_dropped = input.queries_dropped,
-        .writer_failed = input.writer_failed,
-        .refreshes_gated = input.refreshes_gated,
-        .snapshot_generation = input.snapshot_generation,
+        .disk = .{ .state = diskState(input.disk_state), .free_bytes = input.disk_free_bytes },
     };
 }
 
@@ -143,10 +222,11 @@ pub fn handle(
 pub fn collect(state: *server.WebState, io: std.Io) Input {
     var input: Input = .{};
 
+    input.now_s = std.Io.Clock.real.now(io).toSeconds();
+
     if (state.monitor) |monitor| {
         input.disk_state = monitor.state();
-        input.disk = monitor.gauges();
-        input.disk_sample_failures = monitor.sample_failures.load(.monotonic);
+        input.disk_free_bytes = monitor.gauges().free_bytes;
     }
 
     if (state.pool) |pool| {
@@ -160,9 +240,13 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
 
     if (state.logger) |logger| {
         input.queries_dropped = logger.queries_dropped.load(.monotonic);
+        input.last_drop_s = logger.lastDropSeconds();
         input.writer_failed = logger.writer_failed.load(.monotonic);
+        input.gate_episode = logger.gateEpisode();
     }
 
+    if (state.pause) |paused| input.pause_until = paused.until.load(.monotonic);
+
     // Assigned before the `if`, not inside it: the field's benign default is
     // `true`, so the natural `if (state.events) |store|` shape would report an
     // absent store as recording — the one case that must degrade.
@@ -174,15 +258,13 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
         input.diagnostics_active_errors = counts.errors;
     }
 
-    if (state.history) |history| {
-        input.history_flush_failing = history.snapshotStats(io).last_flush_failed;
-    }
-
+    // The generation itself is not in the body — the UI-facing fact is whether
+    // protection has a snapshot at all, and `/metrics` keeps the number.
+    input.snapshot_available = false;
     if (state.manager) |manager| {
-        input.refreshes_gated = manager.refreshesGated();
         if (manager.acquire(io)) |acquired| {
             defer acquired.release(io);
-            input.snapshot_generation = acquired.snapshot.generation;
+            input.snapshot_available = true;
         }
     }
 
@@ -196,104 +278,175 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
 const db = @import("../../storage/db.zig");
 const events_mod = @import("../../storage/events.zig");
 const migrations = @import("../../storage/migrations.zig");
-const history_mod = @import("../../upstream/history.zig");
-const logger_mod = @import("../../storage/logger.zig");
-const upstream_history_repo = @import("../../storage/repositories/upstream_history_repo.zig");
 const testing = std.testing;
 
-/// A box with nothing wrong with it: one upstream up, disk ok, writer alive.
+/// A box with nothing wrong with it: one upstream up, disk ok, writer alive,
+/// a snapshot published and filtering unpaused.
 const healthy: Input = .{
     .disk_state = .ok,
     .upstreams_available = 1,
     .upstreams_total = 1,
     .writer_failed = false,
+    .snapshot_available = true,
 };
 
-test "the degraded matrix covers disk state, availability and the writer" {
-    const cases = [_]struct { input: Input, degraded: bool }{
-        .{ .input = healthy, .degraded = false },
-        .{ .input = withDisk(healthy, .warn), .degraded = true },
-        .{ .input = withDisk(healthy, .critical), .degraded = true },
-        .{ .input = withAvailable(healthy, 0), .degraded = true },
-        .{ .input = withWriterFailed(healthy), .degraded = true },
-        // A failing upstream-history flush is losing the dashboard's numbers
-        // right now, and it recovers on its own the moment a flush succeeds.
-        .{ .input = withHistoryFailing(healthy, true), .degraded = true },
-        .{ .input = withHistoryFailing(healthy, false), .degraded = false },
-        // The operational log not recording is itself a fault an operator must
-        // act on: whatever fails next will leave no record of having failed.
-        .{ .input = withDiagnostics(healthy, false, false), .degraded = true },
-        .{ .input = withDiagnostics(healthy, true, true), .degraded = true },
-        .{ .input = withDiagnostics(healthy, true, false), .degraded = false },
-        // Two faults at once still report one status.
-        .{ .input = withWriterFailed(withDisk(healthy, .critical)), .degraded = true },
-        // Some upstreams down is not degraded while one still answers.
-        .{ .input = .{ .upstreams_available = 1, .upstreams_total = 3 }, .degraded = false },
-    };
+fn with(input: Input, comptime field: []const u8, value: anytype) Input {
+    var out = input;
+    @field(out, field) = value;
+    return out;
+}
 
-    for (cases, 0..) |case, i| {
-        errdefer std.debug.print("case {d}\n", .{i});
-        try testing.expectEqual(case.degraded, degraded(case.input));
-        try testing.expectEqualStrings(
-            if (case.degraded) status_degraded else status_ok,
-            rollup(case.input).status,
-        );
+test "each degrading condition degrades on its own and says so in the body" {
+    try testing.expectEqualStrings(status_ok, rollup(healthy).status);
+
+    {
+        const body = rollup(with(healthy, "snapshot_available", false));
+        try testing.expectEqualStrings(status_degraded, body.status);
+        try testing.expectEqualStrings(protection_unavailable, body.protection.state);
+    }
+    {
+        const body = rollup(with(healthy, "upstreams_available", @as(u32, 0)));
+        try testing.expectEqualStrings(status_degraded, body.status);
+        try testing.expectEqualStrings(upstreams_unavailable, body.upstreams.state);
+    }
+    {
+        const body = rollup(with(healthy, "gate_episode", .losing));
+        try testing.expectEqualStrings(status_degraded, body.status);
+        try testing.expectEqualStrings(query_history_losing, body.query_history.state);
+    }
+    {
+        const body = rollup(with(healthy, "writer_failed", true));
+        try testing.expectEqualStrings(status_degraded, body.status);
+        try testing.expectEqualStrings(query_history_failed, body.query_history.state);
+    }
+    {
+        const body = rollup(with(healthy, "diagnostics_present", false));
+        try testing.expectEqualStrings(status_degraded, body.status);
+        try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state);
+    }
+    {
+        const body = rollup(with(healthy, "diagnostics_write_failed", true));
+        try testing.expectEqualStrings(status_degraded, body.status);
+        try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state);
+    }
+    for ([_]struct { disk_monitor.State, []const u8 }{
+        .{ .warn, disk_low },
+        .{ .critical, disk_critical },
+    }) |case| {
+        const body = rollup(with(healthy, "disk_state", case[0]));
+        try testing.expectEqualStrings(status_degraded, body.status);
+        try testing.expectEqualStrings(case[1], body.disk.state);
     }
 }
 
-fn withDisk(input: Input, state: disk_monitor.State) Input {
-    var out = input;
-    out.disk_state = state;
-    return out;
+test "conditions that are not faults leave the status ok" {
+    // Some upstreams down while one still answers.
+    const partial = rollup(.{ .upstreams_available = 1, .upstreams_total = 3 });
+    try testing.expectEqualStrings(status_ok, partial.status);
+    try testing.expectEqualStrings(upstreams_ok, partial.upstreams.state);
+
+    // A gate holding writes that has not cost a row yet.
+    const holding = rollup(with(healthy, "gate_episode", .gated));
+    try testing.expectEqualStrings(status_ok, holding.status);
+    try testing.expectEqualStrings(query_history_recording, holding.query_history.state);
+
+    // Drops that already happened. The counter is cumulative and the rollup is
+    // stateless, so feeding it in would latch the box to degraded forever.
+    var dropped = with(healthy, "queries_dropped", @as(u64, 9));
+    dropped.last_drop_s = 1_700_000_000;
+    const body = rollup(dropped);
+    try testing.expectEqualStrings(status_ok, body.status);
+    try testing.expectEqualStrings(query_history_recording, body.query_history.state);
+    try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total);
+    try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s);
+
+    // Open diagnostics episodes are what the box is doing, not a fault of the
+    // log that recorded them.
+    var open = healthy;
+    open.diagnostics_active_warnings = 3;
+    open.diagnostics_active_errors = 1;
+    const with_episodes = rollup(open);
+    try testing.expectEqualStrings(status_ok, with_episodes.status);
+    try testing.expectEqualStrings(diagnostics_recording, with_episodes.diagnostics.state);
+    try testing.expectEqual(@as(u32, 3), with_episodes.diagnostics.active_warnings);
+    try testing.expectEqual(@as(u32, 1), with_episodes.diagnostics.active_errors);
 }
 
-fn withAvailable(input: Input, available: u32) Input {
-    var out = input;
-    out.upstreams_available = available;
-    return out;
+test "a pause is surfaced, never alarmed, and an expired one is over" {
+    var indefinite = healthy;
+    indefinite.pause_until = -1;
+    indefinite.now_s = 1_000;
+    const forever = rollup(indefinite);
+    try testing.expectEqualStrings(status_ok, forever.status);
+    try testing.expectEqualStrings(protection_paused, forever.protection.state);
+    try testing.expectEqual(@as(?i64, null), forever.protection.until);
+
+    var timed = healthy;
+    timed.pause_until = 1_060;
+    timed.now_s = 1_000;
+    const live = rollup(timed);
+    try testing.expectEqualStrings(status_ok, live.status);
+    try testing.expectEqualStrings(protection_paused, live.protection.state);
+    try testing.expectEqual(@as(?i64, 1_060), live.protection.until);
+
+    // The stored second is when filtering is back on, so at it the pause is over.
+    timed.now_s = 1_060;
+    const expired = rollup(timed);
+    try testing.expectEqualStrings(protection_active, expired.protection.state);
+    try testing.expectEqual(@as(?i64, null), expired.protection.until);
 }
 
-fn withWriterFailed(input: Input) Input {
-    var out = input;
-    out.writer_failed = true;
-    return out;
+test "protection unavailable wins over a live pause" {
+    var both = healthy;
+    both.snapshot_available = false;
+    both.pause_until = -1;
+    const body = rollup(both);
+    try testing.expectEqualStrings(protection_unavailable, body.protection.state);
+    try testing.expectEqual(@as(?i64, null), body.protection.until);
+    try testing.expectEqualStrings(status_degraded, body.status);
 }
 
-fn withHistoryFailing(input: Input, failing: bool) Input {
-    var out = input;
-    out.history_flush_failing = failing;
-    return out;
+test "a failed writer outranks a losing gate" {
+    var both = healthy;
+    both.writer_failed = true;
+    both.gate_episode = .losing;
+    try testing.expectEqualStrings(query_history_failed, rollup(both).query_history.state);
 }
 
-fn withDiagnostics(input: Input, present: bool, write_failed: bool) Input {
-    var out = input;
-    out.diagnostics_present = present;
-    out.diagnostics_write_failed = write_failed;
-    return out;
+test "two faults at once still report one status" {
+    var both = with(healthy, "disk_state", disk_monitor.State.critical);
+    both.writer_failed = true;
+    try testing.expectEqualStrings(status_degraded, rollup(both).status);
 }
 
-test "the diagnostics block reports the state and the open counts" {
-    const recording = rollup(.{
-        .upstreams_available = 1,
-        .diagnostics_active_warnings = 3,
-        .diagnostics_active_errors = 1,
+test "the body reports every input verbatim" {
+    const body = rollup(.{
+        .disk_state = .warn,
+        .disk_free_bytes = 100,
+        .upstreams_available = 2,
+        .upstreams_total = 4,
+        .queries_dropped = 9,
+        .last_drop_s = 1_700_000_000,
+        .snapshot_available = true,
     });
-    try testing.expectEqualStrings(diagnostics_recording, recording.diagnostics.state);
-    try testing.expectEqual(@as(u32, 3), recording.diagnostics.active_warnings);
-    try testing.expectEqual(@as(u32, 1), recording.diagnostics.active_errors);
-    // Open episodes are what the box is doing, not a fault of the log: they do
-    // not degrade on their own.
-    try testing.expectEqualStrings(status_ok, recording.status);
 
-    // Failing writes: the counts are whatever was last read, and the state is
-    // the honest one.
-    const failing = rollup(.{ .upstreams_available = 1, .diagnostics_write_failed = true });
-    try testing.expectEqualStrings(diagnostics_unavailable, failing.diagnostics.state);
-    try testing.expectEqualStrings(status_degraded, failing.status);
+    try testing.expectEqualStrings(status_degraded, body.status);
+    try testing.expectEqualStrings(disk_low, body.disk.state);
+    try testing.expectEqual(@as(u64, 100), body.disk.free_bytes);
+    try testing.expectEqual(@as(u32, 2), body.upstreams.available);
+    try testing.expectEqual(@as(u32, 4), body.upstreams.total);
+    try testing.expectEqualStrings(upstreams_ok, body.upstreams.state);
+    try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total);
+    try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s);
+    try testing.expectEqualStrings(protection_active, body.protection.state);
+}
 
-    const absent = rollup(.{ .upstreams_available = 1, .diagnostics_present = false });
-    try testing.expectEqualStrings(diagnostics_unavailable, absent.diagnostics.state);
-    try testing.expectEqualStrings(status_degraded, absent.status);
+test "an unstamped drop time serializes as null, not as zero" {
+    var buffer: [1024]u8 = undefined;
+    var writer: std.Io.Writer = .fixed(&buffer);
+    try std.json.Stringify.value(rollup(.{}), .{}, &writer);
+    try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"last_drop_s\":null"));
+    try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"until\":null"));
 }
 
 test "collect reports an absent store as unavailable rather than as recording" {
@@ -307,6 +460,10 @@ test "collect reports an absent store as unavailable rather than as recording" {
     const absent = collect(&state, io);
     try testing.expect(!absent.diagnostics_present);
     try testing.expectEqualStrings(diagnostics_unavailable, rollup(absent).diagnostics.state);
+    // And a bare state has no snapshot either, which is the honest reading of a
+    // process that has published nothing.
+    try testing.expect(!absent.snapshot_available);
+    try testing.expectEqualStrings(protection_unavailable, rollup(absent).protection.state);
 
     var database = try db.Db.open(":memory:", .{ .mode = .memory });
     defer database.close();
@@ -325,76 +482,7 @@ test "collect reports an absent store as unavailable rather than as recording" {
     try testing.expectEqualStrings(diagnostics_recording, rollup(present).diagnostics.state);
 }
 
-test "a history overflow that already happened does not degrade the rollup" {
-    // `rows_dropped` is cumulative and the rollup is stateless, so the only
-    // thing it could do with a drop count is latch on it. The accumulator's
-    // drops reach an operator through `/metrics` and through the per-window
-    // `complete` flag, and never through this.
-    const dropped: Input = .{
-        .upstreams_available = 1,
-        .upstreams_total = 1,
-        .history_flush_failing = false,
-    };
-    try testing.expect(!degraded(dropped));
-    try testing.expectEqualStrings(status_ok, rollup(dropped).status);
-}
-
-test "collect reads the accumulator's current flush state" {
-    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
-    defer threaded.deinit();
-    const io = threaded.io();
-
-    const acc = try testing.allocator.create(history_mod.Accumulator);
-    defer testing.allocator.destroy(acc);
-    acc.* = .init;
-
-    var state: server.WebState = .{ .gpa = testing.allocator, .history = acc };
-    try testing.expect(!collect(&state, io).history_flush_failing);
-
-    var database = try db.Db.open(":memory:", .{ .mode = .memory });
-    defer database.close();
-    acc.recordSuccess(io, "https://a.example", 60);
-    // No schema in this database, so the real write fails and the flag is set
-    // by the production path rather than by a test poking a field.
-    acc.flushOnce(io, &database, upstream_history_repo.flush);
-    try testing.expect(collect(&state, io).history_flush_failing);
-    try testing.expectEqualStrings("degraded", rollup(collect(&state, io)).status);
-}
-
-test "the body reports every input verbatim" {
-    const body = rollup(.{
-        .disk_state = .warn,
-        .disk = .{ .free_bytes = 100, .db_bytes = 20, .log_bytes = 3 },
-        .disk_sample_failures = 2,
-        .upstreams_available = 2,
-        .upstreams_total = 4,
-        .queries_dropped = 9,
-        .writer_failed = false,
-        .refreshes_gated = 1,
-        .snapshot_generation = 12,
-    });
-
-    try testing.expectEqualStrings("degraded", body.status);
-    try testing.expectEqualStrings("warn", body.disk.state);
-    try testing.expectEqual(@as(u64, 100), body.disk.free_bytes);
-    try testing.expectEqual(@as(u64, 20), body.disk.db_bytes);
-    try testing.expectEqual(@as(u64, 3), body.disk.log_bytes);
-    try testing.expectEqual(@as(u64, 2), body.disk.sample_failures);
-    try testing.expectEqual(@as(u32, 2), body.upstreams.available);
-    try testing.expectEqual(@as(u32, 4), body.upstreams.total);
-    try testing.expectEqual(@as(u64, 9), body.queries_dropped);
-    try testing.expectEqual(@as(u64, 1), body.refreshes_gated);
-    try testing.expectEqual(@as(?u64, 12), body.snapshot_generation);
-}
-
-test "an unpublished snapshot serializes as null, not as zero" {
-    var buffer: [512]u8 = undefined;
-    var writer: std.Io.Writer = .fixed(&buffer);
-    try std.json.Stringify.value(rollup(.{}), .{}, &writer);
-    try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"snapshot_generation\":null"));
-}
-
-test "collect reads the logger's counters and reports a bare state as degraded" {
+test "collect reads the logger's counters and the pause flag" {
     var threaded: std.Io.Threaded = .init(testing.allocator, .{});
     defer threaded.deinit();
     const io = threaded.io();
@@ -402,14 +490,25 @@ test "collect reads the logger's counters and reports a bare state as degraded"
     var queue_buf: [2]logger_mod.Entry = undefined;
     var query_logger: logger_mod.Logger = .init(.{}, &queue_buf);
     query_logger.queries_dropped.store(4, .monotonic);
+    query_logger.last_drop_s.store(1_700_000_000, .monotonic);
     query_logger.writer_failed.store(true, .monotonic);
 
-    var state: server.WebState = .{ .gpa = testing.allocator, .logger = &query_logger };
+    var paused: pause_mod.Pause = .{};
+    paused.pauseFor(0, null);
+
+    var state: server.WebState = .{
+        .gpa = testing.allocator,
+        .logger = &query_logger,
+        .pause = &paused,
+    };
     const input = collect(&state, io);
 
     try testing.expectEqual(@as(u64, 4), input.queries_dropped);
+    try testing.expectEqual(@as(?i64, 1_700_000_000), input.last_drop_s);
     try testing.expect(input.writer_failed);
+    try testing.expectEqual(@as(i64, -1), input.pause_until);
     try testing.expectEqual(@as(u32, 0), input.upstreams_total);
-    try testing.expectEqual(@as(?u64, null), input.snapshot_generation);
-    try testing.expectEqualStrings("degraded", rollup(input).status);
+    try testing.expectEqualStrings(status_degraded, rollup(input).status);
+    // No snapshot manager either, so protection outranks the pause here too.
+    try testing.expectEqualStrings(protection_unavailable, rollup(input).protection.state);
 }