milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/web/handlers/health.zig

@@ -25,6 +25,15 @@ pub const Disk = struct {
     sample_failures: u64,
 };
 
+/// The diagnostics store's own state, not a summary of what it holds: `state`
+/// answers "is the operational log recording", and the two counts answer "what
+/// is open right now".
+pub const Diagnostics = struct {
+    state: []const u8,
+    active_warnings: u32,
+    active_errors: u32,
+};
+
 pub const Upstreams = struct {
     available: u32,
     total: u32,
@@ -34,6 +43,7 @@ pub const Body = struct {
     status: []const u8,
     disk: Disk,
     upstreams: Upstreams,
+    diagnostics: Diagnostics,
     queries_dropped: u64,
     writer_failed: bool,
     refreshes_gated: u64,
@@ -61,6 +71,16 @@ pub const Input = struct {
     /// one overflow. Drops surface through the metric and through the API's
     /// per-window `complete` instead.
     history_flush_failing: bool = false,
+    /// The diagnostics store exists. The benign default matches every other
+    /// field here — a half-wired `Input` reports a box with nothing wrong — but
+    /// `collect` must assign it explicitly, because in a serving process an
+    /// absent store means `Store.init` failed.
+    diagnostics_present: bool = true,
+    /// The last diagnostics write failed. Current state, cleared by the next
+    /// write that succeeds, like `history_flush_failing`.
+    diagnostics_write_failed: bool = false,
+    diagnostics_active_warnings: u32 = 0,
+    diagnostics_active_errors: u32 = 0,
     refreshes_gated: u64 = 0,
     snapshot_generation: ?u64 = null,
 };
@@ -68,6 +88,16 @@ pub const Input = struct {
 pub const status_ok = "ok";
 pub const status_degraded = "degraded";
 
+pub const diagnostics_recording = "recording";
+pub const diagnostics_unavailable = "unavailable";
+
+/// The operational log is not recording — either the store never opened or its
+/// writes are failing. Both mean the same thing to an operator: the record of
+/// what went wrong is not being kept.
+pub fn diagnosticsUnavailable(input: Input) bool {
+    return !input.diagnostics_present or input.diagnostics_write_failed;
+}
+
 /// Conditions an operator must act on, and every one of them is a fact about
 /// now rather than a count of the past: a disk that is filling stops the query
 /// log, a pool with nothing available stops resolution, a failed writer means
@@ -76,7 +106,7 @@ pub const status_degraded = "degraded";
 /// the underlying condition does.
 pub fn degraded(input: Input) bool {
     return input.disk_state != .ok or input.upstreams_available == 0 or
-        input.writer_failed or input.history_flush_failing;
+        input.writer_failed or input.history_flush_failing or diagnosticsUnavailable(input);
 }
 
 pub fn rollup(input: Input) Body {
@@ -90,6 +120,11 @@ pub fn rollup(input: Input) Body {
             .sample_failures = input.disk_sample_failures,
         },
         .upstreams = .{ .available = input.upstreams_available, .total = input.upstreams_total },
+        .diagnostics = .{
+            .state = if (diagnosticsUnavailable(input)) diagnostics_unavailable else diagnostics_recording,
+            .active_warnings = input.diagnostics_active_warnings,
+            .active_errors = input.diagnostics_active_errors,
+        },
         .queries_dropped = input.queries_dropped,
         .writer_failed = input.writer_failed,
         .refreshes_gated = input.refreshes_gated,
@@ -128,6 +163,17 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
         input.writer_failed = logger.writer_failed.load(.monotonic);
     }
 
+    // Assigned before the `if`, not inside it: the field's benign default is
+    // `true`, so the natural `if (state.events) |store|` shape would report an
+    // absent store as recording — the one case that must degrade.
+    input.diagnostics_present = state.events != null;
+    if (state.events) |store| {
+        input.diagnostics_write_failed = store.writeFailed();
+        const counts = store.activeCounts(io);
+        input.diagnostics_active_warnings = counts.warnings;
+        input.diagnostics_active_errors = counts.errors;
+    }
+
     if (state.history) |history| {
         input.history_flush_failing = history.snapshotStats(io).last_flush_failed;
     }
@@ -148,6 +194,8 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
 // ---------------------------------------------------------------------------
 
 const db = @import("../../storage/db.zig");
+const events_mod = @import("../../storage/events.zig");
+const migrations = @import("../../storage/migrations.zig");
 const history_mod = @import("../../upstream/history.zig");
 const logger_mod = @import("../../storage/logger.zig");
 const upstream_history_repo = @import("../../storage/repositories/upstream_history_repo.zig");
@@ -172,6 +220,11 @@ test "the degraded matrix covers disk state, availability and the writer" {
         // right now, and it recovers on its own the moment a flush succeeds.
         .{ .input = withHistoryFailing(healthy, true), .degraded = true },
         .{ .input = withHistoryFailing(healthy, false), .degraded = false },
+        // The operational log not recording is itself a fault an operator must
+        // act on: whatever fails next will leave no record of having failed.
+        .{ .input = withDiagnostics(healthy, false, false), .degraded = true },
+        .{ .input = withDiagnostics(healthy, true, true), .degraded = true },
+        .{ .input = withDiagnostics(healthy, true, false), .degraded = false },
         // Two faults at once still report one status.
         .{ .input = withWriterFailed(withDisk(healthy, .critical)), .degraded = true },
         // Some upstreams down is not degraded while one still answers.
@@ -212,6 +265,66 @@ fn withHistoryFailing(input: Input, failing: bool) Input {
     return out;
 }
 
+fn withDiagnostics(input: Input, present: bool, write_failed: bool) Input {
+    var out = input;
+    out.diagnostics_present = present;
+    out.diagnostics_write_failed = write_failed;
+    return out;
+}
+
+test "the diagnostics block reports the state and the open counts" {
+    const recording = rollup(.{
+        .upstreams_available = 1,
+        .diagnostics_active_warnings = 3,
+        .diagnostics_active_errors = 1,
+    });
+    try testing.expectEqualStrings(diagnostics_recording, recording.diagnostics.state);
+    try testing.expectEqual(@as(u32, 3), recording.diagnostics.active_warnings);
+    try testing.expectEqual(@as(u32, 1), recording.diagnostics.active_errors);
+    // Open episodes are what the box is doing, not a fault of the log: they do
+    // not degrade on their own.
+    try testing.expectEqualStrings(status_ok, recording.status);
+
+    // Failing writes: the counts are whatever was last read, and the state is
+    // the honest one.
+    const failing = rollup(.{ .upstreams_available = 1, .diagnostics_write_failed = true });
+    try testing.expectEqualStrings(diagnostics_unavailable, failing.diagnostics.state);
+    try testing.expectEqualStrings(status_degraded, failing.status);
+
+    const absent = rollup(.{ .upstreams_available = 1, .diagnostics_present = false });
+    try testing.expectEqualStrings(diagnostics_unavailable, absent.diagnostics.state);
+    try testing.expectEqualStrings(status_degraded, absent.status);
+}
+
+test "collect reports an absent store as unavailable rather than as recording" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    // `diagnostics_present` defaults to true like every other benign default,
+    // so an assignment `collect` forgot would read as a healthy log here.
+    var state: server.WebState = .{ .gpa = testing.allocator };
+    const absent = collect(&state, io);
+    try testing.expect(!absent.diagnostics_present);
+    try testing.expectEqualStrings(diagnostics_unavailable, rollup(absent).diagnostics.state);
+
+    var database = try db.Db.open(":memory:", .{ .mode = .memory });
+    defer database.close();
+    try db.applyPragmas(&database, .{});
+    _ = try migrations.migrate(&database);
+    var store = try events_mod.Store.init(io, &database, 1000);
+    store.report(io, 1000, .disk_space, "data", "data", .warning, "low");
+    store.report(io, 1000, .listener_start, "doh", "doh", .@"error", "AddressInUse");
+
+    state.events = &store;
+    const present = collect(&state, io);
+    try testing.expect(present.diagnostics_present);
+    try testing.expect(!present.diagnostics_write_failed);
+    try testing.expectEqual(@as(u32, 1), present.diagnostics_active_warnings);
+    try testing.expectEqual(@as(u32, 1), present.diagnostics_active_errors);
+    try testing.expectEqualStrings(diagnostics_recording, rollup(present).diagnostics.state);
+}
+
 test "a history overflow that already happened does not degrade the rollup" {
     // `rows_dropped` is cumulative and the rollup is stateless, so the only
     // thing it could do with a drop count is latch on it. The accumulator's