milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/web/metrics.zig

@@ -30,6 +30,7 @@ const dns_cache = @import("../cache/dns_cache.zig");
 const dns_handler = @import("../server/handler.zig");
 const disk_monitor = @import("../storage/disk_monitor.zig");
 const dot_server = @import("../server/dot_server.zig");
+const events_mod = @import("../storage/events.zig");
 const history_mod = @import("../upstream/history.zig");
 const http_util = @import("http_util.zig");
 const logging = @import("../platform/logging.zig");
@@ -115,6 +116,15 @@ pub const UpstreamSample = struct {
     success_rate: f32,
 };
 
+/// The diagnostics store, as one scrape sees it. Two gauges and a counter,
+/// written out rather than reflected over a stats struct because they are not
+/// all the same kind of number.
+pub const DiagnosticsSample = struct {
+    active_warnings: u32,
+    active_errors: u32,
+    write_failures: u64,
+};
+
 /// Everything one scrape reports. A null section is a collaborator the state
 /// does not have.
 pub const Sample = struct {
@@ -129,6 +139,11 @@ pub const Sample = struct {
     /// The upstream-history flush loop's counters (m26 ruling 7). Absent while
     /// no accumulator is wired, like every other collaborator.
     history: ?history_mod.Accumulator.Stats = null,
+    /// The diagnostics store's open episodes and its failed writes. Absent
+    /// while no store is wired, like every other collaborator — an operator
+    /// distinguishes "no series" from "zero episodes" through `/api/health`,
+    /// which says which of the two it is.
+    diagnostics: ?DiagnosticsSample = null,
     blocklist: ?BlocklistSample = null,
     disk: ?DiskSample = null,
     /// One entry per enabled TLS endpoint (milestone-10 ruling 10). Rendered
@@ -205,6 +220,15 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
 
     if (state.history) |history| sample.history = history.snapshotStats(io);
 
+    if (state.events) |store| {
+        const counts = store.activeCounts(io);
+        sample.diagnostics = .{
+            .active_warnings = counts.warnings,
+            .active_errors = counts.errors,
+            .write_failures = store.writeFailures(),
+        };
+    }
+
     if (state.manager) |manager| {
         const generation: ?u64 = if (manager.acquire(io)) |acquired| gen: {
             defer acquired.release(io);
@@ -388,6 +412,27 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
         );
     }
 
+    if (sample.diagnostics) |diagnostics| {
+        try gauge(
+            w,
+            "nxdns_diagnostics_active_warnings",
+            "Operational event episodes open right now at warning severity.",
+            diagnostics.active_warnings,
+        );
+        try gauge(
+            w,
+            "nxdns_diagnostics_active_errors",
+            "Operational event episodes open right now at error severity.",
+            diagnostics.active_errors,
+        );
+        try counter(
+            w,
+            "nxdns_diagnostics_write_failures_total",
+            "Operational events dropped because the diagnostics database refused the write.",
+            diagnostics.write_failures,
+        );
+    }
+
     if (sample.blocklist) |blocklist| {
         try counter(
             w,
@@ -638,8 +683,10 @@ fn writeLabelValue(w: *std.Io.Writer, value: []const u8) std.Io.Writer.Error!voi
 // tests
 // ---------------------------------------------------------------------------
 
+const db = @import("../storage/db.zig");
 const local_tables = @import("../server/local_tables.zig");
 const logger_mod = @import("../storage/logger.zig");
+const migrations = @import("../storage/migrations.zig");
 const testing = std.testing;
 
 /// A handler with no upstream reachable: every test here reads counters and
@@ -781,6 +828,55 @@ test "the upstream-history family renders three counters and one gauge" {
     try testing.expect(!std.mem.containsAtLeast(u8, bare, 1, "nxdns_upstream_history_"));
 }
 
+test "the diagnostics family renders two gauges and one counter" {
+    const text = try renderToString(testing.allocator, .{
+        .diagnostics = .{ .active_warnings = 3, .active_errors = 1, .write_failures = 7 },
+    });
+    defer testing.allocator.free(text);
+
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_diagnostics_active_warnings gauge\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_diagnostics_active_warnings 3\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_diagnostics_active_errors gauge\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_diagnostics_active_errors 1\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_diagnostics_write_failures_total counter\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_diagnostics_write_failures_total 7\n"));
+
+    // No store is an absent family, not a family of zeros: "no episodes open"
+    // and "nothing is recording them" must not render the same.
+    const bare = try renderToString(testing.allocator, .{});
+    defer testing.allocator.free(bare);
+    try testing.expect(!std.mem.containsAtLeast(u8, bare, 1, "nxdns_diagnostics_"));
+}
+
+test "collect reads the diagnostics store's open episodes and failed writes" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try db.Db.open(":memory:", .{ .mode = .memory });
+    defer database.close();
+    try db.applyPragmas(&database, .{});
+    _ = try migrations.migrate(&database);
+    var store = try events_mod.Store.init(io, &database, 1000);
+    store.report(io, 1000, .disk_space, "data", "data", .warning, "low");
+
+    var state: server.WebState = .{ .gpa = testing.allocator, .events = &store };
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena.deinit();
+
+    const sample = try collect(&state, io, arena.allocator());
+    try testing.expectEqual(@as(u32, 1), sample.diagnostics.?.active_warnings);
+    try testing.expectEqual(@as(u32, 0), sample.diagnostics.?.active_errors);
+    try testing.expectEqual(@as(u64, 0), sample.diagnostics.?.write_failures);
+
+    // A write that cannot land is counted here, through the production path
+    // rather than by poking the field.
+    try database.exec("DROP TABLE operational_events;");
+    store.report(io, 1100, .disk_space, "data", "data", .warning, "low");
+    const after = try collect(&state, io, arena.allocator());
+    try testing.expect(after.diagnostics.?.write_failures >= 1);
+}
+
 test "every HELP line has a TYPE line and a sample, and every sample a name" {
     const text = try renderToString(testing.allocator, .{});
     defer testing.allocator.free(text);