milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/server/clients.zig

@@ -25,6 +25,7 @@ const client_names = @import("client_names.zig");
 const clients_repo = @import("../storage/repositories/clients_repo.zig");
 const db = @import("../storage/db.zig");
 const disk_monitor = @import("../storage/disk_monitor.zig");
+const events = @import("../storage/events.zig");
 const logger = @import("../storage/logger.zig");
 
 const log = std.log.scoped(.clients);
@@ -63,6 +64,9 @@ pub const Tracker = struct {
     count: u32,
     passes: u64,
     stats: Stats,
+    /// Wired by the composition root after `init`, following the
+    /// `gate: ?*disk_monitor.Monitor` idiom. Null in every unit test here.
+    diagnostics: ?*events.Store = null,
 
     /// `retention_days` is `logging.retention_days`, the same knob the query log
     /// prunes by (milestone-7 ruling 16). A client silent for that long is as
@@ -185,6 +189,7 @@ pub const Tracker = struct {
 
         var flushed: u64 = 0;
         var failures: u64 = 0;
+        var last_failure: []const u8 = "";
         for (batch) |entry| {
             // `logger.max_client_len` is the RFC 5952 bound every address text
             // in this program is sized by, so `format` cannot fail here.
@@ -199,9 +204,19 @@ pub const Tracker = struct {
                     log.warn("materialising client {s} failed: {s}", .{ w.buffered(), @errorName(err) });
                 }
                 failures += 1;
+                last_failure = @errorName(err);
             }
         }
 
+        // One aggregated outcome per pass, so the row's `occurrences` counts
+        // failing passes rather than failing addresses. A pass that wrote
+        // nothing resolves nothing: an empty batch is not evidence of success.
+        if (failures != 0) {
+            self.reportStorage(io, now_s, "materialise", "materialising a client failed", last_failure, failures);
+        } else if (flushed != 0) {
+            if (self.diagnostics) |store| store.resolve(io, now_s, .clients_storage, "materialise");
+        }
+
         self.mutex.lockUncancelable(io);
         self.passes += 1;
         self.stats.flushed += flushed;
@@ -215,17 +230,39 @@ pub const Tracker = struct {
                 self.mutex.lockUncancelable(io);
                 self.stats.pruned += deleted;
                 self.mutex.unlock(io);
+                if (self.diagnostics) |store| store.resolve(io, now_s, .clients_storage, "prune");
             } else |err| {
                 log.warn("pruning clients before {d} failed: {s}", .{ cutoff, @errorName(err) });
                 self.mutex.lockUncancelable(io);
                 self.stats.flush_failures += 1;
                 self.mutex.unlock(io);
+                self.reportStorage(io, now_s, "prune", "pruning stale clients failed", @errorName(err), 1);
             }
         }
 
         if (names) |resolver| resolver.runPass(io, database, now_s);
     }
 
+    /// One aggregated report per pass; `count` is how many failures it held.
+    fn reportStorage(
+        self: *Tracker,
+        io: std.Io,
+        now_s: i64,
+        operation: []const u8,
+        message: []const u8,
+        error_name: []const u8,
+        count: u64,
+    ) void {
+        const store = self.diagnostics orelse return;
+        var buf: [events.Store.max_detail_len]u8 = undefined;
+        const detail = std.fmt.bufPrint(&buf, "{s}: {s} ({d} this pass)", .{
+            message,
+            error_name,
+            count,
+        }) catch buf[0..];
+        store.report(io, now_s, .clients_storage, operation, operation, .warning, detail);
+    }
+
     pub fn snapshotStats(self: *Tracker, io: std.Io) Stats {
         self.mutex.lockUncancelable(io);
         defer self.mutex.unlock(io);
@@ -258,6 +295,7 @@ pub const Tracker = struct {
 // tests
 // ---------------------------------------------------------------------------
 
+const events_fixture = @import("../storage/events_fixture.zig");
 const migrations = @import("../storage/migrations.zig");
 
 const testing = std.testing;
@@ -649,3 +687,83 @@ test "a gated pass attempts no naming either" {
     try testing.expectEqual(@as(usize, 0), CountingExchange.calls);
     try testing.expectEqual(@as(u64, 0), names.snapshotStats(io).attempted);
 }
+
+test "a failing materialise opens one episode per pass and a clean pass closes it" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openMigrated();
+    defer database.close();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var tracker: Tracker = .init(30);
+    tracker.diagnostics = &fx.store;
+
+    try database.exec(
+        \\CREATE TRIGGER refuse_insert BEFORE INSERT ON clients
+        \\BEGIN SELECT RAISE(ABORT, 'refused'); END;
+    );
+    _ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000000);
+    _ = tracker.trackAt(io, parsed("192.168.1.11"), 1700000001);
+    tracker.flushOnce(io, &database, true, null);
+
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+    try testing.expectEqualStrings("clients.storage", try fx.text("SELECT code FROM operational_events"));
+    try testing.expectEqualStrings("materialise", try fx.text("SELECT subject_key FROM operational_events"));
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT occurrences FROM operational_events"));
+
+    _ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000060);
+    tracker.flushOnce(io, &database, true, null);
+    try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
+
+    try database.exec("DROP TRIGGER refuse_insert;");
+    _ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000120);
+    tracker.flushOnce(io, &database, true, null);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "a failing prune opens its own episode the next due pass closes" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openMigrated();
+    defer database.close();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var tracker: Tracker = .init(30);
+    tracker.diagnostics = &fx.store;
+    // One pass short of due, so the pass below is the pruning one.
+    tracker.passes = Tracker.prune_every_passes - 1;
+
+    try database.exec(
+        \\CREATE TRIGGER refuse_delete BEFORE DELETE ON clients
+        \\BEGIN SELECT RAISE(ABORT, 'refused'); END;
+    );
+    try clients_repo.upsertSeen(&database, "192.168.1.10", 1);
+    tracker.flushOnce(io, &database, true, null);
+
+    try testing.expectEqualStrings("prune", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+
+    try database.exec("DROP TRIGGER refuse_delete;");
+    tracker.passes = Tracker.prune_every_passes - 1;
+    tracker.flushOnce(io, &database, true, null);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}