milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/server/cert_store.zig

@@ -13,6 +13,8 @@
 //! through the size.
 
 const std = @import("std");
+
+const events = @import("../storage/events.zig");
 const tls_server = @import("../platform/tls_server.zig");
 
 const log = std.log.scoped(.cert_store);
@@ -101,6 +103,11 @@ pub const ReloadHook = struct {
 };
 
 pub const CertStore = struct {
+    /// Which endpoint's certificate this store holds. It is the subject of
+    /// every `certificate.reload` event, and a box serving both DoH and DoT
+    /// runs two stores over two file pairs.
+    pub const Kind = enum { doh, dot };
+
     gpa: std.mem.Allocator,
     /// Borrowed from the config; must outlive the store.
     cert_path: []const u8,
@@ -132,6 +139,12 @@ pub const CertStore = struct {
     /// `reload` synchronously (that would self-deadlock on `reload_mutex`).
     after_load_hook: ?ReloadHook,
 
+    /// Set by the composition root right after `init`, with `diagnostics`.
+    kind: Kind = .doh,
+    /// Wired the same way and for the same reason as every other subsystem's:
+    /// the store is fully usable without it, and `nxdns check` has none.
+    diagnostics: ?*events.Store = null,
+
     reloads: std.atomic.Value(u64),
     reload_failures: std.atomic.Value(u64),
     /// Wall-clock second of the last successful load, including the one in
@@ -243,7 +256,7 @@ pub const CertStore = struct {
         };
         while (true) {
             try interval.sleep(io);
-            self.pollOnce(io);
+            self.pollOnce(io, std.Io.Clock.real.now(io).toSeconds());
         }
     }
 
@@ -252,13 +265,15 @@ pub const CertStore = struct {
     /// changed, and the old one keeps serving either way. A failed reload
     /// warns and counts (`reload_failures`); the signature stays at the loaded
     /// pair, so every subsequent poll retries until the files parse.
-    pub fn pollOnce(self: *CertStore, io: std.Io) void {
-        const cert_sig = statSig(io, self.cert_path) catch {
+    pub fn pollOnce(self: *CertStore, io: std.Io, now_s: i64) void {
+        const cert_sig = statSig(io, self.cert_path) catch |err| {
             log.warn("stat {s} failed; keeping the loaded certificate", .{self.cert_path});
+            self.reportReload(io, now_s, "stat of the certificate failed", @errorName(err));
             return;
         };
-        const key_sig = statSig(io, self.key_path) catch {
+        const key_sig = statSig(io, self.key_path) catch |err| {
             log.warn("stat {s} failed; keeping the loaded certificate", .{self.key_path});
+            self.reportReload(io, now_s, "stat of the private key failed", @errorName(err));
             return;
         };
         const observed: Signature = .{ .cert = cert_sig, .key = key_sig };
@@ -266,18 +281,36 @@ pub const CertStore = struct {
         self.mutex.lockUncancelable(io);
         const loaded = self.loaded;
         self.mutex.unlock(io);
-        if (!changed(loaded, observed)) return;
+        if (!changed(loaded, observed)) {
+            // A poll that stat'ed both files and found nothing to do is a
+            // fully healthy pass, so it closes any episode a transient stat
+            // failure opened. Without this, a file that never changes again
+            // would leave that episode open forever.
+            if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
+            return;
+        }
 
         if (self.reload(io)) {
             log.info("certificate reloaded from {s}", .{self.cert_path});
+            if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
         } else |err| {
             log.warn("certificate reload from {s} failed ({s}); the old certificate keeps serving", .{
                 self.cert_path,
                 humanMessage(err),
             });
+            self.reportReload(io, now_s, "certificate reload failed", humanMessage(err));
         }
     }
 
+    /// A warning, never an error: a stat failure can be a rename window, and a
+    /// failed reload leaves the loaded certificate serving. Nothing is down.
+    fn reportReload(self: *CertStore, io: std.Io, now_s: i64, message: []const u8, reason: []const u8) void {
+        const store = self.diagnostics orelse return;
+        var buf: [events.Store.max_detail_len]u8 = undefined;
+        const detail = std.fmt.bufPrint(&buf, "{s}: {s}", .{ message, reason }) catch buf[0..];
+        store.report(io, now_s, .certificate_reload, @tagName(self.kind), @tagName(self.kind), .warning, detail);
+    }
+
     pub fn snapshotStats(self: *const CertStore) Stats {
         return .{
             .reloads = self.reloads.load(.monotonic),
@@ -406,6 +439,7 @@ fn statSig(io: std.Io, path: []const u8) !FileSig {
 // tests
 // ---------------------------------------------------------------------------
 
+const events_fixture = @import("../storage/events_fixture.zig");
 const fixtures = @import("test_fixtures");
 const testing = std.testing;
 
@@ -699,7 +733,7 @@ test "pollOnce reloads on a changed stat pair and stays put on an unchanged one"
     var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
     defer store.deinit(io);
 
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
     try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
 
     // Same certificate plus a trailing newline: the PEM still parses and the
@@ -710,14 +744,14 @@ test "pollOnce reloads on a changed stat pair and stays put on an unchanged one"
 
     const old = store.acquire(io);
     store.release(io, old);
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
 
     try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
     const fresh = store.acquire(io);
     try testing.expect(fresh != old);
     store.release(io, fresh);
 
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
     try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
 }
 
@@ -734,7 +768,7 @@ test "pollOnce warns and keeps serving when a reload fails" {
     store.release(io, before);
 
     try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
 
     const stats = store.snapshotStats();
     try testing.expectEqual(@as(u64, 0), stats.reloads);
@@ -755,7 +789,7 @@ test "pollOnce does nothing when a file cannot be stat'ed" {
     defer store.deinit(io);
 
     try env.tmp.dir.deleteFile(io, "cert.pem");
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
 
     const stats = store.snapshotStats();
     try testing.expectEqual(@as(u64, 0), stats.reloads);
@@ -873,3 +907,89 @@ test "a reload overlapping another reload's window publishes last" {
     store.mutex.unlock(io);
     try testing.expectEqual(@as(u64, grown.len), final.cert.size);
 }
+
+test "a failed reload opens an episode keyed by endpoint kind and a good one closes it" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+    store.kind = .dot;
+    store.diagnostics = &fx.store;
+
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
+    store.pollOnce(io, 1000);
+
+    try testing.expectEqualStrings("certificate.reload", try fx.text("SELECT code FROM operational_events"));
+    try testing.expectEqualStrings("dot", try fx.text("SELECT subject_key FROM operational_events"));
+    try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
+
+    // The same certificate plus a newline: it parses, and the size differs even
+    // within one timestamp granule.
+    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
+    defer testing.allocator.free(grown);
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = grown });
+    store.pollOnce(io, 1100);
+
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "a failed stat opens the same episode a failed reload would" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+    store.diagnostics = &fx.store;
+
+    try env.tmp.dir.deleteFile(io, "cert.pem");
+    store.pollOnce(io, 1000);
+
+    try testing.expectEqualStrings("doh", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reload_failures);
+}
+
+test "an unchanged poll closes the episode a transient stat failure opened" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+    store.diagnostics = &fx.store;
+
+    // What a stat failure in a rename window left open. The file it named is
+    // back and unchanged, so no reload will ever close this episode.
+    fx.store.report(io, 1000, .certificate_reload, "doh", "doh", .warning, "stat of the certificate failed: FileNotFound");
+
+    store.pollOnce(io, 1100);
+
+    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}

src/server/client_names.zig

@@ -30,6 +30,7 @@ const clients_repo = @import("../storage/repositories/clients_repo.zig");
 const db = @import("../storage/db.zig");
 const dns_header = @import("../dns/header.zig");
 const edns = @import("../dns/edns.zig");
+const events = @import("../storage/events.zig");
 const forward_client = @import("../local/forward_client.zig");
 const local_tables = @import("local_tables.zig");
 const name_mod = @import("../dns/name.zig");
@@ -103,6 +104,9 @@ pub const Resolver = struct {
     tables: *local_tables.LocalTables,
     stats: Stats = .{},
     exchange_fn: ExchangeFn = defaultExchange,
+    /// Wired by the composition root after `init`, following the
+    /// `gate: ?*disk_monitor.Monitor` idiom. Null in every unit test here.
+    diagnostics: ?*events.Store = null,
 
     pub fn init(tables: *local_tables.LocalTables) Resolver {
         return .{ .tables = tables };
@@ -127,10 +131,13 @@ pub const Resolver = struct {
             self.mutex.lockUncancelable(io);
             self.stats.read_failures += 1;
             self.mutex.unlock(io);
+            self.reportStorage(io, now_s, "read", "selecting clients to name failed", @errorName(err), 1);
             return;
         };
+        if (self.diagnostics) |store| store.resolve(io, now_s, .client_names_storage, "read");
 
         var write_failures: u64 = 0;
+        var last_write_error: []const u8 = "";
         for (candidates[0..count]) |*candidate| {
             var learned_buf: [types.max_name_len]u8 = undefined;
             const result = self.attempt(io, candidate.ip(), &learned_buf);
@@ -144,6 +151,7 @@ pub const Resolver = struct {
                     log.warn("recording the name of {s} failed: {s}", .{ candidate.ip(), @errorName(err) });
                 }
                 write_failures += 1;
+                last_write_error = @errorName(err);
             }
 
             self.mutex.lockUncancelable(io);
@@ -158,10 +166,38 @@ pub const Resolver = struct {
             self.mutex.unlock(io);
         }
 
-        if (write_failures == 0) return;
+        // The clean-pass determination is what closes a `write` episode: one
+        // aggregated outcome per pass, so a row's `occurrences` counts failing
+        // passes rather than failing rows.
+        if (write_failures == 0) {
+            if (self.diagnostics) |store| store.resolve(io, now_s, .client_names_storage, "write");
+            return;
+        }
         self.mutex.lockUncancelable(io);
         self.stats.write_failures += write_failures;
         self.mutex.unlock(io);
+        self.reportStorage(io, now_s, "write", "recording a client name failed", last_write_error, write_failures);
+    }
+
+    /// One aggregated report per pass. `count` is how many failures that pass
+    /// held; the detail carries it because the row itself counts passes.
+    fn reportStorage(
+        self: *Resolver,
+        io: std.Io,
+        now_s: i64,
+        operation: []const u8,
+        message: []const u8,
+        error_name: []const u8,
+        count: u64,
+    ) void {
+        const store = self.diagnostics orelse return;
+        var buf: [events.Store.max_detail_len]u8 = undefined;
+        const detail = std.fmt.bufPrint(&buf, "{s}: {s} ({d} this pass)", .{
+            message,
+            error_name,
+            count,
+        }) catch buf[0..];
+        store.report(io, now_s, .client_names_storage, operation, operation, .warning, detail);
     }
 
     const Attempt = struct {
@@ -355,6 +391,7 @@ fn exchangeOnce(
 
 const forward_zones = @import("../local/forward_zones.zig");
 const migrations = @import("../storage/migrations.zig");
+const events_fixture = @import("../storage/events_fixture.zig");
 const testing = std.testing;
 
 /// A stub exchange whose reply and error are set by the test. `calls` is the
@@ -1157,3 +1194,70 @@ test "an extended rcode of zero still reads as the header's rcode" {
     try testing.expectEqualStrings("nas.lan", (try f.learned("192.168.1.10", &buf)).?);
     try testing.expectEqual(@as(u64, 1), f.resolver.snapshotStats(f.io()).answered);
 }
+
+test "a failing write opens one episode per pass and a clean pass closes it" {
+    var f: Fixture = undefined;
+    try fixture(&f, "168.192.in-addr.arpa");
+    defer f.deinit();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(f.io(), 1000);
+    defer fx.deinit();
+    f.resolver.diagnostics = &fx.store;
+
+    // Two addresses outside the declared zone: both reach the write step
+    // without any exchange, which is not what this test is about.
+    try clients_repo.upsertSeen(&f.database, "10.0.0.1", 1700000000);
+    try clients_repo.upsertSeen(&f.database, "10.0.0.2", 1700000000);
+    try f.database.exec(
+        \\CREATE TRIGGER refuse_update BEFORE UPDATE ON clients
+        \\BEGIN SELECT RAISE(ABORT, 'refused'); END;
+    );
+
+    f.resolver.runPass(f.io(), &f.database, 1700000000);
+
+    // Two failing rows, one aggregated report: the row counts failing passes.
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+    try testing.expectEqualStrings("client_names.storage", try fx.text("SELECT code FROM operational_events"));
+    try testing.expectEqualStrings("write", try fx.text("SELECT subject_key FROM operational_events"));
+    try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT occurrences FROM operational_events"));
+
+    f.resolver.runPass(f.io(), &f.database, 1700000060);
+    try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
+
+    try f.database.exec("DROP TRIGGER refuse_update;");
+    f.resolver.runPass(f.io(), &f.database, 1700000120);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "a failing candidate select opens a read episode the next clean pass closes" {
+    var f: Fixture = undefined;
+    try fixture(&f, "168.192.in-addr.arpa");
+    defer f.deinit();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(f.io(), 1000);
+    defer fx.deinit();
+    f.resolver.diagnostics = &fx.store;
+
+    try clients_repo.upsertSeen(&f.database, "10.0.0.1", 1700000000);
+    try f.database.exec("ALTER TABLE clients RENAME TO clients_aside;");
+    f.resolver.runPass(f.io(), &f.database, 1700000000);
+
+    try testing.expectEqualStrings("read", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+
+    try f.database.exec("ALTER TABLE clients_aside RENAME TO clients;");
+    f.resolver.runPass(f.io(), &f.database, 1700000060);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}

src/server/clients.zig

@@ -25,6 +25,7 @@ const client_names = @import("client_names.zig");
 const clients_repo = @import("../storage/repositories/clients_repo.zig");
 const db = @import("../storage/db.zig");
 const disk_monitor = @import("../storage/disk_monitor.zig");
+const events = @import("../storage/events.zig");
 const logger = @import("../storage/logger.zig");
 
 const log = std.log.scoped(.clients);
@@ -63,6 +64,9 @@ pub const Tracker = struct {
     count: u32,
     passes: u64,
     stats: Stats,
+    /// Wired by the composition root after `init`, following the
+    /// `gate: ?*disk_monitor.Monitor` idiom. Null in every unit test here.
+    diagnostics: ?*events.Store = null,
 
     /// `retention_days` is `logging.retention_days`, the same knob the query log
     /// prunes by (milestone-7 ruling 16). A client silent for that long is as
@@ -185,6 +189,7 @@ pub const Tracker = struct {
 
         var flushed: u64 = 0;
         var failures: u64 = 0;
+        var last_failure: []const u8 = "";
         for (batch) |entry| {
             // `logger.max_client_len` is the RFC 5952 bound every address text
             // in this program is sized by, so `format` cannot fail here.
@@ -199,9 +204,19 @@ pub const Tracker = struct {
                     log.warn("materialising client {s} failed: {s}", .{ w.buffered(), @errorName(err) });
                 }
                 failures += 1;
+                last_failure = @errorName(err);
             }
         }
 
+        // One aggregated outcome per pass, so the row's `occurrences` counts
+        // failing passes rather than failing addresses. A pass that wrote
+        // nothing resolves nothing: an empty batch is not evidence of success.
+        if (failures != 0) {
+            self.reportStorage(io, now_s, "materialise", "materialising a client failed", last_failure, failures);
+        } else if (flushed != 0) {
+            if (self.diagnostics) |store| store.resolve(io, now_s, .clients_storage, "materialise");
+        }
+
         self.mutex.lockUncancelable(io);
         self.passes += 1;
         self.stats.flushed += flushed;
@@ -215,17 +230,39 @@ pub const Tracker = struct {
                 self.mutex.lockUncancelable(io);
                 self.stats.pruned += deleted;
                 self.mutex.unlock(io);
+                if (self.diagnostics) |store| store.resolve(io, now_s, .clients_storage, "prune");
             } else |err| {
                 log.warn("pruning clients before {d} failed: {s}", .{ cutoff, @errorName(err) });
                 self.mutex.lockUncancelable(io);
                 self.stats.flush_failures += 1;
                 self.mutex.unlock(io);
+                self.reportStorage(io, now_s, "prune", "pruning stale clients failed", @errorName(err), 1);
             }
         }
 
         if (names) |resolver| resolver.runPass(io, database, now_s);
     }
 
+    /// One aggregated report per pass; `count` is how many failures it held.
+    fn reportStorage(
+        self: *Tracker,
+        io: std.Io,
+        now_s: i64,
+        operation: []const u8,
+        message: []const u8,
+        error_name: []const u8,
+        count: u64,
+    ) void {
+        const store = self.diagnostics orelse return;
+        var buf: [events.Store.max_detail_len]u8 = undefined;
+        const detail = std.fmt.bufPrint(&buf, "{s}: {s} ({d} this pass)", .{
+            message,
+            error_name,
+            count,
+        }) catch buf[0..];
+        store.report(io, now_s, .clients_storage, operation, operation, .warning, detail);
+    }
+
     pub fn snapshotStats(self: *Tracker, io: std.Io) Stats {
         self.mutex.lockUncancelable(io);
         defer self.mutex.unlock(io);
@@ -258,6 +295,7 @@ pub const Tracker = struct {
 // tests
 // ---------------------------------------------------------------------------
 
+const events_fixture = @import("../storage/events_fixture.zig");
 const migrations = @import("../storage/migrations.zig");
 
 const testing = std.testing;
@@ -649,3 +687,83 @@ test "a gated pass attempts no naming either" {
     try testing.expectEqual(@as(usize, 0), CountingExchange.calls);
     try testing.expectEqual(@as(u64, 0), names.snapshotStats(io).attempted);
 }
+
+test "a failing materialise opens one episode per pass and a clean pass closes it" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openMigrated();
+    defer database.close();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var tracker: Tracker = .init(30);
+    tracker.diagnostics = &fx.store;
+
+    try database.exec(
+        \\CREATE TRIGGER refuse_insert BEFORE INSERT ON clients
+        \\BEGIN SELECT RAISE(ABORT, 'refused'); END;
+    );
+    _ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000000);
+    _ = tracker.trackAt(io, parsed("192.168.1.11"), 1700000001);
+    tracker.flushOnce(io, &database, true, null);
+
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+    try testing.expectEqualStrings("clients.storage", try fx.text("SELECT code FROM operational_events"));
+    try testing.expectEqualStrings("materialise", try fx.text("SELECT subject_key FROM operational_events"));
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT occurrences FROM operational_events"));
+
+    _ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000060);
+    tracker.flushOnce(io, &database, true, null);
+    try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
+
+    try database.exec("DROP TRIGGER refuse_insert;");
+    _ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000120);
+    tracker.flushOnce(io, &database, true, null);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "a failing prune opens its own episode the next due pass closes" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openMigrated();
+    defer database.close();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var tracker: Tracker = .init(30);
+    tracker.diagnostics = &fx.store;
+    // One pass short of due, so the pass below is the pruning one.
+    tracker.passes = Tracker.prune_every_passes - 1;
+
+    try database.exec(
+        \\CREATE TRIGGER refuse_delete BEFORE DELETE ON clients
+        \\BEGIN SELECT RAISE(ABORT, 'refused'); END;
+    );
+    try clients_repo.upsertSeen(&database, "192.168.1.10", 1);
+    tracker.flushOnce(io, &database, true, null);
+
+    try testing.expectEqualStrings("prune", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+
+    try database.exec("DROP TRIGGER refuse_delete;");
+    tracker.passes = Tracker.prune_every_passes - 1;
+    tracker.flushOnce(io, &database, true, null);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}