milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/app.zig

@@ -44,6 +44,7 @@ const doh_client = @import("upstream/doh_client.zig");
 const doh_server = @import("server/doh_server.zig");
 const dot_client = @import("upstream/dot_client.zig");
 const dot_server = @import("server/dot_server.zig");
+const events = @import("storage/events.zig");
 const faults = @import("config/faults.zig");
 const fetcher = @import("filter/fetcher.zig");
 const forward_zones = @import("local/forward_zones.zig");
@@ -154,6 +155,7 @@ fn reconcileFromFile(
     config_db: *db.Db,
     dir: std.Io.Dir,
     config_path: []const u8,
+    config_load: ?*ConfigLoad,
 ) !i64 {
     return reconcileFromFileAt(
         r,
@@ -161,6 +163,7 @@ fn reconcileFromFile(
         dir,
         config_path,
         std.Io.Clock.real.now(r.io).toSeconds(),
+        config_load,
     );
 }
 
@@ -184,6 +187,7 @@ fn reconcileFromFileAt(
     dir: std.Io.Dir,
     config_path: []const u8,
     pass_now: i64,
+    config_load: ?*ConfigLoad,
 ) !i64 {
     var arena_state: std.heap.ArenaAllocator = .init(r.gpa);
     defer arena_state.deinit();
@@ -203,9 +207,32 @@ fn reconcileFromFileAt(
     diags.writeAll(r.err) catch {};
     r.err.flush() catch {};
 
+    // Warnings only. A `.fail` rejects the file and the process exits, so there
+    // is nobody left to read a diagnostics row about it; a warning is the case
+    // where the box serves on with a setting the operator did not mean.
+    if (config_load) |collector| {
+        for (diags.problems.items) |problem| {
+            if (problem.severity != .warn) continue;
+            collector.note(problem.path, problem.path, problem.message);
+        }
+    }
+
     return result;
 }
 
+/// An upstream's identity is its url: the whole url is the key, and the
+/// redaction is the label, because a url can carry an account token.
+fn noteUpstream(config_load: *ConfigLoad, url: []const u8, message: []const u8) void {
+    var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
+    const label = std.fmt.bufPrint(&label_buf, "{f}", .{safe_url.redact(url)}) catch &label_buf;
+    var detail_buf: [events.Store.max_detail_len]u8 = undefined;
+    const detail = std.fmt.bufPrint(&detail_buf, "upstream {f} {s}", .{
+        safe_url.redactQuoted(url),
+        message,
+    }) catch &detail_buf;
+    config_load.note(url, label, detail);
+}
+
 /// Returns the moment the transaction committed, which is what the settings
 /// envelope reports as `reconciled_at`.
 fn applyManagedFile(
@@ -318,6 +345,55 @@ fn printSummary(
     try r.out.flush();
 }
 
+/// The `configuration.load` findings of one boot.
+///
+/// This code is boot-finalized: nothing during the run can fix a setting, so a
+/// restart is its recovery. Every finding is reported as it is made and its key
+/// kept, and one `resolveExcept` after the last of them closes the episodes of
+/// settings that were wrong last boot and are not wrong now.
+const ConfigLoad = struct {
+    store: ?*events.Store,
+    io: std.Io,
+    now_s: i64,
+    keys: [events.Store.max_kept_keys][events.Store.max_subject_key_len]u8 = undefined,
+    lens: [events.Store.max_kept_keys]u16 = @splat(0),
+    len: usize = 0,
+    /// Set when a boot produced more distinct findings than `resolveExcept`
+    /// carries. The bulk resolve is then refused rather than truncated: a stale
+    /// episode left open is honest, and one that is still true closed is not.
+    /// The refusal goes through the store, so it is counted and latched.
+    overflowed: bool = false,
+
+    fn note(self: *ConfigLoad, key: []const u8, label: []const u8, detail: []const u8) void {
+        const store = self.store orelse return;
+        store.report(self.io, self.now_s, .configuration_load, key, label, .warning, detail);
+        self.keep(key);
+    }
+
+    fn keep(self: *ConfigLoad, key: []const u8) void {
+        var canon_buf: [events.Store.max_subject_key_len]u8 = undefined;
+        const canon = events.canonicalKey(key, &canon_buf);
+        for (0..self.len) |i| {
+            if (std.mem.eql(u8, self.keys[i][0..self.lens[i]], canon)) return;
+        }
+        if (self.len == self.keys.len) {
+            self.overflowed = true;
+            return;
+        }
+        @memcpy(self.keys[self.len][0..canon.len], canon);
+        self.lens[self.len] = @intCast(canon.len);
+        self.len += 1;
+    }
+
+    fn finalize(self: *ConfigLoad) void {
+        const store = self.store orelse return;
+        if (self.overflowed) return store.refuseResolveExcept(self.io, self.now_s);
+        var kept: [events.Store.max_kept_keys][]const u8 = undefined;
+        for (0..self.len) |i| kept[i] = self.keys[i][0..self.lens[i]];
+        store.resolveExcept(self.io, self.now_s, .configuration_load, kept[0..self.len]);
+    }
+};
+
 fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     const io = r.io;
     const gpa = r.gpa;
@@ -334,13 +410,29 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     defer config_db.close();
     _ = try migrations.migrate(&config_db);
 
+    // Opened unconditionally, not gated on `cfg.web.enabled`: diagnostics record
+    // what went wrong whether or not anyone is running the UI, and the store
+    // owns this connection outright (`storage/events.zig`).
+    var events_db = try data.openConfigDb(io);
+    defer events_db.close();
+    const boot_now_s = std.Io.Clock.real.now(io).toSeconds();
+    var event_store_storage: ?events.Store = events.Store.init(io, &events_db, boot_now_s) catch |err| blk: {
+        // No store rather than a store on a mirror it could not verify: the
+        // latter answers `resolve` with confident no-ops. `/api/health` reports
+        // the absence as `unavailable` and degrades on it.
+        log.warn("diagnostics store unavailable: {s}", .{@errorName(err)});
+        break :blk null;
+    };
+    const event_store: ?*events.Store = if (event_store_storage) |*s| s else null;
+    var config_load: ConfigLoad = .{ .store = event_store, .io = io, .now_s = boot_now_s };
+
     // Ruling 1: the presence of `--config` is the whole authority decision. With
     // it, the file is the sole declarative source and the database is converged
     // onto it here, before anything reads the database. Without it the database
     // is authority and this step does not exist — a file on disk that no flag
     // names changes nothing.
     const reconciled_at: ?i64 = if (args.config) |config_path|
-        try reconcileFromFile(r, &config_db, std.Io.Dir.cwd(), config_path)
+        try reconcileFromFile(r, &config_db, std.Io.Dir.cwd(), config_path, &config_load)
     else
         null;
 
@@ -437,7 +529,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     defer bundle.deinit(gpa);
     var bundle_lock: std.Io.RwLock = .init;
 
-    var upstreams = try Upstreams.build(gpa, cfg.upstreams, &dns_http, &bundle, &bundle_lock);
+    var upstreams = try Upstreams.build(gpa, cfg.upstreams, &dns_http, &bundle, &bundle_lock, &config_load);
     defer upstreams.deinit(gpa);
 
     var pool: pool_mod.Pool = .init(
@@ -455,7 +547,9 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     const history = try gpa.create(history_mod.Accumulator);
     defer gpa.destroy(history);
     history.* = .init;
+    history.diagnostics = event_store;
     pool.history = history;
+    pool.diagnostics = event_store;
 
     // -----------------------------------------------------------------------
     // per-query state
@@ -472,15 +566,18 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
 
     var paused: pause.Pause = .{};
     var tracker: clients.Tracker = .init(cfg.logging.retention_days);
+    tracker.diagnostics = event_store;
     // Naming rides the tracker's pass, on the tracker's task and connection
     // (milestone-25 ruling 1), and reads the live forward zones.
     var client_names_resolver: client_names.Resolver = .init(&tables);
+    client_names_resolver.diagnostics = event_store;
 
     // The queue holds waiting tasks in intrusive lists, so neither the buffer
     // nor the `Logger` may move once a task has touched either.
     const queue_buf = try gpa.alloc(logger_mod.Entry, cfg.logging.query_log_buffer_max);
     defer gpa.free(queue_buf);
     var query_logger: logger_mod.Logger = .init(cfg.logging, queue_buf);
+    query_logger.diagnostics = event_store;
 
     // Milestone 8 fans every logged query out to the SSE hub as well. The hub
     // exists only when the web interface does (ruling 6) — without it the sink
@@ -510,6 +607,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     // Ruling 17. The scheduler consults it before every scheduled pass; the
     // startup `reload` below is an operator action and stays ungated.
     manager.monitor = &monitor;
+    manager.diagnostics = event_store;
 
     // One synchronous sample before anything can consult the gate. `Monitor`
     // initializes to `.ok`, and `Monitor.run` takes its first sample inside the
@@ -525,12 +623,37 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     // fails therefore leaves the state at `.ok` — an unreadable filesystem is
     // not evidence that the disk is full, which is the monitor's documented
     // policy and the right one here too.
-    monitor.sample(io);
+    monitor.sample(io, event_store, boot_now_s);
 
     var retention: retention_mod.Retention = .init(cfg.logging);
 
-    var querylog_writer_db = try data.openQuerylogDb(io);
+    var querylog_opened = try data.openQuerylogDb(io);
+    var querylog_writer_db = querylog_opened.database;
     defer querylog_writer_db.close();
+
+    // One-shot and already over: the file was recreated during this boot, and
+    // there is nothing to recover from. Never emitted for `.missing` — a first
+    // creation renames nothing aside, so the event would carry an aside path
+    // that does not exist and would greet every fresh install with a warning.
+    if (querylog_opened.recreated) |cause| {
+        if (cause != .missing) {
+            if (event_store) |store| {
+                var detail_buf: [events.Store.max_detail_len]u8 = undefined;
+                const detail = std.fmt.bufPrint(&detail_buf, "previous file kept as '{s}'", .{
+                    querylog_opened.aside(),
+                }) catch detail_buf[0..];
+                store.reportResolved(
+                    io,
+                    boot_now_s,
+                    .query_log_recreated,
+                    @tagName(cause),
+                    @tagName(cause),
+                    .warning,
+                    detail,
+                );
+            }
+        }
+    }
     var querylog_retention_db = try data.reopenQuerylogDb(io);
     defer querylog_retention_db.close();
     var querylog_history_db = try data.reopenQuerylogDb(io);
@@ -574,6 +697,17 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
             "loading the blocklist snapshot failed ({s}); serving unfiltered until the next refresh",
             .{@errorName(err)},
         );
+        // No manager lock is held here, so this reports directly rather than
+        // through the manager's collector.
+        if (event_store) |store| {
+            var buf: [events.Store.max_detail_len]u8 = undefined;
+            const detail = std.fmt.bufPrint(
+                &buf,
+                "loading the blocklist snapshot failed: {s}",
+                .{@errorName(err)},
+            ) catch buf[0..];
+            store.report(io, boot_now_s, .blocklist_snapshot, "snapshot", "blocklist snapshot", .@"error", detail);
+        }
     };
 
     // -----------------------------------------------------------------------
@@ -609,12 +743,16 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     defer if (doh_certs) |*store| store.deinit(io);
     if (cfg.doh_server.enabled) {
         doh_certs = try openCertStore(r, gpa, io, cfg.doh_server, "doh_server", doh_server.alpn_protocols);
+        doh_certs.?.kind = .doh;
+        doh_certs.?.diagnostics = event_store;
     }
 
     var dot_certs: ?cert_store.CertStore = null;
     defer if (dot_certs) |*store| store.deinit(io);
     if (cfg.dot_server.enabled) {
         dot_certs = try openCertStore(r, gpa, io, cfg.dot_server, "dot_server", dot_alpn);
+        dot_certs.?.kind = .dot;
+        dot_certs.?.diagnostics = event_store;
     }
 
     // Bound here, in this frame, rather than through doh_server's module-level
@@ -622,13 +760,35 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     // only a listener that lives in this frame has an address to wire there.
     // A failed bind warns and stays off (ruling 1, the web precedent): TLS DNS
     // failing to come up must not stop the plain-DNS side this box exists for.
+    var failed_listeners: [2][]const u8 = undefined;
+    var failed_listener_count: usize = 0;
+
     var doh: ?doh_server.DohServer = null;
     defer if (doh) |*server| server.deinit(io);
-    if (doh_certs) |*store| doh = bindDoh(gpa, io, cfg.doh_server, &h, store);
+    if (doh_certs) |*store| {
+        doh = bindDoh(gpa, io, cfg.doh_server, &h, store, event_store, boot_now_s);
+        if (doh == null) {
+            failed_listeners[failed_listener_count] = "doh";
+            failed_listener_count += 1;
+        }
+    }
 
     var dot: ?dot_server.DotServer = null;
     defer if (dot) |*server| server.deinit(io);
-    if (dot_certs) |*store| dot = bindDot(gpa, io, cfg.dot_server, &h, store);
+    if (dot_certs) |*store| {
+        dot = bindDot(gpa, io, cfg.dot_server, &h, store, event_store, boot_now_s);
+        if (dot == null) {
+            failed_listeners[failed_listener_count] = "dot";
+            failed_listener_count += 1;
+        }
+    }
+
+    // Boot-finalized: one call closes whatever the last boot left open for an
+    // endpoint that started clean this time — including an endpoint now
+    // disabled, which contributes no key and so is not kept.
+    if (event_store) |store| {
+        store.resolveExcept(io, boot_now_s, .listener_start, failed_listeners[0..failed_listener_count]);
+    }
 
     // -----------------------------------------------------------------------
     // web interface (ruling 26)
@@ -668,6 +828,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
         .dot_listener = if (dot) |*server| server else null,
         .config_db = if (web_config_db) |*database| database else null,
         .querylog_db = if (web_querylog_db) |*database| database else null,
+        .events = event_store,
         .version = version.string,
         .started_unix = std.Io.Clock.real.now(io).toSeconds(),
         // Ruling 24: `--admin-dev` serves from disk with no cache headers;
@@ -684,6 +845,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     var udp6: ?udp_server.UdpServer = udp_server.UdpServer.bind(gpa, io, v6_bind, &h, .{}) catch |err| bound: {
         if (!ipv6Unavailable(err)) return reportBind(r, "udp", v6_bind, err);
         log.warn("this system has no IPv6; serving IPv4 only", .{});
+        config_load.note("dns.bind_ipv6", "dns.bind_ipv6", "this system has no IPv6; serving IPv4 only");
         break :bound null;
     };
     defer if (udp6) |*s| s.deinit(gpa, io);
@@ -712,6 +874,12 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     };
     defer if (tcp4) |*s| s.deinit(io);
 
+    // Every `configuration.load` finding of this boot is in by here: the managed
+    // file, the upstream table and the IPv6 bind above. Finalizing any earlier
+    // would resolve an episode this boot is about to reopen, so consecutive
+    // IPv6-less boots would read as a new episode each time.
+    config_load.finalize();
+
     // Ruling 13: `/metrics` sums each transport's listeners into one family, so
     // the web state carries pointers to whichever of the four came up. The
     // arrays are declared here rather than beside `web_state` because a
@@ -793,11 +961,11 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     if (doh_certs) |*store| try group.concurrent(io, cert_store.CertStore.watch, .{ store, io });
     if (dot_certs) |*store| try group.concurrent(io, cert_store.CertStore.watch, .{ store, io });
 
-    try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db, gate });
+    try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db, gate, event_store });
     // Ungated: a flush writes at most one row per upstream per minute, the same
     // category as the query logger's own writes, which are ungated too.
     try group.concurrent(io, history_mod.Accumulator.run, .{ history, io, &querylog_history_db });
-    try group.concurrent(io, disk_monitor.Monitor.run, .{ &monitor, io });
+    try group.concurrent(io, disk_monitor.Monitor.run, .{ &monitor, io, event_store });
     try group.concurrent(io, manager_mod.Manager.runScheduler, .{ &manager, io });
     try group.concurrent(io, clients.Tracker.run, .{ &tracker, io, &tracker_db, gate, &client_names_resolver });
     try group.concurrent(io, runMaintenance, .{ &h, if (web_limiter) |*l| l else null, io });
@@ -882,6 +1050,23 @@ fn openCertStore(
     };
 }
 
+/// An error, not a warning: an endpoint the operator enabled is not serving.
+fn reportListener(
+    store: ?*events.Store,
+    io: std.Io,
+    now_s: i64,
+    kind: []const u8,
+    comptime fmt: []const u8,
+    args: anytype,
+) void {
+    const s = store orelse return;
+    var buf: [events.Store.max_detail_len]u8 = undefined;
+    var w: std.Io.Writer = .fixed(&buf);
+    w.print("{s} listener ", .{kind}) catch {};
+    w.print(fmt, args) catch {};
+    s.report(io, now_s, .listener_start, kind, kind, .@"error", w.buffered());
+}
+
 /// Ruling 1: a listener that cannot bind warns and stays off. The bind text
 /// itself gets the same treatment — `validate` refuses it, but a hand-edited
 /// database can still carry one, and it is not worth taking DNS down over.
@@ -891,13 +1076,17 @@ fn bindDoh(
     endpoint: model.TlsEndpoint,
     h: *handler.Handler,
     store: *cert_store.CertStore,
+    diagnostics: ?*events.Store,
+    now_s: i64,
 ) ?doh_server.DohServer {
     const bind_address = net.IpAddress.parse(endpoint.bind, endpoint.port) catch {
         log.warn("doh_server.bind '{s}' is not an IP address; DoH is disabled", .{endpoint.bind});
+        reportListener(diagnostics, io, now_s, "doh", "bind '{s}' is not an IP address", .{endpoint.bind});
         return null;
     };
     const server = doh_server.DohServer.listen(gpa, io, bind_address, h, store, .{}) catch |err| {
         log.warn("doh listener cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
+        reportListener(diagnostics, io, now_s, "doh", "cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
         return null;
     };
     log.info("doh listener on {f}", .{server.boundAddress()});
@@ -910,13 +1099,17 @@ fn bindDot(
     endpoint: model.TlsEndpoint,
     h: *handler.Handler,
     store: *cert_store.CertStore,
+    diagnostics: ?*events.Store,
+    now_s: i64,
 ) ?dot_server.DotServer {
     const bind_address = net.IpAddress.parse(endpoint.bind, endpoint.port) catch {
         log.warn("dot_server.bind '{s}' is not an IP address; DoT is disabled", .{endpoint.bind});
+        reportListener(diagnostics, io, now_s, "dot", "bind '{s}' is not an IP address", .{endpoint.bind});
         return null;
     };
     const server = dot_server.DotServer.listen(gpa, io, bind_address, h, store, .{}) catch |err| {
         log.warn("dot listener cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
+        reportListener(diagnostics, io, now_s, "dot", "cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
         return null;
     };
     log.info("dot listener on {f}", .{server.boundAddress()});
@@ -1006,6 +1199,7 @@ const Upstreams = struct {
         http: *std.http.Client,
         bundle: *Certificate.Bundle,
         bundle_lock: *std.Io.RwLock,
+        config_load: *ConfigLoad,
     ) (Allocator.Error || error{NoUsableUpstreams})!Upstreams {
         var enabled: usize = 0;
         for (servers) |server| {
@@ -1041,6 +1235,7 @@ const Upstreams = struct {
                     "upstream {f} is not an https:// or tls:// endpoint; skipped",
                     .{safe_url.redactQuoted(server.url)},
                 );
+                noteUpstream(config_load, server.url, "not an https:// or tls:// endpoint; skipped");
                 continue;
             };
 
@@ -1058,6 +1253,7 @@ const Upstreams = struct {
                             "upstream {f} is not a usable DoH url; skipped",
                             .{safe_url.redactQuoted(server.url)},
                         );
+                        noteUpstream(config_load, server.url, "not a usable DoH url; skipped");
                         continue;
                     };
                     doh_count += 1;
@@ -1154,6 +1350,9 @@ fn parseBind(
     return addr;
 }
 
+const events_fixture = @import("storage/events_fixture.zig");
+const testing = std.testing;
+
 test "parseBind refuses a bind address of the wrong family" {
     var out_buf: [8]u8 = undefined;
     var err_buf: [256]u8 = undefined;
@@ -1325,7 +1524,7 @@ test "a start in file mode prints the warnings the file earned" {
 
     // The file is valid, so the start succeeds and the database converges onto
     // it. The returned stamp is what the settings envelope reports.
-    const reconciled_at = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
+    const reconciled_at = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
     try std.testing.expect(reconciled_at > 0);
     try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
 
@@ -1431,7 +1630,7 @@ test "reconciled_at is stamped after the commit, not from the clock the pass wro
 
     const pass_now: i64 = 42;
     const before = std.Io.Clock.real.now(io).toSeconds();
-    const reconciled_at = try reconcileFromFileAt(r, &database, tmp.dir, "config.zon", pass_now);
+    const reconciled_at = try reconcileFromFileAt(r, &database, tmp.dir, "config.zon", pass_now, null);
 
     // The pinned clock reached the engine, so the two values really are separate
     // inputs rather than the same read twice.
@@ -1474,7 +1673,7 @@ test "the startup summary reports what the reconcile changed, then that nothing
     var err: Writer = .fixed(&err_buf);
     const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out_writer.interface, .err = &err };
 
-    _ = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
+    _ = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
     {
         // Read back through the file: `serve` does not return for as long as the
         // service runs, so a summary still in the buffer is a summary nobody
@@ -1491,7 +1690,7 @@ test "the startup summary reports what the reconcile changed, then that nothing
     }
 
     try tmp.dir.writeFile(io, .{ .sub_path = "stdout.txt", .data = "" });
-    _ = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
+    _ = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
     {
         const printed = try tmp.dir.readFileAlloc(io, "stdout.txt", gpa, .limited(8192));
         defer gpa.free(printed);
@@ -1548,7 +1747,7 @@ test "a broken error writer does not replace the reason a managed file was rejec
 
     try std.testing.expectError(
         error.MissingDefaultGroup,
-        reconcileFromFile(r, &database, tmp.dir, "config.zon"),
+        reconcileFromFile(r, &database, tmp.dir, "config.zon", null),
     );
 
     // Empty, so the writer did fail — without this the assertion above would
@@ -1594,7 +1793,7 @@ test "a broken error writer does not stop a start whose file applied" {
     var err_writer = brokenErrWriter(io, err_file, &err_buf);
     const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer.interface };
 
-    _ = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
+    _ = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
     try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
 
     // Nothing reached the file, so the flush really did fail.
@@ -1779,3 +1978,126 @@ test "one maintenance pass drops the api limiter's stale buckets" {
     // A limiter the app did not build is not a reason for the pass to fail.
     try maintenanceOnce(&h, null, io);
 }
+
+test "a failing listener bind opens an error episode a clean boot closes" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    reportListener(&fx.store, io, 1000, "dot", "cannot listen on {s}:{d}: {t}", .{
+        "0.0.0.0",
+        @as(u16, 853),
+        error.AddressInUse,
+    });
+
+    try testing.expectEqualStrings("listener.start", try fx.text("SELECT code FROM operational_events"));
+    try testing.expectEqualStrings("dot", try fx.text("SELECT subject_key FROM operational_events"));
+    try testing.expectEqualStrings("error", try fx.text("SELECT severity FROM operational_events"));
+
+    // The next boot: DoH failed, DoT started clean. One `resolveExcept` over
+    // the failed keys closes the stale DoT episode and leaves the DoH one open.
+    reportListener(&fx.store, io, 1100, "doh", "cannot listen on {s}:{d}: {t}", .{
+        "0.0.0.0",
+        @as(u16, 443),
+        error.AddressInUse,
+    });
+    fx.store.resolveExcept(io, 1100, .listener_start, &.{"doh"});
+
+    try testing.expectEqual(
+        @as(i64, 1),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+    try testing.expectEqualStrings("doh", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+}
+
+test "a boot with no listener finding closes every listener episode" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    fx.store.report(io, 900, .listener_start, "doh", "doh", .@"error", "stale");
+    fx.store.report(io, 900, .listener_start, "dot", "dot", .@"error", "stale");
+
+    fx.store.resolveExcept(io, 1000, .listener_start, &.{});
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "a configuration finding is reported once and finalize closes the rest" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    // Two stale episodes from a previous boot.
+    fx.store.report(io, 900, .configuration_load, "dns.bind_ipv6", "dns.bind_ipv6", .warning, "stale");
+    fx.store.report(io, 900, .configuration_load, "upstreams[0].url", "upstreams[0].url", .warning, "stale");
+
+    var collector: ConfigLoad = .{ .store = &fx.store, .io = io, .now_s = 1000 };
+    collector.note("dns.bind_ipv6", "dns.bind_ipv6", "this system has no IPv6; serving IPv4 only");
+    // The same finding twice is one episode and one kept key.
+    collector.note("dns.bind_ipv6", "dns.bind_ipv6", "this system has no IPv6; serving IPv4 only");
+    try testing.expectEqual(@as(usize, 1), collector.len);
+
+    collector.finalize();
+
+    try testing.expectEqual(
+        @as(i64, 1),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+    try testing.expectEqualStrings("dns.bind_ipv6", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+    // The stale episode is still the same one: this boot bumped it twice.
+    try testing.expectEqual(@as(i64, 3), try fx.count(
+        "SELECT occurrences FROM operational_events WHERE resolved_at IS NULL",
+    ));
+}
+
+test "an over-long boot finding list refuses to finalize rather than truncate" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    fx.store.report(io, 900, .configuration_load, "stale.setting", "stale.setting", .warning, "stale");
+
+    var collector: ConfigLoad = .{ .store = &fx.store, .io = io, .now_s = 1000 };
+    var key_buf: [32]u8 = undefined;
+    for (0..events.Store.max_kept_keys + 1) |i| {
+        const key = try std.fmt.bufPrint(&key_buf, "upstreams[{d}].url", .{i});
+        collector.note(key, key, "not a usable url");
+    }
+    try testing.expect(collector.overflowed);
+
+    collector.finalize();
+
+    // Closing more than the boot meant would resolve episodes that are still
+    // true, so the stale one stays open instead.
+    try testing.expectEqual(@as(i64, 1), try fx.count(
+        "SELECT count(*) FROM operational_events WHERE subject_key = 'stale.setting' AND resolved_at IS NULL",
+    ));
+    // Refused, and refused out loud: the store counted and latched it, so
+    // `/api/health` reports the diagnostics log as not recording.
+    try testing.expect(fx.store.writeFailed());
+    try testing.expectEqual(@as(u64, 1), fx.store.writeFailures());
+}