milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/server/cert_store.zig

@@ -13,6 +13,8 @@
 //! through the size.
 
 const std = @import("std");
+
+const events = @import("../storage/events.zig");
 const tls_server = @import("../platform/tls_server.zig");
 
 const log = std.log.scoped(.cert_store);
@@ -101,6 +103,11 @@ pub const ReloadHook = struct {
 };
 
 pub const CertStore = struct {
+    /// Which endpoint's certificate this store holds. It is the subject of
+    /// every `certificate.reload` event, and a box serving both DoH and DoT
+    /// runs two stores over two file pairs.
+    pub const Kind = enum { doh, dot };
+
     gpa: std.mem.Allocator,
     /// Borrowed from the config; must outlive the store.
     cert_path: []const u8,
@@ -132,6 +139,12 @@ pub const CertStore = struct {
     /// `reload` synchronously (that would self-deadlock on `reload_mutex`).
     after_load_hook: ?ReloadHook,
 
+    /// Set by the composition root right after `init`, with `diagnostics`.
+    kind: Kind = .doh,
+    /// Wired the same way and for the same reason as every other subsystem's:
+    /// the store is fully usable without it, and `nxdns check` has none.
+    diagnostics: ?*events.Store = null,
+
     reloads: std.atomic.Value(u64),
     reload_failures: std.atomic.Value(u64),
     /// Wall-clock second of the last successful load, including the one in
@@ -243,7 +256,7 @@ pub const CertStore = struct {
         };
         while (true) {
             try interval.sleep(io);
-            self.pollOnce(io);
+            self.pollOnce(io, std.Io.Clock.real.now(io).toSeconds());
         }
     }
 
@@ -252,13 +265,15 @@ pub const CertStore = struct {
     /// changed, and the old one keeps serving either way. A failed reload
     /// warns and counts (`reload_failures`); the signature stays at the loaded
     /// pair, so every subsequent poll retries until the files parse.
-    pub fn pollOnce(self: *CertStore, io: std.Io) void {
-        const cert_sig = statSig(io, self.cert_path) catch {
+    pub fn pollOnce(self: *CertStore, io: std.Io, now_s: i64) void {
+        const cert_sig = statSig(io, self.cert_path) catch |err| {
             log.warn("stat {s} failed; keeping the loaded certificate", .{self.cert_path});
+            self.reportReload(io, now_s, "stat of the certificate failed", @errorName(err));
             return;
         };
-        const key_sig = statSig(io, self.key_path) catch {
+        const key_sig = statSig(io, self.key_path) catch |err| {
             log.warn("stat {s} failed; keeping the loaded certificate", .{self.key_path});
+            self.reportReload(io, now_s, "stat of the private key failed", @errorName(err));
             return;
         };
         const observed: Signature = .{ .cert = cert_sig, .key = key_sig };
@@ -266,18 +281,36 @@ pub const CertStore = struct {
         self.mutex.lockUncancelable(io);
         const loaded = self.loaded;
         self.mutex.unlock(io);
-        if (!changed(loaded, observed)) return;
+        if (!changed(loaded, observed)) {
+            // A poll that stat'ed both files and found nothing to do is a
+            // fully healthy pass, so it closes any episode a transient stat
+            // failure opened. Without this, a file that never changes again
+            // would leave that episode open forever.
+            if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
+            return;
+        }
 
         if (self.reload(io)) {
             log.info("certificate reloaded from {s}", .{self.cert_path});
+            if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
         } else |err| {
             log.warn("certificate reload from {s} failed ({s}); the old certificate keeps serving", .{
                 self.cert_path,
                 humanMessage(err),
             });
+            self.reportReload(io, now_s, "certificate reload failed", humanMessage(err));
         }
     }
 
+    /// A warning, never an error: a stat failure can be a rename window, and a
+    /// failed reload leaves the loaded certificate serving. Nothing is down.
+    fn reportReload(self: *CertStore, io: std.Io, now_s: i64, message: []const u8, reason: []const u8) void {
+        const store = self.diagnostics orelse return;
+        var buf: [events.Store.max_detail_len]u8 = undefined;
+        const detail = std.fmt.bufPrint(&buf, "{s}: {s}", .{ message, reason }) catch buf[0..];
+        store.report(io, now_s, .certificate_reload, @tagName(self.kind), @tagName(self.kind), .warning, detail);
+    }
+
     pub fn snapshotStats(self: *const CertStore) Stats {
         return .{
             .reloads = self.reloads.load(.monotonic),
@@ -406,6 +439,7 @@ fn statSig(io: std.Io, path: []const u8) !FileSig {
 // tests
 // ---------------------------------------------------------------------------
 
+const events_fixture = @import("../storage/events_fixture.zig");
 const fixtures = @import("test_fixtures");
 const testing = std.testing;
 
@@ -699,7 +733,7 @@ test "pollOnce reloads on a changed stat pair and stays put on an unchanged one"
     var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
     defer store.deinit(io);
 
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
     try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
 
     // Same certificate plus a trailing newline: the PEM still parses and the
@@ -710,14 +744,14 @@ test "pollOnce reloads on a changed stat pair and stays put on an unchanged one"
 
     const old = store.acquire(io);
     store.release(io, old);
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
 
     try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
     const fresh = store.acquire(io);
     try testing.expect(fresh != old);
     store.release(io, fresh);
 
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
     try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
 }
 
@@ -734,7 +768,7 @@ test "pollOnce warns and keeps serving when a reload fails" {
     store.release(io, before);
 
     try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
 
     const stats = store.snapshotStats();
     try testing.expectEqual(@as(u64, 0), stats.reloads);
@@ -755,7 +789,7 @@ test "pollOnce does nothing when a file cannot be stat'ed" {
     defer store.deinit(io);
 
     try env.tmp.dir.deleteFile(io, "cert.pem");
-    store.pollOnce(io);
+    store.pollOnce(io, 1000);
 
     const stats = store.snapshotStats();
     try testing.expectEqual(@as(u64, 0), stats.reloads);
@@ -873,3 +907,89 @@ test "a reload overlapping another reload's window publishes last" {
     store.mutex.unlock(io);
     try testing.expectEqual(@as(u64, grown.len), final.cert.size);
 }
+
+test "a failed reload opens an episode keyed by endpoint kind and a good one closes it" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+    store.kind = .dot;
+    store.diagnostics = &fx.store;
+
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
+    store.pollOnce(io, 1000);
+
+    try testing.expectEqualStrings("certificate.reload", try fx.text("SELECT code FROM operational_events"));
+    try testing.expectEqualStrings("dot", try fx.text("SELECT subject_key FROM operational_events"));
+    try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
+
+    // The same certificate plus a newline: it parses, and the size differs even
+    // within one timestamp granule.
+    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
+    defer testing.allocator.free(grown);
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = grown });
+    store.pollOnce(io, 1100);
+
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "a failed stat opens the same episode a failed reload would" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+    store.diagnostics = &fx.store;
+
+    try env.tmp.dir.deleteFile(io, "cert.pem");
+    store.pollOnce(io, 1000);
+
+    try testing.expectEqualStrings("doh", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reload_failures);
+}
+
+test "an unchanged poll closes the episode a transient stat failure opened" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+    store.diagnostics = &fx.store;
+
+    // What a stat failure in a rename window left open. The file it named is
+    // back and unchanged, so no reload will ever close this episode.
+    fx.store.report(io, 1000, .certificate_reload, "doh", "doh", .warning, "stat of the certificate failed: FileNotFound");
+
+    store.pollOnce(io, 1100);
+
+    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}