milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/upstream/pool.zig

@@ -47,6 +47,7 @@
 
 const std = @import("std");
 
+const events = @import("../storage/events.zig");
 const health = @import("health.zig");
 const history_mod = @import("history.zig");
 const safe_url = @import("../safe_url.zig");
@@ -136,6 +137,9 @@ pub const Pool = struct {
     /// pool is fully usable without it — `nxdns check` and every unit test here
     /// run with no history at all.
     history: ?*history_mod.Accumulator = null,
+    /// The diagnostics store, wired the same way and for the same reason as
+    /// `history`. Every emit here sits outside `mutex`; see `recordHistory`.
+    diagnostics: ?*events.Store = null,
 
     pub fn init(
         entries: []Entry,
@@ -327,6 +331,7 @@ pub const Pool = struct {
         // constraint: the accumulator takes a mutex of its own, and no task may
         // hold one of the two while it takes the other.
         self.recordHistory(io, entry, .success);
+        self.recordDiagnostics(io, entry, .success);
     }
 
     fn recordFailure(
@@ -344,6 +349,7 @@ pub const Pool = struct {
         // After the pool mutex is released, for the reason `recordSuccess`
         // states.
         self.recordHistory(io, entry, .{ .failure = @errorName(err) });
+        self.recordDiagnostics(io, entry, .{ .failure = @errorName(err) });
     }
 
     const Outcome = union(enum) { success, failure: []const u8 };
@@ -359,9 +365,34 @@ pub const Pool = struct {
             .failure => |name| history.recordFailure(io, entry.endpoint.url, wall_s, name),
         }
     }
+
+    /// The same placement discipline as `recordHistory`: the store takes a
+    /// mutex of its own, so this runs after the pool's is released.
+    ///
+    /// A success is the steady state of the whole program, so `resolve` is
+    /// built to issue no SQL when nothing is open (`storage/events.zig`).
+    fn recordDiagnostics(self: *Pool, io: std.Io, entry: *Entry, outcome: Outcome) void {
+        const store = self.diagnostics orelse return;
+        const url = entry.endpoint.url;
+        const now_s = std.Io.Clock.real.now(io).toSeconds();
+        switch (outcome) {
+            .success => store.resolve(io, now_s, .upstream_exchange, url),
+            .failure => |name| {
+                var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
+                const label = std.fmt.bufPrint(&label_buf, "{f}", .{safe_url.redact(url)}) catch &label_buf;
+                var detail_buf: [events.Store.max_detail_len]u8 = undefined;
+                const detail = std.fmt.bufPrint(&detail_buf, "upstream {f} failed: {s}", .{
+                    safe_url.redactQuoted(url),
+                    name,
+                }) catch &detail_buf;
+                store.report(io, now_s, .upstream_exchange, url, label, .warning, detail);
+            },
+        }
+    }
 };
 
 const db = @import("../storage/db.zig");
+const events_fixture = @import("../storage/events_fixture.zig");
 const querylog_schema = @import("../storage/querylog_schema.zig");
 const upstream_history_repo = @import("../storage/repositories/upstream_history_repo.zig");
 
@@ -948,3 +979,69 @@ test "an entry that enters backoff while a task waits on it is not attempted" {
     try testing.expectEqual(@as(u64, 1), entries[0].health.total_failures);
     try testing.expect(entries[0].health.backoff_until != null);
 }
+
+test "a successful exchange with nothing open costs the store no statement" {
+    if (@FieldType(events.Store, "statements") != u64) return error.SkipZigTest;
+
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
+    var entries = [_]Entry{testEntry("https://good.example/dns-query", &good, 10)};
+    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+    pool.diagnostics = &fx.store;
+
+    var buf: [512]u8 = undefined;
+    const before = fx.store.statements;
+    for (0..20) |_| _ = try pool.exchange(io, query_bytes, &buf);
+    try testing.expectEqual(before, fx.store.statements);
+    try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
+}
+
+test "a failing then recovering upstream leaves exactly one resolved episode" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var flaky: Fake = .{ .behavior = .{ .fail = error.Timeout } };
+    var standby: Fake = .{ .behavior = .{ .reply = response_bytes } };
+    var entries = [_]Entry{
+        testEntry("https://flaky.example/dns-query", &flaky, 10),
+        testEntry("https://standby.example/dns-query", &standby, 20),
+    };
+    var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+    pool.diagnostics = &fx.store;
+
+    var buf: [512]u8 = undefined;
+    _ = try pool.exchange(io, query_bytes, &buf);
+    // Backoff would park the failing entry, so the second failure is driven
+    // through `recordFailure` itself rather than through another exchange.
+    pool.recordFailure(io, &entries[0], std.Io.Clock.awake.now(io), error.ConnectFailed);
+
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+    try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
+    try testing.expectEqualStrings("upstream.exchange", try fx.text("SELECT code FROM operational_events"));
+    try testing.expectEqualStrings(
+        "https://flaky.example/dns-query",
+        try fx.text("SELECT subject_key FROM operational_events"),
+    );
+
+    flaky.behavior = .{ .reply = response_bytes };
+    pool.recordSuccess(io, &entries[0], std.Io.Clock.awake.now(io));
+
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+    try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
+}