milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/storage/retention.zig

@@ -13,6 +13,7 @@ const std = @import("std");
 
 const db = @import("db.zig");
 const disk_monitor = @import("disk_monitor.zig");
+const events = @import("events.zig");
 const model = @import("../config/model.zig");
 const queries_repo = @import("repositories/queries_repo.zig");
 const upstream_history_repo = @import("repositories/upstream_history_repo.zig");
@@ -102,15 +103,23 @@ pub const Retention = struct {
         io: std.Io,
         database: *db.Db,
         monitor: ?*disk_monitor.Monitor,
+        store: ?*events.Store,
     ) void {
         add(&self.counters.passes, 1);
         const now = std.Io.Clock.real.now(io).toSeconds();
         const cutoff = now - model.retentionSeconds(self.cfg);
 
+        // Diagnostics retention rides this pass rather than a schedule of its
+        // own: one daily housekeeping task, and a box restarted every night
+        // still prunes through `Store.init`.
+        if (store) |s| s.prune(io, now);
+
         if (queries_repo.pruneOlderThan(database, cutoff)) |deleted| {
             add(&self.counters.rows_pruned, @intCast(deleted));
+            maintenance(store, io, now, "prune", null);
         } else |err| {
             log.warn("retention prune before {d} failed: {s}", .{ cutoff, @errorName(err) });
+            maintenance(store, io, now, "prune", @errorName(err));
         }
 
         // Before the vacuum-cadence logic below, which returns early on six
@@ -123,14 +132,18 @@ pub const Retention = struct {
         const history_cutoff = now - upstream_history_repo.retention_window_s;
         if (upstream_history_repo.pruneOlderThan(database, history_cutoff)) |deleted| {
             add(&self.counters.upstream_rows_pruned, @intCast(deleted));
+            maintenance(store, io, now, "history_prune", null);
         } else |err| {
             log.warn("upstream history prune before {d} failed: {s}", .{ history_cutoff, @errorName(err) });
+            maintenance(store, io, now, "history_prune", @errorName(err));
         }
 
         if (queries_repo.checkpointTruncate(database)) {
             add(&self.counters.checkpoints, 1);
+            maintenance(store, io, now, "checkpoint", null);
         } else |err| {
             log.warn("retention checkpoint failed: {s}", .{@errorName(err)});
+            maintenance(store, io, now, "checkpoint", @errorName(err));
         }
 
         self.passes_since_vacuum += 1;
@@ -141,17 +154,37 @@ pub const Retention = struct {
         if (monitor) |m| if (!m.writesAllowed()) {
             add(&self.counters.vacuums_gated, 1);
             log.warn("retention vacuum skipped: the disk monitor refuses writes", .{});
+            maintenance(store, io, now, "vacuum", "the disk monitor refuses writes");
             return;
         };
 
         if (queries_repo.vacuum(database)) {
             add(&self.counters.vacuums, 1);
             self.passes_since_vacuum = 0;
+            maintenance(store, io, now, "vacuum", null);
         } else |err| {
             log.warn("retention vacuum failed: {s}", .{@errorName(err)});
+            maintenance(store, io, now, "vacuum", @errorName(err));
         }
     }
 
+    /// One step's outcome. `reason` null is the success branch of that same
+    /// step in that same pass, which is what closes its episode; a gated vacuum
+    /// is a failure of the step, because the work it owes is still owed.
+    fn maintenance(
+        store: ?*events.Store,
+        io: std.Io,
+        now_s: i64,
+        operation: []const u8,
+        reason: ?[]const u8,
+    ) void {
+        const s = store orelse return;
+        const text = reason orelse return s.resolve(io, now_s, .query_log_maintenance, operation);
+        var buf: [events.Store.max_detail_len]u8 = undefined;
+        const detail = std.fmt.bufPrint(&buf, "retention {s} failed: {s}", .{ operation, text }) catch buf[0..];
+        s.report(io, now_s, .query_log_maintenance, operation, operation, .warning, detail);
+    }
+
     fn add(counter: *std.atomic.Value(u64), delta: u64) void {
         _ = counter.fetchAdd(delta, .monotonic);
     }
@@ -182,13 +215,14 @@ pub const Retention = struct {
         io: std.Io,
         database: *db.Db,
         monitor: ?*disk_monitor.Monitor,
+        store: ?*events.Store,
     ) std.Io.Cancelable!void {
         const interval: std.Io.Clock.Duration = .{
             .raw = .fromSeconds(pass_interval_s),
             .clock = .boot,
         };
         while (true) {
-            self.runOnce(io, database, monitor);
+            self.runOnce(io, database, monitor, store);
             try interval.sleep(io);
         }
     }
@@ -198,6 +232,7 @@ pub const Retention = struct {
 // tests
 // ---------------------------------------------------------------------------
 
+const events_fixture = @import("events_fixture.zig");
 const querylog_schema = @import("querylog_schema.zig");
 
 const testing = std.testing;
@@ -243,7 +278,7 @@ test "a pass prunes the rows past the retention window and keeps the rest" {
     try writeRows(&database, &.{ now - 40 * day, now - 31 * day, now - 29 * day, now - 60 });
 
     var retention: Retention = .init(.{ .retention_days = 30 });
-    retention.runOnce(io, &database, null);
+    retention.runOnce(io, &database, null, null);
 
     try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
@@ -267,12 +302,12 @@ test "the cutoff follows retention_days" {
     try writeRows(&database, &.{now - 3 * day});
 
     var keeps: Retention = .init(.{ .retention_days = 7 });
-    keeps.runOnce(io, &database, null);
+    keeps.runOnce(io, &database, null, null);
     try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 0), keeps.snapshotStats().rows_pruned);
 
     var prunes: Retention = .init(.{ .retention_days = 1 });
-    prunes.runOnce(io, &database, null);
+    prunes.runOnce(io, &database, null, null);
     try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 1), prunes.snapshotStats().rows_pruned);
 }
@@ -287,16 +322,16 @@ test "the seventh pass vacuums and the six before it do not" {
 
     var retention: Retention = .init(.{});
     for (0..6) |_| {
-        retention.runOnce(io, &database, null);
+        retention.runOnce(io, &database, null, null);
         try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums);
     }
-    retention.runOnce(io, &database, null);
+    retention.runOnce(io, &database, null, null);
 
     try testing.expectEqual(@as(u64, 7), retention.snapshotStats().passes);
     try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
     try testing.expectEqual(@as(u64, 7), retention.snapshotStats().checkpoints);
 
-    for (0..7) |_| retention.runOnce(io, &database, null);
+    for (0..7) |_| retention.runOnce(io, &database, null, null);
     try testing.expectEqual(@as(u64, 14), retention.snapshotStats().passes);
     try testing.expectEqual(@as(u64, 2), retention.snapshotStats().vacuums);
 }
@@ -315,7 +350,7 @@ test "a gated pass skips the vacuum, counts it, and vacuums on the next pass" {
     monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
 
     var gated: Retention = .init(.{});
-    for (0..vacuum_every_passes) |_| gated.runOnce(io, &database, &monitor);
+    for (0..vacuum_every_passes) |_| gated.runOnce(io, &database, &monitor, null);
 
     // Prune and checkpoint ran on every pass; only the vacuum was refused.
     try testing.expectEqual(@as(u64, vacuum_every_passes), gated.snapshotStats().passes);
@@ -325,12 +360,12 @@ test "a gated pass skips the vacuum, counts it, and vacuums on the next pass" {
 
     // The vacuum is due again immediately, not seven passes later.
     monitor.state_raw.store(@intFromEnum(disk_monitor.State.ok), .monotonic);
-    gated.runOnce(io, &database, &monitor);
+    gated.runOnce(io, &database, &monitor, null);
     try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums);
     try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums_gated);
 
     // And the counter reset, so the next six passes vacuum nothing.
-    for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor);
+    for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor, null);
     try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums);
 }
 
@@ -346,7 +381,7 @@ test "a warn state still allows the vacuum" {
     monitor.state_raw.store(@intFromEnum(disk_monitor.State.warn), .monotonic);
 
     var retention: Retention = .init(.{});
-    for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor);
+    for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor, null);
 
     try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
     try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums_gated);
@@ -361,7 +396,7 @@ test "a pass over an empty database still counts" {
     defer database.close();
 
     var retention: Retention = .init(.{});
-    retention.runOnce(io, &database, null);
+    retention.runOnce(io, &database, null, null);
 
     try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
     try testing.expectEqual(@as(u64, 0), retention.snapshotStats().rows_pruned);
@@ -385,7 +420,7 @@ test "a failing prune counts the pass and leaves the rows alone" {
     );
 
     var retention: Retention = .init(.{ .retention_days = 30 });
-    retention.runOnce(io, &database, null);
+    retention.runOnce(io, &database, null, null);
 
     try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
@@ -412,7 +447,7 @@ test "the upstream-history window is its own, and a one-day query log does not s
 
     // A query log kept for one day, and 30 days of upstream minutes beside it.
     var retention: Retention = .init(.{ .retention_days = 1 });
-    retention.runOnce(io, &database, null);
+    retention.runOnce(io, &database, null, null);
 
     const stats = retention.snapshotStats();
     // One query-log row is older than one day; one minute row is older than the
@@ -450,7 +485,7 @@ test "the history prune runs on the passes where the vacuum logic returns early"
         .{ .url = "https://a.example", .minute_ts = old_minute, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
     });
     var early: Retention = .init(.{});
-    early.runOnce(io, &database, null);
+    early.runOnce(io, &database, null, null);
     try testing.expectEqual(@as(u64, 1), early.snapshotStats().upstream_rows_pruned);
     try testing.expectEqual(@as(i64, 0), try upstream_history_repo.countMinutes(&database));
 
@@ -460,11 +495,11 @@ test "the history prune runs on the passes where the vacuum logic returns early"
     monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
 
     var gated: Retention = .init(.{});
-    for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor);
+    for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor, null);
     try upstream_history_repo.flush(&database, &.{
         .{ .url = "https://b.example", .minute_ts = old_minute, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
     });
-    gated.runOnce(io, &database, &monitor);
+    gated.runOnce(io, &database, &monitor, null);
 
     try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums_gated);
     try testing.expectEqual(@as(u64, 1), gated.snapshotStats().upstream_rows_pruned);
@@ -487,13 +522,108 @@ test "the next pass retries what the failed one could not do" {
     );
 
     var retention: Retention = .init(.{ .retention_days = 30 });
-    retention.runOnce(io, &database, null);
+    retention.runOnce(io, &database, null, null);
     try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
 
     try database.exec("DROP TRIGGER refuse_delete;");
-    retention.runOnce(io, &database, null);
+    retention.runOnce(io, &database, null, null);
 
     try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 2), retention.snapshotStats().passes);
     try testing.expectEqual(@as(u64, 2), retention.snapshotStats().rows_pruned);
 }
+
+test "a failing prune opens a maintenance episode the next clean pass closes" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openLog();
+    defer database.close();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    const now = std.Io.Clock.real.now(io).toSeconds();
+    try writeRows(&database, &.{now - 40 * 86_400});
+    try database.exec(
+        \\CREATE TRIGGER refuse_delete BEFORE DELETE ON query_log
+        \\BEGIN SELECT RAISE(ABORT, 'refused'); END;
+    );
+
+    var retention: Retention = .init(.{});
+    retention.runOnce(io, &database, null, &fx.store);
+
+    // Only the prune failed; checkpoint and history prune succeeded, and a
+    // success writes no row of its own.
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+    try testing.expectEqualStrings("query_log.maintenance", try fx.text("SELECT code FROM operational_events"));
+    try testing.expectEqualStrings("prune", try fx.text("SELECT subject_key FROM operational_events"));
+    try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
+
+    try database.exec("DROP TRIGGER refuse_delete;");
+    retention.runOnce(io, &database, null, &fx.store);
+
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "a gated vacuum is a maintenance failure the next ungated pass closes" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openLog();
+    defer database.close();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null);
+    monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
+
+    var retention: Retention = .init(.{});
+    for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor, &fx.store);
+
+    try testing.expectEqualStrings("vacuum", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+
+    monitor.state_raw.store(@intFromEnum(disk_monitor.State.ok), .monotonic);
+    retention.runOnce(io, &database, &monitor, &fx.store);
+
+    try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "a pass prunes the diagnostics store once" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openLog();
+    defer database.close();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    // Resolved further back than the retention window, so the pass must drop it.
+    const now = std.Io.Clock.real.now(io).toSeconds();
+    const stale = now - events.Store.resolved_retention_s - 86_400;
+    fx.store.reportResolved(io, stale, .query_log_recreated, "one-shot", "one-shot", .warning, "aside kept");
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+
+    var retention: Retention = .init(.{});
+    retention.runOnce(io, &database, null, &fx.store);
+
+    try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
+}