milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/storage/disk_monitor.zig

@@ -9,6 +9,7 @@
 //! milestone-5 file; the gate is pulled, not pushed.
 
 const std = @import("std");
+const events = @import("events.zig");
 const model = @import("../config/model.zig");
 const statfs = @import("../platform/statfs.zig");
 
@@ -89,43 +90,49 @@ pub const Monitor = struct {
     /// evidence that the disk filled — and a failed size scan leaves that one
     /// gauge at its previous reading. Every failure increments
     /// `sample_failures` and logs one line at `warn`.
-    pub fn sample(self: *Monitor, io: std.Io) void {
-        const free = statfs.freeBytes(self.data_path) catch {
+    pub fn sample(self: *Monitor, io: std.Io, store: ?*events.Store, now_s: i64) void {
+        const free = statfs.freeBytes(self.data_path) catch |err| {
             self.countFailure();
             log.warn("statvfs on {s} failed", .{self.data_path});
+            probeFailed(store, io, now_s, "statvfs", "statvfs on the data path failed", err);
             return;
         };
         self.free_bytes.store(free, .monotonic);
+        if (store) |s| s.resolve(io, now_s, .disk_probe, "statvfs");
 
         if (sumDir(io, self.data_dir, isDatabaseFile)) |bytes| {
             self.db_bytes.store(bytes, .monotonic);
+            if (store) |s| s.resolve(io, now_s, .disk_probe, "data_dir");
         } else |err| {
             self.countFailure();
             log.warn("sizing the data directory failed: {s}", .{@errorName(err)});
+            probeFailed(store, io, now_s, "data_dir", "sizing the data directory failed", err);
         }
 
         if (self.log_dir_path) |path| {
             if (self.sumLogDir(io, path)) |bytes| {
                 self.log_bytes.store(bytes, .monotonic);
+                if (store) |s| s.resolve(io, now_s, .disk_probe, "log_dir");
             } else |err| {
                 self.countFailure();
                 log.warn("sizing {s} failed: {s}", .{ path, @errorName(err) });
+                probeFailed(store, io, now_s, "log_dir", "sizing the log directory failed", err);
             }
         }
 
-        self.publish(classify(free, self.cfg), free);
+        self.publish(io, store, now_s, classify(free, self.cfg), free);
     }
 
     /// Sample first, then sleep: a process that starts on a full disk must not
     /// serve a whole interval believing the state is `.ok`. `.boot` so a
     /// suspended box still sees the interval elapse.
-    pub fn run(self: *Monitor, io: std.Io) std.Io.Cancelable!void {
+    pub fn run(self: *Monitor, io: std.Io, store: ?*events.Store) std.Io.Cancelable!void {
         const interval: std.Io.Clock.Duration = .{
             .raw = .fromSeconds(sample_interval_s),
             .clock = .boot,
         };
         while (true) {
-            self.sample(io);
+            self.sample(io, store, std.Io.Clock.real.now(io).toSeconds());
             try interval.sleep(io);
         }
     }
@@ -136,7 +143,14 @@ pub const Monitor = struct {
 
     /// Logs on transitions only. A disk that sits at `.warn` for a week
     /// produces one line, not ten thousand.
-    fn publish(self: *Monitor, next: State, free: u64) void {
+    fn publish(
+        self: *Monitor,
+        io: std.Io,
+        store: ?*events.Store,
+        now_s: i64,
+        next: State,
+        free: u64,
+    ) void {
         const previous: State = @enumFromInt(self.state_raw.swap(@intFromEnum(next), .monotonic));
         if (previous == next) return;
         log.warn("disk state {t} -> {t}: {d} bytes free on {s}", .{
@@ -145,6 +159,24 @@ pub const Monitor = struct {
             free,
             self.data_path,
         });
+
+        const s = store orelse return;
+        if (next == .ok) {
+            s.resolve(io, now_s, .disk_space, disk_space_key);
+            return;
+        }
+        var buf: [events.Store.max_detail_len]u8 = undefined;
+        const detail = std.fmt.bufPrint(&buf, "disk state {t} -> {t}: {d} bytes free on {s}", .{
+            previous,
+            next,
+            free,
+            self.data_path,
+        }) catch buf[0..];
+        s.report(io, now_s, .disk_space, disk_space_key, "data directory", switch (next) {
+            .warn => .warning,
+            .critical => .@"error",
+            .ok => unreachable,
+        }, detail);
     }
 
     fn sumLogDir(self: *Monitor, io: std.Io, path: [:0]const u8) !u64 {
@@ -155,6 +187,26 @@ pub const Monitor = struct {
     }
 };
 
+/// The one subject `disk.space` ever has: this box has exactly one data
+/// directory, and its filesystem is what the thresholds classify.
+const disk_space_key = "data";
+
+/// Every probe failure is a warning, not an error: an unreadable filesystem is
+/// a gap in what the monitor can see, and the state it published last stands.
+fn probeFailed(
+    store: ?*events.Store,
+    io: std.Io,
+    now_s: i64,
+    operation: []const u8,
+    message: []const u8,
+    err: anyerror,
+) void {
+    const s = store orelse return;
+    var buf: [events.Store.max_detail_len]u8 = undefined;
+    const detail = std.fmt.bufPrint(&buf, "{s}: {s}", .{ message, @errorName(err) }) catch buf[0..];
+    s.report(io, now_s, .disk_probe, operation, operation, .warning, detail);
+}
+
 fn everyFile(_: []const u8) bool {
     return true;
 }
@@ -190,6 +242,7 @@ fn sumDir(io: std.Io, dir: std.Io.Dir, accept: *const fn ([]const u8) bool) !u64
 // tests
 // ---------------------------------------------------------------------------
 
+const events_fixture = @import("events_fixture.zig");
 const testing = std.testing;
 
 const mb = 1024 * 1024;
@@ -280,7 +333,7 @@ test "a sample sizes the databases and ignores every other file" {
     try tmp.dir.writeFile(io, .{ .sub_path = "notes.txt", .data = &[_]u8{'d'} ** 4096 });
 
     var monitor: Monitor = .init(.{ .min_free_mb = 0, .warn_free_mb = 0 }, tmp.dir, ".", null);
-    monitor.sample(io);
+    monitor.sample(io, null, 0);
 
     const g = monitor.gauges();
     try testing.expectEqual(@as(u64, 160), g.db_bytes);
@@ -308,7 +361,7 @@ test "a sample sizes every file in the log directory" {
     const log_path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/logs", .{tmp.sub_path});
 
     var monitor: Monitor = .init(.{ .min_free_mb = 0, .warn_free_mb = 0 }, tmp.dir, ".", log_path);
-    monitor.sample(io);
+    monitor.sample(io, null, 0);
 
     try testing.expectEqual(@as(u64, 1000), monitor.gauges().log_bytes);
     try testing.expectEqual(@as(u64, 0), monitor.sample_failures.load(.monotonic));
@@ -321,7 +374,7 @@ test "a failed statvfs counts and keeps the previous state" {
 
     var monitor: Monitor = .init(.{}, std.Io.Dir.cwd(), "./nxdns-no-such-path-7c21", null);
     monitor.state_raw.store(@intFromEnum(State.warn), .monotonic);
-    monitor.sample(io);
+    monitor.sample(io, null, 0);
 
     try testing.expectEqual(State.warn, monitor.state());
     try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
@@ -342,7 +395,7 @@ test "an unreadable log directory counts a failure but still publishes a state"
         ".",
         "./nxdns-no-such-dir-4f8a",
     );
-    monitor.sample(io);
+    monitor.sample(io, null, 0);
 
     try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
     try testing.expectEqual(State.ok, monitor.state());
@@ -386,7 +439,7 @@ test "an unreadable data directory fails the scan and keeps the previous gauge"
     // The handle keeps its read permission from open time, so `iterate` still
     // lists the file, but path resolution under the directory now fails.
     try tmp.dir.setPermissions(io, .fromMode(0o600));
-    monitor.sample(io);
+    monitor.sample(io, null, 0);
     try tmp.dir.setPermissions(io, .fromMode(0o700));
 
     try testing.expectEqual(@as(u64, 4096), monitor.gauges().db_bytes);
@@ -409,13 +462,115 @@ test "a threshold above the real free space drives the state to critical" {
         ".",
         null,
     );
-    monitor.sample(io);
+    monitor.sample(io, null, 0);
     try testing.expectEqual(State.critical, monitor.state());
     try testing.expect(!monitor.writesAllowed());
 
     monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
-    monitor.sample(io);
+    monitor.sample(io, null, 0);
     try testing.expectEqual(State.ok, monitor.state());
     try testing.expect(monitor.writesAllowed());
     try testing.expectEqual(@as(u64, 0), monitor.sample_failures.load(.monotonic));
 }
+
+test "a disk transition records an episode per severity and closes it on recovery" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    const unreachable_mb = std.math.maxInt(u32);
+    var monitor: Monitor = .init(
+        .{ .min_free_mb = unreachable_mb, .warn_free_mb = unreachable_mb },
+        tmp.dir,
+        ".",
+        null,
+    );
+    monitor.sample(io, &fx.store, 1000);
+    try testing.expectEqual(State.critical, monitor.state());
+    try testing.expectEqualStrings("disk.space", try fx.text(
+        "SELECT code FROM operational_events WHERE resolved_at IS NULL",
+    ));
+    try testing.expectEqualStrings("error", try fx.text(
+        "SELECT severity FROM operational_events WHERE resolved_at IS NULL",
+    ));
+    try testing.expectEqualStrings("data", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+
+    // A second critical sample is the same episode, not a second row: `publish`
+    // only reports on a transition.
+    monitor.sample(io, &fx.store, 1060);
+    try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
+
+    monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
+    monitor.sample(io, &fx.store, 1120);
+    try testing.expectEqual(State.ok, monitor.state());
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+    try testing.expectEqual(@as(i64, 1120), try fx.count("SELECT resolved_at FROM operational_events"));
+}
+
+test "a failed probe opens an episode the next clean pass closes" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    var monitor: Monitor = .init(
+        .{ .min_free_mb = 0, .warn_free_mb = 0 },
+        tmp.dir,
+        ".",
+        "./nxdns-no-such-dir-4f8a",
+    );
+    monitor.sample(io, &fx.store, 1000);
+
+    try testing.expectEqualStrings("disk.probe", try fx.text(
+        "SELECT code FROM operational_events WHERE resolved_at IS NULL",
+    ));
+    try testing.expectEqualStrings("log_dir", try fx.text(
+        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
+    ));
+    try testing.expectEqualStrings("warning", try fx.text(
+        "SELECT severity FROM operational_events WHERE resolved_at IS NULL",
+    ));
+
+    try tmp.dir.createDirPath(io, "logs");
+    var path_buf: [256]u8 = undefined;
+    monitor.log_dir_path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/logs", .{tmp.sub_path});
+    monitor.sample(io, &fx.store, 1100);
+
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
+    );
+}
+
+test "every emit site is inert when the store is absent" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var monitor: Monitor = .init(
+        .{ .min_free_mb = std.math.maxInt(u32), .warn_free_mb = std.math.maxInt(u32) },
+        std.Io.Dir.cwd(),
+        "./nxdns-no-such-path-7c21",
+        "./nxdns-no-such-dir-4f8a",
+    );
+    monitor.sample(io, null, 0);
+    try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
+}