milestone 16: behavioral fixes for silent failures, locks, counters and the query log

src/storage/phase6_integration_test.zig

@@ -360,7 +360,7 @@ test "S8 case 5: a retention pass prunes the old rows and truncates the write-ah
     try testing.expect(try f.sizeOf("querylog.db-wal") > 0);
 
     var pass: retention.Retention = .init(.{ .retention_days = 30 });
-    pass.runOnce(io, log_db.database());
+    pass.runOnce(io, log_db.database(), null);
 
     try testing.expectEqual(@as(u64, 1), pass.snapshotStats().passes);
     try testing.expectEqual(@as(u64, 2), pass.snapshotStats().rows_pruned);

src/storage/retention.zig

@@ -12,6 +12,7 @@
 const std = @import("std");
 
 const db = @import("db.zig");
+const disk_monitor = @import("disk_monitor.zig");
 const model = @import("../config/model.zig");
 const queries_repo = @import("repositories/queries_repo.zig");
 
@@ -32,6 +33,9 @@ pub const Stats = struct {
     rows_pruned: u64 = 0,
     checkpoints: u64 = 0,
     vacuums: u64 = 0,
+    /// Vacuums the disk monitor refused. The pass still pruned and
+    /// checkpointed, and the vacuum is due again on the next pass.
+    vacuums_gated: u64 = 0,
 };
 
 /// The live counters. Atomic because the retention task writes them and the web
@@ -42,25 +46,30 @@ const Counters = struct {
     rows_pruned: std.atomic.Value(u64) = .init(0),
     checkpoints: std.atomic.Value(u64) = .init(0),
     vacuums: std.atomic.Value(u64) = .init(0),
+    vacuums_gated: std.atomic.Value(u64) = .init(0),
 };
 
 pub const Retention = struct {
     cfg: model.Logging,
     counters: Counters,
+    /// Passes since the last vacuum that succeeded. Plain rather than atomic:
+    /// only the retention task reads or writes it, and no consumer reports it.
+    passes_since_vacuum: u32,
 
     pub fn init(cfg: model.Logging) Retention {
-        return .{ .cfg = cfg, .counters = .{} };
+        return .{ .cfg = cfg, .counters = .{}, .passes_since_vacuum = 0 };
     }
 
-    /// The four counters, read one at a time. A scrape that lands mid-pass can
-    /// see a pass counted before the rows it pruned are; the alternative is a
-    /// lock on the pass itself, which buys a consistency no consumer needs.
+    /// The counters, read one at a time. A scrape that lands mid-pass can see a
+    /// pass counted before the rows it pruned are; the alternative is a lock on
+    /// the pass itself, which buys a consistency no consumer needs.
     pub fn snapshotStats(self: *const Retention) Stats {
         return .{
             .passes = self.counters.passes.load(.monotonic),
             .rows_pruned = self.counters.rows_pruned.load(.monotonic),
             .checkpoints = self.counters.checkpoints.load(.monotonic),
             .vacuums = self.counters.vacuums.load(.monotonic),
+            .vacuums_gated = self.counters.vacuums_gated.load(.monotonic),
         };
     }
 
@@ -75,35 +84,54 @@ pub const Retention = struct {
     /// on the first failure would still be a day away from its retry.
     ///
     /// `database` must be a connection no other task uses; see `run`.
-    pub fn runOnce(self: *Retention, io: std.Io, database: *db.Db) void {
-        const pass = add(&self.counters.passes, 1) + 1;
+    ///
+    /// `monitor` gates the vacuum only. Prune and checkpoint free space, so a
+    /// full disk is a reason to run them rather than a reason to skip them,
+    /// while a `VACUUM` rewrites the whole file on the very filesystem the
+    /// monitor watches and fails `SQLITE_FULL` there. A null monitor means no
+    /// gate, which is the shape the logger's writer takes.
+    pub fn runOnce(
+        self: *Retention,
+        io: std.Io,
+        database: *db.Db,
+        monitor: ?*disk_monitor.Monitor,
+    ) void {
+        add(&self.counters.passes, 1);
         const cutoff = std.Io.Clock.real.now(io).toSeconds() - model.retentionSeconds(self.cfg);
 
         if (queries_repo.pruneOlderThan(database, cutoff)) |deleted| {
-            _ = add(&self.counters.rows_pruned, @intCast(deleted));
+            add(&self.counters.rows_pruned, @intCast(deleted));
         } else |err| {
             log.warn("retention prune before {d} failed: {s}", .{ cutoff, @errorName(err) });
         }
 
         if (queries_repo.checkpointTruncate(database)) {
-            _ = add(&self.counters.checkpoints, 1);
+            add(&self.counters.checkpoints, 1);
         } else |err| {
             log.warn("retention checkpoint failed: {s}", .{@errorName(err)});
         }
 
-        if (pass % vacuum_every_passes != 0) return;
+        self.passes_since_vacuum += 1;
+        if (self.passes_since_vacuum < vacuum_every_passes) return;
+
+        // The counter is not reset here, so a vacuum the monitor refused is due
+        // again on the very next pass rather than seven passes later.
+        if (monitor) |m| if (!m.writesAllowed()) {
+            add(&self.counters.vacuums_gated, 1);
+            log.warn("retention vacuum skipped: the disk monitor refuses writes", .{});
+            return;
+        };
+
         if (queries_repo.vacuum(database)) {
-            _ = add(&self.counters.vacuums, 1);
+            add(&self.counters.vacuums, 1);
+            self.passes_since_vacuum = 0;
         } else |err| {
             log.warn("retention vacuum failed: {s}", .{@errorName(err)});
         }
     }
 
-    /// Returns the value before the addition, which is what the pass counter
-    /// needs: only this task increments it, so `previous + 1` is this pass's
-    /// number.
-    fn add(counter: *std.atomic.Value(u64), delta: u64) u64 {
-        return counter.fetchAdd(delta, .monotonic);
+    fn add(counter: *std.atomic.Value(u64), delta: u64) void {
+        _ = counter.fetchAdd(delta, .monotonic);
     }
 
     /// Daily loop, first pass immediately. Phase 7 starts it.
@@ -125,13 +153,18 @@ pub const Retention = struct {
     /// SQLite's own — WAL plus the `busy_timeout` of `db.zig`'s open options —
     /// so a pass that still loses a race sees `error.Busy` or `error.Locked`,
     /// logs at `warn`, and repeats the work on the next interval.
-    pub fn run(self: *Retention, io: std.Io, database: *db.Db) std.Io.Cancelable!void {
+    pub fn run(
+        self: *Retention,
+        io: std.Io,
+        database: *db.Db,
+        monitor: ?*disk_monitor.Monitor,
+    ) std.Io.Cancelable!void {
         const interval: std.Io.Clock.Duration = .{
             .raw = .fromSeconds(pass_interval_s),
             .clock = .boot,
         };
         while (true) {
-            self.runOnce(io, database);
+            self.runOnce(io, database, monitor);
             try interval.sleep(io);
         }
     }
@@ -186,7 +219,7 @@ test "a pass prunes the rows past the retention window and keeps the rest" {
     try writeRows(&database, &.{ now - 40 * day, now - 31 * day, now - 29 * day, now - 60 });
 
     var retention: Retention = .init(.{ .retention_days = 30 });
-    retention.runOnce(io, &database);
+    retention.runOnce(io, &database, null);
 
     try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
@@ -210,12 +243,12 @@ test "the cutoff follows retention_days" {
     try writeRows(&database, &.{now - 3 * day});
 
     var keeps: Retention = .init(.{ .retention_days = 7 });
-    keeps.runOnce(io, &database);
+    keeps.runOnce(io, &database, null);
     try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 0), keeps.snapshotStats().rows_pruned);
 
     var prunes: Retention = .init(.{ .retention_days = 1 });
-    prunes.runOnce(io, &database);
+    prunes.runOnce(io, &database, null);
     try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 1), prunes.snapshotStats().rows_pruned);
 }
@@ -230,20 +263,71 @@ test "the seventh pass vacuums and the six before it do not" {
 
     var retention: Retention = .init(.{});
     for (0..6) |_| {
-        retention.runOnce(io, &database);
+        retention.runOnce(io, &database, null);
         try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums);
     }
-    retention.runOnce(io, &database);
+    retention.runOnce(io, &database, null);
 
     try testing.expectEqual(@as(u64, 7), retention.snapshotStats().passes);
     try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
     try testing.expectEqual(@as(u64, 7), retention.snapshotStats().checkpoints);
 
-    for (0..7) |_| retention.runOnce(io, &database);
+    for (0..7) |_| retention.runOnce(io, &database, null);
     try testing.expectEqual(@as(u64, 14), retention.snapshotStats().passes);
     try testing.expectEqual(@as(u64, 2), retention.snapshotStats().vacuums);
 }
 
+test "a gated pass skips the vacuum, counts it, and vacuums on the next pass" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openLog();
+    defer database.close();
+
+    // `.critical` is the one state `writesAllowed` refuses on, and it is
+    // published here directly: no real filesystem has to fill up for it.
+    var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null);
+    monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
+
+    var gated: Retention = .init(.{});
+    for (0..vacuum_every_passes) |_| gated.runOnce(io, &database, &monitor);
+
+    // Prune and checkpoint ran on every pass; only the vacuum was refused.
+    try testing.expectEqual(@as(u64, vacuum_every_passes), gated.snapshotStats().passes);
+    try testing.expectEqual(@as(u64, vacuum_every_passes), gated.snapshotStats().checkpoints);
+    try testing.expectEqual(@as(u64, 0), gated.snapshotStats().vacuums);
+    try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums_gated);
+
+    // The vacuum is due again immediately, not seven passes later.
+    monitor.state_raw.store(@intFromEnum(disk_monitor.State.ok), .monotonic);
+    gated.runOnce(io, &database, &monitor);
+    try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums);
+    try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums_gated);
+
+    // And the counter reset, so the next six passes vacuum nothing.
+    for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor);
+    try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums);
+}
+
+test "a warn state still allows the vacuum" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openLog();
+    defer database.close();
+
+    var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null);
+    monitor.state_raw.store(@intFromEnum(disk_monitor.State.warn), .monotonic);
+
+    var retention: Retention = .init(.{});
+    for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor);
+
+    try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
+    try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums_gated);
+}
+
 test "a pass over an empty database still counts" {
     var threaded: std.Io.Threaded = .init(testing.allocator, .{});
     defer threaded.deinit();
@@ -253,7 +337,7 @@ test "a pass over an empty database still counts" {
     defer database.close();
 
     var retention: Retention = .init(.{});
-    retention.runOnce(io, &database);
+    retention.runOnce(io, &database, null);
 
     try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
     try testing.expectEqual(@as(u64, 0), retention.snapshotStats().rows_pruned);
@@ -277,7 +361,7 @@ test "a failing prune counts the pass and leaves the rows alone" {
     );
 
     var retention: Retention = .init(.{ .retention_days = 30 });
-    retention.runOnce(io, &database);
+    retention.runOnce(io, &database, null);
 
     try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
@@ -302,11 +386,11 @@ test "the next pass retries what the failed one could not do" {
     );
 
     var retention: Retention = .init(.{ .retention_days = 30 });
-    retention.runOnce(io, &database);
+    retention.runOnce(io, &database, null);
     try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
 
     try database.exec("DROP TRIGGER refuse_delete;");
-    retention.runOnce(io, &database);
+    retention.runOnce(io, &database, null);
 
     try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
     try testing.expectEqual(@as(u64, 2), retention.snapshotStats().passes);