milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/filter/manager.zig

@@ -59,6 +59,7 @@ const groups_repo = @import("../storage/repositories/groups_repo.zig");
 const rules_repo = @import("../storage/repositories/rules_repo.zig");
 const sources_repo = @import("../storage/repositories/sources_repo.zig");
 const disk_monitor = @import("../storage/disk_monitor.zig");
+const events = @import("../storage/events.zig");
 const compiler = @import("compiler.zig");
 const fetcher = @import("fetcher.zig");
 const matcher = @import("matcher.zig");
@@ -162,6 +163,51 @@ pub const State = enum {
     }
 };
 
+/// The `blocklist.storage` operations, each its own episode subject.
+///
+/// A fixed set on purpose: several of these fail once per file in a pass, and
+/// one slot per operation is what turns that into one report per pass instead
+/// of an unbounded list of them.
+pub const StorageOp = enum { sweep, directory_read, create_dir, open_dir, delete };
+
+/// One operation's outcome across one pass. `detail` keeps the last failure,
+/// and `failures` says how many that pass held — the row's `occurrences` counts
+/// failing passes, so the count belongs in the text.
+const Aggregate = struct {
+    failures: u32 = 0,
+    succeeded: bool = false,
+    detail: [events.Store.max_detail_len]u8 = @splat(0),
+    detail_len: u16 = 0,
+
+    fn detailText(self: *const Aggregate) []const u8 {
+        return self.detail[0..self.detail_len];
+    }
+};
+
+/// What a locked body observed, held until `Manager.flushDiagnostics` can
+/// report it with no manager lock held.
+///
+/// Two of this file's operations cannot report from where they stand:
+/// `publishRefresh` runs under `writer_lock` by contract and `pruneOrphans`
+/// holds both writer mutexes through its filesystem work. Collect-then-flush is
+/// what keeps their outcomes without holding a lock across a store call, and
+/// nothing here can grow: the storage slots are an enum array and a refresh
+/// outcome rides the status entry the source already has.
+///
+/// The flush still happens inside the lock that serializes passes — a pass
+/// drains its own outcomes before it releases `refresh_lock` (or, for a
+/// standalone `reload`, `writer_lock`). What collect-then-flush avoids is
+/// holding a *manager* lock across a store call, not deferring the report until
+/// the next pass could merge into it.
+const Pending = struct {
+    mutex: std.Io.Mutex = .init,
+    storage: std.EnumArray(StorageOp, Aggregate) = .initFill(.{}),
+    /// Null until a pass observes a snapshot outcome at all.
+    snapshot_failed: ?bool = null,
+    snapshot_detail: [events.Store.max_detail_len]u8 = @splat(0),
+    snapshot_detail_len: u16 = 0,
+};
+
 /// A status is a value with no borrowed memory, so a copy handed to the API
 /// outlives every reload. The url is held inline for that reason.
 pub const SourceStatus = struct {
@@ -182,6 +228,32 @@ pub const SourceStatus = struct {
     url_len: u8 = 0,
     last_error: [max_error_len]u8 = @splat(0),
     last_error_len: u8 = 0,
+    /// The diagnostics identity of this source, canonicalized from the WHOLE
+    /// url by `setUrl`. `url` above is a display copy truncated at
+    /// `max_url_len`, and two urls sharing a 255-byte prefix would share one
+    /// episode if that copy were the key.
+    event_key: [events.Store.max_subject_key_len]u8 = @splat(0),
+    event_key_len: u16 = 0,
+    /// Diagnostics accounting for the pass in progress, cleared by every
+    /// `flushDiagnostics`. `pass_outcome` says this source recorded one at all;
+    /// `pass_failures` counts the failing ones, which a pass can hold more than
+    /// one of (a refresh that failed, then the reload that could not load the
+    /// files it did not write). One flush reports one `blocklist.refresh`
+    /// occurrence per source, so `occurrences` counts failing passes rather
+    /// than flushes, and the detail carries how many failures the pass held.
+    ///
+    /// These two fields live in exactly one copy of the status table at a time,
+    /// which is what makes that count right while reloads replace the table
+    /// underneath: a candidate built by `mergeStatuses` carries none of them,
+    /// `installStatuses` folds the live table's in as it swaps, and the flush
+    /// claims an entry by copying it and zeroing both fields in one locked
+    /// step. Copy them anywhere else and the outcome gets reported twice.
+    pass_outcome: bool = false,
+    pass_failures: u16 = 0,
+
+    pub fn eventKey(self: *const SourceStatus) []const u8 {
+        return self.event_key[0..self.event_key_len];
+    }
 
     pub fn errorText(self: *const SourceStatus) []const u8 {
         return self.last_error[0..self.last_error_len];
@@ -196,9 +268,12 @@ pub const SourceStatus = struct {
         @memcpy(self.url[0..kept], url[0..kept]);
         @memset(self.url[kept..], 0);
         self.url_len = @intCast(kept);
+        self.event_key_len = @intCast(events.canonicalKey(url, &self.event_key).len);
     }
 
     fn fail(self: *SourceStatus, state: State, text: []const u8) void {
+        self.pass_failures +|= 1;
+        self.pass_outcome = true;
         self.state = state;
         const kept = @min(text.len, max_error_len);
         @memcpy(self.last_error[0..kept], text[0..kept]);
@@ -207,6 +282,7 @@ pub const SourceStatus = struct {
     }
 
     fn succeed(self: *SourceStatus, at: i64, counts: compiler.Counts) void {
+        self.pass_outcome = true;
         self.state = .ok;
         self.counts = counts;
         self.last_success = at;
@@ -294,6 +370,14 @@ pub const Manager = struct {
     /// what every test and `nxdns check` want. Only the scheduler consults it —
     /// see `refreshGated`.
     monitor: ?*disk_monitor.Monitor = null,
+    /// The diagnostics store, wired the same way as `monitor` and null
+    /// everywhere else. Never touched while a manager lock is held: see
+    /// `flushDiagnostics`.
+    diagnostics: ?*events.Store = null,
+    /// What the locked bodies observed and could not report from where they
+    /// stood. Bounded by construction — one slot per storage operation, one
+    /// snapshot outcome — and drained by `flushDiagnostics`.
+    pending: Pending = .{},
     /// Scheduled refresh passes skipped by the disk gate. The `/api/health`
     /// rollup reads it through `refreshesGated`.
     refreshes_gated: std.atomic.Value(u64) = .init(0),
@@ -388,6 +472,228 @@ pub const Manager = struct {
         return kept;
     }
 
+    // -----------------------------------------------------------------------
+    // diagnostics
+    // -----------------------------------------------------------------------
+
+    /// Records one storage operation's failure. Callable from anywhere,
+    /// including under both writer mutexes: it touches `pending` only.
+    fn noteStorageFailure(
+        self: *Manager,
+        io: std.Io,
+        op: StorageOp,
+        comptime fmt: []const u8,
+        args: anytype,
+    ) void {
+        if (self.diagnostics == null) return;
+        self.pending.mutex.lockUncancelable(io);
+        defer self.pending.mutex.unlock(io);
+        const slot = self.pending.storage.getPtr(op);
+        slot.failures +|= 1;
+        var w: std.Io.Writer = .fixed(&slot.detail);
+        w.print(fmt, args) catch {};
+        slot.detail_len = @intCast(w.end);
+    }
+
+    fn noteStorageSuccess(self: *Manager, io: std.Io, op: StorageOp) void {
+        if (self.diagnostics == null) return;
+        self.pending.mutex.lockUncancelable(io);
+        defer self.pending.mutex.unlock(io);
+        self.pending.storage.getPtr(op).succeeded = true;
+    }
+
+    /// Records whether a snapshot was published. `reason` null is the post-swap
+    /// success; anything else is the pass that could not publish one.
+    fn noteSnapshot(self: *Manager, io: std.Io, reason: ?[]const u8) void {
+        if (self.diagnostics == null) return;
+        self.pending.mutex.lockUncancelable(io);
+        defer self.pending.mutex.unlock(io);
+        self.pending.snapshot_failed = reason != null;
+        const text = reason orelse "";
+        const kept = @min(text.len, self.pending.snapshot_detail.len);
+        @memcpy(self.pending.snapshot_detail[0..kept], text[0..kept]);
+        self.pending.snapshot_detail_len = @intCast(kept);
+    }
+
+    /// Drains `pending` and the status table into the store, holding no manager
+    /// lock across a store call.
+    ///
+    /// Called by every pass that can fill either one, and *before that pass
+    /// releases the lock serializing it* — `refresh_lock` for a refresh pass,
+    /// `writer_lock` for a standalone `reload`. Draining after the release
+    /// would let the next pass record its own outcomes on the same entries
+    /// first, and two failing passes would reach the store as one occurrence.
+    /// The `defer` that calls this is registered after the unlock `defer` for
+    /// that reason; defers run last-registered-first.
+    ///
+    /// It is idempotent: a drained collector reports nothing.
+    pub fn flushDiagnostics(self: *Manager, io: std.Io) void {
+        const store = self.diagnostics orelse return;
+        const now_s = std.Io.Clock.real.now(io).toSeconds();
+
+        var storage: std.EnumArray(StorageOp, Aggregate) = undefined;
+        var snapshot_failed: ?bool = null;
+        var snapshot_detail: [events.Store.max_detail_len]u8 = undefined;
+        var snapshot_detail_len: u16 = 0;
+        {
+            self.pending.mutex.lockUncancelable(io);
+            defer self.pending.mutex.unlock(io);
+            storage = self.pending.storage;
+            snapshot_failed = self.pending.snapshot_failed;
+            snapshot_detail = self.pending.snapshot_detail;
+            snapshot_detail_len = self.pending.snapshot_detail_len;
+            self.pending.storage = .initFill(.{});
+            self.pending.snapshot_failed = null;
+            self.pending.snapshot_detail_len = 0;
+        }
+
+        var it = storage.iterator();
+        while (it.next()) |kv| {
+            const op = @tagName(kv.key);
+            if (kv.value.failures != 0) {
+                var buf: [events.Store.max_detail_len]u8 = undefined;
+                const detail = std.fmt.bufPrint(&buf, "{s} ({d} this pass)", .{
+                    kv.value.detailText(),
+                    kv.value.failures,
+                }) catch buf[0..];
+                store.report(io, now_s, .blocklist_storage, op, op, .warning, detail);
+            } else if (kv.value.succeeded) {
+                store.resolve(io, now_s, .blocklist_storage, op);
+            }
+        }
+
+        if (snapshot_failed) |failed| {
+            if (failed) {
+                store.report(
+                    io,
+                    now_s,
+                    .blocklist_snapshot,
+                    snapshot_key,
+                    "blocklist snapshot",
+                    .@"error",
+                    snapshot_detail[0..snapshot_detail_len],
+                );
+            } else {
+                store.resolve(io, now_s, .blocklist_snapshot, snapshot_key);
+            }
+        }
+
+        self.flushSourceDiagnostics(io, store, now_s);
+    }
+
+    /// One `blocklist.refresh` episode per source, from the status table.
+    ///
+    /// The table IS the per-source collection the collect-then-flush rule asks
+    /// for: `prepareRefresh`, `publishRefresh` and the reload's load outcomes
+    /// all write their result into the entry, under locks this cannot take. So
+    /// one entry is copied out at a time under the exclusive lock and the store
+    /// is called with nothing held.
+    ///
+    /// The walk is a drain, not an index scan: a reload can replace the whole
+    /// table between two iterations, and an index into the table it replaced
+    /// would skip or repeat entries. Each round takes the lock, claims the
+    /// first entry that still carries pass accounting by copying it out and
+    /// zeroing the two fields, and reports it with nothing held. Claiming and
+    /// clearing are one locked step, so an outcome is reported once: a table
+    /// swapped in mid-drain carries the entries this flush has not claimed yet,
+    /// and `installStatuses` folded them in for exactly that reason. The drain
+    /// ends when a scan finds nothing left to claim.
+    ///
+    /// The drain reaches only the sources the table still holds, so the sweep
+    /// below is what closes the episode of one that is gone.
+    fn flushSourceDiagnostics(self: *Manager, io: std.Io, store: *events.Store, now_s: i64) void {
+        drain: while (true) {
+            var status: SourceStatus = undefined;
+            {
+                self.lock.lockUncancelable(io);
+                defer self.lock.unlock(io);
+                const claimed = for (self.statuses) |*entry| {
+                    if (!entry.pass_outcome) continue;
+                    status = entry.*;
+                    entry.pass_outcome = false;
+                    entry.pass_failures = 0;
+                    // A source with no diagnostics identity has nothing to
+                    // report under, but its accounting is cleared all the same:
+                    // left set, it would make every later scan claim it and the
+                    // drain would never end.
+                    if (entry.event_key_len == 0) continue;
+                    break true;
+                } else false;
+                if (!claimed) break :drain;
+            }
+            if (!status.state.isRefreshFailure() and status.state != .load_failed) {
+                store.resolve(io, now_s, .blocklist_refresh, status.eventKey());
+                continue;
+            }
+            var buf: [events.Store.max_detail_len]u8 = undefined;
+            const detail = std.fmt.bufPrint(&buf, "{t}: {s} ({d} this pass)", .{
+                status.state,
+                status.errorText(),
+                status.pass_failures,
+            }) catch buf[0..];
+            var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
+            const label = std.fmt.bufPrint(&label_buf, "{f}", .{
+                safe_url.redact(status.urlText()),
+            }) catch &label_buf;
+            store.report(io, now_s, .blocklist_refresh, status.eventKey(), label, .warning, detail);
+        }
+
+        self.resolveDeletedSources(io, store, now_s);
+    }
+
+    /// Closes the `blocklist.refresh` episode of a source that no longer exists.
+    ///
+    /// Nothing else can. An episode of this code is closed by its source
+    /// succeeding, and a source deleted through the API or dropped by a config
+    /// import never succeeds again: the drain above walks the status table, the
+    /// deleted source has no entry in it, and the resolved-row pruning never
+    /// touches an active row. Without this the operator keeps a warning about a
+    /// list they removed on purpose, and no restart clears it.
+    ///
+    /// The status table holds every source at every flush site — `refreshAll`
+    /// syncs it before it refreshes anything and a reload rebuilds it from the
+    /// rows — so its keys are exactly the episodes that may stay open.
+    fn resolveDeletedSources(self: *Manager, io: std.Io, store: *events.Store, now_s: i64) void {
+        var storage: [events.Store.max_kept_keys][events.Store.max_subject_key_len]u8 = undefined;
+        var lens: [events.Store.max_kept_keys]u16 = undefined;
+        var len: usize = 0;
+        {
+            // Shared: this reads the table and changes nothing in it. The keys
+            // are copied out because the arena they live in is freed by the
+            // next `installStatuses`, and the store is called below with
+            // nothing held.
+            self.lock.lockSharedUncancelable(io);
+            defer self.lock.unlockShared(io);
+
+            // An empty table before the first published snapshot means "no
+            // source set has been read yet", not "every source was deleted".
+            // Sweeping on it would close every episode the last run left open,
+            // and the pass that follows would reopen each one as a new episode
+            // with its history reset.
+            if (self.generation == 0) return;
+
+            for (self.statuses) |*entry| {
+                if (entry.event_key_len == 0) continue;
+                // `resolveExcept` refuses a kept list longer than
+                // `max_kept_keys`, because it canonicalizes onto the stack.
+                // Over that many keyed sources the sweep is skipped whole: the
+                // alternative is a truncated kept list, which would close
+                // episodes that are still true. A source deleted while the
+                // household is over the cap keeps its episode until the count
+                // falls back under it.
+                if (len == storage.len) return;
+                const key = entry.eventKey();
+                @memcpy(storage[len][0..key.len], key);
+                lens[len] = entry.event_key_len;
+                len += 1;
+            }
+        }
+
+        var kept: [events.Store.max_kept_keys][]const u8 = undefined;
+        for (0..len) |i| kept[i] = storage[i][0..lens[i]];
+        store.resolveExcept(io, now_s, .blocklist_refresh, kept[0..len]);
+    }
+
     // -----------------------------------------------------------------------
     // reload
     // -----------------------------------------------------------------------
@@ -407,12 +713,35 @@ pub const Manager = struct {
     /// table keeps describing that snapshot too — the table is rebuilt off to
     /// the side and the load findings are written into it there, so a reload
     /// that never publishes changes neither.
+    ///
+    /// A standalone reload is its own pass, and `writer_lock` is what serializes
+    /// it against every other writer of the status table. So it flushes inside
+    /// that lock: the load outcomes it wrote at the swap are drained before any
+    /// other pass can add its own to the same entries, which is what keeps two
+    /// failing passes two occurrences instead of one.
     pub fn reload(self: *Manager, io: std.Io) Error!void {
+        // Must not be entered with `writer_lock` held.
         self.writer_lock.lockUncancelable(io);
         defer self.writer_lock.unlock(io);
+        // Registered after the unlock so it runs before it, and `defer` and not
+        // straight-line code after the call: a reload that fails has already
+        // collected the outcomes that explain why, and leaving them pending
+        // would hold them until some later pass flushed them under the wrong
+        // timestamp.
+        defer self.flushDiagnostics(io);
         return self.reloadLocked(io);
     }
 
+    /// `reload` without the flush, for a caller that is inside a pass with a
+    /// flush of its own. One pass flushes once: flushing here as well would
+    /// split the pass's outcomes across two reports.
+    fn reloadCollecting(self: *Manager, io: std.Io) Error!void {
+        // Must not be entered with `writer_lock` held.
+        self.writer_lock.lockUncancelable(io);
+        defer self.writer_lock.unlock(io);
+        try self.reloadLocked(io);
+    }
+
     fn reloadLocked(self: *Manager, io: std.Io) Error!void {
         var rows = try sources_repo.listSourceRows(self.database, self.gpa);
         defer rows.deinit(self.gpa);
@@ -543,6 +872,7 @@ pub const Manager = struct {
             rows.items.len,
             memory_bytes,
         });
+        self.noteSnapshot(io, null);
     }
 
     /// What one enabled source contributes to the snapshot being built. Nothing
@@ -642,6 +972,11 @@ pub const Manager = struct {
     pub fn refreshSource(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool {
         self.refresh_lock.lockUncancelable(io);
         defer self.refresh_lock.unlock(io);
+        // Registered after the unlock, so it runs before it: a pass drains its
+        // own outcomes while it still holds `refresh_lock`. `defer` at all, so
+        // a refresh that fails outright still reports what it collected instead
+        // of leaving it for an unrelated later flush.
+        defer self.flushDiagnostics(io);
         return self.refreshSourceLocked(io, row);
     }
 
@@ -706,6 +1041,9 @@ pub const Manager = struct {
     pub fn refreshAll(self: *Manager, io: std.Io) Error!void {
         self.refresh_lock.lockUncancelable(io);
         defer self.refresh_lock.unlock(io);
+        // Inside `refresh_lock`, by being registered after the unlock: see
+        // `refreshSource`.
+        defer self.flushDiagnostics(io);
 
         var rows = try sources_repo.listSourceRows(self.database, self.gpa);
         defer rows.deinit(self.gpa);
@@ -718,8 +1056,9 @@ pub const Manager = struct {
             _ = try self.refreshSourceLocked(io, row);
         }
         // `reload` takes `writer_lock`, which the pass has been careful not to
-        // hold: the order is `refresh_lock` first, always.
-        return self.reload(io);
+        // hold: the order is `refresh_lock` first, always. The collecting
+        // variant, because the `defer` above is this pass's one flush.
+        return self.reloadCollecting(io);
     }
 
     /// The three temporary files one refresh compiles into, before the header
@@ -1161,9 +1500,16 @@ pub const Manager = struct {
         // ask the same filesystem for.
         try self.sweepOrphans(io);
 
+        // `startupPass` flushes its own outcomes before it releases
+        // `refresh_lock`, so the only flush left here is the one the failure
+        // note below needs.
         self.startupPass(io) catch |err| switch (err) {
             error.Canceled => return error.Canceled,
-            else => log.warn("blocklist startup pass failed: {s}", .{@errorName(err)}),
+            else => {
+                log.warn("blocklist startup pass failed: {s}", .{@errorName(err)});
+                self.noteSnapshot(io, @errorName(err));
+                self.flushDiagnostics(io);
+            },
         };
         if (!self.update.enabled) return;
 
@@ -1175,19 +1521,33 @@ pub const Manager = struct {
         };
         while (true) {
             try interval.sleep(io);
-            // Ahead of the gate as well as ahead of the pass: the sweep only
-            // unlinks, so it is the one thing here that can give a critically
-            // full disk room back, and gating it would keep the residue that
-            // helped fill the disk in the first place.
-            try self.sweepOrphans(io);
-            if (self.refreshGated()) continue;
-            self.refreshAll(io) catch |err| switch (err) {
-                error.Canceled => return error.Canceled,
-                else => log.warn("blocklist refresh pass failed: {s}", .{@errorName(err)}),
-            };
+            try self.scheduledPass(io);
         }
     }
 
+    /// What one elapsed interval does. Split from the loop above so a test can
+    /// run the pass without waiting the interval out; nothing in production
+    /// calls it but `runScheduler`.
+    pub fn scheduledPass(self: *Manager, io: std.Io) std.Io.Cancelable!void {
+        // Ahead of the gate as well as ahead of the refresh: the sweep only
+        // unlinks, so it is the one thing here that can give a critically full
+        // disk room back, and gating it would keep the residue that helped fill
+        // the disk in the first place.
+        try self.sweepOrphans(io);
+        if (self.refreshGated()) return;
+        // `refreshAll` flushes the pass itself, so the only flush left here is
+        // the one the failure note below needs: flushing unconditionally would
+        // report every outcome of the pass a second time.
+        self.refreshAll(io) catch |err| switch (err) {
+            error.Canceled => return error.Canceled,
+            else => {
+                log.warn("blocklist refresh pass failed: {s}", .{@errorName(err)});
+                self.noteSnapshot(io, @errorName(err));
+                self.flushDiagnostics(io);
+            },
+        };
+    }
+
     /// `pruneOrphans` with its failure absorbed. Leftover bytes under
     /// `<data_dir>/blocklists/` are not an outage, and a sweep that could not
     /// read the directory must not cost the household the refresh pass behind
@@ -1197,10 +1557,16 @@ pub const Manager = struct {
     /// Taken from outside every `*Locked` body: `pruneOrphans` takes both
     /// writer mutexes itself and neither is reentrant.
     fn sweepOrphans(self: *Manager, io: std.Io) std.Io.Cancelable!void {
-        self.pruneOrphans(io) catch |err| switch (err) {
+        if (self.pruneOrphans(io)) {
+            self.noteStorageSuccess(io, .sweep);
+        } else |err| switch (err) {
             error.Canceled => return error.Canceled,
-            else => log.warn("pruning orphaned blocklist files failed: {s}", .{@errorName(err)}),
-        };
+            else => {
+                log.warn("pruning orphaned blocklist files failed: {s}", .{@errorName(err)});
+                self.noteStorageFailure(io, .sweep, "pruning orphaned blocklist files failed: {s}", .{@errorName(err)});
+            },
+        }
+        self.flushDiagnostics(io);
     }
 
     /// The §11.6 gate, consulted by scheduled passes only (ruling 17). A
@@ -1232,11 +1598,14 @@ pub const Manager = struct {
     fn startupPass(self: *Manager, io: std.Io) Error!void {
         self.refresh_lock.lockUncancelable(io);
         defer self.refresh_lock.unlock(io);
+        // Inside `refresh_lock`, by being registered after the unlock: see
+        // `refreshSource`.
+        defer self.flushDiagnostics(io);
 
         // Ahead of the gate on purpose: loading the compiled files that already
         // exist is a read. A full disk must not cost the household its
         // filtering as well as its downloads.
-        try self.reload(io);
+        try self.reloadCollecting(io);
 
         if (self.refreshGated()) return;
 
@@ -1251,7 +1620,7 @@ pub const Manager = struct {
             if (!self.needsRefresh(io, row, now)) continue;
             if (try self.refreshSourceLocked(io, row)) refreshed = true;
         }
-        if (refreshed) try self.reload(io);
+        if (refreshed) try self.reloadCollecting(io);
     }
 
     fn needsRefresh(self: *Manager, io: std.Io, row: sources_repo.SourceRow, now: i64) bool {
@@ -1297,6 +1666,13 @@ pub const Manager = struct {
     /// `<data_dir>/blocklists/` if nothing has yet, and an empty directory
     /// sweeps to nothing.
     pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
+        defer self.flushDiagnostics(io);
+        return self.pruneOrphansLocked(io);
+    }
+
+    /// Assumes nothing and takes both writer mutexes itself. Split from
+    /// `pruneOrphans` so the diagnostics flush above happens with neither held.
+    fn pruneOrphansLocked(self: *Manager, io: std.Io) Error!void {
         // `refresh_lock` first, and for the reason it exists: the download and
         // the compile are the only writers of `.raw.tmp`, `.list.tmp`,
         // `.wild.tmp` and `.allow.tmp`, and they hold it for as long as they
@@ -1334,6 +1710,7 @@ pub const Manager = struct {
                 error.Canceled => return error.Canceled,
                 else => {
                     log.warn("pruning blocklists: reading the directory failed: {s}", .{@errorName(err)});
+                    self.noteStorageFailure(io, .directory_read, "reading the blocklist directory failed: {s}", .{@errorName(err)});
                     return error.FileSystem;
                 },
             } orelse break;
@@ -1343,6 +1720,8 @@ pub const Manager = struct {
             try doomed.append(self.gpa, try self.gpa.dupe(u8, entry.name));
         }
 
+        self.noteStorageSuccess(io, .directory_read);
+
         for (doomed.items) |name| {
             self.deleteQuietly(io, dir, name);
             log.info("pruned orphaned blocklist file {s}", .{name});
@@ -1381,8 +1760,21 @@ pub const Manager = struct {
     }
 
     /// Publishes a built table and frees the one it replaces. The caller holds
-    /// the exclusive lock, so no reader is inside the old table.
+    /// the exclusive lock, so no reader is inside the old table and no flush is
+    /// half way through draining it.
+    ///
+    /// The pass accounting the live table still holds is folded into the
+    /// incoming entry of the same id first. `mergeStatuses` left the candidate
+    /// carrying none, so an outcome recorded after the candidate was built —
+    /// and any a flush has not drained yet — survives the swap exactly once. An
+    /// outcome a flush already reported is zero in the live table, so nothing
+    /// here resurrects it.
     fn installStatuses(self: *Manager, table: StatusTable) void {
+        for (table.items) |*incoming| {
+            const live = entryFor(self.statuses, incoming.id) orelse continue;
+            incoming.pass_failures +|= live.pass_failures;
+            incoming.pass_outcome = incoming.pass_outcome or live.pass_outcome;
+        }
         self.status_arena.deinit();
         self.status_arena = table.arena;
         self.statuses = table.items;
@@ -1449,29 +1841,46 @@ pub const Manager = struct {
             error.Canceled => return error.Canceled,
             else => {
                 log.warn("creating {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
+                self.noteStorageFailure(io, .create_dir, "creating {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
                 return error.FileSystem;
             },
         };
-        return self.paths.dir.openDir(io, self.paths.subdir, options) catch |err| switch (err) {
+        self.noteStorageSuccess(io, .create_dir);
+
+        const dir = self.paths.dir.openDir(io, self.paths.subdir, options) catch |err| switch (err) {
             error.Canceled => return error.Canceled,
             else => {
                 log.warn("opening {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
+                self.noteStorageFailure(io, .open_dir, "opening {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
                 return error.FileSystem;
             },
         };
+        self.noteStorageSuccess(io, .open_dir);
+        return dir;
     }
 
     /// A temporary that cannot be removed is not a failure of the operation
     /// that made it, but it is not nothing either: it is left visible.
     fn deleteQuietly(self: *Manager, io: std.Io, dir: std.Io.Dir, name: []const u8) void {
-        _ = self;
-        dir.deleteFile(io, name) catch |err| switch (err) {
-            error.FileNotFound => {},
-            else => log.warn("deleting {s} failed: {s}", .{ name, @errorName(err) }),
-        };
+        if (dir.deleteFile(io, name)) {
+            self.noteStorageSuccess(io, .delete);
+        } else |err| switch (err) {
+            // A name that was never created is the ordinary case: the temporary
+            // deletes are installed before the files exist.
+            error.FileNotFound => self.noteStorageSuccess(io, .delete),
+            else => {
+                log.warn("deleting {s} failed: {s}", .{ name, @errorName(err) });
+                self.noteStorageFailure(io, .delete, "deleting {s} failed: {s}", .{ name, @errorName(err) });
+            },
+        }
     }
 };
 
+/// The one subject `blocklist.snapshot` ever has: a box publishes exactly one
+/// snapshot, and every source that failed to load is its own
+/// `blocklist.refresh` episode.
+const snapshot_key = "snapshot";
+
 /// A status table and the arena holding it. Until `installStatuses` takes it,
 /// it is a candidate nobody can see, and `deinit` frees it whole.
 const StatusTable = struct {
@@ -1488,6 +1897,14 @@ const StatusTable = struct {
 /// over from `previous`. A source deleted since `previous` was built is gone; a
 /// source added since starts blank. `previous` is only read, so the caller's
 /// published table is untouched by this.
+///
+/// The pass accounting is *not* carried: it lives in exactly one table copy at
+/// a time. `previous` is a snapshot of the published table taken outside the
+/// swap, so copying its counters here would leave the same outcomes in two
+/// tables — the live one for a flush to drain, and this candidate for the
+/// reload's own flush to report a second time. A candidate holds only what
+/// `applyLoadOutcomes` writes into it; what the live table holds is folded in
+/// by `installStatuses` under the exclusive lock.
 fn mergeStatuses(
     table: []SourceStatus,
     rows: []const sources_repo.SourceRow,
@@ -1500,6 +1917,8 @@ fn mergeStatuses(
             status.* = prior;
             break;
         }
+        status.pass_outcome = false;
+        status.pass_failures = 0;
         // After the carry-over: a url edited on the row wins over the one the
         // prior entry recorded.
         status.setUrl(row.url);
@@ -1716,6 +2135,7 @@ fn containsId(rows: []const sources_repo.SourceRow, id: i64) bool {
 // real swaps under load are the integration suite's (S9).
 
 const testing = std.testing;
+const events_fixture = @import("../storage/events_fixture.zig");
 const migrations = @import("../storage/migrations.zig");
 
 fn openMigrated() !db.Db {
@@ -2275,6 +2695,13 @@ test "a candidate table carries prior entries over and leaves the published one
     try testing.expectEqual(State.never_fetched, candidate[1].state);
     try testing.expect(!candidate[1].loaded);
 
+    // The one thing a candidate does not carry. `published[0]` is holding a
+    // failure no flush has drained yet; copying its accounting here would leave
+    // the same outcome in two tables, and the flush of each would report it.
+    try testing.expect(published[0].pass_outcome);
+    try testing.expect(!candidate[0].pass_outcome);
+    try testing.expectEqual(@as(u16, 0), candidate[0].pass_failures);
+
     // The published table is untouched, so a reload that fails before the swap
     // leaves it describing the snapshot that is still serving — including the
     // entry of the deleted source, which that snapshot still enforces.
@@ -2284,6 +2711,181 @@ test "a candidate table carries prior entries over and leaves the published one
     try testing.expectEqualStrings("https://lists.example/one.txt", published[0].urlText());
 }
 
+test "installing a table folds the live pass accounting in by id" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openMigrated();
+    defer database.close();
+
+    var f: fetcher.Fetcher = undefined;
+    var mgr = try testManager(&database, &f);
+    defer mgr.deinit(io);
+
+    // Source 1 recorded a failure the flush has not drained; source 2 was
+    // drained already; source 3 recorded one this reload knows nothing about.
+    var live: std.heap.ArenaAllocator = .init(testing.allocator);
+    const live_items = try live.allocator().alloc(SourceStatus, 3);
+    live_items[0] = .{ .id = 1, .pass_outcome = true, .pass_failures = 2 };
+    live_items[1] = .{ .id = 2 };
+    live_items[2] = .{ .id = 3, .pass_outcome = true, .pass_failures = 1 };
+    {
+        mgr.lock.lockUncancelable(io);
+        defer mgr.lock.unlock(io);
+        mgr.installStatuses(.{ .arena = live, .items = live_items });
+    }
+
+    // What a reload built beside it, carrying only its own load outcomes.
+    var incoming: std.heap.ArenaAllocator = .init(testing.allocator);
+    const incoming_items = try incoming.allocator().alloc(SourceStatus, 3);
+    incoming_items[0] = .{ .id = 1, .pass_outcome = true, .pass_failures = 1 };
+    incoming_items[1] = .{ .id = 2, .pass_outcome = true, .pass_failures = 4 };
+    incoming_items[2] = .{ .id = 3 };
+    {
+        mgr.lock.lockUncancelable(io);
+        defer mgr.lock.unlock(io);
+        mgr.installStatuses(.{ .arena = incoming, .items = incoming_items });
+    }
+
+    try testing.expectEqual(@as(u16, 3), mgr.statuses[0].pass_failures);
+    try testing.expect(mgr.statuses[0].pass_outcome);
+
+    // A drained entry adds nothing: what the swap publishes is the reload's own
+    // accounting and no resurrection of what was already reported.
+    try testing.expectEqual(@as(u16, 4), mgr.statuses[1].pass_failures);
+    try testing.expect(mgr.statuses[1].pass_outcome);
+
+    // The half the swap used to lose: an outcome the live table held and the
+    // candidate never saw.
+    try testing.expectEqual(@as(u16, 1), mgr.statuses[2].pass_failures);
+    try testing.expect(mgr.statuses[2].pass_outcome);
+}
+
+test "the flush claims every entry that carries pass accounting, once" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openMigrated();
+    defer database.close();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1_700_000_000);
+    defer fx.deinit();
+
+    var f: fetcher.Fetcher = undefined;
+    var mgr = try testManager(&database, &f);
+    defer mgr.deinit(io);
+    mgr.diagnostics = &fx.store;
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    const items = try arena.allocator().alloc(SourceStatus, 3);
+    items[0] = .{ .id = 1 };
+    items[0].setUrl("https://lists.example/one.txt");
+    items[0].fail(.fetch_failed, "HttpStatus");
+    // No url, so no episode to report under. The drain has to claim it anyway:
+    // an entry left with `pass_outcome` set is the one every later scan finds
+    // first, and the entry behind it would never be reached.
+    items[1] = .{ .id = 2, .pass_outcome = true, .pass_failures = 1 };
+    items[2] = .{ .id = 3 };
+    items[2].setUrl("https://lists.example/three.txt");
+    items[2].succeed(1_700_000_000, .{ .domains = 3 });
+    {
+        mgr.lock.lockUncancelable(io);
+        defer mgr.lock.unlock(io);
+        mgr.installStatuses(.{ .arena = arena, .items = items });
+    }
+
+    mgr.flushDiagnostics(io);
+
+    try testing.expectEqual(@as(i64, 1), try fx.count(
+        "SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh'",
+    ));
+    try testing.expectEqual(@as(i64, 1), try fx.count(
+        "SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
+    ));
+    for (mgr.statuses) |entry| {
+        try testing.expect(!entry.pass_outcome);
+        try testing.expectEqual(@as(u16, 0), entry.pass_failures);
+    }
+
+    // Drained: flushing the same table again reports nothing a second time.
+    mgr.flushDiagnostics(io);
+    try testing.expectEqual(@as(i64, 1), try fx.count(
+        "SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
+    ));
+}
+
+test "the flush closes the episode of a source that is no longer in the table" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openMigrated();
+    defer database.close();
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1_700_000_000);
+    defer fx.deinit();
+
+    var f: fetcher.Fetcher = undefined;
+    var mgr = try testManager(&database, &f);
+    defer mgr.deinit(io);
+    mgr.diagnostics = &fx.store;
+
+    // What the operator deleted while it was failing. Nothing will ever record
+    // a success for it, so nothing but the sweep can close this.
+    fx.store.report(
+        io,
+        1_700_000_000,
+        .blocklist_refresh,
+        "https://lists.example/deleted.txt",
+        "lists.example/deleted.txt",
+        .warning,
+        "fetch_failed: HttpStatus (1 this pass)",
+    );
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    const items = try arena.allocator().alloc(SourceStatus, 1);
+    items[0] = .{ .id = 1 };
+    items[0].setUrl("https://lists.example/one.txt");
+    items[0].fail(.fetch_failed, "HttpStatus");
+    {
+        mgr.lock.lockUncancelable(io);
+        defer mgr.lock.unlock(io);
+        mgr.installStatuses(.{ .arena = arena, .items = items });
+    }
+
+    // No snapshot published yet, so the table is not known to describe the
+    // source set and the sweep must not run: the drain reports the failing
+    // source and the deleted one's episode is left alone.
+    mgr.flushDiagnostics(io);
+    try testing.expectEqual(@as(i64, 2), try fx.count(
+        "SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL",
+    ));
+
+    mgr.generation = 1;
+    mgr.flushDiagnostics(io);
+
+    // One left active, and it is the source that still exists.
+    try testing.expectEqual(@as(i64, 1), try fx.count(
+        "SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL",
+    ));
+    try testing.expectEqualStrings(
+        "https://lists.example/one.txt",
+        try fx.text(
+            "SELECT subject_key FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL",
+        ),
+    );
+    try testing.expectEqualStrings(
+        "https://lists.example/deleted.txt",
+        try fx.text(
+            "SELECT subject_key FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NOT NULL",
+        ),
+    );
+}
+
 test "a disabled source stops being loaded" {
     var statuses = [_]SourceStatus{.{ .id = 1 }};
     statuses[0].succeed(1_700_000_000, .{ .domains = 9 });