milestone 16: behavioral fixes for silent failures, locks, counters and the query log

src/filter/filter_integration_test.zig

@@ -27,6 +27,7 @@ const migrations = @import("../storage/migrations.zig");
 const context = @import("../storage/repositories/context.zig");
 const groups_repo = @import("../storage/repositories/groups_repo.zig");
 const local_repo = @import("../storage/repositories/local_repo.zig");
+const rules_repo = @import("../storage/repositories/rules_repo.zig");
 const sources_repo = @import("../storage/repositories/sources_repo.zig");
 
 const compiler = @import("compiler.zig");
@@ -351,7 +352,15 @@ const Env = struct {
 // fixtures: the loopback http server
 // ---------------------------------------------------------------------------
 
-const Route = enum(u8) { body, redirect, not_found, oversize, chunked };
+const Route = enum(u8) { body, redirect, not_found, oversize, chunked, stall };
+
+/// How long the `stall` route holds a reply open when nothing releases it.
+///
+/// The cases that use it prove a writer ran *beside* a parked download rather
+/// than behind it, so nothing depends on this number being large. It exists so
+/// that a regression fails the run in a second or two instead of hanging it,
+/// which is what an unbounded stall against a single lock would do.
+const stall_budget: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(1_500), .clock = .awake };
 
 const redirect_path = "/redirected.txt";
 
@@ -396,6 +405,16 @@ const HttpFixture = struct {
     /// reports a buffered part as written and would otherwise hide a fixture
     /// that sent everything in one go.
     flushed_parts: std.atomic.Value(u32),
+    /// Set by the `stall` route once the reply is open and the body has
+    /// stopped. A test that waits for this knows the refresh is inside its
+    /// download and not on its way there.
+    stall_reached: std.Io.Event,
+    /// Set by the test to let the `stall` route finish its reply.
+    stall_release: std.Io.Event,
+    /// Whether the `stall` route has stopped waiting — by release or by
+    /// `stall_budget` expiring. A writer that returns while this is still false
+    /// returned with the download unfinished, which is the whole claim.
+    stall_resumed: std.atomic.Value(bool),
 
     fn init(io: std.Io, body: []const u8) !HttpFixture {
         const local: net.IpAddress = try .parse("127.0.0.1", 0);
@@ -404,6 +423,9 @@ const HttpFixture = struct {
             .body = body,
             .route = .init(@intFromEnum(Route.body)),
             .flushed_parts = .init(0),
+            .stall_reached = .unset,
+            .stall_release = .unset,
+            .stall_resumed = .init(false),
         };
     }
 
@@ -435,14 +457,14 @@ const HttpFixture = struct {
             var http: std.http.Server = .init(&reader.interface, &writer.interface);
 
             var request = http.receiveHead() catch continue;
-            self.respond(&request) catch continue;
+            self.respond(io, &request) catch continue;
         }
     }
 
     /// Every reply closes the connection. A keep-alive reply would leave the
     /// fetcher holding the connection open while this server waits to accept a
     /// second one that never comes (milestone-5 spec, S9 note from S8).
-    fn respond(self: *HttpFixture, request: *std.http.Server.Request) !void {
+    fn respond(self: *HttpFixture, io: std.Io, request: *std.http.Server.Request) !void {
         switch (@as(Route, @enumFromInt(self.route.load(.acquire)))) {
             .body => try request.respond(self.body, .{ .keep_alive = false }),
             .redirect => if (std.mem.eql(u8, request.head.target, redirect_path))
@@ -463,9 +485,35 @@ const HttpFixture = struct {
                 .extra_headers = &.{.{ .name = "content-length", .value = oversize_length }},
             }),
             .chunked => try self.respondChunked(request),
+            .stall => try self.respondStalled(io, request),
         }
     }
 
+    /// Opens the reply, sends the first third of the body and then stops until
+    /// `stall_release` is set or `stall_budget` runs out. The fetcher is inside
+    /// its body pump for the whole of that, which is what a slow upstream looks
+    /// like to a refresh pass — and what the refresh used to hold the writer
+    /// lock across.
+    fn respondStalled(self: *HttpFixture, io: std.Io, request: *std.http.Server.Request) !void {
+        var send_buf: [4096]u8 = undefined;
+        var stream = try request.respondStreaming(&send_buf, .{
+            .respond_options = .{ .keep_alive = false },
+        });
+        const parts = thirds(self.body);
+        try stream.writer.writeAll(parts[0]);
+        try stream.flush();
+
+        self.stall_reached.set(io);
+        // A timeout and a cancellation are both "stop waiting": the writes
+        // below then either finish the reply or fail into `serve`'s `continue`.
+        self.stall_release.waitTimeout(io, .{ .duration = stall_budget }) catch {};
+        self.stall_resumed.store(true, .release);
+
+        try stream.writer.writeAll(parts[1]);
+        try stream.writer.writeAll(parts[2]);
+        try stream.end();
+    }
+
     /// Streams the body in three flushed parts instead of one `respond`
     /// (milestone-15 ruling 7). Every other arm sends ~130 bytes in a single
     /// write, which the fetcher consumes in one read: the loop that 35f2324
@@ -1167,6 +1215,163 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
     try testing.expect(decision.blocked);
 }
 
+// ---------------------------------------------------------------------------
+// 10c–10d: a refresh in flight against the writers beside it
+// ---------------------------------------------------------------------------
+
+const RefreshAllTask = struct {
+    mgr: *manager.Manager,
+    /// Read by the test to tell "the pass is still downloading" from "the pass
+    /// has finished". Atomic because the two run on different tasks.
+    done: std.atomic.Value(bool) = .init(false),
+    result: manager.Manager.Error!void = {},
+
+    fn run(self: *RefreshAllTask, io: std.Io) void {
+        self.result = self.mgr.refreshAll(io);
+        self.done.store(true, .release);
+    }
+};
+
+const RefreshSourceTask = struct {
+    mgr: *manager.Manager,
+    row: sources_repo.SourceRow,
+    result: manager.Manager.Error!bool = false,
+
+    fn run(self: *RefreshSourceTask, io: std.Io) void {
+        self.result = self.mgr.refreshSource(io, self.row);
+    }
+};
+
+/// One block rule in the `default` group, written the way the web handler
+/// writes it.
+fn insertBlockRule(database: *db.Db, pattern: []const u8) !void {
+    const group_id = (try groups_repo.groupId(database, "default")) orelse
+        return error.TestGroupMissing;
+    var group_ids: context.IdMap = .empty;
+    defer group_ids.deinit(testing.allocator);
+    try group_ids.put(testing.allocator, "default", group_id);
+
+    try rules_repo.insertRule(database, .{
+        .group = "default",
+        .pattern = pattern,
+        .kind = .exact,
+        .action = .block,
+    }, .{ .group_ids = &group_ids });
+}
+
+test "10c: a rule save completes while a refresh is parked in its download" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    const env = try Env.create(gpa);
+    defer env.destroy();
+    const io = env.io();
+
+    var fixture = try HttpFixture.init(io, http_body);
+    defer fixture.deinit(io);
+    fixture.setRoute(.stall);
+    var group: std.Io.Group = .init;
+    defer group.cancel(io);
+    try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
+
+    var url_buf: [64]u8 = undefined;
+    const url = try fixture.url(&url_buf);
+    const id = try seedSource(&env.database, url);
+    try env.mgr.reload(io);
+
+    var task: RefreshAllTask = .{ .mgr = &env.mgr };
+    var tasks: std.Io.Group = .init;
+    defer tasks.cancel(io);
+    try tasks.concurrent(io, RefreshAllTask.run, .{ &task, io });
+
+    // The pass is now inside the download, holding the refresh lock and
+    // nothing else.
+    try fixture.stall_reached.wait(io);
+
+    // What the web does on every rule save: write the row, then reload. Behind
+    // one lock this waits out the download — on a request path with no timeout,
+    // occupying one of 64 web slots.
+    try insertBlockRule(&env.database, "blocked.example.com");
+    try env.mgr.reload(io);
+
+    // Not a duration: the reply is still open, so the download this reload
+    // returned across has not finished and cannot have been waited out. Behind
+    // one lock the reload would have returned only after `stall_budget` gave
+    // the reply up.
+    const ran_beside = !fixture.stall_resumed.load(.acquire) and !task.done.load(.acquire);
+    fixture.stall_release.set(io);
+
+    try tasks.await(io);
+    try task.result;
+    try testing.expect(ran_beside);
+
+    // Both writers landed: the rule is enforced and the refresh finished the
+    // download the reload ran across.
+    const decision, _ = try env.evaluate("blocked.example.com");
+    try testing.expect(decision.blocked);
+    try testing.expectEqual(matcher.Reason.rule_block_exact, decision.reason);
+    try testing.expectEqual(manager.State.ok, (try env.status(id)).state);
+}
+
+test "10d: a source deleted mid-refresh does not take the refresh's temporary files" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    const env = try Env.create(gpa);
+    defer env.destroy();
+    const io = env.io();
+
+    var fixture = try HttpFixture.init(io, http_body);
+    defer fixture.deinit(io);
+    fixture.setRoute(.stall);
+    var group: std.Io.Group = .init;
+    defer group.cancel(io);
+    try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
+
+    var url_buf: [64]u8 = undefined;
+    const url = try fixture.url(&url_buf);
+    const id = try seedSource(&env.database, url);
+    try env.mgr.reload(io);
+
+    // A `SourceRow` borrows the strings this list owns, and the refresh task
+    // reads them for as long as it runs. The list outlives the task.
+    var rows = try listRows(&env.database);
+    defer rows.deinit();
+    var task: RefreshSourceTask = .{ .mgr = &env.mgr, .row = try rows.byUrl(url) };
+
+    var tasks: std.Io.Group = .init;
+    defer tasks.cancel(io);
+    try tasks.concurrent(io, RefreshSourceTask.run, .{ &task, io });
+
+    try fixture.stall_reached.wait(io);
+
+    var dir = try env.blocklistDir();
+    defer dir.close(io);
+    var raw_buf: [64]u8 = undefined;
+    const raw_name = try std.fmt.bufPrint(&raw_buf, "{d}.raw.tmp", .{id});
+    try dir.access(io, raw_name, .{});
+
+    // The `DELETE /api/blocklists/{id}` sequence: drop the row, then sweep the
+    // directory. The row is gone, so nothing in the sweep itself would spare
+    // this source's raw download — only the refresh lock does, by making the
+    // sweep wait for the pass instead of running across it.
+    try sources_repo.deleteSource(&env.database, id);
+    try env.mgr.pruneOrphans(io);
+
+    try tasks.await(io);
+    // True only if the compile found the raw file the download had written.
+    try testing.expect(try task.result);
+
+    // The sweep did run, once the refresh was out of the way: the compiled
+    // files of a source with no row are gone, and so is every temporary.
+    var it = dir.iterate();
+    while (try it.next(io)) |entry| {
+        try testing.expect(!std.mem.endsWith(u8, entry.name, ".tmp"));
+        try testing.expect(!std.mem.endsWith(u8, entry.name, ".list"));
+        try testing.expect(!std.mem.endsWith(u8, entry.name, ".wild"));
+    }
+}
+
 // ---------------------------------------------------------------------------
 // 11–12: local records, from the database to the wire
 // ---------------------------------------------------------------------------

src/filter/manager.zig

@@ -24,16 +24,28 @@
 //! socket and no file, so the uncancelable lock forms are used: a lock this
 //! code takes is always released within a few instructions.
 //!
-//! A second lock, `writer_lock`, serializes the writers against each other:
-//! `reload`, `refreshSource`, `refreshAll`, the startup pass and
-//! `pruneOrphans`. Two concurrent reloads would otherwise compute the same
-//! generation and each destroy a snapshot the other had just published, and two
-//! concurrent refreshes would share the fetcher's buffers and, for one source,
-//! the same `.raw.tmp` / `.list.tmp` / `.wild.tmp` paths. It is held across
-//! downloads and compiles, so it is a plain mutex rather than the RCU lock:
-//! readers must never wait behind a refresh. The public entry points take it;
-//! the `*Locked` bodies assume it and never take it again, because it is not
-//! reentrant.
+//! Two more locks serialize the writers, and they divide the work by how long
+//! it takes.
+//!
+//! `writer_lock` covers what a writer does to the *published* state: build a
+//! snapshot, install the compiled files, write the runtime columns, record a
+//! status. Two concurrent reloads would otherwise compute the same generation
+//! and each destroy a snapshot the other had just published. Every section it
+//! guards is bounded by local work — a read of the compiled files at worst —
+//! so a web mutation that ends in `reload` never waits out a download.
+//!
+//! `refresh_lock` covers what a writer does *before* it has anything to
+//! publish: the download of one source, at up to 300 s each, and the compile
+//! that follows it. It also covers blocklist-directory maintenance, because
+//! those stages are the only writers of `.raw.tmp` / `.list.tmp` /
+//! `.wild.tmp` and `pruneOrphans` must not sweep the temporaries of a refresh
+//! that is still running. Two concurrent refreshes would share the fetcher's
+//! buffers and, for one source, the same temporary paths.
+//!
+//! **Lock ordering: `refresh_lock` is never acquired while `writer_lock` is
+//! held.** A path that needs both takes `refresh_lock` first. The public entry
+//! points take what they need; the `*Locked` bodies assume it and never take
+//! it again, because neither mutex is reentrant.
 
 const std = @import("std");
 const Allocator = std.mem.Allocator;
@@ -251,8 +263,21 @@ pub const Manager = struct {
     total_budget: std.Io.Clock.Duration,
 
     lock: std.Io.RwLock,
-    /// Serializes the writers against each other. Never taken by a reader.
+    /// Serializes everything that changes the published state — the snapshot,
+    /// the compiled files, the runtime columns, the status table — against
+    /// every other writer. Never taken by a reader.
+    ///
+    /// Lock ordering: `refresh_lock` is never acquired while this is held. A
+    /// caller that needs both takes `refresh_lock` first.
     writer_lock: std.Io.Mutex,
+    /// Serializes refresh passes against each other, and against the
+    /// blocklist-directory maintenance in `pruneOrphans`. Held across a
+    /// download and a compile, which `writer_lock` deliberately is not, so an
+    /// unrelated `reload` never waits out a 300-second fetch.
+    ///
+    /// Lock ordering: this is taken first, and never while `writer_lock` is
+    /// held.
+    refresh_lock: std.Io.Mutex,
     current: ?*matcher.Snapshot,
     generation: u64,
     statuses: []SourceStatus,
@@ -301,6 +326,7 @@ pub const Manager = struct {
             .total_budget = total_budget,
             .lock = .init,
             .writer_lock = .init,
+            .refresh_lock = .init,
             .current = null,
             .generation = 0,
             .statuses = &.{},
@@ -538,14 +564,21 @@ pub const Manager = struct {
         // failing append: `bodies` owns each one from the moment it is read.
         try bodies.ensureUnusedCapacity(self.gpa, 2);
 
+        // `error.Canceled` is the one-shot signal that this task is being torn
+        // down, and it is consumed by whoever catches it. Recording it as a
+        // load failure would spend it on a status row that reads "Canceled",
+        // publish a snapshot with this source missing, and let the shutdown
+        // carry on as if nothing had asked it to stop.
         const list_bytes = dir.readFileAlloc(io, list_name, self.gpa, .limited(max_compiled_bytes)) catch |err| {
             if (err == error.OutOfMemory) return error.OutOfMemory;
+            if (err == error.Canceled) return error.Canceled;
             return loadFailure(row, list_name, err);
         };
         bodies.appendAssumeCapacity(list_bytes);
 
         const wild_bytes = dir.readFileAlloc(io, wild_name, self.gpa, .limited(max_compiled_bytes)) catch |err| {
             if (err == error.OutOfMemory) return error.OutOfMemory;
+            if (err == error.Canceled) return error.Canceled;
             return loadFailure(row, wild_name, err);
         };
         bodies.appendAssumeCapacity(wild_bytes);
@@ -580,11 +613,13 @@ pub const Manager = struct {
     /// before it refreshes anything, which is why it is the entry point Phase 8
     /// and the scheduler use.
     pub fn refreshSource(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool {
-        self.writer_lock.lockUncancelable(io);
-        defer self.writer_lock.unlock(io);
+        self.refresh_lock.lockUncancelable(io);
+        defer self.refresh_lock.unlock(io);
         return self.refreshSourceLocked(io, row);
     }
 
+    /// Assumes `refresh_lock`. Takes `writer_lock` itself, for the publish half
+    /// alone.
     fn refreshSourceLocked(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool {
         // The previous entry describes the compiled files that are still on
         // disk, and a failed refresh leaves them serving. Starting from a blank
@@ -594,41 +629,6 @@ pub const Manager = struct {
         status.setUrl(row.url);
         status.last_attempt = std.Io.Clock.real.now(io).toSeconds();
 
-        const replaced = try self.refreshOne(io, row, &status);
-        self.commitStatus(io, status);
-        return replaced;
-    }
-
-    /// Every enabled source, one at a time, then one `reload`. A failing source
-    /// never stops the pass: it would hide every source behind it.
-    ///
-    /// This returns an error only when nothing could be done at all — out of
-    /// memory, an unreachable database, an unusable blocklist directory. A
-    /// source that failed to download or compile is a successful pass with a
-    /// non-`ok` status.
-    pub fn refreshAll(self: *Manager, io: std.Io) Error!void {
-        self.writer_lock.lockUncancelable(io);
-        defer self.writer_lock.unlock(io);
-
-        var rows = try sources_repo.listSourceRows(self.database, self.gpa);
-        defer rows.deinit(self.gpa);
-        defer sources_repo.freeSourceRows(self.gpa, rows.items);
-
-        try self.syncStatuses(io, rows.items);
-
-        for (rows.items) |row| {
-            if (!row.enabled) continue;
-            _ = try self.refreshSourceLocked(io, row);
-        }
-        return self.reloadLocked(io);
-    }
-
-    fn refreshOne(
-        self: *Manager,
-        io: std.Io,
-        row: sources_repo.SourceRow,
-        status: *SourceStatus,
-    ) Error!bool {
         var dir = try self.openDir(io, .{});
         defer dir.close(io);
 
@@ -647,12 +647,80 @@ pub const Manager = struct {
         defer self.deleteQuietly(io, dir, list_tmp);
         defer self.deleteQuietly(io, dir, wild_tmp);
 
+        // The half that takes the time: one download of up to `total_budget`
+        // and one compile of everything it returned. `refresh_lock` alone is
+        // held here, so a rule save, a settings change or any other web
+        // mutation that ends in `reload` runs beside it instead of behind it.
+        const prepared = try self.prepareRefresh(io, dir, row, &status, raw_name, list_tmp, wild_tmp);
+
+        // The half that publishes. The compiled files, the runtime columns and
+        // the status entry land under one `writer_lock`, so a reload never
+        // reads new files beside a status entry describing the previous ones.
+        self.writer_lock.lockUncancelable(io);
+        defer self.writer_lock.unlock(io);
+
+        const replaced = try self.publishRefresh(io, dir, row, &status, prepared, list_tmp, wild_tmp);
+        self.commitStatus(io, status);
+        return replaced;
+    }
+
+    /// Every enabled source, one at a time, then one `reload`. A failing source
+    /// never stops the pass: it would hide every source behind it.
+    ///
+    /// This returns an error only when nothing could be done at all — out of
+    /// memory, an unreachable database, an unusable blocklist directory. A
+    /// source that failed to download or compile is a successful pass with a
+    /// non-`ok` status.
+    pub fn refreshAll(self: *Manager, io: std.Io) Error!void {
+        self.refresh_lock.lockUncancelable(io);
+        defer self.refresh_lock.unlock(io);
+
+        var rows = try sources_repo.listSourceRows(self.database, self.gpa);
+        defer rows.deinit(self.gpa);
+        defer sources_repo.freeSourceRows(self.gpa, rows.items);
+
+        try self.syncStatuses(io, rows.items);
+
+        for (rows.items) |row| {
+            if (!row.enabled) continue;
+            _ = try self.refreshSourceLocked(io, row);
+        }
+        // `reload` takes `writer_lock`, which the pass has been careful not to
+        // hold: the order is `refresh_lock` first, always.
+        return self.reload(io);
+    }
+
+    /// What the fetch-and-compile half of a refresh produced. `.failed` needs
+    /// no publish and has already recorded why in the status entry.
+    const Prepared = union(enum) {
+        failed,
+        compiled: struct {
+            format: parsers.Format,
+            result: compiler.Result,
+        },
+    };
+
+    /// Downloads one source and compiles it into the temporary files.
+    ///
+    /// Assumes `refresh_lock` and must not be called with `writer_lock` held:
+    /// this is the part that takes seconds, and nothing here touches the
+    /// published state.
+    fn prepareRefresh(
+        self: *Manager,
+        io: std.Io,
+        dir: std.Io.Dir,
+        row: sources_repo.SourceRow,
+        status: *SourceStatus,
+        raw_name: []const u8,
+        list_tmp: []const u8,
+        wild_tmp: []const u8,
+    ) Error!Prepared {
         self.download(io, dir, raw_name, row) catch |err| switch (err) {
             error.OutOfMemory => return error.OutOfMemory,
             error.Canceled => return error.Canceled,
             else => {
                 self.reportFetchFailure(row, status, err);
-                return false;
+                return .failed;
             },
         };
 
@@ -661,7 +729,7 @@ pub const Manager = struct {
             error.Canceled => return error.Canceled,
             else => {
                 self.reportCompileFailure(row, status, err);
-                return false;
+                return .failed;
             },
         };
 
@@ -670,15 +738,38 @@ pub const Manager = struct {
             error.Canceled => return error.Canceled,
             else => {
                 self.reportCompileFailure(row, status, err);
-                return false;
+                return .failed;
             },
         };
 
         if (rejectedWithoutEntries(result.counts)) {
             self.reportEmptyCompile(row, status, result.counts);
-            return false;
+            return .failed;
         }
 
+        return .{ .compiled = .{ .format = format, .result = result } };
+    }
+
+    /// Installs what `prepareRefresh` produced and writes the runtime columns.
+    /// Returns `true` only when the compiled files were replaced.
+    ///
+    /// Assumes `writer_lock`. Reading the files on disk belongs here and not in
+    /// the half above: they are published under this lock.
+    fn publishRefresh(
+        self: *Manager,
+        io: std.Io,
+        dir: std.Io.Dir,
+        row: sources_repo.SourceRow,
+        status: *SourceStatus,
+        prepared: Prepared,
+        list_tmp: []const u8,
+        wild_tmp: []const u8,
+    ) Error!bool {
+        const compiled = switch (prepared) {
+            .failed => return false,
+            .compiled => |value| value,
+        };
+
         const now = std.Io.Clock.real.now(io).toSeconds();
 
         // Recompiling identical content into new files would invalidate the
@@ -688,7 +779,9 @@ pub const Manager = struct {
         // them, and skipping the rewrite here would leave filtering off for
         // good.
         if (row.checksum) |stored| {
-            if (std.mem.eql(u8, stored, &result.checksum) and self.diskBodiesMatch(io, dir, row.id, stored)) {
+            if (std.mem.eql(u8, stored, &compiled.result.checksum) and
+                self.diskBodiesMatch(io, dir, row.id, stored))
+            {
                 try sources_repo.updateSourceStats(self.database, row.id, .{
                     .last_updated = now,
                     .domain_count = row.domain_count,
@@ -696,17 +789,17 @@ pub const Manager = struct {
                     .skipped_regex_count = row.skipped_regex_count,
                     .checksum = stored,
                 });
-                status.succeed(now, result.counts);
+                status.succeed(now, compiled.result.counts);
                 return false;
             }
         }
 
         const header: Header = .{
             .url = row.url,
-            .format = format,
+            .format = compiled.format,
             .fetched_at = now,
-            .counts = result.counts,
-            .checksum = &result.checksum,
+            .counts = compiled.result.counts,
+            .checksum = &compiled.result.checksum,
         };
         self.publish(io, dir, row.id, header, list_tmp, wild_tmp) catch |err| switch (err) {
             error.OutOfMemory => return error.OutOfMemory,
@@ -719,12 +812,12 @@ pub const Manager = struct {
 
         try sources_repo.updateSourceStats(self.database, row.id, .{
             .last_updated = now,
-            .domain_count = result.counts.domains,
-            .wildcard_count = result.counts.wildcards,
-            .skipped_regex_count = result.counts.skipped_regex,
-            .checksum = &result.checksum,
+            .domain_count = compiled.result.counts.domains,
+            .wildcard_count = compiled.result.counts.wildcards,
+            .skipped_regex_count = compiled.result.counts.skipped_regex,
+            .checksum = &compiled.result.checksum,
         });
-        status.succeed(now, result.counts);
+        status.succeed(now, compiled.result.counts);
         return true;
     }
 
@@ -1034,8 +1127,8 @@ pub const Manager = struct {
     /// it — let alone the server. Cancellation is the one outcome that
     /// propagates, because it means shutdown.
     ///
-    /// Taken from outside every `*Locked` body: `pruneOrphans` takes
-    /// `writer_lock` itself and the mutex is not reentrant.
+    /// Taken from outside every `*Locked` body: `pruneOrphans` takes both
+    /// writer mutexes itself and neither is reentrant.
     fn sweepOrphans(self: *Manager, io: std.Io) std.Io.Cancelable!void {
         self.pruneOrphans(io) catch |err| switch (err) {
             error.Canceled => return error.Canceled,
@@ -1069,13 +1162,13 @@ pub const Manager = struct {
     }
 
     fn startupPass(self: *Manager, io: std.Io) Error!void {
-        self.writer_lock.lockUncancelable(io);
-        defer self.writer_lock.unlock(io);
+        self.refresh_lock.lockUncancelable(io);
+        defer self.refresh_lock.unlock(io);
 
         // Ahead of the gate on purpose: loading the compiled files that already
         // exist is a read. A full disk must not cost the household its
         // filtering as well as its downloads.
-        try self.reloadLocked(io);
+        try self.reload(io);
 
         if (self.refreshGated()) return;
 
@@ -1090,7 +1183,7 @@ pub const Manager = struct {
             if (!self.needsRefresh(io, row, now)) continue;
             if (try self.refreshSourceLocked(io, row)) refreshed = true;
         }
-        if (refreshed) try self.reloadLocked(io);
+        if (refreshed) try self.reload(io);
     }
 
     fn needsRefresh(self: *Manager, io: std.Io, row: sources_repo.SourceRow, now: i64) bool {
@@ -1128,10 +1221,19 @@ pub const Manager = struct {
     /// `<data_dir>/blocklists/` if nothing has yet, and an empty directory
     /// sweeps to nothing.
     pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
-        // Every path that writes a temporary holds this lock too, so the sweep
-        // never reads a directory a refresh is halfway through. The temporaries
-        // it can see therefore belong to a finished or a dead refresh, and only
-        // those of a source with no row are removed.
+        // `refresh_lock` first, and for the reason it exists: the download and
+        // the compile are the only writers of `.raw.tmp`, `.list.tmp` and
+        // `.wild.tmp`, and they hold it for as long as they run. Without it
+        // here, a source deleted through the API would sweep the temporaries of
+        // a refresh still writing them — the row is gone, so nothing else in
+        // this function would spare them — and the pass would fail on a raw
+        // file that vanished under it.
+        //
+        // `writer_lock` second, in the one order this file ever takes them,
+        // because the rows this reads and the compiled files it deletes are
+        // what a reload is reading.
+        self.refresh_lock.lockUncancelable(io);
+        defer self.refresh_lock.unlock(io);
         self.writer_lock.lockUncancelable(io);
         defer self.writer_lock.unlock(io);
 
@@ -1247,6 +1349,12 @@ pub const Manager = struct {
             entry.loaded = loaded;
             return;
         }
+        // No entry of that id: the source was added or deleted between the
+        // table this pass started from and this commit. The outcome is lost
+        // either way — the next reload rebuilds the table from the rows — but
+        // a failure that disappears without a line is the one thing milestone
+        // 5 says never happens.
+        log.warn("blocklist source {d}: no status entry to record the refresh outcome in", .{status.id});
     }
 
     /// One id per group, in `listGroups` order.
@@ -1485,10 +1593,10 @@ const source_file_suffixes = [_][]const u8{ ".list.tmp", ".wild.tmp", ".raw.tmp"
 /// The three temporaries count. A refresh that dies between writing one and
 /// renaming it leaves a file no later refresh reuses and no `defer` reaches, so
 /// excluding them from the sweep means nothing ever removes them. Matching them
-/// is safe because `pruneOrphans` holds `writer_lock` for its whole body: every
-/// path that creates a temporary runs under that same lock, so no refresh is in
-/// flight while the sweep reads the directory, and a temporary the sweep sees
-/// belonging to a source that still has a row is kept regardless.
+/// is safe because `pruneOrphans` holds `refresh_lock` for its whole body:
+/// every path that creates a temporary runs under that same lock, so no refresh
+/// is in flight while the sweep reads the directory, and a temporary the sweep
+/// sees belonging to a source that still has a row is kept regardless.
 fn sourceFileId(file_name: []const u8) ?i64 {
     for (source_file_suffixes) |suffix| {
         if (!std.mem.endsWith(u8, file_name, suffix)) continue;
@@ -1509,8 +1617,9 @@ fn containsId(rows: []const sources_repo.SourceRow, id: i64) bool {
 // tests
 // ---------------------------------------------------------------------------
 //
-// Everything here runs against a `:memory:` database and touches no file. Real
-// files, real HTTP and real swaps are the integration suite's (S9).
+// Everything here runs against a `:memory:` database. Only the cancellation
+// cases below reach a file, and they reach a `testing.tmpDir` — real HTTP and
+// real swaps under load are the integration suite's (S9).
 
 const testing = std.testing;
 const migrations = @import("../storage/migrations.zig");
@@ -1527,8 +1636,9 @@ fn testManager(database: *db.Db, fetcher_ptr: *fetcher.Fetcher) !Manager {
     return Manager.init(
         testing.allocator,
         database,
-        // No test in this file reaches the filesystem: `acquire` answers before
-        // any directory is touched, and the header helpers are pure.
+        // `acquire` answers before any directory is touched and the header
+        // helpers are pure, so this directory is never opened unless a test
+        // replaces it with one of its own.
         .{ .dir = std.Io.Dir.cwd() },
         fetcher_ptr,
         .{},
@@ -1623,6 +1733,149 @@ test "statusSnapshot on an empty manager copies nothing" {
     try testing.expectEqual(@as(usize, 0), manager.statusSnapshot(io, &out));
 }
 
+// ---------------------------------------------------------------------------
+// a canceled read of a compiled file
+// ---------------------------------------------------------------------------
+
+/// The one file open a `cancelingIo` turns into `error.Canceled`, and the io it
+/// hands every other open to.
+///
+/// A `std.Io` carries its implementation's `userdata`, so a patched vtable entry
+/// cannot smuggle a receiver of its own through it and has to read its
+/// configuration from here. The test runner runs the tests of one binary in
+/// sequence, so one instance is enough.
+var canceling_read: struct {
+    inner: std.Io = undefined,
+    /// The file-name suffix whose open is canceled.
+    suffix: []const u8 = "",
+} = .{};
+
+/// `inner` with the open of every file whose name ends in `suffix` replaced by
+/// `error.Canceled`. `vtable` is the caller's storage for the patched copy and
+/// must outlive the returned io.
+fn cancelingIo(inner: std.Io, suffix: []const u8, vtable: *std.Io.VTable) std.Io {
+    canceling_read = .{ .inner = inner, .suffix = suffix };
+    vtable.* = inner.vtable.*;
+    vtable.dirOpenFile = cancelingOpenFile;
+    return .{ .userdata = inner.userdata, .vtable = vtable };
+}
+
+fn cancelingOpenFile(
+    userdata: ?*anyopaque,
+    dir: std.Io.Dir,
+    sub_path: []const u8,
+    options: std.Io.Dir.OpenFileOptions,
+) std.Io.File.OpenError!std.Io.File {
+    if (std.mem.endsWith(u8, sub_path, canceling_read.suffix)) return error.Canceled;
+    return canceling_read.inner.vtable.dirOpenFile(userdata, dir, sub_path, options);
+}
+
+test "a canceled compiled-file read cancels the reload instead of recording it" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var tmp = testing.tmpDir(.{});
+    defer tmp.cleanup();
+
+    var database = try openMigrated();
+    defer database.close();
+
+    var f: fetcher.Fetcher = undefined;
+    var mgr = try testManager(&database, &f);
+    defer mgr.deinit(io);
+    mgr.paths = .{ .dir = tmp.dir };
+
+    const url = "https://lists.example/hosts.txt";
+    try sources_repo.insertBlocklistSource(&database, .{ .url = url, .name = "example" }, .{});
+    var rows = try sources_repo.listSourceRows(&database, testing.allocator);
+    defer rows.deinit(testing.allocator);
+    defer sources_repo.freeSourceRows(testing.allocator, rows.items);
+    const id = rows.items[0].id;
+
+    const list_body = "aaa.example.com\n";
+    const wild_body = "";
+    var dir = try tmp.dir.createDirPathOpen(io, "blocklists", .{});
+    defer dir.close(io);
+    var list_buf: [name_buf_len]u8 = undefined;
+    var wild_buf: [name_buf_len]u8 = undefined;
+    try dir.writeFile(io, .{ .sub_path = compiledName(&list_buf, id, ".list"), .data = list_body });
+    try dir.writeFile(io, .{ .sub_path = compiledName(&wild_buf, id, ".wild"), .data = wild_body });
+    try sources_repo.updateSourceStats(&database, id, .{
+        .last_updated = 1_700_000_000,
+        .domain_count = 1,
+        .wildcard_count = 0,
+        .skipped_regex_count = 0,
+        .checksum = &bodyChecksum(list_body, wild_body),
+    });
+
+    // The baseline every assertion below is against: one clean reload, one
+    // status entry that says so.
+    try mgr.reload(io);
+    var out: [4]SourceStatus = undefined;
+    try testing.expectEqual(@as(usize, 1), mgr.statusSnapshot(io, &out));
+    try testing.expectEqual(State.ok, out[0].state);
+    try testing.expect(out[0].loaded);
+    const published = mgr.generation;
+
+    // Both catch sites, in the order `loadSource` reads the two files. A
+    // cancellation is consumed by whoever catches it, so folding it into a load
+    // failure would spend the shutdown signal and leave a status row reading
+    // "Canceled" behind.
+    for ([_][]const u8{ ".list", ".wild" }) |suffix| {
+        var vtable: std.Io.VTable = undefined;
+        const canceling = cancelingIo(io, suffix, &vtable);
+        try testing.expectError(error.Canceled, mgr.reload(canceling));
+
+        try testing.expectEqual(@as(usize, 1), mgr.statusSnapshot(io, &out));
+        try testing.expectEqual(State.ok, out[0].state);
+        try testing.expectEqual(@as(usize, 0), out[0].errorText().len);
+        try testing.expect(out[0].loaded);
+        // Nothing was published either: the snapshot the reload never built
+        // cannot have replaced the one still serving.
+        try testing.expectEqual(published, mgr.generation);
+    }
+}
+
+test "committing a status for an id the table has no entry for is not silent" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var database = try openMigrated();
+    defer database.close();
+
+    var f: fetcher.Fetcher = undefined;
+    var mgr = try testManager(&database, &f);
+    defer mgr.deinit(io);
+
+    const rows = [_]sources_repo.SourceRow{testRow(1, true)};
+    try mgr.syncStatuses(io, &rows);
+
+    // The unknown id: a source inserted through the API after this pass built
+    // its table, or deleted before the pass reached its commit. The outcome
+    // has nowhere to go, and the warning is the only trace it leaves. The log
+    // sink cannot be installed under the test runner — it would eat the
+    // harness's own output — so what is asserted here is that the miss is
+    // survivable and changes nothing.
+    var stranger: SourceStatus = .{ .id = 42 };
+    stranger.fail(.fetch_failed, "Timeout");
+    mgr.commitStatus(io, stranger);
+
+    var out: [4]SourceStatus = undefined;
+    try testing.expectEqual(@as(usize, 1), mgr.statusSnapshot(io, &out));
+    try testing.expectEqual(@as(i64, 1), out[0].id);
+    try testing.expectEqual(State.never_fetched, out[0].state);
+
+    // The same commit against an id the table does know still lands.
+    var known: SourceStatus = .{ .id = 1 };
+    known.fail(.fetch_failed, "Timeout");
+    mgr.commitStatus(io, known);
+    _ = mgr.statusSnapshot(io, &out);
+    try testing.expectEqual(State.fetch_failed, out[0].state);
+    try testing.expectEqualStrings("Timeout", out[0].errorText());
+}
+
 test "the header writer produces the documented text" {
     var buf: [512]u8 = undefined;
     var w: std.Io.Writer = .fixed(&buf);

src/filter/parsers.zig

@@ -85,13 +85,43 @@ pub fn isComment(line: []const u8) bool {
 
 /// The element-hiding separators, which may also follow a domain list
 /// (`example.com##.ad-banner`).
+///
+/// Where the separator sits decides, because `#` is also the hosts comment
+/// marker and a hosts file banner is drawn out of the same two characters. A
+/// `##` counts only where an element-hiding rule can put one: at the start of
+/// the line with a selector behind it, or straight after the domain list it
+/// applies to. `## Title`, `####` and `see ## below` are therefore text, and a
+/// hosts file that opens with a banner keeps sniffing as hosts.
+///
+/// Guarding this with `isComment` instead would decide nothing: `isComment`
+/// asks this function.
 pub fn isElementHiding(line: []const u8) bool {
     for ([_][]const u8{ "##", "#@#", "#?#", "#$#", "#%#" }) |marker| {
-        if (std.mem.find(u8, line, marker) != null) return true;
+        var from: usize = 0;
+        while (std.mem.find(u8, line[from..], marker)) |offset| {
+            const at = from + offset;
+            if (separatorStartsRule(line, at, marker.len)) return true;
+            from = at + 1;
+        }
     }
     return false;
 }
 
+/// Whether the separator of `marker_len` bytes at `at` is a rule's separator
+/// rather than two characters of prose.
+fn separatorStartsRule(line: []const u8, at: usize, marker_len: usize) bool {
+    if (at == 0) {
+        // A generic rule carries its selector here. A banner carries a space,
+        // another `#`, or nothing at all.
+        if (line.len == marker_len) return false;
+        const after = line[marker_len];
+        return after != '#' and !std.ascii.isWhitespace(after);
+    }
+    // A domain list ends where the separator begins, with no space between.
+    const before = line[at - 1];
+    return before != '#' and !std.ascii.isWhitespace(before);
+}
+
 fn hasAbpMarker(line: []const u8) bool {
     if (std.mem.startsWith(u8, line, "||")) return true;
     if (std.mem.startsWith(u8, line, "@@")) return true;
@@ -188,6 +218,79 @@ test "detectFormat is not fooled by a dollar sign in a comment" {
     try testing.expectEqual(Format.hosts, detectFormat(sample));
 }
 
+/// The banner style a hosts list published for malware URLs opens with: a rule
+/// of `#` characters around a titled header block. Every line of it contains
+/// `##`, and reading those as element hiding used to sniff the whole file as
+/// ABP — which put `0.0.0.0` in the domain set and dropped every hosts line
+/// that carried an inline comment.
+const urlhaus_banner_sample =
+    \\################################################################
+    \\# URLhaus Malicious Hosts File                                 #
+    \\# Last updated: 2026-08-05 06:05:04 (UTC)                      #
+    \\#                                                              #
+    \\# Terms Of Use: https://urlhaus.abuse.ch/api/                  #
+    \\################################################################
+    \\0.0.0.0 bad1.example.com # https://urlhaus.abuse.ch/url/1/
+    \\0.0.0.0 bad2.example.net # https://urlhaus.abuse.ch/url/2/
+    \\0.0.0.0 bad3.example.org # https://urlhaus.abuse.ch/url/3/
+    \\
+;
+
+test "detectFormat reads a hosts file behind a hash banner as hosts" {
+    try testing.expectEqual(Format.hosts, detectFormat(urlhaus_banner_sample));
+
+    // The lines the banner is made of are comments, so the sample the format
+    // is decided from is the three hosts lines alone.
+    var it = std.mem.splitScalar(u8, urlhaus_banner_sample, '\n');
+    while (it.next()) |line| {
+        if (line.len == 0) continue;
+        if (line[0] != '#') continue;
+        try testing.expect(isComment(line));
+        try testing.expect(!isElementHiding(line));
+    }
+}
+
+test "detectFormat still recognizes a generic element-hiding rule" {
+    const sample =
+        \\##.ad-banner
+        \\example.com
+        \\
+    ;
+    try testing.expectEqual(Format.abp, detectFormat(sample));
+    try testing.expect(isElementHiding("##.ad-banner"));
+}
+
+test "detectFormat recognizes element hiding after a domain list" {
+    const sample =
+        \\example.com##.ad
+        \\other.example.net
+        \\
+    ;
+    try testing.expectEqual(Format.abp, detectFormat(sample));
+    try testing.expect(isElementHiding("example.com##.ad"));
+}
+
+test "detectFormat recognizes an exception separator after a domain" {
+    const sample =
+        \\example.com#@#.sponsored
+        \\other.example.net
+        \\
+    ;
+    try testing.expectEqual(Format.abp, detectFormat(sample));
+    try testing.expect(isElementHiding("example.com#@#.sponsored"));
+}
+
+test "a comment line that mentions a separator stays a comment" {
+    const line = "# the ##.ad rules live in the other list";
+    try testing.expect(isComment(line));
+    try testing.expect(!isElementHiding(line));
+
+    // A bare separator and a rule of hashes are text as well.
+    try testing.expect(!isElementHiding("##"));
+    try testing.expect(!isElementHiding("####"));
+    try testing.expect(!isElementHiding("## Title"));
+}
+
 test "parseLine dispatches to the hosts parser" {
     const line = parseLine(.hosts, "0.0.0.0 ads.example.com");
     try testing.expectEqual(Kind.domain, line.kind);