//! The blocklist manager (PLAN §4): the compiled files under //! `/blocklists/`, the refresh that produces them, the metadata //! columns it writes back, the snapshot built from them and the swap that //! publishes it. //! //! This is the only file in this milestone that touches both the database and //! the filesystem. Everything it composes — the parsers, the compiler, the //! domain sets, the rule sets and the snapshot — is pure and testable without //! either. //! //! **The swap is an `std.Io.RwLock`, not a lock-free pointer.** PLAN §7.3 says //! "readers lock-free"; this is a deliberate deviation. Freeing the old //! snapshot without a lock needs epoch-based reclamation or hazard pointers: a //! class of code that is very hard to get right and impossible to test //! convincingly, bought for a household resolver whose target is 100 qps. A //! shared lock held for the microseconds of one `evaluate` costs an uncontended //! atomic pair; the writer takes the exclusive lock only on a swap, which //! happens on refresh. The old snapshot is freed *after* `unlock` returns, and //! the `Handle` API makes "do not retain the pointer" the only shape a caller //! can write. //! //! The same lock guards the status table, which is written from the refresh //! task and read by the API. Both critical sections are short and hold no //! socket and no file, so the uncancelable lock forms are used: a lock this //! code takes is always released within a few instructions. //! //! A second lock, `writer_lock`, serializes the writers against each other: //! `reload`, `refreshSource`, `refreshAll`, the startup pass and //! `pruneOrphans`. Two concurrent reloads would otherwise compute the same //! generation and each destroy a snapshot the other had just published, and two //! concurrent refreshes would share the fetcher's buffers and, for one source, //! the same `.raw.tmp` / `.list.tmp` / `.wild.tmp` paths. It is held across //! downloads and compiles, so it is a plain mutex rather than the RCU lock: //! readers must never wait behind a refresh. The public entry points take it; //! the `*Locked` bodies assume it and never take it again, because it is not //! reentrant. const std = @import("std"); const Allocator = std.mem.Allocator; const model = @import("../config/model.zig"); const db = @import("../storage/db.zig"); const clients_repo = @import("../storage/repositories/clients_repo.zig"); const groups_repo = @import("../storage/repositories/groups_repo.zig"); const rules_repo = @import("../storage/repositories/rules_repo.zig"); const sources_repo = @import("../storage/repositories/sources_repo.zig"); const disk_monitor = @import("../storage/disk_monitor.zig"); const compiler = @import("compiler.zig"); const fetcher = @import("fetcher.zig"); const matcher = @import("matcher.zig"); const parsers = @import("parsers.zig"); const log = std.log.scoped(.blocklist_manager); const Sha256 = std.crypto.hash.sha2.Sha256; /// `SourceStatus.last_error` is fixed-size so the failure path allocates /// nothing. pub const max_error_len: usize = 128; /// `SourceStatus.url` is fixed-size so a copied status borrows nothing. A /// blocklist url longer than this is truncated in the status only; the row /// keeps it whole. pub const max_url_len: usize = 255; /// A compiled body larger than this is refused at load. A source that reaches /// it produced more than `fetcher.max_body_bytes` of names, which cannot /// happen from a download this fetcher performed. pub const max_compiled_bytes: usize = 128 * 1024 * 1024; /// Buffer size for every file stream this file opens. One buffer is live per /// stage, and the stages do not overlap. const io_buf_len: usize = 64 * 1024; /// Holds the sniff sample: `parsers.sample_lines` lines of at most /// `compiler.max_line_len` bytes, each with its newline. A fixed byte window /// would be spent by a handful of legal 4096-byte comment lines and the format /// would then be decided by almost no data. const sample_buf_len: usize = parsers.sample_lines * (compiler.max_line_len + 1); /// `` is at most 20 characters and the longest suffix is `.list.tmp`. const name_buf_len: usize = 48; pub const Paths = struct { /// ``, owned by the caller and left open for the manager's life. dir: std.Io.Dir, subdir: []const u8 = "blocklists", }; /// Where one source stands. `.never_fetched` is the state of a source that has /// no compiled files and no stored checksum, which is a fresh install rather /// than a failure. `.no_valid_entries` is a download that compiled cleanly and /// yielded nothing usable — an error page or a compressed body, not a /// blocklist. pub const State = enum { ok, never_fetched, fetch_failed, compile_failed, no_valid_entries, load_failed, /// Whether this state was recorded by a refresh rather than by a load. /// A load outcome never overwrites one: the files a reload just read are /// exactly the files the failed refresh could not replace, and the operator /// still has to see why the update did not land. `SourceStatus.loaded` /// carries the other half — whether the source is filtering at all. pub fn isRefreshFailure(self: State) bool { return switch (self) { .fetch_failed, .compile_failed, .no_valid_entries => true, .ok, .never_fetched, .load_failed => false, }; } }; /// A status is a value with no borrowed memory, so a copy handed to the API /// outlives every reload. The url is held inline for that reason. pub const SourceStatus = struct { id: i64, state: State = .never_fetched, /// Whether this source's compiled files were read into the most recent /// snapshot build — that is, whether it is filtering right now. `state` /// describes the most recent attempt to *produce* those files, which is a /// different fact: a source whose refresh failed keeps serving what the /// refresh did not replace, and reads `.fetch_failed` with `loaded` set. loaded: bool = false, last_attempt: i64 = 0, last_success: i64 = 0, counts: compiler.Counts = .{}, /// A display copy of the source url, truncated at `max_url_len`. The whole /// url is in the `blocklist_sources` row this status shares an `id` with. url: [max_url_len]u8 = @splat(0), url_len: u8 = 0, last_error: [max_error_len]u8 = @splat(0), last_error_len: u8 = 0, pub fn errorText(self: *const SourceStatus) []const u8 { return self.last_error[0..self.last_error_len]; } pub fn urlText(self: *const SourceStatus) []const u8 { return self.url[0..self.url_len]; } fn setUrl(self: *SourceStatus, url: []const u8) void { const kept = @min(url.len, max_url_len); @memcpy(self.url[0..kept], url[0..kept]); @memset(self.url[kept..], 0); self.url_len = @intCast(kept); } fn fail(self: *SourceStatus, state: State, text: []const u8) void { self.state = state; const kept = @min(text.len, max_error_len); @memcpy(self.last_error[0..kept], text[0..kept]); @memset(self.last_error[kept..], 0); self.last_error_len = @intCast(kept); } fn succeed(self: *SourceStatus, at: i64, counts: compiler.Counts) void { self.state = .ok; self.counts = counts; self.last_success = at; self.last_error = @splat(0); self.last_error_len = 0; } }; /// The header every compiled file carries, ahead of the body. The `sha256` /// covers the `.list` body followed by the `.wild` body and **not** the header, /// so it stays stable across a refetch of unchanged content while /// `fetched_at` moves. pub const Header = struct { url: []const u8, format: parsers.Format, fetched_at: i64, counts: compiler.Counts, /// 64 lowercase hex characters. checksum: []const u8, pub fn write(self: Header, w: *std.Io.Writer) std.Io.Writer.Error!void { try w.writeAll("# nxdns blocklist\n"); try w.print("# url {s}\n", .{self.url}); try w.print("# format {s}\n", .{@tagName(self.format)}); try w.print("# fetched_at {d}\n", .{self.fetched_at}); try w.print("# domains {d}\n", .{self.counts.domains}); try w.print("# wildcards {d}\n", .{self.counts.wildcards}); try w.print("# skipped_regex {d}\n", .{self.counts.skipped_regex}); try w.print("# skipped_unsupported {d}\n", .{self.counts.skipped_unsupported}); try w.print("# invalid {d}\n", .{self.counts.invalid}); try w.print("# sha256 {s}\n", .{self.checksum}); } }; /// The body of a compiled file: everything after the leading `#` lines. A file /// with no header is all body, which is what makes a hand-written fixture a /// legal compiled file. pub fn stripHeader(bytes: []const u8) []const u8 { var rest = bytes; while (rest.len != 0 and rest[0] == '#') { const newline = std.mem.indexOfScalar(u8, rest, '\n') orelse return rest[rest.len..]; rest = rest[newline + 1 ..]; } return rest; } pub const Manager = struct { gpa: Allocator, database: *db.Db, paths: Paths, fetcher: *fetcher.Fetcher, update: model.BlocklistUpdate, /// Bounds one download. `std.http.Client` has no per-request deadline, so /// the fetch runs under `io.concurrent` against a sleep of this length. total_budget: std.Io.Clock.Duration, lock: std.Io.RwLock, /// Serializes the writers against each other. Never taken by a reader. writer_lock: std.Io.Mutex, current: ?*matcher.Snapshot, generation: u64, statuses: []SourceStatus, /// Owns the `statuses` table. The entries themselves borrow nothing. status_arena: std.heap.ArenaAllocator, /// The §11.6 disk gate (ruling 17). Set by the composition root after /// `init` and before `runScheduler` starts; null disables gating, which is /// what every test and `nxdns check` want. Only the scheduler consults it — /// see `refreshGated`. monitor: ?*disk_monitor.Monitor = null, /// Scheduled refresh passes skipped by the disk gate. Phase 8's health /// rollup reads it through `refreshesGated`. refreshes_gated: std.atomic.Value(u64) = .init(0), pub const Error = error{ OutOfMemory, Canceled, /// A filesystem operation on the blocklist directory failed. The /// concrete cause is logged at `warn` where it happens: this taxonomy /// would otherwise carry two dozen members no caller can act on /// differently. FileSystem, /// The `groups` table changed between listing the groups and reading /// their ids. Retrying the reload is the answer, and the caller is the /// only one that can decide to. GroupSetChanged, } || db.Error || matcher.Snapshot.Error; /// The result is not copyable afterwards: `status_arena` and `lock` are /// addressed through `self`. pub fn init( gpa: Allocator, database: *db.Db, paths: Paths, fetcher_ptr: *fetcher.Fetcher, update: model.BlocklistUpdate, total_budget: std.Io.Clock.Duration, ) Error!Manager { return .{ .gpa = gpa, .database = database, .paths = paths, .fetcher = fetcher_ptr, .update = update, .total_budget = total_budget, .lock = .init, .writer_lock = .init, .current = null, .generation = 0, .statuses = &.{}, .status_arena = .init(gpa), }; } pub fn deinit(self: *Manager, io: std.Io) void { self.lock.lockUncancelable(io); const old = self.current; self.current = null; self.statuses = &.{}; self.lock.unlock(io); if (old) |snapshot| destroySnapshot(self.gpa, snapshot); self.status_arena.deinit(); self.* = undefined; } /// Reader side of the swap. The handle holds a shared lock: release it, and /// do not retain `snapshot` afterwards. pub const Handle = struct { snapshot: *const matcher.Snapshot, manager: *Manager, pub fn release(self: Handle, io: std.Io) void { self.manager.lock.unlockShared(io); } }; /// `null` before the first successful `reload`. The caller answers /// SERVFAIL, or forwards unfiltered, on its own policy — this file does not /// decide that. pub fn acquire(self: *Manager, io: std.Io) ?Handle { self.lock.lockSharedUncancelable(io); const snapshot = self.current orelse { self.lock.unlockShared(io); return null; }; return .{ .snapshot = snapshot, .manager = self }; } /// Copies the status table for the API and for `nxdns check`. Returns the /// number of entries written, which is `min(out.len, source count)`. /// /// The copies are self-contained: `SourceStatus` holds its url and its /// error text inline, so the caller may keep them for as long as it likes /// and a concurrent reload cannot pull memory out from under them. pub fn statusSnapshot(self: *Manager, io: std.Io, out: []SourceStatus) usize { self.lock.lockSharedUncancelable(io); defer self.lock.unlockShared(io); const kept = @min(out.len, self.statuses.len); @memcpy(out[0..kept], self.statuses[0..kept]); return kept; } // ----------------------------------------------------------------------- // reload // ----------------------------------------------------------------------- /// Reads the database and every compiled file, builds a snapshot and swaps /// it in. /// /// A source whose compiled files are missing, unreadable or checksum /// mismatched is marked `.load_failed` and left out of the snapshot rather /// than failing the whole reload: one bad file must not cost the operator /// every other list. `runScheduler` refreshes exactly those sources, so the /// state is recorded, surfaced and repaired, never silently accepted. /// /// A body that is present and checksum-clean but malformed fails the build /// (`error.NotSorted`), and the previously published snapshot keeps /// serving: nothing is swapped until the new snapshot exists. The status /// table keeps describing that snapshot too — the table is rebuilt off to /// the side and the load findings are written into it there, so a reload /// that never publishes changes neither. pub fn reload(self: *Manager, io: std.Io) Error!void { self.writer_lock.lockUncancelable(io); defer self.writer_lock.unlock(io); return self.reloadLocked(io); } fn reloadLocked(self: *Manager, io: std.Io) Error!void { var rows = try sources_repo.listSourceRows(self.database, self.gpa); defer rows.deinit(self.gpa); defer sources_repo.freeSourceRows(self.gpa, rows.items); // The table this reload will publish, built where no reader can see it. // It is installed in the swap below or freed unpublished, so a reload // that fails leaves the previous table describing the previous // snapshot — including the entry of a source deleted from the database, // which that snapshot still enforces. var candidate: ?StatusTable = try self.buildStatusTable(io, rows.items); errdefer if (candidate) |*table| table.deinit(); var dir = try self.openDir(io, .{}); defer dir.close(io); const sources = try self.gpa.alloc(model.BlocklistSource, rows.items.len); defer self.gpa.free(sources); const source_ids = try self.gpa.alloc(i64, rows.items.len); defer self.gpa.free(source_ids); const compiled = try self.gpa.alloc(?matcher.Snapshot.Compiled, rows.items.len); defer self.gpa.free(compiled); // The file contents outlive the header stripping and are freed once the // snapshot has copied what it needs into its own arena. var bodies: std.ArrayList([]u8) = .empty; defer { for (bodies.items) |body| self.gpa.free(body); bodies.deinit(self.gpa); } // What this reload found, per source. It is applied to the status table // only if the snapshot it describes is published: everything below here // can still fail, and a status table describing a snapshot nobody // serves is worse than one describing the previous one. const outcomes = try self.gpa.alloc(LoadOutcome, rows.items.len); defer self.gpa.free(outcomes); var loaded: usize = 0; for (rows.items, sources, source_ids, compiled, outcomes) |row, *source, *source_id, *slot, *outcome| { source_id.* = row.id; // `is_suggested` is a UI hint the snapshot never reads. source.* = .{ .url = row.url, .name = row.name, .enabled = row.enabled }; outcome.* = if (row.enabled) try self.loadSource(io, dir, row, &bodies) else .disabled; switch (outcome.*) { .loaded => |body| { slot.* = body; loaded += 1; }, // Not loadable and therefore not enforced. Saying so here is // what keeps `Snapshot.build`'s `MissingCompiledSource` for the // case it is meant for: a caller that forgot to read a body. .disabled, .failed => { slot.* = null; source.enabled = false; }, } } var groups = try groups_repo.listGroups(self.database, self.gpa); defer groups.deinit(self.gpa); defer groups_repo.freeGroups(self.gpa, groups.items); const group_ids = try self.groupIds(groups.items); defer self.gpa.free(group_ids); var group_sources = try groups_repo.listGroupSources(self.database, self.gpa); defer group_sources.deinit(self.gpa); defer groups_repo.freeGroupSources(self.gpa, group_sources.items); var rule_rows = try rules_repo.listRules(self.database, self.gpa); defer rule_rows.deinit(self.gpa); defer rules_repo.freeRules(self.gpa, rule_rows.items); var clients = try clients_repo.listClients(self.database, self.gpa); defer clients.deinit(self.gpa); defer clients_repo.freeClients(self.gpa, clients.items); var prefixes = try clients_repo.listClientPrefixes(self.database, self.gpa); defer prefixes.deinit(self.gpa); defer clients_repo.freeClientPrefixes(self.gpa, prefixes.items); // Query names are attacker-supplied, so a fixed seed would make // probe-chain flooding computable offline. var seed_bytes: [8]u8 = undefined; io.random(&seed_bytes); const generation = self.generation + 1; const snapshot = try self.gpa.create(matcher.Snapshot); errdefer self.gpa.destroy(snapshot); snapshot.* = try matcher.Snapshot.build(self.gpa, .{ .groups = groups.items, .group_ids = group_ids, .group_sources = group_sources.items, .sources = sources, .source_ids = source_ids, .rules = rule_rows.items, .clients = clients.items, .prefixes = prefixes.items, .compiled = compiled, .seed = std.mem.readInt(u64, &seed_bytes, .little), .generation = generation, }); // Read before the swap: once `current` points at it, this snapshot // belongs to the readers and to whichever writer replaces it next. const memory_bytes = snapshot.memoryBytes(); // The snapshot, the status table and the load facts land together, so a // reader never sees a status table describing anything but the // published snapshot. self.lock.lockUncancelable(io); const old = self.current; self.current = snapshot; self.generation = generation; applyLoadOutcomes(candidate.?.items, rows.items, outcomes); self.installStatuses(candidate.?); candidate = null; self.lock.unlock(io); // After `unlock`: no reader can still hold the old snapshot here, and // `writer_lock` keeps every other writer out of this sequence. if (old) |previous| destroySnapshot(self.gpa, previous); log.info("blocklist snapshot generation {d}: {d} of {d} sources loaded, {d} bytes", .{ generation, loaded, rows.items.len, memory_bytes, }); } /// What one enabled source contributes to the snapshot being built. Nothing /// here touches the status table: the outcome is data until the swap /// commits it. fn loadSource( self: *Manager, io: std.Io, dir: std.Io.Dir, row: sources_repo.SourceRow, bodies: *std.ArrayList([]u8), ) Error!LoadOutcome { const stored = row.checksum orelse // No stored checksum means no successful compile has ever // happened. A fresh install is here on every source. return .{ .failed = .{ .state = .never_fetched, .text = "" } }; var list_buf: [name_buf_len]u8 = undefined; var wild_buf: [name_buf_len]u8 = undefined; const list_name = compiledName(&list_buf, row.id, ".list"); const wild_name = compiledName(&wild_buf, row.id, ".wild"); // Reserved before the reads, so neither buffer can be orphaned by a // failing append: `bodies` owns each one from the moment it is read. try bodies.ensureUnusedCapacity(self.gpa, 2); const list_bytes = dir.readFileAlloc(io, list_name, self.gpa, .limited(max_compiled_bytes)) catch |err| { if (err == error.OutOfMemory) return error.OutOfMemory; return loadFailure(row, list_name, err); }; bodies.appendAssumeCapacity(list_bytes); const wild_bytes = dir.readFileAlloc(io, wild_name, self.gpa, .limited(max_compiled_bytes)) catch |err| { if (err == error.OutOfMemory) return error.OutOfMemory; return loadFailure(row, wild_name, err); }; bodies.appendAssumeCapacity(wild_bytes); const list_body = stripHeader(list_bytes); const wild_body = stripHeader(wild_bytes); // The checksum covers both bodies together, so a crash between the two // `replace` calls — a new `.list` beside an old `.wild` — is caught // here and refreshed, not served as a half-updated list. if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body))) { log.warn("blocklist {s}: compiled files do not match the stored checksum", .{row.url}); return .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } }; } return .{ .loaded = .{ .list_body = list_body, .wild_body = wild_body } }; } // ----------------------------------------------------------------------- // refresh // ----------------------------------------------------------------------- /// Downloads, compiles and atomically replaces the compiled files of one /// source, then writes its runtime columns. /// /// Returns `true` only when the compiled files were replaced. Unchanged /// content (equal checksum) and every recorded failure return `false`; the /// reason for a failure is in the status table, not in the return value. /// /// The status entry is found by row id, so a source added since the last /// `reload` has nowhere to record its outcome. `refreshAll` syncs the table /// before it refreshes anything, which is why it is the entry point Phase 8 /// and the scheduler use. pub fn refreshSource(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool { self.writer_lock.lockUncancelable(io); defer self.writer_lock.unlock(io); return self.refreshSourceLocked(io, row); } fn refreshSourceLocked(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool { // The previous entry describes the compiled files that are still on // disk, and a failed refresh leaves them serving. Starting from a blank // status would erase `last_success` and the counters of a list that is // still being enforced. var status: SourceStatus = self.priorStatus(io, row.id) orelse .{ .id = row.id }; status.setUrl(row.url); status.last_attempt = std.Io.Clock.real.now(io).toSeconds(); const replaced = try self.refreshOne(io, row, &status); self.commitStatus(io, status); return replaced; } /// Every enabled source, one at a time, then one `reload`. A failing source /// never stops the pass: it would hide every source behind it. /// /// This returns an error only when nothing could be done at all — out of /// memory, an unreachable database, an unusable blocklist directory. A /// source that failed to download or compile is a successful pass with a /// non-`ok` status. pub fn refreshAll(self: *Manager, io: std.Io) Error!void { self.writer_lock.lockUncancelable(io); defer self.writer_lock.unlock(io); var rows = try sources_repo.listSourceRows(self.database, self.gpa); defer rows.deinit(self.gpa); defer sources_repo.freeSourceRows(self.gpa, rows.items); try self.syncStatuses(io, rows.items); for (rows.items) |row| { if (!row.enabled) continue; _ = try self.refreshSourceLocked(io, row); } return self.reloadLocked(io); } fn refreshOne( self: *Manager, io: std.Io, row: sources_repo.SourceRow, status: *SourceStatus, ) Error!bool { var dir = try self.openDir(io, .{}); defer dir.close(io); var raw_buf: [name_buf_len]u8 = undefined; var list_tmp_buf: [name_buf_len]u8 = undefined; var wild_tmp_buf: [name_buf_len]u8 = undefined; const raw_name = compiledName(&raw_buf, row.id, ".raw.tmp"); const list_tmp = compiledName(&list_tmp_buf, row.id, ".list.tmp"); const wild_tmp = compiledName(&wild_tmp_buf, row.id, ".wild.tmp"); // Installed before the calls that create these files, not after: an // `error.Canceled` or `error.OutOfMemory` returned straight out of // `download` or `compileTo` would outrun a later `defer` and leave a // temporary behind. Deleting a name that was never created is a no-op. defer self.deleteQuietly(io, dir, raw_name); defer self.deleteQuietly(io, dir, list_tmp); defer self.deleteQuietly(io, dir, wild_tmp); self.download(io, dir, raw_name, row.url) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.Canceled => return error.Canceled, else => { self.reportFetchFailure(row, status, err); return false; }, }; const format = self.detectFormat(io, dir, raw_name) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.Canceled => return error.Canceled, else => { self.reportCompileFailure(row, status, err); return false; }, }; const result = self.compileTo(io, dir, raw_name, format, list_tmp, wild_tmp) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.Canceled => return error.Canceled, else => { self.reportCompileFailure(row, status, err); return false; }, }; if (rejectedWithoutEntries(result.counts)) { self.reportEmptyCompile(row, status, result.counts); return false; } const now = std.Io.Clock.real.now(io).toSeconds(); // Recompiling identical content into new files would invalidate the // snapshot for nothing. The files on disk must actually carry that // content: if a reload found them corrupt and excluded the source, a // re-download of unchanged upstream bytes is the one chance to repair // them, and skipping the rewrite here would leave filtering off for // good. if (row.checksum) |stored| { if (std.mem.eql(u8, stored, &result.checksum) and self.diskBodiesMatch(io, dir, row.id, stored)) { try sources_repo.updateSourceStats(self.database, row.id, .{ .last_updated = now, .domain_count = row.domain_count, .wildcard_count = row.wildcard_count, .skipped_regex_count = row.skipped_regex_count, .checksum = stored, }); status.succeed(now, result.counts); return false; } } const header: Header = .{ .url = row.url, .format = format, .fetched_at = now, .counts = result.counts, .checksum = &result.checksum, }; self.publish(io, dir, row.id, header, list_tmp, wild_tmp) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.Canceled => return error.Canceled, else => { self.reportCompileFailure(row, status, err); return false; }, }; try sources_repo.updateSourceStats(self.database, row.id, .{ .last_updated = now, .domain_count = result.counts.domains, .wildcard_count = result.counts.wildcards, .skipped_regex_count = result.counts.skipped_regex, .checksum = &result.checksum, }); status.succeed(now, result.counts); return true; } /// The body goes to a temporary file, never to memory: `max_body_bytes` is /// 64 MB and the memory budget has no room for it beside two snapshots. fn download( self: *Manager, io: std.Io, dir: std.Io.Dir, raw_name: []const u8, url: []const u8, ) !void { const file = try dir.createFile(io, raw_name, .{ .permissions = .fromMode(0o600) }); defer file.close(io); const buffer = try self.gpa.alloc(u8, io_buf_len); defer self.gpa.free(buffer); var fw = file.writer(io, buffer); const result = self.fetchWithin(io, url, &fw.interface) catch |err| { // `fetcher.Error.Unexpected` is what a failing sink surfaces as; // the concrete cause is on this writer, which the fetcher does not // own. if (fw.err) |cause| return cause; if (err == error.HttpStatus) { if (self.fetcher.last_status) |status| { log.warn("blocklist {s}: http status {d}", .{ url, @intFromEnum(status) }); } } return err; }; try fw.interface.flush(); // The compile reads this file back; the bytes must be there, not in a // buffer this function is about to drop. try file.sync(io); log.debug("blocklist {s}: downloaded {d} bytes", .{ url, result.bytes_read }); } /// `std.http.Client` has no per-request deadline, so the whole exchange /// races a sleep and the loser is canceled — milestone 3's pattern. fn fetchWithin( self: *Manager, io: std.Io, url: []const u8, w: *std.Io.Writer, ) fetcher.Error!fetcher.Result { var outcomes: [2]Outcome = undefined; var race: std.Io.Select(Outcome) = .init(io, &outcomes); defer race.cancelDiscard(); race.concurrent(.fetch, fetcher.Fetcher.fetch, .{ self.fetcher, io, url, w }) catch |err| switch (err) { error.ConcurrencyUnavailable => return error.SystemResources, }; race.concurrent(.expiry, expire, .{ io, self.total_budget }) catch |err| switch (err) { error.ConcurrencyUnavailable => return error.SystemResources, }; switch (try race.await()) { .fetch => |result| return result, .expiry => |result| { // A canceled sleep means this task is being torn down, not that // the download is slow. try result; return error.Timeout; }, } } fn detectFormat( self: *Manager, io: std.Io, dir: std.Io.Dir, raw_name: []const u8, ) !parsers.Format { const file = try dir.openFile(io, raw_name, .{}); defer file.close(io); const buffers = try self.gpa.alloc(u8, io_buf_len + sample_buf_len); defer self.gpa.free(buffers); var fr = file.reader(io, buffers[0..io_buf_len]); var sample: std.Io.Writer = .fixed(buffers[io_buf_len..]); collectSample(&fr.interface, &sample) catch |err| switch (err) { error.ReadFailed => return fr.err orelse err, // `sample_buf_len` holds every line `collectSample` can emit, so a // full buffer means the sample is complete. error.WriteFailed => {}, }; return parsers.detectFormat(sample.buffered()); } /// Compiles into two plain temporary files. The compiled bodies cannot go /// straight into the final files: the header carries counts that only exist /// once the whole input has been compiled, and the loader requires the /// header first. fn compileTo( self: *Manager, io: std.Io, dir: std.Io.Dir, raw_name: []const u8, format: parsers.Format, list_tmp: []const u8, wild_tmp: []const u8, ) !compiler.Result { const raw = try dir.openFile(io, raw_name, .{}); defer raw.close(io); const list_file = try dir.createFile(io, list_tmp, .{ .permissions = .fromMode(0o600) }); defer list_file.close(io); const wild_file = try dir.createFile(io, wild_tmp, .{ .permissions = .fromMode(0o600) }); defer wild_file.close(io); const buffers = try self.gpa.alloc(u8, 3 * io_buf_len); defer self.gpa.free(buffers); var fr = raw.reader(io, buffers[0..io_buf_len]); var list_w = list_file.writer(io, buffers[io_buf_len .. 2 * io_buf_len]); var wild_w = wild_file.writer(io, buffers[2 * io_buf_len ..]); const result = compiler.compile( self.gpa, &fr.interface, format, &list_w.interface, &wild_w.interface, ) catch |err| switch (err) { // `compiler.Error` names the direction; the concrete cause is on // the stream that failed. error.ReadFailed => return fr.err orelse err, error.WriteFailed => return list_w.err orelse (wild_w.err orelse err), else => return err, }; try list_w.interface.flush(); try wild_w.interface.flush(); try list_file.sync(io); try wild_file.sync(io); return result; } /// Writes header + body into each final file through `createFileAtomic` + /// `replace`, so a crash mid-write can never leave a half-list that would /// load as a valid, shorter blocklist. fn publish( self: *Manager, io: std.Io, dir: std.Io.Dir, id: i64, header: Header, list_tmp: []const u8, wild_tmp: []const u8, ) !void { const buffers = try self.gpa.alloc(u8, 2 * io_buf_len); defer self.gpa.free(buffers); var list_buf: [name_buf_len]u8 = undefined; var wild_buf: [name_buf_len]u8 = undefined; try publishOne(io, dir, compiledName(&list_buf, id, ".list"), list_tmp, header, buffers); try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), wild_tmp, header, buffers); } fn publishOne( io: std.Io, dir: std.Io.Dir, dest: []const u8, body_name: []const u8, header: Header, buffers: []u8, ) !void { const body = try dir.openFile(io, body_name, .{}); defer body.close(io); var af = try dir.createFileAtomic(io, dest, .{ .permissions = .fromMode(0o600), .replace = true, }); defer af.deinit(io); var fr = body.reader(io, buffers[0..io_buf_len]); var fw = af.file.writer(io, buffers[io_buf_len..]); header.write(&fw.interface) catch return fw.err orelse error.WriteFailed; _ = fr.interface.streamRemaining(&fw.interface) catch return fr.err orelse (fw.err orelse error.WriteFailed); fw.interface.flush() catch return fw.err orelse error.WriteFailed; // Before `replace`, which closes the file: the rename must publish // durable bytes, not an empty file with the content still in the page // cache. try af.file.sync(io); try af.replace(io); } /// Whether the two compiled files on disk hash to `expected`. A missing, /// unreadable or corrupt file answers false, which sends the caller down /// the rewrite path — the only path that can repair it. fn diskBodiesMatch(self: *Manager, io: std.Io, dir: std.Io.Dir, id: i64, expected: []const u8) bool { var list_buf: [name_buf_len]u8 = undefined; var wild_buf: [name_buf_len]u8 = undefined; const limit: std.Io.Limit = .limited(max_compiled_bytes); const list_bytes = dir.readFileAlloc(io, compiledName(&list_buf, id, ".list"), self.gpa, limit) catch return false; defer self.gpa.free(list_bytes); const wild_bytes = dir.readFileAlloc(io, compiledName(&wild_buf, id, ".wild"), self.gpa, limit) catch return false; defer self.gpa.free(wild_bytes); return compiledBodiesMatch(list_bytes, wild_bytes, expected); } fn reportFetchFailure( self: *Manager, row: sources_repo.SourceRow, status: *SourceStatus, err: anyerror, ) void { _ = self; log.warn("blocklist {s}: download failed: {s}", .{ row.url, @errorName(err) }); status.fail(.fetch_failed, @errorName(err)); } fn reportCompileFailure( self: *Manager, row: sources_repo.SourceRow, status: *SourceStatus, err: anyerror, ) void { _ = self; log.warn("blocklist {s}: compile failed: {s}", .{ row.url, @errorName(err) }); status.fail(.compile_failed, @errorName(err)); } fn reportEmptyCompile( self: *Manager, row: sources_repo.SourceRow, status: *SourceStatus, counts: compiler.Counts, ) void { _ = self; var buf: [max_error_len]u8 = undefined; const text = std.fmt.bufPrint( &buf, "NoValidEntries invalid={d} unsupported={d} long_lines={d}", .{ counts.invalid, counts.skipped_unsupported, counts.long_lines }, ) catch "NoValidEntries"; log.warn("blocklist {s}: {s}", .{ row.url, text }); status.fail(.no_valid_entries, text); } // ----------------------------------------------------------------------- // scheduling // ----------------------------------------------------------------------- /// Loads at startup, refreshes only what needs it, then sleeps /// `update.interval_hours` between full passes. Returns on /// `error.Canceled`. /// /// A cold restart must not re-download every list and a boot loop must not /// become a download loop, so the startup pass refreshes a source only when /// it has no usable compiled files or its `last_updated` is older than the /// interval. /// /// `update.enabled == false` stops after the startup pass; manual refresh /// through `refreshAll` still works. pub fn runScheduler(self: *Manager, io: std.Io) std.Io.Cancelable!void { self.startupPass(io) catch |err| switch (err) { error.Canceled => return error.Canceled, else => log.warn("blocklist startup pass failed: {s}", .{@errorName(err)}), }; if (!self.update.enabled) return; // `boot` rather than `awake`: a box that suspends overnight should // still see its daily interval elapse. const interval: std.Io.Clock.Duration = .{ .raw = .fromSeconds(model.updateIntervalSeconds(self.update)), .clock = .boot, }; while (true) { try interval.sleep(io); if (self.refreshGated()) continue; self.refreshAll(io) catch |err| switch (err) { error.Canceled => return error.Canceled, else => log.warn("blocklist refresh pass failed: {s}", .{@errorName(err)}), }; } } /// The §11.6 gate, consulted by scheduled passes only (ruling 17). A /// download writes tens of megabytes into the blocklist directory and the /// compile writes as much again, which is exactly the "non-essential write" /// a critically full disk must not take. /// /// `reload` and `refreshAll` are deliberately not gated: both are operator /// actions (the composition root's startup load, Phase 8's manual refresh), /// and an operator who asks for a refresh on a full disk has asked for it. /// /// Counting happens here, so a caller cannot skip a pass without recording /// it. One `warn` line per skipped pass — at a 24-hour interval that is one /// line a day, and the disk monitor already logs the state change itself. fn refreshGated(self: *Manager) bool { const monitor = self.monitor orelse return false; if (monitor.writesAllowed()) return false; _ = self.refreshes_gated.fetchAdd(1, .monotonic); log.warn("free space is critical; skipping the scheduled blocklist refresh", .{}); return true; } /// Scheduled refresh passes the disk gate has skipped. pub fn refreshesGated(self: *const Manager) u64 { return self.refreshes_gated.load(.monotonic); } fn startupPass(self: *Manager, io: std.Io) Error!void { self.writer_lock.lockUncancelable(io); defer self.writer_lock.unlock(io); // Ahead of the gate on purpose: loading the compiled files that already // exist is a read. A full disk must not cost the household its // filtering as well as its downloads. try self.reloadLocked(io); if (self.refreshGated()) return; var rows = try sources_repo.listSourceRows(self.database, self.gpa); defer rows.deinit(self.gpa); defer sources_repo.freeSourceRows(self.gpa, rows.items); const now = std.Io.Clock.real.now(io).toSeconds(); var refreshed = false; for (rows.items) |row| { if (!row.enabled) continue; if (!self.needsRefresh(io, row, now)) continue; if (try self.refreshSourceLocked(io, row)) refreshed = true; } if (refreshed) try self.reloadLocked(io); } fn needsRefresh(self: *Manager, io: std.Io, row: sources_repo.SourceRow, now: i64) bool { self.lock.lockSharedUncancelable(io); const state: State = state: { for (self.statuses) |status| { if (status.id == row.id) break :state status.state; } break :state .never_fetched; }; self.lock.unlockShared(io); if (state != .ok) return true; const last = row.last_updated orelse return true; return now - last >= model.updateIntervalSeconds(self.update); } // ----------------------------------------------------------------------- // orphans // ----------------------------------------------------------------------- /// Deletes `.list` and `.wild` files whose id is no longer a /// `blocklist_sources` row. Files of a live source are left alone, /// whatever their state. pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void { // A refresh in flight owns the temporaries of a live source; the sweep // must not run beside one and decide from a half-written directory. self.writer_lock.lockUncancelable(io); defer self.writer_lock.unlock(io); var rows = try sources_repo.listSourceRows(self.database, self.gpa); defer rows.deinit(self.gpa); defer sources_repo.freeSourceRows(self.gpa, rows.items); var dir = try self.openDir(io, .{ .iterate = true }); defer dir.close(io); // The names are collected first: `Entry.name` is invalidated by the // next `next`, and deleting under an open cursor is not defined. var doomed: std.ArrayList([]u8) = .empty; defer { for (doomed.items) |item| self.gpa.free(item); doomed.deinit(self.gpa); } var it = dir.iterate(); while (true) { const entry = it.next(io) catch |err| switch (err) { error.Canceled => return error.Canceled, else => { log.warn("pruning blocklists: reading the directory failed: {s}", .{@errorName(err)}); return error.FileSystem; }, } orelse break; if (entry.kind != .file) continue; const id = compiledId(entry.name) orelse continue; if (containsId(rows.items, id)) continue; try doomed.append(self.gpa, try self.gpa.dupe(u8, entry.name)); } for (doomed.items) |name| { self.deleteQuietly(io, dir, name); log.info("pruned orphaned compiled file {s}", .{name}); } } // ----------------------------------------------------------------------- // internals // ----------------------------------------------------------------------- /// Builds the status table for `rows`, carrying every existing entry over /// by row id so a recorded failure survives. Nothing is published: the /// caller either installs the result or frees it, which is what lets /// `reloadLocked` decide only once its snapshot exists. fn buildStatusTable( self: *Manager, io: std.Io, rows: []const sources_repo.SourceRow, ) Error!StatusTable { var fresh: std.heap.ArenaAllocator = .init(self.gpa); errdefer fresh.deinit(); const arena = fresh.allocator(); // The published table is copied under the lock. Reading it unlocked // would race the writer that replaces it — and the arena its entries // live in is freed by whoever installs what this builds. const previous = previous: { self.lock.lockSharedUncancelable(io); defer self.lock.unlockShared(io); break :previous try arena.dupe(SourceStatus, self.statuses); }; const table = try arena.alloc(SourceStatus, rows.len); mergeStatuses(table, rows, previous); return .{ .arena = fresh, .items = table }; } /// Publishes a built table and frees the one it replaces. The caller holds /// the exclusive lock, so no reader is inside the old table. fn installStatuses(self: *Manager, table: StatusTable) void { self.status_arena.deinit(); self.status_arena = table.arena; self.statuses = table.items; } /// Rebuilds and publishes the status table from the current source set. /// /// `refreshAll` calls this before it refreshes anything: a source added /// since the last reload needs an entry to record its outcome in, and the /// API has to see the pass advance while it runs. `reloadLocked` does not /// call it — a reload publishes its table together with the snapshot that /// table describes. fn syncStatuses(self: *Manager, io: std.Io, rows: []const sources_repo.SourceRow) Error!void { const table = try self.buildStatusTable(io, rows); self.lock.lockUncancelable(io); defer self.lock.unlock(io); self.installStatuses(table); } /// The recorded entry for one source, or `null` when the table has none. fn priorStatus(self: *Manager, io: std.Io, id: i64) ?SourceStatus { self.lock.lockSharedUncancelable(io); defer self.lock.unlockShared(io); for (self.statuses) |entry| { if (entry.id == id) return entry; } return null; } fn commitStatus(self: *Manager, io: std.Io, status: SourceStatus) void { self.lock.lockUncancelable(io); defer self.lock.unlock(io); for (self.statuses) |*entry| { if (entry.id != status.id) continue; // `loaded` is the reload's fact, not the refresh's: the files this // refresh wrote are not in a snapshot until the next reload reads // them. const loaded = entry.loaded; entry.* = status; entry.loaded = loaded; return; } } /// One id per group, in `listGroups` order. fn groupIds(self: *Manager, groups: []const model.Group) Error![]i64 { const out = try self.gpa.alloc(i64, groups.len); errdefer self.gpa.free(out); for (out, groups) |*slot, group| { slot.* = try groups_repo.groupId(self.database, group.name) orelse return error.GroupSetChanged; } return out; } fn openDir(self: *Manager, io: std.Io, options: std.Io.Dir.OpenOptions) Error!std.Io.Dir { _ = self.paths.dir.createDirPathStatus(io, self.paths.subdir, .fromMode(0o700)) catch |err| switch (err) { error.Canceled => return error.Canceled, else => { log.warn("creating {s} failed: {s}", .{ self.paths.subdir, @errorName(err) }); return error.FileSystem; }, }; return self.paths.dir.openDir(io, self.paths.subdir, options) catch |err| switch (err) { error.Canceled => return error.Canceled, else => { log.warn("opening {s} failed: {s}", .{ self.paths.subdir, @errorName(err) }); return error.FileSystem; }, }; } /// A temporary that cannot be removed is not a failure of the operation /// that made it, but it is not nothing either: it is left visible. fn deleteQuietly(self: *Manager, io: std.Io, dir: std.Io.Dir, name: []const u8) void { _ = self; dir.deleteFile(io, name) catch |err| switch (err) { error.FileNotFound => {}, else => log.warn("deleting {s} failed: {s}", .{ name, @errorName(err) }), }; } }; /// A status table and the arena holding it. Until `installStatuses` takes it, /// it is a candidate nobody can see, and `deinit` frees it whole. const StatusTable = struct { arena: std.heap.ArenaAllocator, items: []SourceStatus, fn deinit(self: *StatusTable) void { self.arena.deinit(); self.items = &.{}; } }; /// Fills `table` with one entry per row, carrying an entry of the same row id /// over from `previous`. A source deleted since `previous` was built is gone; a /// source added since starts blank. `previous` is only read, so the caller's /// published table is untouched by this. fn mergeStatuses( table: []SourceStatus, rows: []const sources_repo.SourceRow, previous: []const SourceStatus, ) void { for (table, rows) |*status, row| { status.* = .{ .id = row.id }; for (previous) |prior| { if (prior.id != row.id) continue; status.* = prior; break; } // After the carry-over: a url edited on the row wins over the one the // prior entry recorded. status.setUrl(row.url); } } /// What one `reload` found for one source. The texts are static, so an outcome /// borrows nothing and stays valid until the swap that commits it. const LoadOutcome = union(enum) { /// The source is switched off. Not in the snapshot, whatever it was before. disabled, /// Its compiled files are in the snapshot. loaded: matcher.Snapshot.Compiled, /// It is not in the snapshot, for this reason. failed: struct { state: State, text: []const u8 }, }; fn loadFailure(row: sources_repo.SourceRow, file_name: []const u8, err: anyerror) LoadOutcome { log.warn("blocklist {s}: reading {s} failed: {s}", .{ row.url, file_name, @errorName(err) }); return .{ .failed = .{ .state = .load_failed, .text = @errorName(err) } }; } /// Writes one reload's findings into the status table. The caller holds the /// exclusive lock: this runs inside the swap so the table and the published /// snapshot describe the same thing. /// /// `rows` and `outcomes` are parallel. A source with no entry is one the table /// was not synced for, which cannot happen from `reloadLocked` and is skipped /// rather than asserted, because the table is rebuilt by row id. fn applyLoadOutcomes( statuses: []SourceStatus, rows: []const sources_repo.SourceRow, outcomes: []const LoadOutcome, ) void { for (rows, outcomes) |row, outcome| { const entry = entryFor(statuses, row.id) orelse continue; switch (outcome) { // A source disabled since it last loaded is no longer filtering, // and its recorded state describes files nothing reads. .disabled => entry.loaded = false, .loaded => { entry.loaded = true; // Two states survive a successful load. `.ok`, because a // refresh in this process already filled the counters the // compile produced and the three database columns are a subset // of them. And any refresh failure, because the files that just // loaded are exactly the ones the failed refresh could not // replace, so the operator must still see why. if (entry.state == .ok or entry.state.isRefreshFailure()) continue; entry.succeed(row.last_updated orelse 0, .{ .domains = countOf(row.domain_count), .wildcards = countOf(row.wildcard_count), .skipped_regex = countOf(row.skipped_regex_count), }); }, .failed => |reason| { entry.loaded = false; // The state follows only when nothing more informative is // there: a refresh failure already says why the files are // missing or stale, and `loaded` already says they are not // filtering. if (entry.state.isRefreshFailure()) continue; entry.fail(reason.state, reason.text); }, } } } fn entryFor(statuses: []SourceStatus, id: i64) ?*SourceStatus { for (statuses) |*entry| { if (entry.id == id) return entry; } return null; } const Outcome = union(enum) { fetch: fetcher.Error!fetcher.Result, expiry: std.Io.Cancelable!void, }; fn expire(io: std.Io, duration: std.Io.Clock.Duration) std.Io.Cancelable!void { return duration.sleep(io); } /// A counter column read back from the database. It is `NOT NULL DEFAULT 0` and /// only this file writes it, so a value outside `u32` means the row was edited /// behind nxdns's back; the status reports 0 rather than trapping. fn countOf(value: i64) u32 { return std.math.cast(u32, value) orelse 0; } fn destroySnapshot(gpa: Allocator, snapshot: *matcher.Snapshot) void { snapshot.deinit(); gpa.destroy(snapshot); } /// Copies the lines `parsers.detectFormat` would count — neither blank nor a /// comment — until it has `parsers.sample_lines` of them, and writes them to /// `w` newline-separated. Sampling by line rather than by a byte window is what /// keeps a run of long comment lines from deciding the format: `detectFormat` /// reads exactly these lines and ignores everything this drops. /// /// A line over `compiler.max_line_len` is skipped, as the compiler skips it. fn collectSample(r: *std.Io.Reader, w: *std.Io.Writer) error{ ReadFailed, WriteFailed }!void { var considered: usize = 0; while (considered < parsers.sample_lines) { const raw = r.takeDelimiter('\n') catch |err| switch (err) { // The stream is left unmodified here, so the line has to be stepped // over or this loop never advances. error.StreamTooLong => { _ = r.discardDelimiterInclusive('\n') catch |discard_err| switch (discard_err) { error.EndOfStream => return, error.ReadFailed => return error.ReadFailed, }; continue; }, error.ReadFailed => return error.ReadFailed, } orelse return; if (raw.len > compiler.max_line_len) continue; const line = std.mem.trim(u8, raw, &std.ascii.whitespace); if (line.len == 0) continue; if (parsers.isComment(line)) continue; considered += 1; try w.writeAll(line); try w.writeByte('\n'); } } /// Whether two compiled files carry the bodies `expected` was taken over. fn compiledBodiesMatch(list_bytes: []const u8, wild_bytes: []const u8, expected: []const u8) bool { return std.mem.eql(u8, expected, &bodyChecksum(stripHeader(list_bytes), stripHeader(wild_bytes))); } /// A compile that produced no entry at all while rejecting lines is an error /// page, a compressed body or a format the sniff got wrong — not a blocklist. /// Publishing it would replace a working list with nothing and report `ok`. An /// input that rejected nothing is an empty list, which is legal. fn rejectedWithoutEntries(counts: compiler.Counts) bool { if (counts.domains != 0 or counts.wildcards != 0) return false; return counts.invalid != 0 or counts.skipped_unsupported != 0 or counts.long_lines != 0; } fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 { var hasher = Sha256.init(.{}); hasher.update(list_body); hasher.update(wild_body); var digest: [Sha256.digest_length]u8 = undefined; hasher.final(&digest); return std.fmt.bytesToHex(digest, .lower); } fn compiledName(buf: *[name_buf_len]u8, id: i64, suffix: []const u8) []const u8 { // An `i64` prints in at most 20 characters and the longest suffix is nine, // so `name_buf_len` cannot be exceeded. return std.fmt.bufPrint(buf, "{d}{s}", .{ id, suffix }) catch unreachable; } /// The source id a compiled file belongs to, or null when the name is not one /// of ours. Temporary files are deliberately not matched: they belong to a /// refresh that may still be running. fn compiledId(file_name: []const u8) ?i64 { const stem = if (std.mem.endsWith(u8, file_name, ".list")) file_name[0 .. file_name.len - ".list".len] else if (std.mem.endsWith(u8, file_name, ".wild")) file_name[0 .. file_name.len - ".wild".len] else return null; return std.fmt.parseInt(i64, stem, 10) catch null; } fn containsId(rows: []const sources_repo.SourceRow, id: i64) bool { for (rows) |row| { if (row.id == id) return true; } return false; } // --------------------------------------------------------------------------- // tests // --------------------------------------------------------------------------- // // Everything here runs against a `:memory:` database and touches no file. Real // files, real HTTP and real swaps are the integration suite's (S9). const testing = std.testing; const migrations = @import("../storage/migrations.zig"); fn openMigrated() !db.Db { var database = try db.Db.open(":memory:", .{ .mode = .memory }); errdefer database.close(); try db.applyPragmas(&database, .{}); _ = try migrations.migrate(&database); return database; } fn testManager(database: *db.Db, fetcher_ptr: *fetcher.Fetcher) !Manager { return Manager.init( testing.allocator, database, // No test in this file reaches the filesystem: `acquire` answers before // any directory is touched, and the header helpers are pure. .{ .dir = std.Io.Dir.cwd() }, fetcher_ptr, .{}, .{ .raw = .fromSeconds(30), .clock = .awake }, ); } test "init leaves the manager with no snapshot and no statuses" { var threaded: std.Io.Threaded = .init(testing.allocator, .{}); defer threaded.deinit(); const io = threaded.io(); var database = try openMigrated(); defer database.close(); var f: fetcher.Fetcher = undefined; var manager = try testManager(&database, &f); defer manager.deinit(io); try testing.expectEqual(@as(u64, 0), manager.generation); try testing.expectEqual(@as(usize, 0), manager.statuses.len); try testing.expect(manager.current == null); } test "acquire before any reload returns null and holds no lock" { var threaded: std.Io.Threaded = .init(testing.allocator, .{}); defer threaded.deinit(); const io = threaded.io(); var database = try openMigrated(); defer database.close(); var f: fetcher.Fetcher = undefined; var manager = try testManager(&database, &f); defer manager.deinit(io); try testing.expect(manager.acquire(io) == null); // A retained shared lock would make this exclusive lock block forever. try testing.expect(manager.lock.tryLock(io)); manager.lock.unlock(io); } test "the disk gate skips a scheduled refresh only while writes are critical" { var threaded: std.Io.Threaded = .init(testing.allocator, .{}); defer threaded.deinit(); const io = threaded.io(); var database = try openMigrated(); defer database.close(); var f: fetcher.Fetcher = undefined; var manager = try testManager(&database, &f); defer manager.deinit(io); // No monitor: every pass runs, which is what the tests and `check` rely on. try testing.expect(!manager.refreshGated()); try testing.expectEqual(@as(u64, 0), manager.refreshesGated()); var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null); manager.monitor = &monitor; // `.ok` and `.warn` both allow writes: only `critical` stops them. try testing.expect(!manager.refreshGated()); monitor.state_raw.store(@intFromEnum(disk_monitor.State.warn), .monotonic); try testing.expect(!manager.refreshGated()); try testing.expectEqual(@as(u64, 0), manager.refreshesGated()); monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic); try testing.expect(manager.refreshGated()); try testing.expect(manager.refreshGated()); try testing.expectEqual(@as(u64, 2), manager.refreshesGated()); // Free space recovers and the schedule resumes; the counter keeps its total. monitor.state_raw.store(@intFromEnum(disk_monitor.State.ok), .monotonic); try testing.expect(!manager.refreshGated()); try testing.expectEqual(@as(u64, 2), manager.refreshesGated()); } test "statusSnapshot on an empty manager copies nothing" { var threaded: std.Io.Threaded = .init(testing.allocator, .{}); defer threaded.deinit(); const io = threaded.io(); var database = try openMigrated(); defer database.close(); var f: fetcher.Fetcher = undefined; var manager = try testManager(&database, &f); defer manager.deinit(io); var out: [4]SourceStatus = undefined; try testing.expectEqual(@as(usize, 0), manager.statusSnapshot(io, &out)); } test "the header writer produces the documented text" { var buf: [512]u8 = undefined; var w: std.Io.Writer = .fixed(&buf); const header: Header = .{ .url = "https://lists.example/hosts.txt", .format = .hosts, .fetched_at = 1_700_000_000, .counts = .{ .domains = 12, .wildcards = 3, .skipped_regex = 2, .skipped_unsupported = 1, .invalid = 5, .long_lines = 9, .duplicates = 4, }, .checksum = "0" ** 64, }; try header.write(&w); try testing.expectEqualStrings( \\# nxdns blocklist \\# url https://lists.example/hosts.txt \\# format hosts \\# fetched_at 1700000000 \\# domains 12 \\# wildcards 3 \\# skipped_regex 2 \\# skipped_unsupported 1 \\# invalid 5 \\ ++ "# sha256 " ++ "0" ** 64 ++ "\n", w.buffered()); } test "stripHeader returns the body of a compiled file" { const file = "# nxdns blocklist\n" ++ "# url https://lists.example/hosts.txt\n" ++ "ads.example.com\ntracker.example.net\n"; try testing.expectEqualStrings("ads.example.com\ntracker.example.net\n", stripHeader(file)); } test "stripHeader returns everything for a file with no header" { try testing.expectEqualStrings("a.example.com\n", stripHeader("a.example.com\n")); } test "stripHeader returns an empty body for a header-only file" { try testing.expectEqualStrings("", stripHeader("# nxdns blocklist\n# sha256 x\n")); } test "stripHeader tolerates an unterminated header line" { try testing.expectEqualStrings("", stripHeader("# nxdns blocklist")); } test "SourceStatus truncates a long error at max_error_len" { var status: SourceStatus = .{ .id = 1 }; const long = "E" ** (max_error_len + 40); status.fail(.fetch_failed, long); try testing.expectEqual(State.fetch_failed, status.state); try testing.expectEqual(@as(u8, max_error_len), status.last_error_len); try testing.expectEqualStrings("E" ** max_error_len, status.errorText()); } test "a success clears the recorded error" { var status: SourceStatus = .{ .id = 7 }; status.fail(.compile_failed, "TooManyDomains"); status.succeed(1_700_000_000, .{ .domains = 3, .wildcards = 1 }); try testing.expectEqual(State.ok, status.state); try testing.expectEqual(@as(i64, 1_700_000_000), status.last_success); try testing.expectEqual(@as(u32, 3), status.counts.domains); try testing.expectEqualStrings("", status.errorText()); } test "compiledName spells the four file names of a source" { var buf: [name_buf_len]u8 = undefined; try testing.expectEqualStrings("42.list", compiledName(&buf, 42, ".list")); try testing.expectEqualStrings("42.wild", compiledName(&buf, 42, ".wild")); try testing.expectEqualStrings("42.raw.tmp", compiledName(&buf, 42, ".raw.tmp")); try testing.expectEqualStrings("42.list.tmp", compiledName(&buf, 42, ".list.tmp")); } test "compiledId matches compiled files and nothing else" { try testing.expectEqual(@as(?i64, 7), compiledId("7.list")); try testing.expectEqual(@as(?i64, 7), compiledId("7.wild")); try testing.expectEqual(@as(?i64, null), compiledId("7.list.tmp")); try testing.expectEqual(@as(?i64, null), compiledId("7.raw.tmp")); try testing.expectEqual(@as(?i64, null), compiledId("notes.list")); try testing.expectEqual(@as(?i64, null), compiledId("README")); } test "a failed refresh keeps the fields of the compiled files still serving" { var status: SourceStatus = .{ .id = 3 }; status.succeed(1_700_000_000, .{ .domains = 5, .wildcards = 2 }); // What `refreshSourceLocked` starts from, and what a download failure does // to it. var next = status; next.last_attempt = 1_700_003_600; next.fail(.fetch_failed, "Timeout"); try testing.expectEqual(State.fetch_failed, next.state); try testing.expectEqualStrings("Timeout", next.errorText()); try testing.expectEqual(@as(i64, 1_700_000_000), next.last_success); try testing.expectEqual(@as(i64, 1_700_003_600), next.last_attempt); try testing.expectEqual(@as(u32, 5), next.counts.domains); try testing.expectEqual(@as(u32, 2), next.counts.wildcards); } test "a load outcome never overwrites a refresh failure" { try testing.expect(State.fetch_failed.isRefreshFailure()); try testing.expect(State.compile_failed.isRefreshFailure()); try testing.expect(State.no_valid_entries.isRefreshFailure()); // The three a load produces. `applyLoadOutcomes` may write over these, // because nothing more informative is there. try testing.expect(!State.ok.isRefreshFailure()); try testing.expect(!State.never_fetched.isRefreshFailure()); try testing.expect(!State.load_failed.isRefreshFailure()); } fn testRow(id: i64, enabled: bool) sources_repo.SourceRow { return .{ .id = id, .url = "https://lists.example/hosts.txt", .name = "example", .enabled = enabled, .last_updated = 1_700_000_000, .domain_count = 9, .wildcard_count = 4, .skipped_regex_count = 1, .checksum = "0" ** 64, }; } test "a candidate table carries prior entries over and leaves the published one alone" { var published = [_]SourceStatus{ .{ .id = 1 }, .{ .id = 2 } }; published[0].setUrl("https://lists.example/one.txt"); published[0].fail(.fetch_failed, "HttpStatus"); published[0].loaded = true; published[1].setUrl("https://lists.example/two.txt"); published[1].succeed(1_700_000_000, .{ .domains = 4 }); // Source 2 was deleted and source 3 added; source 1 kept its id and got a // new url. const rows = [_]sources_repo.SourceRow{ blk: { var row = testRow(1, true); row.url = "https://lists.example/moved.txt"; break :blk row; }, testRow(3, true), }; var candidate: [2]SourceStatus = undefined; mergeStatuses(&candidate, &rows, &published); try testing.expectEqual(@as(i64, 1), candidate[0].id); try testing.expectEqual(State.fetch_failed, candidate[0].state); try testing.expectEqualStrings("HttpStatus", candidate[0].errorText()); try testing.expect(candidate[0].loaded); try testing.expectEqualStrings("https://lists.example/moved.txt", candidate[0].urlText()); try testing.expectEqual(@as(i64, 3), candidate[1].id); try testing.expectEqual(State.never_fetched, candidate[1].state); try testing.expect(!candidate[1].loaded); // The published table is untouched, so a reload that fails before the swap // leaves it describing the snapshot that is still serving — including the // entry of the deleted source, which that snapshot still enforces. try testing.expectEqual(@as(usize, 2), published.len); try testing.expectEqual(@as(i64, 2), published[1].id); try testing.expectEqual(State.ok, published[1].state); try testing.expectEqualStrings("https://lists.example/one.txt", published[0].urlText()); } test "a disabled source stops being loaded" { var statuses = [_]SourceStatus{.{ .id = 1 }}; statuses[0].succeed(1_700_000_000, .{ .domains = 9 }); statuses[0].loaded = true; const rows = [_]sources_repo.SourceRow{testRow(1, false)}; applyLoadOutcomes(&statuses, &rows, &.{.disabled}); // Nothing enforces it any more, and the state that described the files it // used to serve is left as the record of how it last stood. try testing.expect(!statuses[0].loaded); try testing.expectEqual(State.ok, statuses[0].state); } test "a source that failed to refresh keeps its failure while its old files serve" { // What `refreshSourceLocked` records, then what the `reload` that follows // it in `refreshAll` finds: the previous files still load. var statuses = [_]SourceStatus{.{ .id = 1 }}; statuses[0].succeed(1_700_000_000, .{ .domains = 9 }); statuses[0].fail(.fetch_failed, "HttpStatus"); const rows = [_]sources_repo.SourceRow{testRow(1, true)}; const body: matcher.Snapshot.Compiled = .{ .list_body = "", .wild_body = "" }; applyLoadOutcomes(&statuses, &rows, &.{.{ .loaded = body }}); try testing.expect(statuses[0].loaded); try testing.expectEqual(State.fetch_failed, statuses[0].state); try testing.expectEqualStrings("HttpStatus", statuses[0].errorText()); try testing.expectEqual(@as(u32, 9), statuses[0].counts.domains); } test "a load failure is recorded when no refresh failure explains it" { var statuses = [_]SourceStatus{ .{ .id = 1 }, .{ .id = 2 } }; statuses[0].succeed(1_700_000_000, .{ .domains = 9 }); statuses[0].loaded = true; statuses[1].fail(.compile_failed, "TooManyDomains"); statuses[1].loaded = true; const rows = [_]sources_repo.SourceRow{ testRow(1, true), testRow(2, true) }; const reason: LoadOutcome = .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } }; applyLoadOutcomes(&statuses, &rows, &.{ reason, reason }); try testing.expect(!statuses[0].loaded); try testing.expectEqual(State.load_failed, statuses[0].state); try testing.expectEqualStrings("ChecksumMismatch", statuses[0].errorText()); // The compile failure is why the files are unusable; it outranks the // symptom the loader saw. try testing.expect(!statuses[1].loaded); try testing.expectEqual(State.compile_failed, statuses[1].state); try testing.expectEqualStrings("TooManyDomains", statuses[1].errorText()); } test "a load of a source this process never refreshed takes the row counters" { var statuses = [_]SourceStatus{.{ .id = 1 }}; const rows = [_]sources_repo.SourceRow{testRow(1, true)}; const body: matcher.Snapshot.Compiled = .{ .list_body = "", .wild_body = "" }; applyLoadOutcomes(&statuses, &rows, &.{.{ .loaded = body }}); try testing.expect(statuses[0].loaded); try testing.expectEqual(State.ok, statuses[0].state); try testing.expectEqual(@as(i64, 1_700_000_000), statuses[0].last_success); try testing.expectEqual(@as(u32, 9), statuses[0].counts.domains); try testing.expectEqual(@as(u32, 4), statuses[0].counts.wildcards); try testing.expectEqual(@as(u32, 1), statuses[0].counts.skipped_regex); } test "a status borrows nothing, so a copy outlives the table it came from" { var status: SourceStatus = .{ .id = 5 }; status.setUrl("https://lists.example/hosts.txt"); status.fail(.load_failed, "ChecksumMismatch"); const copy = status; // The source of the original is overwritten, as a reload overwrites the // table: a copy that borrowed would read the new bytes or freed memory. status.setUrl("https://other.example/other.txt"); status.fail(.fetch_failed, "Timeout"); try testing.expectEqualStrings("https://lists.example/hosts.txt", copy.urlText()); try testing.expectEqualStrings("ChecksumMismatch", copy.errorText()); try testing.expectEqual(State.load_failed, copy.state); } test "SourceStatus truncates a long url at max_url_len" { var status: SourceStatus = .{ .id = 6 }; status.setUrl("https://lists.example/" ++ "p" ** max_url_len); try testing.expectEqual(@as(u8, max_url_len), status.url_len); try testing.expectEqualStrings( ("https://lists.example/" ++ "p" ** max_url_len)[0..max_url_len], status.urlText(), ); // A shorter url must not leave the tail of the longer one behind it. status.setUrl("https://a.example/x"); try testing.expectEqualStrings("https://a.example/x", status.urlText()); } test "compiledBodiesMatch verifies the bodies, not the presence of the files" { const list_body = "a.example.com\nb.example.com\n"; const wild_body = "c.example.com\n"; const expected = bodyChecksum(list_body, wild_body); const header = "# nxdns blocklist\n" ++ "# url https://lists.example/hosts.txt\n"; try testing.expect(compiledBodiesMatch(header ++ list_body, header ++ wild_body, &expected)); // The corruption a reload reports as `ChecksumMismatch`: the file is there, // its body is not what the checksum was taken over. try testing.expect(!compiledBodiesMatch(header ++ "a.example.com\nb.exa", header ++ wild_body, &expected)); try testing.expect(!compiledBodiesMatch("", "", &expected)); } test "rejectedWithoutEntries fails a compile that produced nothing usable" { // An html error page: every line is rejected, nothing is written. try testing.expect(rejectedWithoutEntries(.{ .invalid = 12, .skipped_unsupported = 3 })); // A compressed body: one long binary run with no newline in it. try testing.expect(rejectedWithoutEntries(.{ .long_lines = 1 })); // An empty list rejects nothing and is legal. try testing.expect(!rejectedWithoutEntries(.{})); // A real list rejects lines and still produces entries. try testing.expect(!rejectedWithoutEntries(.{ .domains = 1000, .invalid = 40 })); try testing.expect(!rejectedWithoutEntries(.{ .wildcards = 7, .skipped_unsupported = 90 })); } fn sampleOf(input: []const u8, out: []u8) ![]const u8 { var r: std.Io.Reader = .fixed(input); var w: std.Io.Writer = .fixed(out); try collectSample(&r, &w); return w.buffered(); } test "collectSample skips comments instead of spending the sample on them" { const gpa = testing.allocator; const long_comment = "# " ++ "c" ** (compiler.max_line_len - 2) ++ "\n"; var input: std.ArrayList(u8) = .empty; defer input.deinit(gpa); // Sixteen of these fill a 64 KiB window on their own. for (0..20) |_| try input.appendSlice(gpa, long_comment); try input.appendSlice(gpa, "0.0.0.0 ads.example.com\n0.0.0.0 tracker.example.net\n"); const out = try gpa.alloc(u8, sample_buf_len); defer gpa.free(out); const sample = try sampleOf(input.items, out); try testing.expectEqualStrings( "0.0.0.0 ads.example.com\n0.0.0.0 tracker.example.net\n", sample, ); try testing.expectEqual(parsers.Format.hosts, parsers.detectFormat(sample)); } test "collectSample stops at sample_lines counted lines" { const gpa = testing.allocator; var input: std.ArrayList(u8) = .empty; defer input.deinit(gpa); var line_buf: [64]u8 = undefined; for (0..parsers.sample_lines + 10) |i| { try input.appendSlice(gpa, try std.fmt.bufPrint(&line_buf, "0.0.0.0 host{d}.example.com\n", .{i})); } const out = try gpa.alloc(u8, sample_buf_len); defer gpa.free(out); const sample = try sampleOf(input.items, out); var lines = std.mem.tokenizeScalar(u8, sample, '\n'); var count: usize = 0; while (lines.next()) |_| count += 1; try testing.expectEqual(parsers.sample_lines, count); } test "collectSample keeps the abp marker a long comment run would have hidden" { const gpa = testing.allocator; const long_comment = "! " ++ "c" ** (compiler.max_line_len - 2) ++ "\n"; var input: std.ArrayList(u8) = .empty; defer input.deinit(gpa); for (0..20) |_| try input.appendSlice(gpa, long_comment); try input.appendSlice(gpa, "||ads.example.com^\n"); const out = try gpa.alloc(u8, sample_buf_len); defer gpa.free(out); const sample = try sampleOf(input.items, out); try testing.expectEqualStrings("||ads.example.com^\n", sample); try testing.expectEqual(parsers.Format.abp, parsers.detectFormat(sample)); } test "collectSample skips a line over max_line_len" { const gpa = testing.allocator; var input: std.ArrayList(u8) = .empty; defer input.deinit(gpa); try input.appendNTimes(gpa, 'x', 8 * compiler.max_line_len); try input.appendSlice(gpa, "\nads.example.com\n"); const out = try gpa.alloc(u8, sample_buf_len); defer gpa.free(out); // A `Reader.fixed` holds the whole input, so the over-long line comes back // rather than being refused. The compiler would skip it, so the sniff does. const sample = try sampleOf(input.items, out); try testing.expectEqualStrings("ads.example.com\n", sample); } test "collectSample steps over a line that does not fit the reader buffer" { const gpa = testing.allocator; var input: std.ArrayList(u8) = .empty; defer input.deinit(gpa); try input.appendNTimes(gpa, 'x', 4 * compiler.max_line_len); try input.appendSlice(gpa, "\nads.example.com\n"); // A reader buffer smaller than the long line makes `takeDelimiter` report // `error.StreamTooLong` and leave the stream where it was, which is the // path that loops forever without the discard. var backing: std.Io.Reader = .fixed(input.items); var reader_buf: [compiler.max_line_len]u8 = undefined; var limited = backing.limited(.unlimited, &reader_buf); const out = try gpa.alloc(u8, sample_buf_len); defer gpa.free(out); var w: std.Io.Writer = .fixed(out); try collectSample(&limited.interface, &w); try testing.expectEqualStrings("ads.example.com\n", w.buffered()); } test "bodyChecksum covers the list body followed by the wild body" { const both = bodyChecksum("a.example.com\n", "b.example.com\n"); var hasher = Sha256.init(.{}); hasher.update("a.example.com\nb.example.com\n"); var digest: [Sha256.digest_length]u8 = undefined; hasher.final(&digest); try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &both); // Order matters: the two halves are not interchangeable. try testing.expect(!std.mem.eql(u8, &both, &bodyChecksum("b.example.com\n", "a.example.com\n"))); }