mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 13 Activity

milestone 21: abp list exceptions and a regex rule kind

Browse Source
This commit is contained in:
mokhtar
2026-08-13 19:14:47 +02:00
parent b340521716
commit 2ab7c1f1de
51 changed files with 4016 additions and 465 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/filter/manager.zig
+219 -69
View File
@@ -38,9 +38,10 @@
//! publish: the download of one source, at up to 300 s each, and the compile
//! that follows it. It also covers blocklist-directory maintenance, because
//! those stages are the only writers of `.raw.tmp` / `.list.tmp` /
//! `.wild.tmp` and `pruneOrphans` must not sweep the temporaries of a refresh
//! that is still running. Two concurrent refreshes would share the fetcher's
//! buffers and, for one source, the same temporary paths.
//! `.wild.tmp` / `.allow.tmp` and `pruneOrphans` must not sweep the
//! temporaries of a refresh that is still running. Two concurrent refreshes
//! would share the fetcher's buffers and, for one source, the same temporary
//! paths.
//!
//! **Lock ordering: `refresh_lock` is never acquired while `writer_lock` is
//! held.** A path that needs both takes `refresh_lock` first. The public entry
@@ -94,7 +95,7 @@ const io_buf_len: usize = 64 * 1024;
/// would then be decided by almost no data.
const sample_buf_len: usize = parsers.sample_lines * (compiler.max_line_len + 1);
/// `<id>` is at most 20 characters and the longest suffix is `.list.tmp`.
/// `<id>` is at most 20 characters and the longest suffix is `.allow.tmp`.
const name_buf_len: usize = 48;
/// How one blocklist source is named in a log line: by its row id and its name,
@@ -215,9 +216,13 @@ pub const SourceStatus = struct {
};
/// The header every compiled file carries, ahead of the body. The `sha256`
/// covers the `.list` body followed by the `.wild` body and **not** the header,
/// so it stays stable across a refetch of unchanged content while
/// `fetched_at` moves.
/// covers the `.list` body, then the `.wild` body, then the `.allow` body, and
/// **not** the header, so it stays stable across a refetch of unchanged content
/// while `fetched_at` moves.
///
/// The `.allow` body is hashed last so that a source with no exceptions keeps
/// the digest it had when only two bodies existed: every checksum written before
/// exceptions were honoured stays valid, and no upgrade forces a refetch.
pub const Header = struct {
url: []const u8,
format: parsers.Format,
@@ -233,6 +238,7 @@ pub const Header = struct {
try w.print("# fetched_at {d}\n", .{self.fetched_at});
try w.print("# domains {d}\n", .{self.counts.domains});
try w.print("# wildcards {d}\n", .{self.counts.wildcards});
try w.print("# exceptions {d}\n", .{self.counts.exceptions});
try w.print("# skipped_regex {d}\n", .{self.counts.skipped_regex});
try w.print("# skipped_unsupported {d}\n", .{self.counts.skipped_unsupported});
try w.print("# invalid {d}\n", .{self.counts.invalid});
@@ -557,12 +563,14 @@ pub const Manager = struct {
var list_buf: [name_buf_len]u8 = undefined;
var wild_buf: [name_buf_len]u8 = undefined;
var allow_buf: [name_buf_len]u8 = undefined;
const list_name = compiledName(&list_buf, row.id, ".list");
const wild_name = compiledName(&wild_buf, row.id, ".wild");
const allow_name = compiledName(&allow_buf, row.id, ".allow");
// Reserved before the reads, so neither buffer can be orphaned by a
// failing append: `bodies` owns each one from the moment it is read.
try bodies.ensureUnusedCapacity(self.gpa, 2);
// Reserved before the reads, so no buffer can be orphaned by a failing
// append: `bodies` owns each one from the moment it is read.
try bodies.ensureUnusedCapacity(self.gpa, 3);
// `error.Canceled` is the one-shot signal that this task is being torn
// down, and it is consumed by whoever catches it. Recording it as a
@@ -583,18 +591,39 @@ pub const Manager = struct {
};
bodies.appendAssumeCapacity(wild_bytes);
// A missing `.allow` file is an empty allow body, not a failure. Two
// sources are in that state and both are ordinary: one compiled before
// exceptions were honoured, and one whose list carries no `@@` line.
// Because the empty body contributes nothing to the checksum, the
// stored digest of either still matches.
const allow_bytes: []const u8 = blk: {
const read = dir.readFileAlloc(io, allow_name, self.gpa, .limited(max_compiled_bytes)) catch |err| {
if (err == error.OutOfMemory) return error.OutOfMemory;
if (err == error.Canceled) return error.Canceled;
if (err == error.FileNotFound) break :blk "";
return loadFailure(row, allow_name, err);
};
bodies.appendAssumeCapacity(read);
break :blk read;
};
const list_body = stripHeader(list_bytes);
const wild_body = stripHeader(wild_bytes);
const allow_body = stripHeader(allow_bytes);
// The checksum covers both bodies together, so a crash between the two
// The checksum covers the three bodies together, so a crash between the
// `replace` calls — a new `.list` beside an old `.wild` — is caught
// here and refreshed, not served as a half-updated list.
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body))) {
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body, allow_body))) {
log.warn("blocklist {f}: compiled files do not match the stored checksum", .{SourceLabel.of(row)});
return .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } };
}
return .{ .loaded = .{ .list_body = list_body, .wild_body = wild_body } };
return .{ .loaded = .{
.list_body = list_body,
.wild_body = wild_body,
.allow_body = allow_body,
} };
}
// -----------------------------------------------------------------------
@@ -635,23 +664,28 @@ pub const Manager = struct {
var raw_buf: [name_buf_len]u8 = undefined;
var list_tmp_buf: [name_buf_len]u8 = undefined;
var wild_tmp_buf: [name_buf_len]u8 = undefined;
var allow_tmp_buf: [name_buf_len]u8 = undefined;
const raw_name = compiledName(&raw_buf, row.id, ".raw.tmp");
const list_tmp = compiledName(&list_tmp_buf, row.id, ".list.tmp");
const wild_tmp = compiledName(&wild_tmp_buf, row.id, ".wild.tmp");
const tmp: TempNames = .{
.list = compiledName(&list_tmp_buf, row.id, ".list.tmp"),
.wild = compiledName(&wild_tmp_buf, row.id, ".wild.tmp"),
.allow = compiledName(&allow_tmp_buf, row.id, ".allow.tmp"),
};
// Installed before the calls that create these files, not after: an
// `error.Canceled` or `error.OutOfMemory` returned straight out of
// `download` or `compileTo` would outrun a later `defer` and leave a
// temporary behind. Deleting a name that was never created is a no-op.
defer self.deleteQuietly(io, dir, raw_name);
defer self.deleteQuietly(io, dir, list_tmp);
defer self.deleteQuietly(io, dir, wild_tmp);
defer self.deleteQuietly(io, dir, tmp.list);
defer self.deleteQuietly(io, dir, tmp.wild);
defer self.deleteQuietly(io, dir, tmp.allow);
// The half that takes the time: one download of up to `total_budget`
// and one compile of everything it returned. `refresh_lock` alone is
// held here, so a rule save, a settings change or any other web
// mutation that ends in `reload` runs beside it instead of behind it.
const prepared = try self.prepareRefresh(io, dir, row, &status, raw_name, list_tmp, wild_tmp);
const prepared = try self.prepareRefresh(io, dir, row, &status, raw_name, tmp);
// The half that publishes. The compiled files, the runtime columns and
// the status entry land under one `writer_lock`, so a reload never
@@ -659,7 +693,7 @@ pub const Manager = struct {
self.writer_lock.lockUncancelable(io);
defer self.writer_lock.unlock(io);
const replaced = try self.publishRefresh(io, dir, row, &status, prepared, list_tmp, wild_tmp);
const replaced = try self.publishRefresh(io, dir, row, &status, prepared, tmp);
self.commitStatus(io, status);
return replaced;
}
@@ -690,6 +724,14 @@ pub const Manager = struct {
return self.reload(io);
}
/// The three temporary files one refresh compiles into, before the header
/// is prepended and each is renamed over the file it replaces.
const TempNames = struct {
list: []const u8,
wild: []const u8,
allow: []const u8,
};
/// What the fetch-and-compile half of a refresh produced. `.failed` needs
/// no publish and has already recorded why in the status entry.
const Prepared = union(enum) {
@@ -712,8 +754,7 @@ pub const Manager = struct {
row: sources_repo.SourceRow,
status: *SourceStatus,
raw_name: []const u8,
list_tmp: []const u8,
wild_tmp: []const u8,
tmp: TempNames,
) Error!Prepared {
self.download(io, dir, raw_name, row) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
@@ -733,7 +774,7 @@ pub const Manager = struct {
},
};
const result = self.compileTo(io, dir, raw_name, format, list_tmp, wild_tmp) catch |err| switch (err) {
const result = self.compileTo(io, dir, raw_name, format, tmp) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.Canceled => return error.Canceled,
else => {
@@ -762,8 +803,7 @@ pub const Manager = struct {
row: sources_repo.SourceRow,
status: *SourceStatus,
prepared: Prepared,
list_tmp: []const u8,
wild_tmp: []const u8,
tmp: TempNames,
) Error!bool {
const compiled = switch (prepared) {
.failed => return false,
@@ -786,6 +826,7 @@ pub const Manager = struct {
.last_updated = now,
.domain_count = row.domain_count,
.wildcard_count = row.wildcard_count,
.exception_count = row.exception_count,
.skipped_regex_count = row.skipped_regex_count,
.checksum = stored,
});
@@ -801,7 +842,7 @@ pub const Manager = struct {
.counts = compiled.result.counts,
.checksum = &compiled.result.checksum,
};
self.publish(io, dir, row.id, header, list_tmp, wild_tmp) catch |err| switch (err) {
self.publish(io, dir, row.id, header, tmp) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.Canceled => return error.Canceled,
else => {
@@ -814,6 +855,7 @@ pub const Manager = struct {
.last_updated = now,
.domain_count = compiled.result.counts.domains,
.wildcard_count = compiled.result.counts.wildcards,
.exception_count = compiled.result.counts.exceptions,
.skipped_regex_count = compiled.result.counts.skipped_regex,
.checksum = &compiled.result.checksum,
});
@@ -913,7 +955,7 @@ pub const Manager = struct {
return parsers.detectFormat(sample.buffered());
}
/// Compiles into two plain temporary files. The compiled bodies cannot go
/// Compiles into three plain temporary files. The compiled bodies cannot go
/// straight into the final files: the header carries counts that only exist
/// once the whole input has been compiled, and the loader requires the
/// header first.
@@ -923,22 +965,24 @@ pub const Manager = struct {
dir: std.Io.Dir,
raw_name: []const u8,
format: parsers.Format,
list_tmp: []const u8,
wild_tmp: []const u8,
tmp: TempNames,
) !compiler.Result {
const raw = try dir.openFile(io, raw_name, .{});
defer raw.close(io);
const list_file = try dir.createFile(io, list_tmp, .{ .permissions = .fromMode(0o600) });
const list_file = try dir.createFile(io, tmp.list, .{ .permissions = .fromMode(0o600) });
defer list_file.close(io);
const wild_file = try dir.createFile(io, wild_tmp, .{ .permissions = .fromMode(0o600) });
const wild_file = try dir.createFile(io, tmp.wild, .{ .permissions = .fromMode(0o600) });
defer wild_file.close(io);
const allow_file = try dir.createFile(io, tmp.allow, .{ .permissions = .fromMode(0o600) });
defer allow_file.close(io);
const buffers = try self.gpa.alloc(u8, 3 * io_buf_len);
const buffers = try self.gpa.alloc(u8, 4 * io_buf_len);
defer self.gpa.free(buffers);
var fr = raw.reader(io, buffers[0..io_buf_len]);
var list_w = list_file.writer(io, buffers[io_buf_len .. 2 * io_buf_len]);
var wild_w = wild_file.writer(io, buffers[2 * io_buf_len ..]);
var wild_w = wild_file.writer(io, buffers[2 * io_buf_len .. 3 * io_buf_len]);
var allow_w = allow_file.writer(io, buffers[3 * io_buf_len ..]);
const result = compiler.compile(
self.gpa,
@@ -946,18 +990,22 @@ pub const Manager = struct {
format,
&list_w.interface,
&wild_w.interface,
&allow_w.interface,
) catch |err| switch (err) {
// `compiler.Error` names the direction; the concrete cause is on
// the stream that failed.
error.ReadFailed => return fr.err orelse err,
error.WriteFailed => return list_w.err orelse (wild_w.err orelse err),
error.WriteFailed => return list_w.err orelse
(wild_w.err orelse (allow_w.err orelse err)),
else => return err,
};
try list_w.interface.flush();
try wild_w.interface.flush();
try allow_w.interface.flush();
try list_file.sync(io);
try wild_file.sync(io);
try allow_file.sync(io);
return result;
}
@@ -970,16 +1018,17 @@ pub const Manager = struct {
dir: std.Io.Dir,
id: i64,
header: Header,
list_tmp: []const u8,
wild_tmp: []const u8,
tmp: TempNames,
) !void {
const buffers = try self.gpa.alloc(u8, 2 * io_buf_len);
defer self.gpa.free(buffers);
var list_buf: [name_buf_len]u8 = undefined;
var wild_buf: [name_buf_len]u8 = undefined;
try publishOne(io, dir, compiledName(&list_buf, id, ".list"), list_tmp, header, buffers);
try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), wild_tmp, header, buffers);
var allow_buf: [name_buf_len]u8 = undefined;
try publishOne(io, dir, compiledName(&list_buf, id, ".list"), tmp.list, header, buffers);
try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), tmp.wild, header, buffers);
try publishOne(io, dir, compiledName(&allow_buf, id, ".allow"), tmp.allow, header, buffers);
}
fn publishOne(
@@ -1014,12 +1063,19 @@ pub const Manager = struct {
try af.replace(io);
}
/// Whether the two compiled files on disk hash to `expected`. A missing,
/// Whether the compiled files on disk hash to `expected`. A missing,
/// unreadable or corrupt file answers false, which sends the caller down
/// the rewrite path — the only path that can repair it.
///
/// A missing `.allow` file is the one exception, and it is the same one
/// `loadSource` makes: a source compiled before exceptions were honoured has
/// no such file, and its stored checksum was taken over an empty allow body.
/// Answering false there would rewrite every list on the first refresh after
/// an upgrade for no change in content.
fn diskBodiesMatch(self: *Manager, io: std.Io, dir: std.Io.Dir, id: i64, expected: []const u8) bool {
var list_buf: [name_buf_len]u8 = undefined;
var wild_buf: [name_buf_len]u8 = undefined;
var allow_buf: [name_buf_len]u8 = undefined;
const limit: std.Io.Limit = .limited(max_compiled_bytes);
const list_bytes = dir.readFileAlloc(io, compiledName(&list_buf, id, ".list"), self.gpa, limit) catch
@@ -1029,7 +1085,11 @@ pub const Manager = struct {
return false;
defer self.gpa.free(wild_bytes);
return compiledBodiesMatch(list_bytes, wild_bytes, expected);
const allow_bytes = dir.readFileAlloc(io, compiledName(&allow_buf, id, ".allow"), self.gpa, limit) catch |err|
if (err == error.FileNotFound) @as([]u8, &.{}) else return false;
defer self.gpa.free(allow_bytes);
return compiledBodiesMatch(list_bytes, wild_bytes, allow_bytes, expected);
}
fn reportFetchFailure(
@@ -1231,12 +1291,12 @@ pub const Manager = struct {
/// sweeps to nothing.
pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
// `refresh_lock` first, and for the reason it exists: the download and
// the compile are the only writers of `.raw.tmp`, `.list.tmp` and
// `.wild.tmp`, and they hold it for as long as they run. Without it
// here, a source deleted through the API would sweep the temporaries of
// a refresh still writing them — the row is gone, so nothing else in
// this function would spare them — and the pass would fail on a raw
// file that vanished under it.
// the compile are the only writers of `.raw.tmp`, `.list.tmp`,
// `.wild.tmp` and `.allow.tmp`, and they hold it for as long as they
// run. Without it here, a source deleted through the API would sweep
// the temporaries of a refresh still writing them — the row is gone, so
// nothing else in this function would spare them — and the pass would
// fail on a raw file that vanished under it.
//
// `writer_lock` second, in the one order this file ever takes them,
// because the rows this reads and the compiled files it deletes are
@@ -1485,6 +1545,7 @@ fn applyLoadOutcomes(
entry.succeed(row.last_updated orelse 0, .{
.domains = countOf(row.domain_count),
.wildcards = countOf(row.wildcard_count),
.exceptions = countOf(row.exception_count),
.skipped_regex = countOf(row.skipped_regex_count),
});
},
@@ -1555,43 +1616,62 @@ fn collectSample(r: *std.Io.Reader, w: *std.Io.Writer) error{ ReadFailed, WriteF
}
}
/// Whether two compiled files carry the bodies `expected` was taken over.
fn compiledBodiesMatch(list_bytes: []const u8, wild_bytes: []const u8, expected: []const u8) bool {
return std.mem.eql(u8, expected, &bodyChecksum(stripHeader(list_bytes), stripHeader(wild_bytes)));
/// Whether three compiled files carry the bodies `expected` was taken over.
fn compiledBodiesMatch(
list_bytes: []const u8,
wild_bytes: []const u8,
allow_bytes: []const u8,
expected: []const u8,
) bool {
return std.mem.eql(u8, expected, &bodyChecksum(
stripHeader(list_bytes),
stripHeader(wild_bytes),
stripHeader(allow_bytes),
));
}
/// A compile that produced no entry at all while rejecting lines is an error
/// page, a compressed body or a format the sniff got wrong — not a blocklist.
/// Publishing it would replace a working list with nothing and report `ok`. An
/// input that rejected nothing is an empty list, which is legal.
///
/// A list of nothing but exceptions is loadable: an allow-only list published
/// beside a blocking one is a shape operators use, and it produces entries.
fn rejectedWithoutEntries(counts: compiler.Counts) bool {
if (counts.domains != 0 or counts.wildcards != 0) return false;
if (counts.domains != 0 or counts.wildcards != 0 or counts.exceptions != 0) return false;
return counts.invalid != 0 or counts.skipped_unsupported != 0 or counts.long_lines != 0;
}
fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 {
/// The digest the `.list`, `.wild` and `.allow` bodies share, in that order.
/// The allow body comes last so that hashing an empty one leaves the digest of
/// the two-body form untouched, which is what keeps every checksum stored before
/// exceptions were honoured valid.
fn bodyChecksum(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) [64]u8 {
var hasher = Sha256.init(.{});
hasher.update(list_body);
hasher.update(wild_body);
hasher.update(allow_body);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
return std.fmt.bytesToHex(digest, .lower);
}
fn compiledName(buf: *[name_buf_len]u8, id: i64, suffix: []const u8) []const u8 {
// An `i64` prints in at most 20 characters and the longest suffix is nine,
// An `i64` prints in at most 20 characters and the longest suffix is ten,
// so `name_buf_len` cannot be exceeded.
return std.fmt.bufPrint(buf, "{d}{s}", .{ id, suffix }) catch unreachable;
}
/// Every name `compiledName` can produce, longest suffix first so `.list.tmp`
/// is never read as `.list`.
const source_file_suffixes = [_][]const u8{ ".list.tmp", ".wild.tmp", ".raw.tmp", ".list", ".wild" };
const source_file_suffixes = [_][]const u8{
".allow.tmp", ".list.tmp", ".wild.tmp", ".raw.tmp", ".allow", ".list", ".wild",
};
/// The source id a file under the blocklist directory belongs to, or null when
/// the name is not one of ours.
///
/// The three temporaries count. A refresh that dies between writing one and
/// The four temporaries count. A refresh that dies between writing one and
/// renaming it leaves a file no later refresh reuses and no `defer` reaches, so
/// excluding them from the sweep means nothing ever removes them. Matching them
/// is safe because `pruneOrphans` holds `refresh_lock` for its whole body:
@@ -1830,18 +1910,24 @@ test "a canceled compiled-file read cancels the reload instead of recording it"
const list_body = "aaa.example.com\n";
const wild_body = "";
const allow_body = "";
var dir = try tmp.dir.createDirPathOpen(io, "blocklists", .{});
defer dir.close(io);
var list_buf: [name_buf_len]u8 = undefined;
var wild_buf: [name_buf_len]u8 = undefined;
var allow_buf: [name_buf_len]u8 = undefined;
try dir.writeFile(io, .{ .sub_path = compiledName(&list_buf, id, ".list"), .data = list_body });
try dir.writeFile(io, .{ .sub_path = compiledName(&wild_buf, id, ".wild"), .data = wild_body });
// Present rather than absent, so the third read is a real one: `loadSource`
// treats a missing `.allow` as an empty body and would never open it.
try dir.writeFile(io, .{ .sub_path = compiledName(&allow_buf, id, ".allow"), .data = allow_body });
try sources_repo.updateSourceStats(&database, id, .{
.last_updated = 1_700_000_000,
.domain_count = 1,
.wildcard_count = 0,
.skipped_regex_count = 0,
.checksum = &bodyChecksum(list_body, wild_body),
.exception_count = 0,
.checksum = &bodyChecksum(list_body, wild_body, allow_body),
});
// The baseline every assertion below is against: one clean reload, one
@@ -1853,11 +1939,12 @@ test "a canceled compiled-file read cancels the reload instead of recording it"
try testing.expect(out[0].loaded);
const published = mgr.generation;
// Both catch sites, in the order `loadSource` reads the two files. A
// Every catch site, in the order `loadSource` reads the three files. A
// cancellation is consumed by whoever catches it, so folding it into a load
// failure would spend the shutdown signal and leave a status row reading
// "Canceled" behind.
for ([_][]const u8{ ".list", ".wild" }) |suffix| {
// "Canceled" behind. The `.allow` read is the one that can get this wrong
// twice over: it also has to keep `FileNotFound` apart from a cancellation.
for ([_][]const u8{ ".list", ".wild", ".allow" }) |suffix| {
var vtable: std.Io.VTable = undefined;
const canceling = cancelingIo(io, suffix, &vtable);
try testing.expectError(error.Canceled, mgr.reload(canceling));
@@ -1921,6 +2008,7 @@ test "the header writer produces the documented text" {
.counts = .{
.domains = 12,
.wildcards = 3,
.exceptions = 7,
.skipped_regex = 2,
.skipped_unsupported = 1,
.invalid = 5,
@@ -1938,6 +2026,7 @@ test "the header writer produces the documented text" {
\\# fetched_at 1700000000
\\# domains 12
\\# wildcards 3
\\# exceptions 7
\\# skipped_regex 2
\\# skipped_unsupported 1
\\# invalid 5
@@ -2042,32 +2131,51 @@ test "a success clears the recorded error" {
try testing.expectEqualStrings("", status.errorText());
}
test "compiledName spells the four file names of a source" {
comptime {
// The two tests below spell every suffix out instead of looping over
// `source_file_suffixes`: a test that reads the table moves with it, so a
// name dropped from the table would take the assertion that covers it along.
// An eighth suffix breaks the build here until both are extended.
std.debug.assert(source_file_suffixes.len == 7);
}
test "compiledName spells every file name of a source" {
var buf: [name_buf_len]u8 = undefined;
try testing.expectEqualStrings("42.list", compiledName(&buf, 42, ".list"));
try testing.expectEqualStrings("42.wild", compiledName(&buf, 42, ".wild"));
try testing.expectEqualStrings("42.allow", compiledName(&buf, 42, ".allow"));
try testing.expectEqualStrings("42.raw.tmp", compiledName(&buf, 42, ".raw.tmp"));
try testing.expectEqualStrings("42.list.tmp", compiledName(&buf, 42, ".list.tmp"));
try testing.expectEqualStrings("42.wild.tmp", compiledName(&buf, 42, ".wild.tmp"));
try testing.expectEqualStrings("42.allow.tmp", compiledName(&buf, 42, ".allow.tmp"));
}
test "sourceFileId matches every name a refresh writes, including the temporaries" {
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.allow"));
// A temporary left by a refresh that died belongs to its source id, so the
// sweep can tell whether that source still has a row.
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.raw.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.allow.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.list"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.allow"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.raw.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.allow.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.raw"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.allowed"));
try testing.expectEqual(@as(?i64, null), sourceFileId("README"));
}
test "every name compiledName writes is a name the sweep can attribute" {
// A round-trip over the table, not a coverage check: this loop reads the
// same array the code reads, so it cannot notice a missing entry. The two
// tests above are what pins the set.
var buf: [name_buf_len]u8 = undefined;
for (source_file_suffixes) |suffix| {
try testing.expectEqual(@as(?i64, 42), sourceFileId(compiledName(&buf, 42, suffix)));
@@ -2113,6 +2221,7 @@ fn testRow(id: i64, enabled: bool) sources_repo.SourceRow {
.last_updated = 1_700_000_000,
.domain_count = 9,
.wildcard_count = 4,
.exception_count = 2,
.skipped_regex_count = 1,
.checksum = "0" ** 64,
};
@@ -2260,17 +2369,49 @@ test "SourceStatus truncates a long url at max_url_len" {
test "compiledBodiesMatch verifies the bodies, not the presence of the files" {
const list_body = "a.example.com\nb.example.com\n";
const wild_body = "c.example.com\n";
const expected = bodyChecksum(list_body, wild_body);
const allow_body = "d.example.com\n";
const expected = bodyChecksum(list_body, wild_body, allow_body);
const header =
"# nxdns blocklist\n" ++
"# url https://lists.example/hosts.txt\n";
try testing.expect(compiledBodiesMatch(header ++ list_body, header ++ wild_body, &expected));
try testing.expect(compiledBodiesMatch(
header ++ list_body,
header ++ wild_body,
header ++ allow_body,
&expected,
));
// The corruption a reload reports as `ChecksumMismatch`: the file is there,
// its body is not what the checksum was taken over.
try testing.expect(!compiledBodiesMatch(header ++ "a.example.com\nb.exa", header ++ wild_body, &expected));
try testing.expect(!compiledBodiesMatch("", "", &expected));
// its body is not what the checksum was taken over. An allow body that lost
// its entry counts, because a dropped exception silently restores a block.
try testing.expect(!compiledBodiesMatch(
header ++ "a.example.com\nb.exa",
header ++ wild_body,
header ++ allow_body,
&expected,
));
try testing.expect(!compiledBodiesMatch(header ++ list_body, header ++ wild_body, "", &expected));
try testing.expect(!compiledBodiesMatch("", "", "", &expected));
}
test "a source with no exceptions keeps the checksum it had before the allow body existed" {
const list_body = "a.example.com\nb.example.com\n";
const wild_body = "c.example.com\n";
// What an older nxdns stored: the digest of the two bodies alone. It is what
// sits in `blocklist_sources.checksum` on every installation being upgraded,
// and the files on disk are the two it was taken over.
var hasher = Sha256.init(.{});
hasher.update(list_body);
hasher.update(wild_body);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
const stored = std.fmt.bytesToHex(digest, .lower);
try testing.expectEqualStrings(&stored, &bodyChecksum(list_body, wild_body, ""));
// No `.allow` file: what `loadSource` and `diskBodiesMatch` pass for one.
try testing.expect(compiledBodiesMatch(list_body, wild_body, "", &stored));
}
test "rejectedWithoutEntries fails a compile that produced nothing usable" {
@@ -2391,14 +2532,23 @@ test "collectSample steps over a line that does not fit the reader buffer" {
try testing.expectEqualStrings("ads.example.com\n", w.buffered());
}
test "bodyChecksum covers the list body followed by the wild body" {
const both = bodyChecksum("a.example.com\n", "b.example.com\n");
test "bodyChecksum covers the list body, then the wild body, then the allow body" {
const all = bodyChecksum("a.example.com\n", "b.example.com\n", "c.example.com\n");
var hasher = Sha256.init(.{});
hasher.update("a.example.com\nb.example.com\n");
hasher.update("a.example.com\nb.example.com\nc.example.com\n");
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &both);
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &all);
// Order matters: the two halves are not interchangeable.
try testing.expect(!std.mem.eql(u8, &both, &bodyChecksum("b.example.com\n", "a.example.com\n")));
// Order matters: the three parts are not interchangeable.
try testing.expect(!std.mem.eql(
u8,
&all,
&bodyChecksum("b.example.com\n", "a.example.com\n", "c.example.com\n"),
));
try testing.expect(!std.mem.eql(
u8,
&all,
&bodyChecksum("a.example.com\n", "c.example.com\n", "b.example.com\n"),
));
}