mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 13 Activity

filter: separate the compiled bodies in the checksum, a name moving between them was invisible

Browse Source
This commit is contained in:
mokhtar
2026-08-14 02:09:25 +02:00
parent 1bce81eea0
commit 21c5ce1f36
6 changed files with 274 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/filter/manager.zig
+52 -35
View File
@@ -220,9 +220,8 @@ pub const SourceStatus = struct {
/// **not** the header, so it stays stable across a refetch of unchanged content
/// while `fetched_at` moves.
///
/// The `.allow` body is hashed last so that a source with no exceptions keeps
/// the digest it had when only two bodies existed: every checksum written before
/// exceptions were honoured stays valid, and no upgrade forces a refetch.
/// Each body is followed by a separator byte, so the digest identifies which
/// body a name sits in rather than only which names were written.
pub const Header = struct {
url: []const u8,
format: parsers.Format,
@@ -591,11 +590,10 @@ pub const Manager = struct {
};
bodies.appendAssumeCapacity(wild_bytes);
// A missing `.allow` file is an empty allow body, not a failure. Two
// sources are in that state and both are ordinary: one compiled before
// exceptions were honoured, and one whose list carries no `@@` line.
// Because the empty body contributes nothing to the checksum, the
// stored digest of either still matches.
// A missing `.allow` file is an empty allow body, not a failure. The
// digest still covers three bodies, the third of them empty, so a
// source whose list carries no `@@` line matches whether its empty
// `.allow` file survived or not.
const allow_bytes: []const u8 = blk: {
const read = dir.readFileAlloc(io, allow_name, self.gpa, .limited(max_compiled_bytes)) catch |err| {
if (err == error.OutOfMemory) return error.OutOfMemory;
@@ -822,19 +820,19 @@ pub const Manager = struct {
if (std.mem.eql(u8, stored, &compiled.result.checksum) and
self.diskBodiesMatch(io, dir, row.id, stored))
{
// The three written counts come from the row: the checksum
// covers the three bodies, so an unchanged checksum means an
// unchanged number of entries in each. The two skip counts do
// not: a skipped line lands in no body, so a list that changed
// only its regex or browser-syntax lines arrives here with a
// stale row and a fresh compile. Taking them from the row would
// show the operator one number and restore another after a
// restart.
// Every count comes from the compile that just ran, not from
// the row. The two skip counts have to: a skipped line lands in
// no body, so a list that changed only its regex or
// browser-syntax lines reaches here with a stale row. The three
// written counts equal the row's anyway once the digest is
// framed, so reading them from the compile costs nothing and
// leaves no field whose freshness rests on an argument about
// what the checksum covers.
try sources_repo.updateSourceStats(self.database, row.id, .{
.last_updated = now,
.domain_count = row.domain_count,
.wildcard_count = row.wildcard_count,
.exception_count = row.exception_count,
.domain_count = compiled.result.counts.domains,
.wildcard_count = compiled.result.counts.wildcards,
.exception_count = compiled.result.counts.exceptions,
.skipped_regex_count = compiled.result.counts.skipped_regex,
.skipped_unsupported_count = compiled.result.counts.skipped_unsupported,
.checksum = stored,
@@ -1078,10 +1076,9 @@ pub const Manager = struct {
/// the rewrite path — the only path that can repair it.
///
/// A missing `.allow` file is the one exception, and it is the same one
/// `loadSource` makes: a source compiled before exceptions were honoured has
/// no such file, and its stored checksum was taken over an empty allow body.
/// Answering false there would rewrite every list on the first refresh after
/// an upgrade for no change in content.
/// `loadSource` makes: it reads as an empty allow body, which is what a list
/// with no `@@` line compiles to anyway. Answering false there would rewrite
/// such a list on every refresh for no change in content.
fn diskBodiesMatch(self: *Manager, io: std.Io, dir: std.Io.Dir, id: i64, expected: []const u8) bool {
var list_buf: [name_buf_len]u8 = undefined;
var wild_buf: [name_buf_len]u8 = undefined;
@@ -1653,15 +1650,20 @@ fn rejectedWithoutEntries(counts: compiler.Counts) bool {
return counts.invalid != 0 or counts.skipped_unsupported != 0 or counts.long_lines != 0;
}
/// The digest the `.list`, `.wild` and `.allow` bodies share, in that order.
/// The allow body comes last so that hashing an empty one leaves the digest of
/// the two-body form untouched, which is what keeps every checksum stored before
/// exceptions were honoured valid.
/// The digest the `.list`, `.wild` and `.allow` bodies share, in that order,
/// each followed by `compiler.body_separator`.
///
/// Must stay byte-for-byte what `compiler.compile` produces, separators
/// included: this is the other half of the same digest, and the two are
/// compared against each other on every refresh.
fn bodyChecksum(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) [64]u8 {
var hasher = Sha256.init(.{});
hasher.update(list_body);
hasher.update(compiler.body_separator);
hasher.update(wild_body);
hasher.update(compiler.body_separator);
hasher.update(allow_body);
hasher.update(compiler.body_separator);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
return std.fmt.bytesToHex(digest, .lower);
@@ -2412,23 +2414,33 @@ test "compiledBodiesMatch verifies the bodies, not the presence of the files" {
try testing.expect(!compiledBodiesMatch("", "", "", &expected));
}
test "a source with no exceptions keeps the checksum it had before the allow body existed" {
test "bodyChecksum separates the three bodies" {
const list_body = "a.example.com\nb.example.com\n";
const wild_body = "c.example.com\n";
// What an older nxdns stored: the digest of the two bodies alone. It is what
// sits in `blocklist_sources.checksum` on every installation being upgraded,
// and the files on disk are the two it was taken over.
var hasher = Sha256.init(.{});
hasher.update(list_body);
hasher.update(compiler.body_separator);
hasher.update(wild_body);
hasher.update(compiler.body_separator);
hasher.update(compiler.body_separator);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
const stored = std.fmt.bytesToHex(digest, .lower);
const expected = std.fmt.bytesToHex(digest, .lower);
try testing.expectEqualStrings(&stored, &bodyChecksum(list_body, wild_body, ""));
try testing.expectEqualStrings(&expected, &bodyChecksum(list_body, wild_body, ""));
// No `.allow` file: what `loadSource` and `diskBodiesMatch` pass for one.
try testing.expect(compiledBodiesMatch(list_body, wild_body, "", &stored));
// It is an empty body, and an empty body still gets its separator.
try testing.expect(compiledBodiesMatch(list_body, wild_body, "", &expected));
// The framing itself: the same bytes in a different body is a different
// digest. Unframed these two are equal, and a stale `.list` survives an
// upstream that switched the name to a wildcard.
try testing.expect(!std.mem.eql(
u8,
&bodyChecksum("a.example\n", "", ""),
&bodyChecksum("", "a.example\n", ""),
));
}
test "rejectedWithoutEntries fails a compile that produced nothing usable" {
@@ -2552,7 +2564,12 @@ test "collectSample steps over a line that does not fit the reader buffer" {
test "bodyChecksum covers the list body, then the wild body, then the allow body" {
const all = bodyChecksum("a.example.com\n", "b.example.com\n", "c.example.com\n");
var hasher = Sha256.init(.{});
hasher.update("a.example.com\nb.example.com\nc.example.com\n");
hasher.update("a.example.com\n");
hasher.update(compiler.body_separator);
hasher.update("b.example.com\n");
hasher.update(compiler.body_separator);
hasher.update("c.example.com\n");
hasher.update(compiler.body_separator);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &all);