mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 13 Activity

milestone 24: persist and surface the unsupported-line count

Browse Source
This commit is contained in:
mokhtar
2026-08-13 20:44:27 +02:00
parent 21571e448e
commit 1bce81eea0
18 changed files with 799 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/filter/filter_integration_test.zig
+172 -1
View File
@@ -121,6 +121,27 @@ const http_domains: i64 = 2;
const http_wildcards: i64 = 1;
const http_exceptions: i64 = 0;
const http_regex: i64 = 1;
/// Zero, and asserted rather than assumed: a hosts list carries no line a DNS
/// sinkhole cannot translate, which is what makes the abp fixture below a
/// separate source instead of two more lines in this one.
const http_unsupported: i64 = 0;
/// An ABP-format list: one element-hiding rule and one `$`-modifier rule that
/// nxdns counts and skips, beside two names it blocks. `detectFormat` assigns
/// one format to a whole source, so these lines cannot join `http_body` — a
/// single `##` there would re-parse every hosts line as ABP.
const abp_body =
"! small abp list\n" ++
"##.ad-banner\n" ++
"||ads.example^$third-party\n" ++
"||blocked.example^\n" ++
"tracker.example\n";
/// `||blocked.example^` covers its own apex, so it writes one `.list` entry
/// beside its `.wild` one; `tracker.example` writes the second.
const abp_domains: i64 = 2;
const abp_wildcards: i64 = 1;
const abp_unsupported: i64 = 2;
/// Compiles `text` into `<base>.list`, `<base>.wild` and `<base>.allow` under
/// `dir`, exactly as the manager's compile stage does, and returns the
@@ -393,7 +414,7 @@ const Env = struct {
// fixtures: the loopback http server
// ---------------------------------------------------------------------------
const Route = enum(u8) { body, redirect, not_found, oversize, chunked, stall };
const Route = enum(u8) { body, redirect, not_found, oversize, chunked, stall, changed };
/// How long the `stall` route holds a reply open when nothing releases it.
///
@@ -440,6 +461,11 @@ const oversize_length = "104857600";
const HttpFixture = struct {
server: net.Server,
body: []const u8,
/// What the `changed` route serves: the same list after its author edited
/// it. A test sets this before the serving task starts and reaches it by
/// switching the route, so the two bodies are read through the atomic that
/// selects them and never written beside a request in flight.
changed_body: []const u8,
route: std.atomic.Value(u8),
/// Connections accepted, whatever came over them. A test that claims a pass
/// downloaded nothing reads this rather than the route counters: a refetch
@@ -466,6 +492,7 @@ const HttpFixture = struct {
return .{
.server = try local.listen(io, .{ .reuse_address = true }),
.body = body,
.changed_body = "",
.route = .init(@intFromEnum(Route.body)),
.accepted = .init(0),
.flushed_parts = .init(0),
@@ -514,6 +541,7 @@ const HttpFixture = struct {
fn respond(self: *HttpFixture, io: std.Io, request: *std.http.Server.Request) !void {
switch (@as(Route, @enumFromInt(self.route.load(.acquire)))) {
.body => try request.respond(self.body, .{ .keep_alive = false }),
.changed => try request.respond(self.changed_body, .{ .keep_alive = false }),
.redirect => if (std.mem.eql(u8, request.head.target, redirect_path))
try request.respond(self.body, .{ .keep_alive = false })
else
@@ -824,6 +852,7 @@ test "3: a damaged compiled file never replaces a serving snapshot with a worse
.wildcard_count = 1,
.exception_count = 0,
.skipped_regex_count = 0,
.skipped_unsupported_count = 0,
.checksum = &bodyChecksum(good_list, good_wild, ""),
});
@@ -856,6 +885,7 @@ test "3: a damaged compiled file never replaces a serving snapshot with a worse
.wildcard_count = 1,
.exception_count = 0,
.skipped_regex_count = 0,
.skipped_unsupported_count = 0,
.checksum = &bodyChecksum(unsorted_list, good_wild, ""),
});
@@ -913,6 +943,7 @@ test "4: a 200 response is fetched, compiled, recorded and served" {
try testing.expectEqual(http_domains, row.domain_count);
try testing.expectEqual(http_wildcards, row.wildcard_count);
try testing.expectEqual(http_regex, row.skipped_regex_count);
try testing.expectEqual(http_unsupported, row.skipped_unsupported_count);
try testing.expectEqual(@as(usize, 64), (row.checksum orelse return error.TestNoChecksum).len);
try testing.expect(row.last_updated != null);
@@ -1073,6 +1104,7 @@ test "8: refetching identical content skips the rewrite and still moves last_upd
.wildcard_count = http_wildcards,
.exception_count = http_exceptions,
.skipped_regex_count = http_regex,
.skipped_unsupported_count = http_unsupported,
.checksum = blk: {
var rows = try listRows(&env.database);
defer rows.deinit();
@@ -1143,6 +1175,7 @@ fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []c
.wildcard_count = 0,
.exception_count = 0,
.skipped_regex_count = 0,
.skipped_unsupported_count = 0,
.checksum = &bodyChecksum(list_body, wild_body, ""),
});
}
@@ -1255,6 +1288,7 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
.wildcard_count = 0,
.exception_count = 0,
.skipped_regex_count = 0,
.skipped_unsupported_count = 0,
.checksum = &bodyChecksum(list_body, "", ""),
});
@@ -2156,6 +2190,7 @@ test "20: a data directory written before exceptions existed loads with no check
.wildcard_count = 1,
.exception_count = 0,
.skipped_regex_count = 0,
.skipped_unsupported_count = 0,
.checksum = &legacy_checksum,
});
@@ -2170,3 +2205,139 @@ test "20: a data directory written before exceptions existed loads with no check
try testing.expect(decision.blocked);
try testing.expect((try env.evaluate("x.ccc.example.com"))[0].blocked);
}
// ---------------------------------------------------------------------------
// 21–22: the unsupported count, from the compile to the database and back
// ---------------------------------------------------------------------------
test "21: an abp list's unsupported lines are counted, persisted and read back" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
var fixture = try HttpFixture.init(io, abp_body);
defer fixture.deinit(io);
var group: std.Io.Group = .init;
defer group.cancel(io);
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
var url_buf: [64]u8 = undefined;
const url = try fixture.url(&url_buf);
const id = try seedSource(&env.database, url);
try testing.expect(try refreshOnce(env, url));
try env.mgr.reload(io);
var rows = try listRows(&env.database);
defer rows.deinit();
const row = try rows.byUrl(url);
try testing.expectEqual(abp_domains, row.domain_count);
try testing.expectEqual(abp_wildcards, row.wildcard_count);
try testing.expectEqual(@as(i64, 0), row.skipped_regex_count);
try testing.expectEqual(abp_unsupported, row.skipped_unsupported_count);
const status = try env.status(id);
try testing.expectEqual(manager.State.ok, status.state);
try testing.expectEqual(@as(u32, @intCast(abp_unsupported)), status.counts.skipped_unsupported);
// What the number costs the operator: the `$`-modifier rule named a domain
// and blocked nothing, while the two lines nxdns could translate did block.
try testing.expect(!(try env.evaluate("ads.example"))[0].blocked);
try testing.expect((try env.evaluate("blocked.example"))[0].blocked);
try testing.expect((try env.evaluate("tracker.example"))[0].blocked);
}
/// The same list before and after its author edited only lines nxdns skips.
/// The written entries are identical in both, so the two compiles produce one
/// checksum and the refresh takes the unchanged-checksum path.
const churn_before =
"! churn fixture\n" ++
"##.ad-one\n" ++
"||blocked.example^\n" ++
"tracker.example\n";
const churn_after =
"! churn fixture\n" ++
"##.ad-one\n" ++
"##.ad-two\n" ++
"/ads[0-9]+/\n" ++
"||blocked.example^\n" ++
"tracker.example\n";
test "22: a list that changed only its skipped lines still updates both skip counters" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
var fixture = try HttpFixture.init(io, churn_before);
fixture.changed_body = churn_after;
defer fixture.deinit(io);
var group: std.Io.Group = .init;
defer group.cancel(io);
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
var url_buf: [64]u8 = undefined;
const url = try fixture.url(&url_buf);
const id = try seedSource(&env.database, url);
try testing.expect(try refreshOnce(env, url));
var first_checksum: [64]u8 = undefined;
{
var rows = try listRows(&env.database);
defer rows.deinit();
const row = try rows.byUrl(url);
try testing.expectEqual(@as(i64, 0), row.skipped_regex_count);
try testing.expectEqual(@as(i64, 1), row.skipped_unsupported_count);
@memcpy(&first_checksum, row.checksum orelse return error.TestNoChecksum);
}
// The edited list. Two more skipped lines and not one written entry moved,
// so the refresh finds its stored checksum and rewrites nothing on disk.
fixture.setRoute(.changed);
try testing.expect(!try refreshOnce(env, url));
{
var rows = try listRows(&env.database);
defer rows.deinit();
const row = try rows.byUrl(url);
try testing.expectEqualStrings(&first_checksum, row.checksum orelse return error.TestNoChecksum);
// The three written counts are the ones an unchanged checksum vouches
// for; the two skip counts are the ones it says nothing about.
try testing.expectEqual(abp_domains, row.domain_count);
try testing.expectEqual(abp_wildcards, row.wildcard_count);
try testing.expectEqual(@as(i64, 1), row.skipped_regex_count);
try testing.expectEqual(@as(i64, 2), row.skipped_unsupported_count);
}
const live = try env.status(id);
try testing.expectEqual(@as(u32, 1), live.counts.skipped_regex);
try testing.expectEqual(@as(u32, 2), live.counts.skipped_unsupported);
// The restart. A new manager over the same database and the same files
// carries nothing across in memory, so the status it publishes is what
// rehydration read out of the row — which is the only reason writing the
// fresh counts above matters.
env.mgr.deinit(io);
env.mgr = try manager.Manager.init(
gpa,
&env.database,
.{ .dir = env.tmp.dir },
&env.f,
.{ .enabled = false },
budget,
);
try env.mgr.reload(io);
const restored = try env.status(id);
try testing.expectEqual(manager.State.ok, restored.state);
try testing.expect(restored.loaded);
try testing.expectEqual(@as(u32, 1), restored.counts.skipped_regex);
try testing.expectEqual(@as(u32, 2), restored.counts.skipped_unsupported);
}
src/filter/manager.zig
+19 -2
View File
@@ -822,12 +822,21 @@ pub const Manager = struct {
if (std.mem.eql(u8, stored, &compiled.result.checksum) and
self.diskBodiesMatch(io, dir, row.id, stored))
{
// The three written counts come from the row: the checksum
// covers the three bodies, so an unchanged checksum means an
// unchanged number of entries in each. The two skip counts do
// not: a skipped line lands in no body, so a list that changed
// only its regex or browser-syntax lines arrives here with a
// stale row and a fresh compile. Taking them from the row would
// show the operator one number and restore another after a
// restart.
try sources_repo.updateSourceStats(self.database, row.id, .{
.last_updated = now,
.domain_count = row.domain_count,
.wildcard_count = row.wildcard_count,
.exception_count = row.exception_count,
.skipped_regex_count = row.skipped_regex_count,
.skipped_regex_count = compiled.result.counts.skipped_regex,
.skipped_unsupported_count = compiled.result.counts.skipped_unsupported,
.checksum = stored,
});
status.succeed(now, compiled.result.counts);
@@ -857,6 +866,7 @@ pub const Manager = struct {
.wildcard_count = compiled.result.counts.wildcards,
.exception_count = compiled.result.counts.exceptions,
.skipped_regex_count = compiled.result.counts.skipped_regex,
.skipped_unsupported_count = compiled.result.counts.skipped_unsupported,
.checksum = &compiled.result.checksum,
});
status.succeed(now, compiled.result.counts);
@@ -1537,7 +1547,7 @@ fn applyLoadOutcomes(
entry.loaded = true;
// Two states survive a successful load. `.ok`, because a
// refresh in this process already filled the counters the
// compile produced and the three database columns are a subset
// compile produced and the five database columns are a subset
// of them. And any refresh failure, because the files that just
// loaded are exactly the ones the failed refresh could not
// replace, so the operator must still see why.
@@ -1547,6 +1557,7 @@ fn applyLoadOutcomes(
.wildcards = countOf(row.wildcard_count),
.exceptions = countOf(row.exception_count),
.skipped_regex = countOf(row.skipped_regex_count),
.skipped_unsupported = countOf(row.skipped_unsupported_count),
});
},
.failed => |reason| {
@@ -1926,6 +1937,7 @@ test "a canceled compiled-file read cancels the reload instead of recording it"
.domain_count = 1,
.wildcard_count = 0,
.skipped_regex_count = 0,
.skipped_unsupported_count = 0,
.exception_count = 0,
.checksum = &bodyChecksum(list_body, wild_body, allow_body),
});
@@ -2049,6 +2061,7 @@ test "the log label names a source without printing what its url carries" {
.domain_count = 0,
.wildcard_count = 0,
.skipped_regex_count = 0,
.skipped_unsupported_count = 0,
.checksum = null,
};
const printed = try std.fmt.bufPrint(&buf, "blocklist {f}: download failed: {s}", .{
@@ -2223,6 +2236,7 @@ fn testRow(id: i64, enabled: bool) sources_repo.SourceRow {
.wildcard_count = 4,
.exception_count = 2,
.skipped_regex_count = 1,
.skipped_unsupported_count = 5,
.checksum = "0" ** 64,
};
}
@@ -2333,6 +2347,9 @@ test "a load of a source this process never refreshed takes the row counters" {
try testing.expectEqual(@as(u32, 9), statuses[0].counts.domains);
try testing.expectEqual(@as(u32, 4), statuses[0].counts.wildcards);
try testing.expectEqual(@as(u32, 1), statuses[0].counts.skipped_regex);
// Rehydration: a restart reads this from the row and nowhere else, because
// no path reparses a compiled file's header.
try testing.expectEqual(@as(u32, 5), statuses[0].counts.skipped_unsupported);
}
test "a status borrows nothing, so a copy outlives the table it came from" {