mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 13 Activity

milestone 21: abp list exceptions and a regex rule kind

Browse Source
This commit is contained in:
mokhtar
2026-08-13 19:14:47 +02:00
parent b340521716
commit 2ab7c1f1de
51 changed files with 4016 additions and 465 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/filter/compiler.zig
+174 -42
View File
@@ -1,14 +1,15 @@
//! Compiles a downloaded blocklist into the two bodies nxdns stores on disk:
//! a `.list` body of exact names and a `.wild` body of suffixes.
//! Compiles a downloaded blocklist into the three bodies nxdns stores on disk:
//! a `.list` body of exact names, a `.wild` body of suffixes and an `.allow`
//! body of the names the list's `@@` exceptions lift.
//!
//! Pure over reader/writer interfaces: an allocator, a `*std.Io.Reader` and two
//! `*std.Io.Writer`. No `std.Io` value, no file, no clock. A compiled body is a
//! pure function of (bytes, format), which is what makes two runs — and two
//! Pure over reader/writer interfaces: an allocator, a `*std.Io.Reader` and
//! three `*std.Io.Writer`. No `std.Io` value, no file, no clock. A compiled body
//! is a pure function of (bytes, format), which is what makes two runs — and two
//! permutations of the same input — byte-identical.
//!
//! Nothing but the sorted, deduplicated names is written: no header, no
//! timestamp, no counts. The header belongs to the caller, and the checksum
//! covers the two bodies only.
//! covers the three bodies only.
const std = @import("std");
const parsers = @import("parsers.zig");
@@ -23,6 +24,12 @@ pub const max_line_len: usize = 4096;
pub const Counts = struct {
domains: u32 = 0,
wildcards: u32 = 0,
/// Written, deduplicated `.allow` entries: the names this list's `@@`
/// exceptions lift out of what other lists block.
exceptions: u32 = 0,
/// Regex lines this list carried, counted and skipped. nxdns has an engine
/// for them now, but it stays reserved for operator rules: a downloaded list
/// is other people's patterns, and PLAN §2.2 keeps them out.
skipped_regex: u32 = 0,
skipped_unsupported: u32 = 0,
/// Not a valid domain name (`dns.name.fromText` rejected it, a non-ASCII
@@ -36,30 +43,35 @@ pub const Counts = struct {
pub const Result = struct {
counts: Counts,
/// Lowercase hex sha256 over the `.list` body followed by the `.wild` body.
/// Lowercase hex sha256 over the `.list` body, then the `.wild` body, then
/// the `.allow` body.
///
/// The `.allow` body came last for a reason: hashing an empty one adds
/// nothing, so a list with no exceptions keeps the digest it had when only
/// two bodies existed. Every checksum published before exceptions were
/// honoured therefore stays valid, and upgrading forces no refetch.
checksum: [64]u8,
};
pub const Error = error{ OutOfMemory, TooManyDomains, ReadFailed, WriteFailed };
/// Reads `r` to end of stream and writes the two compiled bodies.
/// Reads `r` to end of stream and writes the three compiled bodies.
///
/// `counts.domains` and `counts.wildcards` are the written, deduplicated
/// counts: they are what `blocklist_sources.domain_count` and `wildcard_count`
/// store and what the UI shows.
/// `counts.domains`, `counts.wildcards` and `counts.exceptions` are the written,
/// deduplicated counts: they are what `blocklist_sources.domain_count`,
/// `wildcard_count` and `exception_count` store and what the UI shows.
pub fn compile(
gpa: std.mem.Allocator,
r: *std.Io.Reader,
format: parsers.Format,
list_w: *std.Io.Writer,
wild_w: *std.Io.Writer,
allow_w: *std.Io.Writer,
) Error!Result {
var counts: Counts = .{};
var list: Entries = .{};
defer list.deinit(gpa);
var wild: Entries = .{};
defer wild.deinit(gpa);
var bodies: Bodies = .{};
defer bodies.deinit(gpa);
while (try parsers.nextBoundedLine(r, max_line_len)) |event| {
const raw = switch (event) {
@@ -81,32 +93,27 @@ pub fn compile(
.domain => {
var fields = std.mem.tokenizeAny(u8, parsed.text, &std.ascii.whitespace);
while (fields.next()) |field| {
try addCandidate(gpa, field, false, false, &list, &wild, &counts);
try addCandidate(gpa, field, parsed, &bodies, &counts);
}
},
.wildcard => try addCandidate(
gpa,
parsed.text,
true,
parsed.covers_apex,
&list,
&wild,
&counts,
),
.wildcard, .exception => try addCandidate(gpa, parsed.text, parsed, &bodies, &counts),
}
}
// The `.allow` body is hashed last so that a list with no exceptions
// reproduces the digest a two-body compile of the same bytes produced.
var hasher = Sha256.init(.{});
counts.domains = try emit(&list, list_w, &hasher, &counts.duplicates);
counts.wildcards = try emit(&wild, wild_w, &hasher, &counts.duplicates);
counts.domains = try emit(&bodies.list, list_w, &hasher, &counts.duplicates);
counts.wildcards = try emit(&bodies.wild, wild_w, &hasher, &counts.duplicates);
counts.exceptions = try emit(&bodies.allow, allow_w, &hasher, &counts.duplicates);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
return .{ .counts = counts, .checksum = std.fmt.bytesToHex(digest, .lower) };
}
/// Normalizes one whitespace-separated candidate and files it under `.list`,
/// `.wild`, or neither.
/// Normalizes one whitespace-separated candidate of `line` and files it under
/// `.list`, `.wild`, `.allow`, or nowhere.
///
/// The normalization below is deliberately not `dns.name.normalizeText`: this
/// one adds the two-label minimum, rejects control bytes, and reports every
@@ -114,20 +121,19 @@ pub fn compile(
fn addCandidate(
gpa: std.mem.Allocator,
field: []const u8,
from_wildcard_line: bool,
covers_apex: bool,
list: *Entries,
wild: *Entries,
line: parsers.Line,
bodies: *Bodies,
counts: *Counts,
) Error!void {
var candidate = field;
var is_wildcard = from_wildcard_line;
var is_wildcard = line.kind == .wildcard;
if (std.mem.startsWith(u8, candidate, "*.")) {
is_wildcard = true;
candidate = candidate[2..];
}
// A '*' anywhere else makes this a pattern, and patterns belong to the
// `rules` table; a blocklist entry is a name or a suffix.
// `rules` table, where the operator writes them as a `.wildcard` or a
// `.regex`; a blocklist entry is a name or a suffix.
if (std.mem.indexOfScalar(u8, candidate, '*') != null) {
counts.invalid += 1;
return;
@@ -163,12 +169,17 @@ fn addCandidate(
return;
}
if (is_wildcard) {
try wild.append(gpa, normalized);
// An exception needs no apex entry beside its suffix entry: the matcher
// walks the `.allow` set over the full name and every parent, so one entry
// lifts `x` and every subdomain of it at once.
if (line.kind == .exception) {
try bodies.allow.append(gpa, normalized);
} else if (is_wildcard) {
try bodies.wild.append(gpa, normalized);
// An ABP `||x^` rule covers `x` itself as well as its subdomains.
if (covers_apex) try list.append(gpa, normalized);
if (line.covers_apex) try bodies.list.append(gpa, normalized);
} else {
try list.append(gpa, normalized);
try bodies.list.append(gpa, normalized);
}
}
@@ -202,6 +213,20 @@ fn emit(
return written;
}
/// The three bodies under construction, in the order they are written and
/// hashed.
const Bodies = struct {
list: Entries = .{},
wild: Entries = .{},
allow: Entries = .{},
fn deinit(self: *Bodies, gpa: std.mem.Allocator) void {
self.list.deinit(gpa);
self.wild.deinit(gpa);
self.allow.deinit(gpa);
}
};
/// Length-prefixed candidate bytes plus the offsets that index them. Sorting
/// permutes the offsets, so the bytes never move.
const Entries = struct {
@@ -247,10 +272,12 @@ const Compiled = struct {
result: Result,
list_w: std.Io.Writer.Allocating,
wild_w: std.Io.Writer.Allocating,
allow_w: std.Io.Writer.Allocating,
fn deinit(self: *Compiled) void {
self.list_w.deinit();
self.wild_w.deinit();
self.allow_w.deinit();
}
fn list(self: *Compiled) []const u8 {
@@ -260,6 +287,10 @@ const Compiled = struct {
fn wild(self: *Compiled) []const u8 {
return self.wild_w.written();
}
fn allow(self: *Compiled) []const u8 {
return self.allow_w.written();
}
};
fn compileText(gpa: std.mem.Allocator, text: []const u8, format: parsers.Format) Error!Compiled {
@@ -272,8 +303,10 @@ fn compileReader(gpa: std.mem.Allocator, r: *std.Io.Reader, format: parsers.Form
errdefer list_w.deinit();
var wild_w: std.Io.Writer.Allocating = .init(gpa);
errdefer wild_w.deinit();
const result = try compile(gpa, r, format, &list_w.writer, &wild_w.writer);
return .{ .result = result, .list_w = list_w, .wild_w = wild_w };
var allow_w: std.Io.Writer.Allocating = .init(gpa);
errdefer allow_w.deinit();
const result = try compile(gpa, r, format, &list_w.writer, &wild_w.writer, &allow_w.writer);
return .{ .result = result, .list_w = list_w, .wild_w = wild_w, .allow_w = allow_w };
}
const hosts_fixture =
@@ -339,10 +372,106 @@ test "abp apex rule lands in both bodies" {
try testing.expectEqualStrings("bare.com\nx.com\n", c.list());
try testing.expectEqualStrings("x.com\n", c.wild());
try testing.expectEqualStrings("z.com\n", c.allow());
try testing.expectEqual(@as(u32, 2), c.result.counts.domains);
try testing.expectEqual(@as(u32, 1), c.result.counts.wildcards);
try testing.expectEqual(@as(u32, 1), c.result.counts.exceptions);
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_regex);
try testing.expectEqual(@as(u32, 2), c.result.counts.skipped_unsupported);
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_unsupported);
}
test "an abp list's hosts-style lines reach the domain body through the split" {
// The case `parser_abp` defers here: it hands a whitespace-carrying bare
// candidate over whole, and only the tokenization in `compile` files the
// name out of it. A mixed list — `!` header and `||` rules, so `detectFormat`
// calls the whole file `abp`, plus the hosts lines such lists carry — reaches
// a compiled body no other way, and no parser test can see it happen.
const fixture =
"! Title: mixed\n" ++
"||blocked.example^\n" ++
"0.0.0.0 ads.example\n" ++
"127.0.0.1 localhost\n";
var c = try compileText(testing.allocator, fixture, .abp);
defer c.deinit();
// The address field is filed as a name of its own: abp lines have no hosts
// framing, so the compiler cannot know which field is the address. `0.0.0.0`
// and `127.0.0.1` are names nobody resolves, which is why the split is worth
// more than the two spurious entries cost.
try testing.expectEqualStrings(
"0.0.0.0\n127.0.0.1\nads.example\nblocked.example\n",
c.list(),
);
try testing.expectEqualStrings("blocked.example\n", c.wild());
try testing.expectEqual(@as(u32, 4), c.result.counts.domains);
// `localhost` is the one label the two-label minimum drops.
try testing.expectEqual(@as(u32, 1), c.result.counts.invalid);
}
test "the allow body is sorted, deduplicated and normalized like the others" {
const fixture =
"@@||GOOD.ads.example^\n" ++
"@@||a.ads.example^$important\n" ++
"@@||good.ads.example\n" ++
"@@||localhost^\n" ++
"@@||bad*.ads.example^\n" ++
"||ads.example^\n";
var c = try compileText(testing.allocator, fixture, .abp);
defer c.deinit();
try testing.expectEqualStrings("a.ads.example\ngood.ads.example\n", c.allow());
try testing.expectEqual(@as(u32, 2), c.result.counts.exceptions);
try testing.expectEqual(@as(u32, 1), c.result.counts.duplicates);
// `localhost` is one label, and the starred name is not a name at all.
try testing.expectEqual(@as(u32, 1), c.result.counts.invalid);
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_unsupported);
// The exceptions changed neither block body.
try testing.expectEqualStrings("ads.example\n", c.list());
try testing.expectEqualStrings("ads.example\n", c.wild());
}
test "an empty allow body reproduces the checksum of a two-body compile" {
var c = try compileText(testing.allocator, hosts_fixture, .hosts);
defer c.deinit();
try testing.expectEqualStrings("", c.allow());
// What the digest was before the `.allow` body existed: the `.list` body
// followed by the `.wild` body and nothing else. Every checksum stored by an
// older nxdns was taken this way, and this is the equality that keeps them
// valid — without it, every source on every installation would report
// `ChecksumMismatch` at the first reload after the upgrade and re-download.
var hasher = Sha256.init(.{});
hasher.update(c.list());
hasher.update(c.wild());
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &c.result.checksum);
}
test "the checksum of a source with exceptions covers all three bodies in order" {
const fixture =
"||ads.example^\n" ++
"@@||good.ads.example^\n";
var c = try compileText(testing.allocator, fixture, .abp);
defer c.deinit();
var hasher = Sha256.init(.{});
hasher.update(c.list());
hasher.update(c.wild());
hasher.update(c.allow());
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &c.result.checksum);
// A non-empty allow body does move the digest, so a list that gains an
// exception is recompiled rather than silently kept.
var without = try compileText(testing.allocator, "||ads.example^\n", .abp);
defer without.deinit();
try testing.expect(!std.mem.eql(u8, &c.result.checksum, &without.result.checksum));
}
test "two runs of the same input are byte-identical" {
@@ -353,6 +482,7 @@ test "two runs of the same input are byte-identical" {
try testing.expectEqualStrings(a.list(), b.list());
try testing.expectEqualStrings(a.wild(), b.wild());
try testing.expectEqualStrings(a.allow(), b.allow());
try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum);
}
@@ -374,6 +504,7 @@ test "a permutation of the input compiles to the same bodies" {
try testing.expectEqualStrings(a.list(), b.list());
try testing.expectEqualStrings(a.wild(), b.wild());
try testing.expectEqualStrings(a.allow(), b.allow());
try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum);
}
@@ -495,6 +626,7 @@ test "empty input produces empty bodies and the sha256 of the empty string" {
try testing.expectEqualStrings("", c.list());
try testing.expectEqualStrings("", c.wild());
try testing.expectEqualStrings("", c.allow());
try testing.expectEqualStrings(
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
&c.result.checksum,
src/filter/filter_integration_test.zig
+286 -25
View File
@@ -33,10 +33,13 @@ const sources_repo = @import("../storage/repositories/sources_repo.zig");
const compiler = @import("compiler.zig");
const fetcher = @import("fetcher.zig");
const parsers = @import("parsers.zig");
const manager = @import("manager.zig");
const matcher = @import("matcher.zig");
const response = @import("response.zig");
const lookup = @import("../web/handlers/lookup.zig");
const forward_client = @import("../local/forward_client.zig");
const forward_zones = @import("../local/forward_zones.zig");
const records = @import("../local/records.zig");
@@ -60,6 +63,11 @@ const budget: std.Io.Clock.Duration = .{ .raw = .fromSeconds(10), .clock = .awak
/// The forward-zone read timeout. Case 15 asserts a silent resolver gives up
/// inside twice this, so it has to be short enough to keep the run quick and
/// long enough that a loopback answer always beats it.
/// `/api/lookup` reports the local tables beside the filter decision; this
/// suite's cases are about the filter half, so both are empty here.
const empty_records: records.Records = .empty;
const empty_zones: forward_zones.Zones = .empty;
const read_timeout: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(200), .clock = .awake };
const file_limit: std.Io.Limit = .limited(8 * 1024 * 1024);
@@ -111,49 +119,67 @@ const http_body =
const http_domains: i64 = 2;
const http_wildcards: i64 = 1;
const http_exceptions: i64 = 0;
const http_regex: i64 = 1;
/// Compiles `text` into `<base>.list` and `<base>.wild` under `dir`, exactly as
/// the manager's compile stage does, and returns the compiler's own result.
/// Compiles `text` into `<base>.list`, `<base>.wild` and `<base>.allow` under
/// `dir`, exactly as the manager's compile stage does, and returns the
/// compiler's own result.
fn compileToFiles(
gpa: std.mem.Allocator,
io: std.Io,
dir: std.Io.Dir,
base: []const u8,
text: []const u8,
format: parsers.Format,
) !compiler.Result {
var list_name_buf: [64]u8 = undefined;
var wild_name_buf: [64]u8 = undefined;
var allow_name_buf: [64]u8 = undefined;
const list_name = try std.fmt.bufPrint(&list_name_buf, "{s}.list", .{base});
const wild_name = try std.fmt.bufPrint(&wild_name_buf, "{s}.wild", .{base});
const allow_name = try std.fmt.bufPrint(&allow_name_buf, "{s}.allow", .{base});
const list_file = try dir.createFile(io, list_name, .{ .permissions = .fromMode(0o600) });
defer list_file.close(io);
const wild_file = try dir.createFile(io, wild_name, .{ .permissions = .fromMode(0o600) });
defer wild_file.close(io);
const allow_file = try dir.createFile(io, allow_name, .{ .permissions = .fromMode(0o600) });
defer allow_file.close(io);
const buffers = try gpa.alloc(u8, 2 * 16 * 1024);
const buffers = try gpa.alloc(u8, 3 * 16 * 1024);
defer gpa.free(buffers);
var r: std.Io.Reader = .fixed(text);
var list_w = list_file.writer(io, buffers[0 .. 16 * 1024]);
var wild_w = wild_file.writer(io, buffers[16 * 1024 ..]);
var wild_w = wild_file.writer(io, buffers[16 * 1024 .. 32 * 1024]);
var allow_w = allow_file.writer(io, buffers[32 * 1024 ..]);
const result = try compiler.compile(gpa, &r, .hosts, &list_w.interface, &wild_w.interface);
const result = try compiler.compile(
gpa,
&r,
format,
&list_w.interface,
&wild_w.interface,
&allow_w.interface,
);
try list_w.interface.flush();
try wild_w.interface.flush();
try allow_w.interface.flush();
return result;
}
/// The two compiled bodies of one source, read back from disk with their
/// The three compiled bodies of one source, read back from disk with their
/// headers stripped, exactly as `Manager.reload` reads them.
const Bodies = struct {
list: []u8,
wild: []u8,
allow: []u8,
fn read(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, base: []const u8) !Bodies {
var list_name_buf: [64]u8 = undefined;
var wild_name_buf: [64]u8 = undefined;
var allow_name_buf: [64]u8 = undefined;
const list = try dir.readFileAlloc(
io,
try std.fmt.bufPrint(&list_name_buf, "{s}.list", .{base}),
@@ -167,21 +193,34 @@ const Bodies = struct {
gpa,
file_limit,
);
return .{ .list = list, .wild = wild };
errdefer gpa.free(wild);
const allow = try dir.readFileAlloc(
io,
try std.fmt.bufPrint(&allow_name_buf, "{s}.allow", .{base}),
gpa,
file_limit,
);
return .{ .list = list, .wild = wild, .allow = allow };
}
fn deinit(self: *Bodies, gpa: std.mem.Allocator) void {
gpa.free(self.list);
gpa.free(self.wild);
gpa.free(self.allow);
self.* = undefined;
}
};
/// A one-group, one-source snapshot over two compiled bodies.
fn snapshotOver(gpa: std.mem.Allocator, list_body: []const u8, wild_body: []const u8) !matcher.Snapshot {
/// A one-group, one-source snapshot over the three compiled bodies.
fn snapshotOver(
gpa: std.mem.Allocator,
list_body: []const u8,
wild_body: []const u8,
allow_body: []const u8,
) !matcher.Snapshot {
const sources = [_]model.BlocklistSource{.{ .url = source_url, .name = source_name }};
const compiled = [_]?matcher.Snapshot.Compiled{
.{ .list_body = list_body, .wild_body = wild_body },
.{ .list_body = list_body, .wild_body = wild_body, .allow_body = allow_body },
};
return matcher.Snapshot.build(gpa, .{
.groups = &.{.{ .name = "default" }},
@@ -198,10 +237,11 @@ fn snapshotOver(gpa: std.mem.Allocator, list_body: []const u8, wild_body: []cons
});
}
fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 {
fn bodyChecksum(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) [64]u8 {
var hasher = Sha256.init(.{});
hasher.update(list_body);
hasher.update(wild_body);
hasher.update(allow_body);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
return std.fmt.bytesToHex(digest, .lower);
@@ -679,7 +719,7 @@ test "1: a compiled hosts fixture loads into a snapshot that blocks its domains"
const text = try hostsFixture(gpa);
defer gpa.free(text);
const result = try compileToFiles(gpa, io, tmp.dir, "1", text);
const result = try compileToFiles(gpa, io, tmp.dir, "1", text, .hosts);
try testing.expectEqual(@as(u32, fixture_domains), result.counts.domains);
try testing.expectEqual(@as(u32, 1), result.counts.skipped_regex);
// The three single-label names are the only invalid candidates here.
@@ -692,7 +732,7 @@ test "1: a compiled hosts fixture loads into a snapshot that blocks its domains"
try testing.expect(std.mem.find(u8, bodies.list, bare) == null);
}
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild);
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild, bodies.allow);
defer snapshot.deinit();
const group = snapshot.groupIndexByName("default").?;
@@ -732,8 +772,8 @@ test "2: recompiling the same fixture produces byte-identical files and checksum
var second_dir = try tmp.dir.createDirPathOpen(io, "second", .{});
defer second_dir.close(io);
const first = try compileToFiles(gpa, io, first_dir, "1", text);
const second = try compileToFiles(gpa, io, second_dir, "1", text);
const first = try compileToFiles(gpa, io, first_dir, "1", text, .hosts);
const second = try compileToFiles(gpa, io, second_dir, "1", text, .hosts);
try testing.expectEqualStrings(&first.checksum, &second.checksum);
try testing.expectEqual(first.counts, second.counts);
@@ -745,11 +785,12 @@ test "2: recompiling the same fixture produces byte-identical files and checksum
try testing.expectEqualSlices(u8, first_bodies.list, second_bodies.list);
try testing.expectEqualSlices(u8, first_bodies.wild, second_bodies.wild);
try testing.expectEqualSlices(u8, first_bodies.allow, second_bodies.allow);
// The checksum the compiler reported is the one over the two bodies it
// The checksum the compiler reported is the one over the three bodies it
// wrote, which is what the manager stores and compares against.
try testing.expectEqualStrings(
&bodyChecksum(first_bodies.list, first_bodies.wild),
&bodyChecksum(first_bodies.list, first_bodies.wild, first_bodies.allow),
&first.checksum,
);
}
@@ -781,8 +822,9 @@ test "3: a damaged compiled file never replaces a serving snapshot with a worse
.last_updated = 1_700_000_000,
.domain_count = 2,
.wildcard_count = 1,
.exception_count = 0,
.skipped_regex_count = 0,
.checksum = &bodyChecksum(good_list, good_wild),
.checksum = &bodyChecksum(good_list, good_wild, ""),
});
try env.mgr.reload(io);
@@ -812,8 +854,9 @@ test "3: a damaged compiled file never replaces a serving snapshot with a worse
.last_updated = 1_700_000_000,
.domain_count = 2,
.wildcard_count = 1,
.exception_count = 0,
.skipped_regex_count = 0,
.checksum = &bodyChecksum(unsorted_list, good_wild),
.checksum = &bodyChecksum(unsorted_list, good_wild, ""),
});
try testing.expectError(error.NotSorted, env.mgr.reload(io));
@@ -1028,6 +1071,7 @@ test "8: refetching identical content skips the rewrite and still moves last_upd
.last_updated = 1_000,
.domain_count = http_domains,
.wildcard_count = http_wildcards,
.exception_count = http_exceptions,
.skipped_regex_count = http_regex,
.checksum = blk: {
var rows = try listRows(&env.database);
@@ -1069,6 +1113,9 @@ const ReloadTask = struct {
/// Writes one source's compiled files and records their checksum, without any
/// network: the swap and the orphan sweep care about files and rows, not about
/// where the bytes came from.
///
/// All three files, including an empty `.allow`, because that is what a publish
/// leaves: `publishOne` runs once per body and never skips the empty one.
fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []const u8) !void {
const io = env.io();
var dir = try env.blocklistDir();
@@ -1076,6 +1123,7 @@ fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []c
var list_name_buf: [64]u8 = undefined;
var wild_name_buf: [64]u8 = undefined;
var allow_name_buf: [64]u8 = undefined;
try dir.writeFile(io, .{
.sub_path = try std.fmt.bufPrint(&list_name_buf, "{d}.list", .{id}),
.data = list_body,
@@ -1084,13 +1132,18 @@ fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []c
.sub_path = try std.fmt.bufPrint(&wild_name_buf, "{d}.wild", .{id}),
.data = wild_body,
});
try dir.writeFile(io, .{
.sub_path = try std.fmt.bufPrint(&allow_name_buf, "{d}.allow", .{id}),
.data = "",
});
try sources_repo.updateSourceStats(&env.database, id, .{
.last_updated = 1_700_000_000,
.domain_count = 1,
.wildcard_count = 0,
.exception_count = 0,
.skipped_regex_count = 0,
.checksum = &bodyChecksum(list_body, wild_body),
.checksum = &bodyChecksum(list_body, wild_body, ""),
});
}
@@ -1146,14 +1199,21 @@ test "10: pruneOrphans deletes the files of a deleted source and leaves live one
var dir = try env.blocklistDir();
defer dir.close(io);
// Every name a source can own, spelled out: this fixture is what decides
// whether the sweep covers the whole set, so it enumerates
// `manager.source_file_suffixes` by hand rather than sharing it. A suffix
// added to the manager and not added here is swept by nothing and asserted
// by nothing.
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.allow", .data = "lifted.example.com\n" });
// Id 9999 has no `blocklist_sources` row, so no refresh can be writing for
// it: these temporaries are what a refresh that died mid-write leaves
// behind, and the sweep is the only thing that will ever remove them.
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.list.tmp", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.wild.tmp", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.allow.tmp", .data = "" });
// The live source does have a row, so its temporary is a refresh in
// progress and must survive a sweep that runs beside it.
@@ -1165,12 +1225,15 @@ test "10: pruneOrphans deletes the files of a deleted source and leaves live one
var live_buf: [64]u8 = undefined;
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.list", .{id}), .{});
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.allow", .{id}), .{});
try dir.access(io, live_tmp, .{});
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list.tmp", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild.tmp", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow.tmp", .{}));
}
test "10b: the scheduler sweeps orphans on its own, with no operator call" {
@@ -1190,8 +1253,9 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
.last_updated = std.Io.Clock.real.now(io).toSeconds(),
.domain_count = 1,
.wildcard_count = 0,
.exception_count = 0,
.skipped_regex_count = 0,
.checksum = &bodyChecksum(list_body, ""),
.checksum = &bodyChecksum(list_body, "", ""),
});
var dir = try env.blocklistDir();
@@ -1201,7 +1265,9 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
// either.
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.allow", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.allow.tmp", .data = "" });
// `runScheduler` is the entry point `app.zig` hands to `Io.Group`, and the
// only one the server ever calls. A disabled update stops it after the
@@ -1212,7 +1278,9 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow.tmp", .{}));
// The live source kept its files and is still filtering: the sweep did not
// take the snapshot the same pass had just published.
@@ -1376,6 +1444,7 @@ test "10d: a source deleted mid-refresh does not take the refresh's temporary fi
try testing.expect(!std.mem.endsWith(u8, entry.name, ".tmp"));
try testing.expect(!std.mem.endsWith(u8, entry.name, ".list"));
try testing.expect(!std.mem.endsWith(u8, entry.name, ".wild"));
try testing.expect(!std.mem.endsWith(u8, entry.name, ".allow"));
}
}
@@ -1410,10 +1479,13 @@ test "10e: a reconcile then a restart reuses the compiled files and downloads no
defer dir.close(io);
var list_buf: [64]u8 = undefined;
var wild_buf: [64]u8 = undefined;
var allow_buf: [64]u8 = undefined;
const list_name = try std.fmt.bufPrint(&list_buf, "{d}.list", .{id});
const wild_name = try std.fmt.bufPrint(&wild_buf, "{d}.wild", .{id});
const allow_name = try std.fmt.bufPrint(&allow_buf, "{d}.allow", .{id});
const list_before = try dir.statFile(io, list_name, .{});
const wild_before = try dir.statFile(io, wild_name, .{});
const allow_before = try dir.statFile(io, allow_name, .{});
// File mode, declaring exactly what the database already holds. The engine
// has to recognise the source by its url and leave the row where it is:
@@ -1459,14 +1531,19 @@ test "10e: a reconcile then a restart reuses the compiled files and downloads no
// decision the pass made rather than a connection it could not have opened.
try testing.expectEqual(@as(u32, 1), fixture.accepted.load(.monotonic));
// The same two files: not recompiled, and not swept as orphans and written
// back.
// The same three files: not recompiled, and not swept as orphans and
// written back. `.allow` is asserted with the other two because a restart
// that rewrote only the exception body would otherwise leave this test
// green while changing what the snapshot lets through.
const list_after = try dir.statFile(io, list_name, .{});
const wild_after = try dir.statFile(io, wild_name, .{});
const allow_after = try dir.statFile(io, allow_name, .{});
try testing.expectEqual(list_before.inode, list_after.inode);
try testing.expectEqual(list_before.mtime, list_after.mtime);
try testing.expectEqual(wild_before.inode, wild_after.inode);
try testing.expectEqual(wild_before.mtime, wild_after.mtime);
try testing.expectEqual(allow_before.inode, allow_after.inode);
try testing.expectEqual(allow_before.mtime, allow_after.mtime);
// The row kept the id those files are named after, and the snapshot the
// restart published is the one compiled from them.
@@ -1788,12 +1865,12 @@ test "17: each blocking mode synthesizes the documented blocked reply" {
// The reply is synthesized for a name the compiled files actually block, so
// this case covers the decision and the response together.
const result = try compileToFiles(gpa, io, tmp.dir, "1", "0.0.0.0 ads.example.com\n");
const result = try compileToFiles(gpa, io, tmp.dir, "1", "0.0.0.0 ads.example.com\n", .hosts);
try testing.expectEqual(@as(u32, 1), result.counts.domains);
var bodies = try Bodies.read(gpa, io, tmp.dir, "1");
defer bodies.deinit(gpa);
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild);
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild, bodies.allow);
defer snapshot.deinit();
const group = snapshot.groupIndexByName("default").?;
@@ -1909,3 +1986,187 @@ test "18: a body streamed in flushed parts survives the fetcher's multi-read pum
try testing.expectEqual(matcher.Reason.blocklist_domain, decision.reason);
}
}
// ---------------------------------------------------------------------------
// 19–20: list exceptions (milestone 21)
// ---------------------------------------------------------------------------
/// A real ABP list: one domain anchor, the exception that lifts one subtree out
/// of it in each of the two accepted spellings, and two `@@` forms nxdns does
/// not honour.
const abp_exception_body =
"[Adblock Plus 2.0]\n" ++
"! Title: exceptions\n" ++
"||ads.example^\n" ++
"||tracker.example^\n" ++
"@@||good.ads.example^\n" ++
"@@||fine.tracker.example$important\n" ++
"@@||paid.ads.example^$third-party\n" ++
"@@partial.ads.example\n";
const abp_exception_domains: i64 = 2;
const abp_exception_wildcards: i64 = 2;
const abp_exception_exceptions: i64 = 2;
test "19: a downloaded list's exceptions lift its own blocks and nothing else" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
var fixture = try HttpFixture.init(io, abp_exception_body);
defer fixture.deinit(io);
var group: std.Io.Group = .init;
defer group.cancel(io);
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
var url_buf: [64]u8 = undefined;
const url = try fixture.url(&url_buf);
const id = try seedSource(&env.database, url);
try testing.expect(try refreshOnce(env, url));
try env.mgr.reload(io);
// The `.allow` body is a third file beside the two, and the row and the
// status entry both carry its count — which is what a
// `POST /api/blocklists/update` response row reports as `exceptions`.
var dir = try env.blocklistDir();
defer dir.close(io);
var bodies = try Bodies.read(gpa, io, dir, "1");
defer bodies.deinit(gpa);
try testing.expectEqualStrings(
"fine.tracker.example\ngood.ads.example\n",
manager.stripHeader(bodies.allow),
);
var rows = try listRows(&env.database);
defer rows.deinit();
const row = try rows.byUrl(url);
try testing.expectEqual(abp_exception_domains, row.domain_count);
try testing.expectEqual(abp_exception_wildcards, row.wildcard_count);
try testing.expectEqual(abp_exception_exceptions, row.exception_count);
const status = try env.status(id);
try testing.expectEqual(manager.State.ok, status.state);
try testing.expect(status.loaded);
try testing.expectEqual(@as(u32, @intCast(abp_exception_exceptions)), status.counts.exceptions);
// The blocks the list makes still land, apex and subdomain alike.
for ([_][]const u8{ "ads.example", "x.ads.example", "paid.ads.example", "partial.ads.example" }) |blocked| {
const decision, _ = try env.evaluate(blocked);
try testing.expect(decision.blocked);
}
// The two exceptions lift the excepted name and everything under it.
for ([_][]const u8{
"good.ads.example",
"y.good.ads.example",
"fine.tracker.example",
"z.fine.tracker.example",
}) |lifted| {
const decision, _ = try env.evaluate(lifted);
try testing.expect(!decision.blocked);
try testing.expectEqual(matcher.Reason.blocklist_exception, decision.reason);
try testing.expectEqual(@as(?u32, 0), decision.source);
}
// What `/api/lookup` answers, through the same function the handler calls:
// the reason names the exception and the source id names the list.
{
const handle = env.mgr.acquire(io) orelse return error.TestNoSnapshot;
defer handle.release(io);
const group_index = handle.snapshot.groupIndexByName("default") orelse
return error.TestGroupMissing;
const result = lookup.evaluate(
handle.snapshot,
group_index,
"y.good.ads.example",
&empty_records,
&empty_zones,
);
try testing.expect(!result.blocked);
try testing.expectEqual(matcher.Reason.blocklist_exception, result.reason);
try testing.expectEqualStrings("good.ads.example", result.matched);
try testing.expectEqual(@as(?i64, id), result.source_id);
const rendered = lookup.body("y.good.ads.example", result, url);
try testing.expectEqualStrings("blocklist_exception", rendered.reason);
try testing.expectEqualStrings(url, rendered.source_url.?);
}
// An operator block rule outranks the list's exception: a downloaded list
// may cancel what a list decided and never what the operator decided.
const group_id = (try groups_repo.groupId(&env.database, "default")) orelse
return error.TestGroupMissing;
_ = try rules_repo.insertRuleRow(&env.database, .{
.group_id = group_id,
.pattern = "good.ads.example",
.kind = .exact,
.action = .block,
}, 1_700_000_000);
try env.mgr.reload(io);
{
const decision, _ = try env.evaluate("good.ads.example");
try testing.expect(decision.blocked);
try testing.expectEqual(matcher.Reason.rule_block_exact, decision.reason);
}
}
test "20: a data directory written before exceptions existed loads with no checksum mismatch" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
const id = try seedSource(&env.database, source_url);
// Exactly what an older nxdns left behind: two compiled files, no `.allow`
// file, and a checksum taken over the two bodies alone.
const list_body = "aaa.example.com\nbbb.example.com\n";
const wild_body = "ccc.example.com\n";
var dir = try env.blocklistDir();
defer dir.close(io);
var list_name_buf: [64]u8 = undefined;
var wild_name_buf: [64]u8 = undefined;
try dir.writeFile(io, .{
.sub_path = try std.fmt.bufPrint(&list_name_buf, "{d}.list", .{id}),
.data = list_body,
});
try dir.writeFile(io, .{
.sub_path = try std.fmt.bufPrint(&wild_name_buf, "{d}.wild", .{id}),
.data = wild_body,
});
var hasher = Sha256.init(.{});
hasher.update(list_body);
hasher.update(wild_body);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
const legacy_checksum = std.fmt.bytesToHex(digest, .lower);
try sources_repo.updateSourceStats(&env.database, id, .{
.last_updated = 1_700_000_000,
.domain_count = 2,
.wildcard_count = 1,
.exception_count = 0,
.skipped_regex_count = 0,
.checksum = &legacy_checksum,
});
try env.mgr.reload(io);
const status = try env.status(id);
try testing.expectEqual(manager.State.ok, status.state);
try testing.expect(status.loaded);
try testing.expectEqualStrings("", status.errorText());
const decision, _ = try env.evaluate("aaa.example.com");
try testing.expect(decision.blocked);
try testing.expect((try env.evaluate("x.ccc.example.com"))[0].blocked);
}
src/filter/manager.zig
+219 -69
View File
@@ -38,9 +38,10 @@
//! publish: the download of one source, at up to 300 s each, and the compile
//! that follows it. It also covers blocklist-directory maintenance, because
//! those stages are the only writers of `.raw.tmp` / `.list.tmp` /
//! `.wild.tmp` and `pruneOrphans` must not sweep the temporaries of a refresh
//! that is still running. Two concurrent refreshes would share the fetcher's
//! buffers and, for one source, the same temporary paths.
//! `.wild.tmp` / `.allow.tmp` and `pruneOrphans` must not sweep the
//! temporaries of a refresh that is still running. Two concurrent refreshes
//! would share the fetcher's buffers and, for one source, the same temporary
//! paths.
//!
//! **Lock ordering: `refresh_lock` is never acquired while `writer_lock` is
//! held.** A path that needs both takes `refresh_lock` first. The public entry
@@ -94,7 +95,7 @@ const io_buf_len: usize = 64 * 1024;
/// would then be decided by almost no data.
const sample_buf_len: usize = parsers.sample_lines * (compiler.max_line_len + 1);
/// `<id>` is at most 20 characters and the longest suffix is `.list.tmp`.
/// `<id>` is at most 20 characters and the longest suffix is `.allow.tmp`.
const name_buf_len: usize = 48;
/// How one blocklist source is named in a log line: by its row id and its name,
@@ -215,9 +216,13 @@ pub const SourceStatus = struct {
};
/// The header every compiled file carries, ahead of the body. The `sha256`
/// covers the `.list` body followed by the `.wild` body and **not** the header,
/// so it stays stable across a refetch of unchanged content while
/// `fetched_at` moves.
/// covers the `.list` body, then the `.wild` body, then the `.allow` body, and
/// **not** the header, so it stays stable across a refetch of unchanged content
/// while `fetched_at` moves.
///
/// The `.allow` body is hashed last so that a source with no exceptions keeps
/// the digest it had when only two bodies existed: every checksum written before
/// exceptions were honoured stays valid, and no upgrade forces a refetch.
pub const Header = struct {
url: []const u8,
format: parsers.Format,
@@ -233,6 +238,7 @@ pub const Header = struct {
try w.print("# fetched_at {d}\n", .{self.fetched_at});
try w.print("# domains {d}\n", .{self.counts.domains});
try w.print("# wildcards {d}\n", .{self.counts.wildcards});
try w.print("# exceptions {d}\n", .{self.counts.exceptions});
try w.print("# skipped_regex {d}\n", .{self.counts.skipped_regex});
try w.print("# skipped_unsupported {d}\n", .{self.counts.skipped_unsupported});
try w.print("# invalid {d}\n", .{self.counts.invalid});
@@ -557,12 +563,14 @@ pub const Manager = struct {
var list_buf: [name_buf_len]u8 = undefined;
var wild_buf: [name_buf_len]u8 = undefined;
var allow_buf: [name_buf_len]u8 = undefined;
const list_name = compiledName(&list_buf, row.id, ".list");
const wild_name = compiledName(&wild_buf, row.id, ".wild");
const allow_name = compiledName(&allow_buf, row.id, ".allow");
// Reserved before the reads, so neither buffer can be orphaned by a
// failing append: `bodies` owns each one from the moment it is read.
try bodies.ensureUnusedCapacity(self.gpa, 2);
// Reserved before the reads, so no buffer can be orphaned by a failing
// append: `bodies` owns each one from the moment it is read.
try bodies.ensureUnusedCapacity(self.gpa, 3);
// `error.Canceled` is the one-shot signal that this task is being torn
// down, and it is consumed by whoever catches it. Recording it as a
@@ -583,18 +591,39 @@ pub const Manager = struct {
};
bodies.appendAssumeCapacity(wild_bytes);
// A missing `.allow` file is an empty allow body, not a failure. Two
// sources are in that state and both are ordinary: one compiled before
// exceptions were honoured, and one whose list carries no `@@` line.
// Because the empty body contributes nothing to the checksum, the
// stored digest of either still matches.
const allow_bytes: []const u8 = blk: {
const read = dir.readFileAlloc(io, allow_name, self.gpa, .limited(max_compiled_bytes)) catch |err| {
if (err == error.OutOfMemory) return error.OutOfMemory;
if (err == error.Canceled) return error.Canceled;
if (err == error.FileNotFound) break :blk "";
return loadFailure(row, allow_name, err);
};
bodies.appendAssumeCapacity(read);
break :blk read;
};
const list_body = stripHeader(list_bytes);
const wild_body = stripHeader(wild_bytes);
const allow_body = stripHeader(allow_bytes);
// The checksum covers both bodies together, so a crash between the two
// The checksum covers the three bodies together, so a crash between the
// `replace` calls — a new `.list` beside an old `.wild` — is caught
// here and refreshed, not served as a half-updated list.
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body))) {
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body, allow_body))) {
log.warn("blocklist {f}: compiled files do not match the stored checksum", .{SourceLabel.of(row)});
return .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } };
}
return .{ .loaded = .{ .list_body = list_body, .wild_body = wild_body } };
return .{ .loaded = .{
.list_body = list_body,
.wild_body = wild_body,
.allow_body = allow_body,
} };
}
// -----------------------------------------------------------------------
@@ -635,23 +664,28 @@ pub const Manager = struct {
var raw_buf: [name_buf_len]u8 = undefined;
var list_tmp_buf: [name_buf_len]u8 = undefined;
var wild_tmp_buf: [name_buf_len]u8 = undefined;
var allow_tmp_buf: [name_buf_len]u8 = undefined;
const raw_name = compiledName(&raw_buf, row.id, ".raw.tmp");
const list_tmp = compiledName(&list_tmp_buf, row.id, ".list.tmp");
const wild_tmp = compiledName(&wild_tmp_buf, row.id, ".wild.tmp");
const tmp: TempNames = .{
.list = compiledName(&list_tmp_buf, row.id, ".list.tmp"),
.wild = compiledName(&wild_tmp_buf, row.id, ".wild.tmp"),
.allow = compiledName(&allow_tmp_buf, row.id, ".allow.tmp"),
};
// Installed before the calls that create these files, not after: an
// `error.Canceled` or `error.OutOfMemory` returned straight out of
// `download` or `compileTo` would outrun a later `defer` and leave a
// temporary behind. Deleting a name that was never created is a no-op.
defer self.deleteQuietly(io, dir, raw_name);
defer self.deleteQuietly(io, dir, list_tmp);
defer self.deleteQuietly(io, dir, wild_tmp);
defer self.deleteQuietly(io, dir, tmp.list);
defer self.deleteQuietly(io, dir, tmp.wild);
defer self.deleteQuietly(io, dir, tmp.allow);
// The half that takes the time: one download of up to `total_budget`
// and one compile of everything it returned. `refresh_lock` alone is
// held here, so a rule save, a settings change or any other web
// mutation that ends in `reload` runs beside it instead of behind it.
const prepared = try self.prepareRefresh(io, dir, row, &status, raw_name, list_tmp, wild_tmp);
const prepared = try self.prepareRefresh(io, dir, row, &status, raw_name, tmp);
// The half that publishes. The compiled files, the runtime columns and
// the status entry land under one `writer_lock`, so a reload never
@@ -659,7 +693,7 @@ pub const Manager = struct {
self.writer_lock.lockUncancelable(io);
defer self.writer_lock.unlock(io);
const replaced = try self.publishRefresh(io, dir, row, &status, prepared, list_tmp, wild_tmp);
const replaced = try self.publishRefresh(io, dir, row, &status, prepared, tmp);
self.commitStatus(io, status);
return replaced;
}
@@ -690,6 +724,14 @@ pub const Manager = struct {
return self.reload(io);
}
/// The three temporary files one refresh compiles into, before the header
/// is prepended and each is renamed over the file it replaces.
const TempNames = struct {
list: []const u8,
wild: []const u8,
allow: []const u8,
};
/// What the fetch-and-compile half of a refresh produced. `.failed` needs
/// no publish and has already recorded why in the status entry.
const Prepared = union(enum) {
@@ -712,8 +754,7 @@ pub const Manager = struct {
row: sources_repo.SourceRow,
status: *SourceStatus,
raw_name: []const u8,
list_tmp: []const u8,
wild_tmp: []const u8,
tmp: TempNames,
) Error!Prepared {
self.download(io, dir, raw_name, row) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
@@ -733,7 +774,7 @@ pub const Manager = struct {
},
};
const result = self.compileTo(io, dir, raw_name, format, list_tmp, wild_tmp) catch |err| switch (err) {
const result = self.compileTo(io, dir, raw_name, format, tmp) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.Canceled => return error.Canceled,
else => {
@@ -762,8 +803,7 @@ pub const Manager = struct {
row: sources_repo.SourceRow,
status: *SourceStatus,
prepared: Prepared,
list_tmp: []const u8,
wild_tmp: []const u8,
tmp: TempNames,
) Error!bool {
const compiled = switch (prepared) {
.failed => return false,
@@ -786,6 +826,7 @@ pub const Manager = struct {
.last_updated = now,
.domain_count = row.domain_count,
.wildcard_count = row.wildcard_count,
.exception_count = row.exception_count,
.skipped_regex_count = row.skipped_regex_count,
.checksum = stored,
});
@@ -801,7 +842,7 @@ pub const Manager = struct {
.counts = compiled.result.counts,
.checksum = &compiled.result.checksum,
};
self.publish(io, dir, row.id, header, list_tmp, wild_tmp) catch |err| switch (err) {
self.publish(io, dir, row.id, header, tmp) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.Canceled => return error.Canceled,
else => {
@@ -814,6 +855,7 @@ pub const Manager = struct {
.last_updated = now,
.domain_count = compiled.result.counts.domains,
.wildcard_count = compiled.result.counts.wildcards,
.exception_count = compiled.result.counts.exceptions,
.skipped_regex_count = compiled.result.counts.skipped_regex,
.checksum = &compiled.result.checksum,
});
@@ -913,7 +955,7 @@ pub const Manager = struct {
return parsers.detectFormat(sample.buffered());
}
/// Compiles into two plain temporary files. The compiled bodies cannot go
/// Compiles into three plain temporary files. The compiled bodies cannot go
/// straight into the final files: the header carries counts that only exist
/// once the whole input has been compiled, and the loader requires the
/// header first.
@@ -923,22 +965,24 @@ pub const Manager = struct {
dir: std.Io.Dir,
raw_name: []const u8,
format: parsers.Format,
list_tmp: []const u8,
wild_tmp: []const u8,
tmp: TempNames,
) !compiler.Result {
const raw = try dir.openFile(io, raw_name, .{});
defer raw.close(io);
const list_file = try dir.createFile(io, list_tmp, .{ .permissions = .fromMode(0o600) });
const list_file = try dir.createFile(io, tmp.list, .{ .permissions = .fromMode(0o600) });
defer list_file.close(io);
const wild_file = try dir.createFile(io, wild_tmp, .{ .permissions = .fromMode(0o600) });
const wild_file = try dir.createFile(io, tmp.wild, .{ .permissions = .fromMode(0o600) });
defer wild_file.close(io);
const allow_file = try dir.createFile(io, tmp.allow, .{ .permissions = .fromMode(0o600) });
defer allow_file.close(io);
const buffers = try self.gpa.alloc(u8, 3 * io_buf_len);
const buffers = try self.gpa.alloc(u8, 4 * io_buf_len);
defer self.gpa.free(buffers);
var fr = raw.reader(io, buffers[0..io_buf_len]);
var list_w = list_file.writer(io, buffers[io_buf_len .. 2 * io_buf_len]);
var wild_w = wild_file.writer(io, buffers[2 * io_buf_len ..]);
var wild_w = wild_file.writer(io, buffers[2 * io_buf_len .. 3 * io_buf_len]);
var allow_w = allow_file.writer(io, buffers[3 * io_buf_len ..]);
const result = compiler.compile(
self.gpa,
@@ -946,18 +990,22 @@ pub const Manager = struct {
format,
&list_w.interface,
&wild_w.interface,
&allow_w.interface,
) catch |err| switch (err) {
// `compiler.Error` names the direction; the concrete cause is on
// the stream that failed.
error.ReadFailed => return fr.err orelse err,
error.WriteFailed => return list_w.err orelse (wild_w.err orelse err),
error.WriteFailed => return list_w.err orelse
(wild_w.err orelse (allow_w.err orelse err)),
else => return err,
};
try list_w.interface.flush();
try wild_w.interface.flush();
try allow_w.interface.flush();
try list_file.sync(io);
try wild_file.sync(io);
try allow_file.sync(io);
return result;
}
@@ -970,16 +1018,17 @@ pub const Manager = struct {
dir: std.Io.Dir,
id: i64,
header: Header,
list_tmp: []const u8,
wild_tmp: []const u8,
tmp: TempNames,
) !void {
const buffers = try self.gpa.alloc(u8, 2 * io_buf_len);
defer self.gpa.free(buffers);
var list_buf: [name_buf_len]u8 = undefined;
var wild_buf: [name_buf_len]u8 = undefined;
try publishOne(io, dir, compiledName(&list_buf, id, ".list"), list_tmp, header, buffers);
try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), wild_tmp, header, buffers);
var allow_buf: [name_buf_len]u8 = undefined;
try publishOne(io, dir, compiledName(&list_buf, id, ".list"), tmp.list, header, buffers);
try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), tmp.wild, header, buffers);
try publishOne(io, dir, compiledName(&allow_buf, id, ".allow"), tmp.allow, header, buffers);
}
fn publishOne(
@@ -1014,12 +1063,19 @@ pub const Manager = struct {
try af.replace(io);
}
/// Whether the two compiled files on disk hash to `expected`. A missing,
/// Whether the compiled files on disk hash to `expected`. A missing,
/// unreadable or corrupt file answers false, which sends the caller down
/// the rewrite path — the only path that can repair it.
///
/// A missing `.allow` file is the one exception, and it is the same one
/// `loadSource` makes: a source compiled before exceptions were honoured has
/// no such file, and its stored checksum was taken over an empty allow body.
/// Answering false there would rewrite every list on the first refresh after
/// an upgrade for no change in content.
fn diskBodiesMatch(self: *Manager, io: std.Io, dir: std.Io.Dir, id: i64, expected: []const u8) bool {
var list_buf: [name_buf_len]u8 = undefined;
var wild_buf: [name_buf_len]u8 = undefined;
var allow_buf: [name_buf_len]u8 = undefined;
const limit: std.Io.Limit = .limited(max_compiled_bytes);
const list_bytes = dir.readFileAlloc(io, compiledName(&list_buf, id, ".list"), self.gpa, limit) catch
@@ -1029,7 +1085,11 @@ pub const Manager = struct {
return false;
defer self.gpa.free(wild_bytes);
return compiledBodiesMatch(list_bytes, wild_bytes, expected);
const allow_bytes = dir.readFileAlloc(io, compiledName(&allow_buf, id, ".allow"), self.gpa, limit) catch |err|
if (err == error.FileNotFound) @as([]u8, &.{}) else return false;
defer self.gpa.free(allow_bytes);
return compiledBodiesMatch(list_bytes, wild_bytes, allow_bytes, expected);
}
fn reportFetchFailure(
@@ -1231,12 +1291,12 @@ pub const Manager = struct {
/// sweeps to nothing.
pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
// `refresh_lock` first, and for the reason it exists: the download and
// the compile are the only writers of `.raw.tmp`, `.list.tmp` and
// `.wild.tmp`, and they hold it for as long as they run. Without it
// here, a source deleted through the API would sweep the temporaries of
// a refresh still writing them — the row is gone, so nothing else in
// this function would spare them — and the pass would fail on a raw
// file that vanished under it.
// the compile are the only writers of `.raw.tmp`, `.list.tmp`,
// `.wild.tmp` and `.allow.tmp`, and they hold it for as long as they
// run. Without it here, a source deleted through the API would sweep
// the temporaries of a refresh still writing them — the row is gone, so
// nothing else in this function would spare them — and the pass would
// fail on a raw file that vanished under it.
//
// `writer_lock` second, in the one order this file ever takes them,
// because the rows this reads and the compiled files it deletes are
@@ -1485,6 +1545,7 @@ fn applyLoadOutcomes(
entry.succeed(row.last_updated orelse 0, .{
.domains = countOf(row.domain_count),
.wildcards = countOf(row.wildcard_count),
.exceptions = countOf(row.exception_count),
.skipped_regex = countOf(row.skipped_regex_count),
});
},
@@ -1555,43 +1616,62 @@ fn collectSample(r: *std.Io.Reader, w: *std.Io.Writer) error{ ReadFailed, WriteF
}
}
/// Whether two compiled files carry the bodies `expected` was taken over.
fn compiledBodiesMatch(list_bytes: []const u8, wild_bytes: []const u8, expected: []const u8) bool {
return std.mem.eql(u8, expected, &bodyChecksum(stripHeader(list_bytes), stripHeader(wild_bytes)));
/// Whether three compiled files carry the bodies `expected` was taken over.
fn compiledBodiesMatch(
list_bytes: []const u8,
wild_bytes: []const u8,
allow_bytes: []const u8,
expected: []const u8,
) bool {
return std.mem.eql(u8, expected, &bodyChecksum(
stripHeader(list_bytes),
stripHeader(wild_bytes),
stripHeader(allow_bytes),
));
}
/// A compile that produced no entry at all while rejecting lines is an error
/// page, a compressed body or a format the sniff got wrong — not a blocklist.
/// Publishing it would replace a working list with nothing and report `ok`. An
/// input that rejected nothing is an empty list, which is legal.
///
/// A list of nothing but exceptions is loadable: an allow-only list published
/// beside a blocking one is a shape operators use, and it produces entries.
fn rejectedWithoutEntries(counts: compiler.Counts) bool {
if (counts.domains != 0 or counts.wildcards != 0) return false;
if (counts.domains != 0 or counts.wildcards != 0 or counts.exceptions != 0) return false;
return counts.invalid != 0 or counts.skipped_unsupported != 0 or counts.long_lines != 0;
}
fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 {
/// The digest the `.list`, `.wild` and `.allow` bodies share, in that order.
/// The allow body comes last so that hashing an empty one leaves the digest of
/// the two-body form untouched, which is what keeps every checksum stored before
/// exceptions were honoured valid.
fn bodyChecksum(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) [64]u8 {
var hasher = Sha256.init(.{});
hasher.update(list_body);
hasher.update(wild_body);
hasher.update(allow_body);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
return std.fmt.bytesToHex(digest, .lower);
}
fn compiledName(buf: *[name_buf_len]u8, id: i64, suffix: []const u8) []const u8 {
// An `i64` prints in at most 20 characters and the longest suffix is nine,
// An `i64` prints in at most 20 characters and the longest suffix is ten,
// so `name_buf_len` cannot be exceeded.
return std.fmt.bufPrint(buf, "{d}{s}", .{ id, suffix }) catch unreachable;
}
/// Every name `compiledName` can produce, longest suffix first so `.list.tmp`
/// is never read as `.list`.
const source_file_suffixes = [_][]const u8{ ".list.tmp", ".wild.tmp", ".raw.tmp", ".list", ".wild" };
const source_file_suffixes = [_][]const u8{
".allow.tmp", ".list.tmp", ".wild.tmp", ".raw.tmp", ".allow", ".list", ".wild",
};
/// The source id a file under the blocklist directory belongs to, or null when
/// the name is not one of ours.
///
/// The three temporaries count. A refresh that dies between writing one and
/// The four temporaries count. A refresh that dies between writing one and
/// renaming it leaves a file no later refresh reuses and no `defer` reaches, so
/// excluding them from the sweep means nothing ever removes them. Matching them
/// is safe because `pruneOrphans` holds `refresh_lock` for its whole body:
@@ -1830,18 +1910,24 @@ test "a canceled compiled-file read cancels the reload instead of recording it"
const list_body = "aaa.example.com\n";
const wild_body = "";
const allow_body = "";
var dir = try tmp.dir.createDirPathOpen(io, "blocklists", .{});
defer dir.close(io);
var list_buf: [name_buf_len]u8 = undefined;
var wild_buf: [name_buf_len]u8 = undefined;
var allow_buf: [name_buf_len]u8 = undefined;
try dir.writeFile(io, .{ .sub_path = compiledName(&list_buf, id, ".list"), .data = list_body });
try dir.writeFile(io, .{ .sub_path = compiledName(&wild_buf, id, ".wild"), .data = wild_body });
// Present rather than absent, so the third read is a real one: `loadSource`
// treats a missing `.allow` as an empty body and would never open it.
try dir.writeFile(io, .{ .sub_path = compiledName(&allow_buf, id, ".allow"), .data = allow_body });
try sources_repo.updateSourceStats(&database, id, .{
.last_updated = 1_700_000_000,
.domain_count = 1,
.wildcard_count = 0,
.skipped_regex_count = 0,
.checksum = &bodyChecksum(list_body, wild_body),
.exception_count = 0,
.checksum = &bodyChecksum(list_body, wild_body, allow_body),
});
// The baseline every assertion below is against: one clean reload, one
@@ -1853,11 +1939,12 @@ test "a canceled compiled-file read cancels the reload instead of recording it"
try testing.expect(out[0].loaded);
const published = mgr.generation;
// Both catch sites, in the order `loadSource` reads the two files. A
// Every catch site, in the order `loadSource` reads the three files. A
// cancellation is consumed by whoever catches it, so folding it into a load
// failure would spend the shutdown signal and leave a status row reading
// "Canceled" behind.
for ([_][]const u8{ ".list", ".wild" }) |suffix| {
// "Canceled" behind. The `.allow` read is the one that can get this wrong
// twice over: it also has to keep `FileNotFound` apart from a cancellation.
for ([_][]const u8{ ".list", ".wild", ".allow" }) |suffix| {
var vtable: std.Io.VTable = undefined;
const canceling = cancelingIo(io, suffix, &vtable);
try testing.expectError(error.Canceled, mgr.reload(canceling));
@@ -1921,6 +2008,7 @@ test "the header writer produces the documented text" {
.counts = .{
.domains = 12,
.wildcards = 3,
.exceptions = 7,
.skipped_regex = 2,
.skipped_unsupported = 1,
.invalid = 5,
@@ -1938,6 +2026,7 @@ test "the header writer produces the documented text" {
\\# fetched_at 1700000000
\\# domains 12
\\# wildcards 3
\\# exceptions 7
\\# skipped_regex 2
\\# skipped_unsupported 1
\\# invalid 5
@@ -2042,32 +2131,51 @@ test "a success clears the recorded error" {
try testing.expectEqualStrings("", status.errorText());
}
test "compiledName spells the four file names of a source" {
comptime {
// The two tests below spell every suffix out instead of looping over
// `source_file_suffixes`: a test that reads the table moves with it, so a
// name dropped from the table would take the assertion that covers it along.
// An eighth suffix breaks the build here until both are extended.
std.debug.assert(source_file_suffixes.len == 7);
}
test "compiledName spells every file name of a source" {
var buf: [name_buf_len]u8 = undefined;
try testing.expectEqualStrings("42.list", compiledName(&buf, 42, ".list"));
try testing.expectEqualStrings("42.wild", compiledName(&buf, 42, ".wild"));
try testing.expectEqualStrings("42.allow", compiledName(&buf, 42, ".allow"));
try testing.expectEqualStrings("42.raw.tmp", compiledName(&buf, 42, ".raw.tmp"));
try testing.expectEqualStrings("42.list.tmp", compiledName(&buf, 42, ".list.tmp"));
try testing.expectEqualStrings("42.wild.tmp", compiledName(&buf, 42, ".wild.tmp"));
try testing.expectEqualStrings("42.allow.tmp", compiledName(&buf, 42, ".allow.tmp"));
}
test "sourceFileId matches every name a refresh writes, including the temporaries" {
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.allow"));
// A temporary left by a refresh that died belongs to its source id, so the
// sweep can tell whether that source still has a row.
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.raw.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.allow.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.list"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.allow"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.raw.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.allow.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.raw"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.allowed"));
try testing.expectEqual(@as(?i64, null), sourceFileId("README"));
}
test "every name compiledName writes is a name the sweep can attribute" {
// A round-trip over the table, not a coverage check: this loop reads the
// same array the code reads, so it cannot notice a missing entry. The two
// tests above are what pins the set.
var buf: [name_buf_len]u8 = undefined;
for (source_file_suffixes) |suffix| {
try testing.expectEqual(@as(?i64, 42), sourceFileId(compiledName(&buf, 42, suffix)));
@@ -2113,6 +2221,7 @@ fn testRow(id: i64, enabled: bool) sources_repo.SourceRow {
.last_updated = 1_700_000_000,
.domain_count = 9,
.wildcard_count = 4,
.exception_count = 2,
.skipped_regex_count = 1,
.checksum = "0" ** 64,
};
@@ -2260,17 +2369,49 @@ test "SourceStatus truncates a long url at max_url_len" {
test "compiledBodiesMatch verifies the bodies, not the presence of the files" {
const list_body = "a.example.com\nb.example.com\n";
const wild_body = "c.example.com\n";
const expected = bodyChecksum(list_body, wild_body);
const allow_body = "d.example.com\n";
const expected = bodyChecksum(list_body, wild_body, allow_body);
const header =
"# nxdns blocklist\n" ++
"# url https://lists.example/hosts.txt\n";
try testing.expect(compiledBodiesMatch(header ++ list_body, header ++ wild_body, &expected));
try testing.expect(compiledBodiesMatch(
header ++ list_body,
header ++ wild_body,
header ++ allow_body,
&expected,
));
// The corruption a reload reports as `ChecksumMismatch`: the file is there,
// its body is not what the checksum was taken over.
try testing.expect(!compiledBodiesMatch(header ++ "a.example.com\nb.exa", header ++ wild_body, &expected));
try testing.expect(!compiledBodiesMatch("", "", &expected));
// its body is not what the checksum was taken over. An allow body that lost
// its entry counts, because a dropped exception silently restores a block.
try testing.expect(!compiledBodiesMatch(
header ++ "a.example.com\nb.exa",
header ++ wild_body,
header ++ allow_body,
&expected,
));
try testing.expect(!compiledBodiesMatch(header ++ list_body, header ++ wild_body, "", &expected));
try testing.expect(!compiledBodiesMatch("", "", "", &expected));
}
test "a source with no exceptions keeps the checksum it had before the allow body existed" {
const list_body = "a.example.com\nb.example.com\n";
const wild_body = "c.example.com\n";
// What an older nxdns stored: the digest of the two bodies alone. It is what
// sits in `blocklist_sources.checksum` on every installation being upgraded,
// and the files on disk are the two it was taken over.
var hasher = Sha256.init(.{});
hasher.update(list_body);
hasher.update(wild_body);
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
const stored = std.fmt.bytesToHex(digest, .lower);
try testing.expectEqualStrings(&stored, &bodyChecksum(list_body, wild_body, ""));
// No `.allow` file: what `loadSource` and `diskBodiesMatch` pass for one.
try testing.expect(compiledBodiesMatch(list_body, wild_body, "", &stored));
}
test "rejectedWithoutEntries fails a compile that produced nothing usable" {
@@ -2391,14 +2532,23 @@ test "collectSample steps over a line that does not fit the reader buffer" {
try testing.expectEqualStrings("ads.example.com\n", w.buffered());
}
test "bodyChecksum covers the list body followed by the wild body" {
const both = bodyChecksum("a.example.com\n", "b.example.com\n");
test "bodyChecksum covers the list body, then the wild body, then the allow body" {
const all = bodyChecksum("a.example.com\n", "b.example.com\n", "c.example.com\n");
var hasher = Sha256.init(.{});
hasher.update("a.example.com\nb.example.com\n");
hasher.update("a.example.com\nb.example.com\nc.example.com\n");
var digest: [Sha256.digest_length]u8 = undefined;
hasher.final(&digest);
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &both);
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &all);
// Order matters: the two halves are not interchangeable.
try testing.expect(!std.mem.eql(u8, &both, &bodyChecksum("b.example.com\n", "a.example.com\n")));
// Order matters: the three parts are not interchangeable.
try testing.expect(!std.mem.eql(
u8,
&all,
&bodyChecksum("b.example.com\n", "a.example.com\n", "c.example.com\n"),
));
try testing.expect(!std.mem.eql(
u8,
&all,
&bodyChecksum("a.example.com\n", "c.example.com\n", "b.example.com\n"),
));
}
src/filter/matcher.zig
+348 -11
View File
@@ -27,6 +27,11 @@ pub const Reason = enum {
rule_block_exact,
rule_allow_wildcard,
rule_block_wildcard,
rule_allow_regex,
rule_block_regex,
/// An `@@` exception from a downloaded list. It cancels what another list
/// blocks and never what a rule decides — see `evaluate`.
blocklist_exception,
blocklist_domain,
blocklist_wildcard,
};
@@ -34,12 +39,14 @@ pub const Reason = enum {
pub const Decision = struct {
blocked: bool,
reason: Reason,
/// The candidate (for the exact and blocklist levels) or the pattern (for
/// the wildcard levels) that decided it. Borrowed from the caller's
/// normalized buffer or from the snapshot. "" when `reason == .none`.
/// The candidate (for the exact, exception and blocklist levels) or the
/// pattern (for the wildcard and regex levels) that decided it. Borrowed
/// from the caller's normalized buffer or from the snapshot. "" when
/// `reason == .none`.
matched: []const u8,
/// `.blocklist_*` only: index into `Snapshot.sources`, so the query log and
/// the UI can name the list that blocked the query.
/// the UI can name the list that blocked the query — or, for
/// `.blocklist_exception`, the list that lifted it.
source: ?u32 = null,
};
@@ -95,6 +102,10 @@ pub const SourceSets = struct {
name: []const u8,
domains: domain_set.DomainSet,
wildcards: domain_set.DomainSet,
/// The names this source's `@@` exceptions lift. One entry covers the name
/// and every subdomain of it, because `evaluate` walks the full name and
/// each parent against this set.
exceptions: domain_set.DomainSet,
};
pub const Group = struct {
@@ -122,7 +133,15 @@ pub const Snapshot = struct {
/// can tell which generation answered a query.
generation: u64,
pub const Compiled = struct { list_body: []const u8, wild_body: []const u8 };
/// The compiled bodies of one source. `allow_body` defaults to empty
/// because a source compiled before exceptions were honoured has no
/// `.allow` file at all. A source refreshed since then always has one,
/// empty when its list carries no `@@` line.
pub const Compiled = struct {
list_body: []const u8,
wild_body: []const u8,
allow_body: []const u8 = "",
};
pub const Input = struct {
groups: []const model.Group,
@@ -192,6 +211,7 @@ pub const Snapshot = struct {
.name = try arena.dupe(u8, row.name),
.domains = try domain_set.DomainSet.build(arena, bodies.list_body, input.seed),
.wildcards = try domain_set.DomainSet.build(arena, bodies.wild_body, input.seed),
.exceptions = try domain_set.DomainSet.build(arena, bodies.allow_body, input.seed),
};
}
@@ -221,7 +241,11 @@ pub const Snapshot = struct {
.id = id,
.name = try arena.dupe(u8, row.name),
.safe_search = row.safe_search,
.rules = try rules.RuleSet.build(arena, group_rules.items, input.seed),
// `arena` retains, `gpa` scratches: an arena reclaims only its
// most recent allocation, so a rule build's temporaries taken
// from it would outlive the build and go unreported by
// `memoryBytes`.
.rules = try rules.RuleSet.build(arena, gpa, group_rules.items, input.seed),
.sources = try arena.dupe(u32, dedupSorted(group_sources.items)),
};
}
@@ -269,7 +293,9 @@ pub const Snapshot = struct {
/// PLAN §3.10 precedence, allow winning at equal specificity:
/// 1. exact/parent allow rules 2. exact/parent block rules
/// 3. wildcard allow rules 4. wildcard block rules
/// 5. blocklist domains 6. blocklist wildcards
/// 5. regex allow rules 6. regex block rules
/// 7. blocklist exceptions
/// 8. blocklist domains 9. blocklist wildcards
///
/// The order is level-by-level over the whole candidate chain, not
/// candidate-by-candidate over the levels: level 1 is checked against every
@@ -277,10 +303,24 @@ pub const Snapshot = struct {
/// allow rule on the parent beat a block rule on the child, which is the
/// behaviour an allow list is written for.
///
/// Level 5 tests only the full name and level 6 tests only proper parents:
/// Levels 5 and 6 are last among the operator rules because they are the
/// only ones that cost more than a hash lookup or a label walk: a regex is
/// reached only once every set-shaped level has missed. They are matched
/// against the full name alone — a pattern that should cover subdomains
/// says so, which is what an unanchored regex already does.
///
/// Level 7 is where a downloaded list's `@@` exceptions are honoured, and
/// its position is the whole safety argument: every operator rule has
/// already returned by the time it runs, so an exception can cancel a block
/// levels 8 and 9 would have made and nothing else. No downloaded list can
/// open an allow hole the operator did not open. It walks the full name and
/// every parent, because one `@@||x^` entry lifts `x` together with its
/// subdomains.
///
/// Level 8 tests only the full name and level 9 tests only proper parents:
/// a `.list` entry is the domain itself, a `.wild` entry is what `*.x.y`
/// means. Both walk the group's sources in ascending index order, so the
/// reported source is stable for a given snapshot.
/// means. All three list levels walk the group's sources in ascending index
/// order, so the reported source is stable for a given snapshot.
///
/// `domain` is normalized (`normalize`). No allocation, no lock, no clock.
pub fn evaluate(self: *const Snapshot, group: u32, domain: []const u8) Decision {
@@ -307,6 +347,27 @@ pub const Snapshot = struct {
return .{ .blocked = true, .reason = .rule_block_wildcard, .matched = pattern };
}
if (rules.matchRegex(g.rules.regex_allow, domain)) |pattern| {
return .{ .blocked = false, .reason = .rule_allow_regex, .matched = pattern };
}
if (rules.matchRegex(g.rules.regex_block, domain)) |pattern| {
return .{ .blocked = true, .reason = .rule_block_regex, .matched = pattern };
}
var exceptions: Candidates = .init(domain);
while (exceptions.next()) |candidate| {
for (g.sources) |index| {
if (self.sources[index].exceptions.contains(candidate)) {
return .{
.blocked = false,
.reason = .blocklist_exception,
.matched = candidate,
.source = index,
};
}
}
}
for (g.sources) |index| {
if (self.sources[index].domains.contains(domain)) {
return .{
@@ -373,7 +434,8 @@ pub const Snapshot = struct {
var total: usize = 0;
for (self.sources) |*source| {
total += @sizeOf(SourceSets) + source.name.len +
source.domains.memoryBytes() + source.wildcards.memoryBytes();
source.domains.memoryBytes() + source.wildcards.memoryBytes() +
source.exceptions.memoryBytes();
}
for (self.groups) |*group| {
total += @sizeOf(Group) + group.name.len +
@@ -586,6 +648,71 @@ test "precedence: a block wildcard with no allow blocks" {
try testing.expectEqualStrings("*.example.com", decision.matched);
}
test "precedence: a block regex with no allow blocks, and reports its pattern" {
const rows = [_]model.Rule{rule("^ad[0-9]+-", .regex, .block)};
var snapshot = try build(testing.allocator, .{ .rules = &rows });
defer snapshot.deinit();
const decision = snapshot.evaluate(0, "ad42-tracker.example.com");
try testing.expect(decision.blocked);
try testing.expectEqual(Reason.rule_block_regex, decision.reason);
// `matched` is the pattern the operator wrote, which is what the query log
// has to name for the block to be explicable.
try testing.expectEqualStrings("^ad[0-9]+-", decision.matched);
// Unanchored at the tail, anchored at the head: the digits must lead.
try testing.expect(!snapshot.evaluate(0, "x.ad42-tracker.example.com").blocked);
try testing.expect(!snapshot.evaluate(0, "ads.example.com").blocked);
}
test "precedence: an allow regex beats a block regex that matches the same name" {
const rows = [_]model.Rule{
rule("tracker", .regex, .block),
rule("^good\\.", .regex, .allow),
};
var snapshot = try build(testing.allocator, .{ .rules = &rows });
defer snapshot.deinit();
const decision = snapshot.evaluate(0, "good.tracker.example.com");
try testing.expect(!decision.blocked);
try testing.expectEqual(Reason.rule_allow_regex, decision.reason);
try testing.expectEqualStrings("^good\\.", decision.matched);
try testing.expect(snapshot.evaluate(0, "bad.tracker.example.com").blocked);
}
test "precedence: both wildcard levels beat an allow regex that matches" {
// The adjacent pair either side of the wildcard/regex boundary. A regex is
// the most expensive level and therefore the last operator level, so a
// wildcard decides first whichever way it decides.
for ([_]model.Rule{
rule("*.example.com", .wildcard, .allow),
rule("*.example.com", .wildcard, .block),
}) |wild| {
const rows = [_]model.Rule{ wild, rule("example", .regex, .allow) };
var snapshot = try build(testing.allocator, .{ .rules = &rows });
defer snapshot.deinit();
const decision = snapshot.evaluate(0, "a.example.com");
try testing.expectEqual(wild.action == .block, decision.blocked);
try testing.expect(decision.reason == .rule_allow_wildcard or
decision.reason == .rule_block_wildcard);
}
}
test "precedence: an exact allow rule beats a block regex" {
const rows = [_]model.Rule{
rule("tracker", .regex, .block),
rule("good.tracker.example.com", .exact, .allow),
};
var snapshot = try build(testing.allocator, .{ .rules = &rows });
defer snapshot.deinit();
const decision = snapshot.evaluate(0, "good.tracker.example.com");
try testing.expect(!decision.blocked);
try testing.expectEqual(Reason.rule_allow_exact, decision.reason);
}
const one_source = [_]model.BlocklistSource{.{ .url = "https://lists.test/a", .name = "list a" }};
const one_source_id = [_]i64{11};
const one_link = [_]model.GroupSource{
@@ -601,6 +728,14 @@ const Lists = struct {
return .{ .compiled = .{.{ .list_body = list_body, .wild_body = wild_body }} };
}
fn initWithExceptions(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) Lists {
return .{ .compiled = .{.{
.list_body = list_body,
.wild_body = wild_body,
.allow_body = allow_body,
}} };
}
fn fixture(self: *const Lists) Fixture {
return .{
.sources = &one_source,
@@ -669,6 +804,208 @@ test "precedence: an allow rule beats a wild entry" {
try testing.expectEqual(Reason.rule_allow_exact, decision.reason);
}
// --- list exceptions (milestone 21 ruling 2) --------------------------------
/// The fixture ruling 2 is written against: one list that blocks `ads.example`
/// and its subdomains, and lifts `good.ads.example` back out.
const exception_lists: Lists = .initWithExceptions(
"ads.example\n",
"ads.example\n",
"good.ads.example\n",
);
test "precedence: a list exception beats a list domain entry" {
var snapshot = try build(testing.allocator, exception_lists.fixture());
defer snapshot.deinit();
// The apex is blocked by the `.list` entry; the excepted name is not, even
// though the same source blocks it through `.wild`.
try testing.expect(snapshot.evaluate(0, "ads.example").blocked);
const decision = snapshot.evaluate(0, "good.ads.example");
try testing.expect(!decision.blocked);
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
try testing.expectEqualStrings("good.ads.example", decision.matched);
try testing.expectEqual(@as(?u32, 0), decision.source);
}
test "precedence: a list exception beats a list domain entry on the same name" {
// The boundary the fixture above cannot pin: `good.ads.example` is not in
// its `.list` body, so that test compares the exception against the
// wildcard level. Here one name is carried by both `.allow` and `.list`,
// which is the only way level 7 and level 8 are reached by one query.
const lists: Lists = .initWithExceptions(
"good.ads.example\n",
"",
"good.ads.example\n",
);
var snapshot = try build(testing.allocator, lists.fixture());
defer snapshot.deinit();
const decision = snapshot.evaluate(0, "good.ads.example");
try testing.expect(!decision.blocked);
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
try testing.expectEqualStrings("good.ads.example", decision.matched);
try testing.expectEqual(@as(?u32, 0), decision.source);
}
test "precedence: a list domain entry beats a list wildcard entry" {
// Level 8 over level 9: the name is its own `.list` entry and a subdomain
// of a `.wild` entry, so both would block and the reported reason is what
// separates them.
const lists: Lists = .init("x.ads.example\n", "ads.example\n");
var snapshot = try build(testing.allocator, lists.fixture());
defer snapshot.deinit();
const decision = snapshot.evaluate(0, "x.ads.example");
try testing.expect(decision.blocked);
try testing.expectEqual(Reason.blocklist_domain, decision.reason);
try testing.expectEqualStrings("x.ads.example", decision.matched);
try testing.expectEqual(@as(?u32, 0), decision.source);
}
test "precedence: a list exception beats a list wildcard entry" {
var snapshot = try build(testing.allocator, exception_lists.fixture());
defer snapshot.deinit();
try testing.expect(snapshot.evaluate(0, "x.ads.example").blocked);
// The parent walk: one `@@||good.ads.example^` entry covers the subdomains
// of the excepted name as well as the name itself.
const decision = snapshot.evaluate(0, "y.good.ads.example");
try testing.expect(!decision.blocked);
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
try testing.expectEqualStrings("good.ads.example", decision.matched);
try testing.expectEqual(@as(?u32, 0), decision.source);
}
test "precedence: an operator block rule beats a list exception" {
// The property the exception level's position exists for: a downloaded list
// may cancel what another list blocks and may never cancel what the
// operator decided. All three operator block levels are checked, because
// all three sit above the exception level.
for ([_]model.Rule{
rule("good.ads.example", .exact, .block),
rule("*.ads.example", .wildcard, .block),
rule("^good\\.ads\\.example$", .regex, .block),
}) |blocking| {
var fixture = exception_lists.fixture();
const rows = [_]model.Rule{blocking};
fixture.rules = &rows;
var snapshot = try build(testing.allocator, fixture);
defer snapshot.deinit();
const decision = snapshot.evaluate(0, "good.ads.example");
try testing.expect(decision.blocked);
try testing.expect(decision.reason == .rule_block_exact or
decision.reason == .rule_block_wildcard or
decision.reason == .rule_block_regex);
}
}
test "the regex reasons render as the wire strings the API and the query log carry" {
// `web/handlers/lookup.zig` renders a reason as `@tagName`, and
// `storage/logger.zig` stores one in a 32-byte `max_reason_len` buffer.
// Neither can be reached from this file — pure core imports no web and no
// storage — so the tag names and their length are pinned here.
try testing.expectEqualStrings("rule_allow_regex", @tagName(Reason.rule_allow_regex));
try testing.expectEqualStrings("rule_block_regex", @tagName(Reason.rule_block_regex));
inline for (@typeInfo(Reason).@"enum".fields) |field| {
try testing.expect(field.name.len <= 32);
}
}
test "precedence: an allow regex beats every list level" {
// The other side of the same boundary: an operator allow rule lifts a list
// block, whichever of the three list levels made it.
const rows = [_]model.Rule{rule("ads\\.example$", .regex, .allow)};
var fixture = exception_lists.fixture();
fixture.rules = &rows;
var snapshot = try build(testing.allocator, fixture);
defer snapshot.deinit();
// `.list` blocks the apex and `.wild` blocks the subdomains; the regex
// covers both, and answers before either is consulted.
for ([_][]const u8{ "ads.example", "x.ads.example" }) |domain| {
const decision = snapshot.evaluate(0, domain);
try testing.expect(!decision.blocked);
try testing.expectEqual(Reason.rule_allow_regex, decision.reason);
try testing.expectEqual(@as(?u32, null), decision.source);
}
}
test "precedence: a block regex blocks a name no list carries" {
const rows = [_]model.Rule{rule("^ad[0-9]+-", .regex, .block)};
var fixture = exception_lists.fixture();
fixture.rules = &rows;
var snapshot = try build(testing.allocator, fixture);
defer snapshot.deinit();
const decision = snapshot.evaluate(0, "ad7-cdn.other.example");
try testing.expect(decision.blocked);
try testing.expectEqual(Reason.rule_block_regex, decision.reason);
try testing.expectEqual(@as(?u32, null), decision.source);
}
test "precedence: a list exception is scoped to the groups the source is in" {
const groups = [_]model.Group{ .{ .name = "default" }, .{ .name = "kids" } };
const ids = [_]i64{ 1, 2 };
var snapshot = try build(testing.allocator, .{
.groups = &groups,
.group_ids = &ids,
.sources = &one_source,
.source_ids = &one_source_id,
.group_sources = &one_link,
.compiled = &exception_lists.compiled,
});
defer snapshot.deinit();
const kids = snapshot.groupIndexByName("kids").?;
try testing.expectEqual(
Reason.blocklist_exception,
snapshot.evaluate(snapshot.default_group, "good.ads.example").reason,
);
// `kids` is linked to no source, so neither the block nor the exception
// reaches it.
try testing.expectEqual(Reason.none, snapshot.evaluate(kids, "good.ads.example").reason);
}
test "precedence: an exception in one list lifts the block another list made" {
const sources = [_]model.BlocklistSource{
.{ .url = "https://lists.test/a", .name = "list a" },
.{ .url = "https://lists.test/b", .name = "list b" },
};
const source_ids = [_]i64{ 11, 12 };
const links = [_]model.GroupSource{
.{ .group = "default", .source_url = "https://lists.test/a" },
.{ .group = "default", .source_url = "https://lists.test/b" },
};
const compiled = [_]?Snapshot.Compiled{
.{ .list_body = "ads.example\n", .wild_body = "ads.example\n" },
.{ .list_body = "", .wild_body = "", .allow_body = "good.ads.example\n" },
};
var snapshot = try build(testing.allocator, .{
.sources = &sources,
.source_ids = &source_ids,
.group_sources = &links,
.compiled = &compiled,
});
defer snapshot.deinit();
try testing.expect(snapshot.evaluate(0, "ads.example").blocked);
const decision = snapshot.evaluate(0, "good.ads.example");
try testing.expect(!decision.blocked);
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
// The source reported is the one that lifted the block, not the one that
// made it.
try testing.expectEqual(@as(?u32, 1), decision.source);
}
test "precedence: nothing configured allows with reason none" {
var snapshot = try build(testing.allocator, .{});
defer snapshot.deinit();
src/filter/parser_abp.zig
+132 -8
View File
@@ -1,9 +1,17 @@
//! The Adblock Plus filter syntax, restricted to what a DNS sinkhole can
//! honour: domain anchors and bare names. Pure, `std` only.
//!
//! Exception rules (`@@`) are `.unsupported` rather than an allow entry. The
//! allow surface is the `rules` table, and a downloaded list that could quietly
//! allow a domain across every group is a policy hole the operator did not open.
//! An exception rule is `.exception` in exactly two spellings, `@@||name^` and
//! `@@||name`, each of which may carry the literal `$important` behind it. Every
//! other `@@` form stays `.unsupported`: a bare `@@name`, a path, a scheme, any
//! other modifier.
//!
//! What makes honouring them safe is where they land, not what they say. A list
//! exception is evaluated below every operator rule (PLAN §3.10), so it can
//! cancel a block another list made and nothing else. No downloaded list can
//! open an allow hole the operator did not open, which is why the allow surface
//! stays the `rules` table — including its `.regex` kind, which is the one
//! regex dialect nxdns evaluates and which no downloaded list can reach.
const std = @import("std");
const parsers = @import("parsers.zig");
@@ -12,6 +20,31 @@ const parsers = @import("parsers.zig");
/// this syntax and only a trailing one is meaningful for a domain rule.
const rule_tokens = "*^|/$";
/// The one modifier an exception line may carry. AdGuard-authored lists write it
/// on most of their `@@` rules and it changes nothing here: these exceptions
/// already sit below every operator rule, so "important" cannot raise one above
/// the decisions it is not allowed to reach.
const important_modifier = "$important";
/// A candidate the compiler could turn into a name, for the two anchored forms
/// only. `rule_tokens` covers the syntax characters; this also refuses
/// whitespace, which is neither a rule token nor a control byte and so used to
/// survive into a compiled body as an entry only a query carrying the same
/// space could match. That is true of `||name` and `@@||name` because
/// `compiler.zig` hands a `.wildcard` or `.exception` text to `addCandidate`
/// whole. A `.domain` text is tokenized on whitespace first and each field
/// filed separately, so the bare form does not come through here: refusing a
/// space there would drop the hosts-style lines that a mixed list classified
/// `abp` by `detectFormat` still contributes.
fn isNameCandidate(candidate: []const u8) bool {
if (candidate.len == 0) return false;
if (std.mem.findAny(u8, candidate, rule_tokens) != null) return false;
for (candidate) |c| {
if (std.ascii.isWhitespace(c) or std.ascii.isControl(c)) return false;
}
return true;
}
pub fn parseLine(line: []const u8) parsers.Line {
const text = std.mem.trim(u8, line, &std.ascii.whitespace);
if (text.len == 0) return .{ .kind = .ignore };
@@ -19,15 +52,14 @@ pub fn parseLine(line: []const u8) parsers.Line {
if (text[0] == '[') return .{ .kind = .ignore };
if (parsers.isElementHiding(text)) return .{ .kind = .unsupported };
if (text[0] == '#') return .{ .kind = .ignore };
if (std.mem.startsWith(u8, text, "@@")) return .{ .kind = .unsupported };
if (std.mem.startsWith(u8, text, "@@")) return parseException(text[2..]);
if (text[0] == '/') return .{ .kind = .regex };
if (std.mem.findScalar(u8, text, '$') != null) return .{ .kind = .unsupported };
if (std.mem.startsWith(u8, text, "||")) {
var candidate = text[2..];
if (std.mem.endsWith(u8, candidate, "^")) candidate = candidate[0 .. candidate.len - 1];
if (candidate.len == 0) return .{ .kind = .unsupported };
if (std.mem.findAny(u8, candidate, rule_tokens) != null) return .{ .kind = .unsupported };
if (!isNameCandidate(candidate)) return .{ .kind = .unsupported };
// A domain anchor covers the domain itself as well as its subdomains,
// so the compiler emits an apex entry beside the wildcard one.
return .{ .kind = .wildcard, .text = candidate, .covers_apex = true };
@@ -37,6 +69,24 @@ pub fn parseLine(line: []const u8) parsers.Line {
return .{ .kind = .domain, .text = text };
}
/// One exception line, past its `@@`. The domain anchor and the trailing `^` get
/// the same treatment they get on a block rule, so `@@||x^` and `||x^` accept
/// and reject the same names.
///
/// `$important` is stripped before the anchor is read, because the `$` would
/// otherwise be a rule token and refuse the whole line.
fn parseException(rest: []const u8) parsers.Line {
if (!std.mem.startsWith(u8, rest, "||")) return .{ .kind = .unsupported };
var candidate = rest[2..];
if (std.mem.endsWith(u8, candidate, important_modifier)) {
candidate = candidate[0 .. candidate.len - important_modifier.len];
}
if (std.mem.endsWith(u8, candidate, "^")) candidate = candidate[0 .. candidate.len - 1];
if (!isNameCandidate(candidate)) return .{ .kind = .unsupported };
return .{ .kind = .exception, .text = candidate, .covers_apex = true };
}
const testing = std.testing;
test "a bang comment is ignored" {
@@ -69,8 +119,46 @@ test "a modifier list is unsupported" {
try testing.expectEqual(parsers.Kind.unsupported, parseLine("||example.com^$third-party").kind);
}
test "an exception rule is unsupported" {
try testing.expectEqual(parsers.Kind.unsupported, parseLine("@@||example.com^").kind);
test "an exception rule is an exception that covers its apex" {
const line = parseLine("@@||example.com^");
try testing.expectEqual(parsers.Kind.exception, line.kind);
try testing.expectEqualStrings("example.com", line.text);
try testing.expect(line.covers_apex);
}
test "an exception rule without a separator is still an exception" {
const line = parseLine("@@||example.com");
try testing.expectEqual(parsers.Kind.exception, line.kind);
try testing.expectEqualStrings("example.com", line.text);
try testing.expect(line.covers_apex);
}
test "an exception rule tolerates the important modifier" {
for ([_][]const u8{ "@@||example.com^$important", "@@||example.com$important" }) |text| {
const line = parseLine(text);
try testing.expectEqual(parsers.Kind.exception, line.kind);
try testing.expectEqualStrings("example.com", line.text);
try testing.expect(line.covers_apex);
}
}
test "every exception form outside the two anchored ones is unsupported" {
for ([_][]const u8{
// No domain anchor: this is a substring rule in browser syntax, and
// reading it as a name would allow far more than it says.
"@@example.com",
"@@|http://example.com",
"@@||example.com/path^",
"@@||example.com^$third-party",
"@@||example.com^$important$third-party",
"@@||example.com^$dnstype=A",
"@@||^",
"@@||$important",
"@@",
"@@||ads*.example.com^",
}) |text| {
try testing.expectEqual(parsers.Kind.unsupported, parseLine(text).kind);
}
}
test "element hiding is unsupported" {
@@ -94,6 +182,42 @@ test "a bare name is a domain" {
try testing.expectEqualStrings("example.com", line.text);
}
test "a candidate carrying whitespace is unsupported in the anchored forms" {
// A space is not a rule token and it is not a control byte, so it used to
// reach the compiler, which lowercases and length-checks but does not
// reject it. An anchored form's text is filed whole, so the entry it wrote
// could only ever match a query name carrying the same space.
for ([_][]const u8{
"||good.example bad.example^",
"@@||good.example bad.example^",
"||good.example\tbad.example",
"@@||good.example\tbad.example",
}) |text| {
try testing.expectEqual(parsers.Kind.unsupported, parseLine(text).kind);
}
}
test "a bare candidate carrying whitespace stays a domain" {
// Not the same case: `compiler.zig` tokenizes a `.domain` text on
// whitespace and files each field. Refusing it here would drop the
// hosts-style lines of a mixed list, which `detectFormat` classifies `abp`
// as a whole and which only reach a compiled body through that split.
//
// What this test pins is the parser half — the kind and the untouched text.
// The split itself belongs to the compiler and is asserted there, by
// "an abp list's hosts-style lines reach the domain body through the split":
// a tokenizer removed from `compile` would leave every assertion below true.
for ([_][]const u8{
"good.example bad.example",
"0.0.0.0 ads.example",
"good.example\tbad.example",
}) |text| {
const line = parseLine(text);
try testing.expectEqual(parsers.Kind.domain, line.kind);
try testing.expectEqualStrings(text, line.text);
}
}
test "a rule token outside the supported forms is unsupported" {
try testing.expectEqual(parsers.Kind.unsupported, parseLine("ads*.example.com").kind);
try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com^").kind);
src/filter/parsers.zig
+12 -4
View File
@@ -22,10 +22,15 @@ pub const Kind = enum {
domain,
/// `text` holds one candidate suffix; every proper subdomain of it matches.
wildcard,
/// A regex rule. Counted, skipped, never compiled (PLAN §2.2).
/// A regex line in a downloaded list. Counted, skipped, never compiled: the
/// engine exists for rules the operator wrote, not for lists (PLAN §2.2).
regex,
/// `text` holds one candidate name an ABP exception rule (`@@||x^`) lifts:
/// the name itself and every subdomain of it. Only the ABP parser emits it.
exception,
/// Syntactically a rule of this format, but one nxdns cannot honour:
/// an ABP modifier list, an exception rule, element hiding, a scheme anchor.
/// an ABP modifier list, an exception form outside `@@||x^`, element
/// hiding, a scheme anchor.
unsupported,
};
@@ -33,8 +38,11 @@ pub const Line = struct {
kind: Kind,
/// Borrowed from the caller's line. Not lowercased, not validated.
text: []const u8 = "",
/// `.wildcard` only. ABP `||x^` covers `x` itself as well as its subdomains,
/// so the compiler emits an additional `.list` entry when this is set.
/// `.wildcard` and `.exception` only: the rule covers the anchored name
/// itself as well as its subdomains, which is what ABP `||x^` and `@@||x^`
/// mean. The compiler acts on it for a `.wildcard` line, by emitting an
/// additional `.list` entry; an `.exception` line needs no second entry,
/// because the allow walk tests the full name as well as its parents.
covers_apex: bool = false,
};
src/filter/regex.zig
+1101
View File
File diff suppressed because it is too large Load Diff
src/filter/rules.zig
+339 -62
View File
@@ -1,4 +1,4 @@
//! One group's explicit rules (PLAN §3.10 levels 1–4), compiled once into an
//! One group's explicit rules (PLAN §3.10 levels 1–6), compiled once into an
//! immutable form the query path can read without allocating.
//!
//! Exact patterns go into a `DomainSet`; wildcard patterns stay a flat, sorted
@@ -7,7 +7,13 @@
//! that many short patterns is cheaper than an index that would have to be
//! rebuilt on every snapshot swap.
//!
//! Pure: an allocator and plain values, no `std.Io`, no clock, no entropy
//! Regex patterns are compiled here, once per snapshot, into the linear-time
//! programs of `regex.zig` and scanned the same way. `max_regex_per_group` caps
//! them far lower, at 256: a regex costs a whole VM run where a wildcard costs a
//! label comparison, and the matcher reaches them only after every hash and
//! wildcard level has missed.
//!
//! Pure: allocators and plain values, no `std.Io`, no clock, no entropy
//! source. The hash seed arrives as a parameter.
const std = @import("std");
@@ -17,14 +23,40 @@ const model = @import("../config/model.zig");
const name = @import("../dns/name.zig");
const types = @import("../dns/types.zig");
const domain_set = @import("domain_set.zig");
const regex = @import("regex.zig");
const wildcard = @import("wildcard.zig");
pub const Error = error{ OutOfMemory, BadPattern, TooManyWildcards } || domain_set.DomainSet.Error;
pub const Error = error{
OutOfMemory,
BadPattern,
TooManyWildcards,
TooManyRegexRules,
} || domain_set.DomainSet.Error;
/// Both wildcard lists of one group together. The cap exists so a rules table
/// edited into the millions cannot turn every query into a linear scan.
pub const max_wildcards_per_group: usize = 4096;
/// Both regex lists of one group together, capped well below the wildcards: a
/// miss at this level runs every program to its end.
pub const max_regex_per_group: usize = 256;
/// A compiled operator regex beside the text it was written as. The text is what
/// `Decision.matched` reports, so the query log names the rule the operator
/// wrote rather than an instruction count.
pub const RegexRule = struct {
pattern: []const u8,
program: regex.Program,
/// Frees through a copy of the program: the slices holding these rules are
/// `const`, and `Program.deinit` wants a mutable pointer only to blank the
/// struct it is finished with.
fn free(self: RegexRule, gpa: Allocator) void {
var program = self.program;
program.deinit(gpa);
}
};
pub const RuleSet = struct {
exact_allow: domain_set.DomainSet = .empty,
exact_block: domain_set.DomainSet = .empty,
@@ -32,75 +64,115 @@ pub const RuleSet = struct {
/// order and the same first match.
wildcard_allow: []const []const u8 = &.{},
wildcard_block: []const []const u8 = &.{},
/// One block holding the bytes of both wildcard lists; freed as a unit.
wildcard_bytes: []const u8 = &.{},
/// Sorted and deduplicated like the wildcards, so the first regex to match a
/// name is the same one on every rebuild of the same rows.
regex_allow: []const RegexRule = &.{},
regex_block: []const RegexRule = &.{},
/// One block holding the pattern bytes of all four lists; freed as a unit.
pattern_bytes: []const u8 = &.{},
pub const empty: RuleSet = .{};
/// `rows` are one group's rules only; splitting `listRules` output by group
/// belongs to the caller, which is the only holder of the group table.
///
/// Patterns are normalized (lowercase over ASCII, one trailing dot
/// Name patterns are normalized (lowercase over ASCII, one trailing dot
/// stripped) and validated: `.exact` through `dns.name.fromText`,
/// `.wildcard` through `wildcard.validate`. An invalid pattern is
/// `error.BadPattern`, not a skipped row — every pattern passed
/// `config/validate.zig` on the way in, so an invalid one here means the
/// rows were edited underneath nxdns and a silently dropped allow rule
/// `.wildcard` through `wildcard.validate`, `.regex` by compiling it. An
/// invalid pattern is `error.BadPattern`, not a skipped row — every pattern
/// passed `config/validate.zig` on the way in, so an invalid one here means
/// the rows were edited underneath nxdns and a silently dropped allow rule
/// would block a domain the operator unblocked.
pub fn build(gpa: Allocator, rows: []const model.Rule, seed: u64) Error!RuleSet {
///
/// Two allocators, because the caller's `perm` is a snapshot arena: an
/// arena reclaims only its most recent allocation, so every temporary taken
/// from it would live as long as the snapshot and go unreported by
/// `memoryBytes`. `perm` owns what the returned set retains and is what
/// `deinit` frees; `scratch` owns the build's working storage, which is
/// released by the time `build` returns. Passing one allocator as both is
/// correct wherever freeing works normally.
pub fn build(
perm: Allocator,
scratch: Allocator,
rows: []const model.Rule,
seed: u64,
) Error!RuleSet {
if (rows.len == 0) return .empty;
var scratch: std.ArrayList(u8) = .empty;
defer scratch.deinit(gpa);
var spans: [4]std.ArrayList(Span) = .{ .empty, .empty, .empty, .empty };
defer for (&spans) |*bucket| bucket.deinit(gpa);
var joined: std.ArrayList(u8) = .empty;
defer joined.deinit(scratch);
var spans: [6]std.ArrayList(Span) = @splat(.empty);
defer for (&spans) |*bucket| bucket.deinit(scratch);
var wildcards: usize = 0;
var regexes: usize = 0;
var buf: [types.max_name_len]u8 = undefined;
for (rows) |row| {
const pattern = normalize(row.pattern, &buf) catch return error.BadPattern;
switch (row.kind) {
.exact => _ = name.fromText(pattern) catch return error.BadPattern,
.wildcard => {
wildcard.validate(pattern) catch return error.BadPattern;
const pattern = switch (row.kind) {
.exact => blk: {
const text = normalize(row.pattern, &buf) catch return error.BadPattern;
_ = name.fromText(text) catch return error.BadPattern;
break :blk text;
},
.wildcard => blk: {
const text = normalize(row.pattern, &buf) catch return error.BadPattern;
wildcard.validate(text) catch return error.BadPattern;
wildcards += 1;
if (wildcards > max_wildcards_per_group) return error.TooManyWildcards;
break :blk text;
},
}
// A regex is not a name, so `normalize` must not touch it: it
// strips a trailing `.`, which here is the any-byte atom, and it
// lowercases, which turns the rejected `\D` into the accepted
// `\d`. Either would silently change what the rule matches. The
// bytes stay as the operator wrote them — the same bytes
// `config/validate.zig` compiled at the edge. Compiling waits
// until after the sort, so a duplicate is compiled once.
.regex => blk: {
regexes += 1;
if (regexes > max_regex_per_group) return error.TooManyRegexRules;
break :blk row.pattern;
},
};
const bucket = &spans[bucketOf(row.kind, row.action)];
try bucket.append(gpa, .{ .offset = scratch.items.len, .len = pattern.len });
try scratch.appendSlice(gpa, pattern);
try bucket.append(scratch, .{ .offset = joined.items.len, .len = pattern.len });
try joined.appendSlice(scratch, pattern);
}
// `scratch` stops growing here, so spans can become slices of it.
var sorted: [4]std.ArrayList([]const u8) = .{ .empty, .empty, .empty, .empty };
defer for (&sorted) |*bucket| bucket.deinit(gpa);
// `joined` stops growing here, so spans can become slices of it.
var sorted: [6]std.ArrayList([]const u8) = @splat(.empty);
defer for (&sorted) |*bucket| bucket.deinit(scratch);
for (&spans, &sorted) |*bucket, *out| {
try out.ensureTotalCapacityPrecise(gpa, bucket.items.len);
try out.ensureTotalCapacityPrecise(scratch, bucket.items.len);
for (bucket.items) |span| {
out.appendAssumeCapacity(scratch.items[span.offset..][0..span.len]);
out.appendAssumeCapacity(joined.items[span.offset..][0..span.len]);
}
std.mem.sort([]const u8, out.items, {}, lessThanBytes);
dedupSorted(out);
}
var self: RuleSet = .empty;
errdefer self.deinit(gpa);
errdefer self.deinit(perm);
self.exact_allow = try buildSet(gpa, sorted[bucketOf(.exact, .allow)].items, seed);
self.exact_block = try buildSet(gpa, sorted[bucketOf(.exact, .block)].items, seed);
self.exact_allow = try buildSet(perm, scratch, sorted[bucketOf(.exact, .allow)].items, seed);
self.exact_block = try buildSet(perm, scratch, sorted[bucketOf(.exact, .block)].items, seed);
const allow = sorted[bucketOf(.wildcard, .allow)].items;
const block = sorted[bucketOf(.wildcard, .block)].items;
const wild_allow = sorted[bucketOf(.wildcard, .allow)].items;
const wild_block = sorted[bucketOf(.wildcard, .block)].items;
const re_allow = sorted[bucketOf(.regex, .allow)].items;
const re_block = sorted[bucketOf(.regex, .block)].items;
var total: usize = 0;
for (allow) |pattern| total += pattern.len;
for (block) |pattern| total += pattern.len;
for ([_][]const []const u8{ wild_allow, wild_block, re_allow, re_block }) |list| {
for (list) |pattern| total += pattern.len;
}
const bytes = try gpa.alloc(u8, total);
self.wildcard_bytes = bytes;
const bytes = try perm.alloc(u8, total);
self.pattern_bytes = bytes;
var at: usize = 0;
self.wildcard_allow = try copyPatterns(gpa, allow, bytes, &at);
self.wildcard_block = try copyPatterns(gpa, block, bytes, &at);
self.wildcard_allow = try copyPatterns(perm, wild_allow, bytes, &at);
self.wildcard_block = try copyPatterns(perm, wild_block, bytes, &at);
self.regex_allow = try compilePatterns(perm, scratch, re_allow, bytes, &at);
self.regex_block = try compilePatterns(perm, scratch, re_block, bytes, &at);
return self;
}
@@ -110,18 +182,35 @@ pub const RuleSet = struct {
self.exact_block.deinit(gpa);
gpa.free(self.wildcard_allow);
gpa.free(self.wildcard_block);
gpa.free(self.wildcard_bytes);
freeRules(gpa, self.regex_allow);
freeRules(gpa, self.regex_block);
gpa.free(self.pattern_bytes);
self.* = .empty;
}
pub fn memoryBytes(self: *const RuleSet) usize {
var programs: usize = 0;
for (self.regex_allow) |item| programs += item.program.memoryBytes();
for (self.regex_block) |item| programs += item.program.memoryBytes();
return self.exact_allow.memoryBytes() +
self.exact_block.memoryBytes() +
self.wildcard_bytes.len +
(self.wildcard_allow.len + self.wildcard_block.len) * @sizeOf([]const u8);
self.pattern_bytes.len +
programs +
(self.wildcard_allow.len + self.wildcard_block.len) * @sizeOf([]const u8) +
(self.regex_allow.len + self.regex_block.len) * @sizeOf(RegexRule);
}
};
/// The first regex of `list` that matches `domain`, or null. `list` is sorted,
/// so "first" is stable across rebuilds of the same rows. The caller checks the
/// allow list before the block list, as it does for wildcards.
pub fn matchRegex(list: []const RegexRule, domain: []const u8) ?[]const u8 {
for (list) |item| {
if (regex.matches(&item.program, domain)) return item.pattern;
}
return null;
}
// ---------------------------------------------------------------------------
// Internals
// ---------------------------------------------------------------------------
@@ -131,15 +220,16 @@ pub const RuleSet = struct {
const Span = struct { offset: usize, len: usize };
fn bucketOf(kind: model.RuleKind, action: model.RuleAction) usize {
const kind_bit: usize = switch (kind) {
const kind_base: usize = switch (kind) {
.exact => 0,
.wildcard => 2,
.regex => 4,
};
const action_bit: usize = switch (action) {
const action_offset: usize = switch (action) {
.allow => 0,
.block => 1,
};
return kind_bit + action_bit;
return kind_base + action_offset;
}
fn lessThanBytes(_: void, a: []const u8, b: []const u8) bool {
@@ -159,26 +249,31 @@ fn dedupSorted(list: *std.ArrayList([]const u8)) void {
list.shrinkRetainingCapacity(kept);
}
fn buildSet(gpa: Allocator, patterns: []const []const u8, seed: u64) Error!domain_set.DomainSet {
fn buildSet(
perm: Allocator,
scratch: Allocator,
patterns: []const []const u8,
seed: u64,
) Error!domain_set.DomainSet {
if (patterns.len == 0) return .empty;
var body: std.ArrayList(u8) = .empty;
defer body.deinit(gpa);
defer body.deinit(scratch);
for (patterns) |pattern| {
try body.appendSlice(gpa, pattern);
try body.append(gpa, '\n');
try body.appendSlice(scratch, pattern);
try body.append(scratch, '\n');
}
return domain_set.DomainSet.build(gpa, body.items, seed);
return domain_set.DomainSet.build(perm, body.items, seed);
}
fn copyPatterns(
gpa: Allocator,
perm: Allocator,
patterns: []const []const u8,
bytes: []u8,
at: *usize,
) Error![]const []const u8 {
if (patterns.len == 0) return &.{};
const out = try gpa.alloc([]const u8, patterns.len);
const out = try perm.alloc([]const u8, patterns.len);
for (out, patterns) |*slot, pattern| {
@memcpy(bytes[at.*..][0..pattern.len], pattern);
slot.* = bytes[at.*..][0..pattern.len];
@@ -187,6 +282,49 @@ fn copyPatterns(
return out;
}
/// Copies the pattern texts into `bytes` like `copyPatterns` and compiles each
/// one. A compile failure is `error.BadPattern` whichever of the engine's three
/// refusals fired: the pattern already passed `config/validate.zig`, which names
/// the limit, so a row that fails here was written around that check.
///
/// Compiling into `scratch` and cloning across is what keeps a parse-time AST
/// out of `perm`: `regex.compile` builds the AST, the child lists and the
/// growing instruction buffer through the allocator it returns the program on.
fn compilePatterns(
perm: Allocator,
scratch: Allocator,
patterns: []const []const u8,
bytes: []u8,
at: *usize,
) Error![]const RegexRule {
if (patterns.len == 0) return &.{};
const out = try perm.alloc(RegexRule, patterns.len);
var built: usize = 0;
errdefer {
for (out[0..built]) |item| item.free(perm);
perm.free(out);
}
for (out, patterns) |*slot, pattern| {
var compiled = regex.compile(scratch, pattern) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
else => return error.BadPattern,
};
defer compiled.deinit(scratch);
const program = try compiled.clone(perm);
@memcpy(bytes[at.*..][0..pattern.len], pattern);
slot.* = .{ .pattern = bytes[at.*..][0..pattern.len], .program = program };
at.* += pattern.len;
built += 1;
}
return out;
}
fn freeRules(gpa: Allocator, list: []const RegexRule) void {
for (list) |item| item.free(gpa);
gpa.free(list);
}
const NameError = error{BadName};
/// Lowercases over ASCII and strips one trailing dot. A byte ≥ 0x80 is
@@ -223,7 +361,7 @@ test "exact rules land in the matching set" {
rule("ads.example.com", .exact, .block),
rule("good.example.com", .exact, .allow),
};
var set = try RuleSet.build(testing.allocator, &rows, 0x5eed);
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0x5eed);
defer set.deinit(testing.allocator);
try testing.expect(set.exact_block.contains("ads.example.com"));
@@ -239,7 +377,7 @@ test "wildcard rules land in the matching list, sorted" {
rule("*.a.example.com", .wildcard, .block),
rule("*.allowed.example.com", .wildcard, .allow),
};
var set = try RuleSet.build(testing.allocator, &rows, 0x5eed);
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0x5eed);
defer set.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), set.wildcard_block.len);
@@ -254,7 +392,7 @@ test "patterns are normalized to lowercase without a trailing dot" {
rule("ADS.Example.COM.", .exact, .block),
rule("*.Tracker.NET.", .wildcard, .block),
};
var set = try RuleSet.build(testing.allocator, &rows, 0);
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
defer set.deinit(testing.allocator);
try testing.expect(set.exact_block.contains("ads.example.com"));
@@ -268,7 +406,7 @@ test "duplicate rows collapse to one entry" {
rule("*.x.example.com", .wildcard, .block),
rule("*.x.example.com", .wildcard, .block),
};
var set = try RuleSet.build(testing.allocator, &rows, 0);
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
defer set.deinit(testing.allocator);
try testing.expectEqual(@as(u32, 1), set.exact_block.count);
@@ -278,14 +416,98 @@ test "duplicate rows collapse to one entry" {
test "an invalid exact pattern is an error" {
for ([_][]const u8{ "", ".", "a..b", "ads example.com", "ads\u{00e9}.example.com" }) |pattern| {
const rows = [_]model.Rule{rule(pattern, .exact, .block)};
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0));
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &rows, 0));
}
}
test "regex rules land in their own buckets, compiled and sorted" {
const rows = [_]model.Rule{
rule("^zz", .regex, .block),
rule("^aa", .regex, .block),
rule("ok$", .regex, .allow),
};
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0x5eed);
defer set.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 2), set.regex_block.len);
try testing.expectEqualStrings("^aa", set.regex_block[0].pattern);
try testing.expectEqualStrings("^zz", set.regex_block[1].pattern);
try testing.expectEqual(@as(usize, 1), set.regex_allow.len);
try testing.expectEqualStrings("ok$", set.regex_allow[0].pattern);
try testing.expectEqualStrings("^aa", matchRegex(set.regex_block, "aabb.example").?);
try testing.expect(matchRegex(set.regex_block, "bbaa.example") == null);
try testing.expectEqualStrings("ok$", matchRegex(set.regex_allow, "example.ok").?);
}
test "a regex pattern keeps the bytes the operator wrote" {
// `normalize` would strip the trailing dot and lowercase the escape, and
// either edit would change what the pattern matches. The exact and wildcard
// kinds still normalize; only this one is exempt.
const rows = [_]model.Rule{
rule("ADS\\.Example\\.", .regex, .block),
rule("ADS.Example.", .exact, .block),
};
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
defer set.deinit(testing.allocator);
try testing.expectEqualStrings("ADS\\.Example\\.", set.regex_block[0].pattern);
try testing.expect(set.exact_block.contains("ads.example"));
}
test "duplicate regex rows collapse to one compiled program" {
const rows = [_]model.Rule{
rule("^ad[0-9]+-", .regex, .block),
rule("^ad[0-9]+-", .regex, .block),
};
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
defer set.deinit(testing.allocator);
try testing.expectEqual(@as(usize, 1), set.regex_block.len);
}
test "a regex pattern the engine refuses is an error, not a skipped row" {
for ([_][]const u8{ "(", "", "a+?", "[z-a]", "\\s" }) |pattern| {
const rows = [_]model.Rule{rule(pattern, .regex, .block)};
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &rows, 0));
}
// The size limits arrive as `BadPattern` too: which one fired is
// `config/validate.zig`'s to report, and by here the row is simply wrong.
const long = [_]model.Rule{rule("a" ** 300, .regex, .block)};
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &long, 0));
const complex = [_]model.Rule{rule("(abcdefghij){200}", .regex, .block)};
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &complex, 0));
}
test "too many regex rules is an error" {
const gpa = testing.allocator;
const rows = try gpa.alloc(model.Rule, max_regex_per_group + 1);
defer gpa.free(rows);
var patterns: std.ArrayList([]u8) = .empty;
defer {
for (patterns.items) |p| gpa.free(p);
patterns.deinit(gpa);
}
for (rows, 0..) |*row, i| {
const pattern = try std.fmt.allocPrint(gpa, "^n{d}-", .{i});
try patterns.append(gpa, pattern);
row.* = rule(pattern, .regex, .block);
}
try testing.expectError(error.TooManyRegexRules, RuleSet.build(gpa, gpa, rows, 0));
// The cap counts both actions together, like the wildcard one.
rows[0].action = .allow;
try testing.expectError(error.TooManyRegexRules, RuleSet.build(gpa, gpa, rows, 0));
try testing.expectEqual(@as(usize, 256), max_regex_per_group);
}
test "an invalid wildcard pattern is an error" {
for ([_][]const u8{ "example.com", "ad*.example.com", "*..com" }) |pattern| {
const rows = [_]model.Rule{rule(pattern, .wildcard, .block)};
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0));
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &rows, 0));
}
}
@@ -305,11 +527,11 @@ test "too many wildcards is an error" {
row.* = rule(pattern, .wildcard, .block);
}
try testing.expectError(error.TooManyWildcards, RuleSet.build(gpa, rows, 0));
try testing.expectError(error.TooManyWildcards, RuleSet.build(gpa, gpa, rows, 0));
}
test "an empty rule list builds the empty set" {
var set = try RuleSet.build(testing.allocator, &[_]model.Rule{}, 0);
var set = try RuleSet.build(testing.allocator, testing.allocator, &[_]model.Rule{}, 0);
defer set.deinit(testing.allocator);
try testing.expect(!set.exact_block.contains("ads.example.com"));
@@ -328,11 +550,63 @@ test "memoryBytes counts every part" {
rule("ads.example.com", .exact, .block),
rule("*.tracker.net", .wildcard, .block),
};
var set = try RuleSet.build(testing.allocator, &rows, 0);
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
defer set.deinit(testing.allocator);
try testing.expect(set.memoryBytes() > set.exact_block.memoryBytes());
try testing.expect(set.memoryBytes() >= "*.tracker.net".len);
// A compiled program is the largest thing a rule set holds, so leaving it
// out would make the snapshot's memory report a fiction.
const with_regex = [_]model.Rule{ rows[0], rows[1], rule("^ad[0-9]+-", .regex, .block) };
var wider = try RuleSet.build(testing.allocator, testing.allocator, &with_regex, 0);
defer wider.deinit(testing.allocator);
try testing.expect(wider.memoryBytes() > set.memoryBytes() + "^ad[0-9]+-".len);
try testing.expect(wider.memoryBytes() >= wider.regex_block[0].program.memoryBytes());
}
test "the build's temporaries stay out of the permanent allocator" {
// The property the two-allocator split exists for. An arena reclaims only
// its most recent allocation, so a temporary taken from `perm` would live
// as long as the arena and be invisible to `memoryBytes`. Two checks, one
// per direction: `testing.allocator` fails the test if anything the set
// retains was taken from `scratch`, and the arena's capacity fails it if
// the build's working storage was taken from `perm`.
const gpa = testing.allocator;
var patterns: std.ArrayList([]u8) = .empty;
defer {
for (patterns.items) |p| gpa.free(p);
patterns.deinit(gpa);
}
var rows: std.ArrayList(model.Rule) = .empty;
defer rows.deinit(gpa);
var i: usize = 0;
while (i < 64) : (i += 1) {
const regex_pattern = try std.fmt.allocPrint(gpa, "^r{d}-[0-9]+\\.ads\\.invalid$", .{i});
try patterns.append(gpa, regex_pattern);
try rows.append(gpa, rule(regex_pattern, .regex, .block));
const wild = try std.fmt.allocPrint(gpa, "*.w{d:0>5}.example.com", .{i});
try patterns.append(gpa, wild);
try rows.append(gpa, rule(wild, .wildcard, .block));
const exact = try std.fmt.allocPrint(gpa, "e{d:0>5}.example.com", .{i});
try patterns.append(gpa, exact);
try rows.append(gpa, rule(exact, .exact, .block));
}
var arena: std.heap.ArenaAllocator = .init(gpa);
defer arena.deinit();
const set = try RuleSet.build(arena.allocator(), gpa, rows.items, 0x5eed);
try testing.expectEqual(@as(usize, 64), set.regex_block.len);
try testing.expect(set.exact_block.contains("e00007.example.com"));
// Whole-arena capacity against what the set says it holds. The slack is the
// allocator's page rounding; the defect this guards against was a factor of
// twelve.
try testing.expect(arena.queryCapacity() < 2 * set.memoryBytes());
}
fn buildUnderFailure(gpa: Allocator) !void {
@@ -341,11 +615,14 @@ fn buildUnderFailure(gpa: Allocator) !void {
rule("good.example.com", .exact, .allow),
rule("*.tracker.net", .wildcard, .block),
rule("*.ok.tracker.net", .wildcard, .allow),
rule("^ad[0-9]+-", .regex, .block),
rule("\\.ok\\.", .regex, .allow),
};
var set = try RuleSet.build(gpa, &rows, 0x5eed);
var set = try RuleSet.build(gpa, gpa, &rows, 0x5eed);
defer set.deinit(gpa);
try testing.expect(set.exact_block.contains("ads.example.com"));
try testing.expectEqualStrings("*.tracker.net", set.wildcard_block[0]);
try testing.expectEqualStrings("^ad[0-9]+-", set.regex_block[0].pattern);
}
test "build leaks nothing under allocation failure" {
src/filter/wildcard.zig
+5 -3
View File
@@ -4,7 +4,8 @@
//! A pattern is a domain name in which one or more labels are exactly `*`.
//! Each `*` label matches one or more labels of the queried name. Partial-label
//! globbing (`ad*.example.com`) is deliberately absent: it is regex by another
//! name, which PLAN §2.2 rules out.
//! name, and PLAN §2.2 keeps one regex dialect rather than two. An operator who
//! needs one writes a `.regex` rule, which `filter/regex.zig` compiles.
const std = @import("std");
@@ -18,8 +19,9 @@ pub const PatternError = error{
/// No label is exactly "*".
NoWildcard,
/// A label contains '*' but is not exactly "*". Partial-label globbing
/// (`ad*.example.com`) is out of scope: it is regex by another name, and
/// PLAN §3.9 defines the wildcard as a label pattern.
/// (`ad*.example.com`) is out of scope: PLAN §3.9 defines the wildcard as a
/// label pattern, and the `.regex` kind covers what partial globbing was
/// wanted for.
PartialWildcardLabel,
EmptyLabel,
LabelTooLong,