mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 25 Activity

filter: store each blocklist name once, flagged exact and subdomains, instead of in two sets per source

Browse Source
This commit is contained in:
mokhtar
2026-10-21 20:49:55 +02:00
parent 2a06239676
commit 4916b01571
4 changed files with 511 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/filter/domain_set.zig
+183 -39
View File
@@ -1,10 +1,15 @@
//! An immutable exact-match set of domain names, built from a compiled
//! blocklist body.
//! An immutable exact-match set of domain names, built from compiled
//! blocklist bodies, with a flag byte per name.
//!
//! The layout is a flat arena of length-prefixed names plus an open-addressed
//! table of `u32` offsets into it. No Bloom filter and no hash-only key set: a
//! false positive in a DNS sinkhole blocks a real domain for a real household
//! and is undebuggable from the outside, so every probe compares full bytes.
//! The layout is a flat arena of length-prefixed, flag-tagged names plus an
//! open-addressed table of `u32` offsets into it. No Bloom filter and no
//! hash-only key set: a false positive in a DNS sinkhole blocks a real domain
//! for a real household and is undebuggable from the outside, so every probe
//! compares full bytes.
//!
//! The flags let one set stand for two bodies that share names. A source's
//! `.list` and `.wild` bodies are byte-identical for every `||x^` line of an ABP
//! list, so storing them as two sets paid for each such name twice.
//!
//! Pure. Takes an allocator and bytes; no `std.Io`, no clock, no entropy
//! source — the hash seed arrives as a parameter.
@@ -12,7 +17,7 @@
const std = @import("std");
pub const DomainSet = struct {
/// Length-prefixed lowercase names, back to back: [len: u8][bytes]…
/// Back to back: [len: u8][flags: u8][len bytes of lowercase name]…
arena: []const u8,
/// Open-addressed table of offsets into `arena`; `empty_slot` marks a hole.
/// Length is always a power of two.
@@ -23,10 +28,15 @@ pub const DomainSet = struct {
pub const empty_slot: u32 = std.math.maxInt(u32);
pub const max_count: u32 = 4_000_000;
pub const max_arena_bytes: usize = 1 << 31;
/// The flag of a name in the first body passed to `buildMerged`, and of
/// every name `build` stores.
pub const in_first: u8 = 1 << 0;
/// The flag of a name in the second body passed to `buildMerged`.
pub const in_second: u8 = 1 << 1;
pub const Error = error{ OutOfMemory, TooManyDomains, SetTooLarge, NotSorted, NotLowercase };
/// An empty set that owns nothing. `contains` on it is always false.
/// An empty set that owns nothing. `flags` on it is always 0.
pub const empty: DomainSet = .{
.arena = &.{},
.index = &.{},
@@ -34,15 +44,22 @@ pub const DomainSet = struct {
.seed = 0,
};
/// Builds from a compiled body: LF-separated, lowercase, sorted ascending,
/// deduplicated, every line 1–255 bytes.
/// A set over one body, every name flagged `in_first`.
pub fn build(gpa: std.mem.Allocator, body: []const u8, seed: u64) Error!DomainSet {
return buildMerged(gpa, body, "", seed);
}
/// A set over the union of two bodies. A name in both is stored once and
/// carries both flags.
///
/// The sortedness is verified, not assumed: a hand-edited or truncated file
/// must fail loudly rather than produce a set that silently misses entries.
/// Verification is one comparison per line. Because a valid line is at least
/// one byte, comparing against an initial empty `prev` also rejects an empty
/// line and a duplicate line — both are `error.NotSorted`, since neither is
/// strictly ascending.
/// Each body must be a compiled body: LF-separated, lowercase, sorted
/// ascending, deduplicated, every line 1–255 bytes. The sortedness is
/// verified, not assumed: a hand-edited or truncated file must fail loudly
/// rather than produce a set that silently misses entries. Verification is
/// one comparison per line. Because a valid line is at least one byte,
/// comparing against an initial empty `prev` also rejects an empty line and
/// a duplicate line — both are `error.NotSorted`, since neither is strictly
/// ascending.
///
/// A line longer than 255 bytes is `error.SetTooLarge`: the arena stores a
/// `u8` length prefix and must not silently truncate, for the same reason
@@ -50,25 +67,25 @@ pub const DomainSet = struct {
///
/// `seed` randomizes the hash. Query names are attacker-supplied, so a fixed
/// seed would make probe-chain flooding computable offline.
pub fn build(gpa: std.mem.Allocator, body: []const u8, seed: u64) Error!DomainSet {
if (body.len == 0) return empty;
if (body.len > max_arena_bytes) return error.SetTooLarge;
pub fn buildMerged(gpa: std.mem.Allocator, first: []const u8, second: []const u8, seed: u64) Error!DomainSet {
// Ahead of the line scan: a body this large cannot fit the `u32`
// offsets whatever its lines hold.
if (first.len > max_arena_bytes or second.len > max_arena_bytes) return error.SetTooLarge;
try validate(first);
try validate(second);
var count: u32 = 0;
var arena_len: usize = 0;
{
var prev: []const u8 = "";
var it: LineIterator = .{ .rest = body };
while (it.next()) |line| {
for (line) |c| if (c >= 'A' and c <= 'Z') return error.NotLowercase;
if (line.len > std.math.maxInt(u8)) return error.SetTooLarge;
if (std.mem.order(u8, prev, line) != .lt) return error.NotSorted;
var merge: Merge = .init(first, second);
while (merge.next()) |entry| {
if (count == max_count) return error.TooManyDomains;
prev = line;
count += 1;
arena_len += 1 + line.len;
arena_len += 2 + entry.name.len;
}
}
if (count == 0) return empty;
if (arena_len > max_arena_bytes) return error.SetTooLarge;
const capacity = capacityFor(count);
@@ -79,14 +96,15 @@ pub const DomainSet = struct {
const mask = capacity - 1;
var write_at: usize = 0;
var it: LineIterator = .{ .rest = body };
while (it.next()) |line| {
var merge: Merge = .init(first, second);
while (merge.next()) |entry| {
const offset: u32 = @intCast(write_at);
arena[write_at] = @intCast(line.len);
@memcpy(arena[write_at + 1 ..][0..line.len], line);
write_at += 1 + line.len;
arena[write_at] = @intCast(entry.name.len);
arena[write_at + 1] = entry.flags;
@memcpy(arena[write_at + 2 ..][0..entry.name.len], entry.name);
write_at += 2 + entry.name.len;
var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(seed, line))) & mask;
var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(seed, entry.name))) & mask;
while (index[slot] != empty_slot) slot = (slot + 1) & mask;
index[slot] = offset;
}
@@ -100,28 +118,97 @@ pub const DomainSet = struct {
self.* = empty;
}
/// `domain` must be normalized (lowercase, no trailing dot). Allocation-free.
pub fn contains(self: *const DomainSet, domain: []const u8) bool {
if (self.index.len == 0) return false;
/// The flags `domain` carries, 0 when it is not in the set. `domain` must
/// be normalized (lowercase, no trailing dot). Allocation-free.
pub fn flags(self: *const DomainSet, domain: []const u8) u8 {
if (self.index.len == 0) return 0;
const mask = self.index.len - 1;
var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(self.seed, domain))) & mask;
while (true) {
const offset = self.index[slot];
if (offset == empty_slot) return false;
if (offset == empty_slot) return 0;
const len = self.arena[offset];
if (len == domain.len and std.mem.eql(u8, self.arena[offset + 1 ..][0..len], domain)) {
return true;
if (len == domain.len and std.mem.eql(u8, self.arena[offset + 2 ..][0..len], domain)) {
return self.arena[offset + 1];
}
slot = (slot + 1) & mask;
}
}
pub fn contains(self: *const DomainSet, domain: []const u8) bool {
return self.flags(domain) != 0;
}
/// Bytes held, for the memory report in `Snapshot.memoryBytes`.
pub fn memoryBytes(self: *const DomainSet) usize {
return self.arena.len + self.index.len * @sizeOf(u32);
}
};
fn validate(body: []const u8) DomainSet.Error!void {
var prev: []const u8 = "";
var lines: u32 = 0;
var it: LineIterator = .{ .rest = body };
while (it.next()) |line| {
for (line) |c| if (c >= 'A' and c <= 'Z') return error.NotLowercase;
if (line.len > std.math.maxInt(u8)) return error.SetTooLarge;
if (std.mem.order(u8, prev, line) != .lt) return error.NotSorted;
if (lines == DomainSet.max_count) return error.TooManyDomains;
prev = line;
lines += 1;
}
}
/// Walks the union of two sorted, validated bodies in ascending order,
/// yielding each distinct name once with the flags of the bodies holding it.
const Merge = struct {
first: LineIterator,
second: LineIterator,
first_head: ?[]const u8,
second_head: ?[]const u8,
const Entry = struct { name: []const u8, flags: u8 };
fn init(first: []const u8, second: []const u8) Merge {
var merge: Merge = .{
.first = .{ .rest = first },
.second = .{ .rest = second },
.first_head = null,
.second_head = null,
};
merge.first_head = merge.first.next();
merge.second_head = merge.second.next();
return merge;
}
fn next(self: *Merge) ?Entry {
const a = self.first_head orelse {
const b = self.second_head orelse return null;
self.second_head = self.second.next();
return .{ .name = b, .flags = DomainSet.in_second };
};
const b = self.second_head orelse {
self.first_head = self.first.next();
return .{ .name = a, .flags = DomainSet.in_first };
};
switch (std.mem.order(u8, a, b)) {
.lt => {
self.first_head = self.first.next();
return .{ .name = a, .flags = DomainSet.in_first };
},
.gt => {
self.second_head = self.second.next();
return .{ .name = b, .flags = DomainSet.in_second };
},
.eq => {
self.first_head = self.first.next();
self.second_head = self.second.next();
return .{ .name = a, .flags = DomainSet.in_first | DomainSet.in_second };
},
}
}
};
/// Smallest power of two at least `count * 4 / 3`, minimum 16. The load factor
/// stays at or under 0.75, which keeps at least one hole and therefore
/// terminates the probe loop in `contains`.
@@ -283,3 +370,60 @@ test "capacityFor keeps the load factor at or under three quarters" {
try testing.expectEqual(@as(usize, 32), capacityFor(13));
try testing.expectEqual(@as(usize, 2 << 20), capacityFor(1_000_000));
}
const list_part =
"a.example\n" ++
"both.example\n" ++
"m.example\n";
const wild_part =
"both.example\n" ++
"w.example\n" ++
"z.example\n";
test "two bodies merge into one entry per name, carrying each body's flag" {
var set = try DomainSet.buildMerged(testing.allocator, list_part, wild_part, 0x77);
defer set.deinit(testing.allocator);
try testing.expectEqual(@as(u32, 5), set.count);
try testing.expectEqual(@as(u8, 1), set.flags("a.example"));
try testing.expectEqual(@as(u8, 1), set.flags("m.example"));
try testing.expectEqual(@as(u8, 3), set.flags("both.example"));
try testing.expectEqual(@as(u8, 2), set.flags("w.example"));
try testing.expectEqual(@as(u8, 2), set.flags("z.example"));
try testing.expectEqual(@as(u8, 0), set.flags("example"));
try testing.expectEqual(@as(u8, 0), set.flags("x.both.example"));
}
test "two identical bodies cost what one body costs" {
var one = try DomainSet.build(testing.allocator, small_body, 0x77);
defer one.deinit(testing.allocator);
var two = try DomainSet.buildMerged(testing.allocator, small_body, small_body, 0x77);
defer two.deinit(testing.allocator);
try testing.expectEqual(one.count, two.count);
try testing.expectEqual(one.memoryBytes(), two.memoryBytes());
try testing.expectEqual(@as(u8, 3), two.flags("example.com"));
}
test "an empty first body leaves the second body's flags alone" {
var set = try DomainSet.buildMerged(testing.allocator, "", wild_part, 0);
defer set.deinit(testing.allocator);
try testing.expectEqual(@as(u32, 3), set.count);
try testing.expectEqual(@as(u8, 2), set.flags("both.example"));
}
test "both bodies are validated, not just the first" {
try testing.expectError(error.NotSorted, DomainSet.buildMerged(testing.allocator, list_part, "b.example\na.example\n", 0));
try testing.expectError(error.NotLowercase, DomainSet.buildMerged(testing.allocator, list_part, "A.example\n", 0));
}
fn buildMergedUnderFailure(gpa: std.mem.Allocator) !void {
var set = try DomainSet.buildMerged(gpa, list_part, wild_part, 0x1234);
defer set.deinit(gpa);
try testing.expectEqual(@as(u8, 3), set.flags("both.example"));
}
test "buildMerged under allocation failure" {
try testing.checkAllAllocationFailures(testing.allocator, buildMergedUnderFailure, .{});
}
src/filter/list_levels_test.zig
+307
View File
@@ -0,0 +1,307 @@
//! Equivalence of the list levels (7–9 of `Snapshot.evaluate`) against a
//! reference model that keeps every compiled body as its own list of lines.
//!
//! The snapshot stores a source's `.list` and `.wild` names in one flagged set;
//! the reference keeps them apart and answers by scanning lines, the way the
//! bodies read on disk. Real list text goes through the real compiler, so the
//! corpus exercises what the compiler emits, not what a test assumes it emits.
const std = @import("std");
const Allocator = std.mem.Allocator;
const model = @import("../config/model.zig");
const compiler = @import("compiler.zig");
const parsers = @import("parsers.zig");
const matcher = @import("matcher.zig");
const testing = std.testing;
const Raw = struct { url: []const u8, format: parsers.Format, text: []const u8 };
/// Three overlapping sources covering each shape a body can take: `||x^`
/// (both bodies), `*.x` alone, `x` alone, `x` together with `*.x`, `@@`
/// exceptions, an exact entry for a subdomain of another source's wildcard,
/// and nested wildcards `*.x` with `*.y.x`.
const corpus = [_]Raw{
.{ .url = "https://lists.test/abp", .format = .abp, .text =
\\! abp
\\||ads.example^
\\||t.example^
\\||deep.y.x.example^
\\@@||good.ads.example^
\\@@||t.example^
\\
},
.{ .url = "https://lists.test/domains", .format = .domains, .text =
\\# domains
\\x.example
\\v.example
\\*.v.example
\\*.w.example
\\*.y.w.example
\\sub.ads.example
\\ads.example
\\
},
.{ .url = "https://lists.test/hosts", .format = .hosts, .text =
\\0.0.0.0 ads.example
\\0.0.0.0 h.example
\\0.0.0.0 a.w.example
\\0.0.0.0 good.ads.example
\\
},
};
const Compiled = struct {
list: std.Io.Writer.Allocating,
wild: std.Io.Writer.Allocating,
allow: std.Io.Writer.Allocating,
fn init(gpa: Allocator, raw: Raw) !Compiled {
var self: Compiled = .{ .list = .init(gpa), .wild = .init(gpa), .allow = .init(gpa) };
errdefer self.deinit();
var r: std.Io.Reader = .fixed(raw.text);
_ = try compiler.compile(gpa, &r, raw.format, &self.list.writer, &self.wild.writer, &self.allow.writer);
return self;
}
fn deinit(self: *Compiled) void {
self.list.deinit();
self.wild.deinit();
self.allow.deinit();
}
fn bodies(self: *Compiled) matcher.Snapshot.Compiled {
return .{ .list_body = self.list.written(), .wild_body = self.wild.written(), .allow_body = self.allow.written() };
}
};
fn bodyHas(body: []const u8, wanted: []const u8) bool {
var lines = std.mem.tokenizeScalar(u8, body, '\n');
while (lines.next()) |line| {
if (std.mem.eql(u8, line, wanted)) return true;
}
return false;
}
/// Levels 7–9 as documented on `Snapshot.evaluate`, over separate bodies.
fn reference(bodies: []const matcher.Snapshot.Compiled, sources: []const u32, domain: []const u8) matcher.Decision {
var exceptions: matcher.Candidates = .init(domain);
while (exceptions.next()) |candidate| {
for (sources) |index| {
if (bodyHas(bodies[index].allow_body, candidate)) {
return .{ .blocked = false, .reason = .blocklist_exception, .matched = candidate, .source = index };
}
}
}
for (sources) |index| {
if (bodyHas(bodies[index].list_body, domain)) {
return .{ .blocked = true, .reason = .blocklist_domain, .matched = domain, .source = index };
}
}
var parents: matcher.Candidates = .init(domain);
_ = parents.next();
while (parents.next()) |parent| {
for (sources) |index| {
if (bodyHas(bodies[index].wild_body, parent)) {
return .{ .blocked = true, .reason = .blocklist_wildcard, .matched = parent, .source = index };
}
}
}
return .{ .blocked = false, .reason = .none, .matched = "" };
}
fn expectSameDecision(expected: matcher.Decision, actual: matcher.Decision, probe: []const u8) !void {
errdefer std.debug.print("probe {s}: expected {t} {s} {?d}, got {t} {s} {?d}\n", .{
probe, expected.reason, expected.matched, expected.source,
actual.reason, actual.matched, actual.source,
});
try testing.expectEqual(expected.blocked, actual.blocked);
try testing.expectEqual(expected.reason, actual.reason);
try testing.expectEqualStrings(expected.matched, actual.matched);
try testing.expectEqual(expected.source, actual.source);
}
/// Every name in every body, each with two subdomains and its own parents,
/// plus names no list mentions.
fn probes(gpa: Allocator, bodies: []const matcher.Snapshot.Compiled) !std.ArrayList([]u8) {
var out: std.ArrayList([]u8) = .empty;
errdefer {
for (out.items) |probe| gpa.free(probe);
out.deinit(gpa);
}
for (bodies) |b| {
for ([_][]const u8{ b.list_body, b.wild_body, b.allow_body }) |body| {
var lines = std.mem.tokenizeScalar(u8, body, '\n');
while (lines.next()) |line| {
var parents: matcher.Candidates = .init(line);
while (parents.next()) |parent| try out.append(gpa, try gpa.dupe(u8, parent));
try out.append(gpa, try std.fmt.allocPrint(gpa, "p.{s}", .{line}));
try out.append(gpa, try std.fmt.allocPrint(gpa, "q.r.{s}", .{line}));
}
}
}
for ([_][]const u8{ "example", "unlisted.example", "x.unlisted.example", "" }) |name| {
try out.append(gpa, try gpa.dupe(u8, name));
}
return out;
}
test "list levels: the flagged name set decides exactly what separate bodies decide" {
const gpa = testing.allocator;
var compiled: [corpus.len]Compiled = undefined;
var built: usize = 0;
defer for (compiled[0..built]) |*c| c.deinit();
for (corpus, &compiled) |raw, *slot| {
slot.* = try .init(gpa, raw);
built += 1;
}
var bodies: [corpus.len]matcher.Snapshot.Compiled = undefined;
var compiled_opt: [corpus.len]?matcher.Snapshot.Compiled = undefined;
var sources: [corpus.len]model.BlocklistSource = undefined;
var source_ids: [corpus.len]i64 = undefined;
for (&compiled, &bodies, &compiled_opt, &sources, &source_ids, corpus, 0..) |*c, *b, *o, *s, *id, raw, i| {
b.* = c.bodies();
o.* = b.*;
s.* = .{ .url = raw.url, .name = raw.url };
id.* = @intCast(i + 1);
}
// `default` sees all three sources; `narrow` only the last two, so the
// ascending-index walk starts at a different source.
const links = [_]model.GroupSource{
.{ .group = "default", .source_url = corpus[0].url },
.{ .group = "default", .source_url = corpus[1].url },
.{ .group = "default", .source_url = corpus[2].url },
.{ .group = "narrow", .source_url = corpus[2].url },
.{ .group = "narrow", .source_url = corpus[1].url },
};
var snapshot = try matcher.Snapshot.build(gpa, .{
.groups = &.{ .{ .name = "default" }, .{ .name = "narrow" } },
.group_ids = &.{ 1, 2 },
.group_sources = &links,
.sources = &sources,
.source_ids = &source_ids,
.rules = &.{},
.clients = &.{},
.prefixes = &.{},
.compiled = &compiled_opt,
.seed = 0x5eed,
.generation = 1,
});
defer snapshot.deinit();
var probe_list = try probes(gpa, &bodies);
defer {
for (probe_list.items) |probe| gpa.free(probe);
probe_list.deinit(gpa);
}
var decided = [_]u32{0} ** @typeInfo(matcher.Reason).@"enum".fields.len;
for ([_][]const u32{ &.{ 0, 1, 2 }, &.{ 1, 2 } }, 0..) |group_sources, group| {
try testing.expectEqualSlices(u32, group_sources, snapshot.groups[group].sources);
for (probe_list.items) |probe| {
const actual = snapshot.evaluate(@intCast(group), probe);
try expectSameDecision(reference(&bodies, group_sources, probe), actual, probe);
decided[@intFromEnum(actual.reason)] += 1;
}
}
// The corpus reaches every list level, so the comparison is not vacuous.
try testing.expect(decided[@intFromEnum(matcher.Reason.blocklist_exception)] > 0);
try testing.expect(decided[@intFromEnum(matcher.Reason.blocklist_domain)] > 0);
try testing.expect(decided[@intFromEnum(matcher.Reason.blocklist_wildcard)] > 0);
try testing.expect(decided[@intFromEnum(matcher.Reason.none)] > 0);
// The reference and the snapshot read the same compiled bodies, so a
// compiler that dropped a shape would drop it from both. Each shape the
// corpus exists for is therefore also pinned to the decision it must give.
for (named_cases) |case| {
const blocked = case.reason == .blocklist_domain or case.reason == .blocklist_wildcard;
try expectSameDecision(
.{ .blocked = blocked, .reason = case.reason, .matched = case.matched, .source = case.source },
snapshot.evaluate(case.group, case.probe),
case.probe,
);
}
}
const NamedCase = struct {
group: u32,
probe: []const u8,
reason: matcher.Reason,
matched: []const u8 = "",
source: ?u32 = null,
};
const default_group = 0;
const narrow_group = 1;
const named_cases = [_]NamedCase{
// `||x^`: the apex is an exact match, a subdomain a wildcard match.
.{ .group = default_group, .probe = "ads.example", .reason = .blocklist_domain, .matched = "ads.example", .source = 0 },
.{ .group = default_group, .probe = "p.ads.example", .reason = .blocklist_wildcard, .matched = "ads.example", .source = 0 },
// An exact entry for the queried name beats another source's wildcard.
.{ .group = default_group, .probe = "sub.ads.example", .reason = .blocklist_domain, .matched = "sub.ads.example", .source = 1 },
// `@@` beats every block, its own source's and another's, for the name
// and its subdomains.
.{ .group = default_group, .probe = "good.ads.example", .reason = .blocklist_exception, .matched = "good.ads.example", .source = 0 },
.{ .group = default_group, .probe = "x.good.ads.example", .reason = .blocklist_exception, .matched = "good.ads.example", .source = 0 },
.{ .group = default_group, .probe = "t.example", .reason = .blocklist_exception, .matched = "t.example", .source = 0 },
.{ .group = default_group, .probe = "p.t.example", .reason = .blocklist_exception, .matched = "t.example", .source = 0 },
// `x` alone blocks the apex only.
.{ .group = default_group, .probe = "x.example", .reason = .blocklist_domain, .matched = "x.example", .source = 1 },
.{ .group = default_group, .probe = "p.x.example", .reason = .none },
// `*.x` alone blocks subdomains only.
.{ .group = default_group, .probe = "w.example", .reason = .none },
.{ .group = default_group, .probe = "p.w.example", .reason = .blocklist_wildcard, .matched = "w.example", .source = 1 },
// An exact entry for a subdomain of a wildcard reports the exact entry.
.{ .group = default_group, .probe = "a.w.example", .reason = .blocklist_domain, .matched = "a.w.example", .source = 2 },
// Nested wildcards: the nearer parent wins.
.{ .group = default_group, .probe = "p.y.w.example", .reason = .blocklist_wildcard, .matched = "y.w.example", .source = 1 },
.{ .group = default_group, .probe = "y.w.example", .reason = .blocklist_wildcard, .matched = "w.example", .source = 1 },
.{ .group = default_group, .probe = "q.deep.y.x.example", .reason = .blocklist_wildcard, .matched = "deep.y.x.example", .source = 0 },
// `x` with `*.x` in a domains list.
.{ .group = default_group, .probe = "v.example", .reason = .blocklist_domain, .matched = "v.example", .source = 1 },
.{ .group = default_group, .probe = "p.v.example", .reason = .blocklist_wildcard, .matched = "v.example", .source = 1 },
// Overlapping sources: the lowest linked index reports, and a group that
// does not link the ABP source has neither its wildcard nor its `@@`.
.{ .group = narrow_group, .probe = "ads.example", .reason = .blocklist_domain, .matched = "ads.example", .source = 1 },
.{ .group = narrow_group, .probe = "p.ads.example", .reason = .none },
.{ .group = narrow_group, .probe = "good.ads.example", .reason = .blocklist_domain, .matched = "good.ads.example", .source = 2 },
};
test "list levels: a name in both bodies of one source is stored once" {
const gpa = testing.allocator;
var c = try Compiled.init(gpa, corpus[0]);
defer c.deinit();
const b = c.bodies();
// The ABP source compiles each `||x^` into both bodies.
try testing.expectEqualStrings(b.list_body, b.wild_body);
const sources = [_]model.BlocklistSource{.{ .url = corpus[0].url, .name = "abp" }};
const compiled = [_]?matcher.Snapshot.Compiled{b};
var snapshot = try matcher.Snapshot.build(gpa, .{
.groups = &.{.{ .name = "default" }},
.group_ids = &.{1},
.group_sources = &.{.{ .group = "default", .source_url = corpus[0].url }},
.sources = &sources,
.source_ids = &.{1},
.rules = &.{},
.clients = &.{},
.prefixes = &.{},
.compiled = &compiled,
.seed = 0x5eed,
.generation = 1,
});
defer snapshot.deinit();
const names = snapshot.sources[0].names;
try testing.expectEqual(@as(u32, @intCast(std.mem.count(u8, b.list_body, "\n"))), names.count);
try testing.expectEqual(
matcher.SourceSets.exact | matcher.SourceSets.subdomains,
names.flags("ads.example"),
);
}
src/filter/matcher.zig
+20 -12
View File
@@ -100,12 +100,18 @@ pub const SourceSets = struct {
id: i64,
/// Snapshot-arena-owned; the source's display name for the UI.
name: []const u8,
domains: domain_set.DomainSet,
wildcards: domain_set.DomainSet,
/// Every name of the source's `.list` and `.wild` bodies, stored once and
/// flagged with the body it came from: `exact` blocks the name itself,
/// `subdomains` blocks its proper subdomains. An ABP `||x^` line compiles
/// into both bodies, so one entry carries both flags.
names: domain_set.DomainSet,
/// The names this source's `@@` exceptions lift. One entry covers the name
/// and every subdomain of it, because `evaluate` walks the full name and
/// each parent against this set.
exceptions: domain_set.DomainSet,
pub const exact = domain_set.DomainSet.in_first;
pub const subdomains = domain_set.DomainSet.in_second;
};
pub const Group = struct {
@@ -209,8 +215,7 @@ pub const Snapshot = struct {
sources[at] = .{
.id = id,
.name = try arena.dupe(u8, row.name),
.domains = try domain_set.DomainSet.build(arena, bodies.list_body, input.seed),
.wildcards = try domain_set.DomainSet.build(arena, bodies.wild_body, input.seed),
.names = try domain_set.DomainSet.buildMerged(arena, bodies.list_body, bodies.wild_body, input.seed),
.exceptions = try domain_set.DomainSet.build(arena, bodies.allow_body, input.seed),
};
}
@@ -369,7 +374,7 @@ pub const Snapshot = struct {
}
for (g.sources) |index| {
if (self.sources[index].domains.contains(domain)) {
if (self.sources[index].names.flags(domain) & SourceSets.exact != 0) {
return .{
.blocked = true,
.reason = .blocklist_domain,
@@ -384,7 +389,7 @@ pub const Snapshot = struct {
_ = parents.next();
while (parents.next()) |parent| {
for (g.sources) |index| {
if (self.sources[index].wildcards.contains(parent)) {
if (self.sources[index].names.flags(parent) & SourceSets.subdomains != 0) {
return .{
.blocked = true,
.reason = .blocklist_wildcard,
@@ -434,8 +439,7 @@ pub const Snapshot = struct {
var total: usize = 0;
for (self.sources) |*source| {
total += @sizeOf(SourceSets) + source.name.len +
source.domains.memoryBytes() + source.wildcards.memoryBytes() +
source.exceptions.memoryBytes();
source.names.memoryBytes() + source.exceptions.memoryBytes();
}
for (self.groups) |*group| {
total += @sizeOf(Group) + group.name.len +
@@ -1347,14 +1351,18 @@ test "memoryBytes stays within the compiled-data bound" {
try body.appendSlice(gpa, try std.fmt.bufPrint(&line, "d{d:0>5}.example.com\n", .{i}));
}
const lists: Lists = .init(body.items, "");
// Identical bodies, as an ABP list compiles them: the names are stored
// once, so one arena and one index bound the lot.
const lists: Lists = .init(body.items, body.items);
var snapshot = try build(gpa, lists.fixture());
defer snapshot.deinit();
// arena + index, over both bodies, plus a kilobyte of struct overhead.
const index_bytes = 2 * 16 * 1024 * @sizeOf(u32);
try testing.expect(snapshot.memoryBytes() < 2 * body.items.len + index_bytes + 1024);
// Per name: the length prefix in place of the newline, plus the flag byte.
const arena_bytes = body.items.len + count;
const index_bytes = 16 * 1024 * @sizeOf(u32);
try testing.expect(snapshot.memoryBytes() < arena_bytes + index_bytes + 1024);
try testing.expect(snapshot.evaluate(0, "d00042.example.com").blocked);
try testing.expect(snapshot.evaluate(0, "x.d00042.example.com").blocked);
}
fn buildUnderFailure(gpa: Allocator) !void {
src/tests.zig
+1
View File
@@ -70,6 +70,7 @@ comptime {
_ = @import("filter/domain_set.zig");
_ = @import("filter/rules.zig");
_ = @import("filter/matcher.zig");
_ = @import("filter/list_levels_test.zig");
_ = @import("filter/compiler.zig");
_ = @import("filter/safesearch.zig");
_ = @import("filter/response.zig");