diff --git a/src/filter/domain_set.zig b/src/filter/domain_set.zig index d6f1018..95ca1a5 100644 --- a/src/filter/domain_set.zig +++ b/src/filter/domain_set.zig @@ -1,10 +1,15 @@ -//! An immutable exact-match set of domain names, built from a compiled -//! blocklist body. +//! An immutable exact-match set of domain names, built from compiled +//! blocklist bodies, with a flag byte per name. //! -//! The layout is a flat arena of length-prefixed names plus an open-addressed -//! table of `u32` offsets into it. No Bloom filter and no hash-only key set: a -//! false positive in a DNS sinkhole blocks a real domain for a real household -//! and is undebuggable from the outside, so every probe compares full bytes. +//! The layout is a flat arena of length-prefixed, flag-tagged names plus an +//! open-addressed table of `u32` offsets into it. No Bloom filter and no +//! hash-only key set: a false positive in a DNS sinkhole blocks a real domain +//! for a real household and is undebuggable from the outside, so every probe +//! compares full bytes. +//! +//! The flags let one set stand for two bodies that share names. A source's +//! `.list` and `.wild` bodies are byte-identical for every `||x^` line of an ABP +//! list, so storing them as two sets paid for each such name twice. //! //! Pure. Takes an allocator and bytes; no `std.Io`, no clock, no entropy //! source — the hash seed arrives as a parameter. @@ -12,7 +17,7 @@ const std = @import("std"); pub const DomainSet = struct { - /// Length-prefixed lowercase names, back to back: [len: u8][bytes]… + /// Back to back: [len: u8][flags: u8][len bytes of lowercase name]… arena: []const u8, /// Open-addressed table of offsets into `arena`; `empty_slot` marks a hole. /// Length is always a power of two. @@ -23,10 +28,15 @@ pub const DomainSet = struct { pub const empty_slot: u32 = std.math.maxInt(u32); pub const max_count: u32 = 4_000_000; pub const max_arena_bytes: usize = 1 << 31; + /// The flag of a name in the first body passed to `buildMerged`, and of + /// every name `build` stores. + pub const in_first: u8 = 1 << 0; + /// The flag of a name in the second body passed to `buildMerged`. + pub const in_second: u8 = 1 << 1; pub const Error = error{ OutOfMemory, TooManyDomains, SetTooLarge, NotSorted, NotLowercase }; - /// An empty set that owns nothing. `contains` on it is always false. + /// An empty set that owns nothing. `flags` on it is always 0. pub const empty: DomainSet = .{ .arena = &.{}, .index = &.{}, @@ -34,15 +44,22 @@ pub const DomainSet = struct { .seed = 0, }; - /// Builds from a compiled body: LF-separated, lowercase, sorted ascending, - /// deduplicated, every line 1–255 bytes. + /// A set over one body, every name flagged `in_first`. + pub fn build(gpa: std.mem.Allocator, body: []const u8, seed: u64) Error!DomainSet { + return buildMerged(gpa, body, "", seed); + } + + /// A set over the union of two bodies. A name in both is stored once and + /// carries both flags. /// - /// The sortedness is verified, not assumed: a hand-edited or truncated file - /// must fail loudly rather than produce a set that silently misses entries. - /// Verification is one comparison per line. Because a valid line is at least - /// one byte, comparing against an initial empty `prev` also rejects an empty - /// line and a duplicate line — both are `error.NotSorted`, since neither is - /// strictly ascending. + /// Each body must be a compiled body: LF-separated, lowercase, sorted + /// ascending, deduplicated, every line 1–255 bytes. The sortedness is + /// verified, not assumed: a hand-edited or truncated file must fail loudly + /// rather than produce a set that silently misses entries. Verification is + /// one comparison per line. Because a valid line is at least one byte, + /// comparing against an initial empty `prev` also rejects an empty line and + /// a duplicate line — both are `error.NotSorted`, since neither is strictly + /// ascending. /// /// A line longer than 255 bytes is `error.SetTooLarge`: the arena stores a /// `u8` length prefix and must not silently truncate, for the same reason @@ -50,25 +67,25 @@ pub const DomainSet = struct { /// /// `seed` randomizes the hash. Query names are attacker-supplied, so a fixed /// seed would make probe-chain flooding computable offline. - pub fn build(gpa: std.mem.Allocator, body: []const u8, seed: u64) Error!DomainSet { - if (body.len == 0) return empty; - if (body.len > max_arena_bytes) return error.SetTooLarge; + pub fn buildMerged(gpa: std.mem.Allocator, first: []const u8, second: []const u8, seed: u64) Error!DomainSet { + // Ahead of the line scan: a body this large cannot fit the `u32` + // offsets whatever its lines hold. + if (first.len > max_arena_bytes or second.len > max_arena_bytes) return error.SetTooLarge; + try validate(first); + try validate(second); var count: u32 = 0; var arena_len: usize = 0; { - var prev: []const u8 = ""; - var it: LineIterator = .{ .rest = body }; - while (it.next()) |line| { - for (line) |c| if (c >= 'A' and c <= 'Z') return error.NotLowercase; - if (line.len > std.math.maxInt(u8)) return error.SetTooLarge; - if (std.mem.order(u8, prev, line) != .lt) return error.NotSorted; + var merge: Merge = .init(first, second); + while (merge.next()) |entry| { if (count == max_count) return error.TooManyDomains; - prev = line; count += 1; - arena_len += 1 + line.len; + arena_len += 2 + entry.name.len; } } + if (count == 0) return empty; + if (arena_len > max_arena_bytes) return error.SetTooLarge; const capacity = capacityFor(count); @@ -79,14 +96,15 @@ pub const DomainSet = struct { const mask = capacity - 1; var write_at: usize = 0; - var it: LineIterator = .{ .rest = body }; - while (it.next()) |line| { + var merge: Merge = .init(first, second); + while (merge.next()) |entry| { const offset: u32 = @intCast(write_at); - arena[write_at] = @intCast(line.len); - @memcpy(arena[write_at + 1 ..][0..line.len], line); - write_at += 1 + line.len; + arena[write_at] = @intCast(entry.name.len); + arena[write_at + 1] = entry.flags; + @memcpy(arena[write_at + 2 ..][0..entry.name.len], entry.name); + write_at += 2 + entry.name.len; - var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(seed, line))) & mask; + var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(seed, entry.name))) & mask; while (index[slot] != empty_slot) slot = (slot + 1) & mask; index[slot] = offset; } @@ -100,28 +118,97 @@ pub const DomainSet = struct { self.* = empty; } - /// `domain` must be normalized (lowercase, no trailing dot). Allocation-free. - pub fn contains(self: *const DomainSet, domain: []const u8) bool { - if (self.index.len == 0) return false; + /// The flags `domain` carries, 0 when it is not in the set. `domain` must + /// be normalized (lowercase, no trailing dot). Allocation-free. + pub fn flags(self: *const DomainSet, domain: []const u8) u8 { + if (self.index.len == 0) return 0; const mask = self.index.len - 1; var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(self.seed, domain))) & mask; while (true) { const offset = self.index[slot]; - if (offset == empty_slot) return false; + if (offset == empty_slot) return 0; const len = self.arena[offset]; - if (len == domain.len and std.mem.eql(u8, self.arena[offset + 1 ..][0..len], domain)) { - return true; + if (len == domain.len and std.mem.eql(u8, self.arena[offset + 2 ..][0..len], domain)) { + return self.arena[offset + 1]; } slot = (slot + 1) & mask; } } + pub fn contains(self: *const DomainSet, domain: []const u8) bool { + return self.flags(domain) != 0; + } + /// Bytes held, for the memory report in `Snapshot.memoryBytes`. pub fn memoryBytes(self: *const DomainSet) usize { return self.arena.len + self.index.len * @sizeOf(u32); } }; +fn validate(body: []const u8) DomainSet.Error!void { + var prev: []const u8 = ""; + var lines: u32 = 0; + var it: LineIterator = .{ .rest = body }; + while (it.next()) |line| { + for (line) |c| if (c >= 'A' and c <= 'Z') return error.NotLowercase; + if (line.len > std.math.maxInt(u8)) return error.SetTooLarge; + if (std.mem.order(u8, prev, line) != .lt) return error.NotSorted; + if (lines == DomainSet.max_count) return error.TooManyDomains; + prev = line; + lines += 1; + } +} + +/// Walks the union of two sorted, validated bodies in ascending order, +/// yielding each distinct name once with the flags of the bodies holding it. +const Merge = struct { + first: LineIterator, + second: LineIterator, + first_head: ?[]const u8, + second_head: ?[]const u8, + + const Entry = struct { name: []const u8, flags: u8 }; + + fn init(first: []const u8, second: []const u8) Merge { + var merge: Merge = .{ + .first = .{ .rest = first }, + .second = .{ .rest = second }, + .first_head = null, + .second_head = null, + }; + merge.first_head = merge.first.next(); + merge.second_head = merge.second.next(); + return merge; + } + + fn next(self: *Merge) ?Entry { + const a = self.first_head orelse { + const b = self.second_head orelse return null; + self.second_head = self.second.next(); + return .{ .name = b, .flags = DomainSet.in_second }; + }; + const b = self.second_head orelse { + self.first_head = self.first.next(); + return .{ .name = a, .flags = DomainSet.in_first }; + }; + switch (std.mem.order(u8, a, b)) { + .lt => { + self.first_head = self.first.next(); + return .{ .name = a, .flags = DomainSet.in_first }; + }, + .gt => { + self.second_head = self.second.next(); + return .{ .name = b, .flags = DomainSet.in_second }; + }, + .eq => { + self.first_head = self.first.next(); + self.second_head = self.second.next(); + return .{ .name = a, .flags = DomainSet.in_first | DomainSet.in_second }; + }, + } + } +}; + /// Smallest power of two at least `count * 4 / 3`, minimum 16. The load factor /// stays at or under 0.75, which keeps at least one hole and therefore /// terminates the probe loop in `contains`. @@ -283,3 +370,60 @@ test "capacityFor keeps the load factor at or under three quarters" { try testing.expectEqual(@as(usize, 32), capacityFor(13)); try testing.expectEqual(@as(usize, 2 << 20), capacityFor(1_000_000)); } + +const list_part = + "a.example\n" ++ + "both.example\n" ++ + "m.example\n"; +const wild_part = + "both.example\n" ++ + "w.example\n" ++ + "z.example\n"; + +test "two bodies merge into one entry per name, carrying each body's flag" { + var set = try DomainSet.buildMerged(testing.allocator, list_part, wild_part, 0x77); + defer set.deinit(testing.allocator); + + try testing.expectEqual(@as(u32, 5), set.count); + try testing.expectEqual(@as(u8, 1), set.flags("a.example")); + try testing.expectEqual(@as(u8, 1), set.flags("m.example")); + try testing.expectEqual(@as(u8, 3), set.flags("both.example")); + try testing.expectEqual(@as(u8, 2), set.flags("w.example")); + try testing.expectEqual(@as(u8, 2), set.flags("z.example")); + try testing.expectEqual(@as(u8, 0), set.flags("example")); + try testing.expectEqual(@as(u8, 0), set.flags("x.both.example")); +} + +test "two identical bodies cost what one body costs" { + var one = try DomainSet.build(testing.allocator, small_body, 0x77); + defer one.deinit(testing.allocator); + var two = try DomainSet.buildMerged(testing.allocator, small_body, small_body, 0x77); + defer two.deinit(testing.allocator); + + try testing.expectEqual(one.count, two.count); + try testing.expectEqual(one.memoryBytes(), two.memoryBytes()); + try testing.expectEqual(@as(u8, 3), two.flags("example.com")); +} + +test "an empty first body leaves the second body's flags alone" { + var set = try DomainSet.buildMerged(testing.allocator, "", wild_part, 0); + defer set.deinit(testing.allocator); + + try testing.expectEqual(@as(u32, 3), set.count); + try testing.expectEqual(@as(u8, 2), set.flags("both.example")); +} + +test "both bodies are validated, not just the first" { + try testing.expectError(error.NotSorted, DomainSet.buildMerged(testing.allocator, list_part, "b.example\na.example\n", 0)); + try testing.expectError(error.NotLowercase, DomainSet.buildMerged(testing.allocator, list_part, "A.example\n", 0)); +} + +fn buildMergedUnderFailure(gpa: std.mem.Allocator) !void { + var set = try DomainSet.buildMerged(gpa, list_part, wild_part, 0x1234); + defer set.deinit(gpa); + try testing.expectEqual(@as(u8, 3), set.flags("both.example")); +} + +test "buildMerged under allocation failure" { + try testing.checkAllAllocationFailures(testing.allocator, buildMergedUnderFailure, .{}); +} diff --git a/src/filter/list_levels_test.zig b/src/filter/list_levels_test.zig new file mode 100644 index 0000000..edf1ea5 --- /dev/null +++ b/src/filter/list_levels_test.zig @@ -0,0 +1,307 @@ +//! Equivalence of the list levels (7–9 of `Snapshot.evaluate`) against a +//! reference model that keeps every compiled body as its own list of lines. +//! +//! The snapshot stores a source's `.list` and `.wild` names in one flagged set; +//! the reference keeps them apart and answers by scanning lines, the way the +//! bodies read on disk. Real list text goes through the real compiler, so the +//! corpus exercises what the compiler emits, not what a test assumes it emits. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const model = @import("../config/model.zig"); +const compiler = @import("compiler.zig"); +const parsers = @import("parsers.zig"); +const matcher = @import("matcher.zig"); + +const testing = std.testing; + +const Raw = struct { url: []const u8, format: parsers.Format, text: []const u8 }; + +/// Three overlapping sources covering each shape a body can take: `||x^` +/// (both bodies), `*.x` alone, `x` alone, `x` together with `*.x`, `@@` +/// exceptions, an exact entry for a subdomain of another source's wildcard, +/// and nested wildcards `*.x` with `*.y.x`. +const corpus = [_]Raw{ + .{ .url = "https://lists.test/abp", .format = .abp, .text = + \\! abp + \\||ads.example^ + \\||t.example^ + \\||deep.y.x.example^ + \\@@||good.ads.example^ + \\@@||t.example^ + \\ + }, + .{ .url = "https://lists.test/domains", .format = .domains, .text = + \\# domains + \\x.example + \\v.example + \\*.v.example + \\*.w.example + \\*.y.w.example + \\sub.ads.example + \\ads.example + \\ + }, + .{ .url = "https://lists.test/hosts", .format = .hosts, .text = + \\0.0.0.0 ads.example + \\0.0.0.0 h.example + \\0.0.0.0 a.w.example + \\0.0.0.0 good.ads.example + \\ + }, +}; + +const Compiled = struct { + list: std.Io.Writer.Allocating, + wild: std.Io.Writer.Allocating, + allow: std.Io.Writer.Allocating, + + fn init(gpa: Allocator, raw: Raw) !Compiled { + var self: Compiled = .{ .list = .init(gpa), .wild = .init(gpa), .allow = .init(gpa) }; + errdefer self.deinit(); + var r: std.Io.Reader = .fixed(raw.text); + _ = try compiler.compile(gpa, &r, raw.format, &self.list.writer, &self.wild.writer, &self.allow.writer); + return self; + } + + fn deinit(self: *Compiled) void { + self.list.deinit(); + self.wild.deinit(); + self.allow.deinit(); + } + + fn bodies(self: *Compiled) matcher.Snapshot.Compiled { + return .{ .list_body = self.list.written(), .wild_body = self.wild.written(), .allow_body = self.allow.written() }; + } +}; + +fn bodyHas(body: []const u8, wanted: []const u8) bool { + var lines = std.mem.tokenizeScalar(u8, body, '\n'); + while (lines.next()) |line| { + if (std.mem.eql(u8, line, wanted)) return true; + } + return false; +} + +/// Levels 7–9 as documented on `Snapshot.evaluate`, over separate bodies. +fn reference(bodies: []const matcher.Snapshot.Compiled, sources: []const u32, domain: []const u8) matcher.Decision { + var exceptions: matcher.Candidates = .init(domain); + while (exceptions.next()) |candidate| { + for (sources) |index| { + if (bodyHas(bodies[index].allow_body, candidate)) { + return .{ .blocked = false, .reason = .blocklist_exception, .matched = candidate, .source = index }; + } + } + } + for (sources) |index| { + if (bodyHas(bodies[index].list_body, domain)) { + return .{ .blocked = true, .reason = .blocklist_domain, .matched = domain, .source = index }; + } + } + var parents: matcher.Candidates = .init(domain); + _ = parents.next(); + while (parents.next()) |parent| { + for (sources) |index| { + if (bodyHas(bodies[index].wild_body, parent)) { + return .{ .blocked = true, .reason = .blocklist_wildcard, .matched = parent, .source = index }; + } + } + } + return .{ .blocked = false, .reason = .none, .matched = "" }; +} + +fn expectSameDecision(expected: matcher.Decision, actual: matcher.Decision, probe: []const u8) !void { + errdefer std.debug.print("probe {s}: expected {t} {s} {?d}, got {t} {s} {?d}\n", .{ + probe, expected.reason, expected.matched, expected.source, + actual.reason, actual.matched, actual.source, + }); + try testing.expectEqual(expected.blocked, actual.blocked); + try testing.expectEqual(expected.reason, actual.reason); + try testing.expectEqualStrings(expected.matched, actual.matched); + try testing.expectEqual(expected.source, actual.source); +} + +/// Every name in every body, each with two subdomains and its own parents, +/// plus names no list mentions. +fn probes(gpa: Allocator, bodies: []const matcher.Snapshot.Compiled) !std.ArrayList([]u8) { + var out: std.ArrayList([]u8) = .empty; + errdefer { + for (out.items) |probe| gpa.free(probe); + out.deinit(gpa); + } + for (bodies) |b| { + for ([_][]const u8{ b.list_body, b.wild_body, b.allow_body }) |body| { + var lines = std.mem.tokenizeScalar(u8, body, '\n'); + while (lines.next()) |line| { + var parents: matcher.Candidates = .init(line); + while (parents.next()) |parent| try out.append(gpa, try gpa.dupe(u8, parent)); + try out.append(gpa, try std.fmt.allocPrint(gpa, "p.{s}", .{line})); + try out.append(gpa, try std.fmt.allocPrint(gpa, "q.r.{s}", .{line})); + } + } + } + for ([_][]const u8{ "example", "unlisted.example", "x.unlisted.example", "" }) |name| { + try out.append(gpa, try gpa.dupe(u8, name)); + } + return out; +} + +test "list levels: the flagged name set decides exactly what separate bodies decide" { + const gpa = testing.allocator; + + var compiled: [corpus.len]Compiled = undefined; + var built: usize = 0; + defer for (compiled[0..built]) |*c| c.deinit(); + for (corpus, &compiled) |raw, *slot| { + slot.* = try .init(gpa, raw); + built += 1; + } + + var bodies: [corpus.len]matcher.Snapshot.Compiled = undefined; + var compiled_opt: [corpus.len]?matcher.Snapshot.Compiled = undefined; + var sources: [corpus.len]model.BlocklistSource = undefined; + var source_ids: [corpus.len]i64 = undefined; + for (&compiled, &bodies, &compiled_opt, &sources, &source_ids, corpus, 0..) |*c, *b, *o, *s, *id, raw, i| { + b.* = c.bodies(); + o.* = b.*; + s.* = .{ .url = raw.url, .name = raw.url }; + id.* = @intCast(i + 1); + } + + // `default` sees all three sources; `narrow` only the last two, so the + // ascending-index walk starts at a different source. + const links = [_]model.GroupSource{ + .{ .group = "default", .source_url = corpus[0].url }, + .{ .group = "default", .source_url = corpus[1].url }, + .{ .group = "default", .source_url = corpus[2].url }, + .{ .group = "narrow", .source_url = corpus[2].url }, + .{ .group = "narrow", .source_url = corpus[1].url }, + }; + var snapshot = try matcher.Snapshot.build(gpa, .{ + .groups = &.{ .{ .name = "default" }, .{ .name = "narrow" } }, + .group_ids = &.{ 1, 2 }, + .group_sources = &links, + .sources = &sources, + .source_ids = &source_ids, + .rules = &.{}, + .clients = &.{}, + .prefixes = &.{}, + .compiled = &compiled_opt, + .seed = 0x5eed, + .generation = 1, + }); + defer snapshot.deinit(); + + var probe_list = try probes(gpa, &bodies); + defer { + for (probe_list.items) |probe| gpa.free(probe); + probe_list.deinit(gpa); + } + + var decided = [_]u32{0} ** @typeInfo(matcher.Reason).@"enum".fields.len; + for ([_][]const u32{ &.{ 0, 1, 2 }, &.{ 1, 2 } }, 0..) |group_sources, group| { + try testing.expectEqualSlices(u32, group_sources, snapshot.groups[group].sources); + for (probe_list.items) |probe| { + const actual = snapshot.evaluate(@intCast(group), probe); + try expectSameDecision(reference(&bodies, group_sources, probe), actual, probe); + decided[@intFromEnum(actual.reason)] += 1; + } + } + + // The corpus reaches every list level, so the comparison is not vacuous. + try testing.expect(decided[@intFromEnum(matcher.Reason.blocklist_exception)] > 0); + try testing.expect(decided[@intFromEnum(matcher.Reason.blocklist_domain)] > 0); + try testing.expect(decided[@intFromEnum(matcher.Reason.blocklist_wildcard)] > 0); + try testing.expect(decided[@intFromEnum(matcher.Reason.none)] > 0); + + // The reference and the snapshot read the same compiled bodies, so a + // compiler that dropped a shape would drop it from both. Each shape the + // corpus exists for is therefore also pinned to the decision it must give. + for (named_cases) |case| { + const blocked = case.reason == .blocklist_domain or case.reason == .blocklist_wildcard; + try expectSameDecision( + .{ .blocked = blocked, .reason = case.reason, .matched = case.matched, .source = case.source }, + snapshot.evaluate(case.group, case.probe), + case.probe, + ); + } +} + +const NamedCase = struct { + group: u32, + probe: []const u8, + reason: matcher.Reason, + matched: []const u8 = "", + source: ?u32 = null, +}; + +const default_group = 0; +const narrow_group = 1; + +const named_cases = [_]NamedCase{ + // `||x^`: the apex is an exact match, a subdomain a wildcard match. + .{ .group = default_group, .probe = "ads.example", .reason = .blocklist_domain, .matched = "ads.example", .source = 0 }, + .{ .group = default_group, .probe = "p.ads.example", .reason = .blocklist_wildcard, .matched = "ads.example", .source = 0 }, + // An exact entry for the queried name beats another source's wildcard. + .{ .group = default_group, .probe = "sub.ads.example", .reason = .blocklist_domain, .matched = "sub.ads.example", .source = 1 }, + // `@@` beats every block, its own source's and another's, for the name + // and its subdomains. + .{ .group = default_group, .probe = "good.ads.example", .reason = .blocklist_exception, .matched = "good.ads.example", .source = 0 }, + .{ .group = default_group, .probe = "x.good.ads.example", .reason = .blocklist_exception, .matched = "good.ads.example", .source = 0 }, + .{ .group = default_group, .probe = "t.example", .reason = .blocklist_exception, .matched = "t.example", .source = 0 }, + .{ .group = default_group, .probe = "p.t.example", .reason = .blocklist_exception, .matched = "t.example", .source = 0 }, + // `x` alone blocks the apex only. + .{ .group = default_group, .probe = "x.example", .reason = .blocklist_domain, .matched = "x.example", .source = 1 }, + .{ .group = default_group, .probe = "p.x.example", .reason = .none }, + // `*.x` alone blocks subdomains only. + .{ .group = default_group, .probe = "w.example", .reason = .none }, + .{ .group = default_group, .probe = "p.w.example", .reason = .blocklist_wildcard, .matched = "w.example", .source = 1 }, + // An exact entry for a subdomain of a wildcard reports the exact entry. + .{ .group = default_group, .probe = "a.w.example", .reason = .blocklist_domain, .matched = "a.w.example", .source = 2 }, + // Nested wildcards: the nearer parent wins. + .{ .group = default_group, .probe = "p.y.w.example", .reason = .blocklist_wildcard, .matched = "y.w.example", .source = 1 }, + .{ .group = default_group, .probe = "y.w.example", .reason = .blocklist_wildcard, .matched = "w.example", .source = 1 }, + .{ .group = default_group, .probe = "q.deep.y.x.example", .reason = .blocklist_wildcard, .matched = "deep.y.x.example", .source = 0 }, + // `x` with `*.x` in a domains list. + .{ .group = default_group, .probe = "v.example", .reason = .blocklist_domain, .matched = "v.example", .source = 1 }, + .{ .group = default_group, .probe = "p.v.example", .reason = .blocklist_wildcard, .matched = "v.example", .source = 1 }, + // Overlapping sources: the lowest linked index reports, and a group that + // does not link the ABP source has neither its wildcard nor its `@@`. + .{ .group = narrow_group, .probe = "ads.example", .reason = .blocklist_domain, .matched = "ads.example", .source = 1 }, + .{ .group = narrow_group, .probe = "p.ads.example", .reason = .none }, + .{ .group = narrow_group, .probe = "good.ads.example", .reason = .blocklist_domain, .matched = "good.ads.example", .source = 2 }, +}; + +test "list levels: a name in both bodies of one source is stored once" { + const gpa = testing.allocator; + var c = try Compiled.init(gpa, corpus[0]); + defer c.deinit(); + const b = c.bodies(); + // The ABP source compiles each `||x^` into both bodies. + try testing.expectEqualStrings(b.list_body, b.wild_body); + + const sources = [_]model.BlocklistSource{.{ .url = corpus[0].url, .name = "abp" }}; + const compiled = [_]?matcher.Snapshot.Compiled{b}; + var snapshot = try matcher.Snapshot.build(gpa, .{ + .groups = &.{.{ .name = "default" }}, + .group_ids = &.{1}, + .group_sources = &.{.{ .group = "default", .source_url = corpus[0].url }}, + .sources = &sources, + .source_ids = &.{1}, + .rules = &.{}, + .clients = &.{}, + .prefixes = &.{}, + .compiled = &compiled, + .seed = 0x5eed, + .generation = 1, + }); + defer snapshot.deinit(); + + const names = snapshot.sources[0].names; + try testing.expectEqual(@as(u32, @intCast(std.mem.count(u8, b.list_body, "\n"))), names.count); + try testing.expectEqual( + matcher.SourceSets.exact | matcher.SourceSets.subdomains, + names.flags("ads.example"), + ); +} diff --git a/src/filter/matcher.zig b/src/filter/matcher.zig index 72e8db5..b1b81d3 100644 --- a/src/filter/matcher.zig +++ b/src/filter/matcher.zig @@ -100,12 +100,18 @@ pub const SourceSets = struct { id: i64, /// Snapshot-arena-owned; the source's display name for the UI. name: []const u8, - domains: domain_set.DomainSet, - wildcards: domain_set.DomainSet, + /// Every name of the source's `.list` and `.wild` bodies, stored once and + /// flagged with the body it came from: `exact` blocks the name itself, + /// `subdomains` blocks its proper subdomains. An ABP `||x^` line compiles + /// into both bodies, so one entry carries both flags. + names: domain_set.DomainSet, /// The names this source's `@@` exceptions lift. One entry covers the name /// and every subdomain of it, because `evaluate` walks the full name and /// each parent against this set. exceptions: domain_set.DomainSet, + + pub const exact = domain_set.DomainSet.in_first; + pub const subdomains = domain_set.DomainSet.in_second; }; pub const Group = struct { @@ -209,8 +215,7 @@ pub const Snapshot = struct { sources[at] = .{ .id = id, .name = try arena.dupe(u8, row.name), - .domains = try domain_set.DomainSet.build(arena, bodies.list_body, input.seed), - .wildcards = try domain_set.DomainSet.build(arena, bodies.wild_body, input.seed), + .names = try domain_set.DomainSet.buildMerged(arena, bodies.list_body, bodies.wild_body, input.seed), .exceptions = try domain_set.DomainSet.build(arena, bodies.allow_body, input.seed), }; } @@ -369,7 +374,7 @@ pub const Snapshot = struct { } for (g.sources) |index| { - if (self.sources[index].domains.contains(domain)) { + if (self.sources[index].names.flags(domain) & SourceSets.exact != 0) { return .{ .blocked = true, .reason = .blocklist_domain, @@ -384,7 +389,7 @@ pub const Snapshot = struct { _ = parents.next(); while (parents.next()) |parent| { for (g.sources) |index| { - if (self.sources[index].wildcards.contains(parent)) { + if (self.sources[index].names.flags(parent) & SourceSets.subdomains != 0) { return .{ .blocked = true, .reason = .blocklist_wildcard, @@ -434,8 +439,7 @@ pub const Snapshot = struct { var total: usize = 0; for (self.sources) |*source| { total += @sizeOf(SourceSets) + source.name.len + - source.domains.memoryBytes() + source.wildcards.memoryBytes() + - source.exceptions.memoryBytes(); + source.names.memoryBytes() + source.exceptions.memoryBytes(); } for (self.groups) |*group| { total += @sizeOf(Group) + group.name.len + @@ -1347,14 +1351,18 @@ test "memoryBytes stays within the compiled-data bound" { try body.appendSlice(gpa, try std.fmt.bufPrint(&line, "d{d:0>5}.example.com\n", .{i})); } - const lists: Lists = .init(body.items, ""); + // Identical bodies, as an ABP list compiles them: the names are stored + // once, so one arena and one index bound the lot. + const lists: Lists = .init(body.items, body.items); var snapshot = try build(gpa, lists.fixture()); defer snapshot.deinit(); - // arena + index, over both bodies, plus a kilobyte of struct overhead. - const index_bytes = 2 * 16 * 1024 * @sizeOf(u32); - try testing.expect(snapshot.memoryBytes() < 2 * body.items.len + index_bytes + 1024); + // Per name: the length prefix in place of the newline, plus the flag byte. + const arena_bytes = body.items.len + count; + const index_bytes = 16 * 1024 * @sizeOf(u32); + try testing.expect(snapshot.memoryBytes() < arena_bytes + index_bytes + 1024); try testing.expect(snapshot.evaluate(0, "d00042.example.com").blocked); + try testing.expect(snapshot.evaluate(0, "x.d00042.example.com").blocked); } fn buildUnderFailure(gpa: Allocator) !void { diff --git a/src/tests.zig b/src/tests.zig index bcb427d..ce5a256 100644 --- a/src/tests.zig +++ b/src/tests.zig @@ -70,6 +70,7 @@ comptime { _ = @import("filter/domain_set.zig"); _ = @import("filter/rules.zig"); _ = @import("filter/matcher.zig"); + _ = @import("filter/list_levels_test.zig"); _ = @import("filter/compiler.zig"); _ = @import("filter/safesearch.zig"); _ = @import("filter/response.zig");