diff --git a/PLAN.md b/PLAN.md index 10c3d83..06d321f 100644 --- a/PLAN.md +++ b/PLAN.md @@ -271,14 +271,18 @@ Walk chain to depth 8; any target hitting block logic → synthesize blocked res ### 7.1 Evaluation -For `{domain, qtype, group_id}`: +For `{domain, group_id}` (the qtype travels with the query for logging and response synthesis, not +for matching): 1. Normalize: lowercase, trim trailing dot. 2. Build candidate chain (full, parent1, parent2, …). -3. Explicit rules per §3.10 precedence. -4. Group's blocklist domains (hash set over compiled lists). -5. Group's blocklist wildcards. +3. Explicit rules per §3.10 precedence, evaluated against every candidate in the chain. +4. Group's blocklist domains (hash set over compiled lists), matched against the query name only. +5. Group's blocklist wildcards, matched against every proper parent of the query name. 6. No match → allow. +Blocklist entries do not parent-walk; only rules do (§3.9). ABP `||x.y^` emits both a domain entry +`x.y` and a wildcard entry `x.y`, which together give domain-and-subdomains semantics. + ### 7.2 Group Assignment - Per source IP: (1) exact match in `clients`, (2) longest-prefix match in `client_prefixes` (ties: longer prefix, then priority), (3) `default` group. @@ -287,7 +291,12 @@ For `{domain, qtype, group_id}`: ### 7.3 Reload -New immutable matcher built from DB + compiled list files → atomic pointer swap (RCU, generation counter). Readers lock-free. +New immutable matcher built from DB + compiled list files → swap under an `std.Io.RwLock` with a +generation counter. Readers take the shared lock for the microseconds of one evaluate; the writer +takes the exclusive lock only for the swap, and the source status table is installed in the same +critical section, so a failed reload publishes neither. (Deliberate deviation from "readers +lock-free": freeing the old snapshot without a lock needs epoch-based reclamation, unjustifiable at +household scale — see specs/milestone-5.md S8.3.) ### 7.4 Safe-Search diff --git a/build.zig b/build.zig index 3a2239f..03b79c7 100644 --- a/build.zig +++ b/build.zig @@ -79,6 +79,24 @@ pub fn build(b: *std.Build) void { }); test_step.dependOn(&b.addRunArtifact(fuzz_tests).step); + const parsers_mod = b.createModule(.{ + .root_source_file = b.path("src/filter/parsers.zig"), + .target = target, + .optimize = optimize, + }); + const blocklist_fuzz_mod = b.createModule(.{ + .root_source_file = b.path("tests/fuzz/blocklist_fuzz.zig"), + .target = target, + .optimize = optimize, + }); + blocklist_fuzz_mod.addImport("parsers", parsers_mod); + const blocklist_fuzz_tests = b.addTest(.{ + .name = "blocklist-fuzz", + .use_llvm = if (fuzz) true else null, + .root_module = blocklist_fuzz_mod, + }); + test_step.dependOn(&b.addRunArtifact(blocklist_fuzz_tests).step); + const cross = b.step("cross", "Build static musl executables for every deploy target"); for (cross_targets) |triple| { const query = std.Target.Query.parse(.{ .arch_os_abi = triple }) catch |err| { diff --git a/specs/milestone-5.md b/specs/milestone-5.md new file mode 100644 index 0000000..5fc2f54 --- /dev/null +++ b/specs/milestone-5.md @@ -0,0 +1,1798 @@ +# Milestone 5: Filtering + Local DNS + +Goal (PLAN §16 Phase 5): the rule matcher (exact / parent-walk / wildcard); blocklist parsers and the +compiled file format; the fetcher and its scheduled update; the RCU snapshot swap; per-group +safe-search; blocked-response synthesis; local records; forward zones. +Exit: the §3.10 precedence table is validated by tests; local zone answers and conditional forwards +work. + +Read first: `AGENTS.md` (values), `specs/research/zig-0.16-api-notes.md` (verified stdlib facts — +pre-0.16 knowledge is stale and MUST NOT be used), `specs/milestone-1.md`, `specs/milestone-2.md`, +`specs/milestone-3.md`, `specs/milestone-4.md` (module conventions, the "As built" notes, and the +binding logging policy restated below). The Zig source of truth is `/home/mokhtar/app/zig` at tag +`0.16.0`. PLAN §2.2, §3.8–§3.10, §5, §6.2, §6.4, §6.5, §7, §11.2, §12, §18 are the scope authority. + +## What already exists (do not respecify, import it) + +- `src/dns/*` — pure wire format. Used here: `name.Name`, `name.fromText`, `name.formatText`, + `name.eqlIgnoreCase`, `types.Type`, `types.Class`, `types.Rcode`, `types.max_name_len`, + `question.Question`, `header.Header`, `edns.OptRecord`, `packet.ResponseBuilder` + (`init`, `setRcode`, `addAnswer`, `addOptEcho`, `finish`). Do not write a second name parser and do + not add anything to `src/dns/`. +- `src/platform/address.zig` — `NetAddress.parse/format/key/eql`, `Prefix.parse/contains`, + `matchLongest`. Group assignment by client IP uses these; do not write a second IP parser. +- `src/config/model.zig` — `Config`, `Group`, `Rule`, `RuleKind`, `RuleAction`, `BlocklistSource`, + `GroupSource`, `LocalRecord`, `RecordType`, `ForwardZone`, `Client`, `ClientPrefix`, `Blocking`, + `BlockResponse`, `BlocklistUpdate`, `updateIntervalSeconds`. +- `src/config/validate.zig` — `parseResolver(text) ResolverError!Resolver`, `Resolver`, + `ResolverScheme`. Its doc comment already names this phase as the importer. `local/forward_zones.zig` + imports it; nobody writes a second resolver-URL parser. +- `src/storage/repositories/*` — `listGroups`/`freeGroups`, `listGroupSources`/`freeGroupSources`, + `listRules`/`freeRules`, `listBlocklistSources`/`freeBlocklistSources`, `listLocalRecords`, + `listForwardZones`, `listClients`, `listClientPrefixes`, and the `count*` functions. Every list + returns `std.ArrayList(model.X)` with heap-owned strings and a matching `freeX`. **The repositories + gain no new functions in this milestone except the two named in S8.1**, which are additive and + owned by S8. +- `src/storage/db.zig` — `Db`, `Stmt`, `Tx`, `Error`. `db.zig` takes no `std.Io` (milestone 4's + documented exception); everything else that touches the filesystem takes `io: std.Io`. +- `src/upstream/transport.zig` — `Client` (the `exchangeFn` vtable), `ExchangeError`, `PeerFault`, + `LocalResource`, `mapLocal`, `group`, `validateResponse`, `max_message_len`. The forward-zone + client implements this interface; it does not invent a second one. +- `build.zig` — `-Dintegration` (hermetic, loopback/tmpdir only, PR-blocking) and `-Dlive` (leaves + the machine, manual only) reach test files through `@import("build_options")`. A second test + artifact carries the fuzz targets with `dns` as a named module. + +## Sessions + +``` +S1 (parsers + wildcard) S2 (domain_set + compiler) S3 (safesearch + response) +S4 (local records + forward_zones) S6 (fetcher) S7 (forward_client) [all parallel, no deps] + | | + +---------+----------+ + v + S5 (rules + matcher) [needs S1, S2] + v + S8 (manager) [needs S2, S5, S6] + v + S9 (integration + fuzz) [needs everything] +``` + +Six sessions start together. Every later session is written against **this spec**, not against the +previous session's source. The orchestrator — not any session — wires `src/tests.zig` imports and +every `build.zig` change. A session that needs a build change reports the exact change in its +completion report; the one this milestone needs is stated in S9.3. + +## Session verification protocol (read this before starting) + +Unchanged from milestone 4, and it still binds: + +- `zig test ` does not work here: every file imports across `src/` subdirectories or links + against `sqlite3`. Each session verifies its own work with `zig fmt --check ` and + `zig ast-check `. `zig ast-check` reports only syntax and AST-level errors; it does + **not** type-check, so it cannot prove the code compiles. +- The orchestrator wires the files into `src/tests.zig` and runs `zig build test` (and + `zig build test -Dintegration` for S9). That run is the real gate. +- Every session states in its completion report that its tests have not been executed, and lists the + exact test names it wrote so the orchestrator can confirm they ran. +- No session edits `build.zig`, `build.zig.zon`, or `src/tests.zig`. No session edits a milestone + 1–4 file; a needed change there is reported, not made. + +## Design invariants (all sessions) + +- **The pure core stays pure.** `filter/parsers.zig`, `parser_hosts.zig`, `parser_domains.zig`, + `parser_abp.zig`, `wildcard.zig`, `domain_set.zig`, `compiler.zig`, `rules.zig`, `matcher.zig`, + `safesearch.zig`, `response.zig`, `local/records.zig` and `local/forward_zones.zig` take **no + `std.Io` value**. They take bytes, structs, an allocator, and `*std.Io.Reader` / `*std.Io.Writer` + interfaces — which are not `Io` and carry no backend. Clocks, entropy, sockets and files arrive as + parameters or do not arrive at all. +- **`std.Io` lives in exactly three files**: `filter/fetcher.zig`, `filter/manager.zig`, + `local/forward_client.zig`. PLAN §5 marks `filter/` "pure" and then lists `fetcher.zig` and + `compiler.zig` inside it; the split above keeps that promise where it matters — the compiler is + pure over reader/writer interfaces, and only the fetcher and the manager touch the network and the + filesystem. +- **Logging policy (binding, repo-wide, milestone 4 "As built")**: a condition that is returned as a + typed error logs at `warn` at most. `err` is reserved for failures the code swallows. The zig test + runner fails any test that emits `err` logs. In this milestone the only legitimate `err` site is a + background refresh task that has nowhere to return its failure — and even that logs at `warn`, + because the failure is recorded in `SourceStatus` and surfaced (S8.5). **No `std.log.err` call is + written in this milestone.** +- **Every failure mode is counted, not dropped** (AGENTS.md). A skipped regex line, an unsupported + ABP modifier, an over-long line, a single-label name, an unparseable name, a failed download and a + checksum mismatch each increment a named counter that reaches `SourceStatus` and (for the three + columns that exist) `blocklist_sources`. +- **No regex, ever** (PLAN §2.2). Regex lines are recognized, counted, skipped. No engine, no + dependency, no "just a small subset". +- **Nothing in this milestone allocates on the query path.** `Snapshot.evaluate` performs no + allocation, opens no file, and takes no lock beyond the reader lock its caller already holds. It is + callable from a `std.Io` task with a stack buffer and nothing else. +- **Determinism**: a compiled list body is a pure function of the downloaded bytes and the format. + See the determinism contract in §Determinism below; S2 and S9 both test it. +- Unit tests live in-file. Tests that touch the filesystem, loopback sockets or real HTTP live in + `src/filter/filter_integration_test.zig`, guarded by + `if (!build_options.integration) return error.SkipZigTest;`. Tests that leave the machine are + guarded by `build_options.live`. Mirrors milestones 1, 3 and 4. + +--- + +## Resolved PLAN ambiguities (read before writing code) + +These were decided while writing this spec. They are the ruling for this milestone; the orchestrator +carries any of them that change PLAN text back into PLAN. + +1. **Blocklist domain entries match exactly; only rules parent-walk.** PLAN §7.1 builds a candidate + chain and then consults rules (step 3), blocklist domains (step 4) and blocklist wildcards + (step 5). If blocklist domains were also matched against the whole chain, §3.10's separate + "blocklist wildcards" level would be unreachable and one `com`-shaped line in a bad list would + black-hole the internet. §3.9 ties parent-walk to the **rule** model ("parent-walk (implicit via + candidate chain)"). Ruling: exact and wildcard **rules** are evaluated against every candidate in + the chain; **blocklist `.list` entries** are matched against the query name only; **blocklist + `.wild` entries** are matched against every *proper parent* of the query name (that is what + `*.x.y` means). ABP `||x.y^` therefore emits both a `.list` entry `x.y` and a `.wild` entry `x.y`, + which together give the "domain and all subdomains" semantics the syntax promises. +2. **`*` matches one or more labels.** PLAN §3.9 gives both `*.doubleclick.net` and + `ads.*.example.com`. A one-label-only `*` would make the first pattern miss + `a.b.doubleclick.net`, which is not what an operator writing it means. One rule for both + positions: each `*` label matches one **or more** labels. Bounded by the 128-label ceiling a + 255-byte name implies, so the backtracking match cannot blow up. +3. **`evaluate` takes no qtype.** PLAN §7.1 keys evaluation on `{domain, qtype, group_id}` but no + step in §7 or §3.10 reads the qtype. Ruling: the filtering decision is + `{domain, group}`; the qtype travels with the query for logging and for response synthesis, not + for matching. Adding an unused parameter would be generality nobody asked for (AGENTS.md). +4. **Group assignment by client IP is in scope; auto-materialization is not.** PLAN §7.2 belongs to + the filtering engine this phase builds, and the matching half is pure (`clients` exact match → + `client_prefixes` longest-prefix → `default`). The half that **writes** — inserting the unseen + client row, updating `last_seen`, retention of `hand_edited = 0` rows — needs a clock and a + database write on the query path and belongs with the handler pipeline (Phase 7) and retention + (Phase 6). `Snapshot.groupForClient` is built here; nothing inserts a row. +5. **A plain UDP/TCP resolver client is in scope.** PLAN Phase 5's exit criterion says conditional + forwards *work*, and §6.5 permits plain transports. No such client exists: `transport.Endpoint` + knows only `https://` and `tls://`, by design. Ruling: `src/local/forward_client.zig` implements + `transport.Client` over plain UDP with TCP fallback, configured from `validate.Resolver` rather + than from `transport.Endpoint`, so no milestone-3 file is edited. Wiring it into the query path is + Phase 7. +6. **Compressed transfer encoding is out of scope.** `std.http.Client` advertises gzip and deflate by + default (verified, Client.zig:831) and `Response.readerDecompressing` would be needed to read + them. The fetcher overrides `accept_encoding` to identity, exactly as `doh_client.zig` does. A + daily download of a few megabytes does not pay for a decompression buffer and a second failure + surface. A source URL that serves a `.gz` **body** (content, not encoding) fails the compile with + a high invalid-line count and a visible error, rather than silently compiling to nothing. + +## Determinism + +For a given input byte stream and format, `compiler.compile` must produce byte-identical `.list` and +`.wild` bodies on every run, on both target architectures. Concretely: + +- Names are lowercased over ASCII only (`A`–`Z`); a byte ≥ 0x80 makes the candidate invalid. +- A single trailing dot is stripped; the name is stored without it. +- Entries are sorted ascending by `std.mem.order(u8, …)` over the full name and deduplicated. +- Every line ends with a single `\n`, including the last. An empty body is zero bytes. +- The body carries no header, no timestamp, no hostname, no counts. The manager writes the header + (S8.3); the checksum covers the two bodies only. +- Sorting is by byte value, not by locale, not by label. Two runs of the same input in different + input order (S9 shuffles a fixture) produce the same body. + +The snapshot built from those files is deterministic in content but **not** in hash-table layout: the +open-addressing seed is per-snapshot randomness (S2.1). `contains` results do not depend on it, and +that is the property tests assert. + +## Memory budget (PLAN §18: < 100 MB with ~1M blocked domains) + +The compiled in-memory structure is an **exact-match** set — a flat arena of length-prefixed +lowercase names plus an open-addressed table of `u32` offsets into it. No Bloom filter, no +hash-only key set, no probabilistic structure: a false positive in a DNS sinkhole blocks a real +domain for a real household and is undebuggable from the outside. Every probe compares full bytes. + +Sizing for one 1,000,000-entry source, mean name length 22 bytes: + +| Part | Size | +|---|---| +| arena (`1 + len` bytes per name) | ≈ 23 MB | +| index: capacity `2^21` slots at load factor ≤ 0.75, 4 bytes per slot | 8 MiB | +| per-source struct overhead | < 1 KB | +| **total, one 1M-domain source** | **≈ 31 MB** | + +A typical household set (three lists, ~150k entries each, heavy overlap) is under 15 MB. Sets are +**per source, shared between groups** — a group holds indices into `Snapshot.sources`, so assigning +one source to four groups costs four `u32`s, not four copies. + +Refresh peak, which is what the budget has to survive: + +1. Sources are compiled **one at a time**, each into its own arena, and that arena is freed before + the next source starts. Compile working set for a 1M-entry source: 23 MB of candidate bytes plus + 4 MB of `u32` offsets ≈ 27 MB. +2. The new snapshot is built only after every compile has finished, so compile arenas and the new + snapshot never coexist. +3. Peak is therefore `max(old + compile, old + new)` ≈ 62 MB for a 1M-domain configuration, leaving + headroom under the 100 MB target for the servers, the pool and (Phase 6) the cache. + +Hard caps, all of which produce a typed error and a counter rather than a slow death: +`compiler.max_domains = 2_000_000` per source, `fetcher.max_body_bytes = 64 * 1024 * 1024`, +`compiler.max_line_len = 4096`, `rules.max_wildcards_per_group = 4096`. + +Lookup cost, against PLAN §18's "blocklist lookup p95 < 1 ms": a 5-label query walks at most 5 +candidates; per candidate the matcher does 2 rule-set probes plus one probe per assigned source +(`.list` for the full name, `.wild` for parents). With 5 sources that is under 40 Wyhash + memcmp +probes on structures that fit in L2 for typical list sizes. No allocation, no lock beyond the +caller's reader lock. + +--- + +## Verified 0.16.0 stdlib facts used by this milestone + +Read from `/home/mokhtar/app/zig` at tag `0.16.0`. Anything not listed here or in +`specs/research/zig-0.16-api-notes.md` must be re-verified against the source before use. + +### Locks — they are **not** in `std.Thread` + +`lib/std/Thread.zig` declares no `Mutex` and no `RwLock` in 0.16.0. Both live under `std.Io`: + +```zig +pub const RwLock = @import("Io/RwLock.zig"); // Io.zig:48 +pub const init: RwLock = .{ ... }; // Io/RwLock.zig:15 +pub fn lock(rl: *RwLock, io: Io) Io.Cancelable!void; // Io/RwLock.zig:51 +pub fn lockUncancelable(rl: *RwLock, io: Io) void; // Io/RwLock.zig:42 +pub fn unlock(rl: *RwLock, io: Io) void; // Io/RwLock.zig:70 +pub fn lockShared(rl: *RwLock, io: Io) Io.Cancelable!void; // Io/RwLock.zig:115 +pub fn lockSharedUncancelable(rl: *RwLock, io: Io) void; // Io/RwLock.zig:97 +pub fn unlockShared(rl: *RwLock, io: Io) void; // Io/RwLock.zig:133 +pub fn tryLockShared(rl: *RwLock, io: Io) bool; // Io/RwLock.zig:75 +pub const Mutex = extern struct { ... }; // Io.zig:1587 +``` + +### Timers + +```zig +pub fn sleep(io: Io, duration: Duration, clock: Clock) Cancelable!void; // Io.zig:2397 +pub fn sleep(duration: Clock.Duration, io: Io) Cancelable!void; // Io.zig:900 +pub fn fromSeconds(x: i64) Duration; // Io.zig:988 +pub fn fromMilliseconds(x: i64) Duration; // Io.zig:984 +pub fn now(clock: Clock, io: Io) Io.Timestamp; // Io.zig:778 +pub fn toSeconds(t: Timestamp) i64; // Io.zig:943 +``` + +There is no timer wheel and no periodic-callback API. A scheduled refresh is a task that sleeps and +loops; cancellation arrives as `error.Canceled` from the sleep (milestone 3 convention). + +### `std.Io.Reader` line iteration + +```zig +pub fn takeDelimiter(r: *Reader, delimiter: u8) error{ ReadFailed, StreamTooLong }!?[]u8; // Reader.zig:895 +pub fn discardDelimiterInclusive(r: *Reader, delimiter: u8) Error!usize; // Reader.zig:1037 +pub fn discardRemaining(r: *Reader) ShortError!usize; // Reader.zig:270 +``` + +`takeDelimiter` returns `null` at end of stream, excludes the delimiter, and treats end-of-stream as +a delimiter for a final unterminated line. `error.StreamTooLong` means the line did not fit the +reader's buffer and **leaves the stream unmodified** (doc comment, Reader.zig:885) — the compiler +must then `discardDelimiterInclusive('\n')` to step over that line and count it, or it will spin +forever on the same bytes. This is the single most likely infinite loop in this milestone. + +### `std.http.Client` — GET download flow + +```zig +pub fn request(client: *Client, method, uri, options: RequestOptions) RequestError!Request; +pub fn sendBodiless(r: *Request) Writer.Error!void; // Client.zig:912 +pub fn receiveHead(r: *Request, redirect_buffer: []u8) ReceiveHeadError!Response; // Client.zig:1133 +pub fn reader(response: *Response, transfer_buffer: []u8) *Reader; // Client.zig:736 +pub fn readerDecompressing(...) *Reader; // Client.zig:752 (NOT used, ambiguity 6) +redirect_behavior: Request.RedirectBehavior = @enumFromInt(3) // Client.zig:1654 +pub const default_accept_encoding = { gzip, deflate, identity } // Client.zig:831 +``` + +`receiveHead` follows redirects itself and needs the `redirect_buffer` to outlive `Request.uri`; +RFC 9110 recommends ≥ 8000 bytes (doc comment, Client.zig:1128). An empty buffer is legal only when +`redirect_behavior = .not_allowed`. Blocklist URLs redirect in practice, so the fetcher allows +redirects and supplies a real buffer. `Response.reader` returns **compressed** bytes if a compressed +encoding was negotiated — which is why the fetcher pins `accept_encoding` to identity. + +### Hashing and sorting + +```zig +pub fn hash(seed: u64, input: []const u8) u64; // hash/wyhash.zig:178 → std.hash.Wyhash +pub const Sha256 = Sha2x32(iv256, 256); // crypto/sha2.zig:23 +pub fn hash(b: []const u8, out: *[32]u8, options) void; // crypto/sha2.zig:108 +pub fn bytesToHex(input: anytype, case: Case) [input.len * 2]u8; // fmt.zig:1156 +pub fn sortUnstableContext(a: usize, b: usize, context: anytype) void; // mem.zig:648 +``` + +### Filesystem (unchanged from milestone 4, reproduced for the two files that need it) + +```zig +pub fn createFileAtomic(dir: Dir, io: Io, sub_path: []const u8, options: CreateFileAtomicOptions) + CreateFileAtomicError!File.Atomic; // Dir.zig:1924 +pub fn replace(af: *Atomic, io: Io) ReplaceError!void; // File/Atomic.zig:77 +pub fn deinit(af: *Atomic, io: Io) void; // File/Atomic.zig:23 +pub fn readFileAlloc(dir, io, sub_path, gpa, limit) ReadFileAllocError![]u8; // Dir.zig:1326 +pub fn createDirPathStatus(dir, io, sub_path, permissions) ...!CreatePathStatus; // Dir.zig:851 +pub fn deleteFile(dir, io, sub_path) DeleteFileError!void; // Dir.zig:1004 +pub fn writer(file: File, io: Io, buffer: []u8) Writer; // File.zig:600, BY VALUE +``` + +Plain `createDirPath` hardcodes `0o777` (Dir.zig:843) and must not be used; the blocklist directory +is created with `createDirPathStatus(io, path, .fromMode(0o700))`. + +--- + +## Session S1: `src/filter/parsers.zig`, `parser_hosts.zig`, `parser_domains.zig`, `parser_abp.zig`, `wildcard.zig` + +Pure, and **`std` is the only import in all five files**. No `@import("../dns/...")`, no config, no +allocator, no I/O. That constraint is not stylistic: `parsers.zig` is the root of the fuzz module +S9.3 adds, and a module root cannot import across its own directory boundary +(`error.ImportOutsideModulePath`, milestone 2). Domain **validity** is decided in S2's compiler +through `dns.name.fromText`; these files decide **format**. + +### S1.1 `parsers.zig` — the shared vocabulary + +```zig +pub const hosts = @import("parser_hosts.zig"); +pub const domains = @import("parser_domains.zig"); +pub const abp = @import("parser_abp.zig"); +pub const wildcard = @import("wildcard.zig"); + +pub const Format = enum { hosts, domains, abp }; + +pub const Kind = enum { + /// Nothing on the line, or only a comment. + ignore, + /// `text` holds one or more whitespace-separated candidate names. + domain, + /// `text` holds one candidate suffix; every proper subdomain of it matches. + wildcard, + /// A regex rule. Counted, skipped, never compiled (PLAN §2.2). + regex, + /// Syntactically a rule of this format, but one nxdns cannot honour: + /// an ABP modifier list, an exception rule, element hiding, a scheme anchor. + unsupported, +}; + +pub const Line = struct { + kind: Kind, + /// Borrowed from the caller's line. Not lowercased, not validated. + text: []const u8 = "", + /// `.wildcard` only. ABP `||x^` covers `x` itself as well as its subdomains, + /// so the compiler emits an additional `.list` entry when this is set. + covers_apex: bool = false, +}; + +/// Dispatches to the format's parser. The line must not contain '\n' or '\r'; +/// the caller strips them. +pub fn parseLine(format: Format, line: []const u8) Line; + +/// Picks a format from the first `sample_lines` lines that are not blank and +/// not comments: an ABP marker (`||`, `@@`, `##`, `$`) anywhere wins `.abp`; +/// otherwise a majority of lines whose first field parses as an IP literal +/// wins `.hosts`; otherwise `.domains`. +pub const sample_lines = 64; +pub fn detectFormat(sample: []const u8) Format; +``` + +`detectFormat` uses its own minimal IPv4/IPv6-literal recognizer over the first field (digits, dots, +colons and hex only) rather than `NetAddress.parse` — importing `platform/` would break the module +constraint above, and a sniffing heuristic does not need a correct parser. + +### S1.2 `parser_hosts.zig` + +```zig +pub fn parseLine(line: []const u8) parsers.Line; +``` + +1. Truncate at the first `#`. Trim ASCII whitespace. Empty → `.ignore`. +2. A line starting with `/` → `.regex` (a regex smuggled into a hosts list). +3. Split on ASCII whitespace. If the first field looks like an IP literal **and** at least one field + follows, the remaining fields are the candidate names → `.domain` with `text` spanning them. +4. If the first field does not look like an IP literal, the whole line is the candidate name(s) → + `.domain`. Real "hosts" lists are frequently bare domain lists with a hosts extension. +5. A leading `*.` on any candidate is not stripped here; `parsers.parseLine` is per line, and the + compiler splits `text` and re-classifies each name (S2.3). This is the one place a `.domain` line + can yield wildcard entries, and it is spelled out in S2.3 rather than duplicated here. + +The sink address is not checked against a list of "blocking" IPs: a hosts-format blocklist that maps +to `127.0.0.1`, `0.0.0.0` or `::` is the same instruction to nxdns, and a source that maps to a real +address is still a source of names the operator asked to block. + +### S1.3 `parser_domains.zig` + +```zig +pub fn parseLine(line: []const u8) parsers.Line; +``` + +One candidate per line. `#` and `!` start a comment (both appear in the wild); truncate and trim. +A leading `/` → `.regex`. Everything else → `.domain` with the trimmed text. A line containing +whitespace after trimming → `.unsupported` (a domains-format list with two fields is a +mis-detected hosts file, and guessing is worse than counting). + +### S1.4 `parser_abp.zig` + +```zig +pub fn parseLine(line: []const u8) parsers.Line; +``` + +| Input | Result | +|---|---| +| `! comment`, `[Adblock Plus 2.0]`, empty | `.ignore` | +| `\|\|example.com^` | `.wildcard`, `text = "example.com"`, `covers_apex = true` | +| `\|\|example.com^` with a trailing `$…` modifier list | `.unsupported` | +| `@@…` (exception) | `.unsupported` | +| `##…`, `#@#…`, `#?#…` (element hiding) | `.unsupported` | +| `\|http://…`, `\|https://…` (scheme anchor) | `.unsupported` | +| `/regex/` | `.regex` | +| `example.com` (bare) | `.domain` | +| anything containing `*`, `^` or `\|` outside the forms above | `.unsupported` | + +`^` is a separator token in ABP and only the trailing `^` (or a trailing `^` before a modifier) is +meaningful for a domain rule; anything else carrying `^` is `.unsupported`. Exception rules are +`.unsupported` rather than an allow entry: PLAN's allow surface is the `rules` table, and a list +that could quietly allow a domain across every group is a policy hole the operator did not open. + +### S1.5 `wildcard.zig` + +```zig +pub const max_labels = 128; + +pub const PatternError = error{ + /// No label is exactly "*". + NoWildcard, + /// A label contains '*' but is not exactly "*". Partial-label globbing + /// (`ad*.example.com`) is out of scope: it is regex by another name, and + /// PLAN §3.9 defines the wildcard as a label pattern. + PartialWildcardLabel, + EmptyLabel, + LabelTooLong, + PatternTooLong, + TooManyLabels, +}; + +/// Syntax only. A valid pattern has at least one label that is exactly "*", +/// every other label is 1–63 bytes with no '*' inside it, and the whole +/// pattern is at most 253 bytes over at most `max_labels` labels. +pub fn validate(pattern: []const u8) PatternError!void; + +/// `domain` is already normalized: lowercase, no trailing dot. `pattern` is +/// lowercase. Each "*" label matches ONE OR MORE labels (resolved ambiguity 2). +/// Allocation-free; the backtracking is bounded by `max_labels` on both sides. +pub fn matches(pattern: []const u8, domain: []const u8) bool; +``` + +Implementation shape: split both sides into label slices in two fixed `[max_labels][]const u8` +stack arrays, then run the classic two-pointer glob match with a single backtrack point (a `*` +consumes one label and may extend). No recursion, no allocation. A pattern with more than +`max_labels` labels cannot occur because `validate` rejects it and the matcher only holds validated +patterns; `matches` asserts the bound. + +### S1.6 Tests (in-file) + +`parsers.zig`: +- `detectFormat` on a hosts fixture, a domains fixture, an ABP fixture, and a file whose first 64 + lines are all comments (→ `.domains`). +- `parseLine` dispatches to the format's parser (one case each). + +`parser_hosts.zig`: +- `0.0.0.0 ads.example.com` → `.domain`, `"ads.example.com"`. +- `127.0.0.1 a.example.com b.example.com` → `.domain` whose `text` spans both names. +- `::1 ip6-localhost ip6-loopback` → `.domain` with both names (the compiler drops single-label + names, S2.3 — this test asserts the parser does not silently swallow them). +- `0.0.0.0 ads.example.com # tracker` → the comment is gone. +- `# whole line`, ``, ` ` → `.ignore`. +- `/ads\d+/` → `.regex`. +- `example.com` (no IP) → `.domain`. + +`parser_domains.zig`: bare name; `! comment`; `# comment`; inline `example.com # x`; `/re/` → +`.regex`; `0.0.0.0 example.com` → `.unsupported`. + +`parser_abp.zig`: one named test per row of the S1.4 table. + +`wildcard.zig`: +- `validate` accepts `*.doubleclick.net` and `ads.*.example.com`; one named test per `PatternError` + member: `example.com` → `NoWildcard`, `a*b.com` → `PartialWildcardLabel`, `a..b` → `EmptyLabel`, a + 64-byte label → `LabelTooLong`, a 300-byte pattern → `PatternTooLong`, a 200-label pattern → + `TooManyLabels`. +- `matches("*.doubleclick.net", "a.doubleclick.net")` true; + `("*.doubleclick.net", "a.b.doubleclick.net")` true (ambiguity 2); + `("*.doubleclick.net", "doubleclick.net")` false; + `("ads.*.example.com", "ads.eu.example.com")` true; + `("ads.*.example.com", "ads.eu.west.example.com")` true; + `("ads.*.example.com", "ads.example.com")` false; + `("*.example.com", "example.com.evil.net")` false. +- A pathological pattern `*.*.*.*.*.*.*.*.example.com` against a 100-label domain terminates (assert + it returns, which is the runnable form of "the backtracking is bounded"). + +### S1.7 Acceptance criteria + +- [ ] `zig fmt --check` and `zig ast-check` clean on all five files. +- [ ] `grep -n "@import" src/filter/parsers.zig src/filter/parser_*.zig src/filter/wildcard.zig` + shows only `std` and sibling files in `src/filter/`. +- [ ] `PatternError` includes `PartialWildcardLabel` and every member is produced by a named test. +- [ ] No allocator, no `std.Io` value, no clock appears in any of the five files. +- [ ] Every bullet in S1.6 exists as a named test. + +--- + +## Session S2: `src/filter/domain_set.zig`, `src/filter/compiler.zig` + +Pure: an allocator and reader/writer interfaces, no `Io`. `compiler.zig` may import +`../dns/name.zig` (it decides domain validity); `domain_set.zig` imports only `std`. + +### S2.1 `domain_set.zig` + +```zig +pub const DomainSet = struct { + /// Length-prefixed lowercase names, back to back: [len: u8][bytes]… + arena: []const u8, + /// Open-addressed table of offsets into `arena`; `empty` marks a hole. + /// Length is always a power of two. + index: []const u32, + count: u32, + seed: u64, + + pub const empty_slot: u32 = std.math.maxInt(u32); + pub const max_count: u32 = 4_000_000; + pub const max_arena_bytes: usize = 1 << 31; + + pub const Error = error{ OutOfMemory, TooManyDomains, SetTooLarge, NotSorted, NotLowercase }; + + /// Builds from a compiled body: LF-separated, lowercase, sorted ascending, + /// deduplicated, every line 1–255 bytes. The sortedness is VERIFIED, not + /// assumed — a hand-edited or truncated file must fail loudly at load + /// (`error.NotSorted`) rather than produce a set that silently misses + /// entries. Verification is one comparison per line and costs nothing. + /// + /// `seed` randomizes the hash. Query names are attacker-supplied, so a + /// fixed seed would make probe-chain flooding computable offline; the seed + /// arrives as a parameter so this file stays free of entropy sources. + pub fn build(gpa: std.mem.Allocator, body: []const u8, seed: u64) Error!DomainSet; + + /// An empty set that owns nothing. `contains` on it is always false. + pub const empty: DomainSet; + + pub fn deinit(self: *DomainSet, gpa: std.mem.Allocator) void; + + /// `domain` must be normalized (lowercase, no trailing dot). Allocation-free. + pub fn contains(self: *const DomainSet, domain: []const u8) bool; + + /// Bytes held, for the memory report in `Snapshot.memoryBytes`. + pub fn memoryBytes(self: *const DomainSet) usize; +}; +``` + +Rules: + +- Capacity is the smallest power of two ≥ `count * 4 / 3`, minimum 16. Linear probing, insertion in + file order, no tombstones (the set is immutable after `build`). +- Hash is `std.hash.Wyhash.hash(self.seed, domain)`; the slot is `hash & (index.len - 1)`. A probe + compares the full stored bytes with `std.mem.eql` before reporting a hit. **There is no path in + which a hash collision produces a match.** +- `build` makes exactly two allocations (arena, index) and frees both on any error. +- A body of length 0 yields `empty`. +- `count > max_count` → `error.TooManyDomains`; body longer than `max_arena_bytes` → + `error.SetTooLarge` (the index stores `u32` offsets and must not silently truncate). + +Tests (in-file): +- Build from a small sorted body; `contains` true for every member, false for four non-members + including a prefix, a suffix, an uppercase spelling and the empty string. +- A body that is not sorted → `error.NotSorted`; a body with an uppercase byte → `error.NotLowercase`; + a duplicate line → `error.NotSorted` (equal is not ascending, and a duplicate means the compiler + broke its contract). +- Two sets built from the same body with different seeds answer `contains` identically over a + 50-name probe list — the property that matters, stated as a test. +- 10,000 generated names round-trip; `memoryBytes` is within 2× of the naive `body.len` bound. +- `build` under `std.testing.checkAllAllocationFailures` leaks nothing. +- `empty.contains("x") == false` and `deinit` on `empty` is a no-op. + +### S2.2 `compiler.zig` — signatures + +```zig +pub const max_domains: u32 = 2_000_000; +pub const max_line_len: usize = 4096; + +pub const Counts = struct { + domains: u32 = 0, + wildcards: u32 = 0, + skipped_regex: u32 = 0, + skipped_unsupported: u32 = 0, + /// Not a valid domain name (`dns.name.fromText` rejected it, a non-ASCII + /// byte, or fewer than two labels). + invalid: u32 = 0, + /// Lines longer than `max_line_len`, skipped whole. + long_lines: u32 = 0, + /// Duplicates removed by the sort/unique pass. + duplicates: u32 = 0, +}; + +pub const Result = struct { + counts: Counts, + /// Lowercase hex sha256 over the `.list` body followed by the `.wild` body. + checksum: [64]u8, +}; + +pub const Error = error{ OutOfMemory, TooManyDomains, ReadFailed, WriteFailed }; + +/// Reads `r` to end of stream, writes the two compiled bodies. Nothing else is +/// written — headers belong to the caller (S8.3), so this function is a pure +/// function of (bytes, format) and is tested by comparing two runs. +pub fn compile( + gpa: std.mem.Allocator, + r: *std.Io.Reader, + format: parsers.Format, + list_w: *std.Io.Writer, + wild_w: *std.Io.Writer, +) Error!Result; +``` + +### S2.3 `compile` algorithm + +1. Two `std.ArrayList(u8)` arenas (`list_bytes`, `wild_bytes`) and two `std.ArrayList(u32)` offset + lists, all from `gpa`. All four are freed before returning, on every path. +2. Loop with `r.takeDelimiter('\n')`: + - `null` → end. + - `error.StreamTooLong` → `counts.long_lines += 1`, then + `r.discardDelimiterInclusive('\n')` (tolerating `error.EndOfStream` as end of input) and + continue. **Not doing this discard is an infinite loop** — `takeDelimiter` leaves the stream + unmodified on `StreamTooLong` (verified, Reader.zig:885). + - Strip a trailing `\r` (CRLF files are common). +3. `parsers.parseLine(format, line)`; `.ignore` continues, `.regex` and `.unsupported` increment + their counters and continue. +4. For `.domain`, split `line.text` on ASCII whitespace and process each field; for `.wildcard`, + process `line.text` as one field with `covers_apex` remembered. +5. Per candidate field, in order: + a. A leading `*.` makes it a wildcard candidate over the remainder; a `*` anywhere else makes it + `invalid` (blocklist entries are suffixes, not patterns — patterns belong to the `rules` + table). + b. Strip one trailing `.`. + c. Reject any byte ≥ 0x80 or any ASCII control byte → `invalid`. + d. Lowercase ASCII into a `[types.max_name_len]u8` stack buffer. + e. `dns.name.fromText` must accept it → else `invalid`. + f. Fewer than two labels → `invalid`. This is what keeps `localhost`, `local`, `broadcasthost` + and the `ip6-*` names that every hosts list carries from black-holing the loopback names of + every client on the LAN. It is the single highest-consequence rule in this file. + g. Append to `list_bytes` (domain) or `wild_bytes` (wildcard), recording the offset. A + `.wildcard` with `covers_apex` appends to **both**. + h. Either list exceeding `max_domains` entries → `error.TooManyDomains`. +6. Sort each offset list with `std.mem.sortUnstableContext` comparing the stored names bytewise, then + write unique entries to the matching writer, each followed by `\n`, counting duplicates. +7. Hash both bodies as they are written (`std.crypto.hash.sha2.Sha256` streaming: `update` per + emitted line, `final`, `std.fmt.bytesToHex(digest, .lower)`), `.list` first then `.wild`. +8. Return `Result`. `counts.domains` and `counts.wildcards` are the **written, deduplicated** counts — + they are what `blocklist_sources.domain_count` and `wildcard_count` store, and what the UI shows. + +### S2.4 Tests (in-file, `std.Io.Reader.fixed` / `std.Io.Writer.Allocating`) + +- Hosts fixture with sink IPs, comments, a duplicate, `localhost`, an over-long line and a regex line: + every counter has the expected value and the body is exactly the expected sorted text. +- Domains fixture and ABP fixture likewise, the ABP one asserting `||x.com^` produces `x.com` in + **both** bodies. +- **Determinism**: compile the same fixture twice → identical bodies and identical checksums. +- **Order independence**: compile a shuffled permutation of the same fixture → identical bodies. +- Uppercase input compiles to lowercase output; a trailing-dot input compiles without the dot. +- A candidate with a non-ASCII byte, a single-label candidate, and a `a*b.com` candidate each land in + `counts.invalid` and appear in neither body. +- An input of 3,000 lines with a 5,000-byte line in the middle: `long_lines == 1`, the surrounding + lines are all present, and the call **terminates** (the `StreamTooLong` regression test). +- An empty input produces two empty bodies and the sha256 of the empty string. +- `compile` under `checkAllAllocationFailures` leaks nothing. + +### S2.5 Acceptance criteria + +- [ ] `zig fmt --check` and `zig ast-check` clean on both files. +- [ ] Neither file takes a `std.Io` value, opens a file, or reads a clock. +- [ ] `domain_set.zig` imports only `std`. +- [ ] `contains` compares full bytes on every probe; no code path returns a hit from a hash + comparison alone. +- [ ] The `StreamTooLong` discard is implemented and covered by the 3,000-line test. +- [ ] The two-label minimum is implemented and covered by a `localhost` test. +- [ ] Determinism and order-independence tests pass. + +--- + +## Session S3: `src/filter/safesearch.zig`, `src/filter/response.zig` + +Pure. `safesearch.zig` imports `std` and `../dns/name.zig`; `response.zig` imports `std`, `../dns/*` +and `../config/model.zig`. + +### S3.1 `safesearch.zig` + +```zig +pub const Entry = struct { domain: []const u8, target: []const u8 }; + +/// PLAN §7.4. Sorted by `domain`, so the table is searchable and diffable, and +/// asserted sorted by a comptime block. Every target is a name the operator's +/// upstream can resolve; nxdns never hardcodes an address. +pub const table = [_]Entry{ + .{ .domain = "bing.com", .target = "strict.bing.com" }, + .{ .domain = "duckduckgo.com", .target = "safe.duckduckgo.com" }, + .{ .domain = "google.com", .target = "forcesafesearch.google.com" }, + .{ .domain = "pixabay.com", .target = "safesearch.pixabay.com" }, + .{ .domain = "www.bing.com", .target = "strict.bing.com" }, + .{ .domain = "www.duckduckgo.com", .target = "safe.duckduckgo.com" }, + .{ .domain = "www.google.com", .target = "forcesafesearch.google.com" }, + .{ .domain = "www.youtube.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "youtube.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "m.youtube.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "youtubei.googleapis.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "youtube.googleapis.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "www.youtube-nocookie.com", .target = "restrictmoderate.youtube.com" }, +}; + +/// Exact match on a normalized name (lowercase, no trailing dot). Google's +/// country domains (`google.de`, …) are NOT enumerated: the list is unbounded, +/// it goes stale, and `forcesafesearch.google.com` is the documented target for +/// every one of them — an operator who needs a country domain adds a rule. +/// State that reasoning in the file. +pub fn lookup(domain: []const u8) ?[]const u8; + +/// The rewritten question name for a matched query, as a `dns.name.Name`. +/// Applying it — sending the rewritten question upstream and prefixing the +/// answer with a CNAME from the original name to `target` — is the handler's +/// job (Phase 7). Nothing here builds a response. +pub fn rewrite(domain: []const u8) ?name.Name; +``` + +The table is asserted sorted and duplicate-free in a `comptime` block, and `lookup` is a binary +search. A test walks every entry and asserts `dns.name.fromText` accepts both sides. + +### S3.2 `response.zig` — blocked-response synthesis (PLAN §6.2) + +```zig +pub const Options = struct { + mode: model.BlockResponse, // .zero | .nxdomain + ttl: u32, // blocking.ttl +}; + +pub const Error = packet.ResponseBuilder.Error; + +/// Writes a blocked reply for `q` into `buf` and returns a prefix of it. +/// +/// `.zero`: A → 0.0.0.0, AAAA → ::, every other qtype → NOERROR with no answer +/// (NODATA). Synthesizing an address for a qtype that does not carry one is +/// not possible, and NXDOMAIN for, say, an MX query would tell the client the +/// name does not exist while an A query says it does. +/// `.nxdomain`: RCODE = NXDOMAIN, no answer, for every qtype. +/// +/// No SOA is placed in the authority section: nxdns is not authoritative for +/// the name and a synthesized SOA would give resolvers a negative-caching TTL +/// nxdns cannot honour. Document that in the file. +/// +/// `request_opt` echoes EDNS exactly as `handler.zig` does: when the query +/// carried an OPT record, the reply carries one with the same payload size and +/// the DO bit passed through. +pub fn writeBlocked( + buf: []u8, + request: header.Header, + q: question.Question, + request_opt: ?edns.OptRecord, + do_bit: bool, + options: Options, +) Error![]u8; +``` + +Only the question's class `IN` is answered with addresses; a non-`IN` class takes the NODATA path. + +Tests (in-file): for each of the two modes × {A, AAAA, MX, HTTPS} × {with OPT, without OPT}, parse +the produced message back with `packet.parse` and assert the rcode, ancount, arcount, the echoed +question, the rdata bytes and the TTL. One test asserts a `buf` too small returns +`error.WriteFailed` rather than truncating. + +### S3.3 Acceptance criteria + +- [ ] `zig fmt --check` and `zig ast-check` clean on both files. +- [ ] The safe-search table is comptime-asserted sorted and duplicate-free. +- [ ] Neither file takes a `std.Io` value or an allocator. +- [ ] Every blocked-response case in S3.2 is a named test that re-parses the output. + +--- + +## Session S4: `src/local/records.zig`, `src/local/forward_zones.zig` + +Pure. Both build immutable lookup structures from the model types the repositories already return. + +### S4.1 `records.zig` + +```zig +pub const Value = union(enum) { a: [4]u8, aaaa: [16]u8, cname: name.Name }; + +pub const Record = struct { + /// Normalized owner name: lowercase, no trailing dot. + owner: []const u8, + value: Value, + ttl: u32, +}; + +pub const Error = error{ OutOfMemory, BadRecordValue, BadRecordName, TooManyRecords }; +pub const max_records: usize = 10_000; + +pub const Records = struct { + /// Arena-owned, sorted by (owner, rtype) so lookup is a binary search and + /// the answer order for one name is stable across restarts. + items: []const Record, + + pub const empty: Records; + + /// `gpa` owns the result; `deinit` frees it. Values are parsed here, once: + /// an `a` value through `NetAddress.parse` (must be `.ip4`), `aaaa` (`.ip6`), + /// a `cname` through `dns.name.fromText`. A bad value is an error, not a + /// skipped row — `validate.zig` already rejects these, so reaching one here + /// means the database was edited behind nxdns's back and silence would make + /// a record vanish with no signal. + pub fn build(gpa: std.mem.Allocator, rows: []const model.LocalRecord) Error!Records; + pub fn deinit(self: *Records, gpa: std.mem.Allocator) void; + + /// All records for `domain` whose type matches `qtype`, plus any CNAME + /// (RFC 1034 §3.6.2: a CNAME answers every qtype). Empty slice = no local + /// record. Allocation-free. + pub fn lookup(self: *const Records, domain: []const u8, qtype: types.Type) []const Record; + + /// True when the name has any local record of any type. The handler needs + /// this to answer NODATA instead of forwarding a name nxdns owns. + pub fn hasName(self: *const Records, domain: []const u8) bool; +}; + +/// Writes `records` as answers into a builder the caller has already +/// initialized with the request header and question. Mechanism only. +pub fn writeAnswers( + b: *packet.ResponseBuilder, + owner: name.Name, + records: []const Record, +) packet.ResponseBuilder.Error!void; +``` + +Local records are group-independent (PLAN §6.4) and are matched **before** filtering. Resolving a +CNAME target through the pipeline is Phase 7; `writeAnswers` emits the CNAME record and stops. + +### S4.2 `forward_zones.zig` + +```zig +pub const Zone = struct { + /// Normalized: lowercase, no trailing dot. + zone: []const u8, + resolver: validate.Resolver, +}; + +pub const Error = error{ OutOfMemory, BadZone, BadResolver, TooManyZones }; +pub const max_zones: usize = 1_000; + +pub const Zones = struct { + /// Arena-owned, sorted by descending label count then by name, so the first + /// match found by a forward scan is the longest one. + items: []const Zone, + + pub const empty: Zones; + + pub fn build(gpa: std.mem.Allocator, rows: []const model.ForwardZone) Error!Zones; + pub fn deinit(self: *Zones, gpa: std.mem.Allocator) void; + + /// Longest-suffix match on LABEL boundaries: `lan.home` matches `nas.lan.home` + /// and `lan.home`, and does NOT match `notlan.home`. `10.in-addr.arpa` + /// matches every reverse name under it. Allocation-free. + pub fn match(self: *const Zones, domain: []const u8) ?*const Zone; +}; +``` + +`build` parses each `resolver` string with `validate.parseResolver` — the function whose doc comment +already names this file as its Phase 5 importer. There is no second resolver parser. + +### S4.3 Tests (in-file) + +`records.zig`: +- Build from A, AAAA and CNAME rows; `lookup("nas.lan", .a)` returns the A record; `.aaaa` returns + the AAAA; `.mx` returns nothing; with a CNAME present, every qtype returns the CNAME. +- Two A records for one name both come back, in a stable order across two builds. +- Uppercase and trailing-dot owner names normalize to the same key. +- A bad `a` value (`"::1"`), a bad `aaaa` value, a bad CNAME target and an unparseable owner each + produce their typed error. +- `writeAnswers` output re-parses with the expected ancount, types, TTLs and rdata. +- `build` under `checkAllAllocationFailures`. + +`forward_zones.zig`: +- `lan.home` matches `nas.lan.home`, `a.b.lan.home` and `lan.home`; does not match `notlan.home`, + `home` or `lan.home.evil.net`. +- With both `home` and `lan.home` configured, `nas.lan.home` matches `lan.home` (longest wins). +- `10.in-addr.arpa` matches `5.4.3.10.in-addr.arpa`. +- A bad resolver URL → `error.BadResolver`; a bad zone → `error.BadZone`. +- `build` under `checkAllAllocationFailures`. + +### S4.4 Acceptance criteria + +- [ ] `zig fmt --check` and `zig ast-check` clean on both files. +- [ ] Neither file takes a `std.Io` value or reads a clock. +- [ ] `forward_zones.zig` calls `validate.parseResolver`; no second resolver parser exists + (`grep -rn "udp://" src/local/` shows only test text). +- [ ] Suffix matching is on label boundaries, proven by the `notlan.home` test. +- [ ] Every bullet in S4.3 exists as a named test. + +--- + +## Session S5: `src/filter/rules.zig`, `src/filter/matcher.zig` + +Pure, and the heart of the milestone. Depends on S1's `wildcard.zig` and S2's `domain_set.zig`. + +### S5.1 `rules.zig` — one group's explicit rules + +```zig +pub const Error = error{ OutOfMemory, BadPattern, TooManyWildcards } || domain_set.DomainSet.Error; +pub const max_wildcards_per_group: usize = 4096; + +pub const RuleSet = struct { + exact_allow: domain_set.DomainSet, + exact_block: domain_set.DomainSet, + /// Arena-owned, normalized, sorted for determinism. Scanned linearly: + /// these are operator-authored and few, and a linear scan over ≤ 4096 + /// short patterns is far cheaper than any index that would have to be + /// rebuilt on every swap. + wildcard_allow: []const []const u8, + wildcard_block: []const []const u8, + + pub const empty: RuleSet; + + /// `rows` are this group's rules only (already filtered by the caller). + /// Patterns are normalized (lowercase, trailing dot stripped) and validated: + /// `.exact` through `dns.name.fromText`, `.wildcard` through + /// `wildcard.validate`. An invalid pattern is `error.BadPattern` — the + /// database went through `validate.zig`, so an invalid one here means the + /// rows were edited underneath nxdns. + pub fn build(gpa: std.mem.Allocator, rows: []const model.Rule, seed: u64) Error!RuleSet; + pub fn deinit(self: *RuleSet, gpa: std.mem.Allocator) void; + pub fn memoryBytes(self: *const RuleSet) usize; +}; +``` + +`build` must be given only one group's rows; splitting `listRules` output by group is the caller's +job (S5.3's `Snapshot.build`), because only it holds the group table. + +### S5.2 `matcher.zig` — normalization, candidates, decision + +```zig +pub const Reason = enum { + none, + rule_allow_exact, + rule_block_exact, + rule_allow_wildcard, + rule_block_wildcard, + blocklist_domain, + blocklist_wildcard, +}; + +pub const Decision = struct { + blocked: bool, + reason: Reason, + /// The candidate (or pattern) that decided it; borrowed from the caller's + /// normalized buffer or from the snapshot. "" when `reason == .none`. + matched: []const u8, + /// `.blocklist_*` only: index into `Snapshot.sources`, for the block reason + /// the query log (Phase 6) and the UI (Phase 8) will show. + source: ?u32 = null, +}; + +/// Lowercase ASCII, trailing dot stripped, written into `buf`. Returns a slice +/// of `buf`. The root name normalizes to "". +pub fn normalize(qname: name.Name, buf: *[types.max_name_len]u8) []const u8; + +/// Full name, then each parent, ending at the last two-label suffix. The TLD +/// alone is NOT a candidate: a rule or list entry on `com` is a configuration +/// mistake that would take the whole internet with it, and refusing to walk +/// that far costs nothing real. +pub const Candidates = struct { + rest: []const u8, + pub fn init(domain: []const u8) Candidates; + pub fn next(self: *Candidates) ?[]const u8; +}; +``` + +### S5.3 `matcher.zig` — the snapshot + +```zig +pub const SourceSets = struct { + /// Row id, so the manager can map a decision back to `blocklist_sources`. + id: i64, + /// Borrowed from the snapshot arena; the source's display name for the UI. + name: []const u8, + domains: domain_set.DomainSet, + wildcards: domain_set.DomainSet, +}; + +pub const Group = struct { + id: i64, + name: []const u8, + safe_search: bool, + rules: rules.RuleSet, + /// Indices into `Snapshot.sources`, ascending, deduplicated. + sources: []const u32, +}; + +pub const ClientEntry = struct { key: address.NetAddress.Key, group: u32 }; +pub const PrefixEntry = struct { prefix: address.Prefix, group: u32, priority: i32 }; + +pub const Snapshot = struct { + arena: std.heap.ArenaAllocator, + groups: []Group, + sources: []SourceSets, + clients: []ClientEntry, + prefixes: []PrefixEntry, + /// Index into `groups` of the group named "default". Always valid: + /// `build` returns `error.MissingDefaultGroup` otherwise. + default_group: u32, + /// Monotonic, assigned by the manager. Logged on every swap so an operator + /// can tell which generation answered a query. + generation: u64, + + pub const Input = struct { + groups: []const model.Group, + group_sources: []const model.GroupSource, + sources: []const model.BlocklistSource, + rules: []const model.Rule, + clients: []const model.Client, + prefixes: []const model.ClientPrefix, + /// One entry per `sources[i]`, in the same order: the compiled bodies + /// already read from disk with their headers stripped. An enabled source + /// whose bodies are absent is `error.MissingCompiledSource` — a silently + /// unenforced blocklist is exactly the failure PLAN §1.3 exists to + /// prevent. + compiled: []const Compiled, + seed: u64, + generation: u64, + }; + + pub const Compiled = struct { list_body: []const u8, wild_body: []const u8 }; + + pub const Error = error{ + OutOfMemory, MissingDefaultGroup, UnknownGroup, UnknownSource, + MissingCompiledSource, BadClientIp, BadClientPrefix, + } || rules.Error; + + /// Builds an immutable snapshot. Every string is copied into `arena`, so the + /// caller may free the repository lists immediately afterwards. Disabled + /// sources are skipped entirely — they cost no memory. + pub fn build(gpa: std.mem.Allocator, input: Input) Error!Snapshot; + pub fn deinit(self: *Snapshot) void; + + /// PLAN §3.10 precedence, allow wins at equal specificity: + /// 1. exact/parent allow rules 2. exact/parent block rules + /// 3. wildcard allow rules 4. wildcard block rules + /// 5. blocklist domains 6. blocklist wildcards + /// `domain` is normalized (`normalize`). No allocation, no lock, no clock. + pub fn evaluate(self: *const Snapshot, group: u32, domain: []const u8) Decision; + + /// PLAN §7.2 matching half: exact client row, else longest-prefix match + /// (ties broken by longer prefix then higher `priority`), else the default + /// group. Auto-materialization is Phase 7 (resolved ambiguity 4). + pub fn groupForClient(self: *const Snapshot, addr: address.NetAddress) u32; + + pub fn groupIndexById(self: *const Snapshot, id: i64) ?u32; + pub fn groupIndexByName(self: *const Snapshot, name_text: []const u8) ?u32; + pub fn safeSearch(self: *const Snapshot, group: u32) bool; + pub fn memoryBytes(self: *const Snapshot) usize; +}; +``` + +`evaluate` order is the specification, and it is level-by-level over the **whole candidate chain**, +not candidate-by-candidate over the levels: level 1 is checked against every candidate before level 2 +is checked against any. That is what makes "an allow rule on the parent beats a block rule on the +child" true, which is the behaviour operators expect from an allow list. + +Levels 5 and 6 iterate the group's sources in ascending index order and return the first hit, so the +reported `source` is stable for a given snapshot. Level 5 tests **only the full name** and level 6 +tests **only proper parents** (resolved ambiguity 1). + +### S5.4 Tests (in-file) + +Precedence — one named test per row, each asserting `blocked`, `reason` and `matched`: + +| Configuration | Query | Expected | +|---|---|---| +| block rule `ads.example.com`, nothing else | `ads.example.com` | blocked, `rule_block_exact` | +| block rule `example.com` | `ads.example.com` | blocked, `rule_block_exact`, matched `example.com` (parent walk) | +| block rule `example.com`, allow rule `ads.example.com` | `ads.example.com` | allowed, `rule_allow_exact` | +| block rule `ads.example.com`, allow rule `example.com` | `ads.example.com` | allowed (allow level runs first over the whole chain) | +| allow rule `*.example.com`, block rule `ads.example.com` | `ads.example.com` | blocked (exact block, level 2, beats wildcard allow, level 3) | +| allow wildcard `*.example.com`, block wildcard `*.example.com` | `a.example.com` | allowed (tie → allow wins) | +| list entry `tracker.net` | `tracker.net` | blocked, `blocklist_domain` | +| list entry `tracker.net` | `sub.tracker.net` | **allowed** (ambiguity 1) | +| wild entry `tracker.net` | `sub.tracker.net` | blocked, `blocklist_wildcard` | +| wild entry `tracker.net` | `tracker.net` | allowed (a `.wild` entry covers proper subdomains only) | +| allow rule `sub.tracker.net`, wild entry `tracker.net` | `sub.tracker.net` | allowed | +| nothing configured | `example.com` | allowed, `reason == .none` | +| source assigned to group A only | same query in group B | allowed in B, blocked in A | +| disabled source | its entry | allowed (disabled sources are not loaded) | + +Also: +- `normalize` lowercases, strips one trailing dot, and returns `""` for the root. +- `Candidates` over `a.b.example.com` yields exactly `a.b.example.com`, `b.example.com`, + `example.com` — and not `com`. +- `groupForClient`: exact IPv4 hit; exact IPv6 hit through the canonical key; `/24` prefix hit; + overlapping `/16` and `/24` → the `/24` wins; equal-length prefixes → higher priority wins; + no match → default group. +- `Snapshot.build` without a `default` group → `error.MissingDefaultGroup`; with a `group_sources` + row naming an unknown source → `error.UnknownSource`; with an enabled source whose `compiled` entry + is absent → `error.MissingCompiledSource`. +- Two snapshots built from the same input with different seeds produce identical decisions over a + 40-query table. +- `Snapshot.build` under `checkAllAllocationFailures` leaks nothing. +- `memoryBytes` on a snapshot with 10,000 synthetic domains stays under the S§Memory-budget bound + (`count * (avg_len + 1) * 2 + index`), asserted as an inequality so it is a real regression guard. + +### S5.5 Acceptance criteria + +- [ ] `zig fmt --check` and `zig ast-check` clean on both files. +- [ ] Neither file takes a `std.Io` value, opens a file, or reads a clock. +- [ ] `evaluate` performs no allocation (no allocator is reachable from its signature). +- [ ] Every row of the S5.4 precedence table exists as a named test. +- [ ] The candidate chain stops before the TLD, proven by a test. +- [ ] `Snapshot.build` copies every string into its arena; a test frees the input lists before + calling `evaluate`. + +--- + +## Session S6: `src/filter/fetcher.zig` + +The first of the three `std.Io` files. Downloads one source; knows nothing about parsing, files or +the database. + +### S6.1 Signatures + +```zig +pub const max_body_bytes: usize = 64 * 1024 * 1024; +pub const redirect_buffer_len: usize = 8192; // RFC 9110 recommendation (Client.zig:1128) +pub const min_transfer_buf: usize = 16 * 1024; + +pub const Error = error{ + BadUrl, ConnectFailed, TlsFailed, SendFailed, ReceiveFailed, HttpStatus, + BodyTooLarge, Timeout, Canceled, OutOfMemory, SystemResources, Unexpected, +}; + +pub const Result = struct { bytes_read: u64, status: std.http.Status }; + +pub const Fetcher = struct { + http: *std.http.Client, // caller-owned, shared, pools connections + transfer_buf: []u8, // caller-owned, ≥ min_transfer_buf + redirect_buf: []u8, // caller-owned, ≥ redirect_buffer_len + + /// GETs `url` and streams the body into `w`. Nothing is buffered whole: + /// a 64 MB list must not become a 64 MB allocation, and the caller is + /// writing into a temporary file anyway. + /// + /// `budget` bounds the WHOLE exchange. `std.http.Client` has no per-request + /// deadline (verified), so the caller runs this under `io.concurrent` and + /// cancels the future — the same pattern `tls_client_integration_test.zig` + /// established. This function therefore takes no timeout parameter and + /// simply propagates `error.Canceled`; S8 owns the deadline. + pub fn fetch(self: *Fetcher, io: std.Io, url: []const u8, w: *std.Io.Writer) Error!Result; +}; +``` + +Rules: + +- `std.Uri.parse` failure → `error.BadUrl`. A scheme other than `http`/`https` → `error.BadUrl`. +- `accept_encoding` is overridden to identity (resolved ambiguity 6), matching `doh_client.zig`. +- `redirect_behavior` keeps the stdlib default of 3; `receiveHead(self.redirect_buf)` follows them. + `error.TooManyHttpRedirects` maps to `error.HttpStatus`. +- Status other than `.ok` → `error.HttpStatus`. An error return carries no `Result`, so the fetcher + stores the numeric status in a `last_status: ?std.http.Status` field, cleared at the start of each + `fetch` and set from every response head; the caller reads it after `error.HttpStatus`. `content-type` is **not** checked: blocklists are served as `text/plain`, + `application/octet-stream`, `text/html` and worse, and the compiler's invalid-line counters are the + honest signal about content. +- The body is copied to `w` in `transfer_buf`-sized chunks with a running total; exceeding + `max_body_bytes` → `error.BodyTooLarge` (and the caller discards its temporary file). +- Error mapping reuses `transport.mapLocal` first, then classifies by phase, exactly as + `doh_client.zig`'s `mapError` does. Do not invent a second classification vocabulary. + +### S6.2 Tests + +In-file (no sockets): URL rejection table (`ftp://x`, `x`, `https://`), and a `Fetcher` value +constructed against an undefined `std.http.Client` proving the URL check precedes any client use +(the `doh_client.zig` pattern). + +Everything real belongs to S9: a loopback `std.http.Server` serving a fixture, a redirect chain, a +404, and a body that exceeds a lowered cap. + +### S6.3 Acceptance criteria + +- [ ] `zig fmt --check` and `zig ast-check` clean. +- [ ] `accept_encoding` is pinned to identity and the reason is commented. +- [ ] The body is streamed; no `allocRemaining`, no whole-body allocation anywhere in the file. +- [ ] No timeout parameter and no sleep: the deadline is the caller's (S8.4). +- [ ] `transport.mapLocal` is used before any phase classification. + +--- + +## Session S7: `src/local/forward_client.zig` + +Conditional forwarding's transport (resolved ambiguity 5). Implements `transport.Client` so the +Phase 7 handler treats a forward zone exactly like any other exchange. + +### S7.1 Signatures + +```zig +pub const ForwardClient = struct { + resolver: validate.Resolver, + /// Caller-owned scratch for the TCP length-prefixed path. + frame_buf: []u8, + read_timeout: std.Io.Clock.Duration, + stats: Stats = .{}, + + pub const Stats = struct { + queries: u64 = 0, + udp_truncated: u64 = 0, // TC=1 → retried over TCP + failures: u64 = 0, + }; + + pub fn init(resolver: validate.Resolver, frame_buf: []u8, read_timeout: std.Io.Clock.Duration) ForwardClient; + pub fn client(self: *ForwardClient) transport.Client; + + /// UDP: send, `receiveTimeout`, validate. TC=1 → retry over TCP with the + /// RFC 1035 §4.2.2 two-byte length prefix. `.tcp` resolvers skip straight + /// to the TCP path. + pub fn exchange(self: *ForwardClient, io: std.Io, query: []const u8, response_buf: []u8) + transport.ExchangeError![]u8; +}; +``` + +Rules: + +- Every response passes `transport.validateResponse(query, response)` before it is returned — a + forward zone points at LAN infrastructure, which is not a reason to trust its framing. +- UDP uses `Socket.receiveTimeout` (verified implemented on the POSIX Threaded backend, milestone 3). + A datagram from an address other than the resolver's is **discarded and counted**, and the receive + is retried within the remaining budget. +- TCP has no read timeout in 0.16.0: run the exchange under `io.concurrent` and cancel the loser + against a `std.Io.Clock.Duration` sleep, exactly as milestone 3 does. Never set + `ConnectOptions.timeout` — the Threaded backend panics (Threaded.zig:12077). +- Failures map through `transport.mapLocal` first, then to the phase's `PeerFault`. Every socket is + closed on every path. +- No health tracking and no backoff: `upstream/health.zig` and `pool.zig` model the *upstream* pool, + and a forward zone has exactly one designated resolver with no failover partner. Say so in a + comment so its absence does not read as an oversight. + +### S7.2 Tests + +In-file: the `transport.Client` vtable instantiation check (milestone 3's pattern) and the stats +struct defaults. Everything with a socket is S9's: a loopback UDP responder answering an A query, a +loopback responder setting TC=1 followed by a TCP responder returning the full answer, a silent +responder proving the timeout produces `error.Timeout`, and a responder answering with a mismatched +ID proving `error.ResponseMismatch`. + +### S7.3 Acceptance criteria + +- [ ] `zig fmt --check` and `zig ast-check` clean. +- [ ] `ConnectOptions.timeout` appears nowhere. +- [ ] Every response goes through `transport.validateResponse`. +- [ ] A datagram from a foreign source address is counted and discarded, not returned. +- [ ] The file implements `transport.Client` and defines no second client interface. + +--- + +## Session S8: `src/filter/manager.zig` + +The `BlocklistManager` of PLAN §4: compiled-file layout, refresh, metadata, snapshot build and the +RCU swap. The only file in this milestone that touches both the database and the filesystem. + +### S8.1 The two repository additions this session owns + +`blocklist_sources` carries counters and a checksum that milestone 4 deliberately excluded from the +config model (runtime facts, not configuration). Refresh has to write them, so `sources_repo.zig` +gains exactly two functions — additive, no signature of an existing function changes: + +```zig +pub const SourceRow = struct { + id: i64, + url: []const u8, + name: []const u8, + enabled: bool, + last_updated: ?i64, + domain_count: i64, + wildcard_count: i64, + skipped_regex_count: i64, + checksum: ?[]const u8, +}; + +/// Every source with its row id and runtime columns. Strings are heap copies. +/// ORDER BY url (matching `listBlocklistSources`, milestone 4 §S4.2). +pub fn listSourceRows(database: *db.Db, gpa: Allocator) db.Error!std.ArrayList(SourceRow); +pub fn freeSourceRows(gpa: Allocator, items: []const SourceRow) void; + +/// Writes the runtime columns for one source after a successful compile. +pub fn updateSourceStats(database: *db.Db, id: i64, stats: SourceStats) db.Error!void; +``` + +This is the one exception to "no session edits a milestone 1–4 file", and it is scoped to appending +two functions and their tests to `sources_repo.zig`. S8 owns that file for this milestone. No +schema change and no migration: every column already exists (PLAN §11.2). + +### S8.2 Compiled-file layout (PLAN §3.8, §3.13) + +``` +/blocklists/.list +/blocklists/.wild +``` + +Directory created with `createDirPathStatus(io, path, .fromMode(0o700))`; files written 0o600 through +`createFileAtomic` + `replace` (milestone 4's verified helper), so a crash mid-write can never leave +a half-list that would load as a valid, shorter blocklist. `` is the `blocklist_sources` +row id, so a renamed URL keeps its files and a deleted source's files are removed by +`pruneOrphans` (below). + +Header, written by the manager ahead of the body, every line prefixed `# `: + +``` +# nxdns blocklist +# url +# format +# fetched_at +# domains +# wildcards +# skipped_regex +# skipped_unsupported +# invalid +# sha256 <64 hex chars> +``` + +The loader strips every leading `#` line and hands the remainder to `DomainSet.build`. The `sha256` +covers the `.list` body followed by the `.wild` body — **not** the header, so the checksum is stable +across refetches of unchanged content while `fetched_at` moves. + +### S8.3 Signatures + +```zig +pub const Paths = struct { + dir: std.Io.Dir, // + subdir: []const u8 = "blocklists", +}; + +pub const SourceStatus = struct { + id: i64, + url: []const u8, // borrowed from the manager's arena + state: enum { ok, never_fetched, fetch_failed, compile_failed, load_failed }, + last_attempt: i64 = 0, + last_success: i64 = 0, + counts: compiler.Counts = .{}, + /// Fixed-size, no allocation on the failure path. + last_error: [128]u8 = @splat(0), + last_error_len: u8 = 0, +}; + +pub const Manager = struct { + gpa: std.mem.Allocator, + database: *db.Db, + paths: Paths, + fetcher: *fetcher.Fetcher, + update: model.BlocklistUpdate, + total_budget: std.Io.Clock.Duration, + + lock: std.Io.RwLock, + current: ?*matcher.Snapshot, + generation: u64, + statuses: []SourceStatus, + + pub const Error = error{ OutOfMemory, ... } || db.Error || matcher.Snapshot.Error; + + pub fn init(gpa, database, paths, fetcher_ptr, update, total_budget) Error!Manager; + pub fn deinit(self: *Manager, io: std.Io) void; + + /// Reads the database and every compiled file, builds a snapshot and swaps + /// it in. Called at startup and after any refresh. A source whose compiled + /// files are missing or whose checksum does not match is marked + /// `.load_failed` and REFRESHED, not silently skipped. + pub fn reload(self: *Manager, io: std.Io) Error!void; + + /// Fetch + compile + atomically replace the compiled files for one source, + /// then update its row. Returns false when the content was unchanged + /// (checksum equal), in which case only `last_updated` moves. + pub fn refreshSource(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool; + + /// Every enabled source, one at a time, then one `reload`. Never stops at + /// the first failure: a broken source must not hide the rest. + pub fn refreshAll(self: *Manager, io: std.Io) Error!void; + + /// Long-running task: sleeps `update.interval_hours`, refreshes, repeats. + /// Returns on `error.Canceled`. Started with `io.concurrent` by Phase 7's + /// wiring; nothing in this milestone starts it automatically. + pub fn runScheduler(self: *Manager, io: std.Io) std.Io.Cancelable!void; + + /// Reader side of the RCU swap. The handle holds a shared lock; the caller + /// MUST release it and MUST NOT retain the snapshot pointer afterwards. + pub const Handle = struct { + snapshot: *const matcher.Snapshot, + manager: *Manager, + pub fn release(self: Handle, io: std.Io) void; + }; + pub fn acquire(self: *Manager, io: std.Io) ?Handle; + + /// Copies the status table for the API and `nxdns check`. + pub fn statusSnapshot(self: *Manager, io: std.Io, out: []SourceStatus) usize; + + /// Deletes `.list`/`.wild` files whose id is not in the database. + pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void; +}; +``` + +**The swap is an `std.Io.RwLock`, not a lock-free pointer.** PLAN §7.3 says "readers lock-free", and +this is a deliberate, documented deviation. Freeing the old snapshot safely without a lock requires +epoch-based reclamation or hazard pointers — a class of code that is very hard to get right and +impossible to test convincingly, bought for a household resolver whose target is 100 qps. A shared +lock held for the microseconds of one `evaluate` costs an uncontended atomic pair; the writer takes +the exclusive lock only on a swap, which happens on refresh. The old snapshot is freed **after** +`unlock` returns, and the `Handle` API makes "do not retain the pointer" the only shape the caller +can write. State this reasoning in the file, and have the orchestrator amend PLAN §7.3. + +### S8.4 `refreshSource` order — the order is the specification + +1. `now = std.Io.Clock.real.now(io).toSeconds()`; `status.last_attempt = now`. +2. Create `/.list.tmp` and `.wild.tmp` through `createFileAtomic` + (`.permissions = .fromMode(0o600)`, `.replace = true`). +3. Download into a third temporary file (`.raw.tmp`) rather than memory — `max_body_bytes` is + 64 MB and the budget in §Memory has no room for it alongside two snapshots. Run + `fetcher.fetch` under `io.concurrent` with a `total_budget` sleep racing it, and cancel the loser + (milestone 3's pattern). A fetch failure sets `.fetch_failed` with the error name, logs at `warn`, + and returns without touching the live compiled files. +4. Reopen the raw file, sniff the format with `parsers.detectFormat` over the first + `parsers.sample_lines` lines, then `compiler.compile` from a `File.Reader` into the two temporary + writers. Any compile error sets `.compile_failed` and returns; the live files are still untouched. +5. If `result.checksum` equals the stored `checksum`, `deinit` the atomics (which discards the + temporaries), update only `last_updated`, and return `false`. Recompiling identical content into + a new file would invalidate the snapshot for nothing. +6. Otherwise write the header, `replace` both atomics, delete the raw temporary, and + `updateSourceStats` with the counts, the checksum and `last_updated = now`. +7. `status.state = .ok`, `status.counts = result.counts`, `status.last_success = now`. + +Every failure path deletes its temporaries. No path leaves a `.tmp` behind, and no path deletes a +good compiled file. + +### S8.5 Failure visibility + +- Every non-`ok` state records the error name into `SourceStatus.last_error` and logs once at `warn` + with the source URL and the state. No `err` level, per the binding logging policy: the condition is + recorded and surfaced, not swallowed. +- `refreshAll` returns successfully when at least one source failed; the failures live in + `statuses`. It returns an error only when *nothing* could be done (out of memory, the database is + unreachable). Phase 8 exposes the statuses at `GET /api/blocklists`; `nxdns check` gains nothing in + this milestone. +- `reload` with zero enabled sources is normal (a fresh install), produces a valid empty snapshot, + and logs at `info`. + +### S8.6 Startup and scheduling policy + +`runScheduler` refreshes a source at startup only when it needs it: no compiled file, a checksum +mismatch against the file on disk, or `last_updated` older than `interval_hours`. A cold restart of +a Pi must not re-download every list, and a boot loop must not turn into a download loop. After the +initial pass it sleeps `interval_hours` between full passes. `update.enabled == false` means +`runScheduler` returns immediately after the initial load; manual refresh through `refreshAll` still +works (Phase 8's `POST /api/blocklists/update`). + +### S8.7 Tests + +In-file (`:memory:` database, no filesystem): `init`/`deinit`; `acquire` before any `reload` returns +`null`; the header writer produces the exact expected text for a known `Counts`; the header stripper +returns the body for a header-only file and for a file with no header; `SourceStatus.last_error` +truncation at 128 bytes. + +Everything with real files, real HTTP and real swaps is S9's. + +### S8.8 Acceptance criteria + +- [ ] `zig fmt --check` and `zig ast-check` clean on `manager.zig` and `sources_repo.zig`. +- [ ] `sources_repo.zig` gains exactly two public functions plus `SourceRow`/`freeSourceRows`; no + existing signature changes; the new list has a deterministic `ORDER BY url`. +- [ ] No schema change, no migration step, no new column. +- [ ] Compiled files are written through `createFileAtomic` + `replace` at 0o600, in a 0o700 + directory created with `createDirPathStatus`. +- [ ] A fetch or compile failure leaves the previously compiled files byte-identical (test in S9). +- [ ] The RwLock deviation from PLAN §7.3 is documented in the file with its reasoning. +- [ ] The old snapshot is freed after `unlock`, never while a reader could hold it. +- [ ] `refreshAll` does not stop at the first failing source. +- [ ] `grep -n "std.log.err" src/filter/ src/local/` returns nothing. + +--- + +## Session S9: `src/filter/filter_integration_test.zig`, `tests/fuzz/blocklist_fuzz.zig` + +### S9.1 Hermetic integration cases (`-Dintegration`, `std.testing.tmpDir`) + +*compile → load → match, through real files* + +1. Compile a 5,000-line hosts fixture to real `.list`/`.wild` files, build a snapshot from them, and + assert 20 sampled domains match and 20 non-members do not. +2. Recompile the same fixture into a second directory: both files are byte-identical, and so is the + checksum. +3. Truncate a `.list` file mid-line and reload → `error.NotSorted` or `MissingCompiledSource`, and + **the previous snapshot is still serving** (assert through `acquire`). + +*fetcher against a loopback `std.http.Server`* + +4. A 200 response with a hosts body → the compiled files exist, `blocklist_sources` counters and + checksum are written, and `acquire` sees the domains. +5. A 302 to a second path → followed, same result. +6. A 404 → `.fetch_failed`, the previously compiled files are byte-identical, and the snapshot still + blocks what it blocked before. +7. A body larger than a lowered `max_body_bytes` → `error.BodyTooLarge`, no `.tmp` file remains in + the blocklist directory (assert by listing it). +8. Refetching identical content → `refreshSource` returns `false`, the compiled files' mtimes are + unchanged, and `last_updated` moved. + +*swap* + +9. Under a snapshot acquired by one task, a concurrent `reload` completes and the holding task still + reads a consistent snapshot; after `release` and a re-`acquire`, the new generation is visible. +10. `pruneOrphans` deletes files for a deleted source and leaves live ones alone. + +*local records and forward zones, end to end* + +11. Build `Records` from a seeded database (through `local_repo.listLocalRecords`), answer an A query + for `nas.lan`, and re-parse the reply: one answer, correct rdata, correct TTL. +12. A CNAME local record answers an A query with the CNAME record and nothing else. +13. `ForwardClient` against a loopback UDP responder: an A query for `nas.lan.home` returns the + responder's answer, and `Zones.match` selected that resolver. +14. The loopback responder sets TC=1; the client retries over TCP against a loopback TCP responder + and returns the full answer, with `stats.udp_truncated == 1`. +15. A silent responder → `error.Timeout` inside the configured budget (assert the elapsed time is + below twice the budget). +16. A responder answering with a wrong ID → `error.ResponseMismatch`. + +*blocked responses* + +17. For each `blocking.response` mode, synthesize a reply for a blocked A and AAAA query and re-parse + it: `.zero` gives `0.0.0.0` / `::` with `blocking.ttl`; `.nxdomain` gives NXDOMAIN with no answer. + +### S9.2 Fuzz target + +`tests/fuzz/blocklist_fuzz.zig`, in the style of `tests/fuzz/dns_fuzz.zig` (milestone 2): a +`std.testing.fuzz` test per parser plus one for `wildcard.matches`, each filling a buffer through +`Smith.sliceWithHash` and feeding the prefix in. The acceptance property is "does not crash and does +not hang": every parser returns a `Line` for any byte string, and `wildcard.matches` terminates for +any pattern/domain pair (the pattern is first passed through `validate`, and only accepted patterns +reach `matches`, matching how production uses it). Corpus entries: one hosts line, one ABP line, one +regex line, one over-long line. + +### S9.3 The build change (orchestrator, not this session) + +```zig +const parsers_mod = b.createModule(.{ + .root_source_file = b.path("src/filter/parsers.zig"), + .target = target, + .optimize = optimize, +}); +const blocklist_fuzz_mod = b.createModule(.{ + .root_source_file = b.path("tests/fuzz/blocklist_fuzz.zig"), + .target = target, + .optimize = optimize, +}); +blocklist_fuzz_mod.addImport("parsers", parsers_mod); +const blocklist_fuzz_tests = b.addTest(.{ + .name = "blocklist-fuzz", + .use_llvm = if (fuzz) true else null, + .root_module = blocklist_fuzz_mod, +}); +test_step.dependOn(&b.addRunArtifact(blocklist_fuzz_tests).step); +``` + +This is exactly why S1's five files import nothing outside `src/filter/`: `parsers.zig` is a module +root here and in the aggregator, and an import that escaped its directory would fail with +`error.ImportOutsideModulePath` (milestone 2). The in-file tests of those five files still run in the +aggregator artifact; the fuzz artifact runs only `blocklist_fuzz.zig`'s tests. + +### S9.4 Acceptance criteria + +- [ ] `zig fmt --check` and `zig ast-check` clean on both files. +- [ ] All 17 hermetic cases exist as named tests and pass under `zig build test -Dintegration`. +- [ ] No hermetic test reaches a non-loopback address or leaves a file outside its `tmpDir`. +- [ ] The fuzz tests pass under plain `zig build test` (corpus replay + empty input). + +--- + +## Module Layout + +``` +src/filter/parsers.zig S1 format vocabulary + dispatch + sniffing (std only) +src/filter/parser_hosts.zig S1 hosts-file lines (std only) +src/filter/parser_domains.zig S1 one-domain-per-line lists (std only) +src/filter/parser_abp.zig S1 ABP subset; modifiers/exceptions unsupported (std only) +src/filter/wildcard.zig S1 `*`-label pattern validate + match (std only) +src/filter/domain_set.zig S2 arena + open-addressed exact set +src/filter/compiler.zig S2 stream -> sorted, deduped .list/.wild bodies + sha256 +src/filter/safesearch.zig S3 per-group safe-search table + rewrite +src/filter/response.zig S3 blocked-response synthesis (zero | nxdomain) +src/filter/rules.zig S5 one group's compiled allow/block rules +src/filter/matcher.zig S5 Snapshot, §3.10 precedence, group-for-client +src/filter/fetcher.zig S6 std.http.Client GET, streamed, capped +src/filter/manager.zig S8 compiled files, refresh, metadata, RCU swap, scheduler +src/filter/filter_integration_test.zig S9 -Dintegration: real files, loopback HTTP/UDP/TCP +src/local/records.zig S4 local A/AAAA/CNAME lookup + answer writing +src/local/forward_zones.zig S4 zone suffix match -> validate.Resolver +src/local/forward_client.zig S7 plain UDP/TCP resolver, transport.Client +src/storage/repositories/sources_repo.zig S8 += listSourceRows, updateSourceStats +tests/fuzz/blocklist_fuzz.zig S9 parser + wildcard fuzz targets +``` + +## File Ownership + +| Files | Owner | Notes | +|---|---|---| +| `src/filter/parsers.zig`, `parser_hosts.zig`, `parser_domains.zig`, `parser_abp.zig`, `wildcard.zig` | S1 | frozen after S1 verifies; `std`-only imports | +| `src/filter/domain_set.zig`, `src/filter/compiler.zig` | S2 | frozen after S2 verifies | +| `src/filter/safesearch.zig`, `src/filter/response.zig` | S3 | | +| `src/local/records.zig`, `src/local/forward_zones.zig` | S4 | | +| `src/filter/rules.zig`, `src/filter/matcher.zig` | S5 | | +| `src/filter/fetcher.zig` | S6 | | +| `src/local/forward_client.zig` | S7 | | +| `src/filter/manager.zig`, `src/storage/repositories/sources_repo.zig` | S8 | the only milestone-4 file any session edits, and only additively (S8.1) | +| `src/filter/filter_integration_test.zig`, `tests/fuzz/blocklist_fuzz.zig` | S9 | | +| `build.zig`, `build.zig.zon`, `src/tests.zig` | orchestrator | no session edits these | +| `PLAN.md` §7.1, §7.3 | orchestrator | amended per resolved ambiguities 1 and 3, and per S8.3 | + +No session touches `src/dns/`, `src/server/`, `src/upstream/`, `src/platform/`, `src/config/`, +`src/cli.zig` or `src/main.zig`. A needed change there is reported, not made. + +## Acceptance Criteria (Milestone 5 Complete) + +- [ ] `zig build test` exits 0 with every new file wired into `src/tests.zig`, including the new + `blocklist-fuzz` artifact. +- [ ] `zig build test -Dintegration` exits 0: milestone 1's loopback TLS echo, milestone 3's + listener and resolver tests, milestone 4's 21 storage cases, and all 17 filtering cases. +- [ ] `zig build cross` still produces two statically linked executables. +- [ ] The PLAN §3.10 precedence table is proven row by row by the S5.4 tests. +- [ ] Compiling a fixture twice produces byte-identical bodies and checksums; compiling a shuffled + permutation produces the same bodies. +- [ ] `grep -rn "std.log.err" src/filter/ src/local/` returns nothing. +- [ ] `grep -rln "std.Io," src/filter/ src/local/` names only `fetcher.zig`, `manager.zig`, + `forward_client.zig` and the S9 test file `filter_integration_test.zig` (reader/writer + interface parameters do not count; check the match). +- [ ] `grep -rn "regex" src/filter/` shows counting and skipping only — no matching engine. +- [ ] A hosts fixture containing `localhost`, `ip6-localhost` and `broadcasthost` compiles to a body + containing none of them. +- [ ] A snapshot over a 1,000,000-entry synthetic list reports `memoryBytes()` under 40 MB (S5.4's + inequality test, scaled) — the runnable form of the PLAN §18 budget. +- [ ] `zig fmt --check` clean repo-wide; GPG-signed lowercase commits. + +## Anti-Requirements + +- **No handler integration.** `src/server/handler.zig` is not edited. Nothing in this milestone is + called from the query path; composing local records → forward zones → filtering → safe-search → + cache → upstream is Phase 7. The pieces are built and tested standalone. +- **No CNAME uncloaking.** PLAN §6.3 is explicitly Phase 7 (§16). No chain walking, no depth-8 + budget, no re-evaluation of answer-section names anywhere in this milestone. +- **No cache, no rate limiter, no query log, no disk monitor, no retention, no log rotation.** + Phase 6. A `Decision` carries a `reason` for the future query log; nothing writes one. +- **No pause/resume.** Phase 7. +- **No web API, no SSE, no `/metrics`, no auth.** Phase 8. `SourceStatus` exists for that phase to + read; no handler exists. +- **No regex engine, in any form, for any reason.** PLAN §2.2. Regex lines are counted and skipped. +- **No allow rules sourced from blocklists.** ABP exception rules (`@@`) are counted as unsupported. + Allow policy lives in the `rules` table where an operator can see it. +- **No blocklist storage in SQLite.** PLAN Decision A: compiled flat files under + `/blocklists/`, metadata columns only in `config.db`. +- **No schema change and no migration step.** Every column this milestone writes already exists. +- **No HTTP compression, no conditional requests (`ETag`/`If-Modified-Since`), no HTTP/2.** + Unchanged content is detected by comparing the compiled checksum, which also catches a source that + changes its headers without changing its content. +- **No client auto-materialization, no `last_seen` updates.** Phase 7 (resolved ambiguity 4). +- **No snapshot persistence.** The snapshot is rebuilt from the database and the compiled files at + startup; caching it on disk would add a fourth thing that can be stale. +- **No lock-free reclamation scheme.** The documented `RwLock` deviation (S8.3) is the design, not a + placeholder for a later epoch-based rewrite. +- **No third-party Zig packages.** stdlib plus the two pinned C libraries. + +## As built (S1–S4, S6, S7 and orchestrator wiring) + +Deviations from the text above, recorded after the first six sessions verified. Where this section +and the session text disagree, this section wins. + +**S1 parsers + wildcard.** `validate` counts labels before the 253-byte length check, so +`TooManyLabels` is reachable. A hosts line with a sink address and no name is `.unsupported`. +`||example.com` without `^` is `.wildcard` with `covers_apex = true`. `detectFormat` treats `$` as a +marker only on non-comment lines; `||`, `@@` and element-hiding separators are matched anchored. +`looksLikeIpLiteral` lives in `parsers.zig` and is shared with `parser_hosts.zig`. +`wildcard.matches` is total on unvalidated input: the label splitter returns null above `max_labels` +and `matches` returns false — no assert, because `parsers.zig` is the S9.3 fuzz root. + +**S2 domain_set + compiler.** The line-length cap is enforced both ways: `error.StreamTooLong` is +discarded with `discardDelimiterInclusive`, and a returned line over `max_line_len` is skipped; +both increment `long_lines`. `DomainSet.build` maps an empty line to `error.NotSorted` and a line +over 255 bytes to `error.SetTooLarge`; per line, the lowercase check runs before the order check. +`covers_apex` applies only to `.wildcard` lines: `*.x` in a hosts or domains list is a wildcard with +no apex entry. + +**S3 safesearch + response.** The safe-search table as written above was not sorted; the built table +holds the same entry set sorted ascending by `std.mem.order` (`www.youtube-nocookie.com` < +`www.youtube.com`; `youtube.com` < `youtube.googleapis.com` < `youtubei.googleapis.com`). `rewrite` +uses `catch unreachable` on `name.fromText`, upheld by the every-entry-is-valid test. The +mode × qtype × OPT matrix is eight named tests, each looping over OPT presence internally; all +sixteen cases are exercised and re-parsed. + +**S4 records + forward_zones.** A CNAME is exclusive at a name (RFC 1034 §3.6.2): records sort by +(owner, rtype) with rank A, AAAA, CNAME, and a name carrying a CNAME answers with the CNAME run +alone. `validate.zig` does not reject a CNAME coexisting with an A, so the mixed case is reachable +from a hand-edited database. `qtype == .any` returns the whole record run. The root name is +rejected (`BadRecordName` / `BadZone`), as is any byte ≥ 0x80 in an owner, CNAME target or zone. +Each table is `items` plus one flat byte block — two surviving allocations, freed by `deinit`. + +**S6 fetcher.** `Fetcher.last_status: ?std.http.Status` carries the numeric status past +`error.HttpStatus` (see S6.1). `error.WriteFailed` from the output writer maps to +`error.Unexpected`; the caller owns the writer and reads the concrete failure there. A `narrowLocal` +step folds `transport.mapLocal`'s fd-quota members into `error.SystemResources` and the rest into +`error.Unexpected`. A declared `content-length` over `max_body_bytes` returns `error.BodyTooLarge` +before the body streams. + +**S7 forward_client.** `read_timeout` is `std.Io.Clock.Duration` (S9 constructs it as +`.{ .raw = .fromMilliseconds(200), .clock = .awake }`). `Stats` has a fourth field, +`foreign_datagrams`. `failures` counts `.peer_fault` and `.local_resource` only — cancellations are +shutdown, not failure. The TC bit is read only after `validateResponse` matches the reply. +`frame_buf` is split in half between the stream writer and reader; `min_frame_buf = 1024` is +asserted in `init`. + +**Orchestrator wiring.** All thirteen new files are imported by `src/tests.zig` individually +(`wildcard.zig` is unreachable through `parsers.zig` for test collection). The TCP framing helpers +(`prefix_len`, `framePrefix`, `parsePrefix`) moved to `upstream/transport.zig` as their single home; +`dot_client.zig`, `tcp_server.zig`, `forward_client.zig` and both server integration tests now use +the `transport.*` forms, and the canonical framing tests live in `transport.zig`. `zig build test` +exits 0 after the move. + +**S5 rules + matcher.** `Snapshot.Input` gains `group_ids: []const i64` and `source_ids: []const +i64`, parallel to `groups` and `sources` (the model structs carry no row id); length mismatch is an +assert. `compiled` is `[]const ?Compiled` — `null` for an enabled source is +`error.MissingCompiledSource`; a disabled source needs no entry. On a prefix-length tie the *lower* +priority number wins, matching `address.matchLongest` and `pool.zig`. `max_wildcards_per_group` +counts both wildcard lists of a group combined, over rows before deduplication. Duplicate rule rows +are deduplicated before `DomainSet.build`. Level-6 nesting iterates parents outermost (most specific +first), sources innermost. A `group_sources` row naming a disabled source is legal; an unknown URL is +`error.UnknownSource`, an unknown group `error.UnknownGroup`. `RuleSet` carries a fifth field, +`wildcard_bytes`, backing both pattern lists. The snapshot builds every sub-structure from its own +arena, so `Snapshot.deinit` is one `arena.deinit()`. **S8 must fill `group_ids`, `source_ids` and +the optional `compiled` entries accordingly.** + +**S8 manager + sources_repo.** The compile stage writes plain `.list.tmp` / `.wild.tmp`; +`publish` streams header + body into the final files via `createFileAtomic` + `sync` + `replace` at +0o600 — the header carries counts that exist only after the compile, so it cannot go into the same +atomic pass. Group row ids come from `groups_repo.groupId(database, name)` (added by the +orchestrator; a vanished name is `error.GroupSetChanged`) — the manager does not run raw SQL. A +source whose compiled files are missing, unreadable or checksum-mismatched is marked `.load_failed` +and excluded from the snapshot; the reload succeeds without it. A present, checksum-clean but +malformed body still fails the build and the previous snapshot keeps serving. `updateSourceStats` +takes the whole stats struct; the unchanged path re-writes the row and verifies the on-disk bodies +hash to the stored checksum before skipping (a corrupt file takes the rewrite path, which repairs +it). `SourceStatus.state` is a named `pub const State` with a `no_valid_entries` member: a download +that compiles to zero domains and zero wildcards with any nonzero invalid/unsupported/long-line +count fails the refresh, leaves the previous files serving, and records +`NoValidEntries invalid=N unsupported=N long_lines=N`. A failed refresh seeds its status from the +prior entry, so `last_success` and the counts of the still-serving files survive. A second lock, +`writer_lock` (plain `std.Io.Mutex`, separate from the RCU lock so readers never wait behind a +download), serializes `reload`, `refreshSource`, `refreshAll`, `pruneOrphans` and the startup pass. +Format sniffing collects `parsers.sample_lines` countable lines (never a flat byte window). +`SourceStatus` is a value type that borrows nothing: `url` is inline `[max_url_len]u8` + `url_len` +(`max_url_len = 255`, truncation in the status only; the full url lives on the source row), read via +`urlText()`; a copy made under the lock outlives the table it came from. `SourceStatus` also splits +the refresh fact from the load fact: `state` records the latest attempt to produce the files, +`loaded: bool` records whether they are filtering right now, and a load outcome never overwrites a +refresh failure (`State.isRefreshFailure` names the three refresh states) — after `refreshAll`, a +source can honestly read `.fetch_failed` with `loaded = true`. Refresh tmp cleanup defers are +installed before the calls that create the files, so cancellation and OutOfMemory paths leak no +`.tmp`. A reload collects per-source `LoadOutcome`s while it builds the candidate snapshot and +applies them to the status table (`applyLoadOutcomes`) inside the same exclusive-lock section as the +swap — a reload that fails before publishing leaves both the snapshot and the status table +describing the previous generation, and the table and snapshot change atomically for readers. A +disabled source is recorded as not loaded; its `state` keeps the record of how it last stood. The +reload builds its status table off to the side (`StatusTable`, `buildStatusTable`, `mergeStatuses`) +and installs it in the same critical section that swaps the snapshot, so a reload that fails +publishes neither. `refreshAll` still syncs the table up front — a refresh pass needs somewhere to +record per-source outcomes as it goes. +`Manager.Error` adds `Canceled`, `FileSystem`, +`GroupSetChanged`. The snapshot seed comes from `io.random` (`std.crypto.random` does not exist in +0.16). The daily interval sleeps on `.clock = .boot`; S7's short timeouts stay on `.awake`. +`sources_repo` additions: `SourceRow`, `SourceStats`, `listSourceRows`, `freeSourceRows`, +`updateSourceStats` — additive only. + +**S9 note from S8:** a loopback `std.http.Server` answering `request.respond(body, .{})` deadlocks a +second fetch — the fetcher keeps the connection alive while a one-accept server waits for a new one. +Respond with `.{ .keep_alive = false }` or serve a keep-alive loop on the stream. + +**S9 integration + fuzz.** Case 3 splits into the two real failure modes: a checksum-mismatched file +marks the source `.load_failed` and the reload succeeds without it (generation advances); only a +checksum-clean but malformed body fails `reload` and leaves the previous snapshot serving. Case 7 +proves the cap from the response head: an explicit `content-length` of 100 MiB returns +`error.BodyTooLarge` before any body streams. Case 8 back-dates `last_updated` through +`updateSourceStats` and asserts inode and mtime of the compiled file are unchanged. Case 15 asserts +`budget/2 <= elapsed < 2*budget` (the POSIX backend wakes ~0.8 ms early on a 200 ms deadline). The +fuzz corpus is inline — `tests/fuzz/corpus.zig` imports the `dns` module, which the blocklist-fuzz +module does not have. Fuzz targets assert properties, not only absence of crashes: `Line.text` +windows the input (pointer containment), `covers_apex` only on `.wildcard`, `detectFormat`'s answer +survives `parseLine` over the same bytes, and `matches` is exercised on rejected patterns. + +**Final wiring.** `build.zig` gained the `blocklist-fuzz` artifact (module import `parsers` → +`src/filter/parsers.zig`, LLVM backend under `-Dfuzz`), hung off `test_step` beside the dns fuzz +artifact. Evaluation: `zig build test`, `zig build test -Dintegration` and `zig build cross` all +exit 0; both cross executables are statically linked. diff --git a/src/filter/compiler.zig b/src/filter/compiler.zig new file mode 100644 index 0000000..06b777f --- /dev/null +++ b/src/filter/compiler.zig @@ -0,0 +1,500 @@ +//! Compiles a downloaded blocklist into the two bodies nxdns stores on disk: +//! a `.list` body of exact names and a `.wild` body of suffixes. +//! +//! Pure over reader/writer interfaces: an allocator, a `*std.Io.Reader` and two +//! `*std.Io.Writer`. No `std.Io` value, no file, no clock. A compiled body is a +//! pure function of (bytes, format), which is what makes two runs — and two +//! permutations of the same input — byte-identical. +//! +//! Nothing but the sorted, deduplicated names is written: no header, no +//! timestamp, no counts. The header belongs to the caller, and the checksum +//! covers the two bodies only. + +const std = @import("std"); +const parsers = @import("parsers.zig"); +const name = @import("../dns/name.zig"); +const types = @import("../dns/types.zig"); + +const Sha256 = std.crypto.hash.sha2.Sha256; + +pub const max_domains: u32 = 2_000_000; +pub const max_line_len: usize = 4096; + +pub const Counts = struct { + domains: u32 = 0, + wildcards: u32 = 0, + skipped_regex: u32 = 0, + skipped_unsupported: u32 = 0, + /// Not a valid domain name (`dns.name.fromText` rejected it, a non-ASCII + /// byte, or fewer than two labels). + invalid: u32 = 0, + /// Lines longer than `max_line_len`, skipped whole. + long_lines: u32 = 0, + /// Duplicates removed by the sort/unique pass. + duplicates: u32 = 0, +}; + +pub const Result = struct { + counts: Counts, + /// Lowercase hex sha256 over the `.list` body followed by the `.wild` body. + checksum: [64]u8, +}; + +pub const Error = error{ OutOfMemory, TooManyDomains, ReadFailed, WriteFailed }; + +/// Reads `r` to end of stream and writes the two compiled bodies. +/// +/// `counts.domains` and `counts.wildcards` are the written, deduplicated +/// counts: they are what `blocklist_sources.domain_count` and `wildcard_count` +/// store and what the UI shows. +pub fn compile( + gpa: std.mem.Allocator, + r: *std.Io.Reader, + format: parsers.Format, + list_w: *std.Io.Writer, + wild_w: *std.Io.Writer, +) Error!Result { + var counts: Counts = .{}; + + var list: Entries = .{}; + defer list.deinit(gpa); + var wild: Entries = .{}; + defer wild.deinit(gpa); + + while (true) { + const raw = r.takeDelimiter('\n') catch |err| switch (err) { + error.ReadFailed => return error.ReadFailed, + // `takeDelimiter` leaves the stream unmodified on `StreamTooLong` + // (Reader.zig:885). Without this discard the loop re-reads the same + // bytes forever. + error.StreamTooLong => { + counts.long_lines += 1; + _ = r.discardDelimiterInclusive('\n') catch |discard_err| switch (discard_err) { + error.EndOfStream => break, + error.ReadFailed => return error.ReadFailed, + }; + continue; + }, + } orelse break; + + var line = raw; + if (line.len != 0 and line[line.len - 1] == '\r') line = line[0 .. line.len - 1]; + // A reader whose buffer is larger than `max_line_len` reports the + // over-long line here instead of through `error.StreamTooLong`. + if (line.len > max_line_len) { + counts.long_lines += 1; + continue; + } + + const parsed = parsers.parseLine(format, line); + switch (parsed.kind) { + .ignore => {}, + .regex => counts.skipped_regex += 1, + .unsupported => counts.skipped_unsupported += 1, + .domain => { + var fields = std.mem.tokenizeAny(u8, parsed.text, &std.ascii.whitespace); + while (fields.next()) |field| { + try addCandidate(gpa, field, false, false, &list, &wild, &counts); + } + }, + .wildcard => try addCandidate( + gpa, + parsed.text, + true, + parsed.covers_apex, + &list, + &wild, + &counts, + ), + } + } + + var hasher = Sha256.init(.{}); + counts.domains = try emit(&list, list_w, &hasher, &counts.duplicates); + counts.wildcards = try emit(&wild, wild_w, &hasher, &counts.duplicates); + + var digest: [Sha256.digest_length]u8 = undefined; + hasher.final(&digest); + return .{ .counts = counts, .checksum = std.fmt.bytesToHex(digest, .lower) }; +} + +/// Normalizes one whitespace-separated candidate and files it under `.list`, +/// `.wild`, or neither. +fn addCandidate( + gpa: std.mem.Allocator, + field: []const u8, + from_wildcard_line: bool, + covers_apex: bool, + list: *Entries, + wild: *Entries, + counts: *Counts, +) Error!void { + var candidate = field; + var is_wildcard = from_wildcard_line; + if (std.mem.startsWith(u8, candidate, "*.")) { + is_wildcard = true; + candidate = candidate[2..]; + } + // A '*' anywhere else makes this a pattern, and patterns belong to the + // `rules` table; a blocklist entry is a name or a suffix. + if (std.mem.indexOfScalar(u8, candidate, '*') != null) { + counts.invalid += 1; + return; + } + + if (candidate.len != 0 and candidate[candidate.len - 1] == '.') { + candidate = candidate[0 .. candidate.len - 1]; + } + if (candidate.len == 0 or candidate.len > types.max_name_len) { + counts.invalid += 1; + return; + } + + var buf: [types.max_name_len]u8 = undefined; + for (candidate, 0..) |c, i| { + if (c >= 0x80 or std.ascii.isControl(c)) { + counts.invalid += 1; + return; + } + buf[i] = std.ascii.toLower(c); + } + const normalized = buf[0..candidate.len]; + + const parsed_name = name.fromText(normalized) catch { + counts.invalid += 1; + return; + }; + // The two-label minimum is what keeps `localhost`, `local`, `broadcasthost` + // and the `ip6-*` names that every hosts list carries from black-holing the + // loopback names of every client on the LAN. + if (parsed_name.labelCount() < 2) { + counts.invalid += 1; + return; + } + + if (is_wildcard) { + try wild.append(gpa, normalized); + // An ABP `||x^` rule covers `x` itself as well as its subdomains. + if (covers_apex) try list.append(gpa, normalized); + } else { + try list.append(gpa, normalized); + } +} + +/// Sorts, deduplicates, writes and hashes one body. Returns the written count. +fn emit( + entries: *Entries, + w: *std.Io.Writer, + hasher: *Sha256, + duplicates: *u32, +) Error!u32 { + const offsets = entries.offsets.items; + if (offsets.len > 1) std.mem.sortUnstableContext(0, offsets.len, SortContext{ .entries = entries }); + + var written: u32 = 0; + var prev: []const u8 = ""; + var first = true; + for (0..offsets.len) |i| { + const text = entries.get(i); + if (!first and std.mem.eql(u8, prev, text)) { + duplicates.* += 1; + continue; + } + try w.writeAll(text); + try w.writeByte('\n'); + hasher.update(text); + hasher.update("\n"); + prev = text; + first = false; + written += 1; + } + return written; +} + +/// Length-prefixed candidate bytes plus the offsets that index them. Sorting +/// permutes the offsets, so the bytes never move. +const Entries = struct { + bytes: std.ArrayList(u8) = .empty, + offsets: std.ArrayList(u32) = .empty, + + fn deinit(self: *Entries, gpa: std.mem.Allocator) void { + self.bytes.deinit(gpa); + self.offsets.deinit(gpa); + } + + fn append(self: *Entries, gpa: std.mem.Allocator, text: []const u8) Error!void { + if (self.offsets.items.len >= max_domains) return error.TooManyDomains; + const offset = std.math.cast(u32, self.bytes.items.len) orelse return error.TooManyDomains; + try self.bytes.append(gpa, @intCast(text.len)); + try self.bytes.appendSlice(gpa, text); + try self.offsets.append(gpa, offset); + } + + fn get(self: *const Entries, i: usize) []const u8 { + const offset = self.offsets.items[i]; + const len = self.bytes.items[offset]; + return self.bytes.items[offset + 1 ..][0..len]; + } +}; + +const SortContext = struct { + entries: *Entries, + + pub fn lessThan(self: SortContext, a: usize, b: usize) bool { + return std.mem.order(u8, self.entries.get(a), self.entries.get(b)) == .lt; + } + + pub fn swap(self: SortContext, a: usize, b: usize) void { + const offsets = self.entries.offsets.items; + std.mem.swap(u32, &offsets[a], &offsets[b]); + } +}; + +const testing = std.testing; + +const Compiled = struct { + result: Result, + list_w: std.Io.Writer.Allocating, + wild_w: std.Io.Writer.Allocating, + + fn deinit(self: *Compiled) void { + self.list_w.deinit(); + self.wild_w.deinit(); + } + + fn list(self: *Compiled) []const u8 { + return self.list_w.written(); + } + + fn wild(self: *Compiled) []const u8 { + return self.wild_w.written(); + } +}; + +fn compileText(gpa: std.mem.Allocator, text: []const u8, format: parsers.Format) Error!Compiled { + var r: std.Io.Reader = .fixed(text); + return compileReader(gpa, &r, format); +} + +fn compileReader(gpa: std.mem.Allocator, r: *std.Io.Reader, format: parsers.Format) Error!Compiled { + var list_w: std.Io.Writer.Allocating = .init(gpa); + errdefer list_w.deinit(); + var wild_w: std.Io.Writer.Allocating = .init(gpa); + errdefer wild_w.deinit(); + const result = try compile(gpa, r, format, &list_w.writer, &wild_w.writer); + return .{ .result = result, .list_w = list_w, .wild_w = wild_w }; +} + +const hosts_fixture = + "# a comment\n" ++ + "0.0.0.0 ads.example.com\n" ++ + "0.0.0.0 ads.example.com\n" ++ + "127.0.0.1 localhost\n" ++ + "0.0.0.0 tracker.example.org # tracker\n" ++ + "/ads\\d+/\n" ++ + "\n" ++ + "0.0.0.0 EXAMPLE.com.\n"; + +test "hosts fixture compiles to a sorted deduplicated body" { + var c = try compileText(testing.allocator, hosts_fixture, .hosts); + defer c.deinit(); + + try testing.expectEqualStrings( + "ads.example.com\nexample.com\ntracker.example.org\n", + c.list(), + ); + try testing.expectEqualStrings("", c.wild()); + try testing.expectEqual(@as(u32, 3), c.result.counts.domains); + try testing.expectEqual(@as(u32, 0), c.result.counts.wildcards); + try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_regex); + try testing.expectEqual(@as(u32, 0), c.result.counts.skipped_unsupported); + try testing.expectEqual(@as(u32, 1), c.result.counts.invalid); + try testing.expectEqual(@as(u32, 0), c.result.counts.long_lines); + try testing.expectEqual(@as(u32, 1), c.result.counts.duplicates); +} + +test "domains fixture compiles" { + const fixture = + "# a comment\n" ++ + "b.example.com\n" ++ + "! a bang comment\n" ++ + "a.example.com\n" ++ + "/re/\n" ++ + "0.0.0.0 two.fields.com\n" ++ + "localhost\n"; + + var c = try compileText(testing.allocator, fixture, .domains); + defer c.deinit(); + + try testing.expectEqualStrings("a.example.com\nb.example.com\n", c.list()); + try testing.expectEqualStrings("", c.wild()); + try testing.expectEqual(@as(u32, 2), c.result.counts.domains); + try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_regex); + try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_unsupported); + try testing.expectEqual(@as(u32, 1), c.result.counts.invalid); +} + +test "abp apex rule lands in both bodies" { + const fixture = + "! Title: test\n" ++ + "||x.com^\n" ++ + "||y.com^$third-party\n" ++ + "@@||z.com^\n" ++ + "/re/\n" ++ + "bare.com\n"; + + var c = try compileText(testing.allocator, fixture, .abp); + defer c.deinit(); + + try testing.expectEqualStrings("bare.com\nx.com\n", c.list()); + try testing.expectEqualStrings("x.com\n", c.wild()); + try testing.expectEqual(@as(u32, 2), c.result.counts.domains); + try testing.expectEqual(@as(u32, 1), c.result.counts.wildcards); + try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_regex); + try testing.expectEqual(@as(u32, 2), c.result.counts.skipped_unsupported); +} + +test "two runs of the same input are byte-identical" { + var a = try compileText(testing.allocator, hosts_fixture, .hosts); + defer a.deinit(); + var b = try compileText(testing.allocator, hosts_fixture, .hosts); + defer b.deinit(); + + try testing.expectEqualStrings(a.list(), b.list()); + try testing.expectEqualStrings(a.wild(), b.wild()); + try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum); +} + +test "a permutation of the input compiles to the same bodies" { + const shuffled = + "0.0.0.0 EXAMPLE.com.\n" ++ + "/ads\\d+/\n" ++ + "0.0.0.0 tracker.example.org # tracker\n" ++ + "\n" ++ + "0.0.0.0 ads.example.com\n" ++ + "127.0.0.1 localhost\n" ++ + "# a comment\n" ++ + "0.0.0.0 ads.example.com\n"; + + var a = try compileText(testing.allocator, hosts_fixture, .hosts); + defer a.deinit(); + var b = try compileText(testing.allocator, shuffled, .hosts); + defer b.deinit(); + + try testing.expectEqualStrings(a.list(), b.list()); + try testing.expectEqualStrings(a.wild(), b.wild()); + try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum); +} + +test "uppercase and trailing dot normalize" { + var c = try compileText(testing.allocator, "AdS.Example.COM.\n", .domains); + defer c.deinit(); + try testing.expectEqualStrings("ads.example.com\n", c.list()); +} + +test "invalid candidates are counted and written nowhere" { + const fixture = + "caf\xc3\xa9.example.com\n" ++ + "localhost\n" ++ + "a*b.com\n"; + + var c = try compileText(testing.allocator, fixture, .domains); + defer c.deinit(); + + try testing.expectEqual(@as(u32, 3), c.result.counts.invalid); + try testing.expectEqual(@as(u32, 0), c.result.counts.domains); + try testing.expectEqualStrings("", c.list()); + try testing.expectEqualStrings("", c.wild()); +} + +test "a leading star label becomes a wildcard entry" { + var c = try compileText(testing.allocator, "*.ads.example.com\n", .domains); + defer c.deinit(); + try testing.expectEqualStrings("", c.list()); + try testing.expectEqualStrings("ads.example.com\n", c.wild()); + try testing.expectEqual(@as(u32, 1), c.result.counts.wildcards); +} + +test "an over-long line is skipped whole" { + const gpa = testing.allocator; + var text: std.ArrayList(u8) = .empty; + defer text.deinit(gpa); + + var line_buf: [64]u8 = undefined; + var i: usize = 0; + while (i < 3_000) : (i += 1) { + if (i == 1_500) { + try text.appendNTimes(gpa, 'x', 5_000); + try text.append(gpa, '\n'); + } + try text.appendSlice(gpa, try std.fmt.bufPrint(&line_buf, "d{d:0>5}.example.com\n", .{i})); + } + + // A reader buffer smaller than the long line makes `takeDelimiter` report + // `error.StreamTooLong`, which is the path that loops forever without the + // discard. + var backing: std.Io.Reader = .fixed(text.items); + var buf: [max_line_len]u8 = undefined; + var limited = backing.limited(.unlimited, &buf); + + var c = try compileReader(gpa, &limited.interface, .domains); + defer c.deinit(); + + try testing.expectEqual(@as(u32, 1), c.result.counts.long_lines); + try testing.expectEqual(@as(u32, 3_000), c.result.counts.domains); + try testing.expect(std.mem.startsWith(u8, c.list(), "d00000.example.com\n")); + try testing.expect(std.mem.endsWith(u8, c.list(), "d02999.example.com\n")); + try testing.expect(std.mem.indexOf(u8, c.list(), "d01499.example.com\n") != null); + try testing.expect(std.mem.indexOf(u8, c.list(), "d01500.example.com\n") != null); + try testing.expect(std.mem.indexOf(u8, c.list(), "xxxx") == null); +} + +test "an over-long line is skipped when the reader buffer is large" { + const gpa = testing.allocator; + var text: std.ArrayList(u8) = .empty; + defer text.deinit(gpa); + + try text.appendSlice(gpa, "a.example.com\n"); + try text.appendNTimes(gpa, 'x', 5_000); + try text.append(gpa, '\n'); + try text.appendSlice(gpa, "b.example.com\n"); + + var c = try compileText(gpa, text.items, .domains); + defer c.deinit(); + + try testing.expectEqual(@as(u32, 1), c.result.counts.long_lines); + try testing.expectEqualStrings("a.example.com\nb.example.com\n", c.list()); +} + +test "carriage returns are stripped" { + var c = try compileText(testing.allocator, "b.example.com\r\na.example.com\r\n", .domains); + defer c.deinit(); + try testing.expectEqualStrings("a.example.com\nb.example.com\n", c.list()); +} + +test "empty input produces empty bodies and the sha256 of the empty string" { + var c = try compileText(testing.allocator, "", .domains); + defer c.deinit(); + + try testing.expectEqualStrings("", c.list()); + try testing.expectEqualStrings("", c.wild()); + try testing.expectEqualStrings( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + &c.result.checksum, + ); +} + +fn compileUnderFailure(gpa: std.mem.Allocator) !void { + var c = compileText(gpa, hosts_fixture, .hosts) catch |err| switch (err) { + // `Writer.Allocating` reports an exhausted allocator as `WriteFailed`; + // it has no other failure mode. + error.WriteFailed => return error.OutOfMemory, + else => |e| return e, + }; + defer c.deinit(); + try testing.expectEqual(@as(u32, 3), c.result.counts.domains); +} + +test "compile under allocation failure" { + try testing.checkAllAllocationFailures(testing.allocator, compileUnderFailure, .{}); +} diff --git a/src/filter/domain_set.zig b/src/filter/domain_set.zig new file mode 100644 index 0000000..d6f1018 --- /dev/null +++ b/src/filter/domain_set.zig @@ -0,0 +1,285 @@ +//! An immutable exact-match set of domain names, built from a compiled +//! blocklist body. +//! +//! The layout is a flat arena of length-prefixed names plus an open-addressed +//! table of `u32` offsets into it. No Bloom filter and no hash-only key set: a +//! false positive in a DNS sinkhole blocks a real domain for a real household +//! and is undebuggable from the outside, so every probe compares full bytes. +//! +//! Pure. Takes an allocator and bytes; no `std.Io`, no clock, no entropy +//! source — the hash seed arrives as a parameter. + +const std = @import("std"); + +pub const DomainSet = struct { + /// Length-prefixed lowercase names, back to back: [len: u8][bytes]… + arena: []const u8, + /// Open-addressed table of offsets into `arena`; `empty_slot` marks a hole. + /// Length is always a power of two. + index: []const u32, + count: u32, + seed: u64, + + pub const empty_slot: u32 = std.math.maxInt(u32); + pub const max_count: u32 = 4_000_000; + pub const max_arena_bytes: usize = 1 << 31; + + pub const Error = error{ OutOfMemory, TooManyDomains, SetTooLarge, NotSorted, NotLowercase }; + + /// An empty set that owns nothing. `contains` on it is always false. + pub const empty: DomainSet = .{ + .arena = &.{}, + .index = &.{}, + .count = 0, + .seed = 0, + }; + + /// Builds from a compiled body: LF-separated, lowercase, sorted ascending, + /// deduplicated, every line 1–255 bytes. + /// + /// The sortedness is verified, not assumed: a hand-edited or truncated file + /// must fail loudly rather than produce a set that silently misses entries. + /// Verification is one comparison per line. Because a valid line is at least + /// one byte, comparing against an initial empty `prev` also rejects an empty + /// line and a duplicate line — both are `error.NotSorted`, since neither is + /// strictly ascending. + /// + /// A line longer than 255 bytes is `error.SetTooLarge`: the arena stores a + /// `u8` length prefix and must not silently truncate, for the same reason + /// the `u32` offsets bound `max_arena_bytes`. + /// + /// `seed` randomizes the hash. Query names are attacker-supplied, so a fixed + /// seed would make probe-chain flooding computable offline. + pub fn build(gpa: std.mem.Allocator, body: []const u8, seed: u64) Error!DomainSet { + if (body.len == 0) return empty; + if (body.len > max_arena_bytes) return error.SetTooLarge; + + var count: u32 = 0; + var arena_len: usize = 0; + { + var prev: []const u8 = ""; + var it: LineIterator = .{ .rest = body }; + while (it.next()) |line| { + for (line) |c| if (c >= 'A' and c <= 'Z') return error.NotLowercase; + if (line.len > std.math.maxInt(u8)) return error.SetTooLarge; + if (std.mem.order(u8, prev, line) != .lt) return error.NotSorted; + if (count == max_count) return error.TooManyDomains; + prev = line; + count += 1; + arena_len += 1 + line.len; + } + } + + const capacity = capacityFor(count); + + const arena = try gpa.alloc(u8, arena_len); + errdefer gpa.free(arena); + const index = try gpa.alloc(u32, capacity); + @memset(index, empty_slot); + + const mask = capacity - 1; + var write_at: usize = 0; + var it: LineIterator = .{ .rest = body }; + while (it.next()) |line| { + const offset: u32 = @intCast(write_at); + arena[write_at] = @intCast(line.len); + @memcpy(arena[write_at + 1 ..][0..line.len], line); + write_at += 1 + line.len; + + var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(seed, line))) & mask; + while (index[slot] != empty_slot) slot = (slot + 1) & mask; + index[slot] = offset; + } + + return .{ .arena = arena, .index = index, .count = count, .seed = seed }; + } + + pub fn deinit(self: *DomainSet, gpa: std.mem.Allocator) void { + gpa.free(self.arena); + gpa.free(self.index); + self.* = empty; + } + + /// `domain` must be normalized (lowercase, no trailing dot). Allocation-free. + pub fn contains(self: *const DomainSet, domain: []const u8) bool { + if (self.index.len == 0) return false; + const mask = self.index.len - 1; + var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(self.seed, domain))) & mask; + while (true) { + const offset = self.index[slot]; + if (offset == empty_slot) return false; + const len = self.arena[offset]; + if (len == domain.len and std.mem.eql(u8, self.arena[offset + 1 ..][0..len], domain)) { + return true; + } + slot = (slot + 1) & mask; + } + } + + /// Bytes held, for the memory report in `Snapshot.memoryBytes`. + pub fn memoryBytes(self: *const DomainSet) usize { + return self.arena.len + self.index.len * @sizeOf(u32); + } +}; + +/// Smallest power of two at least `count * 4 / 3`, minimum 16. The load factor +/// stays at or under 0.75, which keeps at least one hole and therefore +/// terminates the probe loop in `contains`. +fn capacityFor(count: u32) usize { + const wanted = (@as(u64, count) * 4 + 2) / 3; + var capacity: usize = 16; + while (capacity < wanted) capacity *= 2; + return capacity; +} + +/// Yields LF-separated lines. A final line without a trailing LF is yielded; +/// a trailing LF does not yield a final empty line. +const LineIterator = struct { + rest: []const u8, + + fn next(self: *LineIterator) ?[]const u8 { + if (self.rest.len == 0) return null; + if (std.mem.indexOfScalar(u8, self.rest, '\n')) |nl| { + defer self.rest = self.rest[nl + 1 ..]; + return self.rest[0..nl]; + } + defer self.rest = self.rest[self.rest.len..]; + return self.rest; + } +}; + +const testing = std.testing; + +const small_body = + "ads.example.com\n" ++ + "example.com\n" ++ + "tracker.example.org\n" ++ + "zzz.example.net\n"; + +test "build and contains over a small body" { + var set = try DomainSet.build(testing.allocator, small_body, 0x1234); + defer set.deinit(testing.allocator); + + try testing.expectEqual(@as(u32, 4), set.count); + try testing.expect(set.contains("ads.example.com")); + try testing.expect(set.contains("example.com")); + try testing.expect(set.contains("tracker.example.org")); + try testing.expect(set.contains("zzz.example.net")); + + // A prefix, a suffix, an uppercase spelling and the empty string. + try testing.expect(!set.contains("example.co")); + try testing.expect(!set.contains("example.com.evil.net")); + try testing.expect(!set.contains("Example.com")); + try testing.expect(!set.contains("")); +} + +test "build rejects an unsorted body" { + try testing.expectError( + error.NotSorted, + DomainSet.build(testing.allocator, "b.example.com\na.example.com\n", 0), + ); +} + +test "build rejects an uppercase byte" { + try testing.expectError( + error.NotLowercase, + DomainSet.build(testing.allocator, "a.example.com\nB.example.com\n", 0), + ); +} + +test "build rejects a duplicate line" { + try testing.expectError( + error.NotSorted, + DomainSet.build(testing.allocator, "a.example.com\na.example.com\n", 0), + ); +} + +test "build rejects an over-long line" { + var body: [300]u8 = undefined; + @memset(&body, 'a'); + body[299] = '\n'; + try testing.expectError(error.SetTooLarge, DomainSet.build(testing.allocator, &body, 0)); +} + +test "contains does not depend on the seed" { + var a = try DomainSet.build(testing.allocator, small_body, 0); + defer a.deinit(testing.allocator); + var b = try DomainSet.build(testing.allocator, small_body, 0xdead_beef_cafe_f00d); + defer b.deinit(testing.allocator); + + var buf: [64]u8 = undefined; + var i: usize = 0; + while (i < 50) : (i += 1) { + const probe = try std.fmt.bufPrint(&buf, "n{d}.example.com", .{i}); + try testing.expectEqual(a.contains(probe), b.contains(probe)); + } + for ([_][]const u8{ "ads.example.com", "example.com", "zzz.example.net", "nope.test" }) |probe| { + try testing.expectEqual(a.contains(probe), b.contains(probe)); + } +} + +test "ten thousand names round-trip" { + const gpa = testing.allocator; + var body: std.ArrayList(u8) = .empty; + defer body.deinit(gpa); + + // Fixed-width zero padding makes the generated order the sorted order. + var line_buf: [64]u8 = undefined; + var i: usize = 0; + while (i < 10_000) : (i += 1) { + try body.appendSlice(gpa, try std.fmt.bufPrint(&line_buf, "d{d:0>5}.example.com\n", .{i})); + } + + var set = try DomainSet.build(gpa, body.items, 0x5eed); + defer set.deinit(gpa); + try testing.expectEqual(@as(u32, 10_000), set.count); + + var buf: [64]u8 = undefined; + i = 0; + while (i < 10_000) : (i += 1) { + const nameStr = try std.fmt.bufPrint(&buf, "d{d:0>5}.example.com", .{i}); + try testing.expect(set.contains(nameStr)); + } + try testing.expect(!set.contains("d10000.example.com")); + try testing.expect(set.memoryBytes() < 2 * body.items.len); +} + +fn buildUnderFailure(gpa: std.mem.Allocator) !void { + var set = try DomainSet.build(gpa, small_body, 0x1234); + defer set.deinit(gpa); + try testing.expect(set.contains("example.com")); +} + +test "build under allocation failure" { + try testing.checkAllAllocationFailures(testing.allocator, buildUnderFailure, .{}); +} + +test "the empty set owns nothing" { + var set: DomainSet = .empty; + try testing.expect(!set.contains("x")); + try testing.expect(!set.contains("")); + try testing.expectEqual(@as(usize, 0), set.memoryBytes()); + set.deinit(testing.allocator); + try testing.expect(!set.contains("x")); +} + +test "an empty body builds the empty set" { + var set = try DomainSet.build(testing.allocator, "", 7); + defer set.deinit(testing.allocator); + try testing.expectEqual(@as(u32, 0), set.count); + try testing.expect(!set.contains("x")); +} + +test "a final line without a trailing newline is kept" { + var set = try DomainSet.build(testing.allocator, "a.example.com\nb.example.com", 0); + defer set.deinit(testing.allocator); + try testing.expectEqual(@as(u32, 2), set.count); + try testing.expect(set.contains("b.example.com")); +} + +test "capacityFor keeps the load factor at or under three quarters" { + try testing.expectEqual(@as(usize, 16), capacityFor(0)); + try testing.expectEqual(@as(usize, 16), capacityFor(12)); + try testing.expectEqual(@as(usize, 32), capacityFor(13)); + try testing.expectEqual(@as(usize, 2 << 20), capacityFor(1_000_000)); +} diff --git a/src/filter/fetcher.zig b/src/filter/fetcher.zig new file mode 100644 index 0000000..5a97192 --- /dev/null +++ b/src/filter/fetcher.zig @@ -0,0 +1,265 @@ +//! Blocklist download over HTTP/1.1. +//! +//! One `Fetcher` wraps a caller-owned `std.http.Client`, which owns the +//! connection pool and the CA bundle, exactly as `upstream/doh_client.zig` +//! does. This file knows nothing about parsing, files or the database: it GETs +//! a URL and streams the bytes into a writer the caller supplies. +//! +//! The body is never held whole. A blocklist can reach `max_body_bytes`, and +//! the caller writes into a temporary file anyway, so nothing here allocates. +//! +//! There is no timeout parameter and no sleep. `std.http.Client` has no +//! per-request deadline, so the caller runs `fetch` under `io.concurrent` and +//! cancels the future; this file only propagates `error.Canceled`. + +const std = @import("std"); +const transport = @import("../upstream/transport.zig"); + +pub const max_body_bytes: usize = 64 * 1024 * 1024; + +/// RFC 9110 recommends at least 8000 bytes for the redirect buffer +/// (`std.http.Client` doc comment, Client.zig:1128). +pub const redirect_buffer_len: usize = 8192; + +pub const min_transfer_buf: usize = 16 * 1024; + +pub const Error = error{ + BadUrl, + ConnectFailed, + TlsFailed, + SendFailed, + ReceiveFailed, + HttpStatus, + BodyTooLarge, + Timeout, + Canceled, + OutOfMemory, + SystemResources, + Unexpected, +}; + +pub const Result = struct { + bytes_read: u64, + status: std.http.Status, +}; + +pub const Fetcher = struct { + /// Caller-owned; shared across sources, pools connections. + http: *std.http.Client, + /// Caller-owned HTTP body transfer buffer, at least `min_transfer_buf`. + transfer_buf: []u8, + /// Caller-owned. `receiveHead` follows redirects itself and needs this to + /// outlive `Request.uri`. At least `redirect_buffer_len`. + redirect_buf: []u8, + /// The status of the most recent response head, or null before the first + /// one. `error.HttpStatus` carries no `Result`, and the operator's message + /// needs the number, so it is readable here after a failed `fetch`. + last_status: ?std.http.Status = null, + + /// GETs `url` and streams the body into `w`. + pub fn fetch( + self: *Fetcher, + io: std.Io, + url: []const u8, + w: *std.Io.Writer, + ) Error!Result { + // `std.http.Client` carries the `std.Io` it was constructed with and + // takes none per request. The parameter stays in the signature because + // the manager drives every fetch through one `std.Io`. + _ = io; + + std.debug.assert(self.transfer_buf.len >= min_transfer_buf); + std.debug.assert(self.redirect_buf.len >= redirect_buffer_len); + + const uri = try parseUrl(url); + + self.last_status = null; + + var req = self.http.request(.GET, uri, .{ + .keep_alive = true, + .headers = .{ + // Identity only: a compressed transfer encoding would need + // `Response.readerDecompressing`, a decompression buffer and a + // second failure surface, for a download that runs once a day. + .accept_encoding = .{ .override = "identity" }, + }, + }) catch |err| return mapError(err, .connect); + defer req.deinit(); + + req.sendBodiless() catch |err| return mapError(err, .send); + + var resp = req.receiveHead(self.redirect_buf) catch |err| return mapError(err, .receive); + + self.last_status = resp.head.status; + if (resp.head.status != .ok) return error.HttpStatus; + + // `content-type` is deliberately not checked: blocklists are served as + // text/plain, application/octet-stream and text/html alike, and the + // compiler's invalid-line counters are the honest signal about content. + if (resp.head.content_length) |declared| { + if (declared > max_body_bytes) return error.BodyTooLarge; + } + + const body = resp.reader(self.transfer_buf); + var total: u64 = 0; + while (true) { + const n = body.readSliceShort(self.transfer_buf) catch |err| + return mapError(err, .receive); + if (n == 0) break; + total += n; + if (total > max_body_bytes) return error.BodyTooLarge; + // The caller owns `w` and can read the concrete failure from its + // own writer; this taxonomy has no member for a failing sink. + w.writeAll(self.transfer_buf[0..n]) catch return error.Unexpected; + } + + return .{ .bytes_read = total, .status = resp.head.status }; + } +}; + +/// A scheme other than `http`/`https`, an unparseable URL and a URL with no +/// host are one fault to the operator: the source row is unusable. +fn parseUrl(url: []const u8) Error!std.Uri { + const uri = std.Uri.parse(url) catch return error.BadUrl; + if (!std.mem.eql(u8, uri.scheme, "http") and + !std.mem.eql(u8, uri.scheme, "https")) return error.BadUrl; + const host = uri.host orelse return error.BadUrl; + if (host.isEmpty()) return error.BadUrl; + return uri; +} + +/// Which call failed. The phase is what decides the classification, and only +/// the call site knows it — guessing it from an error name would be wrong the +/// first time two phases shared an error. +const Phase = enum { connect, send, receive }; + +fn mapError(err: anyerror, phase: Phase) Error { + if (transport.mapLocal(err)) |local| return narrowLocal(local); + switch (err) { + error.Timeout => return error.Timeout, + // Both are ruled out by `parseUrl` before the client is touched. + error.UnsupportedUriScheme, error.UriMissingHost => return error.BadUrl, + error.TooManyHttpRedirects => return error.HttpStatus, + else => {}, + } + const err_name = @errorName(err); + if (std.mem.startsWith(u8, err_name, "Tls") or + std.mem.startsWith(u8, err_name, "Certificate")) return error.TlsFailed; + return switch (phase) { + .connect => error.ConnectFailed, + .send => error.SendFailed, + .receive => error.ReceiveFailed, + }; +} + +/// `transport.mapLocal` answers in `transport.ExchangeError`, which is wider +/// than this file's taxonomy. Both file-descriptor quotas are the same +/// exhaustion to a downloader, and `error.BufferTooSmall` cannot occur because +/// this file hands the client no undersized buffer. +fn narrowLocal(local: transport.ExchangeError) Error { + return switch (local) { + error.OutOfMemory => error.OutOfMemory, + error.SystemResources, + error.ProcessFdQuotaExceeded, + error.SystemFdQuotaExceeded, + => error.SystemResources, + error.Canceled => error.Canceled, + else => error.Unexpected, + }; +} + +const testing = std.testing; + +fn undefinedFetcher(transfer_buf: []u8, redirect_buf: []u8) Fetcher { + // `http` is never driven: every test below asserts a rejection that + // happens before the first client call. + const http: *std.http.Client = undefined; + return .{ .http = http, .transfer_buf = transfer_buf, .redirect_buf = redirect_buf }; +} + +test "fetch rejects a non-http scheme before any client use" { + var transfer_buf: [min_transfer_buf]u8 = undefined; + var redirect_buf: [redirect_buffer_len]u8 = undefined; + var f = undefinedFetcher(&transfer_buf, &redirect_buf); + var sink_buf: [0]u8 = .{}; + var discarding: std.Io.Writer.Discarding = .init(&sink_buf); + try testing.expectError( + error.BadUrl, + f.fetch(undefined, "ftp://example.com/list.txt", &discarding.writer), + ); +} + +test "fetch rejects a url with no scheme before any client use" { + var transfer_buf: [min_transfer_buf]u8 = undefined; + var redirect_buf: [redirect_buffer_len]u8 = undefined; + var f = undefinedFetcher(&transfer_buf, &redirect_buf); + var sink_buf: [0]u8 = .{}; + var discarding: std.Io.Writer.Discarding = .init(&sink_buf); + try testing.expectError(error.BadUrl, f.fetch(undefined, "x", &discarding.writer)); +} + +test "fetch rejects a url with no host before any client use" { + var transfer_buf: [min_transfer_buf]u8 = undefined; + var redirect_buf: [redirect_buffer_len]u8 = undefined; + var f = undefinedFetcher(&transfer_buf, &redirect_buf); + var sink_buf: [0]u8 = .{}; + var discarding: std.Io.Writer.Discarding = .init(&sink_buf); + try testing.expectError(error.BadUrl, f.fetch(undefined, "https://", &discarding.writer)); +} + +test "parseUrl accepts http and https urls" { + const plain = try parseUrl("http://example.com/hosts.txt"); + try testing.expectEqualStrings("http", plain.scheme); + const secure = try parseUrl("https://example.com:8443/hosts.txt"); + try testing.expectEqualStrings("https", secure.scheme); + try testing.expectEqual(@as(?u16, 8443), secure.port); +} + +test "parseUrl rejects the url forms the source table can hold" { + try testing.expectError(error.BadUrl, parseUrl("ftp://example.com/list")); + try testing.expectError(error.BadUrl, parseUrl("file:///etc/hosts")); + try testing.expectError(error.BadUrl, parseUrl("x")); + try testing.expectError(error.BadUrl, parseUrl("")); + try testing.expectError(error.BadUrl, parseUrl("https://")); +} + +test "mapError maps local errors before phase errors" { + try testing.expectEqual(error.OutOfMemory, mapError(error.OutOfMemory, .connect)); + try testing.expectEqual(error.Canceled, mapError(error.Canceled, .receive)); + try testing.expectEqual(error.Unexpected, mapError(error.Unexpected, .send)); + try testing.expectEqual(error.SystemResources, mapError(error.SystemResources, .connect)); + try testing.expectEqual( + error.SystemResources, + mapError(error.ProcessFdQuotaExceeded, .connect), + ); + try testing.expectEqual( + error.SystemResources, + mapError(error.SystemFdQuotaExceeded, .connect), + ); +} + +test "mapError maps tls errors regardless of phase" { + try testing.expectEqual(error.TlsFailed, mapError(error.TlsInitializationFailed, .connect)); + try testing.expectEqual(error.TlsFailed, mapError(error.TlsAlert, .receive)); + try testing.expectEqual(error.TlsFailed, mapError(error.CertificateExpired, .connect)); +} + +test "mapError maps a redirect overrun to HttpStatus" { + try testing.expectEqual(error.HttpStatus, mapError(error.TooManyHttpRedirects, .receive)); +} + +test "mapError maps a connect timeout to Timeout" { + try testing.expectEqual(error.Timeout, mapError(error.Timeout, .connect)); +} + +test "mapError maps remaining errors by phase" { + try testing.expectEqual(error.ConnectFailed, mapError(error.ConnectionRefused, .connect)); + try testing.expectEqual(error.SendFailed, mapError(error.WriteFailed, .send)); + try testing.expectEqual(error.ReceiveFailed, mapError(error.ReadFailed, .receive)); + try testing.expectEqual(error.ReceiveFailed, mapError(error.HttpHeadersInvalid, .receive)); +} + +test "caps are the values the memory budget was sized against" { + try testing.expectEqual(@as(usize, 64 * 1024 * 1024), max_body_bytes); + try testing.expectEqual(@as(usize, 8192), redirect_buffer_len); +} diff --git a/src/filter/filter_integration_test.zig b/src/filter/filter_integration_test.zig new file mode 100644 index 0000000..fdefd75 --- /dev/null +++ b/src/filter/filter_integration_test.zig @@ -0,0 +1,1419 @@ +//! Milestone-5 filtering integration tests (spec S9.1): real compiled files, +//! real HTTP over the loopback, real snapshot swaps, and the local-DNS path +//! end to end. +//! +//! This lives in its own file because it needs `@import("build_options")`, which +//! only exists when the compilation is driven by `build.zig`. The body compiles +//! on every `zig build test` run, so it cannot rot, and every test skips at run +//! time unless `-Dintegration` is passed. +//! +//! Hermetic: every file is written inside one `std.testing.tmpDir`, every socket +//! is on 127.0.0.1, and every database is `:memory:`. Nothing leaves the +//! machine and nothing outlives its test. +//! +//! Two `std.Io` instances are in play, as in the milestone-3 and milestone-4 +//! suites. `std.testing.tmpDir` creates and removes its directory against +//! `std.testing.io`; everything under test runs on a `std.Io.Threaded` the test +//! owns, because the manager and the forward client both race tasks against a +//! budget. A `std.Io.Dir` is a handle, so the two agree on what they address. + +const std = @import("std"); +const build_options = @import("build_options"); +const net = std.Io.net; + +const model = @import("../config/model.zig"); +const db = @import("../storage/db.zig"); +const migrations = @import("../storage/migrations.zig"); +const context = @import("../storage/repositories/context.zig"); +const groups_repo = @import("../storage/repositories/groups_repo.zig"); +const local_repo = @import("../storage/repositories/local_repo.zig"); +const sources_repo = @import("../storage/repositories/sources_repo.zig"); + +const compiler = @import("compiler.zig"); +const fetcher = @import("fetcher.zig"); +const manager = @import("manager.zig"); +const matcher = @import("matcher.zig"); +const response = @import("response.zig"); + +const forward_client = @import("../local/forward_client.zig"); +const forward_zones = @import("../local/forward_zones.zig"); +const records = @import("../local/records.zig"); + +const edns = @import("../dns/edns.zig"); +const header = @import("../dns/header.zig"); +const name = @import("../dns/name.zig"); +const packet = @import("../dns/packet.zig"); +const question = @import("../dns/question.zig"); +const record = @import("../dns/record.zig"); +const types = @import("../dns/types.zig"); +const transport = @import("../upstream/transport.zig"); + +const testing = std.testing; +const Sha256 = std.crypto.hash.sha2.Sha256; + +/// Long enough that a loopback exchange cannot lose to scheduling, short enough +/// that a wedged server fails the run instead of hanging it. +const budget: std.Io.Clock.Duration = .{ .raw = .fromSeconds(10), .clock = .awake }; + +/// The forward-zone read timeout. Case 15 asserts a silent resolver gives up +/// inside twice this, so it has to be short enough to keep the run quick and +/// long enough that a loopback answer always beats it. +const read_timeout: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(200), .clock = .awake }; + +const file_limit: std.Io.Limit = .limited(8 * 1024 * 1024); + +const source_url = "http://127.0.0.1/list.txt"; +const source_name = "test list"; + +// --------------------------------------------------------------------------- +// fixtures: compiled bodies +// --------------------------------------------------------------------------- + +/// Names the compiler must drop, whatever list they arrive in: a single-label +/// name in a blocklist black-holes the loopback names of every client on the +/// LAN (spec S2.3f). +const single_label_names = [_][]const u8{ "localhost", "broadcasthost", "ip6-localhost" }; + +const fixture_domains = 5_000; + +/// The sampling stride over the fixture's domains. Coprime with +/// `fixture_domains` so twenty samples never repeat. +const sample_stride = 263; + +/// A hosts-format list with sink addresses, comments, wildcard entries, the +/// single-label names every real hosts file carries, and a regex line. +fn hostsFixture(gpa: std.mem.Allocator) ![]u8 { + var out: std.Io.Writer.Allocating = .init(gpa); + errdefer out.deinit(); + const w = &out.writer; + + try w.writeAll("# nxdns test fixture\n"); + for (single_label_names) |bare| try w.print("127.0.0.1 {s}\n", .{bare}); + for (0..fixture_domains) |i| { + try w.print("0.0.0.0 ad{d}.example.com # tracker\n", .{i}); + if (i % 500 == 0) try w.print("0.0.0.0 *.wild{d}.example.net\n", .{i}); + } + try w.writeAll("/ads[0-9]+/\n"); + return out.toOwnedSlice(); +} + +/// A small list whose every counter is a known number: two domains, one +/// wildcard, one regex line and one single-label name. +const http_body = + "# small hosts list\n" ++ + "0.0.0.0 ads.example.com # advertising\n" ++ + "0.0.0.0 tracker.example.net\n" ++ + "0.0.0.0 *.wild.example.org\n" ++ + "127.0.0.1 localhost\n" ++ + "/regex[0-9]+/\n"; + +const http_domains: i64 = 2; +const http_wildcards: i64 = 1; +const http_regex: i64 = 1; + +/// Compiles `text` into `.list` and `.wild` under `dir`, exactly as +/// the manager's compile stage does, and returns the compiler's own result. +fn compileToFiles( + gpa: std.mem.Allocator, + io: std.Io, + dir: std.Io.Dir, + base: []const u8, + text: []const u8, +) !compiler.Result { + var list_name_buf: [64]u8 = undefined; + var wild_name_buf: [64]u8 = undefined; + const list_name = try std.fmt.bufPrint(&list_name_buf, "{s}.list", .{base}); + const wild_name = try std.fmt.bufPrint(&wild_name_buf, "{s}.wild", .{base}); + + const list_file = try dir.createFile(io, list_name, .{ .permissions = .fromMode(0o600) }); + defer list_file.close(io); + const wild_file = try dir.createFile(io, wild_name, .{ .permissions = .fromMode(0o600) }); + defer wild_file.close(io); + + const buffers = try gpa.alloc(u8, 2 * 16 * 1024); + defer gpa.free(buffers); + + var r: std.Io.Reader = .fixed(text); + var list_w = list_file.writer(io, buffers[0 .. 16 * 1024]); + var wild_w = wild_file.writer(io, buffers[16 * 1024 ..]); + + const result = try compiler.compile(gpa, &r, .hosts, &list_w.interface, &wild_w.interface); + try list_w.interface.flush(); + try wild_w.interface.flush(); + return result; +} + +/// The two compiled bodies of one source, read back from disk with their +/// headers stripped, exactly as `Manager.reload` reads them. +const Bodies = struct { + list: []u8, + wild: []u8, + + fn read(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, base: []const u8) !Bodies { + var list_name_buf: [64]u8 = undefined; + var wild_name_buf: [64]u8 = undefined; + const list = try dir.readFileAlloc( + io, + try std.fmt.bufPrint(&list_name_buf, "{s}.list", .{base}), + gpa, + file_limit, + ); + errdefer gpa.free(list); + const wild = try dir.readFileAlloc( + io, + try std.fmt.bufPrint(&wild_name_buf, "{s}.wild", .{base}), + gpa, + file_limit, + ); + return .{ .list = list, .wild = wild }; + } + + fn deinit(self: *Bodies, gpa: std.mem.Allocator) void { + gpa.free(self.list); + gpa.free(self.wild); + self.* = undefined; + } +}; + +/// A one-group, one-source snapshot over two compiled bodies. +fn snapshotOver(gpa: std.mem.Allocator, list_body: []const u8, wild_body: []const u8) !matcher.Snapshot { + const sources = [_]model.BlocklistSource{.{ .url = source_url, .name = source_name }}; + const compiled = [_]?matcher.Snapshot.Compiled{ + .{ .list_body = list_body, .wild_body = wild_body }, + }; + return matcher.Snapshot.build(gpa, .{ + .groups = &.{.{ .name = "default" }}, + .group_ids = &.{1}, + .group_sources = &.{.{ .group = "default", .source_url = source_url }}, + .sources = &sources, + .source_ids = &.{7}, + .rules = &.{}, + .clients = &.{}, + .prefixes = &.{}, + .compiled = &compiled, + .seed = 0x5eed, + .generation = 1, + }); +} + +fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 { + var hasher = Sha256.init(.{}); + hasher.update(list_body); + hasher.update(wild_body); + var digest: [Sha256.digest_length]u8 = undefined; + hasher.final(&digest); + return std.fmt.bytesToHex(digest, .lower); +} + +// --------------------------------------------------------------------------- +// fixtures: database +// --------------------------------------------------------------------------- + +fn openMigrated() !db.Db { + var database = try db.Db.open(":memory:", .{ .mode = .memory }); + errdefer database.close(); + try db.applyPragmas(&database, .{}); + _ = try migrations.migrate(&database); + return database; +} + +/// One enabled source linked to the `default` group, which migration step 1 +/// already seeded. Returns the source's row id, which is what the compiled +/// files are named after. +fn seedSource(database: *db.Db, url: []const u8) !i64 { + const group_id = (try groups_repo.groupId(database, "default")) orelse + return error.TestGroupMissing; + + try sources_repo.insertBlocklistSource(database, .{ .url = url, .name = source_name }, .{}); + var rows = try listRows(database); + defer rows.deinit(); + const source_id = (try rows.byUrl(url)).id; + + var group_ids: context.IdMap = .empty; + defer group_ids.deinit(testing.allocator); + try group_ids.put(testing.allocator, "default", group_id); + var source_ids: context.IdMap = .empty; + defer source_ids.deinit(testing.allocator); + try source_ids.put(testing.allocator, url, source_id); + + try groups_repo.insertGroupSource( + database, + .{ .group = "default", .source_url = url }, + .{ .group_ids = &group_ids, .source_ids = &source_ids }, + ); + return source_id; +} + +const Rows = struct { + list: std.ArrayList(sources_repo.SourceRow), + + fn deinit(self: *Rows) void { + sources_repo.freeSourceRows(testing.allocator, self.list.items); + self.list.deinit(testing.allocator); + } + + fn byUrl(self: *const Rows, url: []const u8) !sources_repo.SourceRow { + for (self.list.items) |row| { + if (std.mem.eql(u8, row.url, url)) return row; + } + return error.TestSourceMissing; + } +}; + +fn listRows(database: *db.Db) !Rows { + return .{ .list = try sources_repo.listSourceRows(database, testing.allocator) }; +} + +// --------------------------------------------------------------------------- +// fixtures: the manager and everything it needs +// --------------------------------------------------------------------------- + +/// Heap-allocated because the fetcher points at the client and the manager +/// points at the fetcher: moving this struct would dangle both. +const Env = struct { + gpa: std.mem.Allocator, + threaded: std.Io.Threaded, + tmp: testing.TmpDir, + database: db.Db, + http: std.http.Client, + transfer_buf: [fetcher.min_transfer_buf]u8, + redirect_buf: [fetcher.redirect_buffer_len]u8, + f: fetcher.Fetcher, + mgr: manager.Manager, + + fn create(gpa: std.mem.Allocator) !*Env { + const self = try gpa.create(Env); + errdefer gpa.destroy(self); + + self.gpa = gpa; + self.threaded = .init(gpa, .{}); + self.tmp = testing.tmpDir(.{ .iterate = true }); + self.database = try openMigrated(); + self.http = .{ .allocator = gpa, .io = self.threaded.io() }; + self.f = .{ + .http = &self.http, + .transfer_buf = &self.transfer_buf, + .redirect_buf = &self.redirect_buf, + }; + self.mgr = try manager.Manager.init( + gpa, + &self.database, + .{ .dir = self.tmp.dir }, + &self.f, + .{}, + budget, + ); + return self; + } + + fn destroy(self: *Env) void { + const gpa = self.gpa; + self.mgr.deinit(self.io()); + self.http.deinit(); + self.database.close(); + self.threaded.deinit(); + self.tmp.cleanup(); + gpa.destroy(self); + } + + fn io(self: *Env) std.Io { + return self.threaded.io(); + } + + /// The blocklist directory the manager writes into, created if the manager + /// has not created it yet. + fn blocklistDir(self: *Env) !std.Io.Dir { + _ = try self.tmp.dir.createDirPathStatus(self.io(), "blocklists", .fromMode(0o700)); + return self.tmp.dir.openDir(self.io(), "blocklists", .{ .iterate = true }); + } + + fn status(self: *Env, id: i64) !manager.SourceStatus { + var out: [8]manager.SourceStatus = undefined; + const kept = self.mgr.statusSnapshot(self.io(), &out); + for (out[0..kept]) |entry| { + if (entry.id == id) return entry; + } + return error.TestStatusMissing; + } + + /// The decision the published snapshot makes for `domain` in the default + /// group, and the generation that made it. + fn evaluate(self: *Env, domain: []const u8) !struct { matcher.Decision, u64 } { + const handle = self.mgr.acquire(self.io()) orelse return error.TestNoSnapshot; + defer handle.release(self.io()); + const group = handle.snapshot.groupIndexByName("default") orelse + return error.TestGroupMissing; + return .{ handle.snapshot.evaluate(group, domain), handle.snapshot.generation }; + } +}; + +// --------------------------------------------------------------------------- +// fixtures: the loopback http server +// --------------------------------------------------------------------------- + +const Route = enum(u8) { body, redirect, not_found, oversize }; + +const redirect_path = "/redirected.txt"; + +/// Past `fetcher.max_body_bytes`. The cap is a constant, so the only way to +/// reach it in a test is a declared length: the fetcher refuses on the response +/// head, before a byte of body streams. +const oversize_length = "104857600"; + +const HttpFixture = struct { + server: net.Server, + body: []const u8, + route: std.atomic.Value(u8), + + fn init(io: std.Io, body: []const u8) !HttpFixture { + const local: net.IpAddress = try .parse("127.0.0.1", 0); + return .{ + .server = try local.listen(io, .{ .reuse_address = true }), + .body = body, + .route = .init(@intFromEnum(Route.body)), + }; + } + + fn deinit(self: *HttpFixture, io: std.Io) void { + self.server.deinit(io); + } + + fn url(self: *const HttpFixture, buf: []u8) ![]const u8 { + return std.fmt.bufPrint(buf, "http://127.0.0.1:{d}/list.txt", .{ + self.server.socket.address.getPort(), + }); + } + + fn setRoute(self: *HttpFixture, route: Route) void { + self.route.store(@intFromEnum(route), .release); + } + + /// One request per connection, for as many connections as arrive. Returns + /// when the task is canceled, which is what closes the accept. + fn serve(self: *HttpFixture, io: std.Io) void { + while (true) { + var stream = self.server.accept(io) catch return; + defer stream.close(io); + + var read_buf: [8192]u8 = undefined; + var write_buf: [8192]u8 = undefined; + var reader = stream.reader(io, &read_buf); + var writer = stream.writer(io, &write_buf); + var http: std.http.Server = .init(&reader.interface, &writer.interface); + + var request = http.receiveHead() catch continue; + self.respond(&request) catch continue; + } + } + + /// Every reply closes the connection. A keep-alive reply would leave the + /// fetcher holding the connection open while this server waits to accept a + /// second one that never comes (milestone-5 spec, S9 note from S8). + fn respond(self: *HttpFixture, request: *std.http.Server.Request) !void { + switch (@as(Route, @enumFromInt(self.route.load(.acquire)))) { + .body => try request.respond(self.body, .{ .keep_alive = false }), + .redirect => if (std.mem.eql(u8, request.head.target, redirect_path)) + try request.respond(self.body, .{ .keep_alive = false }) + else + try request.respond("", .{ + .keep_alive = false, + .status = .found, + .extra_headers = &.{.{ .name = "location", .value = redirect_path }}, + }), + .not_found => try request.respond("no such list", .{ + .keep_alive = false, + .status = .not_found, + }), + .oversize => try request.respond("", .{ + .keep_alive = false, + .transfer_encoding = .none, + .extra_headers = &.{.{ .name = "content-length", .value = oversize_length }}, + }), + } + } +}; + +// --------------------------------------------------------------------------- +// fixtures: the loopback dns responder +// --------------------------------------------------------------------------- + +const DnsMode = enum { answer, truncate, silent, wrong_id }; + +/// A query for nas.lan.home A: id 0x2222, RD set, one question, no OPT. +const forward_query = + "\x22\x22\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00" ++ + "\x03nas\x03lan\x04home\x00\x00\x01\x00\x01"; + +const forward_rdata = [4]u8{ 192, 168, 1, 10 }; +const forward_ttl: u32 = 120; + +const DnsFixture = struct { + socket: net.Socket, + listener: ?net.Server = null, + mode: DnsMode, + + fn port(self: *const DnsFixture) u16 { + return self.socket.address.getPort(); + } + + fn deinit(self: *DnsFixture, io: std.Io) void { + if (self.listener) |*listener| listener.deinit(io); + self.socket.close(io); + } + + fn serveUdp(self: *DnsFixture, io: std.Io) void { + var in: [2048]u8 = undefined; + var out: [2048]u8 = undefined; + while (true) { + const msg = self.socket.receive(io, &in) catch return; + if (self.mode == .silent) continue; + const reply = answerFor(msg.data, &out, self.mode) catch continue; + self.socket.send(io, &msg.from, reply) catch return; + } + } + + /// The TCP side always answers in full: it exists only because the UDP + /// answer was truncated. + fn serveTcp(self: *DnsFixture, io: std.Io) void { + const listener = &(self.listener.?); + var out: [2048]u8 = undefined; + while (true) { + var stream = listener.accept(io) catch return; + defer stream.close(io); + + var read_buf: [2048]u8 = undefined; + var write_buf: [2048]u8 = undefined; + var reader = stream.reader(io, &read_buf); + var writer = stream.writer(io, &write_buf); + + const framed = reader.interface.takeArray(transport.prefix_len) catch continue; + const len = transport.parsePrefix(framed.*); + const query = reader.interface.take(len) catch continue; + const reply = answerFor(query, &out, .answer) catch continue; + + writer.interface.writeAll(&transport.framePrefix(@intCast(reply.len))) catch continue; + writer.interface.writeAll(reply) catch continue; + writer.interface.flush() catch continue; + } + } +}; + +fn answerFor(query: []const u8, out: []u8, mode: DnsMode) ![]u8 { + const request = try packet.parse(query); + const q = packet.firstQuestion(request) orelse return error.TestMissingQuestion; + + var b = try packet.ResponseBuilder.init(out, request.header, q); + if (mode != .truncate) try b.addAnswer(q.name, .a, .in, forward_ttl, &forward_rdata); + const bytes = b.finish(); + + switch (mode) { + // RFC 1035 §4.1.1: TC is bit 0x02 of the second flags byte. The builder + // has no truncation switch because nothing in nxdns truncates. + .truncate => bytes[2] |= 0x02, + .wrong_id => packet.setId(bytes, request.header.id +% 1), + else => {}, + } + return bytes; +} + +/// A UDP socket and a TCP listener on the same loopback port, which is what a +/// resolver URL names. The two port spaces are independent, so an ephemeral UDP +/// port is nearly always free for TCP; a collision is retried rather than +/// failed, because it is not a defect in anything under test. +fn bindPair(io: std.Io) !struct { net.Socket, net.Server } { + var attempts: usize = 0; + while (attempts < 32) : (attempts += 1) { + const any: net.IpAddress = try .parse("127.0.0.1", 0); + const socket = try any.bind(io, .{ .mode = .dgram }); + const paired: net.IpAddress = try .parse("127.0.0.1", socket.address.getPort()); + const listener = paired.listen(io, .{ .reuse_address = true }) catch |err| switch (err) { + error.AddressInUse => { + socket.close(io); + continue; + }, + else => { + socket.close(io); + return err; + }, + }; + return .{ socket, listener }; + } + return error.TestNoFreePortPair; +} + +fn expectForwardAnswer(reply: []const u8) !void { + const p = try packet.parse(reply); + try testing.expectEqual(@as(u16, 0x2222), p.header.id); + try testing.expect(p.header.flags.qr); + try testing.expectEqual(@as(u16, 1), p.header.ancount); + + var it = packet.answers(p); + const rec = (try it.next()) orelse return error.TestMissingAnswer; + try testing.expectEqual(types.Type.a, rec.rtype); + try testing.expectEqual(forward_ttl, rec.ttl); + try testing.expectEqual(forward_rdata, try record.rdataA(reply, rec)); +} + +// --------------------------------------------------------------------------- +// 1–3: compile, load and match through real files +// --------------------------------------------------------------------------- + +test "1: a compiled hosts fixture loads into a snapshot that blocks its domains" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var tmp = testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + + const text = try hostsFixture(gpa); + defer gpa.free(text); + + const result = try compileToFiles(gpa, io, tmp.dir, "1", text); + try testing.expectEqual(@as(u32, fixture_domains), result.counts.domains); + try testing.expectEqual(@as(u32, 1), result.counts.skipped_regex); + // The three single-label names are the only invalid candidates here. + try testing.expectEqual(@as(u32, single_label_names.len), result.counts.invalid); + + var bodies = try Bodies.read(gpa, io, tmp.dir, "1"); + defer bodies.deinit(gpa); + + for (single_label_names) |bare| { + try testing.expect(std.mem.find(u8, bodies.list, bare) == null); + } + + var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild); + defer snapshot.deinit(); + const group = snapshot.groupIndexByName("default").?; + + var buf: [64]u8 = undefined; + for (0..20) |j| { + const member = try std.fmt.bufPrint(&buf, "ad{d}.example.com", .{j * sample_stride}); + const decision = snapshot.evaluate(group, member); + try testing.expect(decision.blocked); + try testing.expectEqual(matcher.Reason.blocklist_domain, decision.reason); + } + for (0..20) |j| { + const stranger = try std.fmt.bufPrint(&buf, "nomatch{d}.example.org", .{j}); + try testing.expect(!snapshot.evaluate(group, stranger).blocked); + } + + // A `.wild` entry covers proper subdomains and not the apex. + try testing.expect(snapshot.evaluate(group, "a.wild0.example.net").blocked); + try testing.expect(!snapshot.evaluate(group, "wild0.example.net").blocked); +} + +test "2: recompiling the same fixture produces byte-identical files and checksums" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var tmp = testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + + const text = try hostsFixture(gpa); + defer gpa.free(text); + + var first_dir = try tmp.dir.createDirPathOpen(io, "first", .{}); + defer first_dir.close(io); + var second_dir = try tmp.dir.createDirPathOpen(io, "second", .{}); + defer second_dir.close(io); + + const first = try compileToFiles(gpa, io, first_dir, "1", text); + const second = try compileToFiles(gpa, io, second_dir, "1", text); + + try testing.expectEqualStrings(&first.checksum, &second.checksum); + try testing.expectEqual(first.counts, second.counts); + + var first_bodies = try Bodies.read(gpa, io, first_dir, "1"); + defer first_bodies.deinit(gpa); + var second_bodies = try Bodies.read(gpa, io, second_dir, "1"); + defer second_bodies.deinit(gpa); + + try testing.expectEqualSlices(u8, first_bodies.list, second_bodies.list); + try testing.expectEqualSlices(u8, first_bodies.wild, second_bodies.wild); + + // The checksum the compiler reported is the one over the two bodies it + // wrote, which is what the manager stores and compares against. + try testing.expectEqualStrings( + &bodyChecksum(first_bodies.list, first_bodies.wild), + &first.checksum, + ); +} + +test "3: a damaged compiled file never replaces a serving snapshot with a worse one" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + const env = try Env.create(gpa); + defer env.destroy(); + const io = env.io(); + + const id = try seedSource(&env.database, source_url); + + const good_list = "aaa.example.com\nbbb.example.com\n"; + const good_wild = "ccc.example.com\n"; + + var dir = try env.blocklistDir(); + defer dir.close(io); + + var name_buf: [64]u8 = undefined; + const list_name = try std.fmt.bufPrint(&name_buf, "{d}.list", .{id}); + try dir.writeFile(io, .{ .sub_path = list_name, .data = good_list }); + var wild_name_buf: [64]u8 = undefined; + const wild_name = try std.fmt.bufPrint(&wild_name_buf, "{d}.wild", .{id}); + try dir.writeFile(io, .{ .sub_path = wild_name, .data = good_wild }); + + try sources_repo.updateSourceStats(&env.database, id, .{ + .last_updated = 1_700_000_000, + .domain_count = 2, + .wildcard_count = 1, + .skipped_regex_count = 0, + .checksum = &bodyChecksum(good_list, good_wild), + }); + + try env.mgr.reload(io); + { + const decision, const generation = try env.evaluate("aaa.example.com"); + try testing.expect(decision.blocked); + try testing.expectEqual(@as(u64, 1), generation); + } + + // Truncated mid-line: the checksum no longer matches, so the source is + // marked `.load_failed` and left out. The reload still succeeds, because + // one damaged list must not cost the operator every other one. + try dir.writeFile(io, .{ .sub_path = list_name, .data = "aaa.example.com\nbbb.exa" }); + try env.mgr.reload(io); + { + const decision, const generation = try env.evaluate("aaa.example.com"); + try testing.expect(!decision.blocked); + try testing.expectEqual(@as(u64, 2), generation); + try testing.expectEqual(manager.State.load_failed, (try env.status(id)).state); + } + + // Present, checksum-clean and malformed: the build fails and nothing is + // swapped, so the snapshot published above keeps serving. + const unsorted_list = "bbb.example.com\naaa.example.com\n"; + try dir.writeFile(io, .{ .sub_path = list_name, .data = unsorted_list }); + try sources_repo.updateSourceStats(&env.database, id, .{ + .last_updated = 1_700_000_000, + .domain_count = 2, + .wildcard_count = 1, + .skipped_regex_count = 0, + .checksum = &bodyChecksum(unsorted_list, good_wild), + }); + + try testing.expectError(error.NotSorted, env.mgr.reload(io)); + { + _, const generation = try env.evaluate("aaa.example.com"); + try testing.expectEqual(@as(u64, 2), generation); + } +} + +// --------------------------------------------------------------------------- +// 4–8: the fetcher against a loopback http server +// --------------------------------------------------------------------------- + +/// The whole refresh path for one source served by `fixture`, with the status +/// table synced first so `refreshSource` has somewhere to record its outcome. +fn refreshOnce(env: *Env, url: []const u8) !bool { + try env.mgr.reload(env.io()); + var rows = try listRows(&env.database); + defer rows.deinit(); + return env.mgr.refreshSource(env.io(), try rows.byUrl(url)); +} + +test "4: a 200 response is fetched, compiled, recorded and served" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + const env = try Env.create(gpa); + defer env.destroy(); + const io = env.io(); + + var fixture = try HttpFixture.init(io, http_body); + defer fixture.deinit(io); + var group: std.Io.Group = .init; + defer group.cancel(io); + try group.concurrent(io, HttpFixture.serve, .{ &fixture, io }); + + var url_buf: [64]u8 = undefined; + const url = try fixture.url(&url_buf); + const id = try seedSource(&env.database, url); + + try testing.expect(try refreshOnce(env, url)); + try env.mgr.reload(io); + + var dir = try env.blocklistDir(); + defer dir.close(io); + var bodies = try Bodies.read(gpa, io, dir, "1"); + defer bodies.deinit(gpa); + try testing.expect(std.mem.find(u8, bodies.list, "ads.example.com") != null); + + var rows = try listRows(&env.database); + defer rows.deinit(); + const row = try rows.byUrl(url); + try testing.expectEqual(id, row.id); + try testing.expectEqual(http_domains, row.domain_count); + try testing.expectEqual(http_wildcards, row.wildcard_count); + try testing.expectEqual(http_regex, row.skipped_regex_count); + try testing.expectEqual(@as(usize, 64), (row.checksum orelse return error.TestNoChecksum).len); + try testing.expect(row.last_updated != null); + + try testing.expectEqual(manager.State.ok, (try env.status(id)).state); + + { + const decision, _ = try env.evaluate("ads.example.com"); + try testing.expect(decision.blocked); + try testing.expectEqual(matcher.Reason.blocklist_domain, decision.reason); + } + { + const decision, _ = try env.evaluate("a.wild.example.org"); + try testing.expect(decision.blocked); + try testing.expectEqual(matcher.Reason.blocklist_wildcard, decision.reason); + } + { + const decision, _ = try env.evaluate("safe.example.com"); + try testing.expect(!decision.blocked); + } +} + +test "5: a redirect is followed to the same result" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + const env = try Env.create(gpa); + defer env.destroy(); + const io = env.io(); + + var fixture = try HttpFixture.init(io, http_body); + defer fixture.deinit(io); + fixture.setRoute(.redirect); + var group: std.Io.Group = .init; + defer group.cancel(io); + try group.concurrent(io, HttpFixture.serve, .{ &fixture, io }); + + var url_buf: [64]u8 = undefined; + const url = try fixture.url(&url_buf); + const id = try seedSource(&env.database, url); + + try testing.expect(try refreshOnce(env, url)); + try env.mgr.reload(io); + + try testing.expectEqual(manager.State.ok, (try env.status(id)).state); + var rows = try listRows(&env.database); + defer rows.deinit(); + try testing.expectEqual(http_domains, (try rows.byUrl(url)).domain_count); + + const decision, _ = try env.evaluate("tracker.example.net"); + try testing.expect(decision.blocked); +} + +test "6: a 404 leaves the compiled files and the snapshot untouched" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + const env = try Env.create(gpa); + defer env.destroy(); + const io = env.io(); + + var fixture = try HttpFixture.init(io, http_body); + defer fixture.deinit(io); + var group: std.Io.Group = .init; + defer group.cancel(io); + try group.concurrent(io, HttpFixture.serve, .{ &fixture, io }); + + var url_buf: [64]u8 = undefined; + const url = try fixture.url(&url_buf); + const id = try seedSource(&env.database, url); + + try testing.expect(try refreshOnce(env, url)); + try env.mgr.reload(io); + + var dir = try env.blocklistDir(); + defer dir.close(io); + var before = try Bodies.read(gpa, io, dir, "1"); + defer before.deinit(gpa); + + fixture.setRoute(.not_found); + try testing.expect(!try refreshOnce(env, url)); + + const failed = try env.status(id); + try testing.expectEqual(manager.State.fetch_failed, failed.state); + try testing.expectEqualStrings("HttpStatus", failed.errorText()); + + var after = try Bodies.read(gpa, io, dir, "1"); + defer after.deinit(gpa); + try testing.expectEqualSlices(u8, before.list, after.list); + try testing.expectEqualSlices(u8, before.wild, after.wild); + + try env.mgr.reload(io); + const decision, _ = try env.evaluate("ads.example.com"); + try testing.expect(decision.blocked); +} + +test "7: a body over the cap fails the refresh and leaves no temporary file" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + const env = try Env.create(gpa); + defer env.destroy(); + const io = env.io(); + + var fixture = try HttpFixture.init(io, http_body); + defer fixture.deinit(io); + fixture.setRoute(.oversize); + var group: std.Io.Group = .init; + defer group.cancel(io); + try group.concurrent(io, HttpFixture.serve, .{ &fixture, io }); + + var url_buf: [64]u8 = undefined; + const url = try fixture.url(&url_buf); + const id = try seedSource(&env.database, url); + + try testing.expect(!try refreshOnce(env, url)); + + const failed = try env.status(id); + try testing.expectEqual(manager.State.fetch_failed, failed.state); + try testing.expectEqualStrings("BodyTooLarge", failed.errorText()); + + var dir = try env.blocklistDir(); + defer dir.close(io); + var it = dir.iterate(); + while (try it.next(io)) |entry| { + try testing.expect(!std.mem.endsWith(u8, entry.name, ".tmp")); + } +} + +test "8: refetching identical content skips the rewrite and still moves last_updated" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + const env = try Env.create(gpa); + defer env.destroy(); + const io = env.io(); + + var fixture = try HttpFixture.init(io, http_body); + defer fixture.deinit(io); + var group: std.Io.Group = .init; + defer group.cancel(io); + try group.concurrent(io, HttpFixture.serve, .{ &fixture, io }); + + var url_buf: [64]u8 = undefined; + const url = try fixture.url(&url_buf); + const id = try seedSource(&env.database, url); + + try testing.expect(try refreshOnce(env, url)); + + var dir = try env.blocklistDir(); + defer dir.close(io); + const before = try dir.statFile(io, "1.list", .{}); + + // A refresh inside the same wall-clock second would write back the same + // number, so the stored value is moved out of the way first. + try sources_repo.updateSourceStats(&env.database, id, .{ + .last_updated = 1_000, + .domain_count = http_domains, + .wildcard_count = http_wildcards, + .skipped_regex_count = http_regex, + .checksum = blk: { + var rows = try listRows(&env.database); + defer rows.deinit(); + var stored: [64]u8 = undefined; + @memcpy(&stored, (try rows.byUrl(url)).checksum orelse return error.TestNoChecksum); + break :blk &stored; + }, + }); + + try testing.expect(!try refreshOnce(env, url)); + + // `replace` renames a new file over the old one, so an untouched file keeps + // both its inode and its modification time. + const after = try dir.statFile(io, "1.list", .{}); + try testing.expectEqual(before.inode, after.inode); + try testing.expectEqual(before.mtime, after.mtime); + + var rows = try listRows(&env.database); + defer rows.deinit(); + const row = try rows.byUrl(url); + try testing.expect((row.last_updated orelse return error.TestNoTimestamp) > 1_000); + try testing.expectEqual(manager.State.ok, (try env.status(id)).state); +} + +// --------------------------------------------------------------------------- +// 9–10: the swap and the orphan sweep +// --------------------------------------------------------------------------- + +const ReloadTask = struct { + mgr: *manager.Manager, + result: manager.Manager.Error!void = {}, + + fn run(self: *ReloadTask, io: std.Io) void { + self.result = self.mgr.reload(io); + } +}; + +/// Writes one source's compiled files and records their checksum, without any +/// network: the swap and the orphan sweep care about files and rows, not about +/// where the bytes came from. +fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []const u8) !void { + const io = env.io(); + var dir = try env.blocklistDir(); + defer dir.close(io); + + var list_name_buf: [64]u8 = undefined; + var wild_name_buf: [64]u8 = undefined; + try dir.writeFile(io, .{ + .sub_path = try std.fmt.bufPrint(&list_name_buf, "{d}.list", .{id}), + .data = list_body, + }); + try dir.writeFile(io, .{ + .sub_path = try std.fmt.bufPrint(&wild_name_buf, "{d}.wild", .{id}), + .data = wild_body, + }); + + try sources_repo.updateSourceStats(&env.database, id, .{ + .last_updated = 1_700_000_000, + .domain_count = 1, + .wildcard_count = 0, + .skipped_regex_count = 0, + .checksum = &bodyChecksum(list_body, wild_body), + }); +} + +test "9: a reload swaps under a held handle and the new generation follows the release" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + const env = try Env.create(gpa); + defer env.destroy(); + const io = env.io(); + + const id = try seedSource(&env.database, source_url); + try publishFixtureFiles(env, id, "aaa.example.com\n", ""); + try env.mgr.reload(io); + + const handle = env.mgr.acquire(io) orelse return error.TestNoSnapshot; + const group = handle.snapshot.groupIndexByName("default") orelse return error.TestGroupMissing; + try testing.expectEqual(@as(u64, 1), handle.snapshot.generation); + + var task: ReloadTask = .{ .mgr = &env.mgr }; + var tasks: std.Io.Group = .init; + defer tasks.cancel(io); + try tasks.concurrent(io, ReloadTask.run, .{ &task, io }); + + // The reload takes the exclusive lock and therefore waits here. The + // snapshot this handle holds cannot change under it, and cannot be freed + // while it is read. + for (0..1000) |_| { + try testing.expect(handle.snapshot.evaluate(group, "aaa.example.com").blocked); + } + try testing.expectEqual(@as(u64, 1), handle.snapshot.generation); + + handle.release(io); + try tasks.await(io); + try task.result; + + const after = env.mgr.acquire(io) orelse return error.TestNoSnapshot; + defer after.release(io); + try testing.expectEqual(@as(u64, 2), after.snapshot.generation); + try testing.expect(after.snapshot.evaluate(group, "aaa.example.com").blocked); +} + +test "10: pruneOrphans deletes the files of a deleted source and leaves live ones" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + const env = try Env.create(gpa); + defer env.destroy(); + const io = env.io(); + + const id = try seedSource(&env.database, source_url); + try publishFixtureFiles(env, id, "aaa.example.com\n", ""); + + var dir = try env.blocklistDir(); + defer dir.close(io); + try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" }); + try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" }); + // A refresh that is still running owns its temporaries, so they are not + // orphans and must survive. + try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" }); + + try env.mgr.pruneOrphans(io); + + var live_buf: [64]u8 = undefined; + try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.list", .{id}), .{}); + try dir.access(io, "9999.raw.tmp", .{}); + try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{})); + try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{})); +} + +// --------------------------------------------------------------------------- +// 11–12: local records, from the database to the wire +// --------------------------------------------------------------------------- + +/// A query for example.com A with an EDNS(0) OPT record advertising 4096 +/// bytes: id 0x1234, RD set, one question, one additional. Only its header and +/// its OPT record are used; each test supplies its own question. +const opt_query = + "\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x01" ++ + "\x07example\x03com\x00\x00\x01\x00\x01" ++ + "\x00\x00\x29\x10\x00\x00\x00\x00\x00\x00\x00"; + +fn requestHeader() header.Header { + return (packet.parse(opt_query) catch unreachable).header; +} + +/// Answers `qname`/`qtype` from `table`, exactly as the Phase 7 handler will: +/// look the name up, then write the run it returns into a response. +fn answerLocal( + table: *const records.Records, + buf: []u8, + qname: []const u8, + qtype: types.Type, +) ![]u8 { + const owner = try name.fromText(qname); + const q: question.Question = .{ .name = owner, .qtype = qtype, .qclass = .in }; + var b = try packet.ResponseBuilder.init(buf, requestHeader(), q); + try records.writeAnswers(&b, owner, table.lookup(qname, qtype)); + return b.finish(); +} + +fn localTable(gpa: std.mem.Allocator, database: *db.Db) !records.Records { + var rows = try local_repo.listLocalRecords(database, gpa); + defer rows.deinit(gpa); + defer local_repo.freeLocalRecords(gpa, rows.items); + return records.Records.build(gpa, rows.items); +} + +test "11: a local A record read from the database answers an A query" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + var database = try openMigrated(); + defer database.close(); + + try local_repo.insertLocalRecord(&database, .{ + .name = "nas.lan", + .rtype = .a, + .value = "192.168.1.10", + .ttl = 600, + }, .{}); + + var table = try localTable(gpa, &database); + defer table.deinit(gpa); + + var buf: [512]u8 = undefined; + const bytes = try answerLocal(&table, &buf, "nas.lan", .a); + + const p = try packet.parse(bytes); + try testing.expectEqual(@as(u16, 1), p.header.ancount); + var it = packet.answers(p); + const rec = (try it.next()) orelse return error.TestMissingAnswer; + try testing.expectEqual(types.Type.a, rec.rtype); + try testing.expectEqual(@as(u32, 600), rec.ttl); + try testing.expectEqual([4]u8{ 192, 168, 1, 10 }, try record.rdataA(bytes, rec)); + try testing.expect((try it.next()) == null); + + // A type the name has no record of is NODATA, not a wrong answer. + var mx_buf: [512]u8 = undefined; + const mx = try answerLocal(&table, &mx_buf, "nas.lan", .mx); + try testing.expectEqual(@as(u16, 0), (try packet.parse(mx)).header.ancount); + try testing.expect(table.hasName("nas.lan")); +} + +test "12: a local CNAME answers an A query with the CNAME alone" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + var database = try openMigrated(); + defer database.close(); + + try local_repo.insertLocalRecord(&database, .{ + .name = "www.lan", + .rtype = .cname, + .value = "nas.lan", + .ttl = 300, + }, .{}); + + var table = try localTable(gpa, &database); + defer table.deinit(gpa); + + var buf: [512]u8 = undefined; + const bytes = try answerLocal(&table, &buf, "www.lan", .a); + + const p = try packet.parse(bytes); + try testing.expectEqual(@as(u16, 1), p.header.ancount); + var it = packet.answers(p); + const rec = (try it.next()) orelse return error.TestMissingAnswer; + try testing.expectEqual(types.Type.cname, rec.rtype); + try testing.expectEqual(@as(u32, 300), rec.ttl); + try testing.expectEqualSlices( + u8, + (try name.fromText("nas.lan")).wire(), + (try record.rdataCname(bytes, rec)).wire(), + ); + try testing.expect((try it.next()) == null); +} + +// --------------------------------------------------------------------------- +// 13–16: forward zones over the loopback +// --------------------------------------------------------------------------- + +/// Seeds `lan.home` pointing at a loopback resolver and returns the zone table +/// the caller matches against. +fn seedZone( + gpa: std.mem.Allocator, + database: *db.Db, + scheme: []const u8, + port: u16, +) !forward_zones.Zones { + var url_buf: [64]u8 = undefined; + const resolver = try std.fmt.bufPrint(&url_buf, "{s}://127.0.0.1:{d}", .{ scheme, port }); + try local_repo.insertForwardZone(database, .{ .zone = "lan.home", .resolver = resolver }, .{}); + + var rows = try local_repo.listForwardZones(database, gpa); + defer rows.deinit(gpa); + defer local_repo.freeForwardZones(gpa, rows.items); + return forward_zones.Zones.build(gpa, rows.items); +} + +test "13: a forward zone reaches its own resolver over udp" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var database = try openMigrated(); + defer database.close(); + + const local: net.IpAddress = try .parse("127.0.0.1", 0); + var fixture: DnsFixture = .{ + .socket = try local.bind(io, .{ .mode = .dgram }), + .mode = .answer, + }; + defer fixture.deinit(io); + + var group: std.Io.Group = .init; + defer group.cancel(io); + try group.concurrent(io, DnsFixture.serveUdp, .{ &fixture, io }); + + var zones = try seedZone(gpa, &database, "udp", fixture.port()); + defer zones.deinit(gpa); + + const zone = zones.match("nas.lan.home") orelse return error.TestNoZone; + try testing.expectEqualStrings("lan.home", zone.zone); + try testing.expectEqual(fixture.port(), zone.resolver.port); + // A name outside the zone is not this resolver's business. + try testing.expect(zones.match("notlan.home") == null); + + var frame_buf: [forward_client.min_frame_buf]u8 = undefined; + var client: forward_client.ForwardClient = .init(zone.resolver, &frame_buf, read_timeout); + + var reply_buf: [2048]u8 = undefined; + try expectForwardAnswer(try client.exchange(io, forward_query, &reply_buf)); + + try testing.expectEqual(@as(u64, 1), client.stats.queries); + try testing.expectEqual(@as(u64, 0), client.stats.udp_truncated); + try testing.expectEqual(@as(u64, 0), client.stats.failures); +} + +test "14: a truncated udp answer is retried over tcp" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var database = try openMigrated(); + defer database.close(); + + const socket, const listener = try bindPair(io); + var fixture: DnsFixture = .{ .socket = socket, .listener = listener, .mode = .truncate }; + defer fixture.deinit(io); + + var group: std.Io.Group = .init; + defer group.cancel(io); + try group.concurrent(io, DnsFixture.serveUdp, .{ &fixture, io }); + try group.concurrent(io, DnsFixture.serveTcp, .{ &fixture, io }); + + var zones = try seedZone(gpa, &database, "udp", fixture.port()); + defer zones.deinit(gpa); + const zone = zones.match("a.b.lan.home") orelse return error.TestNoZone; + + var frame_buf: [forward_client.min_frame_buf]u8 = undefined; + var client: forward_client.ForwardClient = .init(zone.resolver, &frame_buf, read_timeout); + + var reply_buf: [2048]u8 = undefined; + try expectForwardAnswer(try client.exchange(io, forward_query, &reply_buf)); + + try testing.expectEqual(@as(u64, 1), client.stats.udp_truncated); + try testing.expectEqual(@as(u64, 0), client.stats.failures); +} + +test "15: a silent resolver times out inside the configured budget" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var database = try openMigrated(); + defer database.close(); + + const local: net.IpAddress = try .parse("127.0.0.1", 0); + var fixture: DnsFixture = .{ + .socket = try local.bind(io, .{ .mode = .dgram }), + .mode = .silent, + }; + defer fixture.deinit(io); + + var group: std.Io.Group = .init; + defer group.cancel(io); + try group.concurrent(io, DnsFixture.serveUdp, .{ &fixture, io }); + + var zones = try seedZone(gpa, &database, "udp", fixture.port()); + defer zones.deinit(gpa); + const zone = zones.match("nas.lan.home") orelse return error.TestNoZone; + + var frame_buf: [forward_client.min_frame_buf]u8 = undefined; + var client: forward_client.ForwardClient = .init(zone.resolver, &frame_buf, read_timeout); + + var reply_buf: [2048]u8 = undefined; + const started: std.Io.Timestamp = .now(io, .awake); + try testing.expectError(error.Timeout, client.exchange(io, forward_query, &reply_buf)); + const elapsed = started.durationTo(.now(io, .awake)); + + // The deadline is not asserted to the millisecond: the receive returns a + // fraction of a millisecond early on this backend, and pinning the floor to + // the exact budget would fail on clock resolution rather than on behaviour. + const limit = read_timeout.raw.toMilliseconds(); + try testing.expect(elapsed.toMilliseconds() >= @divTrunc(limit, 2)); + try testing.expect(elapsed.toMilliseconds() < 2 * limit); + try testing.expectEqual(@as(u64, 1), client.stats.failures); +} + +test "16: a resolver answering with the wrong id is rejected" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var database = try openMigrated(); + defer database.close(); + + const local: net.IpAddress = try .parse("127.0.0.1", 0); + var fixture: DnsFixture = .{ + .socket = try local.bind(io, .{ .mode = .dgram }), + .mode = .wrong_id, + }; + defer fixture.deinit(io); + + var group: std.Io.Group = .init; + defer group.cancel(io); + try group.concurrent(io, DnsFixture.serveUdp, .{ &fixture, io }); + + var zones = try seedZone(gpa, &database, "udp", fixture.port()); + defer zones.deinit(gpa); + const zone = zones.match("nas.lan.home") orelse return error.TestNoZone; + + var frame_buf: [forward_client.min_frame_buf]u8 = undefined; + var client: forward_client.ForwardClient = .init(zone.resolver, &frame_buf, read_timeout); + + var reply_buf: [2048]u8 = undefined; + try testing.expectError( + error.ResponseMismatch, + client.exchange(io, forward_query, &reply_buf), + ); + try testing.expectEqual(@as(u64, 1), client.stats.failures); +} + +// --------------------------------------------------------------------------- +// 17: the blocked reply each mode synthesizes +// --------------------------------------------------------------------------- + +test "17: each blocking mode synthesizes the documented blocked reply" { + if (!build_options.integration) return error.SkipZigTest; + + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var tmp = testing.tmpDir(.{ .iterate = true }); + defer tmp.cleanup(); + + // The reply is synthesized for a name the compiled files actually block, so + // this case covers the decision and the response together. + const result = try compileToFiles(gpa, io, tmp.dir, "1", "0.0.0.0 ads.example.com\n"); + try testing.expectEqual(@as(u32, 1), result.counts.domains); + + var bodies = try Bodies.read(gpa, io, tmp.dir, "1"); + defer bodies.deinit(gpa); + var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild); + defer snapshot.deinit(); + + const group = snapshot.groupIndexByName("default").?; + try testing.expect(snapshot.evaluate(group, "ads.example.com").blocked); + + const ttl: u32 = 5; + const blocked = try name.fromText("ads.example.com"); + + for ([_]types.Type{ .a, .aaaa }) |qtype| { + var buf: [512]u8 = undefined; + const zero = try response.writeBlocked( + &buf, + requestHeader(), + .{ .name = blocked, .qtype = qtype, .qclass = .in }, + null, + false, + .{ .mode = .zero, .ttl = ttl }, + ); + + const p = try packet.parse(zero); + try testing.expectEqual(types.Rcode.no_error, p.header.flags.rcode); + try testing.expectEqual(@as(u16, 1), p.header.ancount); + + var it = packet.answers(p); + const rec = (try it.next()) orelse return error.TestMissingAnswer; + try testing.expectEqual(qtype, rec.rtype); + try testing.expectEqual(ttl, rec.ttl); + switch (qtype) { + .a => try testing.expectEqual([4]u8{ 0, 0, 0, 0 }, try record.rdataA(zero, rec)), + .aaaa => try testing.expectEqual( + [16]u8{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + try record.rdataAaaa(zero, rec), + ), + else => unreachable, + } + + var nx_buf: [512]u8 = undefined; + const nx = try response.writeBlocked( + &nx_buf, + requestHeader(), + .{ .name = blocked, .qtype = qtype, .qclass = .in }, + edns.parseOpt(opt_query, packet.findOptRecord(try packet.parse(opt_query)).?) catch + unreachable, + false, + .{ .mode = .nxdomain, .ttl = ttl }, + ); + + const nx_packet = try packet.parse(nx); + try testing.expectEqual(types.Rcode.nx_domain, nx_packet.header.flags.rcode); + try testing.expectEqual(@as(u16, 0), nx_packet.header.ancount); + // The EDNS echo survives the NXDOMAIN path. + try testing.expect(packet.findOptRecord(nx_packet) != null); + } +} diff --git a/src/filter/manager.zig b/src/filter/manager.zig new file mode 100644 index 0000000..da8681a --- /dev/null +++ b/src/filter/manager.zig @@ -0,0 +1,1877 @@ +//! The blocklist manager (PLAN §4): the compiled files under +//! `/blocklists/`, the refresh that produces them, the metadata +//! columns it writes back, the snapshot built from them and the swap that +//! publishes it. +//! +//! This is the only file in this milestone that touches both the database and +//! the filesystem. Everything it composes — the parsers, the compiler, the +//! domain sets, the rule sets and the snapshot — is pure and testable without +//! either. +//! +//! **The swap is an `std.Io.RwLock`, not a lock-free pointer.** PLAN §7.3 says +//! "readers lock-free"; this is a deliberate deviation. Freeing the old +//! snapshot without a lock needs epoch-based reclamation or hazard pointers: a +//! class of code that is very hard to get right and impossible to test +//! convincingly, bought for a household resolver whose target is 100 qps. A +//! shared lock held for the microseconds of one `evaluate` costs an uncontended +//! atomic pair; the writer takes the exclusive lock only on a swap, which +//! happens on refresh. The old snapshot is freed *after* `unlock` returns, and +//! the `Handle` API makes "do not retain the pointer" the only shape a caller +//! can write. +//! +//! The same lock guards the status table, which is written from the refresh +//! task and read by the API. Both critical sections are short and hold no +//! socket and no file, so the uncancelable lock forms are used: a lock this +//! code takes is always released within a few instructions. +//! +//! A second lock, `writer_lock`, serializes the writers against each other: +//! `reload`, `refreshSource`, `refreshAll`, the startup pass and +//! `pruneOrphans`. Two concurrent reloads would otherwise compute the same +//! generation and each destroy a snapshot the other had just published, and two +//! concurrent refreshes would share the fetcher's buffers and, for one source, +//! the same `.raw.tmp` / `.list.tmp` / `.wild.tmp` paths. It is held across +//! downloads and compiles, so it is a plain mutex rather than the RCU lock: +//! readers must never wait behind a refresh. The public entry points take it; +//! the `*Locked` bodies assume it and never take it again, because it is not +//! reentrant. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const model = @import("../config/model.zig"); +const db = @import("../storage/db.zig"); +const clients_repo = @import("../storage/repositories/clients_repo.zig"); +const groups_repo = @import("../storage/repositories/groups_repo.zig"); +const rules_repo = @import("../storage/repositories/rules_repo.zig"); +const sources_repo = @import("../storage/repositories/sources_repo.zig"); +const compiler = @import("compiler.zig"); +const fetcher = @import("fetcher.zig"); +const matcher = @import("matcher.zig"); +const parsers = @import("parsers.zig"); + +const log = std.log.scoped(.blocklist_manager); +const Sha256 = std.crypto.hash.sha2.Sha256; + +/// `SourceStatus.last_error` is fixed-size so the failure path allocates +/// nothing. +pub const max_error_len: usize = 128; + +/// `SourceStatus.url` is fixed-size so a copied status borrows nothing. A +/// blocklist url longer than this is truncated in the status only; the row +/// keeps it whole. +pub const max_url_len: usize = 255; + +/// A compiled body larger than this is refused at load. A source that reaches +/// it produced more than `fetcher.max_body_bytes` of names, which cannot +/// happen from a download this fetcher performed. +pub const max_compiled_bytes: usize = 128 * 1024 * 1024; + +/// Buffer size for every file stream this file opens. One buffer is live per +/// stage, and the stages do not overlap. +const io_buf_len: usize = 64 * 1024; + +/// Holds the sniff sample: `parsers.sample_lines` lines of at most +/// `compiler.max_line_len` bytes, each with its newline. A fixed byte window +/// would be spent by a handful of legal 4096-byte comment lines and the format +/// would then be decided by almost no data. +const sample_buf_len: usize = parsers.sample_lines * (compiler.max_line_len + 1); + +/// `` is at most 20 characters and the longest suffix is `.list.tmp`. +const name_buf_len: usize = 48; + +pub const Paths = struct { + /// ``, owned by the caller and left open for the manager's life. + dir: std.Io.Dir, + subdir: []const u8 = "blocklists", +}; + +/// Where one source stands. `.never_fetched` is the state of a source that has +/// no compiled files and no stored checksum, which is a fresh install rather +/// than a failure. `.no_valid_entries` is a download that compiled cleanly and +/// yielded nothing usable — an error page or a compressed body, not a +/// blocklist. +pub const State = enum { + ok, + never_fetched, + fetch_failed, + compile_failed, + no_valid_entries, + load_failed, + + /// Whether this state was recorded by a refresh rather than by a load. + /// A load outcome never overwrites one: the files a reload just read are + /// exactly the files the failed refresh could not replace, and the operator + /// still has to see why the update did not land. `SourceStatus.loaded` + /// carries the other half — whether the source is filtering at all. + pub fn isRefreshFailure(self: State) bool { + return switch (self) { + .fetch_failed, .compile_failed, .no_valid_entries => true, + .ok, .never_fetched, .load_failed => false, + }; + } +}; + +/// A status is a value with no borrowed memory, so a copy handed to the API +/// outlives every reload. The url is held inline for that reason. +pub const SourceStatus = struct { + id: i64, + state: State = .never_fetched, + /// Whether this source's compiled files were read into the most recent + /// snapshot build — that is, whether it is filtering right now. `state` + /// describes the most recent attempt to *produce* those files, which is a + /// different fact: a source whose refresh failed keeps serving what the + /// refresh did not replace, and reads `.fetch_failed` with `loaded` set. + loaded: bool = false, + last_attempt: i64 = 0, + last_success: i64 = 0, + counts: compiler.Counts = .{}, + /// A display copy of the source url, truncated at `max_url_len`. The whole + /// url is in the `blocklist_sources` row this status shares an `id` with. + url: [max_url_len]u8 = @splat(0), + url_len: u8 = 0, + last_error: [max_error_len]u8 = @splat(0), + last_error_len: u8 = 0, + + pub fn errorText(self: *const SourceStatus) []const u8 { + return self.last_error[0..self.last_error_len]; + } + + pub fn urlText(self: *const SourceStatus) []const u8 { + return self.url[0..self.url_len]; + } + + fn setUrl(self: *SourceStatus, url: []const u8) void { + const kept = @min(url.len, max_url_len); + @memcpy(self.url[0..kept], url[0..kept]); + @memset(self.url[kept..], 0); + self.url_len = @intCast(kept); + } + + fn fail(self: *SourceStatus, state: State, text: []const u8) void { + self.state = state; + const kept = @min(text.len, max_error_len); + @memcpy(self.last_error[0..kept], text[0..kept]); + @memset(self.last_error[kept..], 0); + self.last_error_len = @intCast(kept); + } + + fn succeed(self: *SourceStatus, at: i64, counts: compiler.Counts) void { + self.state = .ok; + self.counts = counts; + self.last_success = at; + self.last_error = @splat(0); + self.last_error_len = 0; + } +}; + +/// The header every compiled file carries, ahead of the body. The `sha256` +/// covers the `.list` body followed by the `.wild` body and **not** the header, +/// so it stays stable across a refetch of unchanged content while +/// `fetched_at` moves. +pub const Header = struct { + url: []const u8, + format: parsers.Format, + fetched_at: i64, + counts: compiler.Counts, + /// 64 lowercase hex characters. + checksum: []const u8, + + pub fn write(self: Header, w: *std.Io.Writer) std.Io.Writer.Error!void { + try w.writeAll("# nxdns blocklist\n"); + try w.print("# url {s}\n", .{self.url}); + try w.print("# format {s}\n", .{@tagName(self.format)}); + try w.print("# fetched_at {d}\n", .{self.fetched_at}); + try w.print("# domains {d}\n", .{self.counts.domains}); + try w.print("# wildcards {d}\n", .{self.counts.wildcards}); + try w.print("# skipped_regex {d}\n", .{self.counts.skipped_regex}); + try w.print("# skipped_unsupported {d}\n", .{self.counts.skipped_unsupported}); + try w.print("# invalid {d}\n", .{self.counts.invalid}); + try w.print("# sha256 {s}\n", .{self.checksum}); + } +}; + +/// The body of a compiled file: everything after the leading `#` lines. A file +/// with no header is all body, which is what makes a hand-written fixture a +/// legal compiled file. +pub fn stripHeader(bytes: []const u8) []const u8 { + var rest = bytes; + while (rest.len != 0 and rest[0] == '#') { + const newline = std.mem.indexOfScalar(u8, rest, '\n') orelse return rest[rest.len..]; + rest = rest[newline + 1 ..]; + } + return rest; +} + +pub const Manager = struct { + gpa: Allocator, + database: *db.Db, + paths: Paths, + fetcher: *fetcher.Fetcher, + update: model.BlocklistUpdate, + /// Bounds one download. `std.http.Client` has no per-request deadline, so + /// the fetch runs under `io.concurrent` against a sleep of this length. + total_budget: std.Io.Clock.Duration, + + lock: std.Io.RwLock, + /// Serializes the writers against each other. Never taken by a reader. + writer_lock: std.Io.Mutex, + current: ?*matcher.Snapshot, + generation: u64, + statuses: []SourceStatus, + /// Owns the `statuses` table. The entries themselves borrow nothing. + status_arena: std.heap.ArenaAllocator, + + pub const Error = error{ + OutOfMemory, + Canceled, + /// A filesystem operation on the blocklist directory failed. The + /// concrete cause is logged at `warn` where it happens: this taxonomy + /// would otherwise carry two dozen members no caller can act on + /// differently. + FileSystem, + /// The `groups` table changed between listing the groups and reading + /// their ids. Retrying the reload is the answer, and the caller is the + /// only one that can decide to. + GroupSetChanged, + } || db.Error || matcher.Snapshot.Error; + + /// The result is not copyable afterwards: `status_arena` and `lock` are + /// addressed through `self`. + pub fn init( + gpa: Allocator, + database: *db.Db, + paths: Paths, + fetcher_ptr: *fetcher.Fetcher, + update: model.BlocklistUpdate, + total_budget: std.Io.Clock.Duration, + ) Error!Manager { + return .{ + .gpa = gpa, + .database = database, + .paths = paths, + .fetcher = fetcher_ptr, + .update = update, + .total_budget = total_budget, + .lock = .init, + .writer_lock = .init, + .current = null, + .generation = 0, + .statuses = &.{}, + .status_arena = .init(gpa), + }; + } + + pub fn deinit(self: *Manager, io: std.Io) void { + self.lock.lockUncancelable(io); + const old = self.current; + self.current = null; + self.statuses = &.{}; + self.lock.unlock(io); + + if (old) |snapshot| destroySnapshot(self.gpa, snapshot); + self.status_arena.deinit(); + self.* = undefined; + } + + /// Reader side of the swap. The handle holds a shared lock: release it, and + /// do not retain `snapshot` afterwards. + pub const Handle = struct { + snapshot: *const matcher.Snapshot, + manager: *Manager, + + pub fn release(self: Handle, io: std.Io) void { + self.manager.lock.unlockShared(io); + } + }; + + /// `null` before the first successful `reload`. The caller answers + /// SERVFAIL, or forwards unfiltered, on its own policy — this file does not + /// decide that. + pub fn acquire(self: *Manager, io: std.Io) ?Handle { + self.lock.lockSharedUncancelable(io); + const snapshot = self.current orelse { + self.lock.unlockShared(io); + return null; + }; + return .{ .snapshot = snapshot, .manager = self }; + } + + /// Copies the status table for the API and for `nxdns check`. Returns the + /// number of entries written, which is `min(out.len, source count)`. + /// + /// The copies are self-contained: `SourceStatus` holds its url and its + /// error text inline, so the caller may keep them for as long as it likes + /// and a concurrent reload cannot pull memory out from under them. + pub fn statusSnapshot(self: *Manager, io: std.Io, out: []SourceStatus) usize { + self.lock.lockSharedUncancelable(io); + defer self.lock.unlockShared(io); + const kept = @min(out.len, self.statuses.len); + @memcpy(out[0..kept], self.statuses[0..kept]); + return kept; + } + + // ----------------------------------------------------------------------- + // reload + // ----------------------------------------------------------------------- + + /// Reads the database and every compiled file, builds a snapshot and swaps + /// it in. + /// + /// A source whose compiled files are missing, unreadable or checksum + /// mismatched is marked `.load_failed` and left out of the snapshot rather + /// than failing the whole reload: one bad file must not cost the operator + /// every other list. `runScheduler` refreshes exactly those sources, so the + /// state is recorded, surfaced and repaired, never silently accepted. + /// + /// A body that is present and checksum-clean but malformed fails the build + /// (`error.NotSorted`), and the previously published snapshot keeps + /// serving: nothing is swapped until the new snapshot exists. The status + /// table keeps describing that snapshot too — the table is rebuilt off to + /// the side and the load findings are written into it there, so a reload + /// that never publishes changes neither. + pub fn reload(self: *Manager, io: std.Io) Error!void { + self.writer_lock.lockUncancelable(io); + defer self.writer_lock.unlock(io); + return self.reloadLocked(io); + } + + fn reloadLocked(self: *Manager, io: std.Io) Error!void { + var rows = try sources_repo.listSourceRows(self.database, self.gpa); + defer rows.deinit(self.gpa); + defer sources_repo.freeSourceRows(self.gpa, rows.items); + + // The table this reload will publish, built where no reader can see it. + // It is installed in the swap below or freed unpublished, so a reload + // that fails leaves the previous table describing the previous + // snapshot — including the entry of a source deleted from the database, + // which that snapshot still enforces. + var candidate: ?StatusTable = try self.buildStatusTable(io, rows.items); + errdefer if (candidate) |*table| table.deinit(); + + var dir = try self.openDir(io, .{}); + defer dir.close(io); + + const sources = try self.gpa.alloc(model.BlocklistSource, rows.items.len); + defer self.gpa.free(sources); + const source_ids = try self.gpa.alloc(i64, rows.items.len); + defer self.gpa.free(source_ids); + const compiled = try self.gpa.alloc(?matcher.Snapshot.Compiled, rows.items.len); + defer self.gpa.free(compiled); + + // The file contents outlive the header stripping and are freed once the + // snapshot has copied what it needs into its own arena. + var bodies: std.ArrayList([]u8) = .empty; + defer { + for (bodies.items) |body| self.gpa.free(body); + bodies.deinit(self.gpa); + } + + // What this reload found, per source. It is applied to the status table + // only if the snapshot it describes is published: everything below here + // can still fail, and a status table describing a snapshot nobody + // serves is worse than one describing the previous one. + const outcomes = try self.gpa.alloc(LoadOutcome, rows.items.len); + defer self.gpa.free(outcomes); + + var loaded: usize = 0; + for (rows.items, sources, source_ids, compiled, outcomes) |row, *source, *source_id, *slot, *outcome| { + source_id.* = row.id; + // `is_suggested` is a UI hint the snapshot never reads. + source.* = .{ .url = row.url, .name = row.name, .enabled = row.enabled }; + outcome.* = if (row.enabled) try self.loadSource(io, dir, row, &bodies) else .disabled; + switch (outcome.*) { + .loaded => |body| { + slot.* = body; + loaded += 1; + }, + // Not loadable and therefore not enforced. Saying so here is + // what keeps `Snapshot.build`'s `MissingCompiledSource` for the + // case it is meant for: a caller that forgot to read a body. + .disabled, .failed => { + slot.* = null; + source.enabled = false; + }, + } + } + + var groups = try groups_repo.listGroups(self.database, self.gpa); + defer groups.deinit(self.gpa); + defer groups_repo.freeGroups(self.gpa, groups.items); + + const group_ids = try self.groupIds(groups.items); + defer self.gpa.free(group_ids); + + var group_sources = try groups_repo.listGroupSources(self.database, self.gpa); + defer group_sources.deinit(self.gpa); + defer groups_repo.freeGroupSources(self.gpa, group_sources.items); + + var rule_rows = try rules_repo.listRules(self.database, self.gpa); + defer rule_rows.deinit(self.gpa); + defer rules_repo.freeRules(self.gpa, rule_rows.items); + + var clients = try clients_repo.listClients(self.database, self.gpa); + defer clients.deinit(self.gpa); + defer clients_repo.freeClients(self.gpa, clients.items); + + var prefixes = try clients_repo.listClientPrefixes(self.database, self.gpa); + defer prefixes.deinit(self.gpa); + defer clients_repo.freeClientPrefixes(self.gpa, prefixes.items); + + // Query names are attacker-supplied, so a fixed seed would make + // probe-chain flooding computable offline. + var seed_bytes: [8]u8 = undefined; + io.random(&seed_bytes); + + const generation = self.generation + 1; + const snapshot = try self.gpa.create(matcher.Snapshot); + errdefer self.gpa.destroy(snapshot); + snapshot.* = try matcher.Snapshot.build(self.gpa, .{ + .groups = groups.items, + .group_ids = group_ids, + .group_sources = group_sources.items, + .sources = sources, + .source_ids = source_ids, + .rules = rule_rows.items, + .clients = clients.items, + .prefixes = prefixes.items, + .compiled = compiled, + .seed = std.mem.readInt(u64, &seed_bytes, .little), + .generation = generation, + }); + + // Read before the swap: once `current` points at it, this snapshot + // belongs to the readers and to whichever writer replaces it next. + const memory_bytes = snapshot.memoryBytes(); + + // The snapshot, the status table and the load facts land together, so a + // reader never sees a status table describing anything but the + // published snapshot. + self.lock.lockUncancelable(io); + const old = self.current; + self.current = snapshot; + self.generation = generation; + applyLoadOutcomes(candidate.?.items, rows.items, outcomes); + self.installStatuses(candidate.?); + candidate = null; + self.lock.unlock(io); + + // After `unlock`: no reader can still hold the old snapshot here, and + // `writer_lock` keeps every other writer out of this sequence. + if (old) |previous| destroySnapshot(self.gpa, previous); + + log.info("blocklist snapshot generation {d}: {d} of {d} sources loaded, {d} bytes", .{ + generation, + loaded, + rows.items.len, + memory_bytes, + }); + } + + /// What one enabled source contributes to the snapshot being built. Nothing + /// here touches the status table: the outcome is data until the swap + /// commits it. + fn loadSource( + self: *Manager, + io: std.Io, + dir: std.Io.Dir, + row: sources_repo.SourceRow, + bodies: *std.ArrayList([]u8), + ) Error!LoadOutcome { + const stored = row.checksum orelse + // No stored checksum means no successful compile has ever + // happened. A fresh install is here on every source. + return .{ .failed = .{ .state = .never_fetched, .text = "" } }; + + var list_buf: [name_buf_len]u8 = undefined; + var wild_buf: [name_buf_len]u8 = undefined; + const list_name = compiledName(&list_buf, row.id, ".list"); + const wild_name = compiledName(&wild_buf, row.id, ".wild"); + + // Reserved before the reads, so neither buffer can be orphaned by a + // failing append: `bodies` owns each one from the moment it is read. + try bodies.ensureUnusedCapacity(self.gpa, 2); + + const list_bytes = dir.readFileAlloc(io, list_name, self.gpa, .limited(max_compiled_bytes)) catch |err| { + if (err == error.OutOfMemory) return error.OutOfMemory; + return loadFailure(row, list_name, err); + }; + bodies.appendAssumeCapacity(list_bytes); + + const wild_bytes = dir.readFileAlloc(io, wild_name, self.gpa, .limited(max_compiled_bytes)) catch |err| { + if (err == error.OutOfMemory) return error.OutOfMemory; + return loadFailure(row, wild_name, err); + }; + bodies.appendAssumeCapacity(wild_bytes); + + const list_body = stripHeader(list_bytes); + const wild_body = stripHeader(wild_bytes); + + // The checksum covers both bodies together, so a crash between the two + // `replace` calls — a new `.list` beside an old `.wild` — is caught + // here and refreshed, not served as a half-updated list. + if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body))) { + log.warn("blocklist {s}: compiled files do not match the stored checksum", .{row.url}); + return .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } }; + } + + return .{ .loaded = .{ .list_body = list_body, .wild_body = wild_body } }; + } + + // ----------------------------------------------------------------------- + // refresh + // ----------------------------------------------------------------------- + + /// Downloads, compiles and atomically replaces the compiled files of one + /// source, then writes its runtime columns. + /// + /// Returns `true` only when the compiled files were replaced. Unchanged + /// content (equal checksum) and every recorded failure return `false`; the + /// reason for a failure is in the status table, not in the return value. + /// + /// The status entry is found by row id, so a source added since the last + /// `reload` has nowhere to record its outcome. `refreshAll` syncs the table + /// before it refreshes anything, which is why it is the entry point Phase 8 + /// and the scheduler use. + pub fn refreshSource(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool { + self.writer_lock.lockUncancelable(io); + defer self.writer_lock.unlock(io); + return self.refreshSourceLocked(io, row); + } + + fn refreshSourceLocked(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool { + // The previous entry describes the compiled files that are still on + // disk, and a failed refresh leaves them serving. Starting from a blank + // status would erase `last_success` and the counters of a list that is + // still being enforced. + var status: SourceStatus = self.priorStatus(io, row.id) orelse .{ .id = row.id }; + status.setUrl(row.url); + status.last_attempt = std.Io.Clock.real.now(io).toSeconds(); + + const replaced = try self.refreshOne(io, row, &status); + self.commitStatus(io, status); + return replaced; + } + + /// Every enabled source, one at a time, then one `reload`. A failing source + /// never stops the pass: it would hide every source behind it. + /// + /// This returns an error only when nothing could be done at all — out of + /// memory, an unreachable database, an unusable blocklist directory. A + /// source that failed to download or compile is a successful pass with a + /// non-`ok` status. + pub fn refreshAll(self: *Manager, io: std.Io) Error!void { + self.writer_lock.lockUncancelable(io); + defer self.writer_lock.unlock(io); + + var rows = try sources_repo.listSourceRows(self.database, self.gpa); + defer rows.deinit(self.gpa); + defer sources_repo.freeSourceRows(self.gpa, rows.items); + + try self.syncStatuses(io, rows.items); + + for (rows.items) |row| { + if (!row.enabled) continue; + _ = try self.refreshSourceLocked(io, row); + } + return self.reloadLocked(io); + } + + fn refreshOne( + self: *Manager, + io: std.Io, + row: sources_repo.SourceRow, + status: *SourceStatus, + ) Error!bool { + var dir = try self.openDir(io, .{}); + defer dir.close(io); + + var raw_buf: [name_buf_len]u8 = undefined; + var list_tmp_buf: [name_buf_len]u8 = undefined; + var wild_tmp_buf: [name_buf_len]u8 = undefined; + const raw_name = compiledName(&raw_buf, row.id, ".raw.tmp"); + const list_tmp = compiledName(&list_tmp_buf, row.id, ".list.tmp"); + const wild_tmp = compiledName(&wild_tmp_buf, row.id, ".wild.tmp"); + + // Installed before the calls that create these files, not after: an + // `error.Canceled` or `error.OutOfMemory` returned straight out of + // `download` or `compileTo` would outrun a later `defer` and leave a + // temporary behind. Deleting a name that was never created is a no-op. + defer self.deleteQuietly(io, dir, raw_name); + defer self.deleteQuietly(io, dir, list_tmp); + defer self.deleteQuietly(io, dir, wild_tmp); + + self.download(io, dir, raw_name, row.url) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.Canceled => return error.Canceled, + else => { + self.reportFetchFailure(row, status, err); + return false; + }, + }; + + const format = self.detectFormat(io, dir, raw_name) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.Canceled => return error.Canceled, + else => { + self.reportCompileFailure(row, status, err); + return false; + }, + }; + + const result = self.compileTo(io, dir, raw_name, format, list_tmp, wild_tmp) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.Canceled => return error.Canceled, + else => { + self.reportCompileFailure(row, status, err); + return false; + }, + }; + + if (rejectedWithoutEntries(result.counts)) { + self.reportEmptyCompile(row, status, result.counts); + return false; + } + + const now = std.Io.Clock.real.now(io).toSeconds(); + + // Recompiling identical content into new files would invalidate the + // snapshot for nothing. The files on disk must actually carry that + // content: if a reload found them corrupt and excluded the source, a + // re-download of unchanged upstream bytes is the one chance to repair + // them, and skipping the rewrite here would leave filtering off for + // good. + if (row.checksum) |stored| { + if (std.mem.eql(u8, stored, &result.checksum) and self.diskBodiesMatch(io, dir, row.id, stored)) { + try sources_repo.updateSourceStats(self.database, row.id, .{ + .last_updated = now, + .domain_count = row.domain_count, + .wildcard_count = row.wildcard_count, + .skipped_regex_count = row.skipped_regex_count, + .checksum = stored, + }); + status.succeed(now, result.counts); + return false; + } + } + + const header: Header = .{ + .url = row.url, + .format = format, + .fetched_at = now, + .counts = result.counts, + .checksum = &result.checksum, + }; + self.publish(io, dir, row.id, header, list_tmp, wild_tmp) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.Canceled => return error.Canceled, + else => { + self.reportCompileFailure(row, status, err); + return false; + }, + }; + + try sources_repo.updateSourceStats(self.database, row.id, .{ + .last_updated = now, + .domain_count = result.counts.domains, + .wildcard_count = result.counts.wildcards, + .skipped_regex_count = result.counts.skipped_regex, + .checksum = &result.checksum, + }); + status.succeed(now, result.counts); + return true; + } + + /// The body goes to a temporary file, never to memory: `max_body_bytes` is + /// 64 MB and the memory budget has no room for it beside two snapshots. + fn download( + self: *Manager, + io: std.Io, + dir: std.Io.Dir, + raw_name: []const u8, + url: []const u8, + ) !void { + const file = try dir.createFile(io, raw_name, .{ .permissions = .fromMode(0o600) }); + defer file.close(io); + + const buffer = try self.gpa.alloc(u8, io_buf_len); + defer self.gpa.free(buffer); + + var fw = file.writer(io, buffer); + const result = self.fetchWithin(io, url, &fw.interface) catch |err| { + // `fetcher.Error.Unexpected` is what a failing sink surfaces as; + // the concrete cause is on this writer, which the fetcher does not + // own. + if (fw.err) |cause| return cause; + if (err == error.HttpStatus) { + if (self.fetcher.last_status) |status| { + log.warn("blocklist {s}: http status {d}", .{ url, @intFromEnum(status) }); + } + } + return err; + }; + try fw.interface.flush(); + // The compile reads this file back; the bytes must be there, not in a + // buffer this function is about to drop. + try file.sync(io); + + log.debug("blocklist {s}: downloaded {d} bytes", .{ url, result.bytes_read }); + } + + /// `std.http.Client` has no per-request deadline, so the whole exchange + /// races a sleep and the loser is canceled — milestone 3's pattern. + fn fetchWithin( + self: *Manager, + io: std.Io, + url: []const u8, + w: *std.Io.Writer, + ) fetcher.Error!fetcher.Result { + var outcomes: [2]Outcome = undefined; + var race: std.Io.Select(Outcome) = .init(io, &outcomes); + defer race.cancelDiscard(); + + race.concurrent(.fetch, fetcher.Fetcher.fetch, .{ self.fetcher, io, url, w }) catch |err| switch (err) { + error.ConcurrencyUnavailable => return error.SystemResources, + }; + race.concurrent(.expiry, expire, .{ io, self.total_budget }) catch |err| switch (err) { + error.ConcurrencyUnavailable => return error.SystemResources, + }; + + switch (try race.await()) { + .fetch => |result| return result, + .expiry => |result| { + // A canceled sleep means this task is being torn down, not that + // the download is slow. + try result; + return error.Timeout; + }, + } + } + + fn detectFormat( + self: *Manager, + io: std.Io, + dir: std.Io.Dir, + raw_name: []const u8, + ) !parsers.Format { + const file = try dir.openFile(io, raw_name, .{}); + defer file.close(io); + + const buffers = try self.gpa.alloc(u8, io_buf_len + sample_buf_len); + defer self.gpa.free(buffers); + + var fr = file.reader(io, buffers[0..io_buf_len]); + var sample: std.Io.Writer = .fixed(buffers[io_buf_len..]); + collectSample(&fr.interface, &sample) catch |err| switch (err) { + error.ReadFailed => return fr.err orelse err, + // `sample_buf_len` holds every line `collectSample` can emit, so a + // full buffer means the sample is complete. + error.WriteFailed => {}, + }; + return parsers.detectFormat(sample.buffered()); + } + + /// Compiles into two plain temporary files. The compiled bodies cannot go + /// straight into the final files: the header carries counts that only exist + /// once the whole input has been compiled, and the loader requires the + /// header first. + fn compileTo( + self: *Manager, + io: std.Io, + dir: std.Io.Dir, + raw_name: []const u8, + format: parsers.Format, + list_tmp: []const u8, + wild_tmp: []const u8, + ) !compiler.Result { + const raw = try dir.openFile(io, raw_name, .{}); + defer raw.close(io); + const list_file = try dir.createFile(io, list_tmp, .{ .permissions = .fromMode(0o600) }); + defer list_file.close(io); + const wild_file = try dir.createFile(io, wild_tmp, .{ .permissions = .fromMode(0o600) }); + defer wild_file.close(io); + + const buffers = try self.gpa.alloc(u8, 3 * io_buf_len); + defer self.gpa.free(buffers); + + var fr = raw.reader(io, buffers[0..io_buf_len]); + var list_w = list_file.writer(io, buffers[io_buf_len .. 2 * io_buf_len]); + var wild_w = wild_file.writer(io, buffers[2 * io_buf_len ..]); + + const result = compiler.compile( + self.gpa, + &fr.interface, + format, + &list_w.interface, + &wild_w.interface, + ) catch |err| switch (err) { + // `compiler.Error` names the direction; the concrete cause is on + // the stream that failed. + error.ReadFailed => return fr.err orelse err, + error.WriteFailed => return list_w.err orelse (wild_w.err orelse err), + else => return err, + }; + + try list_w.interface.flush(); + try wild_w.interface.flush(); + try list_file.sync(io); + try wild_file.sync(io); + return result; + } + + /// Writes header + body into each final file through `createFileAtomic` + + /// `replace`, so a crash mid-write can never leave a half-list that would + /// load as a valid, shorter blocklist. + fn publish( + self: *Manager, + io: std.Io, + dir: std.Io.Dir, + id: i64, + header: Header, + list_tmp: []const u8, + wild_tmp: []const u8, + ) !void { + const buffers = try self.gpa.alloc(u8, 2 * io_buf_len); + defer self.gpa.free(buffers); + + var list_buf: [name_buf_len]u8 = undefined; + var wild_buf: [name_buf_len]u8 = undefined; + try publishOne(io, dir, compiledName(&list_buf, id, ".list"), list_tmp, header, buffers); + try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), wild_tmp, header, buffers); + } + + fn publishOne( + io: std.Io, + dir: std.Io.Dir, + dest: []const u8, + body_name: []const u8, + header: Header, + buffers: []u8, + ) !void { + const body = try dir.openFile(io, body_name, .{}); + defer body.close(io); + + var af = try dir.createFileAtomic(io, dest, .{ + .permissions = .fromMode(0o600), + .replace = true, + }); + defer af.deinit(io); + + var fr = body.reader(io, buffers[0..io_buf_len]); + var fw = af.file.writer(io, buffers[io_buf_len..]); + + header.write(&fw.interface) catch return fw.err orelse error.WriteFailed; + _ = fr.interface.streamRemaining(&fw.interface) catch + return fr.err orelse (fw.err orelse error.WriteFailed); + fw.interface.flush() catch return fw.err orelse error.WriteFailed; + + // Before `replace`, which closes the file: the rename must publish + // durable bytes, not an empty file with the content still in the page + // cache. + try af.file.sync(io); + try af.replace(io); + } + + /// Whether the two compiled files on disk hash to `expected`. A missing, + /// unreadable or corrupt file answers false, which sends the caller down + /// the rewrite path — the only path that can repair it. + fn diskBodiesMatch(self: *Manager, io: std.Io, dir: std.Io.Dir, id: i64, expected: []const u8) bool { + var list_buf: [name_buf_len]u8 = undefined; + var wild_buf: [name_buf_len]u8 = undefined; + const limit: std.Io.Limit = .limited(max_compiled_bytes); + + const list_bytes = dir.readFileAlloc(io, compiledName(&list_buf, id, ".list"), self.gpa, limit) catch + return false; + defer self.gpa.free(list_bytes); + const wild_bytes = dir.readFileAlloc(io, compiledName(&wild_buf, id, ".wild"), self.gpa, limit) catch + return false; + defer self.gpa.free(wild_bytes); + + return compiledBodiesMatch(list_bytes, wild_bytes, expected); + } + + fn reportFetchFailure( + self: *Manager, + row: sources_repo.SourceRow, + status: *SourceStatus, + err: anyerror, + ) void { + _ = self; + log.warn("blocklist {s}: download failed: {s}", .{ row.url, @errorName(err) }); + status.fail(.fetch_failed, @errorName(err)); + } + + fn reportCompileFailure( + self: *Manager, + row: sources_repo.SourceRow, + status: *SourceStatus, + err: anyerror, + ) void { + _ = self; + log.warn("blocklist {s}: compile failed: {s}", .{ row.url, @errorName(err) }); + status.fail(.compile_failed, @errorName(err)); + } + + fn reportEmptyCompile( + self: *Manager, + row: sources_repo.SourceRow, + status: *SourceStatus, + counts: compiler.Counts, + ) void { + _ = self; + var buf: [max_error_len]u8 = undefined; + const text = std.fmt.bufPrint( + &buf, + "NoValidEntries invalid={d} unsupported={d} long_lines={d}", + .{ counts.invalid, counts.skipped_unsupported, counts.long_lines }, + ) catch "NoValidEntries"; + log.warn("blocklist {s}: {s}", .{ row.url, text }); + status.fail(.no_valid_entries, text); + } + + // ----------------------------------------------------------------------- + // scheduling + // ----------------------------------------------------------------------- + + /// Loads at startup, refreshes only what needs it, then sleeps + /// `update.interval_hours` between full passes. Returns on + /// `error.Canceled`. + /// + /// A cold restart must not re-download every list and a boot loop must not + /// become a download loop, so the startup pass refreshes a source only when + /// it has no usable compiled files or its `last_updated` is older than the + /// interval. + /// + /// `update.enabled == false` stops after the startup pass; manual refresh + /// through `refreshAll` still works. + pub fn runScheduler(self: *Manager, io: std.Io) std.Io.Cancelable!void { + self.startupPass(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => log.warn("blocklist startup pass failed: {s}", .{@errorName(err)}), + }; + if (!self.update.enabled) return; + + // `boot` rather than `awake`: a box that suspends overnight should + // still see its daily interval elapse. + const interval: std.Io.Clock.Duration = .{ + .raw = .fromSeconds(model.updateIntervalSeconds(self.update)), + .clock = .boot, + }; + while (true) { + try interval.sleep(io); + self.refreshAll(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => log.warn("blocklist refresh pass failed: {s}", .{@errorName(err)}), + }; + } + } + + fn startupPass(self: *Manager, io: std.Io) Error!void { + self.writer_lock.lockUncancelable(io); + defer self.writer_lock.unlock(io); + + try self.reloadLocked(io); + + var rows = try sources_repo.listSourceRows(self.database, self.gpa); + defer rows.deinit(self.gpa); + defer sources_repo.freeSourceRows(self.gpa, rows.items); + + const now = std.Io.Clock.real.now(io).toSeconds(); + var refreshed = false; + for (rows.items) |row| { + if (!row.enabled) continue; + if (!self.needsRefresh(io, row, now)) continue; + if (try self.refreshSourceLocked(io, row)) refreshed = true; + } + if (refreshed) try self.reloadLocked(io); + } + + fn needsRefresh(self: *Manager, io: std.Io, row: sources_repo.SourceRow, now: i64) bool { + self.lock.lockSharedUncancelable(io); + const state: State = state: { + for (self.statuses) |status| { + if (status.id == row.id) break :state status.state; + } + break :state .never_fetched; + }; + self.lock.unlockShared(io); + + if (state != .ok) return true; + const last = row.last_updated orelse return true; + return now - last >= model.updateIntervalSeconds(self.update); + } + + // ----------------------------------------------------------------------- + // orphans + // ----------------------------------------------------------------------- + + /// Deletes `.list` and `.wild` files whose id is no longer a + /// `blocklist_sources` row. Files of a live source are left alone, + /// whatever their state. + pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void { + // A refresh in flight owns the temporaries of a live source; the sweep + // must not run beside one and decide from a half-written directory. + self.writer_lock.lockUncancelable(io); + defer self.writer_lock.unlock(io); + + var rows = try sources_repo.listSourceRows(self.database, self.gpa); + defer rows.deinit(self.gpa); + defer sources_repo.freeSourceRows(self.gpa, rows.items); + + var dir = try self.openDir(io, .{ .iterate = true }); + defer dir.close(io); + + // The names are collected first: `Entry.name` is invalidated by the + // next `next`, and deleting under an open cursor is not defined. + var doomed: std.ArrayList([]u8) = .empty; + defer { + for (doomed.items) |item| self.gpa.free(item); + doomed.deinit(self.gpa); + } + + var it = dir.iterate(); + while (true) { + const entry = it.next(io) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + log.warn("pruning blocklists: reading the directory failed: {s}", .{@errorName(err)}); + return error.FileSystem; + }, + } orelse break; + if (entry.kind != .file) continue; + const id = compiledId(entry.name) orelse continue; + if (containsId(rows.items, id)) continue; + try doomed.append(self.gpa, try self.gpa.dupe(u8, entry.name)); + } + + for (doomed.items) |name| { + self.deleteQuietly(io, dir, name); + log.info("pruned orphaned compiled file {s}", .{name}); + } + } + + // ----------------------------------------------------------------------- + // internals + // ----------------------------------------------------------------------- + + /// Builds the status table for `rows`, carrying every existing entry over + /// by row id so a recorded failure survives. Nothing is published: the + /// caller either installs the result or frees it, which is what lets + /// `reloadLocked` decide only once its snapshot exists. + fn buildStatusTable( + self: *Manager, + io: std.Io, + rows: []const sources_repo.SourceRow, + ) Error!StatusTable { + var fresh: std.heap.ArenaAllocator = .init(self.gpa); + errdefer fresh.deinit(); + const arena = fresh.allocator(); + + // The published table is copied under the lock. Reading it unlocked + // would race the writer that replaces it — and the arena its entries + // live in is freed by whoever installs what this builds. + const previous = previous: { + self.lock.lockSharedUncancelable(io); + defer self.lock.unlockShared(io); + break :previous try arena.dupe(SourceStatus, self.statuses); + }; + + const table = try arena.alloc(SourceStatus, rows.len); + mergeStatuses(table, rows, previous); + return .{ .arena = fresh, .items = table }; + } + + /// Publishes a built table and frees the one it replaces. The caller holds + /// the exclusive lock, so no reader is inside the old table. + fn installStatuses(self: *Manager, table: StatusTable) void { + self.status_arena.deinit(); + self.status_arena = table.arena; + self.statuses = table.items; + } + + /// Rebuilds and publishes the status table from the current source set. + /// + /// `refreshAll` calls this before it refreshes anything: a source added + /// since the last reload needs an entry to record its outcome in, and the + /// API has to see the pass advance while it runs. `reloadLocked` does not + /// call it — a reload publishes its table together with the snapshot that + /// table describes. + fn syncStatuses(self: *Manager, io: std.Io, rows: []const sources_repo.SourceRow) Error!void { + const table = try self.buildStatusTable(io, rows); + self.lock.lockUncancelable(io); + defer self.lock.unlock(io); + self.installStatuses(table); + } + + /// The recorded entry for one source, or `null` when the table has none. + fn priorStatus(self: *Manager, io: std.Io, id: i64) ?SourceStatus { + self.lock.lockSharedUncancelable(io); + defer self.lock.unlockShared(io); + for (self.statuses) |entry| { + if (entry.id == id) return entry; + } + return null; + } + + fn commitStatus(self: *Manager, io: std.Io, status: SourceStatus) void { + self.lock.lockUncancelable(io); + defer self.lock.unlock(io); + for (self.statuses) |*entry| { + if (entry.id != status.id) continue; + // `loaded` is the reload's fact, not the refresh's: the files this + // refresh wrote are not in a snapshot until the next reload reads + // them. + const loaded = entry.loaded; + entry.* = status; + entry.loaded = loaded; + return; + } + } + + /// One id per group, in `listGroups` order. + fn groupIds(self: *Manager, groups: []const model.Group) Error![]i64 { + const out = try self.gpa.alloc(i64, groups.len); + errdefer self.gpa.free(out); + for (out, groups) |*slot, group| { + slot.* = try groups_repo.groupId(self.database, group.name) orelse + return error.GroupSetChanged; + } + return out; + } + + fn openDir(self: *Manager, io: std.Io, options: std.Io.Dir.OpenOptions) Error!std.Io.Dir { + _ = self.paths.dir.createDirPathStatus(io, self.paths.subdir, .fromMode(0o700)) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + log.warn("creating {s} failed: {s}", .{ self.paths.subdir, @errorName(err) }); + return error.FileSystem; + }, + }; + return self.paths.dir.openDir(io, self.paths.subdir, options) catch |err| switch (err) { + error.Canceled => return error.Canceled, + else => { + log.warn("opening {s} failed: {s}", .{ self.paths.subdir, @errorName(err) }); + return error.FileSystem; + }, + }; + } + + /// A temporary that cannot be removed is not a failure of the operation + /// that made it, but it is not nothing either: it is left visible. + fn deleteQuietly(self: *Manager, io: std.Io, dir: std.Io.Dir, name: []const u8) void { + _ = self; + dir.deleteFile(io, name) catch |err| switch (err) { + error.FileNotFound => {}, + else => log.warn("deleting {s} failed: {s}", .{ name, @errorName(err) }), + }; + } +}; + +/// A status table and the arena holding it. Until `installStatuses` takes it, +/// it is a candidate nobody can see, and `deinit` frees it whole. +const StatusTable = struct { + arena: std.heap.ArenaAllocator, + items: []SourceStatus, + + fn deinit(self: *StatusTable) void { + self.arena.deinit(); + self.items = &.{}; + } +}; + +/// Fills `table` with one entry per row, carrying an entry of the same row id +/// over from `previous`. A source deleted since `previous` was built is gone; a +/// source added since starts blank. `previous` is only read, so the caller's +/// published table is untouched by this. +fn mergeStatuses( + table: []SourceStatus, + rows: []const sources_repo.SourceRow, + previous: []const SourceStatus, +) void { + for (table, rows) |*status, row| { + status.* = .{ .id = row.id }; + for (previous) |prior| { + if (prior.id != row.id) continue; + status.* = prior; + break; + } + // After the carry-over: a url edited on the row wins over the one the + // prior entry recorded. + status.setUrl(row.url); + } +} + +/// What one `reload` found for one source. The texts are static, so an outcome +/// borrows nothing and stays valid until the swap that commits it. +const LoadOutcome = union(enum) { + /// The source is switched off. Not in the snapshot, whatever it was before. + disabled, + /// Its compiled files are in the snapshot. + loaded: matcher.Snapshot.Compiled, + /// It is not in the snapshot, for this reason. + failed: struct { state: State, text: []const u8 }, +}; + +fn loadFailure(row: sources_repo.SourceRow, file_name: []const u8, err: anyerror) LoadOutcome { + log.warn("blocklist {s}: reading {s} failed: {s}", .{ row.url, file_name, @errorName(err) }); + return .{ .failed = .{ .state = .load_failed, .text = @errorName(err) } }; +} + +/// Writes one reload's findings into the status table. The caller holds the +/// exclusive lock: this runs inside the swap so the table and the published +/// snapshot describe the same thing. +/// +/// `rows` and `outcomes` are parallel. A source with no entry is one the table +/// was not synced for, which cannot happen from `reloadLocked` and is skipped +/// rather than asserted, because the table is rebuilt by row id. +fn applyLoadOutcomes( + statuses: []SourceStatus, + rows: []const sources_repo.SourceRow, + outcomes: []const LoadOutcome, +) void { + for (rows, outcomes) |row, outcome| { + const entry = entryFor(statuses, row.id) orelse continue; + switch (outcome) { + // A source disabled since it last loaded is no longer filtering, + // and its recorded state describes files nothing reads. + .disabled => entry.loaded = false, + .loaded => { + entry.loaded = true; + // Two states survive a successful load. `.ok`, because a + // refresh in this process already filled the counters the + // compile produced and the three database columns are a subset + // of them. And any refresh failure, because the files that just + // loaded are exactly the ones the failed refresh could not + // replace, so the operator must still see why. + if (entry.state == .ok or entry.state.isRefreshFailure()) continue; + entry.succeed(row.last_updated orelse 0, .{ + .domains = countOf(row.domain_count), + .wildcards = countOf(row.wildcard_count), + .skipped_regex = countOf(row.skipped_regex_count), + }); + }, + .failed => |reason| { + entry.loaded = false; + // The state follows only when nothing more informative is + // there: a refresh failure already says why the files are + // missing or stale, and `loaded` already says they are not + // filtering. + if (entry.state.isRefreshFailure()) continue; + entry.fail(reason.state, reason.text); + }, + } + } +} + +fn entryFor(statuses: []SourceStatus, id: i64) ?*SourceStatus { + for (statuses) |*entry| { + if (entry.id == id) return entry; + } + return null; +} + +const Outcome = union(enum) { + fetch: fetcher.Error!fetcher.Result, + expiry: std.Io.Cancelable!void, +}; + +fn expire(io: std.Io, duration: std.Io.Clock.Duration) std.Io.Cancelable!void { + return duration.sleep(io); +} + +/// A counter column read back from the database. It is `NOT NULL DEFAULT 0` and +/// only this file writes it, so a value outside `u32` means the row was edited +/// behind nxdns's back; the status reports 0 rather than trapping. +fn countOf(value: i64) u32 { + return std.math.cast(u32, value) orelse 0; +} + +fn destroySnapshot(gpa: Allocator, snapshot: *matcher.Snapshot) void { + snapshot.deinit(); + gpa.destroy(snapshot); +} + +/// Copies the lines `parsers.detectFormat` would count — neither blank nor a +/// comment — until it has `parsers.sample_lines` of them, and writes them to +/// `w` newline-separated. Sampling by line rather than by a byte window is what +/// keeps a run of long comment lines from deciding the format: `detectFormat` +/// reads exactly these lines and ignores everything this drops. +/// +/// A line over `compiler.max_line_len` is skipped, as the compiler skips it. +fn collectSample(r: *std.Io.Reader, w: *std.Io.Writer) error{ ReadFailed, WriteFailed }!void { + var considered: usize = 0; + while (considered < parsers.sample_lines) { + const raw = r.takeDelimiter('\n') catch |err| switch (err) { + // The stream is left unmodified here, so the line has to be stepped + // over or this loop never advances. + error.StreamTooLong => { + _ = r.discardDelimiterInclusive('\n') catch |discard_err| switch (discard_err) { + error.EndOfStream => return, + error.ReadFailed => return error.ReadFailed, + }; + continue; + }, + error.ReadFailed => return error.ReadFailed, + } orelse return; + + if (raw.len > compiler.max_line_len) continue; + const line = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (line.len == 0) continue; + if (parsers.isComment(line)) continue; + + considered += 1; + try w.writeAll(line); + try w.writeByte('\n'); + } +} + +/// Whether two compiled files carry the bodies `expected` was taken over. +fn compiledBodiesMatch(list_bytes: []const u8, wild_bytes: []const u8, expected: []const u8) bool { + return std.mem.eql(u8, expected, &bodyChecksum(stripHeader(list_bytes), stripHeader(wild_bytes))); +} + +/// A compile that produced no entry at all while rejecting lines is an error +/// page, a compressed body or a format the sniff got wrong — not a blocklist. +/// Publishing it would replace a working list with nothing and report `ok`. An +/// input that rejected nothing is an empty list, which is legal. +fn rejectedWithoutEntries(counts: compiler.Counts) bool { + if (counts.domains != 0 or counts.wildcards != 0) return false; + return counts.invalid != 0 or counts.skipped_unsupported != 0 or counts.long_lines != 0; +} + +fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 { + var hasher = Sha256.init(.{}); + hasher.update(list_body); + hasher.update(wild_body); + var digest: [Sha256.digest_length]u8 = undefined; + hasher.final(&digest); + return std.fmt.bytesToHex(digest, .lower); +} + +fn compiledName(buf: *[name_buf_len]u8, id: i64, suffix: []const u8) []const u8 { + // An `i64` prints in at most 20 characters and the longest suffix is nine, + // so `name_buf_len` cannot be exceeded. + return std.fmt.bufPrint(buf, "{d}{s}", .{ id, suffix }) catch unreachable; +} + +/// The source id a compiled file belongs to, or null when the name is not one +/// of ours. Temporary files are deliberately not matched: they belong to a +/// refresh that may still be running. +fn compiledId(file_name: []const u8) ?i64 { + const stem = if (std.mem.endsWith(u8, file_name, ".list")) + file_name[0 .. file_name.len - ".list".len] + else if (std.mem.endsWith(u8, file_name, ".wild")) + file_name[0 .. file_name.len - ".wild".len] + else + return null; + return std.fmt.parseInt(i64, stem, 10) catch null; +} + +fn containsId(rows: []const sources_repo.SourceRow, id: i64) bool { + for (rows) |row| { + if (row.id == id) return true; + } + return false; +} + +// --------------------------------------------------------------------------- +// tests +// --------------------------------------------------------------------------- +// +// Everything here runs against a `:memory:` database and touches no file. Real +// files, real HTTP and real swaps are the integration suite's (S9). + +const testing = std.testing; +const migrations = @import("../storage/migrations.zig"); + +fn openMigrated() !db.Db { + var database = try db.Db.open(":memory:", .{ .mode = .memory }); + errdefer database.close(); + try db.applyPragmas(&database, .{}); + _ = try migrations.migrate(&database); + return database; +} + +fn testManager(database: *db.Db, fetcher_ptr: *fetcher.Fetcher) !Manager { + return Manager.init( + testing.allocator, + database, + // No test in this file reaches the filesystem: `acquire` answers before + // any directory is touched, and the header helpers are pure. + .{ .dir = std.Io.Dir.cwd() }, + fetcher_ptr, + .{}, + .{ .raw = .fromSeconds(30), .clock = .awake }, + ); +} + +test "init leaves the manager with no snapshot and no statuses" { + var threaded: std.Io.Threaded = .init(testing.allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var database = try openMigrated(); + defer database.close(); + + var f: fetcher.Fetcher = undefined; + var manager = try testManager(&database, &f); + defer manager.deinit(io); + + try testing.expectEqual(@as(u64, 0), manager.generation); + try testing.expectEqual(@as(usize, 0), manager.statuses.len); + try testing.expect(manager.current == null); +} + +test "acquire before any reload returns null and holds no lock" { + var threaded: std.Io.Threaded = .init(testing.allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var database = try openMigrated(); + defer database.close(); + + var f: fetcher.Fetcher = undefined; + var manager = try testManager(&database, &f); + defer manager.deinit(io); + + try testing.expect(manager.acquire(io) == null); + // A retained shared lock would make this exclusive lock block forever. + try testing.expect(manager.lock.tryLock(io)); + manager.lock.unlock(io); +} + +test "statusSnapshot on an empty manager copies nothing" { + var threaded: std.Io.Threaded = .init(testing.allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var database = try openMigrated(); + defer database.close(); + + var f: fetcher.Fetcher = undefined; + var manager = try testManager(&database, &f); + defer manager.deinit(io); + + var out: [4]SourceStatus = undefined; + try testing.expectEqual(@as(usize, 0), manager.statusSnapshot(io, &out)); +} + +test "the header writer produces the documented text" { + var buf: [512]u8 = undefined; + var w: std.Io.Writer = .fixed(&buf); + const header: Header = .{ + .url = "https://lists.example/hosts.txt", + .format = .hosts, + .fetched_at = 1_700_000_000, + .counts = .{ + .domains = 12, + .wildcards = 3, + .skipped_regex = 2, + .skipped_unsupported = 1, + .invalid = 5, + .long_lines = 9, + .duplicates = 4, + }, + .checksum = "0" ** 64, + }; + try header.write(&w); + + try testing.expectEqualStrings( + \\# nxdns blocklist + \\# url https://lists.example/hosts.txt + \\# format hosts + \\# fetched_at 1700000000 + \\# domains 12 + \\# wildcards 3 + \\# skipped_regex 2 + \\# skipped_unsupported 1 + \\# invalid 5 + \\ + ++ "# sha256 " ++ "0" ** 64 ++ "\n", w.buffered()); +} + +test "stripHeader returns the body of a compiled file" { + const file = + "# nxdns blocklist\n" ++ + "# url https://lists.example/hosts.txt\n" ++ + "ads.example.com\ntracker.example.net\n"; + try testing.expectEqualStrings("ads.example.com\ntracker.example.net\n", stripHeader(file)); +} + +test "stripHeader returns everything for a file with no header" { + try testing.expectEqualStrings("a.example.com\n", stripHeader("a.example.com\n")); +} + +test "stripHeader returns an empty body for a header-only file" { + try testing.expectEqualStrings("", stripHeader("# nxdns blocklist\n# sha256 x\n")); +} + +test "stripHeader tolerates an unterminated header line" { + try testing.expectEqualStrings("", stripHeader("# nxdns blocklist")); +} + +test "SourceStatus truncates a long error at max_error_len" { + var status: SourceStatus = .{ .id = 1 }; + const long = "E" ** (max_error_len + 40); + status.fail(.fetch_failed, long); + + try testing.expectEqual(State.fetch_failed, status.state); + try testing.expectEqual(@as(u8, max_error_len), status.last_error_len); + try testing.expectEqualStrings("E" ** max_error_len, status.errorText()); +} + +test "a success clears the recorded error" { + var status: SourceStatus = .{ .id = 7 }; + status.fail(.compile_failed, "TooManyDomains"); + status.succeed(1_700_000_000, .{ .domains = 3, .wildcards = 1 }); + + try testing.expectEqual(State.ok, status.state); + try testing.expectEqual(@as(i64, 1_700_000_000), status.last_success); + try testing.expectEqual(@as(u32, 3), status.counts.domains); + try testing.expectEqualStrings("", status.errorText()); +} + +test "compiledName spells the four file names of a source" { + var buf: [name_buf_len]u8 = undefined; + try testing.expectEqualStrings("42.list", compiledName(&buf, 42, ".list")); + try testing.expectEqualStrings("42.wild", compiledName(&buf, 42, ".wild")); + try testing.expectEqualStrings("42.raw.tmp", compiledName(&buf, 42, ".raw.tmp")); + try testing.expectEqualStrings("42.list.tmp", compiledName(&buf, 42, ".list.tmp")); +} + +test "compiledId matches compiled files and nothing else" { + try testing.expectEqual(@as(?i64, 7), compiledId("7.list")); + try testing.expectEqual(@as(?i64, 7), compiledId("7.wild")); + try testing.expectEqual(@as(?i64, null), compiledId("7.list.tmp")); + try testing.expectEqual(@as(?i64, null), compiledId("7.raw.tmp")); + try testing.expectEqual(@as(?i64, null), compiledId("notes.list")); + try testing.expectEqual(@as(?i64, null), compiledId("README")); +} + +test "a failed refresh keeps the fields of the compiled files still serving" { + var status: SourceStatus = .{ .id = 3 }; + status.succeed(1_700_000_000, .{ .domains = 5, .wildcards = 2 }); + + // What `refreshSourceLocked` starts from, and what a download failure does + // to it. + var next = status; + next.last_attempt = 1_700_003_600; + next.fail(.fetch_failed, "Timeout"); + + try testing.expectEqual(State.fetch_failed, next.state); + try testing.expectEqualStrings("Timeout", next.errorText()); + try testing.expectEqual(@as(i64, 1_700_000_000), next.last_success); + try testing.expectEqual(@as(i64, 1_700_003_600), next.last_attempt); + try testing.expectEqual(@as(u32, 5), next.counts.domains); + try testing.expectEqual(@as(u32, 2), next.counts.wildcards); +} + +test "a load outcome never overwrites a refresh failure" { + try testing.expect(State.fetch_failed.isRefreshFailure()); + try testing.expect(State.compile_failed.isRefreshFailure()); + try testing.expect(State.no_valid_entries.isRefreshFailure()); + + // The three a load produces. `applyLoadOutcomes` may write over these, + // because nothing more informative is there. + try testing.expect(!State.ok.isRefreshFailure()); + try testing.expect(!State.never_fetched.isRefreshFailure()); + try testing.expect(!State.load_failed.isRefreshFailure()); +} + +fn testRow(id: i64, enabled: bool) sources_repo.SourceRow { + return .{ + .id = id, + .url = "https://lists.example/hosts.txt", + .name = "example", + .enabled = enabled, + .last_updated = 1_700_000_000, + .domain_count = 9, + .wildcard_count = 4, + .skipped_regex_count = 1, + .checksum = "0" ** 64, + }; +} + +test "a candidate table carries prior entries over and leaves the published one alone" { + var published = [_]SourceStatus{ .{ .id = 1 }, .{ .id = 2 } }; + published[0].setUrl("https://lists.example/one.txt"); + published[0].fail(.fetch_failed, "HttpStatus"); + published[0].loaded = true; + published[1].setUrl("https://lists.example/two.txt"); + published[1].succeed(1_700_000_000, .{ .domains = 4 }); + + // Source 2 was deleted and source 3 added; source 1 kept its id and got a + // new url. + const rows = [_]sources_repo.SourceRow{ + blk: { + var row = testRow(1, true); + row.url = "https://lists.example/moved.txt"; + break :blk row; + }, + testRow(3, true), + }; + + var candidate: [2]SourceStatus = undefined; + mergeStatuses(&candidate, &rows, &published); + + try testing.expectEqual(@as(i64, 1), candidate[0].id); + try testing.expectEqual(State.fetch_failed, candidate[0].state); + try testing.expectEqualStrings("HttpStatus", candidate[0].errorText()); + try testing.expect(candidate[0].loaded); + try testing.expectEqualStrings("https://lists.example/moved.txt", candidate[0].urlText()); + + try testing.expectEqual(@as(i64, 3), candidate[1].id); + try testing.expectEqual(State.never_fetched, candidate[1].state); + try testing.expect(!candidate[1].loaded); + + // The published table is untouched, so a reload that fails before the swap + // leaves it describing the snapshot that is still serving — including the + // entry of the deleted source, which that snapshot still enforces. + try testing.expectEqual(@as(usize, 2), published.len); + try testing.expectEqual(@as(i64, 2), published[1].id); + try testing.expectEqual(State.ok, published[1].state); + try testing.expectEqualStrings("https://lists.example/one.txt", published[0].urlText()); +} + +test "a disabled source stops being loaded" { + var statuses = [_]SourceStatus{.{ .id = 1 }}; + statuses[0].succeed(1_700_000_000, .{ .domains = 9 }); + statuses[0].loaded = true; + + const rows = [_]sources_repo.SourceRow{testRow(1, false)}; + applyLoadOutcomes(&statuses, &rows, &.{.disabled}); + + // Nothing enforces it any more, and the state that described the files it + // used to serve is left as the record of how it last stood. + try testing.expect(!statuses[0].loaded); + try testing.expectEqual(State.ok, statuses[0].state); +} + +test "a source that failed to refresh keeps its failure while its old files serve" { + // What `refreshSourceLocked` records, then what the `reload` that follows + // it in `refreshAll` finds: the previous files still load. + var statuses = [_]SourceStatus{.{ .id = 1 }}; + statuses[0].succeed(1_700_000_000, .{ .domains = 9 }); + statuses[0].fail(.fetch_failed, "HttpStatus"); + + const rows = [_]sources_repo.SourceRow{testRow(1, true)}; + const body: matcher.Snapshot.Compiled = .{ .list_body = "", .wild_body = "" }; + applyLoadOutcomes(&statuses, &rows, &.{.{ .loaded = body }}); + + try testing.expect(statuses[0].loaded); + try testing.expectEqual(State.fetch_failed, statuses[0].state); + try testing.expectEqualStrings("HttpStatus", statuses[0].errorText()); + try testing.expectEqual(@as(u32, 9), statuses[0].counts.domains); +} + +test "a load failure is recorded when no refresh failure explains it" { + var statuses = [_]SourceStatus{ .{ .id = 1 }, .{ .id = 2 } }; + statuses[0].succeed(1_700_000_000, .{ .domains = 9 }); + statuses[0].loaded = true; + statuses[1].fail(.compile_failed, "TooManyDomains"); + statuses[1].loaded = true; + + const rows = [_]sources_repo.SourceRow{ testRow(1, true), testRow(2, true) }; + const reason: LoadOutcome = .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } }; + applyLoadOutcomes(&statuses, &rows, &.{ reason, reason }); + + try testing.expect(!statuses[0].loaded); + try testing.expectEqual(State.load_failed, statuses[0].state); + try testing.expectEqualStrings("ChecksumMismatch", statuses[0].errorText()); + + // The compile failure is why the files are unusable; it outranks the + // symptom the loader saw. + try testing.expect(!statuses[1].loaded); + try testing.expectEqual(State.compile_failed, statuses[1].state); + try testing.expectEqualStrings("TooManyDomains", statuses[1].errorText()); +} + +test "a load of a source this process never refreshed takes the row counters" { + var statuses = [_]SourceStatus{.{ .id = 1 }}; + const rows = [_]sources_repo.SourceRow{testRow(1, true)}; + const body: matcher.Snapshot.Compiled = .{ .list_body = "", .wild_body = "" }; + applyLoadOutcomes(&statuses, &rows, &.{.{ .loaded = body }}); + + try testing.expect(statuses[0].loaded); + try testing.expectEqual(State.ok, statuses[0].state); + try testing.expectEqual(@as(i64, 1_700_000_000), statuses[0].last_success); + try testing.expectEqual(@as(u32, 9), statuses[0].counts.domains); + try testing.expectEqual(@as(u32, 4), statuses[0].counts.wildcards); + try testing.expectEqual(@as(u32, 1), statuses[0].counts.skipped_regex); +} + +test "a status borrows nothing, so a copy outlives the table it came from" { + var status: SourceStatus = .{ .id = 5 }; + status.setUrl("https://lists.example/hosts.txt"); + status.fail(.load_failed, "ChecksumMismatch"); + + const copy = status; + // The source of the original is overwritten, as a reload overwrites the + // table: a copy that borrowed would read the new bytes or freed memory. + status.setUrl("https://other.example/other.txt"); + status.fail(.fetch_failed, "Timeout"); + + try testing.expectEqualStrings("https://lists.example/hosts.txt", copy.urlText()); + try testing.expectEqualStrings("ChecksumMismatch", copy.errorText()); + try testing.expectEqual(State.load_failed, copy.state); +} + +test "SourceStatus truncates a long url at max_url_len" { + var status: SourceStatus = .{ .id = 6 }; + status.setUrl("https://lists.example/" ++ "p" ** max_url_len); + + try testing.expectEqual(@as(u8, max_url_len), status.url_len); + try testing.expectEqualStrings( + ("https://lists.example/" ++ "p" ** max_url_len)[0..max_url_len], + status.urlText(), + ); + + // A shorter url must not leave the tail of the longer one behind it. + status.setUrl("https://a.example/x"); + try testing.expectEqualStrings("https://a.example/x", status.urlText()); +} + +test "compiledBodiesMatch verifies the bodies, not the presence of the files" { + const list_body = "a.example.com\nb.example.com\n"; + const wild_body = "c.example.com\n"; + const expected = bodyChecksum(list_body, wild_body); + + const header = + "# nxdns blocklist\n" ++ + "# url https://lists.example/hosts.txt\n"; + try testing.expect(compiledBodiesMatch(header ++ list_body, header ++ wild_body, &expected)); + + // The corruption a reload reports as `ChecksumMismatch`: the file is there, + // its body is not what the checksum was taken over. + try testing.expect(!compiledBodiesMatch(header ++ "a.example.com\nb.exa", header ++ wild_body, &expected)); + try testing.expect(!compiledBodiesMatch("", "", &expected)); +} + +test "rejectedWithoutEntries fails a compile that produced nothing usable" { + // An html error page: every line is rejected, nothing is written. + try testing.expect(rejectedWithoutEntries(.{ .invalid = 12, .skipped_unsupported = 3 })); + // A compressed body: one long binary run with no newline in it. + try testing.expect(rejectedWithoutEntries(.{ .long_lines = 1 })); + + // An empty list rejects nothing and is legal. + try testing.expect(!rejectedWithoutEntries(.{})); + // A real list rejects lines and still produces entries. + try testing.expect(!rejectedWithoutEntries(.{ .domains = 1000, .invalid = 40 })); + try testing.expect(!rejectedWithoutEntries(.{ .wildcards = 7, .skipped_unsupported = 90 })); +} + +fn sampleOf(input: []const u8, out: []u8) ![]const u8 { + var r: std.Io.Reader = .fixed(input); + var w: std.Io.Writer = .fixed(out); + try collectSample(&r, &w); + return w.buffered(); +} + +test "collectSample skips comments instead of spending the sample on them" { + const gpa = testing.allocator; + const long_comment = "# " ++ "c" ** (compiler.max_line_len - 2) ++ "\n"; + + var input: std.ArrayList(u8) = .empty; + defer input.deinit(gpa); + // Sixteen of these fill a 64 KiB window on their own. + for (0..20) |_| try input.appendSlice(gpa, long_comment); + try input.appendSlice(gpa, "0.0.0.0 ads.example.com\n0.0.0.0 tracker.example.net\n"); + + const out = try gpa.alloc(u8, sample_buf_len); + defer gpa.free(out); + const sample = try sampleOf(input.items, out); + + try testing.expectEqualStrings( + "0.0.0.0 ads.example.com\n0.0.0.0 tracker.example.net\n", + sample, + ); + try testing.expectEqual(parsers.Format.hosts, parsers.detectFormat(sample)); +} + +test "collectSample stops at sample_lines counted lines" { + const gpa = testing.allocator; + + var input: std.ArrayList(u8) = .empty; + defer input.deinit(gpa); + var line_buf: [64]u8 = undefined; + for (0..parsers.sample_lines + 10) |i| { + try input.appendSlice(gpa, try std.fmt.bufPrint(&line_buf, "0.0.0.0 host{d}.example.com\n", .{i})); + } + + const out = try gpa.alloc(u8, sample_buf_len); + defer gpa.free(out); + const sample = try sampleOf(input.items, out); + + var lines = std.mem.tokenizeScalar(u8, sample, '\n'); + var count: usize = 0; + while (lines.next()) |_| count += 1; + try testing.expectEqual(parsers.sample_lines, count); +} + +test "collectSample keeps the abp marker a long comment run would have hidden" { + const gpa = testing.allocator; + const long_comment = "! " ++ "c" ** (compiler.max_line_len - 2) ++ "\n"; + + var input: std.ArrayList(u8) = .empty; + defer input.deinit(gpa); + for (0..20) |_| try input.appendSlice(gpa, long_comment); + try input.appendSlice(gpa, "||ads.example.com^\n"); + + const out = try gpa.alloc(u8, sample_buf_len); + defer gpa.free(out); + const sample = try sampleOf(input.items, out); + + try testing.expectEqualStrings("||ads.example.com^\n", sample); + try testing.expectEqual(parsers.Format.abp, parsers.detectFormat(sample)); +} + +test "collectSample skips a line over max_line_len" { + const gpa = testing.allocator; + + var input: std.ArrayList(u8) = .empty; + defer input.deinit(gpa); + try input.appendNTimes(gpa, 'x', 8 * compiler.max_line_len); + try input.appendSlice(gpa, "\nads.example.com\n"); + + const out = try gpa.alloc(u8, sample_buf_len); + defer gpa.free(out); + + // A `Reader.fixed` holds the whole input, so the over-long line comes back + // rather than being refused. The compiler would skip it, so the sniff does. + const sample = try sampleOf(input.items, out); + try testing.expectEqualStrings("ads.example.com\n", sample); +} + +test "collectSample steps over a line that does not fit the reader buffer" { + const gpa = testing.allocator; + + var input: std.ArrayList(u8) = .empty; + defer input.deinit(gpa); + try input.appendNTimes(gpa, 'x', 4 * compiler.max_line_len); + try input.appendSlice(gpa, "\nads.example.com\n"); + + // A reader buffer smaller than the long line makes `takeDelimiter` report + // `error.StreamTooLong` and leave the stream where it was, which is the + // path that loops forever without the discard. + var backing: std.Io.Reader = .fixed(input.items); + var reader_buf: [compiler.max_line_len]u8 = undefined; + var limited = backing.limited(.unlimited, &reader_buf); + + const out = try gpa.alloc(u8, sample_buf_len); + defer gpa.free(out); + var w: std.Io.Writer = .fixed(out); + try collectSample(&limited.interface, &w); + + try testing.expectEqualStrings("ads.example.com\n", w.buffered()); +} + +test "bodyChecksum covers the list body followed by the wild body" { + const both = bodyChecksum("a.example.com\n", "b.example.com\n"); + var hasher = Sha256.init(.{}); + hasher.update("a.example.com\nb.example.com\n"); + var digest: [Sha256.digest_length]u8 = undefined; + hasher.final(&digest); + try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &both); + + // Order matters: the two halves are not interchangeable. + try testing.expect(!std.mem.eql(u8, &both, &bodyChecksum("b.example.com\n", "a.example.com\n"))); +} diff --git a/src/filter/matcher.zig b/src/filter/matcher.zig new file mode 100644 index 0000000..0efb299 --- /dev/null +++ b/src/filter/matcher.zig @@ -0,0 +1,1044 @@ +//! The filtering decision (PLAN §3.10, §7.1, §7.2): query-name normalization, +//! the candidate chain, and the immutable snapshot every query is evaluated +//! against. +//! +//! `Snapshot.evaluate` allocates nothing, opens nothing and reads no clock. It +//! is callable from an `std.Io` task holding one stack buffer and the reader +//! lock its caller already took. +//! +//! The decision is keyed on `{domain, group}`. The qtype travels with the query +//! for logging and for response synthesis; no step of §3.10 reads it, so it is +//! not a parameter here. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const model = @import("../config/model.zig"); +const address = @import("../platform/address.zig"); +const name = @import("../dns/name.zig"); +const types = @import("../dns/types.zig"); +const domain_set = @import("domain_set.zig"); +const rules = @import("rules.zig"); +const wildcard = @import("wildcard.zig"); + +pub const Reason = enum { + none, + rule_allow_exact, + rule_block_exact, + rule_allow_wildcard, + rule_block_wildcard, + blocklist_domain, + blocklist_wildcard, +}; + +pub const Decision = struct { + blocked: bool, + reason: Reason, + /// The candidate (for the exact and blocklist levels) or the pattern (for + /// the wildcard levels) that decided it. Borrowed from the caller's + /// normalized buffer or from the snapshot. "" when `reason == .none`. + matched: []const u8, + /// `.blocklist_*` only: index into `Snapshot.sources`, so the query log and + /// the UI can name the list that blocked the query. + source: ?u32 = null, +}; + +/// Lowercase ASCII, trailing dot stripped, written into `buf`. Returns a slice +/// of `buf`. The root name normalizes to "". +pub fn normalize(qname: name.Name, buf: *[types.max_name_len]u8) []const u8 { + const wire = qname.wire(); + var at: usize = 0; + var out: usize = 0; + while (at < wire.len) { + const label_len = wire[at]; + if (label_len == 0) break; + if (out != 0) { + buf[out] = '.'; + out += 1; + } + for (wire[at + 1 ..][0..label_len]) |byte| { + buf[out] = std.ascii.toLower(byte); + out += 1; + } + at += 1 + label_len; + } + return buf[0..out]; +} + +/// The full name, then each parent, ending at the last two-label suffix. The +/// TLD alone is not a candidate: a rule or a list entry on `com` is a +/// configuration mistake that would take the whole internet with it, and +/// refusing to walk that far costs nothing real. +pub const Candidates = struct { + rest: []const u8, + + pub fn init(domain: []const u8) Candidates { + return .{ .rest = domain }; + } + + pub fn next(self: *Candidates) ?[]const u8 { + const dot = std.mem.indexOfScalar(u8, self.rest, '.') orelse { + self.rest = ""; + return null; + }; + const current = self.rest; + self.rest = self.rest[dot + 1 ..]; + return current; + } +}; + +pub const SourceSets = struct { + /// `blocklist_sources` row id, so the manager can map a decision back to + /// the source row. + id: i64, + /// Snapshot-arena-owned; the source's display name for the UI. + name: []const u8, + domains: domain_set.DomainSet, + wildcards: domain_set.DomainSet, +}; + +pub const Group = struct { + id: i64, + name: []const u8, + safe_search: bool, + rules: rules.RuleSet, + /// Indices into `Snapshot.sources`, ascending, deduplicated. + sources: []const u32, +}; + +pub const ClientEntry = struct { key: address.NetAddress.Key, group: u32 }; +pub const PrefixEntry = struct { prefix: address.Prefix, group: u32, priority: i32 }; + +pub const Snapshot = struct { + arena: std.heap.ArenaAllocator, + groups: []Group, + sources: []SourceSets, + clients: []ClientEntry, + prefixes: []PrefixEntry, + /// Index into `groups` of the group named "default". Always valid: `build` + /// returns `error.MissingDefaultGroup` otherwise. + default_group: u32, + /// Monotonic, assigned by the manager. Logged on every swap so an operator + /// can tell which generation answered a query. + generation: u64, + + pub const Compiled = struct { list_body: []const u8, wild_body: []const u8 }; + + pub const Input = struct { + groups: []const model.Group, + /// One `groups` row id per `groups[i]`, in the same order. The config + /// model carries no ids, and a decision has to be mappable back to a + /// database row, so the caller — which read both — supplies them. + group_ids: []const i64, + group_sources: []const model.GroupSource, + sources: []const model.BlocklistSource, + /// One `blocklist_sources` row id per `sources[i]`, in the same order. + source_ids: []const i64, + rules: []const model.Rule, + clients: []const model.Client, + prefixes: []const model.ClientPrefix, + /// One entry per `sources[i]`, in the same order: the compiled bodies + /// already read from disk with their headers stripped. `null` means the + /// files were absent or unreadable; for an enabled source that is + /// `error.MissingCompiledSource`, because a silently unenforced + /// blocklist is exactly the failure PLAN §1.3 exists to prevent. A + /// disabled source needs no entry read. + compiled: []const ?Compiled, + seed: u64, + generation: u64, + }; + + pub const Error = error{ + OutOfMemory, + MissingDefaultGroup, + UnknownGroup, + UnknownSource, + MissingCompiledSource, + BadClientIp, + BadClientPrefix, + } || rules.Error; + + /// Builds an immutable snapshot. Every string is copied into the arena, so + /// the caller may free the repository lists immediately afterwards. + /// Disabled sources are skipped entirely — they cost no memory. + pub fn build(gpa: Allocator, input: Input) Error!Snapshot { + std.debug.assert(input.group_ids.len == input.groups.len); + std.debug.assert(input.source_ids.len == input.sources.len); + + var arena_state: std.heap.ArenaAllocator = .init(gpa); + errdefer arena_state.deinit(); + const arena = arena_state.allocator(); + + // `position[i]` is where `input.sources[i]` landed in `sources`, or + // null when the source is disabled and therefore not loaded. + const position = try arena.alloc(?u32, input.sources.len); + var enabled: u32 = 0; + for (input.sources, position) |row, *slot| { + if (!row.enabled) { + slot.* = null; + continue; + } + slot.* = enabled; + enabled += 1; + } + + const sources = try arena.alloc(SourceSets, enabled); + for (input.sources, input.source_ids, position, 0..) |row, id, slot, i| { + const at = slot orelse continue; + if (i >= input.compiled.len) return error.MissingCompiledSource; + const bodies = input.compiled[i] orelse return error.MissingCompiledSource; + sources[at] = .{ + .id = id, + .name = try arena.dupe(u8, row.name), + .domains = try domain_set.DomainSet.build(arena, bodies.list_body, input.seed), + .wildcards = try domain_set.DomainSet.build(arena, bodies.wild_body, input.seed), + }; + } + + const groups = try arena.alloc(Group, input.groups.len); + var group_rules: std.ArrayList(model.Rule) = .empty; + defer group_rules.deinit(gpa); + var group_sources: std.ArrayList(u32) = .empty; + defer group_sources.deinit(gpa); + + for (groups, input.groups, input.group_ids) |*group, row, id| { + group_rules.clearRetainingCapacity(); + for (input.rules) |rule_row| { + if (std.mem.eql(u8, rule_row.group, row.name)) try group_rules.append(gpa, rule_row); + } + + group_sources.clearRetainingCapacity(); + for (input.group_sources) |link| { + if (!std.mem.eql(u8, link.group, row.name)) continue; + const at = try findSource(input.sources, position, link.source_url); + // A link to a disabled source is not an error: the source + // exists, it is simply not loaded. + if (at) |index| try group_sources.append(gpa, index); + } + std.mem.sort(u32, group_sources.items, {}, std.sort.asc(u32)); + + group.* = .{ + .id = id, + .name = try arena.dupe(u8, row.name), + .safe_search = row.safe_search, + .rules = try rules.RuleSet.build(arena, group_rules.items, input.seed), + .sources = try arena.dupe(u32, dedupSorted(group_sources.items)), + }; + } + + // Every `group_sources` row must name a group that exists, whether or + // not that group also owns rules. + for (input.group_sources) |link| { + if (indexOfGroup(groups, link.group) == null) return error.UnknownGroup; + } + + const clients = try arena.alloc(ClientEntry, input.clients.len); + for (clients, input.clients) |*entry, row| { + const addr = address.NetAddress.parse(row.ip) catch return error.BadClientIp; + entry.* = .{ + .key = addr.key(), + .group = indexOfGroup(groups, row.group) orelse return error.UnknownGroup, + }; + } + + const prefixes = try arena.alloc(PrefixEntry, input.prefixes.len); + for (prefixes, input.prefixes) |*entry, row| { + entry.* = .{ + .prefix = address.Prefix.parse(row.prefix) catch return error.BadClientPrefix, + .group = indexOfGroup(groups, row.group) orelse return error.UnknownGroup, + .priority = row.priority, + }; + } + + return .{ + .arena = arena_state, + .groups = groups, + .sources = sources, + .clients = clients, + .prefixes = prefixes, + .default_group = indexOfGroup(groups, "default") orelse return error.MissingDefaultGroup, + .generation = input.generation, + }; + } + + pub fn deinit(self: *Snapshot) void { + self.arena.deinit(); + self.* = undefined; + } + + /// PLAN §3.10 precedence, allow winning at equal specificity: + /// 1. exact/parent allow rules 2. exact/parent block rules + /// 3. wildcard allow rules 4. wildcard block rules + /// 5. blocklist domains 6. blocklist wildcards + /// + /// The order is level-by-level over the whole candidate chain, not + /// candidate-by-candidate over the levels: level 1 is checked against every + /// candidate before level 2 is checked against any. That is what makes an + /// allow rule on the parent beat a block rule on the child, which is the + /// behaviour an allow list is written for. + /// + /// Level 5 tests only the full name and level 6 tests only proper parents: + /// a `.list` entry is the domain itself, a `.wild` entry is what `*.x.y` + /// means. Both walk the group's sources in ascending index order, so the + /// reported source is stable for a given snapshot. + /// + /// `domain` is normalized (`normalize`). No allocation, no lock, no clock. + pub fn evaluate(self: *const Snapshot, group: u32, domain: []const u8) Decision { + const g = &self.groups[group]; + + var level1: Candidates = .init(domain); + while (level1.next()) |candidate| { + if (g.rules.exact_allow.contains(candidate)) { + return .{ .blocked = false, .reason = .rule_allow_exact, .matched = candidate }; + } + } + + var level2: Candidates = .init(domain); + while (level2.next()) |candidate| { + if (g.rules.exact_block.contains(candidate)) { + return .{ .blocked = true, .reason = .rule_block_exact, .matched = candidate }; + } + } + + if (matchWildcard(g.rules.wildcard_allow, domain)) |pattern| { + return .{ .blocked = false, .reason = .rule_allow_wildcard, .matched = pattern }; + } + if (matchWildcard(g.rules.wildcard_block, domain)) |pattern| { + return .{ .blocked = true, .reason = .rule_block_wildcard, .matched = pattern }; + } + + for (g.sources) |index| { + if (self.sources[index].domains.contains(domain)) { + return .{ + .blocked = true, + .reason = .blocklist_domain, + .matched = domain, + .source = index, + }; + } + } + + var parents: Candidates = .init(domain); + // The full name is not a proper parent of itself. + _ = parents.next(); + while (parents.next()) |parent| { + for (g.sources) |index| { + if (self.sources[index].wildcards.contains(parent)) { + return .{ + .blocked = true, + .reason = .blocklist_wildcard, + .matched = parent, + .source = index, + }; + } + } + } + + return .{ .blocked = false, .reason = .none, .matched = "" }; + } + + /// PLAN §7.2 matching half: exact client row, else longest-prefix match + /// (ties broken by longer prefix, then by the preferred `priority`, which + /// is the lower number — the convention `upstreams.priority` already uses), + /// else the default group. Inserting the unseen client row needs a clock + /// and a database write on the query path and belongs to the handler. + pub fn groupForClient(self: *const Snapshot, addr: address.NetAddress) u32 { + const key = addr.key(); + for (self.clients) |entry| { + if (std.mem.eql(u8, &entry.key, &key)) return entry.group; + } + if (address.matchLongest(PrefixEntry, self.prefixes, addr)) |entry| return entry.group; + return self.default_group; + } + + pub fn groupIndexById(self: *const Snapshot, id: i64) ?u32 { + for (self.groups, 0..) |group, i| { + if (group.id == id) return @intCast(i); + } + return null; + } + + pub fn groupIndexByName(self: *const Snapshot, name_text: []const u8) ?u32 { + return indexOfGroup(self.groups, name_text); + } + + pub fn safeSearch(self: *const Snapshot, group: u32) bool { + return self.groups[group].safe_search; + } + + /// The structures this snapshot holds. The arena's own slack is excluded: + /// this number is the regression guard on the compiled data, not a report + /// on the allocator. + pub fn memoryBytes(self: *const Snapshot) usize { + var total: usize = 0; + for (self.sources) |*source| { + total += @sizeOf(SourceSets) + source.name.len + + source.domains.memoryBytes() + source.wildcards.memoryBytes(); + } + for (self.groups) |*group| { + total += @sizeOf(Group) + group.name.len + + group.rules.memoryBytes() + group.sources.len * @sizeOf(u32); + } + total += self.clients.len * @sizeOf(ClientEntry); + total += self.prefixes.len * @sizeOf(PrefixEntry); + return total; + } +}; + +// --------------------------------------------------------------------------- +// Internals +// --------------------------------------------------------------------------- + +fn matchWildcard(patterns: []const []const u8, domain: []const u8) ?[]const u8 { + var it: Candidates = .init(domain); + while (it.next()) |candidate| { + for (patterns) |pattern| { + if (wildcard.matches(pattern, candidate)) return pattern; + } + } + return null; +} + +fn indexOfGroup(groups: []const Group, group_name: []const u8) ?u32 { + for (groups, 0..) |group, i| { + if (std.mem.eql(u8, group.name, group_name)) return @intCast(i); + } + return null; +} + +/// The position in the loaded `sources` array, or null when the named source +/// exists but is disabled. A URL naming no configured source is an error: the +/// group would silently enforce one list fewer than the operator configured. +fn findSource( + rows: []const model.BlocklistSource, + position: []const ?u32, + url: []const u8, +) error{UnknownSource}!?u32 { + for (rows, position) |row, slot| { + if (std.mem.eql(u8, row.url, url)) return slot; + } + return error.UnknownSource; +} + +fn dedupSorted(items: []u32) []const u32 { + var kept: usize = 0; + for (items) |item| { + if (kept > 0 and items[kept - 1] == item) continue; + items[kept] = item; + kept += 1; + } + return items[0..kept]; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +const Fixture = struct { + groups: []const model.Group = &.{.{ .name = "default" }}, + group_ids: []const i64 = &.{1}, + group_sources: []const model.GroupSource = &.{}, + sources: []const model.BlocklistSource = &.{}, + source_ids: []const i64 = &.{}, + compiled: []const ?Snapshot.Compiled = &.{}, + rules: []const model.Rule = &.{}, + clients: []const model.Client = &.{}, + prefixes: []const model.ClientPrefix = &.{}, + seed: u64 = 0x5eed, +}; + +fn build(gpa: Allocator, fixture: Fixture) Snapshot.Error!Snapshot { + return Snapshot.build(gpa, .{ + .groups = fixture.groups, + .group_ids = fixture.group_ids, + .group_sources = fixture.group_sources, + .sources = fixture.sources, + .source_ids = fixture.source_ids, + .rules = fixture.rules, + .clients = fixture.clients, + .prefixes = fixture.prefixes, + .compiled = fixture.compiled, + .seed = fixture.seed, + .generation = 7, + }); +} + +fn rule(pattern: []const u8, kind: model.RuleKind, action: model.RuleAction) model.Rule { + return .{ .group = "default", .pattern = pattern, .kind = kind, .action = action }; +} + +// --- normalization and the candidate chain --------------------------------- + +test "normalize lowercases, strips the trailing dot and empties the root" { + var buf: [types.max_name_len]u8 = undefined; + + try testing.expectEqualStrings( + "ads.example.com", + normalize(try name.fromText("ADS.Example.COM."), &buf), + ); + try testing.expectEqualStrings("", normalize(try name.fromText("."), &buf)); + try testing.expectEqualStrings("com", normalize(try name.fromText("com"), &buf)); +} + +test "the candidate chain stops before the TLD" { + var it: Candidates = .init("a.b.example.com"); + try testing.expectEqualStrings("a.b.example.com", it.next().?); + try testing.expectEqualStrings("b.example.com", it.next().?); + try testing.expectEqualStrings("example.com", it.next().?); + try testing.expect(it.next() == null); + + var single: Candidates = .init("com"); + try testing.expect(single.next() == null); + + var root: Candidates = .init(""); + try testing.expect(root.next() == null); +} + +// --- precedence table (PLAN §3.10) ----------------------------------------- + +test "precedence: an exact block rule blocks its own name" { + const rows = [_]model.Rule{rule("ads.example.com", .exact, .block)}; + var snapshot = try build(testing.allocator, .{ .rules = &rows }); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "ads.example.com"); + try testing.expect(decision.blocked); + try testing.expectEqual(Reason.rule_block_exact, decision.reason); + try testing.expectEqualStrings("ads.example.com", decision.matched); +} + +test "precedence: an exact block rule on a parent blocks the child" { + const rows = [_]model.Rule{rule("example.com", .exact, .block)}; + var snapshot = try build(testing.allocator, .{ .rules = &rows }); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "ads.example.com"); + try testing.expect(decision.blocked); + try testing.expectEqual(Reason.rule_block_exact, decision.reason); + try testing.expectEqualStrings("example.com", decision.matched); +} + +test "precedence: an allow rule on the child beats a block rule on the parent" { + const rows = [_]model.Rule{ + rule("example.com", .exact, .block), + rule("ads.example.com", .exact, .allow), + }; + var snapshot = try build(testing.allocator, .{ .rules = &rows }); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "ads.example.com"); + try testing.expect(!decision.blocked); + try testing.expectEqual(Reason.rule_allow_exact, decision.reason); + try testing.expectEqualStrings("ads.example.com", decision.matched); +} + +test "precedence: an allow rule on the parent beats a block rule on the child" { + const rows = [_]model.Rule{ + rule("ads.example.com", .exact, .block), + rule("example.com", .exact, .allow), + }; + var snapshot = try build(testing.allocator, .{ .rules = &rows }); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "ads.example.com"); + try testing.expect(!decision.blocked); + try testing.expectEqual(Reason.rule_allow_exact, decision.reason); + try testing.expectEqualStrings("example.com", decision.matched); +} + +test "precedence: an exact block rule beats a wildcard allow rule" { + const rows = [_]model.Rule{ + rule("*.example.com", .wildcard, .allow), + rule("ads.example.com", .exact, .block), + }; + var snapshot = try build(testing.allocator, .{ .rules = &rows }); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "ads.example.com"); + try testing.expect(decision.blocked); + try testing.expectEqual(Reason.rule_block_exact, decision.reason); +} + +test "precedence: an allow wildcard beats an identical block wildcard" { + const rows = [_]model.Rule{ + rule("*.example.com", .wildcard, .allow), + rule("*.example.com", .wildcard, .block), + }; + var snapshot = try build(testing.allocator, .{ .rules = &rows }); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "a.example.com"); + try testing.expect(!decision.blocked); + try testing.expectEqual(Reason.rule_allow_wildcard, decision.reason); + try testing.expectEqualStrings("*.example.com", decision.matched); +} + +test "precedence: a block wildcard with no allow blocks" { + const rows = [_]model.Rule{rule("*.example.com", .wildcard, .block)}; + var snapshot = try build(testing.allocator, .{ .rules = &rows }); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "a.example.com"); + try testing.expect(decision.blocked); + try testing.expectEqual(Reason.rule_block_wildcard, decision.reason); + try testing.expectEqualStrings("*.example.com", decision.matched); +} + +const one_source = [_]model.BlocklistSource{.{ .url = "https://lists.test/a", .name = "list a" }}; +const one_source_id = [_]i64{11}; +const one_link = [_]model.GroupSource{ + .{ .group = "default", .source_url = "https://lists.test/a" }, +}; + +/// Holds the `compiled` array itself: a `Fixture` borrows it, so it has to +/// outlive the `build` call rather than live in a helper's frame. +const Lists = struct { + compiled: [1]?Snapshot.Compiled, + + fn init(list_body: []const u8, wild_body: []const u8) Lists { + return .{ .compiled = .{.{ .list_body = list_body, .wild_body = wild_body }} }; + } + + fn fixture(self: *const Lists) Fixture { + return .{ + .sources = &one_source, + .source_ids = &one_source_id, + .group_sources = &one_link, + .compiled = &self.compiled, + }; + } +}; + +test "precedence: a list entry blocks its own name" { + const lists: Lists = .init("tracker.net\n", ""); + var snapshot = try build(testing.allocator, lists.fixture()); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "tracker.net"); + try testing.expect(decision.blocked); + try testing.expectEqual(Reason.blocklist_domain, decision.reason); + try testing.expectEqualStrings("tracker.net", decision.matched); + try testing.expectEqual(@as(?u32, 0), decision.source); +} + +test "precedence: a list entry does not block a subdomain" { + const lists: Lists = .init("tracker.net\n", ""); + var snapshot = try build(testing.allocator, lists.fixture()); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "sub.tracker.net"); + try testing.expect(!decision.blocked); + try testing.expectEqual(Reason.none, decision.reason); +} + +test "precedence: a wild entry blocks a subdomain" { + const lists: Lists = .init("", "tracker.net\n"); + var snapshot = try build(testing.allocator, lists.fixture()); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "sub.tracker.net"); + try testing.expect(decision.blocked); + try testing.expectEqual(Reason.blocklist_wildcard, decision.reason); + try testing.expectEqualStrings("tracker.net", decision.matched); + try testing.expectEqual(@as(?u32, 0), decision.source); +} + +test "precedence: a wild entry does not block the apex" { + const lists: Lists = .init("", "tracker.net\n"); + var snapshot = try build(testing.allocator, lists.fixture()); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "tracker.net"); + try testing.expect(!decision.blocked); + try testing.expectEqual(Reason.none, decision.reason); +} + +test "precedence: an allow rule beats a wild entry" { + const lists: Lists = .init("", "tracker.net\n"); + var fixture = lists.fixture(); + const rows = [_]model.Rule{rule("sub.tracker.net", .exact, .allow)}; + fixture.rules = &rows; + + var snapshot = try build(testing.allocator, fixture); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "sub.tracker.net"); + try testing.expect(!decision.blocked); + try testing.expectEqual(Reason.rule_allow_exact, decision.reason); +} + +test "precedence: nothing configured allows with reason none" { + var snapshot = try build(testing.allocator, .{}); + defer snapshot.deinit(); + + const decision = snapshot.evaluate(0, "example.com"); + try testing.expect(!decision.blocked); + try testing.expectEqual(Reason.none, decision.reason); + try testing.expectEqualStrings("", decision.matched); + try testing.expect(decision.source == null); +} + +test "precedence: a source assigned to one group does not filter another" { + const groups = [_]model.Group{ .{ .name = "default" }, .{ .name = "kids" } }; + const ids = [_]i64{ 1, 2 }; + const links = [_]model.GroupSource{ + .{ .group = "kids", .source_url = "https://lists.test/a" }, + }; + var snapshot = try build(testing.allocator, .{ + .groups = &groups, + .group_ids = &ids, + .sources = &one_source, + .source_ids = &one_source_id, + .group_sources = &links, + .compiled = &.{.{ .list_body = "tracker.net\n", .wild_body = "" }}, + }); + defer snapshot.deinit(); + + const kids = snapshot.groupIndexByName("kids").?; + try testing.expect(snapshot.evaluate(kids, "tracker.net").blocked); + try testing.expect(!snapshot.evaluate(snapshot.default_group, "tracker.net").blocked); +} + +test "precedence: a disabled source filters nothing" { + const sources = [_]model.BlocklistSource{ + .{ .url = "https://lists.test/a", .name = "list a", .enabled = false }, + }; + var snapshot = try build(testing.allocator, .{ + .sources = &sources, + .source_ids = &one_source_id, + .group_sources = &one_link, + .compiled = &.{null}, + }); + defer snapshot.deinit(); + + try testing.expectEqual(@as(usize, 0), snapshot.sources.len); + try testing.expect(!snapshot.evaluate(0, "tracker.net").blocked); +} + +// --- group assignment ------------------------------------------------------ + +const two_groups = [_]model.Group{ .{ .name = "default" }, .{ .name = "kids" } }; +const two_group_ids = [_]i64{ 1, 2 }; + +test "groupForClient matches an exact IPv4 client" { + const clients = [_]model.Client{.{ .ip = "192.168.1.10", .group = "kids" }}; + var snapshot = try build(testing.allocator, .{ + .groups = &two_groups, + .group_ids = &two_group_ids, + .clients = &clients, + }); + defer snapshot.deinit(); + + const kids = snapshot.groupIndexByName("kids").?; + try testing.expectEqual(kids, snapshot.groupForClient(try address.NetAddress.parse("192.168.1.10"))); +} + +test "groupForClient matches an exact IPv6 client through the canonical key" { + const clients = [_]model.Client{.{ .ip = "fd00:0:0::1", .group = "kids" }}; + var snapshot = try build(testing.allocator, .{ + .groups = &two_groups, + .group_ids = &two_group_ids, + .clients = &clients, + }); + defer snapshot.deinit(); + + const kids = snapshot.groupIndexByName("kids").?; + try testing.expectEqual(kids, snapshot.groupForClient(try address.NetAddress.parse("fd00::1"))); +} + +test "groupForClient matches a prefix" { + const prefixes = [_]model.ClientPrefix{.{ .prefix = "192.168.1.0/24", .group = "kids" }}; + var snapshot = try build(testing.allocator, .{ + .groups = &two_groups, + .group_ids = &two_group_ids, + .prefixes = &prefixes, + }); + defer snapshot.deinit(); + + const kids = snapshot.groupIndexByName("kids").?; + try testing.expectEqual(kids, snapshot.groupForClient(try address.NetAddress.parse("192.168.1.7"))); + try testing.expectEqual( + snapshot.default_group, + snapshot.groupForClient(try address.NetAddress.parse("192.168.2.7")), + ); +} + +test "groupForClient prefers the longer prefix" { + const prefixes = [_]model.ClientPrefix{ + .{ .prefix = "192.168.0.0/16", .group = "default" }, + .{ .prefix = "192.168.1.0/24", .group = "kids" }, + }; + var snapshot = try build(testing.allocator, .{ + .groups = &two_groups, + .group_ids = &two_group_ids, + .prefixes = &prefixes, + }); + defer snapshot.deinit(); + + const kids = snapshot.groupIndexByName("kids").?; + try testing.expectEqual(kids, snapshot.groupForClient(try address.NetAddress.parse("192.168.1.7"))); +} + +test "groupForClient breaks a prefix tie by the preferred priority" { + const prefixes = [_]model.ClientPrefix{ + .{ .prefix = "192.168.1.0/24", .group = "default", .priority = 100 }, + .{ .prefix = "192.168.1.0/24", .group = "kids", .priority = 10 }, + }; + var snapshot = try build(testing.allocator, .{ + .groups = &two_groups, + .group_ids = &two_group_ids, + .prefixes = &prefixes, + }); + defer snapshot.deinit(); + + const kids = snapshot.groupIndexByName("kids").?; + try testing.expectEqual(kids, snapshot.groupForClient(try address.NetAddress.parse("192.168.1.7"))); +} + +test "groupForClient prefers an exact client row over a prefix" { + const clients = [_]model.Client{.{ .ip = "192.168.1.7", .group = "default" }}; + const prefixes = [_]model.ClientPrefix{.{ .prefix = "192.168.1.0/24", .group = "kids" }}; + var snapshot = try build(testing.allocator, .{ + .groups = &two_groups, + .group_ids = &two_group_ids, + .clients = &clients, + .prefixes = &prefixes, + }); + defer snapshot.deinit(); + + try testing.expectEqual( + snapshot.default_group, + snapshot.groupForClient(try address.NetAddress.parse("192.168.1.7")), + ); +} + +test "groupForClient falls back to the default group" { + var snapshot = try build(testing.allocator, .{}); + defer snapshot.deinit(); + + try testing.expectEqual( + snapshot.default_group, + snapshot.groupForClient(try address.NetAddress.parse("10.0.0.1")), + ); +} + +// --- build errors ---------------------------------------------------------- + +test "build without a default group is an error" { + const groups = [_]model.Group{.{ .name = "kids" }}; + const ids = [_]i64{2}; + try testing.expectError( + error.MissingDefaultGroup, + build(testing.allocator, .{ .groups = &groups, .group_ids = &ids }), + ); +} + +test "build with a link to an unknown source is an error" { + const links = [_]model.GroupSource{ + .{ .group = "default", .source_url = "https://lists.test/missing" }, + }; + try testing.expectError(error.UnknownSource, build(testing.allocator, .{ + .sources = &one_source, + .source_ids = &one_source_id, + .group_sources = &links, + .compiled = &.{.{ .list_body = "", .wild_body = "" }}, + })); +} + +test "build with a link from an unknown group is an error" { + const links = [_]model.GroupSource{ + .{ .group = "ghosts", .source_url = "https://lists.test/a" }, + }; + try testing.expectError(error.UnknownGroup, build(testing.allocator, .{ + .sources = &one_source, + .source_ids = &one_source_id, + .group_sources = &links, + .compiled = &.{.{ .list_body = "", .wild_body = "" }}, + })); +} + +test "build with an enabled source that has no compiled bodies is an error" { + try testing.expectError(error.MissingCompiledSource, build(testing.allocator, .{ + .sources = &one_source, + .source_ids = &one_source_id, + .group_sources = &one_link, + .compiled = &.{null}, + })); +} + +test "build with a bad client address is an error" { + const clients = [_]model.Client{.{ .ip = "not-an-ip" }}; + try testing.expectError( + error.BadClientIp, + build(testing.allocator, .{ .clients = &clients }), + ); +} + +test "build with a bad client prefix is an error" { + const prefixes = [_]model.ClientPrefix{.{ .prefix = "192.168.1.0/33" }}; + try testing.expectError( + error.BadClientPrefix, + build(testing.allocator, .{ .prefixes = &prefixes }), + ); +} + +test "build with a client in an unknown group is an error" { + const clients = [_]model.Client{.{ .ip = "192.168.1.1", .group = "ghosts" }}; + try testing.expectError( + error.UnknownGroup, + build(testing.allocator, .{ .clients = &clients }), + ); +} + +// --- snapshot properties --------------------------------------------------- + +test "the snapshot owns copies of every input string" { + const gpa = testing.allocator; + + const group_name = try gpa.dupe(u8, "default"); + const source_url = try gpa.dupe(u8, "https://lists.test/a"); + const source_name = try gpa.dupe(u8, "list a"); + const pattern = try gpa.dupe(u8, "ads.example.com"); + const body = try gpa.dupe(u8, "tracker.net\n"); + + var snapshot = blk: { + const groups = [_]model.Group{.{ .name = group_name }}; + const sources = [_]model.BlocklistSource{.{ .url = source_url, .name = source_name }}; + const links = [_]model.GroupSource{.{ .group = group_name, .source_url = source_url }}; + const rows = [_]model.Rule{ + .{ .group = group_name, .pattern = pattern, .kind = .exact, .action = .block }, + }; + break :blk try build(gpa, .{ + .groups = &groups, + .group_ids = &.{1}, + .sources = &sources, + .source_ids = &one_source_id, + .group_sources = &links, + .compiled = &.{.{ .list_body = body, .wild_body = "" }}, + .rules = &rows, + }); + }; + defer snapshot.deinit(); + + for ([_][]u8{ group_name, source_url, source_name, pattern, body }) |owned| { + @memset(owned, 'x'); + gpa.free(owned); + } + + try testing.expectEqualStrings("default", snapshot.groups[0].name); + try testing.expectEqualStrings("list a", snapshot.sources[0].name); + try testing.expect(snapshot.evaluate(0, "ads.example.com").blocked); + try testing.expect(snapshot.evaluate(0, "tracker.net").blocked); +} + +test "decisions do not depend on the seed" { + const rows = [_]model.Rule{ + rule("ads.example.com", .exact, .block), + rule("*.wild.example.com", .wildcard, .block), + rule("ok.wild.example.com", .exact, .allow), + }; + const lists: Lists = .init("tracker.net\n", "wildlist.net\n"); + var fixture = lists.fixture(); + fixture.rules = &rows; + + var a = try build(testing.allocator, fixture); + defer a.deinit(); + fixture.seed = 0xdead_beef_cafe_f00d; + var b = try build(testing.allocator, fixture); + defer b.deinit(); + + var buf: [64]u8 = undefined; + var i: usize = 0; + while (i < 40) : (i += 1) { + const domain = try std.fmt.bufPrint(&buf, "n{d}.wild.example.com", .{i}); + const da = a.evaluate(0, domain); + const db = b.evaluate(0, domain); + try testing.expectEqual(da.blocked, db.blocked); + try testing.expectEqual(da.reason, db.reason); + try testing.expectEqualStrings(da.matched, db.matched); + } + for ([_][]const u8{ "ads.example.com", "ok.wild.example.com", "tracker.net", "x.wildlist.net", "unrelated.org" }) |domain| { + const da = a.evaluate(0, domain); + const db = b.evaluate(0, domain); + try testing.expectEqual(da.blocked, db.blocked); + try testing.expectEqual(da.reason, db.reason); + try testing.expectEqualStrings(da.matched, db.matched); + } +} + +test "group lookup by id and by name" { + var snapshot = try build(testing.allocator, .{ + .groups = &two_groups, + .group_ids = &two_group_ids, + }); + defer snapshot.deinit(); + + try testing.expectEqual(@as(?u32, 0), snapshot.groupIndexById(1)); + try testing.expectEqual(@as(?u32, 1), snapshot.groupIndexById(2)); + try testing.expect(snapshot.groupIndexById(99) == null); + try testing.expectEqual(@as(?u32, 1), snapshot.groupIndexByName("kids")); + try testing.expect(snapshot.groupIndexByName("ghosts") == null); + try testing.expectEqual(@as(u64, 7), snapshot.generation); +} + +test "safeSearch reports the group setting" { + const groups = [_]model.Group{ + .{ .name = "default" }, + .{ .name = "kids", .safe_search = true }, + }; + var snapshot = try build(testing.allocator, .{ + .groups = &groups, + .group_ids = &two_group_ids, + }); + defer snapshot.deinit(); + + try testing.expect(!snapshot.safeSearch(0)); + try testing.expect(snapshot.safeSearch(1)); +} + +test "memoryBytes stays within the compiled-data bound" { + const gpa = testing.allocator; + const count = 10_000; + + var body: std.ArrayList(u8) = .empty; + defer body.deinit(gpa); + var line: [64]u8 = undefined; + var i: usize = 0; + while (i < count) : (i += 1) { + try body.appendSlice(gpa, try std.fmt.bufPrint(&line, "d{d:0>5}.example.com\n", .{i})); + } + + const lists: Lists = .init(body.items, ""); + var snapshot = try build(gpa, lists.fixture()); + defer snapshot.deinit(); + + // arena + index, over both bodies, plus a kilobyte of struct overhead. + const index_bytes = 2 * 16 * 1024 * @sizeOf(u32); + try testing.expect(snapshot.memoryBytes() < 2 * body.items.len + index_bytes + 1024); + try testing.expect(snapshot.evaluate(0, "d00042.example.com").blocked); +} + +fn buildUnderFailure(gpa: Allocator) !void { + const rows = [_]model.Rule{ + rule("ads.example.com", .exact, .block), + rule("*.tracker.net", .wildcard, .block), + }; + const clients = [_]model.Client{.{ .ip = "192.168.1.10" }}; + const prefixes = [_]model.ClientPrefix{.{ .prefix = "192.168.2.0/24" }}; + + const lists: Lists = .init("tracker.net\n", "wildlist.net\n"); + var fixture = lists.fixture(); + fixture.rules = &rows; + fixture.clients = &clients; + fixture.prefixes = &prefixes; + + var snapshot = try build(gpa, fixture); + defer snapshot.deinit(); + try testing.expect(snapshot.evaluate(0, "x.wildlist.net").blocked); +} + +test "build leaks nothing under allocation failure" { + try testing.checkAllAllocationFailures(testing.allocator, buildUnderFailure, .{}); +} diff --git a/src/filter/parser_abp.zig b/src/filter/parser_abp.zig new file mode 100644 index 0000000..c0a7d24 --- /dev/null +++ b/src/filter/parser_abp.zig @@ -0,0 +1,102 @@ +//! The Adblock Plus filter syntax, restricted to what a DNS sinkhole can +//! honour: domain anchors and bare names. Pure, `std` only. +//! +//! Exception rules (`@@`) are `.unsupported` rather than an allow entry. The +//! allow surface is the `rules` table, and a downloaded list that could quietly +//! allow a domain across every group is a policy hole the operator did not open. + +const std = @import("std"); +const parsers = @import("parsers.zig"); + +/// Tokens that a candidate name may never contain. `^` is a separator token in +/// this syntax and only a trailing one is meaningful for a domain rule. +const rule_tokens = "*^|/$"; + +pub fn parseLine(line: []const u8) parsers.Line { + const text = std.mem.trim(u8, line, &std.ascii.whitespace); + if (text.len == 0) return .{ .kind = .ignore }; + if (text[0] == '!') return .{ .kind = .ignore }; + if (text[0] == '[') return .{ .kind = .ignore }; + if (parsers.isElementHiding(text)) return .{ .kind = .unsupported }; + if (text[0] == '#') return .{ .kind = .ignore }; + if (std.mem.startsWith(u8, text, "@@")) return .{ .kind = .unsupported }; + if (text[0] == '/') return .{ .kind = .regex }; + if (std.mem.findScalar(u8, text, '$') != null) return .{ .kind = .unsupported }; + + if (std.mem.startsWith(u8, text, "||")) { + var candidate = text[2..]; + if (std.mem.endsWith(u8, candidate, "^")) candidate = candidate[0 .. candidate.len - 1]; + if (candidate.len == 0) return .{ .kind = .unsupported }; + if (std.mem.findAny(u8, candidate, rule_tokens) != null) return .{ .kind = .unsupported }; + // A domain anchor covers the domain itself as well as its subdomains, + // so the compiler emits an apex entry beside the wildcard one. + return .{ .kind = .wildcard, .text = candidate, .covers_apex = true }; + } + + if (std.mem.findAny(u8, text, rule_tokens) != null) return .{ .kind = .unsupported }; + return .{ .kind = .domain, .text = text }; +} + +const testing = std.testing; + +test "a bang comment is ignored" { + try testing.expectEqual(parsers.Kind.ignore, parseLine("! comment").kind); +} + +test "a list header is ignored" { + try testing.expectEqual(parsers.Kind.ignore, parseLine("[Adblock Plus 2.0]").kind); +} + +test "an empty line is ignored" { + try testing.expectEqual(parsers.Kind.ignore, parseLine("").kind); +} + +test "a domain anchor is a wildcard that covers its apex" { + const line = parseLine("||example.com^"); + try testing.expectEqual(parsers.Kind.wildcard, line.kind); + try testing.expectEqualStrings("example.com", line.text); + try testing.expect(line.covers_apex); +} + +test "a domain anchor without a separator is still a wildcard" { + const line = parseLine("||example.com"); + try testing.expectEqual(parsers.Kind.wildcard, line.kind); + try testing.expectEqualStrings("example.com", line.text); + try testing.expect(line.covers_apex); +} + +test "a modifier list is unsupported" { + try testing.expectEqual(parsers.Kind.unsupported, parseLine("||example.com^$third-party").kind); +} + +test "an exception rule is unsupported" { + try testing.expectEqual(parsers.Kind.unsupported, parseLine("@@||example.com^").kind); +} + +test "element hiding is unsupported" { + try testing.expectEqual(parsers.Kind.unsupported, parseLine("##.ad-banner").kind); + try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com#@#.ad").kind); + try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com#?#.ad").kind); +} + +test "a scheme anchor is unsupported" { + try testing.expectEqual(parsers.Kind.unsupported, parseLine("|http://example.com").kind); + try testing.expectEqual(parsers.Kind.unsupported, parseLine("|https://example.com").kind); +} + +test "a regex rule is recognized" { + try testing.expectEqual(parsers.Kind.regex, parseLine("/ads[0-9]+/").kind); +} + +test "a bare name is a domain" { + const line = parseLine("example.com"); + try testing.expectEqual(parsers.Kind.domain, line.kind); + try testing.expectEqualStrings("example.com", line.text); +} + +test "a rule token outside the supported forms is unsupported" { + try testing.expectEqual(parsers.Kind.unsupported, parseLine("ads*.example.com").kind); + try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com^").kind); + try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com|").kind); + try testing.expectEqual(parsers.Kind.unsupported, parseLine("||example.com/path^").kind); +} diff --git a/src/filter/parser_domains.zig b/src/filter/parser_domains.zig new file mode 100644 index 0000000..f2af7eb --- /dev/null +++ b/src/filter/parser_domains.zig @@ -0,0 +1,46 @@ +//! The plain domain-list format: one candidate name per line. Pure, `std` only. + +const std = @import("std"); +const parsers = @import("parsers.zig"); + +pub fn parseLine(line: []const u8) parsers.Line { + // Both comment markers appear in domain lists in the wild. + const marker = std.mem.findAny(u8, line, "#!") orelse line.len; + const text = std.mem.trim(u8, line[0..marker], &std.ascii.whitespace); + if (text.len == 0) return .{ .kind = .ignore }; + if (text[0] == '/') return .{ .kind = .regex }; + // A second field means the list is a mis-detected hosts file. Counting that + // is honest; guessing which field is the name is not. + if (std.mem.findAny(u8, text, &std.ascii.whitespace) != null) return .{ .kind = .unsupported }; + return .{ .kind = .domain, .text = text }; +} + +const testing = std.testing; + +test "a bare name" { + const line = parseLine("ads.example.com"); + try testing.expectEqual(parsers.Kind.domain, line.kind); + try testing.expectEqualStrings("ads.example.com", line.text); +} + +test "a bang comment line is ignored" { + try testing.expectEqual(parsers.Kind.ignore, parseLine("! comment").kind); +} + +test "a hash comment line is ignored" { + try testing.expectEqual(parsers.Kind.ignore, parseLine("# comment").kind); +} + +test "an inline comment is removed" { + const line = parseLine("example.com # x"); + try testing.expectEqual(parsers.Kind.domain, line.kind); + try testing.expectEqualStrings("example.com", line.text); +} + +test "a regex line is recognized" { + try testing.expectEqual(parsers.Kind.regex, parseLine("/re/").kind); +} + +test "a hosts line in a domains list is unsupported" { + try testing.expectEqual(parsers.Kind.unsupported, parseLine("0.0.0.0 example.com").kind); +} diff --git a/src/filter/parser_hosts.zig b/src/filter/parser_hosts.zig new file mode 100644 index 0000000..6269bf5 --- /dev/null +++ b/src/filter/parser_hosts.zig @@ -0,0 +1,83 @@ +//! The `hosts(5)` blocklist format: an optional sink address followed by one or +//! more names. Pure, `std` only. +//! +//! The sink address is not checked against a list of "blocking" addresses: a +//! list that maps to `127.0.0.1`, `0.0.0.0` or `::` carries the same +//! instruction, and a list that maps to a real address is still a set of names +//! the operator asked to block. + +const std = @import("std"); +const parsers = @import("parsers.zig"); + +pub fn parseLine(line: []const u8) parsers.Line { + const uncommented = if (std.mem.findScalar(u8, line, '#')) |at| line[0..at] else line; + const text = std.mem.trim(u8, uncommented, &std.ascii.whitespace); + if (text.len == 0) return .{ .kind = .ignore }; + if (text[0] == '/') return .{ .kind = .regex }; + + const split = std.mem.findAny(u8, text, &std.ascii.whitespace) orelse { + // A bare address with no name is a hosts line that names nothing to + // block; it is counted rather than compiled into an entry. + if (parsers.looksLikeIpLiteral(text)) return .{ .kind = .unsupported }; + return .{ .kind = .domain, .text = text }; + }; + + // Lists that are bare name lists with a hosts extension are common, so a + // first field that is not an address is a name like any other. + if (!parsers.looksLikeIpLiteral(text[0..split])) return .{ .kind = .domain, .text = text }; + + const names = std.mem.trimStart(u8, text[split..], &std.ascii.whitespace); + return .{ .kind = .domain, .text = names }; +} + +const testing = std.testing; + +test "a sink address and one name" { + const line = parseLine("0.0.0.0 ads.example.com"); + try testing.expectEqual(parsers.Kind.domain, line.kind); + try testing.expectEqualStrings("ads.example.com", line.text); +} + +test "a sink address and several names" { + const line = parseLine("127.0.0.1 a.example.com b.example.com"); + try testing.expectEqual(parsers.Kind.domain, line.kind); + try testing.expectEqualStrings("a.example.com b.example.com", line.text); +} + +test "single-label names survive the parser" { + const line = parseLine("::1 ip6-localhost ip6-loopback"); + try testing.expectEqual(parsers.Kind.domain, line.kind); + try testing.expectEqualStrings("ip6-localhost ip6-loopback", line.text); +} + +test "a trailing comment is removed" { + const line = parseLine("0.0.0.0 ads.example.com # tracker"); + try testing.expectEqual(parsers.Kind.domain, line.kind); + try testing.expectEqualStrings("ads.example.com", line.text); +} + +test "a comment line is ignored" { + try testing.expectEqual(parsers.Kind.ignore, parseLine("# whole line").kind); +} + +test "an empty line is ignored" { + try testing.expectEqual(parsers.Kind.ignore, parseLine("").kind); +} + +test "a blank line is ignored" { + try testing.expectEqual(parsers.Kind.ignore, parseLine(" ").kind); +} + +test "a regex line is recognized" { + try testing.expectEqual(parsers.Kind.regex, parseLine("/ads\\d+/").kind); +} + +test "a bare name without a sink address" { + const line = parseLine("example.com"); + try testing.expectEqual(parsers.Kind.domain, line.kind); + try testing.expectEqualStrings("example.com", line.text); +} + +test "a bare sink address names nothing" { + try testing.expectEqual(parsers.Kind.unsupported, parseLine("0.0.0.0").kind); +} diff --git a/src/filter/parsers.zig b/src/filter/parsers.zig new file mode 100644 index 0000000..163922d --- /dev/null +++ b/src/filter/parsers.zig @@ -0,0 +1,218 @@ +//! Blocklist line parsers: the shared vocabulary and the format sniffer. +//! +//! These files decide **format**, not validity. Whether a candidate is a usable +//! domain name is the compiler's decision, taken through `dns.name.fromText`. +//! `std` is the only import here and in every sibling parser: this file is the +//! root of a separate fuzz module, and a module root cannot import across its +//! own directory boundary. + +const std = @import("std"); + +pub const hosts = @import("parser_hosts.zig"); +pub const domains = @import("parser_domains.zig"); +pub const abp = @import("parser_abp.zig"); +pub const wildcard = @import("wildcard.zig"); + +pub const Format = enum { hosts, domains, abp }; + +pub const Kind = enum { + /// Nothing on the line, or only a comment. + ignore, + /// `text` holds one or more whitespace-separated candidate names. + domain, + /// `text` holds one candidate suffix; every proper subdomain of it matches. + wildcard, + /// A regex rule. Counted, skipped, never compiled (PLAN §2.2). + regex, + /// Syntactically a rule of this format, but one nxdns cannot honour: + /// an ABP modifier list, an exception rule, element hiding, a scheme anchor. + unsupported, +}; + +pub const Line = struct { + kind: Kind, + /// Borrowed from the caller's line. Not lowercased, not validated. + text: []const u8 = "", + /// `.wildcard` only. ABP `||x^` covers `x` itself as well as its subdomains, + /// so the compiler emits an additional `.list` entry when this is set. + covers_apex: bool = false, +}; + +/// Dispatches to the format's parser. The line must not contain '\n' or '\r'; +/// the caller strips them. +pub fn parseLine(format: Format, line: []const u8) Line { + return switch (format) { + .hosts => hosts.parseLine(line), + .domains => domains.parseLine(line), + .abp => abp.parseLine(line), + }; +} + +pub const sample_lines = 64; + +/// Picks a format from the first `sample_lines` lines that are not blank and +/// not comments: an ABP marker (`||`, `@@`, `##`, `$`) wins `.abp`; otherwise a +/// majority of lines whose first field looks like an IP literal wins `.hosts`; +/// otherwise `.domains`. +pub fn detectFormat(sample: []const u8) Format { + var considered: usize = 0; + var ip_first: usize = 0; + + var it = std.mem.splitScalar(u8, sample, '\n'); + while (it.next()) |raw| { + if (considered == sample_lines) break; + const line = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (line.len == 0) continue; + if (hasAbpMarker(line)) return .abp; + if (isComment(line)) continue; + considered += 1; + if (looksLikeIpLiteral(firstField(line))) ip_first += 1; + } + + if (ip_first * 2 > considered) return .hosts; + return .domains; +} + +/// `!` is the ABP comment marker and `#` the hosts one; both appear in every +/// format in the wild. `##`, `#@#` and `#?#` are element-hiding rules, not +/// comments, so they stay visible to `hasAbpMarker`. +pub fn isComment(line: []const u8) bool { + if (line.len == 0) return false; + if (line[0] == '!') return true; + if (line[0] != '#') return false; + return !isElementHiding(line); +} + +/// The element-hiding separators, which may also follow a domain list +/// (`example.com##.ad-banner`). +pub fn isElementHiding(line: []const u8) bool { + for ([_][]const u8{ "##", "#@#", "#?#", "#$#", "#%#" }) |marker| { + if (std.mem.find(u8, line, marker) != null) return true; + } + return false; +} + +fn hasAbpMarker(line: []const u8) bool { + if (std.mem.startsWith(u8, line, "||")) return true; + if (std.mem.startsWith(u8, line, "@@")) return true; + if (isElementHiding(line)) return true; + // A '$' modifier list only counts on a rule line: a hosts file whose + // comments mention a price must not be sniffed as ABP. + if (!isComment(line) and std.mem.findScalar(u8, line, '$') != null) return true; + return false; +} + +/// The line up to the first ASCII whitespace byte. +pub fn firstField(line: []const u8) []const u8 { + const end = std.mem.findAny(u8, line, &std.ascii.whitespace) orelse line.len; + return line[0..end]; +} + +/// A sniffing heuristic, not a parser: it recognizes dotted-quad IPv4 and any +/// hex-and-colon IPv6 spelling. `platform/address.zig` holds the real parser and +/// importing it would break this file's std-only constraint. +pub fn looksLikeIpLiteral(field: []const u8) bool { + if (field.len == 0) return false; + + if (std.mem.findScalar(u8, field, ':') != null) { + for (field) |c| { + if (c != ':' and c != '.' and !std.ascii.isHex(c)) return false; + } + return true; + } + + var parts: usize = 0; + var it = std.mem.splitScalar(u8, field, '.'); + while (it.next()) |part| { + parts += 1; + if (part.len == 0 or part.len > 3) return false; + for (part) |c| { + if (!std.ascii.isDigit(c)) return false; + } + } + return parts == 4; +} + +const testing = std.testing; + +test "detectFormat recognizes a hosts file" { + const sample = + \\# Title: example + \\0.0.0.0 ads.example.com + \\0.0.0.0 track.example.net + \\127.0.0.1 metrics.example.org + \\ + ; + try testing.expectEqual(Format.hosts, detectFormat(sample)); +} + +test "detectFormat recognizes a domains file" { + const sample = + \\# Title: example + \\ads.example.com + \\track.example.net + \\metrics.example.org + \\ + ; + try testing.expectEqual(Format.domains, detectFormat(sample)); +} + +test "detectFormat recognizes an abp file" { + const sample = + \\[Adblock Plus 2.0] + \\! Title: example + \\||ads.example.com^ + \\||track.example.net^ + \\ + ; + try testing.expectEqual(Format.abp, detectFormat(sample)); +} + +test "detectFormat falls back to domains on an all-comment sample" { + var buffer: [64 * 16]u8 = undefined; + var w: usize = 0; + for (0..64) |_| { + @memcpy(buffer[w..][0..14], "# a comment.\n\n"); + w += 14; + } + try testing.expectEqual(Format.domains, detectFormat(buffer[0..w])); +} + +test "detectFormat is not fooled by a dollar sign in a comment" { + const sample = + \\# donations welcome, $5 covers a month + \\0.0.0.0 ads.example.com + \\0.0.0.0 track.example.net + \\ + ; + try testing.expectEqual(Format.hosts, detectFormat(sample)); +} + +test "parseLine dispatches to the hosts parser" { + const line = parseLine(.hosts, "0.0.0.0 ads.example.com"); + try testing.expectEqual(Kind.domain, line.kind); + try testing.expectEqualStrings("ads.example.com", line.text); +} + +test "parseLine dispatches to the domains parser" { + const line = parseLine(.domains, "0.0.0.0 ads.example.com"); + try testing.expectEqual(Kind.unsupported, line.kind); +} + +test "parseLine dispatches to the abp parser" { + const line = parseLine(.abp, "||ads.example.com^"); + try testing.expectEqual(Kind.wildcard, line.kind); + try testing.expectEqualStrings("ads.example.com", line.text); + try testing.expect(line.covers_apex); +} + +test "looksLikeIpLiteral separates addresses from names" { + try testing.expect(looksLikeIpLiteral("0.0.0.0")); + try testing.expect(looksLikeIpLiteral("127.0.0.1")); + try testing.expect(looksLikeIpLiteral("::1")); + try testing.expect(looksLikeIpLiteral("fd00::dead:beef")); + try testing.expect(!looksLikeIpLiteral("example.com")); + try testing.expect(!looksLikeIpLiteral("add.face.cafe")); + try testing.expect(!looksLikeIpLiteral("1.2.3")); + try testing.expect(!looksLikeIpLiteral("")); +} diff --git a/src/filter/response.zig b/src/filter/response.zig new file mode 100644 index 0000000..d65f474 --- /dev/null +++ b/src/filter/response.zig @@ -0,0 +1,279 @@ +//! Blocked-response synthesis (PLAN §6.2). Pure: no allocation, no `std.Io`, +//! no clock. The caller supplies the buffer and gets back a prefix of it. +//! +//! No SOA is placed in the authority section. nxdns is not authoritative for a +//! blocked name, and a synthesized SOA would hand resolvers a negative-caching +//! TTL nxdns cannot honour: the operator can unblock the name at any moment, +//! and a client that cached the negative answer for the SOA's MINIMUM would +//! keep failing long after the block was lifted. + +const std = @import("std"); +const types = @import("../dns/types.zig"); +const header = @import("../dns/header.zig"); +const question = @import("../dns/question.zig"); +const edns = @import("../dns/edns.zig"); +const packet = @import("../dns/packet.zig"); +const model = @import("../config/model.zig"); + +pub const Options = struct { + mode: model.BlockResponse, + ttl: u32, +}; + +pub const Error = packet.ResponseBuilder.Error; + +const zero_a = [_]u8{0} ** 4; +const zero_aaaa = [_]u8{0} ** 16; + +/// Writes a blocked reply for `q` into `buf` and returns a prefix of it. +/// +/// `.zero`: A → 0.0.0.0, AAAA → ::, every other qtype → NOERROR with no answer +/// (NODATA). No address exists to synthesize for a qtype that carries none, +/// and answering NXDOMAIN for, say, an MX query would tell the client the +/// name does not exist while an A query for the same name says it does. +/// `.nxdomain`: RCODE = NXDOMAIN, no answer, for every qtype. +/// +/// Only class `IN` is answered with addresses; any other class takes the +/// NODATA path, because `0.0.0.0` is an IN-class address and means nothing in +/// CH or HS. +/// +/// `request_opt` echoes EDNS exactly as `handler.zig` does: a query that +/// carried an OPT record gets a reply carrying one with the same payload size +/// and the DO bit passed through. +pub fn writeBlocked( + buf: []u8, + request: header.Header, + q: question.Question, + request_opt: ?edns.OptRecord, + do_bit: bool, + options: Options, +) Error![]u8 { + var b = try packet.ResponseBuilder.init(buf, request, q); + + switch (options.mode) { + .nxdomain => b.setRcode(.nx_domain), + .zero => if (q.qclass == .in) switch (q.qtype) { + .a => try b.addAnswer(q.name, .a, .in, options.ttl, &zero_a), + .aaaa => try b.addAnswer(q.name, .aaaa, .in, options.ttl, &zero_aaaa), + else => {}, + }, + } + + if (request_opt) |opt| try b.addOptEcho(opt, do_bit); + return b.finish(); +} + +const testing = std.testing; +const name = @import("../dns/name.zig"); +const record = @import("../dns/record.zig"); + +/// A query for example.com A with an EDNS(0) OPT record advertising 4096 +/// bytes: id 0x1234, RD set, one question, one additional. +const query_bytes = + "\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x01" ++ + "\x07example\x03com\x00\x00\x01\x00\x01" ++ + "\x00\x00\x29\x10\x00\x00\x00\x00\x00\x00\x00"; + +const blocked_name = "ads.example.com"; +const ttl: u32 = 5; + +fn requestHeader() header.Header { + return (packet.parse(query_bytes) catch unreachable).header; +} + +fn requestOpt() edns.OptRecord { + const p = packet.parse(query_bytes) catch unreachable; + return edns.parseOpt(query_bytes, packet.findOptRecord(p).?) catch unreachable; +} + +fn blockedQuestion(qtype: types.Type, qclass: types.Class) !question.Question { + return .{ .name = try name.fromText(blocked_name), .qtype = qtype, .qclass = qclass }; +} + +/// Builds a blocked reply and re-parses it, asserting the parts every case +/// shares: the echoed id, the QR and RA flags, the echoed question, an empty +/// authority section, and an additional section that holds the OPT record only +/// when the query carried one. +fn expectBlocked( + buf: []u8, + mode: model.BlockResponse, + qtype: types.Type, + qclass: types.Class, + with_opt: bool, +) !packet.Packet { + const q = try blockedQuestion(qtype, qclass); + const bytes = try writeBlocked( + buf, + requestHeader(), + q, + if (with_opt) requestOpt() else null, + false, + .{ .mode = mode, .ttl = ttl }, + ); + + const p = try packet.parse(bytes); + try testing.expectEqual(@as(u16, 0x1234), p.header.id); + try testing.expect(p.header.flags.qr); + try testing.expect(p.header.flags.ra); + try testing.expect(p.header.flags.rd); + try testing.expectEqual(@as(u16, 1), p.header.qdcount); + try testing.expectEqual(@as(u16, 0), p.header.nscount); + try testing.expectEqual(@as(u16, if (with_opt) 1 else 0), p.header.arcount); + + const echoed = packet.firstQuestion(p).?; + try testing.expectEqualSlices(u8, q.name.wire(), echoed.name.wire()); + try testing.expectEqual(qtype, echoed.qtype); + try testing.expectEqual(qclass, echoed.qclass); + + if (with_opt) { + const opt = try edns.parseOpt(bytes, packet.findOptRecord(p).?); + try testing.expectEqual(@as(u16, 4096), opt.udp_payload_size); + try testing.expectEqual(false, opt.do_bit); + } else { + try testing.expect(packet.findOptRecord(p) == null); + } + return p; +} + +fn expectNodata(mode: model.BlockResponse, qtype: types.Type, qclass: types.Class) !void { + for ([_]bool{ false, true }) |with_opt| { + var buf: [512]u8 = undefined; + const p = try expectBlocked(&buf, mode, qtype, qclass, with_opt); + try testing.expectEqual(types.Rcode.no_error, p.header.flags.rcode); + try testing.expectEqual(@as(u16, 0), p.header.ancount); + } +} + +fn expectNxdomain(qtype: types.Type) !void { + for ([_]bool{ false, true }) |with_opt| { + var buf: [512]u8 = undefined; + const p = try expectBlocked(&buf, .nxdomain, qtype, .in, with_opt); + try testing.expectEqual(types.Rcode.nx_domain, p.header.flags.rcode); + try testing.expectEqual(@as(u16, 0), p.header.ancount); + } +} + +test "zero mode answers A with 0.0.0.0" { + for ([_]bool{ false, true }) |with_opt| { + var buf: [512]u8 = undefined; + const p = try expectBlocked(&buf, .zero, .a, .in, with_opt); + try testing.expectEqual(types.Rcode.no_error, p.header.flags.rcode); + try testing.expectEqual(@as(u16, 1), p.header.ancount); + + var it = packet.answers(p); + const answer = (try it.next()).?; + try testing.expectEqual(types.Type.a, answer.rtype); + try testing.expectEqual(@as(u16, @intFromEnum(types.Class.in)), answer.class); + try testing.expectEqual(ttl, answer.ttl); + try testing.expectEqualSlices( + u8, + (try name.fromText(blocked_name)).wire(), + answer.name.wire(), + ); + try testing.expectEqual([4]u8{ 0, 0, 0, 0 }, try record.rdataA(p.bytes, answer)); + try testing.expect((try it.next()) == null); + } +} + +test "zero mode answers AAAA with ::" { + for ([_]bool{ false, true }) |with_opt| { + var buf: [512]u8 = undefined; + const p = try expectBlocked(&buf, .zero, .aaaa, .in, with_opt); + try testing.expectEqual(types.Rcode.no_error, p.header.flags.rcode); + try testing.expectEqual(@as(u16, 1), p.header.ancount); + + var it = packet.answers(p); + const answer = (try it.next()).?; + try testing.expectEqual(types.Type.aaaa, answer.rtype); + try testing.expectEqual(ttl, answer.ttl); + try testing.expectEqual(zero_aaaa, try record.rdataAaaa(p.bytes, answer)); + try testing.expect((try it.next()) == null); + } +} + +test "zero mode answers MX with NODATA" { + try expectNodata(.zero, .mx, .in); +} + +test "zero mode answers HTTPS with NODATA" { + try expectNodata(.zero, .https, .in); +} + +test "zero mode answers a non-IN class with NODATA" { + try expectNodata(.zero, .a, .ch); + try expectNodata(.zero, .aaaa, .any); +} + +test "nxdomain mode answers A with NXDOMAIN" { + try expectNxdomain(.a); +} + +test "nxdomain mode answers AAAA with NXDOMAIN" { + try expectNxdomain(.aaaa); +} + +test "nxdomain mode answers MX with NXDOMAIN" { + try expectNxdomain(.mx); +} + +test "nxdomain mode answers HTTPS with NXDOMAIN" { + try expectNxdomain(.https); +} + +test "the DO bit passes through" { + for ([_]bool{ false, true }) |do_bit| { + var buf: [512]u8 = undefined; + const bytes = try writeBlocked( + &buf, + requestHeader(), + try blockedQuestion(.a, .in), + requestOpt(), + do_bit, + .{ .mode = .zero, .ttl = ttl }, + ); + const p = try packet.parse(bytes); + const opt = try edns.parseOpt(bytes, packet.findOptRecord(p).?); + try testing.expectEqual(do_bit, opt.do_bit); + } +} + +test "a ttl of zero survives the round trip" { + var buf: [512]u8 = undefined; + const bytes = try writeBlocked( + &buf, + requestHeader(), + try blockedQuestion(.a, .in), + null, + false, + .{ .mode = .zero, .ttl = 0 }, + ); + const p = try packet.parse(bytes); + var it = packet.answers(p); + try testing.expectEqual(@as(u32, 0), (try it.next()).?.ttl); +} + +test "a buffer too small reports a write failure instead of truncating" { + const q = try blockedQuestion(.a, .in); + const options: Options = .{ .mode = .zero, .ttl = ttl }; + + // Room for the header and the question, but not for the answer record. + var no_room_for_answer: [40]u8 = undefined; + try testing.expectError( + error.WriteFailed, + writeBlocked(&no_room_for_answer, requestHeader(), q, null, false, options), + ); + + // Room for the header and the question and the answer, but not the OPT. + var no_room_for_opt: [72]u8 = undefined; + try testing.expectError( + error.WriteFailed, + writeBlocked(&no_room_for_opt, requestHeader(), q, requestOpt(), false, options), + ); + + // Not even room for the header. + var tiny: [8]u8 = undefined; + try testing.expectError( + error.WriteFailed, + writeBlocked(&tiny, requestHeader(), q, null, false, options), + ); +} diff --git a/src/filter/rules.zig b/src/filter/rules.zig new file mode 100644 index 0000000..9bec2a8 --- /dev/null +++ b/src/filter/rules.zig @@ -0,0 +1,349 @@ +//! One group's explicit rules (PLAN §3.10 levels 1–4), compiled once into an +//! immutable form the query path can read without allocating. +//! +//! Exact patterns go into a `DomainSet`; wildcard patterns stay a flat, sorted +//! array of strings and are scanned linearly. Operator-authored wildcards are +//! few — `max_wildcards_per_group` caps them at 4096 — and a linear scan over +//! that many short patterns is cheaper than an index that would have to be +//! rebuilt on every snapshot swap. +//! +//! Pure: an allocator and plain values, no `std.Io`, no clock, no entropy +//! source. The hash seed arrives as a parameter. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const model = @import("../config/model.zig"); +const name = @import("../dns/name.zig"); +const types = @import("../dns/types.zig"); +const domain_set = @import("domain_set.zig"); +const wildcard = @import("wildcard.zig"); + +pub const Error = error{ OutOfMemory, BadPattern, TooManyWildcards } || domain_set.DomainSet.Error; + +/// Both wildcard lists of one group together. The cap exists so a rules table +/// edited into the millions cannot turn every query into a linear scan. +pub const max_wildcards_per_group: usize = 4096; + +pub const RuleSet = struct { + exact_allow: domain_set.DomainSet = .empty, + exact_block: domain_set.DomainSet = .empty, + /// Normalized and sorted, so a rebuild of the same rows produces the same + /// order and the same first match. + wildcard_allow: []const []const u8 = &.{}, + wildcard_block: []const []const u8 = &.{}, + /// One block holding the bytes of both wildcard lists; freed as a unit. + wildcard_bytes: []const u8 = &.{}, + + pub const empty: RuleSet = .{}; + + /// `rows` are one group's rules only; splitting `listRules` output by group + /// belongs to the caller, which is the only holder of the group table. + /// + /// Patterns are normalized (lowercase over ASCII, one trailing dot + /// stripped) and validated: `.exact` through `dns.name.fromText`, + /// `.wildcard` through `wildcard.validate`. An invalid pattern is + /// `error.BadPattern`, not a skipped row — every pattern passed + /// `config/validate.zig` on the way in, so an invalid one here means the + /// rows were edited underneath nxdns and a silently dropped allow rule + /// would block a domain the operator unblocked. + pub fn build(gpa: Allocator, rows: []const model.Rule, seed: u64) Error!RuleSet { + if (rows.len == 0) return .empty; + + var scratch: std.ArrayList(u8) = .empty; + defer scratch.deinit(gpa); + var spans: [4]std.ArrayList(Span) = .{ .empty, .empty, .empty, .empty }; + defer for (&spans) |*bucket| bucket.deinit(gpa); + + var wildcards: usize = 0; + var buf: [types.max_name_len]u8 = undefined; + for (rows) |row| { + const pattern = normalize(row.pattern, &buf) catch return error.BadPattern; + switch (row.kind) { + .exact => _ = name.fromText(pattern) catch return error.BadPattern, + .wildcard => { + wildcard.validate(pattern) catch return error.BadPattern; + wildcards += 1; + if (wildcards > max_wildcards_per_group) return error.TooManyWildcards; + }, + } + const bucket = &spans[bucketOf(row.kind, row.action)]; + try bucket.append(gpa, .{ .offset = scratch.items.len, .len = pattern.len }); + try scratch.appendSlice(gpa, pattern); + } + + // `scratch` stops growing here, so spans can become slices of it. + var sorted: [4]std.ArrayList([]const u8) = .{ .empty, .empty, .empty, .empty }; + defer for (&sorted) |*bucket| bucket.deinit(gpa); + for (&spans, &sorted) |*bucket, *out| { + try out.ensureTotalCapacityPrecise(gpa, bucket.items.len); + for (bucket.items) |span| { + out.appendAssumeCapacity(scratch.items[span.offset..][0..span.len]); + } + std.mem.sort([]const u8, out.items, {}, lessThanBytes); + dedupSorted(out); + } + + var self: RuleSet = .empty; + errdefer self.deinit(gpa); + + self.exact_allow = try buildSet(gpa, sorted[bucketOf(.exact, .allow)].items, seed); + self.exact_block = try buildSet(gpa, sorted[bucketOf(.exact, .block)].items, seed); + + const allow = sorted[bucketOf(.wildcard, .allow)].items; + const block = sorted[bucketOf(.wildcard, .block)].items; + var total: usize = 0; + for (allow) |pattern| total += pattern.len; + for (block) |pattern| total += pattern.len; + + const bytes = try gpa.alloc(u8, total); + self.wildcard_bytes = bytes; + var at: usize = 0; + self.wildcard_allow = try copyPatterns(gpa, allow, bytes, &at); + self.wildcard_block = try copyPatterns(gpa, block, bytes, &at); + + return self; + } + + pub fn deinit(self: *RuleSet, gpa: Allocator) void { + self.exact_allow.deinit(gpa); + self.exact_block.deinit(gpa); + gpa.free(self.wildcard_allow); + gpa.free(self.wildcard_block); + gpa.free(self.wildcard_bytes); + self.* = .empty; + } + + pub fn memoryBytes(self: *const RuleSet) usize { + return self.exact_allow.memoryBytes() + + self.exact_block.memoryBytes() + + self.wildcard_bytes.len + + (self.wildcard_allow.len + self.wildcard_block.len) * @sizeOf([]const u8); + } +}; + +// --------------------------------------------------------------------------- +// Internals +// --------------------------------------------------------------------------- + +/// Patterns are recorded as offsets because `scratch` reallocates while it +/// grows; they become slices only after the collecting pass ends. +const Span = struct { offset: usize, len: usize }; + +fn bucketOf(kind: model.RuleKind, action: model.RuleAction) usize { + const kind_bit: usize = switch (kind) { + .exact => 0, + .wildcard => 2, + }; + const action_bit: usize = switch (action) { + .allow => 0, + .block => 1, + }; + return kind_bit + action_bit; +} + +fn lessThanBytes(_: void, a: []const u8, b: []const u8) bool { + return std.mem.order(u8, a, b) == .lt; +} + +/// Duplicate rows are removed rather than rejected: two identical block rules +/// are not a corrupt database, and `DomainSet.build` requires a strictly +/// ascending body. +fn dedupSorted(list: *std.ArrayList([]const u8)) void { + var kept: usize = 0; + for (list.items) |item| { + if (kept > 0 and std.mem.eql(u8, list.items[kept - 1], item)) continue; + list.items[kept] = item; + kept += 1; + } + list.shrinkRetainingCapacity(kept); +} + +fn buildSet(gpa: Allocator, patterns: []const []const u8, seed: u64) Error!domain_set.DomainSet { + if (patterns.len == 0) return .empty; + + var body: std.ArrayList(u8) = .empty; + defer body.deinit(gpa); + for (patterns) |pattern| { + try body.appendSlice(gpa, pattern); + try body.append(gpa, '\n'); + } + return domain_set.DomainSet.build(gpa, body.items, seed); +} + +fn copyPatterns( + gpa: Allocator, + patterns: []const []const u8, + bytes: []u8, + at: *usize, +) Error![]const []const u8 { + if (patterns.len == 0) return &.{}; + const out = try gpa.alloc([]const u8, patterns.len); + for (out, patterns) |*slot, pattern| { + @memcpy(bytes[at.*..][0..pattern.len], pattern); + slot.* = bytes[at.*..][0..pattern.len]; + at.* += pattern.len; + } + return out; +} + +const NameError = error{BadName}; + +/// Lowercases over ASCII and strips one trailing dot. A byte ≥ 0x80 is +/// rejected: query names reach the matcher ASCII-lowercased, so a pattern +/// carrying a high byte could never match anything. +fn normalize(text: []const u8, buf: *[types.max_name_len]u8) NameError![]const u8 { + var rest = text; + if (rest.len > 0 and rest[rest.len - 1] == '.') rest = rest[0 .. rest.len - 1]; + if (rest.len == 0 or rest.len > types.max_name_len) return error.BadName; + + for (rest, 0..) |byte, i| { + if (byte >= 0x80 or byte < 0x21) return error.BadName; + buf[i] = std.ascii.toLower(byte); + } + return buf[0..rest.len]; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +fn rule(pattern: []const u8, kind: model.RuleKind, action: model.RuleAction) model.Rule { + return .{ .group = "default", .pattern = pattern, .kind = kind, .action = action }; +} + +test "exact rules land in the matching set" { + const rows = [_]model.Rule{ + rule("ads.example.com", .exact, .block), + rule("good.example.com", .exact, .allow), + }; + var set = try RuleSet.build(testing.allocator, &rows, 0x5eed); + defer set.deinit(testing.allocator); + + try testing.expect(set.exact_block.contains("ads.example.com")); + try testing.expect(!set.exact_block.contains("good.example.com")); + try testing.expect(set.exact_allow.contains("good.example.com")); + try testing.expectEqual(@as(usize, 0), set.wildcard_allow.len); + try testing.expectEqual(@as(usize, 0), set.wildcard_block.len); +} + +test "wildcard rules land in the matching list, sorted" { + const rows = [_]model.Rule{ + rule("*.z.example.com", .wildcard, .block), + rule("*.a.example.com", .wildcard, .block), + rule("*.allowed.example.com", .wildcard, .allow), + }; + var set = try RuleSet.build(testing.allocator, &rows, 0x5eed); + defer set.deinit(testing.allocator); + + try testing.expectEqual(@as(usize, 2), set.wildcard_block.len); + try testing.expectEqualStrings("*.a.example.com", set.wildcard_block[0]); + try testing.expectEqualStrings("*.z.example.com", set.wildcard_block[1]); + try testing.expectEqual(@as(usize, 1), set.wildcard_allow.len); + try testing.expectEqualStrings("*.allowed.example.com", set.wildcard_allow[0]); +} + +test "patterns are normalized to lowercase without a trailing dot" { + const rows = [_]model.Rule{ + rule("ADS.Example.COM.", .exact, .block), + rule("*.Tracker.NET.", .wildcard, .block), + }; + var set = try RuleSet.build(testing.allocator, &rows, 0); + defer set.deinit(testing.allocator); + + try testing.expect(set.exact_block.contains("ads.example.com")); + try testing.expectEqualStrings("*.tracker.net", set.wildcard_block[0]); +} + +test "duplicate rows collapse to one entry" { + const rows = [_]model.Rule{ + rule("ads.example.com", .exact, .block), + rule("ads.example.com.", .exact, .block), + rule("*.x.example.com", .wildcard, .block), + rule("*.x.example.com", .wildcard, .block), + }; + var set = try RuleSet.build(testing.allocator, &rows, 0); + defer set.deinit(testing.allocator); + + try testing.expectEqual(@as(u32, 1), set.exact_block.count); + try testing.expectEqual(@as(usize, 1), set.wildcard_block.len); +} + +test "an invalid exact pattern is an error" { + for ([_][]const u8{ "", ".", "a..b", "ads example.com", "ads\u{00e9}.example.com" }) |pattern| { + const rows = [_]model.Rule{rule(pattern, .exact, .block)}; + try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0)); + } +} + +test "an invalid wildcard pattern is an error" { + for ([_][]const u8{ "example.com", "ad*.example.com", "*..com" }) |pattern| { + const rows = [_]model.Rule{rule(pattern, .wildcard, .block)}; + try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0)); + } +} + +test "too many wildcards is an error" { + const gpa = testing.allocator; + const rows = try gpa.alloc(model.Rule, max_wildcards_per_group + 1); + defer gpa.free(rows); + + var patterns: std.ArrayList([]u8) = .empty; + defer { + for (patterns.items) |p| gpa.free(p); + patterns.deinit(gpa); + } + for (rows, 0..) |*row, i| { + const pattern = try std.fmt.allocPrint(gpa, "*.n{d}.example.com", .{i}); + try patterns.append(gpa, pattern); + row.* = rule(pattern, .wildcard, .block); + } + + try testing.expectError(error.TooManyWildcards, RuleSet.build(gpa, rows, 0)); +} + +test "an empty rule list builds the empty set" { + var set = try RuleSet.build(testing.allocator, &[_]model.Rule{}, 0); + defer set.deinit(testing.allocator); + + try testing.expect(!set.exact_block.contains("ads.example.com")); + try testing.expectEqual(@as(usize, 0), set.memoryBytes()); +} + +test "the empty rule set owns nothing" { + var set: RuleSet = .empty; + try testing.expect(!set.exact_allow.contains("x.example.com")); + try testing.expectEqual(@as(usize, 0), set.memoryBytes()); + set.deinit(testing.allocator); +} + +test "memoryBytes counts every part" { + const rows = [_]model.Rule{ + rule("ads.example.com", .exact, .block), + rule("*.tracker.net", .wildcard, .block), + }; + var set = try RuleSet.build(testing.allocator, &rows, 0); + defer set.deinit(testing.allocator); + + try testing.expect(set.memoryBytes() > set.exact_block.memoryBytes()); + try testing.expect(set.memoryBytes() >= "*.tracker.net".len); +} + +fn buildUnderFailure(gpa: Allocator) !void { + const rows = [_]model.Rule{ + rule("ads.example.com", .exact, .block), + rule("good.example.com", .exact, .allow), + rule("*.tracker.net", .wildcard, .block), + rule("*.ok.tracker.net", .wildcard, .allow), + }; + var set = try RuleSet.build(gpa, &rows, 0x5eed); + defer set.deinit(gpa); + try testing.expect(set.exact_block.contains("ads.example.com")); + try testing.expectEqualStrings("*.tracker.net", set.wildcard_block[0]); +} + +test "build leaks nothing under allocation failure" { + try testing.checkAllAllocationFailures(testing.allocator, buildUnderFailure, .{}); +} diff --git a/src/filter/safesearch.zig b/src/filter/safesearch.zig new file mode 100644 index 0000000..2e39dd9 --- /dev/null +++ b/src/filter/safesearch.zig @@ -0,0 +1,120 @@ +//! Per-group safe search (PLAN §7.4): the table of search and video domains +//! whose queries are rewritten to a provider-operated restricted hostname. +//! Pure: no allocation, no `std.Io`, no clock. +//! +//! Every target is a name the operator's upstream resolves. nxdns never +//! hardcodes an address for one: the providers move these hosts, and a stale +//! literal would send a household's search traffic to somebody else's server. +//! +//! Google's country domains (`google.de`, `google.co.uk`, …) are deliberately +//! not enumerated. The list is unbounded, it goes stale, and +//! `forcesafesearch.google.com` is the documented target for every one of them. +//! An operator who needs a country domain adds a rule. + +const std = @import("std"); +const name = @import("../dns/name.zig"); + +pub const Entry = struct { domain: []const u8, target: []const u8 }; + +/// Sorted ascending by `domain`, so the table is searchable and diffable. The +/// order is a correctness precondition of `lookup`, so it is asserted below at +/// comptime rather than left to review. +pub const table = [_]Entry{ + .{ .domain = "bing.com", .target = "strict.bing.com" }, + .{ .domain = "duckduckgo.com", .target = "safe.duckduckgo.com" }, + .{ .domain = "google.com", .target = "forcesafesearch.google.com" }, + .{ .domain = "m.youtube.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "pixabay.com", .target = "safesearch.pixabay.com" }, + .{ .domain = "www.bing.com", .target = "strict.bing.com" }, + .{ .domain = "www.duckduckgo.com", .target = "safe.duckduckgo.com" }, + .{ .domain = "www.google.com", .target = "forcesafesearch.google.com" }, + .{ .domain = "www.youtube-nocookie.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "www.youtube.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "youtube.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "youtube.googleapis.com", .target = "restrictmoderate.youtube.com" }, + .{ .domain = "youtubei.googleapis.com", .target = "restrictmoderate.youtube.com" }, +}; + +comptime { + for (table[1..], table[0 .. table.len - 1]) |entry, previous| { + if (std.mem.order(u8, previous.domain, entry.domain) != .lt) { + @compileError("safesearch.table must be sorted ascending by domain and duplicate-free"); + } + } +} + +fn orderDomain(query: []const u8, entry: Entry) std.math.Order { + return std.mem.order(u8, query, entry.domain); +} + +/// Exact match. `domain` must be normalized: lowercase, no trailing dot. +pub fn lookup(domain: []const u8) ?[]const u8 { + const index = std.sort.binarySearch(Entry, &table, domain, orderDomain) orelse return null; + return table[index].target; +} + +/// The rewritten question name for a matched query. Applying it — sending the +/// rewritten question upstream and prefixing the answer with a CNAME from the +/// original name to the target — is the handler's job (Phase 7). Nothing here +/// builds a response. +pub fn rewrite(domain: []const u8) ?name.Name { + const target = lookup(domain) orelse return null; + // Every target in `table` is a syntactically valid name, which the + // "every table entry is a valid domain name" test below asserts. + return name.fromText(target) catch unreachable; +} + +const testing = std.testing; + +test "every table entry is a valid domain name" { + for (table) |entry| { + _ = try name.fromText(entry.domain); + _ = try name.fromText(entry.target); + } +} + +test "lookup finds every table entry" { + for (table) |entry| { + try testing.expectEqualStrings(entry.target, lookup(entry.domain).?); + } +} + +test "lookup matches exactly and nothing else" { + try testing.expect(lookup("example.com") == null); + try testing.expect(lookup("") == null); + try testing.expect(lookup("com") == null); + // A subdomain of a listed name is not listed: the table is exact. + try testing.expect(lookup("images.google.com") == null); + // Nor is a parent, nor a name a listed one is a prefix of. + try testing.expect(lookup("google.com.evil.net") == null); + // The caller normalizes; an uppercase spelling is a miss, not a hit. + try testing.expect(lookup("GOOGLE.COM") == null); + // A trailing dot is not stripped here either. + try testing.expect(lookup("google.com.") == null); + // A country domain is deliberately absent. + try testing.expect(lookup("google.de") == null); +} + +test "lookup maps the youtube family to one target" { + const youtube = "restrictmoderate.youtube.com"; + for ([_][]const u8{ + "m.youtube.com", + "www.youtube-nocookie.com", + "www.youtube.com", + "youtube.com", + "youtube.googleapis.com", + "youtubei.googleapis.com", + }) |domain| { + try testing.expectEqualStrings(youtube, lookup(domain).?); + } +} + +test "rewrite returns the target as a wire name" { + const rewritten = rewrite("www.google.com").?; + try testing.expectEqualSlices( + u8, + (try name.fromText("forcesafesearch.google.com")).wire(), + rewritten.wire(), + ); + try testing.expect(rewrite("example.com") == null); +} diff --git a/src/filter/wildcard.zig b/src/filter/wildcard.zig new file mode 100644 index 0000000..73bfb18 --- /dev/null +++ b/src/filter/wildcard.zig @@ -0,0 +1,181 @@ +//! Label-pattern wildcards for filtering rules (PLAN §3.9). Pure: no +//! allocation, no `std.Io`, no recursion. +//! +//! A pattern is a domain name in which one or more labels are exactly `*`. +//! Each `*` label matches one or more labels of the queried name. Partial-label +//! globbing (`ad*.example.com`) is deliberately absent: it is regex by another +//! name, which PLAN §2.2 rules out. + +const std = @import("std"); + +pub const max_labels = 128; + +/// The longest domain name is 255 wire bytes, which is 253 bytes of text. +const max_pattern_len = 253; +const max_label_len = 63; + +pub const PatternError = error{ + /// No label is exactly "*". + NoWildcard, + /// A label contains '*' but is not exactly "*". Partial-label globbing + /// (`ad*.example.com`) is out of scope: it is regex by another name, and + /// PLAN §3.9 defines the wildcard as a label pattern. + PartialWildcardLabel, + EmptyLabel, + LabelTooLong, + PatternTooLong, + TooManyLabels, +}; + +/// Syntax only. A valid pattern has at least one label that is exactly "*", +/// every other label is 1–63 bytes with no '*' inside it, and the whole +/// pattern is at most 253 bytes over at most `max_labels` labels. +pub fn validate(pattern: []const u8) PatternError!void { + // The label count is checked before the byte length so that both bounds + // stay individually reportable: any pattern with more than `max_labels` + // labels also exceeds `max_pattern_len`. + if (std.mem.count(u8, pattern, ".") + 1 > max_labels) return error.TooManyLabels; + if (pattern.len > max_pattern_len) return error.PatternTooLong; + + var star = false; + var it = std.mem.splitScalar(u8, pattern, '.'); + while (it.next()) |label| { + if (label.len == 0) return error.EmptyLabel; + if (label.len > max_label_len) return error.LabelTooLong; + if (std.mem.eql(u8, label, "*")) { + star = true; + } else if (std.mem.findScalar(u8, label, '*') != null) { + return error.PartialWildcardLabel; + } + } + if (!star) return error.NoWildcard; +} + +/// `domain` is already normalized: lowercase, no trailing dot. `pattern` is +/// lowercase. Each "*" label matches ONE OR MORE labels. +/// Allocation-free; the backtracking is bounded by `max_labels` on both sides. +pub fn matches(pattern: []const u8, domain: []const u8) bool { + var pattern_labels: [max_labels][]const u8 = undefined; + var domain_labels: [max_labels][]const u8 = undefined; + + // `validate` rejects a pattern above the label bound and the 253-byte name + // limit bounds the domain, so neither overflow can reach here from the + // matcher. Both are re-checked so that unvalidated input still terminates. + const pattern_len = split(pattern, &pattern_labels) orelse return false; + const domain_len = split(domain, &domain_labels) orelse return false; + + var d: usize = 0; + var p: usize = 0; + var star: ?usize = null; + var star_end: usize = 0; + + while (d < domain_len) { + if (p < pattern_len and isStar(pattern_labels[p])) { + // A '*' takes one label now and grows by one on each backtrack. + star = p; + p += 1; + d += 1; + star_end = d; + } else if (p < pattern_len and std.mem.eql(u8, pattern_labels[p], domain_labels[d])) { + p += 1; + d += 1; + } else if (star) |s| { + p = s + 1; + star_end += 1; + d = star_end; + } else { + return false; + } + } + // A trailing '*' has already consumed its label; nothing may be left over. + return p == pattern_len; +} + +fn isStar(label: []const u8) bool { + return label.len == 1 and label[0] == '*'; +} + +/// Null when `text` holds more than `max_labels` labels. +fn split(text: []const u8, out: *[max_labels][]const u8) ?usize { + var n: usize = 0; + var it = std.mem.splitScalar(u8, text, '.'); + while (it.next()) |label| { + if (n == max_labels) return null; + out[n] = label; + n += 1; + } + return n; +} + +const testing = std.testing; + +test "validate accepts a leading wildcard label" { + try validate("*.doubleclick.net"); +} + +test "validate accepts an interior wildcard label" { + try validate("ads.*.example.com"); +} + +test "validate rejects a pattern with no wildcard label" { + try testing.expectError(error.NoWildcard, validate("example.com")); +} + +test "validate rejects a partial wildcard label" { + try testing.expectError(error.PartialWildcardLabel, validate("a*b.com")); +} + +test "validate rejects an empty label" { + try testing.expectError(error.EmptyLabel, validate("a..b")); +} + +test "validate rejects an oversize label" { + const pattern = "*." ++ ("a" ** 64); + try testing.expectError(error.LabelTooLong, validate(pattern)); +} + +test "validate rejects an oversize pattern" { + const label = "a" ** 60; + const pattern = "*." ++ label ++ "." ++ label ++ "." ++ label ++ "." ++ label ++ "." ++ label; + try testing.expect(pattern.len > 253); + try testing.expectError(error.PatternTooLong, validate(pattern)); +} + +test "validate rejects too many labels" { + const pattern = "*." ++ ("a." ** 199) ++ "com"; + try testing.expectError(error.TooManyLabels, validate(pattern)); +} + +test "matches one label under a leading wildcard" { + try testing.expect(matches("*.doubleclick.net", "a.doubleclick.net")); +} + +test "matches several labels under a leading wildcard" { + try testing.expect(matches("*.doubleclick.net", "a.b.doubleclick.net")); +} + +test "a leading wildcard does not match the apex" { + try testing.expect(!matches("*.doubleclick.net", "doubleclick.net")); +} + +test "matches one label at an interior wildcard" { + try testing.expect(matches("ads.*.example.com", "ads.eu.example.com")); +} + +test "matches several labels at an interior wildcard" { + try testing.expect(matches("ads.*.example.com", "ads.eu.west.example.com")); +} + +test "an interior wildcard requires at least one label" { + try testing.expect(!matches("ads.*.example.com", "ads.example.com")); +} + +test "a pattern does not match a name that only contains it" { + try testing.expect(!matches("*.example.com", "example.com.evil.net")); +} + +test "a pathological pattern terminates" { + const pattern = ("*." ** 8) ++ "example.com"; + const domain = ("a." ** 98) ++ "example.net"; + try testing.expect(!matches(pattern, domain)); +} diff --git a/src/local/forward_client.zig b/src/local/forward_client.zig new file mode 100644 index 0000000..4eb6864 --- /dev/null +++ b/src/local/forward_client.zig @@ -0,0 +1,455 @@ +//! Plain UDP/TCP resolver client for conditional forward zones (PLAN §6.5). +//! +//! A forward zone points at a box on the LAN — a router, a NAS, an internal +//! resolver — which speaks port 53 and nothing else. `transport.Endpoint` knows +//! only `https://` and `tls://` by design, so the configuration for this client +//! comes from `validate.Resolver` instead. The interface it implements is the +//! same `transport.Client` every upstream implements, so the Phase 7 handler +//! treats a forward zone exactly like any other exchange. +//! +//! No health tracking and no backoff live here. `upstream/health.zig` and +//! `upstream/pool.zig` model the upstream *pool*, where failing over to a second +//! endpoint is the whole point. A forward zone has exactly one designated +//! resolver and no failover partner, so a backoff would only add latency to a +//! failure the caller already sees. Their absence is a decision, not an +//! oversight. +//! +//! One `ForwardClient` is used by one task at a time: `stats` is a plain struct +//! and `frame_buf` is not shared. + +const std = @import("std"); +const net = std.Io.net; + +const transport = @import("../upstream/transport.zig"); +const validate = @import("../config/validate.zig"); +const dns_header = @import("../dns/header.zig"); + +const log = std.log.scoped(.forward_client); + +/// RFC 1035 §4.2.2 length prefix for DNS over TCP. +/// The TCP path splits `frame_buf` between the socket writer and the socket +/// reader. Neither half has to hold a whole message — the reply is read +/// straight into the caller's `response_buf` — so this is a floor that keeps +/// each half large enough to frame a query in one write, not a capacity. +pub const min_frame_buf: usize = 1024; + +pub const ForwardClient = struct { + resolver: validate.Resolver, + /// Caller-owned scratch for the TCP length-prefixed path. + frame_buf: []u8, + /// On the `.awake` clock at the caller's choosing, so a suspended host does + /// not burn the budget while it sleeps. + read_timeout: std.Io.Clock.Duration, + stats: Stats = .{}, + + pub const Stats = struct { + queries: u64 = 0, + /// TC=1 over UDP, so the exchange was retried over TCP. + udp_truncated: u64 = 0, + /// A datagram arrived from an address other than the resolver's. It was + /// discarded and the receive retried within the remaining budget, which + /// is invisible to the caller and would otherwise be an unrecorded + /// failure mode. + foreign_datagrams: u64 = 0, + /// Exchanges that returned a peer fault or a local resource error. + /// A cancellation is neither, so it is not counted. + failures: u64 = 0, + }; + + /// An undersized `frame_buf` is a wiring bug in this process, not a runtime + /// condition, so it is an assertion. + pub fn init( + resolver: validate.Resolver, + frame_buf: []u8, + read_timeout: std.Io.Clock.Duration, + ) ForwardClient { + std.debug.assert(frame_buf.len >= min_frame_buf); + return .{ + .resolver = resolver, + .frame_buf = frame_buf, + .read_timeout = read_timeout, + }; + } + + pub fn client(self: *ForwardClient) transport.Client { + return .{ .ptr = self, .exchangeFn = exchangeFn }; + } + + fn exchangeFn( + ptr: *anyopaque, + io: std.Io, + query: []const u8, + response_buf: []u8, + ) transport.ExchangeError![]u8 { + const self: *ForwardClient = @ptrCast(@alignCast(ptr)); + return self.exchange(io, query, response_buf); + } + + /// `.udp` resolvers send one datagram and fall back to TCP when the answer + /// comes back with TC=1. `.tcp` resolvers skip straight to the TCP path. + pub fn exchange( + self: *ForwardClient, + io: std.Io, + query: []const u8, + response_buf: []u8, + ) transport.ExchangeError![]u8 { + // The TCP length prefix is 16-bit, so a longer query cannot be framed. + if (query.len > transport.max_message_len) return error.BufferTooSmall; + if (response_buf.len == 0) return error.BufferTooSmall; + + self.stats.queries += 1; + return self.route(io, query, response_buf) catch |err| { + switch (transport.group(err)) { + .peer_fault, .local_resource => self.stats.failures += 1, + .cancellation => {}, + } + return err; + }; + } + + fn route( + self: *ForwardClient, + io: std.Io, + query: []const u8, + response_buf: []u8, + ) transport.ExchangeError![]u8 { + if (self.resolver.scheme == .udp) { + if (try self.exchangeUdp(io, query, response_buf)) |reply| return reply; + } + return self.exchangeTcp(io, query, response_buf); + } + + /// `null` means the resolver set TC=1 and the caller must retry over TCP. + /// + /// The socket is bound to the wildcard address of the resolver's family on + /// an ephemeral port, so the kernel picks the source port for every + /// exchange rather than this process reusing one. + fn exchangeUdp( + self: *ForwardClient, + io: std.Io, + query: []const u8, + response_buf: []u8, + ) transport.ExchangeError!?[]u8 { + const dest = self.destination(); + const local = wildcardFor(dest); + + const socket = local.bind(io, .{ .mode = .dgram }) catch |err| { + log.debug("forward resolver: udp bind failed: {s}", .{@errorName(err)}); + return mapPhase(err, error.ConnectFailed); + }; + defer closeSocket(io, &socket); + + socket.send(io, &dest, query) catch |err| { + log.debug("forward resolver: udp send failed: {s}", .{@errorName(err)}); + return mapPhase(err, error.SendFailed); + }; + + // A deadline, not a duration: a discarded foreign datagram restarts the + // receive, and a duration would hand each retry the full budget again. + const deadline = (std.Io.Timeout{ .duration = self.read_timeout }).toDeadline(io); + + while (true) { + const msg = socket.receiveTimeout(io, response_buf, deadline) catch |err| switch (err) { + error.Timeout => return error.Timeout, + error.ConcurrencyUnavailable => return error.SystemResources, + else => return mapPhase(err, error.ReceiveFailed), + }; + + // Off-path spoofing is the reason the source address is checked at + // all: the first datagram to arrive is not necessarily the + // resolver's. + if (!msg.from.eql(&dest)) { + self.stats.foreign_datagrams += 1; + continue; + } + + // The kernel threw the tail away because `response_buf` was too + // small, so the message cannot be parsed and TC=1 cannot be read + // out of it. + if (msg.flags.trunc) return error.ResponseTooLarge; + + const reply = response_buf[0..msg.data.len]; + try transport.validateResponse(query, reply); + + // Read after validation: acting on the TC bit of a message that has + // not been matched to the query would let anything that reaches the + // socket force a TCP connection. + const parsed = dns_header.parse(reply) catch return error.BadResponse; + if (parsed.flags.tc) { + self.stats.udp_truncated += 1; + return null; + } + return reply; + } + } + + /// No stream read or write in 0.16.0 takes a timeout, so the budget is a + /// second task and the loser is canceled. `ConnectOptions.timeout` is never + /// set: the Threaded backend panics on it (Threaded.zig:12076). + fn exchangeTcp( + self: *ForwardClient, + io: std.Io, + query: []const u8, + response_buf: []u8, + ) transport.ExchangeError![]u8 { + var outcomes: [2]Outcome = undefined; + var race: std.Io.Select(Outcome) = .init(io, &outcomes); + defer race.cancelDiscard(); + + race.concurrent(.exchange, tcpOnce, .{ self, io, query, response_buf }) catch |err| switch (err) { + error.ConcurrencyUnavailable => return error.SystemResources, + }; + race.concurrent(.expiry, expire, .{ io, self.read_timeout }) catch |err| switch (err) { + error.ConcurrencyUnavailable => return error.SystemResources, + }; + + switch (try race.await()) { + .exchange => |result| return result, + .expiry => |result| { + // A canceled sleep means this whole task is being torn down, + // not that the resolver is slow. + try result; + return error.Timeout; + }, + } + } + + fn tcpOnce( + self: *ForwardClient, + io: std.Io, + query: []const u8, + response_buf: []u8, + ) transport.ExchangeError![]u8 { + const dest = self.destination(); + + const stream = dest.connect(io, .{ .mode = .stream }) catch |err| { + log.debug("forward resolver: tcp connect failed: {s}", .{@errorName(err)}); + return mapPhase(err, error.ConnectFailed); + }; + defer closeStream(io, &stream); + + const split = self.frame_buf.len / 2; + var stream_writer = stream.writer(io, self.frame_buf[0..split]); + var stream_reader = stream.reader(io, self.frame_buf[split..]); + + const w = &stream_writer.interface; + const prefix = transport.framePrefix(@intCast(query.len)); + w.writeAll(&prefix) catch |err| return sendFailure(&stream_writer, err); + w.writeAll(query) catch |err| return sendFailure(&stream_writer, err); + w.flush() catch |err| return sendFailure(&stream_writer, err); + + const r = &stream_reader.interface; + var prefix_bytes: [transport.prefix_len]u8 = undefined; + r.readSliceAll(&prefix_bytes) catch |err| return receiveFailure(&stream_reader, err); + + // RFC 1035 §4.2.2 gives no meaning to a zero-length message. + const len = transport.parsePrefix(prefix_bytes); + if (len == 0) return error.BadResponse; + if (len > response_buf.len) return error.ResponseTooLarge; + r.readSliceAll(response_buf[0..len]) catch |err| return receiveFailure(&stream_reader, err); + + try transport.validateResponse(query, response_buf[0..len]); + return response_buf[0..len]; + } + + fn destination(self: *const ForwardClient) net.IpAddress { + return self.resolver.addr.toIp(self.resolver.port); + } +}; + +const Outcome = union(enum) { + exchange: transport.ExchangeError![]u8, + expiry: std.Io.Cancelable!void, +}; + +fn expire(io: std.Io, duration: std.Io.Clock.Duration) std.Io.Cancelable!void { + return duration.sleep(io); +} + +/// The local address a datagram to `dest` is sent from: same family, port +/// chosen by the kernel. +fn wildcardFor(dest: net.IpAddress) net.IpAddress { + return switch (dest) { + .ip4 => .{ .ip4 = .unspecified(0) }, + .ip6 => .{ .ip6 = .unspecified(0) }, + }; +} + +/// The TCP budget cancels the exchange task. The next cancelable `Io` call in +/// the `defer` chain would then return `error.Canceled` and skip the close, +/// leaking the descriptor, so both closes run with cancellation blocked. +fn closeStream(io: std.Io, stream: *const net.Stream) void { + const prev = io.swapCancelProtection(.blocked); + defer _ = io.swapCancelProtection(prev); + stream.close(io); +} + +fn closeSocket(io: std.Io, socket: *const net.Socket) void { + const prev = io.swapCancelProtection(.blocked); + defer _ = io.swapCancelProtection(prev); + socket.close(io); +} + +fn mapPhase(err: anyerror, phase: transport.PeerFault) transport.ExchangeError { + return transport.mapLocal(err) orelse phase; +} + +/// `Io.Writer` collapses everything to `error.WriteFailed` and stashes the +/// cause. Unwrapping it is what keeps `error.Canceled` and the local resource +/// errors out of the peer fault group. +fn sendFailure(stream_writer: *const net.Stream.Writer, err: anyerror) transport.ExchangeError { + const cause: anyerror = if (err == error.WriteFailed and stream_writer.err != null) + stream_writer.err.? + else + err; + return mapPhase(cause, error.SendFailed); +} + +fn receiveFailure(stream_reader: *const net.Stream.Reader, err: anyerror) transport.ExchangeError { + const cause: anyerror = if (err == error.ReadFailed and stream_reader.err != null) + stream_reader.err.? + else + err; + return mapPhase(cause, error.ReceiveFailed); +} + +const testing = std.testing; + +fn testBuf() [min_frame_buf]u8 { + return undefined; +} + +test "ForwardClient satisfies the Client interface" { + var buf = testBuf(); + var fc: ForwardClient = .init( + try validate.parseResolver("udp://192.168.1.1:53"), + &buf, + .{ .raw = .fromMilliseconds(500), .clock = .awake }, + ); + + const iface: transport.Client = fc.client(); + try testing.expectEqual(@as(*anyopaque, @ptrCast(&fc)), iface.ptr); + try testing.expectEqual(validate.ResolverScheme.udp, fc.resolver.scheme); + try testing.expectEqual(@as(u16, 53), fc.resolver.port); +} + +test "the stats struct starts at zero" { + const stats: ForwardClient.Stats = .{}; + try testing.expectEqual(@as(u64, 0), stats.queries); + try testing.expectEqual(@as(u64, 0), stats.udp_truncated); + try testing.expectEqual(@as(u64, 0), stats.foreign_datagrams); + try testing.expectEqual(@as(u64, 0), stats.failures); +} + +test "init keeps a tcp resolver on the tcp path" { + var buf = testBuf(); + const fc: ForwardClient = .init( + try validate.parseResolver("tcp://[fd00::1]:5353"), + &buf, + .{ .raw = .fromSeconds(2), .clock = .awake }, + ); + try testing.expectEqual(validate.ResolverScheme.tcp, fc.resolver.scheme); + try testing.expectEqual(@as(u16, 5353), fc.resolver.port); + + const dest = fc.destination(); + try testing.expectEqual(net.IpAddress.Family.ip6, std.meta.activeTag(dest)); + try testing.expectEqual(@as(u16, 5353), dest.getPort()); +} + +test "the destination carries the resolver's address and port" { + var buf = testBuf(); + const fc: ForwardClient = .init( + try validate.parseResolver("udp://192.168.1.1:5300"), + &buf, + .{ .raw = .fromSeconds(1), .clock = .awake }, + ); + const dest = fc.destination(); + try testing.expectEqualSlices(u8, &.{ 192, 168, 1, 1 }, &dest.ip4.bytes); + try testing.expectEqual(@as(u16, 5300), dest.ip4.port); +} + +test "only the resolver's own address and port count as its datagram" { + const dest: net.IpAddress = .{ .ip4 = .{ .bytes = .{ 192, 168, 1, 1 }, .port = 53 } }; + + const same: net.IpAddress = .{ .ip4 = .{ .bytes = .{ 192, 168, 1, 1 }, .port = 53 } }; + try testing.expect(same.eql(&dest)); + + // A different host, the right host on a different port, and the right + // address in the wrong family are each a datagram this client discards. + const other_host: net.IpAddress = .{ .ip4 = .{ .bytes = .{ 192, 168, 1, 2 }, .port = 53 } }; + try testing.expect(!other_host.eql(&dest)); + + const other_port: net.IpAddress = .{ .ip4 = .{ .bytes = .{ 192, 168, 1, 1 }, .port = 5353 } }; + try testing.expect(!other_port.eql(&dest)); + + const mapped: net.IpAddress = .{ .ip6 = .fromIp4(.{ .bytes = .{ 192, 168, 1, 1 }, .port = 53 }) }; + try testing.expect(!mapped.eql(&dest)); +} + +test "the local socket matches the resolver's family and takes an ephemeral port" { + const v4 = wildcardFor(.{ .ip4 = .{ .bytes = .{ 1, 1, 1, 1 }, .port = 53 } }); + try testing.expectEqual(net.IpAddress.Family.ip4, std.meta.activeTag(v4)); + try testing.expectEqual(@as(u16, 0), v4.getPort()); + try testing.expectEqualSlices(u8, &.{ 0, 0, 0, 0 }, &v4.ip4.bytes); + + const v6 = wildcardFor(.{ .ip6 = .unspecified(53) }); + try testing.expectEqual(net.IpAddress.Family.ip6, std.meta.activeTag(v6)); + try testing.expectEqual(@as(u16, 0), v6.getPort()); +} + +test "mapPhase keeps local resource and cancellation errors out of the peer fault group" { + const local = [_]anyerror{ + error.OutOfMemory, + error.SystemResources, + error.ProcessFdQuotaExceeded, + error.SystemFdQuotaExceeded, + error.Unexpected, + }; + for (local) |err| { + try testing.expectEqual( + transport.Group.local_resource, + transport.group(mapPhase(err, error.ReceiveFailed)), + ); + } + + try testing.expectEqual( + transport.ExchangeError.Canceled, + mapPhase(error.Canceled, error.ConnectFailed), + ); + + // A refused connection is the resolver's side, so it stays a peer fault. + try testing.expectEqual( + transport.ExchangeError.ConnectFailed, + mapPhase(error.ConnectionRefused, error.ConnectFailed), + ); +} + +test "a stashed stream error is preferred over the collapsed one" { + var stream_writer: net.Stream.Writer = undefined; + stream_writer.err = error.Canceled; + try testing.expectEqual( + transport.ExchangeError.Canceled, + sendFailure(&stream_writer, error.WriteFailed), + ); + + stream_writer.err = error.ConnectionResetByPeer; + try testing.expectEqual( + transport.ExchangeError.SendFailed, + sendFailure(&stream_writer, error.WriteFailed), + ); + + var stream_reader: net.Stream.Reader = undefined; + stream_reader.err = error.SystemResources; + try testing.expectEqual( + transport.ExchangeError.SystemResources, + receiveFailure(&stream_reader, error.ReadFailed), + ); + + // A peer that closes mid-frame never reaches `err`, so the collapsed error + // is what classifies it. + stream_reader.err = null; + try testing.expectEqual( + transport.ExchangeError.ReceiveFailed, + receiveFailure(&stream_reader, error.EndOfStream), + ); +} diff --git a/src/local/forward_zones.zig b/src/local/forward_zones.zig new file mode 100644 index 0000000..b9e866b --- /dev/null +++ b/src/local/forward_zones.zig @@ -0,0 +1,265 @@ +//! Conditional forward zones (PLAN §6.5): an immutable, longest-suffix-first +//! table built once from the `forward_zones` rows and read on the query path +//! without allocating. Pure: an allocator and plain values, no `std.Io`, no +//! clock. +//! +//! Resolver URLs are parsed by `config/validate.zig`'s `parseResolver`, whose +//! doc comment names this file as its importer. There is no second resolver +//! parser. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const model = @import("../config/model.zig"); +const validate = @import("../config/validate.zig"); +const name = @import("../dns/name.zig"); +const types = @import("../dns/types.zig"); + +pub const Zone = struct { + /// Normalized: lowercase, no trailing dot. + zone: []const u8, + resolver: validate.Resolver, +}; + +pub const Error = error{ OutOfMemory, BadZone, BadResolver, TooManyZones }; +pub const max_zones: usize = 1_000; + +pub const Zones = struct { + /// Sorted by descending label count then by name, so the first match found + /// by a forward scan is the longest one. + items: []const Zone, + /// One block holding every `Zone.zone`; freed as a unit. + names: []const u8, + + pub const empty: Zones = .{ .items = &.{}, .names = &.{} }; + + /// `gpa` owns the result; `deinit` frees it. A row that fails to parse is + /// an error, not a skipped row — `validate.zig` already rejects these, so + /// reaching one here means the database was edited behind nxdns's back and + /// silence would send a zone's queries to the wrong resolver. + pub fn build(gpa: Allocator, rows: []const model.ForwardZone) Error!Zones { + if (rows.len == 0) return .empty; + if (rows.len > max_zones) return error.TooManyZones; + + var names: std.ArrayList(u8) = .empty; + defer names.deinit(gpa); + var spans: std.ArrayList(Span) = .empty; + defer spans.deinit(gpa); + + var buf: [types.max_name_len]u8 = undefined; + for (rows) |row| { + const zone = normalizeName(row.zone, &buf) catch return error.BadZone; + const resolver = validate.parseResolver(row.resolver) catch return error.BadResolver; + try spans.append(gpa, .{ + .offset = names.items.len, + .len = zone.len, + .resolver = resolver, + }); + try names.appendSlice(gpa, zone); + } + + const name_bytes = try names.toOwnedSlice(gpa); + errdefer gpa.free(name_bytes); + + const items = try gpa.alloc(Zone, spans.items.len); + for (items, spans.items) |*item, span| item.* = .{ + .zone = name_bytes[span.offset..][0..span.len], + .resolver = span.resolver, + }; + std.mem.sort(Zone, items, {}, lessThan); + + return .{ .items = items, .names = name_bytes }; + } + + pub fn deinit(self: *Zones, gpa: Allocator) void { + gpa.free(self.items); + gpa.free(self.names); + self.* = .empty; + } + + /// Longest-suffix match on label boundaries: `lan.home` matches + /// `nas.lan.home` and `lan.home`, and does not match `notlan.home`. + /// `domain` must be normalized (lowercase, no trailing dot). + /// Allocation-free. + pub fn match(self: *const Zones, domain: []const u8) ?*const Zone { + for (self.items) |*zone| { + if (suffixMatches(zone.zone, domain)) return zone; + } + return null; + } +}; + +// --------------------------------------------------------------------------- +// Internals +// --------------------------------------------------------------------------- + +/// `Zone.zone` slices are cut only after the name block stops growing, so the +/// build pass records offsets instead of pointers. +const Span = struct { + offset: usize, + len: usize, + resolver: validate.Resolver, +}; + +fn labelCount(zone: []const u8) usize { + return std.mem.count(u8, zone, ".") + 1; +} + +fn lessThan(_: void, a: Zone, b: Zone) bool { + const a_labels = labelCount(a.zone); + const b_labels = labelCount(b.zone); + if (a_labels != b_labels) return a_labels > b_labels; + return std.mem.order(u8, a.zone, b.zone) == .lt; +} + +fn suffixMatches(zone: []const u8, domain: []const u8) bool { + if (domain.len == zone.len) return std.mem.eql(u8, domain, zone); + if (domain.len < zone.len + 1) return false; + const start = domain.len - zone.len; + return domain[start - 1] == '.' and std.mem.eql(u8, domain[start..], zone); +} + +const NameError = error{BadName}; + +/// Lowercases over ASCII, strips one trailing dot, and checks the result is a +/// name `dns.name.fromText` accepts. A byte ≥ 0x80 is rejected because query +/// names arrive ASCII-lowercased, so a high byte could never match. The root +/// zone is rejected too: a zone that forwards everything would bypass the +/// upstream pool entirely, which is not what conditional forwarding means. +fn normalizeName(text: []const u8, buf: *[types.max_name_len]u8) NameError![]const u8 { + var rest = text; + if (rest.len > 0 and rest[rest.len - 1] == '.') rest = rest[0 .. rest.len - 1]; + if (rest.len == 0 or rest.len > types.max_name_len) return error.BadName; + + for (rest, 0..) |byte, i| { + if (byte >= 0x80) return error.BadName; + buf[i] = std.ascii.toLower(byte); + } + const normalized = buf[0..rest.len]; + _ = name.fromText(normalized) catch return error.BadName; + return normalized; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +const lan_resolver = "udp://192.168.1.1:53"; +const home_resolver = "tcp://[fd00::1]:5353"; + +test "a zone matches itself and its subdomains on label boundaries" { + const rows = [_]model.ForwardZone{ + .{ .zone = "lan.home", .resolver = lan_resolver }, + }; + var zones = try Zones.build(testing.allocator, &rows); + defer zones.deinit(testing.allocator); + + for ([_][]const u8{ "lan.home", "nas.lan.home", "a.b.lan.home" }) |domain| { + const found = zones.match(domain) orelse return error.TestExpectedMatch; + try testing.expectEqualStrings("lan.home", found.zone); + } + + for ([_][]const u8{ "notlan.home", "home", "lan.home.evil.net", "" }) |domain| { + try testing.expect(zones.match(domain) == null); + } +} + +test "the longest configured zone wins" { + const rows = [_]model.ForwardZone{ + .{ .zone = "home", .resolver = home_resolver }, + .{ .zone = "lan.home", .resolver = lan_resolver }, + }; + var zones = try Zones.build(testing.allocator, &rows); + defer zones.deinit(testing.allocator); + + const nas = zones.match("nas.lan.home") orelse return error.TestExpectedMatch; + try testing.expectEqualStrings("lan.home", nas.zone); + try testing.expectEqual(validate.ResolverScheme.udp, nas.resolver.scheme); + + const printer = zones.match("printer.home") orelse return error.TestExpectedMatch; + try testing.expectEqualStrings("home", printer.zone); + try testing.expectEqual(validate.ResolverScheme.tcp, printer.resolver.scheme); + try testing.expectEqual(@as(u16, 5353), printer.resolver.port); +} + +test "a reverse zone matches every name under it" { + const rows = [_]model.ForwardZone{ + .{ .zone = "10.in-addr.arpa", .resolver = lan_resolver }, + }; + var zones = try Zones.build(testing.allocator, &rows); + defer zones.deinit(testing.allocator); + + const found = zones.match("5.4.3.10.in-addr.arpa") orelse return error.TestExpectedMatch; + try testing.expectEqualStrings("10.in-addr.arpa", found.zone); + try testing.expect(zones.match("5.4.3.11.in-addr.arpa") == null); +} + +test "uppercase and trailing-dot zones normalize to one key" { + const rows = [_]model.ForwardZone{ + .{ .zone = "LAN.Home.", .resolver = lan_resolver }, + }; + var zones = try Zones.build(testing.allocator, &rows); + defer zones.deinit(testing.allocator); + + try testing.expectEqualStrings("lan.home", zones.items[0].zone); + try testing.expect(zones.match("nas.lan.home") != null); +} + +test "the resolver comes from parseResolver" { + const rows = [_]model.ForwardZone{ + .{ .zone = "lan.home", .resolver = lan_resolver }, + }; + var zones = try Zones.build(testing.allocator, &rows); + defer zones.deinit(testing.allocator); + + const expected = try validate.parseResolver(lan_resolver); + const found = zones.match("lan.home") orelse return error.TestExpectedMatch; + try testing.expectEqual(expected.scheme, found.resolver.scheme); + try testing.expectEqual(expected.port, found.resolver.port); + try testing.expect(expected.addr.eql(found.resolver.addr)); +} + +test "a bad resolver URL is an error" { + for ([_][]const u8{ "https://dns.example/dns-query", "udp://192.168.1.1", "udp://nas.lan:53" }) |url| { + const rows = [_]model.ForwardZone{.{ .zone = "lan.home", .resolver = url }}; + try testing.expectError(error.BadResolver, Zones.build(testing.allocator, &rows)); + } +} + +test "a bad zone is an error" { + for ([_][]const u8{ "lan..home", ".", "" }) |zone| { + const rows = [_]model.ForwardZone{.{ .zone = zone, .resolver = lan_resolver }}; + try testing.expectError(error.BadZone, Zones.build(testing.allocator, &rows)); + } +} + +test "too many rows is an error" { + const rows = try testing.allocator.alloc(model.ForwardZone, max_zones + 1); + defer testing.allocator.free(rows); + for (rows) |*row| row.* = .{ .zone = "lan.home", .resolver = lan_resolver }; + try testing.expectError(error.TooManyZones, Zones.build(testing.allocator, rows)); +} + +test "an empty row set builds the empty table" { + var zones = try Zones.build(testing.allocator, &[_]model.ForwardZone{}); + defer zones.deinit(testing.allocator); + + try testing.expectEqual(@as(usize, 0), zones.items.len); + try testing.expect(zones.match("lan.home") == null); +} + +fn buildUnderFailure(gpa: Allocator) !void { + const rows = [_]model.ForwardZone{ + .{ .zone = "home", .resolver = home_resolver }, + .{ .zone = "lan.home", .resolver = lan_resolver }, + }; + var zones = try Zones.build(gpa, &rows); + defer zones.deinit(gpa); + try testing.expectEqualStrings("lan.home", zones.items[0].zone); +} + +test "build leaks nothing under allocation failure" { + try testing.checkAllAllocationFailures(testing.allocator, buildUnderFailure, .{}); +} diff --git a/src/local/records.zig b/src/local/records.zig new file mode 100644 index 0000000..d5cacaf --- /dev/null +++ b/src/local/records.zig @@ -0,0 +1,459 @@ +//! Local DNS records (PLAN §6.4): an immutable, sorted lookup table built once +//! from the `local_records` rows and read on the query path without allocating. +//! Pure: an allocator and plain values, no `std.Io`, no clock. +//! +//! Local records are group-independent and are matched before filtering, so a +//! name that has a record here never reaches the blocklists. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const model = @import("../config/model.zig"); +const header = @import("../dns/header.zig"); +const name = @import("../dns/name.zig"); +const packet = @import("../dns/packet.zig"); +const question = @import("../dns/question.zig"); +const types = @import("../dns/types.zig"); +const address = @import("../platform/address.zig"); + +pub const Value = union(enum) { a: [4]u8, aaaa: [16]u8, cname: name.Name }; + +pub const Record = struct { + /// Normalized owner name: lowercase, no trailing dot. + owner: []const u8, + value: Value, + ttl: u32, +}; + +pub const Error = error{ OutOfMemory, BadRecordValue, BadRecordName, TooManyRecords }; +pub const max_records: usize = 10_000; + +pub const Records = struct { + /// Sorted by (owner, rtype) so lookup is a binary search and the answer + /// order for one name is stable across restarts. + items: []const Record, + /// One block holding every `Record.owner`; freed as a unit. + owners: []const u8, + + pub const empty: Records = .{ .items = &.{}, .owners = &.{} }; + + /// `gpa` owns the result; `deinit` frees it. Values are parsed here, once. + /// A bad value is an error, not a skipped row — `validate.zig` already + /// rejects these, so reaching one here means the database was edited behind + /// nxdns's back and silence would make a record vanish with no signal. + pub fn build(gpa: Allocator, rows: []const model.LocalRecord) Error!Records { + if (rows.len == 0) return .empty; + if (rows.len > max_records) return error.TooManyRecords; + + var owners: std.ArrayList(u8) = .empty; + defer owners.deinit(gpa); + var spans: std.ArrayList(Span) = .empty; + defer spans.deinit(gpa); + + var buf: [types.max_name_len]u8 = undefined; + for (rows) |row| { + const owner = normalizeName(row.name, &buf) catch return error.BadRecordName; + const value = try parseValue(row.rtype, row.value); + try spans.append(gpa, .{ + .offset = owners.items.len, + .len = owner.len, + .value = value, + .ttl = row.ttl, + }); + try owners.appendSlice(gpa, owner); + } + + const owner_bytes = try owners.toOwnedSlice(gpa); + errdefer gpa.free(owner_bytes); + + const items = try gpa.alloc(Record, spans.items.len); + for (items, spans.items) |*item, span| item.* = .{ + .owner = owner_bytes[span.offset..][0..span.len], + .value = span.value, + .ttl = span.ttl, + }; + std.mem.sort(Record, items, {}, lessThan); + + return .{ .items = items, .owners = owner_bytes }; + } + + pub fn deinit(self: *Records, gpa: Allocator) void { + gpa.free(self.items); + gpa.free(self.owners); + self.* = .empty; + } + + /// All records for `domain` whose type matches `qtype`. `domain` must be + /// normalized (lowercase, no trailing dot). An empty slice means the name + /// has no local record of that type. Allocation-free. + /// + /// A CNAME answers every qtype and excludes every other type at the same + /// name (RFC 1034 §3.6.2), so a name carrying one answers with the CNAME + /// alone whatever else the row set holds. + pub fn lookup(self: *const Records, domain: []const u8, qtype: types.Type) []const Record { + const at_name = self.ownerRange(domain); + if (at_name.len == 0) return at_name; + + const cnames = rankRun(at_name, rank_cname); + if (cnames.len != 0) return cnames; + + return switch (qtype) { + .a => rankRun(at_name, rank_a), + .aaaa => rankRun(at_name, rank_aaaa), + .any => at_name, + else => at_name[0..0], + }; + } + + /// True when the name has any local record of any type. The handler needs + /// this to answer NODATA instead of forwarding a name nxdns owns. + pub fn hasName(self: *const Records, domain: []const u8) bool { + return self.ownerRange(domain).len != 0; + } + + fn ownerRange(self: *const Records, domain: []const u8) []const Record { + var low: usize = 0; + var high: usize = self.items.len; + while (low < high) { + const mid = low + (high - low) / 2; + if (std.mem.order(u8, self.items[mid].owner, domain) == .lt) { + low = mid + 1; + } else { + high = mid; + } + } + var end = low; + while (end < self.items.len and std.mem.eql(u8, self.items[end].owner, domain)) end += 1; + return self.items[low..end]; + } +}; + +/// Writes `records` as answers into a builder the caller has already +/// initialized with the request header and question. Mechanism only. +pub fn writeAnswers( + b: *packet.ResponseBuilder, + owner: name.Name, + records: []const Record, +) packet.ResponseBuilder.Error!void { + for (records) |rec| { + switch (rec.value) { + .a => |bytes| try b.addAnswer(owner, .a, .in, rec.ttl, &bytes), + .aaaa => |bytes| try b.addAnswer(owner, .aaaa, .in, rec.ttl, &bytes), + .cname => |target| try b.addAnswer(owner, .cname, .in, rec.ttl, target.wire()), + } + } +} + +// --------------------------------------------------------------------------- +// Internals +// --------------------------------------------------------------------------- + +/// `Record.owner` slices are cut only after the owner block stops growing, so +/// the build pass records offsets instead of pointers. +const Span = struct { + offset: usize, + len: usize, + value: Value, + ttl: u32, +}; + +const rank_a: u2 = 0; +const rank_aaaa: u2 = 1; +const rank_cname: u2 = 2; + +fn rank(value: Value) u2 { + return switch (value) { + .a => rank_a, + .aaaa => rank_aaaa, + .cname => rank_cname, + }; +} + +fn lessThan(_: void, a: Record, b: Record) bool { + return switch (std.mem.order(u8, a.owner, b.owner)) { + .lt => true, + .gt => false, + .eq => rank(a.value) < rank(b.value), + }; +} + +/// The run of one rank inside a single name's records, which the (owner, rtype) +/// sort makes contiguous. +fn rankRun(records: []const Record, wanted: u2) []const Record { + var start: usize = 0; + while (start < records.len and rank(records[start].value) < wanted) start += 1; + var end = start; + while (end < records.len and rank(records[end].value) == wanted) end += 1; + return records[start..end]; +} + +const NameError = error{BadName}; + +/// Lowercases over ASCII, strips one trailing dot, and checks the result is a +/// name `dns.name.fromText` accepts. A byte ≥ 0x80 is rejected: query names +/// arrive ASCII-lowercased, so a high byte here could never be matched and a +/// record that can never answer is a configuration error worth reporting. The +/// root name is rejected for the same reason — nothing can match it. +fn normalizeName(text: []const u8, buf: *[types.max_name_len]u8) NameError![]const u8 { + var rest = text; + if (rest.len > 0 and rest[rest.len - 1] == '.') rest = rest[0 .. rest.len - 1]; + if (rest.len == 0 or rest.len > types.max_name_len) return error.BadName; + + for (rest, 0..) |byte, i| { + if (byte >= 0x80) return error.BadName; + buf[i] = std.ascii.toLower(byte); + } + const normalized = buf[0..rest.len]; + _ = name.fromText(normalized) catch return error.BadName; + return normalized; +} + +fn parseValue(rtype: model.RecordType, text: []const u8) error{BadRecordValue}!Value { + switch (rtype) { + .a => { + const addr = address.NetAddress.parse(text) catch return error.BadRecordValue; + return switch (addr) { + .ip4 => |bytes| Value{ .a = bytes }, + .ip6 => return error.BadRecordValue, + }; + }, + .aaaa => { + const addr = address.NetAddress.parse(text) catch return error.BadRecordValue; + return switch (addr) { + .ip4 => return error.BadRecordValue, + .ip6 => |bytes| Value{ .aaaa = bytes }, + }; + }, + .cname => { + var buf: [types.max_name_len]u8 = undefined; + const target = normalizeName(text, &buf) catch return error.BadRecordValue; + return .{ .cname = name.fromText(target) catch return error.BadRecordValue }; + }, + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +const sample_rows = [_]model.LocalRecord{ + .{ .name = "nas.lan", .rtype = .a, .value = "192.168.1.5", .ttl = 120 }, + .{ .name = "nas.lan", .rtype = .aaaa, .value = "fd00::1", .ttl = 240 }, + .{ .name = "printer.lan", .rtype = .a, .value = "192.168.1.6", .ttl = 60 }, +}; + +test "lookup returns the records of the queried type" { + var records = try Records.build(testing.allocator, &sample_rows); + defer records.deinit(testing.allocator); + + const a = records.lookup("nas.lan", .a); + try testing.expectEqual(@as(usize, 1), a.len); + try testing.expectEqualSlices(u8, &.{ 192, 168, 1, 5 }, &a[0].value.a); + try testing.expectEqual(@as(u32, 120), a[0].ttl); + + const aaaa = records.lookup("nas.lan", .aaaa); + try testing.expectEqual(@as(usize, 1), aaaa.len); + try testing.expectEqual(@as(u32, 240), aaaa[0].ttl); + + try testing.expectEqual(@as(usize, 0), records.lookup("nas.lan", .mx).len); + try testing.expectEqual(@as(usize, 0), records.lookup("other.lan", .a).len); +} + +test "lookup returns the CNAME for every qtype" { + const rows = [_]model.LocalRecord{ + .{ .name = "www.lan", .rtype = .cname, .value = "nas.lan", .ttl = 300 }, + }; + var records = try Records.build(testing.allocator, &rows); + defer records.deinit(testing.allocator); + + for ([_]types.Type{ .a, .aaaa, .mx, .https, .any }) |qtype| { + const found = records.lookup("www.lan", qtype); + try testing.expectEqual(@as(usize, 1), found.len); + try testing.expectEqualSlices( + u8, + (try name.fromText("nas.lan")).wire(), + found[0].value.cname.wire(), + ); + } +} + +test "hasName covers every type at the name" { + var records = try Records.build(testing.allocator, &sample_rows); + defer records.deinit(testing.allocator); + + try testing.expect(records.hasName("nas.lan")); + try testing.expect(records.hasName("printer.lan")); + try testing.expect(!records.hasName("lan")); + try testing.expect(!records.hasName("nas.lan.evil.net")); +} + +test "two A records for one name come back in a stable order" { + const rows = [_]model.LocalRecord{ + .{ .name = "nas.lan", .rtype = .a, .value = "192.168.1.5" }, + .{ .name = "nas.lan", .rtype = .a, .value = "192.168.1.6" }, + }; + + var first = try Records.build(testing.allocator, &rows); + defer first.deinit(testing.allocator); + var second = try Records.build(testing.allocator, &rows); + defer second.deinit(testing.allocator); + + const a = first.lookup("nas.lan", .a); + const b = second.lookup("nas.lan", .a); + try testing.expectEqual(@as(usize, 2), a.len); + try testing.expectEqual(@as(usize, 2), b.len); + try testing.expectEqualSlices(u8, &.{ 192, 168, 1, 5 }, &a[0].value.a); + try testing.expectEqualSlices(u8, &.{ 192, 168, 1, 6 }, &a[1].value.a); + for (a, b) |lhs, rhs| try testing.expectEqualSlices(u8, &lhs.value.a, &rhs.value.a); +} + +test "uppercase and trailing-dot owners normalize to one key" { + const rows = [_]model.LocalRecord{ + .{ .name = "NAS.Lan.", .rtype = .a, .value = "192.168.1.5" }, + }; + var records = try Records.build(testing.allocator, &rows); + defer records.deinit(testing.allocator); + + try testing.expectEqualStrings("nas.lan", records.items[0].owner); + try testing.expectEqual(@as(usize, 1), records.lookup("nas.lan", .a).len); +} + +test "a bad A value is an error" { + const rows = [_]model.LocalRecord{ + .{ .name = "nas.lan", .rtype = .a, .value = "::1" }, + }; + try testing.expectError(error.BadRecordValue, Records.build(testing.allocator, &rows)); +} + +test "a bad AAAA value is an error" { + const rows = [_]model.LocalRecord{ + .{ .name = "nas.lan", .rtype = .aaaa, .value = "192.168.1.5" }, + }; + try testing.expectError(error.BadRecordValue, Records.build(testing.allocator, &rows)); +} + +test "a bad CNAME target is an error" { + const rows = [_]model.LocalRecord{ + .{ .name = "www.lan", .rtype = .cname, .value = "nas..lan" }, + }; + try testing.expectError(error.BadRecordValue, Records.build(testing.allocator, &rows)); +} + +test "an unparseable owner is an error" { + const rows = [_]model.LocalRecord{ + .{ .name = "nas..lan", .rtype = .a, .value = "192.168.1.5" }, + }; + try testing.expectError(error.BadRecordName, Records.build(testing.allocator, &rows)); + + const root = [_]model.LocalRecord{ + .{ .name = ".", .rtype = .a, .value = "192.168.1.5" }, + }; + try testing.expectError(error.BadRecordName, Records.build(testing.allocator, &root)); +} + +test "too many rows is an error" { + const rows = try testing.allocator.alloc(model.LocalRecord, max_records + 1); + defer testing.allocator.free(rows); + for (rows) |*row| row.* = .{ .name = "nas.lan", .rtype = .a, .value = "192.168.1.5" }; + try testing.expectError(error.TooManyRecords, Records.build(testing.allocator, rows)); +} + +test "an empty row set builds the empty table" { + var records = try Records.build(testing.allocator, &[_]model.LocalRecord{}); + defer records.deinit(testing.allocator); + + try testing.expectEqual(@as(usize, 0), records.items.len); + try testing.expect(!records.hasName("nas.lan")); + try testing.expectEqual(@as(usize, 0), records.lookup("nas.lan", .a).len); +} + +fn requestHeader() header.Header { + return .{ + .id = 0x4242, + .flags = .{ + .rcode = .no_error, + .z = 0, + .ra = false, + .rd = true, + .tc = false, + .aa = false, + .opcode = .query, + .qr = false, + }, + .qdcount = 1, + .ancount = 0, + .nscount = 0, + .arcount = 0, + }; +} + +test "writeAnswers emits records the parser reads back" { + var records = try Records.build(testing.allocator, &sample_rows); + defer records.deinit(testing.allocator); + + const owner = try name.fromText("nas.lan"); + const q: question.Question = .{ .name = owner, .qtype = .any, .qclass = .in }; + + var buf: [512]u8 = undefined; + var builder = try packet.ResponseBuilder.init(&buf, requestHeader(), q); + try writeAnswers(&builder, owner, records.lookup("nas.lan", .any)); + const message = builder.finish(); + + const parsed = try packet.parse(message); + try testing.expectEqual(@as(u16, 2), parsed.header.ancount); + + var it = packet.answers(parsed); + const first = (try it.next()).?; + try testing.expectEqual(types.Type.a, first.rtype); + try testing.expectEqual(@as(u16, @intFromEnum(types.Class.in)), first.class); + try testing.expectEqual(@as(u32, 120), first.ttl); + try testing.expectEqualSlices(u8, &.{ 192, 168, 1, 5 }, first.rdata.slice(parsed.bytes)); + + const second = (try it.next()).?; + try testing.expectEqual(types.Type.aaaa, second.rtype); + try testing.expectEqual(@as(u32, 240), second.ttl); + try testing.expectEqual(@as(usize, 16), second.rdata.len); + + try testing.expect((try it.next()) == null); +} + +test "writeAnswers emits a CNAME in wire form" { + const rows = [_]model.LocalRecord{ + .{ .name = "www.lan", .rtype = .cname, .value = "nas.lan", .ttl = 300 }, + }; + var records = try Records.build(testing.allocator, &rows); + defer records.deinit(testing.allocator); + + const owner = try name.fromText("www.lan"); + const q: question.Question = .{ .name = owner, .qtype = .a, .qclass = .in }; + + var buf: [512]u8 = undefined; + var builder = try packet.ResponseBuilder.init(&buf, requestHeader(), q); + try writeAnswers(&builder, owner, records.lookup("www.lan", .a)); + const message = builder.finish(); + + const parsed = try packet.parse(message); + try testing.expectEqual(@as(u16, 1), parsed.header.ancount); + + var it = packet.answers(parsed); + const answer = (try it.next()).?; + try testing.expectEqual(types.Type.cname, answer.rtype); + try testing.expectEqual(@as(u32, 300), answer.ttl); + try testing.expectEqualSlices( + u8, + (try name.fromText("nas.lan")).wire(), + answer.rdata.slice(parsed.bytes), + ); +} + +fn buildUnderFailure(gpa: Allocator) !void { + var records = try Records.build(gpa, &sample_rows); + defer records.deinit(gpa); + try testing.expectEqual(@as(usize, 3), records.items.len); +} + +test "build leaks nothing under allocation failure" { + try testing.checkAllAllocationFailures(testing.allocator, buildUnderFailure, .{}); +} diff --git a/src/server/resolver_integration_test.zig b/src/server/resolver_integration_test.zig index 4dbadda..316f556 100644 --- a/src/server/resolver_integration_test.zig +++ b/src/server/resolver_integration_test.zig @@ -199,11 +199,11 @@ fn tcpQuery(io: std.Io, address: net.IpAddress, id: u16) anyerror!void { var query_buf: [query_bytes.len]u8 = undefined; const query = queryWithId(&query_buf, id); - try writer.interface.writeAll(&tcp_server.framePrefix(@intCast(query.len))); + try writer.interface.writeAll(&transport.framePrefix(@intCast(query.len))); try writer.interface.writeAll(query); try writer.interface.flush(); - const len = tcp_server.parsePrefix((try reader.interface.takeArray(tcp_server.prefix_len)).*); + const len = transport.parsePrefix((try reader.interface.takeArray(transport.prefix_len)).*); try expectAnswer(try reader.interface.take(len), id); } diff --git a/src/server/tcp_server.zig b/src/server/tcp_server.zig index be57e95..73f0a1b 100644 --- a/src/server/tcp_server.zig +++ b/src/server/tcp_server.zig @@ -19,9 +19,6 @@ const transport = @import("../upstream/transport.zig"); const log = std.log.scoped(.tcp_server); -/// RFC 1035 §4.2.2: the message length prefix is two bytes, big-endian. -pub const prefix_len = 2; - /// The stream buffers only stage the framing bytes. A message longer than this /// is read straight into `Conn.query` and written straight from `Conn.reply`, /// so making them larger would buy nothing. @@ -220,7 +217,7 @@ pub const TcpServer = struct { const budget = self.options.idle_timeout; while (true) { - var prefix: [prefix_len]u8 = undefined; + var prefix: [transport.prefix_len]u8 = undefined; var got: usize = 0; switch (race(io, budget, readPrefix, .{ &reader.interface, &prefix, &got })) { .ok => {}, @@ -238,14 +235,14 @@ pub const TcpServer = struct { // A client that closes between messages has finished asking, which // is the normal end of a connection, not a failure. if (got == 0) return; - if (got != prefix_len) { + if (got != transport.prefix_len) { bump(&self.stats.connection_errors); return; } // RFC 1035 §4.2.2 gives no meaning to a zero-length message, and // the prefix is a u16 so it can never exceed `max_message_len`. - const len = parsePrefix(prefix); + const len = transport.parsePrefix(prefix); if (len == 0) { bump(&self.stats.connection_errors); return; @@ -267,7 +264,7 @@ pub const TcpServer = struct { .reply => |b| b, }; - const out = framePrefix(@intCast(bytes.len)); + const out = transport.framePrefix(@intCast(bytes.len)); switch (race(io, budget, writeReply, .{ &writer.interface, &out, bytes })) { .ok => {}, .canceled => return, @@ -333,17 +330,6 @@ pub const TcpServer = struct { } }; -/// RFC 1035 §4.2.2: the message length as a 2-byte big-endian integer. -pub fn framePrefix(len: u16) [prefix_len]u8 { - var out: [prefix_len]u8 = undefined; - std.mem.writeInt(u16, &out, len, .big); - return out; -} - -pub fn parsePrefix(bytes: [prefix_len]u8) u16 { - return std.mem.readInt(u16, &bytes, .big); -} - /// The capacity rule, without the mutex, so it is testable without a backend. fn firstFree(conns: []const TcpServer.Conn) ?usize { for (conns, 0..) |*conn, index| { @@ -402,7 +388,7 @@ fn expire(io: std.Io, budget: std.Io.Clock.Duration) std.Io.Cancelable!void { /// `readSliceShort` rather than `readSliceAll`: a zero-length read is a client /// that closed cleanly between messages, and only a partial prefix is an error. -fn readPrefix(reader: *std.Io.Reader, buf: *[prefix_len]u8, out_len: *usize) anyerror!void { +fn readPrefix(reader: *std.Io.Reader, buf: *[transport.prefix_len]u8, out_len: *usize) anyerror!void { out_len.* = try reader.readSliceShort(buf); } @@ -410,7 +396,7 @@ fn readBody(reader: *std.Io.Reader, buf: []u8) anyerror!void { return reader.readSliceAll(buf); } -fn writeReply(writer: *std.Io.Writer, prefix: *const [prefix_len]u8, bytes: []const u8) anyerror!void { +fn writeReply(writer: *std.Io.Writer, prefix: *const [transport.prefix_len]u8, bytes: []const u8) anyerror!void { try writer.writeAll(prefix); try writer.writeAll(bytes); try writer.flush(); @@ -422,20 +408,6 @@ fn bump(counter: *std.atomic.Value(u64)) void { const testing = std.testing; -test "the length prefix is big-endian and round-trips" { - try testing.expectEqualSlices(u8, &.{ 0x00, 0x00 }, &framePrefix(0)); - try testing.expectEqualSlices(u8, &.{ 0x01, 0x00 }, &framePrefix(256)); - try testing.expectEqualSlices(u8, &.{ 0xff, 0xff }, &framePrefix(65535)); - - for ([_]u16{ 0, 1, 12, 512, 4096, 65534, 65535 }) |len| { - try testing.expectEqual(len, parsePrefix(framePrefix(len))); - } -} - -test "the prefix ceiling is the message ceiling" { - try testing.expectEqual(@as(u16, transport.max_message_len), parsePrefix(.{ 0xff, 0xff })); -} - fn testConns(count: usize) ![]TcpServer.Conn { const conns = try testing.allocator.alloc(TcpServer.Conn, count); for (conns) |*conn| conn.state = .free; diff --git a/src/server/tcp_server_integration_test.zig b/src/server/tcp_server_integration_test.zig index edef8ee..aad8784 100644 --- a/src/server/tcp_server_integration_test.zig +++ b/src/server/tcp_server_integration_test.zig @@ -117,11 +117,11 @@ fn twoQueriesOnOneConnection(io: std.Io, address: net.IpAddress) anyerror!void { var writer = stream.writer(io, &write_buf); for (0..2) |_| { - try writer.interface.writeAll(&tcp_server.framePrefix(@intCast(query_bytes.len))); + try writer.interface.writeAll(&transport.framePrefix(@intCast(query_bytes.len))); try writer.interface.writeAll(query_bytes); try writer.interface.flush(); - const len = tcp_server.parsePrefix((try reader.interface.takeArray(tcp_server.prefix_len)).*); + const len = transport.parsePrefix((try reader.interface.takeArray(transport.prefix_len)).*); try expectAnswersQuery(try reader.interface.take(len)); } } @@ -246,7 +246,7 @@ fn sendZeroLength(io: std.Io, address: net.IpAddress) anyerror!void { var write_buf: [64]u8 = undefined; var writer = stream.writer(io, &write_buf); - try writer.interface.writeAll(&tcp_server.framePrefix(0)); + try writer.interface.writeAll(&transport.framePrefix(0)); try writer.interface.flush(); var read_buf: [64]u8 = undefined; diff --git a/src/storage/repositories/groups_repo.zig b/src/storage/repositories/groups_repo.zig index 194080c..782f5ca 100644 --- a/src/storage/repositories/groups_repo.zig +++ b/src/storage/repositories/groups_repo.zig @@ -65,6 +65,17 @@ pub fn countGroups(database: *db.Db) db.Error!i64 { return database.queryInt("SELECT count(*) FROM groups"); } +/// The row id of one group by name, or null. `listGroups` returns model values +/// without row ids by design (ids are not stable across an import); the filter +/// snapshot needs them to map a decision back to a row. +pub fn groupId(database: *db.Db, group_name: []const u8) db.Error!?i64 { + var stmt = try database.prepare("SELECT id FROM groups WHERE name = ?1"); + defer stmt.deinit(); + try stmt.bindText(1, group_name); + if (!try stmt.step()) return null; + return stmt.columnInt(0); +} + // --------------------------------------------------------------------------- // group_sources // --------------------------------------------------------------------------- diff --git a/src/storage/repositories/sources_repo.zig b/src/storage/repositories/sources_repo.zig index cff4867..b39dea5 100644 --- a/src/storage/repositories/sources_repo.zig +++ b/src/storage/repositories/sources_repo.zig @@ -77,6 +77,106 @@ pub fn countBlocklistSources(database: *db.Db) db.Error!i64 { return database.queryInt("SELECT count(*) FROM blocklist_sources"); } +// --------------------------------------------------------------------------- +// runtime columns (milestone 5 S8.1) +// --------------------------------------------------------------------------- +// +// The blocklist manager needs the row id (the compiled files are named after +// it) and the counters the refresh writes. Neither belongs in `model`: an +// export carries configuration, and these are facts a running server produces. +// Both functions below are additive; no export path reads them. + +pub const SourceRow = struct { + id: i64, + url: []const u8, + name: []const u8, + enabled: bool, + last_updated: ?i64, + domain_count: i64, + wildcard_count: i64, + skipped_regex_count: i64, + checksum: ?[]const u8, +}; + +pub const SourceStats = struct { + last_updated: i64, + domain_count: i64, + wildcard_count: i64, + skipped_regex_count: i64, + /// Lowercase hex sha256 over the `.list` body followed by the `.wild` body. + checksum: []const u8, +}; + +const list_rows_sql = + \\SELECT id, url, name, enabled, last_updated, + \\ domain_count, wildcard_count, skipped_regex_count, checksum + \\ FROM blocklist_sources ORDER BY url +; + +/// Every source with its row id and its runtime columns, in the same `url` +/// order `listBlocklistSources` uses. Every string is a heap copy owned by +/// `gpa`; free the whole list with `freeSourceRows` and then `deinit` the list. +pub fn listSourceRows(database: *db.Db, gpa: Allocator) db.Error!std.ArrayList(SourceRow) { + var stmt = try database.prepare(list_rows_sql); + defer stmt.deinit(); + + var out: std.ArrayList(SourceRow) = .empty; + // `errdefer`s run in reverse: the free pass is declared last so it runs + // before the backing array is released. + errdefer out.deinit(gpa); + errdefer freeSourceRows(gpa, out.items); + + while (try stmt.step()) { + const url = try stmt.columnTextAlloc(gpa, 1); + errdefer gpa.free(url); + const name = try stmt.columnTextAlloc(gpa, 2); + errdefer gpa.free(name); + const checksum = try stmt.columnTextAllocOrNull(gpa, 8); + errdefer if (checksum) |value| gpa.free(value); + try out.append(gpa, .{ + .id = stmt.columnInt(0), + .url = url, + .name = name, + .enabled = stmt.columnBool(3), + .last_updated = if (stmt.isNull(4)) null else stmt.columnInt(4), + .domain_count = stmt.columnInt(5), + .wildcard_count = stmt.columnInt(6), + .skipped_regex_count = stmt.columnInt(7), + .checksum = checksum, + }); + } + return out; +} + +pub fn freeSourceRows(gpa: Allocator, items: []const SourceRow) void { + for (items) |item| { + gpa.free(item.url); + gpa.free(item.name); + if (item.checksum) |value| gpa.free(value); + } +} + +const update_stats_sql = + \\UPDATE blocklist_sources + \\ SET last_updated = ?2, domain_count = ?3, wildcard_count = ?4, + \\ skipped_regex_count = ?5, checksum = ?6 + \\ WHERE id = ?1 +; + +/// Writes the runtime columns for one source after a compile. The configuration +/// columns (`url`, `name`, `enabled`, `is_suggested`) are untouched. +pub fn updateSourceStats(database: *db.Db, id: i64, stats: SourceStats) db.Error!void { + var stmt = try database.prepare(update_stats_sql); + defer stmt.deinit(); + try stmt.bindInt(1, id); + try stmt.bindInt(2, stats.last_updated); + try stmt.bindInt(3, stats.domain_count); + try stmt.bindInt(4, stats.wildcard_count); + try stmt.bindInt(5, stats.skipped_regex_count); + try stmt.bindText(6, stats.checksum); + try stmt.exec(); +} + // --------------------------------------------------------------------------- // tests // --------------------------------------------------------------------------- @@ -175,3 +275,89 @@ fn listBlocklistSourcesUnderFailure(gpa: Allocator) !void { test "listBlocklistSources is leak-safe under allocation failure" { try testing.checkAllAllocationFailures(testing.allocator, listBlocklistSourcesUnderFailure, .{}); } + +test "listSourceRows returns row ids and the runtime columns in url order" { + var database = try openMigrated(); + defer database.close(); + try seedSources(&database); + + var rows = try listSourceRows(&database, testing.allocator); + defer rows.deinit(testing.allocator); + defer freeSourceRows(testing.allocator, rows.items); + + try testing.expectEqual(@as(usize, 3), rows.items.len); + try testing.expectEqualStrings("https://a.example/list.txt", rows.items[0].url); + try testing.expectEqualStrings("https://b.example/list.txt", rows.items[1].url); + try testing.expectEqualStrings("https://c.example/list.txt", rows.items[2].url); + try testing.expectEqualStrings("A list", rows.items[0].name); + try testing.expect(!rows.items[0].enabled); + try testing.expect(rows.items[1].enabled); + + for (rows.items) |row| { + try testing.expect(row.id > 0); + try testing.expectEqual(@as(?i64, null), row.last_updated); + try testing.expectEqual(@as(?[]const u8, null), row.checksum); + try testing.expectEqual(@as(i64, 0), row.domain_count); + try testing.expectEqual(@as(i64, 0), row.wildcard_count); + try testing.expectEqual(@as(i64, 0), row.skipped_regex_count); + } +} + +test "updateSourceStats writes the runtime columns of one source only" { + var database = try openMigrated(); + defer database.close(); + try seedSources(&database); + + var before = try listSourceRows(&database, testing.allocator); + defer before.deinit(testing.allocator); + defer freeSourceRows(testing.allocator, before.items); + + const target = before.items[1]; + try updateSourceStats(&database, target.id, .{ + .last_updated = 1_700_000_000, + .domain_count = 4321, + .wildcard_count = 21, + .skipped_regex_count = 7, + .checksum = "a" ** 64, + }); + + var after = try listSourceRows(&database, testing.allocator); + defer after.deinit(testing.allocator); + defer freeSourceRows(testing.allocator, after.items); + + const updated = after.items[1]; + try testing.expectEqual(target.id, updated.id); + try testing.expectEqualStrings("https://b.example/list.txt", updated.url); + try testing.expectEqualStrings("B list", updated.name); + try testing.expect(updated.enabled); + try testing.expectEqual(@as(?i64, 1_700_000_000), updated.last_updated); + try testing.expectEqual(@as(i64, 4321), updated.domain_count); + try testing.expectEqual(@as(i64, 21), updated.wildcard_count); + try testing.expectEqual(@as(i64, 7), updated.skipped_regex_count); + try testing.expectEqualStrings("a" ** 64, updated.checksum.?); + + // The two untouched rows kept their defaults. + try testing.expectEqual(@as(?i64, null), after.items[0].last_updated); + try testing.expectEqual(@as(?[]const u8, null), after.items[2].checksum); +} + +fn listSourceRowsUnderFailure(gpa: Allocator) !void { + var database = try openMigrated(); + defer database.close(); + try seedSources(&database); + try updateSourceStats(&database, 1, .{ + .last_updated = 1, + .domain_count = 2, + .wildcard_count = 3, + .skipped_regex_count = 4, + .checksum = "b" ** 64, + }); + + var rows = try listSourceRows(&database, gpa); + defer rows.deinit(gpa); + defer freeSourceRows(gpa, rows.items); +} + +test "listSourceRows is leak-safe under allocation failure" { + try testing.checkAllAllocationFailures(testing.allocator, listSourceRowsUnderFailure, .{}); +} diff --git a/src/tests.zig b/src/tests.zig index 9de20ea..8c84b93 100644 --- a/src/tests.zig +++ b/src/tests.zig @@ -46,6 +46,23 @@ comptime { _ = @import("config/bootstrap.zig"); _ = @import("cli.zig"); _ = @import("storage/storage_integration_test.zig"); + _ = @import("filter/parsers.zig"); + _ = @import("filter/parser_hosts.zig"); + _ = @import("filter/parser_domains.zig"); + _ = @import("filter/parser_abp.zig"); + _ = @import("filter/wildcard.zig"); + _ = @import("filter/domain_set.zig"); + _ = @import("filter/rules.zig"); + _ = @import("filter/matcher.zig"); + _ = @import("filter/compiler.zig"); + _ = @import("filter/safesearch.zig"); + _ = @import("filter/response.zig"); + _ = @import("filter/fetcher.zig"); + _ = @import("filter/manager.zig"); + _ = @import("filter/filter_integration_test.zig"); + _ = @import("local/records.zig"); + _ = @import("local/forward_zones.zig"); + _ = @import("local/forward_client.zig"); } extern fn sqlite3_libversion() [*:0]const u8; diff --git a/src/upstream/dot_client.zig b/src/upstream/dot_client.zig index 5c1a4d9..0a101c3 100644 --- a/src/upstream/dot_client.zig +++ b/src/upstream/dot_client.zig @@ -21,19 +21,6 @@ const tls_client = @import("../platform/tls_client.zig"); const log = std.log.scoped(.dot_client); -/// RFC 1035 §4.2.2 length prefix, shared by DNS over TCP and DNS over TLS. -pub const prefix_len = 2; - -pub fn framePrefix(len: u16) [prefix_len]u8 { - var out: [prefix_len]u8 = undefined; - std.mem.writeInt(u16, &out, len, .big); - return out; -} - -pub fn parsePrefix(bytes: [prefix_len]u8) u16 { - return std.mem.readInt(u16, &bytes, .big); -} - pub const ResolveError = error{ConnectFailed}; /// DoT endpoints take IP literals. Name resolution for upstreams is out of @@ -179,7 +166,7 @@ pub const DotClient = struct { defer closeTls(io, &tls_stream); const writer = tls_stream.writer(); - const prefix = framePrefix(@intCast(query.len)); + const prefix = transport.framePrefix(@intCast(query.len)); writer.writeAll(&prefix) catch |err| return sendFailure(&tls_stream, err); writer.writeAll(query) catch |err| return sendFailure(&tls_stream, err); // `TlsStream.flush`, not `writer.flush`: the latter leaves the encrypted @@ -188,10 +175,10 @@ pub const DotClient = struct { tls_stream.flush() catch |err| return sendFailure(&tls_stream, err); const reader = tls_stream.reader(); - var prefix_bytes: [prefix_len]u8 = undefined; + var prefix_bytes: [transport.prefix_len]u8 = undefined; reader.readSliceAll(&prefix_bytes) catch |err| return receiveFailure(&tls_stream, err); - const len = parsePrefix(prefix_bytes); + const len = transport.parsePrefix(prefix_bytes); if (len == 0) return error.BadResponse; if (len > response_buf.len) return error.ResponseTooLarge; reader.readSliceAll(response_buf[0..len]) catch |err| @@ -299,27 +286,6 @@ fn receiveFailure(stream: *tls_client.TlsStream, err: anyerror) transport.Exchan const testing = std.testing; -test "framePrefix writes the length big-endian" { - try testing.expectEqualSlices(u8, &.{ 0x00, 0x00 }, &framePrefix(0)); - try testing.expectEqualSlices(u8, &.{ 0x00, 0x1d }, &framePrefix(29)); - try testing.expectEqualSlices(u8, &.{ 0x01, 0x00 }, &framePrefix(256)); - try testing.expectEqualSlices(u8, &.{ 0xff, 0xff }, &framePrefix(65535)); -} - -test "parsePrefix reads the length big-endian" { - try testing.expectEqual(@as(u16, 0), parsePrefix(.{ 0x00, 0x00 })); - try testing.expectEqual(@as(u16, 29), parsePrefix(.{ 0x00, 0x1d })); - try testing.expectEqual(@as(u16, 256), parsePrefix(.{ 0x01, 0x00 })); - try testing.expectEqual(@as(u16, 65535), parsePrefix(.{ 0xff, 0xff })); -} - -test "framePrefix and parsePrefix round-trip" { - const cases = [_]u16{ 0, 1, 12, 512, 4096, 65534, 65535 }; - for (cases) |len| { - try testing.expectEqual(len, parsePrefix(framePrefix(len))); - } -} - test "resolveAddress accepts IP literals" { const v4 = try resolveAddress(try .parse("tls://1.1.1.1:853")); try testing.expectEqual(@as(u16, 853), v4.ip4.port); diff --git a/src/upstream/transport.zig b/src/upstream/transport.zig index 258b46c..14493df 100644 --- a/src/upstream/transport.zig +++ b/src/upstream/transport.zig @@ -20,6 +20,21 @@ const name = @import("../dns/name.zig"); /// RFC 1035 §4.2.2: the TCP length prefix is 16-bit, so no DNS message can be /// larger than this on any transport nxdns speaks. pub const max_message_len = 65535; + +/// RFC 1035 §4.2.2 two-byte big-endian length prefix, shared by every +/// stream transport (DoT, plain TCP server, forward-zone TCP fallback). +pub const prefix_len = 2; + +pub fn framePrefix(len: u16) [prefix_len]u8 { + var out: [prefix_len]u8 = undefined; + std.mem.writeInt(u16, &out, len, .big); + return out; +} + +pub fn parsePrefix(bytes: [prefix_len]u8) u16 { + return std.mem.readInt(u16, &bytes, .big); +} + pub const doh_default_port = 443; pub const dot_default_port = 853; // RFC 7858 §3.1 pub const doh_default_path = "/dns-query"; // RFC 8484 §4.1 well-known template @@ -264,6 +279,30 @@ pub fn validateResponse(query: []const u8, response: []const u8) ValidateError!v const testing = std.testing; +test "framePrefix writes the length big-endian" { + try testing.expectEqualSlices(u8, &.{ 0x00, 0x00 }, &framePrefix(0)); + try testing.expectEqualSlices(u8, &.{ 0x00, 0x1d }, &framePrefix(29)); + try testing.expectEqualSlices(u8, &.{ 0x01, 0x00 }, &framePrefix(256)); + try testing.expectEqualSlices(u8, &.{ 0xff, 0xff }, &framePrefix(65535)); +} + +test "parsePrefix reads the length big-endian" { + try testing.expectEqual(@as(u16, 0), parsePrefix(.{ 0x00, 0x00 })); + try testing.expectEqual(@as(u16, 29), parsePrefix(.{ 0x00, 0x1d })); + try testing.expectEqual(@as(u16, 256), parsePrefix(.{ 0x01, 0x00 })); + try testing.expectEqual(@as(u16, 65535), parsePrefix(.{ 0xff, 0xff })); +} + +test "framePrefix and parsePrefix round-trip" { + for ([_]u16{ 0, 1, 12, 512, 4096, 65534, 65535 }) |len| { + try testing.expectEqual(len, parsePrefix(framePrefix(len))); + } +} + +test "the prefix ceiling is the message ceiling" { + try testing.expectEqual(@as(u16, max_message_len), parsePrefix(.{ 0xff, 0xff })); +} + test "parse a DoH url with an explicit path" { const e = try Endpoint.parse("https://cloudflare-dns.com/dns-query"); try testing.expectEqual(Scheme.doh, e.scheme); diff --git a/tests/fuzz/blocklist_fuzz.zig b/tests/fuzz/blocklist_fuzz.zig new file mode 100644 index 0000000..e5d8e44 --- /dev/null +++ b/tests/fuzz/blocklist_fuzz.zig @@ -0,0 +1,213 @@ +//! Fuzz targets for the blocklist line parsers and the wildcard matcher +//! (`src/filter/parsers.zig` and its siblings). +//! +//! Every target holds the same contract: any byte string is a legal blocklist +//! line, so a parser may classify it however it likes but may not panic, may +//! not read out of bounds and may not fail to return. Where a classification +//! succeeds the target then checks the invariant the compiler is entitled to +//! rely on: +//! +//! - `Line.text` is always a slice of the caller's line, never a copy and +//! never a dangling pointer into a temporary; +//! - `covers_apex` is set only on a `.wildcard` line, because the compiler +//! reads it only there; +//! - `wildcard.matches` terminates for any pattern, validated or not, and a +//! match implies the domain has at least as many labels as the pattern, +//! since every pattern label consumes at least one domain label. +//! +//! `parsers.parseLine` documents that its line carries no `\n` and no `\r`, so +//! each target cuts the fuzzer's bytes at the first one rather than handing the +//! parser input the compiler could never produce. +//! +//! This file is the root of its own test artifact and reaches the parsers +//! through the `parsers` module, which is why those five files import nothing +//! outside `src/filter/`. +//! +//! Runner semantics: under a plain `zig build test` each target runs once per +//! corpus entry plus once on empty input, which makes the corpus a regression +//! suite. `zig build test --fuzz=` gives each target `n` generated inputs. + +const std = @import("std"); +const parsers = @import("parsers"); + +const wildcard = parsers.wildcard; +const Smith = std.testing.Smith; + +/// Long enough to hold a line past `compiler.max_line_len`, which is the +/// longest line the compiler ever hands a parser. +const max_input = 8192; + +/// `Smith` entity ids. The two-slice target needs stable, distinct ids for its +/// pattern and its domain; the single-slice targets take the first. +const pattern_hash: u32 = 1; +const domain_hash: u32 = 2; + +const fuzz_options: std.testing.FuzzInputOptions = .{ .corpus = &corpus }; + +test "fuzz parser_hosts.parseLine" { + try std.testing.fuzz(parsers.Format.hosts, formatTarget, fuzz_options); +} + +test "fuzz parser_domains.parseLine" { + try std.testing.fuzz(parsers.Format.domains, formatTarget, fuzz_options); +} + +test "fuzz parser_abp.parseLine" { + try std.testing.fuzz(parsers.Format.abp, formatTarget, fuzz_options); +} + +test "fuzz parsers.detectFormat" { + try std.testing.fuzz({}, detectTarget, fuzz_options); +} + +test "fuzz wildcard.validate and wildcard.matches" { + try std.testing.fuzz({}, wildcardTarget, fuzz_options); +} + +/// One format's parser, reached through the dispatcher the compiler uses. +fn formatTarget(format: parsers.Format, smith: *Smith) anyerror!void { + var buf: [max_input]u8 = undefined; + const input = buf[0..smith.sliceWithHash(&buf, pattern_hash)]; + const line = upToNewline(input); + + const parsed = parsers.parseLine(format, line); + try expectBorrowed(parsed, line); +} + +/// The sniffer reads whole files, so this one keeps the line breaks. +fn detectTarget(_: void, smith: *Smith) anyerror!void { + var buf: [max_input]u8 = undefined; + const input = buf[0..smith.sliceWithHash(&buf, pattern_hash)]; + + const format = parsers.detectFormat(input); + + // Whatever the sniffer decides, every line of the same bytes must survive + // that format's parser: this is the pairing the manager performs. + var it = std.mem.splitScalar(u8, input, '\n'); + while (it.next()) |raw| { + const line = upToNewline(raw); + try expectBorrowed(parsers.parseLine(format, line), line); + } +} + +fn wildcardTarget(_: void, smith: *Smith) anyerror!void { + var pattern_buf: [max_input]u8 = undefined; + var domain_buf: [max_input]u8 = undefined; + const pattern = pattern_buf[0..smith.sliceWithHash(&pattern_buf, pattern_hash)]; + const domain = domain_buf[0..smith.sliceWithHash(&domain_buf, domain_hash)]; + + // `matches` is total on unvalidated input by design, so both the accepted + // and the rejected pattern are fed in. Production only ever reaches it with + // an accepted one, which is why the accepted case carries the invariant. + const accepted = if (wildcard.validate(pattern)) |_| true else |_| false; + const matched = wildcard.matches(pattern, domain); + + if (accepted and matched) { + try std.testing.expect(labelCount(domain) >= labelCount(pattern)); + } +} + +/// The parser contract: `text` is a window into the caller's line, so the +/// compiler may keep it for the length of that line and no longer. +fn expectBorrowed(parsed: parsers.Line, line: []const u8) !void { + if (parsed.covers_apex) try std.testing.expectEqual(parsers.Kind.wildcard, parsed.kind); + if (parsed.text.len == 0) return; + + const start = @intFromPtr(parsed.text.ptr); + const line_start = @intFromPtr(line.ptr); + try std.testing.expect(start >= line_start); + try std.testing.expect(start + parsed.text.len <= line_start + line.len); +} + +fn upToNewline(input: []const u8) []const u8 { + const end = std.mem.findAny(u8, input, "\r\n") orelse input.len; + return input[0..end]; +} + +fn labelCount(text: []const u8) usize { + return std.mem.count(u8, text, ".") + 1; +} + +// --------------------------------------------------------------------------- +// corpus +// --------------------------------------------------------------------------- +// +// `Smith` does not consume a corpus entry as raw parser input. It reads a byte +// stream in which a slice is a little-endian `u32` length followed by that many +// bytes, so every entry below is length-prefixed. The five targets share one +// corpus: each starts with a slice, and the wildcard target reads a second one +// that falls back to empty when an entry carries only the first. + +/// A hosts line with a sink address, two names and a trailing comment. +const hosts_line = "0.0.0.0 ads.example.com tracker.example.com # advertising"; + +/// An ABP domain rule, which covers the apex as well as the subdomains. +const abp_line = "||ads.example.net^"; + +/// A regex rule, which every parser counts and skips (PLAN §2.2). +const regex_line = "/^ads[0-9]+\\.example\\.org$/"; + +/// Past `compiler.max_line_len`, so the over-long path is a seed rather than a +/// discovery. +const long_line = "a" ** 5000 ++ ".example.com"; + +/// An element-hiding rule and a scheme anchor: the two `.unsupported` shapes +/// that carry a domain in front of them. +const element_hiding = "example.com##.ad-banner"; +const scheme_anchor = "|https://ads.example.com/track"; + +/// Encodes `bytes` as a single `Smith.slice` value. +fn sliceInput(comptime bytes: []const u8) *const [4 + bytes.len]u8 { + return &struct { + const value: [4 + bytes.len]u8 = blk: { + var buf: [4 + bytes.len]u8 = undefined; + std.mem.writeInt(u32, buf[0..4], @intCast(bytes.len), .little); + buf[4..].* = bytes[0..bytes.len].*; + break :blk buf; + }; + }.value; +} + +/// Encodes two `Smith.slice` values back to back, which is what the wildcard +/// target reads. +fn pairInput(comptime a: []const u8, comptime b: []const u8) *const [8 + a.len + b.len]u8 { + return &struct { + const value: [8 + a.len + b.len]u8 = blk: { + var buf: [8 + a.len + b.len]u8 = undefined; + buf[0 .. 4 + a.len].* = sliceInput(a).*; + buf[4 + a.len ..].* = sliceInput(b).*; + break :blk buf; + }; + }.value; +} + +const corpus = [_][]const u8{ + sliceInput(hosts_line), + sliceInput(abp_line), + sliceInput(regex_line), + sliceInput(long_line), + sliceInput(element_hiding), + sliceInput(scheme_anchor), + // A whole small file, so `detectFormat` sees more than one line. + sliceInput("# a hosts list\n" ++ hosts_line ++ "\n" ++ regex_line ++ "\n"), + // The two wildcard shapes PLAN §3.9 names, each with a name that matches. + pairInput("*.doubleclick.net", "a.b.doubleclick.net"), + pairInput("ads.*.example.com", "ads.eu.west.example.com"), + // A pattern that no name matches, and one `validate` rejects. + pairInput("*.example.com", "example.com.evil.net"), + pairInput("ad*.example.com", "ads.example.com"), +}; + +test "a corpus entry carries its own length" { + const encoded = sliceInput(abp_line); + try std.testing.expectEqual(@as(u32, abp_line.len), std.mem.readInt(u32, encoded[0..4], .little)); + try std.testing.expectEqualSlices(u8, abp_line, encoded[4..]); +} + +test "a paired corpus entry carries both lengths" { + const encoded = pairInput("*.a.b", "x.a.b"); + try std.testing.expectEqual(@as(u32, 5), std.mem.readInt(u32, encoded[0..4], .little)); + try std.testing.expectEqualSlices(u8, "*.a.b", encoded[4..9]); + try std.testing.expectEqual(@as(u32, 5), std.mem.readInt(u32, encoded[9..13], .little)); + try std.testing.expectEqualSlices(u8, "x.a.b", encoded[13..]); +}