milestone 21: abp list exceptions and a regex rule kind
This commit is contained in:
+10
-1
@@ -1527,7 +1527,16 @@ test "bare check with no config database exits 2 and says how to make one" {
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "no config database at "));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, config_db_name));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, db_source_hint));
|
||||
try testing.expectEqual(@as(usize, 0), std.mem.count(u8, text, "OK"));
|
||||
// Not a bare count of "OK" over the whole text: this message embeds the
|
||||
// temporary directory path, and `std.testing.tmpDir` names that directory
|
||||
// with base64 over random bytes, so a run whose name happens to carry those
|
||||
// two letters would fail a test that has nothing to do with naming. Every
|
||||
// verdict this command prints — `OK:` and `OK upstreams[...]` alike — opens
|
||||
// a line, so that is what to assert on.
|
||||
var lines = std.mem.splitScalar(u8, text, '\n');
|
||||
while (lines.next()) |line| {
|
||||
try testing.expect(!std.mem.startsWith(u8, line, "OK"));
|
||||
}
|
||||
try testing.expectEqualStrings("", captured.err.written());
|
||||
}
|
||||
|
||||
|
||||
+32
-8
@@ -1,6 +1,9 @@
|
||||
//! The one configuration model. Bootstrap, import, export, the repositories and
|
||||
//! the running server all speak this struct; nothing else describes nxdns
|
||||
//! configuration.
|
||||
//! The one *declarative* configuration model: loading, reconciliation, import,
|
||||
//! export and the running server all speak this struct, and it is the whole
|
||||
//! shape of a config file. It is not the only shape the repositories accept —
|
||||
//! the API edits rows one at a time through narrower inputs such as
|
||||
//! `RuleInput`, `ClientInput` and `ClientEdit`, so a field added here does not
|
||||
//! reach those paths by itself.
|
||||
//!
|
||||
//! Pure: no `std.Io` value is a parameter anywhere, no SQLite, no clock. The
|
||||
//! only `std.Io` types that appear are `std.Io.Duration` as a conversion result.
|
||||
@@ -8,11 +11,21 @@
|
||||
//! Runtime columns are deliberately absent. `clients.first_seen`,
|
||||
//! `clients.last_seen`, `rules.created_at` and
|
||||
//! `blocklist_sources.{last_updated, domain_count, wildcard_count,
|
||||
//! skipped_regex_count, checksum}` are facts a running server produces, not
|
||||
//! configuration. Including them would make two exports taken minutes apart
|
||||
//! differ, which would make the byte-stable round trip untestable against a
|
||||
//! live server. Import sets the timestamps to the import time and leaves the
|
||||
//! counters at their column defaults.
|
||||
//! exception_count, skipped_regex_count, checksum}` are facts a running server
|
||||
//! produces, not configuration. Including them would make two exports taken
|
||||
//! minutes apart differ, which would make the byte-stable round trip untestable
|
||||
//! against a live server.
|
||||
//!
|
||||
//! Declarative configuration reaches the database through exactly one path:
|
||||
//! `config/reconcile.zig`. `nxdns import` is a thin wrapper over it, and so is
|
||||
//! `run --config`. Reconciliation asks what changed rather than replacing
|
||||
//! wholesale, so a row the input still names keeps the runtime state attached
|
||||
//! to it: a source matched by url keeps its id, checksum and counters, a rule
|
||||
//! keeps its `created_at`, and a client keeps its first-seen and last-seen
|
||||
//! stamps. The source id and checksum are the two that decide whether a
|
||||
//! file-mode restart reuses the compiled bodies or downloads them again:
|
||||
//! `loadSource` names the files after the id and accepts them only against the
|
||||
//! stored checksum. The counters ride along as reported state.
|
||||
|
||||
const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
@@ -256,20 +269,25 @@ pub const BlocklistSource = struct {
|
||||
|
||||
pub const GroupSource = struct { group: []const u8, source_url: []const u8 };
|
||||
|
||||
/// The three spellings `CHECK(kind IN ('exact','wildcard','regex'))` admits
|
||||
/// after migration step 4.
|
||||
pub const RuleKind = enum {
|
||||
exact,
|
||||
wildcard,
|
||||
regex,
|
||||
|
||||
pub fn toDb(self: RuleKind) []const u8 {
|
||||
return switch (self) {
|
||||
.exact => "exact",
|
||||
.wildcard => "wildcard",
|
||||
.regex => "regex",
|
||||
};
|
||||
}
|
||||
|
||||
pub fn fromDb(text: []const u8) ?RuleKind {
|
||||
if (std.mem.eql(u8, text, "exact")) return .exact;
|
||||
if (std.mem.eql(u8, text, "wildcard")) return .wildcard;
|
||||
if (std.mem.eql(u8, text, "regex")) return .regex;
|
||||
return null;
|
||||
}
|
||||
};
|
||||
@@ -789,6 +807,12 @@ test "every toDb and fromDb enum pair round-trips over all tags" {
|
||||
try expectEnumRoundTrip(RecordType);
|
||||
}
|
||||
|
||||
test "RuleKind carries the third kind through export and import" {
|
||||
try testing.expectEqualStrings("regex", RuleKind.regex.toDb());
|
||||
try testing.expectEqual(RuleKind.regex, RuleKind.fromDb("regex").?);
|
||||
try testing.expect(RuleKind.fromDb("Regex") == null);
|
||||
}
|
||||
|
||||
test "RecordType stores the uppercase DDL spelling" {
|
||||
try testing.expectEqualStrings("A", RecordType.a.toDb());
|
||||
try testing.expectEqualStrings("AAAA", RecordType.aaaa.toDb());
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
//!
|
||||
//! The defect this module exists to fix: `import.applyToDb` deletes and
|
||||
//! reinserts every row, `blocklist_sources` included, and the compiled
|
||||
//! blocklists are named after the source row id (`<id>.list` / `<id>.wild`). A
|
||||
//! configuration re-applied on every boot would therefore hand every source a
|
||||
//! new id, orphan every compiled file, and re-download every blocklist on every
|
||||
//! restart.
|
||||
//! blocklists are named after the source row id (`<id>.list` / `<id>.wild` /
|
||||
//! `<id>.allow`). A configuration re-applied on every boot would therefore hand
|
||||
//! every source a new id, orphan every compiled file, and re-download every
|
||||
//! blocklist on every restart.
|
||||
//!
|
||||
//! So nothing is wiped. Every table has an identity; a row the file and the
|
||||
//! database agree on is **updated in place**, keeping its row id and every
|
||||
@@ -1100,6 +1100,7 @@ fn seedSourceStats(database: *db.Db, id: i64) !void {
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 4321,
|
||||
.wildcard_count = 21,
|
||||
.exception_count = 9,
|
||||
.skipped_regex_count = 7,
|
||||
.checksum = "a" ** 64,
|
||||
});
|
||||
@@ -1169,6 +1170,7 @@ test "a source keeps its id, its checksum and its counters across a reconcile" {
|
||||
try testing.expectEqualStrings("advertising", row.name);
|
||||
try testing.expectEqual(@as(?i64, 1_700_000_000), row.last_updated);
|
||||
try testing.expectEqual(@as(i64, 4321), row.domain_count);
|
||||
try testing.expectEqual(@as(i64, 9), row.exception_count);
|
||||
try testing.expectEqualStrings("a" ** 64, row.checksum.?);
|
||||
}
|
||||
|
||||
@@ -1550,6 +1552,50 @@ test "rules keep created_at across a reconcile, duplicates included" {
|
||||
}
|
||||
}
|
||||
|
||||
test "a regex rule declared in the file converges into the table and back out" {
|
||||
var bench: Bench = undefined;
|
||||
try bench.init();
|
||||
defer bench.deinit();
|
||||
|
||||
// Under `.managed_file` authority the API refuses rule writes, so this is
|
||||
// the only way a regex rule reaches the table in that mode. `reconcileRules`
|
||||
// compares the whole tuple and needs no code of its own for the new kind.
|
||||
const with_regex: [:0]const u8 =
|
||||
\\.{
|
||||
\\ .groups = .{ .{ .name = "default" } },
|
||||
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
|
||||
\\ .rules = .{
|
||||
\\ .{ .group = "default", .pattern = "^ad[0-9]+-", .kind = .regex, .action = .block },
|
||||
\\ },
|
||||
\\}
|
||||
;
|
||||
const first = try bench.apply(with_regex, 1_700_000_000);
|
||||
try testing.expectEqual(@as(u32, 1), first.rules.inserted);
|
||||
|
||||
const gpa = testing.allocator;
|
||||
var rows = try rules_repo.listRuleRows(&bench.database, gpa);
|
||||
defer rows.deinit(gpa);
|
||||
defer rules_repo.freeRuleRows(gpa, rows.items);
|
||||
try testing.expectEqual(@as(usize, 1), rows.items.len);
|
||||
try testing.expectEqual(model.RuleKind.regex, rows.items[0].kind);
|
||||
try testing.expectEqualStrings("^ad[0-9]+-", rows.items[0].pattern);
|
||||
|
||||
// Idempotent: the tuple matches itself, so a second pass writes nothing.
|
||||
const second = try bench.apply(with_regex, 1_800_000_000);
|
||||
try testing.expectEqual(@as(u32, 0), second.rules.total());
|
||||
|
||||
// And a file that stops declaring it takes the row with it.
|
||||
const without: [:0]const u8 =
|
||||
\\.{
|
||||
\\ .groups = .{ .{ .name = "default" } },
|
||||
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
|
||||
\\}
|
||||
;
|
||||
const third = try bench.apply(without, 1_900_000_000);
|
||||
try testing.expectEqual(@as(u32, 1), third.rules.deleted);
|
||||
try testing.expectEqual(@as(i64, 0), try rules_repo.countRules(&bench.database));
|
||||
}
|
||||
|
||||
test "dropping one of two identical rules removes exactly one row" {
|
||||
var bench: Bench = undefined;
|
||||
try bench.init();
|
||||
|
||||
+111
-4
@@ -44,6 +44,7 @@ const Writer = std.Io.Writer;
|
||||
const model = @import("model.zig");
|
||||
const address = @import("../platform/address.zig");
|
||||
const dns_name = @import("../dns/name.zig");
|
||||
const regex = @import("../filter/regex.zig");
|
||||
const safe_url = @import("../safe_url.zig");
|
||||
const transport = @import("../upstream/transport.zig");
|
||||
|
||||
@@ -890,13 +891,31 @@ fn checkCollections(cfg: Config, diags: *Diagnostics, scratch: Allocator) error{
|
||||
|
||||
for (cfg.rules, 0..) |rule, i| {
|
||||
try checkGroupRef(diags, &group_names, rule.group, "rules[{d}].group", .{i});
|
||||
if (!try patternIsValid(scratch, rule.pattern, rule.kind)) {
|
||||
// `null` is a good pattern; anything else is the sentence fragment that
|
||||
// says which of the regex engine's limits refused it. The empty string
|
||||
// is a plain syntax refusal, which is the only verdict the exact and
|
||||
// wildcard kinds can reach.
|
||||
const detail: ?[]const u8 = if (patternIsValid(scratch, rule.pattern, rule.kind)) |valid|
|
||||
(if (valid) null else "")
|
||||
else |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
error.BadPattern => "",
|
||||
error.PatternTooLong => std.fmt.comptimePrint(
|
||||
" (over {d} bytes)",
|
||||
.{regex.max_pattern_len},
|
||||
),
|
||||
error.PatternTooComplex => std.fmt.comptimePrint(
|
||||
" (over {d} compiled instructions)",
|
||||
.{regex.max_program_len},
|
||||
),
|
||||
};
|
||||
if (detail) |suffix| {
|
||||
try diags.add(
|
||||
error.BadRulePattern,
|
||||
"rules[{d}].pattern",
|
||||
.{i},
|
||||
"{f} is not a valid {s} pattern",
|
||||
.{ safe_url.quoteText(rule.pattern), rule.kind.toDb() },
|
||||
"{f} is not a valid {s} pattern{s}",
|
||||
.{ safe_url.quoteText(rule.pattern), rule.kind.toDb(), suffix },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1096,11 +1115,18 @@ fn sourceUrlIsValid(url: []const u8) bool {
|
||||
/// Syntax only. Matching semantics are Phase 5's: an exact pattern carries no
|
||||
/// `*` at all, a wildcard pattern carries at least one label that is exactly
|
||||
/// `*`, and every remaining label must survive `dns.name.fromText`.
|
||||
///
|
||||
/// A regex pattern is validated by compiling it, and its three refusals arrive
|
||||
/// as errors rather than as `false` so the caller can name the one that fired.
|
||||
/// The distinction is the operator's, not the compiler's: "not a valid regex
|
||||
/// pattern" sends someone hunting for a typo in a pattern whose only fault is
|
||||
/// that it is longer than `regex.max_pattern_len` or wider than
|
||||
/// `regex.max_program_len`, and neither limit is visible in the pattern text.
|
||||
fn patternIsValid(
|
||||
scratch: Allocator,
|
||||
pattern: []const u8,
|
||||
kind: model.RuleKind,
|
||||
) error{OutOfMemory}!bool {
|
||||
) regex.Error!bool {
|
||||
switch (kind) {
|
||||
.exact => {
|
||||
if (std.mem.findScalar(u8, pattern, '*') != null) return false;
|
||||
@@ -1126,6 +1152,11 @@ fn patternIsValid(
|
||||
_ = dns_name.fromText(substituted.items) catch return false;
|
||||
return true;
|
||||
},
|
||||
.regex => {
|
||||
var program = try regex.compile(scratch, pattern);
|
||||
program.deinit(scratch);
|
||||
return true;
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2105,6 +2136,82 @@ test "rule patterns accept wildcards only when the kind says so" {
|
||||
try expectProblem(bad_label, error.BadRulePattern, "rules[0].pattern");
|
||||
}
|
||||
|
||||
/// The message of the first failure, so a test can assert the sentence an
|
||||
/// operator reads and not only the error tag.
|
||||
fn expectMessage(cfg: Config, expected: ValidateError, expected_message: []const u8) !void {
|
||||
var diags: Diagnostics = .init(testing.allocator);
|
||||
defer diags.deinit();
|
||||
try testing.expectError(expected, validate(cfg, &diags));
|
||||
const failure = diags.firstFailure() orelse return error.TestExpectedFailure;
|
||||
try testing.expectEqualStrings(expected_message, failure.message);
|
||||
}
|
||||
|
||||
/// The tail of the first failure's message. `quoteText` truncates the value it
|
||||
/// quotes at `safe_url.max_len`, so a diagnostic about an over-long pattern
|
||||
/// cannot be matched whole.
|
||||
fn expectMessageSuffix(cfg: Config, expected: ValidateError, expected_suffix: []const u8) !void {
|
||||
var diags: Diagnostics = .init(testing.allocator);
|
||||
defer diags.deinit();
|
||||
try testing.expectError(expected, validate(cfg, &diags));
|
||||
const failure = diags.firstFailure() orelse return error.TestExpectedFailure;
|
||||
if (!std.mem.endsWith(u8, failure.message, expected_suffix)) {
|
||||
std.debug.print("message {s} does not end with {s}\n", .{ failure.message, expected_suffix });
|
||||
return error.TestExpectedEqual;
|
||||
}
|
||||
}
|
||||
|
||||
fn regexRule(pattern: []const u8) [1]model.Rule {
|
||||
return .{.{ .group = "default", .pattern = pattern, .kind = .regex, .action = .block }};
|
||||
}
|
||||
|
||||
test "a regex rule is validated by compiling it" {
|
||||
var cfg = baseConfig();
|
||||
const good = regexRule("^ad[0-9]+-\\.(example|test)\\.com$");
|
||||
cfg.rules = &good;
|
||||
try expectClean(cfg);
|
||||
|
||||
// A regex is not a name: the wildcard and exact kinds reject `*`, and this
|
||||
// one has to accept the characters that make a pattern a pattern.
|
||||
const starred = regexRule("ads.*\\.example");
|
||||
cfg.rules = &starred;
|
||||
try expectClean(cfg);
|
||||
}
|
||||
|
||||
test "each of the regex engine's three refusals names itself in the diagnostic" {
|
||||
var cfg = baseConfig();
|
||||
|
||||
const unclosed = regexRule("(");
|
||||
cfg.rules = &unclosed;
|
||||
try expectProblem(cfg, error.BadRulePattern, "rules[0].pattern");
|
||||
try expectMessage(cfg, error.BadRulePattern, "'(' is not a valid regex pattern");
|
||||
|
||||
// Too long and too complex are the two an operator cannot see by reading
|
||||
// the pattern, so the message has to carry the limit that fired.
|
||||
const too_long = regexRule("a" ** (regex.max_pattern_len + 1));
|
||||
cfg.rules = &too_long;
|
||||
try expectMessageSuffix(
|
||||
cfg,
|
||||
error.BadRulePattern,
|
||||
"is not a valid regex pattern (over 256 bytes)",
|
||||
);
|
||||
|
||||
// Well inside 256 bytes of pattern, well past 1024 instructions of program.
|
||||
const too_complex = regexRule("(abcdefghij){200}");
|
||||
cfg.rules = &too_complex;
|
||||
try expectMessage(
|
||||
cfg,
|
||||
error.BadRulePattern,
|
||||
"'(abcdefghij){200}' is not a valid regex pattern (over 1024 compiled instructions)",
|
||||
);
|
||||
}
|
||||
|
||||
test "an empty regex pattern is refused rather than matching every name" {
|
||||
var cfg = baseConfig();
|
||||
const empty = regexRule("");
|
||||
cfg.rules = ∅
|
||||
try expectProblem(cfg, error.BadRulePattern, "rules[0].pattern");
|
||||
}
|
||||
|
||||
test "parseResolver accepts udp and tcp with an IP literal and a port" {
|
||||
const udp4 = try parseResolver("udp://192.168.1.1:53");
|
||||
try testing.expectEqual(ResolverScheme.udp, udp4.scheme);
|
||||
|
||||
+174
-42
@@ -1,14 +1,15 @@
|
||||
//! Compiles a downloaded blocklist into the two bodies nxdns stores on disk:
|
||||
//! a `.list` body of exact names and a `.wild` body of suffixes.
|
||||
//! Compiles a downloaded blocklist into the three bodies nxdns stores on disk:
|
||||
//! a `.list` body of exact names, a `.wild` body of suffixes and an `.allow`
|
||||
//! body of the names the list's `@@` exceptions lift.
|
||||
//!
|
||||
//! Pure over reader/writer interfaces: an allocator, a `*std.Io.Reader` and two
|
||||
//! `*std.Io.Writer`. No `std.Io` value, no file, no clock. A compiled body is a
|
||||
//! pure function of (bytes, format), which is what makes two runs — and two
|
||||
//! Pure over reader/writer interfaces: an allocator, a `*std.Io.Reader` and
|
||||
//! three `*std.Io.Writer`. No `std.Io` value, no file, no clock. A compiled body
|
||||
//! is a pure function of (bytes, format), which is what makes two runs — and two
|
||||
//! permutations of the same input — byte-identical.
|
||||
//!
|
||||
//! Nothing but the sorted, deduplicated names is written: no header, no
|
||||
//! timestamp, no counts. The header belongs to the caller, and the checksum
|
||||
//! covers the two bodies only.
|
||||
//! covers the three bodies only.
|
||||
|
||||
const std = @import("std");
|
||||
const parsers = @import("parsers.zig");
|
||||
@@ -23,6 +24,12 @@ pub const max_line_len: usize = 4096;
|
||||
pub const Counts = struct {
|
||||
domains: u32 = 0,
|
||||
wildcards: u32 = 0,
|
||||
/// Written, deduplicated `.allow` entries: the names this list's `@@`
|
||||
/// exceptions lift out of what other lists block.
|
||||
exceptions: u32 = 0,
|
||||
/// Regex lines this list carried, counted and skipped. nxdns has an engine
|
||||
/// for them now, but it stays reserved for operator rules: a downloaded list
|
||||
/// is other people's patterns, and PLAN §2.2 keeps them out.
|
||||
skipped_regex: u32 = 0,
|
||||
skipped_unsupported: u32 = 0,
|
||||
/// Not a valid domain name (`dns.name.fromText` rejected it, a non-ASCII
|
||||
@@ -36,30 +43,35 @@ pub const Counts = struct {
|
||||
|
||||
pub const Result = struct {
|
||||
counts: Counts,
|
||||
/// Lowercase hex sha256 over the `.list` body followed by the `.wild` body.
|
||||
/// Lowercase hex sha256 over the `.list` body, then the `.wild` body, then
|
||||
/// the `.allow` body.
|
||||
///
|
||||
/// The `.allow` body came last for a reason: hashing an empty one adds
|
||||
/// nothing, so a list with no exceptions keeps the digest it had when only
|
||||
/// two bodies existed. Every checksum published before exceptions were
|
||||
/// honoured therefore stays valid, and upgrading forces no refetch.
|
||||
checksum: [64]u8,
|
||||
};
|
||||
|
||||
pub const Error = error{ OutOfMemory, TooManyDomains, ReadFailed, WriteFailed };
|
||||
|
||||
/// Reads `r` to end of stream and writes the two compiled bodies.
|
||||
/// Reads `r` to end of stream and writes the three compiled bodies.
|
||||
///
|
||||
/// `counts.domains` and `counts.wildcards` are the written, deduplicated
|
||||
/// counts: they are what `blocklist_sources.domain_count` and `wildcard_count`
|
||||
/// store and what the UI shows.
|
||||
/// `counts.domains`, `counts.wildcards` and `counts.exceptions` are the written,
|
||||
/// deduplicated counts: they are what `blocklist_sources.domain_count`,
|
||||
/// `wildcard_count` and `exception_count` store and what the UI shows.
|
||||
pub fn compile(
|
||||
gpa: std.mem.Allocator,
|
||||
r: *std.Io.Reader,
|
||||
format: parsers.Format,
|
||||
list_w: *std.Io.Writer,
|
||||
wild_w: *std.Io.Writer,
|
||||
allow_w: *std.Io.Writer,
|
||||
) Error!Result {
|
||||
var counts: Counts = .{};
|
||||
|
||||
var list: Entries = .{};
|
||||
defer list.deinit(gpa);
|
||||
var wild: Entries = .{};
|
||||
defer wild.deinit(gpa);
|
||||
var bodies: Bodies = .{};
|
||||
defer bodies.deinit(gpa);
|
||||
|
||||
while (try parsers.nextBoundedLine(r, max_line_len)) |event| {
|
||||
const raw = switch (event) {
|
||||
@@ -81,32 +93,27 @@ pub fn compile(
|
||||
.domain => {
|
||||
var fields = std.mem.tokenizeAny(u8, parsed.text, &std.ascii.whitespace);
|
||||
while (fields.next()) |field| {
|
||||
try addCandidate(gpa, field, false, false, &list, &wild, &counts);
|
||||
try addCandidate(gpa, field, parsed, &bodies, &counts);
|
||||
}
|
||||
},
|
||||
.wildcard => try addCandidate(
|
||||
gpa,
|
||||
parsed.text,
|
||||
true,
|
||||
parsed.covers_apex,
|
||||
&list,
|
||||
&wild,
|
||||
&counts,
|
||||
),
|
||||
.wildcard, .exception => try addCandidate(gpa, parsed.text, parsed, &bodies, &counts),
|
||||
}
|
||||
}
|
||||
|
||||
// The `.allow` body is hashed last so that a list with no exceptions
|
||||
// reproduces the digest a two-body compile of the same bytes produced.
|
||||
var hasher = Sha256.init(.{});
|
||||
counts.domains = try emit(&list, list_w, &hasher, &counts.duplicates);
|
||||
counts.wildcards = try emit(&wild, wild_w, &hasher, &counts.duplicates);
|
||||
counts.domains = try emit(&bodies.list, list_w, &hasher, &counts.duplicates);
|
||||
counts.wildcards = try emit(&bodies.wild, wild_w, &hasher, &counts.duplicates);
|
||||
counts.exceptions = try emit(&bodies.allow, allow_w, &hasher, &counts.duplicates);
|
||||
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
return .{ .counts = counts, .checksum = std.fmt.bytesToHex(digest, .lower) };
|
||||
}
|
||||
|
||||
/// Normalizes one whitespace-separated candidate and files it under `.list`,
|
||||
/// `.wild`, or neither.
|
||||
/// Normalizes one whitespace-separated candidate of `line` and files it under
|
||||
/// `.list`, `.wild`, `.allow`, or nowhere.
|
||||
///
|
||||
/// The normalization below is deliberately not `dns.name.normalizeText`: this
|
||||
/// one adds the two-label minimum, rejects control bytes, and reports every
|
||||
@@ -114,20 +121,19 @@ pub fn compile(
|
||||
fn addCandidate(
|
||||
gpa: std.mem.Allocator,
|
||||
field: []const u8,
|
||||
from_wildcard_line: bool,
|
||||
covers_apex: bool,
|
||||
list: *Entries,
|
||||
wild: *Entries,
|
||||
line: parsers.Line,
|
||||
bodies: *Bodies,
|
||||
counts: *Counts,
|
||||
) Error!void {
|
||||
var candidate = field;
|
||||
var is_wildcard = from_wildcard_line;
|
||||
var is_wildcard = line.kind == .wildcard;
|
||||
if (std.mem.startsWith(u8, candidate, "*.")) {
|
||||
is_wildcard = true;
|
||||
candidate = candidate[2..];
|
||||
}
|
||||
// A '*' anywhere else makes this a pattern, and patterns belong to the
|
||||
// `rules` table; a blocklist entry is a name or a suffix.
|
||||
// `rules` table, where the operator writes them as a `.wildcard` or a
|
||||
// `.regex`; a blocklist entry is a name or a suffix.
|
||||
if (std.mem.indexOfScalar(u8, candidate, '*') != null) {
|
||||
counts.invalid += 1;
|
||||
return;
|
||||
@@ -163,12 +169,17 @@ fn addCandidate(
|
||||
return;
|
||||
}
|
||||
|
||||
if (is_wildcard) {
|
||||
try wild.append(gpa, normalized);
|
||||
// An exception needs no apex entry beside its suffix entry: the matcher
|
||||
// walks the `.allow` set over the full name and every parent, so one entry
|
||||
// lifts `x` and every subdomain of it at once.
|
||||
if (line.kind == .exception) {
|
||||
try bodies.allow.append(gpa, normalized);
|
||||
} else if (is_wildcard) {
|
||||
try bodies.wild.append(gpa, normalized);
|
||||
// An ABP `||x^` rule covers `x` itself as well as its subdomains.
|
||||
if (covers_apex) try list.append(gpa, normalized);
|
||||
if (line.covers_apex) try bodies.list.append(gpa, normalized);
|
||||
} else {
|
||||
try list.append(gpa, normalized);
|
||||
try bodies.list.append(gpa, normalized);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,6 +213,20 @@ fn emit(
|
||||
return written;
|
||||
}
|
||||
|
||||
/// The three bodies under construction, in the order they are written and
|
||||
/// hashed.
|
||||
const Bodies = struct {
|
||||
list: Entries = .{},
|
||||
wild: Entries = .{},
|
||||
allow: Entries = .{},
|
||||
|
||||
fn deinit(self: *Bodies, gpa: std.mem.Allocator) void {
|
||||
self.list.deinit(gpa);
|
||||
self.wild.deinit(gpa);
|
||||
self.allow.deinit(gpa);
|
||||
}
|
||||
};
|
||||
|
||||
/// Length-prefixed candidate bytes plus the offsets that index them. Sorting
|
||||
/// permutes the offsets, so the bytes never move.
|
||||
const Entries = struct {
|
||||
@@ -247,10 +272,12 @@ const Compiled = struct {
|
||||
result: Result,
|
||||
list_w: std.Io.Writer.Allocating,
|
||||
wild_w: std.Io.Writer.Allocating,
|
||||
allow_w: std.Io.Writer.Allocating,
|
||||
|
||||
fn deinit(self: *Compiled) void {
|
||||
self.list_w.deinit();
|
||||
self.wild_w.deinit();
|
||||
self.allow_w.deinit();
|
||||
}
|
||||
|
||||
fn list(self: *Compiled) []const u8 {
|
||||
@@ -260,6 +287,10 @@ const Compiled = struct {
|
||||
fn wild(self: *Compiled) []const u8 {
|
||||
return self.wild_w.written();
|
||||
}
|
||||
|
||||
fn allow(self: *Compiled) []const u8 {
|
||||
return self.allow_w.written();
|
||||
}
|
||||
};
|
||||
|
||||
fn compileText(gpa: std.mem.Allocator, text: []const u8, format: parsers.Format) Error!Compiled {
|
||||
@@ -272,8 +303,10 @@ fn compileReader(gpa: std.mem.Allocator, r: *std.Io.Reader, format: parsers.Form
|
||||
errdefer list_w.deinit();
|
||||
var wild_w: std.Io.Writer.Allocating = .init(gpa);
|
||||
errdefer wild_w.deinit();
|
||||
const result = try compile(gpa, r, format, &list_w.writer, &wild_w.writer);
|
||||
return .{ .result = result, .list_w = list_w, .wild_w = wild_w };
|
||||
var allow_w: std.Io.Writer.Allocating = .init(gpa);
|
||||
errdefer allow_w.deinit();
|
||||
const result = try compile(gpa, r, format, &list_w.writer, &wild_w.writer, &allow_w.writer);
|
||||
return .{ .result = result, .list_w = list_w, .wild_w = wild_w, .allow_w = allow_w };
|
||||
}
|
||||
|
||||
const hosts_fixture =
|
||||
@@ -339,10 +372,106 @@ test "abp apex rule lands in both bodies" {
|
||||
|
||||
try testing.expectEqualStrings("bare.com\nx.com\n", c.list());
|
||||
try testing.expectEqualStrings("x.com\n", c.wild());
|
||||
try testing.expectEqualStrings("z.com\n", c.allow());
|
||||
try testing.expectEqual(@as(u32, 2), c.result.counts.domains);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.wildcards);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.exceptions);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_regex);
|
||||
try testing.expectEqual(@as(u32, 2), c.result.counts.skipped_unsupported);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_unsupported);
|
||||
}
|
||||
|
||||
test "an abp list's hosts-style lines reach the domain body through the split" {
|
||||
// The case `parser_abp` defers here: it hands a whitespace-carrying bare
|
||||
// candidate over whole, and only the tokenization in `compile` files the
|
||||
// name out of it. A mixed list — `!` header and `||` rules, so `detectFormat`
|
||||
// calls the whole file `abp`, plus the hosts lines such lists carry — reaches
|
||||
// a compiled body no other way, and no parser test can see it happen.
|
||||
const fixture =
|
||||
"! Title: mixed\n" ++
|
||||
"||blocked.example^\n" ++
|
||||
"0.0.0.0 ads.example\n" ++
|
||||
"127.0.0.1 localhost\n";
|
||||
|
||||
var c = try compileText(testing.allocator, fixture, .abp);
|
||||
defer c.deinit();
|
||||
|
||||
// The address field is filed as a name of its own: abp lines have no hosts
|
||||
// framing, so the compiler cannot know which field is the address. `0.0.0.0`
|
||||
// and `127.0.0.1` are names nobody resolves, which is why the split is worth
|
||||
// more than the two spurious entries cost.
|
||||
try testing.expectEqualStrings(
|
||||
"0.0.0.0\n127.0.0.1\nads.example\nblocked.example\n",
|
||||
c.list(),
|
||||
);
|
||||
try testing.expectEqualStrings("blocked.example\n", c.wild());
|
||||
try testing.expectEqual(@as(u32, 4), c.result.counts.domains);
|
||||
// `localhost` is the one label the two-label minimum drops.
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.invalid);
|
||||
}
|
||||
|
||||
test "the allow body is sorted, deduplicated and normalized like the others" {
|
||||
const fixture =
|
||||
"@@||GOOD.ads.example^\n" ++
|
||||
"@@||a.ads.example^$important\n" ++
|
||||
"@@||good.ads.example\n" ++
|
||||
"@@||localhost^\n" ++
|
||||
"@@||bad*.ads.example^\n" ++
|
||||
"||ads.example^\n";
|
||||
|
||||
var c = try compileText(testing.allocator, fixture, .abp);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqualStrings("a.ads.example\ngood.ads.example\n", c.allow());
|
||||
try testing.expectEqual(@as(u32, 2), c.result.counts.exceptions);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.duplicates);
|
||||
// `localhost` is one label, and the starred name is not a name at all.
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.invalid);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_unsupported);
|
||||
|
||||
// The exceptions changed neither block body.
|
||||
try testing.expectEqualStrings("ads.example\n", c.list());
|
||||
try testing.expectEqualStrings("ads.example\n", c.wild());
|
||||
}
|
||||
|
||||
test "an empty allow body reproduces the checksum of a two-body compile" {
|
||||
var c = try compileText(testing.allocator, hosts_fixture, .hosts);
|
||||
defer c.deinit();
|
||||
try testing.expectEqualStrings("", c.allow());
|
||||
|
||||
// What the digest was before the `.allow` body existed: the `.list` body
|
||||
// followed by the `.wild` body and nothing else. Every checksum stored by an
|
||||
// older nxdns was taken this way, and this is the equality that keeps them
|
||||
// valid — without it, every source on every installation would report
|
||||
// `ChecksumMismatch` at the first reload after the upgrade and re-download.
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(c.list());
|
||||
hasher.update(c.wild());
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &c.result.checksum);
|
||||
}
|
||||
|
||||
test "the checksum of a source with exceptions covers all three bodies in order" {
|
||||
const fixture =
|
||||
"||ads.example^\n" ++
|
||||
"@@||good.ads.example^\n";
|
||||
|
||||
var c = try compileText(testing.allocator, fixture, .abp);
|
||||
defer c.deinit();
|
||||
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(c.list());
|
||||
hasher.update(c.wild());
|
||||
hasher.update(c.allow());
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &c.result.checksum);
|
||||
|
||||
// A non-empty allow body does move the digest, so a list that gains an
|
||||
// exception is recompiled rather than silently kept.
|
||||
var without = try compileText(testing.allocator, "||ads.example^\n", .abp);
|
||||
defer without.deinit();
|
||||
try testing.expect(!std.mem.eql(u8, &c.result.checksum, &without.result.checksum));
|
||||
}
|
||||
|
||||
test "two runs of the same input are byte-identical" {
|
||||
@@ -353,6 +482,7 @@ test "two runs of the same input are byte-identical" {
|
||||
|
||||
try testing.expectEqualStrings(a.list(), b.list());
|
||||
try testing.expectEqualStrings(a.wild(), b.wild());
|
||||
try testing.expectEqualStrings(a.allow(), b.allow());
|
||||
try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum);
|
||||
}
|
||||
|
||||
@@ -374,6 +504,7 @@ test "a permutation of the input compiles to the same bodies" {
|
||||
|
||||
try testing.expectEqualStrings(a.list(), b.list());
|
||||
try testing.expectEqualStrings(a.wild(), b.wild());
|
||||
try testing.expectEqualStrings(a.allow(), b.allow());
|
||||
try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum);
|
||||
}
|
||||
|
||||
@@ -495,6 +626,7 @@ test "empty input produces empty bodies and the sha256 of the empty string" {
|
||||
|
||||
try testing.expectEqualStrings("", c.list());
|
||||
try testing.expectEqualStrings("", c.wild());
|
||||
try testing.expectEqualStrings("", c.allow());
|
||||
try testing.expectEqualStrings(
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
&c.result.checksum,
|
||||
|
||||
@@ -33,10 +33,13 @@ const sources_repo = @import("../storage/repositories/sources_repo.zig");
|
||||
|
||||
const compiler = @import("compiler.zig");
|
||||
const fetcher = @import("fetcher.zig");
|
||||
const parsers = @import("parsers.zig");
|
||||
const manager = @import("manager.zig");
|
||||
const matcher = @import("matcher.zig");
|
||||
const response = @import("response.zig");
|
||||
|
||||
const lookup = @import("../web/handlers/lookup.zig");
|
||||
|
||||
const forward_client = @import("../local/forward_client.zig");
|
||||
const forward_zones = @import("../local/forward_zones.zig");
|
||||
const records = @import("../local/records.zig");
|
||||
@@ -60,6 +63,11 @@ const budget: std.Io.Clock.Duration = .{ .raw = .fromSeconds(10), .clock = .awak
|
||||
/// The forward-zone read timeout. Case 15 asserts a silent resolver gives up
|
||||
/// inside twice this, so it has to be short enough to keep the run quick and
|
||||
/// long enough that a loopback answer always beats it.
|
||||
/// `/api/lookup` reports the local tables beside the filter decision; this
|
||||
/// suite's cases are about the filter half, so both are empty here.
|
||||
const empty_records: records.Records = .empty;
|
||||
const empty_zones: forward_zones.Zones = .empty;
|
||||
|
||||
const read_timeout: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(200), .clock = .awake };
|
||||
|
||||
const file_limit: std.Io.Limit = .limited(8 * 1024 * 1024);
|
||||
@@ -111,49 +119,67 @@ const http_body =
|
||||
|
||||
const http_domains: i64 = 2;
|
||||
const http_wildcards: i64 = 1;
|
||||
const http_exceptions: i64 = 0;
|
||||
const http_regex: i64 = 1;
|
||||
|
||||
/// Compiles `text` into `<base>.list` and `<base>.wild` under `dir`, exactly as
|
||||
/// the manager's compile stage does, and returns the compiler's own result.
|
||||
/// Compiles `text` into `<base>.list`, `<base>.wild` and `<base>.allow` under
|
||||
/// `dir`, exactly as the manager's compile stage does, and returns the
|
||||
/// compiler's own result.
|
||||
fn compileToFiles(
|
||||
gpa: std.mem.Allocator,
|
||||
io: std.Io,
|
||||
dir: std.Io.Dir,
|
||||
base: []const u8,
|
||||
text: []const u8,
|
||||
format: parsers.Format,
|
||||
) !compiler.Result {
|
||||
var list_name_buf: [64]u8 = undefined;
|
||||
var wild_name_buf: [64]u8 = undefined;
|
||||
var allow_name_buf: [64]u8 = undefined;
|
||||
const list_name = try std.fmt.bufPrint(&list_name_buf, "{s}.list", .{base});
|
||||
const wild_name = try std.fmt.bufPrint(&wild_name_buf, "{s}.wild", .{base});
|
||||
const allow_name = try std.fmt.bufPrint(&allow_name_buf, "{s}.allow", .{base});
|
||||
|
||||
const list_file = try dir.createFile(io, list_name, .{ .permissions = .fromMode(0o600) });
|
||||
defer list_file.close(io);
|
||||
const wild_file = try dir.createFile(io, wild_name, .{ .permissions = .fromMode(0o600) });
|
||||
defer wild_file.close(io);
|
||||
const allow_file = try dir.createFile(io, allow_name, .{ .permissions = .fromMode(0o600) });
|
||||
defer allow_file.close(io);
|
||||
|
||||
const buffers = try gpa.alloc(u8, 2 * 16 * 1024);
|
||||
const buffers = try gpa.alloc(u8, 3 * 16 * 1024);
|
||||
defer gpa.free(buffers);
|
||||
|
||||
var r: std.Io.Reader = .fixed(text);
|
||||
var list_w = list_file.writer(io, buffers[0 .. 16 * 1024]);
|
||||
var wild_w = wild_file.writer(io, buffers[16 * 1024 ..]);
|
||||
var wild_w = wild_file.writer(io, buffers[16 * 1024 .. 32 * 1024]);
|
||||
var allow_w = allow_file.writer(io, buffers[32 * 1024 ..]);
|
||||
|
||||
const result = try compiler.compile(gpa, &r, .hosts, &list_w.interface, &wild_w.interface);
|
||||
const result = try compiler.compile(
|
||||
gpa,
|
||||
&r,
|
||||
format,
|
||||
&list_w.interface,
|
||||
&wild_w.interface,
|
||||
&allow_w.interface,
|
||||
);
|
||||
try list_w.interface.flush();
|
||||
try wild_w.interface.flush();
|
||||
try allow_w.interface.flush();
|
||||
return result;
|
||||
}
|
||||
|
||||
/// The two compiled bodies of one source, read back from disk with their
|
||||
/// The three compiled bodies of one source, read back from disk with their
|
||||
/// headers stripped, exactly as `Manager.reload` reads them.
|
||||
const Bodies = struct {
|
||||
list: []u8,
|
||||
wild: []u8,
|
||||
allow: []u8,
|
||||
|
||||
fn read(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, base: []const u8) !Bodies {
|
||||
var list_name_buf: [64]u8 = undefined;
|
||||
var wild_name_buf: [64]u8 = undefined;
|
||||
var allow_name_buf: [64]u8 = undefined;
|
||||
const list = try dir.readFileAlloc(
|
||||
io,
|
||||
try std.fmt.bufPrint(&list_name_buf, "{s}.list", .{base}),
|
||||
@@ -167,21 +193,34 @@ const Bodies = struct {
|
||||
gpa,
|
||||
file_limit,
|
||||
);
|
||||
return .{ .list = list, .wild = wild };
|
||||
errdefer gpa.free(wild);
|
||||
const allow = try dir.readFileAlloc(
|
||||
io,
|
||||
try std.fmt.bufPrint(&allow_name_buf, "{s}.allow", .{base}),
|
||||
gpa,
|
||||
file_limit,
|
||||
);
|
||||
return .{ .list = list, .wild = wild, .allow = allow };
|
||||
}
|
||||
|
||||
fn deinit(self: *Bodies, gpa: std.mem.Allocator) void {
|
||||
gpa.free(self.list);
|
||||
gpa.free(self.wild);
|
||||
gpa.free(self.allow);
|
||||
self.* = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
/// A one-group, one-source snapshot over two compiled bodies.
|
||||
fn snapshotOver(gpa: std.mem.Allocator, list_body: []const u8, wild_body: []const u8) !matcher.Snapshot {
|
||||
/// A one-group, one-source snapshot over the three compiled bodies.
|
||||
fn snapshotOver(
|
||||
gpa: std.mem.Allocator,
|
||||
list_body: []const u8,
|
||||
wild_body: []const u8,
|
||||
allow_body: []const u8,
|
||||
) !matcher.Snapshot {
|
||||
const sources = [_]model.BlocklistSource{.{ .url = source_url, .name = source_name }};
|
||||
const compiled = [_]?matcher.Snapshot.Compiled{
|
||||
.{ .list_body = list_body, .wild_body = wild_body },
|
||||
.{ .list_body = list_body, .wild_body = wild_body, .allow_body = allow_body },
|
||||
};
|
||||
return matcher.Snapshot.build(gpa, .{
|
||||
.groups = &.{.{ .name = "default" }},
|
||||
@@ -198,10 +237,11 @@ fn snapshotOver(gpa: std.mem.Allocator, list_body: []const u8, wild_body: []cons
|
||||
});
|
||||
}
|
||||
|
||||
fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 {
|
||||
fn bodyChecksum(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) [64]u8 {
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(list_body);
|
||||
hasher.update(wild_body);
|
||||
hasher.update(allow_body);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
return std.fmt.bytesToHex(digest, .lower);
|
||||
@@ -679,7 +719,7 @@ test "1: a compiled hosts fixture loads into a snapshot that blocks its domains"
|
||||
const text = try hostsFixture(gpa);
|
||||
defer gpa.free(text);
|
||||
|
||||
const result = try compileToFiles(gpa, io, tmp.dir, "1", text);
|
||||
const result = try compileToFiles(gpa, io, tmp.dir, "1", text, .hosts);
|
||||
try testing.expectEqual(@as(u32, fixture_domains), result.counts.domains);
|
||||
try testing.expectEqual(@as(u32, 1), result.counts.skipped_regex);
|
||||
// The three single-label names are the only invalid candidates here.
|
||||
@@ -692,7 +732,7 @@ test "1: a compiled hosts fixture loads into a snapshot that blocks its domains"
|
||||
try testing.expect(std.mem.find(u8, bodies.list, bare) == null);
|
||||
}
|
||||
|
||||
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild);
|
||||
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild, bodies.allow);
|
||||
defer snapshot.deinit();
|
||||
const group = snapshot.groupIndexByName("default").?;
|
||||
|
||||
@@ -732,8 +772,8 @@ test "2: recompiling the same fixture produces byte-identical files and checksum
|
||||
var second_dir = try tmp.dir.createDirPathOpen(io, "second", .{});
|
||||
defer second_dir.close(io);
|
||||
|
||||
const first = try compileToFiles(gpa, io, first_dir, "1", text);
|
||||
const second = try compileToFiles(gpa, io, second_dir, "1", text);
|
||||
const first = try compileToFiles(gpa, io, first_dir, "1", text, .hosts);
|
||||
const second = try compileToFiles(gpa, io, second_dir, "1", text, .hosts);
|
||||
|
||||
try testing.expectEqualStrings(&first.checksum, &second.checksum);
|
||||
try testing.expectEqual(first.counts, second.counts);
|
||||
@@ -745,11 +785,12 @@ test "2: recompiling the same fixture produces byte-identical files and checksum
|
||||
|
||||
try testing.expectEqualSlices(u8, first_bodies.list, second_bodies.list);
|
||||
try testing.expectEqualSlices(u8, first_bodies.wild, second_bodies.wild);
|
||||
try testing.expectEqualSlices(u8, first_bodies.allow, second_bodies.allow);
|
||||
|
||||
// The checksum the compiler reported is the one over the two bodies it
|
||||
// The checksum the compiler reported is the one over the three bodies it
|
||||
// wrote, which is what the manager stores and compares against.
|
||||
try testing.expectEqualStrings(
|
||||
&bodyChecksum(first_bodies.list, first_bodies.wild),
|
||||
&bodyChecksum(first_bodies.list, first_bodies.wild, first_bodies.allow),
|
||||
&first.checksum,
|
||||
);
|
||||
}
|
||||
@@ -781,8 +822,9 @@ test "3: a damaged compiled file never replaces a serving snapshot with a worse
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 2,
|
||||
.wildcard_count = 1,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(good_list, good_wild),
|
||||
.checksum = &bodyChecksum(good_list, good_wild, ""),
|
||||
});
|
||||
|
||||
try env.mgr.reload(io);
|
||||
@@ -812,8 +854,9 @@ test "3: a damaged compiled file never replaces a serving snapshot with a worse
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 2,
|
||||
.wildcard_count = 1,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(unsorted_list, good_wild),
|
||||
.checksum = &bodyChecksum(unsorted_list, good_wild, ""),
|
||||
});
|
||||
|
||||
try testing.expectError(error.NotSorted, env.mgr.reload(io));
|
||||
@@ -1028,6 +1071,7 @@ test "8: refetching identical content skips the rewrite and still moves last_upd
|
||||
.last_updated = 1_000,
|
||||
.domain_count = http_domains,
|
||||
.wildcard_count = http_wildcards,
|
||||
.exception_count = http_exceptions,
|
||||
.skipped_regex_count = http_regex,
|
||||
.checksum = blk: {
|
||||
var rows = try listRows(&env.database);
|
||||
@@ -1069,6 +1113,9 @@ const ReloadTask = struct {
|
||||
/// Writes one source's compiled files and records their checksum, without any
|
||||
/// network: the swap and the orphan sweep care about files and rows, not about
|
||||
/// where the bytes came from.
|
||||
///
|
||||
/// All three files, including an empty `.allow`, because that is what a publish
|
||||
/// leaves: `publishOne` runs once per body and never skips the empty one.
|
||||
fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []const u8) !void {
|
||||
const io = env.io();
|
||||
var dir = try env.blocklistDir();
|
||||
@@ -1076,6 +1123,7 @@ fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []c
|
||||
|
||||
var list_name_buf: [64]u8 = undefined;
|
||||
var wild_name_buf: [64]u8 = undefined;
|
||||
var allow_name_buf: [64]u8 = undefined;
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&list_name_buf, "{d}.list", .{id}),
|
||||
.data = list_body,
|
||||
@@ -1084,13 +1132,18 @@ fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []c
|
||||
.sub_path = try std.fmt.bufPrint(&wild_name_buf, "{d}.wild", .{id}),
|
||||
.data = wild_body,
|
||||
});
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&allow_name_buf, "{d}.allow", .{id}),
|
||||
.data = "",
|
||||
});
|
||||
|
||||
try sources_repo.updateSourceStats(&env.database, id, .{
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 1,
|
||||
.wildcard_count = 0,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, wild_body),
|
||||
.checksum = &bodyChecksum(list_body, wild_body, ""),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1146,14 +1199,21 @@ test "10: pruneOrphans deletes the files of a deleted source and leaves live one
|
||||
|
||||
var dir = try env.blocklistDir();
|
||||
defer dir.close(io);
|
||||
// Every name a source can own, spelled out: this fixture is what decides
|
||||
// whether the sweep covers the whole set, so it enumerates
|
||||
// `manager.source_file_suffixes` by hand rather than sharing it. A suffix
|
||||
// added to the manager and not added here is swept by nothing and asserted
|
||||
// by nothing.
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.allow", .data = "lifted.example.com\n" });
|
||||
// Id 9999 has no `blocklist_sources` row, so no refresh can be writing for
|
||||
// it: these temporaries are what a refresh that died mid-write leaves
|
||||
// behind, and the sweep is the only thing that will ever remove them.
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.list.tmp", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.wild.tmp", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.allow.tmp", .data = "" });
|
||||
|
||||
// The live source does have a row, so its temporary is a refresh in
|
||||
// progress and must survive a sweep that runs beside it.
|
||||
@@ -1165,12 +1225,15 @@ test "10: pruneOrphans deletes the files of a deleted source and leaves live one
|
||||
|
||||
var live_buf: [64]u8 = undefined;
|
||||
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.list", .{id}), .{});
|
||||
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.allow", .{id}), .{});
|
||||
try dir.access(io, live_tmp, .{});
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list.tmp", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild.tmp", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow.tmp", .{}));
|
||||
}
|
||||
|
||||
test "10b: the scheduler sweeps orphans on its own, with no operator call" {
|
||||
@@ -1190,8 +1253,9 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
|
||||
.last_updated = std.Io.Clock.real.now(io).toSeconds(),
|
||||
.domain_count = 1,
|
||||
.wildcard_count = 0,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, ""),
|
||||
.checksum = &bodyChecksum(list_body, "", ""),
|
||||
});
|
||||
|
||||
var dir = try env.blocklistDir();
|
||||
@@ -1201,7 +1265,9 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
|
||||
// either.
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.allow", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.allow.tmp", .data = "" });
|
||||
|
||||
// `runScheduler` is the entry point `app.zig` hands to `Io.Group`, and the
|
||||
// only one the server ever calls. A disabled update stops it after the
|
||||
@@ -1212,7 +1278,9 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
|
||||
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow.tmp", .{}));
|
||||
|
||||
// The live source kept its files and is still filtering: the sweep did not
|
||||
// take the snapshot the same pass had just published.
|
||||
@@ -1376,6 +1444,7 @@ test "10d: a source deleted mid-refresh does not take the refresh's temporary fi
|
||||
try testing.expect(!std.mem.endsWith(u8, entry.name, ".tmp"));
|
||||
try testing.expect(!std.mem.endsWith(u8, entry.name, ".list"));
|
||||
try testing.expect(!std.mem.endsWith(u8, entry.name, ".wild"));
|
||||
try testing.expect(!std.mem.endsWith(u8, entry.name, ".allow"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1410,10 +1479,13 @@ test "10e: a reconcile then a restart reuses the compiled files and downloads no
|
||||
defer dir.close(io);
|
||||
var list_buf: [64]u8 = undefined;
|
||||
var wild_buf: [64]u8 = undefined;
|
||||
var allow_buf: [64]u8 = undefined;
|
||||
const list_name = try std.fmt.bufPrint(&list_buf, "{d}.list", .{id});
|
||||
const wild_name = try std.fmt.bufPrint(&wild_buf, "{d}.wild", .{id});
|
||||
const allow_name = try std.fmt.bufPrint(&allow_buf, "{d}.allow", .{id});
|
||||
const list_before = try dir.statFile(io, list_name, .{});
|
||||
const wild_before = try dir.statFile(io, wild_name, .{});
|
||||
const allow_before = try dir.statFile(io, allow_name, .{});
|
||||
|
||||
// File mode, declaring exactly what the database already holds. The engine
|
||||
// has to recognise the source by its url and leave the row where it is:
|
||||
@@ -1459,14 +1531,19 @@ test "10e: a reconcile then a restart reuses the compiled files and downloads no
|
||||
// decision the pass made rather than a connection it could not have opened.
|
||||
try testing.expectEqual(@as(u32, 1), fixture.accepted.load(.monotonic));
|
||||
|
||||
// The same two files: not recompiled, and not swept as orphans and written
|
||||
// back.
|
||||
// The same three files: not recompiled, and not swept as orphans and
|
||||
// written back. `.allow` is asserted with the other two because a restart
|
||||
// that rewrote only the exception body would otherwise leave this test
|
||||
// green while changing what the snapshot lets through.
|
||||
const list_after = try dir.statFile(io, list_name, .{});
|
||||
const wild_after = try dir.statFile(io, wild_name, .{});
|
||||
const allow_after = try dir.statFile(io, allow_name, .{});
|
||||
try testing.expectEqual(list_before.inode, list_after.inode);
|
||||
try testing.expectEqual(list_before.mtime, list_after.mtime);
|
||||
try testing.expectEqual(wild_before.inode, wild_after.inode);
|
||||
try testing.expectEqual(wild_before.mtime, wild_after.mtime);
|
||||
try testing.expectEqual(allow_before.inode, allow_after.inode);
|
||||
try testing.expectEqual(allow_before.mtime, allow_after.mtime);
|
||||
|
||||
// The row kept the id those files are named after, and the snapshot the
|
||||
// restart published is the one compiled from them.
|
||||
@@ -1788,12 +1865,12 @@ test "17: each blocking mode synthesizes the documented blocked reply" {
|
||||
|
||||
// The reply is synthesized for a name the compiled files actually block, so
|
||||
// this case covers the decision and the response together.
|
||||
const result = try compileToFiles(gpa, io, tmp.dir, "1", "0.0.0.0 ads.example.com\n");
|
||||
const result = try compileToFiles(gpa, io, tmp.dir, "1", "0.0.0.0 ads.example.com\n", .hosts);
|
||||
try testing.expectEqual(@as(u32, 1), result.counts.domains);
|
||||
|
||||
var bodies = try Bodies.read(gpa, io, tmp.dir, "1");
|
||||
defer bodies.deinit(gpa);
|
||||
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild);
|
||||
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild, bodies.allow);
|
||||
defer snapshot.deinit();
|
||||
|
||||
const group = snapshot.groupIndexByName("default").?;
|
||||
@@ -1909,3 +1986,187 @@ test "18: a body streamed in flushed parts survives the fetcher's multi-read pum
|
||||
try testing.expectEqual(matcher.Reason.blocklist_domain, decision.reason);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 19–20: list exceptions (milestone 21)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A real ABP list: one domain anchor, the exception that lifts one subtree out
|
||||
/// of it in each of the two accepted spellings, and two `@@` forms nxdns does
|
||||
/// not honour.
|
||||
const abp_exception_body =
|
||||
"[Adblock Plus 2.0]\n" ++
|
||||
"! Title: exceptions\n" ++
|
||||
"||ads.example^\n" ++
|
||||
"||tracker.example^\n" ++
|
||||
"@@||good.ads.example^\n" ++
|
||||
"@@||fine.tracker.example$important\n" ++
|
||||
"@@||paid.ads.example^$third-party\n" ++
|
||||
"@@partial.ads.example\n";
|
||||
|
||||
const abp_exception_domains: i64 = 2;
|
||||
const abp_exception_wildcards: i64 = 2;
|
||||
const abp_exception_exceptions: i64 = 2;
|
||||
|
||||
test "19: a downloaded list's exceptions lift its own blocks and nothing else" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fixture = try HttpFixture.init(io, abp_exception_body);
|
||||
defer fixture.deinit(io);
|
||||
var group: std.Io.Group = .init;
|
||||
defer group.cancel(io);
|
||||
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||
|
||||
var url_buf: [64]u8 = undefined;
|
||||
const url = try fixture.url(&url_buf);
|
||||
const id = try seedSource(&env.database, url);
|
||||
|
||||
try testing.expect(try refreshOnce(env, url));
|
||||
try env.mgr.reload(io);
|
||||
|
||||
// The `.allow` body is a third file beside the two, and the row and the
|
||||
// status entry both carry its count — which is what a
|
||||
// `POST /api/blocklists/update` response row reports as `exceptions`.
|
||||
var dir = try env.blocklistDir();
|
||||
defer dir.close(io);
|
||||
var bodies = try Bodies.read(gpa, io, dir, "1");
|
||||
defer bodies.deinit(gpa);
|
||||
try testing.expectEqualStrings(
|
||||
"fine.tracker.example\ngood.ads.example\n",
|
||||
manager.stripHeader(bodies.allow),
|
||||
);
|
||||
|
||||
var rows = try listRows(&env.database);
|
||||
defer rows.deinit();
|
||||
const row = try rows.byUrl(url);
|
||||
try testing.expectEqual(abp_exception_domains, row.domain_count);
|
||||
try testing.expectEqual(abp_exception_wildcards, row.wildcard_count);
|
||||
try testing.expectEqual(abp_exception_exceptions, row.exception_count);
|
||||
|
||||
const status = try env.status(id);
|
||||
try testing.expectEqual(manager.State.ok, status.state);
|
||||
try testing.expect(status.loaded);
|
||||
try testing.expectEqual(@as(u32, @intCast(abp_exception_exceptions)), status.counts.exceptions);
|
||||
|
||||
// The blocks the list makes still land, apex and subdomain alike.
|
||||
for ([_][]const u8{ "ads.example", "x.ads.example", "paid.ads.example", "partial.ads.example" }) |blocked| {
|
||||
const decision, _ = try env.evaluate(blocked);
|
||||
try testing.expect(decision.blocked);
|
||||
}
|
||||
|
||||
// The two exceptions lift the excepted name and everything under it.
|
||||
for ([_][]const u8{
|
||||
"good.ads.example",
|
||||
"y.good.ads.example",
|
||||
"fine.tracker.example",
|
||||
"z.fine.tracker.example",
|
||||
}) |lifted| {
|
||||
const decision, _ = try env.evaluate(lifted);
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(matcher.Reason.blocklist_exception, decision.reason);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
// What `/api/lookup` answers, through the same function the handler calls:
|
||||
// the reason names the exception and the source id names the list.
|
||||
{
|
||||
const handle = env.mgr.acquire(io) orelse return error.TestNoSnapshot;
|
||||
defer handle.release(io);
|
||||
const group_index = handle.snapshot.groupIndexByName("default") orelse
|
||||
return error.TestGroupMissing;
|
||||
|
||||
const result = lookup.evaluate(
|
||||
handle.snapshot,
|
||||
group_index,
|
||||
"y.good.ads.example",
|
||||
&empty_records,
|
||||
&empty_zones,
|
||||
);
|
||||
try testing.expect(!result.blocked);
|
||||
try testing.expectEqual(matcher.Reason.blocklist_exception, result.reason);
|
||||
try testing.expectEqualStrings("good.ads.example", result.matched);
|
||||
try testing.expectEqual(@as(?i64, id), result.source_id);
|
||||
|
||||
const rendered = lookup.body("y.good.ads.example", result, url);
|
||||
try testing.expectEqualStrings("blocklist_exception", rendered.reason);
|
||||
try testing.expectEqualStrings(url, rendered.source_url.?);
|
||||
}
|
||||
|
||||
// An operator block rule outranks the list's exception: a downloaded list
|
||||
// may cancel what a list decided and never what the operator decided.
|
||||
const group_id = (try groups_repo.groupId(&env.database, "default")) orelse
|
||||
return error.TestGroupMissing;
|
||||
_ = try rules_repo.insertRuleRow(&env.database, .{
|
||||
.group_id = group_id,
|
||||
.pattern = "good.ads.example",
|
||||
.kind = .exact,
|
||||
.action = .block,
|
||||
}, 1_700_000_000);
|
||||
try env.mgr.reload(io);
|
||||
{
|
||||
const decision, _ = try env.evaluate("good.ads.example");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expectEqual(matcher.Reason.rule_block_exact, decision.reason);
|
||||
}
|
||||
}
|
||||
|
||||
test "20: a data directory written before exceptions existed loads with no checksum mismatch" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
const id = try seedSource(&env.database, source_url);
|
||||
|
||||
// Exactly what an older nxdns left behind: two compiled files, no `.allow`
|
||||
// file, and a checksum taken over the two bodies alone.
|
||||
const list_body = "aaa.example.com\nbbb.example.com\n";
|
||||
const wild_body = "ccc.example.com\n";
|
||||
|
||||
var dir = try env.blocklistDir();
|
||||
defer dir.close(io);
|
||||
var list_name_buf: [64]u8 = undefined;
|
||||
var wild_name_buf: [64]u8 = undefined;
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&list_name_buf, "{d}.list", .{id}),
|
||||
.data = list_body,
|
||||
});
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&wild_name_buf, "{d}.wild", .{id}),
|
||||
.data = wild_body,
|
||||
});
|
||||
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(list_body);
|
||||
hasher.update(wild_body);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
const legacy_checksum = std.fmt.bytesToHex(digest, .lower);
|
||||
|
||||
try sources_repo.updateSourceStats(&env.database, id, .{
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 2,
|
||||
.wildcard_count = 1,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &legacy_checksum,
|
||||
});
|
||||
|
||||
try env.mgr.reload(io);
|
||||
|
||||
const status = try env.status(id);
|
||||
try testing.expectEqual(manager.State.ok, status.state);
|
||||
try testing.expect(status.loaded);
|
||||
try testing.expectEqualStrings("", status.errorText());
|
||||
|
||||
const decision, _ = try env.evaluate("aaa.example.com");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expect((try env.evaluate("x.ccc.example.com"))[0].blocked);
|
||||
}
|
||||
|
||||
+219
-69
@@ -38,9 +38,10 @@
|
||||
//! publish: the download of one source, at up to 300 s each, and the compile
|
||||
//! that follows it. It also covers blocklist-directory maintenance, because
|
||||
//! those stages are the only writers of `.raw.tmp` / `.list.tmp` /
|
||||
//! `.wild.tmp` and `pruneOrphans` must not sweep the temporaries of a refresh
|
||||
//! that is still running. Two concurrent refreshes would share the fetcher's
|
||||
//! buffers and, for one source, the same temporary paths.
|
||||
//! `.wild.tmp` / `.allow.tmp` and `pruneOrphans` must not sweep the
|
||||
//! temporaries of a refresh that is still running. Two concurrent refreshes
|
||||
//! would share the fetcher's buffers and, for one source, the same temporary
|
||||
//! paths.
|
||||
//!
|
||||
//! **Lock ordering: `refresh_lock` is never acquired while `writer_lock` is
|
||||
//! held.** A path that needs both takes `refresh_lock` first. The public entry
|
||||
@@ -94,7 +95,7 @@ const io_buf_len: usize = 64 * 1024;
|
||||
/// would then be decided by almost no data.
|
||||
const sample_buf_len: usize = parsers.sample_lines * (compiler.max_line_len + 1);
|
||||
|
||||
/// `<id>` is at most 20 characters and the longest suffix is `.list.tmp`.
|
||||
/// `<id>` is at most 20 characters and the longest suffix is `.allow.tmp`.
|
||||
const name_buf_len: usize = 48;
|
||||
|
||||
/// How one blocklist source is named in a log line: by its row id and its name,
|
||||
@@ -215,9 +216,13 @@ pub const SourceStatus = struct {
|
||||
};
|
||||
|
||||
/// The header every compiled file carries, ahead of the body. The `sha256`
|
||||
/// covers the `.list` body followed by the `.wild` body and **not** the header,
|
||||
/// so it stays stable across a refetch of unchanged content while
|
||||
/// `fetched_at` moves.
|
||||
/// covers the `.list` body, then the `.wild` body, then the `.allow` body, and
|
||||
/// **not** the header, so it stays stable across a refetch of unchanged content
|
||||
/// while `fetched_at` moves.
|
||||
///
|
||||
/// The `.allow` body is hashed last so that a source with no exceptions keeps
|
||||
/// the digest it had when only two bodies existed: every checksum written before
|
||||
/// exceptions were honoured stays valid, and no upgrade forces a refetch.
|
||||
pub const Header = struct {
|
||||
url: []const u8,
|
||||
format: parsers.Format,
|
||||
@@ -233,6 +238,7 @@ pub const Header = struct {
|
||||
try w.print("# fetched_at {d}\n", .{self.fetched_at});
|
||||
try w.print("# domains {d}\n", .{self.counts.domains});
|
||||
try w.print("# wildcards {d}\n", .{self.counts.wildcards});
|
||||
try w.print("# exceptions {d}\n", .{self.counts.exceptions});
|
||||
try w.print("# skipped_regex {d}\n", .{self.counts.skipped_regex});
|
||||
try w.print("# skipped_unsupported {d}\n", .{self.counts.skipped_unsupported});
|
||||
try w.print("# invalid {d}\n", .{self.counts.invalid});
|
||||
@@ -557,12 +563,14 @@ pub const Manager = struct {
|
||||
|
||||
var list_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_buf: [name_buf_len]u8 = undefined;
|
||||
var allow_buf: [name_buf_len]u8 = undefined;
|
||||
const list_name = compiledName(&list_buf, row.id, ".list");
|
||||
const wild_name = compiledName(&wild_buf, row.id, ".wild");
|
||||
const allow_name = compiledName(&allow_buf, row.id, ".allow");
|
||||
|
||||
// Reserved before the reads, so neither buffer can be orphaned by a
|
||||
// failing append: `bodies` owns each one from the moment it is read.
|
||||
try bodies.ensureUnusedCapacity(self.gpa, 2);
|
||||
// Reserved before the reads, so no buffer can be orphaned by a failing
|
||||
// append: `bodies` owns each one from the moment it is read.
|
||||
try bodies.ensureUnusedCapacity(self.gpa, 3);
|
||||
|
||||
// `error.Canceled` is the one-shot signal that this task is being torn
|
||||
// down, and it is consumed by whoever catches it. Recording it as a
|
||||
@@ -583,18 +591,39 @@ pub const Manager = struct {
|
||||
};
|
||||
bodies.appendAssumeCapacity(wild_bytes);
|
||||
|
||||
// A missing `.allow` file is an empty allow body, not a failure. Two
|
||||
// sources are in that state and both are ordinary: one compiled before
|
||||
// exceptions were honoured, and one whose list carries no `@@` line.
|
||||
// Because the empty body contributes nothing to the checksum, the
|
||||
// stored digest of either still matches.
|
||||
const allow_bytes: []const u8 = blk: {
|
||||
const read = dir.readFileAlloc(io, allow_name, self.gpa, .limited(max_compiled_bytes)) catch |err| {
|
||||
if (err == error.OutOfMemory) return error.OutOfMemory;
|
||||
if (err == error.Canceled) return error.Canceled;
|
||||
if (err == error.FileNotFound) break :blk "";
|
||||
return loadFailure(row, allow_name, err);
|
||||
};
|
||||
bodies.appendAssumeCapacity(read);
|
||||
break :blk read;
|
||||
};
|
||||
|
||||
const list_body = stripHeader(list_bytes);
|
||||
const wild_body = stripHeader(wild_bytes);
|
||||
const allow_body = stripHeader(allow_bytes);
|
||||
|
||||
// The checksum covers both bodies together, so a crash between the two
|
||||
// The checksum covers the three bodies together, so a crash between the
|
||||
// `replace` calls — a new `.list` beside an old `.wild` — is caught
|
||||
// here and refreshed, not served as a half-updated list.
|
||||
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body))) {
|
||||
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body, allow_body))) {
|
||||
log.warn("blocklist {f}: compiled files do not match the stored checksum", .{SourceLabel.of(row)});
|
||||
return .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } };
|
||||
}
|
||||
|
||||
return .{ .loaded = .{ .list_body = list_body, .wild_body = wild_body } };
|
||||
return .{ .loaded = .{
|
||||
.list_body = list_body,
|
||||
.wild_body = wild_body,
|
||||
.allow_body = allow_body,
|
||||
} };
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -635,23 +664,28 @@ pub const Manager = struct {
|
||||
var raw_buf: [name_buf_len]u8 = undefined;
|
||||
var list_tmp_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_tmp_buf: [name_buf_len]u8 = undefined;
|
||||
var allow_tmp_buf: [name_buf_len]u8 = undefined;
|
||||
const raw_name = compiledName(&raw_buf, row.id, ".raw.tmp");
|
||||
const list_tmp = compiledName(&list_tmp_buf, row.id, ".list.tmp");
|
||||
const wild_tmp = compiledName(&wild_tmp_buf, row.id, ".wild.tmp");
|
||||
const tmp: TempNames = .{
|
||||
.list = compiledName(&list_tmp_buf, row.id, ".list.tmp"),
|
||||
.wild = compiledName(&wild_tmp_buf, row.id, ".wild.tmp"),
|
||||
.allow = compiledName(&allow_tmp_buf, row.id, ".allow.tmp"),
|
||||
};
|
||||
|
||||
// Installed before the calls that create these files, not after: an
|
||||
// `error.Canceled` or `error.OutOfMemory` returned straight out of
|
||||
// `download` or `compileTo` would outrun a later `defer` and leave a
|
||||
// temporary behind. Deleting a name that was never created is a no-op.
|
||||
defer self.deleteQuietly(io, dir, raw_name);
|
||||
defer self.deleteQuietly(io, dir, list_tmp);
|
||||
defer self.deleteQuietly(io, dir, wild_tmp);
|
||||
defer self.deleteQuietly(io, dir, tmp.list);
|
||||
defer self.deleteQuietly(io, dir, tmp.wild);
|
||||
defer self.deleteQuietly(io, dir, tmp.allow);
|
||||
|
||||
// The half that takes the time: one download of up to `total_budget`
|
||||
// and one compile of everything it returned. `refresh_lock` alone is
|
||||
// held here, so a rule save, a settings change or any other web
|
||||
// mutation that ends in `reload` runs beside it instead of behind it.
|
||||
const prepared = try self.prepareRefresh(io, dir, row, &status, raw_name, list_tmp, wild_tmp);
|
||||
const prepared = try self.prepareRefresh(io, dir, row, &status, raw_name, tmp);
|
||||
|
||||
// The half that publishes. The compiled files, the runtime columns and
|
||||
// the status entry land under one `writer_lock`, so a reload never
|
||||
@@ -659,7 +693,7 @@ pub const Manager = struct {
|
||||
self.writer_lock.lockUncancelable(io);
|
||||
defer self.writer_lock.unlock(io);
|
||||
|
||||
const replaced = try self.publishRefresh(io, dir, row, &status, prepared, list_tmp, wild_tmp);
|
||||
const replaced = try self.publishRefresh(io, dir, row, &status, prepared, tmp);
|
||||
self.commitStatus(io, status);
|
||||
return replaced;
|
||||
}
|
||||
@@ -690,6 +724,14 @@ pub const Manager = struct {
|
||||
return self.reload(io);
|
||||
}
|
||||
|
||||
/// The three temporary files one refresh compiles into, before the header
|
||||
/// is prepended and each is renamed over the file it replaces.
|
||||
const TempNames = struct {
|
||||
list: []const u8,
|
||||
wild: []const u8,
|
||||
allow: []const u8,
|
||||
};
|
||||
|
||||
/// What the fetch-and-compile half of a refresh produced. `.failed` needs
|
||||
/// no publish and has already recorded why in the status entry.
|
||||
const Prepared = union(enum) {
|
||||
@@ -712,8 +754,7 @@ pub const Manager = struct {
|
||||
row: sources_repo.SourceRow,
|
||||
status: *SourceStatus,
|
||||
raw_name: []const u8,
|
||||
list_tmp: []const u8,
|
||||
wild_tmp: []const u8,
|
||||
tmp: TempNames,
|
||||
) Error!Prepared {
|
||||
self.download(io, dir, raw_name, row) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
@@ -733,7 +774,7 @@ pub const Manager = struct {
|
||||
},
|
||||
};
|
||||
|
||||
const result = self.compileTo(io, dir, raw_name, format, list_tmp, wild_tmp) catch |err| switch (err) {
|
||||
const result = self.compileTo(io, dir, raw_name, format, tmp) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
error.Canceled => return error.Canceled,
|
||||
else => {
|
||||
@@ -762,8 +803,7 @@ pub const Manager = struct {
|
||||
row: sources_repo.SourceRow,
|
||||
status: *SourceStatus,
|
||||
prepared: Prepared,
|
||||
list_tmp: []const u8,
|
||||
wild_tmp: []const u8,
|
||||
tmp: TempNames,
|
||||
) Error!bool {
|
||||
const compiled = switch (prepared) {
|
||||
.failed => return false,
|
||||
@@ -786,6 +826,7 @@ pub const Manager = struct {
|
||||
.last_updated = now,
|
||||
.domain_count = row.domain_count,
|
||||
.wildcard_count = row.wildcard_count,
|
||||
.exception_count = row.exception_count,
|
||||
.skipped_regex_count = row.skipped_regex_count,
|
||||
.checksum = stored,
|
||||
});
|
||||
@@ -801,7 +842,7 @@ pub const Manager = struct {
|
||||
.counts = compiled.result.counts,
|
||||
.checksum = &compiled.result.checksum,
|
||||
};
|
||||
self.publish(io, dir, row.id, header, list_tmp, wild_tmp) catch |err| switch (err) {
|
||||
self.publish(io, dir, row.id, header, tmp) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
error.Canceled => return error.Canceled,
|
||||
else => {
|
||||
@@ -814,6 +855,7 @@ pub const Manager = struct {
|
||||
.last_updated = now,
|
||||
.domain_count = compiled.result.counts.domains,
|
||||
.wildcard_count = compiled.result.counts.wildcards,
|
||||
.exception_count = compiled.result.counts.exceptions,
|
||||
.skipped_regex_count = compiled.result.counts.skipped_regex,
|
||||
.checksum = &compiled.result.checksum,
|
||||
});
|
||||
@@ -913,7 +955,7 @@ pub const Manager = struct {
|
||||
return parsers.detectFormat(sample.buffered());
|
||||
}
|
||||
|
||||
/// Compiles into two plain temporary files. The compiled bodies cannot go
|
||||
/// Compiles into three plain temporary files. The compiled bodies cannot go
|
||||
/// straight into the final files: the header carries counts that only exist
|
||||
/// once the whole input has been compiled, and the loader requires the
|
||||
/// header first.
|
||||
@@ -923,22 +965,24 @@ pub const Manager = struct {
|
||||
dir: std.Io.Dir,
|
||||
raw_name: []const u8,
|
||||
format: parsers.Format,
|
||||
list_tmp: []const u8,
|
||||
wild_tmp: []const u8,
|
||||
tmp: TempNames,
|
||||
) !compiler.Result {
|
||||
const raw = try dir.openFile(io, raw_name, .{});
|
||||
defer raw.close(io);
|
||||
const list_file = try dir.createFile(io, list_tmp, .{ .permissions = .fromMode(0o600) });
|
||||
const list_file = try dir.createFile(io, tmp.list, .{ .permissions = .fromMode(0o600) });
|
||||
defer list_file.close(io);
|
||||
const wild_file = try dir.createFile(io, wild_tmp, .{ .permissions = .fromMode(0o600) });
|
||||
const wild_file = try dir.createFile(io, tmp.wild, .{ .permissions = .fromMode(0o600) });
|
||||
defer wild_file.close(io);
|
||||
const allow_file = try dir.createFile(io, tmp.allow, .{ .permissions = .fromMode(0o600) });
|
||||
defer allow_file.close(io);
|
||||
|
||||
const buffers = try self.gpa.alloc(u8, 3 * io_buf_len);
|
||||
const buffers = try self.gpa.alloc(u8, 4 * io_buf_len);
|
||||
defer self.gpa.free(buffers);
|
||||
|
||||
var fr = raw.reader(io, buffers[0..io_buf_len]);
|
||||
var list_w = list_file.writer(io, buffers[io_buf_len .. 2 * io_buf_len]);
|
||||
var wild_w = wild_file.writer(io, buffers[2 * io_buf_len ..]);
|
||||
var wild_w = wild_file.writer(io, buffers[2 * io_buf_len .. 3 * io_buf_len]);
|
||||
var allow_w = allow_file.writer(io, buffers[3 * io_buf_len ..]);
|
||||
|
||||
const result = compiler.compile(
|
||||
self.gpa,
|
||||
@@ -946,18 +990,22 @@ pub const Manager = struct {
|
||||
format,
|
||||
&list_w.interface,
|
||||
&wild_w.interface,
|
||||
&allow_w.interface,
|
||||
) catch |err| switch (err) {
|
||||
// `compiler.Error` names the direction; the concrete cause is on
|
||||
// the stream that failed.
|
||||
error.ReadFailed => return fr.err orelse err,
|
||||
error.WriteFailed => return list_w.err orelse (wild_w.err orelse err),
|
||||
error.WriteFailed => return list_w.err orelse
|
||||
(wild_w.err orelse (allow_w.err orelse err)),
|
||||
else => return err,
|
||||
};
|
||||
|
||||
try list_w.interface.flush();
|
||||
try wild_w.interface.flush();
|
||||
try allow_w.interface.flush();
|
||||
try list_file.sync(io);
|
||||
try wild_file.sync(io);
|
||||
try allow_file.sync(io);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -970,16 +1018,17 @@ pub const Manager = struct {
|
||||
dir: std.Io.Dir,
|
||||
id: i64,
|
||||
header: Header,
|
||||
list_tmp: []const u8,
|
||||
wild_tmp: []const u8,
|
||||
tmp: TempNames,
|
||||
) !void {
|
||||
const buffers = try self.gpa.alloc(u8, 2 * io_buf_len);
|
||||
defer self.gpa.free(buffers);
|
||||
|
||||
var list_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_buf: [name_buf_len]u8 = undefined;
|
||||
try publishOne(io, dir, compiledName(&list_buf, id, ".list"), list_tmp, header, buffers);
|
||||
try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), wild_tmp, header, buffers);
|
||||
var allow_buf: [name_buf_len]u8 = undefined;
|
||||
try publishOne(io, dir, compiledName(&list_buf, id, ".list"), tmp.list, header, buffers);
|
||||
try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), tmp.wild, header, buffers);
|
||||
try publishOne(io, dir, compiledName(&allow_buf, id, ".allow"), tmp.allow, header, buffers);
|
||||
}
|
||||
|
||||
fn publishOne(
|
||||
@@ -1014,12 +1063,19 @@ pub const Manager = struct {
|
||||
try af.replace(io);
|
||||
}
|
||||
|
||||
/// Whether the two compiled files on disk hash to `expected`. A missing,
|
||||
/// Whether the compiled files on disk hash to `expected`. A missing,
|
||||
/// unreadable or corrupt file answers false, which sends the caller down
|
||||
/// the rewrite path — the only path that can repair it.
|
||||
///
|
||||
/// A missing `.allow` file is the one exception, and it is the same one
|
||||
/// `loadSource` makes: a source compiled before exceptions were honoured has
|
||||
/// no such file, and its stored checksum was taken over an empty allow body.
|
||||
/// Answering false there would rewrite every list on the first refresh after
|
||||
/// an upgrade for no change in content.
|
||||
fn diskBodiesMatch(self: *Manager, io: std.Io, dir: std.Io.Dir, id: i64, expected: []const u8) bool {
|
||||
var list_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_buf: [name_buf_len]u8 = undefined;
|
||||
var allow_buf: [name_buf_len]u8 = undefined;
|
||||
const limit: std.Io.Limit = .limited(max_compiled_bytes);
|
||||
|
||||
const list_bytes = dir.readFileAlloc(io, compiledName(&list_buf, id, ".list"), self.gpa, limit) catch
|
||||
@@ -1029,7 +1085,11 @@ pub const Manager = struct {
|
||||
return false;
|
||||
defer self.gpa.free(wild_bytes);
|
||||
|
||||
return compiledBodiesMatch(list_bytes, wild_bytes, expected);
|
||||
const allow_bytes = dir.readFileAlloc(io, compiledName(&allow_buf, id, ".allow"), self.gpa, limit) catch |err|
|
||||
if (err == error.FileNotFound) @as([]u8, &.{}) else return false;
|
||||
defer self.gpa.free(allow_bytes);
|
||||
|
||||
return compiledBodiesMatch(list_bytes, wild_bytes, allow_bytes, expected);
|
||||
}
|
||||
|
||||
fn reportFetchFailure(
|
||||
@@ -1231,12 +1291,12 @@ pub const Manager = struct {
|
||||
/// sweeps to nothing.
|
||||
pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
|
||||
// `refresh_lock` first, and for the reason it exists: the download and
|
||||
// the compile are the only writers of `.raw.tmp`, `.list.tmp` and
|
||||
// `.wild.tmp`, and they hold it for as long as they run. Without it
|
||||
// here, a source deleted through the API would sweep the temporaries of
|
||||
// a refresh still writing them — the row is gone, so nothing else in
|
||||
// this function would spare them — and the pass would fail on a raw
|
||||
// file that vanished under it.
|
||||
// the compile are the only writers of `.raw.tmp`, `.list.tmp`,
|
||||
// `.wild.tmp` and `.allow.tmp`, and they hold it for as long as they
|
||||
// run. Without it here, a source deleted through the API would sweep
|
||||
// the temporaries of a refresh still writing them — the row is gone, so
|
||||
// nothing else in this function would spare them — and the pass would
|
||||
// fail on a raw file that vanished under it.
|
||||
//
|
||||
// `writer_lock` second, in the one order this file ever takes them,
|
||||
// because the rows this reads and the compiled files it deletes are
|
||||
@@ -1485,6 +1545,7 @@ fn applyLoadOutcomes(
|
||||
entry.succeed(row.last_updated orelse 0, .{
|
||||
.domains = countOf(row.domain_count),
|
||||
.wildcards = countOf(row.wildcard_count),
|
||||
.exceptions = countOf(row.exception_count),
|
||||
.skipped_regex = countOf(row.skipped_regex_count),
|
||||
});
|
||||
},
|
||||
@@ -1555,43 +1616,62 @@ fn collectSample(r: *std.Io.Reader, w: *std.Io.Writer) error{ ReadFailed, WriteF
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether two compiled files carry the bodies `expected` was taken over.
|
||||
fn compiledBodiesMatch(list_bytes: []const u8, wild_bytes: []const u8, expected: []const u8) bool {
|
||||
return std.mem.eql(u8, expected, &bodyChecksum(stripHeader(list_bytes), stripHeader(wild_bytes)));
|
||||
/// Whether three compiled files carry the bodies `expected` was taken over.
|
||||
fn compiledBodiesMatch(
|
||||
list_bytes: []const u8,
|
||||
wild_bytes: []const u8,
|
||||
allow_bytes: []const u8,
|
||||
expected: []const u8,
|
||||
) bool {
|
||||
return std.mem.eql(u8, expected, &bodyChecksum(
|
||||
stripHeader(list_bytes),
|
||||
stripHeader(wild_bytes),
|
||||
stripHeader(allow_bytes),
|
||||
));
|
||||
}
|
||||
|
||||
/// A compile that produced no entry at all while rejecting lines is an error
|
||||
/// page, a compressed body or a format the sniff got wrong — not a blocklist.
|
||||
/// Publishing it would replace a working list with nothing and report `ok`. An
|
||||
/// input that rejected nothing is an empty list, which is legal.
|
||||
///
|
||||
/// A list of nothing but exceptions is loadable: an allow-only list published
|
||||
/// beside a blocking one is a shape operators use, and it produces entries.
|
||||
fn rejectedWithoutEntries(counts: compiler.Counts) bool {
|
||||
if (counts.domains != 0 or counts.wildcards != 0) return false;
|
||||
if (counts.domains != 0 or counts.wildcards != 0 or counts.exceptions != 0) return false;
|
||||
return counts.invalid != 0 or counts.skipped_unsupported != 0 or counts.long_lines != 0;
|
||||
}
|
||||
|
||||
fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 {
|
||||
/// The digest the `.list`, `.wild` and `.allow` bodies share, in that order.
|
||||
/// The allow body comes last so that hashing an empty one leaves the digest of
|
||||
/// the two-body form untouched, which is what keeps every checksum stored before
|
||||
/// exceptions were honoured valid.
|
||||
fn bodyChecksum(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) [64]u8 {
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(list_body);
|
||||
hasher.update(wild_body);
|
||||
hasher.update(allow_body);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
return std.fmt.bytesToHex(digest, .lower);
|
||||
}
|
||||
|
||||
fn compiledName(buf: *[name_buf_len]u8, id: i64, suffix: []const u8) []const u8 {
|
||||
// An `i64` prints in at most 20 characters and the longest suffix is nine,
|
||||
// An `i64` prints in at most 20 characters and the longest suffix is ten,
|
||||
// so `name_buf_len` cannot be exceeded.
|
||||
return std.fmt.bufPrint(buf, "{d}{s}", .{ id, suffix }) catch unreachable;
|
||||
}
|
||||
|
||||
/// Every name `compiledName` can produce, longest suffix first so `.list.tmp`
|
||||
/// is never read as `.list`.
|
||||
const source_file_suffixes = [_][]const u8{ ".list.tmp", ".wild.tmp", ".raw.tmp", ".list", ".wild" };
|
||||
const source_file_suffixes = [_][]const u8{
|
||||
".allow.tmp", ".list.tmp", ".wild.tmp", ".raw.tmp", ".allow", ".list", ".wild",
|
||||
};
|
||||
|
||||
/// The source id a file under the blocklist directory belongs to, or null when
|
||||
/// the name is not one of ours.
|
||||
///
|
||||
/// The three temporaries count. A refresh that dies between writing one and
|
||||
/// The four temporaries count. A refresh that dies between writing one and
|
||||
/// renaming it leaves a file no later refresh reuses and no `defer` reaches, so
|
||||
/// excluding them from the sweep means nothing ever removes them. Matching them
|
||||
/// is safe because `pruneOrphans` holds `refresh_lock` for its whole body:
|
||||
@@ -1830,18 +1910,24 @@ test "a canceled compiled-file read cancels the reload instead of recording it"
|
||||
|
||||
const list_body = "aaa.example.com\n";
|
||||
const wild_body = "";
|
||||
const allow_body = "";
|
||||
var dir = try tmp.dir.createDirPathOpen(io, "blocklists", .{});
|
||||
defer dir.close(io);
|
||||
var list_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_buf: [name_buf_len]u8 = undefined;
|
||||
var allow_buf: [name_buf_len]u8 = undefined;
|
||||
try dir.writeFile(io, .{ .sub_path = compiledName(&list_buf, id, ".list"), .data = list_body });
|
||||
try dir.writeFile(io, .{ .sub_path = compiledName(&wild_buf, id, ".wild"), .data = wild_body });
|
||||
// Present rather than absent, so the third read is a real one: `loadSource`
|
||||
// treats a missing `.allow` as an empty body and would never open it.
|
||||
try dir.writeFile(io, .{ .sub_path = compiledName(&allow_buf, id, ".allow"), .data = allow_body });
|
||||
try sources_repo.updateSourceStats(&database, id, .{
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 1,
|
||||
.wildcard_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, wild_body),
|
||||
.exception_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, wild_body, allow_body),
|
||||
});
|
||||
|
||||
// The baseline every assertion below is against: one clean reload, one
|
||||
@@ -1853,11 +1939,12 @@ test "a canceled compiled-file read cancels the reload instead of recording it"
|
||||
try testing.expect(out[0].loaded);
|
||||
const published = mgr.generation;
|
||||
|
||||
// Both catch sites, in the order `loadSource` reads the two files. A
|
||||
// Every catch site, in the order `loadSource` reads the three files. A
|
||||
// cancellation is consumed by whoever catches it, so folding it into a load
|
||||
// failure would spend the shutdown signal and leave a status row reading
|
||||
// "Canceled" behind.
|
||||
for ([_][]const u8{ ".list", ".wild" }) |suffix| {
|
||||
// "Canceled" behind. The `.allow` read is the one that can get this wrong
|
||||
// twice over: it also has to keep `FileNotFound` apart from a cancellation.
|
||||
for ([_][]const u8{ ".list", ".wild", ".allow" }) |suffix| {
|
||||
var vtable: std.Io.VTable = undefined;
|
||||
const canceling = cancelingIo(io, suffix, &vtable);
|
||||
try testing.expectError(error.Canceled, mgr.reload(canceling));
|
||||
@@ -1921,6 +2008,7 @@ test "the header writer produces the documented text" {
|
||||
.counts = .{
|
||||
.domains = 12,
|
||||
.wildcards = 3,
|
||||
.exceptions = 7,
|
||||
.skipped_regex = 2,
|
||||
.skipped_unsupported = 1,
|
||||
.invalid = 5,
|
||||
@@ -1938,6 +2026,7 @@ test "the header writer produces the documented text" {
|
||||
\\# fetched_at 1700000000
|
||||
\\# domains 12
|
||||
\\# wildcards 3
|
||||
\\# exceptions 7
|
||||
\\# skipped_regex 2
|
||||
\\# skipped_unsupported 1
|
||||
\\# invalid 5
|
||||
@@ -2042,32 +2131,51 @@ test "a success clears the recorded error" {
|
||||
try testing.expectEqualStrings("", status.errorText());
|
||||
}
|
||||
|
||||
test "compiledName spells the four file names of a source" {
|
||||
comptime {
|
||||
// The two tests below spell every suffix out instead of looping over
|
||||
// `source_file_suffixes`: a test that reads the table moves with it, so a
|
||||
// name dropped from the table would take the assertion that covers it along.
|
||||
// An eighth suffix breaks the build here until both are extended.
|
||||
std.debug.assert(source_file_suffixes.len == 7);
|
||||
}
|
||||
|
||||
test "compiledName spells every file name of a source" {
|
||||
var buf: [name_buf_len]u8 = undefined;
|
||||
try testing.expectEqualStrings("42.list", compiledName(&buf, 42, ".list"));
|
||||
try testing.expectEqualStrings("42.wild", compiledName(&buf, 42, ".wild"));
|
||||
try testing.expectEqualStrings("42.allow", compiledName(&buf, 42, ".allow"));
|
||||
try testing.expectEqualStrings("42.raw.tmp", compiledName(&buf, 42, ".raw.tmp"));
|
||||
try testing.expectEqualStrings("42.list.tmp", compiledName(&buf, 42, ".list.tmp"));
|
||||
try testing.expectEqualStrings("42.wild.tmp", compiledName(&buf, 42, ".wild.tmp"));
|
||||
try testing.expectEqualStrings("42.allow.tmp", compiledName(&buf, 42, ".allow.tmp"));
|
||||
}
|
||||
|
||||
test "sourceFileId matches every name a refresh writes, including the temporaries" {
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.allow"));
|
||||
|
||||
// A temporary left by a refresh that died belongs to its source id, so the
|
||||
// sweep can tell whether that source still has a row.
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.raw.tmp"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list.tmp"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild.tmp"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.allow.tmp"));
|
||||
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.list"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.allow"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.raw.tmp"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.allow.tmp"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("7.tmp"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("7.raw"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("7.allowed"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("README"));
|
||||
}
|
||||
|
||||
test "every name compiledName writes is a name the sweep can attribute" {
|
||||
// A round-trip over the table, not a coverage check: this loop reads the
|
||||
// same array the code reads, so it cannot notice a missing entry. The two
|
||||
// tests above are what pins the set.
|
||||
var buf: [name_buf_len]u8 = undefined;
|
||||
for (source_file_suffixes) |suffix| {
|
||||
try testing.expectEqual(@as(?i64, 42), sourceFileId(compiledName(&buf, 42, suffix)));
|
||||
@@ -2113,6 +2221,7 @@ fn testRow(id: i64, enabled: bool) sources_repo.SourceRow {
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 9,
|
||||
.wildcard_count = 4,
|
||||
.exception_count = 2,
|
||||
.skipped_regex_count = 1,
|
||||
.checksum = "0" ** 64,
|
||||
};
|
||||
@@ -2260,17 +2369,49 @@ test "SourceStatus truncates a long url at max_url_len" {
|
||||
test "compiledBodiesMatch verifies the bodies, not the presence of the files" {
|
||||
const list_body = "a.example.com\nb.example.com\n";
|
||||
const wild_body = "c.example.com\n";
|
||||
const expected = bodyChecksum(list_body, wild_body);
|
||||
const allow_body = "d.example.com\n";
|
||||
const expected = bodyChecksum(list_body, wild_body, allow_body);
|
||||
|
||||
const header =
|
||||
"# nxdns blocklist\n" ++
|
||||
"# url https://lists.example/hosts.txt\n";
|
||||
try testing.expect(compiledBodiesMatch(header ++ list_body, header ++ wild_body, &expected));
|
||||
try testing.expect(compiledBodiesMatch(
|
||||
header ++ list_body,
|
||||
header ++ wild_body,
|
||||
header ++ allow_body,
|
||||
&expected,
|
||||
));
|
||||
|
||||
// The corruption a reload reports as `ChecksumMismatch`: the file is there,
|
||||
// its body is not what the checksum was taken over.
|
||||
try testing.expect(!compiledBodiesMatch(header ++ "a.example.com\nb.exa", header ++ wild_body, &expected));
|
||||
try testing.expect(!compiledBodiesMatch("", "", &expected));
|
||||
// its body is not what the checksum was taken over. An allow body that lost
|
||||
// its entry counts, because a dropped exception silently restores a block.
|
||||
try testing.expect(!compiledBodiesMatch(
|
||||
header ++ "a.example.com\nb.exa",
|
||||
header ++ wild_body,
|
||||
header ++ allow_body,
|
||||
&expected,
|
||||
));
|
||||
try testing.expect(!compiledBodiesMatch(header ++ list_body, header ++ wild_body, "", &expected));
|
||||
try testing.expect(!compiledBodiesMatch("", "", "", &expected));
|
||||
}
|
||||
|
||||
test "a source with no exceptions keeps the checksum it had before the allow body existed" {
|
||||
const list_body = "a.example.com\nb.example.com\n";
|
||||
const wild_body = "c.example.com\n";
|
||||
|
||||
// What an older nxdns stored: the digest of the two bodies alone. It is what
|
||||
// sits in `blocklist_sources.checksum` on every installation being upgraded,
|
||||
// and the files on disk are the two it was taken over.
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(list_body);
|
||||
hasher.update(wild_body);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
const stored = std.fmt.bytesToHex(digest, .lower);
|
||||
|
||||
try testing.expectEqualStrings(&stored, &bodyChecksum(list_body, wild_body, ""));
|
||||
// No `.allow` file: what `loadSource` and `diskBodiesMatch` pass for one.
|
||||
try testing.expect(compiledBodiesMatch(list_body, wild_body, "", &stored));
|
||||
}
|
||||
|
||||
test "rejectedWithoutEntries fails a compile that produced nothing usable" {
|
||||
@@ -2391,14 +2532,23 @@ test "collectSample steps over a line that does not fit the reader buffer" {
|
||||
try testing.expectEqualStrings("ads.example.com\n", w.buffered());
|
||||
}
|
||||
|
||||
test "bodyChecksum covers the list body followed by the wild body" {
|
||||
const both = bodyChecksum("a.example.com\n", "b.example.com\n");
|
||||
test "bodyChecksum covers the list body, then the wild body, then the allow body" {
|
||||
const all = bodyChecksum("a.example.com\n", "b.example.com\n", "c.example.com\n");
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update("a.example.com\nb.example.com\n");
|
||||
hasher.update("a.example.com\nb.example.com\nc.example.com\n");
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &both);
|
||||
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &all);
|
||||
|
||||
// Order matters: the two halves are not interchangeable.
|
||||
try testing.expect(!std.mem.eql(u8, &both, &bodyChecksum("b.example.com\n", "a.example.com\n")));
|
||||
// Order matters: the three parts are not interchangeable.
|
||||
try testing.expect(!std.mem.eql(
|
||||
u8,
|
||||
&all,
|
||||
&bodyChecksum("b.example.com\n", "a.example.com\n", "c.example.com\n"),
|
||||
));
|
||||
try testing.expect(!std.mem.eql(
|
||||
u8,
|
||||
&all,
|
||||
&bodyChecksum("a.example.com\n", "c.example.com\n", "b.example.com\n"),
|
||||
));
|
||||
}
|
||||
|
||||
+348
-11
@@ -27,6 +27,11 @@ pub const Reason = enum {
|
||||
rule_block_exact,
|
||||
rule_allow_wildcard,
|
||||
rule_block_wildcard,
|
||||
rule_allow_regex,
|
||||
rule_block_regex,
|
||||
/// An `@@` exception from a downloaded list. It cancels what another list
|
||||
/// blocks and never what a rule decides — see `evaluate`.
|
||||
blocklist_exception,
|
||||
blocklist_domain,
|
||||
blocklist_wildcard,
|
||||
};
|
||||
@@ -34,12 +39,14 @@ pub const Reason = enum {
|
||||
pub const Decision = struct {
|
||||
blocked: bool,
|
||||
reason: Reason,
|
||||
/// The candidate (for the exact and blocklist levels) or the pattern (for
|
||||
/// the wildcard levels) that decided it. Borrowed from the caller's
|
||||
/// normalized buffer or from the snapshot. "" when `reason == .none`.
|
||||
/// The candidate (for the exact, exception and blocklist levels) or the
|
||||
/// pattern (for the wildcard and regex levels) that decided it. Borrowed
|
||||
/// from the caller's normalized buffer or from the snapshot. "" when
|
||||
/// `reason == .none`.
|
||||
matched: []const u8,
|
||||
/// `.blocklist_*` only: index into `Snapshot.sources`, so the query log and
|
||||
/// the UI can name the list that blocked the query.
|
||||
/// the UI can name the list that blocked the query — or, for
|
||||
/// `.blocklist_exception`, the list that lifted it.
|
||||
source: ?u32 = null,
|
||||
};
|
||||
|
||||
@@ -95,6 +102,10 @@ pub const SourceSets = struct {
|
||||
name: []const u8,
|
||||
domains: domain_set.DomainSet,
|
||||
wildcards: domain_set.DomainSet,
|
||||
/// The names this source's `@@` exceptions lift. One entry covers the name
|
||||
/// and every subdomain of it, because `evaluate` walks the full name and
|
||||
/// each parent against this set.
|
||||
exceptions: domain_set.DomainSet,
|
||||
};
|
||||
|
||||
pub const Group = struct {
|
||||
@@ -122,7 +133,15 @@ pub const Snapshot = struct {
|
||||
/// can tell which generation answered a query.
|
||||
generation: u64,
|
||||
|
||||
pub const Compiled = struct { list_body: []const u8, wild_body: []const u8 };
|
||||
/// The compiled bodies of one source. `allow_body` defaults to empty
|
||||
/// because a source compiled before exceptions were honoured has no
|
||||
/// `.allow` file at all. A source refreshed since then always has one,
|
||||
/// empty when its list carries no `@@` line.
|
||||
pub const Compiled = struct {
|
||||
list_body: []const u8,
|
||||
wild_body: []const u8,
|
||||
allow_body: []const u8 = "",
|
||||
};
|
||||
|
||||
pub const Input = struct {
|
||||
groups: []const model.Group,
|
||||
@@ -192,6 +211,7 @@ pub const Snapshot = struct {
|
||||
.name = try arena.dupe(u8, row.name),
|
||||
.domains = try domain_set.DomainSet.build(arena, bodies.list_body, input.seed),
|
||||
.wildcards = try domain_set.DomainSet.build(arena, bodies.wild_body, input.seed),
|
||||
.exceptions = try domain_set.DomainSet.build(arena, bodies.allow_body, input.seed),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -221,7 +241,11 @@ pub const Snapshot = struct {
|
||||
.id = id,
|
||||
.name = try arena.dupe(u8, row.name),
|
||||
.safe_search = row.safe_search,
|
||||
.rules = try rules.RuleSet.build(arena, group_rules.items, input.seed),
|
||||
// `arena` retains, `gpa` scratches: an arena reclaims only its
|
||||
// most recent allocation, so a rule build's temporaries taken
|
||||
// from it would outlive the build and go unreported by
|
||||
// `memoryBytes`.
|
||||
.rules = try rules.RuleSet.build(arena, gpa, group_rules.items, input.seed),
|
||||
.sources = try arena.dupe(u32, dedupSorted(group_sources.items)),
|
||||
};
|
||||
}
|
||||
@@ -269,7 +293,9 @@ pub const Snapshot = struct {
|
||||
/// PLAN §3.10 precedence, allow winning at equal specificity:
|
||||
/// 1. exact/parent allow rules 2. exact/parent block rules
|
||||
/// 3. wildcard allow rules 4. wildcard block rules
|
||||
/// 5. blocklist domains 6. blocklist wildcards
|
||||
/// 5. regex allow rules 6. regex block rules
|
||||
/// 7. blocklist exceptions
|
||||
/// 8. blocklist domains 9. blocklist wildcards
|
||||
///
|
||||
/// The order is level-by-level over the whole candidate chain, not
|
||||
/// candidate-by-candidate over the levels: level 1 is checked against every
|
||||
@@ -277,10 +303,24 @@ pub const Snapshot = struct {
|
||||
/// allow rule on the parent beat a block rule on the child, which is the
|
||||
/// behaviour an allow list is written for.
|
||||
///
|
||||
/// Level 5 tests only the full name and level 6 tests only proper parents:
|
||||
/// Levels 5 and 6 are last among the operator rules because they are the
|
||||
/// only ones that cost more than a hash lookup or a label walk: a regex is
|
||||
/// reached only once every set-shaped level has missed. They are matched
|
||||
/// against the full name alone — a pattern that should cover subdomains
|
||||
/// says so, which is what an unanchored regex already does.
|
||||
///
|
||||
/// Level 7 is where a downloaded list's `@@` exceptions are honoured, and
|
||||
/// its position is the whole safety argument: every operator rule has
|
||||
/// already returned by the time it runs, so an exception can cancel a block
|
||||
/// levels 8 and 9 would have made and nothing else. No downloaded list can
|
||||
/// open an allow hole the operator did not open. It walks the full name and
|
||||
/// every parent, because one `@@||x^` entry lifts `x` together with its
|
||||
/// subdomains.
|
||||
///
|
||||
/// Level 8 tests only the full name and level 9 tests only proper parents:
|
||||
/// a `.list` entry is the domain itself, a `.wild` entry is what `*.x.y`
|
||||
/// means. Both walk the group's sources in ascending index order, so the
|
||||
/// reported source is stable for a given snapshot.
|
||||
/// means. All three list levels walk the group's sources in ascending index
|
||||
/// order, so the reported source is stable for a given snapshot.
|
||||
///
|
||||
/// `domain` is normalized (`normalize`). No allocation, no lock, no clock.
|
||||
pub fn evaluate(self: *const Snapshot, group: u32, domain: []const u8) Decision {
|
||||
@@ -307,6 +347,27 @@ pub const Snapshot = struct {
|
||||
return .{ .blocked = true, .reason = .rule_block_wildcard, .matched = pattern };
|
||||
}
|
||||
|
||||
if (rules.matchRegex(g.rules.regex_allow, domain)) |pattern| {
|
||||
return .{ .blocked = false, .reason = .rule_allow_regex, .matched = pattern };
|
||||
}
|
||||
if (rules.matchRegex(g.rules.regex_block, domain)) |pattern| {
|
||||
return .{ .blocked = true, .reason = .rule_block_regex, .matched = pattern };
|
||||
}
|
||||
|
||||
var exceptions: Candidates = .init(domain);
|
||||
while (exceptions.next()) |candidate| {
|
||||
for (g.sources) |index| {
|
||||
if (self.sources[index].exceptions.contains(candidate)) {
|
||||
return .{
|
||||
.blocked = false,
|
||||
.reason = .blocklist_exception,
|
||||
.matched = candidate,
|
||||
.source = index,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (g.sources) |index| {
|
||||
if (self.sources[index].domains.contains(domain)) {
|
||||
return .{
|
||||
@@ -373,7 +434,8 @@ pub const Snapshot = struct {
|
||||
var total: usize = 0;
|
||||
for (self.sources) |*source| {
|
||||
total += @sizeOf(SourceSets) + source.name.len +
|
||||
source.domains.memoryBytes() + source.wildcards.memoryBytes();
|
||||
source.domains.memoryBytes() + source.wildcards.memoryBytes() +
|
||||
source.exceptions.memoryBytes();
|
||||
}
|
||||
for (self.groups) |*group| {
|
||||
total += @sizeOf(Group) + group.name.len +
|
||||
@@ -586,6 +648,71 @@ test "precedence: a block wildcard with no allow blocks" {
|
||||
try testing.expectEqualStrings("*.example.com", decision.matched);
|
||||
}
|
||||
|
||||
test "precedence: a block regex with no allow blocks, and reports its pattern" {
|
||||
const rows = [_]model.Rule{rule("^ad[0-9]+-", .regex, .block)};
|
||||
var snapshot = try build(testing.allocator, .{ .rules = &rows });
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "ad42-tracker.example.com");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_block_regex, decision.reason);
|
||||
// `matched` is the pattern the operator wrote, which is what the query log
|
||||
// has to name for the block to be explicable.
|
||||
try testing.expectEqualStrings("^ad[0-9]+-", decision.matched);
|
||||
|
||||
// Unanchored at the tail, anchored at the head: the digits must lead.
|
||||
try testing.expect(!snapshot.evaluate(0, "x.ad42-tracker.example.com").blocked);
|
||||
try testing.expect(!snapshot.evaluate(0, "ads.example.com").blocked);
|
||||
}
|
||||
|
||||
test "precedence: an allow regex beats a block regex that matches the same name" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("tracker", .regex, .block),
|
||||
rule("^good\\.", .regex, .allow),
|
||||
};
|
||||
var snapshot = try build(testing.allocator, .{ .rules = &rows });
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.tracker.example.com");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_allow_regex, decision.reason);
|
||||
try testing.expectEqualStrings("^good\\.", decision.matched);
|
||||
|
||||
try testing.expect(snapshot.evaluate(0, "bad.tracker.example.com").blocked);
|
||||
}
|
||||
|
||||
test "precedence: both wildcard levels beat an allow regex that matches" {
|
||||
// The adjacent pair either side of the wildcard/regex boundary. A regex is
|
||||
// the most expensive level and therefore the last operator level, so a
|
||||
// wildcard decides first whichever way it decides.
|
||||
for ([_]model.Rule{
|
||||
rule("*.example.com", .wildcard, .allow),
|
||||
rule("*.example.com", .wildcard, .block),
|
||||
}) |wild| {
|
||||
const rows = [_]model.Rule{ wild, rule("example", .regex, .allow) };
|
||||
var snapshot = try build(testing.allocator, .{ .rules = &rows });
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "a.example.com");
|
||||
try testing.expectEqual(wild.action == .block, decision.blocked);
|
||||
try testing.expect(decision.reason == .rule_allow_wildcard or
|
||||
decision.reason == .rule_block_wildcard);
|
||||
}
|
||||
}
|
||||
|
||||
test "precedence: an exact allow rule beats a block regex" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("tracker", .regex, .block),
|
||||
rule("good.tracker.example.com", .exact, .allow),
|
||||
};
|
||||
var snapshot = try build(testing.allocator, .{ .rules = &rows });
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.tracker.example.com");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_allow_exact, decision.reason);
|
||||
}
|
||||
|
||||
const one_source = [_]model.BlocklistSource{.{ .url = "https://lists.test/a", .name = "list a" }};
|
||||
const one_source_id = [_]i64{11};
|
||||
const one_link = [_]model.GroupSource{
|
||||
@@ -601,6 +728,14 @@ const Lists = struct {
|
||||
return .{ .compiled = .{.{ .list_body = list_body, .wild_body = wild_body }} };
|
||||
}
|
||||
|
||||
fn initWithExceptions(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) Lists {
|
||||
return .{ .compiled = .{.{
|
||||
.list_body = list_body,
|
||||
.wild_body = wild_body,
|
||||
.allow_body = allow_body,
|
||||
}} };
|
||||
}
|
||||
|
||||
fn fixture(self: *const Lists) Fixture {
|
||||
return .{
|
||||
.sources = &one_source,
|
||||
@@ -669,6 +804,208 @@ test "precedence: an allow rule beats a wild entry" {
|
||||
try testing.expectEqual(Reason.rule_allow_exact, decision.reason);
|
||||
}
|
||||
|
||||
// --- list exceptions (milestone 21 ruling 2) --------------------------------
|
||||
|
||||
/// The fixture ruling 2 is written against: one list that blocks `ads.example`
|
||||
/// and its subdomains, and lifts `good.ads.example` back out.
|
||||
const exception_lists: Lists = .initWithExceptions(
|
||||
"ads.example\n",
|
||||
"ads.example\n",
|
||||
"good.ads.example\n",
|
||||
);
|
||||
|
||||
test "precedence: a list exception beats a list domain entry" {
|
||||
var snapshot = try build(testing.allocator, exception_lists.fixture());
|
||||
defer snapshot.deinit();
|
||||
|
||||
// The apex is blocked by the `.list` entry; the excepted name is not, even
|
||||
// though the same source blocks it through `.wild`.
|
||||
try testing.expect(snapshot.evaluate(0, "ads.example").blocked);
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.ads.example");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
|
||||
try testing.expectEqualStrings("good.ads.example", decision.matched);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: a list exception beats a list domain entry on the same name" {
|
||||
// The boundary the fixture above cannot pin: `good.ads.example` is not in
|
||||
// its `.list` body, so that test compares the exception against the
|
||||
// wildcard level. Here one name is carried by both `.allow` and `.list`,
|
||||
// which is the only way level 7 and level 8 are reached by one query.
|
||||
const lists: Lists = .initWithExceptions(
|
||||
"good.ads.example\n",
|
||||
"",
|
||||
"good.ads.example\n",
|
||||
);
|
||||
var snapshot = try build(testing.allocator, lists.fixture());
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.ads.example");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
|
||||
try testing.expectEqualStrings("good.ads.example", decision.matched);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: a list domain entry beats a list wildcard entry" {
|
||||
// Level 8 over level 9: the name is its own `.list` entry and a subdomain
|
||||
// of a `.wild` entry, so both would block and the reported reason is what
|
||||
// separates them.
|
||||
const lists: Lists = .init("x.ads.example\n", "ads.example\n");
|
||||
var snapshot = try build(testing.allocator, lists.fixture());
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "x.ads.example");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_domain, decision.reason);
|
||||
try testing.expectEqualStrings("x.ads.example", decision.matched);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: a list exception beats a list wildcard entry" {
|
||||
var snapshot = try build(testing.allocator, exception_lists.fixture());
|
||||
defer snapshot.deinit();
|
||||
|
||||
try testing.expect(snapshot.evaluate(0, "x.ads.example").blocked);
|
||||
|
||||
// The parent walk: one `@@||good.ads.example^` entry covers the subdomains
|
||||
// of the excepted name as well as the name itself.
|
||||
const decision = snapshot.evaluate(0, "y.good.ads.example");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
|
||||
try testing.expectEqualStrings("good.ads.example", decision.matched);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: an operator block rule beats a list exception" {
|
||||
// The property the exception level's position exists for: a downloaded list
|
||||
// may cancel what another list blocks and may never cancel what the
|
||||
// operator decided. All three operator block levels are checked, because
|
||||
// all three sit above the exception level.
|
||||
for ([_]model.Rule{
|
||||
rule("good.ads.example", .exact, .block),
|
||||
rule("*.ads.example", .wildcard, .block),
|
||||
rule("^good\\.ads\\.example$", .regex, .block),
|
||||
}) |blocking| {
|
||||
var fixture = exception_lists.fixture();
|
||||
const rows = [_]model.Rule{blocking};
|
||||
fixture.rules = &rows;
|
||||
|
||||
var snapshot = try build(testing.allocator, fixture);
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.ads.example");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expect(decision.reason == .rule_block_exact or
|
||||
decision.reason == .rule_block_wildcard or
|
||||
decision.reason == .rule_block_regex);
|
||||
}
|
||||
}
|
||||
|
||||
test "the regex reasons render as the wire strings the API and the query log carry" {
|
||||
// `web/handlers/lookup.zig` renders a reason as `@tagName`, and
|
||||
// `storage/logger.zig` stores one in a 32-byte `max_reason_len` buffer.
|
||||
// Neither can be reached from this file — pure core imports no web and no
|
||||
// storage — so the tag names and their length are pinned here.
|
||||
try testing.expectEqualStrings("rule_allow_regex", @tagName(Reason.rule_allow_regex));
|
||||
try testing.expectEqualStrings("rule_block_regex", @tagName(Reason.rule_block_regex));
|
||||
inline for (@typeInfo(Reason).@"enum".fields) |field| {
|
||||
try testing.expect(field.name.len <= 32);
|
||||
}
|
||||
}
|
||||
|
||||
test "precedence: an allow regex beats every list level" {
|
||||
// The other side of the same boundary: an operator allow rule lifts a list
|
||||
// block, whichever of the three list levels made it.
|
||||
const rows = [_]model.Rule{rule("ads\\.example$", .regex, .allow)};
|
||||
var fixture = exception_lists.fixture();
|
||||
fixture.rules = &rows;
|
||||
|
||||
var snapshot = try build(testing.allocator, fixture);
|
||||
defer snapshot.deinit();
|
||||
|
||||
// `.list` blocks the apex and `.wild` blocks the subdomains; the regex
|
||||
// covers both, and answers before either is consulted.
|
||||
for ([_][]const u8{ "ads.example", "x.ads.example" }) |domain| {
|
||||
const decision = snapshot.evaluate(0, domain);
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_allow_regex, decision.reason);
|
||||
try testing.expectEqual(@as(?u32, null), decision.source);
|
||||
}
|
||||
}
|
||||
|
||||
test "precedence: a block regex blocks a name no list carries" {
|
||||
const rows = [_]model.Rule{rule("^ad[0-9]+-", .regex, .block)};
|
||||
var fixture = exception_lists.fixture();
|
||||
fixture.rules = &rows;
|
||||
|
||||
var snapshot = try build(testing.allocator, fixture);
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "ad7-cdn.other.example");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_block_regex, decision.reason);
|
||||
try testing.expectEqual(@as(?u32, null), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: a list exception is scoped to the groups the source is in" {
|
||||
const groups = [_]model.Group{ .{ .name = "default" }, .{ .name = "kids" } };
|
||||
const ids = [_]i64{ 1, 2 };
|
||||
var snapshot = try build(testing.allocator, .{
|
||||
.groups = &groups,
|
||||
.group_ids = &ids,
|
||||
.sources = &one_source,
|
||||
.source_ids = &one_source_id,
|
||||
.group_sources = &one_link,
|
||||
.compiled = &exception_lists.compiled,
|
||||
});
|
||||
defer snapshot.deinit();
|
||||
|
||||
const kids = snapshot.groupIndexByName("kids").?;
|
||||
try testing.expectEqual(
|
||||
Reason.blocklist_exception,
|
||||
snapshot.evaluate(snapshot.default_group, "good.ads.example").reason,
|
||||
);
|
||||
// `kids` is linked to no source, so neither the block nor the exception
|
||||
// reaches it.
|
||||
try testing.expectEqual(Reason.none, snapshot.evaluate(kids, "good.ads.example").reason);
|
||||
}
|
||||
|
||||
test "precedence: an exception in one list lifts the block another list made" {
|
||||
const sources = [_]model.BlocklistSource{
|
||||
.{ .url = "https://lists.test/a", .name = "list a" },
|
||||
.{ .url = "https://lists.test/b", .name = "list b" },
|
||||
};
|
||||
const source_ids = [_]i64{ 11, 12 };
|
||||
const links = [_]model.GroupSource{
|
||||
.{ .group = "default", .source_url = "https://lists.test/a" },
|
||||
.{ .group = "default", .source_url = "https://lists.test/b" },
|
||||
};
|
||||
const compiled = [_]?Snapshot.Compiled{
|
||||
.{ .list_body = "ads.example\n", .wild_body = "ads.example\n" },
|
||||
.{ .list_body = "", .wild_body = "", .allow_body = "good.ads.example\n" },
|
||||
};
|
||||
|
||||
var snapshot = try build(testing.allocator, .{
|
||||
.sources = &sources,
|
||||
.source_ids = &source_ids,
|
||||
.group_sources = &links,
|
||||
.compiled = &compiled,
|
||||
});
|
||||
defer snapshot.deinit();
|
||||
|
||||
try testing.expect(snapshot.evaluate(0, "ads.example").blocked);
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.ads.example");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
|
||||
// The source reported is the one that lifted the block, not the one that
|
||||
// made it.
|
||||
try testing.expectEqual(@as(?u32, 1), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: nothing configured allows with reason none" {
|
||||
var snapshot = try build(testing.allocator, .{});
|
||||
defer snapshot.deinit();
|
||||
|
||||
+132
-8
@@ -1,9 +1,17 @@
|
||||
//! The Adblock Plus filter syntax, restricted to what a DNS sinkhole can
|
||||
//! honour: domain anchors and bare names. Pure, `std` only.
|
||||
//!
|
||||
//! Exception rules (`@@`) are `.unsupported` rather than an allow entry. The
|
||||
//! allow surface is the `rules` table, and a downloaded list that could quietly
|
||||
//! allow a domain across every group is a policy hole the operator did not open.
|
||||
//! An exception rule is `.exception` in exactly two spellings, `@@||name^` and
|
||||
//! `@@||name`, each of which may carry the literal `$important` behind it. Every
|
||||
//! other `@@` form stays `.unsupported`: a bare `@@name`, a path, a scheme, any
|
||||
//! other modifier.
|
||||
//!
|
||||
//! What makes honouring them safe is where they land, not what they say. A list
|
||||
//! exception is evaluated below every operator rule (PLAN §3.10), so it can
|
||||
//! cancel a block another list made and nothing else. No downloaded list can
|
||||
//! open an allow hole the operator did not open, which is why the allow surface
|
||||
//! stays the `rules` table — including its `.regex` kind, which is the one
|
||||
//! regex dialect nxdns evaluates and which no downloaded list can reach.
|
||||
|
||||
const std = @import("std");
|
||||
const parsers = @import("parsers.zig");
|
||||
@@ -12,6 +20,31 @@ const parsers = @import("parsers.zig");
|
||||
/// this syntax and only a trailing one is meaningful for a domain rule.
|
||||
const rule_tokens = "*^|/$";
|
||||
|
||||
/// The one modifier an exception line may carry. AdGuard-authored lists write it
|
||||
/// on most of their `@@` rules and it changes nothing here: these exceptions
|
||||
/// already sit below every operator rule, so "important" cannot raise one above
|
||||
/// the decisions it is not allowed to reach.
|
||||
const important_modifier = "$important";
|
||||
|
||||
/// A candidate the compiler could turn into a name, for the two anchored forms
|
||||
/// only. `rule_tokens` covers the syntax characters; this also refuses
|
||||
/// whitespace, which is neither a rule token nor a control byte and so used to
|
||||
/// survive into a compiled body as an entry only a query carrying the same
|
||||
/// space could match. That is true of `||name` and `@@||name` because
|
||||
/// `compiler.zig` hands a `.wildcard` or `.exception` text to `addCandidate`
|
||||
/// whole. A `.domain` text is tokenized on whitespace first and each field
|
||||
/// filed separately, so the bare form does not come through here: refusing a
|
||||
/// space there would drop the hosts-style lines that a mixed list classified
|
||||
/// `abp` by `detectFormat` still contributes.
|
||||
fn isNameCandidate(candidate: []const u8) bool {
|
||||
if (candidate.len == 0) return false;
|
||||
if (std.mem.findAny(u8, candidate, rule_tokens) != null) return false;
|
||||
for (candidate) |c| {
|
||||
if (std.ascii.isWhitespace(c) or std.ascii.isControl(c)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
pub fn parseLine(line: []const u8) parsers.Line {
|
||||
const text = std.mem.trim(u8, line, &std.ascii.whitespace);
|
||||
if (text.len == 0) return .{ .kind = .ignore };
|
||||
@@ -19,15 +52,14 @@ pub fn parseLine(line: []const u8) parsers.Line {
|
||||
if (text[0] == '[') return .{ .kind = .ignore };
|
||||
if (parsers.isElementHiding(text)) return .{ .kind = .unsupported };
|
||||
if (text[0] == '#') return .{ .kind = .ignore };
|
||||
if (std.mem.startsWith(u8, text, "@@")) return .{ .kind = .unsupported };
|
||||
if (std.mem.startsWith(u8, text, "@@")) return parseException(text[2..]);
|
||||
if (text[0] == '/') return .{ .kind = .regex };
|
||||
if (std.mem.findScalar(u8, text, '$') != null) return .{ .kind = .unsupported };
|
||||
|
||||
if (std.mem.startsWith(u8, text, "||")) {
|
||||
var candidate = text[2..];
|
||||
if (std.mem.endsWith(u8, candidate, "^")) candidate = candidate[0 .. candidate.len - 1];
|
||||
if (candidate.len == 0) return .{ .kind = .unsupported };
|
||||
if (std.mem.findAny(u8, candidate, rule_tokens) != null) return .{ .kind = .unsupported };
|
||||
if (!isNameCandidate(candidate)) return .{ .kind = .unsupported };
|
||||
// A domain anchor covers the domain itself as well as its subdomains,
|
||||
// so the compiler emits an apex entry beside the wildcard one.
|
||||
return .{ .kind = .wildcard, .text = candidate, .covers_apex = true };
|
||||
@@ -37,6 +69,24 @@ pub fn parseLine(line: []const u8) parsers.Line {
|
||||
return .{ .kind = .domain, .text = text };
|
||||
}
|
||||
|
||||
/// One exception line, past its `@@`. The domain anchor and the trailing `^` get
|
||||
/// the same treatment they get on a block rule, so `@@||x^` and `||x^` accept
|
||||
/// and reject the same names.
|
||||
///
|
||||
/// `$important` is stripped before the anchor is read, because the `$` would
|
||||
/// otherwise be a rule token and refuse the whole line.
|
||||
fn parseException(rest: []const u8) parsers.Line {
|
||||
if (!std.mem.startsWith(u8, rest, "||")) return .{ .kind = .unsupported };
|
||||
|
||||
var candidate = rest[2..];
|
||||
if (std.mem.endsWith(u8, candidate, important_modifier)) {
|
||||
candidate = candidate[0 .. candidate.len - important_modifier.len];
|
||||
}
|
||||
if (std.mem.endsWith(u8, candidate, "^")) candidate = candidate[0 .. candidate.len - 1];
|
||||
if (!isNameCandidate(candidate)) return .{ .kind = .unsupported };
|
||||
return .{ .kind = .exception, .text = candidate, .covers_apex = true };
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "a bang comment is ignored" {
|
||||
@@ -69,8 +119,46 @@ test "a modifier list is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("||example.com^$third-party").kind);
|
||||
}
|
||||
|
||||
test "an exception rule is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("@@||example.com^").kind);
|
||||
test "an exception rule is an exception that covers its apex" {
|
||||
const line = parseLine("@@||example.com^");
|
||||
try testing.expectEqual(parsers.Kind.exception, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
try testing.expect(line.covers_apex);
|
||||
}
|
||||
|
||||
test "an exception rule without a separator is still an exception" {
|
||||
const line = parseLine("@@||example.com");
|
||||
try testing.expectEqual(parsers.Kind.exception, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
try testing.expect(line.covers_apex);
|
||||
}
|
||||
|
||||
test "an exception rule tolerates the important modifier" {
|
||||
for ([_][]const u8{ "@@||example.com^$important", "@@||example.com$important" }) |text| {
|
||||
const line = parseLine(text);
|
||||
try testing.expectEqual(parsers.Kind.exception, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
try testing.expect(line.covers_apex);
|
||||
}
|
||||
}
|
||||
|
||||
test "every exception form outside the two anchored ones is unsupported" {
|
||||
for ([_][]const u8{
|
||||
// No domain anchor: this is a substring rule in browser syntax, and
|
||||
// reading it as a name would allow far more than it says.
|
||||
"@@example.com",
|
||||
"@@|http://example.com",
|
||||
"@@||example.com/path^",
|
||||
"@@||example.com^$third-party",
|
||||
"@@||example.com^$important$third-party",
|
||||
"@@||example.com^$dnstype=A",
|
||||
"@@||^",
|
||||
"@@||$important",
|
||||
"@@",
|
||||
"@@||ads*.example.com^",
|
||||
}) |text| {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine(text).kind);
|
||||
}
|
||||
}
|
||||
|
||||
test "element hiding is unsupported" {
|
||||
@@ -94,6 +182,42 @@ test "a bare name is a domain" {
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
}
|
||||
|
||||
test "a candidate carrying whitespace is unsupported in the anchored forms" {
|
||||
// A space is not a rule token and it is not a control byte, so it used to
|
||||
// reach the compiler, which lowercases and length-checks but does not
|
||||
// reject it. An anchored form's text is filed whole, so the entry it wrote
|
||||
// could only ever match a query name carrying the same space.
|
||||
for ([_][]const u8{
|
||||
"||good.example bad.example^",
|
||||
"@@||good.example bad.example^",
|
||||
"||good.example\tbad.example",
|
||||
"@@||good.example\tbad.example",
|
||||
}) |text| {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine(text).kind);
|
||||
}
|
||||
}
|
||||
|
||||
test "a bare candidate carrying whitespace stays a domain" {
|
||||
// Not the same case: `compiler.zig` tokenizes a `.domain` text on
|
||||
// whitespace and files each field. Refusing it here would drop the
|
||||
// hosts-style lines of a mixed list, which `detectFormat` classifies `abp`
|
||||
// as a whole and which only reach a compiled body through that split.
|
||||
//
|
||||
// What this test pins is the parser half — the kind and the untouched text.
|
||||
// The split itself belongs to the compiler and is asserted there, by
|
||||
// "an abp list's hosts-style lines reach the domain body through the split":
|
||||
// a tokenizer removed from `compile` would leave every assertion below true.
|
||||
for ([_][]const u8{
|
||||
"good.example bad.example",
|
||||
"0.0.0.0 ads.example",
|
||||
"good.example\tbad.example",
|
||||
}) |text| {
|
||||
const line = parseLine(text);
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings(text, line.text);
|
||||
}
|
||||
}
|
||||
|
||||
test "a rule token outside the supported forms is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("ads*.example.com").kind);
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com^").kind);
|
||||
|
||||
+12
-4
@@ -22,10 +22,15 @@ pub const Kind = enum {
|
||||
domain,
|
||||
/// `text` holds one candidate suffix; every proper subdomain of it matches.
|
||||
wildcard,
|
||||
/// A regex rule. Counted, skipped, never compiled (PLAN §2.2).
|
||||
/// A regex line in a downloaded list. Counted, skipped, never compiled: the
|
||||
/// engine exists for rules the operator wrote, not for lists (PLAN §2.2).
|
||||
regex,
|
||||
/// `text` holds one candidate name an ABP exception rule (`@@||x^`) lifts:
|
||||
/// the name itself and every subdomain of it. Only the ABP parser emits it.
|
||||
exception,
|
||||
/// Syntactically a rule of this format, but one nxdns cannot honour:
|
||||
/// an ABP modifier list, an exception rule, element hiding, a scheme anchor.
|
||||
/// an ABP modifier list, an exception form outside `@@||x^`, element
|
||||
/// hiding, a scheme anchor.
|
||||
unsupported,
|
||||
};
|
||||
|
||||
@@ -33,8 +38,11 @@ pub const Line = struct {
|
||||
kind: Kind,
|
||||
/// Borrowed from the caller's line. Not lowercased, not validated.
|
||||
text: []const u8 = "",
|
||||
/// `.wildcard` only. ABP `||x^` covers `x` itself as well as its subdomains,
|
||||
/// so the compiler emits an additional `.list` entry when this is set.
|
||||
/// `.wildcard` and `.exception` only: the rule covers the anchored name
|
||||
/// itself as well as its subdomains, which is what ABP `||x^` and `@@||x^`
|
||||
/// mean. The compiler acts on it for a `.wildcard` line, by emitting an
|
||||
/// additional `.list` entry; an `.exception` line needs no second entry,
|
||||
/// because the allow walk tests the full name as well as its parents.
|
||||
covers_apex: bool = false,
|
||||
};
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+339
-62
@@ -1,4 +1,4 @@
|
||||
//! One group's explicit rules (PLAN §3.10 levels 1–4), compiled once into an
|
||||
//! One group's explicit rules (PLAN §3.10 levels 1–6), compiled once into an
|
||||
//! immutable form the query path can read without allocating.
|
||||
//!
|
||||
//! Exact patterns go into a `DomainSet`; wildcard patterns stay a flat, sorted
|
||||
@@ -7,7 +7,13 @@
|
||||
//! that many short patterns is cheaper than an index that would have to be
|
||||
//! rebuilt on every snapshot swap.
|
||||
//!
|
||||
//! Pure: an allocator and plain values, no `std.Io`, no clock, no entropy
|
||||
//! Regex patterns are compiled here, once per snapshot, into the linear-time
|
||||
//! programs of `regex.zig` and scanned the same way. `max_regex_per_group` caps
|
||||
//! them far lower, at 256: a regex costs a whole VM run where a wildcard costs a
|
||||
//! label comparison, and the matcher reaches them only after every hash and
|
||||
//! wildcard level has missed.
|
||||
//!
|
||||
//! Pure: allocators and plain values, no `std.Io`, no clock, no entropy
|
||||
//! source. The hash seed arrives as a parameter.
|
||||
|
||||
const std = @import("std");
|
||||
@@ -17,14 +23,40 @@ const model = @import("../config/model.zig");
|
||||
const name = @import("../dns/name.zig");
|
||||
const types = @import("../dns/types.zig");
|
||||
const domain_set = @import("domain_set.zig");
|
||||
const regex = @import("regex.zig");
|
||||
const wildcard = @import("wildcard.zig");
|
||||
|
||||
pub const Error = error{ OutOfMemory, BadPattern, TooManyWildcards } || domain_set.DomainSet.Error;
|
||||
pub const Error = error{
|
||||
OutOfMemory,
|
||||
BadPattern,
|
||||
TooManyWildcards,
|
||||
TooManyRegexRules,
|
||||
} || domain_set.DomainSet.Error;
|
||||
|
||||
/// Both wildcard lists of one group together. The cap exists so a rules table
|
||||
/// edited into the millions cannot turn every query into a linear scan.
|
||||
pub const max_wildcards_per_group: usize = 4096;
|
||||
|
||||
/// Both regex lists of one group together, capped well below the wildcards: a
|
||||
/// miss at this level runs every program to its end.
|
||||
pub const max_regex_per_group: usize = 256;
|
||||
|
||||
/// A compiled operator regex beside the text it was written as. The text is what
|
||||
/// `Decision.matched` reports, so the query log names the rule the operator
|
||||
/// wrote rather than an instruction count.
|
||||
pub const RegexRule = struct {
|
||||
pattern: []const u8,
|
||||
program: regex.Program,
|
||||
|
||||
/// Frees through a copy of the program: the slices holding these rules are
|
||||
/// `const`, and `Program.deinit` wants a mutable pointer only to blank the
|
||||
/// struct it is finished with.
|
||||
fn free(self: RegexRule, gpa: Allocator) void {
|
||||
var program = self.program;
|
||||
program.deinit(gpa);
|
||||
}
|
||||
};
|
||||
|
||||
pub const RuleSet = struct {
|
||||
exact_allow: domain_set.DomainSet = .empty,
|
||||
exact_block: domain_set.DomainSet = .empty,
|
||||
@@ -32,75 +64,115 @@ pub const RuleSet = struct {
|
||||
/// order and the same first match.
|
||||
wildcard_allow: []const []const u8 = &.{},
|
||||
wildcard_block: []const []const u8 = &.{},
|
||||
/// One block holding the bytes of both wildcard lists; freed as a unit.
|
||||
wildcard_bytes: []const u8 = &.{},
|
||||
/// Sorted and deduplicated like the wildcards, so the first regex to match a
|
||||
/// name is the same one on every rebuild of the same rows.
|
||||
regex_allow: []const RegexRule = &.{},
|
||||
regex_block: []const RegexRule = &.{},
|
||||
/// One block holding the pattern bytes of all four lists; freed as a unit.
|
||||
pattern_bytes: []const u8 = &.{},
|
||||
|
||||
pub const empty: RuleSet = .{};
|
||||
|
||||
/// `rows` are one group's rules only; splitting `listRules` output by group
|
||||
/// belongs to the caller, which is the only holder of the group table.
|
||||
///
|
||||
/// Patterns are normalized (lowercase over ASCII, one trailing dot
|
||||
/// Name patterns are normalized (lowercase over ASCII, one trailing dot
|
||||
/// stripped) and validated: `.exact` through `dns.name.fromText`,
|
||||
/// `.wildcard` through `wildcard.validate`. An invalid pattern is
|
||||
/// `error.BadPattern`, not a skipped row — every pattern passed
|
||||
/// `config/validate.zig` on the way in, so an invalid one here means the
|
||||
/// rows were edited underneath nxdns and a silently dropped allow rule
|
||||
/// `.wildcard` through `wildcard.validate`, `.regex` by compiling it. An
|
||||
/// invalid pattern is `error.BadPattern`, not a skipped row — every pattern
|
||||
/// passed `config/validate.zig` on the way in, so an invalid one here means
|
||||
/// the rows were edited underneath nxdns and a silently dropped allow rule
|
||||
/// would block a domain the operator unblocked.
|
||||
pub fn build(gpa: Allocator, rows: []const model.Rule, seed: u64) Error!RuleSet {
|
||||
///
|
||||
/// Two allocators, because the caller's `perm` is a snapshot arena: an
|
||||
/// arena reclaims only its most recent allocation, so every temporary taken
|
||||
/// from it would live as long as the snapshot and go unreported by
|
||||
/// `memoryBytes`. `perm` owns what the returned set retains and is what
|
||||
/// `deinit` frees; `scratch` owns the build's working storage, which is
|
||||
/// released by the time `build` returns. Passing one allocator as both is
|
||||
/// correct wherever freeing works normally.
|
||||
pub fn build(
|
||||
perm: Allocator,
|
||||
scratch: Allocator,
|
||||
rows: []const model.Rule,
|
||||
seed: u64,
|
||||
) Error!RuleSet {
|
||||
if (rows.len == 0) return .empty;
|
||||
|
||||
var scratch: std.ArrayList(u8) = .empty;
|
||||
defer scratch.deinit(gpa);
|
||||
var spans: [4]std.ArrayList(Span) = .{ .empty, .empty, .empty, .empty };
|
||||
defer for (&spans) |*bucket| bucket.deinit(gpa);
|
||||
var joined: std.ArrayList(u8) = .empty;
|
||||
defer joined.deinit(scratch);
|
||||
var spans: [6]std.ArrayList(Span) = @splat(.empty);
|
||||
defer for (&spans) |*bucket| bucket.deinit(scratch);
|
||||
|
||||
var wildcards: usize = 0;
|
||||
var regexes: usize = 0;
|
||||
var buf: [types.max_name_len]u8 = undefined;
|
||||
for (rows) |row| {
|
||||
const pattern = normalize(row.pattern, &buf) catch return error.BadPattern;
|
||||
switch (row.kind) {
|
||||
.exact => _ = name.fromText(pattern) catch return error.BadPattern,
|
||||
.wildcard => {
|
||||
wildcard.validate(pattern) catch return error.BadPattern;
|
||||
const pattern = switch (row.kind) {
|
||||
.exact => blk: {
|
||||
const text = normalize(row.pattern, &buf) catch return error.BadPattern;
|
||||
_ = name.fromText(text) catch return error.BadPattern;
|
||||
break :blk text;
|
||||
},
|
||||
.wildcard => blk: {
|
||||
const text = normalize(row.pattern, &buf) catch return error.BadPattern;
|
||||
wildcard.validate(text) catch return error.BadPattern;
|
||||
wildcards += 1;
|
||||
if (wildcards > max_wildcards_per_group) return error.TooManyWildcards;
|
||||
break :blk text;
|
||||
},
|
||||
}
|
||||
// A regex is not a name, so `normalize` must not touch it: it
|
||||
// strips a trailing `.`, which here is the any-byte atom, and it
|
||||
// lowercases, which turns the rejected `\D` into the accepted
|
||||
// `\d`. Either would silently change what the rule matches. The
|
||||
// bytes stay as the operator wrote them — the same bytes
|
||||
// `config/validate.zig` compiled at the edge. Compiling waits
|
||||
// until after the sort, so a duplicate is compiled once.
|
||||
.regex => blk: {
|
||||
regexes += 1;
|
||||
if (regexes > max_regex_per_group) return error.TooManyRegexRules;
|
||||
break :blk row.pattern;
|
||||
},
|
||||
};
|
||||
const bucket = &spans[bucketOf(row.kind, row.action)];
|
||||
try bucket.append(gpa, .{ .offset = scratch.items.len, .len = pattern.len });
|
||||
try scratch.appendSlice(gpa, pattern);
|
||||
try bucket.append(scratch, .{ .offset = joined.items.len, .len = pattern.len });
|
||||
try joined.appendSlice(scratch, pattern);
|
||||
}
|
||||
|
||||
// `scratch` stops growing here, so spans can become slices of it.
|
||||
var sorted: [4]std.ArrayList([]const u8) = .{ .empty, .empty, .empty, .empty };
|
||||
defer for (&sorted) |*bucket| bucket.deinit(gpa);
|
||||
// `joined` stops growing here, so spans can become slices of it.
|
||||
var sorted: [6]std.ArrayList([]const u8) = @splat(.empty);
|
||||
defer for (&sorted) |*bucket| bucket.deinit(scratch);
|
||||
for (&spans, &sorted) |*bucket, *out| {
|
||||
try out.ensureTotalCapacityPrecise(gpa, bucket.items.len);
|
||||
try out.ensureTotalCapacityPrecise(scratch, bucket.items.len);
|
||||
for (bucket.items) |span| {
|
||||
out.appendAssumeCapacity(scratch.items[span.offset..][0..span.len]);
|
||||
out.appendAssumeCapacity(joined.items[span.offset..][0..span.len]);
|
||||
}
|
||||
std.mem.sort([]const u8, out.items, {}, lessThanBytes);
|
||||
dedupSorted(out);
|
||||
}
|
||||
|
||||
var self: RuleSet = .empty;
|
||||
errdefer self.deinit(gpa);
|
||||
errdefer self.deinit(perm);
|
||||
|
||||
self.exact_allow = try buildSet(gpa, sorted[bucketOf(.exact, .allow)].items, seed);
|
||||
self.exact_block = try buildSet(gpa, sorted[bucketOf(.exact, .block)].items, seed);
|
||||
self.exact_allow = try buildSet(perm, scratch, sorted[bucketOf(.exact, .allow)].items, seed);
|
||||
self.exact_block = try buildSet(perm, scratch, sorted[bucketOf(.exact, .block)].items, seed);
|
||||
|
||||
const allow = sorted[bucketOf(.wildcard, .allow)].items;
|
||||
const block = sorted[bucketOf(.wildcard, .block)].items;
|
||||
const wild_allow = sorted[bucketOf(.wildcard, .allow)].items;
|
||||
const wild_block = sorted[bucketOf(.wildcard, .block)].items;
|
||||
const re_allow = sorted[bucketOf(.regex, .allow)].items;
|
||||
const re_block = sorted[bucketOf(.regex, .block)].items;
|
||||
var total: usize = 0;
|
||||
for (allow) |pattern| total += pattern.len;
|
||||
for (block) |pattern| total += pattern.len;
|
||||
for ([_][]const []const u8{ wild_allow, wild_block, re_allow, re_block }) |list| {
|
||||
for (list) |pattern| total += pattern.len;
|
||||
}
|
||||
|
||||
const bytes = try gpa.alloc(u8, total);
|
||||
self.wildcard_bytes = bytes;
|
||||
const bytes = try perm.alloc(u8, total);
|
||||
self.pattern_bytes = bytes;
|
||||
var at: usize = 0;
|
||||
self.wildcard_allow = try copyPatterns(gpa, allow, bytes, &at);
|
||||
self.wildcard_block = try copyPatterns(gpa, block, bytes, &at);
|
||||
self.wildcard_allow = try copyPatterns(perm, wild_allow, bytes, &at);
|
||||
self.wildcard_block = try copyPatterns(perm, wild_block, bytes, &at);
|
||||
self.regex_allow = try compilePatterns(perm, scratch, re_allow, bytes, &at);
|
||||
self.regex_block = try compilePatterns(perm, scratch, re_block, bytes, &at);
|
||||
|
||||
return self;
|
||||
}
|
||||
@@ -110,18 +182,35 @@ pub const RuleSet = struct {
|
||||
self.exact_block.deinit(gpa);
|
||||
gpa.free(self.wildcard_allow);
|
||||
gpa.free(self.wildcard_block);
|
||||
gpa.free(self.wildcard_bytes);
|
||||
freeRules(gpa, self.regex_allow);
|
||||
freeRules(gpa, self.regex_block);
|
||||
gpa.free(self.pattern_bytes);
|
||||
self.* = .empty;
|
||||
}
|
||||
|
||||
pub fn memoryBytes(self: *const RuleSet) usize {
|
||||
var programs: usize = 0;
|
||||
for (self.regex_allow) |item| programs += item.program.memoryBytes();
|
||||
for (self.regex_block) |item| programs += item.program.memoryBytes();
|
||||
return self.exact_allow.memoryBytes() +
|
||||
self.exact_block.memoryBytes() +
|
||||
self.wildcard_bytes.len +
|
||||
(self.wildcard_allow.len + self.wildcard_block.len) * @sizeOf([]const u8);
|
||||
self.pattern_bytes.len +
|
||||
programs +
|
||||
(self.wildcard_allow.len + self.wildcard_block.len) * @sizeOf([]const u8) +
|
||||
(self.regex_allow.len + self.regex_block.len) * @sizeOf(RegexRule);
|
||||
}
|
||||
};
|
||||
|
||||
/// The first regex of `list` that matches `domain`, or null. `list` is sorted,
|
||||
/// so "first" is stable across rebuilds of the same rows. The caller checks the
|
||||
/// allow list before the block list, as it does for wildcards.
|
||||
pub fn matchRegex(list: []const RegexRule, domain: []const u8) ?[]const u8 {
|
||||
for (list) |item| {
|
||||
if (regex.matches(&item.program, domain)) return item.pattern;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internals
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -131,15 +220,16 @@ pub const RuleSet = struct {
|
||||
const Span = struct { offset: usize, len: usize };
|
||||
|
||||
fn bucketOf(kind: model.RuleKind, action: model.RuleAction) usize {
|
||||
const kind_bit: usize = switch (kind) {
|
||||
const kind_base: usize = switch (kind) {
|
||||
.exact => 0,
|
||||
.wildcard => 2,
|
||||
.regex => 4,
|
||||
};
|
||||
const action_bit: usize = switch (action) {
|
||||
const action_offset: usize = switch (action) {
|
||||
.allow => 0,
|
||||
.block => 1,
|
||||
};
|
||||
return kind_bit + action_bit;
|
||||
return kind_base + action_offset;
|
||||
}
|
||||
|
||||
fn lessThanBytes(_: void, a: []const u8, b: []const u8) bool {
|
||||
@@ -159,26 +249,31 @@ fn dedupSorted(list: *std.ArrayList([]const u8)) void {
|
||||
list.shrinkRetainingCapacity(kept);
|
||||
}
|
||||
|
||||
fn buildSet(gpa: Allocator, patterns: []const []const u8, seed: u64) Error!domain_set.DomainSet {
|
||||
fn buildSet(
|
||||
perm: Allocator,
|
||||
scratch: Allocator,
|
||||
patterns: []const []const u8,
|
||||
seed: u64,
|
||||
) Error!domain_set.DomainSet {
|
||||
if (patterns.len == 0) return .empty;
|
||||
|
||||
var body: std.ArrayList(u8) = .empty;
|
||||
defer body.deinit(gpa);
|
||||
defer body.deinit(scratch);
|
||||
for (patterns) |pattern| {
|
||||
try body.appendSlice(gpa, pattern);
|
||||
try body.append(gpa, '\n');
|
||||
try body.appendSlice(scratch, pattern);
|
||||
try body.append(scratch, '\n');
|
||||
}
|
||||
return domain_set.DomainSet.build(gpa, body.items, seed);
|
||||
return domain_set.DomainSet.build(perm, body.items, seed);
|
||||
}
|
||||
|
||||
fn copyPatterns(
|
||||
gpa: Allocator,
|
||||
perm: Allocator,
|
||||
patterns: []const []const u8,
|
||||
bytes: []u8,
|
||||
at: *usize,
|
||||
) Error![]const []const u8 {
|
||||
if (patterns.len == 0) return &.{};
|
||||
const out = try gpa.alloc([]const u8, patterns.len);
|
||||
const out = try perm.alloc([]const u8, patterns.len);
|
||||
for (out, patterns) |*slot, pattern| {
|
||||
@memcpy(bytes[at.*..][0..pattern.len], pattern);
|
||||
slot.* = bytes[at.*..][0..pattern.len];
|
||||
@@ -187,6 +282,49 @@ fn copyPatterns(
|
||||
return out;
|
||||
}
|
||||
|
||||
/// Copies the pattern texts into `bytes` like `copyPatterns` and compiles each
|
||||
/// one. A compile failure is `error.BadPattern` whichever of the engine's three
|
||||
/// refusals fired: the pattern already passed `config/validate.zig`, which names
|
||||
/// the limit, so a row that fails here was written around that check.
|
||||
///
|
||||
/// Compiling into `scratch` and cloning across is what keeps a parse-time AST
|
||||
/// out of `perm`: `regex.compile` builds the AST, the child lists and the
|
||||
/// growing instruction buffer through the allocator it returns the program on.
|
||||
fn compilePatterns(
|
||||
perm: Allocator,
|
||||
scratch: Allocator,
|
||||
patterns: []const []const u8,
|
||||
bytes: []u8,
|
||||
at: *usize,
|
||||
) Error![]const RegexRule {
|
||||
if (patterns.len == 0) return &.{};
|
||||
const out = try perm.alloc(RegexRule, patterns.len);
|
||||
var built: usize = 0;
|
||||
errdefer {
|
||||
for (out[0..built]) |item| item.free(perm);
|
||||
perm.free(out);
|
||||
}
|
||||
for (out, patterns) |*slot, pattern| {
|
||||
var compiled = regex.compile(scratch, pattern) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
else => return error.BadPattern,
|
||||
};
|
||||
defer compiled.deinit(scratch);
|
||||
const program = try compiled.clone(perm);
|
||||
|
||||
@memcpy(bytes[at.*..][0..pattern.len], pattern);
|
||||
slot.* = .{ .pattern = bytes[at.*..][0..pattern.len], .program = program };
|
||||
at.* += pattern.len;
|
||||
built += 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
fn freeRules(gpa: Allocator, list: []const RegexRule) void {
|
||||
for (list) |item| item.free(gpa);
|
||||
gpa.free(list);
|
||||
}
|
||||
|
||||
const NameError = error{BadName};
|
||||
|
||||
/// Lowercases over ASCII and strips one trailing dot. A byte ≥ 0x80 is
|
||||
@@ -223,7 +361,7 @@ test "exact rules land in the matching set" {
|
||||
rule("ads.example.com", .exact, .block),
|
||||
rule("good.example.com", .exact, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0x5eed);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0x5eed);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(set.exact_block.contains("ads.example.com"));
|
||||
@@ -239,7 +377,7 @@ test "wildcard rules land in the matching list, sorted" {
|
||||
rule("*.a.example.com", .wildcard, .block),
|
||||
rule("*.allowed.example.com", .wildcard, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0x5eed);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0x5eed);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(usize, 2), set.wildcard_block.len);
|
||||
@@ -254,7 +392,7 @@ test "patterns are normalized to lowercase without a trailing dot" {
|
||||
rule("ADS.Example.COM.", .exact, .block),
|
||||
rule("*.Tracker.NET.", .wildcard, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(set.exact_block.contains("ads.example.com"));
|
||||
@@ -268,7 +406,7 @@ test "duplicate rows collapse to one entry" {
|
||||
rule("*.x.example.com", .wildcard, .block),
|
||||
rule("*.x.example.com", .wildcard, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(u32, 1), set.exact_block.count);
|
||||
@@ -278,14 +416,98 @@ test "duplicate rows collapse to one entry" {
|
||||
test "an invalid exact pattern is an error" {
|
||||
for ([_][]const u8{ "", ".", "a..b", "ads example.com", "ads\u{00e9}.example.com" }) |pattern| {
|
||||
const rows = [_]model.Rule{rule(pattern, .exact, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0));
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &rows, 0));
|
||||
}
|
||||
}
|
||||
|
||||
test "regex rules land in their own buckets, compiled and sorted" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("^zz", .regex, .block),
|
||||
rule("^aa", .regex, .block),
|
||||
rule("ok$", .regex, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0x5eed);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(usize, 2), set.regex_block.len);
|
||||
try testing.expectEqualStrings("^aa", set.regex_block[0].pattern);
|
||||
try testing.expectEqualStrings("^zz", set.regex_block[1].pattern);
|
||||
try testing.expectEqual(@as(usize, 1), set.regex_allow.len);
|
||||
try testing.expectEqualStrings("ok$", set.regex_allow[0].pattern);
|
||||
|
||||
try testing.expectEqualStrings("^aa", matchRegex(set.regex_block, "aabb.example").?);
|
||||
try testing.expect(matchRegex(set.regex_block, "bbaa.example") == null);
|
||||
try testing.expectEqualStrings("ok$", matchRegex(set.regex_allow, "example.ok").?);
|
||||
}
|
||||
|
||||
test "a regex pattern keeps the bytes the operator wrote" {
|
||||
// `normalize` would strip the trailing dot and lowercase the escape, and
|
||||
// either edit would change what the pattern matches. The exact and wildcard
|
||||
// kinds still normalize; only this one is exempt.
|
||||
const rows = [_]model.Rule{
|
||||
rule("ADS\\.Example\\.", .regex, .block),
|
||||
rule("ADS.Example.", .exact, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqualStrings("ADS\\.Example\\.", set.regex_block[0].pattern);
|
||||
try testing.expect(set.exact_block.contains("ads.example"));
|
||||
}
|
||||
|
||||
test "duplicate regex rows collapse to one compiled program" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("^ad[0-9]+-", .regex, .block),
|
||||
rule("^ad[0-9]+-", .regex, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(usize, 1), set.regex_block.len);
|
||||
}
|
||||
|
||||
test "a regex pattern the engine refuses is an error, not a skipped row" {
|
||||
for ([_][]const u8{ "(", "", "a+?", "[z-a]", "\\s" }) |pattern| {
|
||||
const rows = [_]model.Rule{rule(pattern, .regex, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &rows, 0));
|
||||
}
|
||||
|
||||
// The size limits arrive as `BadPattern` too: which one fired is
|
||||
// `config/validate.zig`'s to report, and by here the row is simply wrong.
|
||||
const long = [_]model.Rule{rule("a" ** 300, .regex, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &long, 0));
|
||||
const complex = [_]model.Rule{rule("(abcdefghij){200}", .regex, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &complex, 0));
|
||||
}
|
||||
|
||||
test "too many regex rules is an error" {
|
||||
const gpa = testing.allocator;
|
||||
const rows = try gpa.alloc(model.Rule, max_regex_per_group + 1);
|
||||
defer gpa.free(rows);
|
||||
|
||||
var patterns: std.ArrayList([]u8) = .empty;
|
||||
defer {
|
||||
for (patterns.items) |p| gpa.free(p);
|
||||
patterns.deinit(gpa);
|
||||
}
|
||||
for (rows, 0..) |*row, i| {
|
||||
const pattern = try std.fmt.allocPrint(gpa, "^n{d}-", .{i});
|
||||
try patterns.append(gpa, pattern);
|
||||
row.* = rule(pattern, .regex, .block);
|
||||
}
|
||||
|
||||
try testing.expectError(error.TooManyRegexRules, RuleSet.build(gpa, gpa, rows, 0));
|
||||
|
||||
// The cap counts both actions together, like the wildcard one.
|
||||
rows[0].action = .allow;
|
||||
try testing.expectError(error.TooManyRegexRules, RuleSet.build(gpa, gpa, rows, 0));
|
||||
try testing.expectEqual(@as(usize, 256), max_regex_per_group);
|
||||
}
|
||||
|
||||
test "an invalid wildcard pattern is an error" {
|
||||
for ([_][]const u8{ "example.com", "ad*.example.com", "*..com" }) |pattern| {
|
||||
const rows = [_]model.Rule{rule(pattern, .wildcard, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0));
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &rows, 0));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -305,11 +527,11 @@ test "too many wildcards is an error" {
|
||||
row.* = rule(pattern, .wildcard, .block);
|
||||
}
|
||||
|
||||
try testing.expectError(error.TooManyWildcards, RuleSet.build(gpa, rows, 0));
|
||||
try testing.expectError(error.TooManyWildcards, RuleSet.build(gpa, gpa, rows, 0));
|
||||
}
|
||||
|
||||
test "an empty rule list builds the empty set" {
|
||||
var set = try RuleSet.build(testing.allocator, &[_]model.Rule{}, 0);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &[_]model.Rule{}, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(!set.exact_block.contains("ads.example.com"));
|
||||
@@ -328,11 +550,63 @@ test "memoryBytes counts every part" {
|
||||
rule("ads.example.com", .exact, .block),
|
||||
rule("*.tracker.net", .wildcard, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(set.memoryBytes() > set.exact_block.memoryBytes());
|
||||
try testing.expect(set.memoryBytes() >= "*.tracker.net".len);
|
||||
|
||||
// A compiled program is the largest thing a rule set holds, so leaving it
|
||||
// out would make the snapshot's memory report a fiction.
|
||||
const with_regex = [_]model.Rule{ rows[0], rows[1], rule("^ad[0-9]+-", .regex, .block) };
|
||||
var wider = try RuleSet.build(testing.allocator, testing.allocator, &with_regex, 0);
|
||||
defer wider.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(wider.memoryBytes() > set.memoryBytes() + "^ad[0-9]+-".len);
|
||||
try testing.expect(wider.memoryBytes() >= wider.regex_block[0].program.memoryBytes());
|
||||
}
|
||||
|
||||
test "the build's temporaries stay out of the permanent allocator" {
|
||||
// The property the two-allocator split exists for. An arena reclaims only
|
||||
// its most recent allocation, so a temporary taken from `perm` would live
|
||||
// as long as the arena and be invisible to `memoryBytes`. Two checks, one
|
||||
// per direction: `testing.allocator` fails the test if anything the set
|
||||
// retains was taken from `scratch`, and the arena's capacity fails it if
|
||||
// the build's working storage was taken from `perm`.
|
||||
const gpa = testing.allocator;
|
||||
var patterns: std.ArrayList([]u8) = .empty;
|
||||
defer {
|
||||
for (patterns.items) |p| gpa.free(p);
|
||||
patterns.deinit(gpa);
|
||||
}
|
||||
var rows: std.ArrayList(model.Rule) = .empty;
|
||||
defer rows.deinit(gpa);
|
||||
|
||||
var i: usize = 0;
|
||||
while (i < 64) : (i += 1) {
|
||||
const regex_pattern = try std.fmt.allocPrint(gpa, "^r{d}-[0-9]+\\.ads\\.invalid$", .{i});
|
||||
try patterns.append(gpa, regex_pattern);
|
||||
try rows.append(gpa, rule(regex_pattern, .regex, .block));
|
||||
|
||||
const wild = try std.fmt.allocPrint(gpa, "*.w{d:0>5}.example.com", .{i});
|
||||
try patterns.append(gpa, wild);
|
||||
try rows.append(gpa, rule(wild, .wildcard, .block));
|
||||
|
||||
const exact = try std.fmt.allocPrint(gpa, "e{d:0>5}.example.com", .{i});
|
||||
try patterns.append(gpa, exact);
|
||||
try rows.append(gpa, rule(exact, .exact, .block));
|
||||
}
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(gpa);
|
||||
defer arena.deinit();
|
||||
const set = try RuleSet.build(arena.allocator(), gpa, rows.items, 0x5eed);
|
||||
|
||||
try testing.expectEqual(@as(usize, 64), set.regex_block.len);
|
||||
try testing.expect(set.exact_block.contains("e00007.example.com"));
|
||||
// Whole-arena capacity against what the set says it holds. The slack is the
|
||||
// allocator's page rounding; the defect this guards against was a factor of
|
||||
// twelve.
|
||||
try testing.expect(arena.queryCapacity() < 2 * set.memoryBytes());
|
||||
}
|
||||
|
||||
fn buildUnderFailure(gpa: Allocator) !void {
|
||||
@@ -341,11 +615,14 @@ fn buildUnderFailure(gpa: Allocator) !void {
|
||||
rule("good.example.com", .exact, .allow),
|
||||
rule("*.tracker.net", .wildcard, .block),
|
||||
rule("*.ok.tracker.net", .wildcard, .allow),
|
||||
rule("^ad[0-9]+-", .regex, .block),
|
||||
rule("\\.ok\\.", .regex, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(gpa, &rows, 0x5eed);
|
||||
var set = try RuleSet.build(gpa, gpa, &rows, 0x5eed);
|
||||
defer set.deinit(gpa);
|
||||
try testing.expect(set.exact_block.contains("ads.example.com"));
|
||||
try testing.expectEqualStrings("*.tracker.net", set.wildcard_block[0]);
|
||||
try testing.expectEqualStrings("^ad[0-9]+-", set.regex_block[0].pattern);
|
||||
}
|
||||
|
||||
test "build leaks nothing under allocation failure" {
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
//! A pattern is a domain name in which one or more labels are exactly `*`.
|
||||
//! Each `*` label matches one or more labels of the queried name. Partial-label
|
||||
//! globbing (`ad*.example.com`) is deliberately absent: it is regex by another
|
||||
//! name, which PLAN §2.2 rules out.
|
||||
//! name, and PLAN §2.2 keeps one regex dialect rather than two. An operator who
|
||||
//! needs one writes a `.regex` rule, which `filter/regex.zig` compiles.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
@@ -18,8 +19,9 @@ pub const PatternError = error{
|
||||
/// No label is exactly "*".
|
||||
NoWildcard,
|
||||
/// A label contains '*' but is not exactly "*". Partial-label globbing
|
||||
/// (`ad*.example.com`) is out of scope: it is regex by another name, and
|
||||
/// PLAN §3.9 defines the wildcard as a label pattern.
|
||||
/// (`ad*.example.com`) is out of scope: PLAN §3.9 defines the wildcard as a
|
||||
/// label pattern, and the `.regex` kind covers what partial globbing was
|
||||
/// wanted for.
|
||||
PartialWildcardLabel,
|
||||
EmptyLabel,
|
||||
LabelTooLong,
|
||||
|
||||
+151
-13
@@ -23,6 +23,8 @@ pub const Step = struct { version: u32, sql: [:0]const u8 };
|
||||
pub const steps = [_]Step{
|
||||
.{ .version = 1, .sql = config_schema.ddl_v1 },
|
||||
.{ .version = 2, .sql = ddl_v2 },
|
||||
.{ .version = 3, .sql = ddl_v3 },
|
||||
.{ .version = 4, .sql = ddl_v4 },
|
||||
};
|
||||
|
||||
/// The DoT verification name (`upstreams.tls_name`). Empty keeps the pre-step-2
|
||||
@@ -31,6 +33,42 @@ const ddl_v2: [:0]const u8 =
|
||||
\\ALTER TABLE upstreams ADD COLUMN tls_name TEXT NOT NULL DEFAULT '';
|
||||
;
|
||||
|
||||
/// Written `.allow` entries per source: the `@@||name^` exceptions a downloaded
|
||||
/// list carries. 0 is what every source stands at until its next refresh
|
||||
/// recompiles it, which is also what a list with no exceptions keeps.
|
||||
const ddl_v3: [:0]const u8 =
|
||||
\\ALTER TABLE blocklist_sources ADD COLUMN exception_count INTEGER NOT NULL DEFAULT 0;
|
||||
;
|
||||
|
||||
/// The `regex` rule kind. A `CHECK` constraint cannot be altered in place, and
|
||||
/// `config_schema.ddl_v1` is frozen, so the table is rebuilt: SQLite's
|
||||
/// documented ALTER TABLE procedure, reduced to the steps this table needs.
|
||||
///
|
||||
/// The rebuilt table keeps the name `rules` and its whole v1 shape, ids
|
||||
/// included, because `config_schema.table_names` and the invariant tests below
|
||||
/// assert the schema's table set and a rename would fail all three.
|
||||
///
|
||||
/// The steps this table does not need: no index, trigger or view names `rules`,
|
||||
/// and no other table references it, so nothing outside the four statements has
|
||||
/// to be recreated or repointed. The procedure's `PRAGMA foreign_keys=OFF` is
|
||||
/// deliberately absent — it is a no-op inside a transaction, and `migrate` runs
|
||||
/// every step in one. It is also unneeded here: `rules` is a child of `groups`
|
||||
/// and a parent of nothing, so dropping it violates no reference.
|
||||
const ddl_v4: [:0]const u8 =
|
||||
\\CREATE TABLE rules_v4 (
|
||||
\\ id INTEGER PRIMARY KEY,
|
||||
\\ group_id INTEGER NOT NULL REFERENCES groups(id) ON DELETE CASCADE,
|
||||
\\ pattern TEXT NOT NULL,
|
||||
\\ kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard','regex')),
|
||||
\\ action TEXT NOT NULL CHECK(action IN ('allow','block')),
|
||||
\\ created_at INTEGER NOT NULL
|
||||
\\);
|
||||
\\INSERT INTO rules_v4 (id, group_id, pattern, kind, action, created_at)
|
||||
\\ SELECT id, group_id, pattern, kind, action, created_at FROM rules;
|
||||
\\DROP TABLE rules;
|
||||
\\ALTER TABLE rules_v4 RENAME TO rules;
|
||||
;
|
||||
|
||||
/// The schema version this binary expects. A database `readVersion` reports
|
||||
/// below this needs `nxdns run` to migrate it; above it is `error.SchemaTooNew`
|
||||
/// and needs a newer nxdns.
|
||||
@@ -264,16 +302,26 @@ fn columnExists(database: *db.Db, table: []const u8, column: []const u8) !bool {
|
||||
return stmt.columnInt(0) != 0;
|
||||
}
|
||||
|
||||
test "a fresh database reaches version 2 with the tls_name column" {
|
||||
test "a fresh database reaches version 4 with both added columns and the third rule kind" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
try testing.expectEqual(@as(u32, 2), try migrate(&database));
|
||||
try testing.expectEqual(@as(u32, 2), target_version);
|
||||
try testing.expectEqual(@as(u32, 4), try migrate(&database));
|
||||
try testing.expectEqual(@as(u32, 4), target_version);
|
||||
try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
|
||||
try testing.expect(try columnExists(&database, "blocklist_sources", "exception_count"));
|
||||
|
||||
try database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, '^ad[0-9]+-', 'regex', 'block', 100);
|
||||
);
|
||||
try testing.expectError(error.Constraint, database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, 'x', 'glob', 'block', 100);
|
||||
));
|
||||
}
|
||||
|
||||
test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_name" {
|
||||
test "a version 1 database upgrades and keeps its rows with an empty tls_name" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
@@ -282,8 +330,8 @@ test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_na
|
||||
try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
|
||||
try database.exec("INSERT INTO upstreams (url, priority, enabled) VALUES ('tls://1.1.1.1:853', 10, 1);");
|
||||
|
||||
try testing.expectEqual(@as(u32, 2), try migrate(&database));
|
||||
try testing.expectEqual(@as(u32, 2), try readVersion(&database));
|
||||
try testing.expectEqual(target_version, try migrate(&database));
|
||||
try testing.expectEqual(target_version, try readVersion(&database));
|
||||
try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
|
||||
|
||||
var stmt = try database.prepare("SELECT url, tls_name FROM upstreams");
|
||||
@@ -293,22 +341,112 @@ test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_na
|
||||
try testing.expectEqualStrings("", stmt.columnText(1));
|
||||
}
|
||||
|
||||
test "a failing step after step 2 rolls back the whole upgrade from version 1" {
|
||||
test "a version 2 database upgrades and keeps its sources at exception_count 0" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
const through_two = [_]Step{ steps[0], steps[1] };
|
||||
try testing.expectEqual(@as(u32, 2), try migrateSteps(&database, &through_two));
|
||||
try testing.expect(!try columnExists(&database, "blocklist_sources", "exception_count"));
|
||||
try database.exec(
|
||||
\\INSERT INTO blocklist_sources (url, name, domain_count, wildcard_count, checksum)
|
||||
\\VALUES ('https://lists.example/ads.txt', 'ads', 12, 3, 'abc');
|
||||
);
|
||||
|
||||
try testing.expectEqual(target_version, try migrate(&database));
|
||||
try testing.expect(try columnExists(&database, "blocklist_sources", "exception_count"));
|
||||
|
||||
// The counters and the checksum of a source compiled before this milestone
|
||||
// survive: the checksum is what keeps its compiled files loadable without a
|
||||
// refetch, so a migration that disturbed it would cost every household a
|
||||
// full re-download.
|
||||
var stmt = try database.prepare(
|
||||
"SELECT domain_count, wildcard_count, exception_count, checksum FROM blocklist_sources",
|
||||
);
|
||||
defer stmt.deinit();
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqual(@as(i64, 12), stmt.columnInt(0));
|
||||
try testing.expectEqual(@as(i64, 3), stmt.columnInt(1));
|
||||
try testing.expectEqual(@as(i64, 0), stmt.columnInt(2));
|
||||
try testing.expectEqualStrings("abc", stmt.columnText(3));
|
||||
}
|
||||
|
||||
test "a version 3 database upgrades to 4 with its rules intact and the third kind admitted" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
const through_three = [_]Step{ steps[0], steps[1], steps[2] };
|
||||
try testing.expectEqual(@as(u32, 3), try migrateSteps(&database, &through_three));
|
||||
try database.exec(
|
||||
\\INSERT INTO groups (id, name) VALUES (2, 'kids');
|
||||
\\INSERT INTO rules (id, group_id, pattern, kind, action, created_at) VALUES
|
||||
\\ (7, 1, 'ads.example', 'exact', 'block', 1000),
|
||||
\\ (9, 2, '*.tracker.net', 'wildcard', 'allow', 2000);
|
||||
);
|
||||
// Before step 4 the frozen v1 CHECK admits two kinds only.
|
||||
try testing.expectError(error.Constraint, database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, '^ad', 'regex', 'block', 3000);
|
||||
));
|
||||
|
||||
try testing.expectEqual(@as(u32, 4), try migrate(&database));
|
||||
|
||||
// The rebuild is a copy, so every column of every row survives it — ids
|
||||
// included, because `group_sources` aside, an API client holds rule ids and
|
||||
// a renumbering would silently repoint every bookmark and every ETag.
|
||||
var stmt = try database.prepare(
|
||||
"SELECT id, group_id, pattern, kind, action, created_at FROM rules ORDER BY id",
|
||||
);
|
||||
defer stmt.deinit();
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqual(@as(i64, 7), stmt.columnInt(0));
|
||||
try testing.expectEqual(@as(i64, 1), stmt.columnInt(1));
|
||||
try testing.expectEqualStrings("ads.example", stmt.columnText(2));
|
||||
try testing.expectEqualStrings("exact", stmt.columnText(3));
|
||||
try testing.expectEqualStrings("block", stmt.columnText(4));
|
||||
try testing.expectEqual(@as(i64, 1000), stmt.columnInt(5));
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqual(@as(i64, 9), stmt.columnInt(0));
|
||||
try testing.expectEqual(@as(i64, 2), stmt.columnInt(1));
|
||||
try testing.expectEqualStrings("*.tracker.net", stmt.columnText(2));
|
||||
try testing.expectEqualStrings("wildcard", stmt.columnText(3));
|
||||
try testing.expectEqualStrings("allow", stmt.columnText(4));
|
||||
try testing.expect(!try stmt.step());
|
||||
|
||||
try database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, '^ad[0-9]+-', 'regex', 'block', 3000);
|
||||
);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try database.queryInt("SELECT count(*) FROM rules WHERE kind = 'regex'"),
|
||||
);
|
||||
|
||||
// The scaffolding table is gone and the foreign key came back with the
|
||||
// rebuild: deleting a group still takes its rules with it.
|
||||
try testing.expect(!try tableExists(&database, "rules_v4"));
|
||||
try database.exec("DELETE FROM groups WHERE id = 2;");
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try database.queryInt("SELECT count(*) FROM rules WHERE group_id = 2"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a failing step after the last released one rolls back the whole upgrade from version 1" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
|
||||
_ = try migrateSteps(&database, &first);
|
||||
|
||||
const broken = [_]Step{
|
||||
steps[0],
|
||||
steps[1],
|
||||
.{ .version = 3, .sql = "CREATE TABLE third (" },
|
||||
const broken = steps ++ [_]Step{
|
||||
.{ .version = target_version + 1, .sql = "CREATE TABLE third (" },
|
||||
};
|
||||
try testing.expectError(error.Unexpected, migrateSteps(&database, &broken));
|
||||
|
||||
// One transaction: the ALTER TABLE of step 2 went back with step 3.
|
||||
// One transaction: the ALTER TABLEs of the released steps went back with it.
|
||||
try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
|
||||
try testing.expect(!try columnExists(&database, "blocklist_sources", "exception_count"));
|
||||
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
|
||||
}
|
||||
|
||||
@@ -346,7 +484,7 @@ test "readVersion reads a file database through an immutable open, writing nothi
|
||||
var database = try db.Db.open(path, .{ .mode = .{ .immutable = testing.io } });
|
||||
defer database.close();
|
||||
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
|
||||
try testing.expectEqual(@as(u32, 2), target_version);
|
||||
try testing.expectEqual(@as(u32, 4), target_version);
|
||||
|
||||
// A write through this connection is refused by SQLite, not by convention.
|
||||
try testing.expectError(error.ReadOnly, database.exec("DELETE FROM schema_version;"));
|
||||
|
||||
@@ -281,6 +281,31 @@ test "rules round-trip in group, kind, action, pattern, id order" {
|
||||
);
|
||||
}
|
||||
|
||||
test "a regex rule round-trips through the table with its pattern untouched" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
var ids = try seedGroupIds();
|
||||
defer ids.deinit(testing.allocator);
|
||||
|
||||
// Uppercase, a trailing metacharacter and a backslash escape: everything
|
||||
// the name-shaped kinds normalize away and this one must not.
|
||||
const pattern = "^AD[0-9]+-\\.example\\.";
|
||||
try insertRule(&database, .{
|
||||
.group = "default",
|
||||
.pattern = pattern,
|
||||
.kind = .regex,
|
||||
.action = .block,
|
||||
}, .{ .now = 1700000000, .group_ids = &ids });
|
||||
|
||||
var items = try listRules(&database, testing.allocator);
|
||||
defer items.deinit(testing.allocator);
|
||||
defer freeRules(testing.allocator, items.items);
|
||||
|
||||
try testing.expectEqual(@as(usize, 1), items.items.len);
|
||||
try testing.expectEqual(model.RuleKind.regex, items.items[0].kind);
|
||||
try testing.expectEqualStrings(pattern, items.items[0].pattern);
|
||||
}
|
||||
|
||||
test "a duplicate rule is accepted and stays deterministically ordered by id" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
//! `blocklist_sources`.
|
||||
//!
|
||||
//! Only the four configuration columns are read and written. `last_updated`,
|
||||
//! `domain_count`, `wildcard_count`, `skipped_regex_count` and `checksum` are
|
||||
//! facts a running server produces; an insert leaves them at their column
|
||||
//! defaults so two exports taken minutes apart stay identical.
|
||||
//! `domain_count`, `wildcard_count`, `exception_count`, `skipped_regex_count`
|
||||
//! and `checksum` are facts a running server produces; an insert leaves them at
|
||||
//! their column defaults so two exports taken minutes apart stay identical.
|
||||
//!
|
||||
//! The import path is list / insert / deleteAll / count; the runtime columns and
|
||||
//! the REST surface follow it, both keyed by row id.
|
||||
@@ -88,6 +88,10 @@ pub const SourceRow = struct {
|
||||
last_updated: ?i64,
|
||||
domain_count: i64,
|
||||
wildcard_count: i64,
|
||||
/// Written `.allow` entries: the `@@||name^` exceptions the list carries.
|
||||
/// Defaulted for the same reason `is_suggested` is — the blocklist manager
|
||||
/// builds `SourceRow` values from the refresh columns alone.
|
||||
exception_count: i64 = 0,
|
||||
skipped_regex_count: i64,
|
||||
checksum: ?[]const u8,
|
||||
};
|
||||
@@ -96,15 +100,19 @@ pub const SourceStats = struct {
|
||||
last_updated: i64,
|
||||
domain_count: i64,
|
||||
wildcard_count: i64,
|
||||
exception_count: i64,
|
||||
skipped_regex_count: i64,
|
||||
/// Lowercase hex sha256 over the `.list` body followed by the `.wild` body.
|
||||
/// Lowercase hex sha256 over the `.list` body, then the `.wild` body, then
|
||||
/// the `.allow` body. The allow body is hashed last so an empty one leaves
|
||||
/// the digest of a two-body compile unchanged, which is what keeps a
|
||||
/// checksum written before exceptions were honoured valid.
|
||||
checksum: []const u8,
|
||||
};
|
||||
|
||||
const row_columns_sql =
|
||||
\\SELECT id, url, name, enabled, last_updated,
|
||||
\\ domain_count, wildcard_count, skipped_regex_count, checksum,
|
||||
\\ is_suggested
|
||||
\\ is_suggested, exception_count
|
||||
\\ FROM blocklist_sources
|
||||
;
|
||||
|
||||
@@ -133,6 +141,7 @@ fn readSourceRow(stmt: *db.Stmt, gpa: Allocator) db.Error!SourceRow {
|
||||
.last_updated = if (stmt.isNull(4)) null else stmt.columnInt(4),
|
||||
.domain_count = stmt.columnInt(5),
|
||||
.wildcard_count = stmt.columnInt(6),
|
||||
.exception_count = stmt.columnInt(10),
|
||||
.skipped_regex_count = stmt.columnInt(7),
|
||||
.checksum = checksum,
|
||||
};
|
||||
@@ -150,7 +159,7 @@ pub fn freeSourceRows(gpa: Allocator, items: []const SourceRow) void {
|
||||
const update_stats_sql =
|
||||
\\UPDATE blocklist_sources
|
||||
\\ SET last_updated = ?2, domain_count = ?3, wildcard_count = ?4,
|
||||
\\ skipped_regex_count = ?5, checksum = ?6
|
||||
\\ skipped_regex_count = ?5, checksum = ?6, exception_count = ?7
|
||||
\\ WHERE id = ?1
|
||||
;
|
||||
|
||||
@@ -165,6 +174,7 @@ pub fn updateSourceStats(database: *db.Db, id: i64, stats: SourceStats) db.Error
|
||||
try stmt.bindInt(4, stats.wildcard_count);
|
||||
try stmt.bindInt(5, stats.skipped_regex_count);
|
||||
try stmt.bindText(6, stats.checksum);
|
||||
try stmt.bindInt(7, stats.exception_count);
|
||||
try stmt.exec();
|
||||
}
|
||||
|
||||
@@ -304,7 +314,10 @@ test "insertBlocklistSource leaves the runtime columns at their defaults" {
|
||||
);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try database.queryInt("SELECT sum(domain_count + wildcard_count + skipped_regex_count) FROM blocklist_sources"),
|
||||
try database.queryInt(
|
||||
"SELECT sum(domain_count + wildcard_count + exception_count + skipped_regex_count)" ++
|
||||
" FROM blocklist_sources",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -355,6 +368,7 @@ test "listSourceRows returns row ids and the runtime columns in url order" {
|
||||
try testing.expectEqual(@as(?[]const u8, null), row.checksum);
|
||||
try testing.expectEqual(@as(i64, 0), row.domain_count);
|
||||
try testing.expectEqual(@as(i64, 0), row.wildcard_count);
|
||||
try testing.expectEqual(@as(i64, 0), row.exception_count);
|
||||
try testing.expectEqual(@as(i64, 0), row.skipped_regex_count);
|
||||
}
|
||||
}
|
||||
@@ -373,6 +387,7 @@ test "updateSourceStats writes the runtime columns of one source only" {
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 4321,
|
||||
.wildcard_count = 21,
|
||||
.exception_count = 9,
|
||||
.skipped_regex_count = 7,
|
||||
.checksum = "a" ** 64,
|
||||
});
|
||||
@@ -389,6 +404,7 @@ test "updateSourceStats writes the runtime columns of one source only" {
|
||||
try testing.expectEqual(@as(?i64, 1_700_000_000), updated.last_updated);
|
||||
try testing.expectEqual(@as(i64, 4321), updated.domain_count);
|
||||
try testing.expectEqual(@as(i64, 21), updated.wildcard_count);
|
||||
try testing.expectEqual(@as(i64, 9), updated.exception_count);
|
||||
try testing.expectEqual(@as(i64, 7), updated.skipped_regex_count);
|
||||
try testing.expectEqualStrings("a" ** 64, updated.checksum.?);
|
||||
|
||||
@@ -405,6 +421,7 @@ fn listSourceRowsUnderFailure(gpa: Allocator) !void {
|
||||
.last_updated = 1,
|
||||
.domain_count = 2,
|
||||
.wildcard_count = 3,
|
||||
.exception_count = 5,
|
||||
.skipped_regex_count = 4,
|
||||
.checksum = "b" ** 64,
|
||||
});
|
||||
@@ -470,6 +487,7 @@ test "updateSource leaves the runtime columns where the refresh path left them"
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 12,
|
||||
.wildcard_count = 3,
|
||||
.exception_count = 2,
|
||||
.skipped_regex_count = 1,
|
||||
.checksum = "c" ** 64,
|
||||
});
|
||||
|
||||
@@ -58,6 +58,7 @@ comptime {
|
||||
_ = @import("filter/parser_domains.zig");
|
||||
_ = @import("filter/parser_abp.zig");
|
||||
_ = @import("filter/wildcard.zig");
|
||||
_ = @import("filter/regex.zig");
|
||||
_ = @import("filter/domain_set.zig");
|
||||
_ = @import("filter/rules.zig");
|
||||
_ = @import("filter/matcher.zig");
|
||||
|
||||
@@ -59,6 +59,7 @@ pub const StatusView = struct {
|
||||
last_error: []const u8,
|
||||
domains: u32,
|
||||
wildcards: u32,
|
||||
exceptions: u32,
|
||||
skipped_regex: u32,
|
||||
|
||||
pub fn from(status: *const manager_mod.SourceStatus) StatusView {
|
||||
@@ -72,6 +73,7 @@ pub const StatusView = struct {
|
||||
.last_error = status.errorText(),
|
||||
.domains = status.counts.domains,
|
||||
.wildcards = status.counts.wildcards,
|
||||
.exceptions = status.counts.exceptions,
|
||||
.skipped_regex = status.counts.skipped_regex,
|
||||
};
|
||||
}
|
||||
@@ -317,6 +319,7 @@ test "editing a blocklist keeps the counters the refresh wrote" {
|
||||
.last_updated = 1700,
|
||||
.domain_count = 42,
|
||||
.wildcard_count = 3,
|
||||
.exception_count = 2,
|
||||
.skipped_regex_count = 1,
|
||||
.checksum = "abc",
|
||||
});
|
||||
@@ -380,7 +383,7 @@ test "a status becomes the flat shape the API answers with" {
|
||||
const message = "connection refused";
|
||||
@memcpy(status.last_error[0..message.len], message);
|
||||
status.last_error_len = message.len;
|
||||
status.counts = .{ .domains = 10, .wildcards = 2, .skipped_regex = 1 };
|
||||
status.counts = .{ .domains = 10, .wildcards = 2, .exceptions = 4, .skipped_regex = 1 };
|
||||
|
||||
const view: StatusView = .from(&status);
|
||||
try testing.expectEqual(@as(i64, 7), view.id);
|
||||
@@ -389,4 +392,5 @@ test "a status becomes the flat shape the API answers with" {
|
||||
try testing.expectEqualStrings(url, view.url);
|
||||
try testing.expectEqualStrings(message, view.last_error);
|
||||
try testing.expectEqual(@as(u32, 10), view.domains);
|
||||
try testing.expectEqual(@as(u32, 4), view.exceptions);
|
||||
}
|
||||
|
||||
@@ -38,6 +38,8 @@ pub const Body = struct {
|
||||
reason: []const u8,
|
||||
/// The rule or list entry that decided it; "" when nothing matched.
|
||||
matched: []const u8,
|
||||
/// The list that decided it, which for `blocklist_exception` is the list
|
||||
/// whose `@@` rule lifted the block rather than one that made it.
|
||||
source_url: ?[]const u8,
|
||||
safe_search_rewrite: ?[]const u8,
|
||||
};
|
||||
@@ -50,7 +52,8 @@ pub const Result = struct {
|
||||
blocked: bool,
|
||||
reason: matcher.Reason,
|
||||
matched: []const u8,
|
||||
/// `blocklist_sources` row id of the list that matched.
|
||||
/// `blocklist_sources` row id of the list that matched, whether it blocked
|
||||
/// the name or lifted it through an `@@` exception.
|
||||
source_id: ?i64,
|
||||
safe_search_rewrite: ?[]const u8,
|
||||
};
|
||||
|
||||
@@ -39,7 +39,7 @@ const Created = union(enum) { id: i64, fail: Failure };
|
||||
/// not understood.
|
||||
fn toInput(body: Body) union(enum) { input: rules_repo.RuleInput, fail: Failure } {
|
||||
const kind = model.RuleKind.fromDb(body.kind) orelse
|
||||
return .{ .fail = .{ .invalid = "kind must be 'exact' or 'wildcard'" } };
|
||||
return .{ .fail = .{ .invalid = "kind must be 'exact', 'wildcard' or 'regex'" } };
|
||||
const action = model.RuleAction.fromDb(body.action) orelse
|
||||
return .{ .fail = .{ .invalid = "action must be 'allow' or 'block'" } };
|
||||
return .{ .input = .{
|
||||
@@ -308,7 +308,7 @@ test "an unknown kind or action is a 400 before anything is written" {
|
||||
try testing.expect(toInput(.{
|
||||
.group_id = 1,
|
||||
.pattern = "ads.example",
|
||||
.kind = "regex",
|
||||
.kind = "glob",
|
||||
.action = "block",
|
||||
}).fail == .invalid);
|
||||
|
||||
@@ -327,4 +327,46 @@ test "an unknown kind or action is a 400 before anything is written" {
|
||||
});
|
||||
try testing.expectEqual(model.RuleKind.wildcard, good.input.kind);
|
||||
try testing.expectEqual(model.RuleAction.allow, good.input.action);
|
||||
|
||||
const third = toInput(.{
|
||||
.group_id = 1,
|
||||
.pattern = "^ad[0-9]+-",
|
||||
.kind = "regex",
|
||||
.action = "block",
|
||||
});
|
||||
try testing.expectEqual(model.RuleKind.regex, third.input.kind);
|
||||
}
|
||||
|
||||
test "a regex rule is stored, and a pattern the engine refuses is a 400 that names it" {
|
||||
var bench: mutations.Bench = undefined;
|
||||
try bench.init(testing.allocator);
|
||||
defer bench.deinit(testing.allocator);
|
||||
|
||||
const created = try applyCreate(&bench.state, bench.io(), bench.arena(), .{
|
||||
.group_id = 1,
|
||||
.pattern = "^ad[0-9]+-",
|
||||
.kind = .regex,
|
||||
.action = .block,
|
||||
});
|
||||
const row = (try rules_repo.getRule(&bench.database, bench.arena(), created.id)).?;
|
||||
try testing.expectEqual(model.RuleKind.regex, row.kind);
|
||||
// Stored verbatim: a regex is not a name, so nothing lowercases or
|
||||
// dot-strips it on the way to the table.
|
||||
try testing.expectEqualStrings("^ad[0-9]+-", row.pattern);
|
||||
|
||||
const unclosed = try applyCreate(&bench.state, bench.io(), bench.arena(), .{
|
||||
.group_id = 1,
|
||||
.pattern = "(",
|
||||
.kind = .regex,
|
||||
.action = .block,
|
||||
});
|
||||
try testing.expectEqualStrings(
|
||||
"rules[0].pattern: '(' is not a valid regex pattern",
|
||||
unclosed.fail.invalid,
|
||||
);
|
||||
|
||||
// The refused pattern reached no table, and the good one is still the only
|
||||
// row: a 400 costs no write and no reload.
|
||||
try testing.expectEqual(@as(i64, 1), try bench.queryInt("SELECT count(*) FROM rules"));
|
||||
try testing.expectEqual(@as(usize, 1), bench.reloads);
|
||||
}
|
||||
|
||||
@@ -1876,7 +1876,7 @@ components:
|
||||
|
||||
Blocklist:
|
||||
type: object
|
||||
required: [id, url, name, enabled, is_suggested, last_updated, domain_count, wildcard_count, skipped_regex_count, checksum]
|
||||
required: [id, url, name, enabled, is_suggested, last_updated, domain_count, wildcard_count, exception_count, skipped_regex_count, checksum]
|
||||
properties:
|
||||
id: { type: integer }
|
||||
url: { type: string }
|
||||
@@ -1888,6 +1888,7 @@ components:
|
||||
nullable: true
|
||||
domain_count: { type: integer }
|
||||
wildcard_count: { type: integer }
|
||||
exception_count: { type: integer }
|
||||
skipped_regex_count: { type: integer }
|
||||
checksum:
|
||||
type: string
|
||||
@@ -1918,7 +1919,7 @@ components:
|
||||
|
||||
SourceStatus:
|
||||
type: object
|
||||
required: [id, state, loaded, last_attempt, last_success, url, last_error, domains, wildcards, skipped_regex]
|
||||
required: [id, state, loaded, last_attempt, last_success, url, last_error, domains, wildcards, exceptions, skipped_regex]
|
||||
properties:
|
||||
id: { type: integer }
|
||||
state:
|
||||
@@ -1933,6 +1934,7 @@ components:
|
||||
description: Empty when the last attempt succeeded.
|
||||
domains: { type: integer }
|
||||
wildcards: { type: integer }
|
||||
exceptions: { type: integer }
|
||||
skipped_regex: { type: integer }
|
||||
|
||||
Rule:
|
||||
@@ -1945,7 +1947,7 @@ components:
|
||||
pattern: { type: string }
|
||||
kind:
|
||||
type: string
|
||||
enum: [exact, wildcard]
|
||||
enum: [exact, wildcard, regex]
|
||||
action:
|
||||
type: string
|
||||
enum: [allow, block]
|
||||
@@ -1959,7 +1961,7 @@ components:
|
||||
pattern: { type: string }
|
||||
kind:
|
||||
type: string
|
||||
enum: [exact, wildcard]
|
||||
enum: [exact, wildcard, regex]
|
||||
action:
|
||||
type: string
|
||||
enum: [allow, block]
|
||||
@@ -1973,7 +1975,7 @@ components:
|
||||
pattern: { type: string }
|
||||
kind:
|
||||
type: string
|
||||
enum: [exact, wildcard]
|
||||
enum: [exact, wildcard, regex]
|
||||
action:
|
||||
type: string
|
||||
enum: [allow, block]
|
||||
|
||||
@@ -1665,6 +1665,9 @@ fn createdId(body: []const u8) !i64 {
|
||||
return std.fmt.parseInt(i64, rest[0..end], 10);
|
||||
}
|
||||
|
||||
/// The three files a refresh publishes for one source. The `.allow` file is
|
||||
/// written here too: the delete path has to take every compiled body, and a
|
||||
/// sweep that missed one would leave an orphan this test could not see.
|
||||
fn writeCompiled(io: std.Io, dir: std.Io.Dir, id: i64, body: []const u8) !void {
|
||||
var buf: [64]u8 = undefined;
|
||||
try dir.writeFile(io, .{
|
||||
@@ -1675,6 +1678,10 @@ fn writeCompiled(io: std.Io, dir: std.Io.Dir, id: i64, body: []const u8) !void {
|
||||
.sub_path = try std.fmt.bufPrint(&buf, "{d}.wild", .{id}),
|
||||
.data = "",
|
||||
});
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&buf, "{d}.allow", .{id}),
|
||||
.data = "",
|
||||
});
|
||||
}
|
||||
|
||||
fn accessCompiled(io: std.Io, dir: std.Io.Dir, id: i64) !void {
|
||||
@@ -1726,6 +1733,11 @@ fn deleteSweepsCompiledFiles(io: std.Io, env: *Env) anyerror!void {
|
||||
try std.fmt.bufPrint(&name_buf, "{d}.wild", .{doomed}),
|
||||
.{},
|
||||
));
|
||||
try testing.expectError(error.FileNotFound, dir.access(
|
||||
io,
|
||||
try std.fmt.bufPrint(&name_buf, "{d}.allow", .{doomed}),
|
||||
.{},
|
||||
));
|
||||
try accessCompiled(io, dir, kept);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user