milestone 21: abp list exceptions and a regex rule kind

src/config/model.zig

@@ -1,6 +1,9 @@
-//! The one configuration model. Bootstrap, import, export, the repositories and
-//! the running server all speak this struct; nothing else describes nxdns
-//! configuration.
+//! The one *declarative* configuration model: loading, reconciliation, import,
+//! export and the running server all speak this struct, and it is the whole
+//! shape of a config file. It is not the only shape the repositories accept —
+//! the API edits rows one at a time through narrower inputs such as
+//! `RuleInput`, `ClientInput` and `ClientEdit`, so a field added here does not
+//! reach those paths by itself.
 //!
 //! Pure: no `std.Io` value is a parameter anywhere, no SQLite, no clock. The
 //! only `std.Io` types that appear are `std.Io.Duration` as a conversion result.
@@ -8,11 +11,21 @@
 //! Runtime columns are deliberately absent. `clients.first_seen`,
 //! `clients.last_seen`, `rules.created_at` and
 //! `blocklist_sources.{last_updated, domain_count, wildcard_count,
-//! skipped_regex_count, checksum}` are facts a running server produces, not
-//! configuration. Including them would make two exports taken minutes apart
-//! differ, which would make the byte-stable round trip untestable against a
-//! live server. Import sets the timestamps to the import time and leaves the
-//! counters at their column defaults.
+//! exception_count, skipped_regex_count, checksum}` are facts a running server
+//! produces, not configuration. Including them would make two exports taken
+//! minutes apart differ, which would make the byte-stable round trip untestable
+//! against a live server.
+//!
+//! Declarative configuration reaches the database through exactly one path:
+//! `config/reconcile.zig`. `nxdns import` is a thin wrapper over it, and so is
+//! `run --config`. Reconciliation asks what changed rather than replacing
+//! wholesale, so a row the input still names keeps the runtime state attached
+//! to it: a source matched by url keeps its id, checksum and counters, a rule
+//! keeps its `created_at`, and a client keeps its first-seen and last-seen
+//! stamps. The source id and checksum are the two that decide whether a
+//! file-mode restart reuses the compiled bodies or downloads them again:
+//! `loadSource` names the files after the id and accepts them only against the
+//! stored checksum. The counters ride along as reported state.
 
 const std = @import("std");
 const Allocator = std.mem.Allocator;
@@ -256,20 +269,25 @@ pub const BlocklistSource = struct {
 
 pub const GroupSource = struct { group: []const u8, source_url: []const u8 };
 
+/// The three spellings `CHECK(kind IN ('exact','wildcard','regex'))` admits
+/// after migration step 4.
 pub const RuleKind = enum {
     exact,
     wildcard,
+    regex,
 
     pub fn toDb(self: RuleKind) []const u8 {
         return switch (self) {
             .exact => "exact",
             .wildcard => "wildcard",
+            .regex => "regex",
         };
     }
 
     pub fn fromDb(text: []const u8) ?RuleKind {
         if (std.mem.eql(u8, text, "exact")) return .exact;
         if (std.mem.eql(u8, text, "wildcard")) return .wildcard;
+        if (std.mem.eql(u8, text, "regex")) return .regex;
         return null;
     }
 };
@@ -789,6 +807,12 @@ test "every toDb and fromDb enum pair round-trips over all tags" {
     try expectEnumRoundTrip(RecordType);
 }
 
+test "RuleKind carries the third kind through export and import" {
+    try testing.expectEqualStrings("regex", RuleKind.regex.toDb());
+    try testing.expectEqual(RuleKind.regex, RuleKind.fromDb("regex").?);
+    try testing.expect(RuleKind.fromDb("Regex") == null);
+}
+
 test "RecordType stores the uppercase DDL spelling" {
     try testing.expectEqualStrings("A", RecordType.a.toDb());
     try testing.expectEqualStrings("AAAA", RecordType.aaaa.toDb());

src/config/reconcile.zig

@@ -3,10 +3,10 @@
 //!
 //! The defect this module exists to fix: `import.applyToDb` deletes and
 //! reinserts every row, `blocklist_sources` included, and the compiled
-//! blocklists are named after the source row id (`<id>.list` / `<id>.wild`). A
-//! configuration re-applied on every boot would therefore hand every source a
-//! new id, orphan every compiled file, and re-download every blocklist on every
-//! restart.
+//! blocklists are named after the source row id (`<id>.list` / `<id>.wild` /
+//! `<id>.allow`). A configuration re-applied on every boot would therefore hand
+//! every source a new id, orphan every compiled file, and re-download every
+//! blocklist on every restart.
 //!
 //! So nothing is wiped. Every table has an identity; a row the file and the
 //! database agree on is **updated in place**, keeping its row id and every
@@ -1100,6 +1100,7 @@ fn seedSourceStats(database: *db.Db, id: i64) !void {
         .last_updated = 1_700_000_000,
         .domain_count = 4321,
         .wildcard_count = 21,
+        .exception_count = 9,
         .skipped_regex_count = 7,
         .checksum = "a" ** 64,
     });
@@ -1169,6 +1170,7 @@ test "a source keeps its id, its checksum and its counters across a reconcile" {
     try testing.expectEqualStrings("advertising", row.name);
     try testing.expectEqual(@as(?i64, 1_700_000_000), row.last_updated);
     try testing.expectEqual(@as(i64, 4321), row.domain_count);
+    try testing.expectEqual(@as(i64, 9), row.exception_count);
     try testing.expectEqualStrings("a" ** 64, row.checksum.?);
 }
 
@@ -1550,6 +1552,50 @@ test "rules keep created_at across a reconcile, duplicates included" {
     }
 }
 
+test "a regex rule declared in the file converges into the table and back out" {
+    var bench: Bench = undefined;
+    try bench.init();
+    defer bench.deinit();
+
+    // Under `.managed_file` authority the API refuses rule writes, so this is
+    // the only way a regex rule reaches the table in that mode. `reconcileRules`
+    // compares the whole tuple and needs no code of its own for the new kind.
+    const with_regex: [:0]const u8 =
+        \\.{
+        \\    .groups = .{ .{ .name = "default" } },
+        \\    .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
+        \\    .rules = .{
+        \\        .{ .group = "default", .pattern = "^ad[0-9]+-", .kind = .regex, .action = .block },
+        \\    },
+        \\}
+    ;
+    const first = try bench.apply(with_regex, 1_700_000_000);
+    try testing.expectEqual(@as(u32, 1), first.rules.inserted);
+
+    const gpa = testing.allocator;
+    var rows = try rules_repo.listRuleRows(&bench.database, gpa);
+    defer rows.deinit(gpa);
+    defer rules_repo.freeRuleRows(gpa, rows.items);
+    try testing.expectEqual(@as(usize, 1), rows.items.len);
+    try testing.expectEqual(model.RuleKind.regex, rows.items[0].kind);
+    try testing.expectEqualStrings("^ad[0-9]+-", rows.items[0].pattern);
+
+    // Idempotent: the tuple matches itself, so a second pass writes nothing.
+    const second = try bench.apply(with_regex, 1_800_000_000);
+    try testing.expectEqual(@as(u32, 0), second.rules.total());
+
+    // And a file that stops declaring it takes the row with it.
+    const without: [:0]const u8 =
+        \\.{
+        \\    .groups = .{ .{ .name = "default" } },
+        \\    .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
+        \\}
+    ;
+    const third = try bench.apply(without, 1_900_000_000);
+    try testing.expectEqual(@as(u32, 1), third.rules.deleted);
+    try testing.expectEqual(@as(i64, 0), try rules_repo.countRules(&bench.database));
+}
+
 test "dropping one of two identical rules removes exactly one row" {
     var bench: Bench = undefined;
     try bench.init();

src/config/validate.zig

@@ -44,6 +44,7 @@ const Writer = std.Io.Writer;
 const model = @import("model.zig");
 const address = @import("../platform/address.zig");
 const dns_name = @import("../dns/name.zig");
+const regex = @import("../filter/regex.zig");
 const safe_url = @import("../safe_url.zig");
 const transport = @import("../upstream/transport.zig");
 
@@ -890,13 +891,31 @@ fn checkCollections(cfg: Config, diags: *Diagnostics, scratch: Allocator) error{
 
     for (cfg.rules, 0..) |rule, i| {
         try checkGroupRef(diags, &group_names, rule.group, "rules[{d}].group", .{i});
-        if (!try patternIsValid(scratch, rule.pattern, rule.kind)) {
+        // `null` is a good pattern; anything else is the sentence fragment that
+        // says which of the regex engine's limits refused it. The empty string
+        // is a plain syntax refusal, which is the only verdict the exact and
+        // wildcard kinds can reach.
+        const detail: ?[]const u8 = if (patternIsValid(scratch, rule.pattern, rule.kind)) |valid|
+            (if (valid) null else "")
+        else |err| switch (err) {
+            error.OutOfMemory => return error.OutOfMemory,
+            error.BadPattern => "",
+            error.PatternTooLong => std.fmt.comptimePrint(
+                " (over {d} bytes)",
+                .{regex.max_pattern_len},
+            ),
+            error.PatternTooComplex => std.fmt.comptimePrint(
+                " (over {d} compiled instructions)",
+                .{regex.max_program_len},
+            ),
+        };
+        if (detail) |suffix| {
             try diags.add(
                 error.BadRulePattern,
                 "rules[{d}].pattern",
                 .{i},
-                "{f} is not a valid {s} pattern",
-                .{ safe_url.quoteText(rule.pattern), rule.kind.toDb() },
+                "{f} is not a valid {s} pattern{s}",
+                .{ safe_url.quoteText(rule.pattern), rule.kind.toDb(), suffix },
             );
         }
     }
@@ -1096,11 +1115,18 @@ fn sourceUrlIsValid(url: []const u8) bool {
 /// Syntax only. Matching semantics are Phase 5's: an exact pattern carries no
 /// `*` at all, a wildcard pattern carries at least one label that is exactly
 /// `*`, and every remaining label must survive `dns.name.fromText`.
+///
+/// A regex pattern is validated by compiling it, and its three refusals arrive
+/// as errors rather than as `false` so the caller can name the one that fired.
+/// The distinction is the operator's, not the compiler's: "not a valid regex
+/// pattern" sends someone hunting for a typo in a pattern whose only fault is
+/// that it is longer than `regex.max_pattern_len` or wider than
+/// `regex.max_program_len`, and neither limit is visible in the pattern text.
 fn patternIsValid(
     scratch: Allocator,
     pattern: []const u8,
     kind: model.RuleKind,
-) error{OutOfMemory}!bool {
+) regex.Error!bool {
     switch (kind) {
         .exact => {
             if (std.mem.findScalar(u8, pattern, '*') != null) return false;
@@ -1126,6 +1152,11 @@ fn patternIsValid(
             _ = dns_name.fromText(substituted.items) catch return false;
             return true;
         },
+        .regex => {
+            var program = try regex.compile(scratch, pattern);
+            program.deinit(scratch);
+            return true;
+        },
     }
 }
 
@@ -2105,6 +2136,82 @@ test "rule patterns accept wildcards only when the kind says so" {
     try expectProblem(bad_label, error.BadRulePattern, "rules[0].pattern");
 }
 
+/// The message of the first failure, so a test can assert the sentence an
+/// operator reads and not only the error tag.
+fn expectMessage(cfg: Config, expected: ValidateError, expected_message: []const u8) !void {
+    var diags: Diagnostics = .init(testing.allocator);
+    defer diags.deinit();
+    try testing.expectError(expected, validate(cfg, &diags));
+    const failure = diags.firstFailure() orelse return error.TestExpectedFailure;
+    try testing.expectEqualStrings(expected_message, failure.message);
+}
+
+/// The tail of the first failure's message. `quoteText` truncates the value it
+/// quotes at `safe_url.max_len`, so a diagnostic about an over-long pattern
+/// cannot be matched whole.
+fn expectMessageSuffix(cfg: Config, expected: ValidateError, expected_suffix: []const u8) !void {
+    var diags: Diagnostics = .init(testing.allocator);
+    defer diags.deinit();
+    try testing.expectError(expected, validate(cfg, &diags));
+    const failure = diags.firstFailure() orelse return error.TestExpectedFailure;
+    if (!std.mem.endsWith(u8, failure.message, expected_suffix)) {
+        std.debug.print("message {s} does not end with {s}\n", .{ failure.message, expected_suffix });
+        return error.TestExpectedEqual;
+    }
+}
+
+fn regexRule(pattern: []const u8) [1]model.Rule {
+    return .{.{ .group = "default", .pattern = pattern, .kind = .regex, .action = .block }};
+}
+
+test "a regex rule is validated by compiling it" {
+    var cfg = baseConfig();
+    const good = regexRule("^ad[0-9]+-\\.(example|test)\\.com$");
+    cfg.rules = &good;
+    try expectClean(cfg);
+
+    // A regex is not a name: the wildcard and exact kinds reject `*`, and this
+    // one has to accept the characters that make a pattern a pattern.
+    const starred = regexRule("ads.*\\.example");
+    cfg.rules = &starred;
+    try expectClean(cfg);
+}
+
+test "each of the regex engine's three refusals names itself in the diagnostic" {
+    var cfg = baseConfig();
+
+    const unclosed = regexRule("(");
+    cfg.rules = &unclosed;
+    try expectProblem(cfg, error.BadRulePattern, "rules[0].pattern");
+    try expectMessage(cfg, error.BadRulePattern, "'(' is not a valid regex pattern");
+
+    // Too long and too complex are the two an operator cannot see by reading
+    // the pattern, so the message has to carry the limit that fired.
+    const too_long = regexRule("a" ** (regex.max_pattern_len + 1));
+    cfg.rules = &too_long;
+    try expectMessageSuffix(
+        cfg,
+        error.BadRulePattern,
+        "is not a valid regex pattern (over 256 bytes)",
+    );
+
+    // Well inside 256 bytes of pattern, well past 1024 instructions of program.
+    const too_complex = regexRule("(abcdefghij){200}");
+    cfg.rules = &too_complex;
+    try expectMessage(
+        cfg,
+        error.BadRulePattern,
+        "'(abcdefghij){200}' is not a valid regex pattern (over 1024 compiled instructions)",
+    );
+}
+
+test "an empty regex pattern is refused rather than matching every name" {
+    var cfg = baseConfig();
+    const empty = regexRule("");
+    cfg.rules = ∅
+    try expectProblem(cfg, error.BadRulePattern, "rules[0].pattern");
+}
+
 test "parseResolver accepts udp and tcp with an IP literal and a port" {
     const udp4 = try parseResolver("udp://192.168.1.1:53");
     try testing.expectEqual(ResolverScheme.udp, udp4.scheme);