milestone 21: abp list exceptions and a regex rule kind

src/storage/migrations.zig

@@ -23,6 +23,8 @@ pub const Step = struct { version: u32, sql: [:0]const u8 };
 pub const steps = [_]Step{
     .{ .version = 1, .sql = config_schema.ddl_v1 },
     .{ .version = 2, .sql = ddl_v2 },
+    .{ .version = 3, .sql = ddl_v3 },
+    .{ .version = 4, .sql = ddl_v4 },
 };
 
 /// The DoT verification name (`upstreams.tls_name`). Empty keeps the pre-step-2
@@ -31,6 +33,42 @@ const ddl_v2: [:0]const u8 =
     \\ALTER TABLE upstreams ADD COLUMN tls_name TEXT NOT NULL DEFAULT '';
 ;
 
+/// Written `.allow` entries per source: the `@@||name^` exceptions a downloaded
+/// list carries. 0 is what every source stands at until its next refresh
+/// recompiles it, which is also what a list with no exceptions keeps.
+const ddl_v3: [:0]const u8 =
+    \\ALTER TABLE blocklist_sources ADD COLUMN exception_count INTEGER NOT NULL DEFAULT 0;
+;
+
+/// The `regex` rule kind. A `CHECK` constraint cannot be altered in place, and
+/// `config_schema.ddl_v1` is frozen, so the table is rebuilt: SQLite's
+/// documented ALTER TABLE procedure, reduced to the steps this table needs.
+///
+/// The rebuilt table keeps the name `rules` and its whole v1 shape, ids
+/// included, because `config_schema.table_names` and the invariant tests below
+/// assert the schema's table set and a rename would fail all three.
+///
+/// The steps this table does not need: no index, trigger or view names `rules`,
+/// and no other table references it, so nothing outside the four statements has
+/// to be recreated or repointed. The procedure's `PRAGMA foreign_keys=OFF` is
+/// deliberately absent — it is a no-op inside a transaction, and `migrate` runs
+/// every step in one. It is also unneeded here: `rules` is a child of `groups`
+/// and a parent of nothing, so dropping it violates no reference.
+const ddl_v4: [:0]const u8 =
+    \\CREATE TABLE rules_v4 (
+    \\    id INTEGER PRIMARY KEY,
+    \\    group_id INTEGER NOT NULL REFERENCES groups(id) ON DELETE CASCADE,
+    \\    pattern TEXT NOT NULL,
+    \\    kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard','regex')),
+    \\    action TEXT NOT NULL CHECK(action IN ('allow','block')),
+    \\    created_at INTEGER NOT NULL
+    \\);
+    \\INSERT INTO rules_v4 (id, group_id, pattern, kind, action, created_at)
+    \\    SELECT id, group_id, pattern, kind, action, created_at FROM rules;
+    \\DROP TABLE rules;
+    \\ALTER TABLE rules_v4 RENAME TO rules;
+;
+
 /// The schema version this binary expects. A database `readVersion` reports
 /// below this needs `nxdns run` to migrate it; above it is `error.SchemaTooNew`
 /// and needs a newer nxdns.
@@ -264,16 +302,26 @@ fn columnExists(database: *db.Db, table: []const u8, column: []const u8) !bool {
     return stmt.columnInt(0) != 0;
 }
 
-test "a fresh database reaches version 2 with the tls_name column" {
+test "a fresh database reaches version 4 with both added columns and the third rule kind" {
     var database = try openMigrated();
     defer database.close();
 
-    try testing.expectEqual(@as(u32, 2), try migrate(&database));
-    try testing.expectEqual(@as(u32, 2), target_version);
+    try testing.expectEqual(@as(u32, 4), try migrate(&database));
+    try testing.expectEqual(@as(u32, 4), target_version);
     try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
+    try testing.expect(try columnExists(&database, "blocklist_sources", "exception_count"));
+
+    try database.exec(
+        \\INSERT INTO rules (group_id, pattern, kind, action, created_at)
+        \\VALUES (1, '^ad[0-9]+-', 'regex', 'block', 100);
+    );
+    try testing.expectError(error.Constraint, database.exec(
+        \\INSERT INTO rules (group_id, pattern, kind, action, created_at)
+        \\VALUES (1, 'x', 'glob', 'block', 100);
+    ));
 }
 
-test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_name" {
+test "a version 1 database upgrades and keeps its rows with an empty tls_name" {
     var database = try openMigrated();
     defer database.close();
 
@@ -282,8 +330,8 @@ test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_na
     try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
     try database.exec("INSERT INTO upstreams (url, priority, enabled) VALUES ('tls://1.1.1.1:853', 10, 1);");
 
-    try testing.expectEqual(@as(u32, 2), try migrate(&database));
-    try testing.expectEqual(@as(u32, 2), try readVersion(&database));
+    try testing.expectEqual(target_version, try migrate(&database));
+    try testing.expectEqual(target_version, try readVersion(&database));
     try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
 
     var stmt = try database.prepare("SELECT url, tls_name FROM upstreams");
@@ -293,22 +341,112 @@ test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_na
     try testing.expectEqualStrings("", stmt.columnText(1));
 }
 
-test "a failing step after step 2 rolls back the whole upgrade from version 1" {
+test "a version 2 database upgrades and keeps its sources at exception_count 0" {
+    var database = try openMigrated();
+    defer database.close();
+
+    const through_two = [_]Step{ steps[0], steps[1] };
+    try testing.expectEqual(@as(u32, 2), try migrateSteps(&database, &through_two));
+    try testing.expect(!try columnExists(&database, "blocklist_sources", "exception_count"));
+    try database.exec(
+        \\INSERT INTO blocklist_sources (url, name, domain_count, wildcard_count, checksum)
+        \\VALUES ('https://lists.example/ads.txt', 'ads', 12, 3, 'abc');
+    );
+
+    try testing.expectEqual(target_version, try migrate(&database));
+    try testing.expect(try columnExists(&database, "blocklist_sources", "exception_count"));
+
+    // The counters and the checksum of a source compiled before this milestone
+    // survive: the checksum is what keeps its compiled files loadable without a
+    // refetch, so a migration that disturbed it would cost every household a
+    // full re-download.
+    var stmt = try database.prepare(
+        "SELECT domain_count, wildcard_count, exception_count, checksum FROM blocklist_sources",
+    );
+    defer stmt.deinit();
+    try testing.expect(try stmt.step());
+    try testing.expectEqual(@as(i64, 12), stmt.columnInt(0));
+    try testing.expectEqual(@as(i64, 3), stmt.columnInt(1));
+    try testing.expectEqual(@as(i64, 0), stmt.columnInt(2));
+    try testing.expectEqualStrings("abc", stmt.columnText(3));
+}
+
+test "a version 3 database upgrades to 4 with its rules intact and the third kind admitted" {
+    var database = try openMigrated();
+    defer database.close();
+
+    const through_three = [_]Step{ steps[0], steps[1], steps[2] };
+    try testing.expectEqual(@as(u32, 3), try migrateSteps(&database, &through_three));
+    try database.exec(
+        \\INSERT INTO groups (id, name) VALUES (2, 'kids');
+        \\INSERT INTO rules (id, group_id, pattern, kind, action, created_at) VALUES
+        \\    (7, 1, 'ads.example', 'exact', 'block', 1000),
+        \\    (9, 2, '*.tracker.net', 'wildcard', 'allow', 2000);
+    );
+    // Before step 4 the frozen v1 CHECK admits two kinds only.
+    try testing.expectError(error.Constraint, database.exec(
+        \\INSERT INTO rules (group_id, pattern, kind, action, created_at)
+        \\VALUES (1, '^ad', 'regex', 'block', 3000);
+    ));
+
+    try testing.expectEqual(@as(u32, 4), try migrate(&database));
+
+    // The rebuild is a copy, so every column of every row survives it — ids
+    // included, because `group_sources` aside, an API client holds rule ids and
+    // a renumbering would silently repoint every bookmark and every ETag.
+    var stmt = try database.prepare(
+        "SELECT id, group_id, pattern, kind, action, created_at FROM rules ORDER BY id",
+    );
+    defer stmt.deinit();
+    try testing.expect(try stmt.step());
+    try testing.expectEqual(@as(i64, 7), stmt.columnInt(0));
+    try testing.expectEqual(@as(i64, 1), stmt.columnInt(1));
+    try testing.expectEqualStrings("ads.example", stmt.columnText(2));
+    try testing.expectEqualStrings("exact", stmt.columnText(3));
+    try testing.expectEqualStrings("block", stmt.columnText(4));
+    try testing.expectEqual(@as(i64, 1000), stmt.columnInt(5));
+    try testing.expect(try stmt.step());
+    try testing.expectEqual(@as(i64, 9), stmt.columnInt(0));
+    try testing.expectEqual(@as(i64, 2), stmt.columnInt(1));
+    try testing.expectEqualStrings("*.tracker.net", stmt.columnText(2));
+    try testing.expectEqualStrings("wildcard", stmt.columnText(3));
+    try testing.expectEqualStrings("allow", stmt.columnText(4));
+    try testing.expect(!try stmt.step());
+
+    try database.exec(
+        \\INSERT INTO rules (group_id, pattern, kind, action, created_at)
+        \\VALUES (1, '^ad[0-9]+-', 'regex', 'block', 3000);
+    );
+    try testing.expectEqual(
+        @as(i64, 1),
+        try database.queryInt("SELECT count(*) FROM rules WHERE kind = 'regex'"),
+    );
+
+    // The scaffolding table is gone and the foreign key came back with the
+    // rebuild: deleting a group still takes its rules with it.
+    try testing.expect(!try tableExists(&database, "rules_v4"));
+    try database.exec("DELETE FROM groups WHERE id = 2;");
+    try testing.expectEqual(
+        @as(i64, 0),
+        try database.queryInt("SELECT count(*) FROM rules WHERE group_id = 2"),
+    );
+}
+
+test "a failing step after the last released one rolls back the whole upgrade from version 1" {
     var database = try openMigrated();
     defer database.close();
 
     const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
     _ = try migrateSteps(&database, &first);
 
-    const broken = [_]Step{
-        steps[0],
-        steps[1],
-        .{ .version = 3, .sql = "CREATE TABLE third (" },
+    const broken = steps ++ [_]Step{
+        .{ .version = target_version + 1, .sql = "CREATE TABLE third (" },
     };
     try testing.expectError(error.Unexpected, migrateSteps(&database, &broken));
 
-    // One transaction: the ALTER TABLE of step 2 went back with step 3.
+    // One transaction: the ALTER TABLEs of the released steps went back with it.
     try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
+    try testing.expect(!try columnExists(&database, "blocklist_sources", "exception_count"));
     try testing.expectEqual(@as(u32, 1), try readVersion(&database));
 }
 
@@ -346,7 +484,7 @@ test "readVersion reads a file database through an immutable open, writing nothi
     var database = try db.Db.open(path, .{ .mode = .{ .immutable = testing.io } });
     defer database.close();
     try testing.expectEqual(@as(u32, 1), try readVersion(&database));
-    try testing.expectEqual(@as(u32, 2), target_version);
+    try testing.expectEqual(@as(u32, 4), target_version);
 
     // A write through this connection is refused by SQLite, not by convention.
     try testing.expectError(error.ReadOnly, database.exec("DELETE FROM schema_version;"));

src/storage/repositories/rules_repo.zig

@@ -281,6 +281,31 @@ test "rules round-trip in group, kind, action, pattern, id order" {
     );
 }
 
+test "a regex rule round-trips through the table with its pattern untouched" {
+    var database = try openMigrated();
+    defer database.close();
+    var ids = try seedGroupIds();
+    defer ids.deinit(testing.allocator);
+
+    // Uppercase, a trailing metacharacter and a backslash escape: everything
+    // the name-shaped kinds normalize away and this one must not.
+    const pattern = "^AD[0-9]+-\\.example\\.";
+    try insertRule(&database, .{
+        .group = "default",
+        .pattern = pattern,
+        .kind = .regex,
+        .action = .block,
+    }, .{ .now = 1700000000, .group_ids = &ids });
+
+    var items = try listRules(&database, testing.allocator);
+    defer items.deinit(testing.allocator);
+    defer freeRules(testing.allocator, items.items);
+
+    try testing.expectEqual(@as(usize, 1), items.items.len);
+    try testing.expectEqual(model.RuleKind.regex, items.items[0].kind);
+    try testing.expectEqualStrings(pattern, items.items[0].pattern);
+}
+
 test "a duplicate rule is accepted and stays deterministically ordered by id" {
     var database = try openMigrated();
     defer database.close();

src/storage/repositories/sources_repo.zig

@@ -1,9 +1,9 @@
 //! `blocklist_sources`.
 //!
 //! Only the four configuration columns are read and written. `last_updated`,
-//! `domain_count`, `wildcard_count`, `skipped_regex_count` and `checksum` are
-//! facts a running server produces; an insert leaves them at their column
-//! defaults so two exports taken minutes apart stay identical.
+//! `domain_count`, `wildcard_count`, `exception_count`, `skipped_regex_count`
+//! and `checksum` are facts a running server produces; an insert leaves them at
+//! their column defaults so two exports taken minutes apart stay identical.
 //!
 //! The import path is list / insert / deleteAll / count; the runtime columns and
 //! the REST surface follow it, both keyed by row id.
@@ -88,6 +88,10 @@ pub const SourceRow = struct {
     last_updated: ?i64,
     domain_count: i64,
     wildcard_count: i64,
+    /// Written `.allow` entries: the `@@||name^` exceptions the list carries.
+    /// Defaulted for the same reason `is_suggested` is — the blocklist manager
+    /// builds `SourceRow` values from the refresh columns alone.
+    exception_count: i64 = 0,
     skipped_regex_count: i64,
     checksum: ?[]const u8,
 };
@@ -96,15 +100,19 @@ pub const SourceStats = struct {
     last_updated: i64,
     domain_count: i64,
     wildcard_count: i64,
+    exception_count: i64,
     skipped_regex_count: i64,
-    /// Lowercase hex sha256 over the `.list` body followed by the `.wild` body.
+    /// Lowercase hex sha256 over the `.list` body, then the `.wild` body, then
+    /// the `.allow` body. The allow body is hashed last so an empty one leaves
+    /// the digest of a two-body compile unchanged, which is what keeps a
+    /// checksum written before exceptions were honoured valid.
     checksum: []const u8,
 };
 
 const row_columns_sql =
     \\SELECT id, url, name, enabled, last_updated,
     \\       domain_count, wildcard_count, skipped_regex_count, checksum,
-    \\       is_suggested
+    \\       is_suggested, exception_count
     \\  FROM blocklist_sources
 ;
 
@@ -133,6 +141,7 @@ fn readSourceRow(stmt: *db.Stmt, gpa: Allocator) db.Error!SourceRow {
         .last_updated = if (stmt.isNull(4)) null else stmt.columnInt(4),
         .domain_count = stmt.columnInt(5),
         .wildcard_count = stmt.columnInt(6),
+        .exception_count = stmt.columnInt(10),
         .skipped_regex_count = stmt.columnInt(7),
         .checksum = checksum,
     };
@@ -150,7 +159,7 @@ pub fn freeSourceRows(gpa: Allocator, items: []const SourceRow) void {
 const update_stats_sql =
     \\UPDATE blocklist_sources
     \\   SET last_updated = ?2, domain_count = ?3, wildcard_count = ?4,
-    \\       skipped_regex_count = ?5, checksum = ?6
+    \\       skipped_regex_count = ?5, checksum = ?6, exception_count = ?7
     \\ WHERE id = ?1
 ;
 
@@ -165,6 +174,7 @@ pub fn updateSourceStats(database: *db.Db, id: i64, stats: SourceStats) db.Error
     try stmt.bindInt(4, stats.wildcard_count);
     try stmt.bindInt(5, stats.skipped_regex_count);
     try stmt.bindText(6, stats.checksum);
+    try stmt.bindInt(7, stats.exception_count);
     try stmt.exec();
 }
 
@@ -304,7 +314,10 @@ test "insertBlocklistSource leaves the runtime columns at their defaults" {
     );
     try testing.expectEqual(
         @as(i64, 0),
-        try database.queryInt("SELECT sum(domain_count + wildcard_count + skipped_regex_count) FROM blocklist_sources"),
+        try database.queryInt(
+            "SELECT sum(domain_count + wildcard_count + exception_count + skipped_regex_count)" ++
+                " FROM blocklist_sources",
+        ),
     );
 }
 
@@ -355,6 +368,7 @@ test "listSourceRows returns row ids and the runtime columns in url order" {
         try testing.expectEqual(@as(?[]const u8, null), row.checksum);
         try testing.expectEqual(@as(i64, 0), row.domain_count);
         try testing.expectEqual(@as(i64, 0), row.wildcard_count);
+        try testing.expectEqual(@as(i64, 0), row.exception_count);
         try testing.expectEqual(@as(i64, 0), row.skipped_regex_count);
     }
 }
@@ -373,6 +387,7 @@ test "updateSourceStats writes the runtime columns of one source only" {
         .last_updated = 1_700_000_000,
         .domain_count = 4321,
         .wildcard_count = 21,
+        .exception_count = 9,
         .skipped_regex_count = 7,
         .checksum = "a" ** 64,
     });
@@ -389,6 +404,7 @@ test "updateSourceStats writes the runtime columns of one source only" {
     try testing.expectEqual(@as(?i64, 1_700_000_000), updated.last_updated);
     try testing.expectEqual(@as(i64, 4321), updated.domain_count);
     try testing.expectEqual(@as(i64, 21), updated.wildcard_count);
+    try testing.expectEqual(@as(i64, 9), updated.exception_count);
     try testing.expectEqual(@as(i64, 7), updated.skipped_regex_count);
     try testing.expectEqualStrings("a" ** 64, updated.checksum.?);
 
@@ -405,6 +421,7 @@ fn listSourceRowsUnderFailure(gpa: Allocator) !void {
         .last_updated = 1,
         .domain_count = 2,
         .wildcard_count = 3,
+        .exception_count = 5,
         .skipped_regex_count = 4,
         .checksum = "b" ** 64,
     });
@@ -470,6 +487,7 @@ test "updateSource leaves the runtime columns where the refresh path left them"
         .last_updated = 1_700_000_000,
         .domain_count = 12,
         .wildcard_count = 3,
+        .exception_count = 2,
         .skipped_regex_count = 1,
         .checksum = "c" ** 64,
     });