milestone 21: abp list exceptions and a regex rule kind
This commit is contained in:
+151
-13
@@ -23,6 +23,8 @@ pub const Step = struct { version: u32, sql: [:0]const u8 };
|
||||
pub const steps = [_]Step{
|
||||
.{ .version = 1, .sql = config_schema.ddl_v1 },
|
||||
.{ .version = 2, .sql = ddl_v2 },
|
||||
.{ .version = 3, .sql = ddl_v3 },
|
||||
.{ .version = 4, .sql = ddl_v4 },
|
||||
};
|
||||
|
||||
/// The DoT verification name (`upstreams.tls_name`). Empty keeps the pre-step-2
|
||||
@@ -31,6 +33,42 @@ const ddl_v2: [:0]const u8 =
|
||||
\\ALTER TABLE upstreams ADD COLUMN tls_name TEXT NOT NULL DEFAULT '';
|
||||
;
|
||||
|
||||
/// Written `.allow` entries per source: the `@@||name^` exceptions a downloaded
|
||||
/// list carries. 0 is what every source stands at until its next refresh
|
||||
/// recompiles it, which is also what a list with no exceptions keeps.
|
||||
const ddl_v3: [:0]const u8 =
|
||||
\\ALTER TABLE blocklist_sources ADD COLUMN exception_count INTEGER NOT NULL DEFAULT 0;
|
||||
;
|
||||
|
||||
/// The `regex` rule kind. A `CHECK` constraint cannot be altered in place, and
|
||||
/// `config_schema.ddl_v1` is frozen, so the table is rebuilt: SQLite's
|
||||
/// documented ALTER TABLE procedure, reduced to the steps this table needs.
|
||||
///
|
||||
/// The rebuilt table keeps the name `rules` and its whole v1 shape, ids
|
||||
/// included, because `config_schema.table_names` and the invariant tests below
|
||||
/// assert the schema's table set and a rename would fail all three.
|
||||
///
|
||||
/// The steps this table does not need: no index, trigger or view names `rules`,
|
||||
/// and no other table references it, so nothing outside the four statements has
|
||||
/// to be recreated or repointed. The procedure's `PRAGMA foreign_keys=OFF` is
|
||||
/// deliberately absent — it is a no-op inside a transaction, and `migrate` runs
|
||||
/// every step in one. It is also unneeded here: `rules` is a child of `groups`
|
||||
/// and a parent of nothing, so dropping it violates no reference.
|
||||
const ddl_v4: [:0]const u8 =
|
||||
\\CREATE TABLE rules_v4 (
|
||||
\\ id INTEGER PRIMARY KEY,
|
||||
\\ group_id INTEGER NOT NULL REFERENCES groups(id) ON DELETE CASCADE,
|
||||
\\ pattern TEXT NOT NULL,
|
||||
\\ kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard','regex')),
|
||||
\\ action TEXT NOT NULL CHECK(action IN ('allow','block')),
|
||||
\\ created_at INTEGER NOT NULL
|
||||
\\);
|
||||
\\INSERT INTO rules_v4 (id, group_id, pattern, kind, action, created_at)
|
||||
\\ SELECT id, group_id, pattern, kind, action, created_at FROM rules;
|
||||
\\DROP TABLE rules;
|
||||
\\ALTER TABLE rules_v4 RENAME TO rules;
|
||||
;
|
||||
|
||||
/// The schema version this binary expects. A database `readVersion` reports
|
||||
/// below this needs `nxdns run` to migrate it; above it is `error.SchemaTooNew`
|
||||
/// and needs a newer nxdns.
|
||||
@@ -264,16 +302,26 @@ fn columnExists(database: *db.Db, table: []const u8, column: []const u8) !bool {
|
||||
return stmt.columnInt(0) != 0;
|
||||
}
|
||||
|
||||
test "a fresh database reaches version 2 with the tls_name column" {
|
||||
test "a fresh database reaches version 4 with both added columns and the third rule kind" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
try testing.expectEqual(@as(u32, 2), try migrate(&database));
|
||||
try testing.expectEqual(@as(u32, 2), target_version);
|
||||
try testing.expectEqual(@as(u32, 4), try migrate(&database));
|
||||
try testing.expectEqual(@as(u32, 4), target_version);
|
||||
try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
|
||||
try testing.expect(try columnExists(&database, "blocklist_sources", "exception_count"));
|
||||
|
||||
try database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, '^ad[0-9]+-', 'regex', 'block', 100);
|
||||
);
|
||||
try testing.expectError(error.Constraint, database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, 'x', 'glob', 'block', 100);
|
||||
));
|
||||
}
|
||||
|
||||
test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_name" {
|
||||
test "a version 1 database upgrades and keeps its rows with an empty tls_name" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
@@ -282,8 +330,8 @@ test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_na
|
||||
try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
|
||||
try database.exec("INSERT INTO upstreams (url, priority, enabled) VALUES ('tls://1.1.1.1:853', 10, 1);");
|
||||
|
||||
try testing.expectEqual(@as(u32, 2), try migrate(&database));
|
||||
try testing.expectEqual(@as(u32, 2), try readVersion(&database));
|
||||
try testing.expectEqual(target_version, try migrate(&database));
|
||||
try testing.expectEqual(target_version, try readVersion(&database));
|
||||
try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
|
||||
|
||||
var stmt = try database.prepare("SELECT url, tls_name FROM upstreams");
|
||||
@@ -293,22 +341,112 @@ test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_na
|
||||
try testing.expectEqualStrings("", stmt.columnText(1));
|
||||
}
|
||||
|
||||
test "a failing step after step 2 rolls back the whole upgrade from version 1" {
|
||||
test "a version 2 database upgrades and keeps its sources at exception_count 0" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
const through_two = [_]Step{ steps[0], steps[1] };
|
||||
try testing.expectEqual(@as(u32, 2), try migrateSteps(&database, &through_two));
|
||||
try testing.expect(!try columnExists(&database, "blocklist_sources", "exception_count"));
|
||||
try database.exec(
|
||||
\\INSERT INTO blocklist_sources (url, name, domain_count, wildcard_count, checksum)
|
||||
\\VALUES ('https://lists.example/ads.txt', 'ads', 12, 3, 'abc');
|
||||
);
|
||||
|
||||
try testing.expectEqual(target_version, try migrate(&database));
|
||||
try testing.expect(try columnExists(&database, "blocklist_sources", "exception_count"));
|
||||
|
||||
// The counters and the checksum of a source compiled before this milestone
|
||||
// survive: the checksum is what keeps its compiled files loadable without a
|
||||
// refetch, so a migration that disturbed it would cost every household a
|
||||
// full re-download.
|
||||
var stmt = try database.prepare(
|
||||
"SELECT domain_count, wildcard_count, exception_count, checksum FROM blocklist_sources",
|
||||
);
|
||||
defer stmt.deinit();
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqual(@as(i64, 12), stmt.columnInt(0));
|
||||
try testing.expectEqual(@as(i64, 3), stmt.columnInt(1));
|
||||
try testing.expectEqual(@as(i64, 0), stmt.columnInt(2));
|
||||
try testing.expectEqualStrings("abc", stmt.columnText(3));
|
||||
}
|
||||
|
||||
test "a version 3 database upgrades to 4 with its rules intact and the third kind admitted" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
const through_three = [_]Step{ steps[0], steps[1], steps[2] };
|
||||
try testing.expectEqual(@as(u32, 3), try migrateSteps(&database, &through_three));
|
||||
try database.exec(
|
||||
\\INSERT INTO groups (id, name) VALUES (2, 'kids');
|
||||
\\INSERT INTO rules (id, group_id, pattern, kind, action, created_at) VALUES
|
||||
\\ (7, 1, 'ads.example', 'exact', 'block', 1000),
|
||||
\\ (9, 2, '*.tracker.net', 'wildcard', 'allow', 2000);
|
||||
);
|
||||
// Before step 4 the frozen v1 CHECK admits two kinds only.
|
||||
try testing.expectError(error.Constraint, database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, '^ad', 'regex', 'block', 3000);
|
||||
));
|
||||
|
||||
try testing.expectEqual(@as(u32, 4), try migrate(&database));
|
||||
|
||||
// The rebuild is a copy, so every column of every row survives it — ids
|
||||
// included, because `group_sources` aside, an API client holds rule ids and
|
||||
// a renumbering would silently repoint every bookmark and every ETag.
|
||||
var stmt = try database.prepare(
|
||||
"SELECT id, group_id, pattern, kind, action, created_at FROM rules ORDER BY id",
|
||||
);
|
||||
defer stmt.deinit();
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqual(@as(i64, 7), stmt.columnInt(0));
|
||||
try testing.expectEqual(@as(i64, 1), stmt.columnInt(1));
|
||||
try testing.expectEqualStrings("ads.example", stmt.columnText(2));
|
||||
try testing.expectEqualStrings("exact", stmt.columnText(3));
|
||||
try testing.expectEqualStrings("block", stmt.columnText(4));
|
||||
try testing.expectEqual(@as(i64, 1000), stmt.columnInt(5));
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqual(@as(i64, 9), stmt.columnInt(0));
|
||||
try testing.expectEqual(@as(i64, 2), stmt.columnInt(1));
|
||||
try testing.expectEqualStrings("*.tracker.net", stmt.columnText(2));
|
||||
try testing.expectEqualStrings("wildcard", stmt.columnText(3));
|
||||
try testing.expectEqualStrings("allow", stmt.columnText(4));
|
||||
try testing.expect(!try stmt.step());
|
||||
|
||||
try database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, '^ad[0-9]+-', 'regex', 'block', 3000);
|
||||
);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try database.queryInt("SELECT count(*) FROM rules WHERE kind = 'regex'"),
|
||||
);
|
||||
|
||||
// The scaffolding table is gone and the foreign key came back with the
|
||||
// rebuild: deleting a group still takes its rules with it.
|
||||
try testing.expect(!try tableExists(&database, "rules_v4"));
|
||||
try database.exec("DELETE FROM groups WHERE id = 2;");
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try database.queryInt("SELECT count(*) FROM rules WHERE group_id = 2"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a failing step after the last released one rolls back the whole upgrade from version 1" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
|
||||
_ = try migrateSteps(&database, &first);
|
||||
|
||||
const broken = [_]Step{
|
||||
steps[0],
|
||||
steps[1],
|
||||
.{ .version = 3, .sql = "CREATE TABLE third (" },
|
||||
const broken = steps ++ [_]Step{
|
||||
.{ .version = target_version + 1, .sql = "CREATE TABLE third (" },
|
||||
};
|
||||
try testing.expectError(error.Unexpected, migrateSteps(&database, &broken));
|
||||
|
||||
// One transaction: the ALTER TABLE of step 2 went back with step 3.
|
||||
// One transaction: the ALTER TABLEs of the released steps went back with it.
|
||||
try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
|
||||
try testing.expect(!try columnExists(&database, "blocklist_sources", "exception_count"));
|
||||
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
|
||||
}
|
||||
|
||||
@@ -346,7 +484,7 @@ test "readVersion reads a file database through an immutable open, writing nothi
|
||||
var database = try db.Db.open(path, .{ .mode = .{ .immutable = testing.io } });
|
||||
defer database.close();
|
||||
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
|
||||
try testing.expectEqual(@as(u32, 2), target_version);
|
||||
try testing.expectEqual(@as(u32, 4), target_version);
|
||||
|
||||
// A write through this connection is refused by SQLite, not by convention.
|
||||
try testing.expectError(error.ReadOnly, database.exec("DELETE FROM schema_version;"));
|
||||
|
||||
Reference in New Issue
Block a user