schema: collapse config.db to a single baseline, ddl_v1 stays editable until v0.1

This commit is contained in:
2026-08-13 19:14:54 +02:00
parent 2ab7c1f1de
commit 21571e448e
9 changed files with 67 additions and 189 deletions
+14 -11
View File
@@ -91,7 +91,7 @@ Two SQLite files with opposite write profiles, isolated from each other:
### 3.7 Upgrades: Auto-Migration (Decision J)
- `config.db`: numbered, sequential SQL migration steps compiled into the binary. At startup: read schema version row, apply newer steps inside a transaction, continue. Operator upgrade = install binary, restart.
- `config.db`: numbered, sequential SQL migration steps compiled into the binary. At startup: read schema version row, apply newer steps inside a transaction, continue. Operator upgrade = install binary, restart. Before v0.1 the list holds one step — the baseline of §11.2, edited in place — because nxdns has no installs and a step exists only to reconcile a database somebody already has.
- `querylog.db`: **no migrations.** On schema mismatch: rename aside, recreate fresh.
### 3.8 Blocklist Storage (Decision A)
@@ -105,7 +105,7 @@ Blocklist domains are **not** stored in SQLite — they are a cache of re-downlo
### 3.9 Rule Model (Decision B)
Rule kinds: `exact`, parent-walk (implicit via candidate chain), `wildcard` (`*` segment patterns, e.g. `*.doubleclick.net`, `ads.*.example.com`), `regex` (the linear-time engine of `filter/regex.zig`, matched unanchored against the whole normalized name). Actions: `allow` | `block`. Kind is an explicit column, which is what let the third kind arrive as one migration step rather than a schema break. A regex pattern is stored exactly as written — it is not a name, so it is never lowercased or dot-stripped — and is compiled at both edges: `config/validate.zig` refuses a bad one with the limit it hit, and `filter/rules.zig` compiles it once per snapshot.
Rule kinds: `exact`, parent-walk (implicit via candidate chain), `wildcard` (`*` segment patterns, e.g. `*.doubleclick.net`, `ads.*.example.com`), `regex` (the linear-time engine of `filter/regex.zig`, matched unanchored against the whole normalized name). Actions: `allow` | `block`. Kind is an explicit column, so a fourth kind widens one `CHECK` and touches no other table. A regex pattern is stored exactly as written — it is not a name, so it is never lowercased or dot-stripped — and is compiled at both edges: `config/validate.zig` refuses a bad one with the limit it hit, and `filter/rules.zig` compiles it once per snapshot.
### 3.10 Filtering Precedence
@@ -364,13 +364,14 @@ Per-group boolean. Rewrites known engine domains to their safe-search CNAME targ
### 11.2 config.db Schema (v1 baseline)
The DDL below is the **version 1** schema this plan froze, kept for the table
shapes it argues for. It is not the live schema and must not be implemented
against: that is `src/storage/config_schema.zig` plus the steps in
`src/storage/migrations.zig`, currently at **version 4**. The steps since v1 add
`upstreams.tls_name` (2), `blocklist_sources.exception_count` (3), and the
`regex` rule kind (4) — so the `kind` CHECK below admits two of the three kinds
the database now accepts, lacking `regex`.
The DDL below is the live schema, kept byte-identical to
`src/storage/config_schema.zig`. `src/storage/migrations.zig` carries it as its
one and only step, so a database is at **version 1** or it does not exist.
Until nxdns reaches v0.1 this baseline is **editable**: a schema change edits
this section and `config_schema.zig` together and adds no migration step. nxdns
has no installs, so there is no database for a step to reconcile. At v0.1 the
baseline freezes and every later change becomes an append-only step.
```sql
CREATE TABLE schema_version (version INTEGER NOT NULL);
@@ -403,7 +404,8 @@ CREATE TABLE upstreams (
id INTEGER PRIMARY KEY,
url TEXT NOT NULL UNIQUE,
priority INTEGER NOT NULL DEFAULT 100,
enabled INTEGER NOT NULL DEFAULT 1
enabled INTEGER NOT NULL DEFAULT 1,
tls_name TEXT NOT NULL DEFAULT '' -- DoT verification name; empty verifies against the url host
);
CREATE TABLE blocklist_sources (
@@ -415,6 +417,7 @@ CREATE TABLE blocklist_sources (
last_updated INTEGER,
domain_count INTEGER NOT NULL DEFAULT 0,
wildcard_count INTEGER NOT NULL DEFAULT 0,
exception_count INTEGER NOT NULL DEFAULT 0,
skipped_regex_count INTEGER NOT NULL DEFAULT 0,
checksum TEXT
);
@@ -429,7 +432,7 @@ CREATE TABLE rules (
id INTEGER PRIMARY KEY,
group_id INTEGER NOT NULL REFERENCES groups(id) ON DELETE CASCADE,
pattern TEXT NOT NULL,
kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard')),
kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard','regex')),
action TEXT NOT NULL CHECK(action IN ('allow','block')),
created_at INTEGER NOT NULL
);
+1 -1
View File
@@ -99,7 +99,7 @@ nxdns import /tmp/nxdns-lab/backup.zon --data-dir /tmp/nxdns-lab/data-restored
```
```
info(migrations): config.db migrated from schema version 0 to 4
info(migrations): config.db migrated from schema version 0 to 1
imported /tmp/nxdns-lab/backup.zon
```
+1 -1
View File
@@ -211,7 +211,7 @@ why the container is `docker-nxdns-1`.
A healthy first start logs the reconcile, the authority and the bound sockets:
```
info(migrations): config.db migrated from schema version 0 to 4
info(migrations): config.db migrated from schema version 0 to 1
reconciled '/etc/nxdns/config.zon': upstreams +1 ~0 -0; settings +45 ~0 -0;
settings keys changed: dns.bind_ipv4 dns.bind_ipv6 dns.port web.bind web.port …
web authentication is now enabled
+1 -1
View File
@@ -225,7 +225,7 @@ nxdns import /etc/nxdns/config.zon
```
```
info(migrations): config.db migrated from schema version 0 to 4
info(migrations): config.db migrated from schema version 0 to 1
imported /etc/nxdns/config.zon
```
+1 -1
View File
@@ -424,7 +424,7 @@ asking again reports `rule_block_exact`, `blocked` true and a null `source_url`.
**Symptom.** After putting an older binary back, it will not start:
```
warning(migrations): config.db is at schema version 99; this nxdns binary supports 4
warning(migrations): config.db is at schema version 99; this nxdns binary supports 1
nxdns run failed: SchemaTooNew
```
+5 -6
View File
@@ -307,7 +307,7 @@ opens the database immutable and never migrates, so on a database still one
version behind it reports the mismatch and exits 2 rather than fixing it:
```
FAIL /var/lib/nxdns/config.db: schema version 0, this nxdns expects 4; `nxdns run` migrates it, `check` will not
FAIL /var/lib/nxdns/config.db: schema version 0, this nxdns expects 1; `nxdns run` migrates it, `check` will not
```
That line was reproduced here against a database stamped at version 0; the path
@@ -317,22 +317,21 @@ A fresh database is created at the current schema version; an older one is
stepped up to it. The log line names both versions:
```
info(migrations): config.db migrated from schema version 0 to 4
info(migrations): config.db migrated from schema version 0 to 1
```
> Verified on this host: that exact line is what `nxdns import` printed when it
> created the scratch database used throughout this page. An empty data
> directory is schema version 0, which is why a first run reports a migration
> rather than nothing. The step from a populated older schema to 4 was not
> reproduced here — it needs a database written by an older binary, which this
> host does not have.
> rather than nothing. Version 1 is the only schema nxdns has published, so an
> upgrade from a populated older one is not a case that exists yet.
Rolling back is the case that has no answer. A database stamped by a newer
binary refuses to open, so an older binary against an upgraded data directory
fails to start:
```
warning(migrations): config.db is at schema version 99; this nxdns binary supports 4
warning(migrations): config.db is at schema version 99; this nxdns binary supports 1
nxdns run failed: SchemaTooNew
```
+1 -1
View File
@@ -109,7 +109,7 @@ zig-out/bin/nxdns import ~/nxdns-tutorial/config.zon --data-dir ~/nxdns-tutorial
```
```
info(migrations): config.db migrated from schema version 0 to 4
info(migrations): config.db migrated from schema version 0 to 1
imported /home/you/nxdns-tutorial/config.zon
```
+13 -5
View File
@@ -1,9 +1,15 @@
//! The `config.db` schema, verbatim from PLAN §11.2, plus the table lists every
//! other storage session needs.
//!
//! The DDL text is data, not code: `migrations.zig` carries it as step 1 and
//! never edits it in place. A schema change is a *new* step with new DDL, so
//! this string stays byte-identical to PLAN §11.2 forever.
//! The DDL text is data, not code: `migrations.zig` carries it as step 1.
//!
//! Until nxdns reaches v0.1 this baseline is **editable**. nxdns has no
//! installs, so a schema change edits this string and PLAN §11.2 together — it
//! does not append a migration step. A step exists to reconcile a database
//! somebody already has, and nobody has one.
//!
//! At v0.1 this string freezes and every later change becomes an append-only
//! step. That is a deliberate act, not a rule the code already lives under.
const std = @import("std");
@@ -40,7 +46,8 @@ pub const ddl_v1: [:0]const u8 =
\\ id INTEGER PRIMARY KEY,
\\ url TEXT NOT NULL UNIQUE,
\\ priority INTEGER NOT NULL DEFAULT 100,
\\ enabled INTEGER NOT NULL DEFAULT 1
\\ enabled INTEGER NOT NULL DEFAULT 1,
\\ tls_name TEXT NOT NULL DEFAULT '' -- DoT verification name; empty verifies against the url host
\\);
\\
\\CREATE TABLE blocklist_sources (
@@ -52,6 +59,7 @@ pub const ddl_v1: [:0]const u8 =
\\ last_updated INTEGER,
\\ domain_count INTEGER NOT NULL DEFAULT 0,
\\ wildcard_count INTEGER NOT NULL DEFAULT 0,
\\ exception_count INTEGER NOT NULL DEFAULT 0,
\\ skipped_regex_count INTEGER NOT NULL DEFAULT 0,
\\ checksum TEXT
\\);
@@ -66,7 +74,7 @@ pub const ddl_v1: [:0]const u8 =
\\ id INTEGER PRIMARY KEY,
\\ group_id INTEGER NOT NULL REFERENCES groups(id) ON DELETE CASCADE,
\\ pattern TEXT NOT NULL,
\\ kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard')),
\\ kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard','regex')),
\\ action TEXT NOT NULL CHECK(action IN ('allow','block')),
\\ created_at INTEGER NOT NULL
\\);
+30 -162
View File
@@ -18,57 +18,18 @@ const log = std.log.scoped(.migrations);
pub const Step = struct { version: u32, sql: [:0]const u8 };
/// Append only. Editing a released step — or `config_schema.ddl_v1` — would make
/// a fresh database and an upgraded one disagree, and nothing would detect it.
/// One baseline, no steps. Until v0.1 a schema change edits
/// `config_schema.ddl_v1` in place, because nxdns has no installs and there is
/// no database in the world for a step to reconcile.
///
/// At v0.1 the baseline freezes and this list becomes append-only: editing a
/// released step would make a fresh database and an upgraded one disagree, and
/// nothing would detect it. `migrateSteps` already implements that discipline
/// and its tests already pin it against injected step lists.
pub const steps = [_]Step{
.{ .version = 1, .sql = config_schema.ddl_v1 },
.{ .version = 2, .sql = ddl_v2 },
.{ .version = 3, .sql = ddl_v3 },
.{ .version = 4, .sql = ddl_v4 },
};
/// The DoT verification name (`upstreams.tls_name`). Empty keeps the pre-step-2
/// behavior: verify the certificate against the url host.
const ddl_v2: [:0]const u8 =
\\ALTER TABLE upstreams ADD COLUMN tls_name TEXT NOT NULL DEFAULT '';
;
/// Written `.allow` entries per source: the `@@||name^` exceptions a downloaded
/// list carries. 0 is what every source stands at until its next refresh
/// recompiles it, which is also what a list with no exceptions keeps.
const ddl_v3: [:0]const u8 =
\\ALTER TABLE blocklist_sources ADD COLUMN exception_count INTEGER NOT NULL DEFAULT 0;
;
/// The `regex` rule kind. A `CHECK` constraint cannot be altered in place, and
/// `config_schema.ddl_v1` is frozen, so the table is rebuilt: SQLite's
/// documented ALTER TABLE procedure, reduced to the steps this table needs.
///
/// The rebuilt table keeps the name `rules` and its whole v1 shape, ids
/// included, because `config_schema.table_names` and the invariant tests below
/// assert the schema's table set and a rename would fail all three.
///
/// The steps this table does not need: no index, trigger or view names `rules`,
/// and no other table references it, so nothing outside the four statements has
/// to be recreated or repointed. The procedure's `PRAGMA foreign_keys=OFF` is
/// deliberately absent — it is a no-op inside a transaction, and `migrate` runs
/// every step in one. It is also unneeded here: `rules` is a child of `groups`
/// and a parent of nothing, so dropping it violates no reference.
const ddl_v4: [:0]const u8 =
\\CREATE TABLE rules_v4 (
\\ id INTEGER PRIMARY KEY,
\\ group_id INTEGER NOT NULL REFERENCES groups(id) ON DELETE CASCADE,
\\ pattern TEXT NOT NULL,
\\ kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard','regex')),
\\ action TEXT NOT NULL CHECK(action IN ('allow','block')),
\\ created_at INTEGER NOT NULL
\\);
\\INSERT INTO rules_v4 (id, group_id, pattern, kind, action, created_at)
\\ SELECT id, group_id, pattern, kind, action, created_at FROM rules;
\\DROP TABLE rules;
\\ALTER TABLE rules_v4 RENAME TO rules;
;
/// The schema version this binary expects. A database `readVersion` reports
/// below this needs `nxdns run` to migrate it; above it is `error.SchemaTooNew`
/// and needs a newer nxdns.
@@ -150,7 +111,7 @@ pub fn migrateSteps(database: *db.Db, list: []const Step) Error!u32 {
///
/// Reads only, so it works on a connection opened `.read_only` or
/// `.immutable`. That is what it is public for: `nxdns check` may not migrate
/// (ruling F-c), and "at version 1, this binary expects 2" tells an operator
/// (ruling F-c), and "at version 0, this binary expects 1" tells an operator
/// what to do where a bare SQLite error message does not.
pub fn readVersion(database: *db.Db) Error!u32 {
const present = try database.queryInt(
@@ -302,12 +263,12 @@ fn columnExists(database: *db.Db, table: []const u8, column: []const u8) !bool {
return stmt.columnInt(0) != 0;
}
test "a fresh database reaches version 4 with both added columns and the third rule kind" {
test "a fresh database reaches the baseline with every v1 column and rule kind" {
var database = try openMigrated();
defer database.close();
try testing.expectEqual(@as(u32, 4), try migrate(&database));
try testing.expectEqual(@as(u32, 4), target_version);
try testing.expectEqual(@as(u32, 1), try migrate(&database));
try testing.expectEqual(@as(u32, 1), target_version);
try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
try testing.expect(try columnExists(&database, "blocklist_sources", "exception_count"));
@@ -321,110 +282,16 @@ test "a fresh database reaches version 4 with both added columns and the third r
));
}
test "a version 1 database upgrades and keeps its rows with an empty tls_name" {
test "the baseline rules table cascades from its group" {
var database = try openMigrated();
defer database.close();
_ = try migrate(&database);
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
try testing.expectEqual(@as(u32, 1), try migrateSteps(&database, &first));
try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
try database.exec("INSERT INTO upstreams (url, priority, enabled) VALUES ('tls://1.1.1.1:853', 10, 1);");
try testing.expectEqual(target_version, try migrate(&database));
try testing.expectEqual(target_version, try readVersion(&database));
try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
var stmt = try database.prepare("SELECT url, tls_name FROM upstreams");
defer stmt.deinit();
try testing.expect(try stmt.step());
try testing.expectEqualStrings("tls://1.1.1.1:853", stmt.columnText(0));
try testing.expectEqualStrings("", stmt.columnText(1));
}
test "a version 2 database upgrades and keeps its sources at exception_count 0" {
var database = try openMigrated();
defer database.close();
const through_two = [_]Step{ steps[0], steps[1] };
try testing.expectEqual(@as(u32, 2), try migrateSteps(&database, &through_two));
try testing.expect(!try columnExists(&database, "blocklist_sources", "exception_count"));
try database.exec(
\\INSERT INTO blocklist_sources (url, name, domain_count, wildcard_count, checksum)
\\VALUES ('https://lists.example/ads.txt', 'ads', 12, 3, 'abc');
);
try testing.expectEqual(target_version, try migrate(&database));
try testing.expect(try columnExists(&database, "blocklist_sources", "exception_count"));
// The counters and the checksum of a source compiled before this milestone
// survive: the checksum is what keeps its compiled files loadable without a
// refetch, so a migration that disturbed it would cost every household a
// full re-download.
var stmt = try database.prepare(
"SELECT domain_count, wildcard_count, exception_count, checksum FROM blocklist_sources",
);
defer stmt.deinit();
try testing.expect(try stmt.step());
try testing.expectEqual(@as(i64, 12), stmt.columnInt(0));
try testing.expectEqual(@as(i64, 3), stmt.columnInt(1));
try testing.expectEqual(@as(i64, 0), stmt.columnInt(2));
try testing.expectEqualStrings("abc", stmt.columnText(3));
}
test "a version 3 database upgrades to 4 with its rules intact and the third kind admitted" {
var database = try openMigrated();
defer database.close();
const through_three = [_]Step{ steps[0], steps[1], steps[2] };
try testing.expectEqual(@as(u32, 3), try migrateSteps(&database, &through_three));
try database.exec(
\\INSERT INTO groups (id, name) VALUES (2, 'kids');
\\INSERT INTO rules (id, group_id, pattern, kind, action, created_at) VALUES
\\ (7, 1, 'ads.example', 'exact', 'block', 1000),
\\ (9, 2, '*.tracker.net', 'wildcard', 'allow', 2000);
);
// Before step 4 the frozen v1 CHECK admits two kinds only.
try testing.expectError(error.Constraint, database.exec(
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
\\VALUES (1, '^ad', 'regex', 'block', 3000);
));
try testing.expectEqual(@as(u32, 4), try migrate(&database));
// The rebuild is a copy, so every column of every row survives it — ids
// included, because `group_sources` aside, an API client holds rule ids and
// a renumbering would silently repoint every bookmark and every ETag.
var stmt = try database.prepare(
"SELECT id, group_id, pattern, kind, action, created_at FROM rules ORDER BY id",
);
defer stmt.deinit();
try testing.expect(try stmt.step());
try testing.expectEqual(@as(i64, 7), stmt.columnInt(0));
try testing.expectEqual(@as(i64, 1), stmt.columnInt(1));
try testing.expectEqualStrings("ads.example", stmt.columnText(2));
try testing.expectEqualStrings("exact", stmt.columnText(3));
try testing.expectEqualStrings("block", stmt.columnText(4));
try testing.expectEqual(@as(i64, 1000), stmt.columnInt(5));
try testing.expect(try stmt.step());
try testing.expectEqual(@as(i64, 9), stmt.columnInt(0));
try testing.expectEqual(@as(i64, 2), stmt.columnInt(1));
try testing.expectEqualStrings("*.tracker.net", stmt.columnText(2));
try testing.expectEqualStrings("wildcard", stmt.columnText(3));
try testing.expectEqualStrings("allow", stmt.columnText(4));
try testing.expect(!try stmt.step());
try database.exec(
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
\\VALUES (1, '^ad[0-9]+-', 'regex', 'block', 3000);
);
try testing.expectEqual(
@as(i64, 1),
try database.queryInt("SELECT count(*) FROM rules WHERE kind = 'regex'"),
);
// The scaffolding table is gone and the foreign key came back with the
// rebuild: deleting a group still takes its rules with it.
try testing.expect(!try tableExists(&database, "rules_v4"));
try database.exec("DELETE FROM groups WHERE id = 2;");
try testing.expectEqual(
@as(i64, 0),
@@ -432,22 +299,25 @@ test "a version 3 database upgrades to 4 with its rules intact and the third kin
);
}
test "a failing step after the last released one rolls back the whole upgrade from version 1" {
test "a failing step rolls back an upgrade of a populated database" {
var database = try openMigrated();
defer database.close();
_ = try migrate(&database);
try database.exec("INSERT INTO upstreams (url, priority, enabled) VALUES ('tls://1.1.1.1:853', 10, 1);");
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
_ = try migrateSteps(&database, &first);
// Rollback of an *upgrade* is a different case from rollback of the initial
// creation ("a failing step rolls the whole migration back"): here a
// populated database must come back untouched, not cease to exist.
const broken = steps ++ [_]Step{
.{ .version = target_version + 1, .sql = "CREATE TABLE third (" },
.{ .version = target_version + 1, .sql = "CREATE TABLE second (id INTEGER PRIMARY KEY);" },
.{ .version = target_version + 2, .sql = "CREATE TABLE third (" },
};
try testing.expectError(error.Unexpected, migrateSteps(&database, &broken));
// One transaction: the ALTER TABLEs of the released steps went back with it.
try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
try testing.expect(!try columnExists(&database, "blocklist_sources", "exception_count"));
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
// One transaction: the step that did succeed went back with the one that did not.
try testing.expect(!try tableExists(&database, "second"));
try testing.expectEqual(target_version, try readVersion(&database));
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
}
test "readVersion reports 0 before a migration and target_version after it" {
@@ -471,20 +341,18 @@ test "readVersion reads a file database through an immutable open, writing nothi
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try std.fmt.bufPrintZ(&path_buf, "{s}{s}/config.db", .{ tmp_prefix, &tmp.sub_path });
// A database an older nxdns left at version 1. `check` must report that, not
// migrate it (ruling F-c).
// `check` reads the stamped version without migrating (ruling F-c), so the
// read has to work through a connection that cannot write at all.
{
var database = try db.Db.open(path, .{ .mode = .read_write_create });
defer database.close();
try db.applyPragmas(&database, .{});
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
try testing.expectEqual(@as(u32, 1), try migrateSteps(&database, &first));
try testing.expectEqual(target_version, try migrate(&database));
}
var database = try db.Db.open(path, .{ .mode = .{ .immutable = testing.io } });
defer database.close();
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
try testing.expectEqual(@as(u32, 4), target_version);
try testing.expectEqual(target_version, try readVersion(&database));
// A write through this connection is refused by SQLite, not by convention.
try testing.expectError(error.ReadOnly, database.exec("DELETE FROM schema_version;"));