storage and config: sqlite wrapper, migrations, querylog policy, repositories, zon config with import/export/check cli

This commit is contained in:
2026-08-01 14:21:44 +02:00
parent 17d0401f8a
commit 70bff22d75
23 changed files with 10142 additions and 47 deletions
+621
View File
@@ -0,0 +1,621 @@
//! `nxdns import`: a ZON file becomes the whole content of `config.db`.
//!
//! The order is the specification. Nothing reaches the database until the file
//! has been read, parsed and validated, and every write happens inside one
//! `BEGIN IMMEDIATE` transaction, so a failed import leaves the database
//! byte-for-byte as it was.
//!
//! No filesystem write happens anywhere in this file: the input is opened
//! read-only and the database is SQLite's business.
const std = @import("std");
const Allocator = std.mem.Allocator;
const db = @import("../storage/db.zig");
const config_schema = @import("../storage/config_schema.zig");
const context = @import("../storage/repositories/context.zig");
const clients_repo = @import("../storage/repositories/clients_repo.zig");
const groups_repo = @import("../storage/repositories/groups_repo.zig");
const local_repo = @import("../storage/repositories/local_repo.zig");
const rules_repo = @import("../storage/repositories/rules_repo.zig");
const settings_repo = @import("../storage/repositories/settings_repo.zig");
const sources_repo = @import("../storage/repositories/sources_repo.zig");
const upstreams_repo = @import("../storage/repositories/upstreams_repo.zig");
const address = @import("../platform/address.zig");
const model = @import("model.zig");
const validate = @import("validate.zig");
const log = std.log.scoped(.config_import);
pub const Options = struct { force: bool = false };
pub const Error = db.Error || validate.ValidateError || std.Io.Dir.ReadFileAllocError ||
error{ DatabaseNotEmpty, ConfigTooLarge, ParseZon, PasswordAndHashBothSet };
pub const max_config_bytes = 4 * 1024 * 1024;
/// Holds any PHC-encoded argon2id string comfortably.
const hash_buf_len = 256;
/// The canonical text of an IPv6 prefix, the longest value canonicalised here.
const canonical_buf_len = 64;
// ---------------------------------------------------------------------------
// emptiness
// ---------------------------------------------------------------------------
/// A database is "never configured" when the migrations have run and nothing
/// else has. The migrations themselves create `schema_version` and seed
/// `groups(1, 'default')`, so "no rows anywhere" is the wrong test.
///
/// True iff every table in `config_schema.content_tables` is empty, `groups`
/// holds exactly one row, and that row is the seeded `(1, 'default', 0)`.
///
/// The client count here includes auto-materialised rows: a server that has
/// answered one query is configured enough that a bootstrap file must not
/// overwrite it.
pub fn isEmpty(database: *db.Db) db.Error!bool {
// `inline for` over a comptime table list: every statement below is a
// compile-time string, so no table name is ever concatenated at run time.
inline for (config_schema.content_tables) |table| {
if (try database.queryInt("SELECT count(*) FROM " ++ table) != 0) return false;
}
if (try database.queryInt("SELECT count(*) FROM groups") != 1) return false;
const seeded = try database.queryInt(
"SELECT count(*) FROM groups WHERE id = 1 AND name = 'default' AND safe_search = 0",
);
return seeded == 1;
}
// ---------------------------------------------------------------------------
// import
// ---------------------------------------------------------------------------
/// Reads, parses, validates, then replaces the database contents.
pub fn importFile(
io: std.Io,
gpa: Allocator,
database: *db.Db,
dir: std.Io.Dir,
path: []const u8,
options: Options,
diags: *validate.Diagnostics,
) Error!void {
// `std.zon.parse` needs a sentinel-terminated source and `readFileAlloc`
// cannot supply one.
const source = dir.readFileAllocOptions(
io,
path,
gpa,
.limited(max_config_bytes),
.of(u8),
0,
) catch |e| switch (e) {
error.StreamTooLong => return error.ConfigTooLarge,
else => |other| return other,
};
defer gpa.free(source);
return importSource(io, gpa, database, source, options, diags);
}
/// `importFile` minus the file. It exists because every step from the parse
/// onwards is testable without touching a filesystem, and `nxdns check` (S6)
/// needs the same parse-and-validate half.
pub fn importSource(
io: std.Io,
gpa: Allocator,
database: *db.Db,
source: [:0]const u8,
options: Options,
diags: *validate.Diagnostics,
) Error!void {
// The parsed `Config` is arena-owned and `std.zon.parse.free` is NEVER
// called on it. `Parser.parseStruct` fills an absent field by copying the
// struct's default straight through (parse.zig:874), so a defaulted
// `[]const u8` — and this model has many non-empty string defaults — points
// into the binary's read-only data. `parse.free` keeps no record of which
// fields were parsed and which were defaulted, so it would `@memset` and
// free rodata. Freeing the arena is the only correct release.
var arena_state: std.heap.ArenaAllocator = .init(gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
var zon_diag: std.zon.parse.Diagnostics = .{};
const cfg = std.zon.parse.fromSliceAlloc(model.Config, arena, source, &zon_diag, .{}) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.ParseZon => {
try reportParseFailure(diags, &zon_diag);
return error.ParseZon;
},
};
try validate.validate(cfg, diags);
const now = std.Io.Clock.real.now(io).toSeconds();
return applyToDb(io, gpa, database, cfg, now, options);
}
/// The line and column of a ZON syntax error are the only thing the operator can
/// act on, so they travel the same channel as every other config problem: the
/// caller's `Diagnostics`, which `nxdns import` already renders to stderr. The
/// global log is not that channel — an operator reading command output would see
/// a bare `ParseZon` and nothing else.
///
/// `std.zon.parse.Diagnostics` renders one "line:column: error: text" line per
/// problem, plus a "note:" line each, so each rendered line becomes one
/// `Problem` and the list keeps the parser's order.
fn reportParseFailure(
diags: *validate.Diagnostics,
zon_diag: *const std.zon.parse.Diagnostics,
) error{OutOfMemory}!void {
const rendered = try std.fmt.allocPrint(diags.gpa, "{f}", .{zon_diag});
defer diags.gpa.free(rendered);
var lines = std.mem.splitScalar(u8, rendered, '\n');
while (lines.next()) |line| {
if (line.len == 0) continue;
try diags.add(error.ParseZon, "config", .{}, "{s}", .{line});
}
}
/// The half `bootstrap` reuses: an already-parsed, already-validated config into
/// the database, all or nothing. `now` is the caller's timestamp for the runtime
/// columns the model omits.
pub fn applyToDb(
io: std.Io,
gpa: Allocator,
database: *db.Db,
cfg: model.Config,
now: i64,
options: Options,
) Error!void {
var tx = try db.Tx.begin(database);
errdefer tx.rollback();
// Inside the transaction on purpose. Checking before `BEGIN IMMEDIATE`
// would leave a TOCTOU window against a concurrently starting process;
// `BEGIN IMMEDIATE` already holds the write lock, so the check and the
// writes are one atomic unit.
if (!options.force and !try isEmpty(database)) return error.DatabaseNotEmpty;
inline for (config_schema.delete_order) |table| {
try database.exec("DELETE FROM " ++ table ++ ";");
}
var group_ids: context.IdMap = .empty;
defer group_ids.deinit(gpa);
var source_ids: context.IdMap = .empty;
defer source_ids.deinit(gpa);
try insertGroups(database, gpa, cfg, &group_ids);
try insertSources(database, gpa, cfg, &source_ids);
const ctx: context.InsertContext = .{
.now = now,
.group_ids = &group_ids,
.source_ids = &source_ids,
};
for (cfg.clients) |client| {
var buf: [canonical_buf_len]u8 = undefined;
var canonical = client;
canonical.ip = try canonicalIp(client.ip, &buf);
try clients_repo.insertClient(database, canonical, ctx);
}
for (cfg.client_prefixes) |entry| {
var buf: [canonical_buf_len]u8 = undefined;
var canonical = entry;
canonical.prefix = try canonicalPrefix(entry.prefix, &buf);
try clients_repo.insertClientPrefix(database, canonical, ctx);
}
for (cfg.upstreams) |item| try upstreams_repo.insertUpstream(database, item, ctx);
for (cfg.group_sources) |item| try groups_repo.insertGroupSource(database, item, ctx);
for (cfg.rules) |item| try rules_repo.insertRule(database, item, ctx);
for (cfg.local_records) |item| try local_repo.insertLocalRecord(database, item, ctx);
for (cfg.forward_zones) |item| try local_repo.insertForwardZone(database, item, ctx);
// The buffer must outlive `toSettings`: `effective.web.password_hash` points
// into it.
var hash_buf: [hash_buf_len]u8 = undefined;
var effective = cfg;
if (cfg.web.password.len != 0) {
if (cfg.web.password_hash.len != 0) return error.PasswordAndHashBothSet;
effective.web.password_hash = try hashPassword(io, gpa, cfg.web.password, &hash_buf);
}
// Operator input, never stored. `toSettings` skips the field in both
// directions; clearing it here keeps the in-memory value honest too.
effective.web.password = "";
var pairs: std.ArrayList(model.SettingPair) = .empty;
defer {
model.freeSettings(gpa, pairs.items);
pairs.deinit(gpa);
}
try model.toSettings(effective, gpa, &pairs);
for (pairs.items) |pair| try settings_repo.insertSetting(database, pair, ctx);
try tx.commit();
}
/// `default` goes in first and takes rowid 1. §11.2 seeds group 1 as `default`
/// and §7.2's fallback assignment depends on it; letting an import renumber it
/// would silently move every unassigned client.
///
/// The repositories expose no insert-with-id, so the id is taken rather than
/// given: SQLite assigns rowid 1 to the first row of an empty table, and the
/// table was emptied a few statements ago. The result is checked, not assumed.
fn insertGroups(database: *db.Db, gpa: Allocator, cfg: model.Config, ids: *context.IdMap) Error!void {
const default_index = indexOfGroup(cfg.groups, "default") orelse {
log.warn("the config declares no group named 'default'", .{});
return error.MissingDefaultGroup;
};
try insertGroup(database, gpa, cfg.groups[default_index], ids);
const default_id = ids.get("default").?;
if (default_id != 1) {
log.warn("group 'default' took id {d}, not 1", .{default_id});
return error.Unexpected;
}
for (cfg.groups, 0..) |group, i| {
if (i == default_index) continue;
try insertGroup(database, gpa, group, ids);
}
}
fn insertGroup(database: *db.Db, gpa: Allocator, group: model.Group, ids: *context.IdMap) Error!void {
try groups_repo.insertGroup(database, group, .{});
// The key borrows from `cfg`, which outlives the transaction.
try ids.put(gpa, group.name, database.lastInsertRowid());
}
fn insertSources(database: *db.Db, gpa: Allocator, cfg: model.Config, ids: *context.IdMap) Error!void {
for (cfg.blocklist_sources) |item| {
try sources_repo.insertBlocklistSource(database, item, .{});
try ids.put(gpa, item.url, database.lastInsertRowid());
}
}
fn indexOfGroup(groups: []const model.Group, name: []const u8) ?usize {
for (groups, 0..) |group, i| {
if (std.mem.eql(u8, group.name, name)) return i;
}
return null;
}
/// The validator compares client addresses after canonicalisation, so the row
/// this writes must be canonical too — otherwise `fd00::1` and
/// `FD00:0:0:0:0:0:0:1` pass validation as a duplicate pair and then collide on
/// the column's `UNIQUE`.
fn canonicalIp(text: []const u8, buf: []u8) error{BadClientIp}![]const u8 {
const addr = address.NetAddress.parse(text) catch return error.BadClientIp;
var w: std.Io.Writer = .fixed(buf);
addr.format(&w) catch return error.BadClientIp;
return w.buffered();
}
fn canonicalPrefix(text: []const u8, buf: []u8) error{BadClientPrefix}![]const u8 {
const prefix = address.Prefix.parse(text) catch return error.BadClientPrefix;
var w: std.Io.Writer = .fixed(buf);
prefix.format(&w) catch return error.BadClientPrefix;
return w.buffered();
}
/// argon2id with the OWASP parameters (t=2, m=19 MiB, p=1) rather than the
/// 64 MiB `interactive_2id`, because PLAN §18 budgets under 100 MB total on a
/// Pi 5.
///
/// `strHash`'s error set reaches beyond this module's (it carries
/// `std.Thread.SpawnError` and the PHC encoding errors), so anything that is
/// neither out of memory nor a cancellation is reported as `error.Unexpected`
/// with the real cause logged.
fn hashPassword(io: std.Io, gpa: Allocator, password: []const u8, buf: []u8) Error![]const u8 {
return std.crypto.pwhash.argon2.strHash(password, .{
.allocator = gpa,
.params = .owasp_2id,
.mode = .argon2id,
.encoding = .phc,
}, buf, io) catch |e| switch (e) {
error.OutOfMemory => error.OutOfMemory,
error.Canceled => error.Canceled,
else => {
log.warn("hashing web.password failed: {s}", .{@errorName(e)});
return error.Unexpected;
},
};
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const testing = std.testing;
const migrations = @import("../storage/migrations.zig");
fn openMigrated() !db.Db {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
return database;
}
/// Every row of every config table, rendered in a stable order. Two dumps are
/// equal exactly when the database content is.
fn dump(database: *db.Db, gpa: Allocator) ![]u8 {
var out: std.Io.Writer.Allocating = .init(gpa);
errdefer out.deinit();
const w = &out.writer;
try w.writeAll("groups\n");
var stmt = try database.prepare("SELECT id, name, safe_search FROM groups ORDER BY id");
defer stmt.deinit();
while (try stmt.step()) {
try w.print(" {d} {s} {d}\n", .{ stmt.columnInt(0), stmt.columnText(1), stmt.columnInt(2) });
}
inline for (config_schema.content_tables) |table| {
try w.print("{s}\n", .{table});
var rows = try database.prepare("SELECT * FROM " ++ table ++ " ORDER BY 1, 2");
defer rows.deinit();
const columns = db.c.sqlite3_column_count(rows.handle);
while (try rows.step()) {
var col: c_int = 0;
while (col < columns) : (col += 1) {
try w.print(" {s}", .{rows.columnText(col)});
}
try w.writeAll("\n");
}
}
return out.toOwnedSlice();
}
/// The smallest config that validates: one enabled upstream and the `default`
/// group.
const minimal_source: [:0]const u8 =
\\.{
\\ .groups = .{ .{ .name = "default" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\}
;
/// Exercises every collection and a few non-default scalars.
const full_source: [:0]const u8 =
\\.{
\\ .dns = .{ .port = 5353 },
\\ .logging = .{ .level = .err, .retention_days = 7 },
\\ .groups = .{ .{ .name = "default" }, .{ .name = "kids", .safe_search = true } },
\\ .upstreams = .{
\\ .{ .url = "https://dns.example/dns-query", .priority = 10 },
\\ .{ .url = "tls://dot.example:853", .priority = 20, .enabled = false },
\\ },
\\ .clients = .{ .{ .ip = "FD00:0:0:0:0:0:0:1", .name = "tablet", .group = "kids" } },
\\ .client_prefixes = .{ .{ .prefix = "192.168.1.0/24", .group = "kids", .priority = 50 } },
\\ .blocklist_sources = .{ .{ .url = "https://lists.example/ads.txt", .name = "ads" } },
\\ .group_sources = .{ .{ .group = "kids", .source_url = "https://lists.example/ads.txt" } },
\\ .rules = .{ .{ .group = "kids", .pattern = "*.tracker.example", .kind = .wildcard, .action = .block } },
\\ .local_records = .{ .{ .name = "nas.lan", .rtype = .a, .value = "192.168.1.10", .ttl = 600 } },
\\ .forward_zones = .{ .{ .zone = "lan", .resolver = "udp://192.168.1.1:53" } },
\\}
;
fn importText(io: std.Io, database: *db.Db, source: [:0]const u8, options: Options) !void {
var diags: validate.Diagnostics = .init(testing.allocator);
defer diags.deinit();
return importSource(io, testing.allocator, database, source, options, &diags);
}
test "isEmpty is true on a freshly migrated database" {
var database = try openMigrated();
defer database.close();
try testing.expect(try isEmpty(&database));
}
test "isEmpty is false once a settings row exists" {
var database = try openMigrated();
defer database.close();
try database.exec("INSERT INTO settings (key, value) VALUES ('dns.port', '53');");
try testing.expect(!try isEmpty(&database));
}
test "isEmpty is false once an auto-materialized client exists" {
var database = try openMigrated();
defer database.close();
try database.exec(
\\INSERT INTO clients (ip, name, group_id, hand_edited, first_seen, last_seen)
\\VALUES ('192.168.1.5', NULL, 1, 0, 1, 1);
);
try testing.expect(!try isEmpty(&database));
}
test "isEmpty is false once the seeded group is changed" {
var database = try openMigrated();
defer database.close();
try database.exec("UPDATE groups SET name = 'renamed' WHERE id = 1;");
try testing.expect(!try isEmpty(&database));
}
test "importSource seeds a migrated database and group 'default' keeps id 1" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
try importText(io, &database, full_source, .{});
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT id FROM groups WHERE name = 'default'"));
try testing.expectEqual(@as(i64, 2), try database.queryInt("SELECT count(*) FROM upstreams"));
// The v6 client address was written in canonical form, not as typed.
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM clients WHERE ip = 'fd00::1'"));
}
test "applyToDb without force refuses a configured database and changes nothing" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = testing.allocator;
var database = try openMigrated();
defer database.close();
try importText(io, &database, full_source, .{});
const before = try dump(&database, gpa);
defer gpa.free(before);
const second: model.Config = .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{.{ .url = "https://other.example/dns-query" }},
};
try testing.expectError(error.DatabaseNotEmpty, applyToDb(io, gpa, &database, second, 42, .{}));
const after = try dump(&database, gpa);
defer gpa.free(after);
try testing.expectEqualStrings(before, after);
}
test "applyToDb rolls back completely when an insert fails mid-way" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = testing.allocator;
var database = try openMigrated();
defer database.close();
try importText(io, &database, full_source, .{});
const before = try dump(&database, gpa);
defer gpa.free(before);
// Two identical local records violate `UNIQUE(name, rtype, value)`. The
// validator would catch this, which is exactly why the test calls
// `applyToDb` directly: the all-or-nothing guarantee has to hold on its own.
const broken: model.Config = .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{.{ .url = "https://other.example/dns-query" }},
.local_records = &.{
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
},
};
try testing.expectError(error.Constraint, applyToDb(io, gpa, &database, broken, 42, .{ .force = true }));
const after = try dump(&database, gpa);
defer gpa.free(after);
try testing.expectEqualStrings(before, after);
}
test "importSource writes nothing when validation fails" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
// No `default` group and no enabled upstream.
const bad: [:0]const u8 =
\\.{ .groups = .{ .{ .name = "kids" } } }
;
var diags: validate.Diagnostics = .init(testing.allocator);
defer diags.deinit();
try testing.expectError(
error.MissingDefaultGroup,
importSource(io, testing.allocator, &database, bad, .{}, &diags),
);
try testing.expect(diags.problems.items.len >= 2);
try testing.expect(try isEmpty(&database));
}
test "importSource reports a ZON syntax error and writes nothing" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
var diags: validate.Diagnostics = .init(testing.allocator);
defer diags.deinit();
try testing.expectError(
error.ParseZon,
importSource(io, testing.allocator, &database, ".{ .groups = ", .{}, &diags),
);
try testing.expect(try isEmpty(&database));
// The point of the diagnostic: what the CLI prints must name the line and the
// column, not just `ParseZon`.
try testing.expect(diags.problems.items.len >= 1);
var rendered: std.Io.Writer.Allocating = .init(testing.allocator);
defer rendered.deinit();
try diags.writeAll(&rendered.writer);
const text = rendered.written();
try testing.expect(std.mem.indexOf(u8, text, "1:14: error: ") != null);
}
test "a config omitting every optional field parses into an arena and leaks nothing" {
// The S5.1 rule as a test: `std.zon.parse.free` is never called, the arena
// is the only release, and `std.testing.allocator` fails the test if a
// defaulted rodata string were ever handed to the allocator.
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena_state.deinit();
const cfg = try std.zon.parse.fromSliceAlloc(
model.Config,
arena_state.allocator(),
minimal_source,
null,
.{},
);
try testing.expectEqualStrings("0.0.0.0", cfg.dns.bind_ipv4);
try testing.expectEqual(@as(u16, 53), cfg.dns.port);
try testing.expectEqual(@as(usize, 1), cfg.groups.len);
}
test "a password is hashed into web.password_hash and never stored verbatim" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = testing.allocator;
var database = try openMigrated();
defer database.close();
const cfg: model.Config = .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{.{ .url = "https://dns.example/dns-query" }},
.web = .{ .password = "correct horse battery staple" },
};
try applyToDb(io, gpa, &database, cfg, 42, .{});
var stmt = try database.prepare("SELECT value FROM settings WHERE key = 'web.password_hash'");
defer stmt.deinit();
try testing.expect(try stmt.step());
try testing.expect(std.mem.startsWith(u8, stmt.columnText(0), "$argon2id$"));
try testing.expectEqual(
@as(i64, 0),
try database.queryInt("SELECT count(*) FROM settings WHERE key = 'web.password'"),
);
}
test "a password and a password_hash together are refused" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
const cfg: model.Config = .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{.{ .url = "https://dns.example/dns-query" }},
.web = .{ .password = "plaintext", .password_hash = "$argon2id$v=19$m=19456,t=2,p=1$abc$def" },
};
try testing.expectError(
error.PasswordAndHashBothSet,
applyToDb(io, testing.allocator, &database, cfg, 42, .{}),
);
try testing.expect(try isEmpty(&database));
}