mokhtar/nxdns
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

milestone 13 discrepancies: redact credentials from urls in logs, metrics and cli output

Browse Source
This commit is contained in:
mokhtar
2026-08-07 00:45:17 +02:00
parent 1ff727feb8
commit 8c3328562e
39 changed files with 5734 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config/import.zig
+363 -15
View File
@@ -1,4 +1,13 @@
//! `nxdns import`: a ZON file becomes the whole content of `config.db`.
//! `nxdns import`: a ZON file becomes the whole configuration of `config.db`.
//!
//! Configuration, not content: the `hand_edited = 0` client rows the DNS path
//! materialises from live traffic are runtime state, they are absent from an
//! export, and an import carries them across rather than deleting them.
//!
//! `clients.first_seen` and `clients.last_seen` are runtime state on every client
//! row, configured ones included, and the config model carries neither. So an
//! address the database already knew keeps both across an import, and only an
//! address it has never seen takes the import's clock.
//!
//! The order is the specification. Nothing reaches the database until the file
//! has been read, parsed and validated, and every write happens inside one
@@ -44,21 +53,31 @@ const canonical_buf_len = 64;
// emptiness
// ---------------------------------------------------------------------------
/// A database is "never configured" when the migrations have run and nothing
/// else has. The migrations themselves create `schema_version` and seed
/// `groups(1, 'default')`, so "no rows anywhere" is the wrong test.
/// A database is "never configured" when the migrations have run and the
/// operator has added nothing. The migrations themselves create
/// `schema_version` and seed `groups(1, 'default')`, so "no rows anywhere" is
/// the wrong test.
///
/// True iff every table in `config_schema.content_tables` is empty, `groups`
/// holds exactly one row, and that row is the seeded `(1, 'default', 0)`.
/// True iff no table in `config_schema.content_tables` holds a row the operator
/// put there, `groups` holds exactly one row, and that row is the seeded
/// `(1, 'default', 0)`.
///
/// The client count here includes auto-materialised rows: a server that has
/// answered one query is configured enough that a bootstrap file must not
/// overwrite it.
/// `clients` is the one table a row can reach without an operator: the DNS path
/// materialises `hand_edited = 0` rows straight from live traffic (PLAN §7.2).
/// Counting those made emptiness a function of traffic — a server that had
/// answered a single query silently ignored the seed file its operator dropped
/// next to it. So only `hand_edited = 1` rows count, which is the predicate
/// `clients_repo.listClients` already exports by: this database is empty exactly
/// when its export is empty.
pub fn isEmpty(database: *db.Db) db.Error!bool {
// `inline for` over a comptime table list: every statement below is a
// compile-time string, so no table name is ever concatenated at run time.
inline for (config_schema.content_tables) |table| {
if (try database.queryInt("SELECT count(*) FROM " ++ table) != 0) return false;
const count_sql = comptime if (std.mem.eql(u8, table, "clients"))
"SELECT count(*) FROM clients WHERE hand_edited = 1"
else
"SELECT count(*) FROM " ++ table;
if (try database.queryInt(count_sql) != 0) return false;
}
if (try database.queryInt("SELECT count(*) FROM groups") != 1) return false;
const seeded = try database.queryInt(
@@ -179,6 +198,8 @@ pub fn applyToDb(
// writes are one atomic unit.
if (!options.force and !try isEmpty(database)) return error.DatabaseNotEmpty;
try liftSavedClients(database);
inline for (config_schema.delete_order) |table| {
try database.exec("DELETE FROM " ++ table ++ ";");
}
@@ -203,6 +224,8 @@ pub fn applyToDb(
canonical.ip = try canonicalIp(client.ip, &buf);
try clients_repo.insertClient(database, canonical, ctx);
}
try restoreSavedClients(database, group_ids.get("default").?);
for (cfg.client_prefixes) |entry| {
var buf: [canonical_buf_len]u8 = undefined;
var canonical = entry;
@@ -238,6 +261,102 @@ pub fn applyToDb(
try tx.commit();
}
const lift_saved_clients_sql: [:0]const u8 =
\\DROP TABLE IF EXISTS temp.saved_clients;
\\CREATE TEMP TABLE saved_clients AS
\\ SELECT ip, name, hand_edited, first_seen, last_seen FROM clients;
;
/// Carries what the wipe must not destroy over the wipe that follows: the
/// auto-materialised client rows themselves, and the observed timestamps of
/// every client row whatever its flag.
///
/// A `hand_edited = 0` row is runtime state, not configuration: the DNS path
/// wrote it from live traffic and `clients_repo.listClients` already keeps it out
/// of an export. Replacing the *configuration* must therefore not delete it —
/// but it cannot survive in place either, because `clients.group_id` references
/// `groups(id)` with no cascade, and the wipe empties `groups`. So the rows step
/// aside into the temp database and come back once `groups` holds `default`
/// again.
///
/// The table takes every row, not only the `hand_edited = 0` ones, because
/// `first_seen` and `last_seen` are runtime state on a configured row too — the
/// tracker keeps writing `last_seen` on the clients the operator named. Saving
/// only the materialised rows would keep the observation history of the devices
/// nobody named and destroy it for the devices somebody did. `hand_edited` rides
/// along so the restore can tell the two apart.
///
/// `IF EXISTS` because a rolled-back import must not poison the next one.
fn liftSavedClients(database: *db.Db) Error!void {
return database.exec(lift_saved_clients_sql);
}
const merge_observed_timestamps_sql: [:0]const u8 =
\\UPDATE clients AS c
\\ SET first_seen = m.first_seen, last_seen = m.last_seen
\\ FROM temp.saved_clients m
\\ WHERE m.ip = c.ip
;
const restore_materialised_clients_sql =
\\INSERT INTO clients (ip, name, group_id, hand_edited, first_seen, last_seen)
\\SELECT m.ip, m.name, ?1, 0, m.first_seen, m.last_seen
\\ FROM temp.saved_clients m
\\ WHERE m.hand_edited = 0
\\ AND NOT EXISTS (SELECT 1 FROM clients c WHERE c.ip = m.ip)
;
/// Puts back what `liftSavedClients` set aside, in two steps that must stay in
/// this order: the restore drops the temp table on its way out, so the merge
/// cannot follow it.
///
/// **The merge.** An address the config declares belongs to the config — but
/// `first_seen` and `last_seen` are not the config's to state. The model carries
/// neither field, so `insertClient` writes `now` into both as a placeholder for a
/// device it knows nothing about. When the database already held that address, the
/// placeholder is the worse of the two values and both columns come from the saved
/// row instead. Everything else on the row stays the config's: the name, the
/// group, and `hand_edited = 1`.
///
/// Not "the earlier `first_seen` and the later `last_seen`": the placeholder is
/// the wall clock at import time and every real observation predates it, so
/// "later" would resolve to the placeholder every time and stamp each named
/// device as seen at the moment of the import. An operator restoring a backup
/// would read that as liveness. `first_seen` and `last_seen` mean "when a query
/// from this address arrived", `pruneStale` and the API both read them that way,
/// and an import is not a query. Taking both from the saved row also keeps
/// `first_seen <= last_seen`, which `upsertSeen` guarantees pairwise.
///
/// Only the config's own clients are in the table at this point, so the update
/// needs no filter of its own, and the saved row's flag does not enter into it: a
/// device keeps its history whether the previous row was materialised or
/// configured. A `--force` re-import of the same file is the case that matters —
/// it deletes and rewrites every configured client, and without the merge each
/// one would come back claiming it was first seen at the moment of the import.
///
/// **The restore.** Only `hand_edited = 0` rows come back. A configured client is
/// configuration, so a file that leaves its address out has removed that client
/// and the row must stay gone; its history was saved for the merge, not for a
/// resurrection. `WHERE NOT EXISTS` rather than `INSERT OR IGNORE`: the only
/// materialised row worth skipping is one whose address the config claims — the
/// operator naming a device the server had already discovered, handled by the
/// merge above — and every other constraint failure stays loud.
///
/// The rows return to `default`, the group `upsertSeen` materialises into.
/// `first_seen` and `last_seen` cross unchanged; the row id does not, because the
/// config's clients went in first and hold the low ids now. Nothing references
/// `clients.id` inside `config.db`, and §3.6 keeps the query log out of it.
fn restoreSavedClients(database: *db.Db, default_group_id: i64) Error!void {
try database.exec(merge_observed_timestamps_sql);
{
var stmt = try database.prepare(restore_materialised_clients_sql);
defer stmt.deinit();
try stmt.bindInt(1, default_group_id);
try stmt.exec();
}
return database.exec("DROP TABLE temp.saved_clients;");
}
/// `default` goes in first and takes rowid 1. §11.2 seeds group 1 as `default`
/// and §7.2's fallback assignment depends on it; letting an import renumber it
/// would silently move every unassigned client.
@@ -303,7 +422,7 @@ fn canonicalPrefix(text: []const u8, buf: []u8) error{BadClientPrefix}![]const u
}
/// argon2id with the OWASP parameters (t=2, m=19 MiB, p=1) rather than the
/// 64 MiB `interactive_2id`, because PLAN §18 budgets under 100 MB total on a
/// 64 MiB `interactive_2id`, because PLAN §18 budgets under 100 MiB total on a
/// Pi 5.
///
/// `strHash`'s error set reaches beyond this module's (it carries
@@ -419,16 +538,45 @@ test "isEmpty is false once a settings row exists" {
try testing.expect(!try isEmpty(&database));
}
test "isEmpty is false once an auto-materialized client exists" {
test "isEmpty ignores the client rows live traffic materialises" {
var database = try openMigrated();
defer database.close();
try database.exec(
\\INSERT INTO clients (ip, name, group_id, hand_edited, first_seen, last_seen)
\\VALUES ('192.168.1.5', NULL, 1, 0, 1, 1);
// The real §7.2 write path, not hand-written SQL: what makes these rows
// ignorable is that `upsertSeen` is the thing that wrote them.
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
try clients_repo.upsertSeen(&database, "192.168.1.6", 1700000100);
try testing.expectEqual(@as(i64, 2), try clients_repo.countClients(&database));
try testing.expect(try isEmpty(&database));
}
test "isEmpty is false once a client carries operator intent" {
var database = try openMigrated();
defer database.close();
_ = try clients_repo.insertClientRow(
&database,
.{ .ip = "192.168.1.7", .name = "printer", .group_id = 1 },
1700000000,
);
try testing.expect(!try isEmpty(&database));
}
test "isEmpty is false once an operator edits a materialised client" {
var database = try openMigrated();
defer database.close();
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
try testing.expect(try isEmpty(&database));
// A PUT through the API is what turns a discovered device into policy, and
// that policy is exactly what a seed would replace.
const id = try database.queryInt("SELECT id FROM clients WHERE ip = '192.168.1.5'");
try clients_repo.updateClient(&database, id, .{ .name = "tv", .group_id = 1 });
try testing.expect(!try isEmpty(&database));
}
test "isEmpty is false once the seeded group is changed" {
var database = try openMigrated();
defer database.close();
@@ -459,6 +607,206 @@ test "importSource seeds a migrated database and group 'default' keeps id 1" {
);
}
test "an import keeps the materialised clients it found and lets the config claim an address" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
// Two devices the server discovered. `full_source` names the second one.
// The first is seen twice, so `first_seen` and `last_seen` differ and the
// assertions below cannot pass by carrying one column into both.
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000100);
try clients_repo.upsertSeen(&database, "fd00::1", 1700000200);
try importText(io, &database, full_source, .{});
try testing.expectEqual(@as(i64, 2), try clients_repo.countClients(&database));
// The device the config says nothing about keeps its flag, its group and
// both timestamps: a seed is not a reason to forget when a device appeared.
var stmt = try database.prepare(
"SELECT hand_edited, group_id, first_seen, last_seen FROM clients WHERE ip = '192.168.1.5'",
);
defer stmt.deinit();
try testing.expect(try stmt.step());
try testing.expectEqual(@as(i64, 0), stmt.columnInt(0));
try testing.expectEqual(@as(i64, 1), stmt.columnInt(1));
try testing.expectEqual(@as(i64, 1700000000), stmt.columnInt(2));
try testing.expectEqual(@as(i64, 1700000100), stmt.columnInt(3));
// The one the config names belongs to the config: named, in `kids`, and
// hand-edited, so a later prune leaves it alone.
try testing.expectEqual(@as(i64, 1), try database.queryInt(
\\SELECT count(*) FROM clients c JOIN groups g ON g.id = c.group_id
\\ WHERE c.ip = 'fd00::1' AND c.hand_edited = 1 AND c.name = 'tablet' AND g.name = 'kids'
));
}
test "the config claiming a discovered address keeps that device's observed timestamps" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
// The device is seen twice, so the two timestamps differ and neither
// assertion below can pass by carrying one column into the other.
// `full_source` names this address.
try clients_repo.upsertSeen(&database, "fd00::1", 1700000200);
try clients_repo.upsertSeen(&database, "fd00::1", 1700000900);
try importText(io, &database, full_source, .{});
var stmt = try database.prepare(
"SELECT hand_edited, name, first_seen, last_seen FROM clients WHERE ip = 'fd00::1'",
);
defer stmt.deinit();
try testing.expect(try stmt.step());
// The row is the config's: named, hand-edited.
try testing.expectEqual(@as(i64, 1), stmt.columnInt(0));
try testing.expectEqualStrings("tablet", stmt.columnText(1));
// The observation history is the tracker's. The import clock is `now`, so
// both columns would hold a value far above these if the import had written
// its own.
try testing.expectEqual(@as(i64, 1700000200), stmt.columnInt(2));
try testing.expectEqual(@as(i64, 1700000900), stmt.columnInt(3));
}
test "a failed import leaves the observed timestamps exactly as they were" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = testing.allocator;
var database = try openMigrated();
defer database.close();
// One address the config below claims, one it says nothing about.
try clients_repo.upsertSeen(&database, "fd00::1", 1700000200);
try clients_repo.upsertSeen(&database, "fd00::1", 1700000900);
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
const before = try dump(&database, gpa);
defer gpa.free(before);
// The clients go in, the timestamps merge, and then two identical local
// records violate `UNIQUE(name, rtype, value)`. Everything the import wrote
// must go with the transaction.
const broken: model.Config = .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{.{ .url = "https://dns.example/dns-query" }},
.clients = &.{.{ .ip = "fd00::1", .name = "tablet" }},
.local_records = &.{
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
},
};
try testing.expectError(error.Constraint, applyToDb(io, gpa, &database, broken, 42, .{}));
const after = try dump(&database, gpa);
defer gpa.free(after);
try testing.expectEqualStrings(before, after);
}
test "a forced re-import replaces the configured clients and keeps the materialised ones" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
try importText(io, &database, full_source, .{});
try clients_repo.upsertSeen(&database, "10.0.0.9", 1700000200);
// The device the old file named keeps querying, so its row carries real
// observation history when the wipe reaches it.
try clients_repo.upsertSeen(&database, "fd00::1", 1700003000);
try importText(io, &database, minimal_source, .{ .force = true });
// `full_source`'s hand-edited client went with the rest of the old
// configuration; the discovered one did not.
try testing.expectEqual(@as(i64, 1), try clients_repo.countClients(&database));
try testing.expectEqual(@as(i64, 1700000200), try database.queryInt(
"SELECT first_seen FROM clients WHERE ip = '10.0.0.9' AND hand_edited = 0",
));
// The lift saves a configured client's history so the merge can hand it back
// to the same address. It must never become a reason to resurrect a client
// the new file leaves out: dropping a client from the file removes it.
try testing.expectEqual(@as(i64, 0), try database.queryInt(
"SELECT count(*) FROM clients WHERE ip = 'fd00::1'",
));
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
}
test "a forced re-import keeps the observed timestamps of a client the config names" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
// The device appears in traffic, the operator's file names it, and it keeps
// querying afterwards. The row is now `hand_edited = 1` and carries a real
// `first_seen` and a later `last_seen`.
try clients_repo.upsertSeen(&database, "fd00::1", 1700000200);
try importText(io, &database, full_source, .{});
try clients_repo.upsertSeen(&database, "fd00::1", 1700005000);
try importText(io, &database, full_source, .{ .force = true });
var stmt = try database.prepare(
"SELECT hand_edited, first_seen, last_seen FROM clients WHERE ip = 'fd00::1'",
);
defer stmt.deinit();
try testing.expect(try stmt.step());
try testing.expectEqual(@as(i64, 1), stmt.columnInt(0));
// Re-importing the same file is not an observation of the device.
try testing.expectEqual(@as(i64, 1700000200), stmt.columnInt(1));
try testing.expectEqual(@as(i64, 1700005000), stmt.columnInt(2));
}
test "a failed forced import leaves every client timestamp exactly as it was" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = testing.allocator;
var database = try openMigrated();
defer database.close();
// A configured client with observation history, and a materialised one.
try importText(io, &database, full_source, .{});
try clients_repo.upsertSeen(&database, "fd00::1", 1700005000);
try clients_repo.upsertSeen(&database, "10.0.0.9", 1700000200);
const before = try dump(&database, gpa);
defer gpa.free(before);
const broken: model.Config = .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{.{ .url = "https://dns.example/dns-query" }},
.clients = &.{.{ .ip = "fd00::1", .name = "tablet" }},
.local_records = &.{
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
},
};
try testing.expectError(
error.Constraint,
applyToDb(io, gpa, &database, broken, 42, .{ .force = true }),
);
const after = try dump(&database, gpa);
defer gpa.free(after);
try testing.expectEqualStrings(before, after);
}
test "applyToDb without force refuses a configured database and changes nothing" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();