mokhtar/nxdns
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

milestone 13 discrepancies: redact credentials from urls in logs, metrics and cli output

Browse Source
This commit is contained in:
mokhtar
2026-08-07 00:45:17 +02:00
parent 1ff727feb8
commit 8c3328562e
39 changed files with 5734 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/storage/db.zig
+536 -38
View File
@@ -1,12 +1,17 @@
//! The whole SQLite surface nxdns owns (PLAN Decision G). Nothing above this
//! file calls SQLite directly.
//!
//! **This file takes no `std.Io`.** It is the one deliberate exception to
//! Decision E. SQLite performs its own file I/O through its VFS; routing it
//! through `std.Io` would mean writing a custom SQLite VFS — a large,
//! **SQLite's own file I/O takes no `std.Io`.** It is the one deliberate
//! exception to Decision E. SQLite performs its own file I/O through its VFS;
//! routing it through `std.Io` would mean writing a custom SQLite VFS — a large,
//! security-sensitive component bought for nothing at household scale. Every
//! other storage file that touches the filesystem takes `io: std.Io`.
//!
//! The exception covers SQLite, not nxdns. The one place this file does its own
//! filesystem calls — the probes guarding `OpenMode.immutable`, at the open and
//! again at `Db.verifyImmutable` — follows the ordinary rule and takes an
//! `io: std.Io`, which is why that mode carries one.
//!
//! The C API is declared by hand below. No `@cImport` — the handles stay
//! opaque, matching `src/platform/tls_server.zig`'s Mbed TLS approach.
@@ -135,6 +140,11 @@ pub const Error = error{
SqliteError,
OutOfMemory,
Unexpected,
/// Not a SQLite result code. An `OpenMode.immutable` read would have
/// answered from a stale main file, because `<path>-wal` holds bytes or the
/// files moved while the read ran. Raised by the open and again by
/// `Db.verifyImmutable`. See `OpenMode.immutable`.
WalPending,
};
/// Maps a primary SQLite result code to `Error`. `SQLITE_NOMEM` becomes
@@ -190,19 +200,73 @@ fn check(code: c_int) Error!void {
return mapCode(code);
}
pub const OpenMode = enum { read_write_create, read_write_existing, read_only, memory };
pub const OpenMode = union(enum) {
read_write_create,
read_write_existing,
read_only,
memory,
/// Read a database that this process promises not to change, and that no
/// writer may be touching: `SQLITE_OPEN_READONLY` plus the `immutable=1` URI
/// parameter, which makes the pager treat the file like a temp file — no
/// locking, no rollback journal, no wal-index — so **no `-wal` and no `-shm`
/// appear beside it**.
///
/// This mode exists for `nxdns check` (milestone-13 ruling F-c), which must
/// validate without writing. `.read_only` alone is not enough, and this is
/// measured, not assumed: reading a database whose header says WAL makes
/// SQLite build the wal-index, so `config.db-wal` (0 bytes) and
/// `config.db-shm` (32 KiB) are created, and a read-only connection cannot
/// remove them on close. A command that claims to write nothing must not
/// leave two files behind. Do not "simplify" this back to `.read_only`.
///
/// What `immutable=1` costs: SQLite then **ignores any `-wal` file**. The
/// newest committed rows live there, so an immutable read of a database with
/// an un-checkpointed WAL would answer from stale data and say nothing — a
/// worse failure than the two sidecar files it removes. `Db.open` therefore
/// refuses this mode with `error.WalPending` whenever `<path>-wal` exists and
/// is not empty; the caller reports that as a failure and names `nxdns run`,
/// which opens read-write and checkpoints, as the fix.
///
/// The guard lives inside `open` rather than in a helper callers are trusted
/// to call, because the failure it prevents is silent: a caller that forgets
/// a helper gets a plausible wrong answer, and nothing anywhere reports it.
///
/// **The open is half the guard.** `immutable=1` takes no lock, so nothing
/// keeps a writer out for the duration of the read, and a check made only
/// before the read can say only that the log was empty *then*.
/// `Db.verifyImmutable` makes the other half, and a caller that grades what
/// it read without calling it is back to the stale answer this mode exists
/// to refuse.
///
/// A zero-length `-wal` does not block the open: it holds no frames, so the
/// main file is complete. That is the exact leftover the pre-F-c `check`
/// used to create.
///
/// The `std.Io` is for that probe — the one filesystem call nxdns itself
/// makes in this file. Passing it is what makes the guard unskippable.
immutable: std.Io,
};
pub const OpenOptions = struct {
mode: OpenMode = .read_write_create,
busy_timeout_ms: c_int = 5000,
};
/// SQLite's name for the write-ahead log beside `<path>`. Exported so a caller
/// reporting `error.WalPending` can name the file without hard-coding SQLite's
/// naming convention, which this file owns.
pub const wal_suffix = "-wal";
/// One SQLite connection.
///
/// A `Db` must not move once a `Stmt` prepared from it is alive: every `Stmt`
/// holds a `*Db`.
pub const Db = struct {
handle: *c.Sqlite3,
/// Set by `OpenMode.immutable` and null in every other mode: what the
/// database file and its `-wal` looked like when the read began, for
/// `verifyImmutable` to compare against when it ends.
immutable_guard: ?ImmutableGuard = null,
/// Every mode carries `FULLMUTEX` (serialized mode). Phase 6's query logger
/// and Phase 8's API handlers share one handle across `std.Io` tasks, and a
@@ -220,47 +284,59 @@ pub const Db = struct {
.read_write_create, .memory => base | open_flag.readwrite | open_flag.create,
.read_write_existing => base | open_flag.readwrite,
.read_only => base | open_flag.readonly,
// Its own function: the URI buffer is 12 KiB, and every other open
// in the process would carry it in this frame.
.immutable => |io| return openImmutable(io, path, options.busy_timeout_ms),
};
const filename: [:0]const u8 = switch (options.mode) {
.memory => ":memory:",
else => path,
};
var handle: ?*c.Sqlite3 = null;
const rc = c.sqlite3_open_v2(filename.ptr, &handle, flags, null);
if (rc != result.ok) {
// sqlite3_open_v2 allocates a handle even on failure. Read the
// message from it, then close it; dropping it leaks on every
// failed open.
// Logged at `warn`, not `err`: the failure itself reaches the
// caller as a typed error, and this line only carries the message
// that would otherwise die with the handle.
if (handle) |h| {
log.warn("sqlite3_open_v2 failed for '{s}': {s} (code {d}/{d})", .{
filename,
std.mem.span(c.sqlite3_errmsg(h)),
rc & 0xff,
c.sqlite3_extended_errcode(h),
});
_ = c.sqlite3_close_v2(h);
} else {
log.warn("sqlite3_open_v2 failed for '{s}': {s} (code {d})", .{
filename,
std.mem.span(c.sqlite3_errstr(rc)),
rc,
});
}
return mapCode(rc);
}
const h = handle orelse return error.SqliteError;
const h = try openHandle(filename, flags);
return applyBusyTimeout(h, options.busy_timeout_ms);
}
// A silently ignored busy timeout is how a contended WAL database turns
// into random SQLITE_BUSY failures under load.
check(c.sqlite3_busy_timeout(h, options.busy_timeout_ms)) catch |e| {
_ = c.sqlite3_close_v2(h);
return e;
};
return .{ .handle = h };
/// Proves that the files an `OpenMode.immutable` read answered from stood
/// still while it ran, and fails the read with `error.WalPending` when they
/// did not. Call it after the last read and before anything read is
/// reported.
///
/// `immutable=1` takes no lock at all — that is what stops the pager
/// building a wal-index, and the price is that nothing keeps a writer out.
/// The probe at open time can only say the log was empty at that instant: a
/// writer that appends one frame the instant after leaves the read answering
/// from the older pages of the main file, silently, which is the whole
/// failure `OpenMode.immutable`'s guard exists to prevent.
///
/// Two things are compared, because a writer can hide in either:
///
/// - `<path>-wal` holding bytes now. A log that appeared, one that grew, and
/// one written into the empty file the open accepted all land here.
/// - `<path>` itself moving — size, inode, mtime or ctime. This is the
/// checkpoint the log cannot show: a writer that checkpointed into the main
/// file and truncated its log back to nothing leaves both stats saying "no
/// frames" while the pages the read saw have been replaced.
///
/// Best effort, and honestly so: a filesystem with coarse timestamps can
/// hide a rewrite that lands on the same byte count in the same tick. That
/// cannot be fixed from outside SQLite's locking, and taking a lock is the
/// one thing this mode may not do. What it closes is the window a single
/// stat before the read leaves open for the whole of the read.
///
/// Two stats and nothing else: no `-wal` or `-shm` is created, and neither
/// sidecar is removed or truncated. Calling this on a handle opened in any
/// other mode is a caller bug.
pub fn verifyImmutable(self: *Db) Error!void {
const guard = self.immutable_guard orelse unreachable;
const now = try markImmutable(guard.io, guard.path);
if (!std.meta.eql(now, guard.mark)) {
log.warn(
"'{s}' changed while it was being read without a lock; the read is not trustworthy",
.{guard.path},
);
return error.WalPending;
}
}
pub fn close(self: *Db) void {
@@ -355,6 +431,199 @@ pub const Db = struct {
}
};
fn openHandle(filename: [:0]const u8, flags: c_int) Error!*c.Sqlite3 {
var handle: ?*c.Sqlite3 = null;
const rc = c.sqlite3_open_v2(filename.ptr, &handle, flags, null);
if (rc != result.ok) {
// sqlite3_open_v2 allocates a handle even on failure. Read the
// message from it, then close it; dropping it leaks on every
// failed open.
// Logged at `warn`, not `err`: the failure itself reaches the
// caller as a typed error, and this line only carries the message
// that would otherwise die with the handle.
if (handle) |h| {
log.warn("sqlite3_open_v2 failed for '{s}': {s} (code {d}/{d})", .{
filename,
std.mem.span(c.sqlite3_errmsg(h)),
rc & 0xff,
c.sqlite3_extended_errcode(h),
});
_ = c.sqlite3_close_v2(h);
} else {
log.warn("sqlite3_open_v2 failed for '{s}': {s} (code {d})", .{
filename,
std.mem.span(c.sqlite3_errstr(rc)),
rc,
});
}
return mapCode(rc);
}
return handle orelse error.SqliteError;
}
/// A silently ignored busy timeout is how a contended WAL database turns into
/// random SQLITE_BUSY failures under load.
fn applyBusyTimeout(h: *c.Sqlite3, busy_timeout_ms: c_int) Error!Db {
check(c.sqlite3_busy_timeout(h, busy_timeout_ms)) catch |e| {
_ = c.sqlite3_close_v2(h);
return e;
};
return .{ .handle = h };
}
// ---------------------------------------------------------------------------
// OpenMode.immutable
// ---------------------------------------------------------------------------
const uri_scheme = "file:";
const uri_immutable_query = "?immutable=1";
/// Worst case: every byte of the longest path the platform accepts becomes
/// `%HH`.
const immutable_uri_buf_len =
uri_scheme.len + 3 * std.Io.Dir.max_path_bytes + uri_immutable_query.len + 1;
fn openImmutable(io: std.Io, path: [:0]const u8, busy_timeout_ms: c_int) Error!Db {
const mark = try markImmutable(io, path);
var buf: [immutable_uri_buf_len]u8 = undefined;
const uri = try immutableUri(&buf, path);
// `uri` without `open_flag.uri` would be opened as a filename spelled
// "file:...", creating nothing and finding nothing.
const flags = open_flag.exrescode | open_flag.fullmutex |
open_flag.readonly | open_flag.uri;
const h = try openHandle(uri, flags);
var database = try applyBusyTimeout(h, busy_timeout_ms);
database.immutable_guard = .{ .io = io, .path = path, .mark = mark };
return database;
}
/// What `OpenMode.immutable` recorded at the start of a read so that
/// `Db.verifyImmutable` can prove nothing moved by the end of it.
pub const ImmutableGuard = struct {
io: std.Io,
/// Borrowed. Must outlive the `Db`, which every caller satisfies by owning
/// the path for at least as long as the connection it opened with it.
path: []const u8,
mark: FileMark,
};
/// The main database file at one instant, in the fields an outside observer can
/// compare cheaply. `atime` is deliberately absent: reading the file changes it,
/// so comparing it would report every read as a change.
///
/// All zero when the file does not exist, which is itself a state worth
/// comparing — a database replaced by an unlink is a database that moved.
const FileMark = struct {
present: bool,
size: u64,
inode: std.Io.File.INode,
mtime_ns: i96,
ctime_ns: i96,
};
/// The state an immutable read must find unchanged, or `error.WalPending` when
/// `<path>-wal` already holds bytes.
///
/// The `-wal` rule is deliberately conservative: any non-empty log fails.
/// Deciding whether it really holds committed frames means running WAL recovery
/// — checksums, salt, the wal-index — which is the writing that
/// `OpenMode.immutable` exists to avoid. A live writer, a crash, and a
/// checkpointed-but-retained log all land here, and refusing to answer is the
/// right side to err on: the alternative is a stale answer nobody can see is
/// stale. A zero-length log holds no frames, so the main file is complete and it
/// passes.
///
/// A failed stat is not "no WAL": it means this cannot be known, so it stays a
/// failure.
fn markImmutable(io: std.Io, path: []const u8) Error!FileMark {
var buf: [std.Io.Dir.max_path_bytes + wal_suffix.len]u8 = undefined;
const sidecar = std.fmt.bufPrint(&buf, "{s}{s}", .{ path, wal_suffix }) catch
return error.TooBig;
if (try statOrAbsent(io, sidecar)) |wal| {
if (wal.size > 0) return error.WalPending;
}
const main = try statOrAbsent(io, path) orelse return .{
.present = false,
.size = 0,
.inode = 0,
.mtime_ns = 0,
.ctime_ns = 0,
};
return .{
.present = true,
.size = main.size,
.inode = main.inode,
.mtime_ns = main.mtime.nanoseconds,
.ctime_ns = main.ctime.nanoseconds,
};
}
fn statOrAbsent(io: std.Io, path: []const u8) Error!?std.Io.Dir.Stat {
return std.Io.Dir.cwd().statFile(io, path, .{}) catch |e| switch (e) {
error.FileNotFound => return null,
else => {
log.warn("cannot stat '{s}': {t}", .{ path, e });
return error.IoErr;
},
};
}
/// `file:` + percent-encoded `path` + `?immutable=1`.
///
/// The encoding is load-bearing, not cosmetic. `?` opens SQLite's query section
/// and `#` its fragment, so an unencoded data directory named `dns?db` would
/// silently open a *different* file; `%` must be encoded because SQLite decodes
/// `%HH` on its way back to a filename. `--data-dir` is operator input, so all
/// three are reachable.
///
/// Everything outside the unreserved set (`A-Z a-z 0-9 - . _ ~ /`) is encoded,
/// which is always safe: SQLite decodes every escape in the path before handing
/// the name to its VFS, so the bytes it opens are the bytes passed in.
///
/// `/` stays literal to keep diagnostics readable, with one exception. SQLite
/// reads `file://…` as a URI authority and rejects any authority but the empty
/// one or `localhost` (`sqlite3ParseUri`), so a path beginning with `//` — legal
/// POSIX — has its second slash encoded.
fn immutableUri(buf: []u8, path: []const u8) error{TooBig}![:0]const u8 {
var out: usize = 0;
try appendSlice(buf, &out, uri_scheme);
for (path, 0..) |ch, i| {
const opens_authority = i == 1 and ch == '/' and path[0] == '/';
if (isUriUnreserved(ch) and !opens_authority) {
try appendByte(buf, &out, ch);
} else {
const hex = "0123456789ABCDEF";
try appendByte(buf, &out, '%');
try appendByte(buf, &out, hex[ch >> 4]);
try appendByte(buf, &out, hex[ch & 0xf]);
}
}
try appendSlice(buf, &out, uri_immutable_query);
try appendByte(buf, &out, 0);
return buf[0 .. out - 1 :0];
}
fn isUriUnreserved(ch: u8) bool {
return switch (ch) {
'a'...'z', 'A'...'Z', '0'...'9', '-', '.', '_', '~', '/' => true,
else => false,
};
}
fn appendByte(buf: []u8, out: *usize, ch: u8) error{TooBig}!void {
if (out.* == buf.len) return error.TooBig;
buf[out.*] = ch;
out.* += 1;
}
fn appendSlice(buf: []u8, out: *usize, bytes: []const u8) error{TooBig}!void {
for (bytes) |ch| try appendByte(buf, out, ch);
}
/// One prepared statement.
///
/// There is deliberately **no prepared-statement cache in this milestone**.
@@ -735,6 +1004,235 @@ test "a row-producing statement reports its row through step" {
try testing.expect(try stmt.step());
}
// ---------------------------------------------------------------------------
// OpenMode.immutable
// ---------------------------------------------------------------------------
/// `std.testing.tmpDir` creates its directory under `.zig-cache/tmp/` relative to
/// the process working directory, which is also how SQLite's VFS resolves the
/// filename it is handed (`queries_repo.zig:721`).
const tmp_prefix = ".zig-cache/tmp/";
const sub_path_len = @typeInfo(@FieldType(testing.TmpDir, "sub_path")).array.len;
const test_io = testing.io;
fn tmpPath(buf: []u8, tmp: *const testing.TmpDir, name: []const u8) ![:0]const u8 {
return std.fmt.bufPrintZ(buf, "{s}{s}/{s}", .{ tmp_prefix, &tmp.sub_path, name });
}
/// A file database in WAL mode holding one row, `id = marker`. Closing the last
/// connection checkpoints and unlinks both sidecars, but the header keeps saying
/// WAL — which is what makes a later `.read_only` open recreate them.
fn writeWalDatabase(path: [:0]const u8, marker: i64) !void {
var database = try Db.open(path, .{ .mode = .read_write_create });
defer database.close();
try applyPragmas(&database, .{});
try database.exec("CREATE TABLE t (id INTEGER PRIMARY KEY);");
var stmt = try database.prepare("INSERT INTO t (id) VALUES (?1)");
defer stmt.deinit();
try stmt.bindInt(1, marker);
try stmt.exec();
}
/// A second writer doing what a running nxdns does: it appends to the
/// write-ahead log and, being the last connection, checkpoints into the main
/// file and unlinks both sidecars on close. The blob is what makes the main
/// file grow by whole pages, so a test that watches for the change does not rest
/// on the filesystem's timestamp resolution.
fn checkpointOver(path: [:0]const u8) !void {
var database = try Db.open(path, .{ .mode = .read_write_existing });
defer database.close();
try applyPragmas(&database, .{});
try database.exec("INSERT INTO t (id) VALUES (8);");
try database.exec("CREATE TABLE bulk (v TEXT);");
try database.exec("INSERT INTO bulk (v) VALUES (hex(randomblob(30000)));");
}
fn expectAbsent(dir: std.Io.Dir, name: []const u8) !void {
dir.access(test_io, name, .{}) catch |e| switch (e) {
error.FileNotFound => return,
else => |other| return other,
};
std.debug.print("sidecar '{s}' exists and must not\n", .{name});
return error.SidecarPresent;
}
test "immutableUri encodes what would otherwise change which file is opened" {
var buf: [256]u8 = undefined;
try testing.expectEqualStrings(
"file:/var/lib/nxdns/config.db?immutable=1",
try immutableUri(&buf, "/var/lib/nxdns/config.db"),
);
// '?' would start SQLite's query section, '#' its fragment, '%' an escape.
try testing.expectEqualStrings(
"file:/data%3Fdir/config.db?immutable=1",
try immutableUri(&buf, "/data?dir/config.db"),
);
try testing.expectEqualStrings(
"file:/data%23dir/config.db?immutable=1",
try immutableUri(&buf, "/data#dir/config.db"),
);
try testing.expectEqualStrings(
"file:/data%25dir/config.db?immutable=1",
try immutableUri(&buf, "/data%dir/config.db"),
);
try testing.expectEqualStrings(
"file:/a%20b/c%3Fd%23e%25f.db?immutable=1",
try immutableUri(&buf, "/a b/c?d#e%f.db"),
);
// A relative path stays relative: SQLite's VFS resolves it against the
// working directory, exactly as a bare filename would be.
try testing.expectEqualStrings(
"file:config.db?immutable=1",
try immutableUri(&buf, "config.db"),
);
// A leading "//" would be read as a URI authority and rejected.
try testing.expectEqualStrings(
"file:/%2Fnet/share/config.db?immutable=1",
try immutableUri(&buf, "//net/share/config.db"),
);
// Only the authority position is special: "//" further in stays literal.
try testing.expectEqualStrings(
"file:/net//share/config.db?immutable=1",
try immutableUri(&buf, "/net//share/config.db"),
);
// Non-ASCII bytes survive the round trip because SQLite decodes them back.
try testing.expectEqualStrings(
"file:/caf%C3%A9/config.db?immutable=1",
try immutableUri(&buf, "/café/config.db"),
);
var small: [16]u8 = undefined;
try testing.expectError(error.TooBig, immutableUri(&small, "/var/lib/nxdns/config.db"));
}
test "an immutable open of a WAL database creates no -wal and no -shm" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try tmpPath(&path_buf, &tmp, "config.db");
try writeWalDatabase(path, 7);
try expectAbsent(tmp.dir, "config.db-wal");
try expectAbsent(tmp.dir, "config.db-shm");
{
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
// The point of the mode: a `.read_only` open creates both of these here,
// and cannot delete them on close.
try expectAbsent(tmp.dir, "config.db-wal");
try expectAbsent(tmp.dir, "config.db-shm");
}
try expectAbsent(tmp.dir, "config.db-wal");
try expectAbsent(tmp.dir, "config.db-shm");
}
test "an immutable open of a path holding URI metacharacters opens the intended file" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var decoy_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
var path_buf: [tmp_prefix.len + sub_path_len + 64]u8 = undefined;
// Unencoded, SQLite cuts the filename at the '?' and opens "<tmp>/d". That
// file exists here and holds a different database, so the failure without
// percent-encoding is a wrong answer, not an error.
try tmp.dir.createDirPath(test_io, "d?x#y%z");
try writeWalDatabase(try tmpPath(&decoy_buf, &tmp, "d"), 99);
const path = try tmpPath(&path_buf, &tmp, "d?x#y%z/config.db");
try writeWalDatabase(path, 7);
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
}
test "an immutable open refuses a database whose -wal holds bytes" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try tmpPath(&path_buf, &tmp, "config.db");
try writeWalDatabase(path, 7);
// What an unclean shutdown leaves behind, written directly so the case does
// not depend on when SQLite decides to checkpoint.
try tmp.dir.writeFile(test_io, .{ .sub_path = "config.db" ++ wal_suffix, .data = &([_]u8{0x37} ** 32) });
try testing.expectError(error.WalPending, Db.open(path, .{ .mode = .{ .immutable = test_io } }));
// A zero-length `-wal` holds no frames, so the main file is complete: the
// exact leftover the pre-F-c `check` created must not block a check.
try tmp.dir.writeFile(test_io, .{ .sub_path = "config.db" ++ wal_suffix, .data = "" });
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
}
test "an immutable read refuses a -wal that arrives while it is in flight" {
// The probe at open time can only say the log was empty *then*.
// `immutable=1` takes no lock, so a writer is free to arrive one instant
// later, and the read goes on answering from the older pages of the main
// file with nothing anywhere reporting it.
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try tmpPath(&path_buf, &tmp, "config.db");
try writeWalDatabase(path, 7);
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
// Nothing has moved yet, so the read stands.
try database.verifyImmutable();
// A zero-length log still holds no frames: the rule at the end of the read
// is the rule at the start of it.
try tmp.dir.writeFile(test_io, .{ .sub_path = "config.db" ++ wal_suffix, .data = "" });
try database.verifyImmutable();
// Frames, now, in the log the open accepted as empty.
try tmp.dir.writeFile(test_io, .{
.sub_path = "config.db" ++ wal_suffix,
.data = &([_]u8{0x37} ** 32),
});
try testing.expectError(error.WalPending, database.verifyImmutable());
// Repeatable: reporting the race is all it does.
try testing.expectError(error.WalPending, database.verifyImmutable());
// And it repairs nothing. The operator's log is byte for byte what was
// written, and no wal-index appeared beside it.
const wal = try tmp.dir.statFile(test_io, "config.db" ++ wal_suffix, .{});
try testing.expectEqual(@as(u64, 32), wal.size);
try expectAbsent(tmp.dir, "config.db-shm");
}
test "an immutable read refuses a main file checkpointed under it" {
// The case a `-wal` probe cannot see at either end: a writer checkpointed
// into the main file and, closing, unlinked its log again. Both stats say
// "no frames" while the pages the read answered from have been replaced.
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try tmpPath(&path_buf, &tmp, "config.db");
try writeWalDatabase(path, 7);
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
try database.verifyImmutable();
const before = try tmp.dir.statFile(test_io, "config.db", .{});
try checkpointOver(path);
const after = try tmp.dir.statFile(test_io, "config.db", .{});
// The premise of the test, proved rather than assumed: the main file really
// did move, and the log really is gone again.
try testing.expect(after.size != before.size);
try expectAbsent(tmp.dir, "config.db" ++ wal_suffix);
try testing.expectError(error.WalPending, database.verifyImmutable());
}
test "a duplicate insert into a UNIQUE column returns error.Constraint" {
var db = try openMemory();
defer db.close();
src/storage/migrations.zig
+52 -1
View File
@@ -31,6 +31,9 @@ const ddl_v2: [:0]const u8 =
\\ALTER TABLE upstreams ADD COLUMN tls_name TEXT NOT NULL DEFAULT '';
;
/// The schema version this binary expects. A database `readVersion` reports
/// below this needs `nxdns run` to migrate it; above it is `error.SchemaTooNew`
/// and needs a newer nxdns.
pub const target_version: u32 = steps[steps.len - 1].version;
comptime {
@@ -102,9 +105,16 @@ pub fn migrateSteps(database: *db.Db, list: []const Step) Error!u32 {
return target;
}
/// The schema version stamped in `database`, compared against `target_version`.
///
/// `0` when `schema_version` does not exist yet. Zero rows or more than one row
/// is `error.SchemaCorrupt` — the version of a database is never guessed.
fn readVersion(database: *db.Db) Error!u32 {
///
/// Reads only, so it works on a connection opened `.read_only` or
/// `.immutable`. That is what it is public for: `nxdns check` may not migrate
/// (ruling F-c), and "at version 1, this binary expects 2" tells an operator
/// what to do where a bare SQLite error message does not.
pub fn readVersion(database: *db.Db) Error!u32 {
const present = try database.queryInt(
"SELECT count(*) FROM sqlite_schema WHERE type='table' AND name='schema_version'",
);
@@ -302,6 +312,47 @@ test "a failing step after step 2 rolls back the whole upgrade from version 1" {
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
}
test "readVersion reports 0 before a migration and target_version after it" {
var database = try openMigrated();
defer database.close();
try testing.expectEqual(@as(u32, 0), try readVersion(&database));
_ = try migrate(&database);
try testing.expectEqual(target_version, try readVersion(&database));
}
/// `.zig-cache/tmp/` is where `std.testing.tmpDir` puts its directories, and
/// SQLite's VFS resolves filenames against the same working directory
/// (`db.zig`'s immutable tests).
const tmp_prefix = ".zig-cache/tmp/";
const sub_path_len = @typeInfo(@FieldType(testing.TmpDir, "sub_path")).array.len;
test "readVersion reads a file database through an immutable open, writing nothing" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try std.fmt.bufPrintZ(&path_buf, "{s}{s}/config.db", .{ tmp_prefix, &tmp.sub_path });
// A database an older nxdns left at version 1. `check` must report that, not
// migrate it (ruling F-c).
{
var database = try db.Db.open(path, .{ .mode = .read_write_create });
defer database.close();
try db.applyPragmas(&database, .{});
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
try testing.expectEqual(@as(u32, 1), try migrateSteps(&database, &first));
}
var database = try db.Db.open(path, .{ .mode = .{ .immutable = testing.io } });
defer database.close();
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
try testing.expectEqual(@as(u32, 2), target_version);
// A write through this connection is refused by SQLite, not by convention.
try testing.expectError(error.ReadOnly, database.exec("DELETE FROM schema_version;"));
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
}
test "delete_order and content_tables name exactly the tables the schema creates" {
var database = try openMigrated();
defer database.close();
src/storage/repositories/clients_repo.zig
+7 -3
View File
@@ -2,8 +2,11 @@
//!
//! `listClients` returns only `hand_edited = 1` rows. A client the server
//! materialised from live traffic is runtime state, not configuration, and must
//! not appear in an export. `countClients` counts **all** rows, because S5's
//! "has this database ever been configured" predicate needs the true count.
//! not appear in an export. `hand_edited` is the only marker of operator intent
//! in this table, so it also decides what `import.isEmpty` counts: a database
//! carrying nothing but materialised rows has never been configured, and a seed
//! file must still be able to fill it. `countClients` counts **all** rows and is
//! a test helper — it deliberately does not answer that question.
//!
//! The import path is list / insert / deleteAll / count, plus the two runtime
//! calls `upsertSeen` and `pruneStale` that the Phase 7 client tracker owns.
@@ -134,7 +137,8 @@ pub fn deleteAllClients(database: *db.Db) db.Error!void {
return database.exec("DELETE FROM clients;");
}
/// Counts every row, including the ones `listClients` filters out.
/// Counts every row, including the materialised ones `listClients` filters out.
/// Used by tests; `import.isEmpty` counts operator intent instead.
pub fn countClients(database: *db.Db) db.Error!i64 {
return database.queryInt("SELECT count(*) FROM clients");
}
src/storage/repositories/context.zig
+84 -2
View File
@@ -10,6 +10,8 @@
const std = @import("std");
const safe_url = @import("../../safe_url.zig");
const log = std.log.scoped(.repositories);
/// Group name → `groups.id`, or blocklist source URL → `blocklist_sources.id`.
@@ -17,6 +19,42 @@ pub const IdMap = std.StringHashMapUnmanaged(i64);
const no_ids: IdMap = .empty;
/// The line either lookup writes when the caller's map lacks a name, as a value
/// rather than a format string at each call site.
///
/// Two reasons, in that order. A blocklist source url is operator-supplied and
/// nothing on the way in stops it carrying a credential in its userinfo, its
/// path or its query, so it reaches the log through `safe_url.redact` and
/// through nothing else. And a `std.log` line is not readable from a unit test
/// under the default test runner, which installs its own `std_options`; the
/// tests below read this value instead of stderr.
const MissingId = union(enum) {
/// A group name, out of the configuration file or a `groups` row.
group: []const u8,
/// A blocklist source url, out of the configuration file or a
/// `blocklist_sources` row.
source: []const u8,
pub fn format(self: MissingId, w: *std.Io.Writer) std.Io.Writer.Error!void {
switch (self) {
// A group name holds no credential, so nothing is dropped from it.
// It goes through `quoteText` for what that does to any
// operator-supplied string: it delimits a name with a space in it,
// escapes a `'` that would otherwise close the delimiter, and bounds
// a name of any length. The quotes are the type's own, so this
// format string adds none.
.group => |name| try w.print("no group id for {f}", .{safe_url.quoteText(name)}),
// Redaction costs this line the component that told two sources on
// one host apart, and there is no row id to name the source by
// instead: the id the other call sites print is the one this call
// failed to find. The scheme, the host and the port are what is
// left. The rule against writing a secret to a log does not bend for
// a line that would read better with one.
.source => |url| try w.print("no blocklist source id for {f}", .{safe_url.redact(url)}),
}
}
};
pub const InsertContext = struct {
/// Unix epoch seconds, from `std.Io.Clock.real.now(io).toSeconds()`.
now: i64 = 0,
@@ -29,14 +67,16 @@ pub const InsertContext = struct {
/// `NotFound` is a member of `db.Error`, so it needs no wider error set.
pub fn groupId(self: InsertContext, name: []const u8) error{NotFound}!i64 {
return self.group_ids.get(name) orelse {
log.warn("no group id for '{s}'", .{name});
log.warn("{f}", .{MissingId{ .group = name }});
return error.NotFound;
};
}
/// `error.NotFound` means what it means in `groupId`, for the map keyed by
/// source url.
pub fn sourceId(self: InsertContext, url: []const u8) error{NotFound}!i64 {
return self.source_ids.get(url) orelse {
log.warn("no blocklist source id for '{s}'", .{url});
log.warn("{f}", .{MissingId{ .source = url }});
return error.NotFound;
};
}
@@ -44,6 +84,48 @@ pub const InsertContext = struct {
const testing = std.testing;
fn expectLine(expected: []const u8, missing: MissingId) !void {
var buf: [8 * safe_url.max_len]u8 = undefined;
try testing.expectEqualStrings(expected, try std.fmt.bufPrint(&buf, "{f}", .{missing}));
}
test "a missing source id names where the url points and not what it carries" {
// The three components a blocklist url carries a credential in, on the one
// line that used to print all three: an api key in the query, userinfo, and
// a token in a path segment.
try expectLine(
"no blocklist source id for https://lists.example",
.{ .source = "https://lists.example/hosts.txt?apikey=s3cr3t" },
);
try expectLine(
"no blocklist source id for https://lists.example:8443",
.{ .source = "https://user:pa55@lists.example:8443/hosts.txt" },
);
try expectLine(
"no blocklist source id for https://lists.example",
.{ .source = "https://lists.example/download/token/hunter2/hosts.txt" },
);
// What an operator still gets: the scheme, the host and the port.
try expectLine(
"no blocklist source id for http://10.0.0.2:8080",
.{ .source = "http://10.0.0.2:8080/a/hosts.txt" },
);
}
test "a missing group id quotes, escapes and bounds the name" {
try expectLine("no group id for 'kids'", .{ .group = "kids" });
// A name is database text as well as file text, so a newline in it would end
// this line and start one of the operator's choosing.
try expectLine(
"no group id for 'ads\\n2026-01-01 ERROR forged'",
.{ .group = "ads\n2026-01-01 ERROR forged" },
);
// Nor can a name close the quote around it.
try expectLine("no group id for 'kids\\' --'", .{ .group = "kids' --" });
const long_name = "n" ** (2 * safe_url.max_len);
try expectLine("no group id for '" ++ long_name[0..safe_url.max_len] ++ "...'", .{ .group = long_name });
}
test "an InsertContext with no maps reports a missing id rather than trapping" {
const ctx: InsertContext = .{};
try testing.expectError(error.NotFound, ctx.groupId("default"));