mokhtar/nxdns
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

milestone 13 discrepancies: redact credentials from urls in logs, metrics and cli output

Browse Source
This commit is contained in:
mokhtar
2026-08-07 00:45:17 +02:00
parent 1ff727feb8
commit 8c3328562e
39 changed files with 5734 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app.zig
+358 -37
View File
@@ -44,6 +44,7 @@ const doh_client = @import("upstream/doh_client.zig");
const doh_server = @import("server/doh_server.zig");
const dot_client = @import("upstream/dot_client.zig");
const dot_server = @import("server/dot_server.zig");
const faults = @import("config/faults.zig");
const fetcher = @import("filter/fetcher.zig");
const forward_zones = @import("local/forward_zones.zig");
const handler = @import("server/handler.zig");
@@ -60,6 +61,7 @@ const pool_mod = @import("upstream/pool.zig");
const query_sink = @import("server/query_sink.zig");
const rate_limiter = @import("server/rate_limiter.zig");
const retention_mod = @import("storage/retention.zig");
const safe_url = @import("safe_url.zig");
const shutdown = @import("server/shutdown.zig");
const sse = @import("web/sse.zig");
const static = @import("web/static.zig");
@@ -87,24 +89,14 @@ const download_budget_s = 300;
const doh_request_buf_len = 1024;
const doh_transfer_buf_len = 4096;
/// A configuration fault the operator can fix, as opposed to a runtime one.
/// These are the only failures this file raises itself; everything else comes
/// out of a collaborator.
const ConfigError = error{
NoUsableUpstreams,
BadBindAddress,
BadRateLimit,
BadCertificate,
};
pub fn run(runner: cli.Runner, args: cli.RunArgs) u8 {
const code = serve(runner, args) catch |err| code: {
runner.err.print("nxdns run failed: {s}\n", .{@errorName(err)}) catch {};
if (isConfigFault(err)) {
const mapped = failureExitCode(err);
if (mapped == cli.exit_check) {
runner.err.writeAll("run `nxdns check` to see the configuration in full\n") catch {};
break :code cli.exit_check;
}
break :code cli.exit_runtime;
break :code mapped;
};
// Output the operator never received is not output, so a failed flush
@@ -114,15 +106,75 @@ pub fn run(runner: cli.Runner, args: cli.RunArgs) u8 {
return code;
}
fn isConfigFault(err: anyerror) bool {
return switch (err) {
error.NoUsableUpstreams,
error.BadBindAddress,
error.BadRateLimit,
error.BadCertificate,
=> true,
else => false,
};
/// The one classification, from `config/faults.zig`. This file keeps no list of
/// its own: `run` exiting 1 on a seed file `check` and `import` exit 2 on was
/// exactly the cost of the second list that used to be here.
fn failureExitCode(err: anyerror) u8 {
return if (faults.isConfigFault(err)) cli.exit_check else cli.exit_runtime;
}
/// First run only: the file seeds an empty database and is ignored forever
/// after. Its diagnostics are the operator's one chance to see what the file
/// said, so they are printed the way `check` and `import` print them.
///
/// Printed on the way out either way. A seed file can be accepted and still
/// carry warnings — a blocklist source in no group is downloaded and compiled
/// into nothing — and a warning that only appears when the start fails is a
/// warning nobody ever reads: the start it describes is the one that worked.
/// `check` reported it and `run` did not, which left the same file graded two
/// ways.
///
/// The runner's error writer, not `std.log`: this runs before
/// `logging.install`, and one rendering of a diagnostic across `run`, `check`
/// and `import` is the point of `Diagnostics.writeAll`.
///
/// Flushed here rather than left to `run`'s exit flush. That writer is buffered
/// (`main` gives it 4 KiB) and `serve` does not return for as long as the
/// service runs, so a line left in the buffer reaches the operator when the
/// process stops — days after the start it describes. A failure path flushes
/// anyway because it returns immediately; the successful start is the one that
/// needs this.
fn seedFromFile(
r: cli.Runner,
config_db: *db.Db,
dir: std.Io.Dir,
config_path: []const u8,
) bootstrap.Error!bootstrap.Outcome {
var diags: validate.Diagnostics = .init(r.gpa);
defer diags.deinit();
const result = bootstrap.bootstrap(r.io, r.gpa, config_db, dir, config_path, &diags);
// Neither discard is an oversight, and the two answer different questions.
//
// On a rejected seed file, `result` is returned untouched: the operator gets
// the reason the start failed, never a writer error standing in front of it.
// A broken stderr is not why the configuration was refused.
//
// On a seed that worked, a failure here does not stop the start. The trade
// is one lost warning line against a household with no name resolution, and
// `run` before this point is the only stretch of this program where an
// output failure could take DNS down at all — ruling 4 already says nothing
// after it is fatal. Nor could the failure be reported: this writer *is* the
// error channel, and `logging.install` has not run yet, so `std.log` resolves
// to the same stderr a diagnostic about it would have to travel down.
//
// It is not lost from the process either. A failed drain consumes nothing,
// so whatever the buffer held it still holds — that half is observed, in
// "a broken error writer does not stop a first start that succeeded" below,
// which reads the retained warning back out of the same writer.
//
// What happens to those bytes afterwards is derived, not watched, and is
// labelled so deliberately. `Io.Writer.defaultFlush` drains while `end != 0`
// and `run`'s exit flush maps a failure to exit 1, so a stderr still broken
// at shutdown should carry the condition out in the exit code, and one that
// recovered should deliver the line late. No test drives `run` that far.
// Two limits come with the derivation: an empty buffer flushes clean and
// reports nothing at all, and a failure that recovers ends at exit 0 with a
// line the operator reads days after the start it describes.
diags.writeAll(r.err) catch {};
r.err.flush() catch {};
return result;
}
fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
@@ -141,18 +193,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
defer config_db.close();
_ = try migrations.migrate(&config_db);
{
// First run only: the file seeds an empty database and is ignored
// forever after. Its diagnostics are the operator's one chance to see
// why a config file was rejected, so they are printed like `check`
// prints them.
var diags: validate.Diagnostics = .init(gpa);
defer diags.deinit();
_ = bootstrap.bootstrap(io, gpa, &config_db, std.Io.Dir.cwd(), paths.config, &diags) catch |err| {
diags.writeAll(r.err) catch {};
return err;
};
}
_ = try seedFromFile(r, &config_db, std.Io.Dir.cwd(), paths.config);
// Every string in `cfg` points into this arena, and the pool's endpoints,
// the handler's records and the monitor's paths all keep such strings. It
@@ -717,7 +758,7 @@ const Upstreams = struct {
http: *std.http.Client,
bundle: *Certificate.Bundle,
bundle_lock: *std.Io.RwLock,
) (Allocator.Error || ConfigError)!Upstreams {
) (Allocator.Error || error{NoUsableUpstreams})!Upstreams {
var enabled: usize = 0;
for (servers) |server| {
if (server.enabled) enabled += 1;
@@ -748,7 +789,10 @@ const Upstreams = struct {
if (!server.enabled) continue;
const endpoint = transport.Endpoint.parse(server.url) catch {
log.warn("upstream '{s}' is not an https:// or tls:// endpoint; skipped", .{server.url});
log.warn(
"upstream {f} is not an https:// or tls:// endpoint; skipped",
.{safe_url.redactQuoted(server.url)},
);
continue;
};
@@ -762,7 +806,10 @@ const Upstreams = struct {
self.doh_buf[base..][0..doh_request_buf_len],
self.doh_buf[base + doh_request_buf_len ..][0..doh_transfer_buf_len],
) catch {
log.warn("upstream '{s}' is not a usable DoH url; skipped", .{server.url});
log.warn(
"upstream {f} is not a usable DoH url; skipped",
.{safe_url.redactQuoted(server.url)},
);
continue;
};
doh_count += 1;
@@ -888,6 +935,280 @@ test "parseBind refuses a bind address of the wrong family" {
try std.testing.expect(std.mem.containsAtLeast(u8, printed, 1, "is not an IPv4 address"));
}
test "run maps a rejected configuration to exit 2 and everything else to exit 1" {
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.MissingDefaultGroup));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.ParseZon));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.NoUsableUpstreams));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.BadCertificate));
try std.testing.expectEqual(cli.exit_runtime, failureExitCode(error.AccessDenied));
try std.testing.expectEqual(cli.exit_runtime, failureExitCode(error.OutOfMemory));
}
test "run, check and import agree on a seed file with no default group" {
const config_import = @import("config/import.zig");
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
// The file from discrepancy D1: parseable, one upstream, no group named
// 'default'.
const source: [:0]const u8 =
\\.{
\\ .groups = .{ .{ .name = "kids" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\}
;
// `run`: `serve` seeds through `bootstrap`, which is a wrapper over this
// exact call, so this is the error `run` classifies.
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var diags: validate.Diagnostics = .init(gpa);
defer diags.deinit();
try std.testing.expectError(
error.MissingDefaultGroup,
config_import.importSource(io, gpa, &database, source, .{}, &diags),
);
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.MissingDefaultGroup));
// `import`: `cli.failureExitCode` reaches exit 2 by either route — the
// recorded failures, or the classification `run` just used.
try std.testing.expect(diags.failureCount() != 0);
try std.testing.expect(faults.isConfigFault(error.MissingDefaultGroup));
// `check`: the same file, through the code `nxdns check` runs.
var arena_state: std.heap.ArenaAllocator = .init(gpa);
defer arena_state.deinit();
const cfg = try std.zon.parse.fromSliceAlloc(model.Config, arena_state.allocator(), source, null, .{});
var out_buf: [2048]u8 = undefined;
var err_buf: [256]u8 = undefined;
var out: Writer = .fixed(&out_buf);
var err_writer: Writer = .fixed(&err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer };
try std.testing.expectEqual(cli.exit_check, try cli.checkConfig(r, cfg, false));
}
test "a configuration whose blocklist source is in no group imports and checks clean" {
const config_import = @import("config/import.zig");
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
// A source is created before it is attached — `docs/tutorial/first-run.md`
// POSTs the blocklist and then PUTs the group's sources — so an unattached
// source is a legal intermediate state on every write path. It warns; it
// never fails.
const source: [:0]const u8 =
\\.{
\\ .groups = .{ .{ .name = "default" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\ .blocklist_sources = .{ .{ .url = "https://lists.example/hosts.txt", .name = "ads" } },
\\}
;
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var diags: validate.Diagnostics = .init(gpa);
defer diags.deinit();
try config_import.importSource(io, gpa, &database, source, .{}, &diags);
try std.testing.expectEqual(@as(usize, 0), diags.failureCount());
try std.testing.expectEqual(@as(usize, 1), diags.warningCount());
var arena_state: std.heap.ArenaAllocator = .init(gpa);
defer arena_state.deinit();
const cfg = try std.zon.parse.fromSliceAlloc(model.Config, arena_state.allocator(), source, null, .{});
var check_diags: validate.Diagnostics = .init(gpa);
defer check_diags.deinit();
// No error means no subcommand may reject it; the warning is report-only.
try validate.validate(cfg, &check_diags);
try std.testing.expectEqual(@as(usize, 0), check_diags.failureCount());
try std.testing.expectEqual(@as(usize, 1), check_diags.warningCount());
}
test "a first start that seeds from a file prints the warnings the file earned" {
// D5, second half. The seed file is read once in the life of a database, so
// a warning it earns is printed on that start or never. `run` printed
// diagnostics only when the file was rejected, which made a successful first
// start the one place the finding could not surface.
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try tmp.dir.writeFile(io, .{ .sub_path = "config.zon", .data =
\\.{
\\ .groups = .{ .{ .name = "default" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\ .blocklist_sources = .{ .{ .url = "https://lists.example/hosts.txt", .name = "ads" } },
\\}
});
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
// A buffered file writer, the shape `main` builds over stderr, and not
// `Writer.fixed`: a fixed writer's flush is a no-op, so it counts a line
// still sitting in the buffer as delivered. Reading the file back is the
// only way to ask what the operator can actually see.
var out_buf: [64]u8 = undefined;
var out: Writer = .fixed(&out_buf);
var err_file = try tmp.dir.createFile(io, "stderr.txt", .{});
defer err_file.close(io);
var err_buf: [4096]u8 = undefined;
var err_writer = err_file.writer(io, &err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer.interface };
// The file is valid, so the start succeeds and the database is seeded.
try std.testing.expectEqual(
bootstrap.Outcome.seeded,
try seedFromFile(r, &database, tmp.dir, "config.zon"),
);
try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
// Nothing flushes here on purpose. In production `serve` runs from this
// point until the service stops, so a line that has not reached the file by
// now is a line the operator does not get for days.
const printed = try tmp.dir.readFileAlloc(io, "stderr.txt", gpa, .limited(8192));
defer gpa.free(printed);
try std.testing.expect(std.mem.containsAtLeast(u8, printed, 1, "WARN blocklist_sources[0]: "));
try std.testing.expect(std.mem.containsAtLeast(u8, printed, 1, "belongs to no group"));
// A warning is not a rejection: nothing here claims the start failed.
try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, printed, "FAIL"));
}
/// A broken stderr, in the shape `main` builds: a buffered `File.Writer`, with
/// its drain switched to the mode that fails. `Writer.fixed` cannot stand in —
/// its flush is `noopFlush`, so it has no failure to report and its `written()`
/// counts a line still sitting in the buffer as delivered.
///
/// The file stays empty for as long as the mode is `.failure`, which is what
/// lets a test tell "the writer really failed" from "the writer worked".
fn brokenErrWriter(io: std.Io, file: std.Io.File, buffer: []u8) std.Io.File.Writer {
var w = file.writer(io, buffer);
w.mode = .failure;
return w;
}
test "a broken error writer does not replace the reason a seed file was rejected" {
// The operator has to see why seeding failed, and a broken stderr is not
// that reason.
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// No group named 'default': rejected, and it records a FAIL line on the way.
try tmp.dir.writeFile(io, .{ .sub_path = "config.zon", .data =
\\.{
\\ .groups = .{ .{ .name = "kids" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\}
});
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var out_buf: [64]u8 = undefined;
var out: Writer = .fixed(&out_buf);
var err_file = try tmp.dir.createFile(io, "stderr.txt", .{});
defer err_file.close(io);
// Eight bytes: no FAIL line fits, so `writeAll` must drain mid-line and is
// itself the call that fails. The test below covers the other discard, where
// the line fits and only the flush fails.
var err_buf: [8]u8 = undefined;
var err_writer = brokenErrWriter(io, err_file, &err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer.interface };
try std.testing.expectError(
error.MissingDefaultGroup,
seedFromFile(r, &database, tmp.dir, "config.zon"),
);
// Empty, so the writer did fail — without this the assertion above would
// hold just as well against a writer that worked.
const printed = try tmp.dir.readFileAlloc(io, "stderr.txt", gpa, .limited(8192));
defer gpa.free(printed);
try std.testing.expectEqual(@as(usize, 0), printed.len);
}
test "a broken error writer does not stop a first start that succeeded" {
// The call this file makes: a DNS server for a household does not refuse to
// resolve because stderr is broken. What it must not do is drop the warning
// on the floor, so the second half checks the buffer still holds it.
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// Valid, and it earns one warning: the source belongs to no group.
try tmp.dir.writeFile(io, .{ .sub_path = "config.zon", .data =
\\.{
\\ .groups = .{ .{ .name = "default" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\ .blocklist_sources = .{ .{ .url = "https://lists.example/hosts.txt", .name = "ads" } },
\\}
});
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var out_buf: [64]u8 = undefined;
var out: Writer = .fixed(&out_buf);
var err_file = try tmp.dir.createFile(io, "stderr.txt", .{});
defer err_file.close(io);
// 4 KiB, the buffer `main` gives the real stderr writer: the warning fits,
// so `writeAll` succeeds into the buffer and the flush is what fails. That
// is the shape production hits.
var err_buf: [4096]u8 = undefined;
var err_writer = brokenErrWriter(io, err_file, &err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer.interface };
try std.testing.expectEqual(
bootstrap.Outcome.seeded,
try seedFromFile(r, &database, tmp.dir, "config.zon"),
);
try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
// Nothing reached the file, so the flush really did fail.
const undelivered = try tmp.dir.readFileAlloc(io, "stderr.txt", gpa, .limited(8192));
defer gpa.free(undelivered);
try std.testing.expectEqual(@as(usize, 0), undelivered.len);
// And the warning is still buffered rather than dropped: this is the same
// writer, and these are the bytes `run`'s exit flush meets on the way out.
err_writer.mode = .positional;
try err_writer.interface.flush();
const printed = try tmp.dir.readFileAlloc(io, "stderr.txt", gpa, .limited(8192));
defer gpa.free(printed);
try std.testing.expect(std.mem.containsAtLeast(u8, printed, 1, "WARN blocklist_sources[0]: "));
}
fn reportBind(r: cli.Runner, which: []const u8, addr: net.IpAddress, err: anyerror) anyerror {
r.err.print("cannot bind {s} {f}: {s}\n", .{ which, addr, @errorName(err) }) catch {};
return err;
src/cli.zig
+657 -66
View File
@@ -20,9 +20,11 @@ const tls = std.crypto.tls;
const app = @import("app.zig");
const config_export = @import("config/export.zig");
const faults = @import("config/faults.zig");
const import = @import("config/import.zig");
const model = @import("config/model.zig");
const validate = @import("config/validate.zig");
const cert_store = @import("server/cert_store.zig");
const db = @import("storage/db.zig");
const migrations = @import("storage/migrations.zig");
const querylog_schema = @import("storage/querylog_schema.zig");
@@ -30,6 +32,7 @@ const doh_client = @import("upstream/doh_client.zig");
const dot_client = @import("upstream/dot_client.zig");
const pool = @import("upstream/pool.zig");
const transport = @import("upstream/transport.zig");
const safe_url = @import("safe_url.zig");
const version = @import("version.zig");
pub const exit_ok: u8 = 0;
@@ -436,7 +439,7 @@ pub fn runImport(r: Runner, args: ImportArgs) u8 {
// should need one run to see the whole list.
diags.writeAll(r.err) catch {};
r.err.print("import failed: {s}\n", .{@errorName(e)}) catch {};
return finish(r, failureExitCode(e, diags.problems.items.len));
return finish(r, failureExitCode(e, diags.failureCount()));
};
return finish(r, exit_ok);
}
@@ -458,32 +461,42 @@ fn importImpl(r: Runner, args: ImportArgs, diags: *validate.Diagnostics) !void {
.{ .force = args.force },
diags,
);
// A configuration that imports cleanly can still have recorded warnings — a
// blocklist source in no group is the one that found this. `validate`
// returns nothing for a warning, so printing diagnostics on the failure path
// alone made `import` the command that read the finding and threw it away,
// while `check` printed it from the same file. Only warnings can be here:
// any recorded failure returned above.
try diags.writeAll(r.out);
try r.out.print("imported {s}\n", .{args.file});
}
/// A configuration the operator can fix exits 2; everything else is a runtime
/// failure. `validate` records a diagnostic for every error it returns and then
/// returns the first one, so a non-empty diagnostics list is the reliable
/// discriminator; the named errors below are the config faults that never reach
/// the validator.
/// returns the first one, so a recorded failure is the reliable discriminator;
/// `config/faults.zig` classifies the errors that never reach the validator.
/// That file is the only list — this function keeps none of its own, which is
/// what stops `run`, `check` and `import` drifting apart again (D1).
///
/// `error.OutOfMemory` is matched first, before the list is consulted. Both
/// recording paths — `validate` and import's per-line rendering of a ZON syntax
/// error — add one problem at a time and can run out of memory partway, which
/// leaves problems recorded for a run whose real outcome is a resource failure.
/// A partial report is not a verdict on the configuration, so the runtime exit
/// code wins.
fn failureExitCode(e: anyerror, problems: usize) u8 {
/// `error.DatabaseNotEmpty` is the one exception, and it is deliberate: it
/// reports the state of the database rather than the content of a file, so it
/// is not a configuration fault, and `import` alone decides it is exit 2.
///
/// `error.OutOfMemory` is matched first, before anything else is consulted.
/// Both recording paths — `validate` and import's per-line rendering of a ZON
/// syntax error — add one problem at a time and can run out of memory partway,
/// which leaves problems recorded for a run whose real outcome is a resource
/// failure. A partial report is not a verdict on the configuration, so the
/// runtime exit code wins.
///
/// `failures` counts recorded failures, never warnings: a warning never changes
/// an exit code (F-b).
fn failureExitCode(e: anyerror, failures: usize) u8 {
if (e == error.OutOfMemory) return exit_runtime;
if (problems != 0) return exit_check;
return switch (e) {
error.DatabaseNotEmpty,
error.ConfigTooLarge,
error.ParseZon,
error.PasswordAndHashBothSet,
=> exit_check,
else => exit_runtime,
};
if (failures != 0) return exit_check;
if (e == error.DatabaseNotEmpty) return exit_check;
return if (faults.isConfigFault(e)) exit_check else exit_runtime;
}
// ---------------------------------------------------------------------------
@@ -520,21 +533,10 @@ fn checkImpl(r: Runner, args: CheckArgs, probe: bool) !u8 {
return checkFile(r, arena, args.paths.config, probe);
}
const config_db_path = try std.fs.path.join(arena, &.{ args.paths.data_dir, config_db_name });
const config_db_path = try std.fs.path.joinZ(arena, &.{ args.paths.data_dir, config_db_name });
if (try pathExists(r.io, config_db_path)) {
try r.out.print("checking database {s}\n", .{config_db_path});
var data = try DataDir.open(r.io, r.gpa, args.paths.data_dir, false);
defer data.close(r.io, r.gpa);
// A `check` on a database one schema version behind must still work,
// which is what an operator runs right after an upgrade.
var database = try data.openConfigDb(r.io);
defer database.close();
_ = try migrations.migrate(&database);
const cfg = try config_export.readConfig(&database, arena);
return checkConfig(r, cfg, probe);
return checkDatabase(r, arena, config_db_path, probe);
}
if (try pathExists(r.io, args.paths.config)) {
@@ -550,6 +552,121 @@ fn checkImpl(r: Runner, args: CheckArgs, probe: bool) !u8 {
return exit_check;
}
/// `check` reads `config.db` and writes nothing to it (F-c): no create, no
/// chmod, no `applyPragmas` — which is what would turn WAL on and leave
/// `config.db-wal` and `config.db-shm` behind — and above all no `migrate`. A
/// database behind this binary's schema is reported here; upgrading it is
/// `nxdns run`'s job, and a command that claims to validate without writing may
/// not commit a schema step.
///
/// `.immutable` is the enforcement, not a convention: SQLite refuses every
/// statement that would write, and the mode's `immutable=1` also stops the
/// pager building a wal-index, so no `config.db-wal` and no `config.db-shm`
/// appear beside the file. `.read_only` alone creates both and cannot delete
/// them on close, which is how a "validate without writing" command left two
/// files behind.
fn checkDatabase(r: Runner, arena: Allocator, path: [:0]const u8, probe: bool) !u8 {
var database = db.Db.open(path, .{ .mode = .{ .immutable = r.io } }) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.WalPending => return walPendingFailure(r, path),
else => return unreadable(r, path, e),
};
defer database.close();
const reading = try readDatabase(&database, arena);
// `immutable=1` takes no lock — that is what keeps it from building a
// wal-index and leaving two sidecars behind — so a writer was free to append
// to the log or checkpoint into the main file for the whole of the read
// above. Proved before one word of it is reported: a stale answer and a torn
// read both look exactly like an ordinary finding, which is how this failure
// stays invisible.
database.verifyImmutable() catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
error.WalPending => return walPendingFailure(r, path),
else => return unreadable(r, path, e),
};
switch (reading) {
.version_unreadable => |e| {
try r.out.print("FAIL {s}: the schema version cannot be read ({s})\n", .{ path, @errorName(e) });
return exit_check;
},
// Naming both numbers is the point: "at 1, expects 2" tells an operator
// to run `nxdns run`, where a bare SQLite complaint about a missing
// column tells them nothing.
.version_mismatch => |stamped| {
const fix = if (stamped < migrations.target_version)
"`nxdns run` migrates it, `check` will not"
else
"it was written by a newer nxdns";
try r.out.print(
"FAIL {s}: schema version {d}, this nxdns expects {d}; {s}\n",
.{ path, stamped, migrations.target_version, fix },
);
return exit_check;
},
.config_unreadable => {
// The schema version is already known good, so whatever this is,
// the SQLite message is the only thing that narrows it down.
// `verifyImmutable` makes no SQLite call, so this is still the
// message from the read.
var buf: [256]u8 = undefined;
try r.out.print("FAIL {s}: cannot be read ({s})\n", .{ path, database.lastError(&buf) });
return exit_check;
},
.config => |cfg| return checkConfig(r, cfg, probe),
}
}
/// Everything read out of `config.db`, held rather than reported, because a
/// value from an unlocked read is only worth reporting once `verifyImmutable`
/// has said the files it came from stood still. A wrong schema-version line is
/// as misleading as a wrong setting.
const DbReading = union(enum) {
config: model.Config,
version_unreadable: anyerror,
version_mismatch: u32,
config_unreadable,
};
/// Reads only, so an immutable handle serves it.
fn readDatabase(database: *db.Db, arena: Allocator) error{OutOfMemory}!DbReading {
const stamped = migrations.readVersion(database) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return .{ .version_unreadable = e },
};
if (stamped != migrations.target_version) return .{ .version_mismatch = stamped };
const cfg = config_export.readConfig(database, arena) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => return .config_unreadable,
};
return .{ .config = cfg };
}
/// One wording for `error.WalPending`, whether the log was already there when
/// the read opened or arrived while it ran: from the operator's side those are
/// the same situation, a writer holding changes this read cannot see.
///
/// `immutable=1` ignores the write-ahead log, so the newest committed settings
/// would be invisible and `check` would quietly grade the older ones in the main
/// file. The guard is deliberately conservative — a live writer, an interrupted
/// process and a checkpointed log that was simply kept all look the same from
/// outside — so the line says what to do and does not claim anything is damaged.
fn walPendingFailure(r: Runner, path: []const u8) !u8 {
try r.out.print(
"FAIL {s}: uncheckpointed changes are waiting in {s}{s}, and reading without writing would answer from the older settings in the main file; `nxdns run` applies them. A running nxdns normally holds this log, which is the usual reason to see this line.\n",
.{ path, path, db.wal_suffix },
);
return exit_check;
}
fn unreadable(r: Runner, path: []const u8, e: anyerror) !u8 {
try r.out.print("FAIL {s}: cannot be opened for reading ({s})\n", .{ path, @errorName(e) });
return exit_check;
}
fn pathExists(io: std.Io, path: []const u8) std.Io.Dir.AccessError!bool {
std.Io.Dir.cwd().access(io, path, .{}) catch |e| switch (e) {
error.FileNotFound => return false,
@@ -583,6 +700,19 @@ fn checkFile(r: Runner, arena: Allocator, path: []const u8, probe: bool) !u8 {
try r.out.print("FAIL {s}: larger than {d} bytes\n", .{ path, import.max_config_bytes });
return exit_check;
},
// D4: a named file that is missing or unreadable is the same
// operator-fixable condition as one that fails to parse, so it is
// reported as a finding rather than escaping as a runtime failure. The
// implicit path already exits 2 when it finds nothing to check; naming
// the file must not change the code.
error.FileNotFound => {
try r.out.print("FAIL {s}: no such file\n", .{path});
return exit_check;
},
error.AccessDenied, error.PermissionDenied => {
try r.out.print("FAIL {s}: not readable\n", .{path});
return exit_check;
},
else => |other| return other,
};
@@ -602,48 +732,83 @@ fn checkFile(r: Runner, arena: Allocator, path: []const u8, probe: bool) !u8 {
return checkConfig(r, cfg, probe);
}
/// What one part of `check` found. Failures set exit 2; warnings are printed,
/// counted for the summary, and never change an exit code (F-b) — the service
/// starts either way.
const Tally = struct {
failures: usize = 0,
warnings: usize = 0,
fn plus(self: Tally, other: Tally) Tally {
return .{
.failures = self.failures + other.failures,
.warnings = self.warnings + other.warnings,
};
}
};
/// The half of `check` that has a `Config` already: validate, report every
/// diagnostic, then the certificate and upstream checks. With TLS disabled and
/// `probe` false it touches neither the filesystem nor the network, which is
/// what makes it unit-testable.
///
/// The summary never contradicts the lines above it (D2): a run that printed
/// WARN lines says so instead of claiming no problems were found, and still
/// exits 0.
pub fn checkConfig(r: Runner, cfg: model.Config, probe: bool) !u8 {
var diags: validate.Diagnostics = .init(r.gpa);
defer diags.deinit();
// The returned error is `problems[0].err` — one of the lines about to be
// printed — so it carries nothing the report does not. Only an allocation
// failure means the report itself is incomplete.
// The returned error is the first recorded failure — one of the lines about
// to be printed — so it carries nothing the report does not. Only an
// allocation failure means the report itself is incomplete.
validate.validate(cfg, &diags) catch |e| switch (e) {
error.OutOfMemory => return error.OutOfMemory,
else => {},
};
// `writeAll` prints the "FAIL "/"WARN " prefix itself; the lines below add
// their own because they are not diagnostics.
try diags.writeAll(r.out);
var failures = diags.problems.items.len;
failures += try checkCertificates(r, cfg);
if (probe) failures += try probeUpstreams(r, cfg);
var tally: Tally = .{ .failures = diags.failureCount(), .warnings = diags.warningCount() };
tally = tally.plus(try checkCertificates(r, cfg));
if (probe) tally.failures += try probeUpstreams(r, cfg);
if (failures != 0) return exit_check;
if (tally.failures != 0) return exit_check;
if (tally.warnings != 0) {
try r.out.print("OK: no failures found, {d} warning{s}\n", .{
tally.warnings,
if (tally.warnings == 1) "" else "s",
});
return exit_ok;
}
try r.out.writeAll("OK: no problems found\n");
return exit_ok;
}
fn checkCertificates(r: Runner, cfg: model.Config) !usize {
return try checkTlsFiles(r, cfg.doh_server, "doh_server") +
try checkTlsFiles(r, cfg.dot_server, "dot_server");
fn checkCertificates(r: Runner, cfg: model.Config) !Tally {
const doh = try checkTlsFiles(r, cfg.doh_server, "doh_server");
const dot = try checkTlsFiles(r, cfg.dot_server, "dot_server");
return doh.plus(dot);
}
/// An unreadable certificate or key fails the run; a key readable by anyone
/// beyond its owner is a warning (PLAN §19) and leaves the exit code alone,
/// because the service still starts.
fn checkTlsFiles(r: Runner, endpoint: model.TlsEndpoint, comptime section: []const u8) !usize {
if (!endpoint.enabled) return 0;
var failures: usize = 0;
if (!try pathReadable(r.io, endpoint.cert_path)) {
try r.out.print("FAIL " ++ section ++ ".cert_path: '{s}' is not readable\n", .{endpoint.cert_path});
failures += 1;
}
/// Proves the pair rather than the paths (D3). `CertStore.init` is the load the
/// listeners boot with: it reads both PEM files and builds a
/// `tls_server.ServerContext`, which is where Mbed TLS parses the chain, parses
/// the key and checks that the key belongs to the leaf. Testing readability
/// alone let `check` exit 0 on a certificate and key that do not pair, seconds
/// before `run` exited 2 on `BadCertificate`.
///
/// No listener is bound and nothing is published: the context is built and
/// freed. `alpn` is null because ALPN is negotiated per connection and plays no
/// part in loading a pair; every other input is the server's.
///
/// A key readable by anyone beyond its owner stays a warning (PLAN §19) and is
/// reported even when the pair itself fails, because it is a separate finding
/// about a file that exists.
fn checkTlsFiles(r: Runner, endpoint: model.TlsEndpoint, comptime section: []const u8) !Tally {
if (!endpoint.enabled) return .{};
var tally: Tally = .{};
if (try pathReadable(r.io, endpoint.key_path)) {
const stat = try std.Io.Dir.cwd().statFile(r.io, endpoint.key_path, .{});
@@ -653,13 +818,46 @@ fn checkTlsFiles(r: Runner, endpoint: model.TlsEndpoint, comptime section: []con
"WARN " ++ section ++ ".key_path: '{s}' is mode {o}; a TLS key must be readable by its owner only\n",
.{ endpoint.key_path, mode },
);
tally.warnings += 1;
}
} else {
try r.out.print("FAIL " ++ section ++ ".key_path: '{s}' is not readable\n", .{endpoint.key_path});
failures += 1;
}
return failures;
var store = cert_store.CertStore.init(
r.gpa,
r.io,
endpoint.cert_path,
endpoint.key_path,
null,
) catch |e| {
const at: struct { field: []const u8, path: []const u8 } = switch (e) {
error.OutOfMemory => return error.OutOfMemory,
// Not a verdict on the configuration: the platform's entropy source
// failed, which is the same failure `run` would hit.
error.EntropyFailed => return error.EntropyFailed,
error.CertUnreadable,
error.CertTooLarge,
error.CertParse,
// Mbed TLS rejected the server configuration built from this pair,
// so the certificate is what the operator has to look at.
error.ConfigFailed,
=> .{ .field = "cert_path", .path = endpoint.cert_path },
error.KeyUnreadable,
error.KeyTooLarge,
error.KeyParse,
error.KeyMismatch,
=> .{ .field = "key_path", .path = endpoint.key_path },
};
try r.out.print("FAIL " ++ section ++ ".{s}: '{s}': {s}\n", .{
at.field,
at.path,
cert_store.humanMessage(e),
});
tally.failures += 1;
return tally;
};
store.deinit(r.io);
return tally;
}
/// One `Pool` per upstream, never one pool over all of them. The pool's job is
@@ -697,11 +895,19 @@ fn probeUpstreams(r: Runner, cfg: model.Config) !usize {
// backoff, so the value only has to be a value.
const seed: u64 = @truncate(@as(u96, @bitCast(std.Io.Clock.real.now(r.io).nanoseconds)));
for (cfg.upstreams) |server| {
// Every line below names the upstream by its index into the configuration
// as well as by its redacted url, and the index is the config index rather
// than a count of the upstreams probed — a disabled entry still occupies
// one. The url alone no longer identifies an entry: `safe_url.redact` drops
// the path, and two upstreams on one host commonly differ only there (a
// NextDNS profile is `https://dns.nextdns.io/<profile>`). `upstreams[N]` is
// the same name `config/validate.zig` gives the entry, so a FAIL line here
// and a FAIL line from the validator point at the same place.
for (cfg.upstreams, 0..) |server, i| {
if (!server.enabled) continue;
const endpoint = transport.Endpoint.parse(server.url) catch {
try r.out.print("FAIL {s}: not an https:// or tls:// endpoint\n", .{server.url});
try r.out.print("FAIL upstreams[{d}] {f}: not an https:// or tls:// endpoint\n", .{ i, safe_url.redactQuoted(server.url) });
failures += 1;
continue;
};
@@ -713,7 +919,7 @@ fn probeUpstreams(r: Runner, cfg: model.Config) !usize {
const client: transport.Client = switch (endpoint.scheme) {
.doh => doh: {
doh = doh_client.DohClient.init(&http, endpoint, &request_buf, &transfer_buf) catch {
try r.out.print("FAIL {s}: not a usable DoH url\n", .{server.url});
try r.out.print("FAIL upstreams[{d}] {f}: not a usable DoH url\n", .{ i, safe_url.redactQuoted(server.url) });
failures += 1;
continue;
};
@@ -740,7 +946,7 @@ fn probeUpstreams(r: Runner, cfg: model.Config) !usize {
var single: pool.Pool = .init(&entries, .{}, attempt_timeout, seed);
if (single.exchange(r.io, probe_query, response_buf)) |_| {
try r.out.print("OK {s}\n", .{server.url});
try r.out.print("OK upstreams[{d}] {f}\n", .{ i, safe_url.redact(server.url) });
} else |_| {
// The concrete cause lives in the entry's health, which is where the
// pool put it; `@errorName` of the pool's return value would only
@@ -748,7 +954,7 @@ fn probeUpstreams(r: Runner, cfg: model.Config) !usize {
var snapshots: [1]pool.Snapshot = undefined;
const taken = try single.snapshot(r.io, &snapshots);
const detail = if (taken == 1) snapshots[0].last_error else "no detail recorded";
try r.out.print("FAIL {s}: {s}\n", .{ server.url, detail });
try r.out.print("FAIL upstreams[{d}] {f}: {s}\n", .{ i, safe_url.redactQuoted(server.url), detail });
failures += 1;
}
}
@@ -1019,6 +1225,391 @@ test "an allocation failure while recording diagnostics exits 1, not 2" {
try testing.expect(saw_partial_report);
}
// `runCheck` against real paths, `runExport` and `runImport` all need a real
// data directory, and the upstream probe leaves the machine. Those cases are
// S7's: `src/storage/storage_integration_test.zig` cases 20-22.
test "failureExitCode keeps no list of its own and classifies through config/faults.zig" {
// D1: every one of these reached `import` from a rejected seed file and was
// classified as a runtime failure by the private list this function used to
// carry, while `check` called the same file a configuration fault.
try testing.expectEqual(exit_check, failureExitCode(error.MissingDefaultGroup, 0));
try testing.expectEqual(exit_check, failureExitCode(error.NoUpstreams, 0));
try testing.expectEqual(exit_check, failureExitCode(error.BadUpstreamUrl, 0));
try testing.expectEqual(exit_check, failureExitCode(error.NoUsableUpstreams, 0));
try testing.expectEqual(exit_check, failureExitCode(error.BadCertificate, 0));
// Every member of the shared classification, so a variant added to
// `ValidateError` cannot exit 1 from `import` while exiting 2 from `run`.
inline for (@typeInfo(validate.ValidateError).error_set.?) |member| {
const err = @field(anyerror, member.name);
const expected: u8 = if (faults.isConfigFault(err)) exit_check else exit_runtime;
try testing.expectEqual(expected, failureExitCode(err, 0));
}
// The one config-shaped exit 2 `cli` still decides for itself: it reports
// the state of the database, not the content of a file.
try testing.expect(!faults.isConfigFault(error.DatabaseNotEmpty));
try testing.expectEqual(exit_check, failureExitCode(error.DatabaseNotEmpty, 0));
}
const fixtures = @import("test_fixtures");
/// A tmp directory holding the fixture PEM pair and a data directory, addressed
/// by cwd-relative paths the same way an operator's configuration names them.
/// Must not move after `init`: the slices point into the buffers.
const CheckEnv = struct {
tmp: testing.TmpDir,
cert_path_buf: [160]u8,
key_path_buf: [160]u8,
data_dir_buf: [160]u8,
missing_path_buf: [160]u8,
cert_path: []const u8,
key_path: []const u8,
data_dir: []const u8,
/// A path inside the tmp directory that is never created.
missing_path: []const u8,
fn init(env: *CheckEnv) !void {
env.tmp = testing.tmpDir(.{});
errdefer env.tmp.cleanup();
env.cert_path = try env.path(&env.cert_path_buf, "cert.pem");
env.key_path = try env.path(&env.key_path_buf, "key.pem");
env.data_dir = try env.path(&env.data_dir_buf, "data");
env.missing_path = try env.path(&env.missing_path_buf, "no-such-config.zon");
}
fn deinit(env: *CheckEnv) void {
env.tmp.cleanup();
}
fn path(env: *const CheckEnv, buf: []u8, name: []const u8) ![]const u8 {
return std.fmt.bufPrint(buf, ".zig-cache/tmp/{s}/{s}", .{ env.tmp.sub_path, name });
}
/// The key lands at 0600, so only a test that asks for the permission
/// warning gets one.
fn writePair(env: *CheckEnv, io: std.Io, key_pem: []const u8) !void {
try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = fixtures.cert_pem });
try env.tmp.dir.writeFile(io, .{ .sub_path = "key.pem", .data = key_pem });
try env.tmp.dir.setFilePermissions(io, "key.pem", .fromMode(0o600), .{});
}
/// Valid but for whatever the test broke about the TLS pair.
fn tlsConfig(env: *const CheckEnv) model.Config {
return .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{.{ .url = "https://dns.example/dns-query" }},
.doh_server = .{
.enabled = true,
.cert_path = env.cert_path,
.key_path = env.key_path,
},
};
}
fn expectAbsent(env: *CheckEnv, io: std.Io, name: []const u8) !void {
env.tmp.dir.access(io, name, .{}) catch |e| switch (e) {
error.FileNotFound => return,
else => |other| return other,
};
std.debug.print("expected '{s}' not to exist\n", .{name});
return error.TestUnexpectedResult;
}
};
test "check fails a certificate and a key that do not pair" {
// D3: readability alone passed this configuration, and `run` then exited 2
// on `BadCertificate` seconds later.
var env: CheckEnv = undefined;
try env.init();
defer env.deinit();
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
try env.writePair(r.io, fixtures.mismatched_key_pem);
try testing.expectEqual(exit_check, try checkConfig(r, env.tlsConfig(), false));
const text = captured.out.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "FAIL doh_server.key_path"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "does not belong to the certificate"));
try testing.expectEqual(@as(usize, 0), std.mem.count(u8, text, "OK:"));
}
test "check fails a certificate file that does not parse" {
var env: CheckEnv = undefined;
try env.init();
defer env.deinit();
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
try env.writePair(r.io, fixtures.key_pem);
try env.tmp.dir.writeFile(r.io, .{ .sub_path = "cert.pem", .data = "not a certificate\n" });
try testing.expectEqual(exit_check, try checkConfig(r, env.tlsConfig(), false));
const text = captured.out.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "FAIL doh_server.cert_path"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "could not be parsed"));
}
test "check passes a certificate and a key that pair" {
var env: CheckEnv = undefined;
try env.init();
defer env.deinit();
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
try env.writePair(r.io, fixtures.key_pem);
try testing.expectEqual(exit_ok, try checkConfig(r, env.tlsConfig(), false));
try testing.expectEqualStrings("OK: no problems found\n", captured.out.written());
}
test "a check whose only findings are warnings exits 0 and says so" {
// D2: the summary used to read "OK: no problems found" directly under the
// WARN line it was contradicting.
var env: CheckEnv = undefined;
try env.init();
defer env.deinit();
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
try env.writePair(r.io, fixtures.key_pem);
try env.tmp.dir.setFilePermissions(r.io, "key.pem", .fromMode(0o644), .{});
// A warning never changes an exit code: the service still starts.
try testing.expectEqual(exit_ok, try checkConfig(r, env.tlsConfig(), false));
const text = captured.out.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "WARN doh_server.key_path"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "OK: no failures found, 1 warning\n"));
try testing.expectEqual(@as(usize, 0), std.mem.count(u8, text, "no problems found"));
}
test "check --config naming a missing file is a reported failure at exit 2" {
// D4: this escaped `checkImpl` as `check failed: FileNotFound` at exit 1,
// while the implicit path exits 2 for the same operator-fixable condition.
var env: CheckEnv = undefined;
try env.init();
defer env.deinit();
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
const code = runCheck(r, .{
.paths = .{ .config = env.missing_path },
.config_explicit = true,
}, false);
try testing.expectEqual(exit_check, code);
const text = captured.out.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "FAIL"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "no such file"));
try testing.expectEqualStrings("", captured.err.written());
}
test "check reads config.db without writing to it" {
// D6: the database branch opened read/write, chmod'ed 0600, turned WAL on —
// which is what creates the two sidecars — and committed migration steps,
// from a command whose contract is that it validates without writing.
var env: CheckEnv = undefined;
try env.init();
defer env.deinit();
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
{
var data = try DataDir.open(r.io, r.gpa, env.data_dir, true);
defer data.close(r.io, r.gpa);
var database = try data.openConfigDb(r.io);
defer database.close();
_ = try migrations.migrate(&database);
}
// A mode `openConfigDb` would overwrite, so its chmod cannot hide.
try env.tmp.dir.setFilePermissions(r.io, "data/config.db", .fromMode(0o644), .{});
const before = try env.tmp.dir.statFile(r.io, "data/config.db", .{});
_ = runCheck(r, .{ .paths = .{ .data_dir = env.data_dir } }, false);
try testing.expect(std.mem.containsAtLeast(u8, captured.out.written(), 1, "checking database"));
// Byte for byte the database the writer left, at the mode the writer left:
// no migration step committed, no chmod, no `PRAGMA journal_mode`.
const after = try env.tmp.dir.statFile(r.io, "data/config.db", .{});
try testing.expectEqual(before.size, after.size);
try testing.expectEqual(before.mtime.nanoseconds, after.mtime.nanoseconds);
try testing.expectEqual(
@as(@TypeOf(after.permissions.toMode()), 0o644),
after.permissions.toMode() & 0o777,
);
// Nothing beside it either. A `.read_only` open recreates the wal-index of
// a database whose header says WAL and cannot delete it on close, which
// left `config.db-wal` and `config.db-shm` behind; `.immutable` builds no
// wal-index at all.
try env.expectAbsent(r.io, "data/config.db-wal");
try env.expectAbsent(r.io, "data/config.db-shm");
}
test "check refuses a database whose write-ahead log still holds changes" {
// `immutable=1` ignores the log, so grading the main file would silently
// report settings the operator already replaced.
var env: CheckEnv = undefined;
try env.init();
defer env.deinit();
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
{
var data = try DataDir.open(r.io, r.gpa, env.data_dir, true);
defer data.close(r.io, r.gpa);
var database = try data.openConfigDb(r.io);
defer database.close();
_ = try migrations.migrate(&database);
}
// Bytes are all the guard reads, and it never gets as far as opening this
// as a log: `db.Db.open` refuses on the size alone.
try env.tmp.dir.writeFile(r.io, .{
.sub_path = "data/config.db" ++ db.wal_suffix,
.data = "uncheckpointed frames",
});
const code = runCheck(r, .{ .paths = .{ .data_dir = env.data_dir } }, false);
try testing.expectEqual(exit_check, code);
const text = captured.out.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "FAIL"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "uncheckpointed changes"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "config.db" ++ db.wal_suffix));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns run"));
// Not a claim about damage: an operator reading this must not reach for a
// recovery tool.
try testing.expectEqual(@as(usize, 0), std.mem.count(u8, text, "corrupt"));
}
test "check reports a database behind the schema rather than migrating it" {
var env: CheckEnv = undefined;
try env.init();
defer env.deinit();
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
// Zero bytes is a valid, empty SQLite database: schema version 0, which is
// exactly what an upgrade leaves behind when a step has not run yet.
_ = try std.Io.Dir.cwd().createDirPathStatus(r.io, env.data_dir, .fromMode(0o700));
try env.tmp.dir.writeFile(r.io, .{ .sub_path = "data/config.db", .data = "" });
const code = runCheck(r, .{ .paths = .{ .data_dir = env.data_dir } }, false);
try testing.expectEqual(exit_check, code);
const text = captured.out.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "FAIL"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns run"));
// Both real numbers, not a generic SQLite complaint: an empty database is
// at version 0, and the expected one comes from the migration list rather
// than a literal, so adding a step cannot make this line lie.
var expected_buf: [96]u8 = undefined;
const expected = try std.fmt.bufPrint(
&expected_buf,
"schema version 0, this nxdns expects {d};",
.{migrations.target_version},
);
try testing.expect(std.mem.containsAtLeast(u8, text, 1, expected));
// Not upgraded: the schema `migrate` would have committed is still absent.
const after = try env.tmp.dir.statFile(r.io, "data/config.db", .{});
try testing.expectEqual(@as(u64, 0), after.size);
try env.expectAbsent(r.io, "data/config.db-wal");
try env.expectAbsent(r.io, "data/config.db-shm");
}
test "the upstream probe redacts a url it cannot parse, without leaving the machine" {
// No socket is opened on this branch: `Endpoint.parse` rejects the `@`
// before the loop builds a client, so the leak is reachable in a required
// test rather than only behind a live probe.
//
// It is also the only probe branch a credential can reach. `Endpoint.parse`
// refuses `@`, `?` and `#` in the authority and `?`/`#` in the path, so a
// url carrying userinfo or a query never gets as far as `DohClient.init`
// and its "not a usable DoH url" line. That line's redaction is defensive.
//
// The two upstreams are the NextDNS shape, where the profile id in the path
// is the account credential and is the only thing telling two entries
// apart. Both halves of the contract are asserted at once: neither the
// userinfo nor the profile id may reach stdout, and what is left has to
// still say which of the two entries the operator must go and fix. The
// disabled entry ahead of them is there because the index has to be the
// index into `upstreams`, not a count of the entries probed.
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
const cfg: model.Config = .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{
.{ .url = "https://dns.example/dns-query", .enabled = false },
.{ .url = "https://lists:hunter2@dns.nextdns.io/abcd12" },
.{ .url = "https://lists:hunter2@dns.nextdns.io/efgh34" },
},
};
try testing.expectEqual(@as(usize, 2), try probeUpstreams(r, cfg));
const text = captured.out.written();
try testing.expectEqualStrings(
"FAIL upstreams[1] 'https://dns.nextdns.io': not an https:// or tls:// endpoint\n" ++
"FAIL upstreams[2] 'https://dns.nextdns.io': not an https:// or tls:// endpoint\n",
text,
);
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "hunter2"));
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "abcd12"));
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "efgh34"));
}
test "a successful import prints the warnings the file earned" {
// D5, second half. `check` printed this WARN and `import` recorded it and
// threw it away, because diagnostics were written on the failure path only.
// The operator who imported the file learned nothing about the source they
// had just added, which blocks nothing until a group links it.
var env: CheckEnv = undefined;
try env.init();
defer env.deinit();
var captured: Captured = .init(testing.allocator);
defer captured.deinit();
const r = captured.runner();
try env.tmp.dir.writeFile(r.io, .{ .sub_path = "config.zon", .data =
\\.{
\\ .groups = .{ .{ .name = "default" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\ .blocklist_sources = .{ .{ .url = "https://lists.example/hosts.txt", .name = "ads" } },
\\}
});
var file_buf: [160]u8 = undefined;
const file = try env.path(&file_buf, "config.zon");
const code = runImport(r, .{ .paths = .{ .data_dir = env.data_dir }, .file = file });
// A warning never changes an exit code, and the import really happened.
try testing.expectEqual(exit_ok, code);
const text = captured.out.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "WARN blocklist_sources[0]: "));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "belongs to no group"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "imported "));
try testing.expectEqualStrings("", captured.err.written());
}
// `runExport` needs a real data directory and the upstream probe leaves the
// machine. Those cases are S7's:
// `src/storage/storage_integration_test.zig` cases 20-22.
src/config/bootstrap.zig
+85 -4
View File
@@ -7,7 +7,10 @@
//! The policy is three lines long:
//!
//! - no file → normal steady state, keep the database as it is;
//! - database already configured → the file is ignored, as PLAN §3.5 requires;
//! - database already configured → the file is ignored, as PLAN §3.5 requires.
//! Configured means an operator put something there. A database that has only
//! answered queries is not configured, however many client rows the DNS path
//! materialised into it, and `import.isEmpty` is where that line is drawn;
//! - otherwise → import it, and a file that is unreadable, unparseable or
//! invalid is an error. The operator wrote that file and meant it; starting
//! with silent defaults instead is the exact failure mode PLAN §1.3 exists to
@@ -55,6 +58,84 @@ pub fn bootstrap(
return .seeded;
}
// Every path through `bootstrap` starts with a filesystem access, so all three
// outcomes are exercised in `src/storage/storage_integration_test.zig` (S7)
// against real files. There is nothing here that an in-memory test could reach.
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
//
// All three outcomes are exercised end to end in
// `src/storage/storage_integration_test.zig` (S7) against a real data directory.
// What the two cases below add is the one distinction that decides which outcome
// an operator gets, and it is too important to leave behind a `-Dintegration`
// flag: whether the database has been *configured*, not whether it has been
// *used*.
const testing = std.testing;
const clients_repo = @import("../storage/repositories/clients_repo.zig");
const migrations = @import("../storage/migrations.zig");
const seed_source =
\\.{
\\ .groups = .{ .{ .name = "default" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\}
;
/// Unparseable on purpose: a call that succeeds proves the file was never read.
const broken_source = ".{ .groups = ";
fn openMigrated() !db.Db {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
return database;
}
test "a server that has answered queries still seeds from its configuration file" {
const io = testing.io;
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
try tmp.dir.writeFile(io, .{ .sub_path = "config.zon", .data = seed_source });
var database = try openMigrated();
defer database.close();
// The unattended first boot: the server came up on defaults, answered
// traffic, and the operator dropped a config file in afterwards.
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
var diags: validate.Diagnostics = .init(testing.allocator);
defer diags.deinit();
const outcome = try bootstrap(io, testing.allocator, &database, tmp.dir, "config.zon", &diags);
try testing.expectEqual(Outcome.seeded, outcome);
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
// Seeding did not cost the operator the device list they had been watching.
try testing.expectEqual(@as(i64, 1), try clients_repo.countClients(&database));
}
test "a client the operator has customised keeps the configuration file out" {
const io = testing.io;
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
try tmp.dir.writeFile(io, .{ .sub_path = "config.zon", .data = broken_source });
var database = try openMigrated();
defer database.close();
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
const id = try database.queryInt("SELECT id FROM clients WHERE ip = '192.168.1.5'");
try clients_repo.updateClient(&database, id, .{ .name = "tv", .group_id = 1 });
var diags: validate.Diagnostics = .init(testing.allocator);
defer diags.deinit();
const outcome = try bootstrap(io, testing.allocator, &database, tmp.dir, "config.zon", &diags);
try testing.expectEqual(Outcome.db_already_configured, outcome);
try testing.expectEqual(@as(usize, 0), diags.problems.items.len);
// The name and the flag the operator set are still theirs.
try testing.expectEqual(@as(i64, 1), try database.queryInt(
"SELECT count(*) FROM clients WHERE name = 'tv' AND hand_edited = 1",
));
}
src/config/faults.zig
+115
View File
@@ -0,0 +1,115 @@
//! One definition of "the operator's configuration is wrong".
//!
//! `run`, `check` and `import` all sort a failure into two buckets: the
//! configuration is wrong and the operator can fix it (exit 2, `nxdns check`
//! is the next step), or something else broke (exit 1). Each subcommand used to
//! carry its own list of which errors meant which, and the lists disagreed —
//! the same seed file exited 1 from `run` and 2 from `check`. There is one list
//! now, and it is this file. Nothing else may keep a second one.
const std = @import("std");
const validate = @import("validate.zig");
/// `ValidateError` enters as a whole set rather than variant by variant, so a
/// variant added to the validator cannot silently fall through to exit 1. The
/// four extras are the configuration faults raised outside the validator: the
/// ZON reader (`ParseZon`), the seed-file size limit (`ConfigTooLarge`), the
/// composition root's upstream build (`NoUsableUpstreams`) and its certificate
/// load (`BadCertificate`).
///
/// Not here on purpose: `error.DatabaseNotEmpty`, which reports the state of
/// the database rather than the content of a file, and is the one config-shaped
/// exit 2 `cli` decides for itself.
const ConfigFault = validate.ValidateError || error{
ParseZon,
ConfigTooLarge,
NoUsableUpstreams,
BadCertificate,
};
const faults: []const anyerror = blk: {
const set = @typeInfo(ConfigFault).error_set.?;
var list: [set.len]anyerror = undefined;
for (set, 0..) |member, i| list[i] = @field(anyerror, member.name);
const frozen = list;
break :blk &frozen;
};
/// True when `err` means the configuration the operator supplied is wrong.
/// Linear over a set of about forty errors, on failure paths only.
pub fn isConfigFault(err: anyerror) bool {
for (faults) |fault| {
if (err == fault) return true;
}
return false;
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
const testing = std.testing;
test "every ValidateError variant is a configuration fault, with no exceptions" {
// The guard on the derivation: a variant added to `ValidateError` and left
// out of the classification fails here rather than exiting 1 in the field.
//
// No member is excused. This file used to subtract `error.OutOfMemory`
// here, which made the rule "every ValidateError is a fault, except one" —
// a private exclusion list of exactly the kind this file exists to abolish.
// `validate.ValidateError` no longer carries an allocation failure, so the
// rule is literal again.
inline for (@typeInfo(validate.ValidateError).error_set.?) |member| {
const err = @field(anyerror, member.name);
if (!isConfigFault(err)) {
std.debug.print("isConfigFault(error.{s}) is false, expected true\n", .{member.name});
return error.TestUnexpectedResult;
}
}
}
test "an allocation failure is not a member of the validator's verdict" {
// The root of it: `faults.zig` can only be exception-free while
// `ValidateError` holds nothing that is not a verdict on the file.
inline for (@typeInfo(validate.ValidateError).error_set.?) |member| {
if (std.mem.eql(u8, member.name, "OutOfMemory")) {
std.debug.print("ValidateError carries error.OutOfMemory\n", .{});
return error.TestUnexpectedResult;
}
}
// It is still reachable from `validate`, just not as a finding: the
// allocator can fail and the caller has to handle it.
comptime var reachable = false;
inline for (@typeInfo(validate.Error).error_set.?) |member| {
if (comptime std.mem.eql(u8, member.name, "OutOfMemory")) reachable = true;
}
try testing.expect(reachable);
}
test "the faults raised outside the validator are configuration faults" {
try testing.expect(isConfigFault(error.ParseZon));
try testing.expect(isConfigFault(error.ConfigTooLarge));
try testing.expect(isConfigFault(error.NoUsableUpstreams));
try testing.expect(isConfigFault(error.BadCertificate));
}
test "the seed-file errors that used to exit 1 from run are configuration faults" {
// D1 verbatim: these three reached `run` from a rejected seed file and were
// classified as runtime failures.
try testing.expect(isConfigFault(error.ParseZon));
try testing.expect(isConfigFault(error.MissingDefaultGroup));
try testing.expect(isConfigFault(error.NoUpstreams));
}
test "a runtime failure is not a configuration fault" {
try testing.expect(!isConfigFault(error.OutOfMemory));
try testing.expect(!isConfigFault(error.AccessDenied));
try testing.expect(!isConfigFault(error.FileNotFound));
try testing.expect(!isConfigFault(error.AddressInUse));
// A state conflict, not a bad file: `import` refuses to overwrite a
// configured database and decides that exit code itself.
try testing.expect(!isConfigFault(error.DatabaseNotEmpty));
// Only ever a warning, so it never reaches an exit code by this route.
try testing.expect(!isConfigFault(error.SourceInNoGroup));
}
src/config/import.zig
+363 -15
View File
@@ -1,4 +1,13 @@
//! `nxdns import`: a ZON file becomes the whole content of `config.db`.
//! `nxdns import`: a ZON file becomes the whole configuration of `config.db`.
//!
//! Configuration, not content: the `hand_edited = 0` client rows the DNS path
//! materialises from live traffic are runtime state, they are absent from an
//! export, and an import carries them across rather than deleting them.
//!
//! `clients.first_seen` and `clients.last_seen` are runtime state on every client
//! row, configured ones included, and the config model carries neither. So an
//! address the database already knew keeps both across an import, and only an
//! address it has never seen takes the import's clock.
//!
//! The order is the specification. Nothing reaches the database until the file
//! has been read, parsed and validated, and every write happens inside one
@@ -44,21 +53,31 @@ const canonical_buf_len = 64;
// emptiness
// ---------------------------------------------------------------------------
/// A database is "never configured" when the migrations have run and nothing
/// else has. The migrations themselves create `schema_version` and seed
/// `groups(1, 'default')`, so "no rows anywhere" is the wrong test.
/// A database is "never configured" when the migrations have run and the
/// operator has added nothing. The migrations themselves create
/// `schema_version` and seed `groups(1, 'default')`, so "no rows anywhere" is
/// the wrong test.
///
/// True iff every table in `config_schema.content_tables` is empty, `groups`
/// holds exactly one row, and that row is the seeded `(1, 'default', 0)`.
/// True iff no table in `config_schema.content_tables` holds a row the operator
/// put there, `groups` holds exactly one row, and that row is the seeded
/// `(1, 'default', 0)`.
///
/// The client count here includes auto-materialised rows: a server that has
/// answered one query is configured enough that a bootstrap file must not
/// overwrite it.
/// `clients` is the one table a row can reach without an operator: the DNS path
/// materialises `hand_edited = 0` rows straight from live traffic (PLAN §7.2).
/// Counting those made emptiness a function of traffic — a server that had
/// answered a single query silently ignored the seed file its operator dropped
/// next to it. So only `hand_edited = 1` rows count, which is the predicate
/// `clients_repo.listClients` already exports by: this database is empty exactly
/// when its export is empty.
pub fn isEmpty(database: *db.Db) db.Error!bool {
// `inline for` over a comptime table list: every statement below is a
// compile-time string, so no table name is ever concatenated at run time.
inline for (config_schema.content_tables) |table| {
if (try database.queryInt("SELECT count(*) FROM " ++ table) != 0) return false;
const count_sql = comptime if (std.mem.eql(u8, table, "clients"))
"SELECT count(*) FROM clients WHERE hand_edited = 1"
else
"SELECT count(*) FROM " ++ table;
if (try database.queryInt(count_sql) != 0) return false;
}
if (try database.queryInt("SELECT count(*) FROM groups") != 1) return false;
const seeded = try database.queryInt(
@@ -179,6 +198,8 @@ pub fn applyToDb(
// writes are one atomic unit.
if (!options.force and !try isEmpty(database)) return error.DatabaseNotEmpty;
try liftSavedClients(database);
inline for (config_schema.delete_order) |table| {
try database.exec("DELETE FROM " ++ table ++ ";");
}
@@ -203,6 +224,8 @@ pub fn applyToDb(
canonical.ip = try canonicalIp(client.ip, &buf);
try clients_repo.insertClient(database, canonical, ctx);
}
try restoreSavedClients(database, group_ids.get("default").?);
for (cfg.client_prefixes) |entry| {
var buf: [canonical_buf_len]u8 = undefined;
var canonical = entry;
@@ -238,6 +261,102 @@ pub fn applyToDb(
try tx.commit();
}
const lift_saved_clients_sql: [:0]const u8 =
\\DROP TABLE IF EXISTS temp.saved_clients;
\\CREATE TEMP TABLE saved_clients AS
\\ SELECT ip, name, hand_edited, first_seen, last_seen FROM clients;
;
/// Carries what the wipe must not destroy over the wipe that follows: the
/// auto-materialised client rows themselves, and the observed timestamps of
/// every client row whatever its flag.
///
/// A `hand_edited = 0` row is runtime state, not configuration: the DNS path
/// wrote it from live traffic and `clients_repo.listClients` already keeps it out
/// of an export. Replacing the *configuration* must therefore not delete it —
/// but it cannot survive in place either, because `clients.group_id` references
/// `groups(id)` with no cascade, and the wipe empties `groups`. So the rows step
/// aside into the temp database and come back once `groups` holds `default`
/// again.
///
/// The table takes every row, not only the `hand_edited = 0` ones, because
/// `first_seen` and `last_seen` are runtime state on a configured row too — the
/// tracker keeps writing `last_seen` on the clients the operator named. Saving
/// only the materialised rows would keep the observation history of the devices
/// nobody named and destroy it for the devices somebody did. `hand_edited` rides
/// along so the restore can tell the two apart.
///
/// `IF EXISTS` because a rolled-back import must not poison the next one.
fn liftSavedClients(database: *db.Db) Error!void {
return database.exec(lift_saved_clients_sql);
}
const merge_observed_timestamps_sql: [:0]const u8 =
\\UPDATE clients AS c
\\ SET first_seen = m.first_seen, last_seen = m.last_seen
\\ FROM temp.saved_clients m
\\ WHERE m.ip = c.ip
;
const restore_materialised_clients_sql =
\\INSERT INTO clients (ip, name, group_id, hand_edited, first_seen, last_seen)
\\SELECT m.ip, m.name, ?1, 0, m.first_seen, m.last_seen
\\ FROM temp.saved_clients m
\\ WHERE m.hand_edited = 0
\\ AND NOT EXISTS (SELECT 1 FROM clients c WHERE c.ip = m.ip)
;
/// Puts back what `liftSavedClients` set aside, in two steps that must stay in
/// this order: the restore drops the temp table on its way out, so the merge
/// cannot follow it.
///
/// **The merge.** An address the config declares belongs to the config — but
/// `first_seen` and `last_seen` are not the config's to state. The model carries
/// neither field, so `insertClient` writes `now` into both as a placeholder for a
/// device it knows nothing about. When the database already held that address, the
/// placeholder is the worse of the two values and both columns come from the saved
/// row instead. Everything else on the row stays the config's: the name, the
/// group, and `hand_edited = 1`.
///
/// Not "the earlier `first_seen` and the later `last_seen`": the placeholder is
/// the wall clock at import time and every real observation predates it, so
/// "later" would resolve to the placeholder every time and stamp each named
/// device as seen at the moment of the import. An operator restoring a backup
/// would read that as liveness. `first_seen` and `last_seen` mean "when a query
/// from this address arrived", `pruneStale` and the API both read them that way,
/// and an import is not a query. Taking both from the saved row also keeps
/// `first_seen <= last_seen`, which `upsertSeen` guarantees pairwise.
///
/// Only the config's own clients are in the table at this point, so the update
/// needs no filter of its own, and the saved row's flag does not enter into it: a
/// device keeps its history whether the previous row was materialised or
/// configured. A `--force` re-import of the same file is the case that matters —
/// it deletes and rewrites every configured client, and without the merge each
/// one would come back claiming it was first seen at the moment of the import.
///
/// **The restore.** Only `hand_edited = 0` rows come back. A configured client is
/// configuration, so a file that leaves its address out has removed that client
/// and the row must stay gone; its history was saved for the merge, not for a
/// resurrection. `WHERE NOT EXISTS` rather than `INSERT OR IGNORE`: the only
/// materialised row worth skipping is one whose address the config claims — the
/// operator naming a device the server had already discovered, handled by the
/// merge above — and every other constraint failure stays loud.
///
/// The rows return to `default`, the group `upsertSeen` materialises into.
/// `first_seen` and `last_seen` cross unchanged; the row id does not, because the
/// config's clients went in first and hold the low ids now. Nothing references
/// `clients.id` inside `config.db`, and §3.6 keeps the query log out of it.
fn restoreSavedClients(database: *db.Db, default_group_id: i64) Error!void {
try database.exec(merge_observed_timestamps_sql);
{
var stmt = try database.prepare(restore_materialised_clients_sql);
defer stmt.deinit();
try stmt.bindInt(1, default_group_id);
try stmt.exec();
}
return database.exec("DROP TABLE temp.saved_clients;");
}
/// `default` goes in first and takes rowid 1. §11.2 seeds group 1 as `default`
/// and §7.2's fallback assignment depends on it; letting an import renumber it
/// would silently move every unassigned client.
@@ -303,7 +422,7 @@ fn canonicalPrefix(text: []const u8, buf: []u8) error{BadClientPrefix}![]const u
}
/// argon2id with the OWASP parameters (t=2, m=19 MiB, p=1) rather than the
/// 64 MiB `interactive_2id`, because PLAN §18 budgets under 100 MB total on a
/// 64 MiB `interactive_2id`, because PLAN §18 budgets under 100 MiB total on a
/// Pi 5.
///
/// `strHash`'s error set reaches beyond this module's (it carries
@@ -419,16 +538,45 @@ test "isEmpty is false once a settings row exists" {
try testing.expect(!try isEmpty(&database));
}
test "isEmpty is false once an auto-materialized client exists" {
test "isEmpty ignores the client rows live traffic materialises" {
var database = try openMigrated();
defer database.close();
try database.exec(
\\INSERT INTO clients (ip, name, group_id, hand_edited, first_seen, last_seen)
\\VALUES ('192.168.1.5', NULL, 1, 0, 1, 1);
// The real §7.2 write path, not hand-written SQL: what makes these rows
// ignorable is that `upsertSeen` is the thing that wrote them.
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
try clients_repo.upsertSeen(&database, "192.168.1.6", 1700000100);
try testing.expectEqual(@as(i64, 2), try clients_repo.countClients(&database));
try testing.expect(try isEmpty(&database));
}
test "isEmpty is false once a client carries operator intent" {
var database = try openMigrated();
defer database.close();
_ = try clients_repo.insertClientRow(
&database,
.{ .ip = "192.168.1.7", .name = "printer", .group_id = 1 },
1700000000,
);
try testing.expect(!try isEmpty(&database));
}
test "isEmpty is false once an operator edits a materialised client" {
var database = try openMigrated();
defer database.close();
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
try testing.expect(try isEmpty(&database));
// A PUT through the API is what turns a discovered device into policy, and
// that policy is exactly what a seed would replace.
const id = try database.queryInt("SELECT id FROM clients WHERE ip = '192.168.1.5'");
try clients_repo.updateClient(&database, id, .{ .name = "tv", .group_id = 1 });
try testing.expect(!try isEmpty(&database));
}
test "isEmpty is false once the seeded group is changed" {
var database = try openMigrated();
defer database.close();
@@ -459,6 +607,206 @@ test "importSource seeds a migrated database and group 'default' keeps id 1" {
);
}
test "an import keeps the materialised clients it found and lets the config claim an address" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
// Two devices the server discovered. `full_source` names the second one.
// The first is seen twice, so `first_seen` and `last_seen` differ and the
// assertions below cannot pass by carrying one column into both.
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000100);
try clients_repo.upsertSeen(&database, "fd00::1", 1700000200);
try importText(io, &database, full_source, .{});
try testing.expectEqual(@as(i64, 2), try clients_repo.countClients(&database));
// The device the config says nothing about keeps its flag, its group and
// both timestamps: a seed is not a reason to forget when a device appeared.
var stmt = try database.prepare(
"SELECT hand_edited, group_id, first_seen, last_seen FROM clients WHERE ip = '192.168.1.5'",
);
defer stmt.deinit();
try testing.expect(try stmt.step());
try testing.expectEqual(@as(i64, 0), stmt.columnInt(0));
try testing.expectEqual(@as(i64, 1), stmt.columnInt(1));
try testing.expectEqual(@as(i64, 1700000000), stmt.columnInt(2));
try testing.expectEqual(@as(i64, 1700000100), stmt.columnInt(3));
// The one the config names belongs to the config: named, in `kids`, and
// hand-edited, so a later prune leaves it alone.
try testing.expectEqual(@as(i64, 1), try database.queryInt(
\\SELECT count(*) FROM clients c JOIN groups g ON g.id = c.group_id
\\ WHERE c.ip = 'fd00::1' AND c.hand_edited = 1 AND c.name = 'tablet' AND g.name = 'kids'
));
}
test "the config claiming a discovered address keeps that device's observed timestamps" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
// The device is seen twice, so the two timestamps differ and neither
// assertion below can pass by carrying one column into the other.
// `full_source` names this address.
try clients_repo.upsertSeen(&database, "fd00::1", 1700000200);
try clients_repo.upsertSeen(&database, "fd00::1", 1700000900);
try importText(io, &database, full_source, .{});
var stmt = try database.prepare(
"SELECT hand_edited, name, first_seen, last_seen FROM clients WHERE ip = 'fd00::1'",
);
defer stmt.deinit();
try testing.expect(try stmt.step());
// The row is the config's: named, hand-edited.
try testing.expectEqual(@as(i64, 1), stmt.columnInt(0));
try testing.expectEqualStrings("tablet", stmt.columnText(1));
// The observation history is the tracker's. The import clock is `now`, so
// both columns would hold a value far above these if the import had written
// its own.
try testing.expectEqual(@as(i64, 1700000200), stmt.columnInt(2));
try testing.expectEqual(@as(i64, 1700000900), stmt.columnInt(3));
}
test "a failed import leaves the observed timestamps exactly as they were" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = testing.allocator;
var database = try openMigrated();
defer database.close();
// One address the config below claims, one it says nothing about.
try clients_repo.upsertSeen(&database, "fd00::1", 1700000200);
try clients_repo.upsertSeen(&database, "fd00::1", 1700000900);
try clients_repo.upsertSeen(&database, "192.168.1.5", 1700000000);
const before = try dump(&database, gpa);
defer gpa.free(before);
// The clients go in, the timestamps merge, and then two identical local
// records violate `UNIQUE(name, rtype, value)`. Everything the import wrote
// must go with the transaction.
const broken: model.Config = .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{.{ .url = "https://dns.example/dns-query" }},
.clients = &.{.{ .ip = "fd00::1", .name = "tablet" }},
.local_records = &.{
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
},
};
try testing.expectError(error.Constraint, applyToDb(io, gpa, &database, broken, 42, .{}));
const after = try dump(&database, gpa);
defer gpa.free(after);
try testing.expectEqualStrings(before, after);
}
test "a forced re-import replaces the configured clients and keeps the materialised ones" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
try importText(io, &database, full_source, .{});
try clients_repo.upsertSeen(&database, "10.0.0.9", 1700000200);
// The device the old file named keeps querying, so its row carries real
// observation history when the wipe reaches it.
try clients_repo.upsertSeen(&database, "fd00::1", 1700003000);
try importText(io, &database, minimal_source, .{ .force = true });
// `full_source`'s hand-edited client went with the rest of the old
// configuration; the discovered one did not.
try testing.expectEqual(@as(i64, 1), try clients_repo.countClients(&database));
try testing.expectEqual(@as(i64, 1700000200), try database.queryInt(
"SELECT first_seen FROM clients WHERE ip = '10.0.0.9' AND hand_edited = 0",
));
// The lift saves a configured client's history so the merge can hand it back
// to the same address. It must never become a reason to resurrect a client
// the new file leaves out: dropping a client from the file removes it.
try testing.expectEqual(@as(i64, 0), try database.queryInt(
"SELECT count(*) FROM clients WHERE ip = 'fd00::1'",
));
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
}
test "a forced re-import keeps the observed timestamps of a client the config names" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
// The device appears in traffic, the operator's file names it, and it keeps
// querying afterwards. The row is now `hand_edited = 1` and carries a real
// `first_seen` and a later `last_seen`.
try clients_repo.upsertSeen(&database, "fd00::1", 1700000200);
try importText(io, &database, full_source, .{});
try clients_repo.upsertSeen(&database, "fd00::1", 1700005000);
try importText(io, &database, full_source, .{ .force = true });
var stmt = try database.prepare(
"SELECT hand_edited, first_seen, last_seen FROM clients WHERE ip = 'fd00::1'",
);
defer stmt.deinit();
try testing.expect(try stmt.step());
try testing.expectEqual(@as(i64, 1), stmt.columnInt(0));
// Re-importing the same file is not an observation of the device.
try testing.expectEqual(@as(i64, 1700000200), stmt.columnInt(1));
try testing.expectEqual(@as(i64, 1700005000), stmt.columnInt(2));
}
test "a failed forced import leaves every client timestamp exactly as it was" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = testing.allocator;
var database = try openMigrated();
defer database.close();
// A configured client with observation history, and a materialised one.
try importText(io, &database, full_source, .{});
try clients_repo.upsertSeen(&database, "fd00::1", 1700005000);
try clients_repo.upsertSeen(&database, "10.0.0.9", 1700000200);
const before = try dump(&database, gpa);
defer gpa.free(before);
const broken: model.Config = .{
.groups = &.{.{ .name = "default" }},
.upstreams = &.{.{ .url = "https://dns.example/dns-query" }},
.clients = &.{.{ .ip = "fd00::1", .name = "tablet" }},
.local_records = &.{
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
.{ .name = "dup.lan", .rtype = .a, .value = "10.0.0.1" },
},
};
try testing.expectError(
error.Constraint,
applyToDb(io, gpa, &database, broken, 42, .{ .force = true }),
);
const after = try dump(&database, gpa);
defer gpa.free(after);
try testing.expectEqualStrings(before, after);
}
test "applyToDb without force refuses a configured database and changes nothing" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
src/config/validate.zig
+614 -88
View File
File diff suppressed because it is too large Load Diff
src/filter/filter_integration_test.zig
+64 -3
View File
@@ -1041,17 +1041,78 @@ test "10: pruneOrphans deletes the files of a deleted source and leaves live one
defer dir.close(io);
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
// A refresh that is still running owns its temporaries, so they are not
// orphans and must survive.
// Id 9999 has no `blocklist_sources` row, so no refresh can be writing for
// it: these temporaries are what a refresh that died mid-write leaves
// behind, and the sweep is the only thing that will ever remove them.
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.list.tmp", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.wild.tmp", .data = "" });
// The live source does have a row, so its temporary is a refresh in
// progress and must survive a sweep that runs beside it.
var live_tmp_buf: [64]u8 = undefined;
const live_tmp = try std.fmt.bufPrint(&live_tmp_buf, "{d}.raw.tmp", .{id});
try dir.writeFile(io, .{ .sub_path = live_tmp, .data = "" });
try env.mgr.pruneOrphans(io);
var live_buf: [64]u8 = undefined;
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.list", .{id}), .{});
try dir.access(io, "9999.raw.tmp", .{});
try dir.access(io, live_tmp, .{});
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list.tmp", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild.tmp", .{}));
}
test "10b: the scheduler sweeps orphans on its own, with no operator call" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
const list_body = "aaa.example.com\n";
const id = try seedSource(&env.database, source_url);
try publishFixtureFiles(env, id, list_body, "");
// Dated now, so the startup pass finds nothing to download: this case
// asserts the sweep, and `source_url` points at a port nobody is serving.
try sources_repo.updateSourceStats(&env.database, id, .{
.last_updated = std.Io.Clock.real.now(io).toSeconds(),
.domain_count = 1,
.wildcard_count = 0,
.skipped_regex_count = 0,
.checksum = &bodyChecksum(list_body, ""),
});
var dir = try env.blocklistDir();
defer dir.close(io);
// What a source deleted while the server was down leaves, and what a
// process killed mid-refresh leaves. Nothing else in the tree removes
// either.
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
// `runScheduler` is the entry point `app.zig` hands to `Io.Group`, and the
// only one the server ever calls. A disabled update stops it after the
// startup pass, so the production path runs to completion here with no
// interval to wait out.
env.mgr.update.enabled = false;
try env.mgr.runScheduler(io);
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
// The live source kept its files and is still filtering: the sweep did not
// take the snapshot the same pass had just published.
var live_buf: [64]u8 = undefined;
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.list", .{id}), .{});
const decision, _ = try env.evaluate("aaa.example.com");
try testing.expect(decision.blocked);
}
// ---------------------------------------------------------------------------
src/filter/manager.zig
+197 -35
View File
@@ -39,6 +39,7 @@ const std = @import("std");
const Allocator = std.mem.Allocator;
const model = @import("../config/model.zig");
const safe_url = @import("../safe_url.zig");
const db = @import("../storage/db.zig");
const clients_repo = @import("../storage/repositories/clients_repo.zig");
const groups_repo = @import("../storage/repositories/groups_repo.zig");
@@ -60,6 +61,10 @@ pub const max_error_len: usize = 128;
/// `SourceStatus.url` is fixed-size so a copied status borrows nothing. A
/// blocklist url longer than this is truncated in the status only; the row
/// keeps it whole.
///
/// The log form of a url is bounded separately by `safe_url.max_len`. The two
/// numbers agree today and answer different questions; neither follows the
/// other.
pub const max_url_len: usize = 255;
/// A compiled body larger than this is refused at load. A source that reaches
@@ -80,6 +85,38 @@ const sample_buf_len: usize = parsers.sample_lines * (compiler.max_line_len + 1)
/// `<id>` is at most 20 characters and the longest suffix is `.list.tmp`.
const name_buf_len: usize = 48;
/// How one blocklist source is named in a log line: by its row id and its name,
/// which are its own identity, and by its redacted url, which says where it
/// points and nothing more.
///
/// The url used to carry the identity here on its own. It cannot: `safe_url`
/// drops the path, because a path segment is a place an operator's token lives,
/// and two sources on one host are told apart by exactly that path. The id and
/// the name are on the row every one of these lines already holds, they are
/// what the API and the web UI show, and neither can leak what the url holds.
/// The name is escaped for the same reason the url is — both are database text
/// and a newline in either would forge a log line. It carries its own quotes,
/// out of `safe_url.quoteText`, because a quote this format string added would
/// be a quote the name could close: `ads' (https://decoy.example) --` would then
/// read as a source pointing somewhere it does not.
const SourceLabel = struct {
id: i64,
name: []const u8,
url: []const u8,
fn of(row: sources_repo.SourceRow) SourceLabel {
return .{ .id = row.id, .name = row.name, .url = row.url };
}
pub fn format(self: SourceLabel, w: *std.Io.Writer) std.Io.Writer.Error!void {
try w.print("source {d} {f} {f}", .{
self.id,
safe_url.quoteText(self.name),
safe_url.redactQuoted(self.url),
});
}
};
pub const Paths = struct {
/// `<data_dir>`, owned by the caller and left open for the manager's life.
dir: std.Io.Dir,
@@ -520,7 +557,7 @@ pub const Manager = struct {
// `replace` calls — a new `.list` beside an old `.wild` — is caught
// here and refreshed, not served as a half-updated list.
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body))) {
log.warn("blocklist {s}: compiled files do not match the stored checksum", .{row.url});
log.warn("blocklist {f}: compiled files do not match the stored checksum", .{SourceLabel.of(row)});
return .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } };
}
@@ -610,7 +647,7 @@ pub const Manager = struct {
defer self.deleteQuietly(io, dir, list_tmp);
defer self.deleteQuietly(io, dir, wild_tmp);
self.download(io, dir, raw_name, row.url) catch |err| switch (err) {
self.download(io, dir, raw_name, row) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.Canceled => return error.Canceled,
else => {
@@ -692,13 +729,16 @@ pub const Manager = struct {
}
/// The body goes to a temporary file, never to memory: `max_body_bytes` is
/// 64 MB and the memory budget has no room for it beside two snapshots.
/// 64 MiB and the memory budget has no room for it beside two snapshots.
///
/// It takes the whole row rather than the url alone because its two log
/// lines name the source by its id and name, which only the row carries.
fn download(
self: *Manager,
io: std.Io,
dir: std.Io.Dir,
raw_name: []const u8,
url: []const u8,
row: sources_repo.SourceRow,
) !void {
const file = try dir.createFile(io, raw_name, .{ .permissions = .fromMode(0o600) });
defer file.close(io);
@@ -707,14 +747,14 @@ pub const Manager = struct {
defer self.gpa.free(buffer);
var fw = file.writer(io, buffer);
const result = self.fetchWithin(io, url, &fw.interface) catch |err| {
const result = self.fetchWithin(io, row.url, &fw.interface) catch |err| {
// `fetcher.Error.Unexpected` is what a failing sink surfaces as;
// the concrete cause is on this writer, which the fetcher does not
// own.
if (fw.err) |cause| return cause;
if (err == error.HttpStatus) {
if (self.fetcher.last_status) |status| {
log.warn("blocklist {s}: http status {d}", .{ url, @intFromEnum(status) });
log.warn("blocklist {f}: http status {d}", .{ SourceLabel.of(row), @intFromEnum(status) });
}
}
return err;
@@ -724,7 +764,7 @@ pub const Manager = struct {
// buffer this function is about to drop.
try file.sync(io);
log.debug("blocklist {s}: downloaded {d} bytes", .{ url, result.bytes_read });
log.debug("blocklist {f}: downloaded {d} bytes", .{ SourceLabel.of(row), result.bytes_read });
}
/// `std.http.Client` has no per-request deadline, so the whole exchange
@@ -906,7 +946,7 @@ pub const Manager = struct {
err: anyerror,
) void {
_ = self;
log.warn("blocklist {s}: download failed: {s}", .{ row.url, @errorName(err) });
log.warn("blocklist {f}: download failed: {s}", .{ SourceLabel.of(row), @errorName(err) });
status.fail(.fetch_failed, @errorName(err));
}
@@ -917,7 +957,7 @@ pub const Manager = struct {
err: anyerror,
) void {
_ = self;
log.warn("blocklist {s}: compile failed: {s}", .{ row.url, @errorName(err) });
log.warn("blocklist {f}: compile failed: {s}", .{ SourceLabel.of(row), @errorName(err) });
status.fail(.compile_failed, @errorName(err));
}
@@ -934,7 +974,7 @@ pub const Manager = struct {
"NoValidEntries invalid={d} unsupported={d} long_lines={d}",
.{ counts.invalid, counts.skipped_unsupported, counts.long_lines },
) catch "NoValidEntries";
log.warn("blocklist {s}: {s}", .{ row.url, text });
log.warn("blocklist {f}: {s}", .{ SourceLabel.of(row), text });
status.fail(.no_valid_entries, text);
}
@@ -954,6 +994,13 @@ pub const Manager = struct {
/// `update.enabled == false` stops after the startup pass; manual refresh
/// through `refreshAll` still works.
pub fn runScheduler(self: *Manager, io: std.Io) std.Io.Cancelable!void {
// Ahead of the pass, not after it. This is the sweep that collects what
// a killed process left behind: a `.raw.tmp` as large as the body the
// dead refresh was writing, and the compiled files of a source deleted
// while the server was down. Both are bytes the pass below is about to
// ask the same filesystem for.
try self.sweepOrphans(io);
self.startupPass(io) catch |err| switch (err) {
error.Canceled => return error.Canceled,
else => log.warn("blocklist startup pass failed: {s}", .{@errorName(err)}),
@@ -968,6 +1015,11 @@ pub const Manager = struct {
};
while (true) {
try interval.sleep(io);
// Ahead of the gate as well as ahead of the pass: the sweep only
// unlinks, so it is the one thing here that can give a critically
// full disk room back, and gating it would keep the residue that
// helped fill the disk in the first place.
try self.sweepOrphans(io);
if (self.refreshGated()) continue;
self.refreshAll(io) catch |err| switch (err) {
error.Canceled => return error.Canceled,
@@ -976,6 +1028,21 @@ pub const Manager = struct {
}
}
/// `pruneOrphans` with its failure absorbed. Leftover bytes under
/// `<data_dir>/blocklists/` are not an outage, and a sweep that could not
/// read the directory must not cost the household the refresh pass behind
/// it — let alone the server. Cancellation is the one outcome that
/// propagates, because it means shutdown.
///
/// Taken from outside every `*Locked` body: `pruneOrphans` takes
/// `writer_lock` itself and the mutex is not reentrant.
fn sweepOrphans(self: *Manager, io: std.Io) std.Io.Cancelable!void {
self.pruneOrphans(io) catch |err| switch (err) {
error.Canceled => return error.Canceled,
else => log.warn("pruning orphaned blocklist files failed: {s}", .{@errorName(err)}),
};
}
/// The §11.6 gate, consulted by scheduled passes only (ruling 17). A
/// download writes tens of megabytes into the blocklist directory and the
/// compile writes as much again, which is exactly the "non-essential write"
@@ -1045,12 +1112,26 @@ pub const Manager = struct {
// orphans
// -----------------------------------------------------------------------
/// Deletes `<id>.list` and `<id>.wild` files whose id is no longer a
/// `blocklist_sources` row. Files of a live source are left alone,
/// Deletes the compiled files and the leftover temporaries whose id is no
/// longer a `blocklist_sources` row. Files of a live source are left alone,
/// whatever their state.
///
/// Three callers, and between them they cover every way an orphan is made:
/// `runScheduler` sweeps once before its startup pass — the residue of a
/// process that was killed mid-refresh, and of a source deleted while the
/// server was down — and again before each scheduled pass; the
/// `DELETE /api/blocklists/{id}` handler sweeps as soon as it has removed
/// the row, so the directory follows the table an operator can see instead
/// of waiting out `blocklist_update.interval_hours`.
///
/// It is safe to call on a fresh install: `openDir` creates
/// `<data_dir>/blocklists/` if nothing has yet, and an empty directory
/// sweeps to nothing.
pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
// A refresh in flight owns the temporaries of a live source; the sweep
// must not run beside one and decide from a half-written directory.
// Every path that writes a temporary holds this lock too, so the sweep
// never reads a directory a refresh is halfway through. The temporaries
// it can see therefore belong to a finished or a dead refresh, and only
// those of a source with no row are removed.
self.writer_lock.lockUncancelable(io);
defer self.writer_lock.unlock(io);
@@ -1079,14 +1160,14 @@ pub const Manager = struct {
},
} orelse break;
if (entry.kind != .file) continue;
const id = compiledId(entry.name) orelse continue;
const id = sourceFileId(entry.name) orelse continue;
if (containsId(rows.items, id)) continue;
try doomed.append(self.gpa, try self.gpa.dupe(u8, entry.name));
}
for (doomed.items) |name| {
self.deleteQuietly(io, dir, name);
log.info("pruned orphaned compiled file {s}", .{name});
log.info("pruned orphaned blocklist file {s}", .{name});
}
}
@@ -1253,7 +1334,7 @@ const LoadOutcome = union(enum) {
};
fn loadFailure(row: sources_repo.SourceRow, file_name: []const u8, err: anyerror) LoadOutcome {
log.warn("blocklist {s}: reading {s} failed: {s}", .{ row.url, file_name, @errorName(err) });
log.warn("blocklist {f}: reading {s} failed: {s}", .{ SourceLabel.of(row), file_name, @errorName(err) });
return .{ .failed = .{ .state = .load_failed, .text = @errorName(err) } };
}
@@ -1394,17 +1475,27 @@ fn compiledName(buf: *[name_buf_len]u8, id: i64, suffix: []const u8) []const u8
return std.fmt.bufPrint(buf, "{d}{s}", .{ id, suffix }) catch unreachable;
}
/// The source id a compiled file belongs to, or null when the name is not one
/// of ours. Temporary files are deliberately not matched: they belong to a
/// refresh that may still be running.
fn compiledId(file_name: []const u8) ?i64 {
const stem = if (std.mem.endsWith(u8, file_name, ".list"))
file_name[0 .. file_name.len - ".list".len]
else if (std.mem.endsWith(u8, file_name, ".wild"))
file_name[0 .. file_name.len - ".wild".len]
else
return null;
return std.fmt.parseInt(i64, stem, 10) catch null;
/// Every name `compiledName` can produce, longest suffix first so `.list.tmp`
/// is never read as `.list`.
const source_file_suffixes = [_][]const u8{ ".list.tmp", ".wild.tmp", ".raw.tmp", ".list", ".wild" };
/// The source id a file under the blocklist directory belongs to, or null when
/// the name is not one of ours.
///
/// The three temporaries count. A refresh that dies between writing one and
/// renaming it leaves a file no later refresh reuses and no `defer` reaches, so
/// excluding them from the sweep means nothing ever removes them. Matching them
/// is safe because `pruneOrphans` holds `writer_lock` for its whole body: every
/// path that creates a temporary runs under that same lock, so no refresh is in
/// flight while the sweep reads the directory, and a temporary the sweep sees
/// belonging to a source that still has a row is kept regardless.
fn sourceFileId(file_name: []const u8) ?i64 {
for (source_file_suffixes) |suffix| {
if (!std.mem.endsWith(u8, file_name, suffix)) continue;
const stem = file_name[0 .. file_name.len - suffix.len];
return std.fmt.parseInt(i64, stem, 10) catch null;
}
return null;
}
fn containsId(rows: []const sources_repo.SourceRow, id: i64) bool {
@@ -1566,6 +1657,62 @@ test "the header writer produces the documented text" {
++ "# sha256 " ++ "0" ** 64 ++ "\n", w.buffered());
}
test "the log label names a source without printing what its url carries" {
// Every `log.warn` in this file formats its subject through `SourceLabel`,
// so this is the text of those lines. A `std.log` line is not observable
// from a unit test under the default runner; the label is.
var buf: [1024]u8 = undefined;
const row: sources_repo.SourceRow = .{
.id = 3,
.url = "https://lists.example/download/token/hunter2/hosts.txt?apikey=s3cr3t",
.name = "ads",
.enabled = true,
.last_updated = null,
.domain_count = 0,
.wildcard_count = 0,
.skipped_regex_count = 0,
.checksum = null,
};
const printed = try std.fmt.bufPrint(&buf, "blocklist {f}: download failed: {s}", .{
SourceLabel.of(row),
@errorName(error.ConnectFailed),
});
try testing.expectEqualStrings(
"blocklist source 3 'ads' 'https://lists.example': download failed: ConnectFailed",
printed,
);
try testing.expect(!std.mem.containsAtLeast(u8, printed, 1, "hunter2"));
try testing.expect(!std.mem.containsAtLeast(u8, printed, 1, "s3cr3t"));
// The row is database text, and a path that writes it does not have to
// validate as strictly as the config validator. Neither column may end the
// line and start one of the operator's choosing.
var forged = row;
forged.name = "ads\n2026-01-01 ERROR forged";
forged.url = "https://lists.example\n2026-01-01 ERROR forged/hosts.txt";
const escaped = try std.fmt.bufPrint(&buf, "blocklist {f}", .{SourceLabel.of(forged)});
try testing.expectEqualStrings(
"blocklist source 3 'ads\\n2026-01-01 ERROR forged'" ++
" 'https://lists.example\\n2026-01-01 ERROR forged'",
escaped,
);
try testing.expect(!std.mem.containsAtLeast(u8, escaped, 1, "\n"));
// A name is operator-supplied and reaches the row through the API, so it
// can close the quote this label puts around it and open a decoy that reads
// as the url of a second source. The quote it would close is escaped, and
// the escape is unambiguous because a `\` is escaped too.
var decoy = row;
decoy.name = "ads' (https://decoy.example) --";
decoy.url = "https://lists.example/hosts.txt";
const quoted = try std.fmt.bufPrint(&buf, "blocklist {f}", .{SourceLabel.of(decoy)});
try testing.expectEqualStrings(
"blocklist source 3 'ads\\' (https://decoy.example) --' 'https://lists.example'",
quoted,
);
}
test "stripHeader returns the body of a compiled file" {
const file =
"# nxdns blocklist\n" ++
@@ -1615,13 +1762,28 @@ test "compiledName spells the four file names of a source" {
try testing.expectEqualStrings("42.list.tmp", compiledName(&buf, 42, ".list.tmp"));
}
test "compiledId matches compiled files and nothing else" {
try testing.expectEqual(@as(?i64, 7), compiledId("7.list"));
try testing.expectEqual(@as(?i64, 7), compiledId("7.wild"));
try testing.expectEqual(@as(?i64, null), compiledId("7.list.tmp"));
try testing.expectEqual(@as(?i64, null), compiledId("7.raw.tmp"));
try testing.expectEqual(@as(?i64, null), compiledId("notes.list"));
try testing.expectEqual(@as(?i64, null), compiledId("README"));
test "sourceFileId matches every name a refresh writes, including the temporaries" {
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild"));
// A temporary left by a refresh that died belongs to its source id, so the
// sweep can tell whether that source still has a row.
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.raw.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.list"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.raw.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.raw"));
try testing.expectEqual(@as(?i64, null), sourceFileId("README"));
}
test "every name compiledName writes is a name the sweep can attribute" {
var buf: [name_buf_len]u8 = undefined;
for (source_file_suffixes) |suffix| {
try testing.expectEqual(@as(?i64, 42), sourceFileId(compiledName(&buf, 42, suffix)));
}
}
test "a failed refresh keeps the fields of the compiled files still serving" {
src/main.zig
+60 -2
View File
@@ -18,13 +18,35 @@ comptime {
_ = @import("cli.zig");
}
/// Streaming, never positional, and this is load-bearing rather than a default
/// worth changing back.
///
/// `File.writer` builds a `.positional` writer: it keeps an offset of its own,
/// starting at zero, and pwrites there. `std.log` writes to the same descriptor
/// directly and shares the descriptor's offset. With stderr redirected to a
/// regular file the two disagree about where the end is, so the runner's first
/// flush lands on top of whatever the log already wrote at the start and the
/// operator loses those lines. `writerStreaming` uses the descriptor's own
/// offset, so both writers append.
///
/// A terminal and a pipe are unaffected either way, because neither is seekable
/// and the positional writer falls back to streaming on them. `2>file` is where
/// it shows, and `nxdns run 2>logfile` is an ordinary way to run this program.
/// `>>` needs it too: pwrite ignores `O_APPEND`.
///
/// Takes the `File` rather than naming stdout and stderr inside, so a test can
/// hand it a real file and read back what an operator would have.
fn runnerWriter(file: std.Io.File, io: std.Io, buffer: []u8) std.Io.File.Writer {
return file.writerStreaming(io, buffer);
}
pub fn main(init: std.process.Init) u8 {
// Both `File.Writer` values are self-referential and must not move, so they
// stay in these `var` slots for the whole of `main`.
var out_buffer: [4096]u8 = undefined;
var err_buffer: [4096]u8 = undefined;
var out = std.Io.File.stdout().writer(init.io, &out_buffer);
var err = std.Io.File.stderr().writer(init.io, &err_buffer);
var out = runnerWriter(std.Io.File.stdout(), init.io, &out_buffer);
var err = runnerWriter(std.Io.File.stderr(), init.io, &err_buffer);
const runner: cli.Runner = .{
.io = init.io,
@@ -55,3 +77,39 @@ pub fn main(init: std.process.Init) u8 {
.help => cli.runHelp(runner),
};
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
test "a runner writer appends after what std.log already put in a redirected file" {
// Reproduced live before this test existed: `nxdns run --config <bad file>
// 2> file` lost `info(migrations): config.db migrated from schema version 0
// to 2`, because the runner's error writer pwrote its first flush at offset
// zero, over the line the log had already written. Down a pipe the same run
// kept both lines, which is why a terminal never showed this.
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
var file = try tmp.dir.createFile(io, "stderr.txt", .{});
defer file.close(io);
// What the log sink does: a direct write on the shared descriptor, which
// moves the descriptor's offset. A positional writer does not see it.
const log_line = "info(migrations): config.db migrated from schema version 0 to 2\n";
try file.writeStreamingAll(io, log_line);
const runner_line = "nxdns run failed: MissingDefaultGroup\n";
var buffer: [4096]u8 = undefined;
var w = runnerWriter(file, io, &buffer);
try w.interface.writeAll(runner_line);
try w.interface.flush();
const seen = try tmp.dir.readFileAlloc(io, "stderr.txt", gpa, .limited(4096));
defer gpa.free(seen);
try std.testing.expectEqualStrings(log_line ++ runner_line, seen);
}
src/safe_url.zig
+947
View File
@@ -0,0 +1,947 @@
//! The one redaction an operator-supplied string passes through before it
//! reaches a log line or an operator-facing diagnostic.
//!
//! It lives at the root rather than under `filter/` because both the blocklist
//! manager and the configuration validator print urls, and a `config/` module
//! importing `filter/` would be the wrong dependency direction. It takes bytes
//! and returns bytes: no `Io`, no allocator, no failure mode.
const std = @import("std");
/// The longest text one `SafeUrl` or one `QuotedText` prints, ahead of the `...`
/// that marks a truncation. An operator-supplied url or name has no length
/// limit and a log line must have one. A `QuotedText`'s two quotes fall outside
/// the count: they are the delimiter rather than the text, and a truncated name
/// still closes the one it opened.
///
/// The count is of printed characters, not of source bytes: an escape sequence
/// costs the two or four characters it prints, not the one byte it stands for.
/// A string of control characters therefore truncates early instead of printing
/// several times its length.
///
/// This is deliberately a second number rather than a reuse of the blocklist
/// manager's `max_url_len`, which sizes the inline url copy inside
/// `SourceStatus`. The two answer different questions — how wide one log line
/// may be, and how large a status value a caller may keep is — and they agree
/// on 255 only because that width suits both. Neither may be changed on the
/// other's account.
pub const max_len: usize = 255;
/// The only form of a url that may be written to a log. `format` prints the
/// scheme, the host and the port; it drops the userinfo, the path, the query
/// and the fragment; it escapes every control character; and it bounds the
/// result at `max_len`.
///
/// Where the line falls, and why it falls there. A blocklist source url is
/// operator-supplied and nothing on the way in restricts it to a bare public
/// path: `?apikey=…`, a signed url whose signature is a query parameter,
/// `https://user:pass@host/list.txt` and `https://host/d/<token>/hosts.txt` all
/// validate, import and store. Userinfo, path, query and fragment are the four
/// components where a credential can legally live, and a log line outlives the
/// process — journald and the configured log file keep it, and neither is as
/// protected as the database row the url came from. The rule against writing a
/// secret to a log is absolute, so all four go.
///
/// Dropping the path costs the url the one job it used to do here: telling two
/// sources on one host apart. That job was never the url's. A caller names the
/// source it reports on from the source's own identity — `filter/manager.zig`
/// prints the row id and the name beside this — and an identity read from the
/// row cannot leak what the url holds.
///
/// What it still prints, deliberately: the scheme, the host and the port,
/// because an operator reading a failure has to know where the source points;
/// and, for a url that names no scheme at all, whatever precedes the first `/`
/// once the userinfo is removed, because nothing in such a string distinguishes
/// a host from anything else and that text is the closest thing to one. A
/// string that names a scheme without a `//` after it gets no such treatment,
/// whether or not it holds an `@` — see `redact`, where the text after such a
/// scheme is a path segment under at least one reading and is never printed.
///
/// The host is therefore the one place a credential still survives this, and it
/// is not hypothetical: NextDNS identifies an account by a profile id, which its
/// DoH url carries in the path — `https://dns.nextdns.io/abcd12`, dropped here —
/// and its DoT url carries in the hostname — `tls://abcd12.dns.nextdns.io`,
/// kept. Nothing can be done about the second without printing no host at all,
/// which would leave every line unactionable. An operator whose provider puts a
/// secret in the hostname has published it to every resolver on the way to it
/// long before it reaches this function.
///
/// It scans and never parses, so it cannot fail. `error.BadUrl` — the error
/// `std.Uri.parse` raises — is one of the failures reported through here, so
/// the inputs a parser refuses are exactly the inputs this must still redact.
/// Only `://` introduces an authority, though: a run of one separator, of three
/// or more, or of two that are not both `/`, leaves text that RFC 3986 reads as a
/// path and WHATWG may read as a host, and `redact` prints no host it cannot
/// settle. A `\` still ends an authority and still anchors the scheme scan — see
/// `isSeparator` — it just does not open one.
///
/// Scanning a string a parser refuses means some of those strings have more
/// than one reading, and the readings disagree about which side of an `@` the
/// host is on. `https://lists.example?token=prefix@hunter2` is one: RFC 3986
/// ends the authority at the `?`, so the host is `lists.example` and `hunter2`
/// is part of an api key; read the `@` as a userinfo delimiter instead and the
/// host is `hunter2`. No scan resolves that, and for a while this one chose the
/// second reading and printed `https://hunter2` — a query-string secret printed
/// as a host, which is the exact leak this file exists to stop. `authority` is
/// therefore `null` on such a url and `format` prints
/// `(ambiguous authority omitted)` in place of a host. Withholding the
/// authority is always available and never wrong; choosing a side is wrong half
/// the time, and the half it is wrong in is the half that leaks.
pub const SafeUrl = struct {
/// The scheme without its `:`, or empty when the url carries no scheme
/// delimiter.
scheme: []const u8,
/// The authority without its userinfo — the host and, when present, the
/// port — or `null` when the url admits two readings of where the host is.
/// Empty and `null` are different answers: empty says the url names no
/// authority, `null` says it names one this cannot resolve.
authority: ?[]const u8,
/// Unquoted. A caller that prints this inside a delimiter of its own must
/// use `redactQuoted` instead, or handle the delimiter itself the way
/// `web/metrics.zig` does for a Prometheus label value.
///
/// This renders `scheme://authority` canonically; it does not quote the
/// input's own syntax. A scheme is always followed by `://`, so
/// `tls:\\host` prints `tls://` and `https://?apikey=…` prints `https://`,
/// though only the second contained a `//`. The `://` marks where a host
/// would go, and the operator's remedy is the same either way: the url names
/// no host this can print. Nothing about which bytes separated the scheme
/// from the rest survives redaction, and nothing should — the input is not
/// reproducible from this and is not meant to be.
pub fn format(self: SafeUrl, w: *std.Io.Writer) std.Io.Writer.Error!void {
return self.write(w, .none);
}
/// A truncation ends the value and returns, so a caller that has written an
/// opening delimiter still gets to write the closing one.
fn write(self: SafeUrl, w: *std.Io.Writer, delimiter: Delimiter) std.Io.Writer.Error!void {
var budget: usize = max_len;
if (!try writeEscaped(w, self.scheme, &budget, delimiter)) return w.writeAll("...");
if (self.scheme.len != 0 and !try writeEscaped(w, "://", &budget, delimiter))
return w.writeAll("...");
const authority = self.authority orelse ambiguous_authority;
if (!try writeEscaped(w, authority, &budget, delimiter)) return w.writeAll("...");
}
};
/// A redacted url in the form a caller may print inside a log line or a
/// diagnostic sentence, where it needs a delimiter to keep a host from running
/// into the words around it.
///
/// **`format` writes the quotes**, for the reason `QuotedText` states: a
/// delimiter a caller adds is a delimiter the value can close. Redaction does
/// not make that go away. A `'` is neither a component separator nor a control
/// character, so it survives the scan into the authority — `https://ho'st/x`
/// redacts to `https://ho'st`, and inside a caller's own `'…'` that reads as
/// `'ho'` followed by loose text. The quote and its escape therefore live here,
/// together, and a caller adds none of its own.
///
/// A caller sizing a fixed buffer needs `max_len + 3` for the value, as
/// `SafeUrl` does, plus the two quotes.
pub const QuotedUrl = struct {
url: SafeUrl,
pub fn format(self: QuotedUrl, w: *std.Io.Writer) std.Io.Writer.Error!void {
try w.writeByte('\'');
try self.url.write(w, .single_quote);
try w.writeByte('\'');
}
};
/// What `format` prints where a host would go when it cannot say which text is
/// the host. It is prose rather than a placeholder host because an operator has
/// to read it as a statement about the line and not as an address: `https://`
/// alone already means "this url names no authority", and the two call for
/// different actions.
///
/// It spends the same printing budget the authority would have, so the bound
/// `web/metrics.zig` sizes its buffer against still holds.
///
/// An operator can write a source url whose redaction is this same text, since
/// a url that is not a url prints as itself. That collision costs nothing: it
/// makes one line say less about a source than it could, and it cannot make a
/// credential read as a host, which is the direction that matters.
const ambiguous_authority = "(ambiguous authority omitted)";
/// An operator-supplied string that is not a url — a blocklist source name — in
/// the only form it may be written to a log. It holds no credential by design,
/// so nothing is dropped from it; it comes out of a database row the same way a
/// url does, so a control character in it can forge a log line the same way,
/// and `format` escapes and bounds it for that reason alone.
///
/// **`format` writes the quotes.** A caller printing a name inside a log line
/// has to delimit it, or a name with a space in it runs into the words around
/// it; and a delimiter a caller adds is a delimiter the name can close. `ads'
/// (https://decoy.example) --` inside a caller's quotes produces a line naming a
/// url no source has. So the quote and its escape live in one place, here,
/// where they cannot drift apart. A caller adds none of its own.
pub const QuotedText = struct {
text: []const u8,
pub fn format(self: QuotedText, w: *std.Io.Writer) std.Io.Writer.Error!void {
var budget: usize = max_len;
try w.writeByte('\'');
if (!try writeEscaped(w, self.text, &budget, .single_quote)) try w.writeAll("...");
try w.writeByte('\'');
}
};
/// The `SafeUrl` of `url`. Both fields borrow from `url`, which every caller
/// holds for the length of the call it prints in.
pub fn redact(url: []const u8) SafeUrl {
const first_sep = std.mem.indexOfAny(u8, url, separators) orelse url.len;
const first_colon = std.mem.indexOfScalar(u8, url, ':') orelse url.len;
// A scheme delimiter is a colon immediately before the first separator of
// the whole string. Anchoring on the first separator is what keeps `a/b:/c`
// from reading as a scheme. It anchors on a `\` too, so a url pasted with
// backslashes still has its scheme recognised and echoed, even though a `\`
// no longer opens an authority.
var scheme: []const u8 = "";
var rest = url;
var delimited = false;
if (first_colon + 1 == first_sep and isScheme(url[0..first_colon])) {
scheme = url[0..first_colon];
var after = first_sep;
while (after < url.len and isSeparator(url[after])) after += 1;
// Exactly `//` introduces an authority, and nothing else does. One
// separator leaves an absolute path — RFC 3986 reads `https:/hunter2` as
// the path `/hunter2`, WHATWG reads `hunter2` as the host — and three or
// more is an empty authority to the first and a host to the second.
//
// A run of two that is not two slashes depends on the scheme. WHATWG
// converts a `\` to a `/` only for a *special* scheme, so `https:\\host`
// is contested the same way, while `tls:\\host` — `tls` is not special —
// has no reading at all under which `host` is a host. RFC 3986 gives `\`
// no meaning anywhere.
//
// So the two answers are different answers, and `authority` carries the
// difference: `null` where the readings disagree, empty where they agree
// the url names no authority. Either way the path segment stays out of
// the log, which is the property that matters; this decides only what the
// line then claims about it.
//
// An earlier revision accepted any number of separators, to keep
// `https:/user:pass@host/list` from printing its userinfo. That reason
// expired when the `/` cut moved ahead of the userinfo lookup: the
// authority of a url with no `://` now ends at its first `/`, so it holds
// no userinfo to print. Withholding it is both safe and the honest
// answer.
if (after - first_sep != 2 or url[first_sep] != '/' or url[first_sep + 1] != '/')
return .{ .scheme = scheme, .authority = contestedOrAbsent(scheme, url[after..]) };
rest = url[after..];
delimited = true;
}
// A network-path reference names an authority and no scheme (RFC 3986
// §4.2). Without this the `/` cut below lands at byte zero, the authority is
// empty, and the line reports nothing at all — including for
// `//user:pa55@lists.example/x`, where the host is not in doubt.
// Exactly `//` here too, and for the same reason. RFC 3986 reads an authority
// after `//` and nothing else — `///hunter2/x` is an empty authority and the
// path `/hunter2/x`, and `\\hunter2\x` is a path outright. WHATWG resolves a
// reference against a base, and against a special-scheme base its
// ignore-slashes state reads `hunter2` as the host in both. A run that is not
// exactly `//` is therefore contested, not settled, and it is withheld rather
// than reported as an authority the url does not have.
//
// A run of one is settled: both readings make it a path. So is any run whose
// candidate authority is empty — `\\?\C:\lists\hosts.txt` ends the authority
// at its `?` under the reading that looks for one, so neither finds a host
// and there is nothing to contest.
if (!delimited) {
var run: usize = 0;
while (run < rest.len and isSeparator(rest[run])) run += 1;
if (run >= 2) {
if (candidateAuthority(rest[run..]).len == 0)
return .{ .scheme = scheme, .authority = "" };
if (run != 2 or rest[0] != '/' or rest[1] != '/')
return .{ .scheme = scheme, .authority = null };
rest = rest[2..];
delimited = true;
}
}
// The authority ends at the first `/`. Everything from there on is path,
// query or fragment, and none of the three is printed — so an `@` after
// that slash is not userinfo by any reading, and no longer needs to be one
// to stay out of the log.
const authority = rest[0 .. std.mem.indexOfScalar(u8, rest, '/') orelse rest.len];
// A scheme with no separator after it is an opaque path under RFC 3986 and,
// for a special scheme, a host under WHATWG — so `https:hunter2` is either a
// path segment, which is where a token lives, or a host. The disagreement is
// the whole of the evidence, exactly as it is for `https:a@hunter2`, and this
// check runs before the `@` lookup so both readings reach it.
//
// No exception is made for a suffix that looks like a port. An earlier
// revision took the digits in `localhost:8080` for one, which also let
// `https:123456` through, whose digits are an opaque path and as much a token
// as any other text. Neither is resolved now.
//
// Note what the two cases are not. `https:` is a WHATWG special scheme, so
// `https:hunter2` is contested — an opaque path to RFC 3986, a host to
// WHATWG. `localhost:` is not special, so `localhost:8080` is a scheme and an
// opaque path to *both*, and what an operator meant by it — a host and a
// port — is a reading no parser offers. It is withheld all the same, because
// the text after the colon is a path segment under every reading and a path
// segment is never printed.
//
// So the two get the same treatment and different answers: `https:hunter2` is
// withheld as contested, `localhost:8080` as naming no authority at all.
//
// An `IP:port` is unaffected: a leading digit fails the scheme production, so
// `10.0.0.2:8080` and `[::1]:853` resolve.
if (!delimited) {
const trimmed = cut(authority);
const colon = std.mem.indexOfScalar(u8, trimmed, ':') orelse trimmed.len;
if (colon != trimmed.len and isScheme(trimmed[0..colon])) return .{
.scheme = scheme,
.authority = contestedOrAbsent(trimmed[0..colon], trimmed[colon + 1 ..]),
};
}
// With no `@` there is no userinfo to remove and no side to choose. A `?`,
// a `#` or a `\` ends the authority; each of the three ends it under every
// reading of the text before it.
const at = std.mem.lastIndexOfScalar(u8, authority, '@') orelse
return .{ .scheme = scheme, .authority = cut(authority) };
// A `?`, a `#` or a `\` in front of that `@` makes the authority ambiguous,
// and the two readings put the host on opposite sides of the `@`. On
// `https://lists.example?token=prefix@hunter2` the text after it is an api
// key; on `https://user:pa55?@host` the text before it is a password. Both
// readings are available on both urls and nothing in either string tells
// them apart, so neither side may be printed.
if (std.mem.indexOfAny(u8, authority[0..at], "?#\\") != null)
return .{ .scheme = scheme, .authority = null };
// An `@` is a userinfo delimiter only inside an authority. The scan knows one
// is there in exactly three cases: a scheme delimiter introduced it, a `//`
// did, or the string names no scheme at all, where this file's rule is that
// the text before the first `/` is the closest thing to a host. The remaining
// case — a scheme with no separator after it — was withheld above, before the
// `@` was looked for, because it is ambiguous whether or not an `@` is in it.
return .{ .scheme = scheme, .authority = cut(authority[at + 1 ..]) };
}
/// `text` up to the first `?`, `#` or `\`, each of which ends an authority. A
/// `\` is here rather than in `redact`'s `/` cut because the `/` cut runs before
/// the userinfo is located and a `\` before an `@` is not a separator under
/// every reading — `https://user:pa55\@host` is ambiguous, not hierarchical.
fn cut(text: []const u8) []const u8 {
return text[0 .. std.mem.indexOfAny(u8, text, "?#\\") orelse text.len];
}
/// The `QuotedText` of `text`, which it borrows for the length of the call that
/// prints it. The result prints its own quotes; see `QuotedText`.
pub fn quoteText(text: []const u8) QuotedText {
return .{ .text = text };
}
/// The `redact` of `url`, quoted.
///
/// The rule for choosing between the two, so it does not have to be rediscovered
/// per call site: **use this whenever anything follows the url on the line.** A
/// redacted authority can still hold a space, a `:` and a `'`, so unquoted it can
/// impersonate whatever comes next — `upstream {f} failed: {t}` with a url whose
/// authority is `ok failed: Timeout` reports a failure that did not happen.
///
/// `redact` is for the two cases where that cannot arise: the url ends the line,
/// or the caller owns the escaping for a delimiter of its own, as
/// `web/metrics.zig` does for a Prometheus label value.
///
/// The result prints its own quotes; see `QuotedUrl`.
pub fn redactQuoted(url: []const u8) QuotedUrl {
return .{ .url = redact(url) };
}
/// What ends a url's components. `/` is RFC 3986's. `\` is here because the
/// strings this scan exists for are the ones a parser refuses:
/// `https:\\dns.nextdns.io\abcd12` carries an account identifier after a `\`,
/// and a scan that read only `/` printed the whole of it.
///
/// It ends a component; it does not open an authority. What that url *names* is
/// contested — path text to RFC 3986, which gives `\` no meaning, and a host to
/// WHATWG, which reads `\` as `/` for a special scheme — so `redact` withholds
/// it. The `\` still matters here because both readings agree the account
/// identifier after it is not part of any host.
/// `null` when the readings disagree about whether `after` holds a host, empty
/// when they agree it holds none. Both withhold; they differ in what the line
/// claims, and `SafeUrl.authority` documents that as a real distinction.
///
/// Known imprecision, in the safe direction. Three shapes return `null` where
/// both readings in fact find no host, so the line says "could not resolve"
/// where "names none" is the truth:
///
/// - `https:/user@/x` — emptiness is decided before the userinfo is removed,
/// so `user@` counts as a candidate host when the host after it is empty.
/// - `\path@hunter2` — a leading run of one is settled as a path under both
/// readings, but reaches the late-delimiter rule instead of returning here.
/// - `file:secret` — `file` is in `special_schemes`, but WHATWG gives it its
/// own parsing states in which that input is a local path with no host.
///
/// Each prints *less* than it could, never more; none prints the path segment.
/// Fixing them means modelling more of two standards for inputs no accepted
/// configuration can hold: every url this program takes carries `http`, `https`,
/// `tls`, `udp` or `tcp` and a `//`, so all three shapes reach a line only as
/// something the validator is already rejecting by field path.
///
/// Only a WHATWG special scheme reads an authority out of text a `//` did not
/// introduce, so only a special scheme can disagree with RFC 3986 here. And
/// nothing is contested when the reading that looks for a host finds none —
/// `https:` alone names no authority under either.
fn contestedOrAbsent(scheme: []const u8, after: []const u8) ?[]const u8 {
if (candidateAuthority(after).len == 0) return "";
return if (isSpecialScheme(scheme)) null else "";
}
/// The host a WHATWG-style reading would take out of `after`, used only to tell
/// an empty one from a non-empty one.
fn candidateAuthority(after: []const u8) []const u8 {
return cut(after[0 .. std.mem.indexOfScalar(u8, after, '/') orelse after.len]);
}
/// WHATWG's special schemes: the ones whose urls it reads an authority into
/// without a `//`, and whose backslashes it converts to slashes.
///
/// This is a closed set fixed by the URL Standard, not a list of what this
/// program supports. That is the difference between it and the known-scheme list
/// an earlier revision removed: this one cannot go stale when nxdns learns a new
/// transport, because it never described nxdns in the first place.
const special_schemes = [_][]const u8{ "ftp", "file", "http", "https", "ws", "wss" };
fn isSpecialScheme(scheme: []const u8) bool {
for (special_schemes) |special| {
if (std.ascii.eqlIgnoreCase(scheme, special)) return true;
}
return false;
}
const separators = "/\\";
fn isSeparator(c: u8) bool {
return std.mem.indexOfScalar(u8, separators, c) != null;
}
/// Whether `text` is a url scheme: an ASCII letter followed by letters, digits,
/// `+`, `-` and `.` — RFC 3986's production, which is what a scheme delimiter
/// has to look like before the text in front of it may be dropped as one.
fn isScheme(text: []const u8) bool {
if (text.len == 0) return false;
if (!std.ascii.isAlphabetic(text[0])) return false;
for (text[1..]) |c| {
if (std.ascii.isAlphanumeric(c)) continue;
if (c == '+' or c == '-' or c == '.') continue;
return false;
}
return true;
}
/// Writes `text` with every control character escaped, spending at most
/// `budget` printed characters and never splitting an escape sequence. Returns
/// whether the whole of `text` was written.
///
/// The control escapes are the ones `platform/logging.zig` uses on a whole log
/// message, byte for byte, and that is deliberate. `delimiter` adds the one
/// escape that sink has no reason to make.
///
/// Two layers escape the same bytes here and neither is redundant. Do not
/// delete this one on the grounds that the log sink already covers it: that
/// sink covers every `std.log` line and nothing else, and a redacted url does
/// not only reach a sink. `config/validate.zig` builds `Problem.message` as an
/// allocated string; `web/handlers/mutations.zig` prints that message into the
/// body of the 400 from `POST /api/blocklists`, and `cli.zig` prints it to the
/// stdout of an interactive `nxdns check`. Control bytes baked into that string
/// reach an HTTP response and an operator's terminal, neither of which any log
/// sink is in a position to escape. A url has to arrive safe rather than be
/// made safe by whatever it is written to.
///
/// The cost is that a url inside a log line is escaped twice: `\n` in the row
/// prints as `\\n` in journald and as `\n` on stdout. One notation across both
/// is what keeps that legible.
///
/// A `\` is escaped for the same reason it is there: `\n` in the output then
/// means the byte this function replaced and `\\n` means two characters an
/// operator typed. It is what makes `\'` unambiguous as well.
fn writeEscaped(
w: *std.Io.Writer,
text: []const u8,
budget: *usize,
delimiter: Delimiter,
) std.Io.Writer.Error!bool {
const hex = "0123456789abcdef";
for (text) |byte| {
var hex_buf: [4]u8 = undefined;
const escape: ?[]const u8 = switch (byte) {
'\\' => "\\\\",
'\n' => "\\n",
'\r' => "\\r",
'\t' => "\\t",
'\'' => if (delimiter == .single_quote) "\\'" else null,
0x00...0x08, 0x0b, 0x0c, 0x0e...0x1f, 0x7f => blk: {
hex_buf = .{ '\\', 'x', hex[byte >> 4], hex[byte & 0x0f] };
break :blk &hex_buf;
},
else => null,
};
if (escape) |seq| {
if (budget.* < seq.len) return false;
try w.writeAll(seq);
budget.* -= seq.len;
} else {
if (budget.* < 1) return false;
try w.writeByte(byte);
budget.* -= 1;
}
}
return true;
}
/// The character the caller of `writeEscaped` wraps the escaped text in, which
/// is therefore the one character beyond the control set that has to be escaped
/// inside it. `.none` is a value nothing wraps: a `SafeUrl` is printed bare, and
/// escaping a `'` in a host would say a `'` there means something it does not.
const Delimiter = enum { none, single_quote };
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const testing = std.testing;
fn expectRedacted(expected: []const u8, url: []const u8) !void {
var buf: [8 * max_len]u8 = undefined;
try testing.expectEqualStrings(expected, try std.fmt.bufPrint(&buf, "{f}", .{redact(url)}));
}
test "redact drops the query, the fragment and the userinfo" {
// The credential shapes a source url can hold: an api key in the query, a
// signed url whose signature is a query parameter, and userinfo.
try expectRedacted(
"https://lists.example",
"https://lists.example/hosts.txt?apikey=s3cr3t",
);
try expectRedacted(
"https://cdn.example",
"https://cdn.example/l/hosts.txt?Expires=1700000000&Signature=abc123&Key-Pair-Id=K2",
);
try expectRedacted(
"https://lists.example",
"https://user:pa55@lists.example/hosts.txt",
);
try expectRedacted(
"https://lists.example:8443",
"https://token@lists.example:8443/hosts.txt?t=1#frag",
);
}
test "redact drops a credential carried in a path segment" {
// The path is a place a token lives — a per-subscriber download url is the
// common shape — and the rule against writing a secret to a log does not
// bend for the component it sits in.
try expectRedacted(
"https://lists.example",
"https://lists.example/download/token/hunter2/hosts.txt",
);
// NextDNS: the path segment is the account identifier, so this is the shape
// the rule exists for rather than an invented one.
try expectRedacted("https://dns.nextdns.io", "https://dns.nextdns.io/abcd12");
try expectRedacted("https://dns.nextdns.io", "https://dns.nextdns.io/abcd12/mydevice");
// Its DoT form puts the same id in the hostname, where redaction cannot
// reach it. Pinned so the limit stays visible rather than being discovered.
try expectRedacted("tls://abcd12.dns.nextdns.io", "tls://abcd12.dns.nextdns.io");
try expectRedacted("https://lists.example", "https://lists.example/hunter2");
// An `@` in the path is not userinfo, and no longer has to be told apart
// from it: the path goes either way.
try expectRedacted("https://lists.example", "https://lists.example/@who/hosts.txt");
}
test "redact keeps everything an operator needs to know where a source points" {
try expectRedacted("https://lists.example", "https://lists.example/hosts.txt");
try expectRedacted("http://10.0.0.2:8080", "http://10.0.0.2:8080/a/b.txt");
try expectRedacted("https://lists.example", "https://lists.example?apikey=s3cr3t");
// No host: what is left still names the scheme.
try expectRedacted("https://", "https://?apikey=s3cr3t");
}
test "redact holds on the malformed urls a parser refuses" {
// These reach the log through `error.BadUrl`, so scanning has to hold where
// `std.Uri.parse` gives up. A scheme delimiter of one separator, of three,
// and of none at all: each once carried the userinfo into the log, because
// the scan looked for `://` and found no authority without it.
//
// Each is now withheld rather than resolved. An earlier revision read the
// text after a run of any length as the authority, which drops the userinfo
// on these four but prints the path segment of `https:/hunter2` as a host.
// Only a run of exactly two says an authority is there; what these hold after
// one, or after three, is a path to RFC 3986 and a host to WHATWG. The
// property the line asserts is unchanged — no userinfo reaches the log — and
// it now holds by withholding rather than by resolving.
try expectRedacted("https://(ambiguous authority omitted)", "https:/user:hunter2@host/list");
try expectRedacted("https://(ambiguous authority omitted)", "https:///user:hunter2@host/list");
try expectRedacted("HTTPS://(ambiguous authority omitted)", "HTTPS:/user:hunter2@host/list");
try expectRedacted("https://(ambiguous authority omitted)", "https:\\user:hunter2@host\\list");
// A separator of none at all is withheld for the same reason.
try expectRedacted("(ambiguous authority omitted)", "https:user:hunter2@host/list");
try expectRedacted("", "?apikey=s3cr3t");
try expectRedacted("not a url", "not a url");
try expectRedacted("", "");
// A colon that is not a scheme delimiter does not make one.
try expectRedacted("lists.example", "lists.example/a:/b");
try expectRedacted("", "/download/token/hunter2/hosts.txt");
}
test "redact treats a backslash as a hierarchical separator" {
// A url typed or pasted with backslashes reaches these lines through
// `error.BadUrl`, so the scan has to cut on one. NextDNS again, because the
// path segment it carries is the whole account identifier.
// A `\` ends an authority but never opens one. WHATWG converts it to a `/`
// only for a special scheme, and RFC 3986 gives it no meaning at all, so none
// of these names a host that both readings agree on — and `tls:\\host` has no
// reading at all that makes it one. The account identifier stays out of the
// line either way, which is the property this test is for.
try expectRedacted("https://(ambiguous authority omitted)", "https:\\\\dns.nextdns.io\\abcd12");
try expectRedacted("https://(ambiguous authority omitted)", "https:/\\dns.nextdns.io\\abcd12");
try expectRedacted("https://(ambiguous authority omitted)", "https:\\/dns.nextdns.io\\abcd12");
try expectRedacted("https://(ambiguous authority omitted)", "https:\\dns.nextdns.io\\abcd12");
try expectRedacted("tls://", "tls:\\\\abcd12.dns.nextdns.io");
// A token in a backslash path goes the way a token in a `/` path goes.
try expectRedacted(
"https://lists.example",
"https://lists.example\\download\\token\\hunter2\\hosts.txt",
);
// The `\\` does not open an authority, so this is reported by its scheme
// alone. What the assertion is really for is that the token in front of the
// `@` does not reach the line, and withholding delivers that at least as
// well as resolving did.
try expectRedacted(
"https://(ambiguous authority omitted)",
"https:\\\\token@lists.example:8443\\hosts.txt?t=1#frag",
);
// A backslash separator does not make a scheme out of a colon that is not
// one, exactly as a `/` does not.
try expectRedacted("lists.example", "lists.example\\a:\\b");
try expectRedacted("", "\\download\\token\\hunter2\\hosts.txt");
}
test "redact resolves a backslash authority only where one reading survives" {
// A `\` after the userinfo ends the authority under the WHATWG reading and
// is an illegal host byte under RFC 3986's, so both readings agree that
// nothing after it is a host. Cutting there prints less than either.
try expectRedacted("https://host", "https://user@host\\list");
// But only once a `//` has established that an authority is there at all. A
// leading `\\` does not, so the userinfo is withheld with everything else
// rather than cut out of a host that was never settled.
try expectRedacted(
"https://(ambiguous authority omitted)",
"https:\\\\user:hunter2@host\\list",
);
// A `\` in front of the `@` is the ambiguous shape instead, and used to
// print the text on one side of it: `https:\\lists.example\path@evil` gave
// `https://evil`. See "redact omits an authority it cannot resolve".
try expectRedacted(
"https://(ambiguous authority omitted)",
"https:\\\\lists.example\\path@evil",
);
// A Windows path is not a url and leaves nothing that names a host. The
// line it appears on still carries the source's row id and name.
try expectRedacted("", "\\\\?\\C:\\lists\\hosts.txt");
}
test "redact omits an authority it cannot resolve" {
// The shape this exists for: a query parameter whose value holds an `@`.
// The text after that `@` is the query, which is where an api key lives, and
// a scan that read it as the end of a userinfo printed the key where the
// host goes — having already dropped the real host.
try expectRedacted(
"https://(ambiguous authority omitted)",
"https://lists.example?token=prefix@hunter2",
);
try expectRedacted(
"https://(ambiguous authority omitted)",
"https://lists.example?user=a@b.example&key=hunter2",
);
try expectRedacted(
"https://(ambiguous authority omitted)",
"https://lists.example#f@hunter2",
);
try expectRedacted(
"https://(ambiguous authority omitted)",
"https:\\\\lists.example\\p@hunter2",
);
// The mirror image, which the previous ordering fixed and this keeps fixed:
// the text before the `@` is a credential just as often, so neither side may
// be printed. Both of these once printed `user:pa55` as the host.
try expectRedacted(
"https://(ambiguous authority omitted)",
"https://user:pa55?@host/x",
);
try expectRedacted(
"https://(ambiguous authority omitted)",
"https://user:pa55#@host/x",
);
try expectRedacted(
"https://(ambiguous authority omitted)",
"https://user:pa55\\@host/x",
);
try expectRedacted(
"https://(ambiguous authority omitted)",
"https:\\\\user:pa55?@host\\x",
);
// A url naming a scheme without a separator after it has no authority by
// RFC 3986 and one by the WHATWG parser, so the `@` in it is a path
// separator under the first reading and a userinfo delimiter under the
// second. `hunter2` is a path segment or a host and no scan can say which.
try expectRedacted("(ambiguous authority omitted)", "https:a@hunter2");
try expectRedacted("(ambiguous authority omitted)", "https:user:hunter2@host/list");
// `user` is not one of WHATWG's special schemes, so both readings make this a
// scheme and an opaque path and neither finds a host. It names no authority
// rather than one that cannot be resolved, and the password is withheld by
// the same rule either way.
try expectRedacted("", "user:pa55@lists.example/hosts.txt?apikey=s3cr3t");
// A url with no scheme at all prints the marker on its own, which is still
// not a url and still not a host.
try expectRedacted("(ambiguous authority omitted)", "lists.example?token=a@hunter2");
}
test "redact tells an omitted authority apart from an absent one" {
// Three outcomes an operator has to be able to tell apart, because the
// action each calls for differs: a host, no host, and a host this cannot
// name. Only the third withholds anything.
try expectRedacted("https://lists.example", "https://lists.example/hosts.txt");
try expectRedacted("https://", "https://?apikey=s3cr3t");
try expectRedacted("https://(ambiguous authority omitted)", "https://?u=a@hunter2");
try expectRedacted("lists.example", "lists.example/hosts.txt");
try expectRedacted("", "?apikey=s3cr3t");
try expectRedacted("(ambiguous authority omitted)", "?u=a@hunter2");
}
test "redact still resolves an authority whose delimiters follow the userinfo" {
// The ambiguity is an `@` after a `?`, a `#` or a `\`, not an `@` at all.
// Where the delimiters fall the way a url puts them, the host is not in
// doubt and withholding it would cost an operator the line's whole point.
try expectRedacted("https://lists.example", "https://user@lists.example?apikey=s3cr3t");
try expectRedacted("https://lists.example", "https://user:pa55@lists.example#frag");
try expectRedacted("https://lists.example:8443", "https://token@lists.example:8443\\hosts.txt");
try expectRedacted("https://lists.example", "https://user@lists.example/x?u=a@b");
// No `@` in the authority: a `?` still ends it, and nothing is ambiguous.
try expectRedacted("https://lists.example", "https://lists.example?apikey=s3cr3t");
}
test "redact escapes the control characters that would forge a log line" {
// A row can be written by a path that does not validate as strictly as the
// config validator, so the manager prints whatever the column holds. A
// newline in it would end the line and start one of the operator's
// choosing.
try expectRedacted(
"https://lists.example\\n2026-01-01 ERROR forged",
"https://lists.example\n2026-01-01 ERROR forged/hosts.txt",
);
try expectRedacted("https://a\\rb", "https://a\rb/x");
try expectRedacted("https://a\\tb", "https://a\tb/x");
try expectRedacted("https://a\\x00b", "https://a\x00b/x");
try expectRedacted("https://a\\x7fb", "https://a\x7fb/x");
// An ESC would reach a terminal as a control sequence on the one path the
// log sink does not cover: `cli.zig` prints a diagnostic to stdout.
try expectRedacted("https://a\\x1bb", "https://a\x1bb/x");
// A literal backslash ends the authority, so `redact` cannot print one at
// all. The doubling that keeps an escape sequence unambiguous is exercised
// where a backslash does survive: `quoteText`, below.
try expectRedacted("https://a", "https://a\\nb/x");
}
test "redact bounds the line it prints at max_len" {
const long_host = "h" ** (2 * max_len);
var buf: [8 * max_len]u8 = undefined;
const printed = try std.fmt.bufPrint(&buf, "{f}", .{redact("https://" ++ long_host ++ "/x")});
try testing.expectEqualStrings(("https://" ++ long_host)[0..max_len] ++ "...", printed);
// The bound counts what is printed, so an escape cannot spend four
// characters of a log line per byte of url.
const control_host = "\n" ** max_len;
const escaped = try std.fmt.bufPrint(&buf, "{f}", .{redact("https://" ++ control_host ++ "/x")});
try testing.expect(escaped.len <= max_len + 3);
try testing.expect(std.mem.endsWith(u8, escaped, "..."));
try testing.expect(!std.mem.containsAtLeast(u8, escaped, 1, "\n"));
// A scheme long enough on its own truncates inside the scheme rather than
// printing it whole and starting on the host.
const long_scheme = "s" ** (2 * max_len);
const truncated = try std.fmt.bufPrint(&buf, "{f}", .{redact(long_scheme ++ ":/host")});
try testing.expectEqualStrings(long_scheme[0..max_len] ++ "...", truncated);
}
test "quoteText escapes and bounds an operator-supplied name" {
var buf: [8 * max_len]u8 = undefined;
try testing.expectEqualStrings(
"'ads and trackers'",
try std.fmt.bufPrint(&buf, "{f}", .{quoteText("ads and trackers")}),
);
try testing.expectEqualStrings(
"'ads\\n2026-01-01 ERROR forged'",
try std.fmt.bufPrint(&buf, "{f}", .{quoteText("ads\n2026-01-01 ERROR forged")}),
);
// A name cannot close the quote around it and write what follows as though
// it were another field of the line.
try testing.expectEqualStrings(
"'ads\\' (https://decoy.example) --'",
try std.fmt.bufPrint(&buf, "{f}", .{quoteText("ads' (https://decoy.example) --")}),
);
// Nor by escaping the escape: a `\` before the quote is doubled first, so
// `\'` in the output is this function's and never the operator's.
try testing.expectEqualStrings(
"'ads\\\\\\' (https://decoy.example) --'",
try std.fmt.bufPrint(&buf, "{f}", .{quoteText("ads\\' (https://decoy.example) --")}),
);
// Nor by running past `max_len`: the truncation closes the quote too.
const long_name = "n" ** (2 * max_len);
try testing.expectEqualStrings(
"'" ++ long_name[0..max_len] ++ "...'",
try std.fmt.bufPrint(&buf, "{f}", .{quoteText(long_name)}),
);
try testing.expectEqualStrings(
"'" ++ "\\'" ** (max_len / 2) ++ "...'",
try std.fmt.bufPrint(&buf, "{f}", .{quoteText("'" ** max_len)}),
);
}
test "redact withholds a scheme whose separator is missing, with or without an @" {
// RFC 3986 reads `hunter2` as an opaque path and a path segment is where a
// token lives; WHATWG inserts the missing `//` for a special scheme and reads
// it as the host. An earlier revision withheld this only when an `@` was
// present, so the plainer shape printed the path whole.
try expectRedacted("(ambiguous authority omitted)", "https:hunter2");
try expectRedacted("(ambiguous authority omitted)", "https:a@hunter2");
try expectRedacted("(ambiguous authority omitted)", "https:123456");
// Only a WHATWG special scheme can disagree with RFC 3986 here, because only
// a special scheme reads an authority out of text no `//` introduced. For
// every other scheme both readings say the same thing — a scheme and an
// opaque path — so these name no authority rather than an unresolved one.
// Withheld either way; what differs is what the line then claims.
try expectRedacted("", "mailto:ops@example.com");
try expectRedacted("", "localhost:8080/hosts.txt");
try expectRedacted("", "localhost:8080@evil/x");
// An `IP:port` is unaffected, because a leading digit fails the scheme
// production and there is nothing to disagree about.
try expectRedacted("10.0.0.2:8080", "10.0.0.2:8080/hosts.txt");
try expectRedacted("[::1]:853", "[::1]:853/x");
try expectRedacted("lists.example", "lists.example/hosts.txt");
}
test "redact takes exactly two separators as an authority delimiter" {
// One separator leaves an absolute path: RFC 3986 reads `https:/hunter2` as
// the path `/hunter2`, WHATWG reads `hunter2` as the host. Three or more is
// an empty authority to the first and a host to the second. An earlier
// revision accepted any run and printed the path segment as the host.
try expectRedacted("https://(ambiguous authority omitted)", "https:/hunter2");
try expectRedacted("https://(ambiguous authority omitted)", "https:///hunter2");
try expectRedacted("localhost://", "localhost:/hunter2");
try expectRedacted("https://lists.example", "https://lists.example/hosts.txt");
// The userinfo this tolerance was introduced to protect is protected by
// withholding instead. The `/` cut runs ahead of the userinfo lookup now, so
// the authority of a url with no `://` ends before any `@` it holds.
try expectRedacted("https://(ambiguous authority omitted)", "https:/user:pass@host/list");
}
test "redact resolves a network-path reference instead of reporting nothing" {
// The `/` cut lands at byte zero here, so without the `//` branch every one
// of these redacted to the empty string and the line named no source at all.
try expectRedacted("lists.example", "//lists.example/hosts.txt");
// Exactly two. A longer run is contested — an empty authority to RFC 3986,
// and `lists.example` as the host to WHATWG resolving against a
// special-scheme base — so it is withheld rather than reported as a url that
// names no authority. Those are different answers and this type keeps them
// apart.
try expectRedacted("(ambiguous authority omitted)", "///lists.example/x");
// The authority is not in doubt, so the userinfo is dropped rather than the
// whole of it withheld.
try expectRedacted("lists.example", "//user:pa55@lists.example/hosts.txt");
// An ambiguous one is still withheld: the branch says where the authority
// starts, not that every reading of it is settled.
try expectRedacted("(ambiguous authority omitted)", "//a?b@c");
// A single leading `/` is a path, not an authority, and still names none.
try expectRedacted("", "/path/only");
}
test "the shapes redact over-withholds on print less, never more" {
// The three imprecisions `contestedOrAbsent` documents. Each says "could not
// resolve" where both readings in fact find no host, so each prints less than
// it could. Pinned because the failure that matters is the other direction:
// if one of these ever starts naming a host, this test says so.
try expectRedacted("https://(ambiguous authority omitted)", "https:/user@/x");
try expectRedacted("https://(ambiguous authority omitted)", "https:/user:hunter2@/x");
try expectRedacted("(ambiguous authority omitted)", "\\path@hunter2");
try expectRedacted("(ambiguous authority omitted)", "file:secret");
try expectRedacted("file://(ambiguous authority omitted)", "file:\\secret");
// `file` still resolves where a `//` settles it, which is why the imprecision
// is in the classification and not in the scan.
try expectRedacted("file://host", "file://host/secret");
}
test "redactQuoted writes its own quotes and closes them in every exit" {
var buf: [8 * max_len]u8 = undefined;
try testing.expectEqualStrings(
"'https://lists.example'",
try std.fmt.bufPrint(&buf, "{f}", .{redactQuoted("https://lists.example/hosts.txt")}),
);
// The omitted-authority value is prose with spaces in it, which is the case
// the quotes exist for: unquoted it runs into the words of the sentence
// around it.
try testing.expectEqualStrings(
"'https://(ambiguous authority omitted)'",
try std.fmt.bufPrint(&buf, "{f}", .{redactQuoted("https://lists.example?token=prefix@hunter2")}),
);
// A truncation returns early, and the closing quote still has to be written
// or the rest of the line reads as part of the value.
const long_host = "h" ** (2 * max_len);
try testing.expectEqualStrings(
"'https://" ++ ("h" ** (max_len - "https://".len)) ++ "...'",
try std.fmt.bufPrint(&buf, "{f}", .{redactQuoted("https://" ++ long_host ++ "/x")}),
);
}
test "a redacted authority cannot close the quote a caller would have added" {
var buf: [8 * max_len]u8 = undefined;
// A `'` is neither a component separator nor a control character, so it
// survives redaction into the authority. `redact` leaves it, which is why a
// caller may not supply the quotes itself.
try testing.expectEqualStrings(
"https://ho'st",
try std.fmt.bufPrint(&buf, "{f}", .{redact("https://ho'st/x")}),
);
try testing.expectEqualStrings(
"'https://ho\\'st'",
try std.fmt.bufPrint(&buf, "{f}", .{redactQuoted("https://ho'st/x")}),
);
// The forging shape, whole: an operator-supplied url that ends the value and
// writes what follows as though the line had said it.
try testing.expectEqualStrings(
"'https://ho\\' is fine; upstreams[9] '",
try std.fmt.bufPrint(&buf, "{f}", .{redactQuoted("https://ho' is fine; upstreams[9] /x")}),
);
// The escape-the-escape shape that `QuotedText` has to defend against does
// not arise here, and not because the escaper is different — it is the same
// one. A `\` ends an authority, so it never reaches the value to be doubled.
// This is asserted rather than assumed: it is the property that makes a
// single `\` in the output always this file's and never the operator's.
try testing.expectEqualStrings(
"'https://ho'",
try std.fmt.bufPrint(&buf, "{f}", .{redactQuoted("https://ho\\'st/x")}),
);
}
src/server/udp_server.zig
+2 -2
View File
@@ -77,7 +77,7 @@ pub const UdpServer = struct {
/// handler, so nothing larger than 4096 bytes leaves this socket.
///
/// Cost: 4096 + 65535 + the scratch below ≈ 74 KiB per slot, so the
/// default 64 slots hold ≈ 4.6 MiB. The PLAN §18 budget is 100 MB with
/// default 64 slots hold ≈ 4.6 MiB. The PLAN §18 budget is 100 MiB with
/// ~1M blocked domains, so this pool takes about 5% of it.
reply: [transport.max_message_len]u8,
/// The handler's per-query working memory. It belongs to the slot so
@@ -297,7 +297,7 @@ test "a slot's reply buffer holds a whole DNS message" {
test "the default slot pool stays inside the memory budget" {
// 4096 + 65535 + scratch ≈ 74 KiB per slot; 64 slots ≈ 4.6 MiB, against the
// 100 MB of PLAN §18.
// 100 MiB of PLAN §18.
const options: Options = .{};
const pool_bytes = @sizeOf(UdpServer.Slot) * @as(usize, options.max_in_flight);
try testing.expect(pool_bytes < 8 * 1024 * 1024);
src/storage/db.zig
+536 -38
View File
@@ -1,12 +1,17 @@
//! The whole SQLite surface nxdns owns (PLAN Decision G). Nothing above this
//! file calls SQLite directly.
//!
//! **This file takes no `std.Io`.** It is the one deliberate exception to
//! Decision E. SQLite performs its own file I/O through its VFS; routing it
//! through `std.Io` would mean writing a custom SQLite VFS — a large,
//! **SQLite's own file I/O takes no `std.Io`.** It is the one deliberate
//! exception to Decision E. SQLite performs its own file I/O through its VFS;
//! routing it through `std.Io` would mean writing a custom SQLite VFS — a large,
//! security-sensitive component bought for nothing at household scale. Every
//! other storage file that touches the filesystem takes `io: std.Io`.
//!
//! The exception covers SQLite, not nxdns. The one place this file does its own
//! filesystem calls — the probes guarding `OpenMode.immutable`, at the open and
//! again at `Db.verifyImmutable` — follows the ordinary rule and takes an
//! `io: std.Io`, which is why that mode carries one.
//!
//! The C API is declared by hand below. No `@cImport` — the handles stay
//! opaque, matching `src/platform/tls_server.zig`'s Mbed TLS approach.
@@ -135,6 +140,11 @@ pub const Error = error{
SqliteError,
OutOfMemory,
Unexpected,
/// Not a SQLite result code. An `OpenMode.immutable` read would have
/// answered from a stale main file, because `<path>-wal` holds bytes or the
/// files moved while the read ran. Raised by the open and again by
/// `Db.verifyImmutable`. See `OpenMode.immutable`.
WalPending,
};
/// Maps a primary SQLite result code to `Error`. `SQLITE_NOMEM` becomes
@@ -190,19 +200,73 @@ fn check(code: c_int) Error!void {
return mapCode(code);
}
pub const OpenMode = enum { read_write_create, read_write_existing, read_only, memory };
pub const OpenMode = union(enum) {
read_write_create,
read_write_existing,
read_only,
memory,
/// Read a database that this process promises not to change, and that no
/// writer may be touching: `SQLITE_OPEN_READONLY` plus the `immutable=1` URI
/// parameter, which makes the pager treat the file like a temp file — no
/// locking, no rollback journal, no wal-index — so **no `-wal` and no `-shm`
/// appear beside it**.
///
/// This mode exists for `nxdns check` (milestone-13 ruling F-c), which must
/// validate without writing. `.read_only` alone is not enough, and this is
/// measured, not assumed: reading a database whose header says WAL makes
/// SQLite build the wal-index, so `config.db-wal` (0 bytes) and
/// `config.db-shm` (32 KiB) are created, and a read-only connection cannot
/// remove them on close. A command that claims to write nothing must not
/// leave two files behind. Do not "simplify" this back to `.read_only`.
///
/// What `immutable=1` costs: SQLite then **ignores any `-wal` file**. The
/// newest committed rows live there, so an immutable read of a database with
/// an un-checkpointed WAL would answer from stale data and say nothing — a
/// worse failure than the two sidecar files it removes. `Db.open` therefore
/// refuses this mode with `error.WalPending` whenever `<path>-wal` exists and
/// is not empty; the caller reports that as a failure and names `nxdns run`,
/// which opens read-write and checkpoints, as the fix.
///
/// The guard lives inside `open` rather than in a helper callers are trusted
/// to call, because the failure it prevents is silent: a caller that forgets
/// a helper gets a plausible wrong answer, and nothing anywhere reports it.
///
/// **The open is half the guard.** `immutable=1` takes no lock, so nothing
/// keeps a writer out for the duration of the read, and a check made only
/// before the read can say only that the log was empty *then*.
/// `Db.verifyImmutable` makes the other half, and a caller that grades what
/// it read without calling it is back to the stale answer this mode exists
/// to refuse.
///
/// A zero-length `-wal` does not block the open: it holds no frames, so the
/// main file is complete. That is the exact leftover the pre-F-c `check`
/// used to create.
///
/// The `std.Io` is for that probe — the one filesystem call nxdns itself
/// makes in this file. Passing it is what makes the guard unskippable.
immutable: std.Io,
};
pub const OpenOptions = struct {
mode: OpenMode = .read_write_create,
busy_timeout_ms: c_int = 5000,
};
/// SQLite's name for the write-ahead log beside `<path>`. Exported so a caller
/// reporting `error.WalPending` can name the file without hard-coding SQLite's
/// naming convention, which this file owns.
pub const wal_suffix = "-wal";
/// One SQLite connection.
///
/// A `Db` must not move once a `Stmt` prepared from it is alive: every `Stmt`
/// holds a `*Db`.
pub const Db = struct {
handle: *c.Sqlite3,
/// Set by `OpenMode.immutable` and null in every other mode: what the
/// database file and its `-wal` looked like when the read began, for
/// `verifyImmutable` to compare against when it ends.
immutable_guard: ?ImmutableGuard = null,
/// Every mode carries `FULLMUTEX` (serialized mode). Phase 6's query logger
/// and Phase 8's API handlers share one handle across `std.Io` tasks, and a
@@ -220,47 +284,59 @@ pub const Db = struct {
.read_write_create, .memory => base | open_flag.readwrite | open_flag.create,
.read_write_existing => base | open_flag.readwrite,
.read_only => base | open_flag.readonly,
// Its own function: the URI buffer is 12 KiB, and every other open
// in the process would carry it in this frame.
.immutable => |io| return openImmutable(io, path, options.busy_timeout_ms),
};
const filename: [:0]const u8 = switch (options.mode) {
.memory => ":memory:",
else => path,
};
var handle: ?*c.Sqlite3 = null;
const rc = c.sqlite3_open_v2(filename.ptr, &handle, flags, null);
if (rc != result.ok) {
// sqlite3_open_v2 allocates a handle even on failure. Read the
// message from it, then close it; dropping it leaks on every
// failed open.
// Logged at `warn`, not `err`: the failure itself reaches the
// caller as a typed error, and this line only carries the message
// that would otherwise die with the handle.
if (handle) |h| {
log.warn("sqlite3_open_v2 failed for '{s}': {s} (code {d}/{d})", .{
filename,
std.mem.span(c.sqlite3_errmsg(h)),
rc & 0xff,
c.sqlite3_extended_errcode(h),
});
_ = c.sqlite3_close_v2(h);
} else {
log.warn("sqlite3_open_v2 failed for '{s}': {s} (code {d})", .{
filename,
std.mem.span(c.sqlite3_errstr(rc)),
rc,
});
}
return mapCode(rc);
}
const h = handle orelse return error.SqliteError;
const h = try openHandle(filename, flags);
return applyBusyTimeout(h, options.busy_timeout_ms);
}
// A silently ignored busy timeout is how a contended WAL database turns
// into random SQLITE_BUSY failures under load.
check(c.sqlite3_busy_timeout(h, options.busy_timeout_ms)) catch |e| {
_ = c.sqlite3_close_v2(h);
return e;
};
return .{ .handle = h };
/// Proves that the files an `OpenMode.immutable` read answered from stood
/// still while it ran, and fails the read with `error.WalPending` when they
/// did not. Call it after the last read and before anything read is
/// reported.
///
/// `immutable=1` takes no lock at all — that is what stops the pager
/// building a wal-index, and the price is that nothing keeps a writer out.
/// The probe at open time can only say the log was empty at that instant: a
/// writer that appends one frame the instant after leaves the read answering
/// from the older pages of the main file, silently, which is the whole
/// failure `OpenMode.immutable`'s guard exists to prevent.
///
/// Two things are compared, because a writer can hide in either:
///
/// - `<path>-wal` holding bytes now. A log that appeared, one that grew, and
/// one written into the empty file the open accepted all land here.
/// - `<path>` itself moving — size, inode, mtime or ctime. This is the
/// checkpoint the log cannot show: a writer that checkpointed into the main
/// file and truncated its log back to nothing leaves both stats saying "no
/// frames" while the pages the read saw have been replaced.
///
/// Best effort, and honestly so: a filesystem with coarse timestamps can
/// hide a rewrite that lands on the same byte count in the same tick. That
/// cannot be fixed from outside SQLite's locking, and taking a lock is the
/// one thing this mode may not do. What it closes is the window a single
/// stat before the read leaves open for the whole of the read.
///
/// Two stats and nothing else: no `-wal` or `-shm` is created, and neither
/// sidecar is removed or truncated. Calling this on a handle opened in any
/// other mode is a caller bug.
pub fn verifyImmutable(self: *Db) Error!void {
const guard = self.immutable_guard orelse unreachable;
const now = try markImmutable(guard.io, guard.path);
if (!std.meta.eql(now, guard.mark)) {
log.warn(
"'{s}' changed while it was being read without a lock; the read is not trustworthy",
.{guard.path},
);
return error.WalPending;
}
}
pub fn close(self: *Db) void {
@@ -355,6 +431,199 @@ pub const Db = struct {
}
};
fn openHandle(filename: [:0]const u8, flags: c_int) Error!*c.Sqlite3 {
var handle: ?*c.Sqlite3 = null;
const rc = c.sqlite3_open_v2(filename.ptr, &handle, flags, null);
if (rc != result.ok) {
// sqlite3_open_v2 allocates a handle even on failure. Read the
// message from it, then close it; dropping it leaks on every
// failed open.
// Logged at `warn`, not `err`: the failure itself reaches the
// caller as a typed error, and this line only carries the message
// that would otherwise die with the handle.
if (handle) |h| {
log.warn("sqlite3_open_v2 failed for '{s}': {s} (code {d}/{d})", .{
filename,
std.mem.span(c.sqlite3_errmsg(h)),
rc & 0xff,
c.sqlite3_extended_errcode(h),
});
_ = c.sqlite3_close_v2(h);
} else {
log.warn("sqlite3_open_v2 failed for '{s}': {s} (code {d})", .{
filename,
std.mem.span(c.sqlite3_errstr(rc)),
rc,
});
}
return mapCode(rc);
}
return handle orelse error.SqliteError;
}
/// A silently ignored busy timeout is how a contended WAL database turns into
/// random SQLITE_BUSY failures under load.
fn applyBusyTimeout(h: *c.Sqlite3, busy_timeout_ms: c_int) Error!Db {
check(c.sqlite3_busy_timeout(h, busy_timeout_ms)) catch |e| {
_ = c.sqlite3_close_v2(h);
return e;
};
return .{ .handle = h };
}
// ---------------------------------------------------------------------------
// OpenMode.immutable
// ---------------------------------------------------------------------------
const uri_scheme = "file:";
const uri_immutable_query = "?immutable=1";
/// Worst case: every byte of the longest path the platform accepts becomes
/// `%HH`.
const immutable_uri_buf_len =
uri_scheme.len + 3 * std.Io.Dir.max_path_bytes + uri_immutable_query.len + 1;
fn openImmutable(io: std.Io, path: [:0]const u8, busy_timeout_ms: c_int) Error!Db {
const mark = try markImmutable(io, path);
var buf: [immutable_uri_buf_len]u8 = undefined;
const uri = try immutableUri(&buf, path);
// `uri` without `open_flag.uri` would be opened as a filename spelled
// "file:...", creating nothing and finding nothing.
const flags = open_flag.exrescode | open_flag.fullmutex |
open_flag.readonly | open_flag.uri;
const h = try openHandle(uri, flags);
var database = try applyBusyTimeout(h, busy_timeout_ms);
database.immutable_guard = .{ .io = io, .path = path, .mark = mark };
return database;
}
/// What `OpenMode.immutable` recorded at the start of a read so that
/// `Db.verifyImmutable` can prove nothing moved by the end of it.
pub const ImmutableGuard = struct {
io: std.Io,
/// Borrowed. Must outlive the `Db`, which every caller satisfies by owning
/// the path for at least as long as the connection it opened with it.
path: []const u8,
mark: FileMark,
};
/// The main database file at one instant, in the fields an outside observer can
/// compare cheaply. `atime` is deliberately absent: reading the file changes it,
/// so comparing it would report every read as a change.
///
/// All zero when the file does not exist, which is itself a state worth
/// comparing — a database replaced by an unlink is a database that moved.
const FileMark = struct {
present: bool,
size: u64,
inode: std.Io.File.INode,
mtime_ns: i96,
ctime_ns: i96,
};
/// The state an immutable read must find unchanged, or `error.WalPending` when
/// `<path>-wal` already holds bytes.
///
/// The `-wal` rule is deliberately conservative: any non-empty log fails.
/// Deciding whether it really holds committed frames means running WAL recovery
/// — checksums, salt, the wal-index — which is the writing that
/// `OpenMode.immutable` exists to avoid. A live writer, a crash, and a
/// checkpointed-but-retained log all land here, and refusing to answer is the
/// right side to err on: the alternative is a stale answer nobody can see is
/// stale. A zero-length log holds no frames, so the main file is complete and it
/// passes.
///
/// A failed stat is not "no WAL": it means this cannot be known, so it stays a
/// failure.
fn markImmutable(io: std.Io, path: []const u8) Error!FileMark {
var buf: [std.Io.Dir.max_path_bytes + wal_suffix.len]u8 = undefined;
const sidecar = std.fmt.bufPrint(&buf, "{s}{s}", .{ path, wal_suffix }) catch
return error.TooBig;
if (try statOrAbsent(io, sidecar)) |wal| {
if (wal.size > 0) return error.WalPending;
}
const main = try statOrAbsent(io, path) orelse return .{
.present = false,
.size = 0,
.inode = 0,
.mtime_ns = 0,
.ctime_ns = 0,
};
return .{
.present = true,
.size = main.size,
.inode = main.inode,
.mtime_ns = main.mtime.nanoseconds,
.ctime_ns = main.ctime.nanoseconds,
};
}
fn statOrAbsent(io: std.Io, path: []const u8) Error!?std.Io.Dir.Stat {
return std.Io.Dir.cwd().statFile(io, path, .{}) catch |e| switch (e) {
error.FileNotFound => return null,
else => {
log.warn("cannot stat '{s}': {t}", .{ path, e });
return error.IoErr;
},
};
}
/// `file:` + percent-encoded `path` + `?immutable=1`.
///
/// The encoding is load-bearing, not cosmetic. `?` opens SQLite's query section
/// and `#` its fragment, so an unencoded data directory named `dns?db` would
/// silently open a *different* file; `%` must be encoded because SQLite decodes
/// `%HH` on its way back to a filename. `--data-dir` is operator input, so all
/// three are reachable.
///
/// Everything outside the unreserved set (`A-Z a-z 0-9 - . _ ~ /`) is encoded,
/// which is always safe: SQLite decodes every escape in the path before handing
/// the name to its VFS, so the bytes it opens are the bytes passed in.
///
/// `/` stays literal to keep diagnostics readable, with one exception. SQLite
/// reads `file://…` as a URI authority and rejects any authority but the empty
/// one or `localhost` (`sqlite3ParseUri`), so a path beginning with `//` — legal
/// POSIX — has its second slash encoded.
fn immutableUri(buf: []u8, path: []const u8) error{TooBig}![:0]const u8 {
var out: usize = 0;
try appendSlice(buf, &out, uri_scheme);
for (path, 0..) |ch, i| {
const opens_authority = i == 1 and ch == '/' and path[0] == '/';
if (isUriUnreserved(ch) and !opens_authority) {
try appendByte(buf, &out, ch);
} else {
const hex = "0123456789ABCDEF";
try appendByte(buf, &out, '%');
try appendByte(buf, &out, hex[ch >> 4]);
try appendByte(buf, &out, hex[ch & 0xf]);
}
}
try appendSlice(buf, &out, uri_immutable_query);
try appendByte(buf, &out, 0);
return buf[0 .. out - 1 :0];
}
fn isUriUnreserved(ch: u8) bool {
return switch (ch) {
'a'...'z', 'A'...'Z', '0'...'9', '-', '.', '_', '~', '/' => true,
else => false,
};
}
fn appendByte(buf: []u8, out: *usize, ch: u8) error{TooBig}!void {
if (out.* == buf.len) return error.TooBig;
buf[out.*] = ch;
out.* += 1;
}
fn appendSlice(buf: []u8, out: *usize, bytes: []const u8) error{TooBig}!void {
for (bytes) |ch| try appendByte(buf, out, ch);
}
/// One prepared statement.
///
/// There is deliberately **no prepared-statement cache in this milestone**.
@@ -735,6 +1004,235 @@ test "a row-producing statement reports its row through step" {
try testing.expect(try stmt.step());
}
// ---------------------------------------------------------------------------
// OpenMode.immutable
// ---------------------------------------------------------------------------
/// `std.testing.tmpDir` creates its directory under `.zig-cache/tmp/` relative to
/// the process working directory, which is also how SQLite's VFS resolves the
/// filename it is handed (`queries_repo.zig:721`).
const tmp_prefix = ".zig-cache/tmp/";
const sub_path_len = @typeInfo(@FieldType(testing.TmpDir, "sub_path")).array.len;
const test_io = testing.io;
fn tmpPath(buf: []u8, tmp: *const testing.TmpDir, name: []const u8) ![:0]const u8 {
return std.fmt.bufPrintZ(buf, "{s}{s}/{s}", .{ tmp_prefix, &tmp.sub_path, name });
}
/// A file database in WAL mode holding one row, `id = marker`. Closing the last
/// connection checkpoints and unlinks both sidecars, but the header keeps saying
/// WAL — which is what makes a later `.read_only` open recreate them.
fn writeWalDatabase(path: [:0]const u8, marker: i64) !void {
var database = try Db.open(path, .{ .mode = .read_write_create });
defer database.close();
try applyPragmas(&database, .{});
try database.exec("CREATE TABLE t (id INTEGER PRIMARY KEY);");
var stmt = try database.prepare("INSERT INTO t (id) VALUES (?1)");
defer stmt.deinit();
try stmt.bindInt(1, marker);
try stmt.exec();
}
/// A second writer doing what a running nxdns does: it appends to the
/// write-ahead log and, being the last connection, checkpoints into the main
/// file and unlinks both sidecars on close. The blob is what makes the main
/// file grow by whole pages, so a test that watches for the change does not rest
/// on the filesystem's timestamp resolution.
fn checkpointOver(path: [:0]const u8) !void {
var database = try Db.open(path, .{ .mode = .read_write_existing });
defer database.close();
try applyPragmas(&database, .{});
try database.exec("INSERT INTO t (id) VALUES (8);");
try database.exec("CREATE TABLE bulk (v TEXT);");
try database.exec("INSERT INTO bulk (v) VALUES (hex(randomblob(30000)));");
}
fn expectAbsent(dir: std.Io.Dir, name: []const u8) !void {
dir.access(test_io, name, .{}) catch |e| switch (e) {
error.FileNotFound => return,
else => |other| return other,
};
std.debug.print("sidecar '{s}' exists and must not\n", .{name});
return error.SidecarPresent;
}
test "immutableUri encodes what would otherwise change which file is opened" {
var buf: [256]u8 = undefined;
try testing.expectEqualStrings(
"file:/var/lib/nxdns/config.db?immutable=1",
try immutableUri(&buf, "/var/lib/nxdns/config.db"),
);
// '?' would start SQLite's query section, '#' its fragment, '%' an escape.
try testing.expectEqualStrings(
"file:/data%3Fdir/config.db?immutable=1",
try immutableUri(&buf, "/data?dir/config.db"),
);
try testing.expectEqualStrings(
"file:/data%23dir/config.db?immutable=1",
try immutableUri(&buf, "/data#dir/config.db"),
);
try testing.expectEqualStrings(
"file:/data%25dir/config.db?immutable=1",
try immutableUri(&buf, "/data%dir/config.db"),
);
try testing.expectEqualStrings(
"file:/a%20b/c%3Fd%23e%25f.db?immutable=1",
try immutableUri(&buf, "/a b/c?d#e%f.db"),
);
// A relative path stays relative: SQLite's VFS resolves it against the
// working directory, exactly as a bare filename would be.
try testing.expectEqualStrings(
"file:config.db?immutable=1",
try immutableUri(&buf, "config.db"),
);
// A leading "//" would be read as a URI authority and rejected.
try testing.expectEqualStrings(
"file:/%2Fnet/share/config.db?immutable=1",
try immutableUri(&buf, "//net/share/config.db"),
);
// Only the authority position is special: "//" further in stays literal.
try testing.expectEqualStrings(
"file:/net//share/config.db?immutable=1",
try immutableUri(&buf, "/net//share/config.db"),
);
// Non-ASCII bytes survive the round trip because SQLite decodes them back.
try testing.expectEqualStrings(
"file:/caf%C3%A9/config.db?immutable=1",
try immutableUri(&buf, "/café/config.db"),
);
var small: [16]u8 = undefined;
try testing.expectError(error.TooBig, immutableUri(&small, "/var/lib/nxdns/config.db"));
}
test "an immutable open of a WAL database creates no -wal and no -shm" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try tmpPath(&path_buf, &tmp, "config.db");
try writeWalDatabase(path, 7);
try expectAbsent(tmp.dir, "config.db-wal");
try expectAbsent(tmp.dir, "config.db-shm");
{
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
// The point of the mode: a `.read_only` open creates both of these here,
// and cannot delete them on close.
try expectAbsent(tmp.dir, "config.db-wal");
try expectAbsent(tmp.dir, "config.db-shm");
}
try expectAbsent(tmp.dir, "config.db-wal");
try expectAbsent(tmp.dir, "config.db-shm");
}
test "an immutable open of a path holding URI metacharacters opens the intended file" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var decoy_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
var path_buf: [tmp_prefix.len + sub_path_len + 64]u8 = undefined;
// Unencoded, SQLite cuts the filename at the '?' and opens "<tmp>/d". That
// file exists here and holds a different database, so the failure without
// percent-encoding is a wrong answer, not an error.
try tmp.dir.createDirPath(test_io, "d?x#y%z");
try writeWalDatabase(try tmpPath(&decoy_buf, &tmp, "d"), 99);
const path = try tmpPath(&path_buf, &tmp, "d?x#y%z/config.db");
try writeWalDatabase(path, 7);
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
}
test "an immutable open refuses a database whose -wal holds bytes" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try tmpPath(&path_buf, &tmp, "config.db");
try writeWalDatabase(path, 7);
// What an unclean shutdown leaves behind, written directly so the case does
// not depend on when SQLite decides to checkpoint.
try tmp.dir.writeFile(test_io, .{ .sub_path = "config.db" ++ wal_suffix, .data = &([_]u8{0x37} ** 32) });
try testing.expectError(error.WalPending, Db.open(path, .{ .mode = .{ .immutable = test_io } }));
// A zero-length `-wal` holds no frames, so the main file is complete: the
// exact leftover the pre-F-c `check` created must not block a check.
try tmp.dir.writeFile(test_io, .{ .sub_path = "config.db" ++ wal_suffix, .data = "" });
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
}
test "an immutable read refuses a -wal that arrives while it is in flight" {
// The probe at open time can only say the log was empty *then*.
// `immutable=1` takes no lock, so a writer is free to arrive one instant
// later, and the read goes on answering from the older pages of the main
// file with nothing anywhere reporting it.
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try tmpPath(&path_buf, &tmp, "config.db");
try writeWalDatabase(path, 7);
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
// Nothing has moved yet, so the read stands.
try database.verifyImmutable();
// A zero-length log still holds no frames: the rule at the end of the read
// is the rule at the start of it.
try tmp.dir.writeFile(test_io, .{ .sub_path = "config.db" ++ wal_suffix, .data = "" });
try database.verifyImmutable();
// Frames, now, in the log the open accepted as empty.
try tmp.dir.writeFile(test_io, .{
.sub_path = "config.db" ++ wal_suffix,
.data = &([_]u8{0x37} ** 32),
});
try testing.expectError(error.WalPending, database.verifyImmutable());
// Repeatable: reporting the race is all it does.
try testing.expectError(error.WalPending, database.verifyImmutable());
// And it repairs nothing. The operator's log is byte for byte what was
// written, and no wal-index appeared beside it.
const wal = try tmp.dir.statFile(test_io, "config.db" ++ wal_suffix, .{});
try testing.expectEqual(@as(u64, 32), wal.size);
try expectAbsent(tmp.dir, "config.db-shm");
}
test "an immutable read refuses a main file checkpointed under it" {
// The case a `-wal` probe cannot see at either end: a writer checkpointed
// into the main file and, closing, unlinked its log again. Both stats say
// "no frames" while the pages the read answered from have been replaced.
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try tmpPath(&path_buf, &tmp, "config.db");
try writeWalDatabase(path, 7);
var database = try Db.open(path, .{ .mode = .{ .immutable = test_io } });
defer database.close();
try testing.expectEqual(@as(i64, 7), try database.queryInt("SELECT id FROM t"));
try database.verifyImmutable();
const before = try tmp.dir.statFile(test_io, "config.db", .{});
try checkpointOver(path);
const after = try tmp.dir.statFile(test_io, "config.db", .{});
// The premise of the test, proved rather than assumed: the main file really
// did move, and the log really is gone again.
try testing.expect(after.size != before.size);
try expectAbsent(tmp.dir, "config.db" ++ wal_suffix);
try testing.expectError(error.WalPending, database.verifyImmutable());
}
test "a duplicate insert into a UNIQUE column returns error.Constraint" {
var db = try openMemory();
defer db.close();
src/storage/migrations.zig
+52 -1
View File
@@ -31,6 +31,9 @@ const ddl_v2: [:0]const u8 =
\\ALTER TABLE upstreams ADD COLUMN tls_name TEXT NOT NULL DEFAULT '';
;
/// The schema version this binary expects. A database `readVersion` reports
/// below this needs `nxdns run` to migrate it; above it is `error.SchemaTooNew`
/// and needs a newer nxdns.
pub const target_version: u32 = steps[steps.len - 1].version;
comptime {
@@ -102,9 +105,16 @@ pub fn migrateSteps(database: *db.Db, list: []const Step) Error!u32 {
return target;
}
/// The schema version stamped in `database`, compared against `target_version`.
///
/// `0` when `schema_version` does not exist yet. Zero rows or more than one row
/// is `error.SchemaCorrupt` — the version of a database is never guessed.
fn readVersion(database: *db.Db) Error!u32 {
///
/// Reads only, so it works on a connection opened `.read_only` or
/// `.immutable`. That is what it is public for: `nxdns check` may not migrate
/// (ruling F-c), and "at version 1, this binary expects 2" tells an operator
/// what to do where a bare SQLite error message does not.
pub fn readVersion(database: *db.Db) Error!u32 {
const present = try database.queryInt(
"SELECT count(*) FROM sqlite_schema WHERE type='table' AND name='schema_version'",
);
@@ -302,6 +312,47 @@ test "a failing step after step 2 rolls back the whole upgrade from version 1" {
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
}
test "readVersion reports 0 before a migration and target_version after it" {
var database = try openMigrated();
defer database.close();
try testing.expectEqual(@as(u32, 0), try readVersion(&database));
_ = try migrate(&database);
try testing.expectEqual(target_version, try readVersion(&database));
}
/// `.zig-cache/tmp/` is where `std.testing.tmpDir` puts its directories, and
/// SQLite's VFS resolves filenames against the same working directory
/// (`db.zig`'s immutable tests).
const tmp_prefix = ".zig-cache/tmp/";
const sub_path_len = @typeInfo(@FieldType(testing.TmpDir, "sub_path")).array.len;
test "readVersion reads a file database through an immutable open, writing nothing" {
var tmp = testing.tmpDir(.{});
defer tmp.cleanup();
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
const path = try std.fmt.bufPrintZ(&path_buf, "{s}{s}/config.db", .{ tmp_prefix, &tmp.sub_path });
// A database an older nxdns left at version 1. `check` must report that, not
// migrate it (ruling F-c).
{
var database = try db.Db.open(path, .{ .mode = .read_write_create });
defer database.close();
try db.applyPragmas(&database, .{});
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
try testing.expectEqual(@as(u32, 1), try migrateSteps(&database, &first));
}
var database = try db.Db.open(path, .{ .mode = .{ .immutable = testing.io } });
defer database.close();
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
try testing.expectEqual(@as(u32, 2), target_version);
// A write through this connection is refused by SQLite, not by convention.
try testing.expectError(error.ReadOnly, database.exec("DELETE FROM schema_version;"));
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
}
test "delete_order and content_tables name exactly the tables the schema creates" {
var database = try openMigrated();
defer database.close();
src/storage/repositories/clients_repo.zig
+7 -3
View File
@@ -2,8 +2,11 @@
//!
//! `listClients` returns only `hand_edited = 1` rows. A client the server
//! materialised from live traffic is runtime state, not configuration, and must
//! not appear in an export. `countClients` counts **all** rows, because S5's
//! "has this database ever been configured" predicate needs the true count.
//! not appear in an export. `hand_edited` is the only marker of operator intent
//! in this table, so it also decides what `import.isEmpty` counts: a database
//! carrying nothing but materialised rows has never been configured, and a seed
//! file must still be able to fill it. `countClients` counts **all** rows and is
//! a test helper — it deliberately does not answer that question.
//!
//! The import path is list / insert / deleteAll / count, plus the two runtime
//! calls `upsertSeen` and `pruneStale` that the Phase 7 client tracker owns.
@@ -134,7 +137,8 @@ pub fn deleteAllClients(database: *db.Db) db.Error!void {
return database.exec("DELETE FROM clients;");
}
/// Counts every row, including the ones `listClients` filters out.
/// Counts every row, including the materialised ones `listClients` filters out.
/// Used by tests; `import.isEmpty` counts operator intent instead.
pub fn countClients(database: *db.Db) db.Error!i64 {
return database.queryInt("SELECT count(*) FROM clients");
}
src/storage/repositories/context.zig
+84 -2
View File
@@ -10,6 +10,8 @@
const std = @import("std");
const safe_url = @import("../../safe_url.zig");
const log = std.log.scoped(.repositories);
/// Group name → `groups.id`, or blocklist source URL → `blocklist_sources.id`.
@@ -17,6 +19,42 @@ pub const IdMap = std.StringHashMapUnmanaged(i64);
const no_ids: IdMap = .empty;
/// The line either lookup writes when the caller's map lacks a name, as a value
/// rather than a format string at each call site.
///
/// Two reasons, in that order. A blocklist source url is operator-supplied and
/// nothing on the way in stops it carrying a credential in its userinfo, its
/// path or its query, so it reaches the log through `safe_url.redact` and
/// through nothing else. And a `std.log` line is not readable from a unit test
/// under the default test runner, which installs its own `std_options`; the
/// tests below read this value instead of stderr.
const MissingId = union(enum) {
/// A group name, out of the configuration file or a `groups` row.
group: []const u8,
/// A blocklist source url, out of the configuration file or a
/// `blocklist_sources` row.
source: []const u8,
pub fn format(self: MissingId, w: *std.Io.Writer) std.Io.Writer.Error!void {
switch (self) {
// A group name holds no credential, so nothing is dropped from it.
// It goes through `quoteText` for what that does to any
// operator-supplied string: it delimits a name with a space in it,
// escapes a `'` that would otherwise close the delimiter, and bounds
// a name of any length. The quotes are the type's own, so this
// format string adds none.
.group => |name| try w.print("no group id for {f}", .{safe_url.quoteText(name)}),
// Redaction costs this line the component that told two sources on
// one host apart, and there is no row id to name the source by
// instead: the id the other call sites print is the one this call
// failed to find. The scheme, the host and the port are what is
// left. The rule against writing a secret to a log does not bend for
// a line that would read better with one.
.source => |url| try w.print("no blocklist source id for {f}", .{safe_url.redact(url)}),
}
}
};
pub const InsertContext = struct {
/// Unix epoch seconds, from `std.Io.Clock.real.now(io).toSeconds()`.
now: i64 = 0,
@@ -29,14 +67,16 @@ pub const InsertContext = struct {
/// `NotFound` is a member of `db.Error`, so it needs no wider error set.
pub fn groupId(self: InsertContext, name: []const u8) error{NotFound}!i64 {
return self.group_ids.get(name) orelse {
log.warn("no group id for '{s}'", .{name});
log.warn("{f}", .{MissingId{ .group = name }});
return error.NotFound;
};
}
/// `error.NotFound` means what it means in `groupId`, for the map keyed by
/// source url.
pub fn sourceId(self: InsertContext, url: []const u8) error{NotFound}!i64 {
return self.source_ids.get(url) orelse {
log.warn("no blocklist source id for '{s}'", .{url});
log.warn("{f}", .{MissingId{ .source = url }});
return error.NotFound;
};
}
@@ -44,6 +84,48 @@ pub const InsertContext = struct {
const testing = std.testing;
fn expectLine(expected: []const u8, missing: MissingId) !void {
var buf: [8 * safe_url.max_len]u8 = undefined;
try testing.expectEqualStrings(expected, try std.fmt.bufPrint(&buf, "{f}", .{missing}));
}
test "a missing source id names where the url points and not what it carries" {
// The three components a blocklist url carries a credential in, on the one
// line that used to print all three: an api key in the query, userinfo, and
// a token in a path segment.
try expectLine(
"no blocklist source id for https://lists.example",
.{ .source = "https://lists.example/hosts.txt?apikey=s3cr3t" },
);
try expectLine(
"no blocklist source id for https://lists.example:8443",
.{ .source = "https://user:pa55@lists.example:8443/hosts.txt" },
);
try expectLine(
"no blocklist source id for https://lists.example",
.{ .source = "https://lists.example/download/token/hunter2/hosts.txt" },
);
// What an operator still gets: the scheme, the host and the port.
try expectLine(
"no blocklist source id for http://10.0.0.2:8080",
.{ .source = "http://10.0.0.2:8080/a/hosts.txt" },
);
}
test "a missing group id quotes, escapes and bounds the name" {
try expectLine("no group id for 'kids'", .{ .group = "kids" });
// A name is database text as well as file text, so a newline in it would end
// this line and start one of the operator's choosing.
try expectLine(
"no group id for 'ads\\n2026-01-01 ERROR forged'",
.{ .group = "ads\n2026-01-01 ERROR forged" },
);
// Nor can a name close the quote around it.
try expectLine("no group id for 'kids\\' --'", .{ .group = "kids' --" });
const long_name = "n" ** (2 * safe_url.max_len);
try expectLine("no group id for '" ++ long_name[0..safe_url.max_len] ++ "...'", .{ .group = long_name });
}
test "an InsertContext with no maps reports a missing id rather than trapping" {
const ctx: InsertContext = .{};
try testing.expectError(error.NotFound, ctx.groupId("default"));
src/tests.zig
+2
View File
@@ -4,6 +4,7 @@ comptime {
_ = @import("main.zig");
_ = @import("app.zig");
_ = @import("version.zig");
_ = @import("safe_url.zig");
_ = @import("dns/types.zig");
_ = @import("dns/header.zig");
_ = @import("dns/name.zig");
@@ -31,6 +32,7 @@ comptime {
_ = @import("storage/db.zig");
_ = @import("config/model.zig");
_ = @import("config/validate.zig");
_ = @import("config/faults.zig");
_ = @import("storage/config_schema.zig");
_ = @import("storage/migrations.zig");
_ = @import("storage/querylog_schema.zig");
src/upstream/dot_client.zig
+155 -18
View File
@@ -16,11 +16,59 @@ const net = std.Io.net;
const tls = std.crypto.tls;
const Certificate = std.crypto.Certificate;
const safe_url = @import("../safe_url.zig");
const transport = @import("transport.zig");
const tls_client = @import("../platform/tls_client.zig");
const log = std.log.scoped(.dot_client);
/// One diagnostic line about one client, as a value rather than a format string
/// repeated at each call site.
///
/// Two reasons, in that order. The url is redacted in exactly one place, so a
/// line added later cannot print it whole — the defect this type closes was four
/// call sites each formatting `endpoint.url` with `{s}`, missed by three review
/// rounds because each looked like the three beside it. And a `std.log` line is
/// not readable from a unit test under the default test runner, which installs
/// its own `std_options`; the tests below read this value instead of stderr.
const Diagnostic = struct {
endpoint: transport.Endpoint,
detail: Detail,
const Detail = union(enum) {
/// `resolveAddress` refused the host.
not_an_ip_literal,
connect_failed: anyerror,
handshake_failed: Handshake,
bundle_load_failed: anyerror,
const Handshake = struct {
verify_name: []const u8,
cause: anyerror,
};
};
pub fn format(self: Diagnostic, w: *std.Io.Writer) std.Io.Writer.Error!void {
try w.print("dot upstream {f}: ", .{safe_url.redactQuoted(self.endpoint.url)});
switch (self.detail) {
// The redacted url ends in the host and the port, so naming the host
// again would add nothing but an unredacted copy of it.
.not_an_ip_literal => try w.writeAll("host is not an IP literal"),
.connect_failed => |err| try w.print("connect failed: {s}", .{@errorName(err)}),
// `verify_name` is a host name rather than a url, so it carries no
// component redaction could drop. It goes through `quoteText` for
// what that does to any operator-supplied string: it delimits it,
// escapes it and bounds it.
.handshake_failed => |hs| try w.print("TLS handshake as {f} failed: {s} ({t})", .{
safe_url.quoteText(hs.verify_name),
@errorName(hs.cause),
tls_client.classify(hs.cause),
}),
.bundle_load_failed => |err| try w.print("CA bundle load failed: {s}", .{@errorName(err)}),
}
}
};
pub const ResolveError = error{ConnectFailed};
/// DoT endpoints take IP literals. Name resolution for upstreams is out of
@@ -89,6 +137,10 @@ pub const DotClient = struct {
return .{ .ptr = self, .exchangeFn = exchangeFn };
}
fn diagnose(self: *const DotClient, detail: Diagnostic.Detail) Diagnostic {
return .{ .endpoint = self.endpoint, .detail = detail };
}
fn exchangeFn(
ptr: *anyopaque,
io: std.Io,
@@ -119,20 +171,14 @@ pub const DotClient = struct {
if (query.len > transport.max_message_len) return error.BufferTooSmall;
const address = resolveAddress(self.endpoint) catch |err| {
log.warn("dot upstream {s}: host \"{s}\" is not an IP literal", .{
self.endpoint.url,
self.endpoint.host,
});
log.warn("{f}", .{self.diagnose(.not_an_ip_literal)});
return err;
};
try self.ensureBundle(io);
var stream = address.connect(io, .{ .mode = .stream }) catch |err| {
log.debug("dot upstream {s}: connect failed: {s}", .{
self.endpoint.url,
@errorName(err),
});
log.debug("{f}", .{self.diagnose(.{ .connect_failed = err })});
return mapPhase(err, error.ConnectFailed);
};
defer closeStream(io, &stream);
@@ -155,12 +201,10 @@ pub const DotClient = struct {
.stream_write_buffer = self.buffers.stream_write,
}) catch |err| {
const cause = concreteHandshake(&tls_stream, err);
log.warn("dot upstream {s}: TLS handshake as \"{s}\" failed: {s} ({t})", .{
self.endpoint.url,
self.verify_name,
@errorName(cause),
tls_client.classify(cause),
});
log.warn("{f}", .{self.diagnose(.{ .handshake_failed = .{
.verify_name = self.verify_name,
.cause = cause,
} })});
return mapPhase(cause, error.TlsFailed);
};
defer closeTls(io, &tls_stream);
@@ -214,10 +258,7 @@ pub const DotClient = struct {
self.bundle.rescan(self.gpa, io, std.Io.Clock.real.now(io)) catch |err| {
self.bundle.deinit(self.gpa);
self.bundle.* = .empty;
log.warn("dot upstream {s}: CA bundle load failed: {s}", .{
self.endpoint.url,
@errorName(err),
});
log.warn("{f}", .{self.diagnose(.{ .bundle_load_failed = err })});
return mapPhase(err, error.TlsFailed);
};
}
@@ -286,6 +327,102 @@ fn receiveFailure(stream: *tls_client.TlsStream, err: anyerror) transport.Exchan
const testing = std.testing;
fn expectDiagnostic(expected: []const u8, url: []const u8, detail: Diagnostic.Detail) !void {
var buf: [8 * safe_url.max_len]u8 = undefined;
const line = try std.fmt.bufPrint(&buf, "{f}", .{Diagnostic{
.endpoint = .{ .scheme = .dot, .url = url, .host = "host.example", .port = 853, .path = "/" },
.detail = detail,
}});
try testing.expectEqualStrings(expected, line);
}
test "every diagnostic line redacts the url it names the upstream by" {
// The endpoints are built by hand rather than parsed, on purpose.
// `Endpoint.parse` refuses `@`, `?` and `#` in the authority and refuses a
// `.dot` path other than "/", so no url reaching this client through it can
// carry a credential in a component `redact` drops. That is a property of a
// parser one file away, not of this file, and these lines used to print
// whatever `endpoint.url` held. The redaction is what keeps the parser's
// rules from being load-bearing here.
try expectDiagnostic(
"dot upstream 'tls://dns.example': host is not an IP literal",
"tls://user:hunter2@dns.example/abcd12",
.not_an_ip_literal,
);
try expectDiagnostic(
"dot upstream 'tls://dns.example:8853': connect failed: ConnectionRefused",
"tls://dns.example:8853/abcd12",
.{ .connect_failed = error.ConnectionRefused },
);
try expectDiagnostic(
"dot upstream 'tls://dns.example': CA bundle load failed: FileNotFound",
"tls://dns.example/abcd12?apikey=s3cr3t",
.{ .bundle_load_failed = error.FileNotFound },
);
try expectDiagnostic(
"dot upstream 'tls://dns.example': TLS handshake as 'one.one.one.one' failed: " ++
"CertificateHostMismatch (certificate)",
"tls://token@dns.example/abcd12",
.{ .handshake_failed = .{
.verify_name = "one.one.one.one",
.cause = error.CertificateHostMismatch,
} },
);
}
test "a diagnostic escapes and bounds the operator-supplied text it prints" {
// A control byte in either field would forge a second record on the one
// output `std.log`'s own sink does not reach, and an unbounded host or
// verification name would write an unbounded line.
try expectDiagnostic(
"dot upstream 'tls://dns.example\\n2026-01-01 ERROR forged': host is not an IP literal",
"tls://dns.example\n2026-01-01 ERROR forged",
.not_an_ip_literal,
);
try expectDiagnostic(
"dot upstream 'tls://dns.example': TLS handshake as 'a\\nb' failed: TlsAlert (handshake)",
"tls://dns.example",
.{ .handshake_failed = .{ .verify_name = "a\nb", .cause = error.TlsAlert } },
);
// Doubling every operator-supplied field writes the same line, and each
// field is built from the three byte costs at once: a printable byte spends
// one character of the budget, `\n` spends two and `\x00` spends four. That
// expansion is why `safe_url.max_len` counts printed characters rather than
// source bytes, so the bound holds against the widest escape rather than in
// spite of it.
const long = "h\n\x00" ** (2 * safe_url.max_len);
const longer = long ** 2;
var short_buf: [16 * safe_url.max_len]u8 = undefined;
var long_buf: [16 * safe_url.max_len]u8 = undefined;
try testing.expectEqualStrings(
try std.fmt.bufPrint(&short_buf, "{f}", .{Diagnostic{
.endpoint = .{ .scheme = .dot, .url = "tls://" ++ long, .host = long, .port = 853, .path = "/" },
.detail = .{ .handshake_failed = .{ .verify_name = long, .cause = error.TlsAlert } },
}}),
try std.fmt.bufPrint(&long_buf, "{f}", .{Diagnostic{
.endpoint = .{ .scheme = .dot, .url = "tls://" ++ longer, .host = longer, .port = 853, .path = "/" },
.detail = .{ .handshake_failed = .{ .verify_name = longer, .cause = error.TlsAlert } },
}}),
);
}
test "a diagnostic keeps what a parsed DoT url carries" {
// The limit, pinned so it stays visible: a NextDNS DoT upstream is
// `tls://abcd12.dns.nextdns.io`, whose hostname is the whole account
// identifier. Redaction cannot remove it without leaving no host and an
// unactionable line. See `safe_url.SafeUrl`.
var buf: [8 * safe_url.max_len]u8 = undefined;
const line = try std.fmt.bufPrint(&buf, "{f}", .{Diagnostic{
.endpoint = try .parse("tls://abcd12.dns.nextdns.io"),
.detail = .not_an_ip_literal,
}});
try testing.expectEqualStrings(
"dot upstream 'tls://abcd12.dns.nextdns.io': host is not an IP literal",
line,
);
}
test "resolveAddress accepts IP literals" {
const v4 = try resolveAddress(try .parse("tls://1.1.1.1:853"));
try testing.expectEqual(@as(u16, 853), v4.ip4.port);
src/upstream/pool.zig
+60 -1
View File
@@ -40,10 +40,32 @@
const std = @import("std");
const health = @import("health.zig");
const safe_url = @import("../safe_url.zig");
const transport = @import("transport.zig");
const log = std.log.scoped(.upstream);
/// The one line `exchange` writes about a failed attempt, as a value.
///
/// It is a value rather than a format string at the call site for the same
/// reason `dot_client.Diagnostic` is: a `std.log` line is not readable from a
/// unit test under the default test runner, so the test below reads this
/// instead of stderr. An upstream url is operator-supplied and a DoH one carries
/// its credential in the path — `https://dns.nextdns.io/abcd12` is a whole
/// NextDNS account identifier — so it reaches the line through
/// `safe_url.redactQuoted`. Quoted rather than bare because the error name
/// follows it: a redacted authority may still hold a space and a `:`, so
/// unquoted, a url ending `ok failed: Timeout` would report a failure that did
/// not happen.
const AttemptFailure = struct {
endpoint: transport.Endpoint,
err: transport.ExchangeError,
pub fn format(self: AttemptFailure, w: *std.Io.Writer) std.Io.Writer.Error!void {
try w.print("upstream {f} failed: {t}", .{ safe_url.redactQuoted(self.endpoint.url), self.err });
}
};
pub const Entry = struct {
endpoint: transport.Endpoint,
client: transport.Client,
@@ -61,6 +83,11 @@ pub const Entry = struct {
/// A copy of one entry's health, taken under the mutex. Feeds
/// `GET /api/upstream/health` in Phase 8.
pub const Snapshot = struct {
/// Whole, not redacted. `GET /api/upstream/health` returns this to a session
/// that `GET /api/upstreams` already serves the same url to in full, so
/// redacting here would hide nothing from that reader and would make two
/// responses of one API disagree. A consumer reachable without a session has
/// to redact it itself.
url: []const u8,
enabled: bool,
available: bool,
@@ -165,7 +192,7 @@ pub const Pool = struct {
const response = result catch |err| switch (transport.group(err)) {
.peer_fault => {
log.debug("upstream {s} failed: {t}", .{ entry.endpoint.url, err });
log.debug("{f}", .{AttemptFailure{ .endpoint = entry.endpoint, .err = err }});
self.recordFailure(io, entry, completed_at, err);
last_fault = err;
continue;
@@ -385,6 +412,38 @@ const test_cfg: health.Config = .{
const test_timeout: std.Io.Clock.Duration = .{ .raw = .fromSeconds(10), .clock = .awake };
fn expectFailureLine(expected: []const u8, url: []const u8, err: transport.ExchangeError) !void {
var buf: [8 * safe_url.max_len]u8 = undefined;
const line = try std.fmt.bufPrint(&buf, "{f}", .{AttemptFailure{
.endpoint = try .parse(url),
.err = err,
}});
try testing.expectEqualStrings(expected, line);
}
test "the failed-attempt line names an upstream by a url carrying no credential" {
// A NextDNS DoH upstream puts the whole account identifier in the path, and
// this line ran at `debug` on every peer fault, so a debug-level operator
// persisted it to the journal once per failure.
try expectFailureLine(
"upstream 'https://dns.nextdns.io' failed: Timeout",
"https://dns.nextdns.io/abcd12",
error.Timeout,
);
try expectFailureLine(
"upstream 'https://cdn.example:8443' failed: TlsFailed",
"https://cdn.example:8443/d/hunter2/dns-query",
error.TlsFailed,
);
// What it still says, because an operator reading a failover has to know
// which upstream failed: the scheme, the host and the port.
try expectFailureLine(
"upstream 'tls://9.9.9.9:853' failed: ConnectFailed",
"tls://9.9.9.9:853",
error.ConnectFailed,
);
}
test "Pool satisfies the Client interface" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
src/web/handlers/blocklists.zig
+25 -1
View File
@@ -9,6 +9,10 @@
//! refresh downloads and compiles before the response is written: 202 is
//! "accepted and done as far as this connection is concerned", and the status
//! table in the body is what tells the operator which sources actually landed.
//!
//! `DELETE /api/blocklists/{id}` removes the row, reloads, and then sweeps the
//! compiled files that row named, so `<data_dir>/blocklists/` follows the table
//! the operator is looking at rather than the scheduler's next pass.
const std = @import("std");
const Allocator = std.mem.Allocator;
@@ -130,7 +134,27 @@ pub fn applyDelete(state: *server.WebState, io: std.Io, id: i64) ?Failure {
state.config_lock.unlock(io);
written catch |err| return mutations.dbFailure(err, url_conflict);
return mutations.reload(state, io);
const failure = mutations.reload(state, io);
pruneFiles(state, io);
return failure;
}
/// Removes the compiled files the deleted row leaves behind.
///
/// This is the moment an orphan is made during normal operation, and the only
/// other sweep is the scheduler's — up to `blocklist_update.interval_hours`
/// away. Without this call a deleted list keeps its megabytes on disk for a day.
///
/// After the reload and never part of the response: the row is gone and the
/// snapshot has stopped enforcing the list, so bytes still on disk are not a
/// failed delete. `Manager.pruneOrphans` takes the manager's writer lock, which
/// the reload above has already taken and released — nothing here holds it, and
/// `state.config_lock` was released before either.
fn pruneFiles(state: *server.WebState, io: std.Io) void {
const manager = state.manager orelse return;
manager.pruneOrphans(io) catch |err| {
log.warn("pruning the deleted blocklist's files failed: {s}", .{@errorName(err)});
};
}
/// Refreshes every enabled source, then applies the result (ruling 12).
src/web/handlers/mutations.zig
+6 -3
View File
@@ -184,8 +184,12 @@ fn rebuildFailed(what: []const u8, cause: []const u8) Failure {
const skeleton_group = "default";
const skeleton_upstream: model.UpstreamServer = .{ .url = "https://dns.example/dns-query" };
/// Runs the shipped validator over `cfg` and returns the first problem's text,
/// Runs the shipped validator over `cfg` and returns the first failure's text,
/// or null when the candidate is valid. The text is arena-allocated.
///
/// Failures only: a warning describes a configuration that is legal, and a row
/// this API is about to store cannot be rejected for one. The blocklist source
/// a POST creates is in no group yet — a warning by design, and never a 400.
pub fn firstProblem(arena: Allocator, cfg: model.Config) error{OutOfMemory}!?[]const u8 {
var diags: validate.Diagnostics = .init(arena);
defer diags.deinit();
@@ -194,8 +198,7 @@ pub fn firstProblem(arena: Allocator, cfg: model.Config) error{OutOfMemory}!?[]c
error.OutOfMemory => return error.OutOfMemory,
else => {},
};
if (diags.problems.items.len == 0) return null;
const problem = diags.problems.items[0];
const problem = diags.firstFailure() orelse return null;
return try std.fmt.allocPrint(arena, "{s}: {s}", .{ problem.path, problem.message });
}
src/web/metrics.zig
+386 -23
View File
@@ -2,7 +2,9 @@
//!
//! Two halves, so that neither needs the other to be testable: `collect` walks
//! the live collaborators and copies every number into a `Sample`, and `render`
//! turns a `Sample` into text. Nothing is computed during rendering.
//! turns a `Sample` into text. Nothing is computed during rendering, with one
//! exception: an upstream url is redacted where its label is written rather
//! than where it is copied. `writeUrlLabel` carries the reasoning.
//!
//! Three rules the collection half obeys:
//!
@@ -32,6 +34,7 @@ const logging = @import("../platform/logging.zig");
const pool_mod = @import("../upstream/pool.zig");
const rate_limiter = @import("../server/rate_limiter.zig");
const retention_mod = @import("../storage/retention.zig");
const safe_url = @import("../safe_url.zig");
const server = @import("server.zig");
/// The exposition format version, as the 0.0.4 specification writes it.
@@ -96,6 +99,9 @@ pub const DohListenerSample = struct {
/// One upstream, with every string owned by the caller's arena.
pub const UpstreamSample = struct {
/// The configured url, whole. It reaches the exposition only through
/// `writeUrlLabel`, which redacts it; a reader of this field is reading a
/// credential.
url: []const u8,
enabled: bool,
available: bool,
@@ -377,16 +383,19 @@ fn endpointValue(
fn renderUpstreams(w: *std.Io.Writer, list: []const UpstreamSample) std.Io.Writer.Error!void {
try labeledHead(w, "nxdns_upstream_up", "1 while an upstream is enabled and healthy.", "gauge");
for (list) |entry| try labeledValue(w, "nxdns_upstream_up", entry.url, @intFromBool(entry.available));
for (list, 0..) |entry, i| {
try labeledValue(w, "nxdns_upstream_up", i, entry.url, @intFromBool(entry.available));
}
try labeledHead(w, "nxdns_upstream_enabled", "1 while an upstream is enabled by configuration.", "gauge");
for (list) |entry| try labeledValue(w, "nxdns_upstream_enabled", entry.url, @intFromBool(entry.enabled));
for (list, 0..) |entry, i| {
try labeledValue(w, "nxdns_upstream_enabled", i, entry.url, @intFromBool(entry.enabled));
}
try labeledHead(w, "nxdns_upstream_success_rate", "Share of recent exchanges that succeeded.", "gauge");
for (list) |entry| {
try w.writeAll("nxdns_upstream_success_rate{url=\"");
try writeLabelValue(w, entry.url);
try w.print("\"}} {d:.4}\n", .{entry.success_rate});
for (list, 0..) |entry, i| {
try writeUpstreamLabels(w, "nxdns_upstream_success_rate", i, entry.url);
try w.print(" {d:.4}\n", .{entry.success_rate});
}
try labeledHead(
@@ -395,15 +404,19 @@ fn renderUpstreams(w: *std.Io.Writer, list: []const UpstreamSample) std.Io.Write
"Failures since an upstream last answered.",
"gauge",
);
for (list) |entry| {
try labeledValue(w, "nxdns_upstream_consecutive_failures", entry.url, entry.consecutive_failures);
for (list, 0..) |entry, i| {
try labeledValue(w, "nxdns_upstream_consecutive_failures", i, entry.url, entry.consecutive_failures);
}
try labeledHead(w, "nxdns_upstream_successes_total", "Exchanges an upstream answered.", "counter");
for (list) |entry| try labeledValue(w, "nxdns_upstream_successes_total", entry.url, entry.total_successes);
for (list, 0..) |entry, i| {
try labeledValue(w, "nxdns_upstream_successes_total", i, entry.url, entry.total_successes);
}
try labeledHead(w, "nxdns_upstream_failures_total", "Exchanges an upstream failed.", "counter");
for (list) |entry| try labeledValue(w, "nxdns_upstream_failures_total", entry.url, entry.total_failures);
for (list, 0..) |entry, i| {
try labeledValue(w, "nxdns_upstream_failures_total", i, entry.url, entry.total_failures);
}
}
/// Every field of a plain counter struct, under one prefix.
@@ -438,12 +451,93 @@ fn labeledHead(
fn labeledValue(
w: *std.Io.Writer,
name: []const u8,
index: usize,
url: []const u8,
value: u64,
) std.Io.Writer.Error!void {
try w.print("{s}{{url=\"", .{name});
try writeLabelValue(w, url);
try w.print("\"}} {d}\n", .{value});
try writeUpstreamLabels(w, name, index, url);
try w.print(" {d}\n", .{value});
}
/// The label set every upstream family shares, up to and including the closing
/// brace. One definition, because six families have to agree on it exactly:
/// Prometheus identifies a series by its name and its whole label set, so a
/// family that labelled its samples differently would be a different series.
///
/// `index` is the upstream's position in the pool, in the priority order `Pool`
/// sorts on. It is here because the url alone stopped identifying a series once
/// it was redacted: two upstreams on one host — the shape a NextDNS account with
/// two profiles takes — both print `https://dns.nextdns.io`, and two samples of
/// one name with one label set is a duplicate series a scrape must not contain.
/// The position is read from the rendered slice rather than carried in
/// `UpstreamSample`, so no caller can build two samples that claim one index.
///
/// What the index is not: a durable key, and the difference is an operator's to
/// know. `Pool.Snapshot` carries no row id — threading one out of the repository
/// through the pool to reach here is a larger change than the defect warrants —
/// so the position is all there is. Removing `upstreams[0]` renumbers every
/// upstream after it, and one upstream's history then continues under the label
/// its neighbour used to carry.
///
/// What bounds that: the index is only load-bearing when two upstreams share an
/// origin, which is the case it was added for. Where origins differ, `url`
/// carries the identity on its own and a reorder moves nothing that a query
/// grouping on `url` can see. So group on `url`, and read `index` as the
/// disambiguator between upstreams that group would otherwise merge.
fn writeUpstreamLabels(
w: *std.Io.Writer,
name: []const u8,
index: usize,
url: []const u8,
) std.Io.Writer.Error!void {
try w.print("{s}{{index=\"{d}\",url=\"", .{ name, index });
try writeUrlLabel(w, url);
try w.writeAll("\"}");
}
/// The one place an upstream url becomes exposition text.
///
/// `/metrics` is `.auth = .open` in `web/routes.zig` and `web.bind` defaults to
/// `0.0.0.0`, so a url in a label is readable by anything on the LAN without a
/// session, and a Prometheus that scrapes it keeps that string for as long as it
/// keeps the series. A NextDNS DoH upstream is `https://dns.nextdns.io/abcd12`,
/// where the path segment is the whole account identifier, so the label prints
/// what `safe_url.redact` leaves: the scheme, the host and the port.
///
/// The redaction sits here rather than in `collect` because this is where the
/// open endpoint writes the value. A `Sample` built anywhere else renders
/// through this function too, so the guarantee cannot be one caller away.
/// `UpstreamSample.url` stays whole for the same reason it is safe to: nothing
/// but this function reads it, and the session-authenticated
/// `GET /api/upstream/health` reports the same pool with the same urls whole.
///
/// **`redact` output is not safe to interpolate into a label value, and this
/// function is the reason it never has to be.** Do not delete the second layer
/// on the grounds that the first one escapes.
///
/// What `redact` gives: it escapes every control character and bounds its
/// output, so it emits neither a raw newline nor a lone backslash. What it does
/// not give: it leaves `"` alone. A `"` is legal in the text `redact` keeps, and
/// it is the one character that ends a label value — so a host holding one
/// closes the label and lets the rest of the string write label pairs of its
/// own. That is a defect of this call site, not of `redact`: a `"` needs no
/// escape in a log line, which is what `redact` was written for.
///
/// So `writeLabelValue` runs over the redacted text rather than instead of it,
/// and the order is the whole point. It also doubles a backslash `redact` wrote,
/// which is what keeps `\n` in a host from reading as a newline to a parser: the
/// four bytes `\x1b` arrive at a scrape as `\\x1b`.
///
/// A label value's escapes are not a shell's. `quoteText`, which the log lines
/// use, answers a different question — its delimiter is `'` and it writes its
/// own quotes — and it is not the tool here.
fn writeUrlLabel(w: *std.Io.Writer, url: []const u8) std.Io.Writer.Error!void {
// `SafeUrl.format` prints at most `max_len` characters, plus the `...` that
// marks a truncation. The buffer is that bound, so the write cannot fail.
var buf: [safe_url.max_len + 3]u8 = undefined;
var redacted: std.Io.Writer = .fixed(&buf);
try redacted.print("{f}", .{safe_url.redact(url)});
try writeLabelValue(w, redacted.buffered());
}
/// The three characters the exposition format reserves inside a label value.
@@ -552,22 +646,24 @@ test "a full sample renders the whole exposition, byte for byte" {
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_blocklist_generation 4\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_disk_free_bytes 1000\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_disk_sample_failures_total 1\n"));
// The label set carries the redaction, so the path of the configured url is
// already gone from the golden text.
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_up{url=\"https://dns.example/dns-query\"} 1\n",
"nxdns_upstream_up{index=\"0\",url=\"https://dns.example\"} 1\n",
));
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_success_rate{url=\"https://dns.example/dns-query\"} 0.9000\n",
"nxdns_upstream_success_rate{index=\"0\",url=\"https://dns.example\"} 0.9000\n",
));
try testing.expect(std.mem.endsWith(
u8,
text,
"nxdns_upstream_failures_total{url=\"https://dns.example/dns-query\"} 1\n",
"nxdns_upstream_failures_total{index=\"0\",url=\"https://dns.example\"} 1\n",
));
}
@@ -671,15 +767,281 @@ test "cert reload counters render per endpoint, only for the wired stores" {
try testing.expect(std.mem.containsAtLeast(u8, both, 1, "nxdns_cert_reload_failures_total{endpoint=\"dot\"} 0\n"));
}
/// Reads a sample line's label set the way a scrape does and returns the number
/// of label pairs, or null if the line is not one a parser accepts.
///
/// A checker rather than a golden string, because the input that exercises it is
/// a url no parser accepts, and `safe_url.redact` is entitled to change what it
/// prints for one of those. What may not change is the property: an
/// operator-supplied byte must not close a label value early, end the line, or
/// leave an escape sequence behind that means something else to a parser. Only
/// the format's three escapes are accepted for that reason — a `\x1b` `redact`
/// wrote reaches here as `\\x1b`, whose backslash is escaped and whose `x1b` is
/// three ordinary characters.
fn labelPairs(line: []const u8) ?usize {
var i = (std.mem.indexOfScalar(u8, line, '{') orelse return null) + 1;
var pairs: usize = 0;
while (true) {
const eq = std.mem.indexOfScalarPos(u8, line, i, '=') orelse return null;
if (eq == i) return null;
for (line[i..eq]) |c| if (!std.ascii.isAlphanumeric(c) and c != '_') return null;
if (eq + 1 >= line.len or line[eq + 1] != '"') return null;
i = eq + 2;
while (true) {
if (i >= line.len) return null;
if (line[i] == '"') break;
if (line[i] != '\\') {
i += 1;
continue;
}
if (i + 1 >= line.len) return null;
switch (line[i + 1]) {
'\\', '"', 'n' => i += 2,
else => return null,
}
}
pairs += 1;
i += 1;
if (i >= line.len) return null;
if (line[i] == '}') return pairs;
if (line[i] != ',') return null;
i += 1;
}
}
/// `name{labels}` — what Prometheus identifies a series by. Null for a sample
/// that carries no label set.
fn seriesKey(line: []const u8) ?[]const u8 {
const close = std.mem.lastIndexOfScalar(u8, line, '}') orelse return null;
return line[0 .. close + 1];
}
test "a label value escapes the characters the format reserves" {
// Every shape an operator-supplied url can take that reaches the label with
// a character the format reserves. The assertion is the property, not the
// text: these are urls no parser accepts, and `safe_url.redact` may change
// what it prints for one of them without changing what this test protects.
const hostile = [_][]const u8{
"https://a\"b/dns-query",
"https://a\nb/dns-query",
"https://a\\b/dns-query",
"https://a\x1bb/dns-query",
"https://user:pa55@h\"ost/dns-query",
"https://\"}{=,\"/dns-query",
// The shape `safe_url.redact` is being hardened against in this same
// wave: a `?` before the last `@`. What it prints is that fix's to
// decide; that the label holds it safely is this one's.
"https://lists.example?token=prefix@hunter2",
};
for (hostile) |url| {
const upstream_list = [_]UpstreamSample{.{
.url = url,
.enabled = true,
.available = false,
.consecutive_failures = 2,
.total_successes = 0,
.total_failures = 2,
.success_rate = 0,
}};
const text = try renderToString(testing.allocator, .{ .upstreams = &upstream_list });
defer testing.allocator.free(text);
// The url wrote no line of its own, and lost none: every line is a
// comment or a sample, and the six families contribute six samples.
var samples: usize = 0;
var lines = std.mem.splitScalar(u8, text, '\n');
while (lines.next()) |line| {
if (line.len == 0 or std.mem.startsWith(u8, line, "# ")) continue;
try testing.expect(std.mem.startsWith(u8, line, "nxdns_"));
if (!std.mem.startsWith(u8, line, "nxdns_upstream_")) continue;
samples += 1;
// Both labels are there, and both values close where they opened.
try testing.expectEqual(@as(?usize, 2), labelPairs(line));
}
try testing.expectEqual(@as(usize, 6), samples);
}
}
test "two upstreams on one host stay two series" {
// Redaction costs the url the job of telling two upstreams apart: a NextDNS
// account with two profiles is two urls on one host, and both print
// `https://dns.nextdns.io`. Two samples of one name with one label set is a
// duplicate series, which is a broken scrape rather than a hidden one.
const upstream_list = [_]UpstreamSample{
.{
.url = "https://dns.nextdns.io/abcd12",
.enabled = true,
.available = true,
.consecutive_failures = 0,
.total_successes = 5,
.total_failures = 0,
.success_rate = 1,
},
.{
.url = "https://dns.nextdns.io/efgh34",
.enabled = true,
.available = false,
.consecutive_failures = 3,
.total_successes = 9,
.total_failures = 3,
.success_rate = 0.75,
},
};
const text = try renderToString(testing.allocator, .{ .upstreams = &upstream_list });
defer testing.allocator.free(text);
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_up{index=\"0\",url=\"https://dns.nextdns.io\"} 1\n",
));
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_up{index=\"1\",url=\"https://dns.nextdns.io\"} 0\n",
));
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_successes_total{index=\"1\",url=\"https://dns.nextdns.io\"} 9\n",
));
// Distinguishable, not merely present. `nxdns_upstream_up` is the family
// this matters most in: one of these two upstreams is down and the other is
// up, and a reader has to be able to see which. Under one shared label set
// the two samples say 1 and 0 of the same series, so a scrape either reports
// whichever it read last or rejects the pair — and the down upstream is
// invisible either way, on the endpoint an operator watches to find out.
var up_keys: [2][]const u8 = undefined;
var up_values: [2][]const u8 = undefined;
var found: usize = 0;
var up_lines = std.mem.splitScalar(u8, text, '\n');
while (up_lines.next()) |line| {
if (!std.mem.startsWith(u8, line, "nxdns_upstream_up{")) continue;
try testing.expect(found < up_keys.len);
const key = seriesKey(line).?;
up_keys[found] = key;
up_values[found] = line[key.len + 1 ..];
found += 1;
}
try testing.expectEqual(@as(usize, 2), found);
try testing.expect(!std.mem.eql(u8, up_keys[0], up_keys[1]));
try testing.expectEqualStrings("1", up_values[0]);
try testing.expectEqualStrings("0", up_values[1]);
// No two samples in the scrape share a series key, whatever the urls were.
var keys: [32][]const u8 = undefined;
var count: usize = 0;
var lines = std.mem.splitScalar(u8, text, '\n');
while (lines.next()) |line| {
if (line.len == 0 or std.mem.startsWith(u8, line, "# ")) continue;
const key = seriesKey(line) orelse continue;
for (keys[0..count]) |seen| try testing.expect(!std.mem.eql(u8, seen, key));
keys[count] = key;
count += 1;
}
try testing.expectEqual(@as(usize, 12), count);
}
test "an upstream url is redacted before it reaches an open endpoint's label" {
// `/metrics` is `.auth = .open`, so every label here is readable without a
// session by anything that can reach the bind address. A NextDNS DoH
// upstream carries the whole account identifier in its path, and a scraper
// keeps a label for as long as it keeps the series.
const upstream_list = [_]UpstreamSample{
.{
.url = "https://dns.nextdns.io/abcd12",
.enabled = true,
.available = true,
.consecutive_failures = 0,
.total_successes = 3,
.total_failures = 0,
.success_rate = 1,
},
.{
.url = "https://user:hunter2@dns.example:8443/dns-query?apikey=s3cr3t#frag",
.enabled = false,
.available = false,
.consecutive_failures = 4,
.total_successes = 0,
.total_failures = 4,
.success_rate = 0,
},
};
const text = try renderToString(testing.allocator, .{ .upstreams = &upstream_list });
defer testing.allocator.free(text);
// The four components a credential can live in, none of them exposed.
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "abcd12"));
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "hunter2"));
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "s3cr3t"));
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "frag"));
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "dns-query"));
// Every family carries the label, so none of the six may keep the whole url.
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_up{index=\"0\",url=\"https://dns.nextdns.io\"} 1\n",
));
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_enabled{index=\"0\",url=\"https://dns.nextdns.io\"} 1\n",
));
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_success_rate{index=\"0\",url=\"https://dns.nextdns.io\"} 1.0000\n",
));
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_successes_total{index=\"0\",url=\"https://dns.nextdns.io\"} 3\n",
));
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_failures_total{index=\"0\",url=\"https://dns.nextdns.io\"} 0\n",
));
// The scheme, the host and the port stay: an operator reading a scrape has
// to know which upstream a series is about.
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_enabled{index=\"1\",url=\"https://dns.example:8443\"} 0\n",
));
try testing.expect(std.mem.containsAtLeast(
u8,
text,
1,
"nxdns_upstream_consecutive_failures{index=\"1\",url=\"https://dns.example:8443\"} 4\n",
));
}
test "a url longer than the redaction bound cannot run past it" {
const long_host = "h" ** (4 * safe_url.max_len);
const upstream_list = [_]UpstreamSample{.{
.url = "https://dns.example/a\"b\\c",
.url = "https://" ++ long_host ++ "/dns-query",
.enabled = true,
.available = false,
.consecutive_failures = 2,
.available = true,
.consecutive_failures = 0,
.total_successes = 0,
.total_failures = 2,
.success_rate = 0,
.total_failures = 0,
.success_rate = 1,
}};
const text = try renderToString(testing.allocator, .{ .upstreams = &upstream_list });
defer testing.allocator.free(text);
@@ -688,7 +1050,8 @@ test "a label value escapes the characters the format reserves" {
u8,
text,
1,
"nxdns_upstream_up{url=\"https://dns.example/a\\\"b\\\\c\"} 0\n",
"nxdns_upstream_up{index=\"0\",url=\"" ++
("https://" ++ long_host)[0..safe_url.max_len] ++ "...\"} 1\n",
));
}
src/web/web_integration_test.zig
+87
View File
@@ -1129,6 +1129,93 @@ test "W10 a rule mutation reloads the snapshot and the change is live" {
try bounded(env.io(), default_budget, mutationReloads, .{ env.io(), env });
}
// ---------------------------------------------------------------------------
// deleting a source takes its compiled files with it (m13 ruling F-f)
// ---------------------------------------------------------------------------
/// The id in a `201 Created` body from `/api/blocklists`.
fn createdId(body: []const u8) !i64 {
const marker = "\"id\":";
const at = std.mem.indexOf(u8, body, marker) orelse return error.TestNoId;
const rest = body[at + marker.len ..];
const end = std.mem.indexOfNone(u8, rest, "0123456789") orelse rest.len;
return std.fmt.parseInt(i64, rest[0..end], 10);
}
fn writeCompiled(io: std.Io, dir: std.Io.Dir, id: i64, body: []const u8) !void {
var buf: [64]u8 = undefined;
try dir.writeFile(io, .{
.sub_path = try std.fmt.bufPrint(&buf, "{d}.list", .{id}),
.data = body,
});
try dir.writeFile(io, .{
.sub_path = try std.fmt.bufPrint(&buf, "{d}.wild", .{id}),
.data = "",
});
}
fn accessCompiled(io: std.Io, dir: std.Io.Dir, id: i64) !void {
var buf: [64]u8 = undefined;
return dir.access(io, try std.fmt.bufPrint(&buf, "{d}.list", .{id}), .{});
}
fn deleteSweepsCompiledFiles(io: std.Io, env: *Env) anyerror!void {
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [4096]u8 = undefined;
try conn.request("POST", "/api/blocklists", null, "{\"url\":\"https://doomed.test/a.txt\",\"name\":\"doomed\"}");
var response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 201), response.status);
const doomed = try createdId(response.body);
try conn.request("POST", "/api/blocklists", null, "{\"url\":\"https://kept.test/b.txt\",\"name\":\"kept\"}");
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 201), response.status);
const kept = try createdId(response.body);
// The files a refresh would have produced for each row. Neither row carries
// a checksum, so the reload the delete runs treats both as never fetched
// and reads neither — this case is about the directory, not the snapshot.
_ = try env.tmp.dir.createDirPathStatus(io, "blocklists", .fromMode(0o700));
var dir = try env.tmp.dir.openDir(io, "blocklists", .{ .iterate = true });
defer dir.close(io);
try writeCompiled(io, dir, doomed, "doomed.example\n");
try writeCompiled(io, dir, kept, "kept.example\n");
var target_buf: [64]u8 = undefined;
const target = try std.fmt.bufPrint(&target_buf, "/api/blocklists/{d}", .{doomed});
try conn.request("DELETE", target, null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 204), response.status);
// The row is gone, so its files are orphans; without a sweep on this path
// they would sit here until a restart or the scheduler's next pass.
var name_buf: [64]u8 = undefined;
try testing.expectError(error.FileNotFound, dir.access(
io,
try std.fmt.bufPrint(&name_buf, "{d}.list", .{doomed}),
.{},
));
try testing.expectError(error.FileNotFound, dir.access(
io,
try std.fmt.bufPrint(&name_buf, "{d}.wild", .{doomed}),
.{},
));
try accessCompiled(io, dir, kept);
}
test "W10 deleting a blocklist deletes its compiled files and spares the others" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{});
defer env.destroy();
try bounded(env.io(), default_budget, deleteSweepsCompiledFiles, .{ env.io(), env });
}
// ---------------------------------------------------------------------------
// pause via the API changes a real handler decision (ruling 15)
// ---------------------------------------------------------------------------