mokhtar/nxdns
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

milestone 13 discrepancies: redact credentials from urls in logs, metrics and cli output

Browse Source
This commit is contained in:
mokhtar
2026-08-07 00:45:17 +02:00
parent 1ff727feb8
commit 8c3328562e
39 changed files with 5734 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app.zig
+358 -37
View File
@@ -44,6 +44,7 @@ const doh_client = @import("upstream/doh_client.zig");
const doh_server = @import("server/doh_server.zig");
const dot_client = @import("upstream/dot_client.zig");
const dot_server = @import("server/dot_server.zig");
const faults = @import("config/faults.zig");
const fetcher = @import("filter/fetcher.zig");
const forward_zones = @import("local/forward_zones.zig");
const handler = @import("server/handler.zig");
@@ -60,6 +61,7 @@ const pool_mod = @import("upstream/pool.zig");
const query_sink = @import("server/query_sink.zig");
const rate_limiter = @import("server/rate_limiter.zig");
const retention_mod = @import("storage/retention.zig");
const safe_url = @import("safe_url.zig");
const shutdown = @import("server/shutdown.zig");
const sse = @import("web/sse.zig");
const static = @import("web/static.zig");
@@ -87,24 +89,14 @@ const download_budget_s = 300;
const doh_request_buf_len = 1024;
const doh_transfer_buf_len = 4096;
/// A configuration fault the operator can fix, as opposed to a runtime one.
/// These are the only failures this file raises itself; everything else comes
/// out of a collaborator.
const ConfigError = error{
NoUsableUpstreams,
BadBindAddress,
BadRateLimit,
BadCertificate,
};
pub fn run(runner: cli.Runner, args: cli.RunArgs) u8 {
const code = serve(runner, args) catch |err| code: {
runner.err.print("nxdns run failed: {s}\n", .{@errorName(err)}) catch {};
if (isConfigFault(err)) {
const mapped = failureExitCode(err);
if (mapped == cli.exit_check) {
runner.err.writeAll("run `nxdns check` to see the configuration in full\n") catch {};
break :code cli.exit_check;
}
break :code cli.exit_runtime;
break :code mapped;
};
// Output the operator never received is not output, so a failed flush
@@ -114,15 +106,75 @@ pub fn run(runner: cli.Runner, args: cli.RunArgs) u8 {
return code;
}
fn isConfigFault(err: anyerror) bool {
return switch (err) {
error.NoUsableUpstreams,
error.BadBindAddress,
error.BadRateLimit,
error.BadCertificate,
=> true,
else => false,
};
/// The one classification, from `config/faults.zig`. This file keeps no list of
/// its own: `run` exiting 1 on a seed file `check` and `import` exit 2 on was
/// exactly the cost of the second list that used to be here.
fn failureExitCode(err: anyerror) u8 {
return if (faults.isConfigFault(err)) cli.exit_check else cli.exit_runtime;
}
/// First run only: the file seeds an empty database and is ignored forever
/// after. Its diagnostics are the operator's one chance to see what the file
/// said, so they are printed the way `check` and `import` print them.
///
/// Printed on the way out either way. A seed file can be accepted and still
/// carry warnings — a blocklist source in no group is downloaded and compiled
/// into nothing — and a warning that only appears when the start fails is a
/// warning nobody ever reads: the start it describes is the one that worked.
/// `check` reported it and `run` did not, which left the same file graded two
/// ways.
///
/// The runner's error writer, not `std.log`: this runs before
/// `logging.install`, and one rendering of a diagnostic across `run`, `check`
/// and `import` is the point of `Diagnostics.writeAll`.
///
/// Flushed here rather than left to `run`'s exit flush. That writer is buffered
/// (`main` gives it 4 KiB) and `serve` does not return for as long as the
/// service runs, so a line left in the buffer reaches the operator when the
/// process stops — days after the start it describes. A failure path flushes
/// anyway because it returns immediately; the successful start is the one that
/// needs this.
fn seedFromFile(
r: cli.Runner,
config_db: *db.Db,
dir: std.Io.Dir,
config_path: []const u8,
) bootstrap.Error!bootstrap.Outcome {
var diags: validate.Diagnostics = .init(r.gpa);
defer diags.deinit();
const result = bootstrap.bootstrap(r.io, r.gpa, config_db, dir, config_path, &diags);
// Neither discard is an oversight, and the two answer different questions.
//
// On a rejected seed file, `result` is returned untouched: the operator gets
// the reason the start failed, never a writer error standing in front of it.
// A broken stderr is not why the configuration was refused.
//
// On a seed that worked, a failure here does not stop the start. The trade
// is one lost warning line against a household with no name resolution, and
// `run` before this point is the only stretch of this program where an
// output failure could take DNS down at all — ruling 4 already says nothing
// after it is fatal. Nor could the failure be reported: this writer *is* the
// error channel, and `logging.install` has not run yet, so `std.log` resolves
// to the same stderr a diagnostic about it would have to travel down.
//
// It is not lost from the process either. A failed drain consumes nothing,
// so whatever the buffer held it still holds — that half is observed, in
// "a broken error writer does not stop a first start that succeeded" below,
// which reads the retained warning back out of the same writer.
//
// What happens to those bytes afterwards is derived, not watched, and is
// labelled so deliberately. `Io.Writer.defaultFlush` drains while `end != 0`
// and `run`'s exit flush maps a failure to exit 1, so a stderr still broken
// at shutdown should carry the condition out in the exit code, and one that
// recovered should deliver the line late. No test drives `run` that far.
// Two limits come with the derivation: an empty buffer flushes clean and
// reports nothing at all, and a failure that recovers ends at exit 0 with a
// line the operator reads days after the start it describes.
diags.writeAll(r.err) catch {};
r.err.flush() catch {};
return result;
}
fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
@@ -141,18 +193,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
defer config_db.close();
_ = try migrations.migrate(&config_db);
{
// First run only: the file seeds an empty database and is ignored
// forever after. Its diagnostics are the operator's one chance to see
// why a config file was rejected, so they are printed like `check`
// prints them.
var diags: validate.Diagnostics = .init(gpa);
defer diags.deinit();
_ = bootstrap.bootstrap(io, gpa, &config_db, std.Io.Dir.cwd(), paths.config, &diags) catch |err| {
diags.writeAll(r.err) catch {};
return err;
};
}
_ = try seedFromFile(r, &config_db, std.Io.Dir.cwd(), paths.config);
// Every string in `cfg` points into this arena, and the pool's endpoints,
// the handler's records and the monitor's paths all keep such strings. It
@@ -717,7 +758,7 @@ const Upstreams = struct {
http: *std.http.Client,
bundle: *Certificate.Bundle,
bundle_lock: *std.Io.RwLock,
) (Allocator.Error || ConfigError)!Upstreams {
) (Allocator.Error || error{NoUsableUpstreams})!Upstreams {
var enabled: usize = 0;
for (servers) |server| {
if (server.enabled) enabled += 1;
@@ -748,7 +789,10 @@ const Upstreams = struct {
if (!server.enabled) continue;
const endpoint = transport.Endpoint.parse(server.url) catch {
log.warn("upstream '{s}' is not an https:// or tls:// endpoint; skipped", .{server.url});
log.warn(
"upstream {f} is not an https:// or tls:// endpoint; skipped",
.{safe_url.redactQuoted(server.url)},
);
continue;
};
@@ -762,7 +806,10 @@ const Upstreams = struct {
self.doh_buf[base..][0..doh_request_buf_len],
self.doh_buf[base + doh_request_buf_len ..][0..doh_transfer_buf_len],
) catch {
log.warn("upstream '{s}' is not a usable DoH url; skipped", .{server.url});
log.warn(
"upstream {f} is not a usable DoH url; skipped",
.{safe_url.redactQuoted(server.url)},
);
continue;
};
doh_count += 1;
@@ -888,6 +935,280 @@ test "parseBind refuses a bind address of the wrong family" {
try std.testing.expect(std.mem.containsAtLeast(u8, printed, 1, "is not an IPv4 address"));
}
test "run maps a rejected configuration to exit 2 and everything else to exit 1" {
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.MissingDefaultGroup));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.ParseZon));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.NoUsableUpstreams));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.BadCertificate));
try std.testing.expectEqual(cli.exit_runtime, failureExitCode(error.AccessDenied));
try std.testing.expectEqual(cli.exit_runtime, failureExitCode(error.OutOfMemory));
}
test "run, check and import agree on a seed file with no default group" {
const config_import = @import("config/import.zig");
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
// The file from discrepancy D1: parseable, one upstream, no group named
// 'default'.
const source: [:0]const u8 =
\\.{
\\ .groups = .{ .{ .name = "kids" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\}
;
// `run`: `serve` seeds through `bootstrap`, which is a wrapper over this
// exact call, so this is the error `run` classifies.
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var diags: validate.Diagnostics = .init(gpa);
defer diags.deinit();
try std.testing.expectError(
error.MissingDefaultGroup,
config_import.importSource(io, gpa, &database, source, .{}, &diags),
);
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.MissingDefaultGroup));
// `import`: `cli.failureExitCode` reaches exit 2 by either route — the
// recorded failures, or the classification `run` just used.
try std.testing.expect(diags.failureCount() != 0);
try std.testing.expect(faults.isConfigFault(error.MissingDefaultGroup));
// `check`: the same file, through the code `nxdns check` runs.
var arena_state: std.heap.ArenaAllocator = .init(gpa);
defer arena_state.deinit();
const cfg = try std.zon.parse.fromSliceAlloc(model.Config, arena_state.allocator(), source, null, .{});
var out_buf: [2048]u8 = undefined;
var err_buf: [256]u8 = undefined;
var out: Writer = .fixed(&out_buf);
var err_writer: Writer = .fixed(&err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer };
try std.testing.expectEqual(cli.exit_check, try cli.checkConfig(r, cfg, false));
}
test "a configuration whose blocklist source is in no group imports and checks clean" {
const config_import = @import("config/import.zig");
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
// A source is created before it is attached — `docs/tutorial/first-run.md`
// POSTs the blocklist and then PUTs the group's sources — so an unattached
// source is a legal intermediate state on every write path. It warns; it
// never fails.
const source: [:0]const u8 =
\\.{
\\ .groups = .{ .{ .name = "default" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\ .blocklist_sources = .{ .{ .url = "https://lists.example/hosts.txt", .name = "ads" } },
\\}
;
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var diags: validate.Diagnostics = .init(gpa);
defer diags.deinit();
try config_import.importSource(io, gpa, &database, source, .{}, &diags);
try std.testing.expectEqual(@as(usize, 0), diags.failureCount());
try std.testing.expectEqual(@as(usize, 1), diags.warningCount());
var arena_state: std.heap.ArenaAllocator = .init(gpa);
defer arena_state.deinit();
const cfg = try std.zon.parse.fromSliceAlloc(model.Config, arena_state.allocator(), source, null, .{});
var check_diags: validate.Diagnostics = .init(gpa);
defer check_diags.deinit();
// No error means no subcommand may reject it; the warning is report-only.
try validate.validate(cfg, &check_diags);
try std.testing.expectEqual(@as(usize, 0), check_diags.failureCount());
try std.testing.expectEqual(@as(usize, 1), check_diags.warningCount());
}
test "a first start that seeds from a file prints the warnings the file earned" {
// D5, second half. The seed file is read once in the life of a database, so
// a warning it earns is printed on that start or never. `run` printed
// diagnostics only when the file was rejected, which made a successful first
// start the one place the finding could not surface.
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
try tmp.dir.writeFile(io, .{ .sub_path = "config.zon", .data =
\\.{
\\ .groups = .{ .{ .name = "default" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\ .blocklist_sources = .{ .{ .url = "https://lists.example/hosts.txt", .name = "ads" } },
\\}
});
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
// A buffered file writer, the shape `main` builds over stderr, and not
// `Writer.fixed`: a fixed writer's flush is a no-op, so it counts a line
// still sitting in the buffer as delivered. Reading the file back is the
// only way to ask what the operator can actually see.
var out_buf: [64]u8 = undefined;
var out: Writer = .fixed(&out_buf);
var err_file = try tmp.dir.createFile(io, "stderr.txt", .{});
defer err_file.close(io);
var err_buf: [4096]u8 = undefined;
var err_writer = err_file.writer(io, &err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer.interface };
// The file is valid, so the start succeeds and the database is seeded.
try std.testing.expectEqual(
bootstrap.Outcome.seeded,
try seedFromFile(r, &database, tmp.dir, "config.zon"),
);
try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
// Nothing flushes here on purpose. In production `serve` runs from this
// point until the service stops, so a line that has not reached the file by
// now is a line the operator does not get for days.
const printed = try tmp.dir.readFileAlloc(io, "stderr.txt", gpa, .limited(8192));
defer gpa.free(printed);
try std.testing.expect(std.mem.containsAtLeast(u8, printed, 1, "WARN blocklist_sources[0]: "));
try std.testing.expect(std.mem.containsAtLeast(u8, printed, 1, "belongs to no group"));
// A warning is not a rejection: nothing here claims the start failed.
try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, printed, "FAIL"));
}
/// A broken stderr, in the shape `main` builds: a buffered `File.Writer`, with
/// its drain switched to the mode that fails. `Writer.fixed` cannot stand in —
/// its flush is `noopFlush`, so it has no failure to report and its `written()`
/// counts a line still sitting in the buffer as delivered.
///
/// The file stays empty for as long as the mode is `.failure`, which is what
/// lets a test tell "the writer really failed" from "the writer worked".
fn brokenErrWriter(io: std.Io, file: std.Io.File, buffer: []u8) std.Io.File.Writer {
var w = file.writer(io, buffer);
w.mode = .failure;
return w;
}
test "a broken error writer does not replace the reason a seed file was rejected" {
// The operator has to see why seeding failed, and a broken stderr is not
// that reason.
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// No group named 'default': rejected, and it records a FAIL line on the way.
try tmp.dir.writeFile(io, .{ .sub_path = "config.zon", .data =
\\.{
\\ .groups = .{ .{ .name = "kids" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\}
});
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var out_buf: [64]u8 = undefined;
var out: Writer = .fixed(&out_buf);
var err_file = try tmp.dir.createFile(io, "stderr.txt", .{});
defer err_file.close(io);
// Eight bytes: no FAIL line fits, so `writeAll` must drain mid-line and is
// itself the call that fails. The test below covers the other discard, where
// the line fits and only the flush fails.
var err_buf: [8]u8 = undefined;
var err_writer = brokenErrWriter(io, err_file, &err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer.interface };
try std.testing.expectError(
error.MissingDefaultGroup,
seedFromFile(r, &database, tmp.dir, "config.zon"),
);
// Empty, so the writer did fail — without this the assertion above would
// hold just as well against a writer that worked.
const printed = try tmp.dir.readFileAlloc(io, "stderr.txt", gpa, .limited(8192));
defer gpa.free(printed);
try std.testing.expectEqual(@as(usize, 0), printed.len);
}
test "a broken error writer does not stop a first start that succeeded" {
// The call this file makes: a DNS server for a household does not refuse to
// resolve because stderr is broken. What it must not do is drop the warning
// on the floor, so the second half checks the buffer still holds it.
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = std.testing.allocator;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
// Valid, and it earns one warning: the source belongs to no group.
try tmp.dir.writeFile(io, .{ .sub_path = "config.zon", .data =
\\.{
\\ .groups = .{ .{ .name = "default" } },
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
\\ .blocklist_sources = .{ .{ .url = "https://lists.example/hosts.txt", .name = "ads" } },
\\}
});
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var out_buf: [64]u8 = undefined;
var out: Writer = .fixed(&out_buf);
var err_file = try tmp.dir.createFile(io, "stderr.txt", .{});
defer err_file.close(io);
// 4 KiB, the buffer `main` gives the real stderr writer: the warning fits,
// so `writeAll` succeeds into the buffer and the flush is what fails. That
// is the shape production hits.
var err_buf: [4096]u8 = undefined;
var err_writer = brokenErrWriter(io, err_file, &err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer.interface };
try std.testing.expectEqual(
bootstrap.Outcome.seeded,
try seedFromFile(r, &database, tmp.dir, "config.zon"),
);
try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
// Nothing reached the file, so the flush really did fail.
const undelivered = try tmp.dir.readFileAlloc(io, "stderr.txt", gpa, .limited(8192));
defer gpa.free(undelivered);
try std.testing.expectEqual(@as(usize, 0), undelivered.len);
// And the warning is still buffered rather than dropped: this is the same
// writer, and these are the bytes `run`'s exit flush meets on the way out.
err_writer.mode = .positional;
try err_writer.interface.flush();
const printed = try tmp.dir.readFileAlloc(io, "stderr.txt", gpa, .limited(8192));
defer gpa.free(printed);
try std.testing.expect(std.mem.containsAtLeast(u8, printed, 1, "WARN blocklist_sources[0]: "));
}
fn reportBind(r: cli.Runner, which: []const u8, addr: net.IpAddress, err: anyerror) anyerror {
r.err.print("cannot bind {s} {f}: {s}\n", .{ which, addr, @errorName(err) }) catch {};
return err;