mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 13 Activity

milestone 13 discrepancies: redact credentials from urls in logs, metrics and cli output

Browse Source
This commit is contained in:
mokhtar
2026-08-07 00:45:17 +02:00
parent 1ff727feb8
commit 8c3328562e
39 changed files with 5734 additions and 510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/filter/filter_integration_test.zig
+64 -3
View File
@@ -1041,17 +1041,78 @@ test "10: pruneOrphans deletes the files of a deleted source and leaves live one
defer dir.close(io);
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
// A refresh that is still running owns its temporaries, so they are not
// orphans and must survive.
// Id 9999 has no `blocklist_sources` row, so no refresh can be writing for
// it: these temporaries are what a refresh that died mid-write leaves
// behind, and the sweep is the only thing that will ever remove them.
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.list.tmp", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.wild.tmp", .data = "" });
// The live source does have a row, so its temporary is a refresh in
// progress and must survive a sweep that runs beside it.
var live_tmp_buf: [64]u8 = undefined;
const live_tmp = try std.fmt.bufPrint(&live_tmp_buf, "{d}.raw.tmp", .{id});
try dir.writeFile(io, .{ .sub_path = live_tmp, .data = "" });
try env.mgr.pruneOrphans(io);
var live_buf: [64]u8 = undefined;
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.list", .{id}), .{});
try dir.access(io, "9999.raw.tmp", .{});
try dir.access(io, live_tmp, .{});
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list.tmp", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild.tmp", .{}));
}
test "10b: the scheduler sweeps orphans on its own, with no operator call" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
const list_body = "aaa.example.com\n";
const id = try seedSource(&env.database, source_url);
try publishFixtureFiles(env, id, list_body, "");
// Dated now, so the startup pass finds nothing to download: this case
// asserts the sweep, and `source_url` points at a port nobody is serving.
try sources_repo.updateSourceStats(&env.database, id, .{
.last_updated = std.Io.Clock.real.now(io).toSeconds(),
.domain_count = 1,
.wildcard_count = 0,
.skipped_regex_count = 0,
.checksum = &bodyChecksum(list_body, ""),
});
var dir = try env.blocklistDir();
defer dir.close(io);
// What a source deleted while the server was down leaves, and what a
// process killed mid-refresh leaves. Nothing else in the tree removes
// either.
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
// `runScheduler` is the entry point `app.zig` hands to `Io.Group`, and the
// only one the server ever calls. A disabled update stops it after the
// startup pass, so the production path runs to completion here with no
// interval to wait out.
env.mgr.update.enabled = false;
try env.mgr.runScheduler(io);
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
// The live source kept its files and is still filtering: the sweep did not
// take the snapshot the same pass had just published.
var live_buf: [64]u8 = undefined;
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.list", .{id}), .{});
const decision, _ = try env.evaluate("aaa.example.com");
try testing.expect(decision.blocked);
}
// ---------------------------------------------------------------------------
src/filter/manager.zig
+197 -35
View File
@@ -39,6 +39,7 @@ const std = @import("std");
const Allocator = std.mem.Allocator;
const model = @import("../config/model.zig");
const safe_url = @import("../safe_url.zig");
const db = @import("../storage/db.zig");
const clients_repo = @import("../storage/repositories/clients_repo.zig");
const groups_repo = @import("../storage/repositories/groups_repo.zig");
@@ -60,6 +61,10 @@ pub const max_error_len: usize = 128;
/// `SourceStatus.url` is fixed-size so a copied status borrows nothing. A
/// blocklist url longer than this is truncated in the status only; the row
/// keeps it whole.
///
/// The log form of a url is bounded separately by `safe_url.max_len`. The two
/// numbers agree today and answer different questions; neither follows the
/// other.
pub const max_url_len: usize = 255;
/// A compiled body larger than this is refused at load. A source that reaches
@@ -80,6 +85,38 @@ const sample_buf_len: usize = parsers.sample_lines * (compiler.max_line_len + 1)
/// `<id>` is at most 20 characters and the longest suffix is `.list.tmp`.
const name_buf_len: usize = 48;
/// How one blocklist source is named in a log line: by its row id and its name,
/// which are its own identity, and by its redacted url, which says where it
/// points and nothing more.
///
/// The url used to carry the identity here on its own. It cannot: `safe_url`
/// drops the path, because a path segment is a place an operator's token lives,
/// and two sources on one host are told apart by exactly that path. The id and
/// the name are on the row every one of these lines already holds, they are
/// what the API and the web UI show, and neither can leak what the url holds.
/// The name is escaped for the same reason the url is — both are database text
/// and a newline in either would forge a log line. It carries its own quotes,
/// out of `safe_url.quoteText`, because a quote this format string added would
/// be a quote the name could close: `ads' (https://decoy.example) --` would then
/// read as a source pointing somewhere it does not.
const SourceLabel = struct {
id: i64,
name: []const u8,
url: []const u8,
fn of(row: sources_repo.SourceRow) SourceLabel {
return .{ .id = row.id, .name = row.name, .url = row.url };
}
pub fn format(self: SourceLabel, w: *std.Io.Writer) std.Io.Writer.Error!void {
try w.print("source {d} {f} {f}", .{
self.id,
safe_url.quoteText(self.name),
safe_url.redactQuoted(self.url),
});
}
};
pub const Paths = struct {
/// `<data_dir>`, owned by the caller and left open for the manager's life.
dir: std.Io.Dir,
@@ -520,7 +557,7 @@ pub const Manager = struct {
// `replace` calls — a new `.list` beside an old `.wild` — is caught
// here and refreshed, not served as a half-updated list.
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body))) {
log.warn("blocklist {s}: compiled files do not match the stored checksum", .{row.url});
log.warn("blocklist {f}: compiled files do not match the stored checksum", .{SourceLabel.of(row)});
return .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } };
}
@@ -610,7 +647,7 @@ pub const Manager = struct {
defer self.deleteQuietly(io, dir, list_tmp);
defer self.deleteQuietly(io, dir, wild_tmp);
self.download(io, dir, raw_name, row.url) catch |err| switch (err) {
self.download(io, dir, raw_name, row) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.Canceled => return error.Canceled,
else => {
@@ -692,13 +729,16 @@ pub const Manager = struct {
}
/// The body goes to a temporary file, never to memory: `max_body_bytes` is
/// 64 MB and the memory budget has no room for it beside two snapshots.
/// 64 MiB and the memory budget has no room for it beside two snapshots.
///
/// It takes the whole row rather than the url alone because its two log
/// lines name the source by its id and name, which only the row carries.
fn download(
self: *Manager,
io: std.Io,
dir: std.Io.Dir,
raw_name: []const u8,
url: []const u8,
row: sources_repo.SourceRow,
) !void {
const file = try dir.createFile(io, raw_name, .{ .permissions = .fromMode(0o600) });
defer file.close(io);
@@ -707,14 +747,14 @@ pub const Manager = struct {
defer self.gpa.free(buffer);
var fw = file.writer(io, buffer);
const result = self.fetchWithin(io, url, &fw.interface) catch |err| {
const result = self.fetchWithin(io, row.url, &fw.interface) catch |err| {
// `fetcher.Error.Unexpected` is what a failing sink surfaces as;
// the concrete cause is on this writer, which the fetcher does not
// own.
if (fw.err) |cause| return cause;
if (err == error.HttpStatus) {
if (self.fetcher.last_status) |status| {
log.warn("blocklist {s}: http status {d}", .{ url, @intFromEnum(status) });
log.warn("blocklist {f}: http status {d}", .{ SourceLabel.of(row), @intFromEnum(status) });
}
}
return err;
@@ -724,7 +764,7 @@ pub const Manager = struct {
// buffer this function is about to drop.
try file.sync(io);
log.debug("blocklist {s}: downloaded {d} bytes", .{ url, result.bytes_read });
log.debug("blocklist {f}: downloaded {d} bytes", .{ SourceLabel.of(row), result.bytes_read });
}
/// `std.http.Client` has no per-request deadline, so the whole exchange
@@ -906,7 +946,7 @@ pub const Manager = struct {
err: anyerror,
) void {
_ = self;
log.warn("blocklist {s}: download failed: {s}", .{ row.url, @errorName(err) });
log.warn("blocklist {f}: download failed: {s}", .{ SourceLabel.of(row), @errorName(err) });
status.fail(.fetch_failed, @errorName(err));
}
@@ -917,7 +957,7 @@ pub const Manager = struct {
err: anyerror,
) void {
_ = self;
log.warn("blocklist {s}: compile failed: {s}", .{ row.url, @errorName(err) });
log.warn("blocklist {f}: compile failed: {s}", .{ SourceLabel.of(row), @errorName(err) });
status.fail(.compile_failed, @errorName(err));
}
@@ -934,7 +974,7 @@ pub const Manager = struct {
"NoValidEntries invalid={d} unsupported={d} long_lines={d}",
.{ counts.invalid, counts.skipped_unsupported, counts.long_lines },
) catch "NoValidEntries";
log.warn("blocklist {s}: {s}", .{ row.url, text });
log.warn("blocklist {f}: {s}", .{ SourceLabel.of(row), text });
status.fail(.no_valid_entries, text);
}
@@ -954,6 +994,13 @@ pub const Manager = struct {
/// `update.enabled == false` stops after the startup pass; manual refresh
/// through `refreshAll` still works.
pub fn runScheduler(self: *Manager, io: std.Io) std.Io.Cancelable!void {
// Ahead of the pass, not after it. This is the sweep that collects what
// a killed process left behind: a `.raw.tmp` as large as the body the
// dead refresh was writing, and the compiled files of a source deleted
// while the server was down. Both are bytes the pass below is about to
// ask the same filesystem for.
try self.sweepOrphans(io);
self.startupPass(io) catch |err| switch (err) {
error.Canceled => return error.Canceled,
else => log.warn("blocklist startup pass failed: {s}", .{@errorName(err)}),
@@ -968,6 +1015,11 @@ pub const Manager = struct {
};
while (true) {
try interval.sleep(io);
// Ahead of the gate as well as ahead of the pass: the sweep only
// unlinks, so it is the one thing here that can give a critically
// full disk room back, and gating it would keep the residue that
// helped fill the disk in the first place.
try self.sweepOrphans(io);
if (self.refreshGated()) continue;
self.refreshAll(io) catch |err| switch (err) {
error.Canceled => return error.Canceled,
@@ -976,6 +1028,21 @@ pub const Manager = struct {
}
}
/// `pruneOrphans` with its failure absorbed. Leftover bytes under
/// `<data_dir>/blocklists/` are not an outage, and a sweep that could not
/// read the directory must not cost the household the refresh pass behind
/// it — let alone the server. Cancellation is the one outcome that
/// propagates, because it means shutdown.
///
/// Taken from outside every `*Locked` body: `pruneOrphans` takes
/// `writer_lock` itself and the mutex is not reentrant.
fn sweepOrphans(self: *Manager, io: std.Io) std.Io.Cancelable!void {
self.pruneOrphans(io) catch |err| switch (err) {
error.Canceled => return error.Canceled,
else => log.warn("pruning orphaned blocklist files failed: {s}", .{@errorName(err)}),
};
}
/// The §11.6 gate, consulted by scheduled passes only (ruling 17). A
/// download writes tens of megabytes into the blocklist directory and the
/// compile writes as much again, which is exactly the "non-essential write"
@@ -1045,12 +1112,26 @@ pub const Manager = struct {
// orphans
// -----------------------------------------------------------------------
/// Deletes `<id>.list` and `<id>.wild` files whose id is no longer a
/// `blocklist_sources` row. Files of a live source are left alone,
/// Deletes the compiled files and the leftover temporaries whose id is no
/// longer a `blocklist_sources` row. Files of a live source are left alone,
/// whatever their state.
///
/// Three callers, and between them they cover every way an orphan is made:
/// `runScheduler` sweeps once before its startup pass — the residue of a
/// process that was killed mid-refresh, and of a source deleted while the
/// server was down — and again before each scheduled pass; the
/// `DELETE /api/blocklists/{id}` handler sweeps as soon as it has removed
/// the row, so the directory follows the table an operator can see instead
/// of waiting out `blocklist_update.interval_hours`.
///
/// It is safe to call on a fresh install: `openDir` creates
/// `<data_dir>/blocklists/` if nothing has yet, and an empty directory
/// sweeps to nothing.
pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
// A refresh in flight owns the temporaries of a live source; the sweep
// must not run beside one and decide from a half-written directory.
// Every path that writes a temporary holds this lock too, so the sweep
// never reads a directory a refresh is halfway through. The temporaries
// it can see therefore belong to a finished or a dead refresh, and only
// those of a source with no row are removed.
self.writer_lock.lockUncancelable(io);
defer self.writer_lock.unlock(io);
@@ -1079,14 +1160,14 @@ pub const Manager = struct {
},
} orelse break;
if (entry.kind != .file) continue;
const id = compiledId(entry.name) orelse continue;
const id = sourceFileId(entry.name) orelse continue;
if (containsId(rows.items, id)) continue;
try doomed.append(self.gpa, try self.gpa.dupe(u8, entry.name));
}
for (doomed.items) |name| {
self.deleteQuietly(io, dir, name);
log.info("pruned orphaned compiled file {s}", .{name});
log.info("pruned orphaned blocklist file {s}", .{name});
}
}
@@ -1253,7 +1334,7 @@ const LoadOutcome = union(enum) {
};
fn loadFailure(row: sources_repo.SourceRow, file_name: []const u8, err: anyerror) LoadOutcome {
log.warn("blocklist {s}: reading {s} failed: {s}", .{ row.url, file_name, @errorName(err) });
log.warn("blocklist {f}: reading {s} failed: {s}", .{ SourceLabel.of(row), file_name, @errorName(err) });
return .{ .failed = .{ .state = .load_failed, .text = @errorName(err) } };
}
@@ -1394,17 +1475,27 @@ fn compiledName(buf: *[name_buf_len]u8, id: i64, suffix: []const u8) []const u8
return std.fmt.bufPrint(buf, "{d}{s}", .{ id, suffix }) catch unreachable;
}
/// The source id a compiled file belongs to, or null when the name is not one
/// of ours. Temporary files are deliberately not matched: they belong to a
/// refresh that may still be running.
fn compiledId(file_name: []const u8) ?i64 {
const stem = if (std.mem.endsWith(u8, file_name, ".list"))
file_name[0 .. file_name.len - ".list".len]
else if (std.mem.endsWith(u8, file_name, ".wild"))
file_name[0 .. file_name.len - ".wild".len]
else
return null;
return std.fmt.parseInt(i64, stem, 10) catch null;
/// Every name `compiledName` can produce, longest suffix first so `.list.tmp`
/// is never read as `.list`.
const source_file_suffixes = [_][]const u8{ ".list.tmp", ".wild.tmp", ".raw.tmp", ".list", ".wild" };
/// The source id a file under the blocklist directory belongs to, or null when
/// the name is not one of ours.
///
/// The three temporaries count. A refresh that dies between writing one and
/// renaming it leaves a file no later refresh reuses and no `defer` reaches, so
/// excluding them from the sweep means nothing ever removes them. Matching them
/// is safe because `pruneOrphans` holds `writer_lock` for its whole body: every
/// path that creates a temporary runs under that same lock, so no refresh is in
/// flight while the sweep reads the directory, and a temporary the sweep sees
/// belonging to a source that still has a row is kept regardless.
fn sourceFileId(file_name: []const u8) ?i64 {
for (source_file_suffixes) |suffix| {
if (!std.mem.endsWith(u8, file_name, suffix)) continue;
const stem = file_name[0 .. file_name.len - suffix.len];
return std.fmt.parseInt(i64, stem, 10) catch null;
}
return null;
}
fn containsId(rows: []const sources_repo.SourceRow, id: i64) bool {
@@ -1566,6 +1657,62 @@ test "the header writer produces the documented text" {
++ "# sha256 " ++ "0" ** 64 ++ "\n", w.buffered());
}
test "the log label names a source without printing what its url carries" {
// Every `log.warn` in this file formats its subject through `SourceLabel`,
// so this is the text of those lines. A `std.log` line is not observable
// from a unit test under the default runner; the label is.
var buf: [1024]u8 = undefined;
const row: sources_repo.SourceRow = .{
.id = 3,
.url = "https://lists.example/download/token/hunter2/hosts.txt?apikey=s3cr3t",
.name = "ads",
.enabled = true,
.last_updated = null,
.domain_count = 0,
.wildcard_count = 0,
.skipped_regex_count = 0,
.checksum = null,
};
const printed = try std.fmt.bufPrint(&buf, "blocklist {f}: download failed: {s}", .{
SourceLabel.of(row),
@errorName(error.ConnectFailed),
});
try testing.expectEqualStrings(
"blocklist source 3 'ads' 'https://lists.example': download failed: ConnectFailed",
printed,
);
try testing.expect(!std.mem.containsAtLeast(u8, printed, 1, "hunter2"));
try testing.expect(!std.mem.containsAtLeast(u8, printed, 1, "s3cr3t"));
// The row is database text, and a path that writes it does not have to
// validate as strictly as the config validator. Neither column may end the
// line and start one of the operator's choosing.
var forged = row;
forged.name = "ads\n2026-01-01 ERROR forged";
forged.url = "https://lists.example\n2026-01-01 ERROR forged/hosts.txt";
const escaped = try std.fmt.bufPrint(&buf, "blocklist {f}", .{SourceLabel.of(forged)});
try testing.expectEqualStrings(
"blocklist source 3 'ads\\n2026-01-01 ERROR forged'" ++
" 'https://lists.example\\n2026-01-01 ERROR forged'",
escaped,
);
try testing.expect(!std.mem.containsAtLeast(u8, escaped, 1, "\n"));
// A name is operator-supplied and reaches the row through the API, so it
// can close the quote this label puts around it and open a decoy that reads
// as the url of a second source. The quote it would close is escaped, and
// the escape is unambiguous because a `\` is escaped too.
var decoy = row;
decoy.name = "ads' (https://decoy.example) --";
decoy.url = "https://lists.example/hosts.txt";
const quoted = try std.fmt.bufPrint(&buf, "blocklist {f}", .{SourceLabel.of(decoy)});
try testing.expectEqualStrings(
"blocklist source 3 'ads\\' (https://decoy.example) --' 'https://lists.example'",
quoted,
);
}
test "stripHeader returns the body of a compiled file" {
const file =
"# nxdns blocklist\n" ++
@@ -1615,13 +1762,28 @@ test "compiledName spells the four file names of a source" {
try testing.expectEqualStrings("42.list.tmp", compiledName(&buf, 42, ".list.tmp"));
}
test "compiledId matches compiled files and nothing else" {
try testing.expectEqual(@as(?i64, 7), compiledId("7.list"));
try testing.expectEqual(@as(?i64, 7), compiledId("7.wild"));
try testing.expectEqual(@as(?i64, null), compiledId("7.list.tmp"));
try testing.expectEqual(@as(?i64, null), compiledId("7.raw.tmp"));
try testing.expectEqual(@as(?i64, null), compiledId("notes.list"));
try testing.expectEqual(@as(?i64, null), compiledId("README"));
test "sourceFileId matches every name a refresh writes, including the temporaries" {
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild"));
// A temporary left by a refresh that died belongs to its source id, so the
// sweep can tell whether that source still has a row.
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.raw.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list.tmp"));
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.list"));
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.raw.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.tmp"));
try testing.expectEqual(@as(?i64, null), sourceFileId("7.raw"));
try testing.expectEqual(@as(?i64, null), sourceFileId("README"));
}
test "every name compiledName writes is a name the sweep can attribute" {
var buf: [name_buf_len]u8 = undefined;
for (source_file_suffixes) |suffix| {
try testing.expectEqual(@as(?i64, 42), sourceFileId(compiledName(&buf, 42, suffix)));
}
}
test "a failed refresh keeps the fields of the compiled files still serving" {