milestone 28: query provenance — every logged query is exactly explainable
Gates / frontend (push) Successful in 1m36s
Gates / test (push) Successful in 1m56s
Gates / test-aarch64 (push) Successful in 7m37s
Gates / package (push) Successful in 9m12s
Gates / container (push) Successful in 13s
CI / gates (push) Successful in 19m4s

query rows gain qclass, rcode, group, policy action and reason, the
matched rule or list entry with its source, cname and safe-search
targets, route kind, forward zone, and the resolver that actually
answered — the pool and local markers die. servfails are logged and
name the resolver that lost; post-parse protocol refusals become rows.
a detail page at /queries/:id renders the ordered explanation, and
coverage watermarks distinguish an empty history from a missing one.

the schema fingerprint changes: existing query history is recreated
with the old file kept aside and the reset filed as a resolved
diagnostic. fixes an oversized udp reply being rebuilt as noerror,
which handed clients a truncated nxdomain as success.
This commit is contained in:
2026-08-22 09:16:40 +02:00
parent 7e6cb507d2
commit 0fd6bbd312
65 changed files with 7036 additions and 685 deletions
+245 -23
View File
@@ -61,7 +61,9 @@ const migrations = @import("storage/migrations.zig");
const model = @import("config/model.zig");
const pause = @import("server/pause.zig");
const pool_mod = @import("upstream/pool.zig");
const queries_repo = @import("storage/repositories/queries_repo.zig");
const query_sink = @import("server/query_sink.zig");
const querylog_schema = @import("storage/querylog_schema.zig");
const rate_limiter = @import("server/rate_limiter.zig");
const reconcile = @import("config/reconcile.zig");
const retention_mod = @import("storage/retention.zig");
@@ -631,29 +633,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
var querylog_writer_db = querylog_opened.database;
defer querylog_writer_db.close();
// One-shot and already over: the file was recreated during this boot, and
// there is nothing to recover from. Never emitted for `.missing` — a first
// creation renames nothing aside, so the event would carry an aside path
// that does not exist and would greet every fresh install with a warning.
if (querylog_opened.recreated) |cause| {
if (cause != .missing) {
if (event_store) |store| {
var detail_buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&detail_buf, "previous file kept as '{s}'", .{
querylog_opened.aside(),
}) catch detail_buf[0..];
store.reportResolved(
io,
boot_now_s,
.query_log_recreated,
@tagName(cause),
@tagName(cause),
.warning,
detail,
);
}
}
}
reportQuerylogRecreated(event_store, io, boot_now_s, &querylog_opened, &querylog_writer_db);
var querylog_retention_db = try data.reopenQuerylogDb(io);
defer querylog_retention_db.close();
var querylog_history_db = try data.reopenQuerylogDb(io);
@@ -1357,9 +1337,251 @@ fn parseBind(
return addr;
}
/// Files the one-shot `query_log.recreated` event for a boot that replaced the
/// query log.
///
/// One-shot and already over: the file was recreated during this boot, and
/// there is nothing to recover from. Never emitted for `.missing` — a first
/// creation renames nothing aside, so the event would carry an aside path that
/// does not exist and would greet every fresh install with a warning.
///
/// `database` is the connection to the file that was just created; the coverage
/// start is read from it rather than recomputed, so the event states the value
/// the API will.
fn reportQuerylogRecreated(
store: ?*events.Store,
io: std.Io,
now_s: i64,
opened: *const querylog_schema.OpenResult,
database: *db.Db,
) void {
const cause = opened.recreated orelse return;
if (cause == .missing) return;
const s = store orelse return;
// The coverage start belongs in this detail: the recreate is exactly the
// moment the history the operator had stops existing, and the watermark is
// the answer to "from when can I still ask?".
const coverage_start: ?i64 = queries_repo.availableSince(database) catch null;
var detail_buf: [events.Store.max_detail_len]u8 = undefined;
const detail = recreatedDetail(&detail_buf, opened.aside(), coverage_start);
s.reportResolved(io, now_s, .query_log_recreated, @tagName(cause), @tagName(cause), .warning, detail);
}
/// The `query_log.recreated` detail line: what was kept, and from when the new
/// file can answer.
///
/// The coverage start is the operator's actual remedy information — the event
/// says "this history is gone" and this says "and here is where the new history
/// begins". Null only when the fresh file would not answer, which is already a
/// separate failure; the line still names the aside rather than saying nothing.
///
/// The aside is a full path under the data directory, which can be longer than
/// the whole detail column, so the two facts compete for the buffer. The
/// watermark always wins and the name degrades in whole steps: full path, then
/// basename — which the event's own database directory disambiguates — then no
/// name at all. Never a path cut mid-string, which names no file on disk and
/// reads as if it did.
fn recreatedDetail(
buf: *[events.Store.max_detail_len]u8,
aside: []const u8,
coverage_start: ?i64,
) []const u8 {
const names = [_][]const u8{ aside, std.fs.path.basename(aside) };
const since = coverage_start orelse {
for (names) |name| {
return std.fmt.bufPrint(buf, "previous file kept as '{s}'", .{name}) catch continue;
}
return "previous file kept aside";
};
for (names) |name| {
return std.fmt.bufPrint(
buf,
"previous file kept as '{s}'; query history is available from {d}",
.{ name, since },
) catch continue;
}
// The buffer is `max_detail_len`, which no i64 can overrun on its own.
return std.fmt.bufPrint(buf, "query history is available from {d}", .{since}) catch unreachable;
}
const events_fixture = @import("storage/events_fixture.zig");
const testing = std.testing;
test "the recreated detail names the aside and the new coverage start" {
var buf: [events.Store.max_detail_len]u8 = undefined;
try std.testing.expectEqualStrings(
"previous file kept as 'querylog.db.schema-changed-1700000000'; " ++
"query history is available from 1700000001",
recreatedDetail(&buf, "querylog.db.schema-changed-1700000000", 1700000001),
);
// A fresh file that will not answer is a separate failure; the line still
// says what was kept rather than reporting nothing.
try std.testing.expectEqualStrings(
"previous file kept as 'querylog.db.corrupt-1700000000'",
recreatedDetail(&buf, "querylog.db.corrupt-1700000000", null),
);
// A data directory deep enough that its path alone would fill the column:
// the watermark is complete and the name degrades to the basename, which
// still names a real file.
const deep = "/srv/" ++ ("d" ** 60 ++ "/") ** 8 ++ "querylog.db.corrupt-1700000000";
try std.testing.expectEqualStrings(
"previous file kept as 'querylog.db.corrupt-1700000000'; " ++
"query history is available from 1700000001",
recreatedDetail(&buf, deep, 1700000001),
);
try std.testing.expectEqualStrings(
"previous file kept as 'querylog.db.corrupt-1700000000'",
recreatedDetail(&buf, deep, null),
);
// No filesystem produces a name this long, but a truncated one would name
// nothing: the watermark survives alone rather than half-named.
const unnameable = "/srv/" ++ "n" ** 500;
try std.testing.expectEqualStrings(
"query history is available from 1700000001",
recreatedDetail(&buf, unnameable, 1700000001),
);
try std.testing.expectEqualStrings(
"previous file kept aside",
recreatedDetail(&buf, unnameable, null),
);
}
test "a fingerprint recreate files a resolved event naming the real aside and watermark" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var tmp = testing.tmpDir(.{ .iterate = true });
defer tmp.cleanup();
var path_buf: [256]u8 = undefined;
const path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/querylog.db", .{tmp.sub_path});
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
// A fresh install: the file was missing, nothing was set aside, and the
// event would name a path that does not exist.
var created = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
reportQuerylogRecreated(&fx.store, io, 1000, &created, &created.database);
created.database.close();
try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
// A healthy file this build's DDL no longer matches, which is what an
// upgrade that edits the schema produces.
{
var stamped = try db.Db.open(path, .{ .mode = .read_write_existing });
defer stamped.close();
var sql_buf: [64]u8 = undefined;
try stamped.exec(try std.fmt.bufPrintZ(
&sql_buf,
"PRAGMA user_version = {d};",
.{querylog_schema.fingerprint +% 1},
));
}
var recreated = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer recreated.database.close();
try testing.expectEqual(querylog_schema.RecreateReason.fingerprint_mismatch, recreated.recreated.?);
reportQuerylogRecreated(&fx.store, io, 2000, &recreated, &recreated.database);
try testing.expectEqualStrings("query_log.recreated", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("fingerprint_mismatch", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
// One-shot: already over when it is filed, so it never becomes an open
// episode `/api/health` counts.
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
// The detail carries the path that is actually on disk and the watermark
// the API will serve, both read back from the recreate rather than from
// the arguments the event was built with.
try tmp.dir.access(io, std.fs.path.basename(recreated.aside()), .{});
const coverage = try queries_repo.availableSince(&recreated.database);
var expected_buf: [events.Store.max_detail_len]u8 = undefined;
const expected = try std.fmt.bufPrint(
&expected_buf,
"previous file kept as '{s}'; query history is available from {d}",
.{ recreated.aside(), coverage },
);
try testing.expectEqualStrings(expected, try fx.text("SELECT detail FROM operational_events"));
// A boot with no diagnostics store configured is not a failure path.
reportQuerylogRecreated(null, io, 2000, &recreated, &recreated.database);
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
}
test "a recreate under a long data directory keeps the watermark and a usable name" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var tmp = testing.tmpDir(.{ .iterate = true });
defer tmp.cleanup();
// Deep enough that the aside outgrows the detail column, shallow enough
// that SQLite's unix VFS still opens the file: it caps a path at 512 bytes.
const nested = ("d" ** 60 ++ "/") ** 6 ++ "d" ** 60;
try tmp.dir.createDirPath(io, nested);
var path_buf: [1024]u8 = undefined;
const path = try std.fmt.bufPrintZ(
&path_buf,
".zig-cache/tmp/{s}/{s}/querylog.db",
.{ tmp.sub_path, nested },
);
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
{
var created = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer created.database.close();
var sql_buf: [64]u8 = undefined;
try created.database.exec(try std.fmt.bufPrintZ(
&sql_buf,
"PRAGMA user_version = {d};",
.{querylog_schema.fingerprint +% 1},
));
}
var recreated = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer recreated.database.close();
try testing.expectEqual(querylog_schema.RecreateReason.fingerprint_mismatch, recreated.recreated.?);
const line_overhead = "previous file kept as ''; query history is available from ".len;
try testing.expect(recreated.aside().len + line_overhead > events.Store.max_detail_len);
reportQuerylogRecreated(&fx.store, io, 2000, &recreated, &recreated.database);
const detail = try fx.text("SELECT detail FROM operational_events");
const coverage = try queries_repo.availableSince(&recreated.database);
const name = std.fs.path.basename(recreated.aside());
var expected_buf: [events.Store.max_detail_len]u8 = undefined;
const expected = try std.fmt.bufPrint(
&expected_buf,
"previous file kept as '{s}'; query history is available from {d}",
.{ name, coverage },
);
// The watermark is whole — the fact that would be lost to a mid-string cut
// — and the name it kept is the file's, not a prefix of its path.
try testing.expectEqualStrings(expected, detail);
try testing.expect(detail.len <= events.Store.max_detail_len);
var deep = try tmp.dir.openDir(io, nested, .{});
defer deep.close(io);
try deep.access(io, name, .{});
}
test "parseBind refuses a bind address of the wrong family" {
var out_buf: [8]u8 = undefined;
var err_buf: [256]u8 = undefined;
+2 -1
View File
@@ -1007,7 +1007,8 @@ fn probeUpstreams(r: Runner, cfg: model.Config) !usize {
}};
var single: pool.Pool = .init(&entries, .{}, timeouts, seed);
if (single.exchange(r.io, probe_query, response_buf)) |_| {
var selected: ?[]const u8 = null;
if (single.exchange(r.io, probe_query, response_buf, &selected)) |_| {
try r.out.print("OK upstreams[{d}] {f}\n", .{ i, safe_url.redact(server.url) });
} else |_| {
// The concrete cause lives in the entry's health, which is where the
+19
View File
@@ -0,0 +1,19 @@
//! Length caps on the configuration labels that the query log copies into every
//! row it writes.
//!
//! They live in a module of their own because two files need them and neither
//! may import the other: `config/validate.zig` rejects a name that exceeds a cap
//! and `storage/logger.zig` sizes an `Entry` buffer from it, while `validate`
//! already imports `logger` for the entry-size budget it derives. A cap owned by
//! either file would close that loop.
//!
//! The values are deliberately short. A group or a source name is a label an
//! operator reads in a table cell, and every byte of it is copied into every
//! logged row — the cap is what keeps a pasted paragraph out of an `Entry` that
//! travels by value through the queue.
/// `groups[].name`.
pub const max_group_name_len = 64;
/// `blocklist_sources[].name`.
pub const max_source_name_len = 64;
+104 -5
View File
@@ -6,9 +6,13 @@
//! instead of a line number, so every `UNIQUE` and every foreign key in
//! PLAN §11.2 has a check here.
//!
//! Pure: no `std.Io` value is a parameter anywhere, no SQLite, no clock. The
//! only `std.Io` type used is `std.Io.Writer`, for rendering diagnostics. The
//! allocator exists for diagnostic text and scratch bookkeeping alone.
//! Pure: no `std.Io` value is a parameter anywhere, no SQLite call, no clock.
//! The only `std.Io` type used is `std.Io.Writer`, for rendering diagnostics.
//! The allocator exists for diagnostic text and scratch bookkeeping alone. The
//! `storage/logger.zig` import is a comptime one — `query_log_buffer_max` is
//! derived from `@sizeOf(logger.Entry)`, because the bound this file enforces
//! on the queue is a bound on bytes and only the logger knows how wide a queued
//! entry is. Nothing in this file calls into storage.
//!
//! Parsers are reused, never reimplemented: `transport.Endpoint.parse` for
//! upstream URLs, `NetAddress.parse` / `Prefix.parse` for addresses, and
@@ -44,6 +48,8 @@ const Writer = std.Io.Writer;
const model = @import("model.zig");
const address = @import("../platform/address.zig");
const dns_name = @import("../dns/name.zig");
const limits = @import("limits.zig");
const logger = @import("../storage/logger.zig");
const regex = @import("../filter/regex.zig");
const safe_url = @import("../safe_url.zig");
const transport = @import("../upstream/transport.zig");
@@ -72,6 +78,7 @@ pub const ValidateError = error{
DuplicateGroupName,
UnknownGroup,
EmptyGroupName,
GroupNameTooLong,
BadClientIp,
DuplicateClientIp,
BadClientPrefix,
@@ -79,6 +86,7 @@ pub const ValidateError = error{
BadSourceUrl,
DuplicateSourceUrl,
EmptySourceName,
SourceNameTooLong,
UnknownSource,
DuplicateGroupSource,
BadRulePattern,
@@ -461,13 +469,16 @@ fn checkScalars(cfg: Config, diags: *Diagnostics) error{OutOfMemory}!void {
if (cfg.logging.query_log_buffer_max < 1) {
try diags.add(error.BadRetention, "logging.query_log_buffer_max", .{}, "must be at least 1", .{});
}
if (cfg.logging.query_log_buffer_max > max_boot_entries) {
// Its own ceiling, not `max_boot_entries`: a queued `logger.Entry` carries
// every provenance field by value, so the queue's cost is bytes rather than
// entries and the bound follows the width of the entry.
if (cfg.logging.query_log_buffer_max > logger.query_log_buffer_max) {
try diags.add(
error.BadRetention,
"logging.query_log_buffer_max",
.{},
"must be at most {d}, got {d}",
.{ max_boot_entries, cfg.logging.query_log_buffer_max },
.{ logger.query_log_buffer_max, cfg.logging.query_log_buffer_max },
);
}
// No floor: 0 is the documented "do not wait" setting, not a mistake.
@@ -714,6 +725,29 @@ fn checkDotHost(
};
}
/// The shared shape of the two label caps.
///
/// Both names are copied by value into every `query_log` row that mentions
/// them, so the cap is what keeps a pasted paragraph out of the fixed buffers
/// of `storage/logger.zig`. Bytes, not codepoints: the buffer counts bytes.
fn checkNameLength(
diags: *Diagnostics,
comptime fault: ValidateError,
comptime path: []const u8,
path_args: anytype,
value: []const u8,
cap: usize,
) error{OutOfMemory}!void {
if (value.len <= cap) return;
try diags.add(
fault,
path,
path_args,
"must be at most {d} bytes, got {d}; the name is copied into every logged query",
.{ cap, value.len },
);
}
fn checkCollections(cfg: Config, diags: *Diagnostics, scratch: Allocator) error{OutOfMemory}!void {
var group_names: IndexSet = .empty;
var has_default = false;
@@ -729,6 +763,16 @@ fn checkCollections(cfg: Config, diags: *Diagnostics, scratch: Allocator) error{
.{safe_url.quoteText(group.name)},
);
}
// Independent of the chain above: an over-long name is still a name,
// and a duplicate of one is still a duplicate.
try checkNameLength(
diags,
error.GroupNameTooLong,
"groups[{d}].name",
.{i},
group.name,
limits.max_group_name_len,
);
if (std.mem.eql(u8, group.name, "default")) has_default = true;
}
if (!has_default) {
@@ -859,6 +903,14 @@ fn checkCollections(cfg: Config, diags: *Diagnostics, scratch: Allocator) error{
.{},
);
}
try checkNameLength(
diags,
error.SourceNameTooLong,
"blocklist_sources[{d}].name",
.{i},
source.name,
limits.max_source_name_len,
);
}
var group_source_pairs: IndexSet = .empty;
@@ -1308,6 +1360,24 @@ test "an https:// upstream may name a host" {
try expectClean(cfg);
}
/// The longest host `transport.Endpoint.parse` accepts: four labels, 253 bytes.
const host_at_bound = ("a" ** 63 ++ ".") ** 3 ++ "a" ** 61;
test "an upstream host at the length bound validates cleanly" {
var cfg = baseConfig();
cfg.upstreams = &.{.{ .url = "https://" ++ host_at_bound ++ "/dns-query" }};
try expectClean(cfg);
}
test "error.BadUpstreamUrl on an upstream host one byte past the length bound" {
// The bound is the query log's `upstream` width and every other identity
// built from the endpoint, so an over-long host has to fail here rather
// than be shortened downstream.
var cfg = baseConfig();
cfg.upstreams = &.{.{ .url = "https://" ++ host_at_bound ++ "a/dns-query" }};
try expectProblem(cfg, error.BadUpstreamUrl, "upstreams[0].url");
}
test "an IPv6 literal tls:// upstream validates cleanly" {
// `Endpoint.parse` strips the brackets, so the host reaching the check is
// exactly what the client hands to the address parser.
@@ -1371,6 +1441,35 @@ test "error.EmptyGroupName" {
try expectProblem(cfg, error.EmptyGroupName, "groups[1].name");
}
test "error.GroupNameTooLong" {
const cap = limits.max_group_name_len;
// Exactly at the cap is accepted; one byte past it is not. The cap is what
// `storage/logger.zig` sizes its `Entry` buffer from, so a name that passes
// here is a name a logged row stores whole.
var at_cap = baseConfig();
at_cap.groups = &.{ .{ .name = "default" }, .{ .name = "g" ** cap } };
try expectClean(at_cap);
var over = baseConfig();
over.groups = &.{ .{ .name = "default" }, .{ .name = "g" ** (cap + 1) } };
try expectProblem(over, error.GroupNameTooLong, "groups[1].name");
}
test "error.SourceNameTooLong" {
const cap = limits.max_source_name_len;
const url = "https://lists.example/hosts.txt";
var at_cap = baseConfig();
at_cap.blocklist_sources = &.{.{ .url = url, .name = "s" ** cap }};
at_cap.group_sources = &.{.{ .group = "default", .source_url = url }};
try expectClean(at_cap);
var over = baseConfig();
over.blocklist_sources = &.{.{ .url = url, .name = "s" ** (cap + 1) }};
over.group_sources = &.{.{ .group = "default", .source_url = url }};
try expectProblem(over, error.SourceNameTooLong, "blocklist_sources[0].name");
}
test "error.BadClientIp" {
var cfg = baseConfig();
cfg.clients = &.{.{ .ip = "nonsense" }};
+62 -1
View File
@@ -33,8 +33,17 @@ const log = std.log.scoped(.forward_client);
/// each half large enough to frame a query in one write, not a capacity.
pub const min_frame_buf: usize = 1024;
/// `tcp://[` + the longest IPv6 text form + `]:65535`, the widest spelling
/// `identityText` can produce.
pub const max_identity_len: usize = "tcp://[".len + 45 + "]:65535".len;
pub const ForwardClient = struct {
resolver: validate.Resolver,
/// The resolver as text, owned here so the `transport.Client` out-parameter
/// has something stable to borrow: `validate.Resolver` is a parsed address,
/// and a caller logging the exchange needs its spelling.
identity_buf: [max_identity_len]u8 = undefined,
identity_len: usize = 0,
/// Caller-owned scratch for the TCP length-prefixed path.
frame_buf: []u8,
/// On the `.awake` clock at the caller's choosing, so a suspended host does
@@ -64,11 +73,20 @@ pub const ForwardClient = struct {
read_timeout: std.Io.Clock.Duration,
) ForwardClient {
std.debug.assert(frame_buf.len >= min_frame_buf);
return .{
var self: ForwardClient = .{
.resolver = resolver,
.frame_buf = frame_buf,
.read_timeout = read_timeout,
};
self.identity_len = identityText(resolver, &self.identity_buf).len;
return self;
}
/// `udp://192.168.1.1:53`, `tcp://[fd00::1]:53` — the same spelling
/// `validate.parseResolver` accepts, so a log row names the configured
/// value. Valid for as long as this client is.
pub fn identity(self: *const ForwardClient) []const u8 {
return self.identity_buf[0..self.identity_len];
}
pub fn client(self: *ForwardClient) transport.Client {
@@ -80,8 +98,12 @@ pub const ForwardClient = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
const self: *ForwardClient = @ptrCast(@alignCast(ptr));
// Set before the attempt: a failed forward-zone exchange still names
// the resolver it was sent to.
selected.* = self.identity();
return self.exchange(io, query, response_buf);
}
@@ -238,6 +260,24 @@ pub const ForwardClient = struct {
}
};
fn identityText(resolver: validate.Resolver, buf: *[max_identity_len]u8) []const u8 {
var w: std.Io.Writer = .fixed(buf);
w.writeAll(switch (resolver.scheme) {
.udp => "udp://",
.tcp => "tcp://",
}) catch unreachable;
const bracketed = switch (resolver.addr) {
.ip4 => false,
.ip6 => true,
};
if (bracketed) w.writeByte('[') catch unreachable;
resolver.addr.format(&w) catch unreachable;
if (bracketed) w.writeByte(']') catch unreachable;
w.print(":{d}", .{resolver.port}) catch unreachable;
return w.buffered();
}
/// The local address a datagram to `dest` is sent from: same family, port
/// chosen by the kernel.
fn wildcardFor(dest: net.IpAddress) net.IpAddress {
@@ -286,6 +326,27 @@ test "ForwardClient satisfies the Client interface" {
try testing.expectEqual(@as(u16, 53), fc.resolver.port);
}
test "the client owns its resolver identity in both address families" {
var buf = testBuf();
const v4: ForwardClient = .init(
try validate.parseResolver("udp://192.168.1.1:5300"),
&buf,
.{ .raw = .fromSeconds(1), .clock = .awake },
);
try testing.expectEqualStrings("udp://192.168.1.1:5300", v4.identity());
var buf6 = testBuf();
const v6: ForwardClient = .init(
try validate.parseResolver("tcp://[fd00::1]:5353"),
&buf6,
.{ .raw = .fromSeconds(1), .clock = .awake },
);
try testing.expectEqualStrings("tcp://[fd00::1]:5353", v6.identity());
// The borrow points into the client, not into `init`'s frame.
try testing.expect(@intFromPtr(v6.identity().ptr) >= @intFromPtr(&v6));
}
test "the stats struct starts at zero" {
const stats: ForwardClient.Stats = .{};
try testing.expectEqual(@as(u64, 0), stats.queries);
+2
View File
@@ -592,11 +592,13 @@ const FailingUpstream = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
_ = ptr;
_ = io;
_ = query;
_ = response_buf;
selected.* = "fake://failing-upstream";
return error.ConnectFailed;
}
+2
View File
@@ -351,8 +351,10 @@ const FakeUpstream = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
_ = io;
selected.* = "fake://dot-server-upstream";
const self: *FakeUpstream = @ptrCast(@alignCast(ptr));
if (self.reply.len > response_buf.len) return error.ResponseTooLarge;
@memcpy(response_buf[0..self.reply.len], self.reply);
+1390 -183
View File
File diff suppressed because it is too large Load Diff
+9 -3
View File
@@ -33,6 +33,7 @@ const handler = @import("handler.zig");
const header = @import("../dns/header.zig");
const local_tables = @import("local_tables.zig");
const logger_mod = @import("../storage/logger.zig");
const provenance = @import("../storage/provenance.zig");
const manager = @import("../filter/manager.zig");
const matcher = @import("../filter/matcher.zig");
const migrations = @import("../storage/migrations.zig");
@@ -183,8 +184,10 @@ const FakeUpstream = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
_ = io;
selected.* = "fake://phase7-upstream";
const self: *FakeUpstream = @ptrCast(@alignCast(ptr));
_ = self.calls.fetchAdd(1, .monotonic);
@@ -346,7 +349,8 @@ test "S7 case 1: a blocked domain is answered with the zero address and logged"
try testing.expectEqual(@as(usize, 1), logged.len);
try testing.expectEqual(true, logged[0].blocked);
try testing.expectEqualStrings("ads.example.com", logged[0].domain());
try testing.expectEqualStrings("rule_block_exact", logged[0].blockReason());
try testing.expectEqual(provenance.PolicyAction.block, logged[0].policy_action);
try testing.expectEqual(provenance.PolicyReason.rule_block_exact, logged[0].policy_reason);
try testing.expectEqualStrings("127.0.0.1", logged[0].clientIp());
}
@@ -593,7 +597,7 @@ test "S7 case 5: a cached answer comes back with a fresh id, an aged ttl and a l
const logged = drainLog(&lg, io, &entries);
try testing.expectEqual(@as(usize, 3), logged.len);
try testing.expectEqual(@as(?bool, false), logged[0].cache_hit);
try testing.expectEqualStrings("pool", logged[0].upstream());
try testing.expectEqualStrings("fake://phase7-upstream", logged[0].upstream());
try testing.expectEqual(@as(?bool, true), logged[1].cache_hit);
try testing.expectEqualStrings("", logged[1].upstream());
try testing.expectEqual(@as(?bool, true), logged[2].cache_hit);
@@ -651,7 +655,9 @@ test "S7 case 6: a cname into a blocked target blocks the original question" {
const logged = drainLog(&lg, io, &entries);
try testing.expectEqual(@as(usize, 1), logged.len);
try testing.expectEqual(true, logged[0].blocked);
try testing.expectEqualStrings("cname:rule_block_exact", logged[0].blockReason());
// The reason describes the target's own decision; milestone 28 S3 adds the
// target name that says a CNAME chain was followed.
try testing.expectEqual(provenance.PolicyReason.rule_block_exact, logged[0].policy_reason);
try testing.expectEqualStrings("cdn.example.com", logged[0].domain());
}
+4
View File
@@ -114,8 +114,10 @@ const GoodUpstream = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
_ = io;
selected.* = "fake://good-upstream";
const self: *GoodUpstream = @ptrCast(@alignCast(ptr));
_ = self.calls.fetchAdd(1, .monotonic);
return answerQuery(query, response_buf);
@@ -138,8 +140,10 @@ const FaultyUpstream = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
_ = io;
selected.* = "fake://faulty-upstream";
const self: *FaultyUpstream = @ptrCast(@alignCast(ptr));
const seen = self.calls.fetchAdd(1, .monotonic);
if (seen < self.fail_first) return self.fault;
@@ -72,8 +72,10 @@ const FakeUpstream = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
_ = io;
selected.* = "fake://tcp-server-upstream";
const self: *FakeUpstream = @ptrCast(@alignCast(ptr));
if (self.reply.len > response_buf.len) return error.ResponseTooLarge;
@memcpy(response_buf[0..self.reply.len], self.reply);
@@ -73,8 +73,10 @@ const FakeUpstream = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
_ = io;
selected.* = "fake://udp-server-upstream";
const self: *FakeUpstream = @ptrCast(@alignCast(ptr));
if (self.reply.len > response_buf.len) return error.ResponseTooLarge;
@memcpy(response_buf[0..self.reply.len], self.reply);
+416 -53
View File
@@ -34,8 +34,12 @@ const std = @import("std");
const db = @import("db.zig");
const disk_monitor = @import("disk_monitor.zig");
const events = @import("events.zig");
const limits = @import("../config/limits.zig");
const model = @import("../config/model.zig");
const provenance = @import("provenance.zig");
const queries_repo = @import("repositories/queries_repo.zig");
const regex = @import("../filter/regex.zig");
const safe_url = @import("../safe_url.zig");
/// Named `scope` rather than `log`: `Logger.log` is the enqueue entry point,
/// and the two names collide inside the struct.
@@ -58,10 +62,32 @@ pub const gate_retry_s = 1;
pub const max_domain_len = 253;
/// RFC 5952 text of any IPv6 address, zone identifier included.
pub const max_client_len = 45;
pub const max_reason_len = 32;
pub const max_upstream_len = 64;
/// `matched` holds the rule that decided the query, and the widest rule the
/// configuration accepts is a regex pattern at `regex.max_pattern_len`. It does
/// not fit a `u8` length, which is why this one field carries a `u16`.
pub const max_matched_len = regex.max_pattern_len;
/// The redacted resolver identity of the exchange that actually happened. Two
/// bounds apply and the buffer takes the larger, so neither producer truncates:
/// the longest well-formed `scheme://host:port` with a maximal host, and
/// `safe_url.redact`'s own output bound (`max_len` plus the `...` it appends
/// when it truncates).
pub const max_upstream_len = @max(
"https://".len + max_domain_len + ":65535".len,
safe_url.max_len + 3,
);
/// `cname_target`, `safe_search_target` and `forward_zone` each hold a domain
/// name, so they are all the same width as `domain`.
const max_name_len = max_domain_len;
/// One row on its way to `query_log`, carrying its own bytes.
///
/// Every field is by value: `Io.Queue` copies elements as raw bytes, so nothing
/// here may borrow from the query that produced it. The buffer widths above are
/// therefore the row's real storage cost, multiplied by the queue capacity —
/// see `query_log_buffer_max`.
pub const Entry = struct {
timestamp: i64,
domain_buf: [max_domain_len]u8,
@@ -69,28 +95,68 @@ pub const Entry = struct {
client_buf: [max_client_len]u8,
client_len: u8,
qtype: ?u16,
qclass: u16,
/// The client-visible RCODE. Twelve bits, not four: an EDNS extended code
/// carries eight more bits in the OPT record than the header's four. The
/// type is the enforcement — the column's `CHECK` in `querylog_schema.ddl`
/// bounds the same value against every other writer of the file.
rcode: u12,
blocked: bool,
reason_buf: [max_reason_len]u8,
reason_len: u8,
response_time_us: ?i64,
cache_hit: ?bool,
upstream_buf: [max_upstream_len]u8,
upstream_len: u8,
upstream_len: u16,
group_id: ?i64,
group_buf: [limits.max_group_name_len]u8,
group_len: u8,
policy_action: provenance.PolicyAction,
policy_reason: provenance.PolicyReason,
matched_buf: [max_matched_len]u8,
matched_len: u16,
source_id: ?i64,
source_buf: [limits.max_source_name_len]u8,
source_len: u8,
cname_buf: [max_name_len]u8,
cname_len: u8,
safe_search_buf: [max_name_len]u8,
safe_search_len: u8,
route_kind: provenance.RouteKind,
forward_zone_buf: [max_name_len]u8,
forward_zone_len: u8,
/// The borrowed shape of an entry. `init` copies out of it, so a caller can
/// build one from slices that die with the query.
///
/// Every text field defaults to `""`, which reaches a nullable column as
/// NULL. The three fields with no sensible empty value — the two enums and
/// the route — default to what a query the pipeline has not yet explained
/// would honestly say about itself.
pub const Fields = struct {
timestamp: i64,
domain: []const u8,
client_ip: []const u8,
qtype: ?u16 = null,
qclass: u16 = 0,
rcode: u12 = 0,
blocked: bool = false,
/// Empty means "no reason", which reaches the database as NULL.
block_reason: []const u8 = "",
response_time_us: ?i64 = null,
cache_hit: ?bool = null,
/// Empty means "no upstream", which reaches the database as NULL.
/// Empty means "no upstream was attempted", which reaches the database
/// as NULL. Already redacted by the caller.
upstream: []const u8 = "",
group_id: ?i64 = null,
group_name: []const u8 = "",
policy_action: provenance.PolicyAction = .not_evaluated,
policy_reason: provenance.PolicyReason = .no_match,
matched: []const u8 = "",
source_id: ?i64 = null,
source_name: []const u8 = "",
cname_target: []const u8 = "",
safe_search_target: []const u8 = "",
route_kind: provenance.RouteKind = .upstream,
forward_zone: []const u8 = "",
};
/// Copies each string in, truncated to what its buffer holds. A name longer
@@ -104,27 +170,61 @@ pub const Entry = struct {
.client_buf = undefined,
.client_len = 0,
.qtype = f.qtype,
.qclass = f.qclass,
.rcode = f.rcode,
.blocked = f.blocked,
.reason_buf = undefined,
.reason_len = 0,
.response_time_us = f.response_time_us,
.cache_hit = f.cache_hit,
.upstream_buf = undefined,
.upstream_len = 0,
.group_id = f.group_id,
.group_buf = undefined,
.group_len = 0,
.policy_action = f.policy_action,
.policy_reason = f.policy_reason,
.matched_buf = undefined,
.matched_len = 0,
.source_id = f.source_id,
.source_buf = undefined,
.source_len = 0,
.cname_buf = undefined,
.cname_len = 0,
.safe_search_buf = undefined,
.safe_search_len = 0,
.route_kind = f.route_kind,
.forward_zone_buf = undefined,
.forward_zone_len = 0,
};
entry.setDomain(f.domain);
entry.setClientIp(f.client_ip);
entry.reason_len = copyInto(&entry.reason_buf, f.block_reason);
entry.upstream_len = copyInto(&entry.upstream_buf, f.upstream);
copyInto(&entry.upstream_buf, &entry.upstream_len, f.upstream);
copyInto(&entry.group_buf, &entry.group_len, f.group_name);
entry.setMatched(f.matched);
copyInto(&entry.source_buf, &entry.source_len, f.source_name);
entry.setCnameTarget(f.cname_target);
entry.setSafeSearchTarget(f.safe_search_target);
copyInto(&entry.forward_zone_buf, &entry.forward_zone_len, f.forward_zone);
return entry;
}
pub fn setDomain(self: *Entry, value: []const u8) void {
self.domain_len = copyInto(&self.domain_buf, value);
copyInto(&self.domain_buf, &self.domain_len, value);
}
pub fn setClientIp(self: *Entry, value: []const u8) void {
self.client_len = copyInto(&self.client_buf, value);
copyInto(&self.client_buf, &self.client_len, value);
}
pub fn setMatched(self: *Entry, value: []const u8) void {
copyInto(&self.matched_buf, &self.matched_len, value);
}
pub fn setCnameTarget(self: *Entry, value: []const u8) void {
copyInto(&self.cname_buf, &self.cname_len, value);
}
pub fn setSafeSearchTarget(self: *Entry, value: []const u8) void {
copyInto(&self.safe_search_buf, &self.safe_search_len, value);
}
pub fn domain(self: *const Entry) []const u8 {
@@ -135,19 +235,56 @@ pub const Entry = struct {
return self.client_buf[0..self.client_len];
}
pub fn blockReason(self: *const Entry) []const u8 {
return self.reason_buf[0..self.reason_len];
}
pub fn upstream(self: *const Entry) []const u8 {
return self.upstream_buf[0..self.upstream_len];
}
pub fn groupName(self: *const Entry) []const u8 {
return self.group_buf[0..self.group_len];
}
pub fn matched(self: *const Entry) []const u8 {
return self.matched_buf[0..self.matched_len];
}
pub fn sourceName(self: *const Entry) []const u8 {
return self.source_buf[0..self.source_len];
}
pub fn cnameTarget(self: *const Entry) []const u8 {
return self.cname_buf[0..self.cname_len];
}
pub fn safeSearchTarget(self: *const Entry) []const u8 {
return self.safe_search_buf[0..self.safe_search_len];
}
pub fn forwardZone(self: *const Entry) []const u8 {
return self.forward_zone_buf[0..self.forward_zone_len];
}
};
fn copyInto(buf: []u8, value: []const u8) u8 {
/// The memory budget the queue is allowed to occupy. `Entry` travels by value,
/// so the composition root allocates `query_log_buffer_max` of them in full at
/// boot (`app.zig`) and the SSE hub embeds a ring of them per subscriber.
const queue_budget_bytes = 64 * 1024 * 1024;
/// The ceiling `config/validate.zig` enforces on `logging.query_log_buffer_max`,
/// derived from the width of `Entry` rather than picked.
///
/// The provenance columns of milestone 28 roughly tripled `Entry`, so the bound
/// that matters is bytes, not entries: an operator who asks for a million
/// entries is asking for well over a gigabyte of queue. This is a sanity bound,
/// not a memory-fit guarantee — what actually fits depends on the box.
pub const query_log_buffer_max: u32 = @intCast(queue_budget_bytes / @sizeOf(Entry));
/// Copies as much of `value` as `buf` holds, and stores the length through
/// `len`. `len`'s type never bounds anything — `buf.len` does — so the same
/// helper serves the `u8` fields and the `u16` ones.
fn copyInto(buf: []u8, len: anytype, value: []const u8) void {
const n = @min(buf.len, value.len);
@memcpy(buf[0..n], value[0..n]);
return @intCast(n);
len.* = @intCast(n);
}
/// The row borrows from `entry`, which must outlive the `writeBatch` call.
@@ -157,11 +294,23 @@ fn toRow(entry: *const Entry) queries_repo.Row {
.domain = entry.domain(),
.client_ip = entry.clientIp(),
.qtype = entry.qtype,
.qclass = entry.qclass,
.rcode = entry.rcode,
.blocked = entry.blocked,
.block_reason = emptyAsNull(entry.blockReason()),
.response_time_us = entry.response_time_us,
.cache_hit = entry.cache_hit,
.upstream = emptyAsNull(entry.upstream()),
.group_id = entry.group_id,
.group_name = emptyAsNull(entry.groupName()),
.policy_action = entry.policy_action,
.policy_reason = entry.policy_reason,
.matched = emptyAsNull(entry.matched()),
.source_id = entry.source_id,
.source_name = emptyAsNull(entry.sourceName()),
.cname_target = emptyAsNull(entry.cnameTarget()),
.safe_search_target = emptyAsNull(entry.safeSearchTarget()),
.route_kind = entry.route_kind,
.forward_zone = emptyAsNull(entry.forwardZone()),
};
}
@@ -231,9 +380,23 @@ pub const Logger = struct {
/// and hands the result to every consumer, so nothing downstream — the
/// database or the event stream — can observe a value the operator asked
/// to hide.
/// `hide_domains` covers every field derived from the query name, not just
/// `domain`: a matched wildcard, a CNAME target and a safe-search target
/// each name the very thing the operator asked to keep out of the log.
///
/// `forward_zone`, `group_name` and `source_name` stay visible. They are
/// configuration labels the operator wrote, identical on every row that
/// hits them, and they say nothing about which name a client looked up.
pub fn transformed(self: *const Logger, entry: Entry) Entry {
var out = entry;
if (self.cfg.hide_domains) out.setDomain(hidden_marker);
if (self.cfg.hide_domains) {
out.setDomain(hidden_marker);
// Only where there is something to hide: an empty field means the
// query had no such value, and writing a marker would claim it did.
if (out.matched_len != 0) out.setMatched(hidden_marker);
if (out.cname_len != 0) out.setCnameTarget(hidden_marker);
if (out.safe_search_len != 0) out.setSafeSearchTarget(hidden_marker);
}
if (self.cfg.hide_client_ips) out.setClientIp(hidden_marker);
return out;
}
@@ -574,6 +737,34 @@ fn sampleEntry(timestamp: i64, domain: []const u8) Entry {
});
}
/// Every provenance field set to a distinct recognisable value, so a test that
/// loses one loses it visibly.
fn fullFields(timestamp: i64) Entry.Fields {
return .{
.timestamp = timestamp,
.domain = "ads.example.com",
.client_ip = "2001:db8::1",
.qtype = 28,
.qclass = 1,
.rcode = 3,
.blocked = true,
.response_time_us = 42,
.cache_hit = true,
.upstream = "https://dns.example/dns-query",
.group_id = 7,
.group_name = "kids",
.policy_action = .block,
.policy_reason = .blocklist_wildcard,
.matched = "*.ads.example",
.source_id = 3,
.source_name = "steven black",
.cname_target = "tracker.cdn.example",
.safe_search_target = "forcesafesearch.google.com",
.route_kind = .blocked,
.forward_zone = "home.arpa",
};
}
fn openLog() !db.Db {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer database.close();
@@ -583,44 +774,72 @@ fn openLog() !db.Db {
}
test "an entry carries its own bytes and reads them back" {
const entry: Entry = .init(.{
.timestamp = 1700000000,
.domain = "ads.example.com",
.client_ip = "2001:db8::1",
.qtype = 28,
.blocked = true,
.block_reason = "blocklist",
.response_time_us = 42,
.cache_hit = true,
.upstream = "dns.example",
});
const entry: Entry = .init(fullFields(1700000000));
try testing.expectEqualStrings("ads.example.com", entry.domain());
try testing.expectEqualStrings("2001:db8::1", entry.clientIp());
try testing.expectEqualStrings("blocklist", entry.blockReason());
try testing.expectEqualStrings("dns.example", entry.upstream());
try testing.expectEqualStrings("https://dns.example/dns-query", entry.upstream());
try testing.expectEqual(@as(?u16, 28), entry.qtype);
try testing.expectEqual(@as(u16, 1), entry.qclass);
try testing.expectEqual(@as(u12, 3), entry.rcode);
try testing.expect(entry.blocked);
try testing.expectEqual(@as(?i64, 42), entry.response_time_us);
try testing.expectEqual(@as(?bool, true), entry.cache_hit);
try testing.expectEqual(@as(?i64, 7), entry.group_id);
try testing.expectEqualStrings("kids", entry.groupName());
try testing.expectEqual(provenance.PolicyAction.block, entry.policy_action);
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, entry.policy_reason);
try testing.expectEqualStrings("*.ads.example", entry.matched());
try testing.expectEqual(@as(?i64, 3), entry.source_id);
try testing.expectEqualStrings("steven black", entry.sourceName());
try testing.expectEqualStrings("tracker.cdn.example", entry.cnameTarget());
try testing.expectEqualStrings("forcesafesearch.google.com", entry.safeSearchTarget());
try testing.expectEqual(provenance.RouteKind.blocked, entry.route_kind);
try testing.expectEqualStrings("home.arpa", entry.forwardZone());
}
test "an oversize string is truncated to what its buffer holds" {
const long_domain = "a" ** 400;
const entry: Entry = .init(.{
.timestamp = 1,
.domain = long_domain,
.client_ip = "192.0.2.1",
.block_reason = "r" ** 64,
.upstream = "u" ** 128,
.domain = "a" ** 400,
.client_ip = "c" ** 80,
.upstream = "u" ** 600,
.group_name = "g" ** 200,
.matched = "m" ** 600,
.source_name = "s" ** 200,
.cname_target = "n" ** 400,
.safe_search_target = "f" ** 400,
.forward_zone = "z" ** 400,
});
try testing.expectEqual(@as(usize, max_domain_len), entry.domain().len);
try testing.expectEqual(@as(usize, max_reason_len), entry.blockReason().len);
try testing.expectEqual(@as(usize, max_client_len), entry.clientIp().len);
try testing.expectEqual(@as(usize, max_upstream_len), entry.upstream().len);
try testing.expectEqual(@as(usize, limits.max_group_name_len), entry.groupName().len);
try testing.expectEqual(@as(usize, max_matched_len), entry.matched().len);
try testing.expectEqual(@as(usize, limits.max_source_name_len), entry.sourceName().len);
try testing.expectEqual(@as(usize, max_name_len), entry.cnameTarget().len);
try testing.expectEqual(@as(usize, max_name_len), entry.safeSearchTarget().len);
try testing.expectEqual(@as(usize, max_name_len), entry.forwardZone().len);
try testing.expectEqualStrings("a" ** max_domain_len, entry.domain());
}
test "a 256-byte matched pattern is stored whole" {
// The widest rule the configuration accepts is a regex at
// `regex.max_pattern_len`, and it does not fit a `u8` length — which is the
// whole reason `matched_len` is a `u16`.
const widest = "p" ** regex.max_pattern_len;
const entry: Entry = .init(.{
.timestamp = 1,
.domain = "example.com",
.client_ip = "192.0.2.1",
.matched = widest,
});
try testing.expectEqualStrings(widest, entry.matched());
try testing.expectEqual(@as(u16, regex.max_pattern_len), entry.matched_len);
}
test "toRow maps the empty strings to null and passes the rest through" {
const bare: Entry = .init(.{
.timestamp = 7,
@@ -631,23 +850,85 @@ test "toRow maps the empty strings to null and passes the rest through" {
try testing.expectEqual(@as(i64, 7), bare_row.timestamp);
try testing.expectEqualStrings("example.com", bare_row.domain);
try testing.expectEqualStrings("192.0.2.5", bare_row.client_ip);
try testing.expectEqual(@as(?[]const u8, null), bare_row.block_reason);
try testing.expectEqual(@as(?[]const u8, null), bare_row.upstream);
try testing.expectEqual(@as(?u16, null), bare_row.qtype);
try testing.expectEqual(@as(?bool, null), bare_row.cache_hit);
// Every optional text field of an entry nothing filled in reaches its
// column as NULL rather than as an empty string.
try testing.expectEqual(@as(?[]const u8, null), bare_row.upstream);
try testing.expectEqual(@as(?[]const u8, null), bare_row.group_name);
try testing.expectEqual(@as(?[]const u8, null), bare_row.matched);
try testing.expectEqual(@as(?[]const u8, null), bare_row.source_name);
try testing.expectEqual(@as(?[]const u8, null), bare_row.cname_target);
try testing.expectEqual(@as(?[]const u8, null), bare_row.safe_search_target);
try testing.expectEqual(@as(?[]const u8, null), bare_row.forward_zone);
const full: Entry = .init(.{
.timestamp = 8,
.domain = "blocked.example",
.client_ip = "192.0.2.6",
.blocked = true,
.block_reason = "blocklist",
.upstream = "9.9.9.9",
});
const full: Entry = .init(fullFields(8));
const full_row = toRow(&full);
try testing.expect(full_row.blocked);
try testing.expectEqualStrings("blocklist", full_row.block_reason.?);
try testing.expectEqualStrings("9.9.9.9", full_row.upstream.?);
try testing.expectEqual(@as(u16, 1), full_row.qclass);
try testing.expectEqual(@as(u12, 3), full_row.rcode);
try testing.expectEqualStrings("https://dns.example/dns-query", full_row.upstream.?);
try testing.expectEqual(@as(?i64, 7), full_row.group_id);
try testing.expectEqualStrings("kids", full_row.group_name.?);
try testing.expectEqual(provenance.PolicyAction.block, full_row.policy_action);
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, full_row.policy_reason);
try testing.expectEqualStrings("*.ads.example", full_row.matched.?);
try testing.expectEqual(@as(?i64, 3), full_row.source_id);
try testing.expectEqualStrings("steven black", full_row.source_name.?);
try testing.expectEqualStrings("tracker.cdn.example", full_row.cname_target.?);
try testing.expectEqualStrings("forcesafesearch.google.com", full_row.safe_search_target.?);
try testing.expectEqual(provenance.RouteKind.blocked, full_row.route_kind);
try testing.expectEqualStrings("home.arpa", full_row.forward_zone.?);
}
test "an entry with every provenance field set survives the queue, toRow, insert and detailById" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var writer = try queries_repo.BatchWriter.init(&database);
defer writer.deinit();
var buf: [4]Entry = undefined;
var logger: Logger = .init(.{}, &buf);
// The widest `matched` the configuration accepts, carried the whole way:
// 256 bytes does not fit the `u8` length every other text field uses.
const widest_matched = "p" ** max_matched_len;
var fields = fullFields(1234);
fields.matched = widest_matched;
logger.log(io, .init(fields));
const queued = try logger.queue.getOne(io);
try logger.flush(io, &writer, &.{queued}, null);
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena_state.deinit();
const stored = (try queries_repo.detailById(&database, arena_state.allocator(), 1)).?;
try testing.expectEqual(@as(i64, 1234), stored.ts);
try testing.expectEqualStrings("ads.example.com", stored.domain);
try testing.expectEqualStrings("2001:db8::1", stored.client_ip);
try testing.expectEqual(@as(?u16, 28), stored.qtype);
try testing.expectEqual(@as(u16, 1), stored.qclass);
try testing.expectEqual(@as(u12, 3), stored.rcode);
try testing.expect(stored.blocked);
try testing.expectEqual(@as(?i64, 42), stored.response_time_us);
try testing.expectEqual(@as(?bool, true), stored.cache_hit);
try testing.expectEqualStrings("https://dns.example/dns-query", stored.upstream);
try testing.expectEqual(@as(?i64, 7), stored.group_id);
try testing.expectEqualStrings("kids", stored.group_name);
try testing.expectEqual(provenance.PolicyAction.block, stored.policy_action);
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, stored.policy_reason);
try testing.expectEqualStrings(widest_matched, stored.matched);
try testing.expectEqual(@as(?i64, 3), stored.source_id);
try testing.expectEqualStrings("steven black", stored.source_name);
try testing.expectEqualStrings("tracker.cdn.example", stored.cname_target);
try testing.expectEqualStrings("forcesafesearch.google.com", stored.safe_search_target);
try testing.expectEqual(provenance.RouteKind.blocked, stored.route_kind);
try testing.expectEqualStrings("home.arpa", stored.forward_zone);
}
test "log applies both privacy transforms before the entry reaches the queue" {
@@ -667,6 +948,88 @@ test "log applies both privacy transforms before the entry reaches the queue" {
try testing.expectEqual(@as(u64, 0), logger.queries_dropped.load(.monotonic));
}
test "hide_domains hides every query-derived name and leaves the labels alone" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var buf: [4]Entry = undefined;
var logger: Logger = .init(.{ .hide_domains = true }, &buf);
logger.log(io, .init(fullFields(1)));
const hidden = try logger.queue.getOne(io);
// Every field derived from the name the client asked for.
try testing.expectEqualStrings(hidden_marker, hidden.domain());
try testing.expectEqualStrings(hidden_marker, hidden.matched());
try testing.expectEqualStrings(hidden_marker, hidden.cnameTarget());
try testing.expectEqualStrings(hidden_marker, hidden.safeSearchTarget());
// The client is governed by `hide_client_ips`, not by this flag.
try testing.expectEqualStrings("2001:db8::1", hidden.clientIp());
// Configuration labels the operator wrote. They are identical on every row
// that hits them and say nothing about which name a client looked up.
try testing.expectEqualStrings("kids", hidden.groupName());
try testing.expectEqualStrings("steven black", hidden.sourceName());
try testing.expectEqualStrings("home.arpa", hidden.forwardZone());
try testing.expectEqualStrings("https://dns.example/dns-query", hidden.upstream());
}
test "hide_client_ips hides the client and nothing else" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var buf: [4]Entry = undefined;
var logger: Logger = .init(.{ .hide_client_ips = true }, &buf);
logger.log(io, .init(fullFields(1)));
const hidden = try logger.queue.getOne(io);
try testing.expectEqualStrings(hidden_marker, hidden.clientIp());
try testing.expectEqualStrings("ads.example.com", hidden.domain());
try testing.expectEqualStrings("*.ads.example", hidden.matched());
try testing.expectEqualStrings("tracker.cdn.example", hidden.cnameTarget());
try testing.expectEqualStrings("forcesafesearch.google.com", hidden.safeSearchTarget());
try testing.expectEqualStrings("kids", hidden.groupName());
try testing.expectEqualStrings("steven black", hidden.sourceName());
try testing.expectEqualStrings("home.arpa", hidden.forwardZone());
}
test "hide_domains writes no marker into a field the query never had" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var buf: [4]Entry = undefined;
var logger: Logger = .init(.{ .hide_domains = true }, &buf);
// An ordinary allowed query: no rule matched, no CNAME was uncloaked, no
// safe-search rewrite happened. Marking those "hidden" would claim the
// query had values it did not.
logger.log(io, sampleEntry(1, "plain.example"));
const hidden = try logger.queue.getOne(io);
try testing.expectEqualStrings(hidden_marker, hidden.domain());
try testing.expectEqualStrings("", hidden.matched());
try testing.expectEqualStrings("", hidden.cnameTarget());
try testing.expectEqualStrings("", hidden.safeSearchTarget());
}
test "the entry queue's worst case stays inside its byte budget" {
// The bound `config/validate.zig` enforces is derived from this, so the
// budget is what a maximal configuration can actually cost.
try testing.expect(@as(usize, query_log_buffer_max) * @sizeOf(Entry) <= queue_budget_bytes);
// One more entry than the ceiling would exceed it, so the ceiling is the
// largest value that fits rather than a round number under it.
try testing.expect((@as(usize, query_log_buffer_max) + 1) * @sizeOf(Entry) > queue_budget_bytes);
// The shipped default has to be comfortably inside the budget, or the
// out-of-the-box configuration is the one that spends it. At the widths
// above it costs about 17 MiB, roughly a quarter of the ceiling.
const default_max: usize = (model.Logging{}).query_log_buffer_max;
try testing.expect(default_max <= query_log_buffer_max);
try testing.expect(default_max * @sizeOf(Entry) <= queue_budget_bytes / 2);
}
test "log hides only the field its switch names" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
+13 -1
View File
@@ -154,11 +154,23 @@ fn writeRows(database: *db.Db, timestamps: []const i64, domain: []const u8) !voi
.domain = domain,
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 0,
.blocked = false,
.block_reason = null,
.response_time_us = null,
.cache_hit = null,
.upstream = null,
.group_id = 1,
.group_name = "default",
.policy_action = .allow,
.policy_reason = .no_match,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = .upstream,
.forward_zone = null,
};
}
try writer.writeBatch(rows[0..timestamps.len]);
+137
View File
@@ -0,0 +1,137 @@
//! The closed enums a `query_log` row stores to explain one query: what the
//! policy decided, why, and where the answer came from.
//!
//! They live in a module of their own because everything that touches a logged
//! row needs them — `storage/logger.zig`, `storage/repositories/queries_repo.zig`,
//! `server/handler.zig` and the web serializers — and `logger` already imports
//! `queries_repo`, so enums owned by either would close a loop.
//!
//! Each value is stored as its `@tagName` and read back through `parse`. The
//! read path treats an unrecognised value as a data error rather than passing
//! the text through: the column is a closed set, and a row that disagrees came
//! from something other than this schema.
const std = @import("std");
const matcher = @import("../filter/matcher.zig");
/// Whether the filtering policy reached a verdict on this query, and which one.
///
/// `not_evaluated` is the honest answer for a query the pipeline answered before
/// filtering could apply — a non-IN question, a paused resolver, a protocol
/// refusal — and is not the same as "allowed".
pub const PolicyAction = enum {
not_evaluated,
allow,
block,
};
/// Why the policy landed where it did.
///
/// The first nine are `filter/matcher.zig`'s serializable reasons, one for one.
/// The rest name the pipeline steps that decide a query without consulting the
/// matcher at all.
pub const PolicyReason = enum {
rule_allow_exact,
rule_block_exact,
rule_allow_wildcard,
rule_block_wildcard,
rule_allow_regex,
rule_block_regex,
blocklist_exception,
blocklist_domain,
blocklist_wildcard,
/// Answered from `local_records`, before filtering.
local_record,
/// Answered by a configured forward zone, before filtering.
forward_zone,
/// The question was not class IN, so no rule could apply to it.
non_in_class,
/// Filtering was paused.
paused,
/// No filter snapshot was published yet, so the query went unfiltered.
snapshot_unavailable,
/// The matcher evaluated the name and nothing matched.
no_match,
/// A syntactically parsed request refused on protocol grounds — BADVERS,
/// NOTIMP, a malformed EDNS OPT. It names a question, so it is logged, but
/// no policy ever saw it.
protocol_error,
};
/// Where the answer the client received came from.
pub const RouteKind = enum {
blocked,
local,
forward_zone,
upstream,
cache,
rejected,
};
/// The matcher's verdict in the query log's vocabulary.
///
/// Exhaustive on purpose: a reason added to the matcher must be given a stored
/// name here rather than silently reaching a row as something else. `.none` is
/// the matcher's "nothing matched", which is exactly `no_match`.
pub fn fromMatcherReason(reason: matcher.Reason) PolicyReason {
return switch (reason) {
.none => .no_match,
.rule_allow_exact => .rule_allow_exact,
.rule_block_exact => .rule_block_exact,
.rule_allow_wildcard => .rule_allow_wildcard,
.rule_block_wildcard => .rule_block_wildcard,
.rule_allow_regex => .rule_allow_regex,
.rule_block_regex => .rule_block_regex,
.blocklist_exception => .blocklist_exception,
.blocklist_domain => .blocklist_domain,
.blocklist_wildcard => .blocklist_wildcard,
};
}
/// Reads a stored `@tagName` back. `error.Mismatch` is the same error the
/// repositories return for a column that does not hold what the schema says it
/// holds, which is what an unknown value here is.
pub fn parse(comptime Enum: type, text: []const u8) error{Mismatch}!Enum {
return std.meta.stringToEnum(Enum, text) orelse error.Mismatch;
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const testing = std.testing;
test "every matcher reason has a stored name" {
// The mapping is checked here rather than trusted: a reason added to the
// matcher fails the exhaustive switch at compile time, and a reason
// *renamed* would still compile while changing what a row says.
inline for (@typeInfo(matcher.Reason).@"enum".fields) |field| {
const reason: matcher.Reason = @enumFromInt(field.value);
const mapped = fromMatcherReason(reason);
if (reason == .none) {
try testing.expectEqual(PolicyReason.no_match, mapped);
} else {
try testing.expectEqualStrings(field.name, @tagName(mapped));
}
}
}
test "parse round-trips every value of every enum" {
inline for ([_]type{ PolicyAction, PolicyReason, RouteKind }) |Enum| {
inline for (@typeInfo(Enum).@"enum".fields) |field| {
const value: Enum = @enumFromInt(field.value);
try testing.expectEqual(value, try parse(Enum, @tagName(value)));
}
}
}
test "parse rejects a value the schema does not define" {
try testing.expectError(error.Mismatch, parse(PolicyAction, "allowed"));
try testing.expectError(error.Mismatch, parse(PolicyAction, ""));
// A value that belongs to a different one of the three enums is no more
// acceptable than a typo.
try testing.expectError(error.Mismatch, parse(RouteKind, "allow"));
try testing.expectError(error.Mismatch, parse(PolicyReason, "cache"));
}
+153 -5
View File
@@ -22,8 +22,20 @@ const db = @import("db.zig");
const log = std.log.scoped(.querylog_schema);
/// Verbatim from PLAN §11.3. Multi-statement text — it goes through
/// `db.Db.exec`, never through `prepare`.
/// PLAN §11.3, plus the coverage watermark of milestone 28. Multi-statement
/// text — it goes through `db.Db.exec`, never through `prepare`.
///
/// The trailing INSERT seeds `querylog_meta`, which is part of the schema
/// rather than a later step: a `query_log` with no watermark beside it cannot
/// answer whether an empty result means "no queries" or "no history", and every
/// database this program reads from is created by executing this string.
/// `unixepoch()` is SQLite's own UTC clock, which is the clock every
/// `timestamp` in the file is measured against.
///
/// `available_since` starts one second *after* `created_at` on purpose. A row
/// logged in the same second the file was created is not evidence that the
/// second is completely covered, and the watermark's whole job is to be
/// conservative. From there it only ever advances, in `queries_repo.pruneOlderThan`.
pub const ddl: [:0]const u8 =
\\CREATE TABLE domains (
\\ id INTEGER PRIMARY KEY,
@@ -37,10 +49,23 @@ pub const ddl: [:0]const u8 =
\\ client_ip TEXT NOT NULL, -- text, not a FK: log rows are immutable facts
\\ qtype INTEGER,
\\ blocked INTEGER NOT NULL,
\\ block_reason TEXT,
\\ response_time_us INTEGER,
\\ cache_hit INTEGER,
\\ upstream TEXT
\\ upstream TEXT,
\\ qclass INTEGER NOT NULL,
\\ rcode INTEGER NOT NULL,
\\ group_id INTEGER, -- text/id pairs, not FKs: a renamed
\\ group_name TEXT, -- group must not rewrite history
\\ policy_action TEXT NOT NULL,
\\ policy_reason TEXT NOT NULL,
\\ matched TEXT,
\\ source_id INTEGER,
\\ source_name TEXT,
\\ cname_target TEXT,
\\ safe_search_target TEXT,
\\ route_kind TEXT NOT NULL,
\\ forward_zone TEXT,
\\ CHECK (rcode BETWEEN 0 AND 4095) -- twelve bits (RFC 6891 6.1.3)
\\);
\\CREATE INDEX idx_query_log_ts ON query_log(timestamp);
\\CREATE INDEX idx_query_log_client ON query_log(client_ip);
@@ -63,6 +88,14 @@ pub const ddl: [:0]const u8 =
\\ CHECK (failures >= 0)
\\) WITHOUT ROWID;
\\CREATE INDEX idx_upstream_minute_ts ON upstream_minute(minute_ts);
\\
\\CREATE TABLE querylog_meta (
\\ id INTEGER PRIMARY KEY CHECK (id = 1), -- one row, enforced by the schema
\\ created_at INTEGER NOT NULL,
\\ available_since INTEGER NOT NULL
\\);
\\INSERT INTO querylog_meta (id, created_at, available_since)
\\VALUES (1, unixepoch(), unixepoch() + 1);
;
/// `PRAGMA user_version` is a signed 32-bit field. Deriving the fingerprint from
@@ -299,7 +332,7 @@ test "ddl creates the query-log tables, the upstream-history tables and every in
try database.exec(ddl);
try testing.expectEqual(
@as(i64, 4),
@as(i64, 5),
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
);
const objects = [_][]const u8{
@@ -307,6 +340,7 @@ test "ddl creates the query-log tables, the upstream-history tables and every in
"idx_query_log_ts", "idx_query_log_client",
"idx_query_log_domain", "upstream_targets",
"upstream_minute", "idx_upstream_minute_ts",
"querylog_meta",
};
for (objects) |name| {
var stmt = try database.prepare("SELECT count(*) FROM sqlite_schema WHERE name = ?1");
@@ -317,6 +351,69 @@ test "ddl creates the query-log tables, the upstream-history tables and every in
}
}
test "the schema refuses an rcode outside twelve bits" {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
try database.exec(ddl);
try database.exec("INSERT INTO domains (id, domain) VALUES (1, 'a.example');");
var stmt = try database.prepare(
\\INSERT INTO query_log
\\ (timestamp, domain_id, client_ip, blocked, qclass, rcode,
\\ policy_action, policy_reason, route_kind)
\\VALUES (1, 1, '10.0.0.1', 0, 1, ?1, 'not_evaluated', 'no_match', 'upstream')
);
defer stmt.deinit();
// The whole range an EDNS extended RCODE can express, and nothing wider:
// the producers are `u12`, and this is what stops any other writer — a
// hand-run UPDATE included — from putting a value in the column that the
// read path would have to reject.
for ([_]i64{ 0, 4095 }) |accepted| {
try stmt.reset();
try stmt.bindInt(1, accepted);
try stmt.exec();
}
for ([_]i64{ -1, 4096, 65535 }) |refused| {
// `sqlite3_reset` repeats the error of the statement it is resetting,
// which for every iteration after the first is the constraint failure
// this loop just asserted — the same reason `BatchWriter.resetAll`
// discards it.
stmt.reset() catch {};
try stmt.bindInt(1, refused);
try testing.expectError(error.Constraint, stmt.exec());
}
try testing.expectEqual(@as(i64, 2), try database.queryInt("SELECT count(*) FROM query_log"));
}
test "querylog_meta is seeded with one row the schema will not let a second join" {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
try database.exec(ddl);
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM querylog_meta"));
const created = try database.queryInt("SELECT created_at FROM querylog_meta");
const since = try database.queryInt("SELECT available_since FROM querylog_meta");
// Conservative by exactly one second: a row logged in the creating second
// must not let a query claim that second is completely covered.
try testing.expectEqual(created + 1, since);
try testing.expect(created > 1_700_000_000);
// `CHECK (id = 1)` is what makes "the singleton row" a schema fact rather
// than a convention the read path has to defend against.
try testing.expectError(error.Constraint, database.exec(
"INSERT INTO querylog_meta (id, created_at, available_since) VALUES (2, 1, 1);",
));
try testing.expectError(error.Constraint, database.exec(
"INSERT INTO querylog_meta (id, created_at, available_since) VALUES (1, 1, 1);",
));
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM querylog_meta"));
}
test "the user_version statement stamps the fingerprint" {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
@@ -394,6 +491,57 @@ test "a recreate returns the aside name by value and a fresh create returns none
try tmp.dir.access(io, kept, .{});
}
test "a recreate resets coverage to the new file and keeps the old one aside" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var tmp = testing.tmpDir(.{ .iterate = true });
defer tmp.cleanup();
var path_buf: [path_buf_len]u8 = undefined;
const path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/querylog.db", .{tmp.sub_path});
var created = try open(io, std.Io.Dir.cwd(), path);
const first_coverage = try created.database.queryInt("SELECT available_since FROM querylog_meta");
// A row in the file the operator is about to lose.
try created.database.exec("INSERT INTO domains (domain) VALUES ('old.example');");
created.database.close();
// A healthy file this build's DDL no longer matches — the case milestone
// 28's own schema edit produces on every upgrade.
{
var stamped = try db.Db.open(path, .{ .mode = .read_write_existing });
defer stamped.close();
var sql_buf: [64]u8 = undefined;
try stamped.exec(try std.fmt.bufPrintZ(&sql_buf, "PRAGMA user_version = {d};", .{fingerprint +% 1}));
}
var recreated = try open(io, std.Io.Dir.cwd(), path);
defer recreated.database.close();
try testing.expectEqual(RecreateReason.fingerprint_mismatch, recreated.recreated.?);
// The name says the file was healthy and this build moved, not that it rotted.
try testing.expect(std.mem.indexOf(u8, recreated.aside(), ".schema-changed-") != null);
try tmp.dir.access(io, std.fs.path.basename(recreated.aside()), .{});
// Exactly one meta row, and coverage starts at the recreate rather than
// carrying the replaced file's promise forward.
try testing.expectEqual(
@as(i64, 1),
try recreated.database.queryInt("SELECT count(*) FROM querylog_meta"),
);
const new_coverage = try recreated.database.queryInt("SELECT available_since FROM querylog_meta");
try testing.expect(new_coverage >= first_coverage);
// Nothing of the old file came across: the history is genuinely gone, which
// is what the coverage start has to tell the operator.
try testing.expectEqual(
@as(i64, 0),
try recreated.database.queryInt("SELECT count(*) FROM domains"),
);
}
test "a clean reopen reports no recreate and no aside" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
+528 -72
View File
@@ -19,20 +19,42 @@ const std = @import("std");
const Allocator = std.mem.Allocator;
const db = @import("../db.zig");
const provenance = @import("../provenance.zig");
/// One `query_log` row. The logger applies the privacy transforms of PLAN
/// §11.4 before it builds this, so `domain` and `client_ip` are already
/// §11.4 before it builds this, so every domain-bearing field is already
/// whatever the operator agreed to store.
///
/// A `null` text field is a fact the query did not have — no upstream was
/// attempted, no rule matched, no CNAME was uncloaked — and reaches the column
/// as NULL. The three closed enums have no such state: every logged query has a
/// policy verdict, a reason for it and a route, even when the verdict is
/// "not evaluated".
pub const Row = struct {
timestamp: i64,
domain: []const u8,
client_ip: []const u8,
qtype: ?u16,
qclass: u16,
/// Twelve bits: the EDNS extended RCODE the client saw. The column's
/// `CHECK` bounds it to the same range, so a value this type cannot hold
/// is one the schema would have refused anyway.
rcode: u12,
blocked: bool,
block_reason: ?[]const u8,
response_time_us: ?i64,
cache_hit: ?bool,
upstream: ?[]const u8,
group_id: ?i64,
group_name: ?[]const u8,
policy_action: provenance.PolicyAction,
policy_reason: provenance.PolicyReason,
matched: ?[]const u8,
source_id: ?i64,
source_name: ?[]const u8,
cname_target: ?[]const u8,
safe_search_target: ?[]const u8,
route_kind: provenance.RouteKind,
forward_zone: ?[]const u8,
};
const insert_domain_sql = "INSERT OR IGNORE INTO domains (domain) VALUES (?1)";
@@ -41,9 +63,13 @@ const select_domain_sql = "SELECT id FROM domains WHERE domain = ?1";
const insert_row_sql =
\\INSERT INTO query_log
\\ (timestamp, domain_id, client_ip, qtype, blocked, block_reason,
\\ response_time_us, cache_hit, upstream)
\\VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
\\ (timestamp, domain_id, client_ip, qtype, blocked,
\\ response_time_us, cache_hit, upstream, qclass, rcode,
\\ group_id, group_name, policy_action, policy_reason, matched,
\\ source_id, source_name, cname_target, safe_search_target,
\\ route_kind, forward_zone)
\\VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10,
\\ ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21)
;
/// Owns the prepared statements of the flush loop. Init once, reuse per batch.
@@ -123,10 +149,22 @@ pub const BatchWriter = struct {
try stmt.bindText(3, row.client_ip);
try bindIntOrNull(stmt, 4, if (row.qtype) |v| @as(i64, v) else null);
try stmt.bindBool(5, row.blocked);
try stmt.bindTextOrNull(6, row.block_reason);
try bindIntOrNull(stmt, 7, row.response_time_us);
try bindIntOrNull(stmt, 8, if (row.cache_hit) |v| @as(i64, @intFromBool(v)) else null);
try stmt.bindTextOrNull(9, row.upstream);
try bindIntOrNull(stmt, 6, row.response_time_us);
try bindIntOrNull(stmt, 7, if (row.cache_hit) |v| @as(i64, @intFromBool(v)) else null);
try stmt.bindTextOrNull(8, row.upstream);
try stmt.bindInt(9, row.qclass);
try stmt.bindInt(10, row.rcode);
try bindIntOrNull(stmt, 11, row.group_id);
try stmt.bindTextOrNull(12, row.group_name);
try stmt.bindText(13, @tagName(row.policy_action));
try stmt.bindText(14, @tagName(row.policy_reason));
try stmt.bindTextOrNull(15, row.matched);
try bindIntOrNull(stmt, 16, row.source_id);
try stmt.bindTextOrNull(17, row.source_name);
try stmt.bindTextOrNull(18, row.cname_target);
try stmt.bindTextOrNull(19, row.safe_search_target);
try stmt.bindText(20, @tagName(row.route_kind));
try stmt.bindTextOrNull(21, row.forward_zone);
try stmt.exec();
}
@@ -144,17 +182,55 @@ fn bindIntOrNull(stmt: *db.Stmt, idx: c_int, value: ?i64) db.Error!void {
return stmt.bindNull(idx);
}
/// Deletes every `query_log` row strictly older than `cutoff_ts` and returns
/// how many went.
/// What one prune did, and where coverage now begins.
pub const PruneResult = struct {
deleted: i64,
/// The watermark after the prune, which is what a later `availableSince`
/// will return. Handed back so the caller need not re-read it.
available_since: i64,
};
/// Deletes every `query_log` row strictly older than `cutoff_ts` and advances
/// the coverage watermark to the same cutoff, in one transaction.
///
/// Orphaned `domains` rows stay: it is a dimension table, re-interning a name
/// costs one indexed insert, and §11.3 asks for no collection.
pub fn pruneOlderThan(database: *db.Db, cutoff_ts: i64) db.Error!i64 {
var stmt = try database.prepare("DELETE FROM query_log WHERE timestamp < ?1");
defer stmt.deinit();
try stmt.bindInt(1, cutoff_ts);
try stmt.exec();
return database.changes();
/// **The two are one operation, not two.** The watermark is the promise that
/// every query since it is still in the file; a delete that commits without the
/// advance breaks that promise, and an advance that commits without the delete
/// hides rows the file still holds. Either failure rolls both back, and the
/// caller retries the whole thing on its next pass.
///
/// The watermark never moves backward: `max` is what makes a prune with a
/// cutoff older than the file's own creation a no-op on it rather than a
/// regression. Orphaned `domains` rows stay — it is a dimension table,
/// re-interning a name costs one indexed insert, and §11.3 asks for no
/// collection.
pub fn pruneOlderThan(database: *db.Db, cutoff_ts: i64) db.Error!PruneResult {
var tx = try db.Tx.begin(database);
errdefer tx.rollback();
var deleting = try database.prepare("DELETE FROM query_log WHERE timestamp < ?1");
defer deleting.deinit();
try deleting.bindInt(1, cutoff_ts);
try deleting.exec();
const deleted = database.changes();
var advancing = try database.prepare(
"UPDATE querylog_meta SET available_since = max(available_since, ?1) WHERE id = 1",
);
defer advancing.deinit();
try advancing.bindInt(1, cutoff_ts);
try advancing.exec();
const watermark = try database.queryInt("SELECT available_since FROM querylog_meta WHERE id = 1");
try tx.commit();
return .{ .deleted = deleted, .available_since = watermark };
}
/// The oldest timestamp this file can still answer for. A query window that
/// starts before it is incomplete, and the API says so rather than charting the
/// gap as zero.
pub fn availableSince(database: *db.Db) db.Error!i64 {
return database.queryInt("SELECT available_since FROM querylog_meta WHERE id = 1");
}
/// `PRAGMA wal_checkpoint(TRUNCATE)`: moves the WAL into the database and
@@ -189,21 +265,60 @@ pub fn countDomains(database: *db.Db) db.Error!i64 {
/// One row of `GET /api/queries`, joined back through the `domains` dimension.
///
/// `block_reason` and `upstream` are nullable columns, and a NULL reads as `""`
/// the same convention `Stmt.columnText` already uses. Neither column is ever
/// written as an empty string (a reason is a word, an upstream is a URL), so the
/// mapping loses nothing and the API layer can treat `""` as "absent".
/// A summary projection, deliberately narrower than `QueryDetail`: the list is
/// a table the operator scans, and the full provenance of a row is one request
/// away at `GET /api/queries/{id}`.
///
/// The nullable text columns read a NULL as `""` — the same convention
/// `Stmt.columnText` already uses. None of them is ever written as an empty
/// string, so the mapping loses nothing and the API layer can treat `""` as
/// "absent".
pub const QueryRow = struct {
id: i64,
ts: i64,
domain: []const u8,
client_ip: []const u8,
qtype: ?u16,
qclass: u16,
rcode: u12,
blocked: bool,
block_reason: []const u8,
response_time_us: ?i64,
cache_hit: ?bool,
upstream: []const u8,
policy_action: provenance.PolicyAction,
policy_reason: provenance.PolicyReason,
route_kind: provenance.RouteKind,
};
/// Everything one `query_log` row records about one query, for
/// `GET /api/queries/{id}`.
///
/// Same NULL-reads-as-`""` convention as `QueryRow`, and the same closed enums:
/// a stored value the schema does not define is `error.Mismatch`, never passed
/// through as text.
pub const QueryDetail = struct {
id: i64,
ts: i64,
domain: []const u8,
client_ip: []const u8,
qtype: ?u16,
qclass: u16,
rcode: u12,
blocked: bool,
response_time_us: ?i64,
cache_hit: ?bool,
upstream: []const u8,
group_id: ?i64,
group_name: []const u8,
policy_action: provenance.PolicyAction,
policy_reason: provenance.PolicyReason,
matched: []const u8,
source_id: ?i64,
source_name: []const u8,
cname_target: []const u8,
safe_search_target: []const u8,
route_kind: provenance.RouteKind,
forward_zone: []const u8,
};
/// Every field is an independent narrowing; `null` means "do not filter on it".
@@ -230,7 +345,8 @@ pub const max_limit: u32 = 1000;
const select_head =
\\SELECT q.id, q.timestamp, d.domain, q.client_ip, q.qtype, q.blocked,
\\ q.block_reason, q.response_time_us, q.cache_hit, q.upstream
\\ q.response_time_us, q.cache_hit, q.upstream, q.qclass, q.rcode,
\\ q.policy_action, q.policy_reason, q.route_kind
\\ FROM query_log q JOIN domains d ON d.id = q.domain_id
;
@@ -337,15 +453,81 @@ pub fn selectQueries(database: *db.Db, arena: Allocator, filter: QueryFilter) db
.qtype = if (stmt.isNull(4)) null else std.math.cast(u16, stmt.columnInt(4)) orelse
return error.Mismatch,
.blocked = stmt.columnBool(5),
.block_reason = try stmt.columnTextAlloc(arena, 6),
.response_time_us = if (stmt.isNull(7)) null else stmt.columnInt(7),
.cache_hit = if (stmt.isNull(8)) null else stmt.columnBool(8),
.upstream = try stmt.columnTextAlloc(arena, 9),
.response_time_us = if (stmt.isNull(6)) null else stmt.columnInt(6),
.cache_hit = if (stmt.isNull(7)) null else stmt.columnBool(7),
.upstream = try stmt.columnTextAlloc(arena, 8),
.qclass = try columnU16(&stmt, 9),
.rcode = try columnU12(&stmt, 10),
.policy_action = try provenance.parse(provenance.PolicyAction, stmt.columnText(11)),
.policy_reason = try provenance.parse(provenance.PolicyReason, stmt.columnText(12)),
.route_kind = try provenance.parse(provenance.RouteKind, stmt.columnText(13)),
});
}
return out;
}
/// A `NOT NULL` integer column that the schema bounds to 16 bits. A value
/// outside that range means the row came from something other than this schema.
fn columnU16(stmt: *db.Stmt, col: c_int) db.Error!u16 {
return std.math.cast(u16, stmt.columnInt(col)) orelse error.Mismatch;
}
/// The `rcode` column, which the schema's `CHECK` bounds to twelve bits. This
/// build cannot write a wider value — the field is a `u12` all the way from the
/// handler — so a row that carries one was written by something else, and is
/// `error.Mismatch` rather than a value truncated into shape.
fn columnU12(stmt: *db.Stmt, col: c_int) db.Error!u12 {
return std.math.cast(u12, stmt.columnInt(col)) orelse error.Mismatch;
}
const select_detail_sql =
\\SELECT q.id, q.timestamp, d.domain, q.client_ip, q.qtype, q.blocked,
\\ q.response_time_us, q.cache_hit, q.upstream, q.qclass, q.rcode,
\\ q.group_id, q.group_name, q.policy_action, q.policy_reason,
\\ q.matched, q.source_id, q.source_name, q.cname_target,
\\ q.safe_search_target, q.route_kind, q.forward_zone
\\ FROM query_log q JOIN domains d ON d.id = q.domain_id
\\ WHERE q.id = ?1
;
/// One row's full provenance, or `null` when no row has that id — which is what
/// an id the operator kept from before a retention pass looks like, and is a
/// 404 rather than an error.
///
/// Every string is allocated from `arena`, on the same terms as
/// `selectQueries`.
pub fn detailById(database: *db.Db, arena: Allocator, id: i64) db.Error!?QueryDetail {
var stmt = try database.prepare(select_detail_sql);
defer stmt.deinit();
try stmt.bindInt(1, id);
if (!try stmt.step()) return null;
return .{
.id = stmt.columnInt(0),
.ts = stmt.columnInt(1),
.domain = try stmt.columnTextAlloc(arena, 2),
.client_ip = try stmt.columnTextAlloc(arena, 3),
.qtype = if (stmt.isNull(4)) null else try columnU16(&stmt, 4),
.blocked = stmt.columnBool(5),
.response_time_us = if (stmt.isNull(6)) null else stmt.columnInt(6),
.cache_hit = if (stmt.isNull(7)) null else stmt.columnBool(7),
.upstream = try stmt.columnTextAlloc(arena, 8),
.qclass = try columnU16(&stmt, 9),
.rcode = try columnU12(&stmt, 10),
.group_id = if (stmt.isNull(11)) null else stmt.columnInt(11),
.group_name = try stmt.columnTextAlloc(arena, 12),
.policy_action = try provenance.parse(provenance.PolicyAction, stmt.columnText(13)),
.policy_reason = try provenance.parse(provenance.PolicyReason, stmt.columnText(14)),
.matched = try stmt.columnTextAlloc(arena, 15),
.source_id = if (stmt.isNull(16)) null else stmt.columnInt(16),
.source_name = try stmt.columnTextAlloc(arena, 17),
.cname_target = try stmt.columnTextAlloc(arena, 18),
.safe_search_target = try stmt.columnTextAlloc(arena, 19),
.route_kind = try provenance.parse(provenance.RouteKind, stmt.columnText(20)),
.forward_zone = try stmt.columnTextAlloc(arena, 21),
};
}
/// Wraps `needle` in `%` and neutralises the two `LIKE` metacharacters, so a
/// user searching for `a_b` gets domains containing `a_b` and not domains
/// containing `axb`. The escape character escapes itself.
@@ -493,11 +675,23 @@ fn plainRow(timestamp: i64, domain: []const u8) Row {
.domain = domain,
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 0,
.blocked = false,
.block_reason = null,
.response_time_us = 1200,
.cache_hit = false,
.upstream = "9.9.9.9",
.group_id = 1,
.group_name = "default",
.policy_action = .allow,
.policy_reason = .no_match,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = .upstream,
.forward_zone = null,
};
}
@@ -509,6 +703,43 @@ fn domainIdOf(database: *db.Db, domain: []const u8) !i64 {
return stmt.columnInt(0);
}
test "a foreign row with an rcode wider than twelve bits is refused, not truncated" {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
// The shipped schema's `CHECK` makes this row impossible in a file this
// build created, so the table is built without it. The read path's job is
// to refuse a `querylog.db` that came from somewhere else rather than to
// narrow a value it cannot represent.
try database.exec(
\\CREATE TABLE domains (id INTEGER PRIMARY KEY, domain TEXT NOT NULL UNIQUE);
\\CREATE TABLE query_log (
\\ id INTEGER PRIMARY KEY, timestamp INTEGER NOT NULL,
\\ domain_id INTEGER NOT NULL, client_ip TEXT NOT NULL,
\\ qtype INTEGER, blocked INTEGER NOT NULL, response_time_us INTEGER,
\\ cache_hit INTEGER, upstream TEXT, qclass INTEGER NOT NULL,
\\ rcode INTEGER NOT NULL, group_id INTEGER, group_name TEXT,
\\ policy_action TEXT NOT NULL, policy_reason TEXT NOT NULL,
\\ matched TEXT, source_id INTEGER, source_name TEXT,
\\ cname_target TEXT, safe_search_target TEXT,
\\ route_kind TEXT NOT NULL, forward_zone TEXT
\\);
\\INSERT INTO domains (id, domain) VALUES (1, 'a.example');
\\INSERT INTO query_log
\\ (id, timestamp, domain_id, client_ip, blocked, qclass, rcode,
\\ policy_action, policy_reason, route_kind)
\\VALUES (1, 10, 1, '192.0.2.10', 0, 1, 4096, 'allow', 'no_match', 'upstream');
);
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
try testing.expectError(error.Mismatch, selectQueries(&database, arena, .{}));
try testing.expectError(error.Mismatch, detailById(&database, arena, 1));
}
test "writeBatch inserts every row and interns each domain once" {
var database = try openLog();
defer database.close();
@@ -553,7 +784,7 @@ test "a second batch reuses the interned domain id" {
);
}
test "nullable columns round-trip a value and a null" {
test "every column round-trips a value and a null" {
var database = try openLog();
defer database.close();
var writer = try BatchWriter.init(&database);
@@ -565,54 +796,125 @@ test "nullable columns round-trip a value and a null" {
.domain = "blocked.example",
.client_ip = "2001:db8::1",
.qtype = 28,
.qclass = 1,
.rcode = 3,
.blocked = true,
.block_reason = "blocklist",
.response_time_us = 42,
.cache_hit = true,
.upstream = "dns.example",
.upstream = "https://dns.example/dns-query",
.group_id = 7,
.group_name = "kids",
.policy_action = .block,
.policy_reason = .blocklist_wildcard,
.matched = "*.ads.example",
.source_id = 3,
.source_name = "steven black",
.cname_target = "tracker.cdn.example",
.safe_search_target = "forcesafesearch.google.com",
.route_kind = .blocked,
.forward_zone = "home.arpa",
},
// Every nullable column absent at once, which is the shape of a query
// the pipeline answered before any of them applied.
.{
.timestamp = 11,
.domain = "quiet.example",
.client_ip = "hidden",
.qtype = null,
.qclass = 3,
.rcode = 0,
.blocked = false,
.block_reason = null,
.response_time_us = null,
.cache_hit = null,
.upstream = null,
.group_id = null,
.group_name = null,
.policy_action = .not_evaluated,
.policy_reason = .non_in_class,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = .upstream,
.forward_zone = null,
},
});
var stmt = try database.prepare(
\\SELECT d.domain, q.client_ip, q.qtype, q.blocked, q.block_reason,
\\ q.response_time_us, q.cache_hit, q.upstream
\\ FROM query_log q JOIN domains d ON d.id = q.domain_id
\\ ORDER BY q.timestamp
);
defer stmt.deinit();
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
try testing.expect(try stmt.step());
try testing.expectEqualStrings("blocked.example", stmt.columnText(0));
try testing.expectEqualStrings("2001:db8::1", stmt.columnText(1));
try testing.expectEqual(@as(i64, 28), stmt.columnInt(2));
try testing.expect(stmt.columnBool(3));
try testing.expectEqualStrings("blocklist", stmt.columnText(4));
try testing.expectEqual(@as(i64, 42), stmt.columnInt(5));
try testing.expect(stmt.columnBool(6));
try testing.expectEqualStrings("dns.example", stmt.columnText(7));
const full = (try detailById(&database, arena, 1)).?;
try testing.expectEqualStrings("blocked.example", full.domain);
try testing.expectEqualStrings("2001:db8::1", full.client_ip);
try testing.expectEqual(@as(?u16, 28), full.qtype);
try testing.expectEqual(@as(u16, 1), full.qclass);
try testing.expectEqual(@as(u12, 3), full.rcode);
try testing.expect(full.blocked);
try testing.expectEqual(@as(?i64, 42), full.response_time_us);
try testing.expectEqual(@as(?bool, true), full.cache_hit);
try testing.expectEqualStrings("https://dns.example/dns-query", full.upstream);
try testing.expectEqual(@as(?i64, 7), full.group_id);
try testing.expectEqualStrings("kids", full.group_name);
try testing.expectEqual(provenance.PolicyAction.block, full.policy_action);
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, full.policy_reason);
try testing.expectEqualStrings("*.ads.example", full.matched);
try testing.expectEqual(@as(?i64, 3), full.source_id);
try testing.expectEqualStrings("steven black", full.source_name);
try testing.expectEqualStrings("tracker.cdn.example", full.cname_target);
try testing.expectEqualStrings("forcesafesearch.google.com", full.safe_search_target);
try testing.expectEqual(provenance.RouteKind.blocked, full.route_kind);
try testing.expectEqualStrings("home.arpa", full.forward_zone);
try testing.expect(try stmt.step());
try testing.expectEqualStrings("quiet.example", stmt.columnText(0));
try testing.expectEqualStrings("hidden", stmt.columnText(1));
try testing.expect(stmt.isNull(2));
try testing.expect(!stmt.columnBool(3));
try testing.expect(stmt.isNull(4));
try testing.expect(stmt.isNull(5));
try testing.expect(stmt.isNull(6));
try testing.expect(stmt.isNull(7));
// A NULL text column reads as the empty string, by the documented
// convention; a NULL integer stays null, because 0 is a real id.
const bare = (try detailById(&database, arena, 2)).?;
try testing.expectEqual(@as(?u16, null), bare.qtype);
try testing.expectEqual(@as(u16, 3), bare.qclass);
try testing.expectEqual(@as(?i64, null), bare.response_time_us);
try testing.expectEqual(@as(?bool, null), bare.cache_hit);
try testing.expectEqualStrings("", bare.upstream);
try testing.expectEqual(@as(?i64, null), bare.group_id);
try testing.expectEqualStrings("", bare.group_name);
try testing.expectEqual(provenance.PolicyAction.not_evaluated, bare.policy_action);
try testing.expectEqual(provenance.PolicyReason.non_in_class, bare.policy_reason);
try testing.expectEqualStrings("", bare.matched);
try testing.expectEqual(@as(?i64, null), bare.source_id);
try testing.expectEqualStrings("", bare.source_name);
try testing.expectEqualStrings("", bare.cname_target);
try testing.expectEqualStrings("", bare.safe_search_target);
try testing.expectEqual(provenance.RouteKind.upstream, bare.route_kind);
try testing.expectEqualStrings("", bare.forward_zone);
}
try testing.expect(!try stmt.step());
test "detailById returns null for an id no row has" {
var database = try openLog();
defer database.close();
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena_state.deinit();
try seed(&database, &.{plainRow(10, "a.example")});
// An id from before a retention pass looks exactly like this, and is a 404
// rather than an error.
try testing.expectEqual(@as(?QueryDetail, null), try detailById(&database, arena_state.allocator(), 2));
try testing.expectEqual(@as(?QueryDetail, null), try detailById(&database, arena_state.allocator(), 0));
try testing.expect((try detailById(&database, arena_state.allocator(), 1)) != null);
}
test "a stored enum value the schema does not define is a data error, not a passthrough" {
var database = try openLog();
defer database.close();
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
try seed(&database, &.{plainRow(10, "a.example")});
try database.exec("UPDATE query_log SET policy_reason = 'whatever' WHERE id = 1;");
try testing.expectError(error.Mismatch, detailById(&database, arena, 1));
try testing.expectError(error.Mismatch, selectQueries(&database, arena, .{}));
}
test "an empty batch writes nothing and opens no transaction" {
@@ -644,7 +946,7 @@ test "pruneOlderThan deletes strictly older rows and returns the count" {
plainRow(300, "fresh.example"),
});
try testing.expectEqual(@as(i64, 2), try pruneOlderThan(&database, 200));
try testing.expectEqual(@as(i64, 2), (try pruneOlderThan(&database, 200)).deleted);
try testing.expectEqual(@as(i64, 2), try countRows(&database));
// The row exactly at the cutoff stays.
try testing.expectEqual(
@@ -652,7 +954,123 @@ test "pruneOlderThan deletes strictly older rows and returns the count" {
try database.queryInt("SELECT count(*) FROM query_log WHERE timestamp = 200"),
);
// A second pass over the same cutoff finds nothing left to do.
try testing.expectEqual(@as(i64, 0), try pruneOlderThan(&database, 200));
try testing.expectEqual(@as(i64, 0), (try pruneOlderThan(&database, 200)).deleted);
}
test "a prune advances the coverage watermark to its own cutoff" {
var database = try openLog();
defer database.close();
var writer = try BatchWriter.init(&database);
defer writer.deinit();
// The seeded watermark is `created_at + 1`, which is now-ish; the cutoffs
// below are all in the past, so they start out behind it.
const start = try availableSince(&database);
try writer.writeBatch(&.{ plainRow(start + 100, "a.example"), plainRow(start + 300, "b.example") });
const first = try pruneOlderThan(&database, start + 200);
try testing.expectEqual(@as(i64, 1), first.deleted);
try testing.expectEqual(start + 200, first.available_since);
try testing.expectEqual(start + 200, try availableSince(&database));
// A prune that deletes nothing still advances: the window it swept is
// covered whether or not it held rows.
const second = try pruneOlderThan(&database, start + 250);
try testing.expectEqual(@as(i64, 0), second.deleted);
try testing.expectEqual(start + 250, try availableSince(&database));
}
test "the watermark never moves backward" {
var database = try openLog();
defer database.close();
const start = try availableSince(&database);
const advanced = try pruneOlderThan(&database, start + 1000);
try testing.expectEqual(start + 1000, advanced.available_since);
// A shortened `retention_days`, a clock that stepped back, a pass with a
// stale cutoff: none of them may widen the promise the file makes.
for ([_]i64{ start + 999, start, start - 100_000, 0 }) |older| {
const result = try pruneOlderThan(&database, older);
try testing.expectEqual(start + 1000, result.available_since);
try testing.expectEqual(start + 1000, try availableSince(&database));
}
}
test "a failed delete leaves both the rows and the watermark untouched" {
var database = try openLog();
defer database.close();
var writer = try BatchWriter.init(&database);
defer writer.deinit();
const start = try availableSince(&database);
try writer.writeBatch(&.{plainRow(start - 100, "old.example")});
try database.exec(
\\CREATE TRIGGER refuse_delete BEFORE DELETE ON query_log
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
);
try testing.expectError(error.Constraint, pruneOlderThan(&database, start + 1000));
try testing.expectEqual(@as(i64, 1), try countRows(&database));
try testing.expectEqual(start, try availableSince(&database));
}
test "a failed watermark update leaves the rows it had already deleted" {
var database = try openLog();
defer database.close();
var writer = try BatchWriter.init(&database);
defer writer.deinit();
const start = try availableSince(&database);
try writer.writeBatch(&.{plainRow(start - 100, "old.example")});
// The delete succeeds and the advance does not. Without one transaction
// around the pair, this is the case that loses rows the watermark still
// promises.
try database.exec(
\\CREATE TRIGGER refuse_advance BEFORE UPDATE ON querylog_meta
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
);
try testing.expectError(error.Constraint, pruneOlderThan(&database, start + 1000));
try testing.expectEqual(@as(i64, 1), try countRows(&database));
try testing.expectEqual(start, try availableSince(&database));
}
test "a failed commit rolls back the delete and the watermark together" {
var database = try openLog();
defer database.close();
var writer = try BatchWriter.init(&database);
defer writer.deinit();
const start = try availableSince(&database);
try writer.writeBatch(&.{plainRow(start - 100, "old.example")});
// Both statements succeed and COMMIT is what fails: the advance inserts a
// `query_log` row whose `domain_id` references nothing, and
// `defer_foreign_keys` holds that violation back until the commit checks
// it (SQLite's documented semantics for the pragma; the assertions below
// observe the rollback, not the moment the check ran).
try database.exec(
\\CREATE TRIGGER break_at_commit AFTER UPDATE ON querylog_meta
\\BEGIN INSERT INTO query_log
\\ (timestamp, domain_id, client_ip, blocked, qclass, rcode,
\\ policy_action, policy_reason, route_kind)
\\VALUES (1, 999999, 'x', 0, 1, 0, 'allow', 'no_match', 'upstream'); END;
);
try database.exec("PRAGMA defer_foreign_keys = ON;");
try testing.expectError(error.Constraint, pruneOlderThan(&database, start + 1000));
// Nothing survived: not the delete, not the advance, not the row the
// trigger inserted.
try testing.expectEqual(@as(i64, 1), try countRows(&database));
try testing.expectEqual(start, try availableSince(&database));
try testing.expectEqual(
@as(i64, 0),
try database.queryInt("SELECT count(*) FROM query_log WHERE client_ip = 'x'"),
);
}
test "pruneOlderThan leaves the domains dimension table intact" {
@@ -662,7 +1080,7 @@ test "pruneOlderThan leaves the domains dimension table intact" {
defer writer.deinit();
try writer.writeBatch(&.{ plainRow(10, "a.example"), plainRow(11, "b.example") });
try testing.expectEqual(@as(i64, 2), try pruneOlderThan(&database, 1000));
try testing.expectEqual(@as(i64, 2), (try pruneOlderThan(&database, 1000)).deleted);
try testing.expectEqual(@as(i64, 0), try countRows(&database));
try testing.expectEqual(@as(i64, 2), try countDomains(&database));
@@ -749,7 +1167,7 @@ test "checkpointTruncate and vacuum run against a WAL file database" {
try writer.writeBatch(&.{ plainRow(10, "a.example"), plainRow(20, "b.example") });
try checkpointTruncate(&database);
try testing.expectEqual(@as(i64, 1), try pruneOlderThan(&database, 20));
try testing.expectEqual(@as(i64, 1), (try pruneOlderThan(&database, 20)).deleted);
try checkpointTruncate(&database);
try vacuum(&database);
@@ -785,22 +1203,46 @@ test "selectQueries returns the newest row first and reads every column" {
.domain = "ads.example.net",
.client_ip = "192.0.2.10",
.qtype = 28,
.qclass = 1,
.rcode = 0,
.blocked = true,
.block_reason = "blocklist",
.response_time_us = 4200,
.cache_hit = true,
.upstream = "https://dns.example/dns-query",
.group_id = 2,
.group_name = "kids",
.policy_action = .block,
.policy_reason = .blocklist_domain,
.matched = "ads.example.net",
.source_id = 5,
.source_name = "steven black",
.cname_target = null,
.safe_search_target = null,
.route_kind = .blocked,
.forward_zone = null,
},
.{
.timestamp = 20,
.domain = "quiet.example",
.client_ip = "hidden",
.qtype = null,
.qclass = 1,
.rcode = 2,
.blocked = false,
.block_reason = null,
.response_time_us = null,
.cache_hit = null,
.upstream = null,
.group_id = null,
.group_name = null,
.policy_action = .not_evaluated,
.policy_reason = .snapshot_unavailable,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = .rejected,
.forward_zone = null,
},
});
@@ -813,12 +1255,16 @@ test "selectQueries returns the newest row first and reads every column" {
try testing.expectEqualStrings("quiet.example", newest.domain);
try testing.expectEqualStrings("hidden", newest.client_ip);
try testing.expectEqual(@as(?u16, null), newest.qtype);
try testing.expectEqual(@as(u16, 1), newest.qclass);
try testing.expectEqual(@as(u12, 2), newest.rcode);
try testing.expect(!newest.blocked);
// A NULL text column reads as the empty string, by documented convention.
try testing.expectEqualStrings("", newest.block_reason);
try testing.expectEqual(@as(?i64, null), newest.response_time_us);
try testing.expectEqual(@as(?bool, null), newest.cache_hit);
// A NULL text column reads as the empty string, by documented convention.
try testing.expectEqualStrings("", newest.upstream);
try testing.expectEqual(provenance.PolicyAction.not_evaluated, newest.policy_action);
try testing.expectEqual(provenance.PolicyReason.snapshot_unavailable, newest.policy_reason);
try testing.expectEqual(provenance.RouteKind.rejected, newest.route_kind);
const oldest = rows.items[1];
try testing.expectEqual(@as(i64, 1), oldest.id);
@@ -826,11 +1272,15 @@ test "selectQueries returns the newest row first and reads every column" {
try testing.expectEqualStrings("ads.example.net", oldest.domain);
try testing.expectEqualStrings("192.0.2.10", oldest.client_ip);
try testing.expectEqual(@as(?u16, 28), oldest.qtype);
try testing.expectEqual(@as(u16, 1), oldest.qclass);
try testing.expectEqual(@as(u12, 0), oldest.rcode);
try testing.expect(oldest.blocked);
try testing.expectEqualStrings("blocklist", oldest.block_reason);
try testing.expectEqual(@as(?i64, 4200), oldest.response_time_us);
try testing.expectEqual(@as(?bool, true), oldest.cache_hit);
try testing.expectEqualStrings("https://dns.example/dns-query", oldest.upstream);
try testing.expectEqual(provenance.PolicyAction.block, oldest.policy_action);
try testing.expectEqual(provenance.PolicyReason.blocklist_domain, oldest.policy_reason);
try testing.expectEqual(provenance.RouteKind.blocked, oldest.route_kind);
}
test "selectQueries honours the limit and caps it at max_limit" {
@@ -895,7 +1345,9 @@ test "each filter narrows the result on its own" {
var blocked_row = plainRow(200, "ads.example.net");
blocked_row.client_ip = "192.0.2.20";
blocked_row.blocked = true;
blocked_row.block_reason = "blocklist";
blocked_row.policy_action = .block;
blocked_row.policy_reason = .blocklist_domain;
blocked_row.route_kind = .blocked;
try seed(&database, &.{
plainRow(100, "one.example.com"),
blocked_row,
@@ -1004,7 +1456,9 @@ test "statsTotals aggregates the window and averages only the timed rows" {
timed.response_time_us = 100;
var blocked_row = plainRow(150, "ads.example");
blocked_row.blocked = true;
blocked_row.block_reason = "blocklist";
blocked_row.policy_action = .block;
blocked_row.policy_reason = .blocklist_domain;
blocked_row.route_kind = .blocked;
blocked_row.response_time_us = 200;
var cached = plainRow(199, "b.example");
cached.client_ip = "192.0.2.99";
@@ -1042,7 +1496,9 @@ test "timeseries writes every bucket, including the ones with no rows" {
var blocked_row = plainRow(1020, "ads.example");
blocked_row.blocked = true;
blocked_row.block_reason = "blocklist";
blocked_row.policy_action = .block;
blocked_row.policy_reason = .blocklist_domain;
blocked_row.route_kind = .blocked;
var cached = plainRow(1035, "b.example");
cached.cache_hit = true;
try seed(&database, &.{
+17 -3
View File
@@ -114,8 +114,10 @@ pub const Retention = struct {
// still prunes through `Store.init`.
if (store) |s| s.prune(io, now);
if (queries_repo.pruneOlderThan(database, cutoff)) |deleted| {
add(&self.counters.rows_pruned, @intCast(deleted));
// One operation, not two: the delete and the coverage watermark it
// advances commit together or not at all (`queries_repo`).
if (queries_repo.pruneOlderThan(database, cutoff)) |pruned| {
add(&self.counters.rows_pruned, @intCast(pruned.deleted));
maintenance(store, io, now, "prune", null);
} else |err| {
log.warn("retention prune before {d} failed: {s}", .{ cutoff, @errorName(err) });
@@ -255,11 +257,23 @@ fn writeRows(database: *db.Db, timestamps: []const i64) !void {
.domain = "example.com",
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 0,
.blocked = false,
.block_reason = null,
.response_time_us = null,
.cache_hit = null,
.upstream = null,
.group_id = 1,
.group_name = "default",
.policy_action = .allow,
.policy_reason = .no_match,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = .upstream,
.forward_zone = null,
};
}
try writer.writeBatch(rows[0..timestamps.len]);
+4
View File
@@ -33,11 +33,13 @@ comptime {
_ = @import("server/resolver_integration_test.zig");
_ = @import("storage/db.zig");
_ = @import("config/model.zig");
_ = @import("config/limits.zig");
_ = @import("config/validate.zig");
_ = @import("config/faults.zig");
_ = @import("storage/config_schema.zig");
_ = @import("storage/migrations.zig");
_ = @import("storage/querylog_schema.zig");
_ = @import("storage/provenance.zig");
_ = @import("storage/repositories/context.zig");
_ = @import("storage/repositories/crud.zig");
_ = @import("storage/repositories/groups_repo.zig");
@@ -92,6 +94,8 @@ comptime {
_ = @import("server/shutdown.zig");
_ = @import("server/phase7_integration_test.zig");
_ = @import("web/sse.zig");
_ = @import("web/coverage.zig");
_ = @import("web/provenance_view.zig");
_ = @import("server/query_sink.zig");
_ = @import("web/auth.zig");
_ = @import("web/api_limiter.zig");
+4
View File
@@ -75,8 +75,12 @@ pub const DohClient = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
const self: *DohClient = @ptrCast(@alignCast(ptr));
// The endpoint outlives the client, so the borrow is safe for the whole
// query. Set before the attempt: a failure names this resolver too.
selected.* = self.endpoint.url;
return self.exchange(io, query, response_buf);
}
+3 -1
View File
@@ -44,7 +44,9 @@ fn runExchange(io: std.Io, params: Params) anyerror!usize {
const endpoint = try transport.Endpoint.parse("https://cloudflare-dns.com/dns-query");
var doh = try doh_client.DohClient.init(&http, endpoint, &request_buf, &transfer_buf);
const reply = try doh.client().exchange(io, query_bytes, params.response_buf);
var selected: ?[]const u8 = null;
const reply = try doh.client().exchange(io, query_bytes, params.response_buf, &selected);
std.debug.assert(std.mem.eql(u8, selected.?, endpoint.url));
return reply.len;
}
+4
View File
@@ -146,8 +146,12 @@ pub const DotClient = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
const self: *DotClient = @ptrCast(@alignCast(ptr));
// The endpoint outlives the client, so the borrow is safe for the whole
// query. Set before the attempt: a failure names this resolver too.
selected.* = self.endpoint.url;
return self.exchange(io, query, response_buf);
}
+3 -1
View File
@@ -57,7 +57,9 @@ fn runExchange(io: std.Io, params: Params) anyerror!usize {
params.bundle_lock,
params.buffers,
);
const reply = try client.client().exchange(io, query_bytes, params.response_buf);
var selected: ?[]const u8 = null;
const reply = try client.client().exchange(io, query_bytes, params.response_buf, &selected);
std.debug.assert(std.mem.eql(u8, selected.?, endpoint.url));
return reply.len;
}
+291 -46
View File
@@ -172,9 +172,10 @@ pub const Pool = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
const self: *Pool = @ptrCast(@alignCast(ptr));
return self.exchange(io, query, response_buf);
return self.exchange(io, query, response_buf, selected);
}
/// `response_buf` is handed to each attempt in turn, so a failed attempt
@@ -191,15 +192,16 @@ pub const Pool = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
const len = try transport.raceWithin(io, self.timeouts.total, exchangeLoopLen, .{
self, io, query, response_buf,
self, io, query, response_buf, selected,
});
return response_buf[0..len];
}
/// The two-pass failover loop, as a raceable task. It returns the reply's
/// length rather than its slice for the reason `exchangeLen` in the tests
/// length rather than its slice for the reason `Attributed` in the tests
/// below does: `Io.concurrent` stores the future's return value, so the
/// bytes are read back out of the caller's `response_buf` by `exchange`.
fn exchangeLoopLen(
@@ -207,6 +209,7 @@ pub const Pool = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError!usize {
const now = std.Io.Clock.awake.now(io);
var last_fault: ?transport.ExchangeError = null;
@@ -240,6 +243,13 @@ pub const Pool = struct {
}
attempted = true;
// Before the attempt, not after it: a failover reports whoever
// answered, an all-failed exchange reports the last endpoint
// tried, and a cancellation mid-flight reports the endpoint the
// query was in. The url is owned by the Endpoint, which outlives
// the pool, so the borrow stays valid past this loop.
selected.* = entry.endpoint.url;
const result = self.attempt(io, entry.client, query, response_buf);
const completed_at = std.Io.Clock.awake.now(io);
@@ -294,6 +304,12 @@ pub const Pool = struct {
}
/// One exchange raced against the per-attempt budget.
///
/// The leaf client reports an identity of its own, which the pool discards:
/// the entry's endpoint is the pool's own naming of the same resolver, and
/// it is what the caller was handed. A leaf writing its identity into the
/// caller's slot would let a test fake overwrite the endpoint that actually
/// answered.
fn attempt(
self: *Pool,
io: std.Io,
@@ -301,8 +317,9 @@ pub const Pool = struct {
query: []const u8,
response_buf: []u8,
) transport.ExchangeError![]u8 {
var leaf_selected: ?[]const u8 = null;
return transport.raceWithin(io, self.timeouts.attempt, transport.Client.exchange, .{
entry_client, io, query, response_buf,
entry_client, io, query, response_buf, &leaf_selected,
});
}
@@ -409,11 +426,23 @@ const response_bytes =
"\x07example\x03com\x00\x00\x01\x00\x01" ++
"\xc0\x0c\x00\x01\x00\x01\x00\x00\x01\x2c\x00\x04\x5d\xb8\xd8\x22";
/// The same question answered differently, so a reply identifies the entry that
/// produced it: one A record, a different address.
const alt_response_bytes =
"\x12\x34\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00" ++
"\x07example\x03com\x00\x00\x01\x00\x01" ++
"\xc0\x0c\x00\x01\x00\x01\x00\x00\x01\x2c\x00\x04\x0a\x00\x00\x01";
/// Stands in for a DoH or DoT client. Every behaviour the pool has to react to
/// is one variant, and every call is counted so a test can assert that an entry
/// in backoff was not touched.
const Fake = struct {
behavior: Behavior,
/// Replaces `behavior` after the first call. One entry that fails the task
/// which reaches it first and answers the next is what makes two concurrent
/// exchanges end on different entries; a single behaviour cannot say that.
/// Mutated under the entry's `busy` lock, like `calls`.
then: ?Behavior = null,
calls: usize = 0,
in_flight: std.atomic.Value(u32) = .init(0),
/// The most tasks ever inside `exchangeFn` at once. The per-entry lock is
@@ -437,15 +466,24 @@ const Fake = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
_ = query;
// Deliberately not the endpoint url: the pool must report its own
// entry, so a test can tell the two apart.
selected.* = "fake://leaf";
const self: *Fake = @ptrCast(@alignCast(ptr));
const entrants = self.in_flight.fetchAdd(1, .acq_rel) + 1;
defer _ = self.in_flight.fetchSub(1, .acq_rel);
_ = self.peak_in_flight.fetchMax(entrants, .acq_rel);
self.calls += 1;
switch (self.behavior) {
const behavior = self.behavior;
if (self.then) |next| {
self.behavior = next;
self.then = null;
}
switch (behavior) {
.reply => |bytes| return copy(bytes, response_buf),
.fail => |err| return err,
.slow => |slow| {
@@ -539,10 +577,106 @@ test "Pool satisfies the Client interface" {
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
const reply = try pool.client().exchange(io, query_bytes, &buf);
var selected: ?[]const u8 = null;
const reply = try pool.client().exchange(io, query_bytes, &buf, &selected);
try testing.expectEqualSlices(u8, response_bytes, reply);
}
test "the pool reports the endpoint that answered, not the leaf client's own name" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var fake: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{testEntry("https://a.example/dns-query", &fake, 10)};
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
var selected: ?[]const u8 = null;
_ = try pool.exchange(io, query_bytes, &buf, &selected);
try testing.expectEqualStrings("https://a.example/dns-query", selected.?);
}
test "a failover reports the endpoint that answered, not the first one tried" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var bad: Fake = .{ .behavior = .{ .fail = error.Timeout } };
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://bad.example/dns-query", &bad, 10),
testEntry("https://good.example/dns-query", &good, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
var selected: ?[]const u8 = null;
_ = try pool.exchange(io, query_bytes, &buf, &selected);
try testing.expectEqualStrings("https://good.example/dns-query", selected.?);
}
test "an all-failed exchange reports the last endpoint attempted" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var first: Fake = .{ .behavior = .{ .fail = error.ConnectFailed } };
var last: Fake = .{ .behavior = .{ .fail = error.Timeout } };
var entries = [_]Entry{
testEntry("https://first.example/dns-query", &first, 10),
testEntry("https://last.example/dns-query", &last, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
var selected: ?[]const u8 = null;
try testing.expectError(error.Timeout, pool.exchange(io, query_bytes, &buf, &selected));
try testing.expectEqualStrings("https://last.example/dns-query", selected.?);
}
test "a timeout mid-flight reports the endpoint the query was in" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var stalling: Fake = .{ .behavior = .{ .slow = .{
.duration = .{ .raw = .fromSeconds(30), .clock = .awake },
.reply = response_bytes,
} } };
var untouched: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://stalling.example/dns-query", &stalling, 10),
testEntry("https://untouched.example/dns-query", &untouched, 20),
};
var pool: Pool = .init(&entries, test_cfg, .{
.attempt = .{ .raw = .fromMilliseconds(200), .clock = .awake },
.total = .{ .raw = .fromMilliseconds(60), .clock = .awake },
}, 1);
var buf: [512]u8 = undefined;
var selected: ?[]const u8 = null;
try testing.expectError(error.Timeout, pool.exchange(io, query_bytes, &buf, &selected));
try testing.expectEqualStrings("https://stalling.example/dns-query", selected.?);
try testing.expectEqual(@as(usize, 0), untouched.calls);
}
test "an exchange that attempted nothing reports no endpoint" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var fake: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{testEntry("https://a.example/dns-query", &fake, 10)};
entries[0].enabled = false;
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
var selected: ?[]const u8 = null;
try testing.expectError(error.ConnectFailed, pool.exchange(io, query_bytes, &buf, &selected));
try testing.expect(selected == null);
}
test "entries are tried in ascending priority order" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
@@ -560,7 +694,8 @@ test "entries are tried in ascending priority order" {
try testing.expectEqual(@as(i32, 10), entries[0].priority);
var buf: [512]u8 = undefined;
_ = try pool.exchange(io, query_bytes, &buf);
var selected: ?[]const u8 = null;
_ = try pool.exchange(io, query_bytes, &buf, &selected);
try testing.expectEqual(@as(usize, 1), low.calls);
try testing.expectEqual(@as(usize, 0), high.calls);
}
@@ -579,7 +714,8 @@ test "a peer fault fails over to the next entry and is recorded" {
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
const reply = try pool.exchange(io, query_bytes, &buf);
var selected: ?[]const u8 = null;
const reply = try pool.exchange(io, query_bytes, &buf, &selected);
try testing.expectEqualSlices(u8, response_bytes, reply);
try testing.expectEqual(@as(u32, 1), entries[0].health.consecutive_failures);
@@ -603,12 +739,13 @@ test "an entry in backoff is skipped while another is available" {
var buf: [512]u8 = undefined;
// Two failures reach `failure_threshold` and open a backoff window.
_ = try pool.exchange(io, query_bytes, &buf);
_ = try pool.exchange(io, query_bytes, &buf);
var selected: ?[]const u8 = null;
_ = try pool.exchange(io, query_bytes, &buf, &selected);
_ = try pool.exchange(io, query_bytes, &buf, &selected);
try testing.expectEqual(@as(usize, 2), bad.calls);
try testing.expect(entries[0].health.backoff_until != null);
_ = try pool.exchange(io, query_bytes, &buf);
_ = try pool.exchange(io, query_bytes, &buf, &selected);
try testing.expectEqual(@as(usize, 2), bad.calls);
try testing.expectEqual(@as(usize, 3), good.calls);
}
@@ -627,13 +764,14 @@ test "every entry in backoff is still probed" {
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
var selected: ?[]const u8 = null;
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf, &selected));
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf, &selected));
try testing.expect(entries[0].health.backoff_until != null);
try testing.expect(entries[1].health.backoff_until != null);
// Pass one now has no candidate at all. Pass two probes both anyway.
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf, &selected));
try testing.expectEqual(@as(usize, 3), first.calls);
try testing.expectEqual(@as(usize, 3), second.calls);
}
@@ -652,7 +790,8 @@ test "a local resource error short-circuits and records nothing" {
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
try testing.expectError(error.OutOfMemory, pool.exchange(io, query_bytes, &buf));
var selected: ?[]const u8 = null;
try testing.expectError(error.OutOfMemory, pool.exchange(io, query_bytes, &buf, &selected));
try testing.expectEqual(@as(usize, 0), good.calls);
try testing.expectEqual(@as(u64, 0), entries[0].health.total_failures);
try testing.expectEqual(@as(u32, 0), entries[0].health.consecutive_failures);
@@ -672,7 +811,8 @@ test "a cancellation short-circuits and records nothing" {
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
try testing.expectError(error.Canceled, pool.exchange(io, query_bytes, &buf));
var selected: ?[]const u8 = null;
try testing.expectError(error.Canceled, pool.exchange(io, query_bytes, &buf, &selected));
try testing.expectEqual(@as(usize, 0), good.calls);
try testing.expectEqual(@as(u64, 0), entries[0].health.total_failures);
}
@@ -699,7 +839,8 @@ test "an attempt that outruns the budget is a recorded Timeout" {
}, 1);
var buf: [512]u8 = undefined;
const reply = try pool.exchange(io, query_bytes, &buf);
var selected: ?[]const u8 = null;
const reply = try pool.exchange(io, query_bytes, &buf, &selected);
try testing.expectEqualSlices(u8, response_bytes, reply);
try testing.expectEqual(@as(usize, 1), slow.calls);
@@ -736,7 +877,8 @@ test "two stalling upstreams cost the total budget, not one budget each" {
var buf: [512]u8 = undefined;
const started = std.Io.Clock.awake.now(io);
try testing.expectError(error.Timeout, pool.exchange(io, query_bytes, &buf));
var selected: ?[]const u8 = null;
try testing.expectError(error.Timeout, pool.exchange(io, query_bytes, &buf, &selected));
const elapsed_ns = std.Io.Clock.awake.now(io).nanoseconds - started.nanoseconds;
// Under one attempt budget, so the outer deadline is provably what fired.
@@ -770,7 +912,8 @@ test "every entry disabled yields ConnectFailed without waiting out the total bu
var buf: [512]u8 = undefined;
const started = std.Io.Clock.awake.now(io);
try testing.expectError(error.ConnectFailed, pool.exchange(io, query_bytes, &buf));
var selected: ?[]const u8 = null;
try testing.expectError(error.ConnectFailed, pool.exchange(io, query_bytes, &buf, &selected));
const elapsed_ns = std.Io.Clock.awake.now(io).nanoseconds - started.nanoseconds;
try testing.expect(elapsed_ns < @as(i96, 5) * std.time.ns_per_s);
@@ -799,7 +942,8 @@ test "a wired accumulator receives both outcomes the pool records" {
var buf: [512]u8 = undefined;
// One exchange: the first entry fails over into the second, so this drives
// one failure and one success.
_ = try pool.exchange(io, query_bytes, &buf);
var selected: ?[]const u8 = null;
_ = try pool.exchange(io, query_bytes, &buf, &selected);
// Two cells, one per url, in whatever minute the wall clock is in.
try testing.expectEqual(@as(u32, 2), acc.snapshotStats(io).pending);
@@ -852,8 +996,9 @@ test "snapshot reports the counters in pool order" {
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf: [512]u8 = undefined;
_ = try pool.exchange(io, query_bytes, &buf);
_ = try pool.exchange(io, query_bytes, &buf);
var selected: ?[]const u8 = null;
_ = try pool.exchange(io, query_bytes, &buf, &selected);
_ = try pool.exchange(io, query_bytes, &buf, &selected);
var out: [4]Snapshot = undefined;
const written = try pool.snapshot(io, &out);
@@ -882,13 +1027,30 @@ test "snapshot reports the counters in pool order" {
try testing.expectEqual(@as(usize, 1), try pool.snapshot(io, &one));
}
/// One concurrent call's whole result: how much of its buffer the reply filled,
/// and which resolver the pool said answered it.
///
/// `Io.concurrent` stores the return value in the future, so the reply slice is
/// reduced to its length here and the bytes are read back out of the caller's
/// buffer. Returning a `usize` also lets the test discard a result with
/// `catch 0`.
fn exchangeLen(pool: *Pool, io: std.Io, buf: []u8) transport.ExchangeError!usize {
const reply = try pool.exchange(io, query_bytes, buf);
return reply.len;
/// buffer. `selected` survives the trip because it borrows an `Endpoint.url`,
/// which outlives the pool.
const Attributed = struct {
reply_len: usize,
selected: ?[]const u8,
/// What a teardown `await` discards into once the assertions above it have
/// already taken the value.
const discarded: Attributed = .{ .reply_len = 0, .selected = null };
};
/// Each call keeps its own `selected` out-value rather than dropping it: the
/// pointer is written per call, on the stack of the task that made it, and a
/// pool that hung it off `*Pool` instead would hand one call's identity to
/// another. Nothing but a per-call capture can see that.
fn exchangeAttributed(pool: *Pool, io: std.Io, buf: []u8) transport.ExchangeError!Attributed {
var selected: ?[]const u8 = null;
const reply = try pool.exchange(io, query_bytes, buf, &selected);
return .{ .reply_len = reply.len, .selected = selected };
}
test "concurrent exchanges through one entry do not overlap" {
@@ -909,25 +1071,102 @@ test "concurrent exchanges through one entry do not overlap" {
var buf_a: [512]u8 = undefined;
var buf_b: [512]u8 = undefined;
var first = io.concurrent(exchangeLen, .{ &pool, io, &buf_a }) catch |err| switch (err) {
var first = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_a }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = first.await(io) catch 0;
var second = io.concurrent(exchangeLen, .{ &pool, io, &buf_b }) catch |err| switch (err) {
defer _ = first.await(io) catch Attributed.discarded;
var second = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_b }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = second.await(io) catch 0;
defer _ = second.await(io) catch Attributed.discarded;
const len_a = try first.await(io);
const len_b = try second.await(io);
const result_a = try first.await(io);
const result_b = try second.await(io);
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..len_a]);
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..len_b]);
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..result_a.reply_len]);
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..result_b.reply_len]);
try testing.expectEqualStrings("https://only.example/dns-query", result_a.selected.?);
try testing.expectEqualStrings("https://only.example/dns-query", result_b.selected.?);
try testing.expectEqual(@as(usize, 2), fake.calls);
try testing.expectEqual(@as(u32, 1), fake.peak_in_flight.load(.acquire));
try testing.expectEqual(@as(u64, 2), entries[0].health.total_successes);
}
/// The two entries of the test below, each answering with bytes only it
/// produces so a call's reply proves which entry served it independently of
/// what the pool reported.
const divergent_first_url = "https://first.example/dns-query";
const divergent_second_url = "https://second.example/dns-query";
fn expectAnsweredByReporter(result: Attributed, buf: []const u8) !void {
const url = result.selected orelse return error.TestExpectedSelectedResolver;
const reply = if (std.mem.eql(u8, url, divergent_first_url))
alt_response_bytes
else if (std.mem.eql(u8, url, divergent_second_url))
response_bytes
else
return error.TestUnexpectedResolver;
try testing.expectEqualSlices(u8, reply, buf[0..result.reply_len]);
}
test "overlapping exchanges each report the entry that answered that call" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
// The first entry fails the task that reaches it first, slowly enough that
// the second task is queued behind it, then answers that second task. One
// failure is under `test_cfg`'s threshold of two, so no backoff steers the
// waiting task away and the two calls end on different entries.
var first_entry: Fake = .{
.behavior = .{ .slow_fail = .{
.duration = .{ .raw = .fromMilliseconds(50), .clock = .awake },
.err = error.ConnectFailed,
} },
.then = .{ .reply = alt_response_bytes },
};
// Slow too, so the failed-over call is still in flight while the other call
// is being answered — a shared identity would be overwritten under it.
var second_entry: Fake = .{ .behavior = .{ .slow = .{
.duration = .{ .raw = .fromMilliseconds(50), .clock = .awake },
.reply = response_bytes,
} } };
var entries = [_]Entry{
testEntry(divergent_first_url, &first_entry, 10),
testEntry(divergent_second_url, &second_entry, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var buf_a: [512]u8 = undefined;
var buf_b: [512]u8 = undefined;
var first = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_a }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = first.await(io) catch Attributed.discarded;
var second = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_b }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = second.await(io) catch Attributed.discarded;
const result_a = try first.await(io);
const result_b = try second.await(io);
// Which task lands where depends on the order they take the first entry's
// lock, so the claim is over the pair: one identity each, and each one
// matching the bytes that call received.
try testing.expect(!std.mem.eql(u8, result_a.selected.?, result_b.selected.?));
try expectAnsweredByReporter(result_a, &buf_a);
try expectAnsweredByReporter(result_b, &buf_b);
try testing.expectEqual(@as(usize, 2), first_entry.calls);
try testing.expectEqual(@as(usize, 1), second_entry.calls);
try testing.expectEqual(@as(u32, 1), first_entry.peak_in_flight.load(.acquire));
try testing.expectEqual(@as(u64, 1), entries[0].health.total_failures);
try testing.expectEqual(@as(u64, 1), entries[0].health.total_successes);
try testing.expectEqual(@as(u64, 1), entries[1].health.total_successes);
}
test "an entry that enters backoff while a task waits on it is not attempted" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
@@ -956,21 +1195,25 @@ test "an entry that enters backoff while a task waits on it is not attempted" {
var buf_a: [512]u8 = undefined;
var buf_b: [512]u8 = undefined;
var first = io.concurrent(exchangeLen, .{ &pool, io, &buf_a }) catch |err| switch (err) {
var first = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_a }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = first.await(io) catch 0;
var second = io.concurrent(exchangeLen, .{ &pool, io, &buf_b }) catch |err| switch (err) {
defer _ = first.await(io) catch Attributed.discarded;
var second = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_b }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = second.await(io) catch 0;
defer _ = second.await(io) catch Attributed.discarded;
const len_a = try first.await(io);
const len_b = try second.await(io);
const result_a = try first.await(io);
const result_b = try second.await(io);
// Both tasks fail over to the healthy entry and get an answer.
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..len_a]);
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..len_b]);
// Both tasks fail over to the healthy entry and get an answer, and both
// report it: the identity is the entry that answered, not the one that
// failed on the way there.
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..result_a.reply_len]);
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..result_b.reply_len]);
try testing.expectEqualStrings("https://good.example/dns-query", result_a.selected.?);
try testing.expectEqualStrings("https://good.example/dns-query", result_b.selected.?);
try testing.expectEqual(@as(usize, 2), good.calls);
// The point of the test: the entry was attempted once, not twice. Without
@@ -998,7 +1241,8 @@ test "a successful exchange with nothing open costs the store no statement" {
var buf: [512]u8 = undefined;
const before = fx.store.statements;
for (0..20) |_| _ = try pool.exchange(io, query_bytes, &buf);
var selected: ?[]const u8 = null;
for (0..20) |_| _ = try pool.exchange(io, query_bytes, &buf, &selected);
try testing.expectEqual(before, fx.store.statements);
try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
}
@@ -1022,7 +1266,8 @@ test "a failing then recovering upstream leaves exactly one resolved episode" {
pool.diagnostics = &fx.store;
var buf: [512]u8 = undefined;
_ = try pool.exchange(io, query_bytes, &buf);
var selected: ?[]const u8 = null;
_ = try pool.exchange(io, query_bytes, &buf, &selected);
// Backoff would park the failing entry, so the second failure is driven
// through `recordFailure` itself rather than through another exchange.
pool.recordFailure(io, &entries[0], std.Io.Clock.awake.now(io), error.ConnectFailed);
+55 -4
View File
@@ -35,6 +35,13 @@ pub fn parsePrefix(bytes: [prefix_len]u8) u16 {
return std.mem.readInt(u16, &bytes, .big);
}
/// The longest DNS name in text form, and so the longest host an endpoint url
/// can name. Every consumer of `Endpoint.host` sizes itself from this bound
/// rather than re-deriving it: the query log's `upstream` column is built for
/// the widest `scheme://host:port` this permits, so a longer host would reach
/// storage only as a silently shortened identity.
pub const max_host_len = 253;
pub const doh_default_port = 443;
pub const dot_default_port = 853; // RFC 7858 §3.1
pub const doh_default_path = "/dns-query"; // RFC 8484 §4.1 well-known template
@@ -46,14 +53,15 @@ pub const Endpoint = struct {
scheme: Scheme,
/// The original text, for logs and the health API.
url: []const u8,
/// No brackets, no port. Used for SNI and certificate verification.
/// No brackets, no port, never empty, at most `max_host_len` bytes. Used
/// for SNI and certificate verification.
host: []const u8,
port: u16,
/// DoH only; always starts with '/'; `doh_default_path` when absent. A DoT
/// endpoint has no request path, so it carries "/" and nothing reads it.
path: []const u8,
pub const ParseError = error{ UnsupportedScheme, MissingHost, BadPort, BadUrl };
pub const ParseError = error{ UnsupportedScheme, MissingHost, HostTooLong, BadPort, BadUrl };
const doh_prefix = "https://";
const dot_prefix = "tls://";
@@ -82,6 +90,10 @@ pub const Endpoint = struct {
const host, const port_text = try splitAuthority(authority);
if (host.len == 0) return error.MissingHost;
// Rejected here rather than tolerated: every identity built from this
// endpoint is bounded by `max_host_len`, so a longer host would parse
// clean and then be shortened where it is stored or logged.
if (host.len > max_host_len) return error.HostTooLong;
const port: u16 = if (port_text) |text| blk: {
if (text.len == 0) return error.BadPort;
@@ -329,17 +341,27 @@ pub const Client = struct {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) ExchangeError![]u8,
/// Returns a prefix of `response_buf`. The returned message has already
/// passed `validateResponse` against `query`.
///
/// `selected` names the resolver the exchange used. An implementation
/// writes it *before* each attempt, never after, so a failed exchange still
/// names the last resolver it tried — a SERVFAIL row without its resolver
/// explains nothing. The slice must outlive the call; every implementation
/// borrows storage it owns for at least the query's duration. Callers
/// initialize it to null: a `null` after the call means no resolver was
/// reached at all.
pub fn exchange(
self: Client,
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) ExchangeError![]u8 {
return self.exchangeFn(self.ptr, io, query, response_buf);
return self.exchangeFn(self.ptr, io, query, response_buf, selected);
}
};
@@ -449,6 +471,31 @@ test "parse rejects an empty host" {
try testing.expectError(error.MissingHost, Endpoint.parse("tls://"));
}
test "parse takes a host at the length bound and rejects one past it" {
// Four labels, the widest a 253-byte name allows: 3 * (63 + 1) + 61.
const at_bound = ("a" ** 63 ++ ".") ** 3 ++ "a" ** 61;
comptime std.debug.assert(at_bound.len == max_host_len);
const accepted = try Endpoint.parse("https://" ++ at_bound ++ "/dns-query");
try testing.expectEqualStrings(at_bound, accepted.host);
// One byte more is one byte no consumer of `host` has room for.
try testing.expectError(
error.HostTooLong,
Endpoint.parse("https://" ++ at_bound ++ "a/dns-query"),
);
// The bound is on the host alone, so a port and a path do not spend it,
// and the bracketed form is measured with the brackets removed.
try testing.expectError(
error.HostTooLong,
Endpoint.parse("tls://" ++ at_bound ++ "a:853"),
);
try testing.expectError(
error.HostTooLong,
Endpoint.parse("https://[" ++ at_bound ++ "a]:8443/dns-query"),
);
}
test "parse rejects a bad port" {
try testing.expectError(error.BadPort, Endpoint.parse("https://h:99999/"));
try testing.expectError(error.BadPort, Endpoint.parse("https://h:/"));
@@ -641,8 +688,10 @@ test "a fake client satisfies the Client interface" {
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) ExchangeError![]u8 {
_ = io;
selected.* = "fake://echo";
const self: *@This() = @ptrCast(@alignCast(ptr));
self.calls += 1;
if (query.len > response_buf.len) return error.ResponseTooLarge;
@@ -657,9 +706,11 @@ test "a fake client satisfies the Client interface" {
var fake: Fake = .{};
var buf: [16]u8 = undefined;
const echoed = try fake.client().exchange(undefined, "hello", &buf);
var selected: ?[]const u8 = null;
const echoed = try fake.client().exchange(undefined, "hello", &buf, &selected);
try testing.expectEqualStrings("hello", echoed);
try testing.expectEqual(@as(usize, 1), fake.calls);
try testing.expectEqualStrings("fake://echo", selected.?);
}
/// A query for example.com A: id 0x1234, RD set, one question.
+56
View File
@@ -0,0 +1,56 @@
//! How much of the window a client asked about the query log can still answer
//! for.
//!
//! Retention deletes old rows and advances a watermark in the same transaction
//! (`queries_repo.pruneOlderThan`), so the file knows the oldest instant it is
//! complete for. Without that fact on the wire a chart draws a pruned week as a
//! week of silence, which is the one reading that is certainly wrong.
//!
//! Three endpoints carry it — `/api/queries`, `/api/stats` and
//! `/api/stats/timeseries` — and they judge it against their own effective
//! lower bound: the client's `since` for the query log, the period's aligned
//! window start for the two stats endpoints.
const std = @import("std");
const db = @import("../storage/db.zig");
const queries_repo = @import("../storage/repositories/queries_repo.zig");
pub const Coverage = struct {
/// True only when the whole requested window is inside what the file still
/// holds. A request with no lower bound at all asks about all of history,
/// which no file that has ever pruned can promise.
complete: bool,
/// The oldest instant the file is complete for, unix seconds.
available_since: i64,
};
pub fn of(available_since: i64, since: ?i64) Coverage {
return .{
.complete = if (since) |lower_bound| lower_bound >= available_since else false,
.available_since = available_since,
};
}
/// Reads the watermark for a request that is about to answer.
pub fn read(database: *db.Db, since: ?i64) db.Error!Coverage {
return of(try queries_repo.availableSince(database), since);
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const testing = std.testing;
test "a window that starts at or after the watermark is complete" {
try testing.expect(of(1000, 1000).complete);
try testing.expect(of(1000, 1001).complete);
try testing.expect(!of(1000, 999).complete);
}
test "an unbounded window is never complete" {
const unbounded = of(1000, null);
try testing.expect(!unbounded.complete);
try testing.expectEqual(@as(i64, 1000), unbounded.available_since);
}
+33 -37
View File
@@ -23,7 +23,7 @@ const std = @import("std");
const address = @import("../../platform/address.zig");
const http_util = @import("../http_util.zig");
const queries_repo = @import("../../storage/repositories/queries_repo.zig");
const provenance_view = @import("../provenance_view.zig");
const server = @import("../server.zig");
const sse = @import("../sse.zig");
@@ -36,32 +36,13 @@ pub const heartbeat_interval: std.Io.Clock.Duration = .{
.clock = .awake,
};
/// One event's `data:` payload the `/api/queries` row fields (ruling 20),
/// minus `id`: a live entry precedes persistence, so no row id exists yet.
pub const EventView = struct {
ts: i64,
domain: []const u8,
client_ip: []const u8,
qtype: ?u16,
blocked: bool,
block_reason: []const u8,
response_time_us: ?i64,
cache_hit: ?bool,
upstream: []const u8,
};
/// One event's `data:` payload: the shared full-provenance DTO, exactly. A live
/// event says everything `GET /api/queries/{id}` would say about the same query
/// except its id, which does not exist yet — the entry precedes its own insert.
pub const EventView = provenance_view.Provenance;
pub fn view(entry: *const sse.Entry) EventView {
return .{
.ts = entry.timestamp,
.domain = entry.domain(),
.client_ip = entry.clientIp(),
.qtype = entry.qtype,
.blocked = entry.blocked,
.block_reason = entry.blockReason(),
.response_time_us = entry.response_time_us,
.cache_hit = entry.cache_hit,
.upstream = entry.upstream(),
};
return provenance_view.fromEntry(entry);
}
/// One `event: query` frame. JSON never contains a raw newline, so the whole
@@ -139,14 +120,18 @@ pub fn stream(
const testing = std.testing;
test "the event payload carries the /api/queries row fields, minus id" {
const row_fields = @typeInfo(queries_repo.QueryRow).@"struct".fields;
test "the event payload is the detail body minus its id, name and type for name" {
const detail_fields = @typeInfo(provenance_view.QueryDetail).@"struct".fields;
const view_fields = @typeInfo(EventView).@"struct".fields;
comptime {
std.debug.assert(view_fields.len == row_fields.len - 1);
std.debug.assert(std.mem.eql(u8, row_fields[0].name, "id"));
for (row_fields[1..], view_fields) |row_field, view_field| {
std.debug.assert(std.mem.eql(u8, row_field.name, view_field.name));
std.debug.assert(view_fields.len == detail_fields.len - 1);
std.debug.assert(std.mem.eql(u8, detail_fields[0].name, "id"));
for (detail_fields[1..], view_fields) |detail_field, view_field| {
std.debug.assert(std.mem.eql(u8, detail_field.name, view_field.name));
// Names alone would let a group keep its key while changing what it
// holds, which is the drift a live viewer would see and a detail
// page would not.
std.debug.assert(detail_field.type == view_field.type);
}
}
}
@@ -157,11 +142,19 @@ test "a frame is one event line and one data line of JSON" {
.domain = "ads.example",
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 0,
.blocked = true,
.block_reason = "blocklist_domain",
.group_id = 1,
.group_name = "default",
.policy_action = .block,
.policy_reason = .blocklist_domain,
.matched = "ads.example",
.source_id = 3,
.source_name = "StevenBlack",
.route_kind = .blocked,
.response_time_us = 42,
.cache_hit = false,
.upstream = "https://dns.example/dns-query",
});
var buf: [1024]u8 = undefined;
@@ -172,10 +165,12 @@ test "a frame is one event line and one data line of JSON" {
try testing.expect(std.mem.startsWith(u8, frame, "event: query\ndata: {"));
try testing.expect(std.mem.endsWith(u8, frame, "}\n\n"));
try testing.expectEqual(@as(usize, 3), std.mem.count(u8, frame, "\n"));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"ts\":1700000000"));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"time\":1700000000"));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"domain\":\"ads.example\""));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"blocked\":true"));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"block_reason\":\"blocklist_domain\""));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"group\":{\"id\":1,\"name\":\"default\"}"));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"reason\":\"blocklist_domain\""));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"source_name\":\"StevenBlack\""));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"kind\":\"blocked\""));
}
test "an unlogged field stays null and an empty string stays a string" {
@@ -191,6 +186,7 @@ test "an unlogged field stays null and an empty string stays a string" {
const frame = writer.buffered();
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"qtype\":null"));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"cache_hit\":null"));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"duration_us\":null"));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"upstream\":\"\""));
try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"id\":null"));
}
+148 -10
View File
@@ -1,4 +1,5 @@
//! `GET /api/queries` — the query log, newest first (ruling 11).
//! `GET /api/queries` — the query log, newest first (ruling 11) — and
//! `GET /api/queries/{id}`, one row of it fully explained.
//!
//! Keyset pagination rather than an offset: the table is append-only and the
//! UI reads the head of it, so `id < before` is one index seek no matter how
@@ -13,9 +14,11 @@
const std = @import("std");
const Allocator = std.mem.Allocator;
const coverage = @import("../coverage.zig");
const db = @import("../../storage/db.zig");
const http_util = @import("../http_util.zig");
const logger = @import("../../storage/logger.zig");
const provenance_view = @import("../provenance_view.zig");
const queries_repo = @import("../../storage/repositories/queries_repo.zig");
const server = @import("../server.zig");
@@ -41,6 +44,10 @@ pub const Page = struct {
queries: []const queries_repo.QueryRow,
/// The cursor for the next page, or null when this page is the last one.
next_before: ?i64,
/// Whether the log still covers the window the filter asked for. A client
/// that reads rows without reading this cannot tell an empty window from a
/// pruned one.
coverage: coverage.Coverage,
};
pub const FilterError = error{
@@ -110,6 +117,7 @@ pub fn page(
return .{
.queries = rows.items,
.next_before = if (full and rows.items.len != 0) rows.items[rows.items.len - 1].id else null,
.coverage = try coverage.read(database, filter.since),
};
}
@@ -138,10 +146,37 @@ pub fn list(
return http_util.respondJson(request, .ok, result, &.{});
}
/// `GET /api/queries/{id}` — one query, fully explained.
///
/// The row is the whole answer: every field is a fact recorded when the query
/// was answered, so nothing here is joined against current configuration. A
/// group or blocklist renamed since keeps the name it had.
pub fn detail(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
_ = io;
const database = state.querylog_db orelse
return http_util.respondError(request, .service_unavailable, "query log unavailable");
const row = queries_repo.detailById(database, request.arena, request.id.?) catch |err| {
log.warn("query log read failed: {s}", .{@errorName(err)});
return http_util.respondError(request, .internal_server_error, "internal error");
};
// An id retention has pruned and one that never existed are the same
// answer, and the API does not pretend to tell them apart.
const found = row orelse return http_util.respondError(request, .not_found, "not found");
return http_util.respondJson(request, .ok, provenance_view.fromDetail(found), &.{});
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const provenance = @import("../../storage/provenance.zig");
const querylog_schema = @import("../../storage/querylog_schema.zig");
const testing = std.testing;
@@ -210,21 +245,40 @@ fn openLog() !db.Db {
return database;
}
/// Rows the resolver could actually have written. Ruling 20 ties the three
/// route facts together: a block never consulted the cache, so its `cache_hit`
/// is NULL rather than false; a cache hit has no upstream to name; and only an
/// upstream answer carries one. A fixture that broke those ties would let a
/// serializer regression pass here and fail on real rows.
fn seed(database: *db.Db, count: usize) !void {
var writer = try queries_repo.BatchWriter.init(database);
defer writer.deinit();
var rows: [16]queries_repo.Row = undefined;
for (rows[0..count], 0..) |*row, i| {
const blocked = i % 2 == 0;
const from_cache = i % 4 == 1;
row.* = .{
.timestamp = 1_700_000_000 + @as(i64, @intCast(i)),
.domain = if (i % 2 == 0) "ads.example" else "safe.example",
.domain = if (blocked) "ads.example" else "safe.example",
.client_ip = "192.0.2.10",
.qtype = 1,
.blocked = i % 2 == 0,
.block_reason = if (i % 2 == 0) "blocklist_domain" else null,
.qclass = 1,
.rcode = 0,
.blocked = blocked,
.response_time_us = 500,
.cache_hit = false,
.upstream = null,
.cache_hit = if (blocked) null else from_cache,
.upstream = if (blocked or from_cache) null else "9.9.9.9",
.group_id = 1,
.group_name = "default",
.policy_action = if (blocked) .block else .allow,
.policy_reason = if (blocked) .blocklist_domain else .no_match,
.matched = if (blocked) "ads.example" else null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = if (blocked) .blocked else if (from_cache) .cache else .upstream,
.forward_zone = null,
};
}
try writer.writeBatch(rows[0..count]);
@@ -297,6 +351,37 @@ test "the parsed filters narrow the rows the page returns" {
try testing.expectEqual(@as(usize, 0), nobody.queries.len);
}
test "the seeded rows carry only the route shapes ruling 20 allows" {
var database = try openLog();
defer database.close();
try seed(&database, 4);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
var seen: std.EnumSet(provenance.RouteKind) = .initEmpty();
for ((try page(&database, arena.allocator(), .{})).queries) |row| {
seen.insert(row.route_kind);
switch (row.route_kind) {
.blocked => {
try testing.expectEqual(@as(?bool, null), row.cache_hit);
try testing.expectEqualStrings("", row.upstream);
},
.cache => {
try testing.expectEqual(@as(?bool, true), row.cache_hit);
try testing.expectEqualStrings("", row.upstream);
},
.upstream => {
try testing.expectEqual(@as(?bool, false), row.cache_hit);
try testing.expectEqualStrings("9.9.9.9", row.upstream);
},
.local, .forward_zone, .rejected => return error.UnseededRouteKind,
}
}
// All three, so the serializer tests below read every shape the fixture claims.
try testing.expectEqual(@as(usize, 3), seen.count());
}
test "the page serializes as the envelope ruling 11 defines" {
var database = try openLog();
defer database.close();
@@ -312,14 +397,67 @@ test "the page serializes as the envelope ruling 11 defines" {
const text = allocating.written();
try testing.expect(std.mem.startsWith(u8, text, "{\"queries\":["));
try testing.expect(std.mem.endsWith(u8, text, "\"next_before\":null}"));
for ([_][]const u8{
"\"id\":", "\"ts\":", "\"domain\":", "\"client_ip\":",
"\"qtype\":", "\"blocked\":", "\"cache_hit\":", "\"upstream\":",
"\"upstream\":", "\"response_time_us\":", "\"block_reason\":",
"\"id\":", "\"ts\":", "\"domain\":", "\"client_ip\":",
"\"qtype\":", "\"qclass\":", "\"rcode\":", "\"blocked\":",
"\"cache_hit\":", "\"upstream\":", "\"response_time_us\":", "\"policy_action\":",
"\"policy_reason\":", "\"route_kind\":",
}) |field| {
try testing.expect(std.mem.containsAtLeast(u8, text, 1, field));
}
// W1's ruling: a NULL column reads as "", and "" stays "" on the wire.
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"upstream\":\"\""));
// The column the provenance columns replaced.
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "block_reason"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"coverage\":{\"complete\":"));
}
test "the coverage of a page answers the window the filter asked for" {
var database = try openLog();
defer database.close();
try seed(&database, 1);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const watermark = try queries_repo.availableSince(&database);
const unbounded = try page(&database, arena.allocator(), .{});
try testing.expectEqual(watermark, unbounded.coverage.available_since);
try testing.expect(!unbounded.coverage.complete);
const covered = try page(&database, arena.allocator(), .{ .since = watermark });
try testing.expect(covered.coverage.complete);
const older = try page(&database, arena.allocator(), .{ .since = watermark - 1 });
try testing.expect(!older.coverage.complete);
}
test "a detail row carries every provenance field the row stored" {
var database = try openLog();
defer database.close();
try seed(&database, 1);
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
const row = (try queries_repo.detailById(&database, arena, 1)).?;
const view = provenance_view.fromDetail(row);
try testing.expectEqual(@as(i64, 1), view.id);
try testing.expectEqualStrings("ads.example", view.request.domain);
try testing.expectEqualStrings("192.0.2.10", view.request.client);
try testing.expectEqual(@as(u16, 1), view.request.qclass);
try testing.expectEqualStrings("default", view.group.name);
try testing.expectEqual(provenance.PolicyAction.block, view.policy.action);
try testing.expectEqualStrings("ads.example", view.policy.matched);
try testing.expectEqual(provenance.RouteKind.blocked, view.route.kind);
try testing.expectEqualStrings("", view.route.upstream);
try testing.expectEqual(@as(?i64, 500), view.response.duration_us);
try testing.expectEqual(
@as(?queries_repo.QueryDetail, null),
try queries_repo.detailById(&database, arena, 99),
);
}
+27 -1
View File
@@ -14,6 +14,7 @@
const std = @import("std");
const coverage = @import("../coverage.zig");
const db = @import("../../storage/db.zig");
const http_util = @import("../http_util.zig");
const queries_repo = @import("../../storage/repositories/queries_repo.zig");
@@ -99,6 +100,10 @@ pub const TotalsBody = struct {
cached: u64,
clients: u64,
avg_response_time_us: ?i64,
/// Judged against `since`, which is the window this body reports on — so a
/// dashboard can say "history starts here" instead of charting a pruned
/// stretch as a quiet one.
coverage: coverage.Coverage,
};
pub const TimeseriesBody = struct {
@@ -107,6 +112,7 @@ pub const TimeseriesBody = struct {
until: i64,
bucket_seconds: u32,
buckets: []const queries_repo.Bucket,
coverage: coverage.Coverage,
};
pub fn totals(
@@ -121,6 +127,9 @@ pub fn totals(
const result = queries_repo.statsTotals(database, span.since, span.until) catch |err| {
return internal(request, "stats totals", err);
};
const covered = coverage.read(database, span.since) catch |err| {
return internal(request, "stats coverage", err);
};
return http_util.respondJson(request, .ok, TotalsBody{
.period = period.label(),
@@ -131,6 +140,7 @@ pub fn totals(
.cached = result.cached,
.clients = result.distinct_clients,
.avg_response_time_us = result.avg_response_time_us,
.coverage = covered,
}, &.{});
}
@@ -148,6 +158,9 @@ pub fn timeseries(
const written = queries_repo.timeseries(database, span.since, span.bucket_seconds, out) catch |err| {
return internal(request, "stats timeseries", err);
};
const covered = coverage.read(database, span.since) catch |err| {
return internal(request, "stats coverage", err);
};
return http_util.respondJson(request, .ok, TimeseriesBody{
.period = period.label(),
@@ -155,6 +168,7 @@ pub fn timeseries(
.until = span.until,
.bucket_seconds = span.bucket_seconds,
.buckets = out[0..written],
.coverage = covered,
}, &.{});
}
@@ -268,11 +282,23 @@ fn writeRow(writer: *queries_repo.BatchWriter, timestamp: i64, blocked: bool, ca
.domain = "example.com",
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 0,
.blocked = blocked,
.block_reason = if (blocked) "blocklist_domain" else null,
.response_time_us = 1000,
.cache_hit = cached,
.upstream = null,
.group_id = 1,
.group_name = "default",
.policy_action = if (blocked) .block else .allow,
.policy_reason = if (blocked) .blocklist_domain else .no_match,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = if (blocked) .blocked else .upstream,
.forward_zone = null,
}};
try writer.writeBatch(&rows);
}
+218 -10
View File
@@ -226,14 +226,47 @@ paths:
"503":
$ref: "#/components/responses/Unavailable"
/api/queries/{id}:
get:
summary: One query, fully explained
description: |
The full provenance of one logged query: what was asked, which group's
policy applied, what that policy decided and matched on, what was
rewritten, where the answer came from, and what the client received.
Every field is a fact recorded when the query was answered, so a group
or blocklist renamed since keeps the name it had.
parameters:
- name: id
in: path
required: true
schema: { type: integer, minimum: 1 }
responses:
"200":
description: The query.
content:
application/json:
schema:
$ref: "#/components/schemas/QueryDetail"
"401":
$ref: "#/components/responses/Unauthorized"
"404":
$ref: "#/components/responses/NotFound"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/queries/live:
get:
summary: Live query stream (server-sent events)
description: |
`text/event-stream`. The stream opens with `retry: 3000`, then sends
one `event: query` frame per resolved query whose `data:` line is a
JSON object with the `/api/queries` row fields minus `id` (the entry
precedes persistence). A `: ping` comment goes out every 15 seconds.
`Provenance` object the body of `/api/queries/{id}` without its `id`,
which does not exist yet because the entry precedes its own insert.
A `: ping` comment goes out every 15 seconds.
A client that falls more than 64 events behind is disconnected and
should re-sync via `/api/queries` after reconnecting. Connections
per address are capped by `web.sse_max_connections_per_ip`; the
@@ -1855,9 +1888,33 @@ components:
type: boolean
description: False when no password is configured; no cookie is set.
PolicyAction:
type: string
description: |
Whether the filtering policy reached a verdict. `not_evaluated` is the
honest answer for a query answered before filtering could apply, and is
not the same as `allow`.
enum: [not_evaluated, allow, block]
PolicyReason:
type: string
description: |
Why the policy landed where it did. The first nine are the matcher's own
verdicts; the rest name a pipeline step that decided without consulting
the matcher.
enum: [rule_allow_exact, rule_block_exact, rule_allow_wildcard, rule_block_wildcard, rule_allow_regex, rule_block_regex, blocklist_exception, blocklist_domain, blocklist_wildcard, local_record, forward_zone, non_in_class, paused, snapshot_unavailable, no_match, protocol_error]
RouteKind:
type: string
description: Where the answer the client received came from.
enum: [blocked, local, forward_zone, upstream, cache, rejected]
QueryRow:
type: object
required: [id, ts, domain, client_ip, qtype, blocked, block_reason, response_time_us, cache_hit, upstream]
description: |
The summary projection the query-log table scans. The full provenance of
a row is one request away at `/api/queries/{id}`.
required: [id, ts, domain, client_ip, qtype, qclass, rcode, blocked, response_time_us, cache_hit, upstream, policy_action, policy_reason, route_kind]
properties:
id: { type: integer }
ts:
@@ -1868,10 +1925,11 @@ components:
qtype:
type: integer
nullable: true
qclass: { type: integer }
rcode:
type: integer
description: The twelve-bit EDNS extended code, not the four header bits alone.
blocked: { type: boolean }
block_reason:
type: string
description: Empty when the query was not blocked.
response_time_us:
type: integer
nullable: true
@@ -1880,11 +1938,155 @@ components:
nullable: true
upstream:
type: string
description: Empty for cache hits and local answers.
description: Empty for cache hits, local answers and blocked queries.
policy_action:
$ref: "#/components/schemas/PolicyAction"
policy_reason:
$ref: "#/components/schemas/PolicyReason"
route_kind:
$ref: "#/components/schemas/RouteKind"
ProvenanceRequest:
type: object
required: [time, domain, client, qtype, qclass]
properties:
time:
type: integer
description: Unix seconds.
domain: { type: string }
client: { type: string }
qtype:
type: integer
nullable: true
qclass: { type: integer }
ProvenanceGroup:
type: object
description: |
The client's filtering group at the time of the query, as a historical
fact: the id may name a group since renamed or deleted.
required: [id, name]
properties:
id:
type: integer
nullable: true
name: { type: string }
ProvenancePolicy:
type: object
required: [action, reason, matched, source_id, source_name]
properties:
action:
$ref: "#/components/schemas/PolicyAction"
reason:
$ref: "#/components/schemas/PolicyReason"
matched:
type: string
description: The rule pattern or list entry that decided; empty when nothing matched.
source_id:
type: integer
nullable: true
source_name:
type: string
description: The blocklist the match came from; empty for a rule.
ProvenanceRewrites:
type: object
required: [cname_target, safe_search_target]
properties:
cname_target:
type: string
description: Set when the decision was made about a CNAME target rather than the queried name.
safe_search_target: { type: string }
ProvenanceRoute:
type: object
required: [kind, forward_zone, upstream]
properties:
kind:
$ref: "#/components/schemas/RouteKind"
forward_zone: { type: string }
upstream:
type: string
description: |
Non-empty only for an attempted upstream or forward-zone exchange,
including one that failed. Already redacted: the userinfo, path,
query and fragment of a resolver url never reach here.
ProvenanceResponse:
type: object
required: [rcode, duration_us]
properties:
rcode:
type: integer
description: The twelve-bit EDNS extended code, not the four header bits alone.
duration_us:
type: integer
nullable: true
Provenance:
type: object
description: |
One query, fully explained, in the order a query meets the pipeline. The
`data:` payload of a live-stream `event: query` frame is exactly this.
required: [request, group, policy, rewrites, route, response]
properties:
request:
$ref: "#/components/schemas/ProvenanceRequest"
group:
$ref: "#/components/schemas/ProvenanceGroup"
policy:
$ref: "#/components/schemas/ProvenancePolicy"
rewrites:
$ref: "#/components/schemas/ProvenanceRewrites"
route:
$ref: "#/components/schemas/ProvenanceRoute"
response:
$ref: "#/components/schemas/ProvenanceResponse"
QueryDetail:
type: object
description: |
`Provenance` plus the row id. Written out rather than composed with
`allOf` so the drift guard reads one property list per schema.
required: [id, request, group, policy, rewrites, route, response]
properties:
id: { type: integer }
request:
$ref: "#/components/schemas/ProvenanceRequest"
group:
$ref: "#/components/schemas/ProvenanceGroup"
policy:
$ref: "#/components/schemas/ProvenancePolicy"
rewrites:
$ref: "#/components/schemas/ProvenanceRewrites"
route:
$ref: "#/components/schemas/ProvenanceRoute"
response:
$ref: "#/components/schemas/ProvenanceResponse"
Coverage:
type: object
description: |
How much of the requested window the query log can still answer for.
Retention deletes rows and advances the watermark in one transaction, so
a client can tell an empty window from a pruned one instead of charting
the gap as zero.
required: [complete, available_since]
properties:
complete:
type: boolean
description: |
True only when the window's lower bound is at or after
`available_since`. A request with no lower bound asks about all of
history, which no file that has ever pruned can promise.
available_since:
type: integer
description: The oldest instant the file is complete for, unix seconds.
QueriesPage:
type: object
required: [queries, next_before]
required: [queries, next_before, coverage]
properties:
queries:
type: array
@@ -1894,6 +2096,8 @@ components:
type: integer
nullable: true
description: Cursor for the next page; null on the last page.
coverage:
$ref: "#/components/schemas/Coverage"
DiagnosticEvent:
type: object
@@ -1983,7 +2187,7 @@ components:
StatsTotals:
type: object
required: [period, since, until, queries, blocked, cached, clients, avg_response_time_us]
required: [period, since, until, queries, blocked, cached, clients, avg_response_time_us, coverage]
properties:
period:
type: string
@@ -2004,6 +2208,8 @@ components:
type: integer
nullable: true
description: Null when no query in the window recorded a time.
coverage:
$ref: "#/components/schemas/Coverage"
Bucket:
type: object
@@ -2018,7 +2224,7 @@ components:
StatsTimeseries:
type: object
required: [period, since, until, bucket_seconds, buckets]
required: [period, since, until, bucket_seconds, buckets, coverage]
properties:
period:
type: string
@@ -2030,6 +2236,8 @@ components:
type: array
items:
$ref: "#/components/schemas/Bucket"
coverage:
$ref: "#/components/schemas/Coverage"
Lookup:
type: object
+278
View File
@@ -0,0 +1,278 @@
//! The wire shape of one query's provenance, defined once.
//!
//! Two surfaces answer with it: `GET /api/queries/{id}`, which reads a stored
//! row, and the `event: query` frames of `GET /api/queries/live`, which read a
//! queued entry that has not been written yet. They must describe a query the
//! same way — an operator watching the stream and an operator opening the row
//! afterwards are looking at the same facts — so the live event *is* this DTO
//! and the detail body is this DTO plus the row id.
//!
//! It is nested rather than flat because the six groups answer six different
//! questions, in the order a query meets them: what was asked, which group's
//! policy applied, what that policy decided, what was rewritten on the way,
//! where the answer came from, and what the client got back.
//!
//! The list row (`queries_repo.QueryRow`) stays a separate, flatter summary.
//! A table the operator scans wants columns, not a tree, and the full story is
//! one request away.
//!
//! Every text field follows the repository's convention: a NULL column reads as
//! `""`, and `""` on the wire means "absent". No field is ever written as an
//! empty string that means something else.
const std = @import("std");
const logger = @import("../storage/logger.zig");
const provenance = @import("../storage/provenance.zig");
const queries_repo = @import("../storage/repositories/queries_repo.zig");
/// What the client asked. `time` is unix seconds; `qtype` is null for a
/// question whose type the log never recorded.
pub const Request = struct {
time: i64,
domain: []const u8,
client: []const u8,
qtype: ?u16,
qclass: u16,
};
/// The client's filtering group at the time of the query, as a historical fact:
/// the id may name a group that has since been renamed or deleted, which is why
/// the name is stored beside it rather than joined at read time.
pub const Group = struct {
id: ?i64,
name: []const u8,
};
/// What the policy decided and what it matched on. `matched` is the rule
/// pattern or list entry that decided; `source_id`/`source_name` name the
/// blocklist it came from, and are absent for a rule.
pub const Policy = struct {
action: provenance.PolicyAction,
reason: provenance.PolicyReason,
matched: []const u8,
source_id: ?i64,
source_name: []const u8,
};
/// The two rewrites that can happen between the question and the answer.
/// `cname_target` is set when the decision was made about a CNAME target rather
/// than the queried name.
pub const Rewrites = struct {
cname_target: []const u8,
safe_search_target: []const u8,
};
/// Where the answer came from. `upstream` is non-empty only for an attempted
/// upstream or forward-zone exchange, including one that failed, and is already
/// redacted — the path, query and userinfo of a resolver url never reach here.
pub const Route = struct {
kind: provenance.RouteKind,
forward_zone: []const u8,
upstream: []const u8,
};
/// What the client saw. `rcode` is the twelve-bit EDNS extended code, not the
/// four header bits alone.
pub const Response = struct {
rcode: u16,
duration_us: ?i64,
};
/// One query, fully explained. The live stream's `data:` payload is exactly
/// this.
pub const Provenance = struct {
request: Request,
group: Group,
policy: Policy,
rewrites: Rewrites,
route: Route,
response: Response,
};
/// `GET /api/queries/{id}`: the same six groups, plus the id the caller asked
/// for. Spelled out rather than composed, because a JSON object is flat at its
/// top level and Zig has no field-splicing; the `comptime` block below is what
/// keeps the two from drifting.
pub const QueryDetail = struct {
id: i64,
request: Request,
group: Group,
policy: Policy,
rewrites: Rewrites,
route: Route,
response: Response,
};
comptime {
const detail = @typeInfo(QueryDetail).@"struct".fields;
const shared = @typeInfo(Provenance).@"struct".fields;
std.debug.assert(detail.len == shared.len + 1);
std.debug.assert(std.mem.eql(u8, detail[0].name, "id"));
for (detail[1..], shared) |a, b| {
std.debug.assert(std.mem.eql(u8, a.name, b.name));
std.debug.assert(a.type == b.type);
}
}
/// A stored row, as the detail endpoint answers it. Borrows `row`'s strings,
/// which the caller's arena owns.
pub fn fromDetail(row: queries_repo.QueryDetail) QueryDetail {
return .{
.id = row.id,
.request = .{
.time = row.ts,
.domain = row.domain,
.client = row.client_ip,
.qtype = row.qtype,
.qclass = row.qclass,
},
.group = .{ .id = row.group_id, .name = row.group_name },
.policy = .{
.action = row.policy_action,
.reason = row.policy_reason,
.matched = row.matched,
.source_id = row.source_id,
.source_name = row.source_name,
},
.rewrites = .{
.cname_target = row.cname_target,
.safe_search_target = row.safe_search_target,
},
.route = .{
.kind = row.route_kind,
.forward_zone = row.forward_zone,
.upstream = row.upstream,
},
.response = .{ .rcode = row.rcode, .duration_us = row.response_time_us },
};
}
/// A queued entry, as the live stream sends it. Borrows the entry's buffers, so
/// the result must not outlive the entry it was taken from — in the stream both
/// live in one loop iteration.
///
/// There is no id: the entry precedes its own insert, so no row id exists yet.
pub fn fromEntry(entry: *const logger.Entry) Provenance {
return .{
.request = .{
.time = entry.timestamp,
.domain = entry.domain(),
.client = entry.clientIp(),
.qtype = entry.qtype,
.qclass = entry.qclass,
},
.group = .{ .id = entry.group_id, .name = entry.groupName() },
.policy = .{
.action = entry.policy_action,
.reason = entry.policy_reason,
.matched = entry.matched(),
.source_id = entry.source_id,
.source_name = entry.sourceName(),
},
.rewrites = .{
.cname_target = entry.cnameTarget(),
.safe_search_target = entry.safeSearchTarget(),
},
.route = .{
.kind = entry.route_kind,
.forward_zone = entry.forwardZone(),
.upstream = entry.upstream(),
},
.response = .{ .rcode = entry.rcode, .duration_us = entry.response_time_us },
};
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const testing = std.testing;
test "a stored row and a queued entry describe the same query identically" {
const row: queries_repo.QueryDetail = .{
.id = 7,
.ts = 1_700_000_000,
.domain = "ads.example",
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 3,
.blocked = true,
.response_time_us = 1234,
.cache_hit = false,
.upstream = "https://dns.example",
.group_id = 2,
.group_name = "kids",
.policy_action = .block,
.policy_reason = .blocklist_domain,
.matched = "tracker.example",
.source_id = 5,
.source_name = "StevenBlack",
.cname_target = "tracker.example",
.safe_search_target = "forcesafesearch.example",
.route_kind = .blocked,
.forward_zone = "lan",
};
const entry: logger.Entry = .init(.{
.timestamp = row.ts,
.domain = row.domain,
.client_ip = row.client_ip,
.qtype = row.qtype,
.qclass = row.qclass,
.rcode = row.rcode,
.blocked = row.blocked,
.response_time_us = row.response_time_us,
.cache_hit = row.cache_hit,
.upstream = row.upstream,
.group_id = row.group_id,
.group_name = row.group_name,
.policy_action = row.policy_action,
.policy_reason = row.policy_reason,
.matched = row.matched,
.source_id = row.source_id,
.source_name = row.source_name,
.cname_target = row.cname_target,
.safe_search_target = row.safe_search_target,
.route_kind = row.route_kind,
.forward_zone = row.forward_zone,
});
const from_row = fromDetail(row);
const from_entry = fromEntry(&entry);
try testing.expectEqual(@as(i64, 7), from_row.id);
inline for (@typeInfo(Provenance).@"struct".fields) |field| {
const a = @field(from_row, field.name);
const b = @field(from_entry, field.name);
inline for (@typeInfo(field.type).@"struct".fields) |inner| {
const left = @field(a, inner.name);
const right = @field(b, inner.name);
if (@TypeOf(left) == []const u8) {
try testing.expectEqualStrings(left, right);
} else {
try testing.expectEqual(left, right);
}
}
}
}
test "an unexplained query serializes as nulls and empty strings, not as absent keys" {
const entry: logger.Entry = .init(.{
.timestamp = 1,
.domain = "safe.example",
.client_ip = "192.0.2.11",
});
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
defer allocating.deinit();
try std.json.Stringify.value(fromEntry(&entry), .{}, &allocating.writer);
const text = allocating.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"qtype\":null"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"duration_us\":null"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"id\":null"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"upstream\":\"\""));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"action\":\"not_evaluated\""));
}
+5 -1
View File
@@ -67,6 +67,10 @@ pub const table: []const router.RouteInfo = &.{
// Query log, stats, live stream, lookup.
.{ .method = .GET, .pattern = "/api/queries", .auth = .session, .policy = .read, .handler = queries.list },
.{ .method = .GET, .pattern = "/api/queries/live", .auth = .session, .policy = .read, .handler = live.stream, .rate_limit = .exempt },
// Listed after the literal `live`, which a linear first-match scan reaches
// first — though `{id}` would refuse it anyway, since it captures a
// positive integer and nothing else.
.{ .method = .GET, .pattern = "/api/queries/{id}", .auth = .session, .policy = .read, .handler = queries.detail },
.{ .method = .GET, .pattern = "/api/stats", .auth = .session, .policy = .read, .handler = stats.totals },
.{ .method = .GET, .pattern = "/api/stats/timeseries", .auth = .session, .policy = .read, .handler = stats.timeseries },
.{ .method = .GET, .pattern = "/api/lookup", .auth = .session, .policy = .read, .handler = lookup.handle },
@@ -152,7 +156,7 @@ const std = @import("std");
const testing = std.testing;
test "the table carries every endpoint of the milestone" {
try testing.expectEqual(@as(usize, 60), table.len);
try testing.expectEqual(@as(usize, 61), table.len);
}
test "no two entries claim the same method and pattern" {
+8
View File
@@ -25,6 +25,14 @@ pub const max_subscribers = 32;
/// Entries one subscriber may fall behind by. At household query rates this is
/// several seconds of slack on a stalled TCP connection.
///
/// The ring is embedded in the slot, so this multiplies `@sizeOf(logger.Entry)`
/// — about 1.8 KiB once milestone 28 widened it for provenance. One subscriber
/// therefore costs roughly 115 KiB of ring and the whole hub roughly 3.6 MiB,
/// allocated once for the life of the process. That is the reason to keep both
/// this and `max_subscribers` small: they are paid whether or not anyone is
/// watching. `logger.query_log_buffer_max` bounds the other consumer of the
/// same width, the writer queue.
pub const ring_capacity = 64;
pub const SubscriberId = enum(u8) { _ };
+709 -9
View File
@@ -38,15 +38,20 @@ const header = @import("../dns/header.zig");
const http_util = @import("http_util.zig");
const local_repo = @import("../storage/repositories/local_repo.zig");
const local_tables_mod = @import("../server/local_tables.zig");
const logger_mod = @import("../storage/logger.zig");
const manager_mod = @import("../filter/manager.zig");
const migrations = @import("../storage/migrations.zig");
const name = @import("../dns/name.zig");
const openapi = @import("openapi.zig");
const packet = @import("../dns/packet.zig");
const pause_mod = @import("../server/pause.zig");
const coverage_mod = @import("coverage.zig");
const pool_mod = @import("../upstream/pool.zig");
const provenance = @import("../storage/provenance.zig");
const provenance_view = @import("provenance_view.zig");
const queries_repo = @import("../storage/repositories/queries_repo.zig");
const querylog_schema = @import("../storage/querylog_schema.zig");
const query_sink = @import("../server/query_sink.zig");
const question = @import("../dns/question.zig");
const router = @import("router.zig");
const server = @import("server.zig");
@@ -257,7 +262,10 @@ fn contentLength(head: []const u8) ?usize {
// the environment: the real web stack over in-memory databases
// ---------------------------------------------------------------------------
const seeded_query_rows = 25;
/// The rows the uniform loop writes, before the two provenance-rich ones the
/// detail endpoint and the credential sweep read.
const seeded_plain_query_rows = 25;
const seeded_query_rows = seeded_plain_query_rows + 2;
const EnvOptions = struct {
password_hash: []const u8 = "",
@@ -270,6 +278,10 @@ const EnvOptions = struct {
/// wants; the file-authority tests below name a path.
authority: server.Authority = .database,
reconciled_at: ?i64 = null,
/// False detaches the query log from the web state, which is the box a
/// `logging.query_log = false` operator runs. Every query-log route then
/// answers 503 rather than an empty page, which would be a lie.
querylog: bool = true,
};
/// Heap-allocated because `state` and the listener hold pointers into it.
@@ -400,7 +412,7 @@ const Env = struct {
.limiter = &self.limiter,
.hub = self.hub,
.config_db = &self.config_db,
.querylog_db = &self.querylog_db,
.querylog_db = if (options.querylog) &self.querylog_db else null,
.events = &self.events_store,
.version = "w10-test",
.started_unix = std.Io.Clock.real.now(ioh).toSeconds(),
@@ -472,27 +484,108 @@ fn seedConfig(database: *db.Db) !void {
);
}
/// The oldest instant the seeded log is complete for. Pinned rather than taken
/// from `unixepoch()`, which the schema's own seed uses: the contract samples
/// are byte-compared, so a clock in `coverage.available_since` would make the
/// golden a property of the machine that generated it.
///
/// It equals the oldest seeded row's timestamp, so a request bounded at exactly
/// this instant is complete and one bounded a second earlier is not.
const seeded_available_since: i64 = 1_700_000_000;
fn seedQueryLog(database: *db.Db) !void {
try database.exec(
\\UPDATE querylog_meta SET created_at = 1700000000, available_since = 1700000000 WHERE id = 1
);
var writer = try queries_repo.BatchWriter.init(database);
defer writer.deinit();
var domain_buf: [32]u8 = undefined;
var index: usize = 0;
while (index < seeded_query_rows) : (index += 1) {
while (index < seeded_plain_query_rows) : (index += 1) {
const domain = std.fmt.bufPrint(&domain_buf, "d{d}.example", .{index}) catch unreachable;
const blocked = index % 5 == 0;
// The three states the handler can actually produce (`Context.cacheHit`
// and `route_kind` are set together): a blocked answer consulted no
// cache and named no resolver, a cache hit named no resolver, and only
// an upstream exchange did both.
const from_cache = !blocked and index % 2 == 0;
try writer.writeBatch(&.{.{
.timestamp = 1_700_000_000 + @as(i64, @intCast(index)),
.domain = domain,
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 0,
.blocked = blocked,
.block_reason = if (blocked) "blocklist_domain" else null,
.response_time_us = 250,
.cache_hit = if (blocked) null else (index % 2 == 0),
.upstream = if (blocked) null else "https://dns.example/dns-query",
.cache_hit = if (blocked) null else from_cache,
.upstream = if (blocked or from_cache) null else "https://dns.example/dns-query",
.group_id = 1,
.group_name = "default",
.policy_action = if (blocked) .block else .allow,
.policy_reason = if (blocked) .blocklist_domain else .no_match,
.matched = if (blocked) domain else null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = if (blocked) .blocked else if (from_cache) .cache else .upstream,
.forward_zone = null,
}});
}
// Two rows with provenance the loop above never produces, so the detail
// endpoint and its contract sample have a real row to read. They are the
// newest rows, so a first page shows them.
try writer.writeBatch(&.{.{
.timestamp = 1_700_000_000 + seeded_plain_query_rows,
.domain = "news.example",
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 0,
.blocked = false,
.response_time_us = 18_400,
.cache_hit = false,
.upstream = "https://dns.example/dns-query",
.group_id = 1,
.group_name = "default",
.policy_action = .allow,
.policy_reason = .no_match,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = .upstream,
.forward_zone = null,
}});
try writer.writeBatch(&.{.{
.timestamp = 1_700_000_000 + seeded_plain_query_rows + 1,
.domain = "shop.example",
.client_ip = "192.0.2.11",
.qtype = 1,
.qclass = 1,
.rcode = 3,
.blocked = true,
.response_time_us = 900,
.cache_hit = null,
.upstream = null,
.group_id = 2,
.group_name = "kids",
.policy_action = .block,
.policy_reason = .blocklist_wildcard,
.matched = "||tracker.example^",
.source_id = 4,
.source_name = "StevenBlack",
.cname_target = "cdn.tracker.example",
.safe_search_target = null,
.route_kind = .blocked,
.forward_zone = null,
}});
}
/// A fixed instant, like every other seeded timestamp here: the contract
@@ -653,6 +746,7 @@ const contract = [_]Contract{
// Query log, stats, live stream, upstream health.
.{ .method = .GET, .pattern = "/api/queries", .auth = .session, .policy = .read, .target = "/api/queries?limit=10", .status = 200, .check = jsonShape(handlers_queries.Page) },
.{ .method = .GET, .pattern = "/api/queries/{id}", .auth = .session, .policy = .read, .target = "/api/queries/27", .status = 200, .check = jsonShape(provenance_view.QueryDetail) },
.{ .method = .GET, .pattern = "/api/queries/live", .auth = .session, .policy = .read, .rate_limit = .exempt, .target = "/api/queries/live", .status = 200, .kind = .sse },
.{ .method = .GET, .pattern = "/api/stats", .auth = .session, .policy = .read, .target = "/api/stats?period=1h", .status = 200, .check = jsonShape(handlers_stats.TotalsBody) },
.{ .method = .GET, .pattern = "/api/stats/timeseries", .auth = .session, .policy = .read, .target = "/api/stats/timeseries?period=1h", .status = 200, .check = jsonShape(handlers_stats.TimeseriesBody) },
@@ -1653,12 +1747,15 @@ fn sseStream(io: std.Io, env: *Env) anyerror!void {
.domain = "live.example",
.client_ip = "192.0.2.99",
.qtype = 1,
.qclass = 1,
.blocked = true,
.block_reason = "blocklist_domain",
.policy_action = .block,
.policy_reason = .blocklist_domain,
.route_kind = .blocked,
}));
try conn.readChunkedUntil(&seen, env.gpa, "event: query");
try conn.readChunkedUntil(&seen, env.gpa, "\"domain\":\"live.example\"");
try conn.readChunkedUntil(&seen, env.gpa, "\"blocked\":true");
try conn.readChunkedUntil(&seen, env.gpa, "\"reason\":\"blocklist_domain\"");
// The cap is per address and the environment allows one stream: a second
// subscriber from the same address is refused while the first is open.
@@ -1784,7 +1881,7 @@ fn paginationWalk(io: std.Io, env: *Env) anyerror!void {
try testing.expect(pages < 10);
}
// 25 seeded rows walk as 10, 10 and 5, with the cursor ending exactly
// 27 seeded rows walk as 10, 10 and 7, with the cursor ending exactly
// after the third page.
try testing.expectEqual(@as(usize, seeded_query_rows), total);
try testing.expectEqual(@as(usize, 3), pages);
@@ -1800,6 +1897,333 @@ test "W10 keyset pagination walks the seeded log exactly once, newest first" {
try bounded(env.io(), default_budget, paginationWalk, .{ env.io(), env });
}
// ---------------------------------------------------------------------------
// query provenance: the detail endpoint, coverage, and the credential sweep
// (milestone 28)
// ---------------------------------------------------------------------------
/// The id of the seeded CNAME-uncloaked block, which is the last row written.
const seeded_detail_id = seeded_query_rows;
fn detailWalk(io: std.Io, env: *Env) anyerror!void {
var arena_state: std.heap.ArenaAllocator = .init(env.gpa);
defer arena_state.deinit();
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [64 * 1024]u8 = undefined;
var target_buf: [64]u8 = undefined;
const target = try std.fmt.bufPrint(&target_buf, "/api/queries/{d}", .{seeded_detail_id});
try conn.request("GET", target, null, null);
const response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), response.status);
const detail = try std.json.parseFromSliceLeaky(
provenance_view.QueryDetail,
arena_state.allocator(),
response.body,
.{ .ignore_unknown_fields = false },
);
try testing.expectEqual(@as(i64, seeded_detail_id), detail.id);
try testing.expectEqualStrings("shop.example", detail.request.domain);
try testing.expectEqualStrings("192.0.2.11", detail.request.client);
try testing.expectEqual(@as(u16, 1), detail.request.qclass);
try testing.expectEqual(@as(?i64, 2), detail.group.id);
try testing.expectEqualStrings("kids", detail.group.name);
try testing.expectEqual(provenance.PolicyAction.block, detail.policy.action);
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, detail.policy.reason);
try testing.expectEqualStrings("||tracker.example^", detail.policy.matched);
try testing.expectEqual(@as(?i64, 4), detail.policy.source_id);
try testing.expectEqualStrings("StevenBlack", detail.policy.source_name);
try testing.expectEqualStrings("cdn.tracker.example", detail.rewrites.cname_target);
try testing.expectEqualStrings("", detail.rewrites.safe_search_target);
try testing.expectEqual(provenance.RouteKind.blocked, detail.route.kind);
// A blocked query attempted no exchange, so it names no resolver.
try testing.expectEqualStrings("", detail.route.upstream);
try testing.expectEqual(@as(u16, 3), detail.response.rcode);
try testing.expectEqual(@as(?i64, 900), detail.response.duration_us);
// An id past the end of the log and an id retention would have pruned are
// the same answer.
const missing = try std.fmt.bufPrint(&target_buf, "/api/queries/{d}", .{seeded_query_rows + 1000});
try conn.request("GET", missing, null, null);
const not_found = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 404), not_found.status);
try testing.expect(std.mem.containsAtLeast(u8, not_found.body, 1, "\"error\""));
// A non-positive id never reaches SQL: the pattern captures a positive
// integer or does not match, so this is a routing 404.
try conn.request("GET", "/api/queries/0", null, null);
try testing.expectEqual(@as(u16, 404), (try conn.receive(&body_buf)).status);
}
test "W10 milestone 28: the detail endpoint answers one row and 404s the rest" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{});
defer env.destroy();
try bounded(env.io(), default_budget, detailWalk, .{ env.io(), env });
}
fn detailUnavailable(io: std.Io, env: *Env) anyerror!void {
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [8 * 1024]u8 = undefined;
for ([_][]const u8{ "/api/queries/1", "/api/queries?limit=1", "/api/stats", "/api/stats/timeseries" }) |target| {
try conn.request("GET", target, null, null);
const response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 503), response.status);
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "query log unavailable"));
}
}
test "W10 milestone 28: a box with no query log answers 503, not an empty page" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{ .querylog = false });
defer env.destroy();
try bounded(env.io(), default_budget, detailUnavailable, .{ env.io(), env });
}
fn coverageWalk(io: std.Io, env: *Env) anyerror!void {
var arena_state: std.heap.ArenaAllocator = .init(env.gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [64 * 1024]u8 = undefined;
var target_buf: [64]u8 = undefined;
// No lower bound: the request asks about all of history, which a file that
// may have pruned cannot promise.
try conn.request("GET", "/api/queries?limit=1", null, null);
const unbounded = try std.json.parseFromSliceLeaky(
handlers_queries.Page,
arena,
(try conn.receive(&body_buf)).body,
.{ .ignore_unknown_fields = false },
);
try testing.expectEqual(seeded_available_since, unbounded.coverage.available_since);
try testing.expect(!unbounded.coverage.complete);
// Bounded exactly at the watermark.
const at = try std.fmt.bufPrint(&target_buf, "/api/queries?limit=1&since={d}", .{seeded_available_since});
try conn.request("GET", at, null, null);
const covered = try std.json.parseFromSliceLeaky(
handlers_queries.Page,
arena,
(try conn.receive(&body_buf)).body,
.{ .ignore_unknown_fields = false },
);
try testing.expect(covered.coverage.complete);
// One second earlier, and the window reaches past what the file holds.
const before = try std.fmt.bufPrint(&target_buf, "/api/queries?limit=1&since={d}", .{seeded_available_since - 1});
try conn.request("GET", before, null, null);
const partial = try std.json.parseFromSliceLeaky(
handlers_queries.Page,
arena,
(try conn.receive(&body_buf)).body,
.{ .ignore_unknown_fields = false },
);
try testing.expect(!partial.coverage.complete);
// The stats endpoints judge the same watermark against their own aligned
// window, which for any live period starts well after the seeded rows.
try conn.request("GET", "/api/stats?period=1h", null, null);
const totals = try std.json.parseFromSliceLeaky(
handlers_stats.TotalsBody,
arena,
(try conn.receive(&body_buf)).body,
.{ .ignore_unknown_fields = false },
);
try testing.expectEqual(seeded_available_since, totals.coverage.available_since);
try testing.expectEqual(totals.since >= seeded_available_since, totals.coverage.complete);
try conn.request("GET", "/api/stats/timeseries?period=1h", null, null);
const series = try std.json.parseFromSliceLeaky(
handlers_stats.TimeseriesBody,
arena,
(try conn.receive(&body_buf)).body,
.{ .ignore_unknown_fields = false },
);
try testing.expectEqual(totals.since, series.since);
try testing.expectEqual(totals.coverage.complete, series.coverage.complete);
}
test "W10 milestone 28: every window-bounded endpoint reports its own coverage" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{});
defer env.destroy();
try bounded(env.io(), default_budget, coverageWalk, .{ env.io(), env });
}
/// NextDNS's shape: the account id rides in the path, which is exactly where a
/// credential lives in a url an operator may legitimately configure.
/// `Endpoint.parse` refuses userinfo, so the path is the shape a real
/// configuration can carry a secret in — and the path is what
/// `safe_url.redact` drops.
const sweep_token = "b1c2d3";
const sweep_upstream_url = "https://dns.nextdns.io/" ++ sweep_token;
/// What every surface must show instead. The origin survives redaction — an
/// operator reading a failure has to know where the query went — so each
/// surface is checked for it too: one that showed nothing at all would pass a
/// secret check by saying nothing.
const sweep_redacted_upstream = "https://dns.nextdns.io";
/// The name the swept query asks for, so each surface can be pinned to the row
/// this test produced rather than to a seeded one.
const sweep_domain = "creds.example";
/// Names the resolver and never its token.
fn expectRedacted(text: []const u8) !void {
try testing.expect(std.mem.containsAtLeast(u8, text, 1, sweep_redacted_upstream));
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, sweep_token));
}
/// The cross-surface credential sweep, driven end to end: a real `Handler`
/// answers a real query through a resolver whose url carries a token, and the
/// entry travels the production path — `QuerySink`, then the hub and the
/// logger, then the query log the API reads. Nothing here redacts anything, so
/// a handler that stopped redacting fails this test.
///
/// Four surfaces read the same query back: the stored row, straight out of
/// SQLite, and the three the operator's browser sees — the live frame, the list
/// page and the detail body. A leak on any one of them is a secret in a browser
/// history, and the four are separate code paths to the same text.
fn credentialSweep(
io: std.Io,
env: *Env,
query_logger: *logger_mod.Logger,
handler: *dns_handler.Handler,
) anyerror!void {
var arena_state: std.heap.ArenaAllocator = .init(env.gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
// Subscribed before the query runs: the hub publishes to whoever is
// listening at that moment and keeps nothing for a later reader.
var seen: std.ArrayList(u8) = .empty;
defer seen.deinit(env.gpa);
var stream: Conn = undefined;
try openLiveStream(io, env, &stream, &seen);
defer stream.close(io);
var query_buf: [512]u8 = undefined;
var response_buf: [512]u8 = undefined;
var scratch: dns_handler.Scratch = undefined;
const from = address.NetAddress.fromIp(.{ .ip4 = .loopback(53100) });
const query = queryFor(&query_buf, 0x4444, sweep_domain, .a);
try testing.expect(handler.handle(io, .udp, from, query, &response_buf, &scratch) == .reply);
// The live frame. Waiting on the redacted origin rather than on the whole
// frame is safe in both directions: a leaked url starts with it.
try stream.readChunkedUntil(&seen, env.gpa, sweep_redacted_upstream);
try testing.expect(std.mem.containsAtLeast(u8, seen.items, 1, sweep_domain));
try expectRedacted(seen.items);
// The stored row. The producer has already run, so closing the queue and
// running the writer inline drains it in one call: `runWriter` returns when
// a closed queue is empty, and a zero flush interval makes it commit the
// batch it holds rather than wait for company.
query_logger.shutdown(io);
try query_logger.runWriter(io, &env.querylog_db, null);
try testing.expectEqual(@as(u64, 1), query_logger.rows_written.load(.monotonic));
var stmt = try env.querylog_db.prepare(
\\SELECT query_log.id, query_log.upstream
\\FROM query_log JOIN domains ON domains.id = query_log.domain_id
\\WHERE domains.domain = ?
);
defer stmt.deinit();
try stmt.bindText(1, sweep_domain);
try testing.expect(try stmt.step());
const row_id = stmt.columnInt(0);
try testing.expectEqualStrings(sweep_redacted_upstream, stmt.columnText(1));
try testing.expect(!try stmt.step());
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [64 * 1024]u8 = undefined;
var target_buf: [64]u8 = undefined;
// The list row. The swept query is the newest in the log, so a page of one
// is it.
try conn.request("GET", "/api/queries?limit=1", null, null);
const rows = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), rows.status);
const page = try std.json.parseFromSliceLeaky(
handlers_queries.Page,
arena,
rows.body,
.{ .ignore_unknown_fields = false },
);
try testing.expectEqual(@as(usize, 1), page.queries.len);
try testing.expectEqualStrings(sweep_domain, page.queries[0].domain);
try testing.expectEqualStrings(sweep_redacted_upstream, page.queries[0].upstream);
try expectRedacted(rows.body);
// The detail body, read by the row id the database just handed over.
const one = try std.fmt.bufPrint(&target_buf, "/api/queries/{d}", .{row_id});
try conn.request("GET", one, null, null);
const detail_response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), detail_response.status);
const detail = try std.json.parseFromSliceLeaky(
provenance_view.QueryDetail,
arena,
detail_response.body,
.{ .ignore_unknown_fields = false },
);
try testing.expectEqualStrings(sweep_domain, detail.request.domain);
try testing.expectEqualStrings(sweep_redacted_upstream, detail.route.upstream);
try expectRedacted(detail_response.body);
}
test "W10 milestone 28: no query surface echoes a resolver credential" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{});
defer env.destroy();
const io = env.io();
var queue_buf: [4]logger_mod.Entry = undefined;
// Zero flush interval: the drain below is synchronous, and nothing else
// will ever put an entry on this queue for the writer to wait for.
var query_logger: logger_mod.Logger = .init(.{ .query_log_flush_interval_s = 0 }, &queue_buf);
var sink: query_sink.QuerySink = .init(&query_logger, env.hub);
var fake: FakeUpstream = .{ .identity = sweep_upstream_url };
var handler: dns_handler.Handler = .{
.upstream = fake.client(),
.blocking = .{ .mode = .zero, .ttl = 5 },
.forward_read_timeout = .{ .raw = .fromSeconds(2), .clock = .awake },
.manager = &env.mgr,
.pause = &env.pauser,
.sink = &sink,
};
try bounded(io, default_budget, credentialSweep, .{ io, env, &query_logger, &handler });
try testing.expectEqual(@as(u64, 1), fake.calls.load(.monotonic));
}
// ---------------------------------------------------------------------------
// mutation → reload observed (ruling 12)
// ---------------------------------------------------------------------------
@@ -1980,15 +2404,21 @@ fn queryFor(buf: []u8, id: u16, domain: []const u8, qtype: types.Type) []const u
/// can tell whether the filter let the query through.
const FakeUpstream = struct {
calls: std.atomic.Value(u64) = .init(0),
/// The resolver the handler reports as having answered. Operator-supplied
/// text in production, so the credential sweep points it at a url with a
/// token in its path.
identity: []const u8 = "fake://web-upstream",
fn exchangeFn(
ptr: *anyopaque,
io: std.Io,
query: []const u8,
response_buf: []u8,
selected: *?[]const u8,
) transport.ExchangeError![]u8 {
_ = io;
const self: *FakeUpstream = @ptrCast(@alignCast(ptr));
selected.* = self.identity;
_ = self.calls.fetchAdd(1, .monotonic);
const request = packet.parse(query) catch return error.BadResponse;
@@ -2348,6 +2778,272 @@ test "drift guard a bites: methods swapped between two documented paths fail the
try testing.expectEqual(@as(usize, 2), swapped_routes);
}
// ---------------------------------------------------------------------------
// focused schema drift guards (milestone 28)
// ---------------------------------------------------------------------------
//
// Guard a proves every served route is documented and guard b counts the
// operations, and neither looks inside a schema. A field renamed, retyped, made
// nullable or dropped from `required` passes both while breaking every client
// that reads the document — and the query-log provenance shapes are exactly
// where a rename is easy and a wrong `nullable` is silent.
//
// So these read the schema back and hold it to the Zig struct that produces it:
// the same property names, the same types, the same nullability, the same
// requiredness, and no extra property on either side. A `$ref` recurses, so
// checking `QueryDetail` checks all six of its nested objects.
//
// The YAML reader below understands only the shape this document is written in
// — two-space indentation, schemas at four, properties at eight, inline `{ ... }`
// or an indented block, and single-line flow sequences. It is not a YAML parser
// and must not become one; a document it cannot read is a document that stopped
// matching the house style.
/// One schema's body: everything from its key line to the next schema key.
fn yamlSchema(schema_name: []const u8) ?[]const u8 {
var key_buf: [64]u8 = undefined;
const key = std.fmt.bufPrint(&key_buf, "\n {s}:\n", .{schema_name}) catch return null;
const at = std.mem.indexOf(u8, openapi.yaml, key) orelse return null;
const body = openapi.yaml[at + key.len ..];
var end: usize = 0;
var lines = std.mem.splitScalar(u8, body, '\n');
while (lines.next()) |line| {
if (line.len != 0 and !std.mem.startsWith(u8, line, " ")) break;
end += line.len + 1;
}
return body[0..@min(end, body.len)];
}
/// One property's definition: the rest of its line for the inline form, or the
/// indented block that follows it.
fn yamlProperty(schema: []const u8, property_name: []const u8) ?[]const u8 {
const properties_at = std.mem.indexOf(u8, schema, "\n properties:\n") orelse return null;
const properties = schema[properties_at..];
var key_buf: [64]u8 = undefined;
const key = std.fmt.bufPrint(&key_buf, "\n {s}:", .{property_name}) catch return null;
const at = std.mem.indexOf(u8, properties, key) orelse return null;
const rest = properties[at + key.len ..];
const line_end = std.mem.indexOfScalar(u8, rest, '\n') orelse rest.len;
if (std.mem.trim(u8, rest[0..line_end], " ").len != 0) return rest[0..line_end];
var end: usize = line_end + 1;
var lines = std.mem.splitScalar(u8, rest[line_end + 1 ..], '\n');
while (lines.next()) |line| {
if (line.len != 0 and !std.mem.startsWith(u8, line, " ")) break;
end += line.len + 1;
}
return rest[0..@min(end, rest.len)];
}
/// The comma-separated items of a single-line flow sequence, `key: [a, b, c]`.
fn yamlFlowSeq(schema: []const u8, key: []const u8, out: *std.ArrayList([]const u8), gpa: Allocator) !void {
var key_buf: [32]u8 = undefined;
const needle = try std.fmt.bufPrint(&key_buf, "\n {s}: [", .{key});
const at = std.mem.indexOf(u8, schema, needle) orelse return error.TestUnexpectedResult;
const rest = schema[at + needle.len ..];
const close = std.mem.indexOfScalar(u8, rest, ']') orelse return error.TestUnexpectedResult;
var items = std.mem.splitScalar(u8, rest[0..close], ',');
while (items.next()) |item| try out.append(gpa, std.mem.trim(u8, item, " "));
}
/// The property names the schema declares, in document order.
fn yamlPropertyNames(schema: []const u8, out: *std.ArrayList([]const u8), gpa: Allocator) !void {
const properties_at = std.mem.indexOf(u8, schema, "\n properties:\n") orelse
return error.TestUnexpectedResult;
var lines = std.mem.splitScalar(u8, schema[properties_at + 1 ..], '\n');
_ = lines.next();
while (lines.next()) |line| {
if (line.len != 0 and !std.mem.startsWith(u8, line, " ")) break;
if (!std.mem.startsWith(u8, line, " ") or std.mem.startsWith(u8, line, " ")) continue;
const colon = std.mem.indexOfScalar(u8, line, ':') orelse continue;
try out.append(gpa, line[8..colon]);
}
}
/// The OpenAPI `type` a Zig field must be documented as, or `null` when the
/// field is a nested object and must be a `$ref` instead.
fn documentedType(comptime T: type) ?[]const u8 {
const Payload = switch (@typeInfo(T)) {
.optional => |o| o.child,
else => T,
};
return switch (@typeInfo(Payload)) {
.int => "integer",
.bool => "boolean",
// A closed enum is a string on the wire, documented as its own schema.
.@"enum" => null,
.pointer => "string",
.@"struct" => null,
else => @compileError("no documented type for " ++ @typeName(Payload)),
};
}
fn isOptional(comptime T: type) bool {
return @typeInfo(T) == .optional;
}
/// The schema name a `$ref` property points at.
fn refTarget(property: []const u8) ?[]const u8 {
const marker = "$ref: \"#/components/schemas/";
const at = std.mem.indexOf(u8, property, marker) orelse return null;
const rest = property[at + marker.len ..];
const close = std.mem.indexOfScalar(u8, rest, '"') orelse return null;
return rest[0..close];
}
/// Holds `schema_name` to `T`: same properties, same types, same nullability,
/// same requiredness, nothing extra on either side. Recurses through `$ref`.
fn expectSchemaMatches(gpa: Allocator, comptime T: type, schema_name: []const u8) !void {
const schema = yamlSchema(schema_name) orelse {
std.debug.print("openapi.yaml has no schema {s}\n", .{schema_name});
return error.TestUnexpectedResult;
};
var required: std.ArrayList([]const u8) = .empty;
defer required.deinit(gpa);
try yamlFlowSeq(schema, "required", &required, gpa);
const fields = @typeInfo(T).@"struct".fields;
inline for (fields) |field| {
const property = yamlProperty(schema, field.name) orelse {
std.debug.print("{s}: no property {s}\n", .{ schema_name, field.name });
return error.TestUnexpectedResult;
};
// Ahead of both branches: a `$ref` property is as free to go null as a
// scalar one, and a nested object or enum the server may omit is
// exactly the drift a client reading the document cannot see coming.
const documented_nullable = std.mem.containsAtLeast(u8, property, 1, "nullable: true");
if (documented_nullable != isOptional(field.type)) {
std.debug.print(
"{s}.{s}: nullable is {} in the document and {} in Zig\n",
.{ schema_name, field.name, documented_nullable, isOptional(field.type) },
);
return error.TestUnexpectedResult;
}
if (comptime documentedType(field.type)) |wanted| {
var type_buf: [32]u8 = undefined;
const needle = try std.fmt.bufPrint(&type_buf, "type: {s}", .{wanted});
if (!std.mem.containsAtLeast(u8, property, 1, needle)) {
std.debug.print("{s}.{s}: not documented as {s}\n", .{ schema_name, field.name, wanted });
return error.TestUnexpectedResult;
}
} else {
const target = refTarget(property) orelse {
std.debug.print("{s}.{s}: not a $ref\n", .{ schema_name, field.name });
return error.TestUnexpectedResult;
};
const Payload = switch (@typeInfo(field.type)) {
.optional => |o| o.child,
else => field.type,
};
switch (@typeInfo(Payload)) {
.@"enum" => try expectEnumMatches(gpa, Payload, target),
else => try expectSchemaMatches(gpa, Payload, target),
}
}
var listed = false;
for (required.items) |listed_name| listed = listed or std.mem.eql(u8, listed_name, field.name);
if (!listed) {
std.debug.print("{s}.{s}: not in required\n", .{ schema_name, field.name });
return error.TestUnexpectedResult;
}
}
var documented: std.ArrayList([]const u8) = .empty;
defer documented.deinit(gpa);
try yamlPropertyNames(schema, &documented, gpa);
try testing.expectEqual(fields.len, documented.items.len);
try testing.expectEqual(fields.len, required.items.len);
}
/// Holds an enum schema to its Zig enum: the same values, in the same order.
fn expectEnumMatches(gpa: Allocator, comptime T: type, schema_name: []const u8) !void {
const schema = yamlSchema(schema_name) orelse {
std.debug.print("openapi.yaml has no schema {s}\n", .{schema_name});
return error.TestUnexpectedResult;
};
var values: std.ArrayList([]const u8) = .empty;
defer values.deinit(gpa);
try yamlFlowSeq(schema, "enum", &values, gpa);
const tags = @typeInfo(T).@"enum".fields;
try testing.expectEqual(tags.len, values.items.len);
inline for (tags, 0..) |tag, index| {
try testing.expectEqualStrings(tag.name, values.items[index]);
}
}
test "drift guard c: the query-log schemas match the structs that serialize them" {
const gpa = testing.allocator;
try expectSchemaMatches(gpa, queries_repo.QueryRow, "QueryRow");
try expectSchemaMatches(gpa, provenance_view.QueryDetail, "QueryDetail");
try expectSchemaMatches(gpa, provenance_view.Provenance, "Provenance");
try expectSchemaMatches(gpa, coverage_mod.Coverage, "Coverage");
}
test "drift guard c: the three closed enums are documented value for value" {
const gpa = testing.allocator;
try expectEnumMatches(gpa, provenance.PolicyAction, "PolicyAction");
try expectEnumMatches(gpa, provenance.PolicyReason, "PolicyReason");
try expectEnumMatches(gpa, provenance.RouteKind, "RouteKind");
}
test "drift guard c bites: a renamed, retyped or newly optional field fails it" {
const gpa = testing.allocator;
// A field the document does not name at all.
const Renamed = struct { complete: bool, available_from: i64 };
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, Renamed, "Coverage"));
// A field the document names, with the wrong type.
const Retyped = struct { complete: bool, available_since: []const u8 };
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, Retyped, "Coverage"));
// A field the document names and types correctly, but which Zig may now
// send as null while `nullable` is absent from the document.
const Nullable = struct { complete: bool, available_since: ?i64 };
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, Nullable, "Coverage"));
// The same drift behind a `$ref`, where the property carries no `type:` of
// its own: a nested object the server may now omit.
const NullableObject = struct {
request: provenance_view.Request,
group: ?provenance_view.Group,
policy: provenance_view.Policy,
rewrites: provenance_view.Rewrites,
route: provenance_view.Route,
response: provenance_view.Response,
};
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, NullableObject, "Provenance"));
// And behind a `$ref` to an enum, whose values would still line up.
const NullableEnum = struct {
action: ?provenance.PolicyAction,
reason: provenance.PolicyReason,
matched: []const u8,
source_id: ?i64,
source_name: []const u8,
};
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, NullableEnum, "ProvenancePolicy"));
// A struct short one documented property, which excess-property checking on
// the client side would never catch.
const Narrowed = struct { complete: bool };
try testing.expectError(error.TestExpectedEqual, expectSchemaMatches(gpa, Narrowed, "Coverage"));
// An enum missing one of the document's values.
const Short = enum { not_evaluated, allow };
try testing.expectError(error.TestExpectedEqual, expectEnumMatches(gpa, Short, "PolicyAction"));
}
// ---------------------------------------------------------------------------
// contract samples: the frontend's consumed shapes against real responses
// (milestone-17 ruling 5)
@@ -2443,6 +3139,10 @@ const contract_sample_walk = [_]ContractSample{
// Query log and stats. `limit=5` reaches seeded row 21, the blocked one, so
// the page carries both the null-bearing and the populated row shape.
.{ .name = "get_queries", .ts_type = "QueriesPage", .method = "GET", .target = "/api/queries?limit=5", .status = 200 },
// The newest seeded row: a CNAME-uncloaked block with a group, a source and
// a matched pattern, so the golden exercises every nested object rather
// than a row of nulls.
.{ .name = "get_query_detail", .ts_type = "QueryDetail", .method = "GET", .target = "/api/queries/27", .status = 200 },
.{ .name = "get_stats", .ts_type = "StatsTotals", .method = "GET", .target = "/api/stats?period=1h", .status = 200 },
.{ .name = "get_stats_timeseries", .ts_type = "StatsTimeseries", .method = "GET", .target = "/api/stats/timeseries?period=1h", .status = 200 },