milestone 28: query provenance — every logged query is exactly explainable
Gates / frontend (push) Successful in 1m36s
Gates / test (push) Successful in 1m56s
Gates / test-aarch64 (push) Successful in 7m37s
Gates / package (push) Successful in 9m12s
Gates / container (push) Successful in 13s
CI / gates (push) Successful in 19m4s
Gates / frontend (push) Successful in 1m36s
Gates / test (push) Successful in 1m56s
Gates / test-aarch64 (push) Successful in 7m37s
Gates / package (push) Successful in 9m12s
Gates / container (push) Successful in 13s
CI / gates (push) Successful in 19m4s
query rows gain qclass, rcode, group, policy action and reason, the matched rule or list entry with its source, cname and safe-search targets, route kind, forward zone, and the resolver that actually answered — the pool and local markers die. servfails are logged and name the resolver that lost; post-parse protocol refusals become rows. a detail page at /queries/:id renders the ordered explanation, and coverage watermarks distinguish an empty history from a missing one. the schema fingerprint changes: existing query history is recreated with the old file kept aside and the reset filed as a resolved diagnostic. fixes an oversized udp reply being rebuilt as noerror, which handed clients a truncated nxdomain as success.
This commit is contained in:
@@ -38,15 +38,20 @@ const header = @import("../dns/header.zig");
|
||||
const http_util = @import("http_util.zig");
|
||||
const local_repo = @import("../storage/repositories/local_repo.zig");
|
||||
const local_tables_mod = @import("../server/local_tables.zig");
|
||||
const logger_mod = @import("../storage/logger.zig");
|
||||
const manager_mod = @import("../filter/manager.zig");
|
||||
const migrations = @import("../storage/migrations.zig");
|
||||
const name = @import("../dns/name.zig");
|
||||
const openapi = @import("openapi.zig");
|
||||
const packet = @import("../dns/packet.zig");
|
||||
const pause_mod = @import("../server/pause.zig");
|
||||
const coverage_mod = @import("coverage.zig");
|
||||
const pool_mod = @import("../upstream/pool.zig");
|
||||
const provenance = @import("../storage/provenance.zig");
|
||||
const provenance_view = @import("provenance_view.zig");
|
||||
const queries_repo = @import("../storage/repositories/queries_repo.zig");
|
||||
const querylog_schema = @import("../storage/querylog_schema.zig");
|
||||
const query_sink = @import("../server/query_sink.zig");
|
||||
const question = @import("../dns/question.zig");
|
||||
const router = @import("router.zig");
|
||||
const server = @import("server.zig");
|
||||
@@ -257,7 +262,10 @@ fn contentLength(head: []const u8) ?usize {
|
||||
// the environment: the real web stack over in-memory databases
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const seeded_query_rows = 25;
|
||||
/// The rows the uniform loop writes, before the two provenance-rich ones the
|
||||
/// detail endpoint and the credential sweep read.
|
||||
const seeded_plain_query_rows = 25;
|
||||
const seeded_query_rows = seeded_plain_query_rows + 2;
|
||||
|
||||
const EnvOptions = struct {
|
||||
password_hash: []const u8 = "",
|
||||
@@ -270,6 +278,10 @@ const EnvOptions = struct {
|
||||
/// wants; the file-authority tests below name a path.
|
||||
authority: server.Authority = .database,
|
||||
reconciled_at: ?i64 = null,
|
||||
/// False detaches the query log from the web state, which is the box a
|
||||
/// `logging.query_log = false` operator runs. Every query-log route then
|
||||
/// answers 503 rather than an empty page, which would be a lie.
|
||||
querylog: bool = true,
|
||||
};
|
||||
|
||||
/// Heap-allocated because `state` and the listener hold pointers into it.
|
||||
@@ -400,7 +412,7 @@ const Env = struct {
|
||||
.limiter = &self.limiter,
|
||||
.hub = self.hub,
|
||||
.config_db = &self.config_db,
|
||||
.querylog_db = &self.querylog_db,
|
||||
.querylog_db = if (options.querylog) &self.querylog_db else null,
|
||||
.events = &self.events_store,
|
||||
.version = "w10-test",
|
||||
.started_unix = std.Io.Clock.real.now(ioh).toSeconds(),
|
||||
@@ -472,27 +484,108 @@ fn seedConfig(database: *db.Db) !void {
|
||||
);
|
||||
}
|
||||
|
||||
/// The oldest instant the seeded log is complete for. Pinned rather than taken
|
||||
/// from `unixepoch()`, which the schema's own seed uses: the contract samples
|
||||
/// are byte-compared, so a clock in `coverage.available_since` would make the
|
||||
/// golden a property of the machine that generated it.
|
||||
///
|
||||
/// It equals the oldest seeded row's timestamp, so a request bounded at exactly
|
||||
/// this instant is complete and one bounded a second earlier is not.
|
||||
const seeded_available_since: i64 = 1_700_000_000;
|
||||
|
||||
fn seedQueryLog(database: *db.Db) !void {
|
||||
try database.exec(
|
||||
\\UPDATE querylog_meta SET created_at = 1700000000, available_since = 1700000000 WHERE id = 1
|
||||
);
|
||||
|
||||
var writer = try queries_repo.BatchWriter.init(database);
|
||||
defer writer.deinit();
|
||||
|
||||
var domain_buf: [32]u8 = undefined;
|
||||
var index: usize = 0;
|
||||
while (index < seeded_query_rows) : (index += 1) {
|
||||
while (index < seeded_plain_query_rows) : (index += 1) {
|
||||
const domain = std.fmt.bufPrint(&domain_buf, "d{d}.example", .{index}) catch unreachable;
|
||||
const blocked = index % 5 == 0;
|
||||
// The three states the handler can actually produce (`Context.cacheHit`
|
||||
// and `route_kind` are set together): a blocked answer consulted no
|
||||
// cache and named no resolver, a cache hit named no resolver, and only
|
||||
// an upstream exchange did both.
|
||||
const from_cache = !blocked and index % 2 == 0;
|
||||
try writer.writeBatch(&.{.{
|
||||
.timestamp = 1_700_000_000 + @as(i64, @intCast(index)),
|
||||
.domain = domain,
|
||||
.client_ip = "192.0.2.10",
|
||||
.qtype = 1,
|
||||
.qclass = 1,
|
||||
.rcode = 0,
|
||||
.blocked = blocked,
|
||||
.block_reason = if (blocked) "blocklist_domain" else null,
|
||||
.response_time_us = 250,
|
||||
.cache_hit = if (blocked) null else (index % 2 == 0),
|
||||
.upstream = if (blocked) null else "https://dns.example/dns-query",
|
||||
.cache_hit = if (blocked) null else from_cache,
|
||||
.upstream = if (blocked or from_cache) null else "https://dns.example/dns-query",
|
||||
.group_id = 1,
|
||||
.group_name = "default",
|
||||
.policy_action = if (blocked) .block else .allow,
|
||||
.policy_reason = if (blocked) .blocklist_domain else .no_match,
|
||||
.matched = if (blocked) domain else null,
|
||||
.source_id = null,
|
||||
.source_name = null,
|
||||
.cname_target = null,
|
||||
.safe_search_target = null,
|
||||
.route_kind = if (blocked) .blocked else if (from_cache) .cache else .upstream,
|
||||
.forward_zone = null,
|
||||
}});
|
||||
}
|
||||
|
||||
// Two rows with provenance the loop above never produces, so the detail
|
||||
// endpoint and its contract sample have a real row to read. They are the
|
||||
// newest rows, so a first page shows them.
|
||||
try writer.writeBatch(&.{.{
|
||||
.timestamp = 1_700_000_000 + seeded_plain_query_rows,
|
||||
.domain = "news.example",
|
||||
.client_ip = "192.0.2.10",
|
||||
.qtype = 1,
|
||||
.qclass = 1,
|
||||
.rcode = 0,
|
||||
.blocked = false,
|
||||
.response_time_us = 18_400,
|
||||
.cache_hit = false,
|
||||
.upstream = "https://dns.example/dns-query",
|
||||
.group_id = 1,
|
||||
.group_name = "default",
|
||||
.policy_action = .allow,
|
||||
.policy_reason = .no_match,
|
||||
.matched = null,
|
||||
.source_id = null,
|
||||
.source_name = null,
|
||||
.cname_target = null,
|
||||
.safe_search_target = null,
|
||||
.route_kind = .upstream,
|
||||
.forward_zone = null,
|
||||
}});
|
||||
|
||||
try writer.writeBatch(&.{.{
|
||||
.timestamp = 1_700_000_000 + seeded_plain_query_rows + 1,
|
||||
.domain = "shop.example",
|
||||
.client_ip = "192.0.2.11",
|
||||
.qtype = 1,
|
||||
.qclass = 1,
|
||||
.rcode = 3,
|
||||
.blocked = true,
|
||||
.response_time_us = 900,
|
||||
.cache_hit = null,
|
||||
.upstream = null,
|
||||
.group_id = 2,
|
||||
.group_name = "kids",
|
||||
.policy_action = .block,
|
||||
.policy_reason = .blocklist_wildcard,
|
||||
.matched = "||tracker.example^",
|
||||
.source_id = 4,
|
||||
.source_name = "StevenBlack",
|
||||
.cname_target = "cdn.tracker.example",
|
||||
.safe_search_target = null,
|
||||
.route_kind = .blocked,
|
||||
.forward_zone = null,
|
||||
}});
|
||||
}
|
||||
|
||||
/// A fixed instant, like every other seeded timestamp here: the contract
|
||||
@@ -653,6 +746,7 @@ const contract = [_]Contract{
|
||||
|
||||
// Query log, stats, live stream, upstream health.
|
||||
.{ .method = .GET, .pattern = "/api/queries", .auth = .session, .policy = .read, .target = "/api/queries?limit=10", .status = 200, .check = jsonShape(handlers_queries.Page) },
|
||||
.{ .method = .GET, .pattern = "/api/queries/{id}", .auth = .session, .policy = .read, .target = "/api/queries/27", .status = 200, .check = jsonShape(provenance_view.QueryDetail) },
|
||||
.{ .method = .GET, .pattern = "/api/queries/live", .auth = .session, .policy = .read, .rate_limit = .exempt, .target = "/api/queries/live", .status = 200, .kind = .sse },
|
||||
.{ .method = .GET, .pattern = "/api/stats", .auth = .session, .policy = .read, .target = "/api/stats?period=1h", .status = 200, .check = jsonShape(handlers_stats.TotalsBody) },
|
||||
.{ .method = .GET, .pattern = "/api/stats/timeseries", .auth = .session, .policy = .read, .target = "/api/stats/timeseries?period=1h", .status = 200, .check = jsonShape(handlers_stats.TimeseriesBody) },
|
||||
@@ -1653,12 +1747,15 @@ fn sseStream(io: std.Io, env: *Env) anyerror!void {
|
||||
.domain = "live.example",
|
||||
.client_ip = "192.0.2.99",
|
||||
.qtype = 1,
|
||||
.qclass = 1,
|
||||
.blocked = true,
|
||||
.block_reason = "blocklist_domain",
|
||||
.policy_action = .block,
|
||||
.policy_reason = .blocklist_domain,
|
||||
.route_kind = .blocked,
|
||||
}));
|
||||
try conn.readChunkedUntil(&seen, env.gpa, "event: query");
|
||||
try conn.readChunkedUntil(&seen, env.gpa, "\"domain\":\"live.example\"");
|
||||
try conn.readChunkedUntil(&seen, env.gpa, "\"blocked\":true");
|
||||
try conn.readChunkedUntil(&seen, env.gpa, "\"reason\":\"blocklist_domain\"");
|
||||
|
||||
// The cap is per address and the environment allows one stream: a second
|
||||
// subscriber from the same address is refused while the first is open.
|
||||
@@ -1784,7 +1881,7 @@ fn paginationWalk(io: std.Io, env: *Env) anyerror!void {
|
||||
try testing.expect(pages < 10);
|
||||
}
|
||||
|
||||
// 25 seeded rows walk as 10, 10 and 5, with the cursor ending exactly
|
||||
// 27 seeded rows walk as 10, 10 and 7, with the cursor ending exactly
|
||||
// after the third page.
|
||||
try testing.expectEqual(@as(usize, seeded_query_rows), total);
|
||||
try testing.expectEqual(@as(usize, 3), pages);
|
||||
@@ -1800,6 +1897,333 @@ test "W10 keyset pagination walks the seeded log exactly once, newest first" {
|
||||
try bounded(env.io(), default_budget, paginationWalk, .{ env.io(), env });
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// query provenance: the detail endpoint, coverage, and the credential sweep
|
||||
// (milestone 28)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The id of the seeded CNAME-uncloaked block, which is the last row written.
|
||||
const seeded_detail_id = seeded_query_rows;
|
||||
|
||||
fn detailWalk(io: std.Io, env: *Env) anyerror!void {
|
||||
var arena_state: std.heap.ArenaAllocator = .init(env.gpa);
|
||||
defer arena_state.deinit();
|
||||
|
||||
var conn: Conn = undefined;
|
||||
try conn.connect(io, env.addr);
|
||||
defer conn.close(io);
|
||||
|
||||
var body_buf: [64 * 1024]u8 = undefined;
|
||||
var target_buf: [64]u8 = undefined;
|
||||
|
||||
const target = try std.fmt.bufPrint(&target_buf, "/api/queries/{d}", .{seeded_detail_id});
|
||||
try conn.request("GET", target, null, null);
|
||||
const response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), response.status);
|
||||
|
||||
const detail = try std.json.parseFromSliceLeaky(
|
||||
provenance_view.QueryDetail,
|
||||
arena_state.allocator(),
|
||||
response.body,
|
||||
.{ .ignore_unknown_fields = false },
|
||||
);
|
||||
|
||||
try testing.expectEqual(@as(i64, seeded_detail_id), detail.id);
|
||||
try testing.expectEqualStrings("shop.example", detail.request.domain);
|
||||
try testing.expectEqualStrings("192.0.2.11", detail.request.client);
|
||||
try testing.expectEqual(@as(u16, 1), detail.request.qclass);
|
||||
try testing.expectEqual(@as(?i64, 2), detail.group.id);
|
||||
try testing.expectEqualStrings("kids", detail.group.name);
|
||||
try testing.expectEqual(provenance.PolicyAction.block, detail.policy.action);
|
||||
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, detail.policy.reason);
|
||||
try testing.expectEqualStrings("||tracker.example^", detail.policy.matched);
|
||||
try testing.expectEqual(@as(?i64, 4), detail.policy.source_id);
|
||||
try testing.expectEqualStrings("StevenBlack", detail.policy.source_name);
|
||||
try testing.expectEqualStrings("cdn.tracker.example", detail.rewrites.cname_target);
|
||||
try testing.expectEqualStrings("", detail.rewrites.safe_search_target);
|
||||
try testing.expectEqual(provenance.RouteKind.blocked, detail.route.kind);
|
||||
// A blocked query attempted no exchange, so it names no resolver.
|
||||
try testing.expectEqualStrings("", detail.route.upstream);
|
||||
try testing.expectEqual(@as(u16, 3), detail.response.rcode);
|
||||
try testing.expectEqual(@as(?i64, 900), detail.response.duration_us);
|
||||
|
||||
// An id past the end of the log and an id retention would have pruned are
|
||||
// the same answer.
|
||||
const missing = try std.fmt.bufPrint(&target_buf, "/api/queries/{d}", .{seeded_query_rows + 1000});
|
||||
try conn.request("GET", missing, null, null);
|
||||
const not_found = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 404), not_found.status);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, not_found.body, 1, "\"error\""));
|
||||
|
||||
// A non-positive id never reaches SQL: the pattern captures a positive
|
||||
// integer or does not match, so this is a routing 404.
|
||||
try conn.request("GET", "/api/queries/0", null, null);
|
||||
try testing.expectEqual(@as(u16, 404), (try conn.receive(&body_buf)).status);
|
||||
}
|
||||
|
||||
test "W10 milestone 28: the detail endpoint answers one row and 404s the rest" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
var env = try Env.create(gpa, .{});
|
||||
defer env.destroy();
|
||||
|
||||
try bounded(env.io(), default_budget, detailWalk, .{ env.io(), env });
|
||||
}
|
||||
|
||||
fn detailUnavailable(io: std.Io, env: *Env) anyerror!void {
|
||||
var conn: Conn = undefined;
|
||||
try conn.connect(io, env.addr);
|
||||
defer conn.close(io);
|
||||
|
||||
var body_buf: [8 * 1024]u8 = undefined;
|
||||
for ([_][]const u8{ "/api/queries/1", "/api/queries?limit=1", "/api/stats", "/api/stats/timeseries" }) |target| {
|
||||
try conn.request("GET", target, null, null);
|
||||
const response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 503), response.status);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "query log unavailable"));
|
||||
}
|
||||
}
|
||||
|
||||
test "W10 milestone 28: a box with no query log answers 503, not an empty page" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
var env = try Env.create(gpa, .{ .querylog = false });
|
||||
defer env.destroy();
|
||||
|
||||
try bounded(env.io(), default_budget, detailUnavailable, .{ env.io(), env });
|
||||
}
|
||||
|
||||
fn coverageWalk(io: std.Io, env: *Env) anyerror!void {
|
||||
var arena_state: std.heap.ArenaAllocator = .init(env.gpa);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
var conn: Conn = undefined;
|
||||
try conn.connect(io, env.addr);
|
||||
defer conn.close(io);
|
||||
|
||||
var body_buf: [64 * 1024]u8 = undefined;
|
||||
var target_buf: [64]u8 = undefined;
|
||||
|
||||
// No lower bound: the request asks about all of history, which a file that
|
||||
// may have pruned cannot promise.
|
||||
try conn.request("GET", "/api/queries?limit=1", null, null);
|
||||
const unbounded = try std.json.parseFromSliceLeaky(
|
||||
handlers_queries.Page,
|
||||
arena,
|
||||
(try conn.receive(&body_buf)).body,
|
||||
.{ .ignore_unknown_fields = false },
|
||||
);
|
||||
try testing.expectEqual(seeded_available_since, unbounded.coverage.available_since);
|
||||
try testing.expect(!unbounded.coverage.complete);
|
||||
|
||||
// Bounded exactly at the watermark.
|
||||
const at = try std.fmt.bufPrint(&target_buf, "/api/queries?limit=1&since={d}", .{seeded_available_since});
|
||||
try conn.request("GET", at, null, null);
|
||||
const covered = try std.json.parseFromSliceLeaky(
|
||||
handlers_queries.Page,
|
||||
arena,
|
||||
(try conn.receive(&body_buf)).body,
|
||||
.{ .ignore_unknown_fields = false },
|
||||
);
|
||||
try testing.expect(covered.coverage.complete);
|
||||
|
||||
// One second earlier, and the window reaches past what the file holds.
|
||||
const before = try std.fmt.bufPrint(&target_buf, "/api/queries?limit=1&since={d}", .{seeded_available_since - 1});
|
||||
try conn.request("GET", before, null, null);
|
||||
const partial = try std.json.parseFromSliceLeaky(
|
||||
handlers_queries.Page,
|
||||
arena,
|
||||
(try conn.receive(&body_buf)).body,
|
||||
.{ .ignore_unknown_fields = false },
|
||||
);
|
||||
try testing.expect(!partial.coverage.complete);
|
||||
|
||||
// The stats endpoints judge the same watermark against their own aligned
|
||||
// window, which for any live period starts well after the seeded rows.
|
||||
try conn.request("GET", "/api/stats?period=1h", null, null);
|
||||
const totals = try std.json.parseFromSliceLeaky(
|
||||
handlers_stats.TotalsBody,
|
||||
arena,
|
||||
(try conn.receive(&body_buf)).body,
|
||||
.{ .ignore_unknown_fields = false },
|
||||
);
|
||||
try testing.expectEqual(seeded_available_since, totals.coverage.available_since);
|
||||
try testing.expectEqual(totals.since >= seeded_available_since, totals.coverage.complete);
|
||||
|
||||
try conn.request("GET", "/api/stats/timeseries?period=1h", null, null);
|
||||
const series = try std.json.parseFromSliceLeaky(
|
||||
handlers_stats.TimeseriesBody,
|
||||
arena,
|
||||
(try conn.receive(&body_buf)).body,
|
||||
.{ .ignore_unknown_fields = false },
|
||||
);
|
||||
try testing.expectEqual(totals.since, series.since);
|
||||
try testing.expectEqual(totals.coverage.complete, series.coverage.complete);
|
||||
}
|
||||
|
||||
test "W10 milestone 28: every window-bounded endpoint reports its own coverage" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
var env = try Env.create(gpa, .{});
|
||||
defer env.destroy();
|
||||
|
||||
try bounded(env.io(), default_budget, coverageWalk, .{ env.io(), env });
|
||||
}
|
||||
|
||||
/// NextDNS's shape: the account id rides in the path, which is exactly where a
|
||||
/// credential lives in a url an operator may legitimately configure.
|
||||
/// `Endpoint.parse` refuses userinfo, so the path is the shape a real
|
||||
/// configuration can carry a secret in — and the path is what
|
||||
/// `safe_url.redact` drops.
|
||||
const sweep_token = "b1c2d3";
|
||||
const sweep_upstream_url = "https://dns.nextdns.io/" ++ sweep_token;
|
||||
/// What every surface must show instead. The origin survives redaction — an
|
||||
/// operator reading a failure has to know where the query went — so each
|
||||
/// surface is checked for it too: one that showed nothing at all would pass a
|
||||
/// secret check by saying nothing.
|
||||
const sweep_redacted_upstream = "https://dns.nextdns.io";
|
||||
/// The name the swept query asks for, so each surface can be pinned to the row
|
||||
/// this test produced rather than to a seeded one.
|
||||
const sweep_domain = "creds.example";
|
||||
|
||||
/// Names the resolver and never its token.
|
||||
fn expectRedacted(text: []const u8) !void {
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, sweep_redacted_upstream));
|
||||
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, sweep_token));
|
||||
}
|
||||
|
||||
/// The cross-surface credential sweep, driven end to end: a real `Handler`
|
||||
/// answers a real query through a resolver whose url carries a token, and the
|
||||
/// entry travels the production path — `QuerySink`, then the hub and the
|
||||
/// logger, then the query log the API reads. Nothing here redacts anything, so
|
||||
/// a handler that stopped redacting fails this test.
|
||||
///
|
||||
/// Four surfaces read the same query back: the stored row, straight out of
|
||||
/// SQLite, and the three the operator's browser sees — the live frame, the list
|
||||
/// page and the detail body. A leak on any one of them is a secret in a browser
|
||||
/// history, and the four are separate code paths to the same text.
|
||||
fn credentialSweep(
|
||||
io: std.Io,
|
||||
env: *Env,
|
||||
query_logger: *logger_mod.Logger,
|
||||
handler: *dns_handler.Handler,
|
||||
) anyerror!void {
|
||||
var arena_state: std.heap.ArenaAllocator = .init(env.gpa);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
// Subscribed before the query runs: the hub publishes to whoever is
|
||||
// listening at that moment and keeps nothing for a later reader.
|
||||
var seen: std.ArrayList(u8) = .empty;
|
||||
defer seen.deinit(env.gpa);
|
||||
var stream: Conn = undefined;
|
||||
try openLiveStream(io, env, &stream, &seen);
|
||||
defer stream.close(io);
|
||||
|
||||
var query_buf: [512]u8 = undefined;
|
||||
var response_buf: [512]u8 = undefined;
|
||||
var scratch: dns_handler.Scratch = undefined;
|
||||
const from = address.NetAddress.fromIp(.{ .ip4 = .loopback(53100) });
|
||||
const query = queryFor(&query_buf, 0x4444, sweep_domain, .a);
|
||||
try testing.expect(handler.handle(io, .udp, from, query, &response_buf, &scratch) == .reply);
|
||||
|
||||
// The live frame. Waiting on the redacted origin rather than on the whole
|
||||
// frame is safe in both directions: a leaked url starts with it.
|
||||
try stream.readChunkedUntil(&seen, env.gpa, sweep_redacted_upstream);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, seen.items, 1, sweep_domain));
|
||||
try expectRedacted(seen.items);
|
||||
|
||||
// The stored row. The producer has already run, so closing the queue and
|
||||
// running the writer inline drains it in one call: `runWriter` returns when
|
||||
// a closed queue is empty, and a zero flush interval makes it commit the
|
||||
// batch it holds rather than wait for company.
|
||||
query_logger.shutdown(io);
|
||||
try query_logger.runWriter(io, &env.querylog_db, null);
|
||||
try testing.expectEqual(@as(u64, 1), query_logger.rows_written.load(.monotonic));
|
||||
|
||||
var stmt = try env.querylog_db.prepare(
|
||||
\\SELECT query_log.id, query_log.upstream
|
||||
\\FROM query_log JOIN domains ON domains.id = query_log.domain_id
|
||||
\\WHERE domains.domain = ?
|
||||
);
|
||||
defer stmt.deinit();
|
||||
try stmt.bindText(1, sweep_domain);
|
||||
try testing.expect(try stmt.step());
|
||||
const row_id = stmt.columnInt(0);
|
||||
try testing.expectEqualStrings(sweep_redacted_upstream, stmt.columnText(1));
|
||||
try testing.expect(!try stmt.step());
|
||||
|
||||
var conn: Conn = undefined;
|
||||
try conn.connect(io, env.addr);
|
||||
defer conn.close(io);
|
||||
|
||||
var body_buf: [64 * 1024]u8 = undefined;
|
||||
var target_buf: [64]u8 = undefined;
|
||||
|
||||
// The list row. The swept query is the newest in the log, so a page of one
|
||||
// is it.
|
||||
try conn.request("GET", "/api/queries?limit=1", null, null);
|
||||
const rows = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), rows.status);
|
||||
const page = try std.json.parseFromSliceLeaky(
|
||||
handlers_queries.Page,
|
||||
arena,
|
||||
rows.body,
|
||||
.{ .ignore_unknown_fields = false },
|
||||
);
|
||||
try testing.expectEqual(@as(usize, 1), page.queries.len);
|
||||
try testing.expectEqualStrings(sweep_domain, page.queries[0].domain);
|
||||
try testing.expectEqualStrings(sweep_redacted_upstream, page.queries[0].upstream);
|
||||
try expectRedacted(rows.body);
|
||||
|
||||
// The detail body, read by the row id the database just handed over.
|
||||
const one = try std.fmt.bufPrint(&target_buf, "/api/queries/{d}", .{row_id});
|
||||
try conn.request("GET", one, null, null);
|
||||
const detail_response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), detail_response.status);
|
||||
const detail = try std.json.parseFromSliceLeaky(
|
||||
provenance_view.QueryDetail,
|
||||
arena,
|
||||
detail_response.body,
|
||||
.{ .ignore_unknown_fields = false },
|
||||
);
|
||||
try testing.expectEqualStrings(sweep_domain, detail.request.domain);
|
||||
try testing.expectEqualStrings(sweep_redacted_upstream, detail.route.upstream);
|
||||
try expectRedacted(detail_response.body);
|
||||
}
|
||||
|
||||
test "W10 milestone 28: no query surface echoes a resolver credential" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
var env = try Env.create(gpa, .{});
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var queue_buf: [4]logger_mod.Entry = undefined;
|
||||
// Zero flush interval: the drain below is synchronous, and nothing else
|
||||
// will ever put an entry on this queue for the writer to wait for.
|
||||
var query_logger: logger_mod.Logger = .init(.{ .query_log_flush_interval_s = 0 }, &queue_buf);
|
||||
var sink: query_sink.QuerySink = .init(&query_logger, env.hub);
|
||||
|
||||
var fake: FakeUpstream = .{ .identity = sweep_upstream_url };
|
||||
var handler: dns_handler.Handler = .{
|
||||
.upstream = fake.client(),
|
||||
.blocking = .{ .mode = .zero, .ttl = 5 },
|
||||
.forward_read_timeout = .{ .raw = .fromSeconds(2), .clock = .awake },
|
||||
.manager = &env.mgr,
|
||||
.pause = &env.pauser,
|
||||
.sink = &sink,
|
||||
};
|
||||
|
||||
try bounded(io, default_budget, credentialSweep, .{ io, env, &query_logger, &handler });
|
||||
try testing.expectEqual(@as(u64, 1), fake.calls.load(.monotonic));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// mutation → reload observed (ruling 12)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -1980,15 +2404,21 @@ fn queryFor(buf: []u8, id: u16, domain: []const u8, qtype: types.Type) []const u
|
||||
/// can tell whether the filter let the query through.
|
||||
const FakeUpstream = struct {
|
||||
calls: std.atomic.Value(u64) = .init(0),
|
||||
/// The resolver the handler reports as having answered. Operator-supplied
|
||||
/// text in production, so the credential sweep points it at a url with a
|
||||
/// token in its path.
|
||||
identity: []const u8 = "fake://web-upstream",
|
||||
|
||||
fn exchangeFn(
|
||||
ptr: *anyopaque,
|
||||
io: std.Io,
|
||||
query: []const u8,
|
||||
response_buf: []u8,
|
||||
selected: *?[]const u8,
|
||||
) transport.ExchangeError![]u8 {
|
||||
_ = io;
|
||||
const self: *FakeUpstream = @ptrCast(@alignCast(ptr));
|
||||
selected.* = self.identity;
|
||||
_ = self.calls.fetchAdd(1, .monotonic);
|
||||
|
||||
const request = packet.parse(query) catch return error.BadResponse;
|
||||
@@ -2348,6 +2778,272 @@ test "drift guard a bites: methods swapped between two documented paths fail the
|
||||
try testing.expectEqual(@as(usize, 2), swapped_routes);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// focused schema drift guards (milestone 28)
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// Guard a proves every served route is documented and guard b counts the
|
||||
// operations, and neither looks inside a schema. A field renamed, retyped, made
|
||||
// nullable or dropped from `required` passes both while breaking every client
|
||||
// that reads the document — and the query-log provenance shapes are exactly
|
||||
// where a rename is easy and a wrong `nullable` is silent.
|
||||
//
|
||||
// So these read the schema back and hold it to the Zig struct that produces it:
|
||||
// the same property names, the same types, the same nullability, the same
|
||||
// requiredness, and no extra property on either side. A `$ref` recurses, so
|
||||
// checking `QueryDetail` checks all six of its nested objects.
|
||||
//
|
||||
// The YAML reader below understands only the shape this document is written in
|
||||
// — two-space indentation, schemas at four, properties at eight, inline `{ ... }`
|
||||
// or an indented block, and single-line flow sequences. It is not a YAML parser
|
||||
// and must not become one; a document it cannot read is a document that stopped
|
||||
// matching the house style.
|
||||
|
||||
/// One schema's body: everything from its key line to the next schema key.
|
||||
fn yamlSchema(schema_name: []const u8) ?[]const u8 {
|
||||
var key_buf: [64]u8 = undefined;
|
||||
const key = std.fmt.bufPrint(&key_buf, "\n {s}:\n", .{schema_name}) catch return null;
|
||||
const at = std.mem.indexOf(u8, openapi.yaml, key) orelse return null;
|
||||
const body = openapi.yaml[at + key.len ..];
|
||||
|
||||
var end: usize = 0;
|
||||
var lines = std.mem.splitScalar(u8, body, '\n');
|
||||
while (lines.next()) |line| {
|
||||
if (line.len != 0 and !std.mem.startsWith(u8, line, " ")) break;
|
||||
end += line.len + 1;
|
||||
}
|
||||
return body[0..@min(end, body.len)];
|
||||
}
|
||||
|
||||
/// One property's definition: the rest of its line for the inline form, or the
|
||||
/// indented block that follows it.
|
||||
fn yamlProperty(schema: []const u8, property_name: []const u8) ?[]const u8 {
|
||||
const properties_at = std.mem.indexOf(u8, schema, "\n properties:\n") orelse return null;
|
||||
const properties = schema[properties_at..];
|
||||
|
||||
var key_buf: [64]u8 = undefined;
|
||||
const key = std.fmt.bufPrint(&key_buf, "\n {s}:", .{property_name}) catch return null;
|
||||
const at = std.mem.indexOf(u8, properties, key) orelse return null;
|
||||
const rest = properties[at + key.len ..];
|
||||
|
||||
const line_end = std.mem.indexOfScalar(u8, rest, '\n') orelse rest.len;
|
||||
if (std.mem.trim(u8, rest[0..line_end], " ").len != 0) return rest[0..line_end];
|
||||
|
||||
var end: usize = line_end + 1;
|
||||
var lines = std.mem.splitScalar(u8, rest[line_end + 1 ..], '\n');
|
||||
while (lines.next()) |line| {
|
||||
if (line.len != 0 and !std.mem.startsWith(u8, line, " ")) break;
|
||||
end += line.len + 1;
|
||||
}
|
||||
return rest[0..@min(end, rest.len)];
|
||||
}
|
||||
|
||||
/// The comma-separated items of a single-line flow sequence, `key: [a, b, c]`.
|
||||
fn yamlFlowSeq(schema: []const u8, key: []const u8, out: *std.ArrayList([]const u8), gpa: Allocator) !void {
|
||||
var key_buf: [32]u8 = undefined;
|
||||
const needle = try std.fmt.bufPrint(&key_buf, "\n {s}: [", .{key});
|
||||
const at = std.mem.indexOf(u8, schema, needle) orelse return error.TestUnexpectedResult;
|
||||
const rest = schema[at + needle.len ..];
|
||||
const close = std.mem.indexOfScalar(u8, rest, ']') orelse return error.TestUnexpectedResult;
|
||||
|
||||
var items = std.mem.splitScalar(u8, rest[0..close], ',');
|
||||
while (items.next()) |item| try out.append(gpa, std.mem.trim(u8, item, " "));
|
||||
}
|
||||
|
||||
/// The property names the schema declares, in document order.
|
||||
fn yamlPropertyNames(schema: []const u8, out: *std.ArrayList([]const u8), gpa: Allocator) !void {
|
||||
const properties_at = std.mem.indexOf(u8, schema, "\n properties:\n") orelse
|
||||
return error.TestUnexpectedResult;
|
||||
var lines = std.mem.splitScalar(u8, schema[properties_at + 1 ..], '\n');
|
||||
_ = lines.next();
|
||||
while (lines.next()) |line| {
|
||||
if (line.len != 0 and !std.mem.startsWith(u8, line, " ")) break;
|
||||
if (!std.mem.startsWith(u8, line, " ") or std.mem.startsWith(u8, line, " ")) continue;
|
||||
const colon = std.mem.indexOfScalar(u8, line, ':') orelse continue;
|
||||
try out.append(gpa, line[8..colon]);
|
||||
}
|
||||
}
|
||||
|
||||
/// The OpenAPI `type` a Zig field must be documented as, or `null` when the
|
||||
/// field is a nested object and must be a `$ref` instead.
|
||||
fn documentedType(comptime T: type) ?[]const u8 {
|
||||
const Payload = switch (@typeInfo(T)) {
|
||||
.optional => |o| o.child,
|
||||
else => T,
|
||||
};
|
||||
return switch (@typeInfo(Payload)) {
|
||||
.int => "integer",
|
||||
.bool => "boolean",
|
||||
// A closed enum is a string on the wire, documented as its own schema.
|
||||
.@"enum" => null,
|
||||
.pointer => "string",
|
||||
.@"struct" => null,
|
||||
else => @compileError("no documented type for " ++ @typeName(Payload)),
|
||||
};
|
||||
}
|
||||
|
||||
fn isOptional(comptime T: type) bool {
|
||||
return @typeInfo(T) == .optional;
|
||||
}
|
||||
|
||||
/// The schema name a `$ref` property points at.
|
||||
fn refTarget(property: []const u8) ?[]const u8 {
|
||||
const marker = "$ref: \"#/components/schemas/";
|
||||
const at = std.mem.indexOf(u8, property, marker) orelse return null;
|
||||
const rest = property[at + marker.len ..];
|
||||
const close = std.mem.indexOfScalar(u8, rest, '"') orelse return null;
|
||||
return rest[0..close];
|
||||
}
|
||||
|
||||
/// Holds `schema_name` to `T`: same properties, same types, same nullability,
|
||||
/// same requiredness, nothing extra on either side. Recurses through `$ref`.
|
||||
fn expectSchemaMatches(gpa: Allocator, comptime T: type, schema_name: []const u8) !void {
|
||||
const schema = yamlSchema(schema_name) orelse {
|
||||
std.debug.print("openapi.yaml has no schema {s}\n", .{schema_name});
|
||||
return error.TestUnexpectedResult;
|
||||
};
|
||||
|
||||
var required: std.ArrayList([]const u8) = .empty;
|
||||
defer required.deinit(gpa);
|
||||
try yamlFlowSeq(schema, "required", &required, gpa);
|
||||
|
||||
const fields = @typeInfo(T).@"struct".fields;
|
||||
inline for (fields) |field| {
|
||||
const property = yamlProperty(schema, field.name) orelse {
|
||||
std.debug.print("{s}: no property {s}\n", .{ schema_name, field.name });
|
||||
return error.TestUnexpectedResult;
|
||||
};
|
||||
|
||||
// Ahead of both branches: a `$ref` property is as free to go null as a
|
||||
// scalar one, and a nested object or enum the server may omit is
|
||||
// exactly the drift a client reading the document cannot see coming.
|
||||
const documented_nullable = std.mem.containsAtLeast(u8, property, 1, "nullable: true");
|
||||
if (documented_nullable != isOptional(field.type)) {
|
||||
std.debug.print(
|
||||
"{s}.{s}: nullable is {} in the document and {} in Zig\n",
|
||||
.{ schema_name, field.name, documented_nullable, isOptional(field.type) },
|
||||
);
|
||||
return error.TestUnexpectedResult;
|
||||
}
|
||||
|
||||
if (comptime documentedType(field.type)) |wanted| {
|
||||
var type_buf: [32]u8 = undefined;
|
||||
const needle = try std.fmt.bufPrint(&type_buf, "type: {s}", .{wanted});
|
||||
if (!std.mem.containsAtLeast(u8, property, 1, needle)) {
|
||||
std.debug.print("{s}.{s}: not documented as {s}\n", .{ schema_name, field.name, wanted });
|
||||
return error.TestUnexpectedResult;
|
||||
}
|
||||
} else {
|
||||
const target = refTarget(property) orelse {
|
||||
std.debug.print("{s}.{s}: not a $ref\n", .{ schema_name, field.name });
|
||||
return error.TestUnexpectedResult;
|
||||
};
|
||||
const Payload = switch (@typeInfo(field.type)) {
|
||||
.optional => |o| o.child,
|
||||
else => field.type,
|
||||
};
|
||||
switch (@typeInfo(Payload)) {
|
||||
.@"enum" => try expectEnumMatches(gpa, Payload, target),
|
||||
else => try expectSchemaMatches(gpa, Payload, target),
|
||||
}
|
||||
}
|
||||
|
||||
var listed = false;
|
||||
for (required.items) |listed_name| listed = listed or std.mem.eql(u8, listed_name, field.name);
|
||||
if (!listed) {
|
||||
std.debug.print("{s}.{s}: not in required\n", .{ schema_name, field.name });
|
||||
return error.TestUnexpectedResult;
|
||||
}
|
||||
}
|
||||
|
||||
var documented: std.ArrayList([]const u8) = .empty;
|
||||
defer documented.deinit(gpa);
|
||||
try yamlPropertyNames(schema, &documented, gpa);
|
||||
try testing.expectEqual(fields.len, documented.items.len);
|
||||
try testing.expectEqual(fields.len, required.items.len);
|
||||
}
|
||||
|
||||
/// Holds an enum schema to its Zig enum: the same values, in the same order.
|
||||
fn expectEnumMatches(gpa: Allocator, comptime T: type, schema_name: []const u8) !void {
|
||||
const schema = yamlSchema(schema_name) orelse {
|
||||
std.debug.print("openapi.yaml has no schema {s}\n", .{schema_name});
|
||||
return error.TestUnexpectedResult;
|
||||
};
|
||||
|
||||
var values: std.ArrayList([]const u8) = .empty;
|
||||
defer values.deinit(gpa);
|
||||
try yamlFlowSeq(schema, "enum", &values, gpa);
|
||||
|
||||
const tags = @typeInfo(T).@"enum".fields;
|
||||
try testing.expectEqual(tags.len, values.items.len);
|
||||
inline for (tags, 0..) |tag, index| {
|
||||
try testing.expectEqualStrings(tag.name, values.items[index]);
|
||||
}
|
||||
}
|
||||
|
||||
test "drift guard c: the query-log schemas match the structs that serialize them" {
|
||||
const gpa = testing.allocator;
|
||||
try expectSchemaMatches(gpa, queries_repo.QueryRow, "QueryRow");
|
||||
try expectSchemaMatches(gpa, provenance_view.QueryDetail, "QueryDetail");
|
||||
try expectSchemaMatches(gpa, provenance_view.Provenance, "Provenance");
|
||||
try expectSchemaMatches(gpa, coverage_mod.Coverage, "Coverage");
|
||||
}
|
||||
|
||||
test "drift guard c: the three closed enums are documented value for value" {
|
||||
const gpa = testing.allocator;
|
||||
try expectEnumMatches(gpa, provenance.PolicyAction, "PolicyAction");
|
||||
try expectEnumMatches(gpa, provenance.PolicyReason, "PolicyReason");
|
||||
try expectEnumMatches(gpa, provenance.RouteKind, "RouteKind");
|
||||
}
|
||||
|
||||
test "drift guard c bites: a renamed, retyped or newly optional field fails it" {
|
||||
const gpa = testing.allocator;
|
||||
|
||||
// A field the document does not name at all.
|
||||
const Renamed = struct { complete: bool, available_from: i64 };
|
||||
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, Renamed, "Coverage"));
|
||||
|
||||
// A field the document names, with the wrong type.
|
||||
const Retyped = struct { complete: bool, available_since: []const u8 };
|
||||
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, Retyped, "Coverage"));
|
||||
|
||||
// A field the document names and types correctly, but which Zig may now
|
||||
// send as null while `nullable` is absent from the document.
|
||||
const Nullable = struct { complete: bool, available_since: ?i64 };
|
||||
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, Nullable, "Coverage"));
|
||||
|
||||
// The same drift behind a `$ref`, where the property carries no `type:` of
|
||||
// its own: a nested object the server may now omit.
|
||||
const NullableObject = struct {
|
||||
request: provenance_view.Request,
|
||||
group: ?provenance_view.Group,
|
||||
policy: provenance_view.Policy,
|
||||
rewrites: provenance_view.Rewrites,
|
||||
route: provenance_view.Route,
|
||||
response: provenance_view.Response,
|
||||
};
|
||||
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, NullableObject, "Provenance"));
|
||||
|
||||
// And behind a `$ref` to an enum, whose values would still line up.
|
||||
const NullableEnum = struct {
|
||||
action: ?provenance.PolicyAction,
|
||||
reason: provenance.PolicyReason,
|
||||
matched: []const u8,
|
||||
source_id: ?i64,
|
||||
source_name: []const u8,
|
||||
};
|
||||
try testing.expectError(error.TestUnexpectedResult, expectSchemaMatches(gpa, NullableEnum, "ProvenancePolicy"));
|
||||
|
||||
// A struct short one documented property, which excess-property checking on
|
||||
// the client side would never catch.
|
||||
const Narrowed = struct { complete: bool };
|
||||
try testing.expectError(error.TestExpectedEqual, expectSchemaMatches(gpa, Narrowed, "Coverage"));
|
||||
|
||||
// An enum missing one of the document's values.
|
||||
const Short = enum { not_evaluated, allow };
|
||||
try testing.expectError(error.TestExpectedEqual, expectEnumMatches(gpa, Short, "PolicyAction"));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// contract samples: the frontend's consumed shapes against real responses
|
||||
// (milestone-17 ruling 5)
|
||||
@@ -2443,6 +3139,10 @@ const contract_sample_walk = [_]ContractSample{
|
||||
// Query log and stats. `limit=5` reaches seeded row 21, the blocked one, so
|
||||
// the page carries both the null-bearing and the populated row shape.
|
||||
.{ .name = "get_queries", .ts_type = "QueriesPage", .method = "GET", .target = "/api/queries?limit=5", .status = 200 },
|
||||
// The newest seeded row: a CNAME-uncloaked block with a group, a source and
|
||||
// a matched pattern, so the golden exercises every nested object rather
|
||||
// than a row of nulls.
|
||||
.{ .name = "get_query_detail", .ts_type = "QueryDetail", .method = "GET", .target = "/api/queries/27", .status = 200 },
|
||||
.{ .name = "get_stats", .ts_type = "StatsTotals", .method = "GET", .target = "/api/stats?period=1h", .status = 200 },
|
||||
.{ .name = "get_stats_timeseries", .ts_type = "StatsTimeseries", .method = "GET", .target = "/api/stats/timeseries?period=1h", .status = 200 },
|
||||
|
||||
|
||||
Reference in New Issue
Block a user