milestone 28: query provenance — every logged query is exactly explainable

query rows gain qclass, rcode, group, policy action and reason, the
matched rule or list entry with its source, cname and safe-search
targets, route kind, forward zone, and the resolver that actually
answered — the pool and local markers die. servfails are logged and
name the resolver that lost; post-parse protocol refusals become rows.
a detail page at /queries/:id renders the ordered explanation, and
coverage watermarks distinguish an empty history from a missing one.

the schema fingerprint changes: existing query history is recreated
with the old file kept aside and the reset filed as a resolved
diagnostic. fixes an oversized udp reply being rebuilt as noerror,
which handed clients a truncated nxdomain as success.

src/web/handlers/live.zig

@@ -23,7 +23,7 @@ const std = @import("std");
 
 const address = @import("../../platform/address.zig");
 const http_util = @import("../http_util.zig");
-const queries_repo = @import("../../storage/repositories/queries_repo.zig");
+const provenance_view = @import("../provenance_view.zig");
 const server = @import("../server.zig");
 const sse = @import("../sse.zig");
 
@@ -36,32 +36,13 @@ pub const heartbeat_interval: std.Io.Clock.Duration = .{
     .clock = .awake,
 };
 
-/// One event's `data:` payload — the `/api/queries` row fields (ruling 20),
-/// minus `id`: a live entry precedes persistence, so no row id exists yet.
-pub const EventView = struct {
-    ts: i64,
-    domain: []const u8,
-    client_ip: []const u8,
-    qtype: ?u16,
-    blocked: bool,
-    block_reason: []const u8,
-    response_time_us: ?i64,
-    cache_hit: ?bool,
-    upstream: []const u8,
-};
+/// One event's `data:` payload: the shared full-provenance DTO, exactly. A live
+/// event says everything `GET /api/queries/{id}` would say about the same query
+/// except its id, which does not exist yet — the entry precedes its own insert.
+pub const EventView = provenance_view.Provenance;
 
 pub fn view(entry: *const sse.Entry) EventView {
-    return .{
-        .ts = entry.timestamp,
-        .domain = entry.domain(),
-        .client_ip = entry.clientIp(),
-        .qtype = entry.qtype,
-        .blocked = entry.blocked,
-        .block_reason = entry.blockReason(),
-        .response_time_us = entry.response_time_us,
-        .cache_hit = entry.cache_hit,
-        .upstream = entry.upstream(),
-    };
+    return provenance_view.fromEntry(entry);
 }
 
 /// One `event: query` frame. JSON never contains a raw newline, so the whole
@@ -139,14 +120,18 @@ pub fn stream(
 
 const testing = std.testing;
 
-test "the event payload carries the /api/queries row fields, minus id" {
-    const row_fields = @typeInfo(queries_repo.QueryRow).@"struct".fields;
+test "the event payload is the detail body minus its id, name and type for name" {
+    const detail_fields = @typeInfo(provenance_view.QueryDetail).@"struct".fields;
     const view_fields = @typeInfo(EventView).@"struct".fields;
     comptime {
-        std.debug.assert(view_fields.len == row_fields.len - 1);
-        std.debug.assert(std.mem.eql(u8, row_fields[0].name, "id"));
-        for (row_fields[1..], view_fields) |row_field, view_field| {
-            std.debug.assert(std.mem.eql(u8, row_field.name, view_field.name));
+        std.debug.assert(view_fields.len == detail_fields.len - 1);
+        std.debug.assert(std.mem.eql(u8, detail_fields[0].name, "id"));
+        for (detail_fields[1..], view_fields) |detail_field, view_field| {
+            std.debug.assert(std.mem.eql(u8, detail_field.name, view_field.name));
+            // Names alone would let a group keep its key while changing what it
+            // holds, which is the drift a live viewer would see and a detail
+            // page would not.
+            std.debug.assert(detail_field.type == view_field.type);
         }
     }
 }
@@ -157,11 +142,19 @@ test "a frame is one event line and one data line of JSON" {
         .domain = "ads.example",
         .client_ip = "192.0.2.10",
         .qtype = 1,
+        .qclass = 1,
+        .rcode = 0,
         .blocked = true,
-        .block_reason = "blocklist_domain",
+        .group_id = 1,
+        .group_name = "default",
+        .policy_action = .block,
+        .policy_reason = .blocklist_domain,
+        .matched = "ads.example",
+        .source_id = 3,
+        .source_name = "StevenBlack",
+        .route_kind = .blocked,
         .response_time_us = 42,
         .cache_hit = false,
-        .upstream = "https://dns.example/dns-query",
     });
 
     var buf: [1024]u8 = undefined;
@@ -172,10 +165,12 @@ test "a frame is one event line and one data line of JSON" {
     try testing.expect(std.mem.startsWith(u8, frame, "event: query\ndata: {"));
     try testing.expect(std.mem.endsWith(u8, frame, "}\n\n"));
     try testing.expectEqual(@as(usize, 3), std.mem.count(u8, frame, "\n"));
-    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"ts\":1700000000"));
+    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"time\":1700000000"));
     try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"domain\":\"ads.example\""));
-    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"blocked\":true"));
-    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"block_reason\":\"blocklist_domain\""));
+    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"group\":{\"id\":1,\"name\":\"default\"}"));
+    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"reason\":\"blocklist_domain\""));
+    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"source_name\":\"StevenBlack\""));
+    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"kind\":\"blocked\""));
 }
 
 test "an unlogged field stays null and an empty string stays a string" {
@@ -191,6 +186,7 @@ test "an unlogged field stays null and an empty string stays a string" {
     const frame = writer.buffered();
 
     try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"qtype\":null"));
-    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"cache_hit\":null"));
+    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"duration_us\":null"));
     try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"upstream\":\"\""));
+    try testing.expect(std.mem.containsAtLeast(u8, frame, 1, "\"id\":null"));
 }

src/web/handlers/queries.zig

@@ -1,4 +1,5 @@
-//! `GET /api/queries` — the query log, newest first (ruling 11).
+//! `GET /api/queries` — the query log, newest first (ruling 11) — and
+//! `GET /api/queries/{id}`, one row of it fully explained.
 //!
 //! Keyset pagination rather than an offset: the table is append-only and the
 //! UI reads the head of it, so `id < before` is one index seek no matter how
@@ -13,9 +14,11 @@
 const std = @import("std");
 const Allocator = std.mem.Allocator;
 
+const coverage = @import("../coverage.zig");
 const db = @import("../../storage/db.zig");
 const http_util = @import("../http_util.zig");
 const logger = @import("../../storage/logger.zig");
+const provenance_view = @import("../provenance_view.zig");
 const queries_repo = @import("../../storage/repositories/queries_repo.zig");
 const server = @import("../server.zig");
 
@@ -41,6 +44,10 @@ pub const Page = struct {
     queries: []const queries_repo.QueryRow,
     /// The cursor for the next page, or null when this page is the last one.
     next_before: ?i64,
+    /// Whether the log still covers the window the filter asked for. A client
+    /// that reads rows without reading this cannot tell an empty window from a
+    /// pruned one.
+    coverage: coverage.Coverage,
 };
 
 pub const FilterError = error{
@@ -110,6 +117,7 @@ pub fn page(
     return .{
         .queries = rows.items,
         .next_before = if (full and rows.items.len != 0) rows.items[rows.items.len - 1].id else null,
+        .coverage = try coverage.read(database, filter.since),
     };
 }
 
@@ -138,10 +146,37 @@ pub fn list(
     return http_util.respondJson(request, .ok, result, &.{});
 }
 
+/// `GET /api/queries/{id}` — one query, fully explained.
+///
+/// The row is the whole answer: every field is a fact recorded when the query
+/// was answered, so nothing here is joined against current configuration. A
+/// group or blocklist renamed since keeps the name it had.
+pub fn detail(
+    state: *server.WebState,
+    io: std.Io,
+    request: *http_util.Request,
+) http_util.HandlerError!void {
+    _ = io;
+
+    const database = state.querylog_db orelse
+        return http_util.respondError(request, .service_unavailable, "query log unavailable");
+
+    const row = queries_repo.detailById(database, request.arena, request.id.?) catch |err| {
+        log.warn("query log read failed: {s}", .{@errorName(err)});
+        return http_util.respondError(request, .internal_server_error, "internal error");
+    };
+
+    // An id retention has pruned and one that never existed are the same
+    // answer, and the API does not pretend to tell them apart.
+    const found = row orelse return http_util.respondError(request, .not_found, "not found");
+    return http_util.respondJson(request, .ok, provenance_view.fromDetail(found), &.{});
+}
+
 // ---------------------------------------------------------------------------
 // tests
 // ---------------------------------------------------------------------------
 
+const provenance = @import("../../storage/provenance.zig");
 const querylog_schema = @import("../../storage/querylog_schema.zig");
 const testing = std.testing;
 
@@ -210,21 +245,40 @@ fn openLog() !db.Db {
     return database;
 }
 
+/// Rows the resolver could actually have written. Ruling 20 ties the three
+/// route facts together: a block never consulted the cache, so its `cache_hit`
+/// is NULL rather than false; a cache hit has no upstream to name; and only an
+/// upstream answer carries one. A fixture that broke those ties would let a
+/// serializer regression pass here and fail on real rows.
 fn seed(database: *db.Db, count: usize) !void {
     var writer = try queries_repo.BatchWriter.init(database);
     defer writer.deinit();
     var rows: [16]queries_repo.Row = undefined;
     for (rows[0..count], 0..) |*row, i| {
+        const blocked = i % 2 == 0;
+        const from_cache = i % 4 == 1;
         row.* = .{
             .timestamp = 1_700_000_000 + @as(i64, @intCast(i)),
-            .domain = if (i % 2 == 0) "ads.example" else "safe.example",
+            .domain = if (blocked) "ads.example" else "safe.example",
             .client_ip = "192.0.2.10",
             .qtype = 1,
-            .blocked = i % 2 == 0,
-            .block_reason = if (i % 2 == 0) "blocklist_domain" else null,
+            .qclass = 1,
+            .rcode = 0,
+            .blocked = blocked,
             .response_time_us = 500,
-            .cache_hit = false,
-            .upstream = null,
+            .cache_hit = if (blocked) null else from_cache,
+            .upstream = if (blocked or from_cache) null else "9.9.9.9",
+            .group_id = 1,
+            .group_name = "default",
+            .policy_action = if (blocked) .block else .allow,
+            .policy_reason = if (blocked) .blocklist_domain else .no_match,
+            .matched = if (blocked) "ads.example" else null,
+            .source_id = null,
+            .source_name = null,
+            .cname_target = null,
+            .safe_search_target = null,
+            .route_kind = if (blocked) .blocked else if (from_cache) .cache else .upstream,
+            .forward_zone = null,
         };
     }
     try writer.writeBatch(rows[0..count]);
@@ -297,6 +351,37 @@ test "the parsed filters narrow the rows the page returns" {
     try testing.expectEqual(@as(usize, 0), nobody.queries.len);
 }
 
+test "the seeded rows carry only the route shapes ruling 20 allows" {
+    var database = try openLog();
+    defer database.close();
+    try seed(&database, 4);
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena.deinit();
+
+    var seen: std.EnumSet(provenance.RouteKind) = .initEmpty();
+    for ((try page(&database, arena.allocator(), .{})).queries) |row| {
+        seen.insert(row.route_kind);
+        switch (row.route_kind) {
+            .blocked => {
+                try testing.expectEqual(@as(?bool, null), row.cache_hit);
+                try testing.expectEqualStrings("", row.upstream);
+            },
+            .cache => {
+                try testing.expectEqual(@as(?bool, true), row.cache_hit);
+                try testing.expectEqualStrings("", row.upstream);
+            },
+            .upstream => {
+                try testing.expectEqual(@as(?bool, false), row.cache_hit);
+                try testing.expectEqualStrings("9.9.9.9", row.upstream);
+            },
+            .local, .forward_zone, .rejected => return error.UnseededRouteKind,
+        }
+    }
+    // All three, so the serializer tests below read every shape the fixture claims.
+    try testing.expectEqual(@as(usize, 3), seen.count());
+}
+
 test "the page serializes as the envelope ruling 11 defines" {
     var database = try openLog();
     defer database.close();
@@ -312,14 +397,67 @@ test "the page serializes as the envelope ruling 11 defines" {
     const text = allocating.written();
 
     try testing.expect(std.mem.startsWith(u8, text, "{\"queries\":["));
-    try testing.expect(std.mem.endsWith(u8, text, "\"next_before\":null}"));
     for ([_][]const u8{
-        "\"id\":",       "\"ts\":",               "\"domain\":",       "\"client_ip\":",
-        "\"qtype\":",    "\"blocked\":",          "\"cache_hit\":",    "\"upstream\":",
-        "\"upstream\":", "\"response_time_us\":", "\"block_reason\":",
+        "\"id\":",            "\"ts\":",         "\"domain\":",           "\"client_ip\":",
+        "\"qtype\":",         "\"qclass\":",     "\"rcode\":",            "\"blocked\":",
+        "\"cache_hit\":",     "\"upstream\":",   "\"response_time_us\":", "\"policy_action\":",
+        "\"policy_reason\":", "\"route_kind\":",
     }) |field| {
         try testing.expect(std.mem.containsAtLeast(u8, text, 1, field));
     }
     // W1's ruling: a NULL column reads as "", and "" stays "" on the wire.
     try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"upstream\":\"\""));
+    // The column the provenance columns replaced.
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "block_reason"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"coverage\":{\"complete\":"));
+}
+
+test "the coverage of a page answers the window the filter asked for" {
+    var database = try openLog();
+    defer database.close();
+    try seed(&database, 1);
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena.deinit();
+
+    const watermark = try queries_repo.availableSince(&database);
+
+    const unbounded = try page(&database, arena.allocator(), .{});
+    try testing.expectEqual(watermark, unbounded.coverage.available_since);
+    try testing.expect(!unbounded.coverage.complete);
+
+    const covered = try page(&database, arena.allocator(), .{ .since = watermark });
+    try testing.expect(covered.coverage.complete);
+
+    const older = try page(&database, arena.allocator(), .{ .since = watermark - 1 });
+    try testing.expect(!older.coverage.complete);
+}
+
+test "a detail row carries every provenance field the row stored" {
+    var database = try openLog();
+    defer database.close();
+    try seed(&database, 1);
+
+    var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena_state.deinit();
+    const arena = arena_state.allocator();
+
+    const row = (try queries_repo.detailById(&database, arena, 1)).?;
+    const view = provenance_view.fromDetail(row);
+
+    try testing.expectEqual(@as(i64, 1), view.id);
+    try testing.expectEqualStrings("ads.example", view.request.domain);
+    try testing.expectEqualStrings("192.0.2.10", view.request.client);
+    try testing.expectEqual(@as(u16, 1), view.request.qclass);
+    try testing.expectEqualStrings("default", view.group.name);
+    try testing.expectEqual(provenance.PolicyAction.block, view.policy.action);
+    try testing.expectEqualStrings("ads.example", view.policy.matched);
+    try testing.expectEqual(provenance.RouteKind.blocked, view.route.kind);
+    try testing.expectEqualStrings("", view.route.upstream);
+    try testing.expectEqual(@as(?i64, 500), view.response.duration_us);
+
+    try testing.expectEqual(
+        @as(?queries_repo.QueryDetail, null),
+        try queries_repo.detailById(&database, arena, 99),
+    );
 }

src/web/handlers/stats.zig

@@ -14,6 +14,7 @@
 
 const std = @import("std");
 
+const coverage = @import("../coverage.zig");
 const db = @import("../../storage/db.zig");
 const http_util = @import("../http_util.zig");
 const queries_repo = @import("../../storage/repositories/queries_repo.zig");
@@ -99,6 +100,10 @@ pub const TotalsBody = struct {
     cached: u64,
     clients: u64,
     avg_response_time_us: ?i64,
+    /// Judged against `since`, which is the window this body reports on — so a
+    /// dashboard can say "history starts here" instead of charting a pruned
+    /// stretch as a quiet one.
+    coverage: coverage.Coverage,
 };
 
 pub const TimeseriesBody = struct {
@@ -107,6 +112,7 @@ pub const TimeseriesBody = struct {
     until: i64,
     bucket_seconds: u32,
     buckets: []const queries_repo.Bucket,
+    coverage: coverage.Coverage,
 };
 
 pub fn totals(
@@ -121,6 +127,9 @@ pub fn totals(
     const result = queries_repo.statsTotals(database, span.since, span.until) catch |err| {
         return internal(request, "stats totals", err);
     };
+    const covered = coverage.read(database, span.since) catch |err| {
+        return internal(request, "stats coverage", err);
+    };
 
     return http_util.respondJson(request, .ok, TotalsBody{
         .period = period.label(),
@@ -131,6 +140,7 @@ pub fn totals(
         .cached = result.cached,
         .clients = result.distinct_clients,
         .avg_response_time_us = result.avg_response_time_us,
+        .coverage = covered,
     }, &.{});
 }
 
@@ -148,6 +158,9 @@ pub fn timeseries(
     const written = queries_repo.timeseries(database, span.since, span.bucket_seconds, out) catch |err| {
         return internal(request, "stats timeseries", err);
     };
+    const covered = coverage.read(database, span.since) catch |err| {
+        return internal(request, "stats coverage", err);
+    };
 
     return http_util.respondJson(request, .ok, TimeseriesBody{
         .period = period.label(),
@@ -155,6 +168,7 @@ pub fn timeseries(
         .until = span.until,
         .bucket_seconds = span.bucket_seconds,
         .buckets = out[0..written],
+        .coverage = covered,
     }, &.{});
 }
 
@@ -268,11 +282,23 @@ fn writeRow(writer: *queries_repo.BatchWriter, timestamp: i64, blocked: bool, ca
         .domain = "example.com",
         .client_ip = "192.0.2.10",
         .qtype = 1,
+        .qclass = 1,
+        .rcode = 0,
         .blocked = blocked,
-        .block_reason = if (blocked) "blocklist_domain" else null,
         .response_time_us = 1000,
         .cache_hit = cached,
         .upstream = null,
+        .group_id = 1,
+        .group_name = "default",
+        .policy_action = if (blocked) .block else .allow,
+        .policy_reason = if (blocked) .blocklist_domain else .no_match,
+        .matched = null,
+        .source_id = null,
+        .source_name = null,
+        .cname_target = null,
+        .safe_search_target = null,
+        .route_kind = if (blocked) .blocked else .upstream,
+        .forward_zone = null,
     }};
     try writer.writeBatch(&rows);
 }