milestone 30: overview as a dashboard, explicit health contract, period aggregations

src/app.zig

@@ -49,7 +49,6 @@ const faults = @import("config/faults.zig");
 const fetcher = @import("filter/fetcher.zig");
 const forward_zones = @import("local/forward_zones.zig");
 const handler = @import("server/handler.zig");
-const history_mod = @import("upstream/history.zig");
 const http_util = @import("web/http_util.zig");
 const loader = @import("config/loader.zig");
 const local_records = @import("local/records.zig");
@@ -72,7 +71,6 @@ const shutdown = @import("server/shutdown.zig");
 const sse = @import("web/sse.zig");
 const static = @import("web/static.zig");
 const tcp_server = @import("server/tcp_server.zig");
-const upstream_history_repo = @import("storage/repositories/upstream_history_repo.zig");
 const transport = @import("upstream/transport.zig");
 const udp_server = @import("server/udp_server.zig");
 const validate = @import("config/validate.zig");
@@ -544,13 +542,6 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
         @truncate(@as(u96, @bitCast(std.Io.Clock.real.now(io).nanoseconds))),
     );
 
-    // On the heap, not in this frame: the accumulator carries its pending cells
-    // and the flush task's buffer inline, which is about a megabyte.
-    const history = try gpa.create(history_mod.Accumulator);
-    defer gpa.destroy(history);
-    history.* = .init;
-    history.diagnostics = event_store;
-    pool.history = history;
     pool.diagnostics = event_store;
 
     // -----------------------------------------------------------------------
@@ -636,8 +627,6 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     reportQuerylogRecreated(event_store, io, boot_now_s, &querylog_opened, &querylog_writer_db);
     var querylog_retention_db = try data.reopenQuerylogDb(io);
     defer querylog_retention_db.close();
-    var querylog_history_db = try data.reopenQuerylogDb(io);
-    defer querylog_history_db.close();
     var tracker_db = try data.openConfigDb(io);
     defer tracker_db.close();
 
@@ -797,7 +786,6 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
         .local_tables = &tables,
         .logger = &query_logger,
         .retention = &retention,
-        .history = history,
         .sessions = if (sessions) |*s| s else null,
         .limiter = if (web_limiter) |*l| l else null,
         .hub = hub,
@@ -921,22 +909,19 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
 
     // Ruling 4's shutdown order, on the one path every exit from here takes:
     // every producer stops and is joined, then the queue closes, then the
-    // writer is awaited — so the last batch is written rather than raced — and
-    // only then does the final history flush run, with no recording task left
-    // that could add a cell after it. A writer the disk gate will not let write
-    // counts its batch as dropped instead of holding the exit open
-    // (`logger.zig`), so this wait always ends.
+    // writer is awaited — so the last batch is written rather than raced. A
+    // writer the disk gate will not let write counts its batch as dropped
+    // instead of holding the exit open (`logger.zig`), so this wait always
+    // ends.
     //
     // A `defer` and not straight-line code after `shutdown.wait`, because a
     // `concurrent` spawn below can fail with the DNS listeners already
     // serving; an orderly error teardown owes the operator the same drain a
-    // signal gets. The `querylog_history_db` this flush writes through is
-    // declared above, so its `close` runs after it.
+    // signal gets.
     defer {
         group.cancel(io);
         query_logger.shutdown(io);
         writer_future.await(io) catch {};
-        history.flushOnce(io, &querylog_history_db, upstream_history_repo.flush);
     }
 
     if (udp6) |*s| try group.concurrent(io, udp_server.UdpServer.serve, .{ s, io });
@@ -949,9 +934,6 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     if (dot_certs) |*store| try group.concurrent(io, cert_store.CertStore.watch, .{ store, io });
 
     try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db, gate, event_store });
-    // Ungated: a flush writes at most one row per upstream per minute, the same
-    // category as the query logger's own writes, which are ungated too.
-    try group.concurrent(io, history_mod.Accumulator.run, .{ history, io, &querylog_history_db });
     try group.concurrent(io, disk_monitor.Monitor.run, .{ &monitor, io, event_store });
     try group.concurrent(io, manager_mod.Manager.runScheduler, .{ &manager, io });
     try group.concurrent(io, clients.Tracker.run, .{ &tracker, io, &tracker_db, gate, &client_names_resolver });
@@ -1451,6 +1433,168 @@ test "the recreated detail names the aside and the new coverage start" {
     );
 }
 
+/// The `querylog.db` schema as milestone 29 shipped it, verbatim from
+/// `querylog_schema.zig` at commit fa323c7. A literal and not this build's DDL
+/// with the deleted tables appended: the appended form drifts the moment the
+/// surviving tables change, and its fingerprint was never the one an m29 file
+/// on disk actually carries. The transition under test is that exact byte
+/// sequence meeting this build.
+/// The `PRAGMA user_version` an m29 file on disk carries, written down rather
+/// than recomputed from the literal below. A CRC taken over the fixture
+/// validates whatever the fixture happens to say, so a slip in the "byte-exact"
+/// literal would still self-certify; pinning the historical number turns that
+/// slip into a failure. Its value is `Crc32` over `querylog_schema.ddl` at
+/// commit fa323c7.
+const m29_fingerprint: i32 = 603440875;
+
+/// A watermark from long before this test runs. Both schemas seed
+/// `available_since` from `unixepoch()`, so a fixture left at its own default
+/// would satisfy "the new file's coverage is not older" even if recreation
+/// copied the replaced file's promise straight across.
+const m29_available_since: i64 = 1_600_000_000;
+
+const m29_ddl: [:0]const u8 =
+    \\CREATE TABLE domains (
+    \\    id INTEGER PRIMARY KEY,
+    \\    domain TEXT NOT NULL UNIQUE
+    \\);
+    \\
+    \\CREATE TABLE query_log (
+    \\    id INTEGER PRIMARY KEY,
+    \\    timestamp INTEGER NOT NULL,
+    \\    domain_id INTEGER NOT NULL REFERENCES domains(id),
+    \\    client_ip TEXT NOT NULL,            -- text, not a FK: log rows are immutable facts
+    \\    qtype INTEGER,
+    \\    blocked INTEGER NOT NULL,
+    \\    response_time_us INTEGER,
+    \\    cache_hit INTEGER,
+    \\    upstream TEXT,
+    \\    qclass INTEGER NOT NULL,
+    \\    rcode INTEGER NOT NULL,
+    \\    group_id INTEGER,                   -- text/id pairs, not FKs: a renamed
+    \\    group_name TEXT,                    -- group must not rewrite history
+    \\    policy_action TEXT NOT NULL,
+    \\    policy_reason TEXT NOT NULL,
+    \\    matched TEXT,
+    \\    source_id INTEGER,
+    \\    source_name TEXT,
+    \\    cname_target TEXT,
+    \\    safe_search_target TEXT,
+    \\    route_kind TEXT NOT NULL,
+    \\    forward_zone TEXT,
+    \\    CHECK (rcode BETWEEN 0 AND 4095)    -- twelve bits (RFC 6891 6.1.3)
+    \\);
+    \\CREATE INDEX idx_query_log_ts ON query_log(timestamp);
+    \\CREATE INDEX idx_query_log_client ON query_log(client_ip);
+    \\CREATE INDEX idx_query_log_domain ON query_log(domain_id);
+    \\
+    \\CREATE TABLE upstream_targets (
+    \\    id INTEGER PRIMARY KEY,
+    \\    url TEXT NOT NULL UNIQUE   -- the historical identity: config.db ids cannot cross database files
+    \\);
+    \\
+    \\CREATE TABLE upstream_minute (
+    \\    upstream_id INTEGER NOT NULL REFERENCES upstream_targets(id),
+    \\    minute_ts INTEGER NOT NULL,
+    \\    successes INTEGER NOT NULL,
+    \\    failures INTEGER NOT NULL,
+    \\    last_failure_ts INTEGER,
+    \\    last_error TEXT,
+    \\    PRIMARY KEY (upstream_id, minute_ts),
+    \\    CHECK (successes >= 0),
+    \\    CHECK (failures >= 0)
+    \\) WITHOUT ROWID;
+    \\CREATE INDEX idx_upstream_minute_ts ON upstream_minute(minute_ts);
+    \\
+    \\CREATE TABLE querylog_meta (
+    \\    id INTEGER PRIMARY KEY CHECK (id = 1),   -- one row, enforced by the schema
+    \\    created_at INTEGER NOT NULL,
+    \\    available_since INTEGER NOT NULL
+    \\);
+    \\INSERT INTO querylog_meta (id, created_at, available_since)
+    \\VALUES (1, unixepoch(), unixepoch() + 1);
+;
+
+test "an m29 query log is set aside and recreated without the upstream-history tables" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var tmp = testing.tmpDir(.{ .iterate = true });
+    defer tmp.cleanup();
+
+    var path_buf: [256]u8 = undefined;
+    const path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/querylog.db", .{tmp.sub_path});
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+
+    // The fixture is only worth anything while it is still a *different*
+    // schema from this build's, and one that carries the deleted tables.
+    try testing.expect(!std.mem.eql(u8, m29_ddl, querylog_schema.ddl));
+    try testing.expect(std.mem.indexOf(u8, m29_ddl, "CREATE TABLE upstream_minute") != null);
+    try testing.expect(std.mem.indexOf(u8, querylog_schema.ddl, "upstream_minute") == null);
+    // And only while it is still m29's bytes: this is the one check that an
+    // edit to the literal cannot satisfy by changing what it is compared to.
+    try testing.expectEqual(m29_fingerprint, @as(i32, @bitCast(std.hash.Crc32.hash(m29_ddl))));
+
+    // A healthy m29 file, stamped with the fingerprint m29's own DDL produced
+    // and backdated so its coverage promise is visibly the older one.
+    const m29_coverage = blk: {
+        var m29 = try db.Db.open(path, .{ .mode = .read_write_create });
+        defer m29.close();
+        try db.applyPragmas(&m29, .{});
+        try m29.exec(m29_ddl);
+        try m29.exec("INSERT INTO upstream_targets (url) VALUES ('https://dns.example/dns-query');");
+        var meta_buf: [128]u8 = undefined;
+        try m29.exec(try std.fmt.bufPrintZ(
+            &meta_buf,
+            "UPDATE querylog_meta SET created_at = {d}, available_since = {d};",
+            .{ m29_available_since, m29_available_since },
+        ));
+        var version_buf: [64]u8 = undefined;
+        try m29.exec(try std.fmt.bufPrintZ(
+            &version_buf,
+            "PRAGMA user_version = {d};",
+            .{m29_fingerprint},
+        ));
+        break :blk try m29.queryInt("SELECT available_since FROM querylog_meta");
+    };
+    try testing.expectEqual(m29_available_since, m29_coverage);
+
+    var opened = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
+    defer opened.database.close();
+
+    // Set aside under the name that says the file was healthy and this build
+    // moved, and still on disk for an operator who wants it.
+    try testing.expectEqual(querylog_schema.RecreateReason.fingerprint_mismatch, opened.recreated.?);
+    try testing.expect(std.mem.indexOf(u8, opened.aside(), ".schema-changed-") != null);
+    try tmp.dir.access(io, std.fs.path.basename(opened.aside()), .{});
+
+    // The two tables are gone from the file this process will write to.
+    for ([_][]const u8{ "upstream_targets", "upstream_minute", "idx_upstream_minute_ts" }) |name| {
+        var stmt = try opened.database.prepare("SELECT count(*) FROM sqlite_schema WHERE name = ?1");
+        defer stmt.deinit();
+        try stmt.bindText(1, name);
+        try testing.expect(try stmt.step());
+        try testing.expectEqual(@as(i64, 0), stmt.columnInt(0));
+    }
+
+    // Coverage restarts: the new file does not inherit the replaced one's
+    // promise about what it can answer. Strictly newer, not merely not-older —
+    // a recreation that copied the watermark across would pass the weaker test.
+    const coverage = try queries_repo.availableSince(&opened.database);
+    try testing.expect(coverage > m29_coverage);
+
+    reportQuerylogRecreated(&fx.store, io, 2000, &opened, &opened.database);
+    try testing.expectEqualStrings("query_log.recreated", try fx.text("SELECT code FROM operational_events"));
+    try testing.expectEqualStrings(
+        "fingerprint_mismatch",
+        try fx.text("SELECT subject_key FROM operational_events"),
+    );
+}
+
 test "a fingerprint recreate files a resolved event naming the real aside and watermark" {
     var threaded: std.Io.Threaded = .init(testing.allocator, .{});
     defer threaded.deinit();