overview: one endpoint, live projections and a response cache (m36)

src/storage/logger.zig

@@ -30,6 +30,7 @@
 //! `writer_failed`, so the loss is visible rather than silent.
 
 const std = @import("std");
+const Allocator = std.mem.Allocator;
 const builtin = @import("builtin");
 
 const db = @import("db.zig");
@@ -555,13 +556,17 @@ pub const Logger = struct {
     /// group it cancels for exactly that reason.
     ///
     /// `monitor` is the §11.6 gate. Null disables gating.
+    ///
+    /// `gpa` belongs to the `BatchWriter` for that writer's whole life; it
+    /// allocates the projection deltas of one batch and nothing else.
     pub fn runWriter(
         self: *Logger,
         io: std.Io,
+        gpa: Allocator,
         database: *db.Db,
         monitor: ?*disk_monitor.Monitor,
     ) std.Io.Cancelable!void {
-        var writer = queries_repo.BatchWriter.init(database) catch |err| {
+        var writer = queries_repo.BatchWriter.init(gpa, database) catch |err| {
             scope.warn("query logger: preparing the batch statements failed: {s}", .{@errorName(err)});
             // Without a writer there is no consumer, so leaving the queue open
             // would silently swallow every later entry.
@@ -1134,7 +1139,7 @@ test "an entry with every provenance field set survives the queue, toRow, insert
 
     var database = try openLog();
     defer database.close();
-    var writer = try queries_repo.BatchWriter.init(&database);
+    var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
     defer writer.deinit();
 
     var buf: [4]Entry = undefined;
@@ -1466,6 +1471,7 @@ test "shutdown writes the batch the writer holds and the rest of the queue" {
     var future = try io.concurrent(Logger.runWriter, .{
         &logger,
         io,
+        testing.allocator,
         &database,
         @as(?*disk_monitor.Monitor, null),
     });
@@ -1502,7 +1508,7 @@ test "entries that arrive inside one window reach the database in one batch" {
 
     var database = try openLog();
     defer database.close();
-    var writer = try queries_repo.BatchWriter.init(&database);
+    var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
     defer writer.deinit();
 
     var buf: [16]Entry = undefined;
@@ -1559,6 +1565,7 @@ test "the writer holds an entry for the length of the flush interval" {
     var future = try io.concurrent(Logger.runWriter, .{
         &logger,
         io,
+        testing.allocator,
         &database,
         @as(?*disk_monitor.Monitor, null),
     });
@@ -1611,6 +1618,7 @@ test "a full batch flushes without waiting for the interval" {
     var future = try io.concurrent(Logger.runWriter, .{
         &logger,
         io,
+        testing.allocator,
         &database,
         @as(?*disk_monitor.Monitor, null),
     });
@@ -1658,6 +1666,7 @@ test "the writer's next cycle uses the interval set since its last one" {
     var future = try io.concurrent(Logger.runWriter, .{
         &logger,
         io,
+        testing.allocator,
         &database,
         @as(?*disk_monitor.Monitor, null),
     });
@@ -1700,7 +1709,7 @@ test "a gated flush holds the batch until the disk recovers" {
 
     var database = try openLog();
     defer database.close();
-    var writer = try queries_repo.BatchWriter.init(&database);
+    var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
     defer writer.deinit();
 
     var buf: [4]Entry = undefined;
@@ -1754,7 +1763,7 @@ test "a failing batch is dropped whole and the writer stays usable" {
         \\BEGIN SELECT RAISE(ABORT, 'refused'); END;
     );
 
-    var writer = try queries_repo.BatchWriter.init(&database);
+    var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
     defer writer.deinit();
 
     var buf: [4]Entry = undefined;
@@ -1789,7 +1798,7 @@ test "a writer that cannot prepare closes the queue and counts every entry" {
 
     for (0..3) |i| logger.log(io, sampleEntry(@intCast(i), "early.example"));
 
-    try logger.runWriter(io, &database, null);
+    try logger.runWriter(io, testing.allocator, &database, null);
 
     try testing.expect(logger.writer_failed.load(.acquire));
     try testing.expectEqual(@as(u64, 3), logger.queries_dropped.load(.monotonic));
@@ -1842,6 +1851,7 @@ test "the gating episode opens on the gate, turns losing on a drop, and clears o
     var future = try io.concurrent(Logger.runWriter, .{
         &logger,
         io,
+        testing.allocator,
         &database,
         @as(?*disk_monitor.Monitor, &monitor),
     });
@@ -2023,6 +2033,7 @@ test "a canceled writer counts the batch it was holding" {
     var future = try io.concurrent(Logger.runWriter, .{
         &logger,
         io,
+        testing.allocator,
         &database,
         @as(?*disk_monitor.Monitor, &monitor),
     });
@@ -2072,6 +2083,7 @@ test "a disk-gated writer drops what it holds at shutdown instead of hanging" {
     var future = try io.concurrent(Logger.runWriter, .{
         &logger,
         io,
+        testing.allocator,
         &database,
         @as(?*disk_monitor.Monitor, &monitor),
     });
@@ -2121,7 +2133,7 @@ test "an empty batch touches neither the database nor the counters" {
 
     var database = try openLog();
     defer database.close();
-    var writer = try queries_repo.BatchWriter.init(&database);
+    var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
     defer writer.deinit();
 
     var buf: [4]Entry = undefined;
@@ -2155,7 +2167,7 @@ test "a dropped batch opens an error episode the next good batch closes" {
     try fx.init(io, 1000);
     defer fx.deinit();
 
-    var writer = try queries_repo.BatchWriter.init(&database);
+    var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
     defer writer.deinit();
 
     var buf: [4]Entry = undefined;
@@ -2198,7 +2210,7 @@ test "a writer that cannot prepare leaves an episode no recovery path claims" {
     var logger: Logger = .init(.{}, &buf);
     logger.diagnostics = &fx.store;
 
-    try logger.runWriter(io, &database, null);
+    try logger.runWriter(io, testing.allocator, &database, null);
 
     try testing.expectEqualStrings("writer", try fx.text(
         "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
@@ -2209,7 +2221,7 @@ test "a writer that cannot prepare leaves an episode no recovery path claims" {
 
     // The writer returned, so nothing can ever close this. A second run finds
     // the queue closed and adds no second episode.
-    try logger.runWriter(io, &database, null);
+    try logger.runWriter(io, testing.allocator, &database, null);
     try testing.expectEqual(
         @as(i64, 1),
         try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),

src/storage/logger_controller.zig

@@ -248,6 +248,7 @@ pub const Controller = struct {
         owned.writer = try io.concurrent(logger.Logger.runWriter, .{
             generation.logger,
             io,
+            opts.gpa,
             &owned.database,
             opts.monitor,
         });
@@ -434,7 +435,7 @@ pub const Controller = struct {
         errdefer generation.deinit(self.gpa);
         const owned = &generation.owned.?;
 
-        owned.writer = try io.concurrent(runParkedWriter, .{ generation, io, self.monitor });
+        owned.writer = try io.concurrent(runParkedWriter, .{ generation, io, self.gpa, self.monitor });
 
         // The statements are prepared before anything is published, so a
         // failure here is a refused settings change rather than a writer that
@@ -565,11 +566,12 @@ fn drain(generation: *Generation, io: std.Io) void {
 fn runParkedWriter(
     generation: *Generation,
     io: std.Io,
+    gpa: Allocator,
     monitor: ?*disk_monitor.Monitor,
 ) std.Io.Cancelable!void {
     const owned = &generation.owned.?;
 
-    var writer = queries_repo.BatchWriter.init(&owned.database) catch |err| {
+    var writer = queries_repo.BatchWriter.init(gpa, &owned.database) catch |err| {
         owned.prepare_error = err;
         owned.ready.set(io);
         return;

src/storage/phase6_integration_test.zig

@@ -144,7 +144,7 @@ fn awaitCount(counter: *const std.atomic.Value(u64), target: u64, limit: usize)
 }
 
 fn writeRows(database: *db.Db, timestamps: []const i64, domain: []const u8) !void {
-    var writer = try queries_repo.BatchWriter.init(database);
+    var writer = try queries_repo.BatchWriter.init(testing.allocator, database);
     defer writer.deinit();
 
     var rows: [16]queries_repo.Row = undefined;
@@ -208,6 +208,7 @@ test "S8 case 1: the logger writes a real querylog.db end to end" {
     var future = try io.concurrent(logger.Logger.runWriter, .{
         &query_log,
         io,
+        testing.allocator,
         log_db.database(),
         @as(?*disk_monitor.Monitor, null),
     });
@@ -256,6 +257,7 @@ test "S8 case 2: a single entry reaches the file once the flush interval passes"
     var future = try io.concurrent(logger.Logger.runWriter, .{
         &query_log,
         io,
+        testing.allocator,
         log_db.database(),
         @as(?*disk_monitor.Monitor, null),
     });
@@ -311,6 +313,7 @@ test "S8 case 3: a full queue drops the oldest entries and the newest survive" {
     var future = try io.concurrent(logger.Logger.runWriter, .{
         &query_log,
         io,
+        testing.allocator,
         log_db.database(),
         @as(?*disk_monitor.Monitor, &monitor),
     });
@@ -359,6 +362,7 @@ test "S8 case 4: the privacy transforms reach the stored rows" {
     var future = try io.concurrent(logger.Logger.runWriter, .{
         &query_log,
         io,
+        testing.allocator,
         log_db.database(),
         @as(?*disk_monitor.Monitor, null),
     });
@@ -459,6 +463,7 @@ test "S8 case 6: a critical disk gates the flushes and recovery releases them" {
     var future = try io.concurrent(logger.Logger.runWriter, .{
         &query_log,
         io,
+        testing.allocator,
         log_db.database(),
         @as(?*disk_monitor.Monitor, &monitor),
     });

src/storage/querylog_schema.zig

@@ -25,7 +25,7 @@ const log = std.log.scoped(.querylog_schema);
 /// PLAN §11.3, plus the coverage watermark of milestone 28. Multi-statement
 /// text — it goes through `db.Db.exec`, never through `prepare`.
 ///
-/// The trailing INSERT seeds `querylog_meta`, which is part of the schema
+/// The INSERT seeds `querylog_meta`, which is part of the schema
 /// rather than a later step: a `query_log` with no watermark beside it cannot
 /// answer whether an empty result means "no queries" or "no history", and every
 /// database this program reads from is created by executing this string.
@@ -36,6 +36,13 @@ const log = std.log.scoped(.querylog_schema);
 /// logged in the same second the file was created is not evidence that the
 /// second is completely covered, and the watermark's whole job is to be
 /// conservative. From there it only ever advances, in `queries_repo.pruneOlderThan`.
+///
+/// The four `bucket_*` tables are the Overview projections (milestone 36), on a
+/// 30-minute grain that divides every serving width the API offers. They carry
+/// no history of their own: they are born with the file and maintained in the
+/// same transaction as every insert and every prune, so SQLite's transaction is
+/// the only coherence mechanism there is. There is no backfill path — a file
+/// whose projections could disagree with its rows cannot exist.
 pub const ddl: [:0]const u8 =
     \\CREATE TABLE domains (
     \\    id INTEGER PRIMARY KEY,
@@ -78,6 +85,40 @@ pub const ddl: [:0]const u8 =
     \\);
     \\INSERT INTO querylog_meta (id, created_at, available_since)
     \\VALUES (1, unixepoch(), unixepoch() + 1);
+    \\
+    \\CREATE TABLE bucket_totals (
+    \\    bucket INTEGER PRIMARY KEY,
+    \\    queries INTEGER NOT NULL,
+    \\    blocked INTEGER NOT NULL,
+    \\    cached INTEGER NOT NULL,
+    \\    rt_sum INTEGER NOT NULL,     -- sum(response_time_us) over timed rows
+    \\    rt_count INTEGER NOT NULL    -- count(response_time_us)
+    \\) WITHOUT ROWID;
+    \\
+    \\CREATE TABLE bucket_clients (
+    \\    bucket INTEGER NOT NULL,
+    \\    client_ip TEXT NOT NULL,
+    \\    queries INTEGER NOT NULL,
+    \\    PRIMARY KEY (bucket, client_ip)
+    \\) WITHOUT ROWID;
+    \\
+    \\CREATE TABLE bucket_types (
+    \\    bucket INTEGER NOT NULL,
+    \\    qtype INTEGER NOT NULL,      -- -1 encodes a NULL qtype, losslessly
+    \\    count INTEGER NOT NULL,
+    \\    PRIMARY KEY (bucket, qtype)
+    \\) WITHOUT ROWID;
+    \\
+    \\CREATE TABLE bucket_routes (
+    \\    bucket INTEGER NOT NULL,
+    \\    route_kind TEXT NOT NULL,
+    \\    source_present INTEGER NOT NULL,  -- 0: source NULL; 1: source = source_text
+    \\    source_text TEXT NOT NULL,        -- '' when source_present = 0
+    \\    count INTEGER NOT NULL,
+    \\    PRIMARY KEY (bucket, route_kind, source_present, source_text),
+    \\    CHECK (source_present IN (0, 1)),
+    \\    CHECK (source_present = 1 OR source_text = '')
+    \\) WITHOUT ROWID;
 ;
 
 /// The fingerprint of an arbitrary DDL text. `tools/cut.zig` calls this at
@@ -324,13 +365,23 @@ test "ddl creates the query-log tables and every index" {
     try database.exec(ddl);
 
     try testing.expectEqual(
-        @as(i64, 3),
+        @as(i64, 7),
         try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
     );
+    // The three explicit indexes plus `domains.domain`'s autoindex, and
+    // nothing else: the four projection tables are WITHOUT ROWID, so each
+    // one's PRIMARY KEY *is* its storage rather than a second b-tree to keep
+    // in step on every insert.
+    try testing.expectEqual(
+        @as(i64, 4),
+        try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='index'"),
+    );
     const objects = [_][]const u8{
         "domains",              "query_log",
         "idx_query_log_ts",     "idx_query_log_client",
         "idx_query_log_domain", "querylog_meta",
+        "bucket_totals",        "bucket_clients",
+        "bucket_types",         "bucket_routes",
     };
     for (objects) |name| {
         var stmt = try database.prepare("SELECT count(*) FROM sqlite_schema WHERE name = ?1");

src/storage/retention.zig

@@ -251,7 +251,7 @@ fn openLog() !db.Db {
 }
 
 fn writeRows(database: *db.Db, timestamps: []const i64) !void {
-    var writer = try queries_repo.BatchWriter.init(database);
+    var writer = try queries_repo.BatchWriter.init(testing.allocator, database);
     defer writer.deinit();
     var rows: [8]queries_repo.Row = undefined;
     for (timestamps, rows[0..timestamps.len]) |timestamp, *row| {