mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 21 Activity

overview: one endpoint, live projections and a response cache (m36)

Browse Source
This commit is contained in:
mokhtar
2026-08-27 17:48:20 +02:00
parent 7cdb74ec17
commit c8470724ce
40 changed files with 3298 additions and 2242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app.zig
+3
View File
@@ -817,6 +817,9 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
// Same argument as the live hash: a settings PUT may have installed an
// owned generation, and this runs after `group.cancel`.
defer web_state.proxies.deinit(gpa);
// The Overview response cache owns its bodies from `gpa`. Same argument
// again: no web task can still be reading a slot once the group is cancelled.
defer web_state.overview_cache.deinit(gpa);
if (cfg.web.enabled) web_state = .{
.gpa = gpa,
.web = cfg.web,
src/storage/logger.zig
+22 -10
View File
@@ -30,6 +30,7 @@
//! `writer_failed`, so the loss is visible rather than silent.
const std = @import("std");
const Allocator = std.mem.Allocator;
const builtin = @import("builtin");
const db = @import("db.zig");
@@ -555,13 +556,17 @@ pub const Logger = struct {
/// group it cancels for exactly that reason.
///
/// `monitor` is the §11.6 gate. Null disables gating.
///
/// `gpa` belongs to the `BatchWriter` for that writer's whole life; it
/// allocates the projection deltas of one batch and nothing else.
pub fn runWriter(
self: *Logger,
io: std.Io,
gpa: Allocator,
database: *db.Db,
monitor: ?*disk_monitor.Monitor,
) std.Io.Cancelable!void {
var writer = queries_repo.BatchWriter.init(database) catch |err| {
var writer = queries_repo.BatchWriter.init(gpa, database) catch |err| {
scope.warn("query logger: preparing the batch statements failed: {s}", .{@errorName(err)});
// Without a writer there is no consumer, so leaving the queue open
// would silently swallow every later entry.
@@ -1134,7 +1139,7 @@ test "an entry with every provenance field set survives the queue, toRow, insert
var database = try openLog();
defer database.close();
var writer = try queries_repo.BatchWriter.init(&database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
defer writer.deinit();
var buf: [4]Entry = undefined;
@@ -1466,6 +1471,7 @@ test "shutdown writes the batch the writer holds and the rest of the queue" {
var future = try io.concurrent(Logger.runWriter, .{
&logger,
io,
testing.allocator,
&database,
@as(?*disk_monitor.Monitor, null),
});
@@ -1502,7 +1508,7 @@ test "entries that arrive inside one window reach the database in one batch" {
var database = try openLog();
defer database.close();
var writer = try queries_repo.BatchWriter.init(&database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
defer writer.deinit();
var buf: [16]Entry = undefined;
@@ -1559,6 +1565,7 @@ test "the writer holds an entry for the length of the flush interval" {
var future = try io.concurrent(Logger.runWriter, .{
&logger,
io,
testing.allocator,
&database,
@as(?*disk_monitor.Monitor, null),
});
@@ -1611,6 +1618,7 @@ test "a full batch flushes without waiting for the interval" {
var future = try io.concurrent(Logger.runWriter, .{
&logger,
io,
testing.allocator,
&database,
@as(?*disk_monitor.Monitor, null),
});
@@ -1658,6 +1666,7 @@ test "the writer's next cycle uses the interval set since its last one" {
var future = try io.concurrent(Logger.runWriter, .{
&logger,
io,
testing.allocator,
&database,
@as(?*disk_monitor.Monitor, null),
});
@@ -1700,7 +1709,7 @@ test "a gated flush holds the batch until the disk recovers" {
var database = try openLog();
defer database.close();
var writer = try queries_repo.BatchWriter.init(&database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
defer writer.deinit();
var buf: [4]Entry = undefined;
@@ -1754,7 +1763,7 @@ test "a failing batch is dropped whole and the writer stays usable" {
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
);
var writer = try queries_repo.BatchWriter.init(&database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
defer writer.deinit();
var buf: [4]Entry = undefined;
@@ -1789,7 +1798,7 @@ test "a writer that cannot prepare closes the queue and counts every entry" {
for (0..3) |i| logger.log(io, sampleEntry(@intCast(i), "early.example"));
try logger.runWriter(io, &database, null);
try logger.runWriter(io, testing.allocator, &database, null);
try testing.expect(logger.writer_failed.load(.acquire));
try testing.expectEqual(@as(u64, 3), logger.queries_dropped.load(.monotonic));
@@ -1842,6 +1851,7 @@ test "the gating episode opens on the gate, turns losing on a drop, and clears o
var future = try io.concurrent(Logger.runWriter, .{
&logger,
io,
testing.allocator,
&database,
@as(?*disk_monitor.Monitor, &monitor),
});
@@ -2023,6 +2033,7 @@ test "a canceled writer counts the batch it was holding" {
var future = try io.concurrent(Logger.runWriter, .{
&logger,
io,
testing.allocator,
&database,
@as(?*disk_monitor.Monitor, &monitor),
});
@@ -2072,6 +2083,7 @@ test "a disk-gated writer drops what it holds at shutdown instead of hanging" {
var future = try io.concurrent(Logger.runWriter, .{
&logger,
io,
testing.allocator,
&database,
@as(?*disk_monitor.Monitor, &monitor),
});
@@ -2121,7 +2133,7 @@ test "an empty batch touches neither the database nor the counters" {
var database = try openLog();
defer database.close();
var writer = try queries_repo.BatchWriter.init(&database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
defer writer.deinit();
var buf: [4]Entry = undefined;
@@ -2155,7 +2167,7 @@ test "a dropped batch opens an error episode the next good batch closes" {
try fx.init(io, 1000);
defer fx.deinit();
var writer = try queries_repo.BatchWriter.init(&database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
defer writer.deinit();
var buf: [4]Entry = undefined;
@@ -2198,7 +2210,7 @@ test "a writer that cannot prepare leaves an episode no recovery path claims" {
var logger: Logger = .init(.{}, &buf);
logger.diagnostics = &fx.store;
try logger.runWriter(io, &database, null);
try logger.runWriter(io, testing.allocator, &database, null);
try testing.expectEqualStrings("writer", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
@@ -2209,7 +2221,7 @@ test "a writer that cannot prepare leaves an episode no recovery path claims" {
// The writer returned, so nothing can ever close this. A second run finds
// the queue closed and adds no second episode.
try logger.runWriter(io, &database, null);
try logger.runWriter(io, testing.allocator, &database, null);
try testing.expectEqual(
@as(i64, 1),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
src/storage/logger_controller.zig
+4 -2
View File
@@ -248,6 +248,7 @@ pub const Controller = struct {
owned.writer = try io.concurrent(logger.Logger.runWriter, .{
generation.logger,
io,
opts.gpa,
&owned.database,
opts.monitor,
});
@@ -434,7 +435,7 @@ pub const Controller = struct {
errdefer generation.deinit(self.gpa);
const owned = &generation.owned.?;
owned.writer = try io.concurrent(runParkedWriter, .{ generation, io, self.monitor });
owned.writer = try io.concurrent(runParkedWriter, .{ generation, io, self.gpa, self.monitor });
// The statements are prepared before anything is published, so a
// failure here is a refused settings change rather than a writer that
@@ -565,11 +566,12 @@ fn drain(generation: *Generation, io: std.Io) void {
fn runParkedWriter(
generation: *Generation,
io: std.Io,
gpa: Allocator,
monitor: ?*disk_monitor.Monitor,
) std.Io.Cancelable!void {
const owned = &generation.owned.?;
var writer = queries_repo.BatchWriter.init(&owned.database) catch |err| {
var writer = queries_repo.BatchWriter.init(gpa, &owned.database) catch |err| {
owned.prepare_error = err;
owned.ready.set(io);
return;
src/storage/phase6_integration_test.zig
+6 -1
View File
@@ -144,7 +144,7 @@ fn awaitCount(counter: *const std.atomic.Value(u64), target: u64, limit: usize)
}
fn writeRows(database: *db.Db, timestamps: []const i64, domain: []const u8) !void {
var writer = try queries_repo.BatchWriter.init(database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, database);
defer writer.deinit();
var rows: [16]queries_repo.Row = undefined;
@@ -208,6 +208,7 @@ test "S8 case 1: the logger writes a real querylog.db end to end" {
var future = try io.concurrent(logger.Logger.runWriter, .{
&query_log,
io,
testing.allocator,
log_db.database(),
@as(?*disk_monitor.Monitor, null),
});
@@ -256,6 +257,7 @@ test "S8 case 2: a single entry reaches the file once the flush interval passes"
var future = try io.concurrent(logger.Logger.runWriter, .{
&query_log,
io,
testing.allocator,
log_db.database(),
@as(?*disk_monitor.Monitor, null),
});
@@ -311,6 +313,7 @@ test "S8 case 3: a full queue drops the oldest entries and the newest survive" {
var future = try io.concurrent(logger.Logger.runWriter, .{
&query_log,
io,
testing.allocator,
log_db.database(),
@as(?*disk_monitor.Monitor, &monitor),
});
@@ -359,6 +362,7 @@ test "S8 case 4: the privacy transforms reach the stored rows" {
var future = try io.concurrent(logger.Logger.runWriter, .{
&query_log,
io,
testing.allocator,
log_db.database(),
@as(?*disk_monitor.Monitor, null),
});
@@ -459,6 +463,7 @@ test "S8 case 6: a critical disk gates the flushes and recovery releases them" {
var future = try io.concurrent(logger.Logger.runWriter, .{
&query_log,
io,
testing.allocator,
log_db.database(),
@as(?*disk_monitor.Monitor, &monitor),
});
src/storage/querylog_schema.zig
+53 -2
View File
@@ -25,7 +25,7 @@ const log = std.log.scoped(.querylog_schema);
/// PLAN §11.3, plus the coverage watermark of milestone 28. Multi-statement
/// text — it goes through `db.Db.exec`, never through `prepare`.
///
/// The trailing INSERT seeds `querylog_meta`, which is part of the schema
/// The INSERT seeds `querylog_meta`, which is part of the schema
/// rather than a later step: a `query_log` with no watermark beside it cannot
/// answer whether an empty result means "no queries" or "no history", and every
/// database this program reads from is created by executing this string.
@@ -36,6 +36,13 @@ const log = std.log.scoped(.querylog_schema);
/// logged in the same second the file was created is not evidence that the
/// second is completely covered, and the watermark's whole job is to be
/// conservative. From there it only ever advances, in `queries_repo.pruneOlderThan`.
///
/// The four `bucket_*` tables are the Overview projections (milestone 36), on a
/// 30-minute grain that divides every serving width the API offers. They carry
/// no history of their own: they are born with the file and maintained in the
/// same transaction as every insert and every prune, so SQLite's transaction is
/// the only coherence mechanism there is. There is no backfill path — a file
/// whose projections could disagree with its rows cannot exist.
pub const ddl: [:0]const u8 =
\\CREATE TABLE domains (
\\ id INTEGER PRIMARY KEY,
@@ -78,6 +85,40 @@ pub const ddl: [:0]const u8 =
\\);
\\INSERT INTO querylog_meta (id, created_at, available_since)
\\VALUES (1, unixepoch(), unixepoch() + 1);
\\
\\CREATE TABLE bucket_totals (
\\ bucket INTEGER PRIMARY KEY,
\\ queries INTEGER NOT NULL,
\\ blocked INTEGER NOT NULL,
\\ cached INTEGER NOT NULL,
\\ rt_sum INTEGER NOT NULL, -- sum(response_time_us) over timed rows
\\ rt_count INTEGER NOT NULL -- count(response_time_us)
\\) WITHOUT ROWID;
\\
\\CREATE TABLE bucket_clients (
\\ bucket INTEGER NOT NULL,
\\ client_ip TEXT NOT NULL,
\\ queries INTEGER NOT NULL,
\\ PRIMARY KEY (bucket, client_ip)
\\) WITHOUT ROWID;
\\
\\CREATE TABLE bucket_types (
\\ bucket INTEGER NOT NULL,
\\ qtype INTEGER NOT NULL, -- -1 encodes a NULL qtype, losslessly
\\ count INTEGER NOT NULL,
\\ PRIMARY KEY (bucket, qtype)
\\) WITHOUT ROWID;
\\
\\CREATE TABLE bucket_routes (
\\ bucket INTEGER NOT NULL,
\\ route_kind TEXT NOT NULL,
\\ source_present INTEGER NOT NULL, -- 0: source NULL; 1: source = source_text
\\ source_text TEXT NOT NULL, -- '' when source_present = 0
\\ count INTEGER NOT NULL,
\\ PRIMARY KEY (bucket, route_kind, source_present, source_text),
\\ CHECK (source_present IN (0, 1)),
\\ CHECK (source_present = 1 OR source_text = '')
\\) WITHOUT ROWID;
;
/// The fingerprint of an arbitrary DDL text. `tools/cut.zig` calls this at
@@ -324,13 +365,23 @@ test "ddl creates the query-log tables and every index" {
try database.exec(ddl);
try testing.expectEqual(
@as(i64, 3),
@as(i64, 7),
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
);
// The three explicit indexes plus `domains.domain`'s autoindex, and
// nothing else: the four projection tables are WITHOUT ROWID, so each
// one's PRIMARY KEY *is* its storage rather than a second b-tree to keep
// in step on every insert.
try testing.expectEqual(
@as(i64, 4),
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='index'"),
);
const objects = [_][]const u8{
"domains", "query_log",
"idx_query_log_ts", "idx_query_log_client",
"idx_query_log_domain", "querylog_meta",
"bucket_totals", "bucket_clients",
"bucket_types", "bucket_routes",
};
for (objects) |name| {
var stmt = try database.prepare("SELECT count(*) FROM sqlite_schema WHERE name = ?1");
src/storage/repositories/queries_repo.zig
+1411 -379
View File
File diff suppressed because it is too large Load Diff
src/storage/retention.zig
+1 -1
View File
@@ -251,7 +251,7 @@ fn openLog() !db.Db {
}
fn writeRows(database: *db.Db, timestamps: []const i64) !void {
var writer = try queries_repo.BatchWriter.init(database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, database);
defer writer.deinit();
var rows: [8]queries_repo.Row = undefined;
for (timestamps, rows[0..timestamps.len]) |timestamp, *row| {
src/tests.zig
+1 -1
View File
@@ -106,7 +106,7 @@ comptime {
_ = @import("web/server_integration_test.zig");
_ = @import("server/local_tables.zig");
_ = @import("web/metrics.zig");
_ = @import("web/handlers/stats.zig");
_ = @import("web/handlers/overview.zig");
_ = @import("web/handlers/queries.zig");
_ = @import("web/handlers/diagnostics.zig");
_ = @import("web/handlers/lookup.zig");
src/web/coverage.zig
+3 -4
View File
@@ -6,10 +6,9 @@
//! complete for. Without that fact on the wire a chart draws a pruned week as a
//! week of silence, which is the one reading that is certainly wrong.
//!
//! Three endpoints carry it — `/api/queries`, `/api/stats` and
//! `/api/stats/timeseries` — and they judge it against their own effective
//! lower bound: the client's `since` for the query log, the period's aligned
//! window start for the two stats endpoints.
//! Two endpoints carry it — `/api/queries` and `/api/overview` — and they judge
//! it against their own effective lower bound: the client's `since` for the
//! query log, the period's aligned window start for the overview.
const std = @import("std");
src/web/handlers/overview.zig
+647
View File
@@ -0,0 +1,647 @@
//! `GET /api/overview`: everything the Overview page draws, for one period, in
//! one response.
//!
//! It replaces the five per-panel endpoints milestone 30 shipped. Those cost
//! five scans of every raw row in the window and, being five requests, could
//! only promise a shared *window* — queries logged between two of them moved
//! one panel and not the other. One request over one read transaction promises
//! a shared *snapshot*: the totals, the four breakdowns and the coverage
//! watermark beside them all describe one database state, so the breakdowns sum
//! to the totals for a reason and not by luck.
//!
//! Buckets are aligned to the UTC grid, not to the moment of the request. Every
//! width divides a day, so flooring the current time to a multiple of the width
//! puts each bucket on the same boundary a human reads off a clock, and two
//! requests a second apart return the same bucket starts. The last bucket is
//! the one in progress; it fills as the period runs.
//!
//! The aggregate runs on the web task's own query-log connection (m7 ruling
//! 21), which every connection task shares. SQLite's serialized mode makes one
//! call safe; it does not make a transaction safe, so `WebState.querylog_lock`
//! covers the whole read and a second BEGIN can never land inside the first.
//! The transaction is deferred, not `db.Tx`'s BEGIN IMMEDIATE, which would
//! stall the logger and retention behind an HTTP response.
//!
//! The lock is released before the response is written: the body is already
//! built in the request arena, and holding a database lock across a socket
//! write would let one slow client serialize every other reader.
const std = @import("std");
const coverage = @import("../coverage.zig");
const db = @import("../../storage/db.zig");
const http_util = @import("../http_util.zig");
const queries_repo = @import("../../storage/repositories/queries_repo.zig");
const server = @import("../server.zig");
const log = std.log.scoped(.web_overview);
/// The four periods ruling 13 defines. The tag names are the wire spellings.
pub const Period = enum {
@"1h",
@"24h",
@"7d",
@"30d",
pub fn parse(text: []const u8) ?Period {
return std.meta.stringToEnum(Period, text);
}
/// Ruling 13: 1h→60×1m, 24h→48×30m, 7d→168×1h, 30d→120×6h.
pub fn bucketSeconds(self: Period) u32 {
return switch (self) {
.@"1h" => 60,
.@"24h" => 30 * 60,
.@"7d" => 60 * 60,
.@"30d" => 6 * 60 * 60,
};
}
pub fn bucketCount(self: Period) u32 {
return switch (self) {
.@"1h" => 60,
.@"24h" => 48,
.@"7d" => 168,
.@"30d" => 120,
};
}
pub fn label(self: Period) []const u8 {
return @tagName(self);
}
};
pub const default_period: Period = .@"24h";
/// The widest period's bucket count. Nothing here allocates by it any more —
/// the repository returns arena slices — but it is the bound the response size
/// argument rests on, and the assertion below is what keeps it true.
pub const max_buckets = 168;
comptime {
std.debug.assert(std.enums.values(Period).len == server.OverviewCache.slot_count);
for (std.enums.values(Period)) |period| {
std.debug.assert(period.bucketCount() <= max_buckets);
// The UTC alignment argument holds only while every width divides a day.
std.debug.assert(86_400 % period.bucketSeconds() == 0);
}
}
pub const Window = struct {
/// Inclusive, on the bucket grid.
since: i64,
/// Exclusive: the end of the bucket that `now` falls in.
until: i64,
bucket_seconds: u32,
bucket_count: u32,
};
pub fn window(period: Period, now_unix: i64) Window {
const width: i64 = period.bucketSeconds();
const count: i64 = period.bucketCount();
const until = @divFloor(now_unix, width) * width + width;
return .{
.since = until - width * count,
.until = until,
.bucket_seconds = period.bucketSeconds(),
.bucket_count = period.bucketCount(),
};
}
pub const Totals = struct {
queries: u64,
blocked: u64,
/// Distinct client addresses in the window.
clients: u64,
avg_response_time_us: ?i64,
};
pub const Body = struct {
period: []const u8,
since: i64,
until: i64,
bucket_seconds: u32,
totals: Totals,
buckets: []const queries_repo.Bucket,
clients: []const queries_repo.ClientSeries,
other: []const u64,
types: []const queries_repo.TypeCount,
routes: []const queries_repo.RouteCount,
/// Judged against `since`, which is the window this body reports on — so a
/// dashboard can say "history starts here" instead of charting a pruned
/// stretch as a quiet one.
coverage: coverage.Coverage,
};
pub fn handle(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const body = cachedBody(state, io, database, request.arena, period, span) catch |err| {
return internal(request, err);
};
return http_util.respondBytes(request, .ok, body, http_util.content_type_json, &.{});
}
/// The whole cache decision, start to finish, under one hold of
/// `querylog_lock`. Returns bytes owned by `arena`, so the caller writes the
/// socket with the lock already released.
///
/// `data_version` is sampled inside the lock and the rebuild is published under
/// that same sample: a commit landing on another connection while this task
/// builds moves the pragma, so the entry it installs is keyed to a version the
/// next request will not ask for and that request rebuilds. Stale bytes under a
/// current key are therefore not reachable. A failed build or a failed commit
/// publishes nothing and leaves whatever the slot already held.
fn cachedBody(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
period: Period,
span: Window,
) db.Error![]const u8 {
state.querylog_lock.lockUncancelable(io);
defer state.querylog_lock.unlock(io);
const index = @intFromEnum(period);
const data_version = try database.queryInt("PRAGMA data_version");
if (state.overview_cache.get(index, span.until, data_version)) |cached| {
// The copy is what makes a later rebuild's free-and-replace safe: the
// response is written after the lock is gone, and by then these bytes
// may belong to nobody.
return arena.dupe(u8, cached);
}
const body = try buildBody(state, io, database, arena, period, span);
const owned = try state.gpa.dupe(u8, body);
state.overview_cache.put(state.gpa, index, span.until, data_version, owned);
return body;
}
/// One read transaction, one snapshot, one serialized body in `arena`. The
/// caller holds `querylog_lock` and keeps holding it.
fn buildBody(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
period: Period,
span: Window,
) db.Error![]const u8 {
var scope = try server.QuerylogRead.openLocked(state, io, database);
errdefer scope.abort();
const data = try queries_repo.overview(
database,
arena,
span.since,
span.bucket_seconds,
span.bucket_count,
);
const window_coverage = try coverage.read(database, span.since);
try scope.commit();
var allocating: std.Io.Writer.Allocating = .init(arena);
errdefer allocating.deinit();
std.json.Stringify.value(Body{
.period = period.label(),
.since = span.since,
.until = span.until,
.bucket_seconds = span.bucket_seconds,
.totals = .{
.queries = data.totals.queries,
.blocked = data.totals.blocked,
.clients = data.totals.distinct_clients,
.avg_response_time_us = data.totals.avg_response_time_us,
},
.buckets = data.buckets,
.clients = data.clients.clients,
.other = data.clients.other,
.types = data.types,
.routes = data.routes,
.coverage = window_coverage,
}, .{}, &allocating.writer) catch return error.OutOfMemory;
return allocating.written();
}
pub const PeriodError = error{BadPeriod};
/// An absent `period` is the default; anything else it cannot read is a 400,
/// never a silent fallback — a typo must not return a window nobody asked for.
fn periodParam(query: []const u8) PeriodError!Period {
var buf: [8]u8 = undefined;
const found = http_util.queryValue(query, "period", &buf) catch return error.BadPeriod;
const text = found orelse return default_period;
return Period.parse(text) orelse error.BadPeriod;
}
fn badPeriod(request: *http_util.Request) http_util.HandlerError!void {
return http_util.respondError(request, .bad_request, "period must be one of 1h, 24h, 7d, 30d");
}
fn unavailable(request: *http_util.Request) http_util.HandlerError!void {
return http_util.respondError(request, .service_unavailable, "query log unavailable");
}
/// The one thing this file logs. A failed aggregate is a fault in the box, not
/// a property of the request, and the client is told nothing beyond "internal
/// error" (ruling 8, PLAN §19).
fn internal(request: *http_util.Request, err: db.Error) http_util.HandlerError!void {
log.warn("overview failed: {s}", .{@errorName(err)});
return http_util.respondError(request, .internal_server_error, "internal error");
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const querylog_schema = @import("../../storage/querylog_schema.zig");
const testing = std.testing;
test "the period grammar accepts exactly the four spellings" {
try testing.expectEqual(Period.@"1h", Period.parse("1h").?);
try testing.expectEqual(Period.@"24h", Period.parse("24h").?);
try testing.expectEqual(Period.@"7d", Period.parse("7d").?);
try testing.expectEqual(Period.@"30d", Period.parse("30d").?);
try testing.expectEqual(@as(?Period, null), Period.parse("12h"));
try testing.expectEqual(@as(?Period, null), Period.parse("1H"));
try testing.expectEqual(@as(?Period, null), Period.parse(""));
}
test "an absent period defaults and a bad one is rejected" {
try testing.expectEqual(default_period, try periodParam(""));
try testing.expectEqual(default_period, try periodParam("limit=5"));
try testing.expectEqual(Period.@"7d", try periodParam("period=7d"));
try testing.expectError(error.BadPeriod, periodParam("period=12h"));
try testing.expectError(error.BadPeriod, periodParam("period=%2"));
// Longer than any spelling: rejected rather than truncated to "1h".
try testing.expectError(error.BadPeriod, periodParam("period=1hhhhhhhhhh"));
}
test "each period spans its own bucket width times its count" {
for (std.enums.values(Period)) |period| {
const span = window(period, 1_700_000_000);
const width: i64 = period.bucketSeconds();
try testing.expectEqual(width * @as(i64, period.bucketCount()), span.until - span.since);
}
}
test "the window sits on the UTC grid and ends with the bucket in progress" {
// 2023-11-14T22:13:20Z, which is not on any bucket boundary.
const now: i64 = 1_700_000_000;
const span = window(.@"24h", now);
try testing.expectEqual(@as(i64, 0), @rem(span.since, 1800));
try testing.expectEqual(@as(i64, 0), @rem(span.until, 1800));
try testing.expect(span.until > now);
try testing.expect(span.until - now <= 1800);
try testing.expectEqual(@as(u32, 48), span.bucket_count);
}
test "two requests inside one bucket see the same window" {
// A bucket boundary, so the offsets below stay inside one minute.
const boundary: i64 = 1_700_000_000 - @rem(1_700_000_000, 60);
const first = window(.@"1h", boundary);
const second = window(.@"1h", boundary + 59);
try testing.expectEqual(first.since, second.since);
try testing.expectEqual(first.until, second.until);
const next = window(.@"1h", boundary + 60);
try testing.expectEqual(first.until + 60, next.until);
}
test "a timestamp exactly on a boundary starts a new bucket" {
const span = window(.@"7d", 1_700_000_000 - 1_700_000_000 % 3600);
try testing.expectEqual(@as(i64, 0), @rem(span.since, 3600));
try testing.expectEqual(@as(u32, 168), span.bucket_count);
}
test "every period's window is a window the repository will serve" {
// The projection path needs the window start on the 30-minute grid and the
// width a whole number of grains; `window` is the only producer of either.
for (std.enums.values(Period)) |period| {
const span = window(period, 1_700_000_123);
if (span.bucket_seconds < 1800) continue;
try testing.expectEqual(@as(i64, 0), @mod(span.since, 1800));
try testing.expectEqual(@as(u32, 0), span.bucket_seconds % 1800);
}
}
fn openLog() !db.Db {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer database.close();
try db.applyPragmas(&database, .{});
try database.exec(querylog_schema.ddl);
return database;
}
fn writeRow(writer: *queries_repo.BatchWriter, timestamp: i64, blocked: bool, cached: ?bool) !void {
const rows = [_]queries_repo.Row{.{
.timestamp = timestamp,
.domain = "example.com",
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 0,
.blocked = blocked,
.response_time_us = 1000,
.cache_hit = cached,
.upstream = null,
.group_id = 1,
.group_name = "default",
.policy_action = if (blocked) .block else .allow,
.policy_reason = if (blocked) .blocklist_domain else .no_match,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = if (blocked) .blocked else .upstream,
.forward_zone = null,
}};
try writer.writeBatch(&rows);
}
test "one overview answers totals and buckets that agree over the same window" {
var database = try openLog();
defer database.close();
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena_state.deinit();
const span = window(.@"1h", 1_700_000_000);
var writer = try queries_repo.BatchWriter.init(testing.allocator, &database);
defer writer.deinit();
// One row in the first bucket, two in the last, one just outside.
try writeRow(&writer, span.since, false, false);
try writeRow(&writer, span.until - 1, true, false);
try writeRow(&writer, span.until - 2, false, true);
try writeRow(&writer, span.since - 1, false, false);
const data = try queries_repo.overview(
&database,
arena_state.allocator(),
span.since,
span.bucket_seconds,
span.bucket_count,
);
try testing.expectEqual(@as(u64, 3), data.totals.queries);
try testing.expectEqual(@as(u64, 1), data.totals.blocked);
try testing.expectEqual(@as(u64, 1), data.totals.distinct_clients);
try testing.expectEqual(@as(?i64, 1000), data.totals.avg_response_time_us);
try testing.expectEqual(@as(usize, 60), data.buckets.len);
var summed: u64 = 0;
var blocked: u64 = 0;
for (data.buckets) |bucket| {
summed += bucket.queries;
blocked += bucket.blocked;
}
try testing.expectEqual(data.totals.queries, summed);
try testing.expectEqual(data.totals.blocked, blocked);
try testing.expectEqual(span.since, data.buckets[0].ts);
try testing.expectEqual(@as(u64, 1), data.buckets[0].queries);
try testing.expectEqual(@as(u64, 2), data.buckets[59].queries);
try testing.expectEqual(span.until - span.bucket_seconds, data.buckets[59].ts);
}
test "an empty window reports zeros with a null mean" {
var database = try openLog();
defer database.close();
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena_state.deinit();
const span = window(.@"30d", 1_700_000_000);
const data = try queries_repo.overview(
&database,
arena_state.allocator(),
span.since,
span.bucket_seconds,
span.bucket_count,
);
try testing.expectEqual(@as(u64, 0), data.totals.queries);
try testing.expectEqual(@as(?i64, null), data.totals.avg_response_time_us);
try testing.expectEqual(@as(usize, 120), data.buckets.len);
for (data.buckets) |bucket| try testing.expectEqual(@as(u64, 0), bucket.queries);
// `other` is bucket-count sized even here: a chart must never have to
// invent the residual series.
try testing.expectEqual(@as(usize, 120), data.clients.other.len);
}
test "the cache serves one period's bytes and rebuilds when the key moves" {
const gpa = testing.allocator;
var cache: server.OverviewCache = .{};
defer cache.deinit(gpa);
try testing.expectEqual(@as(?[]const u8, null), cache.get(0, 100, 7));
cache.put(gpa, 0, 100, 7, try gpa.dupe(u8, "first"));
try testing.expectEqualStrings("first", cache.get(0, 100, 7).?);
// A different period, a rolled window and a bumped data version are three
// different keys, and none of them hits.
try testing.expectEqual(@as(?[]const u8, null), cache.get(1, 100, 7));
try testing.expectEqual(@as(?[]const u8, null), cache.get(0, 101, 7));
try testing.expectEqual(@as(?[]const u8, null), cache.get(0, 100, 8));
// A rebuild replaces the entry and frees the old body; the leak checker in
// `testing.allocator` is the assertion.
cache.put(gpa, 0, 100, 8, try gpa.dupe(u8, "second"));
try testing.expectEqualStrings("second", cache.get(0, 100, 8).?);
}
/// Two connections onto one file, which is the only arrangement in which
/// `PRAGMA data_version` moves at all: it reports commits by *other*
/// connections, so an in-memory database — where there is no other connection —
/// could never witness the invalidation these tests are about.
const CacheFixture = struct {
threaded: std.Io.Threaded,
tmp: std.testing.TmpDir,
/// The web task's connection, the one the cache is keyed on.
reader: db.Db,
/// Stands in for the logger and for retention.
writer: db.Db,
state: server.WebState,
arena_state: std.heap.ArenaAllocator,
fn init(self: *CacheFixture, gpa: std.mem.Allocator) !void {
self.threaded = .init(gpa, .{});
errdefer self.threaded.deinit();
self.tmp = std.testing.tmpDir(.{});
errdefer self.tmp.cleanup();
var path_buf: [256]u8 = undefined;
const path = try std.fmt.bufPrintZ(
&path_buf,
".zig-cache/tmp/{s}/querylog.db",
.{self.tmp.sub_path},
);
self.writer = try db.Db.open(path, .{ .mode = .read_write_create });
errdefer self.writer.close();
try db.applyPragmas(&self.writer, .{});
try self.writer.exec(querylog_schema.ddl);
// The DDL stamps `created_at` from the wall clock, and the watermark
// with it. These tests work over a fixed 2023 window, so a 2026
// watermark would report every one of them as uncovered and the prune
// below would not move it.
try self.writer.exec(
"UPDATE querylog_meta SET created_at = 1600000000, available_since = 1600000000 WHERE id = 1",
);
self.reader = try db.Db.open(path, .{ .mode = .read_write_existing });
errdefer self.reader.close();
try db.applyPragmas(&self.reader, .{});
self.state = .{ .gpa = gpa, .querylog_db = &self.reader };
self.arena_state = .init(gpa);
}
fn deinit(self: *CacheFixture) void {
self.arena_state.deinit();
self.state.overview_cache.deinit(self.state.gpa);
self.reader.close();
self.writer.close();
self.tmp.cleanup();
self.threaded.deinit();
}
fn io(self: *CacheFixture) std.Io {
return self.threaded.io();
}
fn body(self: *CacheFixture, period: Period, span: Window) db.Error![]const u8 {
return cachedBody(
&self.state,
self.io(),
&self.reader,
self.arena_state.allocator(),
period,
span,
);
}
/// One row through the writer connection, which commits and so moves the
/// reader's `PRAGMA data_version`.
fn log(self: *CacheFixture, timestamp: i64) !void {
var batch = try queries_repo.BatchWriter.init(self.state.gpa, &self.writer);
defer batch.deinit();
try writeRow(&batch, timestamp, false, false);
}
};
test "a cache hit answers without opening a read transaction" {
var fx: CacheFixture = undefined;
try fx.init(testing.allocator);
defer fx.deinit();
const span = window(.@"1h", 1_700_000_000);
try fx.log(span.since + 10);
const first = try fx.body(.@"1h", span);
try testing.expect(first.len > 0);
// A hit never reaches the database, so a fault armed on the next commit is
// never spent — and the bytes are the stored ones, not a rebuild's.
db.read_tx_faults.failNextCommit();
const second = try fx.body(.@"1h", span);
try testing.expectEqualStrings(first, second);
try testing.expect(first.ptr != second.ptr);
// Spend the armed fault so it cannot leak into a later test. A miss does
// reach the database, so this one trips.
db.read_tx_faults.beginCapture();
defer _ = db.read_tx_faults.endCapture();
try testing.expectError(error.Internal, fx.body(.@"24h", window(.@"24h", 1_700_000_000)));
}
test "a commit on another connection invalidates the cached body" {
var fx: CacheFixture = undefined;
try fx.init(testing.allocator);
defer fx.deinit();
const span = window(.@"1h", 1_700_000_000);
try fx.log(span.since + 10);
const before = try fx.body(.@"1h", span);
try fx.log(span.since + 20);
const after = try fx.body(.@"1h", span);
try testing.expect(!std.mem.eql(u8, before, after));
try testing.expect(std.mem.containsAtLeast(u8, after, 1, "\"queries\":2"));
}
test "a retention prune through another connection replaces the body and the watermark" {
var fx: CacheFixture = undefined;
try fx.init(testing.allocator);
defer fx.deinit();
const span = window(.@"1h", 1_700_000_000);
try fx.log(span.since + 10);
const before = try fx.body(.@"1h", span);
try testing.expect(std.mem.containsAtLeast(u8, before, 1, "\"queries\":1"));
// Past the whole window: the row goes and the watermark advances, and both
// halves of the response must move together.
_ = try queries_repo.pruneOlderThan(&fx.writer, span.until);
const after = try fx.body(.@"1h", span);
try testing.expect(std.mem.containsAtLeast(u8, after, 1, "\"queries\":0"));
var watermark_buf: [64]u8 = undefined;
const watermark = try std.fmt.bufPrint(
&watermark_buf,
"\"available_since\":{d}",
.{span.until},
);
try testing.expect(std.mem.containsAtLeast(u8, after, 1, watermark));
}
test "a window roll rebuilds even with the data unchanged" {
var fx: CacheFixture = undefined;
try fx.init(testing.allocator);
defer fx.deinit();
const now: i64 = 1_700_000_000;
const first = try fx.body(.@"1h", window(.@"1h", now));
// One bucket later: same data, a different window, and so a different body.
const rolled = try fx.body(.@"1h", window(.@"1h", now + 60));
try testing.expect(!std.mem.eql(u8, first, rolled));
// The slot now holds the rolled window; asking for the earlier one again
// rebuilds rather than answering from a key that no longer matches.
const again = try fx.body(.@"1h", window(.@"1h", now));
try testing.expectEqualStrings(first, again);
}
test "a failed commit installs nothing and leaves the stored entry alone" {
var fx: CacheFixture = undefined;
try fx.init(testing.allocator);
defer fx.deinit();
const span = window(.@"1h", 1_700_000_000);
try fx.log(span.since + 10);
const stored = try fx.body(.@"1h", span);
// A commit that fails on a rebuild: the key has moved, so this is a miss.
try fx.log(span.since + 20);
db.read_tx_faults.failNextCommit();
db.read_tx_faults.beginCapture();
try testing.expectError(error.Internal, fx.body(.@"1h", span));
try testing.expectEqual(@as(usize, 1), db.read_tx_faults.endCapture());
// Nothing was published under the new key: the next request rebuilds and
// sees the second row, rather than being served the failed read's work or
// the first row's body under a key that now describes two.
const rebuilt = try fx.body(.@"1h", span);
try testing.expect(!std.mem.eql(u8, stored, rebuilt));
try testing.expect(std.mem.containsAtLeast(u8, rebuilt, 1, "\"queries\":2"));
}
src/web/handlers/queries.zig
+1 -1
View File
@@ -281,7 +281,7 @@ fn openLog() !db.Db {
/// upstream answer carries one. A fixture that broke those ties would let a
/// serializer regression pass here and fail on real rows.
fn seed(database: *db.Db, count: usize) !void {
var writer = try queries_repo.BatchWriter.init(database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, database);
defer writer.deinit();
var rows: [16]queries_repo.Row = undefined;
for (rows[0..count], 0..) |*row, i| {
src/web/handlers/stats.zig
-574
View File
@@ -1,574 +0,0 @@
//! The five period endpoints: `GET /api/stats` and `/api/stats/timeseries`
//! (ruling 13), and `/api/stats/types`, `/api/stats/routes` and
//! `/api/stats/clients` (milestone 30).
//!
//! One period grammar, four widths, and one window shared by all five: a
//! request for the same period gets the same `since`/`until` from every
//! endpoint, so the totals describe exactly the span the charts draw rather
//! than a neighbouring one.
//!
//! That is window coherence, not identical counts. Each endpoint is its own
//! request against its own snapshot, so queries logged between two of them move
//! one panel and not the other. Only a box with nothing writing to it — a test
//! — can expect the breakdowns to sum to the totals exactly.
//!
//! Buckets are aligned to the UTC grid, not to the moment of the request. Every
//! width divides a day, so flooring the current time to a multiple of the width
//! puts each bucket on the same boundary a human reads off a clock, and two
//! requests a second apart return the same bucket starts. The last bucket is
//! the one in progress; it fills as the period runs.
//!
//! The aggregates run on the web task's own query-log connection (m7 ruling 21),
//! which every connection task shares. SQLite's serialized mode makes one call
//! safe; it does not make a transaction safe, so `WebState.querylog_lock` covers
//! the whole read and a second BEGIN can never land inside the first. Each
//! response takes one deferred read transaction, so its aggregate and the
//! `coverage` beside it describe one database state: retention cannot prune
//! between them and hand a client pre-prune rows tagged with a post-prune
//! watermark. Deferred, not `db.Tx`'s BEGIN IMMEDIATE, which would stall the
//! logger and retention behind an HTTP response.
//!
//! The lock is released before the response is written: the body is already
//! built in the request arena, and holding a database lock across a socket
//! write would let one slow client serialize every other reader.
const std = @import("std");
const coverage = @import("../coverage.zig");
const db = @import("../../storage/db.zig");
const http_util = @import("../http_util.zig");
const queries_repo = @import("../../storage/repositories/queries_repo.zig");
const server = @import("../server.zig");
const log = std.log.scoped(.web_stats);
/// The four periods ruling 13 defines. The tag names are the wire spellings.
pub const Period = enum {
@"1h",
@"24h",
@"7d",
@"30d",
pub fn parse(text: []const u8) ?Period {
return std.meta.stringToEnum(Period, text);
}
/// Ruling 13: 1h→60×1m, 24h→48×30m, 7d→168×1h, 30d→120×6h.
pub fn bucketSeconds(self: Period) u32 {
return switch (self) {
.@"1h" => 60,
.@"24h" => 30 * 60,
.@"7d" => 60 * 60,
.@"30d" => 6 * 60 * 60,
};
}
pub fn bucketCount(self: Period) u32 {
return switch (self) {
.@"1h" => 60,
.@"24h" => 48,
.@"7d" => 168,
.@"30d" => 120,
};
}
pub fn label(self: Period) []const u8 {
return @tagName(self);
}
};
pub const default_period: Period = .@"24h";
/// The widest period's bucket count, so one stack array serves every request.
pub const max_buckets = 168;
comptime {
for (std.enums.values(Period)) |period| {
std.debug.assert(period.bucketCount() <= max_buckets);
// The UTC alignment argument holds only while every width divides a day.
std.debug.assert(86_400 % period.bucketSeconds() == 0);
}
}
pub const Window = struct {
/// Inclusive, on the bucket grid.
since: i64,
/// Exclusive: the end of the bucket that `now` falls in.
until: i64,
bucket_seconds: u32,
bucket_count: u32,
};
pub fn window(period: Period, now_unix: i64) Window {
const width: i64 = period.bucketSeconds();
const count: i64 = period.bucketCount();
const until = @divFloor(now_unix, width) * width + width;
return .{
.since = until - width * count,
.until = until,
.bucket_seconds = period.bucketSeconds(),
.bucket_count = period.bucketCount(),
};
}
pub const TotalsBody = struct {
period: []const u8,
since: i64,
until: i64,
queries: u64,
blocked: u64,
clients: u64,
avg_response_time_us: ?i64,
/// Judged against `since`, which is the window this body reports on — so a
/// dashboard can say "history starts here" instead of charting a pruned
/// stretch as a quiet one.
coverage: coverage.Coverage,
};
pub const TimeseriesBody = struct {
period: []const u8,
since: i64,
until: i64,
bucket_seconds: u32,
buckets: []const queries_repo.Bucket,
coverage: coverage.Coverage,
};
pub const TypesBody = struct {
period: []const u8,
since: i64,
until: i64,
types: []const queries_repo.TypeCount,
coverage: coverage.Coverage,
};
pub const RoutesBody = struct {
period: []const u8,
since: i64,
until: i64,
routes: []const queries_repo.RouteCount,
coverage: coverage.Coverage,
};
pub const ClientsBody = struct {
period: []const u8,
since: i64,
until: i64,
bucket_seconds: u32,
clients: []const queries_repo.ClientSeries,
other: []const u64,
coverage: coverage.Coverage,
};
/// Everything one response reads from the query log, so the caller can end the
/// transaction and drop the lock before it serializes anything.
fn Read(comptime T: type) type {
return struct {
data: T,
coverage: coverage.Coverage,
};
}
const ReadScope = server.QuerylogRead;
fn readTotals(
state: *server.WebState,
io: std.Io,
database: *db.Db,
span: Window,
) db.Error!Read(queries_repo.StatsTotals) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const read: Read(queries_repo.StatsTotals) = .{
.data = try queries_repo.statsTotals(database, span.since, span.until),
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readTimeseries(
state: *server.WebState,
io: std.Io,
database: *db.Db,
span: Window,
out: []queries_repo.Bucket,
) db.Error!Read(usize) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const read: Read(usize) = .{
.data = try queries_repo.timeseries(database, span.since, span.bucket_seconds, out),
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readTypes(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
span: Window,
) db.Error!Read([]const queries_repo.TypeCount) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const list = try queries_repo.statsTypes(database, arena, span.since, span.until);
const read: Read([]const queries_repo.TypeCount) = .{
.data = list.items,
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readRoutes(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
span: Window,
) db.Error!Read([]const queries_repo.RouteCount) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const list = try queries_repo.statsRoutes(database, arena, span.since, span.until);
const read: Read([]const queries_repo.RouteCount) = .{
.data = list.items,
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readClients(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
span: Window,
) db.Error!Read(queries_repo.ClientsBreakdown) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const read: Read(queries_repo.ClientsBreakdown) = .{
.data = try queries_repo.statsClients(
database,
arena,
span.since,
span.bucket_seconds,
span.bucket_count,
),
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
pub fn totals(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readTotals(state, io, database, span) catch |err| {
return internal(request, "stats totals", err);
};
return http_util.respondJson(request, .ok, TotalsBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.queries = read.data.queries,
.blocked = read.data.blocked,
.clients = read.data.distinct_clients,
.avg_response_time_us = read.data.avg_response_time_us,
.coverage = read.coverage,
}, &.{});
}
pub fn timeseries(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
var buckets: [max_buckets]queries_repo.Bucket = undefined;
const out = buckets[0..span.bucket_count];
const read = readTimeseries(state, io, database, span, out) catch |err| {
return internal(request, "stats timeseries", err);
};
return http_util.respondJson(request, .ok, TimeseriesBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.bucket_seconds = span.bucket_seconds,
.buckets = out[0..read.data],
.coverage = read.coverage,
}, &.{});
}
pub fn types(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readTypes(state, io, database, request.arena, span) catch |err| {
return internal(request, "stats types", err);
};
return http_util.respondJson(request, .ok, TypesBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.types = read.data,
.coverage = read.coverage,
}, &.{});
}
pub fn routes(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readRoutes(state, io, database, request.arena, span) catch |err| {
return internal(request, "stats routes", err);
};
return http_util.respondJson(request, .ok, RoutesBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.routes = read.data,
.coverage = read.coverage,
}, &.{});
}
pub fn clients(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readClients(state, io, database, request.arena, span) catch |err| {
return internal(request, "stats clients", err);
};
return http_util.respondJson(request, .ok, ClientsBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.bucket_seconds = span.bucket_seconds,
.clients = read.data.clients,
.other = read.data.other,
.coverage = read.coverage,
}, &.{});
}
pub const PeriodError = error{BadPeriod};
/// An absent `period` is the default; anything else it cannot read is a 400,
/// never a silent fallback — a typo must not return a window nobody asked for.
fn periodParam(query: []const u8) PeriodError!Period {
var buf: [8]u8 = undefined;
const found = http_util.queryValue(query, "period", &buf) catch return error.BadPeriod;
const text = found orelse return default_period;
return Period.parse(text) orelse error.BadPeriod;
}
fn badPeriod(request: *http_util.Request) http_util.HandlerError!void {
return http_util.respondError(request, .bad_request, "period must be one of 1h, 24h, 7d, 30d");
}
fn unavailable(request: *http_util.Request) http_util.HandlerError!void {
return http_util.respondError(request, .service_unavailable, "query log unavailable");
}
/// The one thing this file logs. A failed aggregate is a fault in the box, not
/// a property of the request, and the client is told nothing beyond "internal
/// error" (ruling 8, PLAN §19).
fn internal(
request: *http_util.Request,
what: []const u8,
err: db.Error,
) http_util.HandlerError!void {
log.warn("{s} failed: {s}", .{ what, @errorName(err) });
return http_util.respondError(request, .internal_server_error, "internal error");
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const querylog_schema = @import("../../storage/querylog_schema.zig");
const testing = std.testing;
test "the period grammar accepts exactly the four spellings" {
try testing.expectEqual(Period.@"1h", Period.parse("1h").?);
try testing.expectEqual(Period.@"24h", Period.parse("24h").?);
try testing.expectEqual(Period.@"7d", Period.parse("7d").?);
try testing.expectEqual(Period.@"30d", Period.parse("30d").?);
try testing.expectEqual(@as(?Period, null), Period.parse("12h"));
try testing.expectEqual(@as(?Period, null), Period.parse("1H"));
try testing.expectEqual(@as(?Period, null), Period.parse(""));
}
test "an absent period defaults and a bad one is rejected" {
try testing.expectEqual(default_period, try periodParam(""));
try testing.expectEqual(default_period, try periodParam("limit=5"));
try testing.expectEqual(Period.@"7d", try periodParam("period=7d"));
try testing.expectError(error.BadPeriod, periodParam("period=12h"));
try testing.expectError(error.BadPeriod, periodParam("period=%2"));
// Longer than any spelling: rejected rather than truncated to "1h".
try testing.expectError(error.BadPeriod, periodParam("period=1hhhhhhhhhh"));
}
test "each period spans its own bucket width times its count" {
for (std.enums.values(Period)) |period| {
const span = window(period, 1_700_000_000);
const width: i64 = period.bucketSeconds();
try testing.expectEqual(width * @as(i64, period.bucketCount()), span.until - span.since);
}
}
test "the window sits on the UTC grid and ends with the bucket in progress" {
// 2023-11-14T22:13:20Z, which is not on any bucket boundary.
const now: i64 = 1_700_000_000;
const span = window(.@"24h", now);
try testing.expectEqual(@as(i64, 0), @rem(span.since, 1800));
try testing.expectEqual(@as(i64, 0), @rem(span.until, 1800));
try testing.expect(span.until > now);
try testing.expect(span.until - now <= 1800);
try testing.expectEqual(@as(u32, 48), span.bucket_count);
}
test "two requests inside one bucket see the same window" {
// A bucket boundary, so the offsets below stay inside one minute.
const boundary: i64 = 1_700_000_000 - @rem(1_700_000_000, 60);
const first = window(.@"1h", boundary);
const second = window(.@"1h", boundary + 59);
try testing.expectEqual(first.since, second.since);
try testing.expectEqual(first.until, second.until);
const next = window(.@"1h", boundary + 60);
try testing.expectEqual(first.until + 60, next.until);
}
test "a timestamp exactly on a boundary starts a new bucket" {
const span = window(.@"7d", 1_700_000_000 - 1_700_000_000 % 3600);
try testing.expectEqual(@as(i64, 0), @rem(span.since, 3600));
try testing.expectEqual(@as(u32, 168), span.bucket_count);
}
fn openLog() !db.Db {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer database.close();
try db.applyPragmas(&database, .{});
try database.exec(querylog_schema.ddl);
return database;
}
fn writeRow(writer: *queries_repo.BatchWriter, timestamp: i64, blocked: bool, cached: ?bool) !void {
const rows = [_]queries_repo.Row{.{
.timestamp = timestamp,
.domain = "example.com",
.client_ip = "192.0.2.10",
.qtype = 1,
.qclass = 1,
.rcode = 0,
.blocked = blocked,
.response_time_us = 1000,
.cache_hit = cached,
.upstream = null,
.group_id = 1,
.group_name = "default",
.policy_action = if (blocked) .block else .allow,
.policy_reason = if (blocked) .blocklist_domain else .no_match,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = if (blocked) .blocked else .upstream,
.forward_zone = null,
}};
try writer.writeBatch(&rows);
}
test "the totals and the buckets agree over the same window" {
var database = try openLog();
defer database.close();
const now: i64 = 1_700_000_000;
const span = window(.@"1h", now);
var writer = try queries_repo.BatchWriter.init(&database);
defer writer.deinit();
// One row in the first bucket, two in the last, one just outside.
try writeRow(&writer, span.since, false, false);
try writeRow(&writer, span.until - 1, true, false);
try writeRow(&writer, span.until - 2, false, true);
try writeRow(&writer, span.since - 1, false, false);
const result = try queries_repo.statsTotals(&database, span.since, span.until);
try testing.expectEqual(@as(u64, 3), result.queries);
try testing.expectEqual(@as(u64, 1), result.blocked);
try testing.expectEqual(@as(u64, 1), result.distinct_clients);
try testing.expectEqual(@as(?i64, 1000), result.avg_response_time_us);
var buckets: [max_buckets]queries_repo.Bucket = undefined;
const out = buckets[0..span.bucket_count];
const written = try queries_repo.timeseries(&database, span.since, span.bucket_seconds, out);
try testing.expectEqual(@as(usize, 60), written);
var summed: u64 = 0;
var blocked: u64 = 0;
for (out) |bucket| {
summed += bucket.queries;
blocked += bucket.blocked;
}
try testing.expectEqual(result.queries, summed);
try testing.expectEqual(result.blocked, blocked);
try testing.expectEqual(span.since, out[0].ts);
try testing.expectEqual(@as(u64, 1), out[0].queries);
try testing.expectEqual(@as(u64, 2), out[59].queries);
try testing.expectEqual(span.until - span.bucket_seconds, out[59].ts);
}
test "an empty window reports zeros with a null mean" {
var database = try openLog();
defer database.close();
const span = window(.@"30d", 1_700_000_000);
const result = try queries_repo.statsTotals(&database, span.since, span.until);
try testing.expectEqual(@as(u64, 0), result.queries);
try testing.expectEqual(@as(?i64, null), result.avg_response_time_us);
var buckets: [max_buckets]queries_repo.Bucket = undefined;
const out = buckets[0..span.bucket_count];
try testing.expectEqual(@as(usize, 120), try queries_repo.timeseries(
&database,
span.since,
span.bucket_seconds,
out,
));
for (out) |bucket| try testing.expectEqual(@as(u64, 0), bucket.queries);
}
src/web/openapi.yaml
+70 -201
View File
@@ -422,139 +422,29 @@ paths:
"503":
$ref: "#/components/responses/Unavailable"
/api/stats:
/api/overview:
get:
summary: Totals for a period
parameters:
- $ref: "#/components/parameters/Period"
responses:
"200":
description: Totals over the period's UTC-aligned window.
content:
application/json:
schema:
$ref: "#/components/schemas/StatsTotals"
"400":
$ref: "#/components/responses/BadRequest"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/stats/timeseries:
get:
summary: Bucketed counts for a period
summary: Everything the Overview page draws, for one period
description: |
Fixed-width UTC buckets covering the same window `/api/stats`
reports for the period: 1h into 60 one-minute buckets, 24h into 48
half-hour buckets, 7d into 168 one-hour buckets, 30d into 120
six-hour buckets. Empty buckets are zero-filled.
parameters:
- $ref: "#/components/parameters/Period"
responses:
"200":
description: The bucket series.
content:
application/json:
schema:
$ref: "#/components/schemas/StatsTimeseries"
"400":
$ref: "#/components/responses/BadRequest"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
One response over one read transaction: the period's totals, its
fixed-width UTC buckets, the per-client series, the query-type
breakdown and the answering-route breakdown, plus the coverage
watermark judged against the same window. The panels therefore describe
one database state rather than five, so the breakdowns sum to the
totals on a quiet box.
/api/stats/types:
get:
summary: Query-type breakdown for a period
description: |
How many queries of each DNS type the period's window holds, over the
same UTC-aligned window `/api/stats` reports for. Rows carry the numeric
type only: the type-name table lives in the admin, and a second copy
here would drift out of agreement with it. `qtype` is nullable in the
query log, so the rows that carry no type group into a row of their own
rather than vanishing from a breakdown that claims to add up. Ordered by
count descending, then type ascending with the null row last. Types
absent from the window are absent from the list.
Buckets are UTC-aligned and zero-filled: 1h into 60 one-minute buckets,
24h into 48 half-hour buckets, 7d into 168 one-hour buckets, 30d into
120 six-hour buckets. The last bucket is the one in progress.
parameters:
- $ref: "#/components/parameters/Period"
responses:
"200":
description: The type breakdown.
description: The period's overview.
content:
application/json:
schema:
$ref: "#/components/schemas/StatsTypes"
"400":
$ref: "#/components/responses/BadRequest"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/stats/routes:
get:
summary: How the period's queries were answered
description: |
A breakdown by answering route over the same window `/api/stats`
reports for. `source` is the answering resolver's identity — the
upstream url on `upstream` rows, the zone on `forward_zone` rows, null
on every other kind and on rows whose identity the log did not record.
It is not the blocklist a block came from. Ordered by count descending,
then route ascending, then source ascending with nulls last.
parameters:
- $ref: "#/components/parameters/Period"
responses:
"200":
description: The route breakdown.
content:
application/json:
schema:
$ref: "#/components/schemas/StatsRoutes"
"400":
$ref: "#/components/responses/BadRequest"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/stats/clients:
get:
summary: Per-client bucketed counts for a period
description: |
One zero-filled series per client, bucketed exactly like
`/api/stats/timeseries` so the two charts share an x-axis. The eight
clients with the most queries in the window are named, ranked by count
descending then address ascending; every other client sums into
`other`, which is always present and always holds one entry per bucket
in the window — including when `clients` is empty, when no client fell
outside the named eight, and when the window holds no queries at all.
parameters:
- $ref: "#/components/parameters/Period"
responses:
"200":
description: The per-client series.
content:
application/json:
schema:
$ref: "#/components/schemas/StatsClients"
$ref: "#/components/schemas/Overview"
"400":
$ref: "#/components/responses/BadRequest"
"401":
@@ -2316,31 +2206,6 @@ components:
type: integer
description: How many resolved events the purge removed; zero when there were none.
StatsTotals:
type: object
required: [period, since, until, queries, blocked, clients, avg_response_time_us, coverage]
properties:
period:
type: string
enum: [1h, 24h, 7d, 30d]
since:
type: integer
description: Window start, unix seconds, inclusive.
until:
type: integer
description: Window end, unix seconds, exclusive.
queries: { type: integer }
blocked: { type: integer }
clients:
type: integer
description: Distinct client addresses in the window.
avg_response_time_us:
type: integer
nullable: true
description: Null when no query in the window recorded a time.
coverage:
$ref: "#/components/schemas/Coverage"
Bucket:
type: object
required: [ts, queries, blocked, cached]
@@ -2352,23 +2217,6 @@ components:
blocked: { type: integer }
cached: { type: integer }
StatsTimeseries:
type: object
required: [period, since, until, bucket_seconds, buckets, coverage]
properties:
period:
type: string
enum: [1h, 24h, 7d, 30d]
since: { type: integer }
until: { type: integer }
bucket_seconds: { type: integer }
buckets:
type: array
items:
$ref: "#/components/schemas/Bucket"
coverage:
$ref: "#/components/schemas/Coverage"
TypeCount:
type: object
required: [qtype, count]
@@ -2381,22 +2229,6 @@ components:
recorded no type, not an absent row.
count: { type: integer }
StatsTypes:
type: object
required: [period, since, until, types, coverage]
properties:
period:
type: string
enum: [1h, 24h, 7d, 30d]
since: { type: integer }
until: { type: integer }
types:
type: array
items:
$ref: "#/components/schemas/TypeCount"
coverage:
$ref: "#/components/schemas/Coverage"
RouteCount:
type: object
required: [route, source, count]
@@ -2412,22 +2244,6 @@ components:
row recorded no identity.
count: { type: integer }
StatsRoutes:
type: object
required: [period, since, until, routes, coverage]
properties:
period:
type: string
enum: [1h, 24h, 7d, 30d]
since: { type: integer }
until: { type: integer }
routes:
type: array
items:
$ref: "#/components/schemas/RouteCount"
coverage:
$ref: "#/components/schemas/Coverage"
ClientSeries:
type: object
required: [client, buckets]
@@ -2443,18 +2259,47 @@ components:
items:
type: integer
StatsClients:
OverviewTotals:
type: object
required: [period, since, until, bucket_seconds, clients, other, coverage]
required: [queries, blocked, clients, avg_response_time_us]
properties:
queries: { type: integer }
blocked: { type: integer }
clients:
type: integer
description: Distinct client addresses in the window.
avg_response_time_us:
type: integer
nullable: true
description: Null when no query in the window recorded a time.
Overview:
type: object
required: [period, since, until, bucket_seconds, totals, buckets, clients, other, types, routes, coverage]
properties:
period:
type: string
enum: [1h, 24h, 7d, 30d]
since: { type: integer }
until: { type: integer }
since:
type: integer
description: Window start, unix seconds, inclusive.
until:
type: integer
description: Window end, unix seconds, exclusive.
bucket_seconds: { type: integer }
totals:
$ref: "#/components/schemas/OverviewTotals"
buckets:
type: array
description: One entry per bucket in the window, zero-filled.
items:
$ref: "#/components/schemas/Bucket"
clients:
type: array
description: |
The eight clients with the most queries in the window, ranked by
count descending then address ascending. Every other client sums
into `other`.
items:
$ref: "#/components/schemas/ClientSeries"
other:
@@ -2466,6 +2311,30 @@ components:
eight, and when the window holds no queries at all.
items:
type: integer
types:
type: array
description: |
How many queries of each DNS type the window holds. Rows carry the
numeric type only: the type-name table lives in the admin, and a
second copy here would drift out of agreement with it. `qtype` is
nullable in the query log, so the rows that carry no type group
into a row of their own rather than vanishing from a breakdown that
claims to add up. Ordered by count descending, then type ascending
with the null row last. Types absent from the window are absent
from the list.
items:
$ref: "#/components/schemas/TypeCount"
routes:
type: array
description: |
A breakdown by answering route. `source` is the answering
resolver's identity — the upstream url on `upstream` rows, the zone
on `forward_zone` rows, null on every other kind and on rows whose
identity the log did not record. It is not the blocklist a block
came from. Ordered by count descending, then route ascending, then
source ascending with nulls last.
items:
$ref: "#/components/schemas/RouteCount"
coverage:
$ref: "#/components/schemas/Coverage"
src/web/routes.zig
+4 -8
View File
@@ -45,11 +45,11 @@ const local = @import("handlers/local.zig");
const lookup = @import("handlers/lookup.zig");
const metrics = @import("metrics.zig");
const openapi = @import("openapi.zig");
const overview = @import("handlers/overview.zig");
const pause = @import("handlers/pause.zig");
const queries = @import("handlers/queries.zig");
const rules = @import("handlers/rules.zig");
const settings = @import("handlers/settings.zig");
const stats = @import("handlers/stats.zig");
const upstreams = @import("handlers/upstreams.zig");
const version = @import("handlers/version.zig");
@@ -64,18 +64,14 @@ pub const table: []const router.RouteInfo = &.{
.{ .method = .POST, .pattern = "/api/auth/login", .auth = .open, .policy = .runtime_action, .handler = auth.login },
.{ .method = .POST, .pattern = "/api/auth/logout", .auth = .session, .policy = .runtime_action, .handler = auth.logout },
// Query log, stats, live stream, lookup.
// Query log, overview, live stream, lookup.
.{ .method = .GET, .pattern = "/api/queries", .auth = .session, .policy = .read, .handler = queries.list },
.{ .method = .GET, .pattern = "/api/queries/live", .auth = .session, .policy = .read, .handler = live.stream, .rate_limit = .exempt },
// Listed after the literal `live`, which a linear first-match scan reaches
// first — though `{id}` would refuse it anyway, since it captures a
// positive integer and nothing else.
.{ .method = .GET, .pattern = "/api/queries/{id}", .auth = .session, .policy = .read, .handler = queries.detail },
.{ .method = .GET, .pattern = "/api/stats", .auth = .session, .policy = .read, .handler = stats.totals },
.{ .method = .GET, .pattern = "/api/stats/timeseries", .auth = .session, .policy = .read, .handler = stats.timeseries },
.{ .method = .GET, .pattern = "/api/stats/types", .auth = .session, .policy = .read, .handler = stats.types },
.{ .method = .GET, .pattern = "/api/stats/routes", .auth = .session, .policy = .read, .handler = stats.routes },
.{ .method = .GET, .pattern = "/api/stats/clients", .auth = .session, .policy = .read, .handler = stats.clients },
.{ .method = .GET, .pattern = "/api/overview", .auth = .session, .policy = .read, .handler = overview.handle },
.{ .method = .GET, .pattern = "/api/lookup", .auth = .session, .policy = .read, .handler = lookup.handle },
// Diagnostics: the operational event log (milestone 27). The two purges are
@@ -161,7 +157,7 @@ const std = @import("std");
const testing = std.testing;
test "the table carries every endpoint of the milestone" {
try testing.expectEqual(@as(usize, 64), table.len);
try testing.expectEqual(@as(usize, 60), table.len);
}
test "no two entries claim the same method and pattern" {
src/web/server.zig
+78 -1
View File
@@ -175,6 +175,67 @@ pub const UpstreamBuild = struct {
bundle_lock: *std.Io.RwLock,
};
/// The Overview response cache: one already-serialized body per period.
///
/// A slot is valid for exactly one `(window.until, data_version)` pair, so it
/// expires both ways a stale Overview can arise — the window rolls onto the
/// next bucket, or another connection (the logger, retention) commits and moves
/// `PRAGMA data_version`. There is no time-to-live and no background refresh:
/// nothing here can serve bytes that describe a database state the reader could
/// not have seen.
///
/// Every field is read and written under `WebState.querylog_lock`, which is
/// also what makes the cache single-flight: a second request for the same key
/// waits for the first rebuild and then hits. The type carries no lock of its
/// own precisely so that nobody can touch it without the one that matters.
pub const OverviewCache = struct {
/// One per `overview.Period`, indexed by `@intFromEnum`. The handler asserts
/// the two counts agree.
pub const slot_count = 4;
const Slot = struct {
/// Empty until the first successful build; never a valid empty body,
/// since every response carries at least the period and the window.
body: []u8 = &.{},
until: i64 = 0,
data_version: i64 = 0,
};
slots: [slot_count]Slot = @splat(.{}),
/// The stored bytes for this key, or null. The caller copies them into its
/// request arena before releasing the lock: a later rebuild frees this
/// allocation.
pub fn get(self: *const OverviewCache, period_index: usize, until: i64, data_version: i64) ?[]const u8 {
const slot = &self.slots[period_index];
if (slot.body.len == 0) return null;
if (slot.until != until or slot.data_version != data_version) return null;
return slot.body;
}
/// Takes ownership of `body`, which must be a `gpa` allocation, and frees
/// whatever the slot held.
pub fn put(
self: *OverviewCache,
gpa: Allocator,
period_index: usize,
until: i64,
data_version: i64,
body: []u8,
) void {
const slot = &self.slots[period_index];
gpa.free(slot.body);
slot.* = .{ .body = body, .until = until, .data_version = data_version };
}
pub fn deinit(self: *OverviewCache, gpa: Allocator) void {
for (&self.slots) |*slot| {
gpa.free(slot.body);
slot.* = .{};
}
}
};
pub const WebState = struct {
gpa: Allocator,
web: model.Web = .{},
@@ -272,6 +333,9 @@ pub const WebState = struct {
/// the shared connection would fail and a third task's reads would land
/// inside someone else's snapshot.
querylog_lock: std.Io.Mutex = .init,
/// The Overview response cache, guarded by `querylog_lock` above. Whoever
/// owns the `WebState` calls `overview_cache.deinit`.
overview_cache: OverviewCache = .{},
/// The diagnostics event store, which owns a third connection of its own
/// and serializes every access — read and write — through its mutex. Null
/// when `Store.init` failed, which `/api/health` reports as `unavailable`
@@ -350,11 +414,24 @@ pub const QuerylogRead = struct {
pub fn open(state: *WebState, io: std.Io, database: *db.Db) db.Error!QuerylogRead {
state.querylog_lock.lockUncancelable(io);
errdefer state.querylog_lock.unlock(io);
var scope = try openLocked(state, io, database);
scope.held = true;
return scope;
}
/// The transaction alone, for a caller that already holds `querylog_lock`
/// and keeps holding it past `commit` — the overview handler, which decides
/// its response cache under the same one hold. Calling `open` there would
/// deadlock on a mutex the task already owns.
///
/// The returned scope releases nothing: `commit` and `abort` end the
/// transaction and leave the lock to whoever took it.
pub fn openLocked(state: *WebState, io: std.Io, database: *db.Db) db.Error!QuerylogRead {
return .{
.state = state,
.io = io,
.tx = try db.ReadTx.begin(database),
.held = true,
.held = false,
};
}
src/web/web_integration_test.zig
+77 -97
View File
@@ -83,7 +83,7 @@ const handlers_lookup = @import("handlers/lookup.zig");
const handlers_pause = @import("handlers/pause.zig");
const handlers_queries = @import("handlers/queries.zig");
const handlers_settings = @import("handlers/settings.zig");
const handlers_stats = @import("handlers/stats.zig");
const handlers_overview = @import("handlers/overview.zig");
const handlers_version = @import("handlers/version.zig");
const Certificate = std.crypto.Certificate;
@@ -680,6 +680,7 @@ const Env = struct {
self.state.live_hash.deinit(gpa);
self.state.proxies.deinit(gpa);
self.state.overview_cache.deinit(gpa);
self.tables.deinit(gpa);
gpa.destroy(self.hub);
self.limiter.deinit();
@@ -738,7 +739,7 @@ fn seedQueryLog(database: *db.Db) !void {
\\UPDATE querylog_meta SET created_at = 1700000000, available_since = 1700000000 WHERE id = 1
);
var writer = try queries_repo.BatchWriter.init(database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, database);
defer writer.deinit();
var domain_buf: [32]u8 = undefined;
@@ -838,7 +839,7 @@ fn seedQueryLog(database: *db.Db) !void {
const recent_clients = 3;
fn seedRecentTraffic(database: *db.Db, now: i64) !void {
var writer = try queries_repo.BatchWriter.init(database);
var writer = try queries_repo.BatchWriter.init(testing.allocator, database);
defer writer.deinit();
const Shape = struct {
@@ -1048,15 +1049,11 @@ const contract = [_]Contract{
// Refresh-all before any source row exists: nothing to fetch, 202 anyway.
.{ .method = .POST, .pattern = "/api/blocklists/update", .auth = .session, .policy = .runtime_action, .target = "/api/blocklists/update", .status = 202, .check = jsonShape(StatusList) },
// Query log, stats, live stream, upstream health.
// Query log, overview, live stream, upstream health.
.{ .method = .GET, .pattern = "/api/queries", .auth = .session, .policy = .read, .target = "/api/queries?limit=10", .status = 200, .check = jsonShape(handlers_queries.Page) },
.{ .method = .GET, .pattern = "/api/queries/{id}", .auth = .session, .policy = .read, .target = "/api/queries/27", .status = 200, .check = jsonShape(provenance_view.QueryDetail) },
.{ .method = .GET, .pattern = "/api/queries/live", .auth = .session, .policy = .read, .rate_limit = .exempt, .target = "/api/queries/live", .status = 200, .kind = .sse },
.{ .method = .GET, .pattern = "/api/stats", .auth = .session, .policy = .read, .target = "/api/stats?period=1h", .status = 200, .check = jsonShape(handlers_stats.TotalsBody) },
.{ .method = .GET, .pattern = "/api/stats/timeseries", .auth = .session, .policy = .read, .target = "/api/stats/timeseries?period=1h", .status = 200, .check = jsonShape(handlers_stats.TimeseriesBody) },
.{ .method = .GET, .pattern = "/api/stats/types", .auth = .session, .policy = .read, .target = "/api/stats/types?period=1h", .status = 200, .check = jsonShape(handlers_stats.TypesBody) },
.{ .method = .GET, .pattern = "/api/stats/routes", .auth = .session, .policy = .read, .target = "/api/stats/routes?period=1h", .status = 200, .check = jsonShape(handlers_stats.RoutesBody) },
.{ .method = .GET, .pattern = "/api/stats/clients", .auth = .session, .policy = .read, .target = "/api/stats/clients?period=1h", .status = 200, .check = jsonShape(handlers_stats.ClientsBody) },
.{ .method = .GET, .pattern = "/api/overview", .auth = .session, .policy = .read, .target = "/api/overview?period=1h", .status = 200, .check = jsonShape(handlers_overview.Body) },
// Diagnostics. The seeded store holds one active episode (id 1) and one
// resolved one, so both the page and the detail answer with real rows.
@@ -2587,11 +2584,7 @@ fn detailUnavailable(io: std.Io, env: *Env) anyerror!void {
const targets = [_][]const u8{
"/api/queries/1",
"/api/queries?limit=1",
"/api/stats",
"/api/stats/timeseries",
"/api/stats/types",
"/api/stats/routes",
"/api/stats/clients",
"/api/overview",
};
for (targets) |target| {
try conn.request("GET", target, null, null);
@@ -2657,27 +2650,20 @@ fn coverageWalk(io: std.Io, env: *Env) anyerror!void {
);
try testing.expect(!partial.coverage.complete);
// The stats endpoints judge the same watermark against their own aligned
// window, which for any live period starts well after the seeded rows.
try conn.request("GET", "/api/stats?period=1h", null, null);
const totals = try std.json.parseFromSliceLeaky(
handlers_stats.TotalsBody,
// The overview judges the same watermark against its own aligned window,
// which for any live period starts well after the seeded rows.
try conn.request("GET", "/api/overview?period=1h", null, null);
const overview_body = try std.json.parseFromSliceLeaky(
handlers_overview.Body,
arena,
(try conn.receive(&body_buf)).body,
.{ .ignore_unknown_fields = false },
);
try testing.expectEqual(seeded_available_since, totals.coverage.available_since);
try testing.expectEqual(totals.since >= seeded_available_since, totals.coverage.complete);
try conn.request("GET", "/api/stats/timeseries?period=1h", null, null);
const series = try std.json.parseFromSliceLeaky(
handlers_stats.TimeseriesBody,
arena,
(try conn.receive(&body_buf)).body,
.{ .ignore_unknown_fields = false },
try testing.expectEqual(seeded_available_since, overview_body.coverage.available_since);
try testing.expectEqual(
overview_body.since >= seeded_available_since,
overview_body.coverage.complete,
);
try testing.expectEqual(totals.since, series.since);
try testing.expectEqual(totals.coverage.complete, series.coverage.complete);
}
fn getJson(
@@ -2708,33 +2694,43 @@ fn emptyAggregations(io: std.Io, env: *Env) anyerror!void {
var body_buf: [256 * 1024]u8 = undefined;
// This environment's only rows are the fixed 2023 seed, so every live
// window is empty. The empty bodies are exact, not merely parseable.
const types_body = try getJson(handlers_stats.TypesBody, arena, &conn, "/api/stats/types?period=1h", &body_buf);
try testing.expectEqualStrings("1h", types_body.period);
try testing.expectEqual(@as(usize, 0), types_body.types.len);
const routes_body = try getJson(handlers_stats.RoutesBody, arena, &conn, "/api/stats/routes?period=1h", &body_buf);
try testing.expectEqual(@as(usize, 0), routes_body.routes.len);
// window is empty. The empty body is exact, not merely parseable.
const body = try getJson(handlers_overview.Body, arena, &conn, "/api/overview?period=1h", &body_buf);
try testing.expectEqualStrings("1h", body.period);
try testing.expectEqual(@as(u64, 0), body.totals.queries);
try testing.expectEqual(@as(?i64, null), body.totals.avg_response_time_us);
try testing.expectEqual(@as(usize, 0), body.types.len);
try testing.expectEqual(@as(usize, 0), body.routes.len);
// `other` is present and bucket-count sized even here: a chart must never
// have to invent the residual series.
const clients = try getJson(handlers_stats.ClientsBody, arena, &conn, "/api/stats/clients?period=1h", &body_buf);
try testing.expectEqual(@as(usize, 0), clients.clients.len);
try testing.expectEqual(@as(u32, 60), clients.bucket_seconds);
try testing.expectEqual(@as(usize, 60), clients.other.len);
for (clients.other) |count| try testing.expectEqual(@as(u64, 0), count);
try testing.expectEqual(@as(usize, 0), body.clients.len);
try testing.expectEqual(@as(u32, 60), body.bucket_seconds);
try testing.expectEqual(@as(usize, 60), body.buckets.len);
try testing.expectEqual(@as(usize, 60), body.other.len);
for (body.other) |count| try testing.expectEqual(@as(u64, 0), count);
// A window nobody covers is still reported as such, not as a quiet hour.
try testing.expectEqual(seeded_available_since, types_body.coverage.available_since);
try testing.expect(types_body.coverage.complete);
try testing.expectEqual(seeded_available_since, body.coverage.available_since);
try testing.expect(body.coverage.complete);
for ([_][]const u8{ "/api/stats/types", "/api/stats/routes", "/api/stats/clients" }) |path| {
var target_buf: [64]u8 = undefined;
const target = try std.fmt.bufPrint(&target_buf, "{s}?period=12h", .{path});
try conn.request("GET", target, null, null);
const bad = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 400), bad.status);
try testing.expect(std.mem.containsAtLeast(u8, bad.body, 1, "period must be one of"));
try conn.request("GET", "/api/overview?period=12h", null, null);
const bad = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 400), bad.status);
try testing.expect(std.mem.containsAtLeast(u8, bad.body, 1, "period must be one of"));
// Milestone 36 removed the five per-panel endpoints. They are gone from the
// table, not merely unreferenced by the admin, so the server refuses them.
for ([_][]const u8{
"/api/stats",
"/api/stats/timeseries",
"/api/stats/types",
"/api/stats/routes",
"/api/stats/clients",
}) |gone| {
try conn.request("GET", gone, null, null);
const missing = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 404), missing.status);
}
}
@@ -2759,23 +2755,16 @@ fn populatedAggregations(io: std.Io, env: *Env) anyerror!void {
var body_buf: [256 * 1024]u8 = undefined;
const totals = try getJson(handlers_stats.TotalsBody, arena, &conn, "/api/stats?period=1h", &body_buf);
const series = try getJson(handlers_stats.TimeseriesBody, arena, &conn, "/api/stats/timeseries?period=1h", &body_buf);
const types_body = try getJson(handlers_stats.TypesBody, arena, &conn, "/api/stats/types?period=1h", &body_buf);
const routes_body = try getJson(handlers_stats.RoutesBody, arena, &conn, "/api/stats/routes?period=1h", &body_buf);
const clients = try getJson(handlers_stats.ClientsBody, arena, &conn, "/api/stats/clients?period=1h", &body_buf);
const body = try getJson(handlers_overview.Body, arena, &conn, "/api/overview?period=1h", &body_buf);
// Nothing writes to this box between the five requests, so the window is
// one state and conservation is a real assertion rather than a race.
try testing.expectEqual(totals.since, series.since);
try testing.expectEqual(totals.since, types_body.since);
try testing.expectEqual(totals.since, routes_body.since);
try testing.expectEqual(totals.since, clients.since);
try testing.expect(totals.queries > 0);
// One response over one snapshot, so conservation is a property of the
// payload rather than of a quiet box between five requests.
try testing.expect(body.totals.queries > 0);
const totals = body.totals;
var typed: u64 = 0;
var null_qtype_rows: usize = 0;
for (types_body.types) |row| {
for (body.types) |row| {
typed += row.count;
if (row.qtype == null) null_qtype_rows += 1;
}
@@ -2786,7 +2775,7 @@ fn populatedAggregations(io: std.Io, env: *Env) anyerror!void {
var routed: u64 = 0;
var null_source_upstreams: usize = 0;
var named_upstreams: usize = 0;
for (routes_body.routes) |row| {
for (body.routes) |row| {
routed += row.count;
if (row.route != .upstream) continue;
if (row.source == null) null_source_upstreams += 1 else named_upstreams += 1;
@@ -2795,17 +2784,20 @@ fn populatedAggregations(io: std.Io, env: *Env) anyerror!void {
try testing.expectEqual(@as(usize, 1), null_source_upstreams);
try testing.expectEqual(@as(usize, 2), named_upstreams);
try testing.expectEqual(@as(usize, recent_clients), clients.clients.len);
try testing.expectEqual(series.buckets.len, clients.other.len);
for (clients.clients) |entry| try testing.expectEqual(series.buckets.len, entry.buckets.len);
try testing.expectEqual(@as(usize, recent_clients), body.clients.len);
try testing.expectEqual(body.buckets.len, body.other.len);
for (body.clients) |entry| try testing.expectEqual(body.buckets.len, entry.buckets.len);
// Per bucket, not just over the window: a series off by one bucket would
// still sum correctly in total.
for (series.buckets, 0..) |bucket, at| {
var summed: u64 = clients.other[at];
for (clients.clients) |entry| summed += entry.buckets[at];
var bucketed: u64 = 0;
for (body.buckets, 0..) |bucket, at| {
bucketed += bucket.queries;
var summed: u64 = body.other[at];
for (body.clients) |entry| summed += entry.buckets[at];
try testing.expectEqual(bucket.queries, summed);
}
try testing.expectEqual(totals.queries, bucketed);
}
test "W10 milestone 30: the three breakdowns conserve the totals over one window" {
@@ -2826,11 +2818,8 @@ fn hammerQuerylog(io: std.Io, env: *Env) anyerror!void {
var body_buf: [256 * 1024]u8 = undefined;
const targets = [_][]const u8{
"/api/stats?period=1h",
"/api/stats/timeseries?period=1h",
"/api/stats/types?period=1h",
"/api/stats/routes?period=1h",
"/api/stats/clients?period=1h",
"/api/overview?period=1h",
"/api/overview?period=24h",
"/api/queries?limit=5",
"/api/queries/27",
};
@@ -2875,7 +2864,7 @@ fn failedCommitIsBounded(io: std.Io, env: *Env) anyerror!void {
// the connection recovers (the rollback attempt worked, so the next
// `BEGIN` is not refused).
db.read_tx_faults.failNextCommit();
try conn.request("GET", "/api/stats/types?period=1h", null, null);
try conn.request("GET", "/api/overview?period=1h", null, null);
const failed = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 500), failed.status);
try testing.expect(std.mem.containsAtLeast(u8, failed.body, 1, "internal error"));
@@ -2883,16 +2872,13 @@ fn failedCommitIsBounded(io: std.Io, env: *Env) anyerror!void {
// Same connection, same shared query-log handle: a request after the fault
// is an ordinary 200. This is the assertion the double-unlock bug failed —
// it panicked here instead of answering.
try conn.request("GET", "/api/stats/types?period=1h", null, null);
try conn.request("GET", "/api/overview?period=1h", null, null);
const recovered = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), recovered.status);
// And every other query-log route still works on that connection.
for ([_][]const u8{
"/api/stats?period=1h",
"/api/stats/timeseries?period=1h",
"/api/stats/routes?period=1h",
"/api/stats/clients?period=1h",
"/api/overview?period=24h",
"/api/queries?limit=5",
"/api/queries/27",
}) |target| {
@@ -3012,7 +2998,7 @@ fn credentialSweep(
// a closed queue is empty, and a zero flush interval makes it commit the
// batch it holds rather than wait for company.
query_logger.shutdown(io);
try query_logger.runWriter(io, &env.querylog_db, null);
try query_logger.runWriter(io, testing.allocator, &env.querylog_db, null);
try testing.expectEqual(@as(u64, 1), query_logger.rows_written.load(.monotonic));
var stmt = try env.querylog_db.prepare(
@@ -3905,15 +3891,13 @@ test "drift guard c: the health rollup matches the five objects it documents" {
try expectSchemaMatches(gpa, handlers_health.Body, "Health");
}
test "drift guard c: the stats schemas match the structs that serialize them" {
test "drift guard c: the overview schema matches the struct that serializes it" {
// Guard b counts operations and guard a matches paths, so neither noticed
// that `cached` outlived the field it documented. This one would have.
// It recurses, so `Bucket`, `ClientSeries`, `TypeCount` and `RouteCount`
// are held to their schemas here too.
const gpa = testing.allocator;
try expectSchemaMatches(gpa, handlers_stats.TotalsBody, "StatsTotals");
try expectSchemaMatches(gpa, handlers_stats.TimeseriesBody, "StatsTimeseries");
try expectSchemaMatches(gpa, handlers_stats.TypesBody, "StatsTypes");
try expectSchemaMatches(gpa, handlers_stats.RoutesBody, "StatsRoutes");
try expectSchemaMatches(gpa, handlers_stats.ClientsBody, "StatsClients");
try expectSchemaMatches(gpa, handlers_overview.Body, "Overview");
}
test "drift guard c: the query-log schemas match the structs that serialize them" {
@@ -4094,8 +4078,6 @@ const contract_sample_walk = [_]ContractSample{
// a matched pattern, so the golden exercises every nested object rather
// than a row of nulls.
.{ .name = "get_query_detail", .ts_type = "QueryDetail", .method = "GET", .target = "/api/queries/27", .status = 200 },
.{ .name = "get_stats", .ts_type = "StatsTotals", .method = "GET", .target = "/api/stats?period=1h", .status = 200 },
.{ .name = "get_stats_timeseries", .ts_type = "StatsTimeseries", .method = "GET", .target = "/api/stats/timeseries?period=1h", .status = 200 },
// Pause: the GET before the POST, so one sample carries `until: null` and
// the other the deadline.
@@ -4122,13 +4104,11 @@ const contract_sample_walk = [_]ContractSample{
.{ .name = "error_not_found", .ts_type = "ErrorEnvelope", .method = "GET", .target = "/api/nope", .status = 404 },
};
/// The three period aggregations, captured against an environment with live
/// traffic in it: over the fixed 2023 seed every one of them would answer with
/// an empty array, which describes no field at all.
/// The overview, captured against an environment with live traffic in it: over
/// the fixed 2023 seed its four breakdowns would every one answer with an empty
/// array, which describes no field at all.
const stats_sample_walk = [_]ContractSample{
.{ .name = "get_stats_types", .ts_type = "StatsTypes", .method = "GET", .target = "/api/stats/types?period=1h", .status = 200 },
.{ .name = "get_stats_routes", .ts_type = "StatsRoutes", .method = "GET", .target = "/api/stats/routes?period=1h", .status = 200 },
.{ .name = "get_stats_clients", .ts_type = "StatsClients", .method = "GET", .target = "/api/stats/clients?period=1h", .status = 200 },
.{ .name = "get_overview", .ts_type = "Overview", .method = "GET", .target = "/api/overview?period=1h", .status = 200 },
};
/// A session-authenticated environment answers this without a cookie.