mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 13 Activity

milestone 30: overview as a dashboard, explicit health contract, period aggregations
Gates / frontend (push) Successful in 1m32s
Details
Gates / test (push) Successful in 1m54s
Details
Gates / package (push) Successful in 5m28s
Details
Gates / container (push) Successful in 14s
Details
Gates / test-aarch64 (push) Failing after 3h10m0s
Details
CI / gates (push) Failing after 3h11m55s
Details

Browse Source
This commit is contained in:
mokhtar
2026-08-22 16:45:15 +02:00
parent 17422fac21
commit 648d9b4496
89 changed files with 7222 additions and 4239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/web/handlers/health.zig
+299 -200
View File
@@ -6,6 +6,13 @@
//!
//! Unauthenticated and rate-limit exempt, like `/metrics`.
//!
//! **Nothing may degrade the rollup without appearing in the response.** The
//! body is five condition objects and a status computed from exactly their
//! states, so an operator reading it can always name the condition that
//! degraded the box. Anything a counter already records — gated refreshes,
//! disk sample failures, the snapshot generation — stays in `/metrics` and in
//! Diagnostics rather than becoming a sixth hidden input here.
//!
//! `rollup` is pure so the whole degraded matrix is testable without a running
//! server; `handle` only gathers the inputs.
@@ -13,16 +20,39 @@ const std = @import("std");
const disk_monitor = @import("../../storage/disk_monitor.zig");
const http_util = @import("../http_util.zig");
const logger_mod = @import("../../storage/logger.zig");
const metrics = @import("../metrics.zig");
const pause_mod = @import("../../server/pause.zig");
const pool_mod = @import("../../upstream/pool.zig");
const server = @import("../server.zig");
pub const Disk = struct {
/// Is filtering in force. `unavailable` is not an operator's doing: it is the
/// state in which the query path has no filter snapshot to evaluate against,
/// which `handler.zig` records as the `snapshot_unavailable` provenance.
pub const Protection = struct {
state: []const u8,
free_bytes: u64,
db_bytes: u64,
log_bytes: u64,
sample_failures: u64,
/// The second filtering resumes at. Null for both an indefinite pause and
/// every non-paused state, which the state field already tells apart.
until: ?i64,
};
pub const Upstreams = struct {
state: []const u8,
available: u32,
/// Enabled upstreams: the pool is built from those alone, so this is what
/// `available` is out of.
total: u32,
};
/// Whether Activity can be trusted. `dropped_total` is cumulative and does not
/// decide the state — a drop that happened an hour ago is not a fault now.
pub const QueryHistory = struct {
state: []const u8,
dropped_total: u64,
/// The newest drop, or null while nothing has been dropped. Stamped by a
/// separate atomic from the count, so a reader can momentarily see a
/// non-zero `dropped_total` beside a null here.
last_drop_s: ?i64,
};
/// The diagnostics store's own state, not a summary of what it holds: `state`
@@ -34,21 +64,18 @@ pub const Diagnostics = struct {
active_errors: u32,
};
pub const Upstreams = struct {
available: u32,
total: u32,
pub const Disk = struct {
state: []const u8,
free_bytes: u64,
};
pub const Body = struct {
status: []const u8,
disk: Disk,
protection: Protection,
upstreams: Upstreams,
query_history: QueryHistory,
diagnostics: Diagnostics,
queries_dropped: u64,
writer_failed: bool,
refreshes_gated: u64,
/// Null before the first filter snapshot is published.
snapshot_generation: ?u64,
disk: Disk,
};
/// What the rollup is computed from. Every field has a defined value even when
@@ -56,41 +83,79 @@ pub const Body = struct {
/// server should report: no disk reading, no upstreams, nothing published.
pub const Input = struct {
disk_state: disk_monitor.State = .ok,
disk: disk_monitor.Gauges = .{ .free_bytes = 0, .db_bytes = 0, .log_bytes = 0 },
disk_sample_failures: u64 = 0,
disk_free_bytes: u64 = 0,
upstreams_available: u32 = 0,
upstreams_total: u32 = 0,
queries_dropped: u64 = 0,
last_drop_s: ?i64 = null,
writer_failed: bool = false,
/// Current state, not a count: the upstream-history flush is failing right
/// now. Cleared by the next flush that succeeds (m26 ruling 7).
///
/// `rows_dropped` deliberately does not appear here. It is cumulative, and
/// a rollup that is computed statelessly cannot ask whether a counter grew
/// — so feeding it in would latch `/api/health` to degraded forever after
/// one overflow. Drops surface through the metric and through the API's
/// per-window `complete` instead.
history_flush_failing: bool = false,
/// The logger's disk-gating episode. `losing` is the only state that
/// degrades: it means the gate is holding writes back *and* has already
/// cost rows in the episode that is open now.
gate_episode: logger_mod.GateEpisode = .open,
/// A filter snapshot exists for the query path to evaluate against. The
/// benign default matches every other field here, and `collect` assigns it
/// explicitly for the same reason `diagnostics_present` is assigned there.
snapshot_available: bool = true,
/// `pause.Pause.until` verbatim: 0 not paused, -1 indefinite, otherwise the
/// second filtering resumes at. Raw rather than a decided boolean so the
/// expiry rule stays `pause.Pause`'s and is exercised by these tests.
pause_until: i64 = 0,
/// The clock the pause is compared against.
now_s: i64 = 0,
/// The diagnostics store exists. The benign default matches every other
/// field here — a half-wired `Input` reports a box with nothing wrong — but
/// `collect` must assign it explicitly, because in a serving process an
/// absent store means `Store.init` failed.
diagnostics_present: bool = true,
/// The last diagnostics write failed. Current state, cleared by the next
/// write that succeeds, like `history_flush_failing`.
/// write that succeeds.
diagnostics_write_failed: bool = false,
diagnostics_active_warnings: u32 = 0,
diagnostics_active_errors: u32 = 0,
refreshes_gated: u64 = 0,
snapshot_generation: ?u64 = null,
};
pub const status_ok = "ok";
pub const status_degraded = "degraded";
pub const protection_active = "active";
pub const protection_paused = "paused";
pub const protection_unavailable = "unavailable";
pub const upstreams_ok = "ok";
pub const upstreams_unavailable = "unavailable";
pub const query_history_recording = "recording";
pub const query_history_losing = "losing";
pub const query_history_failed = "failed";
pub const diagnostics_recording = "recording";
pub const diagnostics_unavailable = "unavailable";
pub const disk_ok = "ok";
pub const disk_low = "low";
pub const disk_critical = "critical";
/// Precedence `unavailable` → `paused` → `active`. With no snapshot the pause
/// flag says nothing an operator can act on: filtering is off either way, and
/// resuming would not turn it back on.
pub fn protection(input: Input) Protection {
if (!input.snapshot_available) return .{ .state = protection_unavailable, .until = null };
const state: pause_mod.Pause = .{ .until = .init(input.pause_until) };
if (!state.isPaused(input.now_s)) return .{ .state = protection_active, .until = null };
return .{
.state = protection_paused,
.until = if (input.pause_until > 0) input.pause_until else null,
};
}
pub fn queryHistoryState(input: Input) []const u8 {
if (input.writer_failed) return query_history_failed;
if (input.gate_episode == .losing) return query_history_losing;
return query_history_recording;
}
/// The operational log is not recording — either the store never opened or its
/// writes are failing. Both mean the same thing to an operator: the record of
/// what went wrong is not being kept.
@@ -98,37 +163,51 @@ pub fn diagnosticsUnavailable(input: Input) bool {
return !input.diagnostics_present or input.diagnostics_write_failed;
}
/// Conditions an operator must act on, and every one of them is a fact about
/// now rather than a count of the past: a disk that is filling stops the query
/// log, a pool with nothing available stops resolution, a failed writer means
/// rows are being lost right now, and a failing history flush means the
/// dashboard's upstream numbers are not being recorded. Each clears itself when
/// the underlying condition does.
/// `warn` reads as a log level rather than as a quantity of disk. The monitor
/// keeps its own name; the wire says what an operator sees on the page.
pub fn diskState(state: disk_monitor.State) []const u8 {
return switch (state) {
.ok => disk_ok,
.warn => disk_low,
.critical => disk_critical,
};
}
/// The degrading set, named rather than gestured at: protection `unavailable`,
/// upstreams `unavailable`, query history `losing` or `failed`, diagnostics
/// `unavailable`, disk `low` or `critical`.
///
/// A pause is deliberately not in it. It is an operator's own choice, and a
/// monitor that pages on it would be paging on a button the operator pressed.
pub fn degraded(input: Input) bool {
return input.disk_state != .ok or input.upstreams_available == 0 or
input.writer_failed or input.history_flush_failing or diagnosticsUnavailable(input);
const history = queryHistoryState(input);
return !input.snapshot_available or
input.upstreams_available == 0 or
!std.mem.eql(u8, history, query_history_recording) or
diagnosticsUnavailable(input) or
input.disk_state != .ok;
}
pub fn rollup(input: Input) Body {
return .{
.status = if (degraded(input)) status_degraded else status_ok,
.disk = .{
.state = @tagName(input.disk_state),
.free_bytes = input.disk.free_bytes,
.db_bytes = input.disk.db_bytes,
.log_bytes = input.disk.log_bytes,
.sample_failures = input.disk_sample_failures,
.protection = protection(input),
.upstreams = .{
.state = if (input.upstreams_available == 0) upstreams_unavailable else upstreams_ok,
.available = input.upstreams_available,
.total = input.upstreams_total,
},
.query_history = .{
.state = queryHistoryState(input),
.dropped_total = input.queries_dropped,
.last_drop_s = input.last_drop_s,
},
.upstreams = .{ .available = input.upstreams_available, .total = input.upstreams_total },
.diagnostics = .{
.state = if (diagnosticsUnavailable(input)) diagnostics_unavailable else diagnostics_recording,
.active_warnings = input.diagnostics_active_warnings,
.active_errors = input.diagnostics_active_errors,
},
.queries_dropped = input.queries_dropped,
.writer_failed = input.writer_failed,
.refreshes_gated = input.refreshes_gated,
.snapshot_generation = input.snapshot_generation,
.disk = .{ .state = diskState(input.disk_state), .free_bytes = input.disk_free_bytes },
};
}
@@ -143,10 +222,11 @@ pub fn handle(
pub fn collect(state: *server.WebState, io: std.Io) Input {
var input: Input = .{};
input.now_s = std.Io.Clock.real.now(io).toSeconds();
if (state.monitor) |monitor| {
input.disk_state = monitor.state();
input.disk = monitor.gauges();
input.disk_sample_failures = monitor.sample_failures.load(.monotonic);
input.disk_free_bytes = monitor.gauges().free_bytes;
}
if (state.pool) |pool| {
@@ -160,9 +240,13 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
if (state.logger) |logger| {
input.queries_dropped = logger.queries_dropped.load(.monotonic);
input.last_drop_s = logger.lastDropSeconds();
input.writer_failed = logger.writer_failed.load(.monotonic);
input.gate_episode = logger.gateEpisode();
}
if (state.pause) |paused| input.pause_until = paused.until.load(.monotonic);
// Assigned before the `if`, not inside it: the field's benign default is
// `true`, so the natural `if (state.events) |store|` shape would report an
// absent store as recording — the one case that must degrade.
@@ -174,15 +258,13 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
input.diagnostics_active_errors = counts.errors;
}
if (state.history) |history| {
input.history_flush_failing = history.snapshotStats(io).last_flush_failed;
}
// The generation itself is not in the body — the UI-facing fact is whether
// protection has a snapshot at all, and `/metrics` keeps the number.
input.snapshot_available = false;
if (state.manager) |manager| {
input.refreshes_gated = manager.refreshesGated();
if (manager.acquire(io)) |acquired| {
defer acquired.release(io);
input.snapshot_generation = acquired.snapshot.generation;
input.snapshot_available = true;
}
}
@@ -196,104 +278,175 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
const db = @import("../../storage/db.zig");
const events_mod = @import("../../storage/events.zig");
const migrations = @import("../../storage/migrations.zig");
const history_mod = @import("../../upstream/history.zig");
const logger_mod = @import("../../storage/logger.zig");
const upstream_history_repo = @import("../../storage/repositories/upstream_history_repo.zig");
const testing = std.testing;
/// A box with nothing wrong with it: one upstream up, disk ok, writer alive.
/// A box with nothing wrong with it: one upstream up, disk ok, writer alive,
/// a snapshot published and filtering unpaused.
const healthy: Input = .{
.disk_state = .ok,
.upstreams_available = 1,
.upstreams_total = 1,
.writer_failed = false,
.snapshot_available = true,
};
test "the degraded matrix covers disk state, availability and the writer" {
const cases = [_]struct { input: Input, degraded: bool }{
.{ .input = healthy, .degraded = false },
.{ .input = withDisk(healthy, .warn), .degraded = true },
.{ .input = withDisk(healthy, .critical), .degraded = true },
.{ .input = withAvailable(healthy, 0), .degraded = true },
.{ .input = withWriterFailed(healthy), .degraded = true },
// A failing upstream-history flush is losing the dashboard's numbers
// right now, and it recovers on its own the moment a flush succeeds.
.{ .input = withHistoryFailing(healthy, true), .degraded = true },
.{ .input = withHistoryFailing(healthy, false), .degraded = false },
// The operational log not recording is itself a fault an operator must
// act on: whatever fails next will leave no record of having failed.
.{ .input = withDiagnostics(healthy, false, false), .degraded = true },
.{ .input = withDiagnostics(healthy, true, true), .degraded = true },
.{ .input = withDiagnostics(healthy, true, false), .degraded = false },
// Two faults at once still report one status.
.{ .input = withWriterFailed(withDisk(healthy, .critical)), .degraded = true },
// Some upstreams down is not degraded while one still answers.
.{ .input = .{ .upstreams_available = 1, .upstreams_total = 3 }, .degraded = false },
};
fn with(input: Input, comptime field: []const u8, value: anytype) Input {
var out = input;
@field(out, field) = value;
return out;
}
for (cases, 0..) |case, i| {
errdefer std.debug.print("case {d}\n", .{i});
try testing.expectEqual(case.degraded, degraded(case.input));
try testing.expectEqualStrings(
if (case.degraded) status_degraded else status_ok,
rollup(case.input).status,
);
test "each degrading condition degrades on its own and says so in the body" {
try testing.expectEqualStrings(status_ok, rollup(healthy).status);
{
const body = rollup(with(healthy, "snapshot_available", false));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(protection_unavailable, body.protection.state);
}
{
const body = rollup(with(healthy, "upstreams_available", @as(u32, 0)));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(upstreams_unavailable, body.upstreams.state);
}
{
const body = rollup(with(healthy, "gate_episode", .losing));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(query_history_losing, body.query_history.state);
}
{
const body = rollup(with(healthy, "writer_failed", true));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(query_history_failed, body.query_history.state);
}
{
const body = rollup(with(healthy, "diagnostics_present", false));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state);
}
{
const body = rollup(with(healthy, "diagnostics_write_failed", true));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state);
}
for ([_]struct { disk_monitor.State, []const u8 }{
.{ .warn, disk_low },
.{ .critical, disk_critical },
}) |case| {
const body = rollup(with(healthy, "disk_state", case[0]));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(case[1], body.disk.state);
}
}
fn withDisk(input: Input, state: disk_monitor.State) Input {
var out = input;
out.disk_state = state;
return out;
test "conditions that are not faults leave the status ok" {
// Some upstreams down while one still answers.
const partial = rollup(.{ .upstreams_available = 1, .upstreams_total = 3 });
try testing.expectEqualStrings(status_ok, partial.status);
try testing.expectEqualStrings(upstreams_ok, partial.upstreams.state);
// A gate holding writes that has not cost a row yet.
const holding = rollup(with(healthy, "gate_episode", .gated));
try testing.expectEqualStrings(status_ok, holding.status);
try testing.expectEqualStrings(query_history_recording, holding.query_history.state);
// Drops that already happened. The counter is cumulative and the rollup is
// stateless, so feeding it in would latch the box to degraded forever.
var dropped = with(healthy, "queries_dropped", @as(u64, 9));
dropped.last_drop_s = 1_700_000_000;
const body = rollup(dropped);
try testing.expectEqualStrings(status_ok, body.status);
try testing.expectEqualStrings(query_history_recording, body.query_history.state);
try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total);
try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s);
// Open diagnostics episodes are what the box is doing, not a fault of the
// log that recorded them.
var open = healthy;
open.diagnostics_active_warnings = 3;
open.diagnostics_active_errors = 1;
const with_episodes = rollup(open);
try testing.expectEqualStrings(status_ok, with_episodes.status);
try testing.expectEqualStrings(diagnostics_recording, with_episodes.diagnostics.state);
try testing.expectEqual(@as(u32, 3), with_episodes.diagnostics.active_warnings);
try testing.expectEqual(@as(u32, 1), with_episodes.diagnostics.active_errors);
}
fn withAvailable(input: Input, available: u32) Input {
var out = input;
out.upstreams_available = available;
return out;
test "a pause is surfaced, never alarmed, and an expired one is over" {
var indefinite = healthy;
indefinite.pause_until = -1;
indefinite.now_s = 1_000;
const forever = rollup(indefinite);
try testing.expectEqualStrings(status_ok, forever.status);
try testing.expectEqualStrings(protection_paused, forever.protection.state);
try testing.expectEqual(@as(?i64, null), forever.protection.until);
var timed = healthy;
timed.pause_until = 1_060;
timed.now_s = 1_000;
const live = rollup(timed);
try testing.expectEqualStrings(status_ok, live.status);
try testing.expectEqualStrings(protection_paused, live.protection.state);
try testing.expectEqual(@as(?i64, 1_060), live.protection.until);
// The stored second is when filtering is back on, so at it the pause is over.
timed.now_s = 1_060;
const expired = rollup(timed);
try testing.expectEqualStrings(protection_active, expired.protection.state);
try testing.expectEqual(@as(?i64, null), expired.protection.until);
}
fn withWriterFailed(input: Input) Input {
var out = input;
out.writer_failed = true;
return out;
test "protection unavailable wins over a live pause" {
var both = healthy;
both.snapshot_available = false;
both.pause_until = -1;
const body = rollup(both);
try testing.expectEqualStrings(protection_unavailable, body.protection.state);
try testing.expectEqual(@as(?i64, null), body.protection.until);
try testing.expectEqualStrings(status_degraded, body.status);
}
fn withHistoryFailing(input: Input, failing: bool) Input {
var out = input;
out.history_flush_failing = failing;
return out;
test "a failed writer outranks a losing gate" {
var both = healthy;
both.writer_failed = true;
both.gate_episode = .losing;
try testing.expectEqualStrings(query_history_failed, rollup(both).query_history.state);
}
fn withDiagnostics(input: Input, present: bool, write_failed: bool) Input {
var out = input;
out.diagnostics_present = present;
out.diagnostics_write_failed = write_failed;
return out;
test "two faults at once still report one status" {
var both = with(healthy, "disk_state", disk_monitor.State.critical);
both.writer_failed = true;
try testing.expectEqualStrings(status_degraded, rollup(both).status);
}
test "the diagnostics block reports the state and the open counts" {
const recording = rollup(.{
.upstreams_available = 1,
.diagnostics_active_warnings = 3,
.diagnostics_active_errors = 1,
test "the body reports every input verbatim" {
const body = rollup(.{
.disk_state = .warn,
.disk_free_bytes = 100,
.upstreams_available = 2,
.upstreams_total = 4,
.queries_dropped = 9,
.last_drop_s = 1_700_000_000,
.snapshot_available = true,
});
try testing.expectEqualStrings(diagnostics_recording, recording.diagnostics.state);
try testing.expectEqual(@as(u32, 3), recording.diagnostics.active_warnings);
try testing.expectEqual(@as(u32, 1), recording.diagnostics.active_errors);
// Open episodes are what the box is doing, not a fault of the log: they do
// not degrade on their own.
try testing.expectEqualStrings(status_ok, recording.status);
// Failing writes: the counts are whatever was last read, and the state is
// the honest one.
const failing = rollup(.{ .upstreams_available = 1, .diagnostics_write_failed = true });
try testing.expectEqualStrings(diagnostics_unavailable, failing.diagnostics.state);
try testing.expectEqualStrings(status_degraded, failing.status);
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(disk_low, body.disk.state);
try testing.expectEqual(@as(u64, 100), body.disk.free_bytes);
try testing.expectEqual(@as(u32, 2), body.upstreams.available);
try testing.expectEqual(@as(u32, 4), body.upstreams.total);
try testing.expectEqualStrings(upstreams_ok, body.upstreams.state);
try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total);
try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s);
try testing.expectEqualStrings(protection_active, body.protection.state);
}
const absent = rollup(.{ .upstreams_available = 1, .diagnostics_present = false });
try testing.expectEqualStrings(diagnostics_unavailable, absent.diagnostics.state);
try testing.expectEqualStrings(status_degraded, absent.status);
test "an unstamped drop time serializes as null, not as zero" {
var buffer: [1024]u8 = undefined;
var writer: std.Io.Writer = .fixed(&buffer);
try std.json.Stringify.value(rollup(.{}), .{}, &writer);
try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"last_drop_s\":null"));
try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"until\":null"));
}
test "collect reports an absent store as unavailable rather than as recording" {
@@ -307,6 +460,10 @@ test "collect reports an absent store as unavailable rather than as recording" {
const absent = collect(&state, io);
try testing.expect(!absent.diagnostics_present);
try testing.expectEqualStrings(diagnostics_unavailable, rollup(absent).diagnostics.state);
// And a bare state has no snapshot either, which is the honest reading of a
// process that has published nothing.
try testing.expect(!absent.snapshot_available);
try testing.expectEqualStrings(protection_unavailable, rollup(absent).protection.state);
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
@@ -325,76 +482,7 @@ test "collect reports an absent store as unavailable rather than as recording" {
try testing.expectEqualStrings(diagnostics_recording, rollup(present).diagnostics.state);
}
test "a history overflow that already happened does not degrade the rollup" {
// `rows_dropped` is cumulative and the rollup is stateless, so the only
// thing it could do with a drop count is latch on it. The accumulator's
// drops reach an operator through `/metrics` and through the per-window
// `complete` flag, and never through this.
const dropped: Input = .{
.upstreams_available = 1,
.upstreams_total = 1,
.history_flush_failing = false,
};
try testing.expect(!degraded(dropped));
try testing.expectEqualStrings(status_ok, rollup(dropped).status);
}
test "collect reads the accumulator's current flush state" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const acc = try testing.allocator.create(history_mod.Accumulator);
defer testing.allocator.destroy(acc);
acc.* = .init;
var state: server.WebState = .{ .gpa = testing.allocator, .history = acc };
try testing.expect(!collect(&state, io).history_flush_failing);
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
acc.recordSuccess(io, "https://a.example", 60);
// No schema in this database, so the real write fails and the flag is set
// by the production path rather than by a test poking a field.
acc.flushOnce(io, &database, upstream_history_repo.flush);
try testing.expect(collect(&state, io).history_flush_failing);
try testing.expectEqualStrings("degraded", rollup(collect(&state, io)).status);
}
test "the body reports every input verbatim" {
const body = rollup(.{
.disk_state = .warn,
.disk = .{ .free_bytes = 100, .db_bytes = 20, .log_bytes = 3 },
.disk_sample_failures = 2,
.upstreams_available = 2,
.upstreams_total = 4,
.queries_dropped = 9,
.writer_failed = false,
.refreshes_gated = 1,
.snapshot_generation = 12,
});
try testing.expectEqualStrings("degraded", body.status);
try testing.expectEqualStrings("warn", body.disk.state);
try testing.expectEqual(@as(u64, 100), body.disk.free_bytes);
try testing.expectEqual(@as(u64, 20), body.disk.db_bytes);
try testing.expectEqual(@as(u64, 3), body.disk.log_bytes);
try testing.expectEqual(@as(u64, 2), body.disk.sample_failures);
try testing.expectEqual(@as(u32, 2), body.upstreams.available);
try testing.expectEqual(@as(u32, 4), body.upstreams.total);
try testing.expectEqual(@as(u64, 9), body.queries_dropped);
try testing.expectEqual(@as(u64, 1), body.refreshes_gated);
try testing.expectEqual(@as(?u64, 12), body.snapshot_generation);
}
test "an unpublished snapshot serializes as null, not as zero" {
var buffer: [512]u8 = undefined;
var writer: std.Io.Writer = .fixed(&buffer);
try std.json.Stringify.value(rollup(.{}), .{}, &writer);
try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"snapshot_generation\":null"));
}
test "collect reads the logger's counters and reports a bare state as degraded" {
test "collect reads the logger's counters and the pause flag" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
@@ -402,14 +490,25 @@ test "collect reads the logger's counters and reports a bare state as degraded"
var queue_buf: [2]logger_mod.Entry = undefined;
var query_logger: logger_mod.Logger = .init(.{}, &queue_buf);
query_logger.queries_dropped.store(4, .monotonic);
query_logger.last_drop_s.store(1_700_000_000, .monotonic);
query_logger.writer_failed.store(true, .monotonic);
var state: server.WebState = .{ .gpa = testing.allocator, .logger = &query_logger };
var paused: pause_mod.Pause = .{};
paused.pauseFor(0, null);
var state: server.WebState = .{
.gpa = testing.allocator,
.logger = &query_logger,
.pause = &paused,
};
const input = collect(&state, io);
try testing.expectEqual(@as(u64, 4), input.queries_dropped);
try testing.expectEqual(@as(?i64, 1_700_000_000), input.last_drop_s);
try testing.expect(input.writer_failed);
try testing.expectEqual(@as(i64, -1), input.pause_until);
try testing.expectEqual(@as(u32, 0), input.upstreams_total);
try testing.expectEqual(@as(?u64, null), input.snapshot_generation);
try testing.expectEqualStrings("degraded", rollup(input).status);
try testing.expectEqualStrings(status_degraded, rollup(input).status);
// No snapshot manager either, so protection outranks the pause here too.
try testing.expectEqualStrings(protection_unavailable, rollup(input).protection.state);
}
src/web/handlers/queries.zig
+36 -6
View File
@@ -126,8 +126,6 @@ pub fn list(
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
_ = io;
var buffers: Buffers = .{};
const filter = parseFilter(request.query, &buffers) catch |err| {
return http_util.respondError(request, .bad_request, message(err));
@@ -136,7 +134,7 @@ pub fn list(
const database = state.querylog_db orelse
return http_util.respondError(request, .service_unavailable, "query log unavailable");
const result = page(database, request.arena, filter) catch |err| {
const result = readPage(state, io, database, request.arena, filter) catch |err| {
// The one thing this handler logs: a database fault is a property of
// the box, not of the request, and the client is told nothing about it.
log.warn("query log read failed: {s}", .{@errorName(err)});
@@ -156,12 +154,10 @@ pub fn detail(
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
_ = io;
const database = state.querylog_db orelse
return http_util.respondError(request, .service_unavailable, "query log unavailable");
const row = queries_repo.detailById(database, request.arena, request.id.?) catch |err| {
const row = detailRow(state, io, database, request.arena, request.id.?) catch |err| {
log.warn("query log read failed: {s}", .{@errorName(err)});
return http_util.respondError(request, .internal_server_error, "internal error");
};
@@ -172,6 +168,40 @@ pub fn detail(
return http_util.respondJson(request, .ok, provenance_view.fromDetail(found), &.{});
}
/// The rows and the coverage watermark come from one database state, so a
/// prune between them cannot tag pre-prune rows with a post-prune
/// `available_since`.
fn readPage(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: Allocator,
filter: queries_repo.QueryFilter,
) db.Error!Page {
var scope = try server.QuerylogRead.open(state, io, database);
errdefer scope.abort();
const result = try page(database, arena, filter);
try scope.commit();
return result;
}
/// One row, read under the shared lock. There is nothing to keep consistent
/// with a second statement here; the lock is what keeps this read out of
/// another response's open transaction.
fn detailRow(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: Allocator,
id: i64,
) db.Error!?queries_repo.QueryDetail {
var scope = try server.QuerylogRead.open(state, io, database);
errdefer scope.abort();
const row = try queries_repo.detailById(database, arena, id);
try scope.commit();
return row;
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
src/web/handlers/stats.zig
+231 -23
View File
@@ -1,8 +1,16 @@
//! `GET /api/stats` and `GET /api/stats/timeseries` (ruling 13).
//! The five period endpoints: `GET /api/stats` and `/api/stats/timeseries`
//! (ruling 13), and `/api/stats/types`, `/api/stats/routes` and
//! `/api/stats/clients` (milestone 30).
//!
//! One period grammar, four widths, and one window shared by both endpoints:
//! the totals cover exactly the span the chart draws, so a dashboard cannot
//! show a sum that disagrees with the bars above it.
//! One period grammar, four widths, and one window shared by all five: a
//! request for the same period gets the same `since`/`until` from every
//! endpoint, so the totals describe exactly the span the charts draw rather
//! than a neighbouring one.
//!
//! That is window coherence, not identical counts. Each endpoint is its own
//! request against its own snapshot, so queries logged between two of them move
//! one panel and not the other. Only a box with nothing writing to it — a test
//! — can expect the breakdowns to sum to the totals exactly.
//!
//! Buckets are aligned to the UTC grid, not to the moment of the request. Every
//! width divides a day, so flooring the current time to a multiple of the width
@@ -10,7 +18,19 @@
//! requests a second apart return the same bucket starts. The last bucket is
//! the one in progress; it fills as the period runs.
//!
//! The aggregates run on the web task's own query-log connection (m7 ruling 21).
//! The aggregates run on the web task's own query-log connection (m7 ruling 21),
//! which every connection task shares. SQLite's serialized mode makes one call
//! safe; it does not make a transaction safe, so `WebState.querylog_lock` covers
//! the whole read and a second BEGIN can never land inside the first. Each
//! response takes one deferred read transaction, so its aggregate and the
//! `coverage` beside it describe one database state: retention cannot prune
//! between them and hand a client pre-prune rows tagged with a post-prune
//! watermark. Deferred, not `db.Tx`'s BEGIN IMMEDIATE, which would stall the
//! logger and retention behind an HTTP response.
//!
//! The lock is released before the response is written: the body is already
//! built in the request arena, and holding a database lock across a socket
//! write would let one slow client serialize every other reader.
const std = @import("std");
@@ -97,7 +117,6 @@ pub const TotalsBody = struct {
until: i64,
queries: u64,
blocked: u64,
cached: u64,
clients: u64,
avg_response_time_us: ?i64,
/// Judged against `since`, which is the window this body reports on — so a
@@ -115,6 +134,135 @@ pub const TimeseriesBody = struct {
coverage: coverage.Coverage,
};
pub const TypesBody = struct {
period: []const u8,
since: i64,
until: i64,
types: []const queries_repo.TypeCount,
coverage: coverage.Coverage,
};
pub const RoutesBody = struct {
period: []const u8,
since: i64,
until: i64,
routes: []const queries_repo.RouteCount,
coverage: coverage.Coverage,
};
pub const ClientsBody = struct {
period: []const u8,
since: i64,
until: i64,
bucket_seconds: u32,
clients: []const queries_repo.ClientSeries,
other: []const u64,
coverage: coverage.Coverage,
};
/// Everything one response reads from the query log, so the caller can end the
/// transaction and drop the lock before it serializes anything.
fn Read(comptime T: type) type {
return struct {
data: T,
coverage: coverage.Coverage,
};
}
const ReadScope = server.QuerylogRead;
fn readTotals(
state: *server.WebState,
io: std.Io,
database: *db.Db,
span: Window,
) db.Error!Read(queries_repo.StatsTotals) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const read: Read(queries_repo.StatsTotals) = .{
.data = try queries_repo.statsTotals(database, span.since, span.until),
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readTimeseries(
state: *server.WebState,
io: std.Io,
database: *db.Db,
span: Window,
out: []queries_repo.Bucket,
) db.Error!Read(usize) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const read: Read(usize) = .{
.data = try queries_repo.timeseries(database, span.since, span.bucket_seconds, out),
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readTypes(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
span: Window,
) db.Error!Read([]const queries_repo.TypeCount) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const list = try queries_repo.statsTypes(database, arena, span.since, span.until);
const read: Read([]const queries_repo.TypeCount) = .{
.data = list.items,
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readRoutes(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
span: Window,
) db.Error!Read([]const queries_repo.RouteCount) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const list = try queries_repo.statsRoutes(database, arena, span.since, span.until);
const read: Read([]const queries_repo.RouteCount) = .{
.data = list.items,
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readClients(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
span: Window,
) db.Error!Read(queries_repo.ClientsBreakdown) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const read: Read(queries_repo.ClientsBreakdown) = .{
.data = try queries_repo.statsClients(
database,
arena,
span.since,
span.bucket_seconds,
span.bucket_count,
),
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
pub fn totals(
state: *server.WebState,
io: std.Io,
@@ -124,23 +272,19 @@ pub fn totals(
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const result = queries_repo.statsTotals(database, span.since, span.until) catch |err| {
const read = readTotals(state, io, database, span) catch |err| {
return internal(request, "stats totals", err);
};
const covered = coverage.read(database, span.since) catch |err| {
return internal(request, "stats coverage", err);
};
return http_util.respondJson(request, .ok, TotalsBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.queries = result.queries,
.blocked = result.blocked,
.cached = result.cached,
.clients = result.distinct_clients,
.avg_response_time_us = result.avg_response_time_us,
.coverage = covered,
.queries = read.data.queries,
.blocked = read.data.blocked,
.clients = read.data.distinct_clients,
.avg_response_time_us = read.data.avg_response_time_us,
.coverage = read.coverage,
}, &.{});
}
@@ -155,20 +299,85 @@ pub fn timeseries(
var buckets: [max_buckets]queries_repo.Bucket = undefined;
const out = buckets[0..span.bucket_count];
const written = queries_repo.timeseries(database, span.since, span.bucket_seconds, out) catch |err| {
const read = readTimeseries(state, io, database, span, out) catch |err| {
return internal(request, "stats timeseries", err);
};
const covered = coverage.read(database, span.since) catch |err| {
return internal(request, "stats coverage", err);
};
return http_util.respondJson(request, .ok, TimeseriesBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.bucket_seconds = span.bucket_seconds,
.buckets = out[0..written],
.coverage = covered,
.buckets = out[0..read.data],
.coverage = read.coverage,
}, &.{});
}
pub fn types(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readTypes(state, io, database, request.arena, span) catch |err| {
return internal(request, "stats types", err);
};
return http_util.respondJson(request, .ok, TypesBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.types = read.data,
.coverage = read.coverage,
}, &.{});
}
pub fn routes(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readRoutes(state, io, database, request.arena, span) catch |err| {
return internal(request, "stats routes", err);
};
return http_util.respondJson(request, .ok, RoutesBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.routes = read.data,
.coverage = read.coverage,
}, &.{});
}
pub fn clients(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readClients(state, io, database, request.arena, span) catch |err| {
return internal(request, "stats clients", err);
};
return http_util.respondJson(request, .ok, ClientsBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.bucket_seconds = span.bucket_seconds,
.clients = read.data.clients,
.other = read.data.other,
.coverage = read.coverage,
}, &.{});
}
@@ -321,7 +530,6 @@ test "the totals and the buckets agree over the same window" {
const result = try queries_repo.statsTotals(&database, span.since, span.until);
try testing.expectEqual(@as(u64, 3), result.queries);
try testing.expectEqual(@as(u64, 1), result.blocked);
try testing.expectEqual(@as(u64, 1), result.cached);
try testing.expectEqual(@as(u64, 1), result.distinct_clients);
try testing.expectEqual(@as(?i64, 1000), result.avg_response_time_us);
src/web/handlers/upstream_health.zig
-585
View File
@@ -1,585 +0,0 @@
//! `GET /api/upstream/health?period=` — the pool's upstreams over the window
//! the dashboard's period picker selected (milestone-26 ruling 6).
//!
//! Two kinds of fact, kept apart on the wire because they answer different
//! questions. `enabled`/`available` are live routing state, read from the pool
//! under its mutex: what the resolver would do with this upstream right now.
//! Everything under `period` is history, aggregated out of `upstream_minute`
//! over `[since, until)` — the same window `/api/stats` reports, so a page
//! cannot show a rate that disagrees with the chart beside it.
//!
//! Nothing here reads a process-lifetime counter. The lifetime totals, the
//! last-32-exchange window and the consecutive-failure count still live in
//! `health.State` for routing and in `/metrics`; they are not this response's
//! business, because a number that starts at process start cannot be scoped to
//! a period and a dashboard that shows one beside a picker lies about it.
//!
//! The aggregation runs on the web task's own query-log connection (m7 ruling
//! 21) and this file owns no SQL: `upstream_history_repo` does.
const std = @import("std");
const Allocator = std.mem.Allocator;
const db = @import("../../storage/db.zig");
const history_mod = @import("../../upstream/history.zig");
const http_util = @import("../http_util.zig");
const metrics = @import("../metrics.zig");
const pool_mod = @import("../../upstream/pool.zig");
const server = @import("../server.zig");
const stats = @import("stats.zig");
const upstream_history_repo = @import("../../storage/repositories/upstream_history_repo.zig");
const log = std.log.scoped(.web_upstream_health);
/// One upstream's outcomes inside the selected window.
pub const PeriodStats = struct {
attempts: u64,
successes: u64,
failures: u64,
/// Null when `attempts == 0`. No observations is not perfect reliability,
/// and a `100.0%` from an idle upstream is the exact misreading this
/// milestone exists to remove.
success_rate: ?f32,
/// The newest failure inside the window, on the wall clock the minute rows
/// are stamped with. Null when the window holds no failure, even if the
/// upstream failed before it.
last_failure_at: ?i64,
/// The error name belonging to `last_failure_at`; null exactly when it is.
last_failure_error: ?[]const u8,
};
pub const Upstream = struct {
url: []const u8,
/// Live: configuration, not history.
enabled: bool,
/// Live: false while the upstream is backing off.
available: bool,
period: PeriodStats,
};
pub const Body = struct {
period: []const u8,
since: i64,
until: i64,
available: u32,
total: u32,
/// See `isComplete`.
complete: bool,
upstreams: []const Upstream,
};
/// The same text `/api/stats` sends (`stats.zig`'s `badPeriod`). One period
/// grammar serves the whole dashboard, so the two routes must not disagree
/// about what a typo means.
pub const bad_period_message = "period must be one of 1h, 24h, 7d, 30d";
pub fn handle(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = stats.periodParam(request.query) catch
return http_util.respondError(request, .bad_request, bad_period_message);
const pool = state.pool orelse
return http_util.respondError(request, .service_unavailable, "no upstream pool");
const database = state.querylog_db orelse
return http_util.respondError(request, .service_unavailable, "query log unavailable");
const now = std.Io.Clock.real.now(io).toSeconds();
const body = collect(request.arena, io, pool, state.history, database, period, now) catch |err| {
if (err == error.OutOfMemory) return error.OutOfMemory;
// A failed aggregate is a fault in the box, not a property of the
// request (ruling 8, PLAN §19).
log.warn("upstream health window failed: {s}", .{@errorName(err)});
return http_util.respondError(request, .internal_server_error, "internal error");
};
return http_util.respondJson(request, .ok, body, &.{});
}
/// `db.Error` already carries `OutOfMemory`, so the arena's failures and
/// SQLite's share one set.
pub const Error = Allocator.Error || db.Error;
pub fn collect(
arena: Allocator,
io: std.Io,
pool: *pool_mod.Pool,
history: ?*history_mod.Accumulator,
database: *db.Db,
period: stats.Period,
now_unix: i64,
) Error!Body {
const span = stats.window(period, now_unix);
var raw: [metrics.max_upstreams]pool_mod.Snapshot = undefined;
const count = metrics.poolSnapshot(pool, io, &raw);
const out = try arena.alloc(Upstream, count);
var available: u32 = 0;
for (raw[0..count], out) |entry, *slot| {
if (entry.available) available += 1;
// Rows come from the current pool only: an upstream deleted from the
// configuration keeps its history in storage until retention takes it,
// and nothing joins it back into this response.
const window_stats = try upstream_history_repo.windowStats(
database,
entry.url,
span.since,
span.until,
);
slot.* = .{
.url = try arena.dupe(u8, entry.url),
.enabled = entry.enabled,
.available = entry.available,
.period = .{
.attempts = window_stats.attempts,
.successes = window_stats.successes,
.failures = window_stats.failures,
.success_rate = successRate(window_stats),
.last_failure_at = window_stats.last_failure_ts,
// `WindowStats` carries its error name by value, in storage this
// loop is done with as soon as the iteration ends. The copy into
// the arena is what keeps the response from pointing at bytes
// the next upstream's row overwrites.
.last_failure_error = if (window_stats.last_failure_ts == null)
null
else
try arena.dupe(u8, window_stats.lastFailureError()),
},
};
}
return .{
.period = period.label(),
.since = span.since,
.until = span.until,
.available = available,
.total = @intCast(count),
.complete = isComplete(history, io, span.since),
.upstreams = out,
};
}
fn successRate(window_stats: upstream_history_repo.WindowStats) ?f32 {
if (window_stats.attempts == 0) return null;
const successes: f32 = @floatFromInt(window_stats.successes);
const attempts: f32 = @floatFromInt(window_stats.attempts);
return successes / attempts;
}
/// Per-window and stateless (ruling 6): false iff capacity has cost this
/// process a minute that falls inside the window. A window that starts after
/// the newest such minute is complete again, so one historical overflow does
/// not mark every later response.
///
/// It says nothing about the newest outcomes, which may not have flushed yet,
/// and nothing about an unclean shutdown, which is not detectable here — the
/// openapi description spells both out.
fn isComplete(history: ?*history_mod.Accumulator, io: std.Io, since: i64) bool {
const accumulator = history orelse return true;
const dropped = accumulator.snapshotStats(io).last_drop_minute orelse return true;
return dropped < since;
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const querylog_schema = @import("../../storage/querylog_schema.zig");
const transport = @import("../../upstream/transport.zig");
const testing = std.testing;
/// The client is never called: every test here reads health, not answers.
fn testEntry(url: []const u8, enabled: bool) pool_mod.Entry {
return .{
.endpoint = transport.Endpoint.parse(url) catch unreachable,
.client = .{ .ptr = undefined, .exchangeFn = undefined },
.priority = 1,
.enabled = enabled,
.health = .init,
};
}
fn testPool(entries: []pool_mod.Entry) pool_mod.Pool {
return .init(entries, .{}, .{
.attempt = .{ .raw = .fromMilliseconds(50), .clock = .awake },
.total = .{ .raw = .fromMilliseconds(100), .clock = .awake },
}, 1);
}
fn openLog() !db.Db {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer database.close();
try db.applyPragmas(&database, .{});
try database.exec(querylog_schema.ddl);
return database;
}
const url_a = "https://a.test/dns-query";
const url_b = "https://b.test/dns-query";
/// A minute-aligned instant, so a window derived from it lands on round
/// numbers the assertions below can name.
const aligned_now: i64 = 1_699_999_980;
test "the window sums the minutes inside it and nothing outside" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
try upstream_history_repo.flush(&database, &.{
// One minute before the window.
.{ .url = url_a, .minute_ts = span.since - 60, .successes = 100, .failures = 100, .last_failure_ts = span.since - 30, .last_error = "Outside" },
.{ .url = url_a, .minute_ts = span.since, .successes = 3, .failures = 1, .last_failure_ts = span.since + 10, .last_error = "Timeout" },
.{ .url = url_a, .minute_ts = span.until - 60, .successes = 5, .failures = 0, .last_failure_ts = null, .last_error = "" },
// The window's exclusive end.
.{ .url = url_a, .minute_ts = span.until, .successes = 200, .failures = 200, .last_failure_ts = span.until + 5, .last_error = "After" },
});
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
try testing.expectEqualStrings("1h", body.period);
try testing.expectEqual(span.since, body.since);
try testing.expectEqual(span.until, body.until);
try testing.expectEqual(@as(u32, 1), body.total);
try testing.expectEqual(@as(u32, 1), body.available);
const period = body.upstreams[0].period;
try testing.expectEqual(@as(u64, 8), period.successes);
try testing.expectEqual(@as(u64, 1), period.failures);
try testing.expectEqual(@as(u64, 9), period.attempts);
try testing.expectEqual(@as(?i64, span.since + 10), period.last_failure_at);
try testing.expectEqualStrings("Timeout", period.last_failure_error.?);
}
test "an upstream with no attempts in the window reports null, never a perfect rate" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
// Only `b` has history, and only outside the window.
try upstream_history_repo.flush(&database, &.{
.{ .url = url_b, .minute_ts = span.since - 600, .successes = 4, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
var entries = [_]pool_mod.Entry{ testEntry(url_a, true), testEntry(url_b, true) };
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
for (body.upstreams) |upstream| {
try testing.expectEqual(@as(u64, 0), upstream.period.attempts);
try testing.expectEqual(@as(u64, 0), upstream.period.successes);
try testing.expectEqual(@as(u64, 0), upstream.period.failures);
try testing.expectEqual(@as(?f32, null), upstream.period.success_rate);
try testing.expectEqual(@as(?i64, null), upstream.period.last_failure_at);
try testing.expectEqual(@as(?[]const u8, null), upstream.period.last_failure_error);
}
}
test "the success rate is the window's own, not a lifetime one" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
try upstream_history_repo.flush(&database, &.{
// A clean past that a lifetime rate would average into the window.
.{ .url = url_a, .minute_ts = span.since - 600, .successes = 1000, .failures = 0, .last_failure_ts = null, .last_error = "" },
.{ .url = url_a, .minute_ts = span.since, .successes = 1, .failures = 3, .last_failure_ts = span.since + 1, .last_error = "Timeout" },
});
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
try testing.expectEqual(@as(?f32, 0.25), body.upstreams[0].period.success_rate);
}
test "the newest failure inside the window wins over an older one outside it" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
try upstream_history_repo.flush(&database, &.{
.{ .url = url_a, .minute_ts = span.since - 120, .successes = 0, .failures = 1, .last_failure_ts = span.since - 100, .last_error = "Older" },
.{ .url = url_a, .minute_ts = span.since, .successes = 0, .failures = 1, .last_failure_ts = span.since + 5, .last_error = "Newer" },
// A later minute with no failure at all must not blank the error.
.{ .url = url_a, .minute_ts = span.since + 60, .successes = 2, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
try testing.expectEqual(@as(?i64, span.since + 5), body.upstreams[0].period.last_failure_at);
try testing.expectEqualStrings("Newer", body.upstreams[0].period.last_failure_error.?);
}
test "two upstreams keep their own last-failure errors" {
// The by-value `WindowStats` buffer is reused per iteration, so a response
// that borrowed it would show the second upstream's error on the first, or
// point at stack storage that is gone by the time it is serialized.
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
try upstream_history_repo.flush(&database, &.{
.{ .url = url_a, .minute_ts = span.since, .successes = 1, .failures = 1, .last_failure_ts = span.since + 1, .last_error = "ConnectFailed" },
.{ .url = url_b, .minute_ts = span.since, .successes = 0, .failures = 2, .last_failure_ts = span.since + 2, .last_error = "TlsHandshakeFailed" },
});
var entries = [_]pool_mod.Entry{ testEntry(url_a, true), testEntry(url_b, true) };
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
try testing.expectEqual(@as(usize, 2), body.upstreams.len);
try testing.expectEqualStrings(url_a, body.upstreams[0].url);
try testing.expectEqualStrings("ConnectFailed", body.upstreams[0].period.last_failure_error.?);
try testing.expectEqualStrings(url_b, body.upstreams[1].url);
try testing.expectEqualStrings("TlsHandshakeFailed", body.upstreams[1].period.last_failure_error.?);
// Serializing after every row is read is what a real response does; the
// texts must still be the ones their own rows carried.
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
defer allocating.deinit();
try std.json.Stringify.value(body, .{}, &allocating.writer);
const text = allocating.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"last_failure_error\":\"ConnectFailed\""));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"last_failure_error\":\"TlsHandshakeFailed\""));
}
test "a disabled upstream is not counted available and still gets its window" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"24h", aligned_now);
try upstream_history_repo.flush(&database, &.{
.{ .url = url_b, .minute_ts = span.since, .successes = 2, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
var entries = [_]pool_mod.Entry{ testEntry(url_a, true), testEntry(url_b, false) };
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"24h", aligned_now);
try testing.expectEqual(@as(u32, 2), body.total);
try testing.expectEqual(@as(u32, 1), body.available);
try testing.expect(!body.upstreams[1].enabled);
try testing.expect(!body.upstreams[1].available);
try testing.expectEqual(@as(u64, 2), body.upstreams[1].period.attempts);
}
/// Fills the accumulator and then overflows it, so `last_drop_minute` is
/// `minute` — the only way to set it, because the accumulator's fields are
/// private to its module and `snapshotStats` is the read surface.
fn accumulatorDroppingAt(
io: std.Io,
minute: i64,
names: *[history_mod.max_pending][8]u8,
) !*history_mod.Accumulator {
const accumulator = try testing.allocator.create(history_mod.Accumulator);
accumulator.* = .init;
for (names, 0..) |*name, i| {
const url = std.fmt.bufPrint(name, "u{d:0>6}", .{i}) catch unreachable;
accumulator.recordSuccess(io, url, minute);
}
// One more cell than capacity: the oldest minute goes, and every cell above
// holds `minute`.
accumulator.recordSuccess(io, "https://overflow.test", minute + 60);
return accumulator;
}
test "complete is false only while a dropped minute falls inside the window" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
// Two `now`s one minute apart, so the same drop is inside the first
// window and one minute before the second.
const inside_now = aligned_now;
const inside = stats.window(.@"1h", inside_now);
const after = stats.window(.@"1h", inside_now + 60);
try testing.expectEqual(inside.since + 60, after.since);
var names: [history_mod.max_pending][8]u8 = undefined;
const accumulator = try accumulatorDroppingAt(io, inside.since, &names);
defer testing.allocator.destroy(accumulator);
try testing.expectEqual(@as(?i64, inside.since), accumulator.snapshotStats(io).last_drop_minute);
const flagged = try collect(arena.allocator(), io, &pool, accumulator, &database, .@"1h", inside_now);
try testing.expect(!flagged.complete);
const recovered = try collect(arena.allocator(), io, &pool, accumulator, &database, .@"1h", inside_now + 60);
try testing.expect(recovered.complete);
// No accumulator at all is no known drop, not an incomplete window.
const unwired = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", inside_now);
try testing.expect(unwired.complete);
}
test "a drop with no overflow leaves every window complete" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
const accumulator = try testing.allocator.create(history_mod.Accumulator);
defer testing.allocator.destroy(accumulator);
accumulator.* = .init;
accumulator.recordFailure(io, url_a, aligned_now, "Timeout");
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, accumulator, &database, .@"1h", aligned_now);
try testing.expect(body.complete);
}
test "the route's period grammar and its 400 text are the ones /api/stats serves" {
// The picker scopes the whole page, so one bad spelling must mean the same
// thing on every route it drives.
try testing.expectEqual(stats.default_period, try stats.periodParam(""));
try testing.expectEqual(stats.Period.@"24h", try stats.periodParam(""));
try testing.expectEqual(stats.Period.@"7d", try stats.periodParam("period=7d"));
try testing.expectError(error.BadPeriod, stats.periodParam("period=12h"));
try testing.expectError(error.BadPeriod, stats.periodParam("period=1hhhhhhhhhh"));
try testing.expectEqualStrings("period must be one of 1h, 24h, 7d, 30d", bad_period_message);
}
test "every period the grammar accepts produces the window that period names" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
for (std.enums.values(stats.Period)) |period| {
const span = stats.window(period, aligned_now);
const body = try collect(arena.allocator(), io, &pool, null, &database, period, aligned_now);
try testing.expectEqualStrings(period.label(), body.period);
try testing.expectEqual(span.since, body.since);
try testing.expectEqual(span.until, body.until);
}
}
test "the body serializes exactly the ranged field set" {
const upstreams = [_]Upstream{ .{
.url = url_a,
.enabled = true,
.available = false,
.period = .{
.attempts = 8,
.successes = 6,
.failures = 2,
.success_rate = 0.75,
.last_failure_at = 1_700_000_000,
.last_failure_error = "ConnectFailed",
},
}, .{
.url = url_b,
.enabled = false,
.available = false,
.period = .{
.attempts = 0,
.successes = 0,
.failures = 0,
.success_rate = null,
.last_failure_at = null,
.last_failure_error = null,
},
} };
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
defer allocating.deinit();
try std.json.Stringify.value(Body{
.period = "1h",
.since = 1_699_996_400,
.until = 1_700_000_000,
.available = 0,
.total = 2,
.complete = true,
.upstreams = &upstreams,
}, .{}, &allocating.writer);
try testing.expectEqualStrings(
\\{"period":"1h","since":1699996400,"until":1700000000,"available":0,"total":2,"complete":true,"upstreams":[{"url":"https://a.test/dns-query","enabled":true,"available":false,"period":{"attempts":8,"successes":6,"failures":2,"success_rate":0.75,"last_failure_at":1700000000,"last_failure_error":"ConnectFailed"}},{"url":"https://b.test/dns-query","enabled":false,"available":false,"period":{"attempts":0,"successes":0,"failures":0,"success_rate":null,"last_failure_at":null,"last_failure_error":null}}]}
, allocating.written());
// The lifetime fields m26 removed. They still exist in `health.State` and in
// `/metrics`; a client of this route must not find them here and start
// reading them as if they were scoped to the period.
for ([_][]const u8{
"consecutive_failures",
"total_successes",
"total_failures",
"last_error_age_s",
"\"last_error\"",
}) |gone| {
try testing.expect(!std.mem.containsAtLeast(u8, allocating.written(), 1, gone));
}
}
src/web/metrics.zig
+1 -62
View File
@@ -31,7 +31,6 @@ const dns_handler = @import("../server/handler.zig");
const disk_monitor = @import("../storage/disk_monitor.zig");
const dot_server = @import("../server/dot_server.zig");
const events_mod = @import("../storage/events.zig");
const history_mod = @import("../upstream/history.zig");
const http_util = @import("http_util.zig");
const logging = @import("../platform/logging.zig");
const pool_mod = @import("../upstream/pool.zig");
@@ -136,9 +135,6 @@ pub const Sample = struct {
tracker: ?TrackerSample = null,
client_names: ?client_names.Resolver.Stats = null,
retention: ?retention_mod.Stats = null,
/// The upstream-history flush loop's counters (m26 ruling 7). Absent while
/// no accumulator is wired, like every other collaborator.
history: ?history_mod.Accumulator.Stats = null,
/// The diagnostics store's open episodes and its failed writes. Absent
/// while no store is wired, like every other collaborator — an operator
/// distinguishes "no series" from "zero episodes" through `/api/health`,
@@ -218,8 +214,6 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
if (state.retention) |retention| sample.retention = retention.snapshotStats();
if (state.history) |history| sample.history = history.snapshotStats(io);
if (state.events) |store| {
const counts = store.activeCounts(io);
sample.diagnostics = .{
@@ -382,36 +376,6 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
try counterGroup(w, "nxdns_retention_", "Query log retention counter", retention);
}
if (sample.history) |history| {
// Written out rather than reflected over `Accumulator.Stats`: three of
// its fields are counters, one is a gauge, and two — the drop watermark
// and the current flush state — are not exposition numbers at all.
try counter(
w,
"nxdns_upstream_history_flushes_total",
"Upstream history flush transactions that committed.",
history.flushes,
);
try counter(
w,
"nxdns_upstream_history_flush_failures_total",
"Upstream history flush attempts that failed; the rows are retried on the next pass.",
history.flush_failures,
);
try counter(
w,
"nxdns_upstream_history_rows_dropped_total",
"Upstream history minutes dropped because the accumulator was full.",
history.rows_dropped,
);
try gauge(
w,
"nxdns_upstream_history_pending",
"Upstream history minutes recorded but not yet flushed.",
history.pending,
);
}
if (sample.diagnostics) |diagnostics| {
try gauge(
w,
@@ -638,7 +602,7 @@ fn writeUpstreamLabels(
/// through this function too, so the guarantee cannot be one caller away.
/// `UpstreamSample.url` stays whole for the same reason it is safe to: nothing
/// but this function reads it, and the session-authenticated
/// `GET /api/upstream/health` reports the same pool with the same urls whole.
/// `GET /api/upstreams` reports the same urls whole.
///
/// **`redact` output is not safe to interpolate into a label value, and this
/// function is the reason it never has to be.** Do not delete the second layer
@@ -803,31 +767,6 @@ test "a full sample renders the whole exposition, byte for byte" {
));
}
test "the upstream-history family renders three counters and one gauge" {
const text = try renderToString(testing.allocator, .{
.history = .{
.flushes = 12,
.flush_failures = 2,
.rows_dropped = 5,
.pending = 3,
.last_drop_minute = 1_700_000_040,
.last_flush_failed = true,
},
});
defer testing.allocator.free(text);
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_history_flushes_total 12\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_history_flush_failures_total 2\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_history_rows_dropped_total 5\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_upstream_history_pending gauge\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_history_pending 3\n"));
// No accumulator is an absent family, not a family of zeros.
const bare = try renderToString(testing.allocator, .{});
defer testing.allocator.free(bare);
try testing.expect(!std.mem.containsAtLeast(u8, bare, 1, "nxdns_upstream_history_"));
}
test "the diagnostics family renders two gauges and one counter" {
const text = try renderToString(testing.allocator, .{
.diagnostics = .{ .active_warnings = 3, .active_errors = 1, .write_failures = 7 },
src/web/openapi.yaml
+289 -131
View File
@@ -65,8 +65,11 @@ paths:
get:
summary: Health rollup
description: |
Always 200; `status` is `degraded` when the disk is not ok, no
upstream is available, or the query-log writer failed. Always
Always 200. `status` is `degraded` when, and only when, one of the five
condition objects is in a degrading state: protection `unavailable`,
upstreams `unavailable`, query history `losing` or `failed`,
diagnostics `unavailable`, or disk `low` or `critical`. A paused
protection is an operator's own choice and does not degrade. Always
unauthenticated and never rate limited.
security: []
responses:
@@ -465,6 +468,99 @@ paths:
"503":
$ref: "#/components/responses/Unavailable"
/api/stats/types:
get:
summary: Query-type breakdown for a period
description: |
How many queries of each DNS type the period's window holds, over the
same UTC-aligned window `/api/stats` reports for. Rows carry the numeric
type only: the type-name table lives in the admin, and a second copy
here would drift out of agreement with it. `qtype` is nullable in the
query log, so the rows that carry no type group into a row of their own
rather than vanishing from a breakdown that claims to add up. Ordered by
count descending, then type ascending with the null row last. Types
absent from the window are absent from the list.
parameters:
- $ref: "#/components/parameters/Period"
responses:
"200":
description: The type breakdown.
content:
application/json:
schema:
$ref: "#/components/schemas/StatsTypes"
"400":
$ref: "#/components/responses/BadRequest"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/stats/routes:
get:
summary: How the period's queries were answered
description: |
A breakdown by answering route over the same window `/api/stats`
reports for. `source` is the answering resolver's identity — the
upstream url on `upstream` rows, the zone on `forward_zone` rows, null
on every other kind and on rows whose identity the log did not record.
It is not the blocklist a block came from. Ordered by count descending,
then route ascending, then source ascending with nulls last.
parameters:
- $ref: "#/components/parameters/Period"
responses:
"200":
description: The route breakdown.
content:
application/json:
schema:
$ref: "#/components/schemas/StatsRoutes"
"400":
$ref: "#/components/responses/BadRequest"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/stats/clients:
get:
summary: Per-client bucketed counts for a period
description: |
One zero-filled series per client, bucketed exactly like
`/api/stats/timeseries` so the two charts share an x-axis. The eight
clients with the most queries in the window are named, ranked by count
descending then address ascending; every other client sums into
`other`, which is always present and always holds one entry per bucket
in the window — including when `clients` is empty, when no client fell
outside the named eight, and when the window holds no queries at all.
parameters:
- $ref: "#/components/parameters/Period"
responses:
"200":
description: The per-client series.
content:
application/json:
schema:
$ref: "#/components/schemas/StatsClients"
"400":
$ref: "#/components/responses/BadRequest"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/lookup:
get:
summary: Explain a domain
@@ -498,35 +594,6 @@ paths:
"503":
$ref: "#/components/responses/Unavailable"
/api/upstream/health:
get:
summary: Upstream pool health for a period
description: |
Each upstream's live routing state beside its recorded outcomes over
the period's window, which is the same UTC-aligned window `/api/stats`
reports for that period. The outcome counts come from per-minute
history in the query log, not from process-lifetime counters, so they
scope to the period and survive a restart.
parameters:
- $ref: "#/components/parameters/Period"
responses:
"200":
description: Per-upstream state and the availability rollup.
content:
application/json:
schema:
$ref: "#/components/schemas/UpstreamHealth"
"400":
$ref: "#/components/responses/BadRequest"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/groups:
get:
summary: List groups
@@ -1822,45 +1889,100 @@ components:
Health:
type: object
required: [status, disk, upstreams, diagnostics, queries_dropped, writer_failed, refreshes_gated, snapshot_generation]
required: [status, protection, upstreams, query_history, diagnostics, disk]
properties:
status:
type: string
enum: [ok, degraded]
diagnostics:
type: object
required: [state, active_warnings, active_errors]
properties:
state:
type: string
enum: [recording, unavailable]
description: unavailable when the event store failed to open or its writes are failing; either state degrades health.
active_warnings: { type: integer }
active_errors: { type: integer }
disk:
type: object
required: [state, free_bytes, db_bytes, log_bytes, sample_failures]
properties:
state:
type: string
enum: [ok, warn, critical]
free_bytes: { type: integer }
db_bytes: { type: integer }
log_bytes: { type: integer }
sample_failures: { type: integer }
protection:
$ref: "#/components/schemas/HealthProtection"
upstreams:
type: object
required: [available, total]
properties:
available: { type: integer }
total: { type: integer }
queries_dropped: { type: integer }
writer_failed: { type: boolean }
refreshes_gated: { type: integer }
snapshot_generation:
$ref: "#/components/schemas/HealthUpstreams"
query_history:
$ref: "#/components/schemas/HealthQueryHistory"
diagnostics:
$ref: "#/components/schemas/HealthDiagnostics"
disk:
$ref: "#/components/schemas/HealthDisk"
HealthProtection:
type: object
required: [state, until]
properties:
state:
type: string
enum: [active, paused, unavailable]
description: >
unavailable when no filter snapshot exists for the query path to
evaluate against, which outranks any pause and is the only one of
the three that degrades health. An expired timed pause is active.
until:
type: integer
nullable: true
description: Null until the first filter snapshot is published.
description: >
The second filtering resumes at. Null for an indefinite pause and
for every state other than paused.
HealthUpstreams:
type: object
required: [state, available, total]
properties:
state:
type: string
enum: [ok, unavailable]
description: unavailable exactly when `available` is 0; that degrades health.
available: { type: integer }
total:
type: integer
description: Enabled upstreams, which is what the routing pool is built from.
HealthQueryHistory:
type: object
required: [state, dropped_total, last_drop_s]
properties:
state:
type: string
enum: [recording, losing, failed]
description: >
failed when the query-log writer never started; losing while the
disk gate is holding writes back and has already cost rows in the
episode open now. Both degrade health. Drops from an earlier
episode do not change the state - they are reported by the two
fields below.
dropped_total:
type: integer
description: Query rows lost since this process started, cumulative.
last_drop_s:
type: integer
nullable: true
description: >
The newest drop, unix seconds; null until one happens. Stamped by a
separate atomic from the count, so a non-zero `dropped_total` beside
a null here is a legal momentary answer.
HealthDiagnostics:
type: object
required: [state, active_warnings, active_errors]
properties:
state:
type: string
enum: [recording, unavailable]
description: unavailable when the event store failed to open or its writes are failing; either state degrades health.
active_warnings: { type: integer }
active_errors: { type: integer }
HealthDisk:
type: object
required: [state, free_bytes]
properties:
state:
type: string
enum: [ok, low, critical]
description: >
The disk monitor's own states; its `warn` is renamed `low` here,
because `warn` reads as a log level rather than as a quantity of
disk. Both `low` and `critical` degrade health.
free_bytes: { type: integer }
Version:
type: object
@@ -2119,6 +2241,9 @@ components:
- query_log.write
- query_log.maintenance
- query_log.recreated
# Legacy: nothing emits this any more (milestone 30 deleted the
# upstream-minute history subsystem), but stored rows survive and
# the list endpoint passes their code through.
- upstream_history.write
- upstream.exchange
- client_names.storage
@@ -2187,7 +2312,7 @@ components:
StatsTotals:
type: object
required: [period, since, until, queries, blocked, cached, clients, avg_response_time_us, coverage]
required: [period, since, until, queries, blocked, clients, avg_response_time_us, coverage]
properties:
period:
type: string
@@ -2200,7 +2325,6 @@ components:
description: Window end, unix seconds, exclusive.
queries: { type: integer }
blocked: { type: integer }
cached: { type: integer }
clients:
type: integer
description: Distinct client addresses in the window.
@@ -2239,6 +2363,106 @@ components:
coverage:
$ref: "#/components/schemas/Coverage"
TypeCount:
type: object
required: [qtype, count]
properties:
qtype:
type: integer
nullable: true
description: |
The numeric DNS type. Null is the group of logged queries that
recorded no type, not an absent row.
count: { type: integer }
StatsTypes:
type: object
required: [period, since, until, types, coverage]
properties:
period:
type: string
enum: [1h, 24h, 7d, 30d]
since: { type: integer }
until: { type: integer }
types:
type: array
items:
$ref: "#/components/schemas/TypeCount"
coverage:
$ref: "#/components/schemas/Coverage"
RouteCount:
type: object
required: [route, source, count]
properties:
route:
$ref: "#/components/schemas/RouteKind"
source:
type: string
nullable: true
description: |
The answering upstream's url or the forward zone, and null on every
other route kind. Also null when an `upstream` or `forward_zone`
row recorded no identity.
count: { type: integer }
StatsRoutes:
type: object
required: [period, since, until, routes, coverage]
properties:
period:
type: string
enum: [1h, 24h, 7d, 30d]
since: { type: integer }
until: { type: integer }
routes:
type: array
items:
$ref: "#/components/schemas/RouteCount"
coverage:
$ref: "#/components/schemas/Coverage"
ClientSeries:
type: object
required: [client, buckets]
properties:
client:
type: string
description: The client address as the log recorded it, redaction included.
buckets:
type: array
description: |
One count per bucket in the window, zero-filled. Every series in a
response has this same length, `other` included.
items:
type: integer
StatsClients:
type: object
required: [period, since, until, bucket_seconds, clients, other, coverage]
properties:
period:
type: string
enum: [1h, 24h, 7d, 30d]
since: { type: integer }
until: { type: integer }
bucket_seconds: { type: integer }
clients:
type: array
items:
$ref: "#/components/schemas/ClientSeries"
other:
type: array
description: |
Every client outside the named eight, summed per bucket. Always
present, and always one entry per bucket in the window — including
when `clients` is empty, when no client fell outside the named
eight, and when the window holds no queries at all.
items:
type: integer
coverage:
$ref: "#/components/schemas/Coverage"
Lookup:
type: object
required: [domain, group_id, local_records, forward_zone, blocked, reason, matched, source_url, safe_search_rewrite]
@@ -2267,72 +2491,6 @@ components:
type: string
nullable: true
UpstreamPeriodStats:
type: object
required: [attempts, successes, failures, success_rate, last_failure_at, last_failure_error]
properties:
attempts:
type: integer
description: Exchanges recorded against this upstream inside the window.
successes: { type: integer }
failures: { type: integer }
success_rate:
type: number
nullable: true
description: >
`successes / attempts`, from 0 to 1. Null when `attempts` is 0: no
observations is not perfect reliability.
last_failure_at:
type: integer
nullable: true
description: >
The newest failure inside the window, unix seconds. Null when the
window holds no failure, even if the upstream failed before it.
last_failure_error:
type: string
nullable: true
description: The error name belonging to `last_failure_at`; null exactly when it is.
UpstreamHealth:
type: object
required: [period, since, until, available, total, complete, upstreams]
properties:
period:
type: string
enum: [1h, 24h, 7d, 30d]
since:
type: integer
description: Window start, unix seconds, inclusive.
until:
type: integer
description: Window end, unix seconds, exclusive.
available:
type: integer
description: How many upstreams the pool would route to right now.
total: { type: integer }
complete:
type: boolean
description: >
No capacity drops known in this process within the selected window;
up to about a minute of the newest outcomes may not have flushed
yet, and outcomes lost in an unclean shutdown are not detectable.
upstreams:
type: array
description: The upstreams configured now; a deleted upstream's history is not returned.
items:
type: object
required: [url, enabled, available, period]
properties:
url: { type: string }
enabled:
type: boolean
description: Live configuration, not history.
available:
type: boolean
description: Live state, not history; false while the upstream is backing off.
period:
$ref: "#/components/schemas/UpstreamPeriodStats"
Group:
type: object
required: [id, name, safe_search]
src/web/routes.zig
+4 -3
View File
@@ -49,7 +49,6 @@ const queries = @import("handlers/queries.zig");
const rules = @import("handlers/rules.zig");
const settings = @import("handlers/settings.zig");
const stats = @import("handlers/stats.zig");
const upstream_health = @import("handlers/upstream_health.zig");
const upstreams = @import("handlers/upstreams.zig");
const version = @import("handlers/version.zig");
@@ -73,8 +72,10 @@ pub const table: []const router.RouteInfo = &.{
.{ .method = .GET, .pattern = "/api/queries/{id}", .auth = .session, .policy = .read, .handler = queries.detail },
.{ .method = .GET, .pattern = "/api/stats", .auth = .session, .policy = .read, .handler = stats.totals },
.{ .method = .GET, .pattern = "/api/stats/timeseries", .auth = .session, .policy = .read, .handler = stats.timeseries },
.{ .method = .GET, .pattern = "/api/stats/types", .auth = .session, .policy = .read, .handler = stats.types },
.{ .method = .GET, .pattern = "/api/stats/routes", .auth = .session, .policy = .read, .handler = stats.routes },
.{ .method = .GET, .pattern = "/api/stats/clients", .auth = .session, .policy = .read, .handler = stats.clients },
.{ .method = .GET, .pattern = "/api/lookup", .auth = .session, .policy = .read, .handler = lookup.handle },
.{ .method = .GET, .pattern = "/api/upstream/health", .auth = .session, .policy = .read, .handler = upstream_health.handle },
// Diagnostics: the operational event log (milestone 27). The two purges are
// `runtime_action` — the event log is runtime state no configuration file
@@ -156,7 +157,7 @@ const std = @import("std");
const testing = std.testing;
test "the table carries every endpoint of the milestone" {
try testing.expectEqual(@as(usize, 61), table.len);
try testing.expectEqual(@as(usize, 63), table.len);
}
test "no two entries claim the same method and pattern" {
src/web/server.zig
+77 -5
View File
@@ -40,7 +40,6 @@ const logger_mod = @import("../storage/logger.zig");
const manager_mod = @import("../filter/manager.zig");
const model = @import("../config/model.zig");
const pause_mod = @import("../server/pause.zig");
const history_mod = @import("../upstream/history.zig");
const pool_mod = @import("../upstream/pool.zig");
const query_sink = @import("../server/query_sink.zig");
const retention_mod = @import("../storage/retention.zig");
@@ -137,10 +136,6 @@ pub const WebState = struct {
client_names: ?*client_names.Resolver = null,
manager: ?*manager_mod.Manager = null,
pool: ?*pool_mod.Pool = null,
/// The upstream-outcome accumulator, for `metrics.collect` and the
/// `/api/health` rollup (m26 ruling 7). The ranged endpoint reads the
/// flushed rows through `querylog_db`, not through this.
history: ?*history_mod.Accumulator = null,
monitor: ?*disk_monitor.Monitor = null,
/// The local records and forward zones the DNS path reads. The
/// local-records and forward-zones handlers rebuild and swap them
@@ -184,6 +179,13 @@ pub const WebState = struct {
/// concurrent writes would misread each other's row counts.
config_lock: std.Io.Mutex = .init,
querylog_db: ?*db.Db = null,
/// Serializes the web layer's work on `querylog_db`, for the same reason
/// `config_lock` exists and one more: the read handlers wrap their several
/// statements in a transaction, and SQLite's serialized mode protects a
/// single call, not a transaction. Without this, two concurrent BEGINs on
/// the shared connection would fail and a third task's reads would land
/// inside someone else's snapshot.
querylog_lock: std.Io.Mutex = .init,
/// The diagnostics event store, which owns a third connection of its own
/// and serializes every access — read and write — through its mutex. Null
/// when `Store.init` failed, which `/api/health` reports as `unavailable`
@@ -219,6 +221,76 @@ pub const WebState = struct {
reload_fn: ?ReloadFn = null,
};
/// One response's hold on the query log: `querylog_lock` plus one deferred read
/// transaction, opened and closed together so no reader can hold one without
/// the other.
///
/// Every web-layer read of `querylog_db` goes through this. The transaction is
/// what makes an aggregate and the coverage watermark beside it describe one
/// database state, and the lock is what makes the transaction meaningful on a
/// connection several tasks share.
///
/// `commit` is fallible and must be called before the response is written: a
/// connection still inside a transaction refuses the next `BEGIN`, so a handler
/// that answered 200 over a failed commit would leave every later query-log
/// request failing for a reason nothing on the wire ever named.
///
/// **The lock is always released, even when the transaction could not be
/// ended.** `lockUncancelable` cannot be interrupted, so holding it against a
/// connection that will not leave its transaction would park every later
/// query-log task forever, with no status and no way out but a kill. Releasing
/// it turns the same fault into a 500 per request: bounded, visible, and
/// recoverable by a restart.
/// **The lock is released exactly once, on every path.** The usage shape below
/// runs `abort` after a failed `commit` — an `errdefer` cannot know the error
/// came from the commit itself — so `release` is the single owner of the
/// unlock and `held` is what makes the second call a no-op. Unlocking an
/// already-unlocked `std.Io.Mutex` is `unreachable`, and under contention it
/// would hand away a hold another task had just taken, so the bounded 500 this
/// type promises would instead be a crash or a corrupted mutex.
///
/// ```zig
/// var scope = try QuerylogRead.open(state, io, database);
/// errdefer scope.abort();
/// ... // reads only
/// try scope.commit();
/// ```
pub const QuerylogRead = struct {
state: *WebState,
io: std.Io,
tx: db.ReadTx,
held: bool,
pub fn open(state: *WebState, io: std.Io, database: *db.Db) db.Error!QuerylogRead {
state.querylog_lock.lockUncancelable(io);
errdefer state.querylog_lock.unlock(io);
return .{
.state = state,
.io = io,
.tx = try db.ReadTx.begin(database),
.held = true,
};
}
pub fn commit(self: *QuerylogRead) db.Error!void {
defer self.release();
return self.tx.commit();
}
/// Safe in `errdefer`, and safe after `commit` however that ended: both the
/// rollback and the release are idempotent.
pub fn abort(self: *QuerylogRead) void {
self.tx.rollback();
self.release();
}
fn release(self: *QuerylogRead) void {
if (!self.held) return;
self.held = false;
self.state.querylog_lock.unlock(self.io);
}
};
/// Ruling 17. Authentication is enabled iff a password hash is set — the live
/// one, so a password set through the API locks the routes without a restart.
/// With it set but no session store wired, every session route is refused: the
src/web/web_integration_test.zig
+438 -11
View File
@@ -32,6 +32,7 @@ const clients_repo = @import("../storage/repositories/clients_repo.zig");
const db = @import("../storage/db.zig");
const dns_handler = @import("../server/handler.zig");
const events_mod = @import("../storage/events.zig");
const events_repo = @import("../storage/repositories/events_repo.zig");
const fetcher = @import("../filter/fetcher.zig");
const groups_repo = @import("../storage/repositories/groups_repo.zig");
const header = @import("../dns/header.zig");
@@ -72,7 +73,6 @@ const handlers_pause = @import("handlers/pause.zig");
const handlers_queries = @import("handlers/queries.zig");
const handlers_settings = @import("handlers/settings.zig");
const handlers_stats = @import("handlers/stats.zig");
const handlers_upstream_health = @import("handlers/upstream_health.zig");
const handlers_version = @import("handlers/version.zig");
const testing = std.testing;
@@ -282,6 +282,13 @@ const EnvOptions = struct {
/// `logging.query_log = false` operator runs. Every query-log route then
/// answers 503 rather than an empty page, which would be a lie.
querylog: bool = true,
/// Seeds a handful of rows inside the *live* period window, on top of the
/// fixed 2023 seed. The stats windows are cut from the real clock, so an
/// aggregation over a fixed seed is always an empty window — and an empty
/// array witnesses no field at all. Only the tests that need populated
/// aggregations ask for it: the rows are newer than every fixed row, so
/// they would otherwise move the query-log page out from under its golden.
recent_traffic: bool = false,
};
/// Heap-allocated because `state` and the listener hold pointers into it.
@@ -334,13 +341,14 @@ const Env = struct {
errdefer self.querylog_db.close();
try self.querylog_db.exec(querylog_schema.ddl);
try seedQueryLog(&self.querylog_db);
if (options.recent_traffic) try seedRecentTraffic(&self.querylog_db, std.Io.Clock.real.now(ioh).toSeconds());
self.events_db = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer self.events_db.close();
try db.applyPragmas(&self.events_db, .{});
_ = try migrations.migrate(&self.events_db);
self.events_store = try events_mod.Store.init(ioh, &self.events_db, seeded_now);
seedEvents(ioh, &self.events_store);
try seedEvents(ioh, &self.events_store, &self.events_db);
// Real fetcher wiring; nothing in this suite downloads (the one
// refreshAll in the contract walk runs with zero source rows).
@@ -381,7 +389,8 @@ const Env = struct {
self.pool_entries = .{.{
.endpoint = transport.Endpoint.parse("https://dns.example/dns-query") catch unreachable,
// Never exchanged with: the pool feeds /api/upstream/health only.
// Never exchanged with: the pool feeds `/metrics` and the
// `/api/health` upstream condition only.
.client = .{ .ptr = undefined, .exchangeFn = undefined },
.priority = 1,
.enabled = true,
@@ -588,6 +597,66 @@ fn seedQueryLog(database: *db.Db) !void {
}});
}
/// The matrix the three period aggregations are read against: three clients,
/// three query types including a row with none, five route kinds, two named
/// upstreams and one upstream row whose resolver the log did not record.
///
/// `now` is the real clock, so these rows land in the live window of every
/// period. Only their timestamps come from it; the counts are fixed, and the
/// contract samples canonicalize every number to zero anyway.
const recent_clients = 3;
fn seedRecentTraffic(database: *db.Db, now: i64) !void {
var writer = try queries_repo.BatchWriter.init(database);
defer writer.deinit();
const Shape = struct {
client: []const u8,
qtype: ?u16,
kind: provenance.RouteKind,
source: ?[]const u8,
};
const shapes = [_]Shape{
.{ .client = "192.0.2.30", .qtype = 1, .kind = .upstream, .source = "https://dns.example/dns-query" },
.{ .client = "192.0.2.30", .qtype = 1, .kind = .upstream, .source = "https://dns.example/dns-query" },
.{ .client = "192.0.2.30", .qtype = 28, .kind = .upstream, .source = "https://dns2.example/dns-query" },
.{ .client = "192.0.2.30", .qtype = 1, .kind = .upstream, .source = null },
.{ .client = "192.0.2.31", .qtype = 28, .kind = .blocked, .source = null },
.{ .client = "192.0.2.31", .qtype = 1, .kind = .cache, .source = null },
.{ .client = "192.0.2.31", .qtype = null, .kind = .local, .source = null },
.{ .client = "192.0.2.32", .qtype = 1, .kind = .forward_zone, .source = "lan" },
.{ .client = "192.0.2.32", .qtype = 1, .kind = .rejected, .source = null },
};
for (shapes, 0..) |shape, index| {
// Inside the narrowest bucket of the narrowest period, so every period
// sees the whole matrix however close to a boundary the clock is.
try writer.writeBatch(&.{.{
.timestamp = now - @as(i64, @intCast(index)) - 1,
.domain = "recent.example",
.client_ip = shape.client,
.qtype = shape.qtype,
.qclass = 1,
.rcode = 0,
.blocked = shape.kind == .blocked,
.response_time_us = 1500,
.cache_hit = shape.kind == .cache,
.upstream = if (shape.kind == .upstream) shape.source else null,
.group_id = 1,
.group_name = "default",
.policy_action = if (shape.kind == .blocked) .block else .allow,
.policy_reason = if (shape.kind == .blocked) .blocklist_domain else .no_match,
.matched = null,
.source_id = null,
.source_name = null,
.cname_target = null,
.safe_search_target = null,
.route_kind = shape.kind,
.forward_zone = if (shape.kind == .forward_zone) shape.source else null,
}});
}
}
/// A fixed instant, like every other seeded timestamp here: the contract
/// samples are byte-compared, so nothing the walk writes may come from a clock.
const seeded_now: i64 = 1_787_118_000;
@@ -595,12 +664,17 @@ const seeded_now: i64 = 1_787_118_000;
/// One active episode and one resolved one, so `/api/diagnostics` answers with
/// both states and the committed contract sample describes a real page rather
/// than an empty one.
fn seedEvents(io: std.Io, store: *events_mod.Store) void {
fn seedEvents(io: std.Io, store: *events_mod.Store, database: *db.Db) !void {
store.report(io, seeded_now, .blocklist_refresh, "https://lists.example/ads.txt", "StevenBlack", .warning, "download failed: ConnectionTimedOut");
store.report(io, seeded_now + 300, .blocklist_refresh, "https://lists.example/ads.txt", "StevenBlack", .warning, "download failed: ConnectionTimedOut");
store.report(io, seeded_now + 60, .upstream_history_write, "history", "history", .warning, "Busy");
store.resolve(io, seeded_now + 120, .upstream_history_write, "history");
// A legacy code no producer emits any more. Rows written by an m29 process
// survive, and the read path has to keep passing their code through — this
// is the resolved episode that proves it. Written through the repository
// because the emitter enum no longer has the code at all.
const legacy = events_mod.legacy_wire_codes[0];
_ = try events_repo.insertActive(database, seeded_now + 60, legacy, "history", "history", "warning", "Busy");
_ = try events_repo.resolveActiveByCode(database, seeded_now + 120, legacy);
}
// ---------------------------------------------------------------------------
@@ -750,7 +824,9 @@ const contract = [_]Contract{
.{ .method = .GET, .pattern = "/api/queries/live", .auth = .session, .policy = .read, .rate_limit = .exempt, .target = "/api/queries/live", .status = 200, .kind = .sse },
.{ .method = .GET, .pattern = "/api/stats", .auth = .session, .policy = .read, .target = "/api/stats?period=1h", .status = 200, .check = jsonShape(handlers_stats.TotalsBody) },
.{ .method = .GET, .pattern = "/api/stats/timeseries", .auth = .session, .policy = .read, .target = "/api/stats/timeseries?period=1h", .status = 200, .check = jsonShape(handlers_stats.TimeseriesBody) },
.{ .method = .GET, .pattern = "/api/upstream/health", .auth = .session, .policy = .read, .target = "/api/upstream/health", .status = 200, .check = jsonShape(handlers_upstream_health.Body) },
.{ .method = .GET, .pattern = "/api/stats/types", .auth = .session, .policy = .read, .target = "/api/stats/types?period=1h", .status = 200, .check = jsonShape(handlers_stats.TypesBody) },
.{ .method = .GET, .pattern = "/api/stats/routes", .auth = .session, .policy = .read, .target = "/api/stats/routes?period=1h", .status = 200, .check = jsonShape(handlers_stats.RoutesBody) },
.{ .method = .GET, .pattern = "/api/stats/clients", .auth = .session, .policy = .read, .target = "/api/stats/clients?period=1h", .status = 200, .check = jsonShape(handlers_stats.ClientsBody) },
// Diagnostics. The seeded store holds one active episode (id 1) and one
// resolved one, so both the page and the detail answer with real rows.
@@ -1977,7 +2053,16 @@ fn detailUnavailable(io: std.Io, env: *Env) anyerror!void {
defer conn.close(io);
var body_buf: [8 * 1024]u8 = undefined;
for ([_][]const u8{ "/api/queries/1", "/api/queries?limit=1", "/api/stats", "/api/stats/timeseries" }) |target| {
const targets = [_][]const u8{
"/api/queries/1",
"/api/queries?limit=1",
"/api/stats",
"/api/stats/timeseries",
"/api/stats/types",
"/api/stats/routes",
"/api/stats/clients",
};
for (targets) |target| {
try conn.request("GET", target, null, null);
const response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 503), response.status);
@@ -2064,6 +2149,260 @@ fn coverageWalk(io: std.Io, env: *Env) anyerror!void {
try testing.expectEqual(totals.coverage.complete, series.coverage.complete);
}
fn getJson(
comptime T: type,
arena: Allocator,
conn: *Conn,
target: []const u8,
body_buf: []u8,
) !T {
try conn.request("GET", target, null, null);
const response = try conn.receive(body_buf);
if (response.status != 200) {
std.debug.print("{s}: status {d}: {s}\n", .{ target, response.status, response.body });
return error.TestUnexpectedResult;
}
return std.json.parseFromSliceLeaky(T, arena, response.body, .{ .ignore_unknown_fields = false });
}
fn emptyAggregations(io: std.Io, env: *Env) anyerror!void {
var arena_state: std.heap.ArenaAllocator = .init(env.gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [256 * 1024]u8 = undefined;
// This environment's only rows are the fixed 2023 seed, so every live
// window is empty. The empty bodies are exact, not merely parseable.
const types_body = try getJson(handlers_stats.TypesBody, arena, &conn, "/api/stats/types?period=1h", &body_buf);
try testing.expectEqualStrings("1h", types_body.period);
try testing.expectEqual(@as(usize, 0), types_body.types.len);
const routes_body = try getJson(handlers_stats.RoutesBody, arena, &conn, "/api/stats/routes?period=1h", &body_buf);
try testing.expectEqual(@as(usize, 0), routes_body.routes.len);
// `other` is present and bucket-count sized even here: a chart must never
// have to invent the residual series.
const clients = try getJson(handlers_stats.ClientsBody, arena, &conn, "/api/stats/clients?period=1h", &body_buf);
try testing.expectEqual(@as(usize, 0), clients.clients.len);
try testing.expectEqual(@as(u32, 60), clients.bucket_seconds);
try testing.expectEqual(@as(usize, 60), clients.other.len);
for (clients.other) |count| try testing.expectEqual(@as(u64, 0), count);
// A window nobody covers is still reported as such, not as a quiet hour.
try testing.expectEqual(seeded_available_since, types_body.coverage.available_since);
try testing.expect(types_body.coverage.complete);
for ([_][]const u8{ "/api/stats/types", "/api/stats/routes", "/api/stats/clients" }) |path| {
var target_buf: [64]u8 = undefined;
const target = try std.fmt.bufPrint(&target_buf, "{s}?period=12h", .{path});
try conn.request("GET", target, null, null);
const bad = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 400), bad.status);
try testing.expect(std.mem.containsAtLeast(u8, bad.body, 1, "period must be one of"));
}
}
test "W10 milestone 30: an empty window answers exact empty aggregations, and a bad period is a 400" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{});
defer env.destroy();
try bounded(env.io(), default_budget, emptyAggregations, .{ env.io(), env });
}
fn populatedAggregations(io: std.Io, env: *Env) anyerror!void {
var arena_state: std.heap.ArenaAllocator = .init(env.gpa);
defer arena_state.deinit();
const arena = arena_state.allocator();
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [256 * 1024]u8 = undefined;
const totals = try getJson(handlers_stats.TotalsBody, arena, &conn, "/api/stats?period=1h", &body_buf);
const series = try getJson(handlers_stats.TimeseriesBody, arena, &conn, "/api/stats/timeseries?period=1h", &body_buf);
const types_body = try getJson(handlers_stats.TypesBody, arena, &conn, "/api/stats/types?period=1h", &body_buf);
const routes_body = try getJson(handlers_stats.RoutesBody, arena, &conn, "/api/stats/routes?period=1h", &body_buf);
const clients = try getJson(handlers_stats.ClientsBody, arena, &conn, "/api/stats/clients?period=1h", &body_buf);
// Nothing writes to this box between the five requests, so the window is
// one state and conservation is a real assertion rather than a race.
try testing.expectEqual(totals.since, series.since);
try testing.expectEqual(totals.since, types_body.since);
try testing.expectEqual(totals.since, routes_body.since);
try testing.expectEqual(totals.since, clients.since);
try testing.expect(totals.queries > 0);
var typed: u64 = 0;
var null_qtype_rows: usize = 0;
for (types_body.types) |row| {
typed += row.count;
if (row.qtype == null) null_qtype_rows += 1;
}
try testing.expectEqual(totals.queries, typed);
// The seeded matrix holds one typeless row, and it must be its own group.
try testing.expectEqual(@as(usize, 1), null_qtype_rows);
var routed: u64 = 0;
var null_source_upstreams: usize = 0;
var named_upstreams: usize = 0;
for (routes_body.routes) |row| {
routed += row.count;
if (row.route != .upstream) continue;
if (row.source == null) null_source_upstreams += 1 else named_upstreams += 1;
}
try testing.expectEqual(totals.queries, routed);
try testing.expectEqual(@as(usize, 1), null_source_upstreams);
try testing.expectEqual(@as(usize, 2), named_upstreams);
try testing.expectEqual(@as(usize, recent_clients), clients.clients.len);
try testing.expectEqual(series.buckets.len, clients.other.len);
for (clients.clients) |entry| try testing.expectEqual(series.buckets.len, entry.buckets.len);
// Per bucket, not just over the window: a series off by one bucket would
// still sum correctly in total.
for (series.buckets, 0..) |bucket, at| {
var summed: u64 = clients.other[at];
for (clients.clients) |entry| summed += entry.buckets[at];
try testing.expectEqual(bucket.queries, summed);
}
}
test "W10 milestone 30: the three breakdowns conserve the totals over one window" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{ .recent_traffic = true });
defer env.destroy();
try bounded(env.io(), default_budget, populatedAggregations, .{ env.io(), env });
}
/// One connection walking every query-log endpoint several times over.
fn hammerQuerylog(io: std.Io, env: *Env) anyerror!void {
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [256 * 1024]u8 = undefined;
const targets = [_][]const u8{
"/api/stats?period=1h",
"/api/stats/timeseries?period=1h",
"/api/stats/types?period=1h",
"/api/stats/routes?period=1h",
"/api/stats/clients?period=1h",
"/api/queries?limit=5",
"/api/queries/27",
};
for (0..3) |_| {
for (targets) |target| {
try conn.request("GET", target, null, null);
const response = try conn.receive(&body_buf);
if (response.status != 200) {
std.debug.print("{s}: status {d}: {s}\n", .{ target, response.status, response.body });
return error.TestUnexpectedResult;
}
}
}
}
fn concurrentQuerylogReads(io: std.Io, env: *Env) anyerror!void {
// Six tasks on six connections against the one shared query-log
// connection. Without `querylog_lock` this is exactly the shape that makes
// a second BEGIN fail and a foreign read land inside someone else's
// transaction; every response here must still be a 200.
var futures: [6]std.Io.Future(anyerror!void) = undefined;
for (&futures) |*future| future.* = try io.concurrent(hammerQuerylog, .{ io, env });
var failure: ?anyerror = null;
for (&futures) |*future| future.await(io) catch |err| {
failure = err;
};
if (failure) |err| return err;
}
fn failedCommitIsBounded(io: std.Io, env: *Env) anyerror!void {
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [256 * 1024]u8 = undefined;
// A read that cannot end its transaction. The three things that must hold
// are all observable from here: the client is told (500, not a 200 over a
// state nobody can name), the process survives (the lock is released
// exactly once — releasing twice is `unreachable` in `std.Io.Mutex`), and
// the connection recovers (the rollback attempt worked, so the next
// `BEGIN` is not refused).
db.read_tx_faults.failNextCommit();
try conn.request("GET", "/api/stats/types?period=1h", null, null);
const failed = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 500), failed.status);
try testing.expect(std.mem.containsAtLeast(u8, failed.body, 1, "internal error"));
// Same connection, same shared query-log handle: a request after the fault
// is an ordinary 200. This is the assertion the double-unlock bug failed —
// it panicked here instead of answering.
try conn.request("GET", "/api/stats/types?period=1h", null, null);
const recovered = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), recovered.status);
// And every other query-log route still works on that connection.
for ([_][]const u8{
"/api/stats?period=1h",
"/api/stats/timeseries?period=1h",
"/api/stats/routes?period=1h",
"/api/stats/clients?period=1h",
"/api/queries?limit=5",
"/api/queries/27",
}) |target| {
try conn.request("GET", target, null, null);
const response = try conn.receive(&body_buf);
if (response.status != 200) {
std.debug.print("{s} after the fault: status {d}\n", .{ target, response.status });
return error.TestUnexpectedResult;
}
}
}
test "W10 milestone 30: a read that cannot commit answers 500 and leaves the connection usable" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{ .recent_traffic = true });
defer env.destroy();
// The teardown fault is reported at `err`, which the test runner counts as
// a failure; this test causes it deliberately and asserts the count.
db.read_tx_faults.beginCapture();
defer _ = db.read_tx_faults.endCapture();
try bounded(env.io(), default_budget, failedCommitIsBounded, .{ env.io(), env });
// Exactly the one COMMIT fault: the ROLLBACK behind it succeeded, and no
// later request tripped a fault of its own.
try testing.expectEqual(@as(usize, 1), db.read_tx_faults.endCapture());
}
test "W10 milestone 30: concurrent query-log reads all answer 200 on the shared connection" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{ .recent_traffic = true });
defer env.destroy();
try bounded(env.io(), default_budget, concurrentQuerylogReads, .{ env.io(), env });
}
test "W10 milestone 28: every window-bounded endpoint reports its own coverage" {
if (!build_options.integration) return error.SkipZigTest;
@@ -2876,12 +3215,33 @@ fn documentedType(comptime T: type) ?[]const u8 {
.bool => "boolean",
// A closed enum is a string on the wire, documented as its own schema.
.@"enum" => null,
.pointer => "string",
// `[]const u8` is a string; every other slice is a JSON array, whose
// element type `elementType` below holds the `items:` block to.
.pointer => |ptr| if (ptr.child == u8) "string" else "array",
.@"struct" => null,
else => @compileError("no documented type for " ++ @typeName(Payload)),
};
}
/// The element type of a field that serializes as a JSON array, or null when
/// the field is not one. `[]const u8` is a string, not an array of integers.
fn elementType(comptime T: type) ?type {
const Payload = switch (@typeInfo(T)) {
.optional => |o| o.child,
else => T,
};
return switch (@typeInfo(Payload)) {
.pointer => |ptr| if (ptr.child == u8) null else ptr.child,
else => null,
};
}
/// The `items:` sub-block of an array property.
fn yamlItems(property: []const u8) ?[]const u8 {
const at = std.mem.indexOf(u8, property, "items:") orelse return null;
return property[at..];
}
fn isOptional(comptime T: type) bool {
return @typeInfo(T) == .optional;
}
@@ -2933,6 +3293,30 @@ fn expectSchemaMatches(gpa: Allocator, comptime T: type, schema_name: []const u8
std.debug.print("{s}.{s}: not documented as {s}\n", .{ schema_name, field.name, wanted });
return error.TestUnexpectedResult;
}
// An array is only as documented as its elements are: without this
// an array of one object would match an array of another.
if (comptime elementType(field.type)) |Element| {
const items = yamlItems(property) orelse {
std.debug.print("{s}.{s}: array with no items\n", .{ schema_name, field.name });
return error.TestUnexpectedResult;
};
switch (@typeInfo(Element)) {
.int => if (!std.mem.containsAtLeast(u8, items, 1, "type: integer")) {
std.debug.print("{s}.{s}: items not documented as integer\n", .{ schema_name, field.name });
return error.TestUnexpectedResult;
},
else => {
const target = refTarget(items) orelse {
std.debug.print("{s}.{s}: items are not a $ref\n", .{ schema_name, field.name });
return error.TestUnexpectedResult;
};
switch (@typeInfo(Element)) {
.@"enum" => try expectEnumMatches(gpa, Element, target),
else => try expectSchemaMatches(gpa, Element, target),
}
},
}
}
} else {
const target = refTarget(property) orelse {
std.debug.print("{s}.{s}: not a $ref\n", .{ schema_name, field.name });
@@ -2981,6 +3365,25 @@ fn expectEnumMatches(gpa: Allocator, comptime T: type, schema_name: []const u8)
}
}
test "drift guard c: the health rollup matches the five objects it documents" {
const gpa = testing.allocator;
// Recurses through the five `$ref`s, so a condition object that gains,
// loses or retypes a field fails here — which is the whole contract: no
// condition may degrade the rollup without appearing in the response.
try expectSchemaMatches(gpa, handlers_health.Body, "Health");
}
test "drift guard c: the stats schemas match the structs that serialize them" {
// Guard b counts operations and guard a matches paths, so neither noticed
// that `cached` outlived the field it documented. This one would have.
const gpa = testing.allocator;
try expectSchemaMatches(gpa, handlers_stats.TotalsBody, "StatsTotals");
try expectSchemaMatches(gpa, handlers_stats.TimeseriesBody, "StatsTimeseries");
try expectSchemaMatches(gpa, handlers_stats.TypesBody, "StatsTypes");
try expectSchemaMatches(gpa, handlers_stats.RoutesBody, "StatsRoutes");
try expectSchemaMatches(gpa, handlers_stats.ClientsBody, "StatsClients");
}
test "drift guard c: the query-log schemas match the structs that serialize them" {
const gpa = testing.allocator;
try expectSchemaMatches(gpa, queries_repo.QueryRow, "QueryRow");
@@ -3134,7 +3537,6 @@ const contract_sample_walk = [_]ContractSample{
.{ .name = "list_upstreams", .ts_type = "{ upstreams: Upstream[] }", .method = "GET", .target = "/api/upstreams", .status = 200 },
.{ .name = "create_upstream", .ts_type = "UpstreamEcho", .method = "POST", .target = "/api/upstreams", .body = "{\"url\":\"https://dns2.example/dns-query\"}", .status = 201 },
.{ .name = "update_upstream", .ts_type = "UpstreamEcho", .method = "PUT", .target = "/api/upstreams/1", .body = "{\"url\":\"https://dns.example/dns-query\",\"priority\":5}", .status = 200 },
.{ .name = "get_upstream_health", .ts_type = "UpstreamHealth", .method = "GET", .target = "/api/upstream/health", .status = 200 },
// Query log and stats. `limit=5` reaches seeded row 21, the blocked one, so
// the page carries both the null-bearing and the populated row shape.
@@ -3163,6 +3565,15 @@ const contract_sample_walk = [_]ContractSample{
.{ .name = "error_not_found", .ts_type = "ErrorEnvelope", .method = "GET", .target = "/api/nope", .status = 404 },
};
/// The three period aggregations, captured against an environment with live
/// traffic in it: over the fixed 2023 seed every one of them would answer with
/// an empty array, which describes no field at all.
const stats_sample_walk = [_]ContractSample{
.{ .name = "get_stats_types", .ts_type = "StatsTypes", .method = "GET", .target = "/api/stats/types?period=1h", .status = 200 },
.{ .name = "get_stats_routes", .ts_type = "StatsRoutes", .method = "GET", .target = "/api/stats/routes?period=1h", .status = 200 },
.{ .name = "get_stats_clients", .ts_type = "StatsClients", .method = "GET", .target = "/api/stats/clients?period=1h", .status = 200 },
};
/// A session-authenticated environment answers this without a cookie.
const unauthorized_sample: ContractSample = .{
.name = "error_unauthorized",
@@ -3314,7 +3725,9 @@ const regen_command =
/// exactly the import list the generated file needs.
fn writeSampleImports(arena: Allocator, w: *std.Io.Writer) !void {
var names: std.ArrayList([]const u8) = .empty;
for (contract_sample_walk ++ [_]ContractSample{ unauthorized_sample, rate_limited_sample }) |sample| {
for (contract_sample_walk ++ stats_sample_walk ++
[_]ContractSample{ unauthorized_sample, rate_limited_sample }) |sample|
{
var index: usize = 0;
while (index < sample.ts_type.len) {
if (!std.ascii.isUpper(sample.ts_type[index])) {
@@ -3397,6 +3810,15 @@ fn sampleWalk(io: std.Io, env: *Env, out: *std.Io.Writer) anyerror!void {
for (contract_sample_walk) |sample| try captureSample(env.gpa, &conn, out, sample, &body_buf);
}
fn statsSampleWalk(io: std.Io, env: *Env, out: *std.Io.Writer) anyerror!void {
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
var body_buf: [128 * 1024]u8 = undefined;
for (stats_sample_walk) |sample| try captureSample(env.gpa, &conn, out, sample, &body_buf);
}
fn sampleUnauthorized(io: std.Io, env: *Env, out: *std.Io.Writer) anyerror!void {
var conn: Conn = undefined;
try conn.connect(io, env.addr);
@@ -3437,6 +3859,11 @@ test "W10 milestone 17: the committed contract samples still describe live respo
defer env.destroy();
try bounded(env.io(), default_budget, sampleWalk, .{ env.io(), env, &rendered.writer });
}
{
var env = try Env.create(gpa, .{ .recent_traffic = true });
defer env.destroy();
try bounded(env.io(), default_budget, statsSampleWalk, .{ env.io(), env, &rendered.writer });
}
{
var hash_buf: [256]u8 = undefined;
const hash = try hashTestPassword(gpa, &hash_buf);