mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 25 Activity

milestone 30: overview as a dashboard, explicit health contract, period aggregations

Browse Source
This commit is contained in:
mokhtar
2026-08-22 16:45:15 +02:00
parent 0e83477d80
commit 623667e475
89 changed files with 7222 additions and 4239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/web/handlers/health.zig
+299 -200
View File
@@ -6,6 +6,13 @@
//!
//! Unauthenticated and rate-limit exempt, like `/metrics`.
//!
//! **Nothing may degrade the rollup without appearing in the response.** The
//! body is five condition objects and a status computed from exactly their
//! states, so an operator reading it can always name the condition that
//! degraded the box. Anything a counter already records — gated refreshes,
//! disk sample failures, the snapshot generation — stays in `/metrics` and in
//! Diagnostics rather than becoming a sixth hidden input here.
//!
//! `rollup` is pure so the whole degraded matrix is testable without a running
//! server; `handle` only gathers the inputs.
@@ -13,16 +20,39 @@ const std = @import("std");
const disk_monitor = @import("../../storage/disk_monitor.zig");
const http_util = @import("../http_util.zig");
const logger_mod = @import("../../storage/logger.zig");
const metrics = @import("../metrics.zig");
const pause_mod = @import("../../server/pause.zig");
const pool_mod = @import("../../upstream/pool.zig");
const server = @import("../server.zig");
pub const Disk = struct {
/// Is filtering in force. `unavailable` is not an operator's doing: it is the
/// state in which the query path has no filter snapshot to evaluate against,
/// which `handler.zig` records as the `snapshot_unavailable` provenance.
pub const Protection = struct {
state: []const u8,
free_bytes: u64,
db_bytes: u64,
log_bytes: u64,
sample_failures: u64,
/// The second filtering resumes at. Null for both an indefinite pause and
/// every non-paused state, which the state field already tells apart.
until: ?i64,
};
pub const Upstreams = struct {
state: []const u8,
available: u32,
/// Enabled upstreams: the pool is built from those alone, so this is what
/// `available` is out of.
total: u32,
};
/// Whether Activity can be trusted. `dropped_total` is cumulative and does not
/// decide the state — a drop that happened an hour ago is not a fault now.
pub const QueryHistory = struct {
state: []const u8,
dropped_total: u64,
/// The newest drop, or null while nothing has been dropped. Stamped by a
/// separate atomic from the count, so a reader can momentarily see a
/// non-zero `dropped_total` beside a null here.
last_drop_s: ?i64,
};
/// The diagnostics store's own state, not a summary of what it holds: `state`
@@ -34,21 +64,18 @@ pub const Diagnostics = struct {
active_errors: u32,
};
pub const Upstreams = struct {
available: u32,
total: u32,
pub const Disk = struct {
state: []const u8,
free_bytes: u64,
};
pub const Body = struct {
status: []const u8,
disk: Disk,
protection: Protection,
upstreams: Upstreams,
query_history: QueryHistory,
diagnostics: Diagnostics,
queries_dropped: u64,
writer_failed: bool,
refreshes_gated: u64,
/// Null before the first filter snapshot is published.
snapshot_generation: ?u64,
disk: Disk,
};
/// What the rollup is computed from. Every field has a defined value even when
@@ -56,41 +83,79 @@ pub const Body = struct {
/// server should report: no disk reading, no upstreams, nothing published.
pub const Input = struct {
disk_state: disk_monitor.State = .ok,
disk: disk_monitor.Gauges = .{ .free_bytes = 0, .db_bytes = 0, .log_bytes = 0 },
disk_sample_failures: u64 = 0,
disk_free_bytes: u64 = 0,
upstreams_available: u32 = 0,
upstreams_total: u32 = 0,
queries_dropped: u64 = 0,
last_drop_s: ?i64 = null,
writer_failed: bool = false,
/// Current state, not a count: the upstream-history flush is failing right
/// now. Cleared by the next flush that succeeds (m26 ruling 7).
///
/// `rows_dropped` deliberately does not appear here. It is cumulative, and
/// a rollup that is computed statelessly cannot ask whether a counter grew
/// — so feeding it in would latch `/api/health` to degraded forever after
/// one overflow. Drops surface through the metric and through the API's
/// per-window `complete` instead.
history_flush_failing: bool = false,
/// The logger's disk-gating episode. `losing` is the only state that
/// degrades: it means the gate is holding writes back *and* has already
/// cost rows in the episode that is open now.
gate_episode: logger_mod.GateEpisode = .open,
/// A filter snapshot exists for the query path to evaluate against. The
/// benign default matches every other field here, and `collect` assigns it
/// explicitly for the same reason `diagnostics_present` is assigned there.
snapshot_available: bool = true,
/// `pause.Pause.until` verbatim: 0 not paused, -1 indefinite, otherwise the
/// second filtering resumes at. Raw rather than a decided boolean so the
/// expiry rule stays `pause.Pause`'s and is exercised by these tests.
pause_until: i64 = 0,
/// The clock the pause is compared against.
now_s: i64 = 0,
/// The diagnostics store exists. The benign default matches every other
/// field here — a half-wired `Input` reports a box with nothing wrong — but
/// `collect` must assign it explicitly, because in a serving process an
/// absent store means `Store.init` failed.
diagnostics_present: bool = true,
/// The last diagnostics write failed. Current state, cleared by the next
/// write that succeeds, like `history_flush_failing`.
/// write that succeeds.
diagnostics_write_failed: bool = false,
diagnostics_active_warnings: u32 = 0,
diagnostics_active_errors: u32 = 0,
refreshes_gated: u64 = 0,
snapshot_generation: ?u64 = null,
};
pub const status_ok = "ok";
pub const status_degraded = "degraded";
pub const protection_active = "active";
pub const protection_paused = "paused";
pub const protection_unavailable = "unavailable";
pub const upstreams_ok = "ok";
pub const upstreams_unavailable = "unavailable";
pub const query_history_recording = "recording";
pub const query_history_losing = "losing";
pub const query_history_failed = "failed";
pub const diagnostics_recording = "recording";
pub const diagnostics_unavailable = "unavailable";
pub const disk_ok = "ok";
pub const disk_low = "low";
pub const disk_critical = "critical";
/// Precedence `unavailable` → `paused` → `active`. With no snapshot the pause
/// flag says nothing an operator can act on: filtering is off either way, and
/// resuming would not turn it back on.
pub fn protection(input: Input) Protection {
if (!input.snapshot_available) return .{ .state = protection_unavailable, .until = null };
const state: pause_mod.Pause = .{ .until = .init(input.pause_until) };
if (!state.isPaused(input.now_s)) return .{ .state = protection_active, .until = null };
return .{
.state = protection_paused,
.until = if (input.pause_until > 0) input.pause_until else null,
};
}
pub fn queryHistoryState(input: Input) []const u8 {
if (input.writer_failed) return query_history_failed;
if (input.gate_episode == .losing) return query_history_losing;
return query_history_recording;
}
/// The operational log is not recording — either the store never opened or its
/// writes are failing. Both mean the same thing to an operator: the record of
/// what went wrong is not being kept.
@@ -98,37 +163,51 @@ pub fn diagnosticsUnavailable(input: Input) bool {
return !input.diagnostics_present or input.diagnostics_write_failed;
}
/// Conditions an operator must act on, and every one of them is a fact about
/// now rather than a count of the past: a disk that is filling stops the query
/// log, a pool with nothing available stops resolution, a failed writer means
/// rows are being lost right now, and a failing history flush means the
/// dashboard's upstream numbers are not being recorded. Each clears itself when
/// the underlying condition does.
/// `warn` reads as a log level rather than as a quantity of disk. The monitor
/// keeps its own name; the wire says what an operator sees on the page.
pub fn diskState(state: disk_monitor.State) []const u8 {
return switch (state) {
.ok => disk_ok,
.warn => disk_low,
.critical => disk_critical,
};
}
/// The degrading set, named rather than gestured at: protection `unavailable`,
/// upstreams `unavailable`, query history `losing` or `failed`, diagnostics
/// `unavailable`, disk `low` or `critical`.
///
/// A pause is deliberately not in it. It is an operator's own choice, and a
/// monitor that pages on it would be paging on a button the operator pressed.
pub fn degraded(input: Input) bool {
return input.disk_state != .ok or input.upstreams_available == 0 or
input.writer_failed or input.history_flush_failing or diagnosticsUnavailable(input);
const history = queryHistoryState(input);
return !input.snapshot_available or
input.upstreams_available == 0 or
!std.mem.eql(u8, history, query_history_recording) or
diagnosticsUnavailable(input) or
input.disk_state != .ok;
}
pub fn rollup(input: Input) Body {
return .{
.status = if (degraded(input)) status_degraded else status_ok,
.disk = .{
.state = @tagName(input.disk_state),
.free_bytes = input.disk.free_bytes,
.db_bytes = input.disk.db_bytes,
.log_bytes = input.disk.log_bytes,
.sample_failures = input.disk_sample_failures,
.protection = protection(input),
.upstreams = .{
.state = if (input.upstreams_available == 0) upstreams_unavailable else upstreams_ok,
.available = input.upstreams_available,
.total = input.upstreams_total,
},
.query_history = .{
.state = queryHistoryState(input),
.dropped_total = input.queries_dropped,
.last_drop_s = input.last_drop_s,
},
.upstreams = .{ .available = input.upstreams_available, .total = input.upstreams_total },
.diagnostics = .{
.state = if (diagnosticsUnavailable(input)) diagnostics_unavailable else diagnostics_recording,
.active_warnings = input.diagnostics_active_warnings,
.active_errors = input.diagnostics_active_errors,
},
.queries_dropped = input.queries_dropped,
.writer_failed = input.writer_failed,
.refreshes_gated = input.refreshes_gated,
.snapshot_generation = input.snapshot_generation,
.disk = .{ .state = diskState(input.disk_state), .free_bytes = input.disk_free_bytes },
};
}
@@ -143,10 +222,11 @@ pub fn handle(
pub fn collect(state: *server.WebState, io: std.Io) Input {
var input: Input = .{};
input.now_s = std.Io.Clock.real.now(io).toSeconds();
if (state.monitor) |monitor| {
input.disk_state = monitor.state();
input.disk = monitor.gauges();
input.disk_sample_failures = monitor.sample_failures.load(.monotonic);
input.disk_free_bytes = monitor.gauges().free_bytes;
}
if (state.pool) |pool| {
@@ -160,9 +240,13 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
if (state.logger) |logger| {
input.queries_dropped = logger.queries_dropped.load(.monotonic);
input.last_drop_s = logger.lastDropSeconds();
input.writer_failed = logger.writer_failed.load(.monotonic);
input.gate_episode = logger.gateEpisode();
}
if (state.pause) |paused| input.pause_until = paused.until.load(.monotonic);
// Assigned before the `if`, not inside it: the field's benign default is
// `true`, so the natural `if (state.events) |store|` shape would report an
// absent store as recording — the one case that must degrade.
@@ -174,15 +258,13 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
input.diagnostics_active_errors = counts.errors;
}
if (state.history) |history| {
input.history_flush_failing = history.snapshotStats(io).last_flush_failed;
}
// The generation itself is not in the body — the UI-facing fact is whether
// protection has a snapshot at all, and `/metrics` keeps the number.
input.snapshot_available = false;
if (state.manager) |manager| {
input.refreshes_gated = manager.refreshesGated();
if (manager.acquire(io)) |acquired| {
defer acquired.release(io);
input.snapshot_generation = acquired.snapshot.generation;
input.snapshot_available = true;
}
}
@@ -196,104 +278,175 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
const db = @import("../../storage/db.zig");
const events_mod = @import("../../storage/events.zig");
const migrations = @import("../../storage/migrations.zig");
const history_mod = @import("../../upstream/history.zig");
const logger_mod = @import("../../storage/logger.zig");
const upstream_history_repo = @import("../../storage/repositories/upstream_history_repo.zig");
const testing = std.testing;
/// A box with nothing wrong with it: one upstream up, disk ok, writer alive.
/// A box with nothing wrong with it: one upstream up, disk ok, writer alive,
/// a snapshot published and filtering unpaused.
const healthy: Input = .{
.disk_state = .ok,
.upstreams_available = 1,
.upstreams_total = 1,
.writer_failed = false,
.snapshot_available = true,
};
test "the degraded matrix covers disk state, availability and the writer" {
const cases = [_]struct { input: Input, degraded: bool }{
.{ .input = healthy, .degraded = false },
.{ .input = withDisk(healthy, .warn), .degraded = true },
.{ .input = withDisk(healthy, .critical), .degraded = true },
.{ .input = withAvailable(healthy, 0), .degraded = true },
.{ .input = withWriterFailed(healthy), .degraded = true },
// A failing upstream-history flush is losing the dashboard's numbers
// right now, and it recovers on its own the moment a flush succeeds.
.{ .input = withHistoryFailing(healthy, true), .degraded = true },
.{ .input = withHistoryFailing(healthy, false), .degraded = false },
// The operational log not recording is itself a fault an operator must
// act on: whatever fails next will leave no record of having failed.
.{ .input = withDiagnostics(healthy, false, false), .degraded = true },
.{ .input = withDiagnostics(healthy, true, true), .degraded = true },
.{ .input = withDiagnostics(healthy, true, false), .degraded = false },
// Two faults at once still report one status.
.{ .input = withWriterFailed(withDisk(healthy, .critical)), .degraded = true },
// Some upstreams down is not degraded while one still answers.
.{ .input = .{ .upstreams_available = 1, .upstreams_total = 3 }, .degraded = false },
};
fn with(input: Input, comptime field: []const u8, value: anytype) Input {
var out = input;
@field(out, field) = value;
return out;
}
for (cases, 0..) |case, i| {
errdefer std.debug.print("case {d}\n", .{i});
try testing.expectEqual(case.degraded, degraded(case.input));
try testing.expectEqualStrings(
if (case.degraded) status_degraded else status_ok,
rollup(case.input).status,
);
test "each degrading condition degrades on its own and says so in the body" {
try testing.expectEqualStrings(status_ok, rollup(healthy).status);
{
const body = rollup(with(healthy, "snapshot_available", false));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(protection_unavailable, body.protection.state);
}
{
const body = rollup(with(healthy, "upstreams_available", @as(u32, 0)));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(upstreams_unavailable, body.upstreams.state);
}
{
const body = rollup(with(healthy, "gate_episode", .losing));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(query_history_losing, body.query_history.state);
}
{
const body = rollup(with(healthy, "writer_failed", true));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(query_history_failed, body.query_history.state);
}
{
const body = rollup(with(healthy, "diagnostics_present", false));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state);
}
{
const body = rollup(with(healthy, "diagnostics_write_failed", true));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state);
}
for ([_]struct { disk_monitor.State, []const u8 }{
.{ .warn, disk_low },
.{ .critical, disk_critical },
}) |case| {
const body = rollup(with(healthy, "disk_state", case[0]));
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(case[1], body.disk.state);
}
}
fn withDisk(input: Input, state: disk_monitor.State) Input {
var out = input;
out.disk_state = state;
return out;
test "conditions that are not faults leave the status ok" {
// Some upstreams down while one still answers.
const partial = rollup(.{ .upstreams_available = 1, .upstreams_total = 3 });
try testing.expectEqualStrings(status_ok, partial.status);
try testing.expectEqualStrings(upstreams_ok, partial.upstreams.state);
// A gate holding writes that has not cost a row yet.
const holding = rollup(with(healthy, "gate_episode", .gated));
try testing.expectEqualStrings(status_ok, holding.status);
try testing.expectEqualStrings(query_history_recording, holding.query_history.state);
// Drops that already happened. The counter is cumulative and the rollup is
// stateless, so feeding it in would latch the box to degraded forever.
var dropped = with(healthy, "queries_dropped", @as(u64, 9));
dropped.last_drop_s = 1_700_000_000;
const body = rollup(dropped);
try testing.expectEqualStrings(status_ok, body.status);
try testing.expectEqualStrings(query_history_recording, body.query_history.state);
try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total);
try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s);
// Open diagnostics episodes are what the box is doing, not a fault of the
// log that recorded them.
var open = healthy;
open.diagnostics_active_warnings = 3;
open.diagnostics_active_errors = 1;
const with_episodes = rollup(open);
try testing.expectEqualStrings(status_ok, with_episodes.status);
try testing.expectEqualStrings(diagnostics_recording, with_episodes.diagnostics.state);
try testing.expectEqual(@as(u32, 3), with_episodes.diagnostics.active_warnings);
try testing.expectEqual(@as(u32, 1), with_episodes.diagnostics.active_errors);
}
fn withAvailable(input: Input, available: u32) Input {
var out = input;
out.upstreams_available = available;
return out;
test "a pause is surfaced, never alarmed, and an expired one is over" {
var indefinite = healthy;
indefinite.pause_until = -1;
indefinite.now_s = 1_000;
const forever = rollup(indefinite);
try testing.expectEqualStrings(status_ok, forever.status);
try testing.expectEqualStrings(protection_paused, forever.protection.state);
try testing.expectEqual(@as(?i64, null), forever.protection.until);
var timed = healthy;
timed.pause_until = 1_060;
timed.now_s = 1_000;
const live = rollup(timed);
try testing.expectEqualStrings(status_ok, live.status);
try testing.expectEqualStrings(protection_paused, live.protection.state);
try testing.expectEqual(@as(?i64, 1_060), live.protection.until);
// The stored second is when filtering is back on, so at it the pause is over.
timed.now_s = 1_060;
const expired = rollup(timed);
try testing.expectEqualStrings(protection_active, expired.protection.state);
try testing.expectEqual(@as(?i64, null), expired.protection.until);
}
fn withWriterFailed(input: Input) Input {
var out = input;
out.writer_failed = true;
return out;
test "protection unavailable wins over a live pause" {
var both = healthy;
both.snapshot_available = false;
both.pause_until = -1;
const body = rollup(both);
try testing.expectEqualStrings(protection_unavailable, body.protection.state);
try testing.expectEqual(@as(?i64, null), body.protection.until);
try testing.expectEqualStrings(status_degraded, body.status);
}
fn withHistoryFailing(input: Input, failing: bool) Input {
var out = input;
out.history_flush_failing = failing;
return out;
test "a failed writer outranks a losing gate" {
var both = healthy;
both.writer_failed = true;
both.gate_episode = .losing;
try testing.expectEqualStrings(query_history_failed, rollup(both).query_history.state);
}
fn withDiagnostics(input: Input, present: bool, write_failed: bool) Input {
var out = input;
out.diagnostics_present = present;
out.diagnostics_write_failed = write_failed;
return out;
test "two faults at once still report one status" {
var both = with(healthy, "disk_state", disk_monitor.State.critical);
both.writer_failed = true;
try testing.expectEqualStrings(status_degraded, rollup(both).status);
}
test "the diagnostics block reports the state and the open counts" {
const recording = rollup(.{
.upstreams_available = 1,
.diagnostics_active_warnings = 3,
.diagnostics_active_errors = 1,
test "the body reports every input verbatim" {
const body = rollup(.{
.disk_state = .warn,
.disk_free_bytes = 100,
.upstreams_available = 2,
.upstreams_total = 4,
.queries_dropped = 9,
.last_drop_s = 1_700_000_000,
.snapshot_available = true,
});
try testing.expectEqualStrings(diagnostics_recording, recording.diagnostics.state);
try testing.expectEqual(@as(u32, 3), recording.diagnostics.active_warnings);
try testing.expectEqual(@as(u32, 1), recording.diagnostics.active_errors);
// Open episodes are what the box is doing, not a fault of the log: they do
// not degrade on their own.
try testing.expectEqualStrings(status_ok, recording.status);
// Failing writes: the counts are whatever was last read, and the state is
// the honest one.
const failing = rollup(.{ .upstreams_available = 1, .diagnostics_write_failed = true });
try testing.expectEqualStrings(diagnostics_unavailable, failing.diagnostics.state);
try testing.expectEqualStrings(status_degraded, failing.status);
try testing.expectEqualStrings(status_degraded, body.status);
try testing.expectEqualStrings(disk_low, body.disk.state);
try testing.expectEqual(@as(u64, 100), body.disk.free_bytes);
try testing.expectEqual(@as(u32, 2), body.upstreams.available);
try testing.expectEqual(@as(u32, 4), body.upstreams.total);
try testing.expectEqualStrings(upstreams_ok, body.upstreams.state);
try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total);
try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s);
try testing.expectEqualStrings(protection_active, body.protection.state);
}
const absent = rollup(.{ .upstreams_available = 1, .diagnostics_present = false });
try testing.expectEqualStrings(diagnostics_unavailable, absent.diagnostics.state);
try testing.expectEqualStrings(status_degraded, absent.status);
test "an unstamped drop time serializes as null, not as zero" {
var buffer: [1024]u8 = undefined;
var writer: std.Io.Writer = .fixed(&buffer);
try std.json.Stringify.value(rollup(.{}), .{}, &writer);
try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"last_drop_s\":null"));
try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"until\":null"));
}
test "collect reports an absent store as unavailable rather than as recording" {
@@ -307,6 +460,10 @@ test "collect reports an absent store as unavailable rather than as recording" {
const absent = collect(&state, io);
try testing.expect(!absent.diagnostics_present);
try testing.expectEqualStrings(diagnostics_unavailable, rollup(absent).diagnostics.state);
// And a bare state has no snapshot either, which is the honest reading of a
// process that has published nothing.
try testing.expect(!absent.snapshot_available);
try testing.expectEqualStrings(protection_unavailable, rollup(absent).protection.state);
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
@@ -325,76 +482,7 @@ test "collect reports an absent store as unavailable rather than as recording" {
try testing.expectEqualStrings(diagnostics_recording, rollup(present).diagnostics.state);
}
test "a history overflow that already happened does not degrade the rollup" {
// `rows_dropped` is cumulative and the rollup is stateless, so the only
// thing it could do with a drop count is latch on it. The accumulator's
// drops reach an operator through `/metrics` and through the per-window
// `complete` flag, and never through this.
const dropped: Input = .{
.upstreams_available = 1,
.upstreams_total = 1,
.history_flush_failing = false,
};
try testing.expect(!degraded(dropped));
try testing.expectEqualStrings(status_ok, rollup(dropped).status);
}
test "collect reads the accumulator's current flush state" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const acc = try testing.allocator.create(history_mod.Accumulator);
defer testing.allocator.destroy(acc);
acc.* = .init;
var state: server.WebState = .{ .gpa = testing.allocator, .history = acc };
try testing.expect(!collect(&state, io).history_flush_failing);
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
acc.recordSuccess(io, "https://a.example", 60);
// No schema in this database, so the real write fails and the flag is set
// by the production path rather than by a test poking a field.
acc.flushOnce(io, &database, upstream_history_repo.flush);
try testing.expect(collect(&state, io).history_flush_failing);
try testing.expectEqualStrings("degraded", rollup(collect(&state, io)).status);
}
test "the body reports every input verbatim" {
const body = rollup(.{
.disk_state = .warn,
.disk = .{ .free_bytes = 100, .db_bytes = 20, .log_bytes = 3 },
.disk_sample_failures = 2,
.upstreams_available = 2,
.upstreams_total = 4,
.queries_dropped = 9,
.writer_failed = false,
.refreshes_gated = 1,
.snapshot_generation = 12,
});
try testing.expectEqualStrings("degraded", body.status);
try testing.expectEqualStrings("warn", body.disk.state);
try testing.expectEqual(@as(u64, 100), body.disk.free_bytes);
try testing.expectEqual(@as(u64, 20), body.disk.db_bytes);
try testing.expectEqual(@as(u64, 3), body.disk.log_bytes);
try testing.expectEqual(@as(u64, 2), body.disk.sample_failures);
try testing.expectEqual(@as(u32, 2), body.upstreams.available);
try testing.expectEqual(@as(u32, 4), body.upstreams.total);
try testing.expectEqual(@as(u64, 9), body.queries_dropped);
try testing.expectEqual(@as(u64, 1), body.refreshes_gated);
try testing.expectEqual(@as(?u64, 12), body.snapshot_generation);
}
test "an unpublished snapshot serializes as null, not as zero" {
var buffer: [512]u8 = undefined;
var writer: std.Io.Writer = .fixed(&buffer);
try std.json.Stringify.value(rollup(.{}), .{}, &writer);
try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"snapshot_generation\":null"));
}
test "collect reads the logger's counters and reports a bare state as degraded" {
test "collect reads the logger's counters and the pause flag" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
@@ -402,14 +490,25 @@ test "collect reads the logger's counters and reports a bare state as degraded"
var queue_buf: [2]logger_mod.Entry = undefined;
var query_logger: logger_mod.Logger = .init(.{}, &queue_buf);
query_logger.queries_dropped.store(4, .monotonic);
query_logger.last_drop_s.store(1_700_000_000, .monotonic);
query_logger.writer_failed.store(true, .monotonic);
var state: server.WebState = .{ .gpa = testing.allocator, .logger = &query_logger };
var paused: pause_mod.Pause = .{};
paused.pauseFor(0, null);
var state: server.WebState = .{
.gpa = testing.allocator,
.logger = &query_logger,
.pause = &paused,
};
const input = collect(&state, io);
try testing.expectEqual(@as(u64, 4), input.queries_dropped);
try testing.expectEqual(@as(?i64, 1_700_000_000), input.last_drop_s);
try testing.expect(input.writer_failed);
try testing.expectEqual(@as(i64, -1), input.pause_until);
try testing.expectEqual(@as(u32, 0), input.upstreams_total);
try testing.expectEqual(@as(?u64, null), input.snapshot_generation);
try testing.expectEqualStrings("degraded", rollup(input).status);
try testing.expectEqualStrings(status_degraded, rollup(input).status);
// No snapshot manager either, so protection outranks the pause here too.
try testing.expectEqualStrings(protection_unavailable, rollup(input).protection.state);
}
src/web/handlers/queries.zig
+36 -6
View File
@@ -126,8 +126,6 @@ pub fn list(
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
_ = io;
var buffers: Buffers = .{};
const filter = parseFilter(request.query, &buffers) catch |err| {
return http_util.respondError(request, .bad_request, message(err));
@@ -136,7 +134,7 @@ pub fn list(
const database = state.querylog_db orelse
return http_util.respondError(request, .service_unavailable, "query log unavailable");
const result = page(database, request.arena, filter) catch |err| {
const result = readPage(state, io, database, request.arena, filter) catch |err| {
// The one thing this handler logs: a database fault is a property of
// the box, not of the request, and the client is told nothing about it.
log.warn("query log read failed: {s}", .{@errorName(err)});
@@ -156,12 +154,10 @@ pub fn detail(
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
_ = io;
const database = state.querylog_db orelse
return http_util.respondError(request, .service_unavailable, "query log unavailable");
const row = queries_repo.detailById(database, request.arena, request.id.?) catch |err| {
const row = detailRow(state, io, database, request.arena, request.id.?) catch |err| {
log.warn("query log read failed: {s}", .{@errorName(err)});
return http_util.respondError(request, .internal_server_error, "internal error");
};
@@ -172,6 +168,40 @@ pub fn detail(
return http_util.respondJson(request, .ok, provenance_view.fromDetail(found), &.{});
}
/// The rows and the coverage watermark come from one database state, so a
/// prune between them cannot tag pre-prune rows with a post-prune
/// `available_since`.
fn readPage(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: Allocator,
filter: queries_repo.QueryFilter,
) db.Error!Page {
var scope = try server.QuerylogRead.open(state, io, database);
errdefer scope.abort();
const result = try page(database, arena, filter);
try scope.commit();
return result;
}
/// One row, read under the shared lock. There is nothing to keep consistent
/// with a second statement here; the lock is what keeps this read out of
/// another response's open transaction.
fn detailRow(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: Allocator,
id: i64,
) db.Error!?queries_repo.QueryDetail {
var scope = try server.QuerylogRead.open(state, io, database);
errdefer scope.abort();
const row = try queries_repo.detailById(database, arena, id);
try scope.commit();
return row;
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
src/web/handlers/stats.zig
+231 -23
View File
@@ -1,8 +1,16 @@
//! `GET /api/stats` and `GET /api/stats/timeseries` (ruling 13).
//! The five period endpoints: `GET /api/stats` and `/api/stats/timeseries`
//! (ruling 13), and `/api/stats/types`, `/api/stats/routes` and
//! `/api/stats/clients` (milestone 30).
//!
//! One period grammar, four widths, and one window shared by both endpoints:
//! the totals cover exactly the span the chart draws, so a dashboard cannot
//! show a sum that disagrees with the bars above it.
//! One period grammar, four widths, and one window shared by all five: a
//! request for the same period gets the same `since`/`until` from every
//! endpoint, so the totals describe exactly the span the charts draw rather
//! than a neighbouring one.
//!
//! That is window coherence, not identical counts. Each endpoint is its own
//! request against its own snapshot, so queries logged between two of them move
//! one panel and not the other. Only a box with nothing writing to it — a test
//! — can expect the breakdowns to sum to the totals exactly.
//!
//! Buckets are aligned to the UTC grid, not to the moment of the request. Every
//! width divides a day, so flooring the current time to a multiple of the width
@@ -10,7 +18,19 @@
//! requests a second apart return the same bucket starts. The last bucket is
//! the one in progress; it fills as the period runs.
//!
//! The aggregates run on the web task's own query-log connection (m7 ruling 21).
//! The aggregates run on the web task's own query-log connection (m7 ruling 21),
//! which every connection task shares. SQLite's serialized mode makes one call
//! safe; it does not make a transaction safe, so `WebState.querylog_lock` covers
//! the whole read and a second BEGIN can never land inside the first. Each
//! response takes one deferred read transaction, so its aggregate and the
//! `coverage` beside it describe one database state: retention cannot prune
//! between them and hand a client pre-prune rows tagged with a post-prune
//! watermark. Deferred, not `db.Tx`'s BEGIN IMMEDIATE, which would stall the
//! logger and retention behind an HTTP response.
//!
//! The lock is released before the response is written: the body is already
//! built in the request arena, and holding a database lock across a socket
//! write would let one slow client serialize every other reader.
const std = @import("std");
@@ -97,7 +117,6 @@ pub const TotalsBody = struct {
until: i64,
queries: u64,
blocked: u64,
cached: u64,
clients: u64,
avg_response_time_us: ?i64,
/// Judged against `since`, which is the window this body reports on — so a
@@ -115,6 +134,135 @@ pub const TimeseriesBody = struct {
coverage: coverage.Coverage,
};
pub const TypesBody = struct {
period: []const u8,
since: i64,
until: i64,
types: []const queries_repo.TypeCount,
coverage: coverage.Coverage,
};
pub const RoutesBody = struct {
period: []const u8,
since: i64,
until: i64,
routes: []const queries_repo.RouteCount,
coverage: coverage.Coverage,
};
pub const ClientsBody = struct {
period: []const u8,
since: i64,
until: i64,
bucket_seconds: u32,
clients: []const queries_repo.ClientSeries,
other: []const u64,
coverage: coverage.Coverage,
};
/// Everything one response reads from the query log, so the caller can end the
/// transaction and drop the lock before it serializes anything.
fn Read(comptime T: type) type {
return struct {
data: T,
coverage: coverage.Coverage,
};
}
const ReadScope = server.QuerylogRead;
fn readTotals(
state: *server.WebState,
io: std.Io,
database: *db.Db,
span: Window,
) db.Error!Read(queries_repo.StatsTotals) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const read: Read(queries_repo.StatsTotals) = .{
.data = try queries_repo.statsTotals(database, span.since, span.until),
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readTimeseries(
state: *server.WebState,
io: std.Io,
database: *db.Db,
span: Window,
out: []queries_repo.Bucket,
) db.Error!Read(usize) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const read: Read(usize) = .{
.data = try queries_repo.timeseries(database, span.since, span.bucket_seconds, out),
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readTypes(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
span: Window,
) db.Error!Read([]const queries_repo.TypeCount) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const list = try queries_repo.statsTypes(database, arena, span.since, span.until);
const read: Read([]const queries_repo.TypeCount) = .{
.data = list.items,
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readRoutes(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
span: Window,
) db.Error!Read([]const queries_repo.RouteCount) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const list = try queries_repo.statsRoutes(database, arena, span.since, span.until);
const read: Read([]const queries_repo.RouteCount) = .{
.data = list.items,
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
fn readClients(
state: *server.WebState,
io: std.Io,
database: *db.Db,
arena: std.mem.Allocator,
span: Window,
) db.Error!Read(queries_repo.ClientsBreakdown) {
var scope = try ReadScope.open(state, io, database);
errdefer scope.abort();
const read: Read(queries_repo.ClientsBreakdown) = .{
.data = try queries_repo.statsClients(
database,
arena,
span.since,
span.bucket_seconds,
span.bucket_count,
),
.coverage = try coverage.read(database, span.since),
};
try scope.commit();
return read;
}
pub fn totals(
state: *server.WebState,
io: std.Io,
@@ -124,23 +272,19 @@ pub fn totals(
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const result = queries_repo.statsTotals(database, span.since, span.until) catch |err| {
const read = readTotals(state, io, database, span) catch |err| {
return internal(request, "stats totals", err);
};
const covered = coverage.read(database, span.since) catch |err| {
return internal(request, "stats coverage", err);
};
return http_util.respondJson(request, .ok, TotalsBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.queries = result.queries,
.blocked = result.blocked,
.cached = result.cached,
.clients = result.distinct_clients,
.avg_response_time_us = result.avg_response_time_us,
.coverage = covered,
.queries = read.data.queries,
.blocked = read.data.blocked,
.clients = read.data.distinct_clients,
.avg_response_time_us = read.data.avg_response_time_us,
.coverage = read.coverage,
}, &.{});
}
@@ -155,20 +299,85 @@ pub fn timeseries(
var buckets: [max_buckets]queries_repo.Bucket = undefined;
const out = buckets[0..span.bucket_count];
const written = queries_repo.timeseries(database, span.since, span.bucket_seconds, out) catch |err| {
const read = readTimeseries(state, io, database, span, out) catch |err| {
return internal(request, "stats timeseries", err);
};
const covered = coverage.read(database, span.since) catch |err| {
return internal(request, "stats coverage", err);
};
return http_util.respondJson(request, .ok, TimeseriesBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.bucket_seconds = span.bucket_seconds,
.buckets = out[0..written],
.coverage = covered,
.buckets = out[0..read.data],
.coverage = read.coverage,
}, &.{});
}
pub fn types(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readTypes(state, io, database, request.arena, span) catch |err| {
return internal(request, "stats types", err);
};
return http_util.respondJson(request, .ok, TypesBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.types = read.data,
.coverage = read.coverage,
}, &.{});
}
pub fn routes(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readRoutes(state, io, database, request.arena, span) catch |err| {
return internal(request, "stats routes", err);
};
return http_util.respondJson(request, .ok, RoutesBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.routes = read.data,
.coverage = read.coverage,
}, &.{});
}
pub fn clients(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = periodParam(request.query) catch return badPeriod(request);
const database = state.querylog_db orelse return unavailable(request);
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
const read = readClients(state, io, database, request.arena, span) catch |err| {
return internal(request, "stats clients", err);
};
return http_util.respondJson(request, .ok, ClientsBody{
.period = period.label(),
.since = span.since,
.until = span.until,
.bucket_seconds = span.bucket_seconds,
.clients = read.data.clients,
.other = read.data.other,
.coverage = read.coverage,
}, &.{});
}
@@ -321,7 +530,6 @@ test "the totals and the buckets agree over the same window" {
const result = try queries_repo.statsTotals(&database, span.since, span.until);
try testing.expectEqual(@as(u64, 3), result.queries);
try testing.expectEqual(@as(u64, 1), result.blocked);
try testing.expectEqual(@as(u64, 1), result.cached);
try testing.expectEqual(@as(u64, 1), result.distinct_clients);
try testing.expectEqual(@as(?i64, 1000), result.avg_response_time_us);
src/web/handlers/upstream_health.zig
-585
View File
@@ -1,585 +0,0 @@
//! `GET /api/upstream/health?period=` — the pool's upstreams over the window
//! the dashboard's period picker selected (milestone-26 ruling 6).
//!
//! Two kinds of fact, kept apart on the wire because they answer different
//! questions. `enabled`/`available` are live routing state, read from the pool
//! under its mutex: what the resolver would do with this upstream right now.
//! Everything under `period` is history, aggregated out of `upstream_minute`
//! over `[since, until)` — the same window `/api/stats` reports, so a page
//! cannot show a rate that disagrees with the chart beside it.
//!
//! Nothing here reads a process-lifetime counter. The lifetime totals, the
//! last-32-exchange window and the consecutive-failure count still live in
//! `health.State` for routing and in `/metrics`; they are not this response's
//! business, because a number that starts at process start cannot be scoped to
//! a period and a dashboard that shows one beside a picker lies about it.
//!
//! The aggregation runs on the web task's own query-log connection (m7 ruling
//! 21) and this file owns no SQL: `upstream_history_repo` does.
const std = @import("std");
const Allocator = std.mem.Allocator;
const db = @import("../../storage/db.zig");
const history_mod = @import("../../upstream/history.zig");
const http_util = @import("../http_util.zig");
const metrics = @import("../metrics.zig");
const pool_mod = @import("../../upstream/pool.zig");
const server = @import("../server.zig");
const stats = @import("stats.zig");
const upstream_history_repo = @import("../../storage/repositories/upstream_history_repo.zig");
const log = std.log.scoped(.web_upstream_health);
/// One upstream's outcomes inside the selected window.
pub const PeriodStats = struct {
attempts: u64,
successes: u64,
failures: u64,
/// Null when `attempts == 0`. No observations is not perfect reliability,
/// and a `100.0%` from an idle upstream is the exact misreading this
/// milestone exists to remove.
success_rate: ?f32,
/// The newest failure inside the window, on the wall clock the minute rows
/// are stamped with. Null when the window holds no failure, even if the
/// upstream failed before it.
last_failure_at: ?i64,
/// The error name belonging to `last_failure_at`; null exactly when it is.
last_failure_error: ?[]const u8,
};
pub const Upstream = struct {
url: []const u8,
/// Live: configuration, not history.
enabled: bool,
/// Live: false while the upstream is backing off.
available: bool,
period: PeriodStats,
};
pub const Body = struct {
period: []const u8,
since: i64,
until: i64,
available: u32,
total: u32,
/// See `isComplete`.
complete: bool,
upstreams: []const Upstream,
};
/// The same text `/api/stats` sends (`stats.zig`'s `badPeriod`). One period
/// grammar serves the whole dashboard, so the two routes must not disagree
/// about what a typo means.
pub const bad_period_message = "period must be one of 1h, 24h, 7d, 30d";
pub fn handle(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const period = stats.periodParam(request.query) catch
return http_util.respondError(request, .bad_request, bad_period_message);
const pool = state.pool orelse
return http_util.respondError(request, .service_unavailable, "no upstream pool");
const database = state.querylog_db orelse
return http_util.respondError(request, .service_unavailable, "query log unavailable");
const now = std.Io.Clock.real.now(io).toSeconds();
const body = collect(request.arena, io, pool, state.history, database, period, now) catch |err| {
if (err == error.OutOfMemory) return error.OutOfMemory;
// A failed aggregate is a fault in the box, not a property of the
// request (ruling 8, PLAN §19).
log.warn("upstream health window failed: {s}", .{@errorName(err)});
return http_util.respondError(request, .internal_server_error, "internal error");
};
return http_util.respondJson(request, .ok, body, &.{});
}
/// `db.Error` already carries `OutOfMemory`, so the arena's failures and
/// SQLite's share one set.
pub const Error = Allocator.Error || db.Error;
pub fn collect(
arena: Allocator,
io: std.Io,
pool: *pool_mod.Pool,
history: ?*history_mod.Accumulator,
database: *db.Db,
period: stats.Period,
now_unix: i64,
) Error!Body {
const span = stats.window(period, now_unix);
var raw: [metrics.max_upstreams]pool_mod.Snapshot = undefined;
const count = metrics.poolSnapshot(pool, io, &raw);
const out = try arena.alloc(Upstream, count);
var available: u32 = 0;
for (raw[0..count], out) |entry, *slot| {
if (entry.available) available += 1;
// Rows come from the current pool only: an upstream deleted from the
// configuration keeps its history in storage until retention takes it,
// and nothing joins it back into this response.
const window_stats = try upstream_history_repo.windowStats(
database,
entry.url,
span.since,
span.until,
);
slot.* = .{
.url = try arena.dupe(u8, entry.url),
.enabled = entry.enabled,
.available = entry.available,
.period = .{
.attempts = window_stats.attempts,
.successes = window_stats.successes,
.failures = window_stats.failures,
.success_rate = successRate(window_stats),
.last_failure_at = window_stats.last_failure_ts,
// `WindowStats` carries its error name by value, in storage this
// loop is done with as soon as the iteration ends. The copy into
// the arena is what keeps the response from pointing at bytes
// the next upstream's row overwrites.
.last_failure_error = if (window_stats.last_failure_ts == null)
null
else
try arena.dupe(u8, window_stats.lastFailureError()),
},
};
}
return .{
.period = period.label(),
.since = span.since,
.until = span.until,
.available = available,
.total = @intCast(count),
.complete = isComplete(history, io, span.since),
.upstreams = out,
};
}
fn successRate(window_stats: upstream_history_repo.WindowStats) ?f32 {
if (window_stats.attempts == 0) return null;
const successes: f32 = @floatFromInt(window_stats.successes);
const attempts: f32 = @floatFromInt(window_stats.attempts);
return successes / attempts;
}
/// Per-window and stateless (ruling 6): false iff capacity has cost this
/// process a minute that falls inside the window. A window that starts after
/// the newest such minute is complete again, so one historical overflow does
/// not mark every later response.
///
/// It says nothing about the newest outcomes, which may not have flushed yet,
/// and nothing about an unclean shutdown, which is not detectable here — the
/// openapi description spells both out.
fn isComplete(history: ?*history_mod.Accumulator, io: std.Io, since: i64) bool {
const accumulator = history orelse return true;
const dropped = accumulator.snapshotStats(io).last_drop_minute orelse return true;
return dropped < since;
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const querylog_schema = @import("../../storage/querylog_schema.zig");
const transport = @import("../../upstream/transport.zig");
const testing = std.testing;
/// The client is never called: every test here reads health, not answers.
fn testEntry(url: []const u8, enabled: bool) pool_mod.Entry {
return .{
.endpoint = transport.Endpoint.parse(url) catch unreachable,
.client = .{ .ptr = undefined, .exchangeFn = undefined },
.priority = 1,
.enabled = enabled,
.health = .init,
};
}
fn testPool(entries: []pool_mod.Entry) pool_mod.Pool {
return .init(entries, .{}, .{
.attempt = .{ .raw = .fromMilliseconds(50), .clock = .awake },
.total = .{ .raw = .fromMilliseconds(100), .clock = .awake },
}, 1);
}
fn openLog() !db.Db {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer database.close();
try db.applyPragmas(&database, .{});
try database.exec(querylog_schema.ddl);
return database;
}
const url_a = "https://a.test/dns-query";
const url_b = "https://b.test/dns-query";
/// A minute-aligned instant, so a window derived from it lands on round
/// numbers the assertions below can name.
const aligned_now: i64 = 1_699_999_980;
test "the window sums the minutes inside it and nothing outside" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
try upstream_history_repo.flush(&database, &.{
// One minute before the window.
.{ .url = url_a, .minute_ts = span.since - 60, .successes = 100, .failures = 100, .last_failure_ts = span.since - 30, .last_error = "Outside" },
.{ .url = url_a, .minute_ts = span.since, .successes = 3, .failures = 1, .last_failure_ts = span.since + 10, .last_error = "Timeout" },
.{ .url = url_a, .minute_ts = span.until - 60, .successes = 5, .failures = 0, .last_failure_ts = null, .last_error = "" },
// The window's exclusive end.
.{ .url = url_a, .minute_ts = span.until, .successes = 200, .failures = 200, .last_failure_ts = span.until + 5, .last_error = "After" },
});
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
try testing.expectEqualStrings("1h", body.period);
try testing.expectEqual(span.since, body.since);
try testing.expectEqual(span.until, body.until);
try testing.expectEqual(@as(u32, 1), body.total);
try testing.expectEqual(@as(u32, 1), body.available);
const period = body.upstreams[0].period;
try testing.expectEqual(@as(u64, 8), period.successes);
try testing.expectEqual(@as(u64, 1), period.failures);
try testing.expectEqual(@as(u64, 9), period.attempts);
try testing.expectEqual(@as(?i64, span.since + 10), period.last_failure_at);
try testing.expectEqualStrings("Timeout", period.last_failure_error.?);
}
test "an upstream with no attempts in the window reports null, never a perfect rate" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
// Only `b` has history, and only outside the window.
try upstream_history_repo.flush(&database, &.{
.{ .url = url_b, .minute_ts = span.since - 600, .successes = 4, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
var entries = [_]pool_mod.Entry{ testEntry(url_a, true), testEntry(url_b, true) };
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
for (body.upstreams) |upstream| {
try testing.expectEqual(@as(u64, 0), upstream.period.attempts);
try testing.expectEqual(@as(u64, 0), upstream.period.successes);
try testing.expectEqual(@as(u64, 0), upstream.period.failures);
try testing.expectEqual(@as(?f32, null), upstream.period.success_rate);
try testing.expectEqual(@as(?i64, null), upstream.period.last_failure_at);
try testing.expectEqual(@as(?[]const u8, null), upstream.period.last_failure_error);
}
}
test "the success rate is the window's own, not a lifetime one" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
try upstream_history_repo.flush(&database, &.{
// A clean past that a lifetime rate would average into the window.
.{ .url = url_a, .minute_ts = span.since - 600, .successes = 1000, .failures = 0, .last_failure_ts = null, .last_error = "" },
.{ .url = url_a, .minute_ts = span.since, .successes = 1, .failures = 3, .last_failure_ts = span.since + 1, .last_error = "Timeout" },
});
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
try testing.expectEqual(@as(?f32, 0.25), body.upstreams[0].period.success_rate);
}
test "the newest failure inside the window wins over an older one outside it" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
try upstream_history_repo.flush(&database, &.{
.{ .url = url_a, .minute_ts = span.since - 120, .successes = 0, .failures = 1, .last_failure_ts = span.since - 100, .last_error = "Older" },
.{ .url = url_a, .minute_ts = span.since, .successes = 0, .failures = 1, .last_failure_ts = span.since + 5, .last_error = "Newer" },
// A later minute with no failure at all must not blank the error.
.{ .url = url_a, .minute_ts = span.since + 60, .successes = 2, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
try testing.expectEqual(@as(?i64, span.since + 5), body.upstreams[0].period.last_failure_at);
try testing.expectEqualStrings("Newer", body.upstreams[0].period.last_failure_error.?);
}
test "two upstreams keep their own last-failure errors" {
// The by-value `WindowStats` buffer is reused per iteration, so a response
// that borrowed it would show the second upstream's error on the first, or
// point at stack storage that is gone by the time it is serialized.
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"1h", aligned_now);
try upstream_history_repo.flush(&database, &.{
.{ .url = url_a, .minute_ts = span.since, .successes = 1, .failures = 1, .last_failure_ts = span.since + 1, .last_error = "ConnectFailed" },
.{ .url = url_b, .minute_ts = span.since, .successes = 0, .failures = 2, .last_failure_ts = span.since + 2, .last_error = "TlsHandshakeFailed" },
});
var entries = [_]pool_mod.Entry{ testEntry(url_a, true), testEntry(url_b, true) };
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
try testing.expectEqual(@as(usize, 2), body.upstreams.len);
try testing.expectEqualStrings(url_a, body.upstreams[0].url);
try testing.expectEqualStrings("ConnectFailed", body.upstreams[0].period.last_failure_error.?);
try testing.expectEqualStrings(url_b, body.upstreams[1].url);
try testing.expectEqualStrings("TlsHandshakeFailed", body.upstreams[1].period.last_failure_error.?);
// Serializing after every row is read is what a real response does; the
// texts must still be the ones their own rows carried.
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
defer allocating.deinit();
try std.json.Stringify.value(body, .{}, &allocating.writer);
const text = allocating.written();
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"last_failure_error\":\"ConnectFailed\""));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"last_failure_error\":\"TlsHandshakeFailed\""));
}
test "a disabled upstream is not counted available and still gets its window" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const span = stats.window(.@"24h", aligned_now);
try upstream_history_repo.flush(&database, &.{
.{ .url = url_b, .minute_ts = span.since, .successes = 2, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
var entries = [_]pool_mod.Entry{ testEntry(url_a, true), testEntry(url_b, false) };
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, null, &database, .@"24h", aligned_now);
try testing.expectEqual(@as(u32, 2), body.total);
try testing.expectEqual(@as(u32, 1), body.available);
try testing.expect(!body.upstreams[1].enabled);
try testing.expect(!body.upstreams[1].available);
try testing.expectEqual(@as(u64, 2), body.upstreams[1].period.attempts);
}
/// Fills the accumulator and then overflows it, so `last_drop_minute` is
/// `minute` — the only way to set it, because the accumulator's fields are
/// private to its module and `snapshotStats` is the read surface.
fn accumulatorDroppingAt(
io: std.Io,
minute: i64,
names: *[history_mod.max_pending][8]u8,
) !*history_mod.Accumulator {
const accumulator = try testing.allocator.create(history_mod.Accumulator);
accumulator.* = .init;
for (names, 0..) |*name, i| {
const url = std.fmt.bufPrint(name, "u{d:0>6}", .{i}) catch unreachable;
accumulator.recordSuccess(io, url, minute);
}
// One more cell than capacity: the oldest minute goes, and every cell above
// holds `minute`.
accumulator.recordSuccess(io, "https://overflow.test", minute + 60);
return accumulator;
}
test "complete is false only while a dropped minute falls inside the window" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
// Two `now`s one minute apart, so the same drop is inside the first
// window and one minute before the second.
const inside_now = aligned_now;
const inside = stats.window(.@"1h", inside_now);
const after = stats.window(.@"1h", inside_now + 60);
try testing.expectEqual(inside.since + 60, after.since);
var names: [history_mod.max_pending][8]u8 = undefined;
const accumulator = try accumulatorDroppingAt(io, inside.since, &names);
defer testing.allocator.destroy(accumulator);
try testing.expectEqual(@as(?i64, inside.since), accumulator.snapshotStats(io).last_drop_minute);
const flagged = try collect(arena.allocator(), io, &pool, accumulator, &database, .@"1h", inside_now);
try testing.expect(!flagged.complete);
const recovered = try collect(arena.allocator(), io, &pool, accumulator, &database, .@"1h", inside_now + 60);
try testing.expect(recovered.complete);
// No accumulator at all is no known drop, not an incomplete window.
const unwired = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", inside_now);
try testing.expect(unwired.complete);
}
test "a drop with no overflow leaves every window complete" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
const accumulator = try testing.allocator.create(history_mod.Accumulator);
defer testing.allocator.destroy(accumulator);
accumulator.* = .init;
accumulator.recordFailure(io, url_a, aligned_now, "Timeout");
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const body = try collect(arena.allocator(), io, &pool, accumulator, &database, .@"1h", aligned_now);
try testing.expect(body.complete);
}
test "the route's period grammar and its 400 text are the ones /api/stats serves" {
// The picker scopes the whole page, so one bad spelling must mean the same
// thing on every route it drives.
try testing.expectEqual(stats.default_period, try stats.periodParam(""));
try testing.expectEqual(stats.Period.@"24h", try stats.periodParam(""));
try testing.expectEqual(stats.Period.@"7d", try stats.periodParam("period=7d"));
try testing.expectError(error.BadPeriod, stats.periodParam("period=12h"));
try testing.expectError(error.BadPeriod, stats.periodParam("period=1hhhhhhhhhh"));
try testing.expectEqualStrings("period must be one of 1h, 24h, 7d, 30d", bad_period_message);
}
test "every period the grammar accepts produces the window that period names" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
var pool = testPool(&entries);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
for (std.enums.values(stats.Period)) |period| {
const span = stats.window(period, aligned_now);
const body = try collect(arena.allocator(), io, &pool, null, &database, period, aligned_now);
try testing.expectEqualStrings(period.label(), body.period);
try testing.expectEqual(span.since, body.since);
try testing.expectEqual(span.until, body.until);
}
}
test "the body serializes exactly the ranged field set" {
const upstreams = [_]Upstream{ .{
.url = url_a,
.enabled = true,
.available = false,
.period = .{
.attempts = 8,
.successes = 6,
.failures = 2,
.success_rate = 0.75,
.last_failure_at = 1_700_000_000,
.last_failure_error = "ConnectFailed",
},
}, .{
.url = url_b,
.enabled = false,
.available = false,
.period = .{
.attempts = 0,
.successes = 0,
.failures = 0,
.success_rate = null,
.last_failure_at = null,
.last_failure_error = null,
},
} };
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
defer allocating.deinit();
try std.json.Stringify.value(Body{
.period = "1h",
.since = 1_699_996_400,
.until = 1_700_000_000,
.available = 0,
.total = 2,
.complete = true,
.upstreams = &upstreams,
}, .{}, &allocating.writer);
try testing.expectEqualStrings(
\\{"period":"1h","since":1699996400,"until":1700000000,"available":0,"total":2,"complete":true,"upstreams":[{"url":"https://a.test/dns-query","enabled":true,"available":false,"period":{"attempts":8,"successes":6,"failures":2,"success_rate":0.75,"last_failure_at":1700000000,"last_failure_error":"ConnectFailed"}},{"url":"https://b.test/dns-query","enabled":false,"available":false,"period":{"attempts":0,"successes":0,"failures":0,"success_rate":null,"last_failure_at":null,"last_failure_error":null}}]}
, allocating.written());
// The lifetime fields m26 removed. They still exist in `health.State` and in
// `/metrics`; a client of this route must not find them here and start
// reading them as if they were scoped to the period.
for ([_][]const u8{
"consecutive_failures",
"total_successes",
"total_failures",
"last_error_age_s",
"\"last_error\"",
}) |gone| {
try testing.expect(!std.mem.containsAtLeast(u8, allocating.written(), 1, gone));
}
}