milestone 8: web server, rest api, sse, auth, metrics and static assets
This commit is contained in:
@@ -0,0 +1,236 @@
|
||||
//! `GET /api/health` — the rollup a monitor scrapes (ruling 22).
|
||||
//!
|
||||
//! Always 200. "degraded" is a fact about the box, not a failure of the
|
||||
//! request, and answering 503 would make an uptime check flap on a full disk
|
||||
//! while nxdns is still resolving perfectly well.
|
||||
//!
|
||||
//! Unauthenticated and rate-limit exempt, like `/metrics`.
|
||||
//!
|
||||
//! `rollup` is pure so the whole degraded matrix is testable without a running
|
||||
//! server; `handle` only gathers the inputs.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const disk_monitor = @import("../../storage/disk_monitor.zig");
|
||||
const http_util = @import("../http_util.zig");
|
||||
const metrics = @import("../metrics.zig");
|
||||
const pool_mod = @import("../../upstream/pool.zig");
|
||||
const server = @import("../server.zig");
|
||||
|
||||
pub const Disk = struct {
|
||||
state: []const u8,
|
||||
free_bytes: u64,
|
||||
db_bytes: u64,
|
||||
log_bytes: u64,
|
||||
sample_failures: u64,
|
||||
};
|
||||
|
||||
pub const Upstreams = struct {
|
||||
available: u32,
|
||||
total: u32,
|
||||
};
|
||||
|
||||
pub const Body = struct {
|
||||
status: []const u8,
|
||||
disk: Disk,
|
||||
upstreams: Upstreams,
|
||||
queries_dropped: u64,
|
||||
writer_failed: bool,
|
||||
refreshes_gated: u64,
|
||||
/// Null before the first filter snapshot is published.
|
||||
snapshot_generation: ?u64,
|
||||
};
|
||||
|
||||
/// What the rollup is computed from. Every field has a defined value even when
|
||||
/// its collaborator is missing, and the defaults are the ones a half-wired
|
||||
/// server should report: no disk reading, no upstreams, nothing published.
|
||||
pub const Input = struct {
|
||||
disk_state: disk_monitor.State = .ok,
|
||||
disk: disk_monitor.Gauges = .{ .free_bytes = 0, .db_bytes = 0, .log_bytes = 0 },
|
||||
disk_sample_failures: u64 = 0,
|
||||
upstreams_available: u32 = 0,
|
||||
upstreams_total: u32 = 0,
|
||||
queries_dropped: u64 = 0,
|
||||
writer_failed: bool = false,
|
||||
refreshes_gated: u64 = 0,
|
||||
snapshot_generation: ?u64 = null,
|
||||
};
|
||||
|
||||
pub const status_ok = "ok";
|
||||
pub const status_degraded = "degraded";
|
||||
|
||||
/// Ruling 22's three conditions. Each one is something an operator must act on:
|
||||
/// a disk that is filling stops the query log, a pool with nothing available
|
||||
/// stops resolution, and a failed writer means rows are being lost right now.
|
||||
pub fn degraded(input: Input) bool {
|
||||
return input.disk_state != .ok or input.upstreams_available == 0 or input.writer_failed;
|
||||
}
|
||||
|
||||
pub fn rollup(input: Input) Body {
|
||||
return .{
|
||||
.status = if (degraded(input)) status_degraded else status_ok,
|
||||
.disk = .{
|
||||
.state = @tagName(input.disk_state),
|
||||
.free_bytes = input.disk.free_bytes,
|
||||
.db_bytes = input.disk.db_bytes,
|
||||
.log_bytes = input.disk.log_bytes,
|
||||
.sample_failures = input.disk_sample_failures,
|
||||
},
|
||||
.upstreams = .{ .available = input.upstreams_available, .total = input.upstreams_total },
|
||||
.queries_dropped = input.queries_dropped,
|
||||
.writer_failed = input.writer_failed,
|
||||
.refreshes_gated = input.refreshes_gated,
|
||||
.snapshot_generation = input.snapshot_generation,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn handle(
|
||||
state: *server.WebState,
|
||||
io: std.Io,
|
||||
request: *http_util.Request,
|
||||
) http_util.HandlerError!void {
|
||||
return http_util.respondJson(request, .ok, rollup(collect(state, io)), &.{});
|
||||
}
|
||||
|
||||
pub fn collect(state: *server.WebState, io: std.Io) Input {
|
||||
var input: Input = .{};
|
||||
|
||||
if (state.monitor) |monitor| {
|
||||
input.disk_state = monitor.state();
|
||||
input.disk = monitor.gauges();
|
||||
input.disk_sample_failures = monitor.sample_failures.load(.monotonic);
|
||||
}
|
||||
|
||||
if (state.pool) |pool| {
|
||||
var raw: [metrics.max_upstreams]pool_mod.Snapshot = undefined;
|
||||
const count = metrics.poolSnapshot(pool, io, &raw);
|
||||
input.upstreams_total = @intCast(count);
|
||||
for (raw[0..count]) |entry| {
|
||||
if (entry.available) input.upstreams_available += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (state.logger) |logger| {
|
||||
input.queries_dropped = logger.queries_dropped.load(.monotonic);
|
||||
input.writer_failed = logger.writer_failed.load(.monotonic);
|
||||
}
|
||||
|
||||
if (state.manager) |manager| {
|
||||
input.refreshes_gated = manager.refreshesGated();
|
||||
if (manager.acquire(io)) |acquired| {
|
||||
defer acquired.release(io);
|
||||
input.snapshot_generation = acquired.snapshot.generation;
|
||||
}
|
||||
}
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const logger_mod = @import("../../storage/logger.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
/// A box with nothing wrong with it: one upstream up, disk ok, writer alive.
|
||||
const healthy: Input = .{
|
||||
.disk_state = .ok,
|
||||
.upstreams_available = 1,
|
||||
.upstreams_total = 1,
|
||||
.writer_failed = false,
|
||||
};
|
||||
|
||||
test "the degraded matrix covers disk state, availability and the writer" {
|
||||
const cases = [_]struct { input: Input, degraded: bool }{
|
||||
.{ .input = healthy, .degraded = false },
|
||||
.{ .input = withDisk(healthy, .warn), .degraded = true },
|
||||
.{ .input = withDisk(healthy, .critical), .degraded = true },
|
||||
.{ .input = withAvailable(healthy, 0), .degraded = true },
|
||||
.{ .input = withWriterFailed(healthy), .degraded = true },
|
||||
// Two faults at once still report one status.
|
||||
.{ .input = withWriterFailed(withDisk(healthy, .critical)), .degraded = true },
|
||||
// Some upstreams down is not degraded while one still answers.
|
||||
.{ .input = .{ .upstreams_available = 1, .upstreams_total = 3 }, .degraded = false },
|
||||
};
|
||||
|
||||
for (cases, 0..) |case, i| {
|
||||
errdefer std.debug.print("case {d}\n", .{i});
|
||||
try testing.expectEqual(case.degraded, degraded(case.input));
|
||||
try testing.expectEqualStrings(
|
||||
if (case.degraded) status_degraded else status_ok,
|
||||
rollup(case.input).status,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn withDisk(input: Input, state: disk_monitor.State) Input {
|
||||
var out = input;
|
||||
out.disk_state = state;
|
||||
return out;
|
||||
}
|
||||
|
||||
fn withAvailable(input: Input, available: u32) Input {
|
||||
var out = input;
|
||||
out.upstreams_available = available;
|
||||
return out;
|
||||
}
|
||||
|
||||
fn withWriterFailed(input: Input) Input {
|
||||
var out = input;
|
||||
out.writer_failed = true;
|
||||
return out;
|
||||
}
|
||||
|
||||
test "the body reports every input verbatim" {
|
||||
const body = rollup(.{
|
||||
.disk_state = .warn,
|
||||
.disk = .{ .free_bytes = 100, .db_bytes = 20, .log_bytes = 3 },
|
||||
.disk_sample_failures = 2,
|
||||
.upstreams_available = 2,
|
||||
.upstreams_total = 4,
|
||||
.queries_dropped = 9,
|
||||
.writer_failed = false,
|
||||
.refreshes_gated = 1,
|
||||
.snapshot_generation = 12,
|
||||
});
|
||||
|
||||
try testing.expectEqualStrings("degraded", body.status);
|
||||
try testing.expectEqualStrings("warn", body.disk.state);
|
||||
try testing.expectEqual(@as(u64, 100), body.disk.free_bytes);
|
||||
try testing.expectEqual(@as(u64, 20), body.disk.db_bytes);
|
||||
try testing.expectEqual(@as(u64, 3), body.disk.log_bytes);
|
||||
try testing.expectEqual(@as(u64, 2), body.disk.sample_failures);
|
||||
try testing.expectEqual(@as(u32, 2), body.upstreams.available);
|
||||
try testing.expectEqual(@as(u32, 4), body.upstreams.total);
|
||||
try testing.expectEqual(@as(u64, 9), body.queries_dropped);
|
||||
try testing.expectEqual(@as(u64, 1), body.refreshes_gated);
|
||||
try testing.expectEqual(@as(?u64, 12), body.snapshot_generation);
|
||||
}
|
||||
|
||||
test "an unpublished snapshot serializes as null, not as zero" {
|
||||
var buffer: [512]u8 = undefined;
|
||||
var writer: std.Io.Writer = .fixed(&buffer);
|
||||
try std.json.Stringify.value(rollup(.{}), .{}, &writer);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"snapshot_generation\":null"));
|
||||
}
|
||||
|
||||
test "collect reads the logger's counters and reports a bare state as degraded" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var queue_buf: [2]logger_mod.Entry = undefined;
|
||||
var query_logger: logger_mod.Logger = .init(.{}, &queue_buf);
|
||||
query_logger.queries_dropped.store(4, .monotonic);
|
||||
query_logger.writer_failed.store(true, .monotonic);
|
||||
|
||||
var state: server.WebState = .{ .gpa = testing.allocator, .logger = &query_logger };
|
||||
const input = collect(&state, io);
|
||||
|
||||
try testing.expectEqual(@as(u64, 4), input.queries_dropped);
|
||||
try testing.expect(input.writer_failed);
|
||||
try testing.expectEqual(@as(u32, 0), input.upstreams_total);
|
||||
try testing.expectEqual(@as(?u64, null), input.snapshot_generation);
|
||||
try testing.expectEqualStrings("degraded", rollup(input).status);
|
||||
}
|
||||
Reference in New Issue
Block a user