//! `GET /api/health` — the rollup a monitor scrapes (ruling 22). //! //! Always 200. "degraded" is a fact about the box, not a failure of the //! request, and answering 503 would make an uptime check flap on a full disk //! while nxdns is still resolving perfectly well. //! //! Unauthenticated and rate-limit exempt, like `/metrics`. //! //! `rollup` is pure so the whole degraded matrix is testable without a running //! server; `handle` only gathers the inputs. const std = @import("std"); const disk_monitor = @import("../../storage/disk_monitor.zig"); const http_util = @import("../http_util.zig"); const metrics = @import("../metrics.zig"); const pool_mod = @import("../../upstream/pool.zig"); const server = @import("../server.zig"); pub const Disk = struct { state: []const u8, free_bytes: u64, db_bytes: u64, log_bytes: u64, sample_failures: u64, }; pub const Upstreams = struct { available: u32, total: u32, }; pub const Body = struct { status: []const u8, disk: Disk, upstreams: Upstreams, queries_dropped: u64, writer_failed: bool, refreshes_gated: u64, /// Null before the first filter snapshot is published. snapshot_generation: ?u64, }; /// What the rollup is computed from. Every field has a defined value even when /// its collaborator is missing, and the defaults are the ones a half-wired /// server should report: no disk reading, no upstreams, nothing published. pub const Input = struct { disk_state: disk_monitor.State = .ok, disk: disk_monitor.Gauges = .{ .free_bytes = 0, .db_bytes = 0, .log_bytes = 0 }, disk_sample_failures: u64 = 0, upstreams_available: u32 = 0, upstreams_total: u32 = 0, queries_dropped: u64 = 0, writer_failed: bool = false, refreshes_gated: u64 = 0, snapshot_generation: ?u64 = null, }; pub const status_ok = "ok"; pub const status_degraded = "degraded"; /// Ruling 22's three conditions. Each one is something an operator must act on: /// a disk that is filling stops the query log, a pool with nothing available /// stops resolution, and a failed writer means rows are being lost right now. pub fn degraded(input: Input) bool { return input.disk_state != .ok or input.upstreams_available == 0 or input.writer_failed; } pub fn rollup(input: Input) Body { return .{ .status = if (degraded(input)) status_degraded else status_ok, .disk = .{ .state = @tagName(input.disk_state), .free_bytes = input.disk.free_bytes, .db_bytes = input.disk.db_bytes, .log_bytes = input.disk.log_bytes, .sample_failures = input.disk_sample_failures, }, .upstreams = .{ .available = input.upstreams_available, .total = input.upstreams_total }, .queries_dropped = input.queries_dropped, .writer_failed = input.writer_failed, .refreshes_gated = input.refreshes_gated, .snapshot_generation = input.snapshot_generation, }; } pub fn handle( state: *server.WebState, io: std.Io, request: *http_util.Request, ) http_util.HandlerError!void { return http_util.respondJson(request, .ok, rollup(collect(state, io)), &.{}); } pub fn collect(state: *server.WebState, io: std.Io) Input { var input: Input = .{}; if (state.monitor) |monitor| { input.disk_state = monitor.state(); input.disk = monitor.gauges(); input.disk_sample_failures = monitor.sample_failures.load(.monotonic); } if (state.pool) |pool| { var raw: [metrics.max_upstreams]pool_mod.Snapshot = undefined; const count = metrics.poolSnapshot(pool, io, &raw); input.upstreams_total = @intCast(count); for (raw[0..count]) |entry| { if (entry.available) input.upstreams_available += 1; } } if (state.logger) |logger| { input.queries_dropped = logger.queries_dropped.load(.monotonic); input.writer_failed = logger.writer_failed.load(.monotonic); } if (state.manager) |manager| { input.refreshes_gated = manager.refreshesGated(); if (manager.acquire(io)) |acquired| { defer acquired.release(io); input.snapshot_generation = acquired.snapshot.generation; } } return input; } // --------------------------------------------------------------------------- // tests // --------------------------------------------------------------------------- const logger_mod = @import("../../storage/logger.zig"); const testing = std.testing; /// A box with nothing wrong with it: one upstream up, disk ok, writer alive. const healthy: Input = .{ .disk_state = .ok, .upstreams_available = 1, .upstreams_total = 1, .writer_failed = false, }; test "the degraded matrix covers disk state, availability and the writer" { const cases = [_]struct { input: Input, degraded: bool }{ .{ .input = healthy, .degraded = false }, .{ .input = withDisk(healthy, .warn), .degraded = true }, .{ .input = withDisk(healthy, .critical), .degraded = true }, .{ .input = withAvailable(healthy, 0), .degraded = true }, .{ .input = withWriterFailed(healthy), .degraded = true }, // Two faults at once still report one status. .{ .input = withWriterFailed(withDisk(healthy, .critical)), .degraded = true }, // Some upstreams down is not degraded while one still answers. .{ .input = .{ .upstreams_available = 1, .upstreams_total = 3 }, .degraded = false }, }; for (cases, 0..) |case, i| { errdefer std.debug.print("case {d}\n", .{i}); try testing.expectEqual(case.degraded, degraded(case.input)); try testing.expectEqualStrings( if (case.degraded) status_degraded else status_ok, rollup(case.input).status, ); } } fn withDisk(input: Input, state: disk_monitor.State) Input { var out = input; out.disk_state = state; return out; } fn withAvailable(input: Input, available: u32) Input { var out = input; out.upstreams_available = available; return out; } fn withWriterFailed(input: Input) Input { var out = input; out.writer_failed = true; return out; } test "the body reports every input verbatim" { const body = rollup(.{ .disk_state = .warn, .disk = .{ .free_bytes = 100, .db_bytes = 20, .log_bytes = 3 }, .disk_sample_failures = 2, .upstreams_available = 2, .upstreams_total = 4, .queries_dropped = 9, .writer_failed = false, .refreshes_gated = 1, .snapshot_generation = 12, }); try testing.expectEqualStrings("degraded", body.status); try testing.expectEqualStrings("warn", body.disk.state); try testing.expectEqual(@as(u64, 100), body.disk.free_bytes); try testing.expectEqual(@as(u64, 20), body.disk.db_bytes); try testing.expectEqual(@as(u64, 3), body.disk.log_bytes); try testing.expectEqual(@as(u64, 2), body.disk.sample_failures); try testing.expectEqual(@as(u32, 2), body.upstreams.available); try testing.expectEqual(@as(u32, 4), body.upstreams.total); try testing.expectEqual(@as(u64, 9), body.queries_dropped); try testing.expectEqual(@as(u64, 1), body.refreshes_gated); try testing.expectEqual(@as(?u64, 12), body.snapshot_generation); } test "an unpublished snapshot serializes as null, not as zero" { var buffer: [512]u8 = undefined; var writer: std.Io.Writer = .fixed(&buffer); try std.json.Stringify.value(rollup(.{}), .{}, &writer); try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"snapshot_generation\":null")); } test "collect reads the logger's counters and reports a bare state as degraded" { var threaded: std.Io.Threaded = .init(testing.allocator, .{}); defer threaded.deinit(); const io = threaded.io(); var queue_buf: [2]logger_mod.Entry = undefined; var query_logger: logger_mod.Logger = .init(.{}, &queue_buf); query_logger.queries_dropped.store(4, .monotonic); query_logger.writer_failed.store(true, .monotonic); var state: server.WebState = .{ .gpa = testing.allocator, .logger = &query_logger }; const input = collect(&state, io); try testing.expectEqual(@as(u64, 4), input.queries_dropped); try testing.expect(input.writer_failed); try testing.expectEqual(@as(u32, 0), input.upstreams_total); try testing.expectEqual(@as(?u64, null), input.snapshot_generation); try testing.expectEqualStrings("degraded", rollup(input).status); }