//! `GET /api/health` — the rollup a monitor scrapes (ruling 22). //! //! Always 200. "degraded" is a fact about the box, not a failure of the //! request, and answering 503 would make an uptime check flap on a full disk //! while nxdns is still resolving perfectly well. //! //! Unauthenticated and rate-limit exempt, like `/metrics`. //! //! **Nothing may degrade the rollup without appearing in the response.** The //! body is five condition objects and a status computed from exactly their //! states, so an operator reading it can always name the condition that //! degraded the box. Anything a counter already records — gated refreshes, //! disk sample failures, the snapshot generation — stays in `/metrics` and in //! Diagnostics rather than becoming a sixth hidden input here. //! //! `rollup` is pure so the whole degraded matrix is testable without a running //! server; `handle` only gathers the inputs. const std = @import("std"); const disk_monitor = @import("../../storage/disk_monitor.zig"); const http_util = @import("../http_util.zig"); const logger_controller = @import("../../storage/logger_controller.zig"); const logger_mod = @import("../../storage/logger.zig"); const metrics = @import("../metrics.zig"); const pause_mod = @import("../../server/pause.zig"); const pool_mod = @import("../../upstream/pool.zig"); const server = @import("../server.zig"); /// Is filtering in force. `unavailable` is not an operator's doing: it is the /// state in which the query path has no filter snapshot to evaluate against, /// which `handler.zig` records as the `snapshot_unavailable` provenance. pub const Protection = struct { state: []const u8, /// The second filtering resumes at. Null for both an indefinite pause and /// every non-paused state, which the state field already tells apart. until: ?i64, }; pub const Upstreams = struct { state: []const u8, available: u32, /// Enabled upstreams: the pool is built from those alone, so this is what /// `available` is out of. total: u32, }; /// Whether Activity can be trusted. `dropped_total` is cumulative and does not /// decide the state — a drop that happened an hour ago is not a fault now. pub const QueryHistory = struct { state: []const u8, dropped_total: u64, /// The newest drop, or null while nothing has been dropped. Stamped by a /// separate atomic from the count, so a reader can momentarily see a /// non-zero `dropped_total` beside a null here. last_drop_s: ?i64, }; /// The diagnostics store's own state, not a summary of what it holds: `state` /// answers "is the operational log recording", and the two counts answer "what /// is open right now". pub const Diagnostics = struct { state: []const u8, active_warnings: u32, active_errors: u32, }; pub const Disk = struct { state: []const u8, free_bytes: u64, }; pub const Body = struct { status: []const u8, protection: Protection, upstreams: Upstreams, query_history: QueryHistory, diagnostics: Diagnostics, disk: Disk, }; /// What the rollup is computed from. Every field has a defined value even when /// its collaborator is missing, and the defaults are the ones a half-wired /// server should report: no disk reading, no upstreams, nothing published. pub const Input = struct { disk_state: disk_monitor.State = .ok, disk_free_bytes: u64 = 0, upstreams_available: u32 = 0, upstreams_total: u32 = 0, queries_dropped: u64 = 0, last_drop_s: ?i64 = null, writer_failed: bool = false, /// The logger's disk-gating episode. `losing` is the only state that /// degrades: it means the gate is holding writes back *and* has already /// cost rows in the episode that is open now. gate_episode: logger_mod.GateEpisode = .open, /// A filter snapshot exists for the query path to evaluate against. The /// benign default matches every other field here, and `collect` assigns it /// explicitly for the same reason `diagnostics_present` is assigned there. snapshot_available: bool = true, /// `pause.Pause.until` verbatim: 0 not paused, -1 indefinite, otherwise the /// second filtering resumes at. Raw rather than a decided boolean so the /// expiry rule stays `pause.Pause`'s and is exercised by these tests. pause_until: i64 = 0, /// The clock the pause is compared against. now_s: i64 = 0, /// The diagnostics store exists. The benign default matches every other /// field here — a half-wired `Input` reports a box with nothing wrong — but /// `collect` must assign it explicitly, because in a serving process an /// absent store means `Store.init` failed. diagnostics_present: bool = true, /// The last diagnostics write failed. Current state, cleared by the next /// write that succeeds. diagnostics_write_failed: bool = false, diagnostics_active_warnings: u32 = 0, diagnostics_active_errors: u32 = 0, }; pub const status_ok = "ok"; pub const status_degraded = "degraded"; pub const protection_active = "active"; pub const protection_paused = "paused"; pub const protection_unavailable = "unavailable"; pub const upstreams_ok = "ok"; pub const upstreams_unavailable = "unavailable"; pub const query_history_recording = "recording"; pub const query_history_losing = "losing"; pub const query_history_failed = "failed"; pub const diagnostics_recording = "recording"; pub const diagnostics_unavailable = "unavailable"; pub const disk_ok = "ok"; pub const disk_low = "low"; pub const disk_critical = "critical"; /// Precedence `unavailable` → `paused` → `active`. With no snapshot the pause /// flag says nothing an operator can act on: filtering is off either way, and /// resuming would not turn it back on. pub fn protection(input: Input) Protection { if (!input.snapshot_available) return .{ .state = protection_unavailable, .until = null }; const state: pause_mod.Pause = .{ .until = .init(input.pause_until) }; if (!state.isPaused(input.now_s)) return .{ .state = protection_active, .until = null }; return .{ .state = protection_paused, .until = if (input.pause_until > 0) input.pause_until else null, }; } fn queryHistoryState(input: Input) []const u8 { if (input.writer_failed) return query_history_failed; if (input.gate_episode == .losing) return query_history_losing; return query_history_recording; } /// The operational log is not recording — either the store never opened or its /// writes are failing. Both mean the same thing to an operator: the record of /// what went wrong is not being kept. fn diagnosticsUnavailable(input: Input) bool { return !input.diagnostics_present or input.diagnostics_write_failed; } /// `warn` reads as a log level rather than as a quantity of disk. The monitor /// keeps its own name; the wire says what an operator sees on the page. fn diskState(state: disk_monitor.State) []const u8 { return switch (state) { .ok => disk_ok, .warn => disk_low, .critical => disk_critical, }; } /// The degrading set, named rather than gestured at: protection `unavailable`, /// upstreams `unavailable`, query history `losing` or `failed`, diagnostics /// `unavailable`, disk `low` or `critical`. /// /// A pause is deliberately not in it. It is an operator's own choice, and a /// monitor that pages on it would be paging on a button the operator pressed. pub fn degraded(input: Input) bool { const history = queryHistoryState(input); return !input.snapshot_available or input.upstreams_available == 0 or !std.mem.eql(u8, history, query_history_recording) or diagnosticsUnavailable(input) or input.disk_state != .ok; } pub fn rollup(input: Input) Body { return .{ .status = if (degraded(input)) status_degraded else status_ok, .protection = protection(input), .upstreams = .{ .state = if (input.upstreams_available == 0) upstreams_unavailable else upstreams_ok, .available = input.upstreams_available, .total = input.upstreams_total, }, .query_history = .{ .state = queryHistoryState(input), .dropped_total = input.queries_dropped, .last_drop_s = input.last_drop_s, }, .diagnostics = .{ .state = if (diagnosticsUnavailable(input)) diagnostics_unavailable else diagnostics_recording, .active_warnings = input.diagnostics_active_warnings, .active_errors = input.diagnostics_active_errors, }, .disk = .{ .state = diskState(input.disk_state), .free_bytes = input.disk_free_bytes }, }; } pub fn handle( state: *server.WebState, io: std.Io, request: *http_util.Request, ) http_util.HandlerError!void { return http_util.respondJson(request, .ok, rollup(collect(state, io)), &.{}); } pub fn collect(state: *server.WebState, io: std.Io) Input { var input: Input = .{}; input.now_s = std.Io.Clock.real.now(io).toSeconds(); if (state.monitor) |monitor| { input.disk_state = monitor.state(); input.disk_free_bytes = monitor.gauges().free_bytes; } if (state.upstreams) |owner| { const generation = owner.acquire(io); defer owner.release(io, generation); if (generation.pool) |pool| { var raw: [metrics.max_upstreams]pool_mod.Snapshot = undefined; const count = metrics.poolSnapshot(pool, io, &raw); input.upstreams_total = @intCast(count); for (raw[0..count]) |entry| { if (entry.available) input.upstreams_available += 1; } } } if (state.logger) |controller| { const reading = controller.sample(io); input.queries_dropped = reading.queries_dropped; input.last_drop_s = reading.last_drop_s; input.writer_failed = reading.writer_failed; input.gate_episode = reading.gate_episode; } if (state.pause) |paused| input.pause_until = paused.until.load(.monotonic); // Assigned before the `if`, not inside it: the field's benign default is // `true`, so the natural `if (state.events) |store|` shape would report an // absent store as recording — the one case that must degrade. input.diagnostics_present = state.events != null; if (state.events) |store| { input.diagnostics_write_failed = store.writeFailed(); const counts = store.activeCounts(io); input.diagnostics_active_warnings = counts.warnings; input.diagnostics_active_errors = counts.errors; } // The generation itself is not in the body — the UI-facing fact is whether // protection has a snapshot at all, and `/metrics` keeps the number. input.snapshot_available = false; if (state.manager) |manager| { if (manager.acquire(io)) |acquired| { defer acquired.release(io); input.snapshot_available = true; } } return input; } // --------------------------------------------------------------------------- // tests // --------------------------------------------------------------------------- const db = @import("../../storage/db.zig"); const events_mod = @import("../../storage/events.zig"); const migrations = @import("../../storage/migrations.zig"); const testing = std.testing; /// A box with nothing wrong with it: one upstream up, disk ok, writer alive, /// a snapshot published and filtering unpaused. const healthy: Input = .{ .disk_state = .ok, .upstreams_available = 1, .upstreams_total = 1, .writer_failed = false, .snapshot_available = true, }; fn with(input: Input, comptime field: []const u8, value: anytype) Input { var out = input; @field(out, field) = value; return out; } test "each degrading condition degrades on its own and says so in the body" { try testing.expectEqualStrings(status_ok, rollup(healthy).status); { const body = rollup(with(healthy, "snapshot_available", false)); try testing.expectEqualStrings(status_degraded, body.status); try testing.expectEqualStrings(protection_unavailable, body.protection.state); } { const body = rollup(with(healthy, "upstreams_available", @as(u32, 0))); try testing.expectEqualStrings(status_degraded, body.status); try testing.expectEqualStrings(upstreams_unavailable, body.upstreams.state); } { const body = rollup(with(healthy, "gate_episode", .losing)); try testing.expectEqualStrings(status_degraded, body.status); try testing.expectEqualStrings(query_history_losing, body.query_history.state); } { const body = rollup(with(healthy, "writer_failed", true)); try testing.expectEqualStrings(status_degraded, body.status); try testing.expectEqualStrings(query_history_failed, body.query_history.state); } { const body = rollup(with(healthy, "diagnostics_present", false)); try testing.expectEqualStrings(status_degraded, body.status); try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state); } { const body = rollup(with(healthy, "diagnostics_write_failed", true)); try testing.expectEqualStrings(status_degraded, body.status); try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state); } for ([_]struct { disk_monitor.State, []const u8 }{ .{ .warn, disk_low }, .{ .critical, disk_critical }, }) |case| { const body = rollup(with(healthy, "disk_state", case[0])); try testing.expectEqualStrings(status_degraded, body.status); try testing.expectEqualStrings(case[1], body.disk.state); } } test "conditions that are not faults leave the status ok" { // Some upstreams down while one still answers. const partial = rollup(.{ .upstreams_available = 1, .upstreams_total = 3 }); try testing.expectEqualStrings(status_ok, partial.status); try testing.expectEqualStrings(upstreams_ok, partial.upstreams.state); // A gate holding writes that has not cost a row yet. const holding = rollup(with(healthy, "gate_episode", .gated)); try testing.expectEqualStrings(status_ok, holding.status); try testing.expectEqualStrings(query_history_recording, holding.query_history.state); // Drops that already happened. The counter is cumulative and the rollup is // stateless, so feeding it in would latch the box to degraded forever. var dropped = with(healthy, "queries_dropped", @as(u64, 9)); dropped.last_drop_s = 1_700_000_000; const body = rollup(dropped); try testing.expectEqualStrings(status_ok, body.status); try testing.expectEqualStrings(query_history_recording, body.query_history.state); try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total); try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s); // Open diagnostics episodes are what the box is doing, not a fault of the // log that recorded them. var open = healthy; open.diagnostics_active_warnings = 3; open.diagnostics_active_errors = 1; const with_episodes = rollup(open); try testing.expectEqualStrings(status_ok, with_episodes.status); try testing.expectEqualStrings(diagnostics_recording, with_episodes.diagnostics.state); try testing.expectEqual(@as(u32, 3), with_episodes.diagnostics.active_warnings); try testing.expectEqual(@as(u32, 1), with_episodes.diagnostics.active_errors); } test "a pause is surfaced, never alarmed, and an expired one is over" { var indefinite = healthy; indefinite.pause_until = -1; indefinite.now_s = 1_000; const forever = rollup(indefinite); try testing.expectEqualStrings(status_ok, forever.status); try testing.expectEqualStrings(protection_paused, forever.protection.state); try testing.expectEqual(@as(?i64, null), forever.protection.until); var timed = healthy; timed.pause_until = 1_060; timed.now_s = 1_000; const live = rollup(timed); try testing.expectEqualStrings(status_ok, live.status); try testing.expectEqualStrings(protection_paused, live.protection.state); try testing.expectEqual(@as(?i64, 1_060), live.protection.until); // The stored second is when filtering is back on, so at it the pause is over. timed.now_s = 1_060; const expired = rollup(timed); try testing.expectEqualStrings(protection_active, expired.protection.state); try testing.expectEqual(@as(?i64, null), expired.protection.until); } test "protection unavailable wins over a live pause" { var both = healthy; both.snapshot_available = false; both.pause_until = -1; const body = rollup(both); try testing.expectEqualStrings(protection_unavailable, body.protection.state); try testing.expectEqual(@as(?i64, null), body.protection.until); try testing.expectEqualStrings(status_degraded, body.status); } test "a failed writer outranks a losing gate" { var both = healthy; both.writer_failed = true; both.gate_episode = .losing; try testing.expectEqualStrings(query_history_failed, rollup(both).query_history.state); } test "two faults at once still report one status" { var both = with(healthy, "disk_state", disk_monitor.State.critical); both.writer_failed = true; try testing.expectEqualStrings(status_degraded, rollup(both).status); } test "the body reports every input verbatim" { const body = rollup(.{ .disk_state = .warn, .disk_free_bytes = 100, .upstreams_available = 2, .upstreams_total = 4, .queries_dropped = 9, .last_drop_s = 1_700_000_000, .snapshot_available = true, }); try testing.expectEqualStrings(status_degraded, body.status); try testing.expectEqualStrings(disk_low, body.disk.state); try testing.expectEqual(@as(u64, 100), body.disk.free_bytes); try testing.expectEqual(@as(u32, 2), body.upstreams.available); try testing.expectEqual(@as(u32, 4), body.upstreams.total); try testing.expectEqualStrings(upstreams_ok, body.upstreams.state); try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total); try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s); try testing.expectEqualStrings(protection_active, body.protection.state); } test "an unstamped drop time serializes as null, not as zero" { var buffer: [1024]u8 = undefined; var writer: std.Io.Writer = .fixed(&buffer); try std.json.Stringify.value(rollup(.{}), .{}, &writer); try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"last_drop_s\":null")); try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"until\":null")); } test "collect reports an absent store as unavailable rather than as recording" { var threaded: std.Io.Threaded = .init(testing.allocator, .{}); defer threaded.deinit(); const io = threaded.io(); // `diagnostics_present` defaults to true like every other benign default, // so an assignment `collect` forgot would read as a healthy log here. var state: server.WebState = .{ .gpa = testing.allocator }; const absent = collect(&state, io); try testing.expect(!absent.diagnostics_present); try testing.expectEqualStrings(diagnostics_unavailable, rollup(absent).diagnostics.state); // And a bare state has no snapshot either, which is the honest reading of a // process that has published nothing. try testing.expect(!absent.snapshot_available); try testing.expectEqualStrings(protection_unavailable, rollup(absent).protection.state); var database = try db.Db.open(":memory:", .{ .mode = .memory }); defer database.close(); try db.applyPragmas(&database, .{}); _ = try migrations.migrate(&database); var store = try events_mod.Store.init(io, &database, 1000); store.report(io, 1000, .disk_space, "data", "data", .warning, "low"); store.report(io, 1000, .listener_start, "doh", "doh", .@"error", "AddressInUse"); state.events = &store; const present = collect(&state, io); try testing.expect(present.diagnostics_present); try testing.expect(!present.diagnostics_write_failed); try testing.expectEqual(@as(u32, 1), present.diagnostics_active_warnings); try testing.expectEqual(@as(u32, 1), present.diagnostics_active_errors); try testing.expectEqualStrings(diagnostics_recording, rollup(present).diagnostics.state); } test "collect reads the logger's counters and the pause flag" { var threaded: std.Io.Threaded = .init(testing.allocator, .{}); defer threaded.deinit(); const io = threaded.io(); var queue_buf: [2]logger_mod.Entry = undefined; var query_logger: logger_mod.Logger = .init(.{}, &queue_buf); query_logger.queries_dropped.store(4, .monotonic); query_logger.last_drop_s.store(1_700_000_000, .monotonic); query_logger.writer_failed.store(true, .monotonic); var log_owner: logger_controller.Borrowed = .{}; var paused: pause_mod.Pause = .{}; paused.pauseFor(0, null); var state: server.WebState = .{ .gpa = testing.allocator, .logger = log_owner.over(&query_logger), .pause = &paused, }; const input = collect(&state, io); try testing.expectEqual(@as(u64, 4), input.queries_dropped); try testing.expectEqual(@as(?i64, 1_700_000_000), input.last_drop_s); try testing.expect(input.writer_failed); try testing.expectEqual(@as(i64, -1), input.pause_until); try testing.expectEqual(@as(u32, 0), input.upstreams_total); try testing.expectEqualStrings(status_degraded, rollup(input).status); // No snapshot manager either, so protection outranks the pause here too. try testing.expectEqualStrings(protection_unavailable, rollup(input).protection.state); }