nxdns/src/web/handlers/health.zig

//! `GET /api/health` — the rollup a monitor scrapes (ruling 22).
//!
//! Always 200. "degraded" is a fact about the box, not a failure of the
//! request, and answering 503 would make an uptime check flap on a full disk
//! while nxdns is still resolving perfectly well.
//!
//! Unauthenticated and rate-limit exempt, like `/metrics`.
//!
//! **Nothing may degrade the rollup without appearing in the response.** The
//! body is five condition objects and a status computed from exactly their
//! states, so an operator reading it can always name the condition that
//! degraded the box. Anything a counter already records — gated refreshes,
//! disk sample failures, the snapshot generation — stays in `/metrics` and in
//! Diagnostics rather than becoming a sixth hidden input here.
//!
//! `rollup` is pure so the whole degraded matrix is testable without a running
//! server; `handle` only gathers the inputs.

const std = @import("std");

const disk_monitor = @import("../../storage/disk_monitor.zig");
const http_util = @import("../http_util.zig");
const logger_controller = @import("../../storage/logger_controller.zig");
const logger_mod = @import("../../storage/logger.zig");
const metrics = @import("../metrics.zig");
const pause_mod = @import("../../server/pause.zig");
const pool_mod = @import("../../upstream/pool.zig");
const server = @import("../server.zig");

/// Is filtering in force. `unavailable` is not an operator's doing: it is the
/// state in which the query path has no filter snapshot to evaluate against,
/// which `handler.zig` records as the `snapshot_unavailable` provenance.
pub const Protection = struct {
    state: []const u8,
    /// The second filtering resumes at. Null for both an indefinite pause and
    /// every non-paused state, which the state field already tells apart.
    until: ?i64,
};

pub const Upstreams = struct {
    state: []const u8,
    available: u32,
    /// Enabled upstreams: the pool is built from those alone, so this is what
    /// `available` is out of.
    total: u32,
};

/// Whether Activity can be trusted. `dropped_total` is cumulative and does not
/// decide the state — a drop that happened an hour ago is not a fault now.
pub const QueryHistory = struct {
    state: []const u8,
    dropped_total: u64,
    /// The newest drop, or null while nothing has been dropped. Stamped by a
    /// separate atomic from the count, so a reader can momentarily see a
    /// non-zero `dropped_total` beside a null here.
    last_drop_s: ?i64,
};

/// The diagnostics store's own state, not a summary of what it holds: `state`
/// answers "is the operational log recording", and the two counts answer "what
/// is open right now".
pub const Diagnostics = struct {
    state: []const u8,
    active_warnings: u32,
    active_errors: u32,
};

pub const Disk = struct {
    state: []const u8,
    free_bytes: u64,
};

pub const Body = struct {
    status: []const u8,
    protection: Protection,
    upstreams: Upstreams,
    query_history: QueryHistory,
    diagnostics: Diagnostics,
    disk: Disk,
};

/// What the rollup is computed from. Every field has a defined value even when
/// its collaborator is missing, and the defaults are the ones a half-wired
/// server should report: no disk reading, no upstreams, nothing published.
pub const Input = struct {
    disk_state: disk_monitor.State = .ok,
    disk_free_bytes: u64 = 0,
    upstreams_available: u32 = 0,
    upstreams_total: u32 = 0,
    queries_dropped: u64 = 0,
    last_drop_s: ?i64 = null,
    writer_failed: bool = false,
    /// The logger's disk-gating episode. `losing` is the only state that
    /// degrades: it means the gate is holding writes back *and* has already
    /// cost rows in the episode that is open now.
    gate_episode: logger_mod.GateEpisode = .open,
    /// A filter snapshot exists for the query path to evaluate against. The
    /// benign default matches every other field here, and `collect` assigns it
    /// explicitly for the same reason `diagnostics_present` is assigned there.
    snapshot_available: bool = true,
    /// `pause.Pause.until` verbatim: 0 not paused, -1 indefinite, otherwise the
    /// second filtering resumes at. Raw rather than a decided boolean so the
    /// expiry rule stays `pause.Pause`'s and is exercised by these tests.
    pause_until: i64 = 0,
    /// The clock the pause is compared against.
    now_s: i64 = 0,
    /// The diagnostics store exists. The benign default matches every other
    /// field here — a half-wired `Input` reports a box with nothing wrong — but
    /// `collect` must assign it explicitly, because in a serving process an
    /// absent store means `Store.init` failed.
    diagnostics_present: bool = true,
    /// The last diagnostics write failed. Current state, cleared by the next
    /// write that succeeds.
    diagnostics_write_failed: bool = false,
    diagnostics_active_warnings: u32 = 0,
    diagnostics_active_errors: u32 = 0,
};

pub const status_ok = "ok";
pub const status_degraded = "degraded";

pub const protection_active = "active";
pub const protection_paused = "paused";
pub const protection_unavailable = "unavailable";

pub const upstreams_ok = "ok";
pub const upstreams_unavailable = "unavailable";

pub const query_history_recording = "recording";
pub const query_history_losing = "losing";
pub const query_history_failed = "failed";

pub const diagnostics_recording = "recording";
pub const diagnostics_unavailable = "unavailable";

pub const disk_ok = "ok";
pub const disk_low = "low";
pub const disk_critical = "critical";

/// Precedence `unavailable` → `paused` → `active`. With no snapshot the pause
/// flag says nothing an operator can act on: filtering is off either way, and
/// resuming would not turn it back on.
pub fn protection(input: Input) Protection {
    if (!input.snapshot_available) return .{ .state = protection_unavailable, .until = null };

    const state: pause_mod.Pause = .{ .until = .init(input.pause_until) };
    if (!state.isPaused(input.now_s)) return .{ .state = protection_active, .until = null };
    return .{
        .state = protection_paused,
        .until = if (input.pause_until > 0) input.pause_until else null,
    };
}

fn queryHistoryState(input: Input) []const u8 {
    if (input.writer_failed) return query_history_failed;
    if (input.gate_episode == .losing) return query_history_losing;
    return query_history_recording;
}

/// The operational log is not recording — either the store never opened or its
/// writes are failing. Both mean the same thing to an operator: the record of
/// what went wrong is not being kept.
fn diagnosticsUnavailable(input: Input) bool {
    return !input.diagnostics_present or input.diagnostics_write_failed;
}

/// `warn` reads as a log level rather than as a quantity of disk. The monitor
/// keeps its own name; the wire says what an operator sees on the page.
fn diskState(state: disk_monitor.State) []const u8 {
    return switch (state) {
        .ok => disk_ok,
        .warn => disk_low,
        .critical => disk_critical,
    };
}

/// The degrading set, named rather than gestured at: protection `unavailable`,
/// upstreams `unavailable`, query history `losing` or `failed`, diagnostics
/// `unavailable`, disk `low` or `critical`.
///
/// A pause is deliberately not in it. It is an operator's own choice, and a
/// monitor that pages on it would be paging on a button the operator pressed.
pub fn degraded(input: Input) bool {
    const history = queryHistoryState(input);
    return !input.snapshot_available or
        input.upstreams_available == 0 or
        !std.mem.eql(u8, history, query_history_recording) or
        diagnosticsUnavailable(input) or
        input.disk_state != .ok;
}

pub fn rollup(input: Input) Body {
    return .{
        .status = if (degraded(input)) status_degraded else status_ok,
        .protection = protection(input),
        .upstreams = .{
            .state = if (input.upstreams_available == 0) upstreams_unavailable else upstreams_ok,
            .available = input.upstreams_available,
            .total = input.upstreams_total,
        },
        .query_history = .{
            .state = queryHistoryState(input),
            .dropped_total = input.queries_dropped,
            .last_drop_s = input.last_drop_s,
        },
        .diagnostics = .{
            .state = if (diagnosticsUnavailable(input)) diagnostics_unavailable else diagnostics_recording,
            .active_warnings = input.diagnostics_active_warnings,
            .active_errors = input.diagnostics_active_errors,
        },
        .disk = .{ .state = diskState(input.disk_state), .free_bytes = input.disk_free_bytes },
    };
}

pub fn handle(
    state: *server.WebState,
    io: std.Io,
    request: *http_util.Request,
) http_util.HandlerError!void {
    return http_util.respondJson(request, .ok, rollup(collect(state, io)), &.{});
}

pub fn collect(state: *server.WebState, io: std.Io) Input {
    var input: Input = .{};

    input.now_s = std.Io.Clock.real.now(io).toSeconds();

    if (state.monitor) |monitor| {
        input.disk_state = monitor.state();
        input.disk_free_bytes = monitor.gauges().free_bytes;
    }

    if (state.upstreams) |owner| {
        const generation = owner.acquire(io);
        defer owner.release(io, generation);
        if (generation.pool) |pool| {
            var raw: [metrics.max_upstreams]pool_mod.Snapshot = undefined;
            const count = metrics.poolSnapshot(pool, io, &raw);
            input.upstreams_total = @intCast(count);
            for (raw[0..count]) |entry| {
                if (entry.available) input.upstreams_available += 1;
            }
        }
    }

    if (state.logger) |controller| {
        const reading = controller.sample(io);
        input.queries_dropped = reading.queries_dropped;
        input.last_drop_s = reading.last_drop_s;
        input.writer_failed = reading.writer_failed;
        input.gate_episode = reading.gate_episode;
    }

    if (state.pause) |paused| input.pause_until = paused.until.load(.monotonic);

    // Assigned before the `if`, not inside it: the field's benign default is
    // `true`, so the natural `if (state.events) |store|` shape would report an
    // absent store as recording — the one case that must degrade.
    input.diagnostics_present = state.events != null;
    if (state.events) |store| {
        input.diagnostics_write_failed = store.writeFailed();
        const counts = store.activeCounts(io);
        input.diagnostics_active_warnings = counts.warnings;
        input.diagnostics_active_errors = counts.errors;
    }

    // The generation itself is not in the body — the UI-facing fact is whether
    // protection has a snapshot at all, and `/metrics` keeps the number.
    input.snapshot_available = false;
    if (state.manager) |manager| {
        if (manager.acquire(io)) |acquired| {
            defer acquired.release(io);
            input.snapshot_available = true;
        }
    }

    return input;
}

// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------

const db = @import("../../storage/db.zig");
const events_mod = @import("../../storage/events.zig");
const migrations = @import("../../storage/migrations.zig");
const testing = std.testing;

/// A box with nothing wrong with it: one upstream up, disk ok, writer alive,
/// a snapshot published and filtering unpaused.
const healthy: Input = .{
    .disk_state = .ok,
    .upstreams_available = 1,
    .upstreams_total = 1,
    .writer_failed = false,
    .snapshot_available = true,
};

fn with(input: Input, comptime field: []const u8, value: anytype) Input {
    var out = input;
    @field(out, field) = value;
    return out;
}

test "each degrading condition degrades on its own and says so in the body" {
    try testing.expectEqualStrings(status_ok, rollup(healthy).status);

    {
        const body = rollup(with(healthy, "snapshot_available", false));
        try testing.expectEqualStrings(status_degraded, body.status);
        try testing.expectEqualStrings(protection_unavailable, body.protection.state);
    }
    {
        const body = rollup(with(healthy, "upstreams_available", @as(u32, 0)));
        try testing.expectEqualStrings(status_degraded, body.status);
        try testing.expectEqualStrings(upstreams_unavailable, body.upstreams.state);
    }
    {
        const body = rollup(with(healthy, "gate_episode", .losing));
        try testing.expectEqualStrings(status_degraded, body.status);
        try testing.expectEqualStrings(query_history_losing, body.query_history.state);
    }
    {
        const body = rollup(with(healthy, "writer_failed", true));
        try testing.expectEqualStrings(status_degraded, body.status);
        try testing.expectEqualStrings(query_history_failed, body.query_history.state);
    }
    {
        const body = rollup(with(healthy, "diagnostics_present", false));
        try testing.expectEqualStrings(status_degraded, body.status);
        try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state);
    }
    {
        const body = rollup(with(healthy, "diagnostics_write_failed", true));
        try testing.expectEqualStrings(status_degraded, body.status);
        try testing.expectEqualStrings(diagnostics_unavailable, body.diagnostics.state);
    }
    for ([_]struct { disk_monitor.State, []const u8 }{
        .{ .warn, disk_low },
        .{ .critical, disk_critical },
    }) |case| {
        const body = rollup(with(healthy, "disk_state", case[0]));
        try testing.expectEqualStrings(status_degraded, body.status);
        try testing.expectEqualStrings(case[1], body.disk.state);
    }
}

test "conditions that are not faults leave the status ok" {
    // Some upstreams down while one still answers.
    const partial = rollup(.{ .upstreams_available = 1, .upstreams_total = 3 });
    try testing.expectEqualStrings(status_ok, partial.status);
    try testing.expectEqualStrings(upstreams_ok, partial.upstreams.state);

    // A gate holding writes that has not cost a row yet.
    const holding = rollup(with(healthy, "gate_episode", .gated));
    try testing.expectEqualStrings(status_ok, holding.status);
    try testing.expectEqualStrings(query_history_recording, holding.query_history.state);

    // Drops that already happened. The counter is cumulative and the rollup is
    // stateless, so feeding it in would latch the box to degraded forever.
    var dropped = with(healthy, "queries_dropped", @as(u64, 9));
    dropped.last_drop_s = 1_700_000_000;
    const body = rollup(dropped);
    try testing.expectEqualStrings(status_ok, body.status);
    try testing.expectEqualStrings(query_history_recording, body.query_history.state);
    try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total);
    try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s);

    // Open diagnostics episodes are what the box is doing, not a fault of the
    // log that recorded them.
    var open = healthy;
    open.diagnostics_active_warnings = 3;
    open.diagnostics_active_errors = 1;
    const with_episodes = rollup(open);
    try testing.expectEqualStrings(status_ok, with_episodes.status);
    try testing.expectEqualStrings(diagnostics_recording, with_episodes.diagnostics.state);
    try testing.expectEqual(@as(u32, 3), with_episodes.diagnostics.active_warnings);
    try testing.expectEqual(@as(u32, 1), with_episodes.diagnostics.active_errors);
}

test "a pause is surfaced, never alarmed, and an expired one is over" {
    var indefinite = healthy;
    indefinite.pause_until = -1;
    indefinite.now_s = 1_000;
    const forever = rollup(indefinite);
    try testing.expectEqualStrings(status_ok, forever.status);
    try testing.expectEqualStrings(protection_paused, forever.protection.state);
    try testing.expectEqual(@as(?i64, null), forever.protection.until);

    var timed = healthy;
    timed.pause_until = 1_060;
    timed.now_s = 1_000;
    const live = rollup(timed);
    try testing.expectEqualStrings(status_ok, live.status);
    try testing.expectEqualStrings(protection_paused, live.protection.state);
    try testing.expectEqual(@as(?i64, 1_060), live.protection.until);

    // The stored second is when filtering is back on, so at it the pause is over.
    timed.now_s = 1_060;
    const expired = rollup(timed);
    try testing.expectEqualStrings(protection_active, expired.protection.state);
    try testing.expectEqual(@as(?i64, null), expired.protection.until);
}

test "protection unavailable wins over a live pause" {
    var both = healthy;
    both.snapshot_available = false;
    both.pause_until = -1;
    const body = rollup(both);
    try testing.expectEqualStrings(protection_unavailable, body.protection.state);
    try testing.expectEqual(@as(?i64, null), body.protection.until);
    try testing.expectEqualStrings(status_degraded, body.status);
}

test "a failed writer outranks a losing gate" {
    var both = healthy;
    both.writer_failed = true;
    both.gate_episode = .losing;
    try testing.expectEqualStrings(query_history_failed, rollup(both).query_history.state);
}

test "two faults at once still report one status" {
    var both = with(healthy, "disk_state", disk_monitor.State.critical);
    both.writer_failed = true;
    try testing.expectEqualStrings(status_degraded, rollup(both).status);
}

test "the body reports every input verbatim" {
    const body = rollup(.{
        .disk_state = .warn,
        .disk_free_bytes = 100,
        .upstreams_available = 2,
        .upstreams_total = 4,
        .queries_dropped = 9,
        .last_drop_s = 1_700_000_000,
        .snapshot_available = true,
    });

    try testing.expectEqualStrings(status_degraded, body.status);
    try testing.expectEqualStrings(disk_low, body.disk.state);
    try testing.expectEqual(@as(u64, 100), body.disk.free_bytes);
    try testing.expectEqual(@as(u32, 2), body.upstreams.available);
    try testing.expectEqual(@as(u32, 4), body.upstreams.total);
    try testing.expectEqualStrings(upstreams_ok, body.upstreams.state);
    try testing.expectEqual(@as(u64, 9), body.query_history.dropped_total);
    try testing.expectEqual(@as(?i64, 1_700_000_000), body.query_history.last_drop_s);
    try testing.expectEqualStrings(protection_active, body.protection.state);
}

test "an unstamped drop time serializes as null, not as zero" {
    var buffer: [1024]u8 = undefined;
    var writer: std.Io.Writer = .fixed(&buffer);
    try std.json.Stringify.value(rollup(.{}), .{}, &writer);
    try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"last_drop_s\":null"));
    try testing.expect(std.mem.containsAtLeast(u8, writer.buffered(), 1, "\"until\":null"));
}

test "collect reports an absent store as unavailable rather than as recording" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    // `diagnostics_present` defaults to true like every other benign default,
    // so an assignment `collect` forgot would read as a healthy log here.
    var state: server.WebState = .{ .gpa = testing.allocator };
    const absent = collect(&state, io);
    try testing.expect(!absent.diagnostics_present);
    try testing.expectEqualStrings(diagnostics_unavailable, rollup(absent).diagnostics.state);
    // And a bare state has no snapshot either, which is the honest reading of a
    // process that has published nothing.
    try testing.expect(!absent.snapshot_available);
    try testing.expectEqualStrings(protection_unavailable, rollup(absent).protection.state);

    var database = try db.Db.open(":memory:", .{ .mode = .memory });
    defer database.close();
    try db.applyPragmas(&database, .{});
    _ = try migrations.migrate(&database);
    var store = try events_mod.Store.init(io, &database, 1000);
    store.report(io, 1000, .disk_space, "data", "data", .warning, "low");
    store.report(io, 1000, .listener_start, "doh", "doh", .@"error", "AddressInUse");

    state.events = &store;
    const present = collect(&state, io);
    try testing.expect(present.diagnostics_present);
    try testing.expect(!present.diagnostics_write_failed);
    try testing.expectEqual(@as(u32, 1), present.diagnostics_active_warnings);
    try testing.expectEqual(@as(u32, 1), present.diagnostics_active_errors);
    try testing.expectEqualStrings(diagnostics_recording, rollup(present).diagnostics.state);
}

test "collect reads the logger's counters and the pause flag" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var queue_buf: [2]logger_mod.Entry = undefined;
    var query_logger: logger_mod.Logger = .init(.{}, &queue_buf);
    query_logger.queries_dropped.store(4, .monotonic);
    query_logger.last_drop_s.store(1_700_000_000, .monotonic);
    query_logger.writer_failed.store(true, .monotonic);
    var log_owner: logger_controller.Borrowed = .{};

    var paused: pause_mod.Pause = .{};
    paused.pauseFor(0, null);

    var state: server.WebState = .{
        .gpa = testing.allocator,
        .logger = log_owner.over(&query_logger),
        .pause = &paused,
    };
    const input = collect(&state, io);

    try testing.expectEqual(@as(u64, 4), input.queries_dropped);
    try testing.expectEqual(@as(?i64, 1_700_000_000), input.last_drop_s);
    try testing.expect(input.writer_failed);
    try testing.expectEqual(@as(i64, -1), input.pause_until);
    try testing.expectEqual(@as(u32, 0), input.upstreams_total);
    try testing.expectEqualStrings(status_degraded, rollup(input).status);
    // No snapshot manager either, so protection outranks the pause here too.
    try testing.expectEqualStrings(protection_unavailable, rollup(input).protection.state);
}