//! `GET /api/upstream/health?period=` — the pool's upstreams over the window
//! the dashboard's period picker selected (milestone-26 ruling 6).
//!
//! Two kinds of fact, kept apart on the wire because they answer different
//! questions. `enabled`/`available` are live routing state, read from the pool
//! under its mutex: what the resolver would do with this upstream right now.
//! Everything under `period` is history, aggregated out of `upstream_minute`
//! over `[since, until)` — the same window `/api/stats` reports, so a page
//! cannot show a rate that disagrees with the chart beside it.
//!
//! Nothing here reads a process-lifetime counter. The lifetime totals, the
//! last-32-exchange window and the consecutive-failure count still live in
//! `health.State` for routing and in `/metrics`; they are not this response's
//! business, because a number that starts at process start cannot be scoped to
//! a period and a dashboard that shows one beside a picker lies about it.
//!
//! The aggregation runs on the web task's own query-log connection (m7 ruling
//! 21) and this file owns no SQL: `upstream_history_repo` does.

const std = @import("std");
const Allocator = std.mem.Allocator;

const db = @import("../../storage/db.zig");
const history_mod = @import("../../upstream/history.zig");
const http_util = @import("../http_util.zig");
const metrics = @import("../metrics.zig");
const pool_mod = @import("../../upstream/pool.zig");
const server = @import("../server.zig");
const stats = @import("stats.zig");
const upstream_history_repo = @import("../../storage/repositories/upstream_history_repo.zig");

const log = std.log.scoped(.web_upstream_health);

/// One upstream's outcomes inside the selected window.
pub const PeriodStats = struct {
    attempts: u64,
    successes: u64,
    failures: u64,
    /// Null when `attempts == 0`. No observations is not perfect reliability,
    /// and a `100.0%` from an idle upstream is the exact misreading this
    /// milestone exists to remove.
    success_rate: ?f32,
    /// The newest failure inside the window, on the wall clock the minute rows
    /// are stamped with. Null when the window holds no failure, even if the
    /// upstream failed before it.
    last_failure_at: ?i64,
    /// The error name belonging to `last_failure_at`; null exactly when it is.
    last_failure_error: ?[]const u8,
};

pub const Upstream = struct {
    url: []const u8,
    /// Live: configuration, not history.
    enabled: bool,
    /// Live: false while the upstream is backing off.
    available: bool,
    period: PeriodStats,
};

pub const Body = struct {
    period: []const u8,
    since: i64,
    until: i64,
    available: u32,
    total: u32,
    /// See `isComplete`.
    complete: bool,
    upstreams: []const Upstream,
};

/// The same text `/api/stats` sends (`stats.zig`'s `badPeriod`). One period
/// grammar serves the whole dashboard, so the two routes must not disagree
/// about what a typo means.
pub const bad_period_message = "period must be one of 1h, 24h, 7d, 30d";

pub fn handle(
    state: *server.WebState,
    io: std.Io,
    request: *http_util.Request,
) http_util.HandlerError!void {
    const period = stats.periodParam(request.query) catch
        return http_util.respondError(request, .bad_request, bad_period_message);
    const pool = state.pool orelse
        return http_util.respondError(request, .service_unavailable, "no upstream pool");
    const database = state.querylog_db orelse
        return http_util.respondError(request, .service_unavailable, "query log unavailable");

    const now = std.Io.Clock.real.now(io).toSeconds();
    const body = collect(request.arena, io, pool, state.history, database, period, now) catch |err| {
        if (err == error.OutOfMemory) return error.OutOfMemory;
        // A failed aggregate is a fault in the box, not a property of the
        // request (ruling 8, PLAN §19).
        log.warn("upstream health window failed: {s}", .{@errorName(err)});
        return http_util.respondError(request, .internal_server_error, "internal error");
    };
    return http_util.respondJson(request, .ok, body, &.{});
}

/// `db.Error` already carries `OutOfMemory`, so the arena's failures and
/// SQLite's share one set.
pub const Error = Allocator.Error || db.Error;

pub fn collect(
    arena: Allocator,
    io: std.Io,
    pool: *pool_mod.Pool,
    history: ?*history_mod.Accumulator,
    database: *db.Db,
    period: stats.Period,
    now_unix: i64,
) Error!Body {
    const span = stats.window(period, now_unix);

    var raw: [metrics.max_upstreams]pool_mod.Snapshot = undefined;
    const count = metrics.poolSnapshot(pool, io, &raw);

    const out = try arena.alloc(Upstream, count);
    var available: u32 = 0;
    for (raw[0..count], out) |entry, *slot| {
        if (entry.available) available += 1;

        // Rows come from the current pool only: an upstream deleted from the
        // configuration keeps its history in storage until retention takes it,
        // and nothing joins it back into this response.
        const window_stats = try upstream_history_repo.windowStats(
            database,
            entry.url,
            span.since,
            span.until,
        );

        slot.* = .{
            .url = try arena.dupe(u8, entry.url),
            .enabled = entry.enabled,
            .available = entry.available,
            .period = .{
                .attempts = window_stats.attempts,
                .successes = window_stats.successes,
                .failures = window_stats.failures,
                .success_rate = successRate(window_stats),
                .last_failure_at = window_stats.last_failure_ts,
                // `WindowStats` carries its error name by value, in storage this
                // loop is done with as soon as the iteration ends. The copy into
                // the arena is what keeps the response from pointing at bytes
                // the next upstream's row overwrites.
                .last_failure_error = if (window_stats.last_failure_ts == null)
                    null
                else
                    try arena.dupe(u8, window_stats.lastFailureError()),
            },
        };
    }

    return .{
        .period = period.label(),
        .since = span.since,
        .until = span.until,
        .available = available,
        .total = @intCast(count),
        .complete = isComplete(history, io, span.since),
        .upstreams = out,
    };
}

fn successRate(window_stats: upstream_history_repo.WindowStats) ?f32 {
    if (window_stats.attempts == 0) return null;
    const successes: f32 = @floatFromInt(window_stats.successes);
    const attempts: f32 = @floatFromInt(window_stats.attempts);
    return successes / attempts;
}

/// Per-window and stateless (ruling 6): false iff capacity has cost this
/// process a minute that falls inside the window. A window that starts after
/// the newest such minute is complete again, so one historical overflow does
/// not mark every later response.
///
/// It says nothing about the newest outcomes, which may not have flushed yet,
/// and nothing about an unclean shutdown, which is not detectable here — the
/// openapi description spells both out.
fn isComplete(history: ?*history_mod.Accumulator, io: std.Io, since: i64) bool {
    const accumulator = history orelse return true;
    const dropped = accumulator.snapshotStats(io).last_drop_minute orelse return true;
    return dropped < since;
}

// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------

const querylog_schema = @import("../../storage/querylog_schema.zig");
const transport = @import("../../upstream/transport.zig");
const testing = std.testing;

/// The client is never called: every test here reads health, not answers.
fn testEntry(url: []const u8, enabled: bool) pool_mod.Entry {
    return .{
        .endpoint = transport.Endpoint.parse(url) catch unreachable,
        .client = .{ .ptr = undefined, .exchangeFn = undefined },
        .priority = 1,
        .enabled = enabled,
        .health = .init,
    };
}

fn testPool(entries: []pool_mod.Entry) pool_mod.Pool {
    return .init(entries, .{}, .{
        .attempt = .{ .raw = .fromMilliseconds(50), .clock = .awake },
        .total = .{ .raw = .fromMilliseconds(100), .clock = .awake },
    }, 1);
}

fn openLog() !db.Db {
    var database = try db.Db.open(":memory:", .{ .mode = .memory });
    errdefer database.close();
    try db.applyPragmas(&database, .{});
    try database.exec(querylog_schema.ddl);
    return database;
}

const url_a = "https://a.test/dns-query";
const url_b = "https://b.test/dns-query";

/// A minute-aligned instant, so a window derived from it lands on round
/// numbers the assertions below can name.
const aligned_now: i64 = 1_699_999_980;

test "the window sums the minutes inside it and nothing outside" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var database = try openLog();
    defer database.close();

    const span = stats.window(.@"1h", aligned_now);
    try upstream_history_repo.flush(&database, &.{
        // One minute before the window.
        .{ .url = url_a, .minute_ts = span.since - 60, .successes = 100, .failures = 100, .last_failure_ts = span.since - 30, .last_error = "Outside" },
        .{ .url = url_a, .minute_ts = span.since, .successes = 3, .failures = 1, .last_failure_ts = span.since + 10, .last_error = "Timeout" },
        .{ .url = url_a, .minute_ts = span.until - 60, .successes = 5, .failures = 0, .last_failure_ts = null, .last_error = "" },
        // The window's exclusive end.
        .{ .url = url_a, .minute_ts = span.until, .successes = 200, .failures = 200, .last_failure_ts = span.until + 5, .last_error = "After" },
    });

    var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
    var pool = testPool(&entries);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
    try testing.expectEqualStrings("1h", body.period);
    try testing.expectEqual(span.since, body.since);
    try testing.expectEqual(span.until, body.until);
    try testing.expectEqual(@as(u32, 1), body.total);
    try testing.expectEqual(@as(u32, 1), body.available);

    const period = body.upstreams[0].period;
    try testing.expectEqual(@as(u64, 8), period.successes);
    try testing.expectEqual(@as(u64, 1), period.failures);
    try testing.expectEqual(@as(u64, 9), period.attempts);
    try testing.expectEqual(@as(?i64, span.since + 10), period.last_failure_at);
    try testing.expectEqualStrings("Timeout", period.last_failure_error.?);
}

test "an upstream with no attempts in the window reports null, never a perfect rate" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var database = try openLog();
    defer database.close();

    const span = stats.window(.@"1h", aligned_now);
    // Only `b` has history, and only outside the window.
    try upstream_history_repo.flush(&database, &.{
        .{ .url = url_b, .minute_ts = span.since - 600, .successes = 4, .failures = 0, .last_failure_ts = null, .last_error = "" },
    });

    var entries = [_]pool_mod.Entry{ testEntry(url_a, true), testEntry(url_b, true) };
    var pool = testPool(&entries);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
    for (body.upstreams) |upstream| {
        try testing.expectEqual(@as(u64, 0), upstream.period.attempts);
        try testing.expectEqual(@as(u64, 0), upstream.period.successes);
        try testing.expectEqual(@as(u64, 0), upstream.period.failures);
        try testing.expectEqual(@as(?f32, null), upstream.period.success_rate);
        try testing.expectEqual(@as(?i64, null), upstream.period.last_failure_at);
        try testing.expectEqual(@as(?[]const u8, null), upstream.period.last_failure_error);
    }
}

test "the success rate is the window's own, not a lifetime one" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var database = try openLog();
    defer database.close();

    const span = stats.window(.@"1h", aligned_now);
    try upstream_history_repo.flush(&database, &.{
        // A clean past that a lifetime rate would average into the window.
        .{ .url = url_a, .minute_ts = span.since - 600, .successes = 1000, .failures = 0, .last_failure_ts = null, .last_error = "" },
        .{ .url = url_a, .minute_ts = span.since, .successes = 1, .failures = 3, .last_failure_ts = span.since + 1, .last_error = "Timeout" },
    });

    var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
    var pool = testPool(&entries);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
    try testing.expectEqual(@as(?f32, 0.25), body.upstreams[0].period.success_rate);
}

test "the newest failure inside the window wins over an older one outside it" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var database = try openLog();
    defer database.close();

    const span = stats.window(.@"1h", aligned_now);
    try upstream_history_repo.flush(&database, &.{
        .{ .url = url_a, .minute_ts = span.since - 120, .successes = 0, .failures = 1, .last_failure_ts = span.since - 100, .last_error = "Older" },
        .{ .url = url_a, .minute_ts = span.since, .successes = 0, .failures = 1, .last_failure_ts = span.since + 5, .last_error = "Newer" },
        // A later minute with no failure at all must not blank the error.
        .{ .url = url_a, .minute_ts = span.since + 60, .successes = 2, .failures = 0, .last_failure_ts = null, .last_error = "" },
    });

    var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
    var pool = testPool(&entries);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
    try testing.expectEqual(@as(?i64, span.since + 5), body.upstreams[0].period.last_failure_at);
    try testing.expectEqualStrings("Newer", body.upstreams[0].period.last_failure_error.?);
}

test "two upstreams keep their own last-failure errors" {
    // The by-value `WindowStats` buffer is reused per iteration, so a response
    // that borrowed it would show the second upstream's error on the first, or
    // point at stack storage that is gone by the time it is serialized.
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var database = try openLog();
    defer database.close();

    const span = stats.window(.@"1h", aligned_now);
    try upstream_history_repo.flush(&database, &.{
        .{ .url = url_a, .minute_ts = span.since, .successes = 1, .failures = 1, .last_failure_ts = span.since + 1, .last_error = "ConnectFailed" },
        .{ .url = url_b, .minute_ts = span.since, .successes = 0, .failures = 2, .last_failure_ts = span.since + 2, .last_error = "TlsHandshakeFailed" },
    });

    var entries = [_]pool_mod.Entry{ testEntry(url_a, true), testEntry(url_b, true) };
    var pool = testPool(&entries);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const body = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", aligned_now);
    try testing.expectEqual(@as(usize, 2), body.upstreams.len);
    try testing.expectEqualStrings(url_a, body.upstreams[0].url);
    try testing.expectEqualStrings("ConnectFailed", body.upstreams[0].period.last_failure_error.?);
    try testing.expectEqualStrings(url_b, body.upstreams[1].url);
    try testing.expectEqualStrings("TlsHandshakeFailed", body.upstreams[1].period.last_failure_error.?);

    // Serializing after every row is read is what a real response does; the
    // texts must still be the ones their own rows carried.
    var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
    defer allocating.deinit();
    try std.json.Stringify.value(body, .{}, &allocating.writer);
    const text = allocating.written();
    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"last_failure_error\":\"ConnectFailed\""));
    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"last_failure_error\":\"TlsHandshakeFailed\""));
}

test "a disabled upstream is not counted available and still gets its window" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var database = try openLog();
    defer database.close();

    const span = stats.window(.@"24h", aligned_now);
    try upstream_history_repo.flush(&database, &.{
        .{ .url = url_b, .minute_ts = span.since, .successes = 2, .failures = 0, .last_failure_ts = null, .last_error = "" },
    });

    var entries = [_]pool_mod.Entry{ testEntry(url_a, true), testEntry(url_b, false) };
    var pool = testPool(&entries);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const body = try collect(arena.allocator(), io, &pool, null, &database, .@"24h", aligned_now);
    try testing.expectEqual(@as(u32, 2), body.total);
    try testing.expectEqual(@as(u32, 1), body.available);
    try testing.expect(!body.upstreams[1].enabled);
    try testing.expect(!body.upstreams[1].available);
    try testing.expectEqual(@as(u64, 2), body.upstreams[1].period.attempts);
}

/// Fills the accumulator and then overflows it, so `last_drop_minute` is
/// `minute` — the only way to set it, because the accumulator's fields are
/// private to its module and `snapshotStats` is the read surface.
fn accumulatorDroppingAt(
    io: std.Io,
    minute: i64,
    names: *[history_mod.max_pending][8]u8,
) !*history_mod.Accumulator {
    const accumulator = try testing.allocator.create(history_mod.Accumulator);
    accumulator.* = .init;
    for (names, 0..) |*name, i| {
        const url = std.fmt.bufPrint(name, "u{d:0>6}", .{i}) catch unreachable;
        accumulator.recordSuccess(io, url, minute);
    }
    // One more cell than capacity: the oldest minute goes, and every cell above
    // holds `minute`.
    accumulator.recordSuccess(io, "https://overflow.test", minute + 60);
    return accumulator;
}

test "complete is false only while a dropped minute falls inside the window" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var database = try openLog();
    defer database.close();

    var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
    var pool = testPool(&entries);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    // Two `now`s one minute apart, so the same drop is inside the first
    // window and one minute before the second.
    const inside_now = aligned_now;
    const inside = stats.window(.@"1h", inside_now);
    const after = stats.window(.@"1h", inside_now + 60);
    try testing.expectEqual(inside.since + 60, after.since);

    var names: [history_mod.max_pending][8]u8 = undefined;
    const accumulator = try accumulatorDroppingAt(io, inside.since, &names);
    defer testing.allocator.destroy(accumulator);
    try testing.expectEqual(@as(?i64, inside.since), accumulator.snapshotStats(io).last_drop_minute);

    const flagged = try collect(arena.allocator(), io, &pool, accumulator, &database, .@"1h", inside_now);
    try testing.expect(!flagged.complete);

    const recovered = try collect(arena.allocator(), io, &pool, accumulator, &database, .@"1h", inside_now + 60);
    try testing.expect(recovered.complete);

    // No accumulator at all is no known drop, not an incomplete window.
    const unwired = try collect(arena.allocator(), io, &pool, null, &database, .@"1h", inside_now);
    try testing.expect(unwired.complete);
}

test "a drop with no overflow leaves every window complete" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var database = try openLog();
    defer database.close();

    var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
    var pool = testPool(&entries);

    const accumulator = try testing.allocator.create(history_mod.Accumulator);
    defer testing.allocator.destroy(accumulator);
    accumulator.* = .init;
    accumulator.recordFailure(io, url_a, aligned_now, "Timeout");

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const body = try collect(arena.allocator(), io, &pool, accumulator, &database, .@"1h", aligned_now);
    try testing.expect(body.complete);
}

test "the route's period grammar and its 400 text are the ones /api/stats serves" {
    // The picker scopes the whole page, so one bad spelling must mean the same
    // thing on every route it drives.
    try testing.expectEqual(stats.default_period, try stats.periodParam(""));
    try testing.expectEqual(stats.Period.@"24h", try stats.periodParam(""));
    try testing.expectEqual(stats.Period.@"7d", try stats.periodParam("period=7d"));
    try testing.expectError(error.BadPeriod, stats.periodParam("period=12h"));
    try testing.expectError(error.BadPeriod, stats.periodParam("period=1hhhhhhhhhh"));
    try testing.expectEqualStrings("period must be one of 1h, 24h, 7d, 30d", bad_period_message);
}

test "every period the grammar accepts produces the window that period names" {
    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
    defer threaded.deinit();
    const io = threaded.io();

    var database = try openLog();
    defer database.close();

    var entries = [_]pool_mod.Entry{testEntry(url_a, true)};
    var pool = testPool(&entries);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    for (std.enums.values(stats.Period)) |period| {
        const span = stats.window(period, aligned_now);
        const body = try collect(arena.allocator(), io, &pool, null, &database, period, aligned_now);
        try testing.expectEqualStrings(period.label(), body.period);
        try testing.expectEqual(span.since, body.since);
        try testing.expectEqual(span.until, body.until);
    }
}

test "the body serializes exactly the ranged field set" {
    const upstreams = [_]Upstream{ .{
        .url = url_a,
        .enabled = true,
        .available = false,
        .period = .{
            .attempts = 8,
            .successes = 6,
            .failures = 2,
            .success_rate = 0.75,
            .last_failure_at = 1_700_000_000,
            .last_failure_error = "ConnectFailed",
        },
    }, .{
        .url = url_b,
        .enabled = false,
        .available = false,
        .period = .{
            .attempts = 0,
            .successes = 0,
            .failures = 0,
            .success_rate = null,
            .last_failure_at = null,
            .last_failure_error = null,
        },
    } };

    var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
    defer allocating.deinit();
    try std.json.Stringify.value(Body{
        .period = "1h",
        .since = 1_699_996_400,
        .until = 1_700_000_000,
        .available = 0,
        .total = 2,
        .complete = true,
        .upstreams = &upstreams,
    }, .{}, &allocating.writer);

    try testing.expectEqualStrings(
        \\{"period":"1h","since":1699996400,"until":1700000000,"available":0,"total":2,"complete":true,"upstreams":[{"url":"https://a.test/dns-query","enabled":true,"available":false,"period":{"attempts":8,"successes":6,"failures":2,"success_rate":0.75,"last_failure_at":1700000000,"last_failure_error":"ConnectFailed"}},{"url":"https://b.test/dns-query","enabled":false,"available":false,"period":{"attempts":0,"successes":0,"failures":0,"success_rate":null,"last_failure_at":null,"last_failure_error":null}}]}
    , allocating.written());

    // The lifetime fields m26 removed. They still exist in `health.State` and in
    // `/metrics`; a client of this route must not find them here and start
    // reading them as if they were scoped to the period.
    for ([_][]const u8{
        "consecutive_failures",
        "total_successes",
        "total_failures",
        "last_error_age_s",
        "\"last_error\"",
    }) |gone| {
        try testing.expect(!std.mem.containsAtLeast(u8, allocating.written(), 1, gone));
    }
}
