nxdns/src/web/handlers/stats.zig

//! The five period endpoints: `GET /api/stats` and `/api/stats/timeseries`
//! (ruling 13), and `/api/stats/types`, `/api/stats/routes` and
//! `/api/stats/clients` (milestone 30).
//!
//! One period grammar, four widths, and one window shared by all five: a
//! request for the same period gets the same `since`/`until` from every
//! endpoint, so the totals describe exactly the span the charts draw rather
//! than a neighbouring one.
//!
//! That is window coherence, not identical counts. Each endpoint is its own
//! request against its own snapshot, so queries logged between two of them move
//! one panel and not the other. Only a box with nothing writing to it — a test
//! — can expect the breakdowns to sum to the totals exactly.
//!
//! Buckets are aligned to the UTC grid, not to the moment of the request. Every
//! width divides a day, so flooring the current time to a multiple of the width
//! puts each bucket on the same boundary a human reads off a clock, and two
//! requests a second apart return the same bucket starts. The last bucket is
//! the one in progress; it fills as the period runs.
//!
//! The aggregates run on the web task's own query-log connection (m7 ruling 21),
//! which every connection task shares. SQLite's serialized mode makes one call
//! safe; it does not make a transaction safe, so `WebState.querylog_lock` covers
//! the whole read and a second BEGIN can never land inside the first. Each
//! response takes one deferred read transaction, so its aggregate and the
//! `coverage` beside it describe one database state: retention cannot prune
//! between them and hand a client pre-prune rows tagged with a post-prune
//! watermark. Deferred, not `db.Tx`'s BEGIN IMMEDIATE, which would stall the
//! logger and retention behind an HTTP response.
//!
//! The lock is released before the response is written: the body is already
//! built in the request arena, and holding a database lock across a socket
//! write would let one slow client serialize every other reader.

const std = @import("std");

const coverage = @import("../coverage.zig");
const db = @import("../../storage/db.zig");
const http_util = @import("../http_util.zig");
const queries_repo = @import("../../storage/repositories/queries_repo.zig");
const server = @import("../server.zig");

const log = std.log.scoped(.web_stats);

/// The four periods ruling 13 defines. The tag names are the wire spellings.
pub const Period = enum {
    @"1h",
    @"24h",
    @"7d",
    @"30d",

    pub fn parse(text: []const u8) ?Period {
        return std.meta.stringToEnum(Period, text);
    }

    /// Ruling 13: 1h→60×1m, 24h→48×30m, 7d→168×1h, 30d→120×6h.
    pub fn bucketSeconds(self: Period) u32 {
        return switch (self) {
            .@"1h" => 60,
            .@"24h" => 30 * 60,
            .@"7d" => 60 * 60,
            .@"30d" => 6 * 60 * 60,
        };
    }

    pub fn bucketCount(self: Period) u32 {
        return switch (self) {
            .@"1h" => 60,
            .@"24h" => 48,
            .@"7d" => 168,
            .@"30d" => 120,
        };
    }

    pub fn label(self: Period) []const u8 {
        return @tagName(self);
    }
};

pub const default_period: Period = .@"24h";

/// The widest period's bucket count, so one stack array serves every request.
pub const max_buckets = 168;

comptime {
    for (std.enums.values(Period)) |period| {
        std.debug.assert(period.bucketCount() <= max_buckets);
        // The UTC alignment argument holds only while every width divides a day.
        std.debug.assert(86_400 % period.bucketSeconds() == 0);
    }
}

pub const Window = struct {
    /// Inclusive, on the bucket grid.
    since: i64,
    /// Exclusive: the end of the bucket that `now` falls in.
    until: i64,
    bucket_seconds: u32,
    bucket_count: u32,
};

pub fn window(period: Period, now_unix: i64) Window {
    const width: i64 = period.bucketSeconds();
    const count: i64 = period.bucketCount();
    const until = @divFloor(now_unix, width) * width + width;
    return .{
        .since = until - width * count,
        .until = until,
        .bucket_seconds = period.bucketSeconds(),
        .bucket_count = period.bucketCount(),
    };
}

pub const TotalsBody = struct {
    period: []const u8,
    since: i64,
    until: i64,
    queries: u64,
    blocked: u64,
    clients: u64,
    avg_response_time_us: ?i64,
    /// Judged against `since`, which is the window this body reports on — so a
    /// dashboard can say "history starts here" instead of charting a pruned
    /// stretch as a quiet one.
    coverage: coverage.Coverage,
};

pub const TimeseriesBody = struct {
    period: []const u8,
    since: i64,
    until: i64,
    bucket_seconds: u32,
    buckets: []const queries_repo.Bucket,
    coverage: coverage.Coverage,
};

pub const TypesBody = struct {
    period: []const u8,
    since: i64,
    until: i64,
    types: []const queries_repo.TypeCount,
    coverage: coverage.Coverage,
};

pub const RoutesBody = struct {
    period: []const u8,
    since: i64,
    until: i64,
    routes: []const queries_repo.RouteCount,
    coverage: coverage.Coverage,
};

pub const ClientsBody = struct {
    period: []const u8,
    since: i64,
    until: i64,
    bucket_seconds: u32,
    clients: []const queries_repo.ClientSeries,
    other: []const u64,
    coverage: coverage.Coverage,
};

/// Everything one response reads from the query log, so the caller can end the
/// transaction and drop the lock before it serializes anything.
fn Read(comptime T: type) type {
    return struct {
        data: T,
        coverage: coverage.Coverage,
    };
}

const ReadScope = server.QuerylogRead;

fn readTotals(
    state: *server.WebState,
    io: std.Io,
    database: *db.Db,
    span: Window,
) db.Error!Read(queries_repo.StatsTotals) {
    var scope = try ReadScope.open(state, io, database);
    errdefer scope.abort();
    const read: Read(queries_repo.StatsTotals) = .{
        .data = try queries_repo.statsTotals(database, span.since, span.until),
        .coverage = try coverage.read(database, span.since),
    };
    try scope.commit();
    return read;
}

fn readTimeseries(
    state: *server.WebState,
    io: std.Io,
    database: *db.Db,
    span: Window,
    out: []queries_repo.Bucket,
) db.Error!Read(usize) {
    var scope = try ReadScope.open(state, io, database);
    errdefer scope.abort();
    const read: Read(usize) = .{
        .data = try queries_repo.timeseries(database, span.since, span.bucket_seconds, out),
        .coverage = try coverage.read(database, span.since),
    };
    try scope.commit();
    return read;
}

fn readTypes(
    state: *server.WebState,
    io: std.Io,
    database: *db.Db,
    arena: std.mem.Allocator,
    span: Window,
) db.Error!Read([]const queries_repo.TypeCount) {
    var scope = try ReadScope.open(state, io, database);
    errdefer scope.abort();
    const list = try queries_repo.statsTypes(database, arena, span.since, span.until);
    const read: Read([]const queries_repo.TypeCount) = .{
        .data = list.items,
        .coverage = try coverage.read(database, span.since),
    };
    try scope.commit();
    return read;
}

fn readRoutes(
    state: *server.WebState,
    io: std.Io,
    database: *db.Db,
    arena: std.mem.Allocator,
    span: Window,
) db.Error!Read([]const queries_repo.RouteCount) {
    var scope = try ReadScope.open(state, io, database);
    errdefer scope.abort();
    const list = try queries_repo.statsRoutes(database, arena, span.since, span.until);
    const read: Read([]const queries_repo.RouteCount) = .{
        .data = list.items,
        .coverage = try coverage.read(database, span.since),
    };
    try scope.commit();
    return read;
}

fn readClients(
    state: *server.WebState,
    io: std.Io,
    database: *db.Db,
    arena: std.mem.Allocator,
    span: Window,
) db.Error!Read(queries_repo.ClientsBreakdown) {
    var scope = try ReadScope.open(state, io, database);
    errdefer scope.abort();
    const read: Read(queries_repo.ClientsBreakdown) = .{
        .data = try queries_repo.statsClients(
            database,
            arena,
            span.since,
            span.bucket_seconds,
            span.bucket_count,
        ),
        .coverage = try coverage.read(database, span.since),
    };
    try scope.commit();
    return read;
}

pub fn totals(
    state: *server.WebState,
    io: std.Io,
    request: *http_util.Request,
) http_util.HandlerError!void {
    const period = periodParam(request.query) catch return badPeriod(request);
    const database = state.querylog_db orelse return unavailable(request);
    const span = window(period, std.Io.Clock.real.now(io).toSeconds());

    const read = readTotals(state, io, database, span) catch |err| {
        return internal(request, "stats totals", err);
    };

    return http_util.respondJson(request, .ok, TotalsBody{
        .period = period.label(),
        .since = span.since,
        .until = span.until,
        .queries = read.data.queries,
        .blocked = read.data.blocked,
        .clients = read.data.distinct_clients,
        .avg_response_time_us = read.data.avg_response_time_us,
        .coverage = read.coverage,
    }, &.{});
}

pub fn timeseries(
    state: *server.WebState,
    io: std.Io,
    request: *http_util.Request,
) http_util.HandlerError!void {
    const period = periodParam(request.query) catch return badPeriod(request);
    const database = state.querylog_db orelse return unavailable(request);
    const span = window(period, std.Io.Clock.real.now(io).toSeconds());

    var buckets: [max_buckets]queries_repo.Bucket = undefined;
    const out = buckets[0..span.bucket_count];
    const read = readTimeseries(state, io, database, span, out) catch |err| {
        return internal(request, "stats timeseries", err);
    };

    return http_util.respondJson(request, .ok, TimeseriesBody{
        .period = period.label(),
        .since = span.since,
        .until = span.until,
        .bucket_seconds = span.bucket_seconds,
        .buckets = out[0..read.data],
        .coverage = read.coverage,
    }, &.{});
}

pub fn types(
    state: *server.WebState,
    io: std.Io,
    request: *http_util.Request,
) http_util.HandlerError!void {
    const period = periodParam(request.query) catch return badPeriod(request);
    const database = state.querylog_db orelse return unavailable(request);
    const span = window(period, std.Io.Clock.real.now(io).toSeconds());

    const read = readTypes(state, io, database, request.arena, span) catch |err| {
        return internal(request, "stats types", err);
    };

    return http_util.respondJson(request, .ok, TypesBody{
        .period = period.label(),
        .since = span.since,
        .until = span.until,
        .types = read.data,
        .coverage = read.coverage,
    }, &.{});
}

pub fn routes(
    state: *server.WebState,
    io: std.Io,
    request: *http_util.Request,
) http_util.HandlerError!void {
    const period = periodParam(request.query) catch return badPeriod(request);
    const database = state.querylog_db orelse return unavailable(request);
    const span = window(period, std.Io.Clock.real.now(io).toSeconds());

    const read = readRoutes(state, io, database, request.arena, span) catch |err| {
        return internal(request, "stats routes", err);
    };

    return http_util.respondJson(request, .ok, RoutesBody{
        .period = period.label(),
        .since = span.since,
        .until = span.until,
        .routes = read.data,
        .coverage = read.coverage,
    }, &.{});
}

pub fn clients(
    state: *server.WebState,
    io: std.Io,
    request: *http_util.Request,
) http_util.HandlerError!void {
    const period = periodParam(request.query) catch return badPeriod(request);
    const database = state.querylog_db orelse return unavailable(request);
    const span = window(period, std.Io.Clock.real.now(io).toSeconds());

    const read = readClients(state, io, database, request.arena, span) catch |err| {
        return internal(request, "stats clients", err);
    };

    return http_util.respondJson(request, .ok, ClientsBody{
        .period = period.label(),
        .since = span.since,
        .until = span.until,
        .bucket_seconds = span.bucket_seconds,
        .clients = read.data.clients,
        .other = read.data.other,
        .coverage = read.coverage,
    }, &.{});
}

pub const PeriodError = error{BadPeriod};

/// An absent `period` is the default; anything else it cannot read is a 400,
/// never a silent fallback — a typo must not return a window nobody asked for.
fn periodParam(query: []const u8) PeriodError!Period {
    var buf: [8]u8 = undefined;
    const found = http_util.queryValue(query, "period", &buf) catch return error.BadPeriod;
    const text = found orelse return default_period;
    return Period.parse(text) orelse error.BadPeriod;
}

fn badPeriod(request: *http_util.Request) http_util.HandlerError!void {
    return http_util.respondError(request, .bad_request, "period must be one of 1h, 24h, 7d, 30d");
}

fn unavailable(request: *http_util.Request) http_util.HandlerError!void {
    return http_util.respondError(request, .service_unavailable, "query log unavailable");
}

/// The one thing this file logs. A failed aggregate is a fault in the box, not
/// a property of the request, and the client is told nothing beyond "internal
/// error" (ruling 8, PLAN §19).
fn internal(
    request: *http_util.Request,
    what: []const u8,
    err: db.Error,
) http_util.HandlerError!void {
    log.warn("{s} failed: {s}", .{ what, @errorName(err) });
    return http_util.respondError(request, .internal_server_error, "internal error");
}

// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------

const querylog_schema = @import("../../storage/querylog_schema.zig");
const testing = std.testing;

test "the period grammar accepts exactly the four spellings" {
    try testing.expectEqual(Period.@"1h", Period.parse("1h").?);
    try testing.expectEqual(Period.@"24h", Period.parse("24h").?);
    try testing.expectEqual(Period.@"7d", Period.parse("7d").?);
    try testing.expectEqual(Period.@"30d", Period.parse("30d").?);
    try testing.expectEqual(@as(?Period, null), Period.parse("12h"));
    try testing.expectEqual(@as(?Period, null), Period.parse("1H"));
    try testing.expectEqual(@as(?Period, null), Period.parse(""));
}

test "an absent period defaults and a bad one is rejected" {
    try testing.expectEqual(default_period, try periodParam(""));
    try testing.expectEqual(default_period, try periodParam("limit=5"));
    try testing.expectEqual(Period.@"7d", try periodParam("period=7d"));
    try testing.expectError(error.BadPeriod, periodParam("period=12h"));
    try testing.expectError(error.BadPeriod, periodParam("period=%2"));
    // Longer than any spelling: rejected rather than truncated to "1h".
    try testing.expectError(error.BadPeriod, periodParam("period=1hhhhhhhhhh"));
}

test "each period spans its own bucket width times its count" {
    for (std.enums.values(Period)) |period| {
        const span = window(period, 1_700_000_000);
        const width: i64 = period.bucketSeconds();
        try testing.expectEqual(width * @as(i64, period.bucketCount()), span.until - span.since);
    }
}

test "the window sits on the UTC grid and ends with the bucket in progress" {
    // 2023-11-14T22:13:20Z, which is not on any bucket boundary.
    const now: i64 = 1_700_000_000;
    const span = window(.@"24h", now);

    try testing.expectEqual(@as(i64, 0), @rem(span.since, 1800));
    try testing.expectEqual(@as(i64, 0), @rem(span.until, 1800));
    try testing.expect(span.until > now);
    try testing.expect(span.until - now <= 1800);
    try testing.expectEqual(@as(u32, 48), span.bucket_count);
}

test "two requests inside one bucket see the same window" {
    // A bucket boundary, so the offsets below stay inside one minute.
    const boundary: i64 = 1_700_000_000 - @rem(1_700_000_000, 60);
    const first = window(.@"1h", boundary);
    const second = window(.@"1h", boundary + 59);
    try testing.expectEqual(first.since, second.since);
    try testing.expectEqual(first.until, second.until);

    const next = window(.@"1h", boundary + 60);
    try testing.expectEqual(first.until + 60, next.until);
}

test "a timestamp exactly on a boundary starts a new bucket" {
    const span = window(.@"7d", 1_700_000_000 - 1_700_000_000 % 3600);
    try testing.expectEqual(@as(i64, 0), @rem(span.since, 3600));
    try testing.expectEqual(@as(u32, 168), span.bucket_count);
}

fn openLog() !db.Db {
    var database = try db.Db.open(":memory:", .{ .mode = .memory });
    errdefer database.close();
    try db.applyPragmas(&database, .{});
    try database.exec(querylog_schema.ddl);
    return database;
}

fn writeRow(writer: *queries_repo.BatchWriter, timestamp: i64, blocked: bool, cached: ?bool) !void {
    const rows = [_]queries_repo.Row{.{
        .timestamp = timestamp,
        .domain = "example.com",
        .client_ip = "192.0.2.10",
        .qtype = 1,
        .qclass = 1,
        .rcode = 0,
        .blocked = blocked,
        .response_time_us = 1000,
        .cache_hit = cached,
        .upstream = null,
        .group_id = 1,
        .group_name = "default",
        .policy_action = if (blocked) .block else .allow,
        .policy_reason = if (blocked) .blocklist_domain else .no_match,
        .matched = null,
        .source_id = null,
        .source_name = null,
        .cname_target = null,
        .safe_search_target = null,
        .route_kind = if (blocked) .blocked else .upstream,
        .forward_zone = null,
    }};
    try writer.writeBatch(&rows);
}

test "the totals and the buckets agree over the same window" {
    var database = try openLog();
    defer database.close();

    const now: i64 = 1_700_000_000;
    const span = window(.@"1h", now);

    var writer = try queries_repo.BatchWriter.init(&database);
    defer writer.deinit();
    // One row in the first bucket, two in the last, one just outside.
    try writeRow(&writer, span.since, false, false);
    try writeRow(&writer, span.until - 1, true, false);
    try writeRow(&writer, span.until - 2, false, true);
    try writeRow(&writer, span.since - 1, false, false);

    const result = try queries_repo.statsTotals(&database, span.since, span.until);
    try testing.expectEqual(@as(u64, 3), result.queries);
    try testing.expectEqual(@as(u64, 1), result.blocked);
    try testing.expectEqual(@as(u64, 1), result.distinct_clients);
    try testing.expectEqual(@as(?i64, 1000), result.avg_response_time_us);

    var buckets: [max_buckets]queries_repo.Bucket = undefined;
    const out = buckets[0..span.bucket_count];
    const written = try queries_repo.timeseries(&database, span.since, span.bucket_seconds, out);
    try testing.expectEqual(@as(usize, 60), written);

    var summed: u64 = 0;
    var blocked: u64 = 0;
    for (out) |bucket| {
        summed += bucket.queries;
        blocked += bucket.blocked;
    }
    try testing.expectEqual(result.queries, summed);
    try testing.expectEqual(result.blocked, blocked);

    try testing.expectEqual(span.since, out[0].ts);
    try testing.expectEqual(@as(u64, 1), out[0].queries);
    try testing.expectEqual(@as(u64, 2), out[59].queries);
    try testing.expectEqual(span.until - span.bucket_seconds, out[59].ts);
}

test "an empty window reports zeros with a null mean" {
    var database = try openLog();
    defer database.close();

    const span = window(.@"30d", 1_700_000_000);
    const result = try queries_repo.statsTotals(&database, span.since, span.until);
    try testing.expectEqual(@as(u64, 0), result.queries);
    try testing.expectEqual(@as(?i64, null), result.avg_response_time_us);

    var buckets: [max_buckets]queries_repo.Bucket = undefined;
    const out = buckets[0..span.bucket_count];
    try testing.expectEqual(@as(usize, 120), try queries_repo.timeseries(
        &database,
        span.since,
        span.bucket_seconds,
        out,
    ));
    for (out) |bucket| try testing.expectEqual(@as(u64, 0), bucket.queries);
}