nxdns/src/web/handlers/queries.zig

//! `GET /api/queries` — the query log, newest first (ruling 11) — and
//! `GET /api/queries/{id}`, one row of it fully explained.
//!
//! Keyset pagination rather than an offset: the table is append-only and the
//! UI reads the head of it, so `id < before` is one index seek no matter how
//! deep the client has scrolled, and rows arriving between two pages cannot
//! shift the window and duplicate a row.
//!
//! Filter parsing is separated from fetching, because parsing is where the
//! input validation of PLAN §19 lives and it is worth testing on its own. Every
//! value is length-capped here and bound as a SQL parameter by the repository;
//! nothing this file reads is ever concatenated into a statement.

const std = @import("std");
const Allocator = std.mem.Allocator;

const coverage = @import("../coverage.zig");
const db = @import("../../storage/db.zig");
const http_util = @import("../http_util.zig");
const logger = @import("../../storage/logger.zig");
const provenance_view = @import("../provenance_view.zig");
const queries_repo = @import("../../storage/repositories/queries_repo.zig");
const server = @import("../server.zig");

const log = std.log.scoped(.web_queries);

pub const default_limit: u32 = 100;
pub const max_limit: u32 = queries_repo.max_limit;

/// The filter widths are the stored widths: a filter wider than the column it
/// compares against could only match a row the query log cannot hold.
pub const max_domain_len = logger.max_domain_len;
pub const max_client_len = logger.max_client_len;

/// Where the two string filters are copied to. The parsed filter borrows them,
/// so it must not outlive the buffers — in the handler both live in the same
/// stack frame.
pub const Buffers = struct {
    domain: [max_domain_len]u8 = undefined,
    client: [max_client_len]u8 = undefined,
};

pub const Page = struct {
    queries: []const queries_repo.QueryRow,
    /// The cursor for the next page, or null when this page is the last one.
    next_before: ?i64,
    /// Whether the log still covers the window the filter asked for. A client
    /// that reads rows without reading this cannot tell an empty window from a
    /// pruned one.
    coverage: coverage.Coverage,
};

pub const FilterError = error{
    BadLimit,
    BadBefore,
    BadDomain,
    BadClient,
    BadBlocked,
    BadSince,
    BadUntil,
};

/// Ruling 11's query string. An absent parameter drops the filter; a malformed
/// one is a 400 rather than a filter silently left off, which would answer a
/// question the client did not ask.
pub fn parseFilter(query: []const u8, buffers: *Buffers) FilterError!queries_repo.QueryFilter {
    var filter: queries_repo.QueryFilter = .{};

    if (http_util.queryInt(u32, query, "limit") catch return error.BadLimit) |limit| {
        if (limit == 0 or limit > max_limit) return error.BadLimit;
        filter.limit = limit;
    }

    if (http_util.queryInt(i64, query, "before") catch return error.BadBefore) |before| {
        // Row ids are positive, so a non-positive cursor is a client bug, not
        // an empty page.
        if (before <= 0) return error.BadBefore;
        filter.before = before;
    }

    if (http_util.queryValue(query, "domain", &buffers.domain) catch return error.BadDomain) |domain| {
        if (domain.len != 0) filter.domain_substring = domain;
    }

    if (http_util.queryValue(query, "client", &buffers.client) catch return error.BadClient) |client| {
        if (client.len != 0) filter.client = client;
    }

    filter.blocked = http_util.queryBool(query, "blocked") catch return error.BadBlocked;
    filter.since = http_util.queryInt(i64, query, "since") catch return error.BadSince;
    filter.until = http_util.queryInt(i64, query, "until") catch return error.BadUntil;

    return filter;
}

pub fn message(err: FilterError) []const u8 {
    return switch (err) {
        error.BadLimit => "limit must be between 1 and 1000",
        error.BadBefore => "before must be a positive row id",
        error.BadDomain => "domain is not a valid filter",
        error.BadClient => "client is not a valid filter",
        error.BadBlocked => "blocked must be true or false",
        error.BadSince => "since must be a unix timestamp in seconds",
        error.BadUntil => "until must be a unix timestamp in seconds",
    };
}

/// A full page carries a cursor and a short one does not: a client stops when
/// `next_before` is null, without a count query telling it how many rows exist.
pub fn page(
    database: *db.Db,
    arena: Allocator,
    filter: queries_repo.QueryFilter,
) db.Error!Page {
    const rows = try queries_repo.selectQueries(database, arena, filter);
    const full = rows.items.len == @min(filter.limit, max_limit);
    return .{
        .queries = rows.items,
        .next_before = if (full and rows.items.len != 0) rows.items[rows.items.len - 1].id else null,
        .coverage = try coverage.read(database, filter.since),
    };
}

pub fn list(
    state: *server.WebState,
    io: std.Io,
    request: *http_util.Request,
) http_util.HandlerError!void {
    var buffers: Buffers = .{};
    const filter = parseFilter(request.query, &buffers) catch |err| {
        return http_util.respondError(request, .bad_request, message(err));
    };

    const database = state.querylog_db orelse
        return http_util.respondError(request, .service_unavailable, "query log unavailable");

    const result = readPage(state, io, database, request.arena, filter) catch |err| {
        // The one thing this handler logs: a database fault is a property of
        // the box, not of the request, and the client is told nothing about it.
        log.warn("query log read failed: {s}", .{@errorName(err)});
        return http_util.respondError(request, .internal_server_error, "internal error");
    };

    return http_util.respondJson(request, .ok, result, &.{});
}

/// `GET /api/queries/{id}` — one query, fully explained.
///
/// The row is the whole answer: every field is a fact recorded when the query
/// was answered, so nothing here is joined against current configuration. A
/// group or blocklist renamed since keeps the name it had.
pub fn detail(
    state: *server.WebState,
    io: std.Io,
    request: *http_util.Request,
) http_util.HandlerError!void {
    const database = state.querylog_db orelse
        return http_util.respondError(request, .service_unavailable, "query log unavailable");

    const row = detailRow(state, io, database, request.arena, request.id.?) catch |err| {
        log.warn("query log read failed: {s}", .{@errorName(err)});
        return http_util.respondError(request, .internal_server_error, "internal error");
    };

    // An id retention has pruned and one that never existed are the same
    // answer, and the API does not pretend to tell them apart.
    const found = row orelse return http_util.respondError(request, .not_found, "not found");
    return http_util.respondJson(request, .ok, provenance_view.fromDetail(found), &.{});
}

/// The rows and the coverage watermark come from one database state, so a
/// prune between them cannot tag pre-prune rows with a post-prune
/// `available_since`.
fn readPage(
    state: *server.WebState,
    io: std.Io,
    database: *db.Db,
    arena: Allocator,
    filter: queries_repo.QueryFilter,
) db.Error!Page {
    var scope = try server.QuerylogRead.open(state, io, database);
    errdefer scope.abort();
    const result = try page(database, arena, filter);
    try scope.commit();
    return result;
}

/// One row, read under the shared lock. There is nothing to keep consistent
/// with a second statement here; the lock is what keeps this read out of
/// another response's open transaction.
fn detailRow(
    state: *server.WebState,
    io: std.Io,
    database: *db.Db,
    arena: Allocator,
    id: i64,
) db.Error!?queries_repo.QueryDetail {
    var scope = try server.QuerylogRead.open(state, io, database);
    errdefer scope.abort();
    const row = try queries_repo.detailById(database, arena, id);
    try scope.commit();
    return row;
}

// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------

const provenance = @import("../../storage/provenance.zig");
const querylog_schema = @import("../../storage/querylog_schema.zig");
const testing = std.testing;

test "an empty query string is the default page" {
    var buffers: Buffers = .{};
    const filter = try parseFilter("", &buffers);
    try testing.expectEqual(default_limit, filter.limit);
    try testing.expectEqual(@as(?i64, null), filter.before);
    try testing.expectEqual(@as(?[]const u8, null), filter.domain_substring);
    try testing.expectEqual(@as(?bool, null), filter.blocked);
}

test "every filter reaches the repository untouched" {
    var buffers: Buffers = .{};
    const filter = try parseFilter(
        "limit=250&before=900&domain=ads.example&client=192.0.2.10&blocked=true&since=100&until=200",
        &buffers,
    );
    try testing.expectEqual(@as(u32, 250), filter.limit);
    try testing.expectEqual(@as(?i64, 900), filter.before);
    try testing.expectEqualStrings("ads.example", filter.domain_substring.?);
    try testing.expectEqualStrings("192.0.2.10", filter.client.?);
    try testing.expectEqual(@as(?bool, true), filter.blocked);
    try testing.expectEqual(@as(?i64, 100), filter.since);
    try testing.expectEqual(@as(?i64, 200), filter.until);
}

test "an empty string filter is no filter at all" {
    var buffers: Buffers = .{};
    const filter = try parseFilter("domain=&client=", &buffers);
    try testing.expectEqual(@as(?[]const u8, null), filter.domain_substring);
    try testing.expectEqual(@as(?[]const u8, null), filter.client);
}

test "each malformed parameter names itself in a 400" {
    var buffers: Buffers = .{};
    try testing.expectError(error.BadLimit, parseFilter("limit=0", &buffers));
    try testing.expectError(error.BadLimit, parseFilter("limit=1001", &buffers));
    try testing.expectError(error.BadLimit, parseFilter("limit=ten", &buffers));
    try testing.expectError(error.BadBefore, parseFilter("before=0", &buffers));
    try testing.expectError(error.BadBefore, parseFilter("before=-4", &buffers));
    try testing.expectError(error.BadBlocked, parseFilter("blocked=maybe", &buffers));
    try testing.expectError(error.BadSince, parseFilter("since=yesterday", &buffers));
    try testing.expectError(error.BadUntil, parseFilter("until=", &buffers));
    try testing.expectError(error.BadDomain, parseFilter("domain=%zz", &buffers));

    var long: [max_domain_len + 8]u8 = @splat('a');
    var text: std.ArrayList(u8) = .empty;
    defer text.deinit(testing.allocator);
    try text.appendSlice(testing.allocator, "domain=");
    try text.appendSlice(testing.allocator, &long);
    try testing.expectError(error.BadDomain, parseFilter(text.items, &buffers));
}

test "the limit cap is the repository's" {
    var buffers: Buffers = .{};
    try testing.expectEqual(max_limit, (try parseFilter("limit=1000", &buffers)).limit);
    try testing.expectEqual(@as(u32, 1000), queries_repo.max_limit);
}

fn openLog() !db.Db {
    var database = try db.Db.open(":memory:", .{ .mode = .memory });
    errdefer database.close();
    try db.applyPragmas(&database, .{});
    try database.exec(querylog_schema.ddl);
    return database;
}

/// Rows the resolver could actually have written. Ruling 20 ties the three
/// route facts together: a block never consulted the cache, so its `cache_hit`
/// is NULL rather than false; a cache hit has no upstream to name; and only an
/// upstream answer carries one. A fixture that broke those ties would let a
/// serializer regression pass here and fail on real rows.
fn seed(database: *db.Db, count: usize) !void {
    var writer = try queries_repo.BatchWriter.init(testing.allocator, database);
    defer writer.deinit();
    var rows: [16]queries_repo.Row = undefined;
    for (rows[0..count], 0..) |*row, i| {
        const blocked = i % 2 == 0;
        const from_cache = i % 4 == 1;
        row.* = .{
            .timestamp = 1_700_000_000 + @as(i64, @intCast(i)),
            .domain = if (blocked) "ads.example" else "safe.example",
            .client_ip = "192.0.2.10",
            .qtype = 1,
            .qclass = 1,
            .rcode = 0,
            .blocked = blocked,
            .response_time_us = 500,
            .cache_hit = if (blocked) null else from_cache,
            .upstream = if (blocked or from_cache) null else "9.9.9.9",
            .group_id = 1,
            .group_name = "default",
            .policy_action = if (blocked) .block else .allow,
            .policy_reason = if (blocked) .blocklist_domain else .no_match,
            .matched = if (blocked) "ads.example" else null,
            .source_id = null,
            .source_name = null,
            .cname_target = null,
            .safe_search_target = null,
            .route_kind = if (blocked) .blocked else if (from_cache) .cache else .upstream,
            .forward_zone = null,
        };
    }
    try writer.writeBatch(rows[0..count]);
}

test "a full page carries a cursor and the last page does not" {
    var database = try openLog();
    defer database.close();
    try seed(&database, 5);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const first = try page(&database, arena.allocator(), .{ .limit = 2 });
    try testing.expectEqual(@as(usize, 2), first.queries.len);
    try testing.expectEqual(first.queries[1].id, first.next_before.?);
    // Newest first.
    try testing.expect(first.queries[0].id > first.queries[1].id);

    const second = try page(&database, arena.allocator(), .{ .limit = 2, .before = first.next_before });
    try testing.expect(second.queries[0].id < first.queries[1].id);

    const third = try page(&database, arena.allocator(), .{ .limit = 2, .before = second.next_before });
    try testing.expectEqual(@as(usize, 1), third.queries.len);
    try testing.expectEqual(@as(?i64, null), third.next_before);
}

test "an empty result is a page with no cursor" {
    var database = try openLog();
    defer database.close();

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const empty = try page(&database, arena.allocator(), .{});
    try testing.expectEqual(@as(usize, 0), empty.queries.len);
    try testing.expectEqual(@as(?i64, null), empty.next_before);
}

test "the parsed filters narrow the rows the page returns" {
    var database = try openLog();
    defer database.close();
    try seed(&database, 6);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    var buffers: Buffers = .{};
    const blocked = try page(
        &database,
        arena.allocator(),
        try parseFilter("blocked=true", &buffers),
    );
    try testing.expectEqual(@as(usize, 3), blocked.queries.len);
    for (blocked.queries) |row| try testing.expect(row.blocked);

    const by_domain = try page(
        &database,
        arena.allocator(),
        try parseFilter("domain=safe", &buffers),
    );
    try testing.expectEqual(@as(usize, 3), by_domain.queries.len);
    for (by_domain.queries) |row| try testing.expectEqualStrings("safe.example", row.domain);

    const nobody = try page(
        &database,
        arena.allocator(),
        try parseFilter("client=198.51.100.1", &buffers),
    );
    try testing.expectEqual(@as(usize, 0), nobody.queries.len);
}

test "the seeded rows carry only the route shapes ruling 20 allows" {
    var database = try openLog();
    defer database.close();
    try seed(&database, 4);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    var seen: std.EnumSet(provenance.RouteKind) = .initEmpty();
    for ((try page(&database, arena.allocator(), .{})).queries) |row| {
        seen.insert(row.route_kind);
        switch (row.route_kind) {
            .blocked => {
                try testing.expectEqual(@as(?bool, null), row.cache_hit);
                try testing.expectEqualStrings("", row.upstream);
            },
            .cache => {
                try testing.expectEqual(@as(?bool, true), row.cache_hit);
                try testing.expectEqualStrings("", row.upstream);
            },
            .upstream => {
                try testing.expectEqual(@as(?bool, false), row.cache_hit);
                try testing.expectEqualStrings("9.9.9.9", row.upstream);
            },
            .local, .forward_zone, .rejected => return error.UnseededRouteKind,
        }
    }
    // All three, so the serializer tests below read every shape the fixture claims.
    try testing.expectEqual(@as(usize, 3), seen.count());
}

test "the page serializes as the envelope ruling 11 defines" {
    var database = try openLog();
    defer database.close();
    try seed(&database, 1);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const result = try page(&database, arena.allocator(), .{ .limit = 100 });
    var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
    defer allocating.deinit();
    try std.json.Stringify.value(result, .{}, &allocating.writer);
    const text = allocating.written();

    try testing.expect(std.mem.startsWith(u8, text, "{\"queries\":["));
    for ([_][]const u8{
        "\"id\":",            "\"ts\":",         "\"domain\":",           "\"client_ip\":",
        "\"qtype\":",         "\"qclass\":",     "\"rcode\":",            "\"blocked\":",
        "\"cache_hit\":",     "\"upstream\":",   "\"response_time_us\":", "\"policy_action\":",
        "\"policy_reason\":", "\"route_kind\":",
    }) |field| {
        try testing.expect(std.mem.containsAtLeast(u8, text, 1, field));
    }
    // W1's ruling: a NULL column reads as "", and "" stays "" on the wire.
    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"upstream\":\"\""));
    // The column the provenance columns replaced.
    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "block_reason"));
    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"coverage\":{\"complete\":"));
}

test "the coverage of a page answers the window the filter asked for" {
    var database = try openLog();
    defer database.close();
    try seed(&database, 1);

    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena.deinit();

    const watermark = try queries_repo.availableSince(&database);

    const unbounded = try page(&database, arena.allocator(), .{});
    try testing.expectEqual(watermark, unbounded.coverage.available_since);
    try testing.expect(!unbounded.coverage.complete);

    const covered = try page(&database, arena.allocator(), .{ .since = watermark });
    try testing.expect(covered.coverage.complete);

    const older = try page(&database, arena.allocator(), .{ .since = watermark - 1 });
    try testing.expect(!older.coverage.complete);
}

test "a detail row carries every provenance field the row stored" {
    var database = try openLog();
    defer database.close();
    try seed(&database, 1);

    var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
    defer arena_state.deinit();
    const arena = arena_state.allocator();

    const row = (try queries_repo.detailById(&database, arena, 1)).?;
    const view = provenance_view.fromDetail(row);

    try testing.expectEqual(@as(i64, 1), view.id);
    try testing.expectEqualStrings("ads.example", view.request.domain);
    try testing.expectEqualStrings("192.0.2.10", view.request.client);
    try testing.expectEqual(@as(u16, 1), view.request.qclass);
    try testing.expectEqualStrings("default", view.group.name);
    try testing.expectEqual(provenance.PolicyAction.block, view.policy.action);
    try testing.expectEqualStrings("ads.example", view.policy.matched);
    try testing.expectEqual(provenance.RouteKind.blocked, view.route.kind);
    try testing.expectEqualStrings("", view.route.upstream);
    try testing.expectEqual(@as(?i64, 500), view.response.duration_us);

    try testing.expectEqual(
        @as(?queries_repo.QueryDetail, null),
        try queries_repo.detailById(&database, arena, 99),
    );
}