//! `GET /api/queries` — the query log, newest first (ruling 11). //! //! Keyset pagination rather than an offset: the table is append-only and the //! UI reads the head of it, so `id < before` is one index seek no matter how //! deep the client has scrolled, and rows arriving between two pages cannot //! shift the window and duplicate a row. //! //! Filter parsing is separated from fetching, because parsing is where the //! input validation of PLAN §19 lives and it is worth testing on its own. Every //! value is length-capped here and bound as a SQL parameter by the repository; //! nothing this file reads is ever concatenated into a statement. const std = @import("std"); const Allocator = std.mem.Allocator; const db = @import("../../storage/db.zig"); const http_util = @import("../http_util.zig"); const logger = @import("../../storage/logger.zig"); const queries_repo = @import("../../storage/repositories/queries_repo.zig"); const server = @import("../server.zig"); const log = std.log.scoped(.web_queries); pub const default_limit: u32 = 100; pub const max_limit: u32 = queries_repo.max_limit; /// The filter widths are the stored widths: a filter wider than the column it /// compares against could only match a row the query log cannot hold. pub const max_domain_len = logger.max_domain_len; pub const max_client_len = logger.max_client_len; /// Where the two string filters are copied to. The parsed filter borrows them, /// so it must not outlive the buffers — in the handler both live in the same /// stack frame. pub const Buffers = struct { domain: [max_domain_len]u8 = undefined, client: [max_client_len]u8 = undefined, }; pub const Page = struct { queries: []const queries_repo.QueryRow, /// The cursor for the next page, or null when this page is the last one. next_before: ?i64, }; pub const FilterError = error{ BadLimit, BadBefore, BadDomain, BadClient, BadBlocked, BadSince, BadUntil, }; /// Ruling 11's query string. An absent parameter drops the filter; a malformed /// one is a 400 rather than a filter silently left off, which would answer a /// question the client did not ask. pub fn parseFilter(query: []const u8, buffers: *Buffers) FilterError!queries_repo.QueryFilter { var filter: queries_repo.QueryFilter = .{}; if (http_util.queryInt(u32, query, "limit") catch return error.BadLimit) |limit| { if (limit == 0 or limit > max_limit) return error.BadLimit; filter.limit = limit; } if (http_util.queryInt(i64, query, "before") catch return error.BadBefore) |before| { // Row ids are positive, so a non-positive cursor is a client bug, not // an empty page. if (before <= 0) return error.BadBefore; filter.before = before; } if (http_util.queryValue(query, "domain", &buffers.domain) catch return error.BadDomain) |domain| { if (domain.len != 0) filter.domain_substring = domain; } if (http_util.queryValue(query, "client", &buffers.client) catch return error.BadClient) |client| { if (client.len != 0) filter.client = client; } filter.blocked = http_util.queryBool(query, "blocked") catch return error.BadBlocked; filter.since = http_util.queryInt(i64, query, "since") catch return error.BadSince; filter.until = http_util.queryInt(i64, query, "until") catch return error.BadUntil; return filter; } pub fn message(err: FilterError) []const u8 { return switch (err) { error.BadLimit => "limit must be between 1 and 1000", error.BadBefore => "before must be a positive row id", error.BadDomain => "domain is not a valid filter", error.BadClient => "client is not a valid filter", error.BadBlocked => "blocked must be true or false", error.BadSince => "since must be a unix timestamp in seconds", error.BadUntil => "until must be a unix timestamp in seconds", }; } /// A full page carries a cursor and a short one does not: a client stops when /// `next_before` is null, without a count query telling it how many rows exist. pub fn page( database: *db.Db, arena: Allocator, filter: queries_repo.QueryFilter, ) db.Error!Page { const rows = try queries_repo.selectQueries(database, arena, filter); const full = rows.items.len == @min(filter.limit, max_limit); return .{ .queries = rows.items, .next_before = if (full and rows.items.len != 0) rows.items[rows.items.len - 1].id else null, }; } pub fn list( state: *server.WebState, io: std.Io, request: *http_util.Request, ) http_util.HandlerError!void { _ = io; var buffers: Buffers = .{}; const filter = parseFilter(request.query, &buffers) catch |err| { return http_util.respondError(request, .bad_request, message(err)); }; const database = state.querylog_db orelse return http_util.respondError(request, .service_unavailable, "query log unavailable"); const result = page(database, request.arena, filter) catch |err| { // The one thing this handler logs: a database fault is a property of // the box, not of the request, and the client is told nothing about it. log.warn("query log read failed: {s}", .{@errorName(err)}); return http_util.respondError(request, .internal_server_error, "internal error"); }; return http_util.respondJson(request, .ok, result, &.{}); } // --------------------------------------------------------------------------- // tests // --------------------------------------------------------------------------- const querylog_schema = @import("../../storage/querylog_schema.zig"); const testing = std.testing; test "an empty query string is the default page" { var buffers: Buffers = .{}; const filter = try parseFilter("", &buffers); try testing.expectEqual(default_limit, filter.limit); try testing.expectEqual(@as(?i64, null), filter.before); try testing.expectEqual(@as(?[]const u8, null), filter.domain_substring); try testing.expectEqual(@as(?bool, null), filter.blocked); } test "every filter reaches the repository untouched" { var buffers: Buffers = .{}; const filter = try parseFilter( "limit=250&before=900&domain=ads.example&client=192.0.2.10&blocked=true&since=100&until=200", &buffers, ); try testing.expectEqual(@as(u32, 250), filter.limit); try testing.expectEqual(@as(?i64, 900), filter.before); try testing.expectEqualStrings("ads.example", filter.domain_substring.?); try testing.expectEqualStrings("192.0.2.10", filter.client.?); try testing.expectEqual(@as(?bool, true), filter.blocked); try testing.expectEqual(@as(?i64, 100), filter.since); try testing.expectEqual(@as(?i64, 200), filter.until); } test "an empty string filter is no filter at all" { var buffers: Buffers = .{}; const filter = try parseFilter("domain=&client=", &buffers); try testing.expectEqual(@as(?[]const u8, null), filter.domain_substring); try testing.expectEqual(@as(?[]const u8, null), filter.client); } test "each malformed parameter names itself in a 400" { var buffers: Buffers = .{}; try testing.expectError(error.BadLimit, parseFilter("limit=0", &buffers)); try testing.expectError(error.BadLimit, parseFilter("limit=1001", &buffers)); try testing.expectError(error.BadLimit, parseFilter("limit=ten", &buffers)); try testing.expectError(error.BadBefore, parseFilter("before=0", &buffers)); try testing.expectError(error.BadBefore, parseFilter("before=-4", &buffers)); try testing.expectError(error.BadBlocked, parseFilter("blocked=maybe", &buffers)); try testing.expectError(error.BadSince, parseFilter("since=yesterday", &buffers)); try testing.expectError(error.BadUntil, parseFilter("until=", &buffers)); try testing.expectError(error.BadDomain, parseFilter("domain=%zz", &buffers)); var long: [max_domain_len + 8]u8 = @splat('a'); var text: std.ArrayList(u8) = .empty; defer text.deinit(testing.allocator); try text.appendSlice(testing.allocator, "domain="); try text.appendSlice(testing.allocator, &long); try testing.expectError(error.BadDomain, parseFilter(text.items, &buffers)); } test "the limit cap is the repository's" { var buffers: Buffers = .{}; try testing.expectEqual(max_limit, (try parseFilter("limit=1000", &buffers)).limit); try testing.expectEqual(@as(u32, 1000), queries_repo.max_limit); } fn openLog() !db.Db { var database = try db.Db.open(":memory:", .{ .mode = .memory }); errdefer database.close(); try db.applyPragmas(&database, .{}); try database.exec(querylog_schema.ddl); return database; } fn seed(database: *db.Db, count: usize) !void { var writer = try queries_repo.BatchWriter.init(database); defer writer.deinit(); var rows: [16]queries_repo.Row = undefined; for (rows[0..count], 0..) |*row, i| { row.* = .{ .timestamp = 1_700_000_000 + @as(i64, @intCast(i)), .domain = if (i % 2 == 0) "ads.example" else "safe.example", .client_ip = "192.0.2.10", .qtype = 1, .blocked = i % 2 == 0, .block_reason = if (i % 2 == 0) "blocklist_domain" else null, .response_time_us = 500, .cache_hit = false, .upstream = null, }; } try writer.writeBatch(rows[0..count]); } test "a full page carries a cursor and the last page does not" { var database = try openLog(); defer database.close(); try seed(&database, 5); var arena: std.heap.ArenaAllocator = .init(testing.allocator); defer arena.deinit(); const first = try page(&database, arena.allocator(), .{ .limit = 2 }); try testing.expectEqual(@as(usize, 2), first.queries.len); try testing.expectEqual(first.queries[1].id, first.next_before.?); // Newest first. try testing.expect(first.queries[0].id > first.queries[1].id); const second = try page(&database, arena.allocator(), .{ .limit = 2, .before = first.next_before }); try testing.expect(second.queries[0].id < first.queries[1].id); const third = try page(&database, arena.allocator(), .{ .limit = 2, .before = second.next_before }); try testing.expectEqual(@as(usize, 1), third.queries.len); try testing.expectEqual(@as(?i64, null), third.next_before); } test "an empty result is a page with no cursor" { var database = try openLog(); defer database.close(); var arena: std.heap.ArenaAllocator = .init(testing.allocator); defer arena.deinit(); const empty = try page(&database, arena.allocator(), .{}); try testing.expectEqual(@as(usize, 0), empty.queries.len); try testing.expectEqual(@as(?i64, null), empty.next_before); } test "the parsed filters narrow the rows the page returns" { var database = try openLog(); defer database.close(); try seed(&database, 6); var arena: std.heap.ArenaAllocator = .init(testing.allocator); defer arena.deinit(); var buffers: Buffers = .{}; const blocked = try page( &database, arena.allocator(), try parseFilter("blocked=true", &buffers), ); try testing.expectEqual(@as(usize, 3), blocked.queries.len); for (blocked.queries) |row| try testing.expect(row.blocked); const by_domain = try page( &database, arena.allocator(), try parseFilter("domain=safe", &buffers), ); try testing.expectEqual(@as(usize, 3), by_domain.queries.len); for (by_domain.queries) |row| try testing.expectEqualStrings("safe.example", row.domain); const nobody = try page( &database, arena.allocator(), try parseFilter("client=198.51.100.1", &buffers), ); try testing.expectEqual(@as(usize, 0), nobody.queries.len); } test "the page serializes as the envelope ruling 11 defines" { var database = try openLog(); defer database.close(); try seed(&database, 1); var arena: std.heap.ArenaAllocator = .init(testing.allocator); defer arena.deinit(); const result = try page(&database, arena.allocator(), .{ .limit = 100 }); var allocating: std.Io.Writer.Allocating = .init(testing.allocator); defer allocating.deinit(); try std.json.Stringify.value(result, .{}, &allocating.writer); const text = allocating.written(); try testing.expect(std.mem.startsWith(u8, text, "{\"queries\":[")); try testing.expect(std.mem.endsWith(u8, text, "\"next_before\":null}")); for ([_][]const u8{ "\"id\":", "\"ts\":", "\"domain\":", "\"client_ip\":", "\"qtype\":", "\"blocked\":", "\"cache_hit\":", "\"upstream\":", "\"upstream\":", "\"response_time_us\":", "\"block_reason\":", }) |field| { try testing.expect(std.mem.containsAtLeast(u8, text, 1, field)); } // W1's ruling: a NULL column reads as "", and "" stays "" on the wire. try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"upstream\":\"\"")); }