milestone 8: web server, rest api, sse, auth, metrics and static assets
This commit is contained in:
@@ -0,0 +1,325 @@
|
||||
//! `GET /api/queries` — the query log, newest first (ruling 11).
|
||||
//!
|
||||
//! Keyset pagination rather than an offset: the table is append-only and the
|
||||
//! UI reads the head of it, so `id < before` is one index seek no matter how
|
||||
//! deep the client has scrolled, and rows arriving between two pages cannot
|
||||
//! shift the window and duplicate a row.
|
||||
//!
|
||||
//! Filter parsing is separated from fetching, because parsing is where the
|
||||
//! input validation of PLAN §19 lives and it is worth testing on its own. Every
|
||||
//! value is length-capped here and bound as a SQL parameter by the repository;
|
||||
//! nothing this file reads is ever concatenated into a statement.
|
||||
|
||||
const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
const db = @import("../../storage/db.zig");
|
||||
const http_util = @import("../http_util.zig");
|
||||
const queries_repo = @import("../../storage/repositories/queries_repo.zig");
|
||||
const server = @import("../server.zig");
|
||||
|
||||
const log = std.log.scoped(.web_queries);
|
||||
|
||||
pub const default_limit: u32 = 100;
|
||||
pub const max_limit: u32 = queries_repo.max_limit;
|
||||
|
||||
/// A domain filter longer than the longest legal domain name matches nothing.
|
||||
pub const max_domain_len = 253;
|
||||
|
||||
/// Long enough for an IPv6 address with a zone identifier.
|
||||
pub const max_client_len = 64;
|
||||
|
||||
/// Where the two string filters are copied to. The parsed filter borrows them,
|
||||
/// so it must not outlive the buffers — in the handler both live in the same
|
||||
/// stack frame.
|
||||
pub const Buffers = struct {
|
||||
domain: [max_domain_len]u8 = undefined,
|
||||
client: [max_client_len]u8 = undefined,
|
||||
};
|
||||
|
||||
pub const Page = struct {
|
||||
queries: []const queries_repo.QueryRow,
|
||||
/// The cursor for the next page, or null when this page is the last one.
|
||||
next_before: ?i64,
|
||||
};
|
||||
|
||||
pub const FilterError = error{
|
||||
BadLimit,
|
||||
BadBefore,
|
||||
BadDomain,
|
||||
BadClient,
|
||||
BadBlocked,
|
||||
BadSince,
|
||||
BadUntil,
|
||||
};
|
||||
|
||||
/// Ruling 11's query string. An absent parameter drops the filter; a malformed
|
||||
/// one is a 400 rather than a filter silently left off, which would answer a
|
||||
/// question the client did not ask.
|
||||
pub fn parseFilter(query: []const u8, buffers: *Buffers) FilterError!queries_repo.QueryFilter {
|
||||
var filter: queries_repo.QueryFilter = .{};
|
||||
|
||||
if (http_util.queryInt(u32, query, "limit") catch return error.BadLimit) |limit| {
|
||||
if (limit == 0 or limit > max_limit) return error.BadLimit;
|
||||
filter.limit = limit;
|
||||
}
|
||||
|
||||
if (http_util.queryInt(i64, query, "before") catch return error.BadBefore) |before| {
|
||||
// Row ids are positive, so a non-positive cursor is a client bug, not
|
||||
// an empty page.
|
||||
if (before <= 0) return error.BadBefore;
|
||||
filter.before = before;
|
||||
}
|
||||
|
||||
if (http_util.queryValue(query, "domain", &buffers.domain) catch return error.BadDomain) |domain| {
|
||||
if (domain.len != 0) filter.domain_substring = domain;
|
||||
}
|
||||
|
||||
if (http_util.queryValue(query, "client", &buffers.client) catch return error.BadClient) |client| {
|
||||
if (client.len != 0) filter.client = client;
|
||||
}
|
||||
|
||||
filter.blocked = http_util.queryBool(query, "blocked") catch return error.BadBlocked;
|
||||
filter.since = http_util.queryInt(i64, query, "since") catch return error.BadSince;
|
||||
filter.until = http_util.queryInt(i64, query, "until") catch return error.BadUntil;
|
||||
|
||||
return filter;
|
||||
}
|
||||
|
||||
pub fn message(err: FilterError) []const u8 {
|
||||
return switch (err) {
|
||||
error.BadLimit => "limit must be between 1 and 1000",
|
||||
error.BadBefore => "before must be a positive row id",
|
||||
error.BadDomain => "domain is not a valid filter",
|
||||
error.BadClient => "client is not a valid filter",
|
||||
error.BadBlocked => "blocked must be true or false",
|
||||
error.BadSince => "since must be a unix timestamp in seconds",
|
||||
error.BadUntil => "until must be a unix timestamp in seconds",
|
||||
};
|
||||
}
|
||||
|
||||
/// A full page carries a cursor and a short one does not: a client stops when
|
||||
/// `next_before` is null, without a count query telling it how many rows exist.
|
||||
pub fn page(
|
||||
database: *db.Db,
|
||||
arena: Allocator,
|
||||
filter: queries_repo.QueryFilter,
|
||||
) db.Error!Page {
|
||||
const rows = try queries_repo.selectQueries(database, arena, filter);
|
||||
const full = rows.items.len == @min(filter.limit, max_limit);
|
||||
return .{
|
||||
.queries = rows.items,
|
||||
.next_before = if (full and rows.items.len != 0) rows.items[rows.items.len - 1].id else null,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn list(
|
||||
state: *server.WebState,
|
||||
io: std.Io,
|
||||
request: *http_util.Request,
|
||||
) http_util.HandlerError!void {
|
||||
_ = io;
|
||||
|
||||
var buffers: Buffers = .{};
|
||||
const filter = parseFilter(request.query, &buffers) catch |err| {
|
||||
return http_util.respondError(request, .bad_request, message(err));
|
||||
};
|
||||
|
||||
const database = state.querylog_db orelse
|
||||
return http_util.respondError(request, .service_unavailable, "query log unavailable");
|
||||
|
||||
const result = page(database, request.arena, filter) catch |err| {
|
||||
// The one thing this handler logs: a database fault is a property of
|
||||
// the box, not of the request, and the client is told nothing about it.
|
||||
log.warn("query log read failed: {s}", .{@errorName(err)});
|
||||
return http_util.respondError(request, .internal_server_error, "internal error");
|
||||
};
|
||||
|
||||
return http_util.respondJson(request, .ok, result, &.{});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const querylog_schema = @import("../../storage/querylog_schema.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
test "an empty query string is the default page" {
|
||||
var buffers: Buffers = .{};
|
||||
const filter = try parseFilter("", &buffers);
|
||||
try testing.expectEqual(default_limit, filter.limit);
|
||||
try testing.expectEqual(@as(?i64, null), filter.before);
|
||||
try testing.expectEqual(@as(?[]const u8, null), filter.domain_substring);
|
||||
try testing.expectEqual(@as(?bool, null), filter.blocked);
|
||||
}
|
||||
|
||||
test "every filter reaches the repository untouched" {
|
||||
var buffers: Buffers = .{};
|
||||
const filter = try parseFilter(
|
||||
"limit=250&before=900&domain=ads.example&client=192.0.2.10&blocked=true&since=100&until=200",
|
||||
&buffers,
|
||||
);
|
||||
try testing.expectEqual(@as(u32, 250), filter.limit);
|
||||
try testing.expectEqual(@as(?i64, 900), filter.before);
|
||||
try testing.expectEqualStrings("ads.example", filter.domain_substring.?);
|
||||
try testing.expectEqualStrings("192.0.2.10", filter.client.?);
|
||||
try testing.expectEqual(@as(?bool, true), filter.blocked);
|
||||
try testing.expectEqual(@as(?i64, 100), filter.since);
|
||||
try testing.expectEqual(@as(?i64, 200), filter.until);
|
||||
}
|
||||
|
||||
test "an empty string filter is no filter at all" {
|
||||
var buffers: Buffers = .{};
|
||||
const filter = try parseFilter("domain=&client=", &buffers);
|
||||
try testing.expectEqual(@as(?[]const u8, null), filter.domain_substring);
|
||||
try testing.expectEqual(@as(?[]const u8, null), filter.client);
|
||||
}
|
||||
|
||||
test "each malformed parameter names itself in a 400" {
|
||||
var buffers: Buffers = .{};
|
||||
try testing.expectError(error.BadLimit, parseFilter("limit=0", &buffers));
|
||||
try testing.expectError(error.BadLimit, parseFilter("limit=1001", &buffers));
|
||||
try testing.expectError(error.BadLimit, parseFilter("limit=ten", &buffers));
|
||||
try testing.expectError(error.BadBefore, parseFilter("before=0", &buffers));
|
||||
try testing.expectError(error.BadBefore, parseFilter("before=-4", &buffers));
|
||||
try testing.expectError(error.BadBlocked, parseFilter("blocked=maybe", &buffers));
|
||||
try testing.expectError(error.BadSince, parseFilter("since=yesterday", &buffers));
|
||||
try testing.expectError(error.BadUntil, parseFilter("until=", &buffers));
|
||||
try testing.expectError(error.BadDomain, parseFilter("domain=%zz", &buffers));
|
||||
|
||||
var long: [max_domain_len + 8]u8 = @splat('a');
|
||||
var text: std.ArrayList(u8) = .empty;
|
||||
defer text.deinit(testing.allocator);
|
||||
try text.appendSlice(testing.allocator, "domain=");
|
||||
try text.appendSlice(testing.allocator, &long);
|
||||
try testing.expectError(error.BadDomain, parseFilter(text.items, &buffers));
|
||||
}
|
||||
|
||||
test "the limit cap is the repository's" {
|
||||
var buffers: Buffers = .{};
|
||||
try testing.expectEqual(max_limit, (try parseFilter("limit=1000", &buffers)).limit);
|
||||
try testing.expectEqual(@as(u32, 1000), queries_repo.max_limit);
|
||||
}
|
||||
|
||||
fn openLog() !db.Db {
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
errdefer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
try database.exec(querylog_schema.ddl);
|
||||
return database;
|
||||
}
|
||||
|
||||
fn seed(database: *db.Db, count: usize) !void {
|
||||
var writer = try queries_repo.BatchWriter.init(database);
|
||||
defer writer.deinit();
|
||||
var rows: [16]queries_repo.Row = undefined;
|
||||
for (rows[0..count], 0..) |*row, i| {
|
||||
row.* = .{
|
||||
.timestamp = 1_700_000_000 + @as(i64, @intCast(i)),
|
||||
.domain = if (i % 2 == 0) "ads.example" else "safe.example",
|
||||
.client_ip = "192.0.2.10",
|
||||
.qtype = 1,
|
||||
.blocked = i % 2 == 0,
|
||||
.block_reason = if (i % 2 == 0) "blocklist_domain" else null,
|
||||
.response_time_us = 500,
|
||||
.cache_hit = false,
|
||||
.upstream = null,
|
||||
};
|
||||
}
|
||||
try writer.writeBatch(rows[0..count]);
|
||||
}
|
||||
|
||||
test "a full page carries a cursor and the last page does not" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
try seed(&database, 5);
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const first = try page(&database, arena.allocator(), .{ .limit = 2 });
|
||||
try testing.expectEqual(@as(usize, 2), first.queries.len);
|
||||
try testing.expectEqual(first.queries[1].id, first.next_before.?);
|
||||
// Newest first.
|
||||
try testing.expect(first.queries[0].id > first.queries[1].id);
|
||||
|
||||
const second = try page(&database, arena.allocator(), .{ .limit = 2, .before = first.next_before });
|
||||
try testing.expect(second.queries[0].id < first.queries[1].id);
|
||||
|
||||
const third = try page(&database, arena.allocator(), .{ .limit = 2, .before = second.next_before });
|
||||
try testing.expectEqual(@as(usize, 1), third.queries.len);
|
||||
try testing.expectEqual(@as(?i64, null), third.next_before);
|
||||
}
|
||||
|
||||
test "an empty result is a page with no cursor" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const empty = try page(&database, arena.allocator(), .{});
|
||||
try testing.expectEqual(@as(usize, 0), empty.queries.len);
|
||||
try testing.expectEqual(@as(?i64, null), empty.next_before);
|
||||
}
|
||||
|
||||
test "the parsed filters narrow the rows the page returns" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
try seed(&database, 6);
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
var buffers: Buffers = .{};
|
||||
const blocked = try page(
|
||||
&database,
|
||||
arena.allocator(),
|
||||
try parseFilter("blocked=true", &buffers),
|
||||
);
|
||||
try testing.expectEqual(@as(usize, 3), blocked.queries.len);
|
||||
for (blocked.queries) |row| try testing.expect(row.blocked);
|
||||
|
||||
const by_domain = try page(
|
||||
&database,
|
||||
arena.allocator(),
|
||||
try parseFilter("domain=safe", &buffers),
|
||||
);
|
||||
try testing.expectEqual(@as(usize, 3), by_domain.queries.len);
|
||||
for (by_domain.queries) |row| try testing.expectEqualStrings("safe.example", row.domain);
|
||||
|
||||
const nobody = try page(
|
||||
&database,
|
||||
arena.allocator(),
|
||||
try parseFilter("client=198.51.100.1", &buffers),
|
||||
);
|
||||
try testing.expectEqual(@as(usize, 0), nobody.queries.len);
|
||||
}
|
||||
|
||||
test "the page serializes as the envelope ruling 11 defines" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
try seed(&database, 1);
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const result = try page(&database, arena.allocator(), .{ .limit = 100 });
|
||||
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer allocating.deinit();
|
||||
try std.json.Stringify.value(result, .{}, &allocating.writer);
|
||||
const text = allocating.written();
|
||||
|
||||
try testing.expect(std.mem.startsWith(u8, text, "{\"queries\":["));
|
||||
try testing.expect(std.mem.endsWith(u8, text, "\"next_before\":null}"));
|
||||
for ([_][]const u8{
|
||||
"\"id\":", "\"ts\":", "\"domain\":", "\"client_ip\":",
|
||||
"\"qtype\":", "\"blocked\":", "\"cache_hit\":", "\"upstream\":",
|
||||
"\"upstream\":", "\"response_time_us\":", "\"block_reason\":",
|
||||
}) |field| {
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, field));
|
||||
}
|
||||
// W1's ruling: a NULL column reads as "", and "" stays "" on the wire.
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "\"upstream\":\"\""));
|
||||
}
|
||||
Reference in New Issue
Block a user