milestone 8: web server, rest api, sse, auth, metrics and static assets
This commit is contained in:
@@ -2,8 +2,9 @@
|
||||
//!
|
||||
//! Two shapes live here. The free functions follow the milestone-4 repository
|
||||
//! idiom — prepare, use, finalize — because retention runs them a handful of
|
||||
//! times per day. The flush loop is the one hot path in the program, so it gets
|
||||
//! `BatchWriter`, which owns its three statements for its whole life
|
||||
//! times per day, and the API read layer at the bottom of the file runs once
|
||||
//! per HTTP request. The flush loop is the one hot path in the program, so it
|
||||
//! gets `BatchWriter`, which owns its three statements for its whole life
|
||||
//! (`db.zig:360` names this file as the reason `db.zig` carries no statement
|
||||
//! cache).
|
||||
//!
|
||||
@@ -15,6 +16,7 @@
|
||||
//! decides what a failed batch means.
|
||||
|
||||
const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
const db = @import("../db.zig");
|
||||
|
||||
@@ -181,6 +183,294 @@ pub fn countDomains(database: *db.Db) db.Error!i64 {
|
||||
return database.queryInt("SELECT count(*) FROM domains");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// the API read layer (`GET /api/queries`, `/api/stats`, `/api/stats/timeseries`)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One row of `GET /api/queries`, joined back through the `domains` dimension.
|
||||
///
|
||||
/// `block_reason` and `upstream` are nullable columns, and a NULL reads as `""`
|
||||
/// — the same convention `Stmt.columnText` already uses. Neither column is ever
|
||||
/// written as an empty string (a reason is a word, an upstream is a URL), so the
|
||||
/// mapping loses nothing and the API layer can treat `""` as "absent".
|
||||
pub const QueryRow = struct {
|
||||
id: i64,
|
||||
ts: i64,
|
||||
domain: []const u8,
|
||||
client_ip: []const u8,
|
||||
qtype: ?u16,
|
||||
blocked: bool,
|
||||
block_reason: []const u8,
|
||||
response_time_us: ?i64,
|
||||
cache_hit: ?bool,
|
||||
upstream: []const u8,
|
||||
};
|
||||
|
||||
/// Every field is an independent narrowing; `null` means "do not filter on it".
|
||||
///
|
||||
/// `since` is inclusive and `until` is exclusive, so adjacent windows tile
|
||||
/// without double-counting a row on the boundary.
|
||||
pub const QueryFilter = struct {
|
||||
limit: u32 = 100,
|
||||
/// Keyset cursor: only rows with a strictly smaller `id`. Rows come back
|
||||
/// newest-first, so this is the id of the last row of the previous page.
|
||||
before: ?i64 = null,
|
||||
/// Matched case-insensitively for ASCII, which is what SQLite's `LIKE`
|
||||
/// does and what a domain search wants.
|
||||
domain_substring: ?[]const u8 = null,
|
||||
client: ?[]const u8 = null,
|
||||
blocked: ?bool = null,
|
||||
since: ?i64 = null,
|
||||
until: ?i64 = null,
|
||||
};
|
||||
|
||||
/// Ruling 11 caps the page at 1000; the repository enforces it too, so a caller
|
||||
/// that forgets cannot ask this connection for the whole table.
|
||||
pub const max_limit: u32 = 1000;
|
||||
|
||||
const select_head =
|
||||
\\SELECT q.id, q.timestamp, d.domain, q.client_ip, q.qtype, q.blocked,
|
||||
\\ q.block_reason, q.response_time_us, q.cache_hit, q.upstream
|
||||
\\ FROM query_log q JOIN domains d ON d.id = q.domain_id
|
||||
;
|
||||
|
||||
/// The escape character of `where_domain`. SQLite does not give string literals
|
||||
/// C escapes, so `'\'` in the SQL text is one backslash.
|
||||
const like_escape = '\\';
|
||||
|
||||
const where_before = " q.id < ?";
|
||||
const where_domain = " d.domain LIKE ? ESCAPE '\\'";
|
||||
const where_client = " q.client_ip = ?";
|
||||
const where_blocked = " q.blocked = ?";
|
||||
const where_since = " q.timestamp >= ?";
|
||||
const where_until = " q.timestamp < ?";
|
||||
const select_tail = " ORDER BY q.id DESC LIMIT ?";
|
||||
|
||||
const where_keyword = " WHERE";
|
||||
const and_keyword = " AND";
|
||||
|
||||
/// Assembles the statement from the fixed fragments above and nothing else.
|
||||
///
|
||||
/// **No value ever reaches this buffer.** Every filter contributes a `?` and is
|
||||
/// bound afterwards, in the order the predicates were appended: an unnumbered
|
||||
/// parameter takes the next free index, so append order and bind order are the
|
||||
/// same single contract.
|
||||
const Sql = struct {
|
||||
/// `where_keyword` is longer than `and_keyword` and is used at most once,
|
||||
/// so counting six of it bounds every reachable combination.
|
||||
const capacity = select_head.len + 6 * where_keyword.len + select_tail.len +
|
||||
where_before.len + where_domain.len + where_client.len +
|
||||
where_blocked.len + where_since.len + where_until.len;
|
||||
|
||||
buf: [capacity]u8 = undefined,
|
||||
len: usize = 0,
|
||||
has_where: bool = false,
|
||||
|
||||
fn put(self: *Sql, fragment: []const u8) void {
|
||||
@memcpy(self.buf[self.len..][0..fragment.len], fragment);
|
||||
self.len += fragment.len;
|
||||
}
|
||||
|
||||
fn predicate(self: *Sql, fragment: []const u8) void {
|
||||
self.put(if (self.has_where) and_keyword else where_keyword);
|
||||
self.has_where = true;
|
||||
self.put(fragment);
|
||||
}
|
||||
|
||||
fn text(self: *const Sql) []const u8 {
|
||||
return self.buf[0..self.len];
|
||||
}
|
||||
};
|
||||
|
||||
/// Rows come back newest-first (`id DESC`). Every string is allocated from
|
||||
/// `arena`, including the list's own storage, so the caller frees the whole
|
||||
/// result by resetting the arena — there is nothing to unwind on failure.
|
||||
pub fn selectQueries(database: *db.Db, arena: Allocator, filter: QueryFilter) db.Error!std.ArrayList(QueryRow) {
|
||||
var sql: Sql = .{};
|
||||
sql.put(select_head);
|
||||
if (filter.before != null) sql.predicate(where_before);
|
||||
if (filter.domain_substring != null) sql.predicate(where_domain);
|
||||
if (filter.client != null) sql.predicate(where_client);
|
||||
if (filter.blocked != null) sql.predicate(where_blocked);
|
||||
if (filter.since != null) sql.predicate(where_since);
|
||||
if (filter.until != null) sql.predicate(where_until);
|
||||
sql.put(select_tail);
|
||||
|
||||
var stmt = try database.prepare(sql.text());
|
||||
defer stmt.deinit();
|
||||
|
||||
var idx: c_int = 0;
|
||||
if (filter.before) |v| {
|
||||
idx += 1;
|
||||
try stmt.bindInt(idx, v);
|
||||
}
|
||||
if (filter.domain_substring) |v| {
|
||||
idx += 1;
|
||||
try stmt.bindText(idx, try likePattern(arena, v));
|
||||
}
|
||||
if (filter.client) |v| {
|
||||
idx += 1;
|
||||
try stmt.bindText(idx, v);
|
||||
}
|
||||
if (filter.blocked) |v| {
|
||||
idx += 1;
|
||||
try stmt.bindBool(idx, v);
|
||||
}
|
||||
if (filter.since) |v| {
|
||||
idx += 1;
|
||||
try stmt.bindInt(idx, v);
|
||||
}
|
||||
if (filter.until) |v| {
|
||||
idx += 1;
|
||||
try stmt.bindInt(idx, v);
|
||||
}
|
||||
idx += 1;
|
||||
try stmt.bindInt(idx, @min(filter.limit, max_limit));
|
||||
|
||||
var out: std.ArrayList(QueryRow) = .empty;
|
||||
while (try stmt.step()) {
|
||||
try out.append(arena, .{
|
||||
.id = stmt.columnInt(0),
|
||||
.ts = stmt.columnInt(1),
|
||||
.domain = try stmt.columnTextAlloc(arena, 2),
|
||||
.client_ip = try stmt.columnTextAlloc(arena, 3),
|
||||
.qtype = if (stmt.isNull(4)) null else std.math.cast(u16, stmt.columnInt(4)) orelse
|
||||
return error.Mismatch,
|
||||
.blocked = stmt.columnBool(5),
|
||||
.block_reason = try stmt.columnTextAlloc(arena, 6),
|
||||
.response_time_us = if (stmt.isNull(7)) null else stmt.columnInt(7),
|
||||
.cache_hit = if (stmt.isNull(8)) null else stmt.columnBool(8),
|
||||
.upstream = try stmt.columnTextAlloc(arena, 9),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/// Wraps `needle` in `%` and neutralises the two `LIKE` metacharacters, so a
|
||||
/// user searching for `a_b` gets domains containing `a_b` and not domains
|
||||
/// containing `axb`. The escape character escapes itself.
|
||||
fn likePattern(arena: Allocator, needle: []const u8) Allocator.Error![]const u8 {
|
||||
var out: std.ArrayList(u8) = try .initCapacity(arena, needle.len * 2 + 2);
|
||||
out.appendAssumeCapacity('%');
|
||||
for (needle) |ch| {
|
||||
if (ch == '%' or ch == '_' or ch == like_escape) out.appendAssumeCapacity(like_escape);
|
||||
out.appendAssumeCapacity(ch);
|
||||
}
|
||||
out.appendAssumeCapacity('%');
|
||||
return out.items;
|
||||
}
|
||||
|
||||
/// The `/api/stats` rollup for one period. `avg_response_time_us` is `null` when
|
||||
/// no row in the window recorded a response time.
|
||||
pub const StatsTotals = struct {
|
||||
queries: u64,
|
||||
blocked: u64,
|
||||
cached: u64,
|
||||
distinct_clients: u64,
|
||||
avg_response_time_us: ?i64,
|
||||
};
|
||||
|
||||
/// The mean is derived from a sum and a count rather than SQL's `avg`, which
|
||||
/// returns REAL: `Stmt` reads integers, and integer microseconds are exact.
|
||||
const stats_totals_sql =
|
||||
\\SELECT count(*),
|
||||
\\ coalesce(sum(blocked <> 0), 0),
|
||||
\\ coalesce(sum(cache_hit = 1), 0),
|
||||
\\ count(DISTINCT client_ip),
|
||||
\\ coalesce(sum(response_time_us), 0),
|
||||
\\ count(response_time_us)
|
||||
\\ FROM query_log
|
||||
\\ WHERE timestamp >= ?1 AND timestamp < ?2
|
||||
;
|
||||
|
||||
/// Aggregates `[since, until)`. An empty window is all zeros with a null mean,
|
||||
/// not an error.
|
||||
pub fn statsTotals(database: *db.Db, since: i64, until: i64) db.Error!StatsTotals {
|
||||
var stmt = try database.prepare(stats_totals_sql);
|
||||
defer stmt.deinit();
|
||||
try stmt.bindInt(1, since);
|
||||
try stmt.bindInt(2, until);
|
||||
|
||||
// A bare aggregate always produces exactly one row; no row means the
|
||||
// statement is not the one this function prepared.
|
||||
if (!try stmt.step()) return error.Misuse;
|
||||
|
||||
const timed = stmt.columnInt(5);
|
||||
return .{
|
||||
.queries = try countOf(stmt.columnInt(0)),
|
||||
.blocked = try countOf(stmt.columnInt(1)),
|
||||
.cached = try countOf(stmt.columnInt(2)),
|
||||
.distinct_clients = try countOf(stmt.columnInt(3)),
|
||||
.avg_response_time_us = if (timed == 0) null else @divTrunc(stmt.columnInt(4), timed),
|
||||
};
|
||||
}
|
||||
|
||||
/// `count` and `sum` over non-negative columns cannot go negative; a negative
|
||||
/// value means the row came from something other than this schema.
|
||||
fn countOf(value: i64) db.Error!u64 {
|
||||
if (value < 0) return error.Mismatch;
|
||||
return @intCast(value);
|
||||
}
|
||||
|
||||
/// One bucket of `/api/stats/timeseries`. `ts` is the bucket's inclusive start.
|
||||
pub const Bucket = struct {
|
||||
ts: i64,
|
||||
queries: u64,
|
||||
blocked: u64,
|
||||
cached: u64,
|
||||
};
|
||||
|
||||
const timeseries_sql =
|
||||
\\SELECT (timestamp - ?1) / ?2,
|
||||
\\ count(*),
|
||||
\\ coalesce(sum(blocked <> 0), 0),
|
||||
\\ coalesce(sum(cache_hit = 1), 0)
|
||||
\\ FROM query_log
|
||||
\\ WHERE timestamp >= ?1 AND timestamp < ?3
|
||||
\\ GROUP BY 1
|
||||
;
|
||||
|
||||
/// Fills `out` with `out.len` buckets of `bucket_seconds` each, covering
|
||||
/// `[since, since + bucket_seconds * out.len)`, and returns how many it wrote.
|
||||
///
|
||||
/// Every bucket is present: a window with no rows in it is written with zeros
|
||||
/// rather than skipped, so the caller charts a contiguous axis without
|
||||
/// reconstructing the gaps. Buckets are aligned to `since`, so the caller —
|
||||
/// which knows the period grammar of ruling 13 — owns UTC alignment by choosing
|
||||
/// `since`.
|
||||
pub fn timeseries(database: *db.Db, since: i64, bucket_seconds: u32, out: []Bucket) db.Error!usize {
|
||||
if (out.len == 0) return 0;
|
||||
// Both are caller bugs, not runtime conditions: a zero width would make the
|
||||
// SQL divide by zero (SQLite yields NULL, silently emptying the chart), and
|
||||
// a window that does not fit i64 cannot be asked about.
|
||||
if (bucket_seconds == 0) return error.Misuse;
|
||||
const width: i64 = bucket_seconds;
|
||||
const span = std.math.mul(i64, width, std.math.cast(i64, out.len) orelse
|
||||
return error.Misuse) catch return error.Misuse;
|
||||
const until = std.math.add(i64, since, span) catch return error.Misuse;
|
||||
|
||||
for (out, 0..) |*bucket, i| {
|
||||
bucket.* = .{ .ts = since + width * @as(i64, @intCast(i)), .queries = 0, .blocked = 0, .cached = 0 };
|
||||
}
|
||||
|
||||
var stmt = try database.prepare(timeseries_sql);
|
||||
defer stmt.deinit();
|
||||
try stmt.bindInt(1, since);
|
||||
try stmt.bindInt(2, width);
|
||||
try stmt.bindInt(3, until);
|
||||
|
||||
while (try stmt.step()) {
|
||||
// The WHERE clause already bounds the index to `out`; the check is
|
||||
// cheap and keeps a schema surprise from writing past the slice.
|
||||
const index = std.math.cast(usize, stmt.columnInt(0)) orelse return error.Mismatch;
|
||||
if (index >= out.len) return error.Mismatch;
|
||||
out[index].queries = try countOf(stmt.columnInt(1));
|
||||
out[index].blocked = try countOf(stmt.columnInt(2));
|
||||
out[index].cached = try countOf(stmt.columnInt(3));
|
||||
}
|
||||
return out.len;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -466,3 +756,379 @@ test "checkpointTruncate and vacuum run against a WAL file database" {
|
||||
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
||||
try testing.expectEqual(@as(i64, 2), try countDomains(&database));
|
||||
}
|
||||
|
||||
// --- the read layer -------------------------------------------------------
|
||||
|
||||
/// `BatchWriter` assigns `query_log.id` in the order it is handed the rows, so
|
||||
/// every test below knows the id of each seeded row: the nth row of the nth
|
||||
/// batch has id n.
|
||||
fn seed(database: *db.Db, rows: []const Row) !void {
|
||||
var writer = try BatchWriter.init(database);
|
||||
defer writer.deinit();
|
||||
try writer.writeBatch(rows);
|
||||
}
|
||||
|
||||
fn ids(rows: []const QueryRow, out: []i64) []const i64 {
|
||||
for (rows, 0..) |row, i| out[i] = row.id;
|
||||
return out[0..rows.len];
|
||||
}
|
||||
|
||||
test "selectQueries returns the newest row first and reads every column" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
|
||||
try seed(&database, &.{
|
||||
.{
|
||||
.timestamp = 10,
|
||||
.domain = "ads.example.net",
|
||||
.client_ip = "192.0.2.10",
|
||||
.qtype = 28,
|
||||
.blocked = true,
|
||||
.block_reason = "blocklist",
|
||||
.response_time_us = 4200,
|
||||
.cache_hit = true,
|
||||
.upstream = "https://dns.example/dns-query",
|
||||
},
|
||||
.{
|
||||
.timestamp = 20,
|
||||
.domain = "quiet.example",
|
||||
.client_ip = "hidden",
|
||||
.qtype = null,
|
||||
.blocked = false,
|
||||
.block_reason = null,
|
||||
.response_time_us = null,
|
||||
.cache_hit = null,
|
||||
.upstream = null,
|
||||
},
|
||||
});
|
||||
|
||||
const rows = try selectQueries(&database, arena_state.allocator(), .{});
|
||||
try testing.expectEqual(@as(usize, 2), rows.items.len);
|
||||
|
||||
const newest = rows.items[0];
|
||||
try testing.expectEqual(@as(i64, 2), newest.id);
|
||||
try testing.expectEqual(@as(i64, 20), newest.ts);
|
||||
try testing.expectEqualStrings("quiet.example", newest.domain);
|
||||
try testing.expectEqualStrings("hidden", newest.client_ip);
|
||||
try testing.expectEqual(@as(?u16, null), newest.qtype);
|
||||
try testing.expect(!newest.blocked);
|
||||
// A NULL text column reads as the empty string, by documented convention.
|
||||
try testing.expectEqualStrings("", newest.block_reason);
|
||||
try testing.expectEqual(@as(?i64, null), newest.response_time_us);
|
||||
try testing.expectEqual(@as(?bool, null), newest.cache_hit);
|
||||
try testing.expectEqualStrings("", newest.upstream);
|
||||
|
||||
const oldest = rows.items[1];
|
||||
try testing.expectEqual(@as(i64, 1), oldest.id);
|
||||
try testing.expectEqual(@as(i64, 10), oldest.ts);
|
||||
try testing.expectEqualStrings("ads.example.net", oldest.domain);
|
||||
try testing.expectEqualStrings("192.0.2.10", oldest.client_ip);
|
||||
try testing.expectEqual(@as(?u16, 28), oldest.qtype);
|
||||
try testing.expect(oldest.blocked);
|
||||
try testing.expectEqualStrings("blocklist", oldest.block_reason);
|
||||
try testing.expectEqual(@as(?i64, 4200), oldest.response_time_us);
|
||||
try testing.expectEqual(@as(?bool, true), oldest.cache_hit);
|
||||
try testing.expectEqualStrings("https://dns.example/dns-query", oldest.upstream);
|
||||
}
|
||||
|
||||
test "selectQueries honours the limit and caps it at max_limit" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
var rows: [1005]Row = undefined;
|
||||
for (&rows, 0..) |*row, i| row.* = plainRow(@intCast(i), "example.com");
|
||||
try seed(&database, &rows);
|
||||
|
||||
const few = try selectQueries(&database, arena, .{ .limit = 3 });
|
||||
try testing.expectEqual(@as(usize, 3), few.items.len);
|
||||
|
||||
// Asked for more than the cap, and for more rows than the cap, so the cap
|
||||
// is what bounds the answer rather than the table.
|
||||
const capped = try selectQueries(&database, arena, .{ .limit = 5000 });
|
||||
try testing.expectEqual(@as(usize, max_limit), capped.items.len);
|
||||
|
||||
const none = try selectQueries(&database, arena, .{ .limit = 0 });
|
||||
try testing.expectEqual(@as(usize, 0), none.items.len);
|
||||
}
|
||||
|
||||
test "keyset paging walks every row exactly once across the page boundaries" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
var seeded: [7]Row = undefined;
|
||||
for (&seeded, 0..) |*row, i| row.* = plainRow(@intCast(i), "example.com");
|
||||
try seed(&database, &seeded);
|
||||
|
||||
var seen: std.ArrayList(i64) = .empty;
|
||||
defer seen.deinit(testing.allocator);
|
||||
|
||||
var before: ?i64 = null;
|
||||
var pages: usize = 0;
|
||||
while (pages < 10) : (pages += 1) {
|
||||
const page = try selectQueries(&database, arena, .{ .limit = 3, .before = before });
|
||||
if (page.items.len == 0) break;
|
||||
for (page.items) |row| try seen.append(testing.allocator, row.id);
|
||||
before = page.items[page.items.len - 1].id;
|
||||
}
|
||||
|
||||
// Two full pages and one short page; the fourth call returns nothing and
|
||||
// breaks before the counter, which is how the walk knows it is done.
|
||||
try testing.expectEqual(@as(usize, 3), pages);
|
||||
try testing.expectEqualSlices(i64, &.{ 7, 6, 5, 4, 3, 2, 1 }, seen.items);
|
||||
}
|
||||
|
||||
test "each filter narrows the result on its own" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
var blocked_row = plainRow(200, "ads.example.net");
|
||||
blocked_row.client_ip = "192.0.2.20";
|
||||
blocked_row.blocked = true;
|
||||
blocked_row.block_reason = "blocklist";
|
||||
try seed(&database, &.{
|
||||
plainRow(100, "one.example.com"),
|
||||
blocked_row,
|
||||
plainRow(300, "two.example.com"),
|
||||
});
|
||||
|
||||
var buf: [8]i64 = undefined;
|
||||
|
||||
const by_domain = try selectQueries(&database, arena, .{ .domain_substring = "example.com" });
|
||||
try testing.expectEqualSlices(i64, &.{ 3, 1 }, ids(by_domain.items, &buf));
|
||||
|
||||
const by_client = try selectQueries(&database, arena, .{ .client = "192.0.2.20" });
|
||||
try testing.expectEqualSlices(i64, &.{2}, ids(by_client.items, &buf));
|
||||
|
||||
// An exact match, not a prefix: the seeded clients share the first octets.
|
||||
const no_client = try selectQueries(&database, arena, .{ .client = "192.0.2" });
|
||||
try testing.expectEqual(@as(usize, 0), no_client.items.len);
|
||||
|
||||
const only_blocked = try selectQueries(&database, arena, .{ .blocked = true });
|
||||
try testing.expectEqualSlices(i64, &.{2}, ids(only_blocked.items, &buf));
|
||||
|
||||
const only_allowed = try selectQueries(&database, arena, .{ .blocked = false });
|
||||
try testing.expectEqualSlices(i64, &.{ 3, 1 }, ids(only_allowed.items, &buf));
|
||||
|
||||
// Every filter at once, all satisfied by the one blocked row.
|
||||
const combined = try selectQueries(&database, arena, .{
|
||||
.limit = 10,
|
||||
.before = 3,
|
||||
.domain_substring = "ads",
|
||||
.client = "192.0.2.20",
|
||||
.blocked = true,
|
||||
.since = 200,
|
||||
.until = 300,
|
||||
});
|
||||
try testing.expectEqualSlices(i64, &.{2}, ids(combined.items, &buf));
|
||||
}
|
||||
|
||||
test "since is inclusive, until is exclusive, and an empty range selects nothing" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
try seed(&database, &.{
|
||||
plainRow(100, "a.example"),
|
||||
plainRow(200, "b.example"),
|
||||
plainRow(300, "c.example"),
|
||||
});
|
||||
|
||||
var buf: [8]i64 = undefined;
|
||||
|
||||
const window = try selectQueries(&database, arena, .{ .since = 100, .until = 300 });
|
||||
try testing.expectEqualSlices(i64, &.{ 2, 1 }, ids(window.items, &buf));
|
||||
|
||||
const after = try selectQueries(&database, arena, .{ .since = 300 });
|
||||
try testing.expectEqualSlices(i64, &.{3}, ids(after.items, &buf));
|
||||
|
||||
const empty = try selectQueries(&database, arena, .{ .since = 300, .until = 300 });
|
||||
try testing.expectEqual(@as(usize, 0), empty.items.len);
|
||||
|
||||
const beyond = try selectQueries(&database, arena, .{ .since = 1000 });
|
||||
try testing.expectEqual(@as(usize, 0), beyond.items.len);
|
||||
}
|
||||
|
||||
test "a domain substring matches % and _ as literal characters" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
try seed(&database, &.{
|
||||
plainRow(10, "a_b.example"),
|
||||
plainRow(20, "axb.example"),
|
||||
plainRow(30, "a%b.example"),
|
||||
plainRow(40, "azzb.example"),
|
||||
plainRow(50, "back\\slash.example"),
|
||||
});
|
||||
|
||||
var buf: [8]i64 = undefined;
|
||||
|
||||
// Unescaped, `_` is LIKE's single-character wildcard and would also match
|
||||
// "axb"; escaped, it matches only the underscore.
|
||||
const underscore = try selectQueries(&database, arena, .{ .domain_substring = "a_b" });
|
||||
try testing.expectEqualSlices(i64, &.{1}, ids(underscore.items, &buf));
|
||||
|
||||
// Unescaped, `%` would match everything from "a" to "b", so "azzb" too.
|
||||
const percent = try selectQueries(&database, arena, .{ .domain_substring = "a%b" });
|
||||
try testing.expectEqualSlices(i64, &.{3}, ids(percent.items, &buf));
|
||||
|
||||
// The escape character escapes itself, so it is searchable as well.
|
||||
const backslash = try selectQueries(&database, arena, .{ .domain_substring = "k\\s" });
|
||||
try testing.expectEqualSlices(i64, &.{5}, ids(backslash.items, &buf));
|
||||
|
||||
// An empty needle is `%%`, which matches every row rather than none.
|
||||
const all = try selectQueries(&database, arena, .{ .domain_substring = "" });
|
||||
try testing.expectEqual(@as(usize, 5), all.items.len);
|
||||
}
|
||||
|
||||
test "statsTotals aggregates the window and averages only the timed rows" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var timed = plainRow(100, "a.example");
|
||||
timed.response_time_us = 100;
|
||||
var blocked_row = plainRow(150, "ads.example");
|
||||
blocked_row.blocked = true;
|
||||
blocked_row.block_reason = "blocklist";
|
||||
blocked_row.response_time_us = 200;
|
||||
var cached = plainRow(199, "b.example");
|
||||
cached.client_ip = "192.0.2.99";
|
||||
cached.cache_hit = true;
|
||||
cached.response_time_us = null;
|
||||
try seed(&database, &.{ timed, blocked_row, cached, plainRow(200, "outside.example") });
|
||||
|
||||
const totals = try statsTotals(&database, 100, 200);
|
||||
try testing.expectEqual(@as(u64, 3), totals.queries);
|
||||
try testing.expectEqual(@as(u64, 1), totals.blocked);
|
||||
try testing.expectEqual(@as(u64, 1), totals.cached);
|
||||
try testing.expectEqual(@as(u64, 2), totals.distinct_clients);
|
||||
// (100 + 200) / 2 — the untimed row is not in the divisor.
|
||||
try testing.expectEqual(@as(?i64, 150), totals.avg_response_time_us);
|
||||
}
|
||||
|
||||
test "statsTotals over an empty window is zeros with a null average" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
try seed(&database, &.{plainRow(100, "a.example")});
|
||||
|
||||
for ([_][2]i64{ .{ 500, 600 }, .{ 100, 100 } }) |window| {
|
||||
const totals = try statsTotals(&database, window[0], window[1]);
|
||||
try testing.expectEqual(@as(u64, 0), totals.queries);
|
||||
try testing.expectEqual(@as(u64, 0), totals.blocked);
|
||||
try testing.expectEqual(@as(u64, 0), totals.cached);
|
||||
try testing.expectEqual(@as(u64, 0), totals.distinct_clients);
|
||||
try testing.expectEqual(@as(?i64, null), totals.avg_response_time_us);
|
||||
}
|
||||
}
|
||||
|
||||
test "timeseries writes every bucket, including the ones with no rows" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var blocked_row = plainRow(1020, "ads.example");
|
||||
blocked_row.blocked = true;
|
||||
blocked_row.block_reason = "blocklist";
|
||||
var cached = plainRow(1035, "b.example");
|
||||
cached.cache_hit = true;
|
||||
try seed(&database, &.{
|
||||
plainRow(995, "before.example"),
|
||||
plainRow(1000, "a.example"),
|
||||
plainRow(1009, "a.example"),
|
||||
blocked_row,
|
||||
cached,
|
||||
plainRow(1040, "after.example"),
|
||||
});
|
||||
|
||||
var buckets: [4]Bucket = undefined;
|
||||
try testing.expectEqual(@as(usize, 4), try timeseries(&database, 1000, 10, &buckets));
|
||||
|
||||
// The row at 995 is before the window and the row at 1040 is past its end;
|
||||
// neither lands in a bucket.
|
||||
try testing.expectEqualSlices(Bucket, &.{
|
||||
.{ .ts = 1000, .queries = 2, .blocked = 0, .cached = 0 },
|
||||
.{ .ts = 1010, .queries = 0, .blocked = 0, .cached = 0 },
|
||||
.{ .ts = 1020, .queries = 1, .blocked = 1, .cached = 0 },
|
||||
.{ .ts = 1030, .queries = 1, .blocked = 0, .cached = 1 },
|
||||
}, &buckets);
|
||||
}
|
||||
|
||||
test "timeseries over an empty table still writes the whole axis" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var buckets: [3]Bucket = undefined;
|
||||
try testing.expectEqual(@as(usize, 3), try timeseries(&database, 0, 60, &buckets));
|
||||
try testing.expectEqualSlices(Bucket, &.{
|
||||
.{ .ts = 0, .queries = 0, .blocked = 0, .cached = 0 },
|
||||
.{ .ts = 60, .queries = 0, .blocked = 0, .cached = 0 },
|
||||
.{ .ts = 120, .queries = 0, .blocked = 0, .cached = 0 },
|
||||
}, &buckets);
|
||||
}
|
||||
|
||||
test "timeseries rejects a zero-width bucket and accepts an empty slice" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var buckets: [2]Bucket = undefined;
|
||||
try testing.expectError(error.Misuse, timeseries(&database, 0, 0, &buckets));
|
||||
|
||||
var none: [0]Bucket = undefined;
|
||||
try testing.expectEqual(@as(usize, 0), try timeseries(&database, 0, 0, &none));
|
||||
}
|
||||
|
||||
test "timeseries reports a window that does not fit an i64 rather than wrapping" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var buckets: [4]Bucket = undefined;
|
||||
try testing.expectError(
|
||||
error.Misuse,
|
||||
timeseries(&database, std.math.maxInt(i64) - 1, 3600, &buckets),
|
||||
);
|
||||
}
|
||||
|
||||
test "the built SQL never carries a filter value and fits its buffer" {
|
||||
var sql: Sql = .{};
|
||||
sql.put(select_head);
|
||||
sql.predicate(where_before);
|
||||
sql.predicate(where_domain);
|
||||
sql.predicate(where_client);
|
||||
sql.predicate(where_blocked);
|
||||
sql.predicate(where_since);
|
||||
sql.predicate(where_until);
|
||||
sql.put(select_tail);
|
||||
|
||||
// Every predicate present is the longest reachable statement.
|
||||
try testing.expect(sql.len <= Sql.capacity);
|
||||
try testing.expectEqual(@as(usize, 1), std.mem.count(u8, sql.text(), " WHERE"));
|
||||
try testing.expectEqual(@as(usize, 5), std.mem.count(u8, sql.text(), " AND"));
|
||||
// Six filters plus the LIMIT, each a bare parameter.
|
||||
try testing.expectEqual(@as(usize, 7), std.mem.count(u8, sql.text(), "?"));
|
||||
}
|
||||
|
||||
test "likePattern wraps the needle and neutralises every metacharacter" {
|
||||
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
try testing.expectEqualStrings("%plain%", try likePattern(arena, "plain"));
|
||||
try testing.expectEqualStrings("%a\\_b%", try likePattern(arena, "a_b"));
|
||||
try testing.expectEqualStrings("%a\\%b%", try likePattern(arena, "a%b"));
|
||||
try testing.expectEqualStrings("%a\\\\b%", try likePattern(arena, "a\\b"));
|
||||
try testing.expectEqualStrings("%%", try likePattern(arena, ""));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user