mokhtar
b774b05456
the freetype client field is gone. the picker is a select-only menu of known clients with multi-select, chips for the active set, and a 32-client cap shared with the server. the api accepts a comma-separated client list and filters any-of with bound parameters. the trigger carets come from phosphor icons, newly adopted. bundle budget rises to 900000 bytes for the picker and the icon dependency.
3255 lines
136 KiB
Zig
3255 lines
136 KiB
Zig
//! `query_log` and its `domains` dimension table in `querylog.db`.
|
|
//!
|
|
//! Two shapes live here. The free functions follow the milestone-4 repository
|
|
//! idiom — prepare, use, finalize — because retention runs them a handful of
|
|
//! times per day, and the API read layer at the bottom of the file runs once
|
|
//! per HTTP request. The flush loop is the one hot path in the program, so it
|
|
//! gets `BatchWriter`, which owns its three statements for its whole life
|
|
//! (`db.zig:360` names this file as the reason `db.zig` carries no statement
|
|
//! cache).
|
|
//!
|
|
//! Every string in a `Row` is borrowed for the duration of the call only:
|
|
//! `Stmt.bindText` binds with `SQLITE_TRANSIENT`, so SQLite copies before
|
|
//! `writeBatch` returns.
|
|
//!
|
|
//! The rows are expendable log data. Nothing here retries, and the caller
|
|
//! decides what a failed batch means.
|
|
|
|
const std = @import("std");
|
|
const Allocator = std.mem.Allocator;
|
|
|
|
const db = @import("../db.zig");
|
|
const provenance = @import("../provenance.zig");
|
|
|
|
/// One `query_log` row. The logger applies the privacy transforms of PLAN
|
|
/// §11.4 before it builds this, so every domain-bearing field is already
|
|
/// whatever the operator agreed to store.
|
|
///
|
|
/// A `null` text field is a fact the query did not have — no upstream was
|
|
/// attempted, no rule matched, no CNAME was uncloaked — and reaches the column
|
|
/// as NULL. The three closed enums have no such state: every logged query has a
|
|
/// policy verdict, a reason for it and a route, even when the verdict is
|
|
/// "not evaluated".
|
|
pub const Row = struct {
|
|
timestamp: i64,
|
|
domain: []const u8,
|
|
client_ip: []const u8,
|
|
qtype: ?u16,
|
|
qclass: u16,
|
|
/// Twelve bits: the EDNS extended RCODE the client saw. The column's
|
|
/// `CHECK` bounds it to the same range, so a value this type cannot hold
|
|
/// is one the schema would have refused anyway.
|
|
rcode: u12,
|
|
blocked: bool,
|
|
response_time_us: ?i64,
|
|
cache_hit: ?bool,
|
|
upstream: ?[]const u8,
|
|
group_id: ?i64,
|
|
group_name: ?[]const u8,
|
|
policy_action: provenance.PolicyAction,
|
|
policy_reason: provenance.PolicyReason,
|
|
matched: ?[]const u8,
|
|
source_id: ?i64,
|
|
source_name: ?[]const u8,
|
|
cname_target: ?[]const u8,
|
|
safe_search_target: ?[]const u8,
|
|
route_kind: provenance.RouteKind,
|
|
forward_zone: ?[]const u8,
|
|
};
|
|
|
|
/// The projection grain of the four `bucket_*` tables, in seconds.
|
|
///
|
|
/// 1800 divides every serving width the Overview offers at or above half an
|
|
/// hour (1800, 3600, 21600), which is what lets one grain answer the 24h, 7d
|
|
/// and 30d windows exactly. The 1h window is 60-second buckets and is served
|
|
/// from the raw rows instead.
|
|
pub const grain: i64 = 1800;
|
|
|
|
/// Floors a timestamp onto the projection grid.
|
|
///
|
|
/// `@divFloor`, never `@divTrunc` or SQLite's `/`: both truncate toward zero,
|
|
/// which for a negative timestamp names the bucket *after* the one the row
|
|
/// belongs to. Every producer and every consumer of a `bucket` value in this
|
|
/// file goes through this function or an equivalent floor expression.
|
|
pub fn bucketOf(timestamp: i64) i64 {
|
|
return @divFloor(timestamp, grain) * grain;
|
|
}
|
|
|
|
/// `bucket_types.qtype` is NOT NULL, so the rows that carry no query type need
|
|
/// a value of their own. No qtype is a `u16`, so -1 cannot collide with one.
|
|
pub const null_qtype: i64 = -1;
|
|
|
|
/// The answering resolver's identity, as the routes breakdown defines it: the
|
|
/// upstream url for `upstream` rows, the zone for `forward_zone` rows, null
|
|
/// everywhere else. The Zig twin of `overview_raw_sql`'s CASE, and the only
|
|
/// place the writer decides what a route's source is.
|
|
fn routeSourceOf(row: Row) ?[]const u8 {
|
|
return switch (row.route_kind) {
|
|
.upstream => row.upstream,
|
|
.forward_zone => row.forward_zone,
|
|
else => null,
|
|
};
|
|
}
|
|
|
|
const insert_domain_sql = "INSERT OR IGNORE INTO domains (domain) VALUES (?1)";
|
|
|
|
const select_domain_sql = "SELECT id FROM domains WHERE domain = ?1";
|
|
|
|
const insert_row_sql =
|
|
\\INSERT INTO query_log
|
|
\\ (timestamp, domain_id, client_ip, qtype, blocked,
|
|
\\ response_time_us, cache_hit, upstream, qclass, rcode,
|
|
\\ group_id, group_name, policy_action, policy_reason, matched,
|
|
\\ source_id, source_name, cname_target, safe_search_target,
|
|
\\ route_kind, forward_zone)
|
|
\\VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10,
|
|
\\ ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21)
|
|
;
|
|
|
|
/// One `bucket_totals` row's worth of change from a single batch.
|
|
const TotalsDelta = struct {
|
|
queries: i64 = 0,
|
|
blocked: i64 = 0,
|
|
cached: i64 = 0,
|
|
rt_sum: i64 = 0,
|
|
rt_count: i64 = 0,
|
|
};
|
|
|
|
/// The delta-map keys borrow the batch's strings. They live only from the
|
|
/// aggregation pass to the end of the `writeBatch` that produced them, which is
|
|
/// inside the borrow `Row` already documents.
|
|
const ClientKey = struct { bucket: i64, client_ip: []const u8 };
|
|
const TypeKey = struct { bucket: i64, qtype: i64 };
|
|
const RouteKey = struct { bucket: i64, route: provenance.RouteKind, source: ?[]const u8 };
|
|
|
|
const ClientKeyContext = struct {
|
|
pub fn hash(_: ClientKeyContext, key: ClientKey) u64 {
|
|
var hasher: std.hash.Wyhash = .init(@bitCast(key.bucket));
|
|
hasher.update(key.client_ip);
|
|
return hasher.final();
|
|
}
|
|
pub fn eql(_: ClientKeyContext, a: ClientKey, b: ClientKey) bool {
|
|
return a.bucket == b.bucket and std.mem.eql(u8, a.client_ip, b.client_ip);
|
|
}
|
|
};
|
|
|
|
const RouteKeyContext = struct {
|
|
pub fn hash(_: RouteKeyContext, key: RouteKey) u64 {
|
|
var hasher: std.hash.Wyhash = .init(@bitCast(key.bucket));
|
|
hasher.update(@tagName(key.route));
|
|
hasher.update(&[_]u8{@intFromBool(key.source != null)});
|
|
hasher.update(key.source orelse "");
|
|
return hasher.final();
|
|
}
|
|
pub fn eql(_: RouteKeyContext, a: RouteKey, b: RouteKey) bool {
|
|
return a.bucket == b.bucket and a.route == b.route and sameSource(a.source, b.source);
|
|
}
|
|
};
|
|
|
|
const load_percentage = std.hash_map.default_max_load_percentage;
|
|
|
|
const upsert_totals_sql =
|
|
\\INSERT INTO bucket_totals (bucket, queries, blocked, cached, rt_sum, rt_count)
|
|
\\VALUES (?1, ?2, ?3, ?4, ?5, ?6)
|
|
\\ON CONFLICT(bucket) DO UPDATE SET
|
|
\\ queries = queries + excluded.queries,
|
|
\\ blocked = blocked + excluded.blocked,
|
|
\\ cached = cached + excluded.cached,
|
|
\\ rt_sum = rt_sum + excluded.rt_sum,
|
|
\\ rt_count = rt_count + excluded.rt_count
|
|
;
|
|
|
|
const upsert_clients_sql =
|
|
\\INSERT INTO bucket_clients (bucket, client_ip, queries)
|
|
\\VALUES (?1, ?2, ?3)
|
|
\\ON CONFLICT(bucket, client_ip) DO UPDATE SET queries = queries + excluded.queries
|
|
;
|
|
|
|
const upsert_types_sql =
|
|
\\INSERT INTO bucket_types (bucket, qtype, count)
|
|
\\VALUES (?1, ?2, ?3)
|
|
\\ON CONFLICT(bucket, qtype) DO UPDATE SET count = count + excluded.count
|
|
;
|
|
|
|
const upsert_routes_sql =
|
|
\\INSERT INTO bucket_routes (bucket, route_kind, source_present, source_text, count)
|
|
\\VALUES (?1, ?2, ?3, ?4, ?5)
|
|
\\ON CONFLICT(bucket, route_kind, source_present, source_text)
|
|
\\DO UPDATE SET count = count + excluded.count
|
|
;
|
|
|
|
/// Owns the prepared statements of the flush loop. Init once, reuse per batch.
|
|
///
|
|
/// `database` must outlive the writer and must not move: every `Stmt` holds a
|
|
/// `*Db`. Neither `Db` nor `Stmt` is thread-safe, so one writer belongs to one
|
|
/// task.
|
|
///
|
|
/// The writer also maintains the four `bucket_*` projections, in the same
|
|
/// transaction as the rows they summarise. `gpa` is owned for the writer's life
|
|
/// and is used for nothing but the per-batch delta maps, whose capacity is
|
|
/// retained across batches and whose contents are dropped at the end of every
|
|
/// `writeBatch`.
|
|
pub const BatchWriter = struct {
|
|
gpa: Allocator,
|
|
database: *db.Db,
|
|
insert_domain: db.Stmt,
|
|
select_domain: db.Stmt,
|
|
insert_row: db.Stmt,
|
|
upsert_totals: db.Stmt,
|
|
upsert_clients: db.Stmt,
|
|
upsert_types: db.Stmt,
|
|
upsert_routes: db.Stmt,
|
|
|
|
totals_deltas: std.AutoHashMapUnmanaged(i64, TotalsDelta) = .empty,
|
|
client_deltas: std.HashMapUnmanaged(ClientKey, i64, ClientKeyContext, load_percentage) = .empty,
|
|
type_deltas: std.AutoHashMapUnmanaged(TypeKey, i64) = .empty,
|
|
route_deltas: std.HashMapUnmanaged(RouteKey, i64, RouteKeyContext, load_percentage) = .empty,
|
|
|
|
pub fn init(gpa: Allocator, database: *db.Db) db.Error!BatchWriter {
|
|
var insert_domain = try database.prepare(insert_domain_sql);
|
|
errdefer insert_domain.deinit();
|
|
var select_domain = try database.prepare(select_domain_sql);
|
|
errdefer select_domain.deinit();
|
|
var insert_row = try database.prepare(insert_row_sql);
|
|
errdefer insert_row.deinit();
|
|
var upsert_totals = try database.prepare(upsert_totals_sql);
|
|
errdefer upsert_totals.deinit();
|
|
var upsert_clients = try database.prepare(upsert_clients_sql);
|
|
errdefer upsert_clients.deinit();
|
|
var upsert_types = try database.prepare(upsert_types_sql);
|
|
errdefer upsert_types.deinit();
|
|
const upsert_routes = try database.prepare(upsert_routes_sql);
|
|
return .{
|
|
.gpa = gpa,
|
|
.database = database,
|
|
.insert_domain = insert_domain,
|
|
.select_domain = select_domain,
|
|
.insert_row = insert_row,
|
|
.upsert_totals = upsert_totals,
|
|
.upsert_clients = upsert_clients,
|
|
.upsert_types = upsert_types,
|
|
.upsert_routes = upsert_routes,
|
|
};
|
|
}
|
|
|
|
pub fn deinit(self: *BatchWriter) void {
|
|
self.route_deltas.deinit(self.gpa);
|
|
self.type_deltas.deinit(self.gpa);
|
|
self.client_deltas.deinit(self.gpa);
|
|
self.totals_deltas.deinit(self.gpa);
|
|
self.upsert_routes.deinit();
|
|
self.upsert_types.deinit();
|
|
self.upsert_clients.deinit();
|
|
self.upsert_totals.deinit();
|
|
self.insert_row.deinit();
|
|
self.select_domain.deinit();
|
|
self.insert_domain.deinit();
|
|
}
|
|
|
|
/// One transaction for the whole batch: the raw rows and every projection
|
|
/// they touch commit together or not at all. Domains are interned through
|
|
/// `INSERT OR IGNORE` followed by `SELECT id`.
|
|
///
|
|
/// On any failure the transaction rolls back, so a batch is all or
|
|
/// nothing, and the writer stays usable for the next batch.
|
|
pub fn writeBatch(self: *BatchWriter, rows: []const Row) db.Error!void {
|
|
if (rows.len == 0) return;
|
|
|
|
// Declared before the aggregation so it runs after every `errdefer`
|
|
// below, and placed before `BEGIN` so an `error.OutOfMemory` fails the
|
|
// batch with no transaction outstanding.
|
|
defer self.clearDeltas();
|
|
try self.aggregate(rows);
|
|
|
|
var tx = try db.Tx.begin(self.database);
|
|
// `errdefer`s run in reverse: the statements are released before the
|
|
// ROLLBACK, so no read cursor is still open when it runs.
|
|
errdefer tx.rollback();
|
|
errdefer self.resetAll();
|
|
|
|
for (rows) |row| {
|
|
const domain_id = try self.internDomain(row.domain);
|
|
try self.write(row, domain_id);
|
|
}
|
|
try self.applyProjections();
|
|
try tx.commit();
|
|
}
|
|
|
|
/// Folds the batch into per-key deltas, so the transaction below runs one
|
|
/// UPSERT per touched key rather than per row. Any slice length: the
|
|
/// logger's 100-row batching is its own policy, not this API's.
|
|
fn aggregate(self: *BatchWriter, rows: []const Row) Allocator.Error!void {
|
|
for (rows) |row| {
|
|
const bucket = bucketOf(row.timestamp);
|
|
|
|
const totals = try self.totals_deltas.getOrPut(self.gpa, bucket);
|
|
if (!totals.found_existing) totals.value_ptr.* = .{};
|
|
totals.value_ptr.queries += 1;
|
|
if (row.blocked) totals.value_ptr.blocked += 1;
|
|
if (row.cache_hit orelse false) totals.value_ptr.cached += 1;
|
|
if (row.response_time_us) |us| {
|
|
totals.value_ptr.rt_sum += us;
|
|
totals.value_ptr.rt_count += 1;
|
|
}
|
|
|
|
try bump(&self.client_deltas, self.gpa, ClientKey{
|
|
.bucket = bucket,
|
|
.client_ip = row.client_ip,
|
|
});
|
|
try bump(&self.type_deltas, self.gpa, TypeKey{
|
|
.bucket = bucket,
|
|
.qtype = if (row.qtype) |v| @as(i64, v) else null_qtype,
|
|
});
|
|
try bump(&self.route_deltas, self.gpa, RouteKey{
|
|
.bucket = bucket,
|
|
.route = row.route_kind,
|
|
.source = routeSourceOf(row),
|
|
});
|
|
}
|
|
}
|
|
|
|
/// One more row on `key`. `map` is any of the three count maps — they
|
|
/// differ only in their key type and its hashing.
|
|
fn bump(map: anytype, gpa: Allocator, key: anytype) Allocator.Error!void {
|
|
const entry = try map.getOrPut(gpa, key);
|
|
if (!entry.found_existing) entry.value_ptr.* = 0;
|
|
entry.value_ptr.* += 1;
|
|
}
|
|
|
|
fn applyProjections(self: *BatchWriter) db.Error!void {
|
|
var totals = self.totals_deltas.iterator();
|
|
while (totals.next()) |entry| {
|
|
const stmt = &self.upsert_totals;
|
|
try stmt.reset();
|
|
try stmt.bindInt(1, entry.key_ptr.*);
|
|
try stmt.bindInt(2, entry.value_ptr.queries);
|
|
try stmt.bindInt(3, entry.value_ptr.blocked);
|
|
try stmt.bindInt(4, entry.value_ptr.cached);
|
|
try stmt.bindInt(5, entry.value_ptr.rt_sum);
|
|
try stmt.bindInt(6, entry.value_ptr.rt_count);
|
|
try stmt.exec();
|
|
}
|
|
|
|
var clients = self.client_deltas.iterator();
|
|
while (clients.next()) |entry| {
|
|
const stmt = &self.upsert_clients;
|
|
try stmt.reset();
|
|
try stmt.bindInt(1, entry.key_ptr.bucket);
|
|
try stmt.bindText(2, entry.key_ptr.client_ip);
|
|
try stmt.bindInt(3, entry.value_ptr.*);
|
|
try stmt.exec();
|
|
}
|
|
|
|
var types = self.type_deltas.iterator();
|
|
while (types.next()) |entry| {
|
|
const stmt = &self.upsert_types;
|
|
try stmt.reset();
|
|
try stmt.bindInt(1, entry.key_ptr.bucket);
|
|
try stmt.bindInt(2, entry.key_ptr.qtype);
|
|
try stmt.bindInt(3, entry.value_ptr.*);
|
|
try stmt.exec();
|
|
}
|
|
|
|
var routes = self.route_deltas.iterator();
|
|
while (routes.next()) |entry| {
|
|
const stmt = &self.upsert_routes;
|
|
try stmt.reset();
|
|
try stmt.bindInt(1, entry.key_ptr.bucket);
|
|
try stmt.bindText(2, @tagName(entry.key_ptr.route));
|
|
try stmt.bindInt(3, @intFromBool(entry.key_ptr.source != null));
|
|
try stmt.bindText(4, entry.key_ptr.source orelse "");
|
|
try stmt.bindInt(5, entry.value_ptr.*);
|
|
try stmt.exec();
|
|
}
|
|
}
|
|
|
|
/// Capacity is retained; only the entries go. The keys borrow the batch's
|
|
/// strings, so nothing may outlive the `writeBatch` that filled them.
|
|
fn clearDeltas(self: *BatchWriter) void {
|
|
self.totals_deltas.clearRetainingCapacity();
|
|
self.client_deltas.clearRetainingCapacity();
|
|
self.type_deltas.clearRetainingCapacity();
|
|
self.route_deltas.clearRetainingCapacity();
|
|
}
|
|
|
|
fn internDomain(self: *BatchWriter, domain: []const u8) db.Error!i64 {
|
|
try self.insert_domain.reset();
|
|
try self.insert_domain.bindText(1, domain);
|
|
try self.insert_domain.exec();
|
|
|
|
try self.select_domain.reset();
|
|
try self.select_domain.bindText(1, domain);
|
|
// The insert above either created the row or found it already there,
|
|
// so a miss means the table changed under this connection.
|
|
if (!try self.select_domain.step()) return error.NotFound;
|
|
const id = self.select_domain.columnInt(0);
|
|
// A statement stopped on a row keeps its cursor open until it is
|
|
// reset; the transaction must not carry that to the next row.
|
|
try self.select_domain.reset();
|
|
return id;
|
|
}
|
|
|
|
fn write(self: *BatchWriter, row: Row, domain_id: i64) db.Error!void {
|
|
var stmt = &self.insert_row;
|
|
try stmt.reset();
|
|
try stmt.bindInt(1, row.timestamp);
|
|
try stmt.bindInt(2, domain_id);
|
|
try stmt.bindText(3, row.client_ip);
|
|
try bindIntOrNull(stmt, 4, if (row.qtype) |v| @as(i64, v) else null);
|
|
try stmt.bindBool(5, row.blocked);
|
|
try bindIntOrNull(stmt, 6, row.response_time_us);
|
|
try bindIntOrNull(stmt, 7, if (row.cache_hit) |v| @as(i64, @intFromBool(v)) else null);
|
|
try stmt.bindTextOrNull(8, row.upstream);
|
|
try stmt.bindInt(9, row.qclass);
|
|
try stmt.bindInt(10, row.rcode);
|
|
try bindIntOrNull(stmt, 11, row.group_id);
|
|
try stmt.bindTextOrNull(12, row.group_name);
|
|
try stmt.bindText(13, @tagName(row.policy_action));
|
|
try stmt.bindText(14, @tagName(row.policy_reason));
|
|
try stmt.bindTextOrNull(15, row.matched);
|
|
try bindIntOrNull(stmt, 16, row.source_id);
|
|
try stmt.bindTextOrNull(17, row.source_name);
|
|
try stmt.bindTextOrNull(18, row.cname_target);
|
|
try stmt.bindTextOrNull(19, row.safe_search_target);
|
|
try stmt.bindText(20, @tagName(row.route_kind));
|
|
try stmt.bindTextOrNull(21, row.forward_zone);
|
|
try stmt.exec();
|
|
}
|
|
|
|
/// Best effort: this runs on the failure path, where the error that
|
|
/// matters is the one already on its way to the caller.
|
|
fn resetAll(self: *BatchWriter) void {
|
|
self.upsert_routes.reset() catch {};
|
|
self.upsert_types.reset() catch {};
|
|
self.upsert_clients.reset() catch {};
|
|
self.upsert_totals.reset() catch {};
|
|
self.insert_row.reset() catch {};
|
|
self.select_domain.reset() catch {};
|
|
self.insert_domain.reset() catch {};
|
|
}
|
|
};
|
|
|
|
fn bindIntOrNull(stmt: *db.Stmt, idx: c_int, value: ?i64) db.Error!void {
|
|
if (value) |v| return stmt.bindInt(idx, v);
|
|
return stmt.bindNull(idx);
|
|
}
|
|
|
|
/// What one prune did, and where coverage now begins.
|
|
pub const PruneResult = struct {
|
|
deleted: i64,
|
|
/// The watermark after the prune, which is what a later `availableSince`
|
|
/// will return. Handed back so the caller need not re-read it.
|
|
available_since: i64,
|
|
};
|
|
|
|
/// Deletes every `query_log` row strictly older than `cutoff_ts` and advances
|
|
/// the coverage watermark to the same cutoff, in one transaction.
|
|
///
|
|
/// **The two are one operation, not two.** The watermark is the promise that
|
|
/// every query since it is still in the file; a delete that commits without the
|
|
/// advance breaks that promise, and an advance that commits without the delete
|
|
/// hides rows the file still holds. Either failure rolls both back, and the
|
|
/// caller retries the whole thing on its next pass.
|
|
///
|
|
/// The watermark never moves backward: `max` is what makes a prune with a
|
|
/// cutoff older than the file's own creation a no-op on it rather than a
|
|
/// regression. Orphaned `domains` rows stay — it is a dimension table,
|
|
/// re-interning a name costs one indexed insert, and §11.3 asks for no
|
|
/// collection.
|
|
///
|
|
/// The four projections are pruned in the same transaction. Buckets entirely
|
|
/// before the cutoff's own bucket are deleted outright; a cutoff that does not
|
|
/// land on the grid leaves one straddling bucket, which is recomputed from the
|
|
/// rows that survived rather than reduced by an estimate. Any failure in any of
|
|
/// that rolls back the raw delete and the watermark advance with it.
|
|
pub fn pruneOlderThan(database: *db.Db, cutoff_ts: i64) db.Error!PruneResult {
|
|
var tx = try db.Tx.begin(database);
|
|
errdefer tx.rollback();
|
|
|
|
var deleting = try database.prepare("DELETE FROM query_log WHERE timestamp < ?1");
|
|
defer deleting.deinit();
|
|
try deleting.bindInt(1, cutoff_ts);
|
|
try deleting.exec();
|
|
const deleted = database.changes();
|
|
|
|
const grid = bucketOf(cutoff_ts);
|
|
for (delete_before_bucket_sql) |sql| {
|
|
var stmt = try database.prepare(sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, grid);
|
|
try stmt.exec();
|
|
}
|
|
// On the grid there is nothing partial to fix: the delete above took whole
|
|
// buckets and the loop took their projection rows.
|
|
if (cutoff_ts != grid) try recomputeBucket(database, grid);
|
|
|
|
var advancing = try database.prepare(
|
|
"UPDATE querylog_meta SET available_since = max(available_since, ?1) WHERE id = 1",
|
|
);
|
|
defer advancing.deinit();
|
|
try advancing.bindInt(1, cutoff_ts);
|
|
try advancing.exec();
|
|
|
|
const watermark = try database.queryInt("SELECT available_since FROM querylog_meta WHERE id = 1");
|
|
try tx.commit();
|
|
return .{ .deleted = deleted, .available_since = watermark };
|
|
}
|
|
|
|
const delete_before_bucket_sql = [_][]const u8{
|
|
"DELETE FROM bucket_totals WHERE bucket < ?1",
|
|
"DELETE FROM bucket_clients WHERE bucket < ?1",
|
|
"DELETE FROM bucket_types WHERE bucket < ?1",
|
|
"DELETE FROM bucket_routes WHERE bucket < ?1",
|
|
};
|
|
|
|
const delete_one_bucket_sql = [_][]const u8{
|
|
"DELETE FROM bucket_totals WHERE bucket = ?1",
|
|
"DELETE FROM bucket_clients WHERE bucket = ?1",
|
|
"DELETE FROM bucket_types WHERE bucket = ?1",
|
|
"DELETE FROM bucket_routes WHERE bucket = ?1",
|
|
};
|
|
|
|
/// `?1` is the bucket start and `?2` its exclusive end. Each statement rebuilds
|
|
/// one projection table's rows for that bucket from the raw rows still in it.
|
|
///
|
|
/// `HAVING count(*) > 0` on the totals statement is what keeps an emptied
|
|
/// bucket from being written back as a row of zeros: a bare aggregate always
|
|
/// produces one row, and a from-scratch recomputation produces none. The other
|
|
/// three group by a key, so an empty range already yields nothing.
|
|
const recompute_bucket_sql = [_][]const u8{
|
|
\\INSERT INTO bucket_totals (bucket, queries, blocked, cached, rt_sum, rt_count)
|
|
\\SELECT ?1, count(*), coalesce(sum(blocked <> 0), 0), coalesce(sum(cache_hit = 1), 0),
|
|
\\ coalesce(sum(response_time_us), 0), count(response_time_us)
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?2
|
|
\\HAVING count(*) > 0
|
|
,
|
|
\\INSERT INTO bucket_clients (bucket, client_ip, queries)
|
|
\\SELECT ?1, client_ip, count(*)
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?2
|
|
\\ GROUP BY client_ip
|
|
,
|
|
\\INSERT INTO bucket_types (bucket, qtype, count)
|
|
\\SELECT ?1, coalesce(qtype, -1), count(*)
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?2
|
|
\\ GROUP BY coalesce(qtype, -1)
|
|
,
|
|
\\INSERT INTO bucket_routes (bucket, route_kind, source_present, source_text, count)
|
|
\\SELECT ?1, route_kind, source IS NOT NULL, coalesce(source, ''), count(*)
|
|
\\ FROM (SELECT route_kind,
|
|
\\ CASE route_kind
|
|
\\ WHEN 'upstream' THEN upstream
|
|
\\ WHEN 'forward_zone' THEN forward_zone
|
|
\\ END AS source
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?2)
|
|
\\ GROUP BY route_kind, source
|
|
,
|
|
};
|
|
|
|
/// Replaces one bucket's projection rows with a recomputation from the raw rows
|
|
/// that are still in it. The caller owns the transaction.
|
|
fn recomputeBucket(database: *db.Db, bucket: i64) db.Error!void {
|
|
for (delete_one_bucket_sql) |sql| {
|
|
var stmt = try database.prepare(sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, bucket);
|
|
try stmt.exec();
|
|
}
|
|
for (recompute_bucket_sql) |sql| {
|
|
var stmt = try database.prepare(sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, bucket);
|
|
try stmt.bindInt(2, bucket + grain);
|
|
try stmt.exec();
|
|
}
|
|
}
|
|
|
|
/// The oldest timestamp this file can still answer for. A query window that
|
|
/// starts before it is incomplete, and the API says so rather than charting the
|
|
/// gap as zero.
|
|
pub fn availableSince(database: *db.Db) db.Error!i64 {
|
|
return database.queryInt("SELECT available_since FROM querylog_meta WHERE id = 1");
|
|
}
|
|
|
|
/// `PRAGMA wal_checkpoint(TRUNCATE)`: moves the WAL into the database and
|
|
/// truncates it to zero bytes, which is what keeps a day of log writes from
|
|
/// growing the WAL past the free space the disk monitor watches.
|
|
///
|
|
/// SQLite reports a checkpoint blocked by a concurrent reader in the row it
|
|
/// returns, not as an error code, so a blocked checkpoint is not an error
|
|
/// here. Retention checkpoints after every prune, so the next pass retries.
|
|
/// On a database that is not in WAL mode the pragma is a no-op.
|
|
pub fn checkpointTruncate(database: *db.Db) db.Error!void {
|
|
return database.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
}
|
|
|
|
/// Rewrites the whole file. Retention runs this rarely by design — on an SD
|
|
/// card a full rewrite is the most expensive thing this program does.
|
|
pub fn vacuum(database: *db.Db) db.Error!void {
|
|
return database.exec("VACUUM;");
|
|
}
|
|
|
|
pub fn countRows(database: *db.Db) db.Error!i64 {
|
|
return database.queryInt("SELECT count(*) FROM query_log");
|
|
}
|
|
|
|
pub fn countDomains(database: *db.Db) db.Error!i64 {
|
|
return database.queryInt("SELECT count(*) FROM domains");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// the API read layer (`GET /api/queries`, `GET /api/overview`)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// One row of `GET /api/queries`, joined back through the `domains` dimension.
|
|
///
|
|
/// A summary projection, deliberately narrower than `QueryDetail`: the list is
|
|
/// a table the operator scans, and the full provenance of a row is one request
|
|
/// away at `GET /api/queries/{id}`.
|
|
///
|
|
/// The nullable text columns read a NULL as `""` — the same convention
|
|
/// `Stmt.columnText` already uses. None of them is ever written as an empty
|
|
/// string, so the mapping loses nothing and the API layer can treat `""` as
|
|
/// "absent".
|
|
pub const QueryRow = struct {
|
|
id: i64,
|
|
ts: i64,
|
|
domain: []const u8,
|
|
client_ip: []const u8,
|
|
qtype: ?u16,
|
|
qclass: u16,
|
|
rcode: u12,
|
|
blocked: bool,
|
|
response_time_us: ?i64,
|
|
cache_hit: ?bool,
|
|
upstream: []const u8,
|
|
policy_action: provenance.PolicyAction,
|
|
policy_reason: provenance.PolicyReason,
|
|
route_kind: provenance.RouteKind,
|
|
};
|
|
|
|
/// Everything one `query_log` row records about one query, for
|
|
/// `GET /api/queries/{id}`.
|
|
///
|
|
/// Same NULL-reads-as-`""` convention as `QueryRow`, and the same closed enums:
|
|
/// a stored value the schema does not define is `error.Mismatch`, never passed
|
|
/// through as text.
|
|
pub const QueryDetail = struct {
|
|
id: i64,
|
|
ts: i64,
|
|
domain: []const u8,
|
|
client_ip: []const u8,
|
|
qtype: ?u16,
|
|
qclass: u16,
|
|
rcode: u12,
|
|
blocked: bool,
|
|
response_time_us: ?i64,
|
|
cache_hit: ?bool,
|
|
upstream: []const u8,
|
|
group_id: ?i64,
|
|
group_name: []const u8,
|
|
policy_action: provenance.PolicyAction,
|
|
policy_reason: provenance.PolicyReason,
|
|
matched: []const u8,
|
|
source_id: ?i64,
|
|
source_name: []const u8,
|
|
cname_target: []const u8,
|
|
safe_search_target: []const u8,
|
|
route_kind: provenance.RouteKind,
|
|
forward_zone: []const u8,
|
|
};
|
|
|
|
/// Every field is an independent narrowing; `null` means "do not filter on it".
|
|
///
|
|
/// `since` is inclusive and `until` is exclusive, so adjacent windows tile
|
|
/// without double-counting a row on the boundary.
|
|
pub const QueryFilter = struct {
|
|
limit: u32 = 100,
|
|
/// Keyset cursor: only rows with a strictly smaller `id`. Rows come back
|
|
/// newest-first, so this is the id of the last row of the previous page.
|
|
before: ?i64 = null,
|
|
/// Matched case-insensitively for ASCII, which is what SQLite's `LIKE`
|
|
/// does and what a domain search wants.
|
|
domain_substring: ?[]const u8 = null,
|
|
/// Exact client addresses, matched any-of. Empty means no client filter;
|
|
/// one address is the common case and reads the same as the old single
|
|
/// filter did. Never more than `max_clients`.
|
|
clients: []const []const u8 = &.{},
|
|
blocked: ?bool = null,
|
|
since: ?i64 = null,
|
|
until: ?i64 = null,
|
|
};
|
|
|
|
/// Ruling 11 caps the page at 1000; the repository enforces it too, so a caller
|
|
/// that forgets cannot ask this connection for the whole table.
|
|
pub const max_limit: u32 = 1000;
|
|
|
|
/// How many addresses one client filter may name. The statement is assembled
|
|
/// into a fixed buffer, so this is a hard bound rather than a preference: a
|
|
/// household picking more than this from a list is not a case worth widening
|
|
/// the buffer for.
|
|
pub const max_clients: usize = 32;
|
|
|
|
const select_head =
|
|
\\SELECT q.id, q.timestamp, d.domain, q.client_ip, q.qtype, q.blocked,
|
|
\\ q.response_time_us, q.cache_hit, q.upstream, q.qclass, q.rcode,
|
|
\\ q.policy_action, q.policy_reason, q.route_kind
|
|
\\ FROM query_log q JOIN domains d ON d.id = q.domain_id
|
|
;
|
|
|
|
/// The escape character of `where_domain`. SQLite does not give string literals
|
|
/// C escapes, so `'\'` in the SQL text is one backslash.
|
|
const like_escape = '\\';
|
|
|
|
const where_before = " q.id < ?";
|
|
const where_domain = " d.domain LIKE ? ESCAPE '\\'";
|
|
const where_client_head = " q.client_ip IN (";
|
|
const where_client_tail = ")";
|
|
/// `?` per address with a comma between, at the widest the cap allows.
|
|
const where_client_max = where_client_head.len + 2 * max_clients + where_client_tail.len;
|
|
const where_blocked = " q.blocked = ?";
|
|
const where_since = " q.timestamp >= ?";
|
|
const where_until = " q.timestamp < ?";
|
|
const select_tail = " ORDER BY q.id DESC LIMIT ?";
|
|
|
|
const where_keyword = " WHERE";
|
|
const and_keyword = " AND";
|
|
|
|
/// Assembles the statement from the fixed fragments above and nothing else.
|
|
///
|
|
/// **No value ever reaches this buffer.** Every filter contributes a `?` and is
|
|
/// bound afterwards, in the order the predicates were appended: an unnumbered
|
|
/// parameter takes the next free index, so append order and bind order are the
|
|
/// same single contract.
|
|
const Sql = struct {
|
|
/// `where_keyword` is longer than `and_keyword` and is used at most once,
|
|
/// so counting six of it bounds every reachable combination.
|
|
const capacity = select_head.len + 6 * where_keyword.len + select_tail.len +
|
|
where_before.len + where_domain.len + where_client_max +
|
|
where_blocked.len + where_since.len + where_until.len;
|
|
|
|
buf: [capacity]u8 = undefined,
|
|
len: usize = 0,
|
|
has_where: bool = false,
|
|
|
|
fn put(self: *Sql, fragment: []const u8) void {
|
|
@memcpy(self.buf[self.len..][0..fragment.len], fragment);
|
|
self.len += fragment.len;
|
|
}
|
|
|
|
fn predicate(self: *Sql, fragment: []const u8) void {
|
|
self.put(if (self.has_where) and_keyword else where_keyword);
|
|
self.has_where = true;
|
|
self.put(fragment);
|
|
}
|
|
|
|
/// One placeholder per address. The addresses themselves are bound, like
|
|
/// every other value; only their count reaches this buffer.
|
|
fn clientPredicate(self: *Sql, count: usize) void {
|
|
self.put(if (self.has_where) and_keyword else where_keyword);
|
|
self.has_where = true;
|
|
self.put(where_client_head);
|
|
for (0..count) |i| {
|
|
if (i > 0) self.put(",");
|
|
self.put("?");
|
|
}
|
|
self.put(where_client_tail);
|
|
}
|
|
|
|
fn text(self: *const Sql) []const u8 {
|
|
return self.buf[0..self.len];
|
|
}
|
|
};
|
|
|
|
/// Rows come back newest-first (`id DESC`). Every string is allocated from
|
|
/// `arena`, including the list's own storage, so the caller frees the whole
|
|
/// result by resetting the arena — there is nothing to unwind on failure.
|
|
pub fn selectQueries(database: *db.Db, arena: Allocator, filter: QueryFilter) db.Error!std.ArrayList(QueryRow) {
|
|
// The statement buffer is sized for the cap, so a longer list would be a
|
|
// buffer overrun rather than a slow query. The handler rejects it first;
|
|
// this is the wall behind that, for a caller that skips the handler.
|
|
if (filter.clients.len > max_clients) return error.Misuse;
|
|
|
|
var sql: Sql = .{};
|
|
sql.put(select_head);
|
|
if (filter.before != null) sql.predicate(where_before);
|
|
if (filter.domain_substring != null) sql.predicate(where_domain);
|
|
if (filter.clients.len > 0) sql.clientPredicate(filter.clients.len);
|
|
if (filter.blocked != null) sql.predicate(where_blocked);
|
|
if (filter.since != null) sql.predicate(where_since);
|
|
if (filter.until != null) sql.predicate(where_until);
|
|
sql.put(select_tail);
|
|
|
|
var stmt = try database.prepare(sql.text());
|
|
defer stmt.deinit();
|
|
|
|
var idx: c_int = 0;
|
|
if (filter.before) |v| {
|
|
idx += 1;
|
|
try stmt.bindInt(idx, v);
|
|
}
|
|
if (filter.domain_substring) |v| {
|
|
idx += 1;
|
|
try stmt.bindText(idx, try likePattern(arena, v));
|
|
}
|
|
for (filter.clients) |client| {
|
|
idx += 1;
|
|
try stmt.bindText(idx, client);
|
|
}
|
|
if (filter.blocked) |v| {
|
|
idx += 1;
|
|
try stmt.bindBool(idx, v);
|
|
}
|
|
if (filter.since) |v| {
|
|
idx += 1;
|
|
try stmt.bindInt(idx, v);
|
|
}
|
|
if (filter.until) |v| {
|
|
idx += 1;
|
|
try stmt.bindInt(idx, v);
|
|
}
|
|
idx += 1;
|
|
try stmt.bindInt(idx, @min(filter.limit, max_limit));
|
|
|
|
var out: std.ArrayList(QueryRow) = .empty;
|
|
while (try stmt.step()) {
|
|
try out.append(arena, .{
|
|
.id = stmt.columnInt(0),
|
|
.ts = stmt.columnInt(1),
|
|
.domain = try stmt.columnTextAlloc(arena, 2),
|
|
.client_ip = try stmt.columnTextAlloc(arena, 3),
|
|
.qtype = if (stmt.isNull(4)) null else std.math.cast(u16, stmt.columnInt(4)) orelse
|
|
return error.Mismatch,
|
|
.blocked = stmt.columnBool(5),
|
|
.response_time_us = if (stmt.isNull(6)) null else stmt.columnInt(6),
|
|
.cache_hit = if (stmt.isNull(7)) null else stmt.columnBool(7),
|
|
.upstream = try stmt.columnTextAlloc(arena, 8),
|
|
.qclass = try columnU16(&stmt, 9),
|
|
.rcode = try columnU12(&stmt, 10),
|
|
.policy_action = try provenance.parse(provenance.PolicyAction, stmt.columnText(11)),
|
|
.policy_reason = try provenance.parse(provenance.PolicyReason, stmt.columnText(12)),
|
|
.route_kind = try provenance.parse(provenance.RouteKind, stmt.columnText(13)),
|
|
});
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/// A `NOT NULL` integer column that the schema bounds to 16 bits. A value
|
|
/// outside that range means the row came from something other than this schema.
|
|
fn columnU16(stmt: *db.Stmt, col: c_int) db.Error!u16 {
|
|
return std.math.cast(u16, stmt.columnInt(col)) orelse error.Mismatch;
|
|
}
|
|
|
|
/// The `rcode` column, which the schema's `CHECK` bounds to twelve bits. This
|
|
/// build cannot write a wider value — the field is a `u12` all the way from the
|
|
/// handler — so a row that carries one was written by something else, and is
|
|
/// `error.Mismatch` rather than a value truncated into shape.
|
|
fn columnU12(stmt: *db.Stmt, col: c_int) db.Error!u12 {
|
|
return std.math.cast(u12, stmt.columnInt(col)) orelse error.Mismatch;
|
|
}
|
|
|
|
const select_detail_sql =
|
|
\\SELECT q.id, q.timestamp, d.domain, q.client_ip, q.qtype, q.blocked,
|
|
\\ q.response_time_us, q.cache_hit, q.upstream, q.qclass, q.rcode,
|
|
\\ q.group_id, q.group_name, q.policy_action, q.policy_reason,
|
|
\\ q.matched, q.source_id, q.source_name, q.cname_target,
|
|
\\ q.safe_search_target, q.route_kind, q.forward_zone
|
|
\\ FROM query_log q JOIN domains d ON d.id = q.domain_id
|
|
\\ WHERE q.id = ?1
|
|
;
|
|
|
|
/// One row's full provenance, or `null` when no row has that id — which is what
|
|
/// an id the operator kept from before a retention pass looks like, and is a
|
|
/// 404 rather than an error.
|
|
///
|
|
/// Every string is allocated from `arena`, on the same terms as
|
|
/// `selectQueries`.
|
|
pub fn detailById(database: *db.Db, arena: Allocator, id: i64) db.Error!?QueryDetail {
|
|
var stmt = try database.prepare(select_detail_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, id);
|
|
if (!try stmt.step()) return null;
|
|
|
|
return .{
|
|
.id = stmt.columnInt(0),
|
|
.ts = stmt.columnInt(1),
|
|
.domain = try stmt.columnTextAlloc(arena, 2),
|
|
.client_ip = try stmt.columnTextAlloc(arena, 3),
|
|
.qtype = if (stmt.isNull(4)) null else try columnU16(&stmt, 4),
|
|
.blocked = stmt.columnBool(5),
|
|
.response_time_us = if (stmt.isNull(6)) null else stmt.columnInt(6),
|
|
.cache_hit = if (stmt.isNull(7)) null else stmt.columnBool(7),
|
|
.upstream = try stmt.columnTextAlloc(arena, 8),
|
|
.qclass = try columnU16(&stmt, 9),
|
|
.rcode = try columnU12(&stmt, 10),
|
|
.group_id = if (stmt.isNull(11)) null else stmt.columnInt(11),
|
|
.group_name = try stmt.columnTextAlloc(arena, 12),
|
|
.policy_action = try provenance.parse(provenance.PolicyAction, stmt.columnText(13)),
|
|
.policy_reason = try provenance.parse(provenance.PolicyReason, stmt.columnText(14)),
|
|
.matched = try stmt.columnTextAlloc(arena, 15),
|
|
.source_id = if (stmt.isNull(16)) null else stmt.columnInt(16),
|
|
.source_name = try stmt.columnTextAlloc(arena, 17),
|
|
.cname_target = try stmt.columnTextAlloc(arena, 18),
|
|
.safe_search_target = try stmt.columnTextAlloc(arena, 19),
|
|
.route_kind = try provenance.parse(provenance.RouteKind, stmt.columnText(20)),
|
|
.forward_zone = try stmt.columnTextAlloc(arena, 21),
|
|
};
|
|
}
|
|
|
|
/// Wraps `needle` in `%` and neutralises the two `LIKE` metacharacters, so a
|
|
/// user searching for `a_b` gets domains containing `a_b` and not domains
|
|
/// containing `axb`. The escape character escapes itself.
|
|
fn likePattern(arena: Allocator, needle: []const u8) Allocator.Error![]const u8 {
|
|
var out: std.ArrayList(u8) = try .initCapacity(arena, needle.len * 2 + 2);
|
|
out.appendAssumeCapacity('%');
|
|
for (needle) |ch| {
|
|
if (ch == '%' or ch == '_' or ch == like_escape) out.appendAssumeCapacity(like_escape);
|
|
out.appendAssumeCapacity(ch);
|
|
}
|
|
out.appendAssumeCapacity('%');
|
|
return out.items;
|
|
}
|
|
|
|
/// The Overview rollup for one window. `avg_response_time_us` is `null` when no
|
|
/// row in the window recorded a response time. The mean is derived from a sum
|
|
/// and a count rather than SQL's `avg`, which returns REAL: `Stmt` reads
|
|
/// integers, and integer microseconds are exact.
|
|
pub const StatsTotals = struct {
|
|
queries: u64,
|
|
blocked: u64,
|
|
distinct_clients: u64,
|
|
avg_response_time_us: ?i64,
|
|
};
|
|
|
|
/// `count` and `sum` over non-negative columns cannot go negative; a negative
|
|
/// value means the row came from something other than this schema.
|
|
fn countOf(value: i64) db.Error!u64 {
|
|
if (value < 0) return error.Mismatch;
|
|
return @intCast(value);
|
|
}
|
|
|
|
/// One bucket of the Overview timeseries. `ts` is the bucket's inclusive start.
|
|
pub const Bucket = struct {
|
|
ts: i64,
|
|
queries: u64,
|
|
blocked: u64,
|
|
cached: u64,
|
|
};
|
|
|
|
/// One row of the Overview type breakdown. `qtype` is nullable in the schema, so
|
|
/// the rows that carry no type group into a row of their own rather than
|
|
/// disappearing from a breakdown that claims to add up.
|
|
///
|
|
/// No name field: the only qtype-name table lives in the admin, and a second
|
|
/// copy here would drift out of agreement with it.
|
|
///
|
|
/// The list carries no zero rows: a type absent from the window is absent from
|
|
/// it. Its order is an order, not an identity — a caller keys on the `qtype`
|
|
/// value, never on a row's position, because a rank change between refreshes
|
|
/// moves rows and must not move what they mean.
|
|
pub const TypeCount = struct {
|
|
qtype: ?u16,
|
|
count: u64,
|
|
};
|
|
|
|
/// One row of the Overview route breakdown: how a slice of the window was
|
|
/// answered.
|
|
///
|
|
/// `source` is the answering resolver's identity and nothing else — the
|
|
/// upstream url for `upstream` rows, the zone for `forward_zone` rows, null
|
|
/// everywhere else. It is deliberately not `source_name`, which names the
|
|
/// blocklist a block came from and would read as an upstream here.
|
|
///
|
|
/// A null source on an `upstream` row is its own group, not a dropped row: it
|
|
/// is a real state of the log and the caller labels it.
|
|
pub const RouteCount = struct {
|
|
route: provenance.RouteKind,
|
|
source: ?[]const u8,
|
|
count: u64,
|
|
};
|
|
|
|
/// How many clients the Overview names before the rest become `other`. Eight is
|
|
/// what one legend can carry without becoming a second table.
|
|
pub const max_client_series = 8;
|
|
|
|
/// One named client's series. `buckets` is always the caller's bucket count
|
|
/// long, zero-filled, and aligned exactly like `Overview.buckets`.
|
|
pub const ClientSeries = struct {
|
|
client: []const u8,
|
|
buckets: []const u64,
|
|
};
|
|
|
|
/// Named clients are ranked by in-window total, ties broken by address, so the
|
|
/// cut at `max_client_series` is the same cut on every read over the same data.
|
|
///
|
|
/// `other` is always present and always bucket-count sized — including for an
|
|
/// empty window and for a window with eight clients or fewer. A caller charting
|
|
/// a stack must not have to invent the residual series.
|
|
pub const ClientsBreakdown = struct {
|
|
clients: []const ClientSeries,
|
|
other: []const u64,
|
|
};
|
|
|
|
fn zeroedBuckets(arena: Allocator, bucket_count: u32) Allocator.Error![]u64 {
|
|
const buckets = try arena.alloc(u64, bucket_count);
|
|
@memset(buckets, 0);
|
|
return buckets;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// the Overview read path (milestone 36)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Everything one Overview response reports about one window, from one read of
|
|
/// one already-open transaction.
|
|
///
|
|
/// Storage owns these scalars: `since`, `bucket_seconds` and `bucket_count` are
|
|
/// numbers, not a web module's period type, so nothing here has to know the
|
|
/// period grammar the handler enforces.
|
|
pub const Overview = struct {
|
|
totals: StatsTotals,
|
|
/// `bucket_count` entries, one per bucket of the window, zero-filled and
|
|
/// aligned to `since`.
|
|
buckets: []const Bucket,
|
|
clients: ClientsBreakdown,
|
|
types: []const TypeCount,
|
|
routes: []const RouteCount,
|
|
};
|
|
|
|
/// One client's running window total beside its per-bucket series, so the cut
|
|
/// at `max_client_series` can be made after the whole window is folded rather
|
|
/// than by a second pass over the data.
|
|
const ClientAccum = struct {
|
|
client: []const u8,
|
|
total: u64,
|
|
buckets: []u64,
|
|
};
|
|
|
|
/// The shared shape both paths fold into. Every allocation is from `arena`, so
|
|
/// there is nothing to unwind on failure.
|
|
const Accum = struct {
|
|
arena: Allocator,
|
|
since: i64,
|
|
width: i64,
|
|
bucket_count: u32,
|
|
buckets: []Bucket,
|
|
rt_sum: i64 = 0,
|
|
rt_count: i64 = 0,
|
|
clients: std.ArrayList(ClientAccum) = .empty,
|
|
/// `client_ip` to its index in `clients`. The names are arena copies, so
|
|
/// they outlive the statement that produced them.
|
|
client_index: std.StringHashMapUnmanaged(u32) = .empty,
|
|
types: std.ArrayList(TypeCount) = .empty,
|
|
type_index: std.AutoHashMapUnmanaged(i64, u32) = .empty,
|
|
/// Household cardinality: a handful of upstreams plus six route kinds. A
|
|
/// linear scan is the whole index this needs.
|
|
routes: std.ArrayList(RouteCount) = .empty,
|
|
|
|
fn init(arena: Allocator, since: i64, width: i64, bucket_count: u32) Allocator.Error!Accum {
|
|
const buckets = try arena.alloc(Bucket, bucket_count);
|
|
for (buckets, 0..) |*bucket, i| bucket.* = .{
|
|
.ts = since + width * @as(i64, @intCast(i)),
|
|
.queries = 0,
|
|
.blocked = 0,
|
|
.cached = 0,
|
|
};
|
|
return .{ .arena = arena, .since = since, .width = width, .bucket_count = bucket_count, .buckets = buckets };
|
|
}
|
|
|
|
/// The serving-bucket index a grid bucket or a raw timestamp falls in.
|
|
/// Both paths bound their reads by the window already; the check is what
|
|
/// keeps a schema surprise from writing past the slice.
|
|
fn indexOf(self: *const Accum, timestamp: i64) db.Error!usize {
|
|
const offset = @divFloor(timestamp - self.since, self.width);
|
|
const index = std.math.cast(usize, offset) orelse return error.Mismatch;
|
|
if (index >= self.buckets.len) return error.Mismatch;
|
|
return index;
|
|
}
|
|
|
|
fn addTotals(self: *Accum, index: usize, queries: u64, blocked: u64, cached: u64) void {
|
|
self.buckets[index].queries += queries;
|
|
self.buckets[index].blocked += blocked;
|
|
self.buckets[index].cached += cached;
|
|
}
|
|
|
|
fn addClient(self: *Accum, client: []const u8, index: usize, count: u64) Allocator.Error!void {
|
|
const slot = try self.client_index.getOrPut(self.arena, client);
|
|
if (!slot.found_existing) {
|
|
const owned = try self.arena.dupe(u8, client);
|
|
slot.key_ptr.* = owned;
|
|
slot.value_ptr.* = @intCast(self.clients.items.len);
|
|
try self.clients.append(self.arena, .{
|
|
.client = owned,
|
|
.total = 0,
|
|
.buckets = try zeroedBuckets(self.arena, self.bucket_count),
|
|
});
|
|
}
|
|
const accum = &self.clients.items[slot.value_ptr.*];
|
|
accum.total += count;
|
|
accum.buckets[index] += count;
|
|
}
|
|
|
|
/// `qtype` is the stored encoding: `null_qtype` for the rows that carry no
|
|
/// query type.
|
|
fn addType(self: *Accum, qtype: i64, count: u64) db.Error!void {
|
|
const slot = try self.type_index.getOrPut(self.arena, qtype);
|
|
if (!slot.found_existing) {
|
|
slot.value_ptr.* = @intCast(self.types.items.len);
|
|
try self.types.append(self.arena, .{
|
|
.qtype = if (qtype == null_qtype) null else std.math.cast(u16, qtype) orelse
|
|
return error.Mismatch,
|
|
.count = 0,
|
|
});
|
|
}
|
|
self.types.items[slot.value_ptr.*].count += count;
|
|
}
|
|
|
|
fn addRoute(self: *Accum, route: provenance.RouteKind, source: ?[]const u8, count: u64) Allocator.Error!void {
|
|
for (self.routes.items) |*existing| {
|
|
if (existing.route == route and sameSource(existing.source, source)) {
|
|
existing.count += count;
|
|
return;
|
|
}
|
|
}
|
|
try self.routes.append(self.arena, .{
|
|
.route = route,
|
|
.source = if (source) |s| try self.arena.dupe(u8, s) else null,
|
|
.count = count,
|
|
});
|
|
}
|
|
|
|
fn finish(self: *Accum) Allocator.Error!Overview {
|
|
var queries: u64 = 0;
|
|
var blocked: u64 = 0;
|
|
for (self.buckets) |bucket| {
|
|
queries += bucket.queries;
|
|
blocked += bucket.blocked;
|
|
}
|
|
|
|
// Every comparator below is a total order on a unique key, so an
|
|
// unstable sort still produces one list for one window.
|
|
std.sort.pdq(TypeCount, self.types.items, {}, typeBefore);
|
|
std.sort.pdq(RouteCount, self.routes.items, {}, routeBefore);
|
|
std.sort.pdq(ClientAccum, self.clients.items, {}, clientBefore);
|
|
|
|
const named = @min(self.clients.items.len, max_client_series);
|
|
const other = try zeroedBuckets(self.arena, self.bucket_count);
|
|
for (self.clients.items[named..]) |folded| {
|
|
for (other, folded.buckets) |*slot, count| slot.* += count;
|
|
}
|
|
const series = try self.arena.alloc(ClientSeries, named);
|
|
for (series, self.clients.items[0..named]) |*entry, accum| {
|
|
entry.* = .{ .client = accum.client, .buckets = accum.buckets };
|
|
}
|
|
|
|
return .{
|
|
.totals = .{
|
|
.queries = queries,
|
|
.blocked = blocked,
|
|
.distinct_clients = self.clients.items.len,
|
|
.avg_response_time_us = if (self.rt_count == 0)
|
|
null
|
|
else
|
|
@divTrunc(self.rt_sum, self.rt_count),
|
|
},
|
|
.buckets = self.buckets,
|
|
.clients = .{ .clients = series, .other = other },
|
|
.types = self.types.items,
|
|
.routes = self.routes.items,
|
|
};
|
|
}
|
|
};
|
|
|
|
/// Absent and present are different sources, and so are two different present
|
|
/// ones: an `upstream` row the log recorded no resolver for is its own group,
|
|
/// exactly as `GROUP BY ... source` makes it.
|
|
fn sameSource(a: ?[]const u8, b: ?[]const u8) bool {
|
|
const left = a orelse return b == null;
|
|
const right = b orelse return false;
|
|
return std.mem.eql(u8, left, right);
|
|
}
|
|
|
|
/// The oracle's `ORDER BY count(*) DESC, qtype IS NULL, qtype ASC`.
|
|
fn typeBefore(_: void, a: TypeCount, b: TypeCount) bool {
|
|
if (a.count != b.count) return a.count > b.count;
|
|
const a_type = a.qtype orelse return false;
|
|
const b_type = b.qtype orelse return true;
|
|
return a_type < b_type;
|
|
}
|
|
|
|
/// The oracle's `ORDER BY count(*) DESC, route_kind ASC, source IS NULL,
|
|
/// source ASC`. `route_kind` is compared as the stored text, not as the
|
|
/// enum's declaration order — the two disagree, and the stored text is what the
|
|
/// SQL sorted.
|
|
fn routeBefore(_: void, a: RouteCount, b: RouteCount) bool {
|
|
if (a.count != b.count) return a.count > b.count;
|
|
switch (std.mem.order(u8, @tagName(a.route), @tagName(b.route))) {
|
|
.lt => return true,
|
|
.gt => return false,
|
|
.eq => {},
|
|
}
|
|
const a_source = a.source orelse return false;
|
|
const b_source = b.source orelse return true;
|
|
return std.mem.order(u8, a_source, b_source) == .lt;
|
|
}
|
|
|
|
/// The oracle's `ORDER BY count(*) DESC, client_ip ASC`, which is what makes
|
|
/// the cut at `max_client_series` the same cut on every read.
|
|
fn clientBefore(_: void, a: ClientAccum, b: ClientAccum) bool {
|
|
if (a.total != b.total) return a.total > b.total;
|
|
return std.mem.order(u8, a.client, b.client) == .lt;
|
|
}
|
|
|
|
const overview_projection_totals_sql =
|
|
\\SELECT bucket, queries, blocked, cached, rt_sum, rt_count
|
|
\\ FROM bucket_totals
|
|
\\ WHERE bucket >= ?1 AND bucket < ?2
|
|
;
|
|
|
|
const overview_projection_clients_sql =
|
|
\\SELECT bucket, client_ip, queries
|
|
\\ FROM bucket_clients
|
|
\\ WHERE bucket >= ?1 AND bucket < ?2
|
|
;
|
|
|
|
const overview_projection_types_sql =
|
|
\\SELECT qtype, sum(count)
|
|
\\ FROM bucket_types
|
|
\\ WHERE bucket >= ?1 AND bucket < ?2
|
|
\\ GROUP BY qtype
|
|
;
|
|
|
|
const overview_projection_routes_sql =
|
|
\\SELECT route_kind, source_present, source_text, sum(count)
|
|
\\ FROM bucket_routes
|
|
\\ WHERE bucket >= ?1 AND bucket < ?2
|
|
\\ GROUP BY route_kind, source_present, source_text
|
|
;
|
|
|
|
const overview_raw_sql =
|
|
\\SELECT timestamp, client_ip, qtype, blocked, cache_hit, response_time_us,
|
|
\\ route_kind,
|
|
\\ CASE route_kind
|
|
\\ WHEN 'upstream' THEN upstream
|
|
\\ WHEN 'forward_zone' THEN forward_zone
|
|
\\ END AS source
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?2
|
|
;
|
|
|
|
/// The whole Overview payload for `[since, since + bucket_seconds *
|
|
/// bucket_count)`, from one already-open read transaction — the caller owns the
|
|
/// transaction and the lock, as it does for every other read here.
|
|
///
|
|
/// Two implementations behind one entry point. At or above the projection grain
|
|
/// the four `bucket_*` tables answer the window, and the cost stops depending
|
|
/// on how many raw rows it holds; below it (the 1h window, on 60-second
|
|
/// buckets) one pass over the raw rows in the window does the same work in Zig.
|
|
/// Both produce the same contracts, which is what the equivalence test asserts.
|
|
///
|
|
/// Preconditions are caller bugs, not runtime conditions: a zero width or count
|
|
/// is `error.Misuse`, and so is a projection-path window that is not on the
|
|
/// grid, because the projections cannot express it. The handler's `window()`
|
|
/// guarantees all of them.
|
|
pub fn overview(
|
|
database: *db.Db,
|
|
arena: Allocator,
|
|
since: i64,
|
|
bucket_seconds: u32,
|
|
bucket_count: u32,
|
|
) db.Error!Overview {
|
|
if (bucket_seconds == 0 or bucket_count == 0) return error.Misuse;
|
|
const width: i64 = bucket_seconds;
|
|
const span = std.math.mul(i64, width, bucket_count) catch return error.Misuse;
|
|
const until = std.math.add(i64, since, span) catch return error.Misuse;
|
|
const from_projections = width >= grain;
|
|
if (from_projections and (@mod(since, grain) != 0 or @rem(width, grain) != 0)) return error.Misuse;
|
|
|
|
var accum = try Accum.init(arena, since, width, bucket_count);
|
|
if (from_projections) {
|
|
try readProjections(database, &accum, since, until);
|
|
} else {
|
|
try readRaw(database, &accum, since, until);
|
|
}
|
|
return try accum.finish();
|
|
}
|
|
|
|
fn readProjections(database: *db.Db, accum: *Accum, since: i64, until: i64) db.Error!void {
|
|
{
|
|
var stmt = try database.prepare(overview_projection_totals_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, until);
|
|
while (try stmt.step()) {
|
|
const index = try accum.indexOf(stmt.columnInt(0));
|
|
accum.addTotals(
|
|
index,
|
|
try countOf(stmt.columnInt(1)),
|
|
try countOf(stmt.columnInt(2)),
|
|
try countOf(stmt.columnInt(3)),
|
|
);
|
|
accum.rt_sum += stmt.columnInt(4);
|
|
accum.rt_count += stmt.columnInt(5);
|
|
}
|
|
}
|
|
{
|
|
var stmt = try database.prepare(overview_projection_clients_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, until);
|
|
while (try stmt.step()) {
|
|
const index = try accum.indexOf(stmt.columnInt(0));
|
|
try accum.addClient(stmt.columnText(1), index, try countOf(stmt.columnInt(2)));
|
|
}
|
|
}
|
|
{
|
|
var stmt = try database.prepare(overview_projection_types_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, until);
|
|
while (try stmt.step()) {
|
|
try accum.addType(stmt.columnInt(0), try countOf(stmt.columnInt(1)));
|
|
}
|
|
}
|
|
{
|
|
var stmt = try database.prepare(overview_projection_routes_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, until);
|
|
while (try stmt.step()) {
|
|
const route = try provenance.parse(provenance.RouteKind, stmt.columnText(0));
|
|
const source: ?[]const u8 = if (stmt.columnInt(1) == 0) null else stmt.columnText(2);
|
|
try accum.addRoute(route, source, try countOf(stmt.columnInt(3)));
|
|
}
|
|
}
|
|
}
|
|
|
|
fn readRaw(database: *db.Db, accum: *Accum, since: i64, until: i64) db.Error!void {
|
|
var stmt = try database.prepare(overview_raw_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, until);
|
|
|
|
while (try stmt.step()) {
|
|
const index = try accum.indexOf(stmt.columnInt(0));
|
|
const blocked: u64 = @intFromBool(stmt.columnBool(3));
|
|
// `cache_hit = 1`, not "non-zero": the SQL aggregates this replaces
|
|
// compare against the literal, and a NULL compares to neither.
|
|
const cached: u64 = @intFromBool(!stmt.isNull(4) and stmt.columnInt(4) == 1);
|
|
accum.addTotals(index, 1, blocked, cached);
|
|
if (!stmt.isNull(5)) {
|
|
accum.rt_sum += stmt.columnInt(5);
|
|
accum.rt_count += 1;
|
|
}
|
|
try accum.addClient(stmt.columnText(1), index, 1);
|
|
try accum.addType(if (stmt.isNull(2)) null_qtype else stmt.columnInt(2), 1);
|
|
try accum.addRoute(
|
|
try provenance.parse(provenance.RouteKind, stmt.columnText(6)),
|
|
stmt.columnTextOrNull(7),
|
|
1,
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const logger = @import("../logger.zig");
|
|
const querylog_schema = @import("../querylog_schema.zig");
|
|
|
|
const testing = std.testing;
|
|
|
|
fn openLog() !db.Db {
|
|
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
|
errdefer database.close();
|
|
try db.applyPragmas(&database, .{});
|
|
try database.exec(querylog_schema.ddl);
|
|
return database;
|
|
}
|
|
|
|
fn plainRow(timestamp: i64, domain: []const u8) Row {
|
|
return .{
|
|
.timestamp = timestamp,
|
|
.domain = domain,
|
|
.client_ip = "192.0.2.10",
|
|
.qtype = 1,
|
|
.qclass = 1,
|
|
.rcode = 0,
|
|
.blocked = false,
|
|
.response_time_us = 1200,
|
|
.cache_hit = false,
|
|
.upstream = "9.9.9.9",
|
|
.group_id = 1,
|
|
.group_name = "default",
|
|
.policy_action = .allow,
|
|
.policy_reason = .no_match,
|
|
.matched = null,
|
|
.source_id = null,
|
|
.source_name = null,
|
|
.cname_target = null,
|
|
.safe_search_target = null,
|
|
.route_kind = .upstream,
|
|
.forward_zone = null,
|
|
};
|
|
}
|
|
|
|
fn domainIdOf(database: *db.Db, domain: []const u8) !i64 {
|
|
var stmt = try database.prepare(select_domain_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindText(1, domain);
|
|
try testing.expect(try stmt.step());
|
|
return stmt.columnInt(0);
|
|
}
|
|
|
|
test "a foreign row with an rcode wider than twelve bits is refused, not truncated" {
|
|
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
|
defer database.close();
|
|
try db.applyPragmas(&database, .{});
|
|
|
|
// The shipped schema's `CHECK` makes this row impossible in a file this
|
|
// build created, so the table is built without it. The read path's job is
|
|
// to refuse a `querylog.db` that came from somewhere else rather than to
|
|
// narrow a value it cannot represent.
|
|
try database.exec(
|
|
\\CREATE TABLE domains (id INTEGER PRIMARY KEY, domain TEXT NOT NULL UNIQUE);
|
|
\\CREATE TABLE query_log (
|
|
\\ id INTEGER PRIMARY KEY, timestamp INTEGER NOT NULL,
|
|
\\ domain_id INTEGER NOT NULL, client_ip TEXT NOT NULL,
|
|
\\ qtype INTEGER, blocked INTEGER NOT NULL, response_time_us INTEGER,
|
|
\\ cache_hit INTEGER, upstream TEXT, qclass INTEGER NOT NULL,
|
|
\\ rcode INTEGER NOT NULL, group_id INTEGER, group_name TEXT,
|
|
\\ policy_action TEXT NOT NULL, policy_reason TEXT NOT NULL,
|
|
\\ matched TEXT, source_id INTEGER, source_name TEXT,
|
|
\\ cname_target TEXT, safe_search_target TEXT,
|
|
\\ route_kind TEXT NOT NULL, forward_zone TEXT
|
|
\\);
|
|
\\INSERT INTO domains (id, domain) VALUES (1, 'a.example');
|
|
\\INSERT INTO query_log
|
|
\\ (id, timestamp, domain_id, client_ip, blocked, qclass, rcode,
|
|
\\ policy_action, policy_reason, route_kind)
|
|
\\VALUES (1, 10, 1, '192.0.2.10', 0, 1, 4096, 'allow', 'no_match', 'upstream');
|
|
);
|
|
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
try testing.expectError(error.Mismatch, selectQueries(&database, arena, .{}));
|
|
try testing.expectError(error.Mismatch, detailById(&database, arena, 1));
|
|
}
|
|
|
|
test "writeBatch inserts every row and interns each domain once" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
try writer.writeBatch(&.{
|
|
plainRow(100, "example.com"),
|
|
plainRow(101, "example.com"),
|
|
plainRow(102, "ads.example.net"),
|
|
});
|
|
|
|
try testing.expectEqual(@as(i64, 3), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 2), try countDomains(&database));
|
|
|
|
const first = try domainIdOf(&database, "example.com");
|
|
try testing.expectEqual(
|
|
@as(i64, 2),
|
|
try database.queryInt("SELECT count(*) FROM query_log WHERE domain_id = 1"),
|
|
);
|
|
try testing.expectEqual(@as(i64, 1), first);
|
|
}
|
|
|
|
test "a second batch reuses the interned domain id" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
try writer.writeBatch(&.{plainRow(100, "example.com")});
|
|
const before = try domainIdOf(&database, "example.com");
|
|
|
|
try writer.writeBatch(&.{ plainRow(200, "example.com"), plainRow(201, "other.example") });
|
|
const after = try domainIdOf(&database, "example.com");
|
|
|
|
try testing.expectEqual(before, after);
|
|
try testing.expectEqual(@as(i64, 3), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 2), try countDomains(&database));
|
|
try testing.expectEqual(
|
|
@as(i64, 2),
|
|
try database.queryInt("SELECT count(*) FROM query_log WHERE domain_id = 1"),
|
|
);
|
|
}
|
|
|
|
test "every column round-trips a value and a null" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
try writer.writeBatch(&.{
|
|
.{
|
|
.timestamp = 10,
|
|
.domain = "blocked.example",
|
|
.client_ip = "2001:db8::1",
|
|
.qtype = 28,
|
|
.qclass = 1,
|
|
.rcode = 3,
|
|
.blocked = true,
|
|
.response_time_us = 42,
|
|
.cache_hit = true,
|
|
.upstream = "https://dns.example/dns-query",
|
|
.group_id = 7,
|
|
.group_name = "kids",
|
|
.policy_action = .block,
|
|
.policy_reason = .blocklist_wildcard,
|
|
.matched = "*.ads.example",
|
|
.source_id = 3,
|
|
.source_name = "steven black",
|
|
.cname_target = "tracker.cdn.example",
|
|
.safe_search_target = "forcesafesearch.google.com",
|
|
.route_kind = .blocked,
|
|
.forward_zone = "home.arpa",
|
|
},
|
|
// Every nullable column absent at once, which is the shape of a query
|
|
// the pipeline answered before any of them applied.
|
|
.{
|
|
.timestamp = 11,
|
|
.domain = "quiet.example",
|
|
.client_ip = "hidden",
|
|
.qtype = null,
|
|
.qclass = 3,
|
|
.rcode = 0,
|
|
.blocked = false,
|
|
.response_time_us = null,
|
|
.cache_hit = null,
|
|
.upstream = null,
|
|
.group_id = null,
|
|
.group_name = null,
|
|
.policy_action = .not_evaluated,
|
|
.policy_reason = .non_in_class,
|
|
.matched = null,
|
|
.source_id = null,
|
|
.source_name = null,
|
|
.cname_target = null,
|
|
.safe_search_target = null,
|
|
.route_kind = .upstream,
|
|
.forward_zone = null,
|
|
},
|
|
});
|
|
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
const full = (try detailById(&database, arena, 1)).?;
|
|
try testing.expectEqualStrings("blocked.example", full.domain);
|
|
try testing.expectEqualStrings("2001:db8::1", full.client_ip);
|
|
try testing.expectEqual(@as(?u16, 28), full.qtype);
|
|
try testing.expectEqual(@as(u16, 1), full.qclass);
|
|
try testing.expectEqual(@as(u12, 3), full.rcode);
|
|
try testing.expect(full.blocked);
|
|
try testing.expectEqual(@as(?i64, 42), full.response_time_us);
|
|
try testing.expectEqual(@as(?bool, true), full.cache_hit);
|
|
try testing.expectEqualStrings("https://dns.example/dns-query", full.upstream);
|
|
try testing.expectEqual(@as(?i64, 7), full.group_id);
|
|
try testing.expectEqualStrings("kids", full.group_name);
|
|
try testing.expectEqual(provenance.PolicyAction.block, full.policy_action);
|
|
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, full.policy_reason);
|
|
try testing.expectEqualStrings("*.ads.example", full.matched);
|
|
try testing.expectEqual(@as(?i64, 3), full.source_id);
|
|
try testing.expectEqualStrings("steven black", full.source_name);
|
|
try testing.expectEqualStrings("tracker.cdn.example", full.cname_target);
|
|
try testing.expectEqualStrings("forcesafesearch.google.com", full.safe_search_target);
|
|
try testing.expectEqual(provenance.RouteKind.blocked, full.route_kind);
|
|
try testing.expectEqualStrings("home.arpa", full.forward_zone);
|
|
|
|
// A NULL text column reads as the empty string, by the documented
|
|
// convention; a NULL integer stays null, because 0 is a real id.
|
|
const bare = (try detailById(&database, arena, 2)).?;
|
|
try testing.expectEqual(@as(?u16, null), bare.qtype);
|
|
try testing.expectEqual(@as(u16, 3), bare.qclass);
|
|
try testing.expectEqual(@as(?i64, null), bare.response_time_us);
|
|
try testing.expectEqual(@as(?bool, null), bare.cache_hit);
|
|
try testing.expectEqualStrings("", bare.upstream);
|
|
try testing.expectEqual(@as(?i64, null), bare.group_id);
|
|
try testing.expectEqualStrings("", bare.group_name);
|
|
try testing.expectEqual(provenance.PolicyAction.not_evaluated, bare.policy_action);
|
|
try testing.expectEqual(provenance.PolicyReason.non_in_class, bare.policy_reason);
|
|
try testing.expectEqualStrings("", bare.matched);
|
|
try testing.expectEqual(@as(?i64, null), bare.source_id);
|
|
try testing.expectEqualStrings("", bare.source_name);
|
|
try testing.expectEqualStrings("", bare.cname_target);
|
|
try testing.expectEqualStrings("", bare.safe_search_target);
|
|
try testing.expectEqual(provenance.RouteKind.upstream, bare.route_kind);
|
|
try testing.expectEqualStrings("", bare.forward_zone);
|
|
}
|
|
|
|
test "detailById returns null for an id no row has" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
|
|
try seed(&database, &.{plainRow(10, "a.example")});
|
|
|
|
// An id from before a retention pass looks exactly like this, and is a 404
|
|
// rather than an error.
|
|
try testing.expectEqual(@as(?QueryDetail, null), try detailById(&database, arena_state.allocator(), 2));
|
|
try testing.expectEqual(@as(?QueryDetail, null), try detailById(&database, arena_state.allocator(), 0));
|
|
try testing.expect((try detailById(&database, arena_state.allocator(), 1)) != null);
|
|
}
|
|
|
|
test "a stored enum value the schema does not define is a data error, not a passthrough" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
try seed(&database, &.{plainRow(10, "a.example")});
|
|
try database.exec("UPDATE query_log SET policy_reason = 'whatever' WHERE id = 1;");
|
|
|
|
try testing.expectError(error.Mismatch, detailById(&database, arena, 1));
|
|
try testing.expectError(error.Mismatch, selectQueries(&database, arena, .{}));
|
|
}
|
|
|
|
test "an empty batch writes nothing and opens no transaction" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
// A transaction is already open, so a `BEGIN IMMEDIATE` from `writeBatch`
|
|
// would fail: this is what proves the empty batch returns before it.
|
|
var tx = try db.Tx.begin(&database);
|
|
try writer.writeBatch(&.{});
|
|
tx.rollback();
|
|
|
|
try testing.expectEqual(@as(i64, 0), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 0), try countDomains(&database));
|
|
}
|
|
|
|
test "pruneOlderThan deletes strictly older rows and returns the count" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
try writer.writeBatch(&.{
|
|
plainRow(100, "old.example"),
|
|
plainRow(199, "old.example"),
|
|
plainRow(200, "edge.example"),
|
|
plainRow(300, "fresh.example"),
|
|
});
|
|
|
|
try testing.expectEqual(@as(i64, 2), (try pruneOlderThan(&database, 200)).deleted);
|
|
try testing.expectEqual(@as(i64, 2), try countRows(&database));
|
|
// The row exactly at the cutoff stays.
|
|
try testing.expectEqual(
|
|
@as(i64, 1),
|
|
try database.queryInt("SELECT count(*) FROM query_log WHERE timestamp = 200"),
|
|
);
|
|
// A second pass over the same cutoff finds nothing left to do.
|
|
try testing.expectEqual(@as(i64, 0), (try pruneOlderThan(&database, 200)).deleted);
|
|
}
|
|
|
|
test "a prune advances the coverage watermark to its own cutoff" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
// The seeded watermark is `created_at + 1`, which is now-ish; the cutoffs
|
|
// below are all in the past, so they start out behind it.
|
|
const start = try availableSince(&database);
|
|
try writer.writeBatch(&.{ plainRow(start + 100, "a.example"), plainRow(start + 300, "b.example") });
|
|
|
|
const first = try pruneOlderThan(&database, start + 200);
|
|
try testing.expectEqual(@as(i64, 1), first.deleted);
|
|
try testing.expectEqual(start + 200, first.available_since);
|
|
try testing.expectEqual(start + 200, try availableSince(&database));
|
|
|
|
// A prune that deletes nothing still advances: the window it swept is
|
|
// covered whether or not it held rows.
|
|
const second = try pruneOlderThan(&database, start + 250);
|
|
try testing.expectEqual(@as(i64, 0), second.deleted);
|
|
try testing.expectEqual(start + 250, try availableSince(&database));
|
|
}
|
|
|
|
test "the watermark never moves backward" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
|
|
const start = try availableSince(&database);
|
|
const advanced = try pruneOlderThan(&database, start + 1000);
|
|
try testing.expectEqual(start + 1000, advanced.available_since);
|
|
|
|
// A shortened `retention_days`, a clock that stepped back, a pass with a
|
|
// stale cutoff: none of them may widen the promise the file makes.
|
|
for ([_]i64{ start + 999, start, start - 100_000, 0 }) |older| {
|
|
const result = try pruneOlderThan(&database, older);
|
|
try testing.expectEqual(start + 1000, result.available_since);
|
|
try testing.expectEqual(start + 1000, try availableSince(&database));
|
|
}
|
|
}
|
|
|
|
test "a failed delete leaves both the rows and the watermark untouched" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
const start = try availableSince(&database);
|
|
try writer.writeBatch(&.{plainRow(start - 100, "old.example")});
|
|
try database.exec(
|
|
\\CREATE TRIGGER refuse_delete BEFORE DELETE ON query_log
|
|
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
|
);
|
|
|
|
try testing.expectError(error.Constraint, pruneOlderThan(&database, start + 1000));
|
|
|
|
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
|
try testing.expectEqual(start, try availableSince(&database));
|
|
}
|
|
|
|
test "a failed watermark update leaves the rows it had already deleted" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
const start = try availableSince(&database);
|
|
try writer.writeBatch(&.{plainRow(start - 100, "old.example")});
|
|
// The delete succeeds and the advance does not. Without one transaction
|
|
// around the pair, this is the case that loses rows the watermark still
|
|
// promises.
|
|
try database.exec(
|
|
\\CREATE TRIGGER refuse_advance BEFORE UPDATE ON querylog_meta
|
|
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
|
);
|
|
|
|
try testing.expectError(error.Constraint, pruneOlderThan(&database, start + 1000));
|
|
|
|
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
|
try testing.expectEqual(start, try availableSince(&database));
|
|
}
|
|
|
|
test "a failed commit rolls back the delete and the watermark together" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
const start = try availableSince(&database);
|
|
try writer.writeBatch(&.{plainRow(start - 100, "old.example")});
|
|
|
|
// Both statements succeed and COMMIT is what fails: the advance inserts a
|
|
// `query_log` row whose `domain_id` references nothing, and
|
|
// `defer_foreign_keys` holds that violation back until the commit checks
|
|
// it (SQLite's documented semantics for the pragma; the assertions below
|
|
// observe the rollback, not the moment the check ran).
|
|
try database.exec(
|
|
\\CREATE TRIGGER break_at_commit AFTER UPDATE ON querylog_meta
|
|
\\BEGIN INSERT INTO query_log
|
|
\\ (timestamp, domain_id, client_ip, blocked, qclass, rcode,
|
|
\\ policy_action, policy_reason, route_kind)
|
|
\\VALUES (1, 999999, 'x', 0, 1, 0, 'allow', 'no_match', 'upstream'); END;
|
|
);
|
|
try database.exec("PRAGMA defer_foreign_keys = ON;");
|
|
|
|
try testing.expectError(error.Constraint, pruneOlderThan(&database, start + 1000));
|
|
|
|
// Nothing survived: not the delete, not the advance, not the row the
|
|
// trigger inserted.
|
|
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
|
try testing.expectEqual(start, try availableSince(&database));
|
|
try testing.expectEqual(
|
|
@as(i64, 0),
|
|
try database.queryInt("SELECT count(*) FROM query_log WHERE client_ip = 'x'"),
|
|
);
|
|
}
|
|
|
|
test "pruneOlderThan leaves the domains dimension table intact" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
try writer.writeBatch(&.{ plainRow(10, "a.example"), plainRow(11, "b.example") });
|
|
try testing.expectEqual(@as(i64, 2), (try pruneOlderThan(&database, 1000)).deleted);
|
|
|
|
try testing.expectEqual(@as(i64, 0), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 2), try countDomains(&database));
|
|
}
|
|
|
|
test "a failing row rolls the whole batch back and the writer survives it" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
try database.exec(
|
|
\\CREATE TRIGGER refuse_boom BEFORE INSERT ON query_log
|
|
\\WHEN new.client_ip = 'boom'
|
|
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
|
);
|
|
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
var doomed = plainRow(20, "second.example");
|
|
doomed.client_ip = "boom";
|
|
try testing.expectError(error.Constraint, writer.writeBatch(&.{
|
|
plainRow(10, "first.example"),
|
|
doomed,
|
|
}));
|
|
|
|
// The interned domain of the row that did insert is gone with it.
|
|
try testing.expectEqual(@as(i64, 0), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 0), try countDomains(&database));
|
|
|
|
try writer.writeBatch(&.{plainRow(30, "third.example")});
|
|
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 1), try countDomains(&database));
|
|
}
|
|
|
|
test "countRows and countDomains agree with what the batches wrote" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
try testing.expectEqual(@as(i64, 0), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 0), try countDomains(&database));
|
|
|
|
var rows: [50]Row = undefined;
|
|
var names: [50][16]u8 = undefined;
|
|
for (&rows, &names, 0..) |*row, *name, i| {
|
|
const written = std.fmt.bufPrint(name, "d{d}.example", .{i % 7}) catch unreachable;
|
|
row.* = plainRow(@intCast(i), written);
|
|
}
|
|
try writer.writeBatch(&rows);
|
|
|
|
try testing.expectEqual(@as(i64, 50), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 7), try countDomains(&database));
|
|
}
|
|
|
|
// `PRAGMA wal_checkpoint` needs a real WAL, which an in-memory database cannot
|
|
// have. `std.testing.tmpDir` creates its directory under `.zig-cache/tmp/`
|
|
// relative to the process working directory, which is also how SQLite's VFS
|
|
// resolves the filename it is handed (`storage_integration_test.zig:44`).
|
|
const tmp_prefix = ".zig-cache/tmp/";
|
|
const sub_path_len = @typeInfo(@FieldType(testing.TmpDir, "sub_path")).array.len;
|
|
|
|
test "checkpointTruncate and vacuum run against a WAL file database" {
|
|
var tmp = testing.tmpDir(.{});
|
|
defer tmp.cleanup();
|
|
|
|
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
|
|
const path = try std.fmt.bufPrintZ(&path_buf, "{s}{s}/querylog.db", .{ tmp_prefix, &tmp.sub_path });
|
|
|
|
var database = try db.Db.open(path, .{ .mode = .read_write_create });
|
|
defer database.close();
|
|
try db.applyPragmas(&database, .{});
|
|
{
|
|
var stmt = try database.prepare("PRAGMA journal_mode");
|
|
defer stmt.deinit();
|
|
try testing.expect(try stmt.step());
|
|
// `columnText` is borrowed until the next call on the statement, so it
|
|
// is compared here rather than carried out of this block.
|
|
try testing.expectEqualStrings("wal", stmt.columnText(0));
|
|
}
|
|
try database.exec(querylog_schema.ddl);
|
|
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
try writer.writeBatch(&.{ plainRow(10, "a.example"), plainRow(20, "b.example") });
|
|
|
|
try checkpointTruncate(&database);
|
|
try testing.expectEqual(@as(i64, 1), (try pruneOlderThan(&database, 20)).deleted);
|
|
try checkpointTruncate(&database);
|
|
try vacuum(&database);
|
|
|
|
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 2), try countDomains(&database));
|
|
}
|
|
|
|
// --- the read layer -------------------------------------------------------
|
|
|
|
/// `BatchWriter` assigns `query_log.id` in the order it is handed the rows, so
|
|
/// every test below knows the id of each seeded row: the nth row of the nth
|
|
/// batch has id n.
|
|
fn seed(database: *db.Db, rows: []const Row) !void {
|
|
var writer = try BatchWriter.init(testing.allocator, database);
|
|
defer writer.deinit();
|
|
try writer.writeBatch(rows);
|
|
}
|
|
|
|
fn ids(rows: []const QueryRow, out: []i64) []const i64 {
|
|
for (rows, 0..) |row, i| out[i] = row.id;
|
|
return out[0..rows.len];
|
|
}
|
|
|
|
test "selectQueries returns the newest row first and reads every column" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
|
|
try seed(&database, &.{
|
|
.{
|
|
.timestamp = 10,
|
|
.domain = "ads.example.net",
|
|
.client_ip = "192.0.2.10",
|
|
.qtype = 28,
|
|
.qclass = 1,
|
|
.rcode = 0,
|
|
.blocked = true,
|
|
.response_time_us = 4200,
|
|
.cache_hit = true,
|
|
.upstream = "https://dns.example/dns-query",
|
|
.group_id = 2,
|
|
.group_name = "kids",
|
|
.policy_action = .block,
|
|
.policy_reason = .blocklist_domain,
|
|
.matched = "ads.example.net",
|
|
.source_id = 5,
|
|
.source_name = "steven black",
|
|
.cname_target = null,
|
|
.safe_search_target = null,
|
|
.route_kind = .blocked,
|
|
.forward_zone = null,
|
|
},
|
|
.{
|
|
.timestamp = 20,
|
|
.domain = "quiet.example",
|
|
.client_ip = "hidden",
|
|
.qtype = null,
|
|
.qclass = 1,
|
|
.rcode = 2,
|
|
.blocked = false,
|
|
.response_time_us = null,
|
|
.cache_hit = null,
|
|
.upstream = null,
|
|
.group_id = null,
|
|
.group_name = null,
|
|
.policy_action = .not_evaluated,
|
|
.policy_reason = .snapshot_unavailable,
|
|
.matched = null,
|
|
.source_id = null,
|
|
.source_name = null,
|
|
.cname_target = null,
|
|
.safe_search_target = null,
|
|
.route_kind = .rejected,
|
|
.forward_zone = null,
|
|
},
|
|
});
|
|
|
|
const rows = try selectQueries(&database, arena_state.allocator(), .{});
|
|
try testing.expectEqual(@as(usize, 2), rows.items.len);
|
|
|
|
const newest = rows.items[0];
|
|
try testing.expectEqual(@as(i64, 2), newest.id);
|
|
try testing.expectEqual(@as(i64, 20), newest.ts);
|
|
try testing.expectEqualStrings("quiet.example", newest.domain);
|
|
try testing.expectEqualStrings("hidden", newest.client_ip);
|
|
try testing.expectEqual(@as(?u16, null), newest.qtype);
|
|
try testing.expectEqual(@as(u16, 1), newest.qclass);
|
|
try testing.expectEqual(@as(u12, 2), newest.rcode);
|
|
try testing.expect(!newest.blocked);
|
|
try testing.expectEqual(@as(?i64, null), newest.response_time_us);
|
|
try testing.expectEqual(@as(?bool, null), newest.cache_hit);
|
|
// A NULL text column reads as the empty string, by documented convention.
|
|
try testing.expectEqualStrings("", newest.upstream);
|
|
try testing.expectEqual(provenance.PolicyAction.not_evaluated, newest.policy_action);
|
|
try testing.expectEqual(provenance.PolicyReason.snapshot_unavailable, newest.policy_reason);
|
|
try testing.expectEqual(provenance.RouteKind.rejected, newest.route_kind);
|
|
|
|
const oldest = rows.items[1];
|
|
try testing.expectEqual(@as(i64, 1), oldest.id);
|
|
try testing.expectEqual(@as(i64, 10), oldest.ts);
|
|
try testing.expectEqualStrings("ads.example.net", oldest.domain);
|
|
try testing.expectEqualStrings("192.0.2.10", oldest.client_ip);
|
|
try testing.expectEqual(@as(?u16, 28), oldest.qtype);
|
|
try testing.expectEqual(@as(u16, 1), oldest.qclass);
|
|
try testing.expectEqual(@as(u12, 0), oldest.rcode);
|
|
try testing.expect(oldest.blocked);
|
|
try testing.expectEqual(@as(?i64, 4200), oldest.response_time_us);
|
|
try testing.expectEqual(@as(?bool, true), oldest.cache_hit);
|
|
try testing.expectEqualStrings("https://dns.example/dns-query", oldest.upstream);
|
|
try testing.expectEqual(provenance.PolicyAction.block, oldest.policy_action);
|
|
try testing.expectEqual(provenance.PolicyReason.blocklist_domain, oldest.policy_reason);
|
|
try testing.expectEqual(provenance.RouteKind.blocked, oldest.route_kind);
|
|
}
|
|
|
|
test "selectQueries honours the limit and caps it at max_limit" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
var rows: [1005]Row = undefined;
|
|
for (&rows, 0..) |*row, i| row.* = plainRow(@intCast(i), "example.com");
|
|
try seed(&database, &rows);
|
|
|
|
const few = try selectQueries(&database, arena, .{ .limit = 3 });
|
|
try testing.expectEqual(@as(usize, 3), few.items.len);
|
|
|
|
// Asked for more than the cap, and for more rows than the cap, so the cap
|
|
// is what bounds the answer rather than the table.
|
|
const capped = try selectQueries(&database, arena, .{ .limit = 5000 });
|
|
try testing.expectEqual(@as(usize, max_limit), capped.items.len);
|
|
|
|
const none = try selectQueries(&database, arena, .{ .limit = 0 });
|
|
try testing.expectEqual(@as(usize, 0), none.items.len);
|
|
}
|
|
|
|
test "keyset paging walks every row exactly once across the page boundaries" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
var seeded: [7]Row = undefined;
|
|
for (&seeded, 0..) |*row, i| row.* = plainRow(@intCast(i), "example.com");
|
|
try seed(&database, &seeded);
|
|
|
|
var seen: std.ArrayList(i64) = .empty;
|
|
defer seen.deinit(testing.allocator);
|
|
|
|
var before: ?i64 = null;
|
|
var pages: usize = 0;
|
|
while (pages < 10) : (pages += 1) {
|
|
const page = try selectQueries(&database, arena, .{ .limit = 3, .before = before });
|
|
if (page.items.len == 0) break;
|
|
for (page.items) |row| try seen.append(testing.allocator, row.id);
|
|
before = page.items[page.items.len - 1].id;
|
|
}
|
|
|
|
// Two full pages and one short page; the fourth call returns nothing and
|
|
// breaks before the counter, which is how the walk knows it is done.
|
|
try testing.expectEqual(@as(usize, 3), pages);
|
|
try testing.expectEqualSlices(i64, &.{ 7, 6, 5, 4, 3, 2, 1 }, seen.items);
|
|
}
|
|
|
|
test "each filter narrows the result on its own" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
var blocked_row = plainRow(200, "ads.example.net");
|
|
blocked_row.client_ip = "192.0.2.20";
|
|
blocked_row.blocked = true;
|
|
blocked_row.policy_action = .block;
|
|
blocked_row.policy_reason = .blocklist_domain;
|
|
blocked_row.route_kind = .blocked;
|
|
try seed(&database, &.{
|
|
plainRow(100, "one.example.com"),
|
|
blocked_row,
|
|
plainRow(300, "two.example.com"),
|
|
});
|
|
|
|
var buf: [8]i64 = undefined;
|
|
|
|
const by_domain = try selectQueries(&database, arena, .{ .domain_substring = "example.com" });
|
|
try testing.expectEqualSlices(i64, &.{ 3, 1 }, ids(by_domain.items, &buf));
|
|
|
|
const by_client = try selectQueries(&database, arena, .{ .clients = &.{"192.0.2.20"} });
|
|
try testing.expectEqualSlices(i64, &.{2}, ids(by_client.items, &buf));
|
|
|
|
// An exact match, not a prefix: the seeded clients share the first octets.
|
|
const no_client = try selectQueries(&database, arena, .{ .clients = &.{"192.0.2"} });
|
|
try testing.expectEqual(@as(usize, 0), no_client.items.len);
|
|
|
|
const only_blocked = try selectQueries(&database, arena, .{ .blocked = true });
|
|
try testing.expectEqualSlices(i64, &.{2}, ids(only_blocked.items, &buf));
|
|
|
|
const only_allowed = try selectQueries(&database, arena, .{ .blocked = false });
|
|
try testing.expectEqualSlices(i64, &.{ 3, 1 }, ids(only_allowed.items, &buf));
|
|
|
|
// Every filter at once, all satisfied by the one blocked row.
|
|
const combined = try selectQueries(&database, arena, .{
|
|
.limit = 10,
|
|
.before = 3,
|
|
.domain_substring = "ads",
|
|
.clients = &.{"192.0.2.20"},
|
|
.blocked = true,
|
|
.since = 200,
|
|
.until = 300,
|
|
});
|
|
try testing.expectEqualSlices(i64, &.{2}, ids(combined.items, &buf));
|
|
}
|
|
|
|
test "since is inclusive, until is exclusive, and an empty range selects nothing" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
try seed(&database, &.{
|
|
plainRow(100, "a.example"),
|
|
plainRow(200, "b.example"),
|
|
plainRow(300, "c.example"),
|
|
});
|
|
|
|
var buf: [8]i64 = undefined;
|
|
|
|
const window = try selectQueries(&database, arena, .{ .since = 100, .until = 300 });
|
|
try testing.expectEqualSlices(i64, &.{ 2, 1 }, ids(window.items, &buf));
|
|
|
|
const after = try selectQueries(&database, arena, .{ .since = 300 });
|
|
try testing.expectEqualSlices(i64, &.{3}, ids(after.items, &buf));
|
|
|
|
const empty = try selectQueries(&database, arena, .{ .since = 300, .until = 300 });
|
|
try testing.expectEqual(@as(usize, 0), empty.items.len);
|
|
|
|
const beyond = try selectQueries(&database, arena, .{ .since = 1000 });
|
|
try testing.expectEqual(@as(usize, 0), beyond.items.len);
|
|
}
|
|
|
|
test "a domain substring matches % and _ as literal characters" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
try seed(&database, &.{
|
|
plainRow(10, "a_b.example"),
|
|
plainRow(20, "axb.example"),
|
|
plainRow(30, "a%b.example"),
|
|
plainRow(40, "azzb.example"),
|
|
plainRow(50, "back\\slash.example"),
|
|
});
|
|
|
|
var buf: [8]i64 = undefined;
|
|
|
|
// Unescaped, `_` is LIKE's single-character wildcard and would also match
|
|
// "axb"; escaped, it matches only the underscore.
|
|
const underscore = try selectQueries(&database, arena, .{ .domain_substring = "a_b" });
|
|
try testing.expectEqualSlices(i64, &.{1}, ids(underscore.items, &buf));
|
|
|
|
// Unescaped, `%` would match everything from "a" to "b", so "azzb" too.
|
|
const percent = try selectQueries(&database, arena, .{ .domain_substring = "a%b" });
|
|
try testing.expectEqualSlices(i64, &.{3}, ids(percent.items, &buf));
|
|
|
|
// The escape character escapes itself, so it is searchable as well.
|
|
const backslash = try selectQueries(&database, arena, .{ .domain_substring = "k\\s" });
|
|
try testing.expectEqualSlices(i64, &.{5}, ids(backslash.items, &buf));
|
|
|
|
// An empty needle is `%%`, which matches every row rather than none.
|
|
const all = try selectQueries(&database, arena, .{ .domain_substring = "" });
|
|
try testing.expectEqual(@as(usize, 5), all.items.len);
|
|
}
|
|
|
|
test "the built SQL never carries a filter value and fits its buffer" {
|
|
var sql: Sql = .{};
|
|
sql.put(select_head);
|
|
sql.predicate(where_before);
|
|
sql.predicate(where_domain);
|
|
sql.clientPredicate(max_clients);
|
|
sql.predicate(where_blocked);
|
|
sql.predicate(where_since);
|
|
sql.predicate(where_until);
|
|
sql.put(select_tail);
|
|
|
|
// Every predicate present, with the client list at its cap, is the longest
|
|
// reachable statement — which is what the buffer is sized against.
|
|
try testing.expect(sql.len <= Sql.capacity);
|
|
try testing.expectEqual(@as(usize, 1), std.mem.count(u8, sql.text(), " WHERE"));
|
|
try testing.expectEqual(@as(usize, 5), std.mem.count(u8, sql.text(), " AND"));
|
|
// Five scalar filters plus the LIMIT, plus one per address, each a bare
|
|
// parameter: no value is ever spelled into the statement.
|
|
try testing.expectEqual(@as(usize, 6 + max_clients), std.mem.count(u8, sql.text(), "?"));
|
|
// The select list has commas of its own, so the client list is measured as
|
|
// the difference against the same statement without it.
|
|
var without: Sql = .{};
|
|
without.put(select_head);
|
|
without.predicate(where_before);
|
|
without.predicate(where_domain);
|
|
without.predicate(where_blocked);
|
|
without.predicate(where_since);
|
|
without.predicate(where_until);
|
|
without.put(select_tail);
|
|
try testing.expectEqual(
|
|
@as(usize, max_clients - 1),
|
|
std.mem.count(u8, sql.text(), ",") - std.mem.count(u8, without.text(), ","),
|
|
);
|
|
}
|
|
|
|
test "one client reads as an exact match and several read as any-of" {
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
var database = try openLog();
|
|
defer database.close();
|
|
|
|
var second = plainRow(200, "b.example");
|
|
second.client_ip = "192.0.2.11";
|
|
var third = plainRow(300, "c.example");
|
|
third.client_ip = "192.0.2.12";
|
|
try seed(&database, &.{ plainRow(100, "a.example"), second, third });
|
|
|
|
var buf: [8]i64 = undefined;
|
|
|
|
const one = try selectQueries(&database, arena, .{ .clients = &.{"192.0.2.11"} });
|
|
try testing.expectEqualSlices(i64, &.{2}, ids(one.items, &buf));
|
|
|
|
// Newest-first, so the higher id leads however the addresses are ordered.
|
|
const two = try selectQueries(&database, arena, .{ .clients = &.{ "192.0.2.12", "192.0.2.10" } });
|
|
try testing.expectEqualSlices(i64, &.{ 3, 1 }, ids(two.items, &buf));
|
|
|
|
// An address nothing was logged from narrows to nothing rather than being
|
|
// ignored, which is the difference between a filter and a suggestion.
|
|
const absent = try selectQueries(&database, arena, .{ .clients = &.{"198.51.100.1"} });
|
|
try testing.expectEqual(@as(usize, 0), absent.items.len);
|
|
|
|
// No addresses at all is no client filter.
|
|
const none = try selectQueries(&database, arena, .{});
|
|
try testing.expectEqual(@as(usize, 3), none.items.len);
|
|
}
|
|
|
|
test "a client list past the cap is refused rather than truncated" {
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
var database = try openLog();
|
|
defer database.close();
|
|
|
|
var too_many: [max_clients + 1][]const u8 = undefined;
|
|
for (&too_many) |*slot| slot.* = "192.0.2.10";
|
|
|
|
// Dropping the overflow would narrow the filter silently, which answers a
|
|
// question the caller did not ask; the statement buffer could not hold it
|
|
// either way.
|
|
try testing.expectError(error.Misuse, selectQueries(&database, arena, .{ .clients = &too_many }));
|
|
}
|
|
|
|
test "likePattern wraps the needle and neutralises every metacharacter" {
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
try testing.expectEqualStrings("%plain%", try likePattern(arena, "plain"));
|
|
try testing.expectEqualStrings("%a\\_b%", try likePattern(arena, "a_b"));
|
|
try testing.expectEqualStrings("%a\\%b%", try likePattern(arena, "a%b"));
|
|
try testing.expectEqualStrings("%a\\\\b%", try likePattern(arena, "a\\b"));
|
|
try testing.expectEqualStrings("%%", try likePattern(arena, ""));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// period aggregations (milestone 30)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const agg_since: i64 = 1_700_000_000;
|
|
const agg_width: u32 = 60;
|
|
const agg_buckets: u32 = 10;
|
|
|
|
/// One row of the aggregation fixtures. Everything the three breakdowns read
|
|
/// is a parameter; everything else is the same on every row, so a test that
|
|
/// changes an outcome names the reason it changed.
|
|
fn aggRow(offset: i64, client: []const u8, qtype: ?u16, kind: provenance.RouteKind, source: ?[]const u8) Row {
|
|
return .{
|
|
.timestamp = agg_since + offset,
|
|
.domain = "example.com",
|
|
.client_ip = client,
|
|
.qtype = qtype,
|
|
.qclass = 1,
|
|
.rcode = 0,
|
|
.blocked = kind == .blocked,
|
|
.response_time_us = 1000,
|
|
.cache_hit = kind == .cache,
|
|
.upstream = if (kind == .upstream) source else null,
|
|
.group_id = 1,
|
|
.group_name = "default",
|
|
.policy_action = if (kind == .blocked) .block else .allow,
|
|
.policy_reason = if (kind == .blocked) .blocklist_domain else .no_match,
|
|
.matched = null,
|
|
.source_id = null,
|
|
// Blocklist provenance, deliberately set on every row: the routes
|
|
// breakdown must never group by it.
|
|
.source_name = "StevenBlack",
|
|
.cname_target = null,
|
|
.safe_search_target = null,
|
|
.route_kind = kind,
|
|
.forward_zone = if (kind == .forward_zone) source else null,
|
|
};
|
|
}
|
|
|
|
test "the type breakdown groups by qtype, keeps the null row and orders it last" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
try seed(&database, &.{
|
|
aggRow(0, "192.0.2.10", 1, .upstream, "9.9.9.9"),
|
|
aggRow(1, "192.0.2.10", 1, .upstream, "9.9.9.9"),
|
|
aggRow(2, "192.0.2.10", 1, .upstream, "9.9.9.9"),
|
|
aggRow(3, "192.0.2.10", 28, .upstream, "9.9.9.9"),
|
|
aggRow(4, "192.0.2.10", 28, .upstream, "9.9.9.9"),
|
|
// Ties with qtype 28, so the tie-break puts the lower code first.
|
|
aggRow(5, "192.0.2.10", 16, .upstream, "9.9.9.9"),
|
|
aggRow(6, "192.0.2.10", 16, .upstream, "9.9.9.9"),
|
|
// A row with no type at all: its own group, never a dropped row.
|
|
aggRow(7, "192.0.2.10", null, .upstream, "9.9.9.9"),
|
|
aggRow(8, "192.0.2.10", null, .upstream, "9.9.9.9"),
|
|
// Outside the window.
|
|
aggRow(-1, "192.0.2.10", 255, .upstream, "9.9.9.9"),
|
|
});
|
|
|
|
const rows = (try overview(&database, arena, agg_since, agg_width, agg_buckets)).types;
|
|
try testing.expectEqual(@as(usize, 4), rows.len);
|
|
try testing.expectEqual(@as(?u16, 1), rows[0].qtype);
|
|
try testing.expectEqual(@as(u64, 3), rows[0].count);
|
|
try testing.expectEqual(@as(?u16, 16), rows[1].qtype);
|
|
try testing.expectEqual(@as(?u16, 28), rows[2].qtype);
|
|
try testing.expectEqual(@as(?u16, null), rows[3].qtype);
|
|
try testing.expectEqual(@as(u64, 2), rows[3].count);
|
|
}
|
|
|
|
test "an empty window has no type rows at all" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
|
|
const rows = (try overview(&database, arena_state.allocator(), agg_since, agg_width, agg_buckets)).types;
|
|
try testing.expectEqual(@as(usize, 0), rows.len);
|
|
}
|
|
|
|
test "the route breakdown keys on the answering resolver, not on blocklist provenance" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
try seed(&database, &.{
|
|
aggRow(0, "192.0.2.10", 1, .upstream, "https://a.example/dns-query"),
|
|
aggRow(1, "192.0.2.10", 1, .upstream, "https://a.example/dns-query"),
|
|
aggRow(2, "192.0.2.10", 1, .upstream, "https://b.example/dns-query"),
|
|
// An upstream row whose resolver the log did not record: its own group.
|
|
aggRow(3, "192.0.2.10", 1, .upstream, null),
|
|
aggRow(4, "192.0.2.10", 1, .forward_zone, "lan"),
|
|
aggRow(5, "192.0.2.10", 1, .blocked, null),
|
|
aggRow(6, "192.0.2.10", 1, .cache, null),
|
|
aggRow(7, "192.0.2.10", 1, .local, null),
|
|
aggRow(8, "192.0.2.10", 1, .rejected, null),
|
|
});
|
|
|
|
const rows = (try overview(&database, arena, agg_since, agg_width, agg_buckets)).routes;
|
|
// Two upstreams, one null-source upstream, one forward zone and four
|
|
// source-less kinds. Every row carries the same `source_name`, so a
|
|
// breakdown that grouped by it would collapse to one row.
|
|
try testing.expectEqual(@as(usize, 8), rows.len);
|
|
|
|
try testing.expectEqual(provenance.RouteKind.upstream, rows[0].route);
|
|
try testing.expectEqualStrings("https://a.example/dns-query", rows[0].source.?);
|
|
try testing.expectEqual(@as(u64, 2), rows[0].count);
|
|
|
|
// The seven remaining rows all count 1, so the tie-break orders them:
|
|
// route ascending, then source ascending with nulls last.
|
|
for (rows[1..]) |row| try testing.expectEqual(@as(u64, 1), row.count);
|
|
try testing.expectEqual(provenance.RouteKind.blocked, rows[1].route);
|
|
try testing.expectEqual(@as(?[]const u8, null), rows[1].source);
|
|
try testing.expectEqual(provenance.RouteKind.cache, rows[2].route);
|
|
try testing.expectEqual(provenance.RouteKind.forward_zone, rows[3].route);
|
|
try testing.expectEqualStrings("lan", rows[3].source.?);
|
|
try testing.expectEqual(provenance.RouteKind.local, rows[4].route);
|
|
try testing.expectEqual(provenance.RouteKind.rejected, rows[5].route);
|
|
try testing.expectEqual(provenance.RouteKind.upstream, rows[6].route);
|
|
try testing.expectEqualStrings("https://b.example/dns-query", rows[6].source.?);
|
|
try testing.expectEqual(provenance.RouteKind.upstream, rows[7].route);
|
|
try testing.expectEqual(@as(?[]const u8, null), rows[7].source);
|
|
}
|
|
|
|
test "an empty window has no route rows at all" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
|
|
const rows = (try overview(&database, arena_state.allocator(), agg_since, agg_width, agg_buckets)).routes;
|
|
try testing.expectEqual(@as(usize, 0), rows.len);
|
|
}
|
|
|
|
test "an empty window still has a zero-filled other series and no named clients" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
|
|
const result = (try overview(&database, arena_state.allocator(), agg_since, agg_width, agg_buckets)).clients;
|
|
try testing.expectEqual(@as(usize, 0), result.clients.len);
|
|
try testing.expectEqual(@as(usize, agg_buckets), result.other.len);
|
|
for (result.other) |count| try testing.expectEqual(@as(u64, 0), count);
|
|
}
|
|
|
|
test "client series are bucket-aligned, zero-filled and ranked by in-window total" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
|
|
try seed(&database, &.{
|
|
aggRow(0, "192.0.2.20", 1, .upstream, "9.9.9.9"),
|
|
aggRow(1, "192.0.2.20", 1, .upstream, "9.9.9.9"),
|
|
aggRow(agg_width * 3, "192.0.2.20", 1, .upstream, "9.9.9.9"),
|
|
aggRow(agg_width * 3, "192.0.2.10", 1, .upstream, "9.9.9.9"),
|
|
// Outside the window on both sides.
|
|
aggRow(-1, "192.0.2.20", 1, .upstream, "9.9.9.9"),
|
|
aggRow(agg_width * agg_buckets, "192.0.2.10", 1, .upstream, "9.9.9.9"),
|
|
});
|
|
|
|
const result = (try overview(&database, arena_state.allocator(), agg_since, agg_width, agg_buckets)).clients;
|
|
try testing.expectEqual(@as(usize, 2), result.clients.len);
|
|
try testing.expectEqualStrings("192.0.2.20", result.clients[0].client);
|
|
try testing.expectEqualStrings("192.0.2.10", result.clients[1].client);
|
|
|
|
for (result.clients) |series| try testing.expectEqual(@as(usize, agg_buckets), series.buckets.len);
|
|
try testing.expectEqual(@as(u64, 2), result.clients[0].buckets[0]);
|
|
try testing.expectEqual(@as(u64, 1), result.clients[0].buckets[3]);
|
|
try testing.expectEqual(@as(u64, 0), result.clients[0].buckets[9]);
|
|
try testing.expectEqual(@as(u64, 1), result.clients[1].buckets[3]);
|
|
for (result.other) |count| try testing.expectEqual(@as(u64, 0), count);
|
|
}
|
|
|
|
test "the ninth client folds into other and the cut is the same on every read" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
// Nine clients, each with one more query than the next, so the ranking is
|
|
// total and the ninth is unambiguously the one that folds.
|
|
var address: [16]u8 = undefined;
|
|
var index: u32 = 0;
|
|
while (index < 9) : (index += 1) {
|
|
const client = try std.fmt.bufPrint(&address, "192.0.2.{d}", .{100 + index});
|
|
var repeat: u32 = 0;
|
|
while (repeat <= index) : (repeat += 1) {
|
|
try seed(&database, &.{aggRow(@intCast(repeat), client, 1, .upstream, "9.9.9.9")});
|
|
}
|
|
}
|
|
|
|
const result = (try overview(&database, arena, agg_since, agg_width, agg_buckets)).clients;
|
|
try testing.expectEqual(@as(usize, max_client_series), result.clients.len);
|
|
// The busiest is 192.0.2.108 with nine rows; the lone folded client is
|
|
// 192.0.2.100 with one.
|
|
try testing.expectEqualStrings("192.0.2.108", result.clients[0].client);
|
|
for (result.clients) |series| {
|
|
try testing.expect(!std.mem.eql(u8, "192.0.2.100", series.client));
|
|
}
|
|
|
|
var other_total: u64 = 0;
|
|
for (result.other) |count| other_total += count;
|
|
try testing.expectEqual(@as(u64, 1), other_total);
|
|
}
|
|
|
|
test "the three breakdowns conserve the window's total" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
// A matrix that exercises every branch the aggregations partition on: null
|
|
// and non-null qtypes, every route kind, a null resolver identity, and ten
|
|
// clients so the top-eight cut has a residual to carry.
|
|
const kinds = [_]provenance.RouteKind{ .blocked, .local, .forward_zone, .upstream, .cache, .rejected };
|
|
var address: [16]u8 = undefined;
|
|
var index: u32 = 0;
|
|
while (index < 40) : (index += 1) {
|
|
const client = try std.fmt.bufPrint(&address, "192.0.2.{d}", .{100 + index % 10});
|
|
const kind = kinds[index % kinds.len];
|
|
try seed(&database, &.{aggRow(
|
|
@intCast(index % (agg_width * agg_buckets)),
|
|
client,
|
|
if (index % 7 == 0) null else @intCast(1 + index % 3),
|
|
kind,
|
|
if (index % 11 == 0) null else "9.9.9.9",
|
|
)});
|
|
}
|
|
|
|
const result = try overview(&database, arena, agg_since, agg_width, agg_buckets);
|
|
try testing.expect(result.totals.queries > 0);
|
|
|
|
var typed: u64 = 0;
|
|
for (result.types) |row| typed += row.count;
|
|
try testing.expectEqual(result.totals.queries, typed);
|
|
|
|
var routed: u64 = 0;
|
|
for (result.routes) |row| routed += row.count;
|
|
try testing.expectEqual(result.totals.queries, routed);
|
|
|
|
// Per bucket, not just in total: a series misaligned by one bucket would
|
|
// still sum correctly over the window.
|
|
for (result.buckets, 0..) |bucket, at| {
|
|
var summed: u64 = result.clients.other[at];
|
|
for (result.clients.clients) |series| summed += series.buckets[at];
|
|
try testing.expectEqual(bucket.queries, summed);
|
|
}
|
|
}
|
|
|
|
test "the aggregations pass a redacted client through as the log stored it" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
|
|
// `hide_client_ips` is applied by the logger before the row is written, so
|
|
// the read path has nothing to transform — and must not invent anything
|
|
// either. The marker is the client's whole identity here.
|
|
try seed(&database, &.{
|
|
aggRow(0, logger.hidden_marker, 1, .upstream, "9.9.9.9"),
|
|
aggRow(1, logger.hidden_marker, 1, .upstream, "9.9.9.9"),
|
|
});
|
|
|
|
const result = (try overview(&database, arena_state.allocator(), agg_since, agg_width, agg_buckets)).clients;
|
|
try testing.expectEqual(@as(usize, 1), result.clients.len);
|
|
try testing.expectEqualStrings(logger.hidden_marker, result.clients[0].client);
|
|
try testing.expectEqual(@as(u64, 2), result.clients[0].buckets[0]);
|
|
}
|
|
|
|
test "a prune committed mid-read is invisible to the reader's transaction" {
|
|
var tmp = testing.tmpDir(.{});
|
|
defer tmp.cleanup();
|
|
|
|
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
|
|
const path = try std.fmt.bufPrintZ(&path_buf, "{s}{s}/querylog.db", .{ tmp_prefix, &tmp.sub_path });
|
|
|
|
var reader = try db.Db.open(path, .{ .mode = .read_write_create });
|
|
defer reader.close();
|
|
try db.applyPragmas(&reader, .{});
|
|
try reader.exec(querylog_schema.ddl);
|
|
// The seeded watermark is the file's creation second, which is now; the
|
|
// fixtures below are in 2023, so the prune's cutoff would never advance it.
|
|
try reader.exec(
|
|
\\UPDATE querylog_meta SET created_at = 1700000000, available_since = 1700000000 WHERE id = 1
|
|
);
|
|
try seed(&reader, &.{
|
|
plainRow(agg_since, "a.example"),
|
|
plainRow(agg_since + 1, "b.example"),
|
|
plainRow(agg_since + 300, "c.example"),
|
|
});
|
|
|
|
// A second connection to the same file, as retention has in production.
|
|
var pruner = try db.Db.open(path, .{ .mode = .read_write_create });
|
|
defer pruner.close();
|
|
try db.applyPragmas(&pruner, .{});
|
|
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
var tx = try db.ReadTx.begin(&reader);
|
|
const before = (try overview(&reader, arena, agg_since, 1000, 1)).totals;
|
|
const watermark_before = try availableSince(&reader);
|
|
try testing.expectEqual(@as(u64, 3), before.queries);
|
|
|
|
// The prune commits while the reader's transaction is open.
|
|
const pruned = try pruneOlderThan(&pruner, agg_since + 200);
|
|
try testing.expectEqual(@as(i64, 2), pruned.deleted);
|
|
try testing.expect(pruned.available_since > watermark_before);
|
|
|
|
// Neither half of the answer moved: the rows the reader would report and
|
|
// the watermark it would tag them with still describe one state.
|
|
const during = (try overview(&reader, arena, agg_since, 1000, 1)).totals;
|
|
try testing.expectEqual(before.queries, during.queries);
|
|
try testing.expectEqual(watermark_before, try availableSince(&reader));
|
|
try tx.commit();
|
|
|
|
// The next response sees the prune — both halves of it.
|
|
const after = (try overview(&reader, arena, agg_since, 1000, 1)).totals;
|
|
try testing.expectEqual(@as(u64, 1), after.queries);
|
|
try testing.expectEqual(pruned.available_since, try availableSince(&reader));
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// the projections and the Overview read path (milestone 36)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Floors a `query_log.timestamp` onto the projection grid in SQL. SQLite's `/`
|
|
/// truncates toward zero, so a negative timestamp needs the bias — the same
|
|
/// reason `bucketOf` is `@divFloor`.
|
|
const bucket_expr = "(CASE WHEN timestamp >= 0 THEN timestamp / 1800 * 1800" ++
|
|
" ELSE (timestamp - 1799) / 1800 * 1800 END)";
|
|
|
|
/// Each projection table beside a from-scratch recomputation of it. Both sides
|
|
/// have unique keys, so `EXCEPT` in both directions is an exact equality test
|
|
/// rather than a containment one.
|
|
const recompute_checks = [_]struct { projection: []const u8, recompute: []const u8 }{
|
|
.{
|
|
.projection = "SELECT bucket, queries, blocked, cached, rt_sum, rt_count FROM bucket_totals",
|
|
.recompute = "SELECT " ++ bucket_expr ++ ", count(*), coalesce(sum(blocked <> 0), 0)," ++
|
|
" coalesce(sum(cache_hit = 1), 0), coalesce(sum(response_time_us), 0)," ++
|
|
" count(response_time_us) FROM query_log GROUP BY 1",
|
|
},
|
|
.{
|
|
.projection = "SELECT bucket, client_ip, queries FROM bucket_clients",
|
|
.recompute = "SELECT " ++ bucket_expr ++ ", client_ip, count(*) FROM query_log GROUP BY 1, 2",
|
|
},
|
|
.{
|
|
.projection = "SELECT bucket, qtype, count FROM bucket_types",
|
|
.recompute = "SELECT " ++ bucket_expr ++ ", coalesce(qtype, -1), count(*) FROM query_log GROUP BY 1, 2",
|
|
},
|
|
.{
|
|
.projection = "SELECT bucket, route_kind, source_present, source_text, count FROM bucket_routes",
|
|
.recompute = "SELECT " ++ bucket_expr ++ ", route_kind, source IS NOT NULL, coalesce(source, ''), count(*)" ++
|
|
" FROM (SELECT timestamp, route_kind, CASE route_kind WHEN 'upstream' THEN upstream" ++
|
|
" WHEN 'forward_zone' THEN forward_zone END AS source FROM query_log) GROUP BY 1, 2, 3, 4",
|
|
},
|
|
};
|
|
|
|
/// Exported for `querylog_migrations.zig`'s fixture and migration tests: the
|
|
/// authority on projection coherence is this file, and a second copy of the
|
|
/// recompute SQL there would be free to drift from the writer it checks.
|
|
pub fn expectProjectionsMatchRecompute(database: *db.Db) !void {
|
|
for (recompute_checks) |check| {
|
|
var buf: [4096]u8 = undefined;
|
|
const sql = try std.fmt.bufPrint(
|
|
&buf,
|
|
"SELECT (SELECT count(*) FROM ({s} EXCEPT {s})) + (SELECT count(*) FROM ({s} EXCEPT {s}))",
|
|
.{ check.projection, check.recompute, check.recompute, check.projection },
|
|
);
|
|
try testing.expectEqual(@as(i64, 0), try database.queryInt(sql));
|
|
}
|
|
}
|
|
|
|
test "bucketOf floors onto the grid on both sides of the epoch" {
|
|
try testing.expectEqual(@as(i64, 0), bucketOf(0));
|
|
try testing.expectEqual(@as(i64, 0), bucketOf(1799));
|
|
try testing.expectEqual(@as(i64, grain), bucketOf(grain));
|
|
try testing.expectEqual(@as(i64, grain), bucketOf(grain + 1));
|
|
// Truncation toward zero would name bucket 0 for all three of these, which
|
|
// is the bucket *after* the one the row belongs to.
|
|
try testing.expectEqual(@as(i64, -grain), bucketOf(-1));
|
|
try testing.expectEqual(@as(i64, -grain), bucketOf(-grain));
|
|
try testing.expectEqual(@as(i64, -2 * grain), bucketOf(-grain - 1));
|
|
}
|
|
|
|
test "a batch maintains all four projections in the transaction that writes the rows" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
var blocked_row = aggRow(0, "192.0.2.20", null, .blocked, null);
|
|
blocked_row.response_time_us = null;
|
|
try writer.writeBatch(&.{
|
|
aggRow(0, "192.0.2.10", 1, .upstream, "9.9.9.9"),
|
|
aggRow(5, "192.0.2.10", 1, .cache, null),
|
|
blocked_row,
|
|
// The next grid bucket, so the floor is what separates them.
|
|
aggRow(grain, "192.0.2.10", 28, .upstream, null),
|
|
});
|
|
|
|
// Two grid buckets: three rows floored into the first, one into the second.
|
|
try testing.expectEqual(@as(i64, 2), try database.queryInt("SELECT count(*) FROM bucket_totals"));
|
|
try testing.expectEqual(bucketOf(agg_since), try database.queryInt("SELECT min(bucket) FROM bucket_totals"));
|
|
try testing.expectEqual(@as(i64, 3), try database.queryInt("SELECT queries FROM bucket_totals ORDER BY bucket LIMIT 1"));
|
|
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT blocked FROM bucket_totals ORDER BY bucket LIMIT 1"));
|
|
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT cached FROM bucket_totals ORDER BY bucket LIMIT 1"));
|
|
// The blocked row carries no response time, so two of the three are timed.
|
|
try testing.expectEqual(@as(i64, 2), try database.queryInt("SELECT rt_count FROM bucket_totals ORDER BY bucket LIMIT 1"));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
|
|
// A second batch accumulates onto the same keys rather than replacing them.
|
|
try writer.writeBatch(&.{aggRow(7, "192.0.2.10", 1, .upstream, "9.9.9.9")});
|
|
try testing.expectEqual(@as(i64, 4), try database.queryInt("SELECT queries FROM bucket_totals ORDER BY bucket LIMIT 1"));
|
|
try testing.expectEqual(
|
|
@as(i64, 3),
|
|
try database.queryInt("SELECT queries FROM bucket_clients ORDER BY bucket, client_ip LIMIT 1"),
|
|
);
|
|
try expectProjectionsMatchRecompute(&database);
|
|
}
|
|
|
|
test "a NULL qtype and a NULL route source are stored losslessly" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
try writer.writeBatch(&.{
|
|
aggRow(0, "192.0.2.10", null, .upstream, null),
|
|
aggRow(1, "192.0.2.10", 1, .upstream, ""),
|
|
});
|
|
|
|
try testing.expectEqual(@as(i64, null_qtype), try database.queryInt("SELECT min(qtype) FROM bucket_types"));
|
|
// A row whose resolver the log did not record, and a row whose resolver is
|
|
// the empty string, are different keys — which is what `source_present`
|
|
// buys over "empty means absent".
|
|
try testing.expectEqual(@as(i64, 2), try database.queryInt("SELECT count(*) FROM bucket_routes"));
|
|
try testing.expectEqual(
|
|
@as(i64, 1),
|
|
try database.queryInt("SELECT count(*) FROM bucket_routes WHERE source_present = 0 AND source_text = ''"),
|
|
);
|
|
try testing.expectEqual(
|
|
@as(i64, 1),
|
|
try database.queryInt("SELECT count(*) FROM bucket_routes WHERE source_present = 1 AND source_text = ''"),
|
|
);
|
|
try expectProjectionsMatchRecompute(&database);
|
|
}
|
|
|
|
test "a failed projection update rolls the whole batch back and the writer survives it" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
try writer.writeBatch(&.{aggRow(0, "192.0.2.10", 1, .upstream, "9.9.9.9")});
|
|
try database.exec(
|
|
\\CREATE TRIGGER refuse_projection BEFORE INSERT ON bucket_clients
|
|
\\WHEN new.client_ip = 'boom'
|
|
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
|
);
|
|
|
|
try testing.expectError(error.Constraint, writer.writeBatch(&.{
|
|
aggRow(1, "boom", 1, .upstream, "9.9.9.9"),
|
|
}));
|
|
|
|
// The raw row went in before the projection statement failed; neither
|
|
// survived, because they were one transaction.
|
|
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
|
|
try database.exec("DROP TRIGGER refuse_projection;");
|
|
try writer.writeBatch(&.{aggRow(2, "192.0.2.11", 1, .upstream, "9.9.9.9")});
|
|
try testing.expectEqual(@as(i64, 2), try countRows(&database));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
}
|
|
|
|
test "a prune on the grid drops whole projection buckets and leaves the rest" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
const base = bucketOf(agg_since);
|
|
try seedAt(&database, &.{ base, base + grain, base + 2 * grain });
|
|
|
|
_ = try pruneOlderThan(&database, base + 2 * grain);
|
|
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM bucket_totals"));
|
|
try testing.expectEqual(base + 2 * grain, try database.queryInt("SELECT bucket FROM bucket_totals"));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
}
|
|
|
|
test "a prune off the grid recomputes the straddling bucket instead of estimating it" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
|
|
const base = bucketOf(agg_since);
|
|
// Four rows in one bucket; the cutoff falls between the second and third.
|
|
try seedAt(&database, &.{ base, base + 100, base + 500, base + 900 });
|
|
|
|
_ = try pruneOlderThan(&database, base + 400);
|
|
try testing.expectEqual(@as(i64, 2), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM bucket_totals"));
|
|
try testing.expectEqual(@as(i64, 2), try database.queryInt("SELECT queries FROM bucket_totals"));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
|
|
// Emptying the straddling bucket removes its row rather than writing zeros.
|
|
_ = try pruneOlderThan(&database, base + grain - 1);
|
|
try testing.expectEqual(@as(i64, 0), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 0), try database.queryInt("SELECT count(*) FROM bucket_totals"));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
}
|
|
|
|
test "a failed projection prune rolls back the raw delete and the watermark with it" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
|
|
const base = bucketOf(agg_since);
|
|
try seedAt(&database, &.{ base, base + grain });
|
|
const watermark = try availableSince(&database);
|
|
|
|
try database.exec(
|
|
\\CREATE TRIGGER refuse_bucket_prune BEFORE DELETE ON bucket_clients
|
|
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
|
);
|
|
|
|
try testing.expectError(error.Constraint, pruneOlderThan(&database, base + grain));
|
|
|
|
try testing.expectEqual(@as(i64, 2), try countRows(&database));
|
|
try testing.expectEqual(watermark, try availableSince(&database));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
|
|
try database.exec("DROP TRIGGER refuse_bucket_prune;");
|
|
_ = try pruneOlderThan(&database, base + grain);
|
|
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
}
|
|
|
|
test "a failed straddling-bucket replacement rolls back the raw delete and the watermark with it" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
|
|
const base = bucketOf(agg_since);
|
|
// One whole bucket to delete outright, and one the cutoff cuts in half.
|
|
try seedAt(&database, &.{ base, base + grain, base + grain + 100, base + grain + 900 });
|
|
const watermark = try availableSince(&database);
|
|
const straddling_totals = "SELECT queries FROM bucket_totals ORDER BY bucket DESC LIMIT 1";
|
|
const straddling_queries = try database.queryInt(straddling_totals);
|
|
|
|
// Only the recompute inserts into a projection table; the bulk prune above
|
|
// it deletes. So this trigger fires during the straddling-bucket
|
|
// replacement and nowhere else in the pass.
|
|
try database.exec(
|
|
\\CREATE TRIGGER refuse_recompute BEFORE INSERT ON bucket_totals
|
|
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
|
);
|
|
|
|
try testing.expectError(error.Constraint, pruneOlderThan(&database, base + grain + 500));
|
|
|
|
// The raw delete, the watermark advance, the whole-bucket projection
|
|
// deletes and the straddling bucket's own delete all went back together.
|
|
try testing.expectEqual(@as(i64, 4), try countRows(&database));
|
|
try testing.expectEqual(watermark, try availableSince(&database));
|
|
try testing.expectEqual(@as(i64, 2), try database.queryInt("SELECT count(*) FROM bucket_totals"));
|
|
try testing.expectEqual(straddling_queries, try database.queryInt(straddling_totals));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
|
|
try database.exec("DROP TRIGGER refuse_recompute;");
|
|
_ = try pruneOlderThan(&database, base + grain + 500);
|
|
try testing.expectEqual(@as(i64, 1), try countRows(&database));
|
|
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT queries FROM bucket_totals"));
|
|
try expectProjectionsMatchRecompute(&database);
|
|
}
|
|
|
|
/// One row per timestamp, all in the same batch, with the fixture defaults.
|
|
fn seedAt(database: *db.Db, timestamps: []const i64) !void {
|
|
var writer = try BatchWriter.init(testing.allocator, database);
|
|
defer writer.deinit();
|
|
|
|
var rows: [16]Row = undefined;
|
|
for (rows[0..timestamps.len], timestamps) |*row, ts| {
|
|
row.* = plainRow(ts, "example.com");
|
|
}
|
|
try writer.writeBatch(rows[0..timestamps.len]);
|
|
}
|
|
|
|
test "an arbitrary interleaving of batches and prunes leaves every projection exact" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
var prng: std.Random.DefaultPrng = .init(0x36a5_0000_36a5);
|
|
const random = prng.random();
|
|
|
|
const kinds = [_]provenance.RouteKind{ .blocked, .local, .forward_zone, .upstream, .cache, .rejected };
|
|
const sources = [_]?[]const u8{ "9.9.9.9", "https://a.example/dns-query", null, "" };
|
|
const base = bucketOf(agg_since);
|
|
|
|
var rows: [24]Row = undefined;
|
|
var addresses: [24][20]u8 = undefined;
|
|
|
|
var step: usize = 0;
|
|
while (step < 60) : (step += 1) {
|
|
if (step % 5 == 4) {
|
|
// Half the cutoffs land off the grid on purpose: the straddling
|
|
// bucket is the only case a delete alone cannot get right.
|
|
const offset = random.intRangeAtMost(i64, 0, 6 * grain);
|
|
const skew: i64 = if (random.boolean()) 0 else random.intRangeLessThan(i64, 1, grain);
|
|
_ = try pruneOlderThan(&database, base + offset + skew);
|
|
} else {
|
|
const count = random.intRangeAtMost(usize, 1, rows.len);
|
|
for (rows[0..count], addresses[0..count]) |*row, *address| {
|
|
const client = try std.fmt.bufPrint(address, "192.0.2.{d}", .{random.intRangeAtMost(u8, 1, 5)});
|
|
const kind = kinds[random.uintLessThan(usize, kinds.len)];
|
|
row.* = plainRow(base + random.intRangeAtMost(i64, 0, 8 * grain), "example.com");
|
|
row.client_ip = client;
|
|
row.qtype = if (random.boolean()) null else random.intRangeAtMost(u16, 1, 3);
|
|
row.blocked = kind == .blocked;
|
|
row.cache_hit = if (random.boolean()) null else kind == .cache;
|
|
row.response_time_us = if (random.boolean()) null else random.intRangeAtMost(i64, 0, 5000);
|
|
row.route_kind = kind;
|
|
const source = sources[random.uintLessThan(usize, sources.len)];
|
|
row.upstream = if (kind == .upstream) source else null;
|
|
row.forward_zone = if (kind == .forward_zone) source else null;
|
|
row.policy_action = if (kind == .blocked) .block else .allow;
|
|
row.policy_reason = if (kind == .blocked) .blocklist_domain else .no_match;
|
|
}
|
|
try writer.writeBatch(rows[0..count]);
|
|
}
|
|
try expectProjectionsMatchRecompute(&database);
|
|
}
|
|
|
|
// The walk has to have exercised both halves of it.
|
|
try testing.expect((try countRows(&database)) > 0);
|
|
}
|
|
|
|
// --- the Overview equivalence oracle --------------------------------------
|
|
|
|
/// A test-only copy of the five SQL aggregates `overview` replaces, kept here
|
|
/// so it survives their deletion. It is the independent statement of the
|
|
/// contract: if `overview` and this disagree on any window, one of them is
|
|
/// wrong, and this one is the one the goldens were written against.
|
|
const oracle = struct {
|
|
const totals_sql =
|
|
\\SELECT count(*),
|
|
\\ coalesce(sum(blocked <> 0), 0),
|
|
\\ count(DISTINCT client_ip),
|
|
\\ coalesce(sum(response_time_us), 0),
|
|
\\ count(response_time_us)
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?2
|
|
;
|
|
|
|
const buckets_sql =
|
|
\\SELECT (timestamp - ?1) / ?2,
|
|
\\ count(*),
|
|
\\ coalesce(sum(blocked <> 0), 0),
|
|
\\ coalesce(sum(cache_hit = 1), 0)
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?3
|
|
\\ GROUP BY 1
|
|
;
|
|
|
|
const types_sql =
|
|
\\SELECT qtype, count(*)
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?2
|
|
\\ GROUP BY qtype
|
|
\\ ORDER BY count(*) DESC, qtype IS NULL, qtype ASC
|
|
;
|
|
|
|
const routes_sql =
|
|
\\SELECT route_kind,
|
|
\\ CASE route_kind
|
|
\\ WHEN 'upstream' THEN upstream
|
|
\\ WHEN 'forward_zone' THEN forward_zone
|
|
\\ END AS source,
|
|
\\ count(*)
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?2
|
|
\\ GROUP BY route_kind, source
|
|
\\ ORDER BY count(*) DESC, route_kind ASC, source IS NULL, source ASC
|
|
;
|
|
|
|
const clients_rank_sql =
|
|
\\SELECT client_ip
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?2
|
|
\\ GROUP BY client_ip
|
|
\\ ORDER BY count(*) DESC, client_ip ASC
|
|
\\ LIMIT ?3
|
|
;
|
|
|
|
const clients_buckets_sql =
|
|
\\SELECT client_ip, (timestamp - ?1) / ?2, count(*)
|
|
\\ FROM query_log
|
|
\\ WHERE timestamp >= ?1 AND timestamp < ?3
|
|
\\ GROUP BY 1, 2
|
|
;
|
|
|
|
fn build(
|
|
database: *db.Db,
|
|
arena: Allocator,
|
|
since: i64,
|
|
bucket_seconds: u32,
|
|
bucket_count: u32,
|
|
) !Overview {
|
|
const width: i64 = bucket_seconds;
|
|
const until = since + width * bucket_count;
|
|
|
|
var totals: StatsTotals = undefined;
|
|
{
|
|
var stmt = try database.prepare(totals_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, until);
|
|
try testing.expect(try stmt.step());
|
|
const timed = stmt.columnInt(4);
|
|
totals = .{
|
|
.queries = try countOf(stmt.columnInt(0)),
|
|
.blocked = try countOf(stmt.columnInt(1)),
|
|
.distinct_clients = try countOf(stmt.columnInt(2)),
|
|
.avg_response_time_us = if (timed == 0) null else @divTrunc(stmt.columnInt(3), timed),
|
|
};
|
|
}
|
|
|
|
const buckets = try arena.alloc(Bucket, bucket_count);
|
|
for (buckets, 0..) |*bucket, i| bucket.* = .{
|
|
.ts = since + width * @as(i64, @intCast(i)),
|
|
.queries = 0,
|
|
.blocked = 0,
|
|
.cached = 0,
|
|
};
|
|
{
|
|
var stmt = try database.prepare(buckets_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, width);
|
|
try stmt.bindInt(3, until);
|
|
while (try stmt.step()) {
|
|
const index: usize = @intCast(stmt.columnInt(0));
|
|
buckets[index].queries = try countOf(stmt.columnInt(1));
|
|
buckets[index].blocked = try countOf(stmt.columnInt(2));
|
|
buckets[index].cached = try countOf(stmt.columnInt(3));
|
|
}
|
|
}
|
|
|
|
var types: std.ArrayList(TypeCount) = .empty;
|
|
{
|
|
var stmt = try database.prepare(types_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, until);
|
|
while (try stmt.step()) {
|
|
try types.append(arena, .{
|
|
.qtype = if (stmt.isNull(0)) null else try columnU16(&stmt, 0),
|
|
.count = try countOf(stmt.columnInt(1)),
|
|
});
|
|
}
|
|
}
|
|
|
|
var routes: std.ArrayList(RouteCount) = .empty;
|
|
{
|
|
var stmt = try database.prepare(routes_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, until);
|
|
while (try stmt.step()) {
|
|
try routes.append(arena, .{
|
|
.route = try provenance.parse(provenance.RouteKind, stmt.columnText(0)),
|
|
.source = try stmt.columnTextAllocOrNull(arena, 1),
|
|
.count = try countOf(stmt.columnInt(2)),
|
|
});
|
|
}
|
|
}
|
|
|
|
var names: std.ArrayList([]const u8) = .empty;
|
|
var series: std.ArrayList([]u64) = .empty;
|
|
{
|
|
var stmt = try database.prepare(clients_rank_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, until);
|
|
try stmt.bindInt(3, max_client_series);
|
|
while (try stmt.step()) {
|
|
try names.append(arena, try stmt.columnTextAlloc(arena, 0));
|
|
try series.append(arena, try zeroedBuckets(arena, bucket_count));
|
|
}
|
|
}
|
|
const other = try zeroedBuckets(arena, bucket_count);
|
|
{
|
|
var stmt = try database.prepare(clients_buckets_sql);
|
|
defer stmt.deinit();
|
|
try stmt.bindInt(1, since);
|
|
try stmt.bindInt(2, width);
|
|
try stmt.bindInt(3, until);
|
|
while (try stmt.step()) {
|
|
const index: usize = @intCast(stmt.columnInt(1));
|
|
const count = try countOf(stmt.columnInt(2));
|
|
const client = stmt.columnText(0);
|
|
const target = for (names.items, series.items) |name, target_buckets| {
|
|
if (std.mem.eql(u8, name, client)) break target_buckets;
|
|
} else other;
|
|
target[index] += count;
|
|
}
|
|
}
|
|
const clients = try arena.alloc(ClientSeries, names.items.len);
|
|
for (clients, names.items, series.items) |*entry, name, entry_buckets| {
|
|
entry.* = .{ .client = name, .buckets = entry_buckets };
|
|
}
|
|
|
|
return .{
|
|
.totals = totals,
|
|
.buckets = buckets,
|
|
.clients = .{ .clients = clients, .other = other },
|
|
.types = types.items,
|
|
.routes = routes.items,
|
|
};
|
|
}
|
|
};
|
|
|
|
fn expectOverviewEqual(expected: Overview, actual: Overview) !void {
|
|
try testing.expectEqual(expected.totals.queries, actual.totals.queries);
|
|
try testing.expectEqual(expected.totals.blocked, actual.totals.blocked);
|
|
try testing.expectEqual(expected.totals.distinct_clients, actual.totals.distinct_clients);
|
|
try testing.expectEqual(expected.totals.avg_response_time_us, actual.totals.avg_response_time_us);
|
|
try testing.expectEqualSlices(Bucket, expected.buckets, actual.buckets);
|
|
|
|
try testing.expectEqual(expected.types.len, actual.types.len);
|
|
for (expected.types, actual.types) |want, got| {
|
|
try testing.expectEqual(want.qtype, got.qtype);
|
|
try testing.expectEqual(want.count, got.count);
|
|
}
|
|
|
|
try testing.expectEqual(expected.routes.len, actual.routes.len);
|
|
for (expected.routes, actual.routes) |want, got| {
|
|
try testing.expectEqual(want.route, got.route);
|
|
try testing.expectEqual(want.count, got.count);
|
|
if (want.source) |source| {
|
|
try testing.expectEqualStrings(source, got.source orelse return error.TestExpectedEqual);
|
|
} else {
|
|
try testing.expectEqual(@as(?[]const u8, null), got.source);
|
|
}
|
|
}
|
|
|
|
try testing.expectEqual(expected.clients.clients.len, actual.clients.clients.len);
|
|
for (expected.clients.clients, actual.clients.clients) |want, got| {
|
|
try testing.expectEqualStrings(want.client, got.client);
|
|
try testing.expectEqualSlices(u64, want.buckets, got.buckets);
|
|
}
|
|
try testing.expectEqualSlices(u64, expected.clients.other, actual.clients.other);
|
|
}
|
|
|
|
/// Every window the equivalence test walks. The first is the raw path (60 s
|
|
/// buckets, the 1h period); the rest are the projection path at each serving
|
|
/// width the period grammar offers.
|
|
const overview_windows = [_]struct { bucket_seconds: u32, bucket_count: u32 }{
|
|
.{ .bucket_seconds = 60, .bucket_count = 60 },
|
|
.{ .bucket_seconds = 1800, .bucket_count = 48 },
|
|
.{ .bucket_seconds = 3600, .bucket_count = 168 },
|
|
.{ .bucket_seconds = 21600, .bucket_count = 120 },
|
|
};
|
|
|
|
fn expectOverviewMatchesOracle(database: *db.Db, since: i64) !void {
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
for (overview_windows) |window| {
|
|
const want = try oracle.build(database, arena, since, window.bucket_seconds, window.bucket_count);
|
|
const got = try overview(database, arena, since, window.bucket_seconds, window.bucket_count);
|
|
expectOverviewEqual(want, got) catch |err| {
|
|
std.debug.print("overview mismatch at bucket_seconds={d}\n", .{window.bucket_seconds});
|
|
return err;
|
|
};
|
|
}
|
|
}
|
|
|
|
test "overview equals the oracle on an empty database" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
try expectOverviewMatchesOracle(&database, bucketOf(agg_since));
|
|
}
|
|
|
|
test "overview equals the oracle over a window with no rows in it" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
try seedAt(&database, &.{ agg_since, agg_since + 10 });
|
|
|
|
// Two full 30-day spans later: every panel is empty and the axis is still
|
|
// whole.
|
|
try expectOverviewMatchesOracle(&database, bucketOf(agg_since) + 240 * grain);
|
|
}
|
|
|
|
test "overview equals the oracle across nulls, ties and a bucket still in progress" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
const since = bucketOf(agg_since);
|
|
const kinds = [_]provenance.RouteKind{ .blocked, .local, .forward_zone, .upstream, .cache, .rejected };
|
|
|
|
// Ten clients so the top-eight cut has a residual, and the counts are
|
|
// deliberately equal in pairs so the `client_ip` tie-break decides.
|
|
var rows: [120]Row = undefined;
|
|
var addresses: [120][20]u8 = undefined;
|
|
for (&rows, &addresses, 0..) |*row, *address, i| {
|
|
const client = try std.fmt.bufPrint(address, "192.0.2.{d}", .{100 + (i / 2) % 10});
|
|
const kind = kinds[i % kinds.len];
|
|
row.* = plainRow(since + @as(i64, @intCast(i)) * 137, "example.com");
|
|
row.client_ip = client;
|
|
// Every seventh row carries no query type, and the types collide in
|
|
// count so the null-last tie-break is exercised.
|
|
row.qtype = if (i % 7 == 0) null else @intCast(1 + i % 3);
|
|
row.blocked = kind == .blocked;
|
|
row.cache_hit = if (i % 5 == 0) null else kind == .cache;
|
|
row.response_time_us = if (i % 4 == 0) null else @intCast(100 + i);
|
|
row.route_kind = kind;
|
|
// Every eleventh upstream row records no resolver: the NULL-source
|
|
// group the routes breakdown must keep rather than drop.
|
|
const source: ?[]const u8 = if (i % 11 == 0) null else if (i % 2 == 0) "9.9.9.9" else "https://a.example/dns-query";
|
|
row.upstream = if (kind == .upstream) source else null;
|
|
row.forward_zone = if (kind == .forward_zone) source else null;
|
|
row.policy_action = if (kind == .blocked) .block else .allow;
|
|
row.policy_reason = if (kind == .blocked) .blocklist_domain else .no_match;
|
|
}
|
|
try writer.writeBatch(&rows);
|
|
|
|
try expectOverviewMatchesOracle(&database, since);
|
|
|
|
// A window whose last bucket is only partly filled: `since` moved so the
|
|
// newest rows land inside the final bucket rather than closing it.
|
|
try expectOverviewMatchesOracle(&database, since - 40 * grain);
|
|
|
|
// And after a prune off the grid, so the projections the read path uses are
|
|
// the recomputed ones.
|
|
_ = try pruneOlderThan(&database, since + 900);
|
|
try expectProjectionsMatchRecompute(&database);
|
|
try expectOverviewMatchesOracle(&database, since);
|
|
}
|
|
|
|
test "overview equals the oracle after an interleaving of batches and prunes" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var writer = try BatchWriter.init(testing.allocator, &database);
|
|
defer writer.deinit();
|
|
|
|
const since = bucketOf(agg_since);
|
|
var prng: std.Random.DefaultPrng = .init(0x0e17_3a99);
|
|
const random = prng.random();
|
|
|
|
var rows: [16]Row = undefined;
|
|
var addresses: [16][20]u8 = undefined;
|
|
var step: usize = 0;
|
|
while (step < 12) : (step += 1) {
|
|
const count = random.intRangeAtMost(usize, 1, rows.len);
|
|
for (rows[0..count], addresses[0..count]) |*row, *address| {
|
|
const client = try std.fmt.bufPrint(address, "10.0.0.{d}", .{random.intRangeAtMost(u8, 1, 12)});
|
|
row.* = plainRow(since + random.intRangeAtMost(i64, 0, 20 * grain), "example.com");
|
|
row.client_ip = client;
|
|
row.qtype = if (random.boolean()) null else random.intRangeAtMost(u16, 1, 4);
|
|
row.response_time_us = if (random.boolean()) null else random.intRangeAtMost(i64, 0, 9000);
|
|
row.cache_hit = if (random.boolean()) null else random.boolean();
|
|
}
|
|
try writer.writeBatch(rows[0..count]);
|
|
if (step % 4 == 3) {
|
|
_ = try pruneOlderThan(&database, since + random.intRangeAtMost(i64, 0, 10 * grain));
|
|
}
|
|
try expectOverviewMatchesOracle(&database, since);
|
|
}
|
|
}
|
|
|
|
test "overview refuses a window the projections cannot express" {
|
|
var database = try openLog();
|
|
defer database.close();
|
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
const since = bucketOf(agg_since);
|
|
|
|
try testing.expectError(error.Misuse, overview(&database, arena, since, 0, 48));
|
|
try testing.expectError(error.Misuse, overview(&database, arena, since, 1800, 0));
|
|
try testing.expectError(error.Misuse, overview(&database, arena, std.math.maxInt(i64) - 1, 3600, 48));
|
|
|
|
// On the projection path the grid is part of the contract: an unaligned
|
|
// start or a width that is not a whole number of grains cannot be answered
|
|
// from 30-minute rows, and guessing would be worse than refusing.
|
|
try testing.expectError(error.Misuse, overview(&database, arena, since + 1, 1800, 48));
|
|
try testing.expectError(error.Misuse, overview(&database, arena, since, 2700, 48));
|
|
|
|
// Below the grain none of that applies: the raw rows carry every second.
|
|
_ = try overview(&database, arena, since + 1, 60, 60);
|
|
}
|