milestone 8: web server, rest api, sse, auth, metrics and static assets
This commit is contained in:
@@ -0,0 +1,340 @@
|
||||
//! `GET /api/stats` and `GET /api/stats/timeseries` (ruling 13).
|
||||
//!
|
||||
//! One period grammar, four widths, and one window shared by both endpoints:
|
||||
//! the totals cover exactly the span the chart draws, so a dashboard cannot
|
||||
//! show a sum that disagrees with the bars above it.
|
||||
//!
|
||||
//! Buckets are aligned to the UTC grid, not to the moment of the request. Every
|
||||
//! width divides a day, so flooring the current time to a multiple of the width
|
||||
//! puts each bucket on the same boundary a human reads off a clock, and two
|
||||
//! requests a second apart return the same bucket starts. The last bucket is
|
||||
//! the one in progress; it fills as the period runs.
|
||||
//!
|
||||
//! The aggregates run on the web task's own query-log connection (m7 ruling 21).
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const db = @import("../../storage/db.zig");
|
||||
const http_util = @import("../http_util.zig");
|
||||
const queries_repo = @import("../../storage/repositories/queries_repo.zig");
|
||||
const server = @import("../server.zig");
|
||||
|
||||
const log = std.log.scoped(.web_stats);
|
||||
|
||||
/// The four periods ruling 13 defines. The tag names are the wire spellings.
|
||||
pub const Period = enum {
|
||||
@"1h",
|
||||
@"24h",
|
||||
@"7d",
|
||||
@"30d",
|
||||
|
||||
pub fn parse(text: []const u8) ?Period {
|
||||
return std.meta.stringToEnum(Period, text);
|
||||
}
|
||||
|
||||
/// Ruling 13: 1h→60×1m, 24h→48×30m, 7d→168×1h, 30d→120×6h.
|
||||
pub fn bucketSeconds(self: Period) u32 {
|
||||
return switch (self) {
|
||||
.@"1h" => 60,
|
||||
.@"24h" => 30 * 60,
|
||||
.@"7d" => 60 * 60,
|
||||
.@"30d" => 6 * 60 * 60,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn bucketCount(self: Period) u32 {
|
||||
return switch (self) {
|
||||
.@"1h" => 60,
|
||||
.@"24h" => 48,
|
||||
.@"7d" => 168,
|
||||
.@"30d" => 120,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn label(self: Period) []const u8 {
|
||||
return @tagName(self);
|
||||
}
|
||||
};
|
||||
|
||||
pub const default_period: Period = .@"24h";
|
||||
|
||||
/// The widest period's bucket count, so one stack array serves every request.
|
||||
pub const max_buckets = 168;
|
||||
|
||||
comptime {
|
||||
for (std.enums.values(Period)) |period| {
|
||||
std.debug.assert(period.bucketCount() <= max_buckets);
|
||||
// The UTC alignment argument holds only while every width divides a day.
|
||||
std.debug.assert(86_400 % period.bucketSeconds() == 0);
|
||||
}
|
||||
}
|
||||
|
||||
pub const Window = struct {
|
||||
/// Inclusive, on the bucket grid.
|
||||
since: i64,
|
||||
/// Exclusive: the end of the bucket that `now` falls in.
|
||||
until: i64,
|
||||
bucket_seconds: u32,
|
||||
bucket_count: u32,
|
||||
};
|
||||
|
||||
pub fn window(period: Period, now_unix: i64) Window {
|
||||
const width: i64 = period.bucketSeconds();
|
||||
const count: i64 = period.bucketCount();
|
||||
const until = @divFloor(now_unix, width) * width + width;
|
||||
return .{
|
||||
.since = until - width * count,
|
||||
.until = until,
|
||||
.bucket_seconds = period.bucketSeconds(),
|
||||
.bucket_count = period.bucketCount(),
|
||||
};
|
||||
}
|
||||
|
||||
pub const TotalsBody = struct {
|
||||
period: []const u8,
|
||||
since: i64,
|
||||
until: i64,
|
||||
queries: u64,
|
||||
blocked: u64,
|
||||
cached: u64,
|
||||
clients: u64,
|
||||
avg_response_time_us: ?i64,
|
||||
};
|
||||
|
||||
pub const TimeseriesBody = struct {
|
||||
period: []const u8,
|
||||
since: i64,
|
||||
until: i64,
|
||||
bucket_seconds: u32,
|
||||
buckets: []const queries_repo.Bucket,
|
||||
};
|
||||
|
||||
pub fn totals(
|
||||
state: *server.WebState,
|
||||
io: std.Io,
|
||||
request: *http_util.Request,
|
||||
) http_util.HandlerError!void {
|
||||
const period = periodParam(request.query) catch return badPeriod(request);
|
||||
const database = state.querylog_db orelse return unavailable(request);
|
||||
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
|
||||
|
||||
const result = queries_repo.statsTotals(database, span.since, span.until) catch |err| {
|
||||
return internal(request, "stats totals", err);
|
||||
};
|
||||
|
||||
return http_util.respondJson(request, .ok, TotalsBody{
|
||||
.period = period.label(),
|
||||
.since = span.since,
|
||||
.until = span.until,
|
||||
.queries = result.queries,
|
||||
.blocked = result.blocked,
|
||||
.cached = result.cached,
|
||||
.clients = result.distinct_clients,
|
||||
.avg_response_time_us = result.avg_response_time_us,
|
||||
}, &.{});
|
||||
}
|
||||
|
||||
pub fn timeseries(
|
||||
state: *server.WebState,
|
||||
io: std.Io,
|
||||
request: *http_util.Request,
|
||||
) http_util.HandlerError!void {
|
||||
const period = periodParam(request.query) catch return badPeriod(request);
|
||||
const database = state.querylog_db orelse return unavailable(request);
|
||||
const span = window(period, std.Io.Clock.real.now(io).toSeconds());
|
||||
|
||||
var buckets: [max_buckets]queries_repo.Bucket = undefined;
|
||||
const out = buckets[0..span.bucket_count];
|
||||
const written = queries_repo.timeseries(database, span.since, span.bucket_seconds, out) catch |err| {
|
||||
return internal(request, "stats timeseries", err);
|
||||
};
|
||||
|
||||
return http_util.respondJson(request, .ok, TimeseriesBody{
|
||||
.period = period.label(),
|
||||
.since = span.since,
|
||||
.until = span.until,
|
||||
.bucket_seconds = span.bucket_seconds,
|
||||
.buckets = out[0..written],
|
||||
}, &.{});
|
||||
}
|
||||
|
||||
pub const PeriodError = error{BadPeriod};
|
||||
|
||||
/// An absent `period` is the default; anything else it cannot read is a 400,
|
||||
/// never a silent fallback — a typo must not return a window nobody asked for.
|
||||
pub fn periodParam(query: []const u8) PeriodError!Period {
|
||||
var buf: [8]u8 = undefined;
|
||||
const found = http_util.queryValue(query, "period", &buf) catch return error.BadPeriod;
|
||||
const text = found orelse return default_period;
|
||||
return Period.parse(text) orelse error.BadPeriod;
|
||||
}
|
||||
|
||||
fn badPeriod(request: *http_util.Request) http_util.HandlerError!void {
|
||||
return http_util.respondError(request, .bad_request, "period must be one of 1h, 24h, 7d, 30d");
|
||||
}
|
||||
|
||||
fn unavailable(request: *http_util.Request) http_util.HandlerError!void {
|
||||
return http_util.respondError(request, .service_unavailable, "query log unavailable");
|
||||
}
|
||||
|
||||
/// The one thing this file logs. A failed aggregate is a fault in the box, not
|
||||
/// a property of the request, and the client is told nothing beyond "internal
|
||||
/// error" (ruling 8, PLAN §19).
|
||||
fn internal(
|
||||
request: *http_util.Request,
|
||||
what: []const u8,
|
||||
err: db.Error,
|
||||
) http_util.HandlerError!void {
|
||||
log.warn("{s} failed: {s}", .{ what, @errorName(err) });
|
||||
return http_util.respondError(request, .internal_server_error, "internal error");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const querylog_schema = @import("../../storage/querylog_schema.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
test "the period grammar accepts exactly the four spellings" {
|
||||
try testing.expectEqual(Period.@"1h", Period.parse("1h").?);
|
||||
try testing.expectEqual(Period.@"24h", Period.parse("24h").?);
|
||||
try testing.expectEqual(Period.@"7d", Period.parse("7d").?);
|
||||
try testing.expectEqual(Period.@"30d", Period.parse("30d").?);
|
||||
try testing.expectEqual(@as(?Period, null), Period.parse("12h"));
|
||||
try testing.expectEqual(@as(?Period, null), Period.parse("1H"));
|
||||
try testing.expectEqual(@as(?Period, null), Period.parse(""));
|
||||
}
|
||||
|
||||
test "an absent period defaults and a bad one is rejected" {
|
||||
try testing.expectEqual(default_period, try periodParam(""));
|
||||
try testing.expectEqual(default_period, try periodParam("limit=5"));
|
||||
try testing.expectEqual(Period.@"7d", try periodParam("period=7d"));
|
||||
try testing.expectError(error.BadPeriod, periodParam("period=12h"));
|
||||
try testing.expectError(error.BadPeriod, periodParam("period=%2"));
|
||||
// Longer than any spelling: rejected rather than truncated to "1h".
|
||||
try testing.expectError(error.BadPeriod, periodParam("period=1hhhhhhhhhh"));
|
||||
}
|
||||
|
||||
test "each period spans its own bucket width times its count" {
|
||||
for (std.enums.values(Period)) |period| {
|
||||
const span = window(period, 1_700_000_000);
|
||||
const width: i64 = period.bucketSeconds();
|
||||
try testing.expectEqual(width * @as(i64, period.bucketCount()), span.until - span.since);
|
||||
}
|
||||
}
|
||||
|
||||
test "the window sits on the UTC grid and ends with the bucket in progress" {
|
||||
// 2023-11-14T22:13:20Z, which is not on any bucket boundary.
|
||||
const now: i64 = 1_700_000_000;
|
||||
const span = window(.@"24h", now);
|
||||
|
||||
try testing.expectEqual(@as(i64, 0), @rem(span.since, 1800));
|
||||
try testing.expectEqual(@as(i64, 0), @rem(span.until, 1800));
|
||||
try testing.expect(span.until > now);
|
||||
try testing.expect(span.until - now <= 1800);
|
||||
try testing.expectEqual(@as(u32, 48), span.bucket_count);
|
||||
}
|
||||
|
||||
test "two requests inside one bucket see the same window" {
|
||||
// A bucket boundary, so the offsets below stay inside one minute.
|
||||
const boundary: i64 = 1_700_000_000 - @rem(1_700_000_000, 60);
|
||||
const first = window(.@"1h", boundary);
|
||||
const second = window(.@"1h", boundary + 59);
|
||||
try testing.expectEqual(first.since, second.since);
|
||||
try testing.expectEqual(first.until, second.until);
|
||||
|
||||
const next = window(.@"1h", boundary + 60);
|
||||
try testing.expectEqual(first.until + 60, next.until);
|
||||
}
|
||||
|
||||
test "a timestamp exactly on a boundary starts a new bucket" {
|
||||
const span = window(.@"7d", 1_700_000_000 - 1_700_000_000 % 3600);
|
||||
try testing.expectEqual(@as(i64, 0), @rem(span.since, 3600));
|
||||
try testing.expectEqual(@as(u32, 168), span.bucket_count);
|
||||
}
|
||||
|
||||
fn openLog() !db.Db {
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
errdefer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
try database.exec(querylog_schema.ddl);
|
||||
return database;
|
||||
}
|
||||
|
||||
fn writeRow(writer: *queries_repo.BatchWriter, timestamp: i64, blocked: bool, cached: ?bool) !void {
|
||||
const rows = [_]queries_repo.Row{.{
|
||||
.timestamp = timestamp,
|
||||
.domain = "example.com",
|
||||
.client_ip = "192.0.2.10",
|
||||
.qtype = 1,
|
||||
.blocked = blocked,
|
||||
.block_reason = if (blocked) "blocklist_domain" else null,
|
||||
.response_time_us = 1000,
|
||||
.cache_hit = cached,
|
||||
.upstream = null,
|
||||
}};
|
||||
try writer.writeBatch(&rows);
|
||||
}
|
||||
|
||||
test "the totals and the buckets agree over the same window" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
const now: i64 = 1_700_000_000;
|
||||
const span = window(.@"1h", now);
|
||||
|
||||
var writer = try queries_repo.BatchWriter.init(&database);
|
||||
defer writer.deinit();
|
||||
// One row in the first bucket, two in the last, one just outside.
|
||||
try writeRow(&writer, span.since, false, false);
|
||||
try writeRow(&writer, span.until - 1, true, false);
|
||||
try writeRow(&writer, span.until - 2, false, true);
|
||||
try writeRow(&writer, span.since - 1, false, false);
|
||||
|
||||
const result = try queries_repo.statsTotals(&database, span.since, span.until);
|
||||
try testing.expectEqual(@as(u64, 3), result.queries);
|
||||
try testing.expectEqual(@as(u64, 1), result.blocked);
|
||||
try testing.expectEqual(@as(u64, 1), result.cached);
|
||||
try testing.expectEqual(@as(u64, 1), result.distinct_clients);
|
||||
try testing.expectEqual(@as(?i64, 1000), result.avg_response_time_us);
|
||||
|
||||
var buckets: [max_buckets]queries_repo.Bucket = undefined;
|
||||
const out = buckets[0..span.bucket_count];
|
||||
const written = try queries_repo.timeseries(&database, span.since, span.bucket_seconds, out);
|
||||
try testing.expectEqual(@as(usize, 60), written);
|
||||
|
||||
var summed: u64 = 0;
|
||||
var blocked: u64 = 0;
|
||||
for (out) |bucket| {
|
||||
summed += bucket.queries;
|
||||
blocked += bucket.blocked;
|
||||
}
|
||||
try testing.expectEqual(result.queries, summed);
|
||||
try testing.expectEqual(result.blocked, blocked);
|
||||
|
||||
try testing.expectEqual(span.since, out[0].ts);
|
||||
try testing.expectEqual(@as(u64, 1), out[0].queries);
|
||||
try testing.expectEqual(@as(u64, 2), out[59].queries);
|
||||
try testing.expectEqual(span.until - span.bucket_seconds, out[59].ts);
|
||||
}
|
||||
|
||||
test "an empty window reports zeros with a null mean" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
const span = window(.@"30d", 1_700_000_000);
|
||||
const result = try queries_repo.statsTotals(&database, span.since, span.until);
|
||||
try testing.expectEqual(@as(u64, 0), result.queries);
|
||||
try testing.expectEqual(@as(?i64, null), result.avg_response_time_us);
|
||||
|
||||
var buckets: [max_buckets]queries_repo.Bucket = undefined;
|
||||
const out = buckets[0..span.bucket_count];
|
||||
try testing.expectEqual(@as(usize, 120), try queries_repo.timeseries(
|
||||
&database,
|
||||
span.since,
|
||||
span.bucket_seconds,
|
||||
out,
|
||||
));
|
||||
for (out) |bucket| try testing.expectEqual(@as(u64, 0), bucket.queries);
|
||||
}
|
||||
Reference in New Issue
Block a user