milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable

src/web/handlers/diagnostics.zig

@@ -0,0 +1,448 @@
+//! `GET /api/diagnostics` and `GET /api/diagnostics/{id}` — the operational
+//! event log, newest first.
+//!
+//! Keyset pagination and the same page envelope as `/api/queries`, for the same
+//! reason: the table is append-only at the head, so `id < before` is one index
+//! seek however deep a client has scrolled, and rows arriving between two pages
+//! cannot shift the window and duplicate one.
+//!
+//! Filter parsing is separated from fetching, because parsing is where PLAN
+//! §19's input validation lives and it is worth testing on its own. Every value
+//! is length-capped here and bound as a SQL parameter by the repository;
+//! nothing this file reads is ever concatenated into a statement.
+//!
+//! No SQL and no connection of its own: `events.Store` owns the one diagnostics
+//! connection and locks its mutex around every read, so this file cannot race
+//! the emitters writing through it.
+
+const std = @import("std");
+
+const db = @import("../../storage/db.zig");
+const events = @import("../../storage/events.zig");
+const http_util = @import("../http_util.zig");
+const server = @import("../server.zig");
+
+const log = std.log.scoped(.web_diagnostics);
+
+pub const default_limit: u32 = 100;
+pub const max_limit: u32 = events.max_limit;
+
+/// Wide enough for every component `events.component` can produce, and for a
+/// mistyped one to still be reported as a bad filter rather than a truncated
+/// match.
+pub const max_component_len = 64;
+
+/// Where the string filters are copied to. The parsed filter borrows them, so
+/// it must not outlive the buffers — in the handler both live in the same stack
+/// frame.
+pub const Buffers = struct {
+    state: [16]u8 = undefined,
+    severity: [16]u8 = undefined,
+    component: [max_component_len]u8 = undefined,
+};
+
+pub const FilterError = error{
+    BadState,
+    BadSeverity,
+    BadComponent,
+    BadSince,
+    BadUntil,
+    BadLimit,
+    BadBefore,
+};
+
+/// An absent parameter drops the filter; a malformed one is a 400 rather than a
+/// filter silently left off, which would answer a question the client did not
+/// ask.
+pub fn parseFilter(query: []const u8, buffers: *Buffers) FilterError!events.Filter {
+    var filter: events.Filter = .{ .limit = default_limit };
+
+    if (http_util.queryValue(query, "state", &buffers.state) catch return error.BadState) |text| {
+        filter.state = std.meta.stringToEnum(events.State, text) orelse return error.BadState;
+    }
+
+    if (http_util.queryValue(query, "severity", &buffers.severity) catch return error.BadSeverity) |text| {
+        // Bound as text by the repository, so it is normalised to one of the
+        // two stored spellings here rather than passed through.
+        const severity = std.meta.stringToEnum(events.Severity, text) orelse return error.BadSeverity;
+        filter.severity = severity.text();
+    }
+
+    if (http_util.queryValue(query, "component", &buffers.component) catch return error.BadComponent) |text| {
+        if (text.len != 0) filter.component = text;
+    }
+
+    filter.since = http_util.queryInt(i64, query, "since") catch return error.BadSince;
+    filter.until = http_util.queryInt(i64, query, "until") catch return error.BadUntil;
+
+    if (http_util.queryInt(u32, query, "limit") catch return error.BadLimit) |limit| {
+        if (limit == 0 or limit > max_limit) return error.BadLimit;
+        filter.limit = limit;
+    }
+
+    if (http_util.queryInt(i64, query, "before") catch return error.BadBefore) |before| {
+        // Row ids are positive, so a non-positive cursor is a client bug, not
+        // an empty page.
+        if (before <= 0) return error.BadBefore;
+        filter.before = before;
+    }
+
+    return filter;
+}
+
+pub fn message(err: FilterError) []const u8 {
+    return switch (err) {
+        error.BadState => "state must be active, resolved or all",
+        error.BadSeverity => "severity must be warning or error",
+        error.BadComponent => "component is not a valid filter",
+        error.BadSince => "since must be a unix timestamp in seconds",
+        error.BadUntil => "until must be a unix timestamp in seconds",
+        error.BadLimit => "limit must be between 1 and 1000",
+        error.BadBefore => "before must be a positive row id",
+    };
+}
+
+const unavailable_message = "diagnostics unavailable";
+
+pub fn list(
+    state: *server.WebState,
+    io: std.Io,
+    request: *http_util.Request,
+) http_util.HandlerError!void {
+    var buffers: Buffers = .{};
+    const filter = parseFilter(request.query, &buffers) catch |err| {
+        return http_util.respondError(request, .bad_request, message(err));
+    };
+
+    const store = state.events orelse
+        return http_util.respondError(request, .service_unavailable, unavailable_message);
+
+    const page = store.selectEvents(io, request.arena, filter) catch |err| {
+        // The one thing this handler logs: a database fault is a property of
+        // the box, not of the request, and the client is told nothing about it.
+        log.warn("diagnostics read failed: {s}", .{@errorName(err)});
+        return http_util.respondError(request, .internal_server_error, "internal error");
+    };
+
+    return http_util.respondJson(request, .ok, page, &.{});
+}
+
+pub fn get(
+    state: *server.WebState,
+    io: std.Io,
+    request: *http_util.Request,
+) http_util.HandlerError!void {
+    const store = state.events orelse
+        return http_util.respondError(request, .service_unavailable, unavailable_message);
+
+    const row = store.selectOne(io, request.arena, request.id.?) catch |err| {
+        log.warn("diagnostics read failed: {s}", .{@errorName(err)});
+        return http_util.respondError(request, .internal_server_error, "internal error");
+    };
+
+    // An id retention has removed and one that never existed are the same
+    // answer, and the API does not pretend to tell them apart.
+    const event = row orelse return http_util.respondError(request, .not_found, "not found");
+    return http_util.respondJson(request, .ok, event, &.{});
+}
+
+/// An open episode is the current state of the box, so it is not history to
+/// throw away — and the message says what would make it purgeable.
+const still_active_message = "the event is still active; it can be purged once it resolves";
+
+/// `DELETE /api/diagnostics` — how many resolved events went.
+pub const PurgeResult = struct {
+    purged: i64,
+};
+
+/// `DELETE /api/diagnostics/{id}`. Resolution stays automatic; this is only
+/// about when the history disappears, which is the operator's call.
+///
+/// Classified `runtime_action` in the route table, not `config_write`: the
+/// event log is runtime state that no configuration file declares, so file
+/// authority has nothing to say about it and the router lets this through in
+/// both modes.
+pub fn purge(
+    state: *server.WebState,
+    io: std.Io,
+    request: *http_util.Request,
+) http_util.HandlerError!void {
+    const store = state.events orelse
+        return http_util.respondError(request, .service_unavailable, unavailable_message);
+
+    // No log here: the store already latches and logs the false→true
+    // transition, and a warning per retried request would spam.
+    const outcome = store.purge(io, request.id.?) catch
+        return http_util.respondError(request, .internal_server_error, "internal error");
+
+    return switch (outcome) {
+        .deleted => http_util.respondEmpty(request, .no_content),
+        .active => http_util.respondError(request, .conflict, still_active_message),
+        .absent => http_util.respondError(request, .not_found, "not found"),
+    };
+}
+
+/// `DELETE /api/diagnostics` — the whole resolved history at once. Active
+/// episodes are never touched, so an operator clearing the page cannot lose the
+/// events that are still true.
+pub fn purgeAll(
+    state: *server.WebState,
+    io: std.Io,
+    request: *http_util.Request,
+) http_util.HandlerError!void {
+    const store = state.events orelse
+        return http_util.respondError(request, .service_unavailable, unavailable_message);
+
+    const purged = store.purgeAll(io) catch
+        return http_util.respondError(request, .internal_server_error, "internal error");
+
+    return http_util.respondJson(request, .ok, PurgeResult{ .purged = purged }, &.{});
+}
+
+// ---------------------------------------------------------------------------
+// tests
+// ---------------------------------------------------------------------------
+
+const migrations = @import("../../storage/migrations.zig");
+const testing = std.testing;
+
+test "an empty query string is the default page over everything" {
+    var buffers: Buffers = .{};
+    const filter = try parseFilter("", &buffers);
+    try testing.expectEqual(default_limit, filter.limit);
+    try testing.expectEqual(events.State.all, filter.state);
+    try testing.expectEqual(@as(?[]const u8, null), filter.severity);
+    try testing.expectEqual(@as(?[]const u8, null), filter.component);
+    try testing.expectEqual(@as(?i64, null), filter.before);
+    try testing.expectEqual(@as(?i64, null), filter.since);
+}
+
+test "every filter reaches the store untouched" {
+    var buffers: Buffers = .{};
+    const filter = try parseFilter(
+        "state=resolved&severity=error&component=query_log&since=100&until=200&limit=250&before=900",
+        &buffers,
+    );
+    try testing.expectEqual(events.State.resolved, filter.state);
+    try testing.expectEqualStrings("error", filter.severity.?);
+    try testing.expectEqualStrings("query_log", filter.component.?);
+    try testing.expectEqual(@as(?i64, 100), filter.since);
+    try testing.expectEqual(@as(?i64, 200), filter.until);
+    try testing.expectEqual(@as(u32, 250), filter.limit);
+    try testing.expectEqual(@as(?i64, 900), filter.before);
+}
+
+test "an empty component is no filter at all" {
+    var buffers: Buffers = .{};
+    try testing.expectEqual(@as(?[]const u8, null), (try parseFilter("component=", &buffers)).component);
+}
+
+test "each malformed parameter names itself in a 400" {
+    var buffers: Buffers = .{};
+    try testing.expectError(error.BadState, parseFilter("state=open", &buffers));
+    try testing.expectError(error.BadState, parseFilter("state=", &buffers));
+    try testing.expectError(error.BadSeverity, parseFilter("severity=info", &buffers));
+    try testing.expectError(error.BadSeverity, parseFilter("severity=WARNING", &buffers));
+    try testing.expectError(error.BadSince, parseFilter("since=yesterday", &buffers));
+    try testing.expectError(error.BadUntil, parseFilter("until=", &buffers));
+    try testing.expectError(error.BadLimit, parseFilter("limit=0", &buffers));
+    try testing.expectError(error.BadLimit, parseFilter("limit=1001", &buffers));
+    try testing.expectError(error.BadLimit, parseFilter("limit=ten", &buffers));
+    try testing.expectError(error.BadBefore, parseFilter("before=0", &buffers));
+    try testing.expectError(error.BadBefore, parseFilter("before=-4", &buffers));
+    try testing.expectError(error.BadComponent, parseFilter("component=%zz", &buffers));
+
+    var long: [max_component_len + 8]u8 = @splat('a');
+    var text: std.ArrayList(u8) = .empty;
+    defer text.deinit(testing.allocator);
+    try text.appendSlice(testing.allocator, "component=");
+    try text.appendSlice(testing.allocator, &long);
+    try testing.expectError(error.BadComponent, parseFilter(text.items, &buffers));
+
+    // Every member of the error set has its own wording, and none of them is
+    // the empty string.
+    inline for (comptime std.meta.fieldNames(FilterError)) |name| {
+        try testing.expect(message(@field(FilterError, name)).len != 0);
+    }
+}
+
+test "the limit cap is the store's" {
+    var buffers: Buffers = .{};
+    try testing.expectEqual(max_limit, (try parseFilter("limit=1000", &buffers)).limit);
+    try testing.expectEqual(@as(u32, 1000), events.max_limit);
+}
+
+/// A store over a migrated in-memory `config.db`, built in place: a `Store`
+/// holds a `*db.Db`, so a fixture that moved after `init` would leave that
+/// pointer behind.
+const Fixture = struct {
+    threaded: std.Io.Threaded = undefined,
+    io: std.Io = undefined,
+    database: db.Db = undefined,
+    store: events.Store = undefined,
+    state: server.WebState = undefined,
+
+    fn init(self: *Fixture) !void {
+        self.threaded = .init(testing.allocator, .{});
+        errdefer self.threaded.deinit();
+        self.io = self.threaded.io();
+
+        self.database = try db.Db.open(":memory:", .{ .mode = .memory });
+        errdefer self.database.close();
+        try db.applyPragmas(&self.database, .{});
+        _ = try migrations.migrate(&self.database);
+
+        self.store = try events.Store.init(self.io, &self.database, 1000);
+        self.state = .{ .gpa = testing.allocator, .events = &self.store };
+    }
+
+    fn deinit(self: *Fixture) void {
+        self.database.close();
+        self.threaded.deinit();
+    }
+
+    fn seed(self: *Fixture) void {
+        const store = &self.store;
+        store.report(self.io, 1000, .blocklist_refresh, "https://a.example", "StevenBlack", .warning, "ConnectionTimedOut");
+        store.report(self.io, 1100, .blocklist_refresh, "https://a.example", "StevenBlack", .warning, "ConnectionTimedOut");
+        store.report(self.io, 1200, .listener_start, "doh", "doh", .@"error", "AddressInUse");
+        store.report(self.io, 1300, .query_log_write, "batch", "batch", .@"error", "Busy");
+        store.resolve(self.io, 1400, .query_log_write, "batch");
+    }
+};
+
+test "a page carries the events, the cursor and the active counts" {
+    var fx: Fixture = .{};
+    try fx.init();
+    defer fx.deinit();
+    fx.seed();
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena.deinit();
+
+    const page = try fx.store.selectEvents(fx.io, arena.allocator(), .{ .limit = 100 });
+    try testing.expectEqual(@as(usize, 3), page.events.len);
+    try testing.expectEqual(@as(?i64, null), page.next_before);
+    try testing.expectEqual(events.Counts{ .warnings = 1, .errors = 1 }, page.active);
+
+    // Newest first, and the episode that repeated counts rather than repeats.
+    try testing.expectEqualStrings("query_log.write", page.events[0].code);
+    try testing.expectEqualStrings("blocklist.refresh", page.events[2].code);
+    try testing.expectEqual(@as(i64, 2), page.events[2].occurrences);
+    try testing.expectEqualStrings("StevenBlack", page.events[2].subject);
+}
+
+test "a full page carries a cursor and the last page does not" {
+    var fx: Fixture = .{};
+    try fx.init();
+    defer fx.deinit();
+    fx.seed();
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena.deinit();
+    const gpa = arena.allocator();
+
+    const first = try fx.store.selectEvents(fx.io, gpa, .{ .limit = 2 });
+    try testing.expectEqual(@as(usize, 2), first.events.len);
+    try testing.expectEqual(first.events[1].id, first.next_before.?);
+
+    const second = try fx.store.selectEvents(fx.io, gpa, .{ .limit = 2, .before = first.next_before });
+    try testing.expectEqual(@as(usize, 1), second.events.len);
+    try testing.expectEqual(@as(?i64, null), second.next_before);
+}
+
+test "the parsed filters narrow the page the store answers with" {
+    var fx: Fixture = .{};
+    try fx.init();
+    defer fx.deinit();
+    fx.seed();
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena.deinit();
+    const gpa = arena.allocator();
+    var buffers: Buffers = .{};
+
+    const active = try fx.store.selectEvents(fx.io, gpa, try parseFilter("state=active", &buffers));
+    try testing.expectEqual(@as(usize, 2), active.events.len);
+
+    const resolved = try fx.store.selectEvents(fx.io, gpa, try parseFilter("state=resolved", &buffers));
+    try testing.expectEqual(@as(usize, 1), resolved.events.len);
+    try testing.expectEqual(@as(?i64, 1400), resolved.events[0].resolved_at);
+
+    const errors = try fx.store.selectEvents(fx.io, gpa, try parseFilter("severity=error", &buffers));
+    try testing.expectEqual(@as(usize, 2), errors.events.len);
+
+    const blocklist = try fx.store.selectEvents(fx.io, gpa, try parseFilter("component=blocklist", &buffers));
+    try testing.expectEqual(@as(usize, 1), blocklist.events.len);
+    try testing.expectEqualStrings("blocklist", blocklist.events[0].component);
+}
+
+test "the wire object carries the label and never the subject key" {
+    var fx: Fixture = .{};
+    try fx.init();
+    defer fx.deinit();
+
+    // A key that would be unmistakable on the wire if it ever leaked: the
+    // upstream urls this store keys on can carry a token.
+    fx.store.report(fx.io, 1000, .upstream_exchange, "https://dns.example/secret-token", "dns.example", .warning, "Timeout");
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena.deinit();
+    const page = try fx.store.selectEvents(fx.io, arena.allocator(), .{});
+
+    var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
+    defer allocating.deinit();
+    try std.json.Stringify.value(page, .{}, &allocating.writer);
+    const text = allocating.written();
+
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "secret-token"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "subject_key"));
+    try testing.expect(std.mem.startsWith(u8, text, "{\"events\":["));
+    for ([_][]const u8{
+        "\"id\":",                    "\"code\":\"upstream.exchange\"",
+        "\"component\":\"upstream\"", "\"subject\":\"dns.example\"",
+        "\"severity\":\"warning\"",   "\"first_seen\":",
+        "\"last_seen\":",             "\"occurrences\":",
+        "\"resolved_at\":null",       "\"detail\":\"Timeout\"",
+        "\"next_before\":null",       "\"active\":{\"warnings\":1,\"errors\":0}",
+    }) |field| {
+        errdefer std.debug.print("missing {s} in {s}\n", .{ field, text });
+        try testing.expect(std.mem.containsAtLeast(u8, text, 1, field));
+    }
+}
+
+test "a detail page answers by id and reports an unknown one as absent" {
+    var fx: Fixture = .{};
+    try fx.init();
+    defer fx.deinit();
+    fx.seed();
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena.deinit();
+
+    const found = (try fx.store.selectOne(fx.io, arena.allocator(), 1)).?;
+    try testing.expectEqualStrings("blocklist.refresh", found.code);
+    // The 404 the handler answers with is this null.
+    try testing.expect((try fx.store.selectOne(fx.io, arena.allocator(), 9999)) == null);
+}
+
+test "the purge-all body is the count and nothing else" {
+    var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
+    defer allocating.deinit();
+    try std.json.Stringify.value(PurgeResult{ .purged = 3 }, .{}, &allocating.writer);
+    try testing.expectEqualStrings("{\"purged\":3}", allocating.written());
+
+    // The 409 says what would make the event purgeable, so it is not the
+    // generic conflict wording.
+    try testing.expect(still_active_message.len != 0);
+    try testing.expect(std.mem.containsAtLeast(u8, still_active_message, 1, "resolves"));
+}
+
+test "a state with no store answers 503 rather than an empty page" {
+    // The handler's only branch that does not need a request: a `WebState`
+    // whose store failed to open reports the endpoint unavailable, and never
+    // an empty list that would read as "nothing is wrong".
+    const state: server.WebState = .{ .gpa = testing.allocator };
+    try testing.expectEqual(@as(?*events.Store, null), state.events);
+    try testing.expect(unavailable_message.len != 0);
+}