mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 13 Activity

milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable
Gates / frontend (push) Successful in 1m33s
Details
Gates / test (push) Successful in 1m48s
Details
Gates / test-aarch64 (push) Successful in 7m10s
Details
Gates / package (push) Successful in 5m31s
Details
Gates / container (push) Successful in 15s
Details
CI / gates (push) Successful in 14m51s
Details

Browse Source
This commit is contained in:
mokhtar
2026-08-20 20:05:59 +02:00
parent 3dd8214ef2
commit 037f209179
50 changed files with 8608 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app.zig
+336 -14
View File
@@ -44,6 +44,7 @@ const doh_client = @import("upstream/doh_client.zig");
const doh_server = @import("server/doh_server.zig");
const dot_client = @import("upstream/dot_client.zig");
const dot_server = @import("server/dot_server.zig");
const events = @import("storage/events.zig");
const faults = @import("config/faults.zig");
const fetcher = @import("filter/fetcher.zig");
const forward_zones = @import("local/forward_zones.zig");
@@ -154,6 +155,7 @@ fn reconcileFromFile(
config_db: *db.Db,
dir: std.Io.Dir,
config_path: []const u8,
config_load: ?*ConfigLoad,
) !i64 {
return reconcileFromFileAt(
r,
@@ -161,6 +163,7 @@ fn reconcileFromFile(
dir,
config_path,
std.Io.Clock.real.now(r.io).toSeconds(),
config_load,
);
}
@@ -184,6 +187,7 @@ fn reconcileFromFileAt(
dir: std.Io.Dir,
config_path: []const u8,
pass_now: i64,
config_load: ?*ConfigLoad,
) !i64 {
var arena_state: std.heap.ArenaAllocator = .init(r.gpa);
defer arena_state.deinit();
@@ -203,9 +207,32 @@ fn reconcileFromFileAt(
diags.writeAll(r.err) catch {};
r.err.flush() catch {};
// Warnings only. A `.fail` rejects the file and the process exits, so there
// is nobody left to read a diagnostics row about it; a warning is the case
// where the box serves on with a setting the operator did not mean.
if (config_load) |collector| {
for (diags.problems.items) |problem| {
if (problem.severity != .warn) continue;
collector.note(problem.path, problem.path, problem.message);
}
}
return result;
}
/// An upstream's identity is its url: the whole url is the key, and the
/// redaction is the label, because a url can carry an account token.
fn noteUpstream(config_load: *ConfigLoad, url: []const u8, message: []const u8) void {
var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
const label = std.fmt.bufPrint(&label_buf, "{f}", .{safe_url.redact(url)}) catch &label_buf;
var detail_buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&detail_buf, "upstream {f} {s}", .{
safe_url.redactQuoted(url),
message,
}) catch &detail_buf;
config_load.note(url, label, detail);
}
/// Returns the moment the transaction committed, which is what the settings
/// envelope reports as `reconciled_at`.
fn applyManagedFile(
@@ -318,6 +345,55 @@ fn printSummary(
try r.out.flush();
}
/// The `configuration.load` findings of one boot.
///
/// This code is boot-finalized: nothing during the run can fix a setting, so a
/// restart is its recovery. Every finding is reported as it is made and its key
/// kept, and one `resolveExcept` after the last of them closes the episodes of
/// settings that were wrong last boot and are not wrong now.
const ConfigLoad = struct {
store: ?*events.Store,
io: std.Io,
now_s: i64,
keys: [events.Store.max_kept_keys][events.Store.max_subject_key_len]u8 = undefined,
lens: [events.Store.max_kept_keys]u16 = @splat(0),
len: usize = 0,
/// Set when a boot produced more distinct findings than `resolveExcept`
/// carries. The bulk resolve is then refused rather than truncated: a stale
/// episode left open is honest, and one that is still true closed is not.
/// The refusal goes through the store, so it is counted and latched.
overflowed: bool = false,
fn note(self: *ConfigLoad, key: []const u8, label: []const u8, detail: []const u8) void {
const store = self.store orelse return;
store.report(self.io, self.now_s, .configuration_load, key, label, .warning, detail);
self.keep(key);
}
fn keep(self: *ConfigLoad, key: []const u8) void {
var canon_buf: [events.Store.max_subject_key_len]u8 = undefined;
const canon = events.canonicalKey(key, &canon_buf);
for (0..self.len) |i| {
if (std.mem.eql(u8, self.keys[i][0..self.lens[i]], canon)) return;
}
if (self.len == self.keys.len) {
self.overflowed = true;
return;
}
@memcpy(self.keys[self.len][0..canon.len], canon);
self.lens[self.len] = @intCast(canon.len);
self.len += 1;
}
fn finalize(self: *ConfigLoad) void {
const store = self.store orelse return;
if (self.overflowed) return store.refuseResolveExcept(self.io, self.now_s);
var kept: [events.Store.max_kept_keys][]const u8 = undefined;
for (0..self.len) |i| kept[i] = self.keys[i][0..self.lens[i]];
store.resolveExcept(self.io, self.now_s, .configuration_load, kept[0..self.len]);
}
};
fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
const io = r.io;
const gpa = r.gpa;
@@ -334,13 +410,29 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
defer config_db.close();
_ = try migrations.migrate(&config_db);
// Opened unconditionally, not gated on `cfg.web.enabled`: diagnostics record
// what went wrong whether or not anyone is running the UI, and the store
// owns this connection outright (`storage/events.zig`).
var events_db = try data.openConfigDb(io);
defer events_db.close();
const boot_now_s = std.Io.Clock.real.now(io).toSeconds();
var event_store_storage: ?events.Store = events.Store.init(io, &events_db, boot_now_s) catch |err| blk: {
// No store rather than a store on a mirror it could not verify: the
// latter answers `resolve` with confident no-ops. `/api/health` reports
// the absence as `unavailable` and degrades on it.
log.warn("diagnostics store unavailable: {s}", .{@errorName(err)});
break :blk null;
};
const event_store: ?*events.Store = if (event_store_storage) |*s| s else null;
var config_load: ConfigLoad = .{ .store = event_store, .io = io, .now_s = boot_now_s };
// Ruling 1: the presence of `--config` is the whole authority decision. With
// it, the file is the sole declarative source and the database is converged
// onto it here, before anything reads the database. Without it the database
// is authority and this step does not exist — a file on disk that no flag
// names changes nothing.
const reconciled_at: ?i64 = if (args.config) |config_path|
try reconcileFromFile(r, &config_db, std.Io.Dir.cwd(), config_path)
try reconcileFromFile(r, &config_db, std.Io.Dir.cwd(), config_path, &config_load)
else
null;
@@ -437,7 +529,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
defer bundle.deinit(gpa);
var bundle_lock: std.Io.RwLock = .init;
var upstreams = try Upstreams.build(gpa, cfg.upstreams, &dns_http, &bundle, &bundle_lock);
var upstreams = try Upstreams.build(gpa, cfg.upstreams, &dns_http, &bundle, &bundle_lock, &config_load);
defer upstreams.deinit(gpa);
var pool: pool_mod.Pool = .init(
@@ -455,7 +547,9 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
const history = try gpa.create(history_mod.Accumulator);
defer gpa.destroy(history);
history.* = .init;
history.diagnostics = event_store;
pool.history = history;
pool.diagnostics = event_store;
// -----------------------------------------------------------------------
// per-query state
@@ -472,15 +566,18 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
var paused: pause.Pause = .{};
var tracker: clients.Tracker = .init(cfg.logging.retention_days);
tracker.diagnostics = event_store;
// Naming rides the tracker's pass, on the tracker's task and connection
// (milestone-25 ruling 1), and reads the live forward zones.
var client_names_resolver: client_names.Resolver = .init(&tables);
client_names_resolver.diagnostics = event_store;
// The queue holds waiting tasks in intrusive lists, so neither the buffer
// nor the `Logger` may move once a task has touched either.
const queue_buf = try gpa.alloc(logger_mod.Entry, cfg.logging.query_log_buffer_max);
defer gpa.free(queue_buf);
var query_logger: logger_mod.Logger = .init(cfg.logging, queue_buf);
query_logger.diagnostics = event_store;
// Milestone 8 fans every logged query out to the SSE hub as well. The hub
// exists only when the web interface does (ruling 6) — without it the sink
@@ -510,6 +607,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
// Ruling 17. The scheduler consults it before every scheduled pass; the
// startup `reload` below is an operator action and stays ungated.
manager.monitor = &monitor;
manager.diagnostics = event_store;
// One synchronous sample before anything can consult the gate. `Monitor`
// initializes to `.ok`, and `Monitor.run` takes its first sample inside the
@@ -525,12 +623,37 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
// fails therefore leaves the state at `.ok` — an unreadable filesystem is
// not evidence that the disk is full, which is the monitor's documented
// policy and the right one here too.
monitor.sample(io);
monitor.sample(io, event_store, boot_now_s);
var retention: retention_mod.Retention = .init(cfg.logging);
var querylog_writer_db = try data.openQuerylogDb(io);
var querylog_opened = try data.openQuerylogDb(io);
var querylog_writer_db = querylog_opened.database;
defer querylog_writer_db.close();
// One-shot and already over: the file was recreated during this boot, and
// there is nothing to recover from. Never emitted for `.missing` — a first
// creation renames nothing aside, so the event would carry an aside path
// that does not exist and would greet every fresh install with a warning.
if (querylog_opened.recreated) |cause| {
if (cause != .missing) {
if (event_store) |store| {
var detail_buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&detail_buf, "previous file kept as '{s}'", .{
querylog_opened.aside(),
}) catch detail_buf[0..];
store.reportResolved(
io,
boot_now_s,
.query_log_recreated,
@tagName(cause),
@tagName(cause),
.warning,
detail,
);
}
}
}
var querylog_retention_db = try data.reopenQuerylogDb(io);
defer querylog_retention_db.close();
var querylog_history_db = try data.reopenQuerylogDb(io);
@@ -574,6 +697,17 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
"loading the blocklist snapshot failed ({s}); serving unfiltered until the next refresh",
.{@errorName(err)},
);
// No manager lock is held here, so this reports directly rather than
// through the manager's collector.
if (event_store) |store| {
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(
&buf,
"loading the blocklist snapshot failed: {s}",
.{@errorName(err)},
) catch buf[0..];
store.report(io, boot_now_s, .blocklist_snapshot, "snapshot", "blocklist snapshot", .@"error", detail);
}
};
// -----------------------------------------------------------------------
@@ -609,12 +743,16 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
defer if (doh_certs) |*store| store.deinit(io);
if (cfg.doh_server.enabled) {
doh_certs = try openCertStore(r, gpa, io, cfg.doh_server, "doh_server", doh_server.alpn_protocols);
doh_certs.?.kind = .doh;
doh_certs.?.diagnostics = event_store;
}
var dot_certs: ?cert_store.CertStore = null;
defer if (dot_certs) |*store| store.deinit(io);
if (cfg.dot_server.enabled) {
dot_certs = try openCertStore(r, gpa, io, cfg.dot_server, "dot_server", dot_alpn);
dot_certs.?.kind = .dot;
dot_certs.?.diagnostics = event_store;
}
// Bound here, in this frame, rather than through doh_server's module-level
@@ -622,13 +760,35 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
// only a listener that lives in this frame has an address to wire there.
// A failed bind warns and stays off (ruling 1, the web precedent): TLS DNS
// failing to come up must not stop the plain-DNS side this box exists for.
var failed_listeners: [2][]const u8 = undefined;
var failed_listener_count: usize = 0;
var doh: ?doh_server.DohServer = null;
defer if (doh) |*server| server.deinit(io);
if (doh_certs) |*store| doh = bindDoh(gpa, io, cfg.doh_server, &h, store);
if (doh_certs) |*store| {
doh = bindDoh(gpa, io, cfg.doh_server, &h, store, event_store, boot_now_s);
if (doh == null) {
failed_listeners[failed_listener_count] = "doh";
failed_listener_count += 1;
}
}
var dot: ?dot_server.DotServer = null;
defer if (dot) |*server| server.deinit(io);
if (dot_certs) |*store| dot = bindDot(gpa, io, cfg.dot_server, &h, store);
if (dot_certs) |*store| {
dot = bindDot(gpa, io, cfg.dot_server, &h, store, event_store, boot_now_s);
if (dot == null) {
failed_listeners[failed_listener_count] = "dot";
failed_listener_count += 1;
}
}
// Boot-finalized: one call closes whatever the last boot left open for an
// endpoint that started clean this time — including an endpoint now
// disabled, which contributes no key and so is not kept.
if (event_store) |store| {
store.resolveExcept(io, boot_now_s, .listener_start, failed_listeners[0..failed_listener_count]);
}
// -----------------------------------------------------------------------
// web interface (ruling 26)
@@ -668,6 +828,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
.dot_listener = if (dot) |*server| server else null,
.config_db = if (web_config_db) |*database| database else null,
.querylog_db = if (web_querylog_db) |*database| database else null,
.events = event_store,
.version = version.string,
.started_unix = std.Io.Clock.real.now(io).toSeconds(),
// Ruling 24: `--admin-dev` serves from disk with no cache headers;
@@ -684,6 +845,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
var udp6: ?udp_server.UdpServer = udp_server.UdpServer.bind(gpa, io, v6_bind, &h, .{}) catch |err| bound: {
if (!ipv6Unavailable(err)) return reportBind(r, "udp", v6_bind, err);
log.warn("this system has no IPv6; serving IPv4 only", .{});
config_load.note("dns.bind_ipv6", "dns.bind_ipv6", "this system has no IPv6; serving IPv4 only");
break :bound null;
};
defer if (udp6) |*s| s.deinit(gpa, io);
@@ -712,6 +874,12 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
};
defer if (tcp4) |*s| s.deinit(io);
// Every `configuration.load` finding of this boot is in by here: the managed
// file, the upstream table and the IPv6 bind above. Finalizing any earlier
// would resolve an episode this boot is about to reopen, so consecutive
// IPv6-less boots would read as a new episode each time.
config_load.finalize();
// Ruling 13: `/metrics` sums each transport's listeners into one family, so
// the web state carries pointers to whichever of the four came up. The
// arrays are declared here rather than beside `web_state` because a
@@ -793,11 +961,11 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
if (doh_certs) |*store| try group.concurrent(io, cert_store.CertStore.watch, .{ store, io });
if (dot_certs) |*store| try group.concurrent(io, cert_store.CertStore.watch, .{ store, io });
try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db, gate });
try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db, gate, event_store });
// Ungated: a flush writes at most one row per upstream per minute, the same
// category as the query logger's own writes, which are ungated too.
try group.concurrent(io, history_mod.Accumulator.run, .{ history, io, &querylog_history_db });
try group.concurrent(io, disk_monitor.Monitor.run, .{ &monitor, io });
try group.concurrent(io, disk_monitor.Monitor.run, .{ &monitor, io, event_store });
try group.concurrent(io, manager_mod.Manager.runScheduler, .{ &manager, io });
try group.concurrent(io, clients.Tracker.run, .{ &tracker, io, &tracker_db, gate, &client_names_resolver });
try group.concurrent(io, runMaintenance, .{ &h, if (web_limiter) |*l| l else null, io });
@@ -882,6 +1050,23 @@ fn openCertStore(
};
}
/// An error, not a warning: an endpoint the operator enabled is not serving.
fn reportListener(
store: ?*events.Store,
io: std.Io,
now_s: i64,
kind: []const u8,
comptime fmt: []const u8,
args: anytype,
) void {
const s = store orelse return;
var buf: [events.Store.max_detail_len]u8 = undefined;
var w: std.Io.Writer = .fixed(&buf);
w.print("{s} listener ", .{kind}) catch {};
w.print(fmt, args) catch {};
s.report(io, now_s, .listener_start, kind, kind, .@"error", w.buffered());
}
/// Ruling 1: a listener that cannot bind warns and stays off. The bind text
/// itself gets the same treatment — `validate` refuses it, but a hand-edited
/// database can still carry one, and it is not worth taking DNS down over.
@@ -891,13 +1076,17 @@ fn bindDoh(
endpoint: model.TlsEndpoint,
h: *handler.Handler,
store: *cert_store.CertStore,
diagnostics: ?*events.Store,
now_s: i64,
) ?doh_server.DohServer {
const bind_address = net.IpAddress.parse(endpoint.bind, endpoint.port) catch {
log.warn("doh_server.bind '{s}' is not an IP address; DoH is disabled", .{endpoint.bind});
reportListener(diagnostics, io, now_s, "doh", "bind '{s}' is not an IP address", .{endpoint.bind});
return null;
};
const server = doh_server.DohServer.listen(gpa, io, bind_address, h, store, .{}) catch |err| {
log.warn("doh listener cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
reportListener(diagnostics, io, now_s, "doh", "cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
return null;
};
log.info("doh listener on {f}", .{server.boundAddress()});
@@ -910,13 +1099,17 @@ fn bindDot(
endpoint: model.TlsEndpoint,
h: *handler.Handler,
store: *cert_store.CertStore,
diagnostics: ?*events.Store,
now_s: i64,
) ?dot_server.DotServer {
const bind_address = net.IpAddress.parse(endpoint.bind, endpoint.port) catch {
log.warn("dot_server.bind '{s}' is not an IP address; DoT is disabled", .{endpoint.bind});
reportListener(diagnostics, io, now_s, "dot", "bind '{s}' is not an IP address", .{endpoint.bind});
return null;
};
const server = dot_server.DotServer.listen(gpa, io, bind_address, h, store, .{}) catch |err| {
log.warn("dot listener cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
reportListener(diagnostics, io, now_s, "dot", "cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
return null;
};
log.info("dot listener on {f}", .{server.boundAddress()});
@@ -1006,6 +1199,7 @@ const Upstreams = struct {
http: *std.http.Client,
bundle: *Certificate.Bundle,
bundle_lock: *std.Io.RwLock,
config_load: *ConfigLoad,
) (Allocator.Error || error{NoUsableUpstreams})!Upstreams {
var enabled: usize = 0;
for (servers) |server| {
@@ -1041,6 +1235,7 @@ const Upstreams = struct {
"upstream {f} is not an https:// or tls:// endpoint; skipped",
.{safe_url.redactQuoted(server.url)},
);
noteUpstream(config_load, server.url, "not an https:// or tls:// endpoint; skipped");
continue;
};
@@ -1058,6 +1253,7 @@ const Upstreams = struct {
"upstream {f} is not a usable DoH url; skipped",
.{safe_url.redactQuoted(server.url)},
);
noteUpstream(config_load, server.url, "not a usable DoH url; skipped");
continue;
};
doh_count += 1;
@@ -1154,6 +1350,9 @@ fn parseBind(
return addr;
}
const events_fixture = @import("storage/events_fixture.zig");
const testing = std.testing;
test "parseBind refuses a bind address of the wrong family" {
var out_buf: [8]u8 = undefined;
var err_buf: [256]u8 = undefined;
@@ -1325,7 +1524,7 @@ test "a start in file mode prints the warnings the file earned" {
// The file is valid, so the start succeeds and the database converges onto
// it. The returned stamp is what the settings envelope reports.
const reconciled_at = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
const reconciled_at = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
try std.testing.expect(reconciled_at > 0);
try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
@@ -1431,7 +1630,7 @@ test "reconciled_at is stamped after the commit, not from the clock the pass wro
const pass_now: i64 = 42;
const before = std.Io.Clock.real.now(io).toSeconds();
const reconciled_at = try reconcileFromFileAt(r, &database, tmp.dir, "config.zon", pass_now);
const reconciled_at = try reconcileFromFileAt(r, &database, tmp.dir, "config.zon", pass_now, null);
// The pinned clock reached the engine, so the two values really are separate
// inputs rather than the same read twice.
@@ -1474,7 +1673,7 @@ test "the startup summary reports what the reconcile changed, then that nothing
var err: Writer = .fixed(&err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out_writer.interface, .err = &err };
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
{
// Read back through the file: `serve` does not return for as long as the
// service runs, so a summary still in the buffer is a summary nobody
@@ -1491,7 +1690,7 @@ test "the startup summary reports what the reconcile changed, then that nothing
}
try tmp.dir.writeFile(io, .{ .sub_path = "stdout.txt", .data = "" });
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
{
const printed = try tmp.dir.readFileAlloc(io, "stdout.txt", gpa, .limited(8192));
defer gpa.free(printed);
@@ -1548,7 +1747,7 @@ test "a broken error writer does not replace the reason a managed file was rejec
try std.testing.expectError(
error.MissingDefaultGroup,
reconcileFromFile(r, &database, tmp.dir, "config.zon"),
reconcileFromFile(r, &database, tmp.dir, "config.zon", null),
);
// Empty, so the writer did fail — without this the assertion above would
@@ -1594,7 +1793,7 @@ test "a broken error writer does not stop a start whose file applied" {
var err_writer = brokenErrWriter(io, err_file, &err_buf);
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer.interface };
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
// Nothing reached the file, so the flush really did fail.
@@ -1779,3 +1978,126 @@ test "one maintenance pass drops the api limiter's stale buckets" {
// A limiter the app did not build is not a reason for the pass to fail.
try maintenanceOnce(&h, null, io);
}
test "a failing listener bind opens an error episode a clean boot closes" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
reportListener(&fx.store, io, 1000, "dot", "cannot listen on {s}:{d}: {t}", .{
"0.0.0.0",
@as(u16, 853),
error.AddressInUse,
});
try testing.expectEqualStrings("listener.start", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("dot", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqualStrings("error", try fx.text("SELECT severity FROM operational_events"));
// The next boot: DoH failed, DoT started clean. One `resolveExcept` over
// the failed keys closes the stale DoT episode and leaves the DoH one open.
reportListener(&fx.store, io, 1100, "doh", "cannot listen on {s}:{d}: {t}", .{
"0.0.0.0",
@as(u16, 443),
error.AddressInUse,
});
fx.store.resolveExcept(io, 1100, .listener_start, &.{"doh"});
try testing.expectEqual(
@as(i64, 1),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
try testing.expectEqualStrings("doh", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
}
test "a boot with no listener finding closes every listener episode" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
fx.store.report(io, 900, .listener_start, "doh", "doh", .@"error", "stale");
fx.store.report(io, 900, .listener_start, "dot", "dot", .@"error", "stale");
fx.store.resolveExcept(io, 1000, .listener_start, &.{});
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
test "a configuration finding is reported once and finalize closes the rest" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
// Two stale episodes from a previous boot.
fx.store.report(io, 900, .configuration_load, "dns.bind_ipv6", "dns.bind_ipv6", .warning, "stale");
fx.store.report(io, 900, .configuration_load, "upstreams[0].url", "upstreams[0].url", .warning, "stale");
var collector: ConfigLoad = .{ .store = &fx.store, .io = io, .now_s = 1000 };
collector.note("dns.bind_ipv6", "dns.bind_ipv6", "this system has no IPv6; serving IPv4 only");
// The same finding twice is one episode and one kept key.
collector.note("dns.bind_ipv6", "dns.bind_ipv6", "this system has no IPv6; serving IPv4 only");
try testing.expectEqual(@as(usize, 1), collector.len);
collector.finalize();
try testing.expectEqual(
@as(i64, 1),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
try testing.expectEqualStrings("dns.bind_ipv6", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
// The stale episode is still the same one: this boot bumped it twice.
try testing.expectEqual(@as(i64, 3), try fx.count(
"SELECT occurrences FROM operational_events WHERE resolved_at IS NULL",
));
}
test "an over-long boot finding list refuses to finalize rather than truncate" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
fx.store.report(io, 900, .configuration_load, "stale.setting", "stale.setting", .warning, "stale");
var collector: ConfigLoad = .{ .store = &fx.store, .io = io, .now_s = 1000 };
var key_buf: [32]u8 = undefined;
for (0..events.Store.max_kept_keys + 1) |i| {
const key = try std.fmt.bufPrint(&key_buf, "upstreams[{d}].url", .{i});
collector.note(key, key, "not a usable url");
}
try testing.expect(collector.overflowed);
collector.finalize();
// Closing more than the boot meant would resolve episodes that are still
// true, so the stale one stays open instead.
try testing.expectEqual(@as(i64, 1), try fx.count(
"SELECT count(*) FROM operational_events WHERE subject_key = 'stale.setting' AND resolved_at IS NULL",
));
// Refused, and refused out loud: the store counted and latched it, so
// `/api/health` reports the diagnostics log as not recording.
try testing.expect(fx.store.writeFailed());
try testing.expectEqual(@as(u64, 1), fx.store.writeFailures());
}
src/cli.zig
+7 -5
View File
@@ -332,12 +332,14 @@ pub const DataDir = struct {
/// `querylog_schema.open` resolves the path through SQLite's VFS as well as
/// through the directory handle, so it is given `cwd` and the joined path
/// rather than `self.dir` and a name (see its doc comment).
pub fn openQuerylogDb(self: *const DataDir, io: std.Io) !db.Db {
const opened = try querylog_schema.open(io, std.Io.Dir.cwd(), self.querylog_db_path);
var database = opened.database;
errdefer database.close();
/// Returns the whole `OpenResult`, recreate reason and aside name included:
/// the composition root records a recreate as a one-shot diagnostics event,
/// and the aside name is what tells an operator where the old file went.
pub fn openQuerylogDb(self: *const DataDir, io: std.Io) !querylog_schema.OpenResult {
var opened = try querylog_schema.open(io, std.Io.Dir.cwd(), self.querylog_db_path);
errdefer opened.database.close();
try self.restrictQuerylogPermissions(io);
return database;
return opened;
}
/// An additional connection to a `querylog.db` that `openQuerylogDb` has
src/config/import.zig
+50
View File
@@ -151,6 +151,7 @@ fn reportDeletes(diags: *validate.Diagnostics, summary: reconcile.Summary) error
const testing = std.testing;
const config_schema = @import("../storage/config_schema.zig");
const export_mod = @import("export.zig");
const migrations = @import("../storage/migrations.zig");
fn openMigrated() !db.Db {
@@ -282,6 +283,55 @@ test "importSource converges a migrated database and group 'default' keeps id 1"
);
}
test "an export and re-import leaves the operational_events log untouched" {
// `operational_events` is runtime state, not configuration: it is out of
// `table_names` and out of `delete_order`, so an export must not emit it and
// an import's wipe must not reach it. A diagnostics log destroyed by a
// routine `nxdns import` would take the record of what the box has been
// doing with it.
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
const gpa = testing.allocator;
var database = try openMigrated();
defer database.close();
try importText(io, &database, full_source, .{});
try database.exec(
\\INSERT INTO operational_events
\\ (code, subject_key, subject_label, severity, first_seen, last_seen, occurrences, resolved_at)
\\VALUES ('disk.space', 'data', 'data', 'warning', 100, 200, 3, NULL),
\\ ('blocklist.refresh', 'https://a.example', 'A', 'warning', 100, 150, 1, 300);
);
var rendered: std.Io.Writer.Allocating = .init(gpa);
defer rendered.deinit();
try export_mod.writeToWriter(gpa, &database, &rendered.writer);
const source = try gpa.dupeZ(u8, rendered.written());
defer gpa.free(source);
// The export is the whole declared configuration and says nothing about
// the log.
try testing.expect(!std.mem.containsAtLeast(u8, source, 1, "operational_events"));
try testing.expect(!std.mem.containsAtLeast(u8, source, 1, "disk.space"));
// A round trip is by definition delete-free for the config tables, and the
// events survive it untouched, resolved and active alike.
try importText(io, &database, source, .{ .allow_delete = true });
var stmt = try database.prepare(
"SELECT code, occurrences, resolved_at FROM operational_events ORDER BY id",
);
defer stmt.deinit();
try testing.expect(try stmt.step());
try testing.expectEqualStrings("disk.space", stmt.columnText(0));
try testing.expectEqual(@as(i64, 3), stmt.columnInt(1));
try testing.expect(stmt.isNull(2));
try testing.expect(try stmt.step());
try testing.expectEqualStrings("blocklist.refresh", stmt.columnText(0));
try testing.expectEqual(@as(i64, 300), stmt.columnInt(2));
try testing.expect(!try stmt.step());
}
test "an import whose diff deletes rows is refused, names the tables, and changes nothing" {
// Ruling 6: the emptiness guard is gone, so this is what stops
// `nxdns import ./wrong.zon` from emptying a configured database.
src/filter/filter_integration_test.zig
+160 -1
View File
@@ -24,6 +24,7 @@ const net = std.Io.net;
const model = @import("../config/model.zig");
const reconcile = @import("../config/reconcile.zig");
const db = @import("../storage/db.zig");
const events_fixture = @import("../storage/events_fixture.zig");
const migrations = @import("../storage/migrations.zig");
const context = @import("../storage/repositories/context.zig");
const groups_repo = @import("../storage/repositories/groups_repo.zig");
@@ -2328,7 +2329,6 @@ test "23: a name moving from the list body to the wild body forces a republish"
try testing.expectEqual(@as(i64, 0), row.wildcard_count);
@memcpy(&first_checksum, row.checksum orelse return error.TestNoChecksum);
}
}
test "22: a list that changed only its skipped lines still updates both skip counters" {
@@ -2405,3 +2405,162 @@ test "22: a list that changed only its skipped lines still updates both skip cou
try testing.expectEqual(@as(u32, 1), restored.counts.skipped_regex);
try testing.expectEqual(@as(u32, 2), restored.counts.skipped_unsupported);
}
// ---------------------------------------------------------------------------
// diagnostics: the events the manager records (milestone 27)
// ---------------------------------------------------------------------------
test "27: a failing refresh opens a blocklist.refresh episode a good one closes" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
env.mgr.diagnostics = &fx.store;
var fixture = try HttpFixture.init(io, http_body);
defer fixture.deinit(io);
fixture.setRoute(.oversize);
var group: std.Io.Group = .init;
defer group.cancel(io);
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
var url_buf: [64]u8 = undefined;
const url = try fixture.url(&url_buf);
_ = try seedSource(&env.database, url);
try testing.expect(!try refreshOnce(env, url));
try testing.expectEqualStrings("blocklist.refresh", try fx.text(
"SELECT code FROM operational_events WHERE resolved_at IS NULL",
));
// The whole url, not the display copy: `subject_key` is the identity.
try testing.expectEqualStrings(url, try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
try testing.expectEqualStrings("warning", try fx.text(
"SELECT severity FROM operational_events WHERE resolved_at IS NULL",
));
fixture.setRoute(.body);
try testing.expect(try refreshOnce(env, url));
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL"),
);
}
test "27: one refreshAll pass records one occurrence of a failing source" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
env.mgr.diagnostics = &fx.store;
var fixture = try HttpFixture.init(io, http_body);
defer fixture.deinit(io);
fixture.setRoute(.oversize);
var group: std.Io.Group = .init;
defer group.cancel(io);
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
var url_buf: [64]u8 = undefined;
const url = try fixture.url(&url_buf);
_ = try seedSource(&env.database, url);
// `scheduledPass` is what an elapsed interval runs, and it ends in
// `refreshAll`, which ends in a reload. One pass is one flush: a reload
// that flushed on its own, or a scheduler that flushed after a pass that
// already had, would report this failure twice.
try env.mgr.scheduledPass(io);
try testing.expectEqual(@as(i64, 1), try fx.count(
"SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
));
// A second pass on a still-failing source is one more occurrence: the
// failures one pass held ride the detail, and the flushes a pass makes are
// not what `occurrences` counts.
try env.mgr.scheduledPass(io);
try testing.expectEqual(@as(i64, 2), try fx.count(
"SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
));
const detail = try fx.text("SELECT detail FROM operational_events WHERE code = 'blocklist.refresh'");
try testing.expect(std.mem.indexOf(u8, detail, "this pass") != null);
}
test "27: a clean reload resolves blocklist.snapshot and records no storage failure" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
// A stale episode from a previous boot, which a published snapshot closes.
fx.store.report(io, 900, .blocklist_snapshot, "snapshot", "blocklist snapshot", .@"error", "stale");
env.mgr.diagnostics = &fx.store;
try env.mgr.reload(io);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
try testing.expectEqual(@as(i64, 900), try fx.count("SELECT first_seen FROM operational_events"));
}
test "27: an unreadable blocklist directory opens a blocklist.storage episode" {
if (!build_options.integration) return error.SkipZigTest;
// Mode bits do not apply to root, so the denial the test needs cannot happen.
if (std.c.geteuid() == 0) return error.SkipZigTest;
const gpa = testing.allocator;
const env = try Env.create(gpa);
defer env.destroy();
const io = env.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
env.mgr.diagnostics = &fx.store;
var dir = try env.blocklistDir();
dir.close(io);
try env.tmp.dir.setPermissions(io, .fromMode(0o600));
const failed = env.mgr.pruneOrphans(io);
try env.tmp.dir.setPermissions(io, .fromMode(0o700));
try testing.expectError(error.FileSystem, failed);
try testing.expectEqualStrings("blocklist.storage", try fx.text(
"SELECT code FROM operational_events WHERE resolved_at IS NULL",
));
// `createDirPathStatus` is the first call to touch the unreadable parent,
// so it is the operation that fails.
try testing.expectEqualStrings("create_dir", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
// A pass that can read the directory again closes it.
try env.mgr.pruneOrphans(io);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
src/filter/manager.zig
+627 -25
View File
@@ -59,6 +59,7 @@ const groups_repo = @import("../storage/repositories/groups_repo.zig");
const rules_repo = @import("../storage/repositories/rules_repo.zig");
const sources_repo = @import("../storage/repositories/sources_repo.zig");
const disk_monitor = @import("../storage/disk_monitor.zig");
const events = @import("../storage/events.zig");
const compiler = @import("compiler.zig");
const fetcher = @import("fetcher.zig");
const matcher = @import("matcher.zig");
@@ -162,6 +163,51 @@ pub const State = enum {
}
};
/// The `blocklist.storage` operations, each its own episode subject.
///
/// A fixed set on purpose: several of these fail once per file in a pass, and
/// one slot per operation is what turns that into one report per pass instead
/// of an unbounded list of them.
pub const StorageOp = enum { sweep, directory_read, create_dir, open_dir, delete };
/// One operation's outcome across one pass. `detail` keeps the last failure,
/// and `failures` says how many that pass held — the row's `occurrences` counts
/// failing passes, so the count belongs in the text.
const Aggregate = struct {
failures: u32 = 0,
succeeded: bool = false,
detail: [events.Store.max_detail_len]u8 = @splat(0),
detail_len: u16 = 0,
fn detailText(self: *const Aggregate) []const u8 {
return self.detail[0..self.detail_len];
}
};
/// What a locked body observed, held until `Manager.flushDiagnostics` can
/// report it with no manager lock held.
///
/// Two of this file's operations cannot report from where they stand:
/// `publishRefresh` runs under `writer_lock` by contract and `pruneOrphans`
/// holds both writer mutexes through its filesystem work. Collect-then-flush is
/// what keeps their outcomes without holding a lock across a store call, and
/// nothing here can grow: the storage slots are an enum array and a refresh
/// outcome rides the status entry the source already has.
///
/// The flush still happens inside the lock that serializes passes — a pass
/// drains its own outcomes before it releases `refresh_lock` (or, for a
/// standalone `reload`, `writer_lock`). What collect-then-flush avoids is
/// holding a *manager* lock across a store call, not deferring the report until
/// the next pass could merge into it.
const Pending = struct {
mutex: std.Io.Mutex = .init,
storage: std.EnumArray(StorageOp, Aggregate) = .initFill(.{}),
/// Null until a pass observes a snapshot outcome at all.
snapshot_failed: ?bool = null,
snapshot_detail: [events.Store.max_detail_len]u8 = @splat(0),
snapshot_detail_len: u16 = 0,
};
/// A status is a value with no borrowed memory, so a copy handed to the API
/// outlives every reload. The url is held inline for that reason.
pub const SourceStatus = struct {
@@ -182,6 +228,32 @@ pub const SourceStatus = struct {
url_len: u8 = 0,
last_error: [max_error_len]u8 = @splat(0),
last_error_len: u8 = 0,
/// The diagnostics identity of this source, canonicalized from the WHOLE
/// url by `setUrl`. `url` above is a display copy truncated at
/// `max_url_len`, and two urls sharing a 255-byte prefix would share one
/// episode if that copy were the key.
event_key: [events.Store.max_subject_key_len]u8 = @splat(0),
event_key_len: u16 = 0,
/// Diagnostics accounting for the pass in progress, cleared by every
/// `flushDiagnostics`. `pass_outcome` says this source recorded one at all;
/// `pass_failures` counts the failing ones, which a pass can hold more than
/// one of (a refresh that failed, then the reload that could not load the
/// files it did not write). One flush reports one `blocklist.refresh`
/// occurrence per source, so `occurrences` counts failing passes rather
/// than flushes, and the detail carries how many failures the pass held.
///
/// These two fields live in exactly one copy of the status table at a time,
/// which is what makes that count right while reloads replace the table
/// underneath: a candidate built by `mergeStatuses` carries none of them,
/// `installStatuses` folds the live table's in as it swaps, and the flush
/// claims an entry by copying it and zeroing both fields in one locked
/// step. Copy them anywhere else and the outcome gets reported twice.
pass_outcome: bool = false,
pass_failures: u16 = 0,
pub fn eventKey(self: *const SourceStatus) []const u8 {
return self.event_key[0..self.event_key_len];
}
pub fn errorText(self: *const SourceStatus) []const u8 {
return self.last_error[0..self.last_error_len];
@@ -196,9 +268,12 @@ pub const SourceStatus = struct {
@memcpy(self.url[0..kept], url[0..kept]);
@memset(self.url[kept..], 0);
self.url_len = @intCast(kept);
self.event_key_len = @intCast(events.canonicalKey(url, &self.event_key).len);
}
fn fail(self: *SourceStatus, state: State, text: []const u8) void {
self.pass_failures +|= 1;
self.pass_outcome = true;
self.state = state;
const kept = @min(text.len, max_error_len);
@memcpy(self.last_error[0..kept], text[0..kept]);
@@ -207,6 +282,7 @@ pub const SourceStatus = struct {
}
fn succeed(self: *SourceStatus, at: i64, counts: compiler.Counts) void {
self.pass_outcome = true;
self.state = .ok;
self.counts = counts;
self.last_success = at;
@@ -294,6 +370,14 @@ pub const Manager = struct {
/// what every test and `nxdns check` want. Only the scheduler consults it —
/// see `refreshGated`.
monitor: ?*disk_monitor.Monitor = null,
/// The diagnostics store, wired the same way as `monitor` and null
/// everywhere else. Never touched while a manager lock is held: see
/// `flushDiagnostics`.
diagnostics: ?*events.Store = null,
/// What the locked bodies observed and could not report from where they
/// stood. Bounded by construction — one slot per storage operation, one
/// snapshot outcome — and drained by `flushDiagnostics`.
pending: Pending = .{},
/// Scheduled refresh passes skipped by the disk gate. The `/api/health`
/// rollup reads it through `refreshesGated`.
refreshes_gated: std.atomic.Value(u64) = .init(0),
@@ -388,6 +472,228 @@ pub const Manager = struct {
return kept;
}
// -----------------------------------------------------------------------
// diagnostics
// -----------------------------------------------------------------------
/// Records one storage operation's failure. Callable from anywhere,
/// including under both writer mutexes: it touches `pending` only.
fn noteStorageFailure(
self: *Manager,
io: std.Io,
op: StorageOp,
comptime fmt: []const u8,
args: anytype,
) void {
if (self.diagnostics == null) return;
self.pending.mutex.lockUncancelable(io);
defer self.pending.mutex.unlock(io);
const slot = self.pending.storage.getPtr(op);
slot.failures +|= 1;
var w: std.Io.Writer = .fixed(&slot.detail);
w.print(fmt, args) catch {};
slot.detail_len = @intCast(w.end);
}
fn noteStorageSuccess(self: *Manager, io: std.Io, op: StorageOp) void {
if (self.diagnostics == null) return;
self.pending.mutex.lockUncancelable(io);
defer self.pending.mutex.unlock(io);
self.pending.storage.getPtr(op).succeeded = true;
}
/// Records whether a snapshot was published. `reason` null is the post-swap
/// success; anything else is the pass that could not publish one.
fn noteSnapshot(self: *Manager, io: std.Io, reason: ?[]const u8) void {
if (self.diagnostics == null) return;
self.pending.mutex.lockUncancelable(io);
defer self.pending.mutex.unlock(io);
self.pending.snapshot_failed = reason != null;
const text = reason orelse "";
const kept = @min(text.len, self.pending.snapshot_detail.len);
@memcpy(self.pending.snapshot_detail[0..kept], text[0..kept]);
self.pending.snapshot_detail_len = @intCast(kept);
}
/// Drains `pending` and the status table into the store, holding no manager
/// lock across a store call.
///
/// Called by every pass that can fill either one, and *before that pass
/// releases the lock serializing it* — `refresh_lock` for a refresh pass,
/// `writer_lock` for a standalone `reload`. Draining after the release
/// would let the next pass record its own outcomes on the same entries
/// first, and two failing passes would reach the store as one occurrence.
/// The `defer` that calls this is registered after the unlock `defer` for
/// that reason; defers run last-registered-first.
///
/// It is idempotent: a drained collector reports nothing.
pub fn flushDiagnostics(self: *Manager, io: std.Io) void {
const store = self.diagnostics orelse return;
const now_s = std.Io.Clock.real.now(io).toSeconds();
var storage: std.EnumArray(StorageOp, Aggregate) = undefined;
var snapshot_failed: ?bool = null;
var snapshot_detail: [events.Store.max_detail_len]u8 = undefined;
var snapshot_detail_len: u16 = 0;
{
self.pending.mutex.lockUncancelable(io);
defer self.pending.mutex.unlock(io);
storage = self.pending.storage;
snapshot_failed = self.pending.snapshot_failed;
snapshot_detail = self.pending.snapshot_detail;
snapshot_detail_len = self.pending.snapshot_detail_len;
self.pending.storage = .initFill(.{});
self.pending.snapshot_failed = null;
self.pending.snapshot_detail_len = 0;
}
var it = storage.iterator();
while (it.next()) |kv| {
const op = @tagName(kv.key);
if (kv.value.failures != 0) {
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "{s} ({d} this pass)", .{
kv.value.detailText(),
kv.value.failures,
}) catch buf[0..];
store.report(io, now_s, .blocklist_storage, op, op, .warning, detail);
} else if (kv.value.succeeded) {
store.resolve(io, now_s, .blocklist_storage, op);
}
}
if (snapshot_failed) |failed| {
if (failed) {
store.report(
io,
now_s,
.blocklist_snapshot,
snapshot_key,
"blocklist snapshot",
.@"error",
snapshot_detail[0..snapshot_detail_len],
);
} else {
store.resolve(io, now_s, .blocklist_snapshot, snapshot_key);
}
}
self.flushSourceDiagnostics(io, store, now_s);
}
/// One `blocklist.refresh` episode per source, from the status table.
///
/// The table IS the per-source collection the collect-then-flush rule asks
/// for: `prepareRefresh`, `publishRefresh` and the reload's load outcomes
/// all write their result into the entry, under locks this cannot take. So
/// one entry is copied out at a time under the exclusive lock and the store
/// is called with nothing held.
///
/// The walk is a drain, not an index scan: a reload can replace the whole
/// table between two iterations, and an index into the table it replaced
/// would skip or repeat entries. Each round takes the lock, claims the
/// first entry that still carries pass accounting by copying it out and
/// zeroing the two fields, and reports it with nothing held. Claiming and
/// clearing are one locked step, so an outcome is reported once: a table
/// swapped in mid-drain carries the entries this flush has not claimed yet,
/// and `installStatuses` folded them in for exactly that reason. The drain
/// ends when a scan finds nothing left to claim.
///
/// The drain reaches only the sources the table still holds, so the sweep
/// below is what closes the episode of one that is gone.
fn flushSourceDiagnostics(self: *Manager, io: std.Io, store: *events.Store, now_s: i64) void {
drain: while (true) {
var status: SourceStatus = undefined;
{
self.lock.lockUncancelable(io);
defer self.lock.unlock(io);
const claimed = for (self.statuses) |*entry| {
if (!entry.pass_outcome) continue;
status = entry.*;
entry.pass_outcome = false;
entry.pass_failures = 0;
// A source with no diagnostics identity has nothing to
// report under, but its accounting is cleared all the same:
// left set, it would make every later scan claim it and the
// drain would never end.
if (entry.event_key_len == 0) continue;
break true;
} else false;
if (!claimed) break :drain;
}
if (!status.state.isRefreshFailure() and status.state != .load_failed) {
store.resolve(io, now_s, .blocklist_refresh, status.eventKey());
continue;
}
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "{t}: {s} ({d} this pass)", .{
status.state,
status.errorText(),
status.pass_failures,
}) catch buf[0..];
var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
const label = std.fmt.bufPrint(&label_buf, "{f}", .{
safe_url.redact(status.urlText()),
}) catch &label_buf;
store.report(io, now_s, .blocklist_refresh, status.eventKey(), label, .warning, detail);
}
self.resolveDeletedSources(io, store, now_s);
}
/// Closes the `blocklist.refresh` episode of a source that no longer exists.
///
/// Nothing else can. An episode of this code is closed by its source
/// succeeding, and a source deleted through the API or dropped by a config
/// import never succeeds again: the drain above walks the status table, the
/// deleted source has no entry in it, and the resolved-row pruning never
/// touches an active row. Without this the operator keeps a warning about a
/// list they removed on purpose, and no restart clears it.
///
/// The status table holds every source at every flush site — `refreshAll`
/// syncs it before it refreshes anything and a reload rebuilds it from the
/// rows — so its keys are exactly the episodes that may stay open.
fn resolveDeletedSources(self: *Manager, io: std.Io, store: *events.Store, now_s: i64) void {
var storage: [events.Store.max_kept_keys][events.Store.max_subject_key_len]u8 = undefined;
var lens: [events.Store.max_kept_keys]u16 = undefined;
var len: usize = 0;
{
// Shared: this reads the table and changes nothing in it. The keys
// are copied out because the arena they live in is freed by the
// next `installStatuses`, and the store is called below with
// nothing held.
self.lock.lockSharedUncancelable(io);
defer self.lock.unlockShared(io);
// An empty table before the first published snapshot means "no
// source set has been read yet", not "every source was deleted".
// Sweeping on it would close every episode the last run left open,
// and the pass that follows would reopen each one as a new episode
// with its history reset.
if (self.generation == 0) return;
for (self.statuses) |*entry| {
if (entry.event_key_len == 0) continue;
// `resolveExcept` refuses a kept list longer than
// `max_kept_keys`, because it canonicalizes onto the stack.
// Over that many keyed sources the sweep is skipped whole: the
// alternative is a truncated kept list, which would close
// episodes that are still true. A source deleted while the
// household is over the cap keeps its episode until the count
// falls back under it.
if (len == storage.len) return;
const key = entry.eventKey();
@memcpy(storage[len][0..key.len], key);
lens[len] = entry.event_key_len;
len += 1;
}
}
var kept: [events.Store.max_kept_keys][]const u8 = undefined;
for (0..len) |i| kept[i] = storage[i][0..lens[i]];
store.resolveExcept(io, now_s, .blocklist_refresh, kept[0..len]);
}
// -----------------------------------------------------------------------
// reload
// -----------------------------------------------------------------------
@@ -407,12 +713,35 @@ pub const Manager = struct {
/// table keeps describing that snapshot too — the table is rebuilt off to
/// the side and the load findings are written into it there, so a reload
/// that never publishes changes neither.
///
/// A standalone reload is its own pass, and `writer_lock` is what serializes
/// it against every other writer of the status table. So it flushes inside
/// that lock: the load outcomes it wrote at the swap are drained before any
/// other pass can add its own to the same entries, which is what keeps two
/// failing passes two occurrences instead of one.
pub fn reload(self: *Manager, io: std.Io) Error!void {
// Must not be entered with `writer_lock` held.
self.writer_lock.lockUncancelable(io);
defer self.writer_lock.unlock(io);
// Registered after the unlock so it runs before it, and `defer` and not
// straight-line code after the call: a reload that fails has already
// collected the outcomes that explain why, and leaving them pending
// would hold them until some later pass flushed them under the wrong
// timestamp.
defer self.flushDiagnostics(io);
return self.reloadLocked(io);
}
/// `reload` without the flush, for a caller that is inside a pass with a
/// flush of its own. One pass flushes once: flushing here as well would
/// split the pass's outcomes across two reports.
fn reloadCollecting(self: *Manager, io: std.Io) Error!void {
// Must not be entered with `writer_lock` held.
self.writer_lock.lockUncancelable(io);
defer self.writer_lock.unlock(io);
try self.reloadLocked(io);
}
fn reloadLocked(self: *Manager, io: std.Io) Error!void {
var rows = try sources_repo.listSourceRows(self.database, self.gpa);
defer rows.deinit(self.gpa);
@@ -543,6 +872,7 @@ pub const Manager = struct {
rows.items.len,
memory_bytes,
});
self.noteSnapshot(io, null);
}
/// What one enabled source contributes to the snapshot being built. Nothing
@@ -642,6 +972,11 @@ pub const Manager = struct {
pub fn refreshSource(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool {
self.refresh_lock.lockUncancelable(io);
defer self.refresh_lock.unlock(io);
// Registered after the unlock, so it runs before it: a pass drains its
// own outcomes while it still holds `refresh_lock`. `defer` at all, so
// a refresh that fails outright still reports what it collected instead
// of leaving it for an unrelated later flush.
defer self.flushDiagnostics(io);
return self.refreshSourceLocked(io, row);
}
@@ -706,6 +1041,9 @@ pub const Manager = struct {
pub fn refreshAll(self: *Manager, io: std.Io) Error!void {
self.refresh_lock.lockUncancelable(io);
defer self.refresh_lock.unlock(io);
// Inside `refresh_lock`, by being registered after the unlock: see
// `refreshSource`.
defer self.flushDiagnostics(io);
var rows = try sources_repo.listSourceRows(self.database, self.gpa);
defer rows.deinit(self.gpa);
@@ -718,8 +1056,9 @@ pub const Manager = struct {
_ = try self.refreshSourceLocked(io, row);
}
// `reload` takes `writer_lock`, which the pass has been careful not to
// hold: the order is `refresh_lock` first, always.
return self.reload(io);
// hold: the order is `refresh_lock` first, always. The collecting
// variant, because the `defer` above is this pass's one flush.
return self.reloadCollecting(io);
}
/// The three temporary files one refresh compiles into, before the header
@@ -1161,9 +1500,16 @@ pub const Manager = struct {
// ask the same filesystem for.
try self.sweepOrphans(io);
// `startupPass` flushes its own outcomes before it releases
// `refresh_lock`, so the only flush left here is the one the failure
// note below needs.
self.startupPass(io) catch |err| switch (err) {
error.Canceled => return error.Canceled,
else => log.warn("blocklist startup pass failed: {s}", .{@errorName(err)}),
else => {
log.warn("blocklist startup pass failed: {s}", .{@errorName(err)});
self.noteSnapshot(io, @errorName(err));
self.flushDiagnostics(io);
},
};
if (!self.update.enabled) return;
@@ -1175,19 +1521,33 @@ pub const Manager = struct {
};
while (true) {
try interval.sleep(io);
// Ahead of the gate as well as ahead of the pass: the sweep only
// unlinks, so it is the one thing here that can give a critically
// full disk room back, and gating it would keep the residue that
// helped fill the disk in the first place.
try self.sweepOrphans(io);
if (self.refreshGated()) continue;
self.refreshAll(io) catch |err| switch (err) {
error.Canceled => return error.Canceled,
else => log.warn("blocklist refresh pass failed: {s}", .{@errorName(err)}),
};
try self.scheduledPass(io);
}
}
/// What one elapsed interval does. Split from the loop above so a test can
/// run the pass without waiting the interval out; nothing in production
/// calls it but `runScheduler`.
pub fn scheduledPass(self: *Manager, io: std.Io) std.Io.Cancelable!void {
// Ahead of the gate as well as ahead of the refresh: the sweep only
// unlinks, so it is the one thing here that can give a critically full
// disk room back, and gating it would keep the residue that helped fill
// the disk in the first place.
try self.sweepOrphans(io);
if (self.refreshGated()) return;
// `refreshAll` flushes the pass itself, so the only flush left here is
// the one the failure note below needs: flushing unconditionally would
// report every outcome of the pass a second time.
self.refreshAll(io) catch |err| switch (err) {
error.Canceled => return error.Canceled,
else => {
log.warn("blocklist refresh pass failed: {s}", .{@errorName(err)});
self.noteSnapshot(io, @errorName(err));
self.flushDiagnostics(io);
},
};
}
/// `pruneOrphans` with its failure absorbed. Leftover bytes under
/// `<data_dir>/blocklists/` are not an outage, and a sweep that could not
/// read the directory must not cost the household the refresh pass behind
@@ -1197,10 +1557,16 @@ pub const Manager = struct {
/// Taken from outside every `*Locked` body: `pruneOrphans` takes both
/// writer mutexes itself and neither is reentrant.
fn sweepOrphans(self: *Manager, io: std.Io) std.Io.Cancelable!void {
self.pruneOrphans(io) catch |err| switch (err) {
if (self.pruneOrphans(io)) {
self.noteStorageSuccess(io, .sweep);
} else |err| switch (err) {
error.Canceled => return error.Canceled,
else => log.warn("pruning orphaned blocklist files failed: {s}", .{@errorName(err)}),
};
else => {
log.warn("pruning orphaned blocklist files failed: {s}", .{@errorName(err)});
self.noteStorageFailure(io, .sweep, "pruning orphaned blocklist files failed: {s}", .{@errorName(err)});
},
}
self.flushDiagnostics(io);
}
/// The §11.6 gate, consulted by scheduled passes only (ruling 17). A
@@ -1232,11 +1598,14 @@ pub const Manager = struct {
fn startupPass(self: *Manager, io: std.Io) Error!void {
self.refresh_lock.lockUncancelable(io);
defer self.refresh_lock.unlock(io);
// Inside `refresh_lock`, by being registered after the unlock: see
// `refreshSource`.
defer self.flushDiagnostics(io);
// Ahead of the gate on purpose: loading the compiled files that already
// exist is a read. A full disk must not cost the household its
// filtering as well as its downloads.
try self.reload(io);
try self.reloadCollecting(io);
if (self.refreshGated()) return;
@@ -1251,7 +1620,7 @@ pub const Manager = struct {
if (!self.needsRefresh(io, row, now)) continue;
if (try self.refreshSourceLocked(io, row)) refreshed = true;
}
if (refreshed) try self.reload(io);
if (refreshed) try self.reloadCollecting(io);
}
fn needsRefresh(self: *Manager, io: std.Io, row: sources_repo.SourceRow, now: i64) bool {
@@ -1297,6 +1666,13 @@ pub const Manager = struct {
/// `<data_dir>/blocklists/` if nothing has yet, and an empty directory
/// sweeps to nothing.
pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
defer self.flushDiagnostics(io);
return self.pruneOrphansLocked(io);
}
/// Assumes nothing and takes both writer mutexes itself. Split from
/// `pruneOrphans` so the diagnostics flush above happens with neither held.
fn pruneOrphansLocked(self: *Manager, io: std.Io) Error!void {
// `refresh_lock` first, and for the reason it exists: the download and
// the compile are the only writers of `.raw.tmp`, `.list.tmp`,
// `.wild.tmp` and `.allow.tmp`, and they hold it for as long as they
@@ -1334,6 +1710,7 @@ pub const Manager = struct {
error.Canceled => return error.Canceled,
else => {
log.warn("pruning blocklists: reading the directory failed: {s}", .{@errorName(err)});
self.noteStorageFailure(io, .directory_read, "reading the blocklist directory failed: {s}", .{@errorName(err)});
return error.FileSystem;
},
} orelse break;
@@ -1343,6 +1720,8 @@ pub const Manager = struct {
try doomed.append(self.gpa, try self.gpa.dupe(u8, entry.name));
}
self.noteStorageSuccess(io, .directory_read);
for (doomed.items) |name| {
self.deleteQuietly(io, dir, name);
log.info("pruned orphaned blocklist file {s}", .{name});
@@ -1381,8 +1760,21 @@ pub const Manager = struct {
}
/// Publishes a built table and frees the one it replaces. The caller holds
/// the exclusive lock, so no reader is inside the old table.
/// the exclusive lock, so no reader is inside the old table and no flush is
/// half way through draining it.
///
/// The pass accounting the live table still holds is folded into the
/// incoming entry of the same id first. `mergeStatuses` left the candidate
/// carrying none, so an outcome recorded after the candidate was built —
/// and any a flush has not drained yet — survives the swap exactly once. An
/// outcome a flush already reported is zero in the live table, so nothing
/// here resurrects it.
fn installStatuses(self: *Manager, table: StatusTable) void {
for (table.items) |*incoming| {
const live = entryFor(self.statuses, incoming.id) orelse continue;
incoming.pass_failures +|= live.pass_failures;
incoming.pass_outcome = incoming.pass_outcome or live.pass_outcome;
}
self.status_arena.deinit();
self.status_arena = table.arena;
self.statuses = table.items;
@@ -1449,29 +1841,46 @@ pub const Manager = struct {
error.Canceled => return error.Canceled,
else => {
log.warn("creating {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
self.noteStorageFailure(io, .create_dir, "creating {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
return error.FileSystem;
},
};
return self.paths.dir.openDir(io, self.paths.subdir, options) catch |err| switch (err) {
self.noteStorageSuccess(io, .create_dir);
const dir = self.paths.dir.openDir(io, self.paths.subdir, options) catch |err| switch (err) {
error.Canceled => return error.Canceled,
else => {
log.warn("opening {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
self.noteStorageFailure(io, .open_dir, "opening {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
return error.FileSystem;
},
};
self.noteStorageSuccess(io, .open_dir);
return dir;
}
/// A temporary that cannot be removed is not a failure of the operation
/// that made it, but it is not nothing either: it is left visible.
fn deleteQuietly(self: *Manager, io: std.Io, dir: std.Io.Dir, name: []const u8) void {
_ = self;
dir.deleteFile(io, name) catch |err| switch (err) {
error.FileNotFound => {},
else => log.warn("deleting {s} failed: {s}", .{ name, @errorName(err) }),
};
if (dir.deleteFile(io, name)) {
self.noteStorageSuccess(io, .delete);
} else |err| switch (err) {
// A name that was never created is the ordinary case: the temporary
// deletes are installed before the files exist.
error.FileNotFound => self.noteStorageSuccess(io, .delete),
else => {
log.warn("deleting {s} failed: {s}", .{ name, @errorName(err) });
self.noteStorageFailure(io, .delete, "deleting {s} failed: {s}", .{ name, @errorName(err) });
},
}
}
};
/// The one subject `blocklist.snapshot` ever has: a box publishes exactly one
/// snapshot, and every source that failed to load is its own
/// `blocklist.refresh` episode.
const snapshot_key = "snapshot";
/// A status table and the arena holding it. Until `installStatuses` takes it,
/// it is a candidate nobody can see, and `deinit` frees it whole.
const StatusTable = struct {
@@ -1488,6 +1897,14 @@ const StatusTable = struct {
/// over from `previous`. A source deleted since `previous` was built is gone; a
/// source added since starts blank. `previous` is only read, so the caller's
/// published table is untouched by this.
///
/// The pass accounting is *not* carried: it lives in exactly one table copy at
/// a time. `previous` is a snapshot of the published table taken outside the
/// swap, so copying its counters here would leave the same outcomes in two
/// tables — the live one for a flush to drain, and this candidate for the
/// reload's own flush to report a second time. A candidate holds only what
/// `applyLoadOutcomes` writes into it; what the live table holds is folded in
/// by `installStatuses` under the exclusive lock.
fn mergeStatuses(
table: []SourceStatus,
rows: []const sources_repo.SourceRow,
@@ -1500,6 +1917,8 @@ fn mergeStatuses(
status.* = prior;
break;
}
status.pass_outcome = false;
status.pass_failures = 0;
// After the carry-over: a url edited on the row wins over the one the
// prior entry recorded.
status.setUrl(row.url);
@@ -1716,6 +2135,7 @@ fn containsId(rows: []const sources_repo.SourceRow, id: i64) bool {
// real swaps under load are the integration suite's (S9).
const testing = std.testing;
const events_fixture = @import("../storage/events_fixture.zig");
const migrations = @import("../storage/migrations.zig");
fn openMigrated() !db.Db {
@@ -2275,6 +2695,13 @@ test "a candidate table carries prior entries over and leaves the published one
try testing.expectEqual(State.never_fetched, candidate[1].state);
try testing.expect(!candidate[1].loaded);
// The one thing a candidate does not carry. `published[0]` is holding a
// failure no flush has drained yet; copying its accounting here would leave
// the same outcome in two tables, and the flush of each would report it.
try testing.expect(published[0].pass_outcome);
try testing.expect(!candidate[0].pass_outcome);
try testing.expectEqual(@as(u16, 0), candidate[0].pass_failures);
// The published table is untouched, so a reload that fails before the swap
// leaves it describing the snapshot that is still serving — including the
// entry of the deleted source, which that snapshot still enforces.
@@ -2284,6 +2711,181 @@ test "a candidate table carries prior entries over and leaves the published one
try testing.expectEqualStrings("https://lists.example/one.txt", published[0].urlText());
}
test "installing a table folds the live pass accounting in by id" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
var f: fetcher.Fetcher = undefined;
var mgr = try testManager(&database, &f);
defer mgr.deinit(io);
// Source 1 recorded a failure the flush has not drained; source 2 was
// drained already; source 3 recorded one this reload knows nothing about.
var live: std.heap.ArenaAllocator = .init(testing.allocator);
const live_items = try live.allocator().alloc(SourceStatus, 3);
live_items[0] = .{ .id = 1, .pass_outcome = true, .pass_failures = 2 };
live_items[1] = .{ .id = 2 };
live_items[2] = .{ .id = 3, .pass_outcome = true, .pass_failures = 1 };
{
mgr.lock.lockUncancelable(io);
defer mgr.lock.unlock(io);
mgr.installStatuses(.{ .arena = live, .items = live_items });
}
// What a reload built beside it, carrying only its own load outcomes.
var incoming: std.heap.ArenaAllocator = .init(testing.allocator);
const incoming_items = try incoming.allocator().alloc(SourceStatus, 3);
incoming_items[0] = .{ .id = 1, .pass_outcome = true, .pass_failures = 1 };
incoming_items[1] = .{ .id = 2, .pass_outcome = true, .pass_failures = 4 };
incoming_items[2] = .{ .id = 3 };
{
mgr.lock.lockUncancelable(io);
defer mgr.lock.unlock(io);
mgr.installStatuses(.{ .arena = incoming, .items = incoming_items });
}
try testing.expectEqual(@as(u16, 3), mgr.statuses[0].pass_failures);
try testing.expect(mgr.statuses[0].pass_outcome);
// A drained entry adds nothing: what the swap publishes is the reload's own
// accounting and no resurrection of what was already reported.
try testing.expectEqual(@as(u16, 4), mgr.statuses[1].pass_failures);
try testing.expect(mgr.statuses[1].pass_outcome);
// The half the swap used to lose: an outcome the live table held and the
// candidate never saw.
try testing.expectEqual(@as(u16, 1), mgr.statuses[2].pass_failures);
try testing.expect(mgr.statuses[2].pass_outcome);
}
test "the flush claims every entry that carries pass accounting, once" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1_700_000_000);
defer fx.deinit();
var f: fetcher.Fetcher = undefined;
var mgr = try testManager(&database, &f);
defer mgr.deinit(io);
mgr.diagnostics = &fx.store;
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
const items = try arena.allocator().alloc(SourceStatus, 3);
items[0] = .{ .id = 1 };
items[0].setUrl("https://lists.example/one.txt");
items[0].fail(.fetch_failed, "HttpStatus");
// No url, so no episode to report under. The drain has to claim it anyway:
// an entry left with `pass_outcome` set is the one every later scan finds
// first, and the entry behind it would never be reached.
items[1] = .{ .id = 2, .pass_outcome = true, .pass_failures = 1 };
items[2] = .{ .id = 3 };
items[2].setUrl("https://lists.example/three.txt");
items[2].succeed(1_700_000_000, .{ .domains = 3 });
{
mgr.lock.lockUncancelable(io);
defer mgr.lock.unlock(io);
mgr.installStatuses(.{ .arena = arena, .items = items });
}
mgr.flushDiagnostics(io);
try testing.expectEqual(@as(i64, 1), try fx.count(
"SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh'",
));
try testing.expectEqual(@as(i64, 1), try fx.count(
"SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
));
for (mgr.statuses) |entry| {
try testing.expect(!entry.pass_outcome);
try testing.expectEqual(@as(u16, 0), entry.pass_failures);
}
// Drained: flushing the same table again reports nothing a second time.
mgr.flushDiagnostics(io);
try testing.expectEqual(@as(i64, 1), try fx.count(
"SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
));
}
test "the flush closes the episode of a source that is no longer in the table" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1_700_000_000);
defer fx.deinit();
var f: fetcher.Fetcher = undefined;
var mgr = try testManager(&database, &f);
defer mgr.deinit(io);
mgr.diagnostics = &fx.store;
// What the operator deleted while it was failing. Nothing will ever record
// a success for it, so nothing but the sweep can close this.
fx.store.report(
io,
1_700_000_000,
.blocklist_refresh,
"https://lists.example/deleted.txt",
"lists.example/deleted.txt",
.warning,
"fetch_failed: HttpStatus (1 this pass)",
);
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
const items = try arena.allocator().alloc(SourceStatus, 1);
items[0] = .{ .id = 1 };
items[0].setUrl("https://lists.example/one.txt");
items[0].fail(.fetch_failed, "HttpStatus");
{
mgr.lock.lockUncancelable(io);
defer mgr.lock.unlock(io);
mgr.installStatuses(.{ .arena = arena, .items = items });
}
// No snapshot published yet, so the table is not known to describe the
// source set and the sweep must not run: the drain reports the failing
// source and the deleted one's episode is left alone.
mgr.flushDiagnostics(io);
try testing.expectEqual(@as(i64, 2), try fx.count(
"SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL",
));
mgr.generation = 1;
mgr.flushDiagnostics(io);
// One left active, and it is the source that still exists.
try testing.expectEqual(@as(i64, 1), try fx.count(
"SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL",
));
try testing.expectEqualStrings(
"https://lists.example/one.txt",
try fx.text(
"SELECT subject_key FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL",
),
);
try testing.expectEqualStrings(
"https://lists.example/deleted.txt",
try fx.text(
"SELECT subject_key FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NOT NULL",
),
);
}
test "a disabled source stops being loaded" {
var statuses = [_]SourceStatus{.{ .id = 1 }};
statuses[0].succeed(1_700_000_000, .{ .domains = 9 });
src/server/cert_store.zig
+130 -10
View File
@@ -13,6 +13,8 @@
//! through the size.
const std = @import("std");
const events = @import("../storage/events.zig");
const tls_server = @import("../platform/tls_server.zig");
const log = std.log.scoped(.cert_store);
@@ -101,6 +103,11 @@ pub const ReloadHook = struct {
};
pub const CertStore = struct {
/// Which endpoint's certificate this store holds. It is the subject of
/// every `certificate.reload` event, and a box serving both DoH and DoT
/// runs two stores over two file pairs.
pub const Kind = enum { doh, dot };
gpa: std.mem.Allocator,
/// Borrowed from the config; must outlive the store.
cert_path: []const u8,
@@ -132,6 +139,12 @@ pub const CertStore = struct {
/// `reload` synchronously (that would self-deadlock on `reload_mutex`).
after_load_hook: ?ReloadHook,
/// Set by the composition root right after `init`, with `diagnostics`.
kind: Kind = .doh,
/// Wired the same way and for the same reason as every other subsystem's:
/// the store is fully usable without it, and `nxdns check` has none.
diagnostics: ?*events.Store = null,
reloads: std.atomic.Value(u64),
reload_failures: std.atomic.Value(u64),
/// Wall-clock second of the last successful load, including the one in
@@ -243,7 +256,7 @@ pub const CertStore = struct {
};
while (true) {
try interval.sleep(io);
self.pollOnce(io);
self.pollOnce(io, std.Io.Clock.real.now(io).toSeconds());
}
}
@@ -252,13 +265,15 @@ pub const CertStore = struct {
/// changed, and the old one keeps serving either way. A failed reload
/// warns and counts (`reload_failures`); the signature stays at the loaded
/// pair, so every subsequent poll retries until the files parse.
pub fn pollOnce(self: *CertStore, io: std.Io) void {
const cert_sig = statSig(io, self.cert_path) catch {
pub fn pollOnce(self: *CertStore, io: std.Io, now_s: i64) void {
const cert_sig = statSig(io, self.cert_path) catch |err| {
log.warn("stat {s} failed; keeping the loaded certificate", .{self.cert_path});
self.reportReload(io, now_s, "stat of the certificate failed", @errorName(err));
return;
};
const key_sig = statSig(io, self.key_path) catch {
const key_sig = statSig(io, self.key_path) catch |err| {
log.warn("stat {s} failed; keeping the loaded certificate", .{self.key_path});
self.reportReload(io, now_s, "stat of the private key failed", @errorName(err));
return;
};
const observed: Signature = .{ .cert = cert_sig, .key = key_sig };
@@ -266,18 +281,36 @@ pub const CertStore = struct {
self.mutex.lockUncancelable(io);
const loaded = self.loaded;
self.mutex.unlock(io);
if (!changed(loaded, observed)) return;
if (!changed(loaded, observed)) {
// A poll that stat'ed both files and found nothing to do is a
// fully healthy pass, so it closes any episode a transient stat
// failure opened. Without this, a file that never changes again
// would leave that episode open forever.
if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
return;
}
if (self.reload(io)) {
log.info("certificate reloaded from {s}", .{self.cert_path});
if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
} else |err| {
log.warn("certificate reload from {s} failed ({s}); the old certificate keeps serving", .{
self.cert_path,
humanMessage(err),
});
self.reportReload(io, now_s, "certificate reload failed", humanMessage(err));
}
}
/// A warning, never an error: a stat failure can be a rename window, and a
/// failed reload leaves the loaded certificate serving. Nothing is down.
fn reportReload(self: *CertStore, io: std.Io, now_s: i64, message: []const u8, reason: []const u8) void {
const store = self.diagnostics orelse return;
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "{s}: {s}", .{ message, reason }) catch buf[0..];
store.report(io, now_s, .certificate_reload, @tagName(self.kind), @tagName(self.kind), .warning, detail);
}
pub fn snapshotStats(self: *const CertStore) Stats {
return .{
.reloads = self.reloads.load(.monotonic),
@@ -406,6 +439,7 @@ fn statSig(io: std.Io, path: []const u8) !FileSig {
// tests
// ---------------------------------------------------------------------------
const events_fixture = @import("../storage/events_fixture.zig");
const fixtures = @import("test_fixtures");
const testing = std.testing;
@@ -699,7 +733,7 @@ test "pollOnce reloads on a changed stat pair and stays put on an unchanged one"
var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
defer store.deinit(io);
store.pollOnce(io);
store.pollOnce(io, 1000);
try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
// Same certificate plus a trailing newline: the PEM still parses and the
@@ -710,14 +744,14 @@ test "pollOnce reloads on a changed stat pair and stays put on an unchanged one"
const old = store.acquire(io);
store.release(io, old);
store.pollOnce(io);
store.pollOnce(io, 1000);
try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
const fresh = store.acquire(io);
try testing.expect(fresh != old);
store.release(io, fresh);
store.pollOnce(io);
store.pollOnce(io, 1000);
try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
}
@@ -734,7 +768,7 @@ test "pollOnce warns and keeps serving when a reload fails" {
store.release(io, before);
try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
store.pollOnce(io);
store.pollOnce(io, 1000);
const stats = store.snapshotStats();
try testing.expectEqual(@as(u64, 0), stats.reloads);
@@ -755,7 +789,7 @@ test "pollOnce does nothing when a file cannot be stat'ed" {
defer store.deinit(io);
try env.tmp.dir.deleteFile(io, "cert.pem");
store.pollOnce(io);
store.pollOnce(io, 1000);
const stats = store.snapshotStats();
try testing.expectEqual(@as(u64, 0), stats.reloads);
@@ -873,3 +907,89 @@ test "a reload overlapping another reload's window publishes last" {
store.mutex.unlock(io);
try testing.expectEqual(@as(u64, grown.len), final.cert.size);
}
test "a failed reload opens an episode keyed by endpoint kind and a good one closes it" {
var env: TestEnv = undefined;
try env.init();
defer env.deinit();
const io = env.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
defer store.deinit(io);
store.kind = .dot;
store.diagnostics = &fx.store;
try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
store.pollOnce(io, 1000);
try testing.expectEqualStrings("certificate.reload", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("dot", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
// The same certificate plus a newline: it parses, and the size differs even
// within one timestamp granule.
const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
defer testing.allocator.free(grown);
try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = grown });
store.pollOnce(io, 1100);
try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
test "a failed stat opens the same episode a failed reload would" {
var env: TestEnv = undefined;
try env.init();
defer env.deinit();
const io = env.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
defer store.deinit(io);
store.diagnostics = &fx.store;
try env.tmp.dir.deleteFile(io, "cert.pem");
store.pollOnce(io, 1000);
try testing.expectEqualStrings("doh", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
try testing.expectEqual(@as(u64, 0), store.snapshotStats().reload_failures);
}
test "an unchanged poll closes the episode a transient stat failure opened" {
var env: TestEnv = undefined;
try env.init();
defer env.deinit();
const io = env.io();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
defer store.deinit(io);
store.diagnostics = &fx.store;
// What a stat failure in a rename window left open. The file it named is
// back and unchanged, so no reload will ever close this episode.
fx.store.report(io, 1000, .certificate_reload, "doh", "doh", .warning, "stat of the certificate failed: FileNotFound");
store.pollOnce(io, 1100);
try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
src/server/client_names.zig
+105 -1
View File
@@ -30,6 +30,7 @@ const clients_repo = @import("../storage/repositories/clients_repo.zig");
const db = @import("../storage/db.zig");
const dns_header = @import("../dns/header.zig");
const edns = @import("../dns/edns.zig");
const events = @import("../storage/events.zig");
const forward_client = @import("../local/forward_client.zig");
const local_tables = @import("local_tables.zig");
const name_mod = @import("../dns/name.zig");
@@ -103,6 +104,9 @@ pub const Resolver = struct {
tables: *local_tables.LocalTables,
stats: Stats = .{},
exchange_fn: ExchangeFn = defaultExchange,
/// Wired by the composition root after `init`, following the
/// `gate: ?*disk_monitor.Monitor` idiom. Null in every unit test here.
diagnostics: ?*events.Store = null,
pub fn init(tables: *local_tables.LocalTables) Resolver {
return .{ .tables = tables };
@@ -127,10 +131,13 @@ pub const Resolver = struct {
self.mutex.lockUncancelable(io);
self.stats.read_failures += 1;
self.mutex.unlock(io);
self.reportStorage(io, now_s, "read", "selecting clients to name failed", @errorName(err), 1);
return;
};
if (self.diagnostics) |store| store.resolve(io, now_s, .client_names_storage, "read");
var write_failures: u64 = 0;
var last_write_error: []const u8 = "";
for (candidates[0..count]) |*candidate| {
var learned_buf: [types.max_name_len]u8 = undefined;
const result = self.attempt(io, candidate.ip(), &learned_buf);
@@ -144,6 +151,7 @@ pub const Resolver = struct {
log.warn("recording the name of {s} failed: {s}", .{ candidate.ip(), @errorName(err) });
}
write_failures += 1;
last_write_error = @errorName(err);
}
self.mutex.lockUncancelable(io);
@@ -158,10 +166,38 @@ pub const Resolver = struct {
self.mutex.unlock(io);
}
if (write_failures == 0) return;
// The clean-pass determination is what closes a `write` episode: one
// aggregated outcome per pass, so a row's `occurrences` counts failing
// passes rather than failing rows.
if (write_failures == 0) {
if (self.diagnostics) |store| store.resolve(io, now_s, .client_names_storage, "write");
return;
}
self.mutex.lockUncancelable(io);
self.stats.write_failures += write_failures;
self.mutex.unlock(io);
self.reportStorage(io, now_s, "write", "recording a client name failed", last_write_error, write_failures);
}
/// One aggregated report per pass. `count` is how many failures that pass
/// held; the detail carries it because the row itself counts passes.
fn reportStorage(
self: *Resolver,
io: std.Io,
now_s: i64,
operation: []const u8,
message: []const u8,
error_name: []const u8,
count: u64,
) void {
const store = self.diagnostics orelse return;
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "{s}: {s} ({d} this pass)", .{
message,
error_name,
count,
}) catch buf[0..];
store.report(io, now_s, .client_names_storage, operation, operation, .warning, detail);
}
const Attempt = struct {
@@ -355,6 +391,7 @@ fn exchangeOnce(
const forward_zones = @import("../local/forward_zones.zig");
const migrations = @import("../storage/migrations.zig");
const events_fixture = @import("../storage/events_fixture.zig");
const testing = std.testing;
/// A stub exchange whose reply and error are set by the test. `calls` is the
@@ -1157,3 +1194,70 @@ test "an extended rcode of zero still reads as the header's rcode" {
try testing.expectEqualStrings("nas.lan", (try f.learned("192.168.1.10", &buf)).?);
try testing.expectEqual(@as(u64, 1), f.resolver.snapshotStats(f.io()).answered);
}
test "a failing write opens one episode per pass and a clean pass closes it" {
var f: Fixture = undefined;
try fixture(&f, "168.192.in-addr.arpa");
defer f.deinit();
var fx: events_fixture.Fixture = .{};
try fx.init(f.io(), 1000);
defer fx.deinit();
f.resolver.diagnostics = &fx.store;
// Two addresses outside the declared zone: both reach the write step
// without any exchange, which is not what this test is about.
try clients_repo.upsertSeen(&f.database, "10.0.0.1", 1700000000);
try clients_repo.upsertSeen(&f.database, "10.0.0.2", 1700000000);
try f.database.exec(
\\CREATE TRIGGER refuse_update BEFORE UPDATE ON clients
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
);
f.resolver.runPass(f.io(), &f.database, 1700000000);
// Two failing rows, one aggregated report: the row counts failing passes.
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqualStrings("client_names.storage", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("write", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT occurrences FROM operational_events"));
f.resolver.runPass(f.io(), &f.database, 1700000060);
try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
try f.database.exec("DROP TRIGGER refuse_update;");
f.resolver.runPass(f.io(), &f.database, 1700000120);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
test "a failing candidate select opens a read episode the next clean pass closes" {
var f: Fixture = undefined;
try fixture(&f, "168.192.in-addr.arpa");
defer f.deinit();
var fx: events_fixture.Fixture = .{};
try fx.init(f.io(), 1000);
defer fx.deinit();
f.resolver.diagnostics = &fx.store;
try clients_repo.upsertSeen(&f.database, "10.0.0.1", 1700000000);
try f.database.exec("ALTER TABLE clients RENAME TO clients_aside;");
f.resolver.runPass(f.io(), &f.database, 1700000000);
try testing.expectEqualStrings("read", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
try f.database.exec("ALTER TABLE clients_aside RENAME TO clients;");
f.resolver.runPass(f.io(), &f.database, 1700000060);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
src/server/clients.zig
+118
View File
@@ -25,6 +25,7 @@ const client_names = @import("client_names.zig");
const clients_repo = @import("../storage/repositories/clients_repo.zig");
const db = @import("../storage/db.zig");
const disk_monitor = @import("../storage/disk_monitor.zig");
const events = @import("../storage/events.zig");
const logger = @import("../storage/logger.zig");
const log = std.log.scoped(.clients);
@@ -63,6 +64,9 @@ pub const Tracker = struct {
count: u32,
passes: u64,
stats: Stats,
/// Wired by the composition root after `init`, following the
/// `gate: ?*disk_monitor.Monitor` idiom. Null in every unit test here.
diagnostics: ?*events.Store = null,
/// `retention_days` is `logging.retention_days`, the same knob the query log
/// prunes by (milestone-7 ruling 16). A client silent for that long is as
@@ -185,6 +189,7 @@ pub const Tracker = struct {
var flushed: u64 = 0;
var failures: u64 = 0;
var last_failure: []const u8 = "";
for (batch) |entry| {
// `logger.max_client_len` is the RFC 5952 bound every address text
// in this program is sized by, so `format` cannot fail here.
@@ -199,9 +204,19 @@ pub const Tracker = struct {
log.warn("materialising client {s} failed: {s}", .{ w.buffered(), @errorName(err) });
}
failures += 1;
last_failure = @errorName(err);
}
}
// One aggregated outcome per pass, so the row's `occurrences` counts
// failing passes rather than failing addresses. A pass that wrote
// nothing resolves nothing: an empty batch is not evidence of success.
if (failures != 0) {
self.reportStorage(io, now_s, "materialise", "materialising a client failed", last_failure, failures);
} else if (flushed != 0) {
if (self.diagnostics) |store| store.resolve(io, now_s, .clients_storage, "materialise");
}
self.mutex.lockUncancelable(io);
self.passes += 1;
self.stats.flushed += flushed;
@@ -215,17 +230,39 @@ pub const Tracker = struct {
self.mutex.lockUncancelable(io);
self.stats.pruned += deleted;
self.mutex.unlock(io);
if (self.diagnostics) |store| store.resolve(io, now_s, .clients_storage, "prune");
} else |err| {
log.warn("pruning clients before {d} failed: {s}", .{ cutoff, @errorName(err) });
self.mutex.lockUncancelable(io);
self.stats.flush_failures += 1;
self.mutex.unlock(io);
self.reportStorage(io, now_s, "prune", "pruning stale clients failed", @errorName(err), 1);
}
}
if (names) |resolver| resolver.runPass(io, database, now_s);
}
/// One aggregated report per pass; `count` is how many failures it held.
fn reportStorage(
self: *Tracker,
io: std.Io,
now_s: i64,
operation: []const u8,
message: []const u8,
error_name: []const u8,
count: u64,
) void {
const store = self.diagnostics orelse return;
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "{s}: {s} ({d} this pass)", .{
message,
error_name,
count,
}) catch buf[0..];
store.report(io, now_s, .clients_storage, operation, operation, .warning, detail);
}
pub fn snapshotStats(self: *Tracker, io: std.Io) Stats {
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
@@ -258,6 +295,7 @@ pub const Tracker = struct {
// tests
// ---------------------------------------------------------------------------
const events_fixture = @import("../storage/events_fixture.zig");
const migrations = @import("../storage/migrations.zig");
const testing = std.testing;
@@ -649,3 +687,83 @@ test "a gated pass attempts no naming either" {
try testing.expectEqual(@as(usize, 0), CountingExchange.calls);
try testing.expectEqual(@as(u64, 0), names.snapshotStats(io).attempted);
}
test "a failing materialise opens one episode per pass and a clean pass closes it" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
var tracker: Tracker = .init(30);
tracker.diagnostics = &fx.store;
try database.exec(
\\CREATE TRIGGER refuse_insert BEFORE INSERT ON clients
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
);
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000000);
_ = tracker.trackAt(io, parsed("192.168.1.11"), 1700000001);
tracker.flushOnce(io, &database, true, null);
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqualStrings("clients.storage", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("materialise", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT occurrences FROM operational_events"));
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000060);
tracker.flushOnce(io, &database, true, null);
try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
try database.exec("DROP TRIGGER refuse_insert;");
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000120);
tracker.flushOnce(io, &database, true, null);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
test "a failing prune opens its own episode the next due pass closes" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openMigrated();
defer database.close();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
var tracker: Tracker = .init(30);
tracker.diagnostics = &fx.store;
// One pass short of due, so the pass below is the pruning one.
tracker.passes = Tracker.prune_every_passes - 1;
try database.exec(
\\CREATE TRIGGER refuse_delete BEFORE DELETE ON clients
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
);
try clients_repo.upsertSeen(&database, "192.168.1.10", 1);
tracker.flushOnce(io, &database, true, null);
try testing.expectEqualStrings("prune", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
try database.exec("DROP TRIGGER refuse_delete;");
tracker.passes = Tracker.prune_every_passes - 1;
tracker.flushOnce(io, &database, true, null);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
src/storage/config_schema.zig
+101 -1
View File
@@ -98,6 +98,57 @@ pub const ddl_v1: [:0]const u8 =
\\);
\\
\\CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL);
\\
\\CREATE TABLE operational_events (
\\ id INTEGER PRIMARY KEY,
\\ code TEXT NOT NULL,
\\ subject_key TEXT NOT NULL,
\\ subject_label TEXT NOT NULL,
\\ severity TEXT NOT NULL CHECK (severity IN ('warning', 'error')),
\\ first_seen INTEGER NOT NULL,
\\ last_seen INTEGER NOT NULL,
\\ occurrences INTEGER NOT NULL CHECK (occurrences > 0),
\\ resolved_at INTEGER,
\\ detail TEXT NOT NULL DEFAULT '',
\\ CHECK (resolved_at IS NULL OR resolved_at >= first_seen)
\\);
\\CREATE UNIQUE INDEX idx_operational_events_active
\\ ON operational_events(code, subject_key) WHERE resolved_at IS NULL;
\\CREATE INDEX idx_operational_events_last_seen
\\ ON operational_events(last_seen DESC);
;
/// The same three statements, each made conditional, for the bridge at the end
/// of `migrations.migrate`.
///
/// A database stamped version 1 before `operational_events` joined `ddl_v1`
/// never runs step 1 again, so nothing would ever create the table there. The
/// bridge closes that divergence for the pre-0.1 installs that exist; it is
/// **removable the moment the v0.1 adoption gate lands**, because from then on
/// a schema change is an append-only migration step and this hazard cannot
/// recur.
///
/// It must not be folded into `ddl_v1`: a fresh database would then create the
/// table twice, and the unconditional `CREATE TABLE` above is what proves the
/// baseline and this text stay in step.
pub const operational_events_bridge: [:0]const u8 =
\\CREATE TABLE IF NOT EXISTS operational_events (
\\ id INTEGER PRIMARY KEY,
\\ code TEXT NOT NULL,
\\ subject_key TEXT NOT NULL,
\\ subject_label TEXT NOT NULL,
\\ severity TEXT NOT NULL CHECK (severity IN ('warning', 'error')),
\\ first_seen INTEGER NOT NULL,
\\ last_seen INTEGER NOT NULL,
\\ occurrences INTEGER NOT NULL CHECK (occurrences > 0),
\\ resolved_at INTEGER,
\\ detail TEXT NOT NULL DEFAULT '',
\\ CHECK (resolved_at IS NULL OR resolved_at >= first_seen)
\\);
\\CREATE UNIQUE INDEX IF NOT EXISTS idx_operational_events_active
\\ ON operational_events(code, subject_key) WHERE resolved_at IS NULL;
\\CREATE INDEX IF NOT EXISTS idx_operational_events_last_seen
\\ ON operational_events(last_seen DESC);
;
/// Child-before-parent, and correct under `foreign_keys = ON`. The reconcile
@@ -108,7 +159,10 @@ pub const ddl_v1: [:0]const u8 =
/// `upstreams`, `local_records`, `forward_zones` and `settings` have no foreign
/// keys, so their position is free; `groups` and `blocklist_sources` must come
/// last, after every referrer. `schema_version` is deliberately absent — an
/// import must never erase the stamped migration version.
/// import must never erase the stamped migration version — and so is
/// `operational_events`, which is runtime state rather than configuration: an
/// import that wiped the diagnostics log would destroy the record of what the
/// box has been doing.
pub const delete_order = [_][]const u8{
"group_sources", "rules", "client_prefixes", "clients",
"upstreams", "local_records", "forward_zones", "settings",
@@ -120,12 +174,16 @@ pub const delete_order = [_][]const u8{
/// that renumbered a group.
///
/// `schema_version` is absent: it is the migration's, not the operator's.
/// `operational_events` is absent for the same class of reason: it is the
/// program's own record of its failures, and an export of it would be a log
/// dump, not a configuration.
pub const table_names = [_][]const u8{
"groups", "clients", "client_prefixes", "upstreams",
"blocklist_sources", "group_sources", "rules", "local_records",
"forward_zones", "settings",
};
const db = @import("db.zig");
const testing = std.testing;
test "table_names names exactly the tables delete_order does" {
@@ -135,6 +193,48 @@ test "table_names names exactly the tables delete_order does" {
}
try testing.expect(indexOf(&table_names, "groups") != null);
try testing.expect(indexOf(&table_names, "schema_version") == null);
// Runtime state, not configuration: neither list may reach it, or an
// import would wipe the diagnostics log and an export would emit it.
try testing.expect(indexOf(&table_names, "operational_events") == null);
try testing.expect(indexOf(&delete_order, "operational_events") == null);
}
test "the bridge creates exactly what the baseline does" {
// The two texts are separate on purpose (a fresh database must not create
// the table twice), which is exactly how they could drift. Applying each to
// its own database and comparing `sqlite_schema` is what keeps them equal.
const baseline = try schemaOf(ddl_v1);
defer testing.allocator.free(baseline);
const bridged = try schemaOf(operational_events_bridge);
defer testing.allocator.free(bridged);
// SQLite stores the `CREATE` text verbatim, so the conditional is the one
// difference the two are allowed to have.
const size = std.mem.replacementSize(u8, bridged, " IF NOT EXISTS", "");
const plain = try testing.allocator.alloc(u8, size);
defer testing.allocator.free(plain);
_ = std.mem.replace(u8, bridged, " IF NOT EXISTS", "", plain);
try testing.expectEqualStrings(baseline, plain);
}
/// Every `sqlite_schema` row of `operational_events`, after applying `sql`.
fn schemaOf(sql: [:0]const u8) ![]u8 {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try database.exec(sql);
var out: std.Io.Writer.Allocating = .init(testing.allocator);
errdefer out.deinit();
var stmt = try database.prepare(
\\SELECT type, name, sql FROM sqlite_schema
\\ WHERE tbl_name = 'operational_events' ORDER BY name
);
defer stmt.deinit();
while (try stmt.step()) {
try out.writer.print("{s} {s}\n{s}\n", .{ stmt.columnText(0), stmt.columnText(1), stmt.columnText(2) });
}
return out.toOwnedSlice();
}
test "delete_order lists every referrer before the table it references" {
src/storage/disk_monitor.zig
+168 -13
View File
@@ -9,6 +9,7 @@
//! milestone-5 file; the gate is pulled, not pushed.
const std = @import("std");
const events = @import("events.zig");
const model = @import("../config/model.zig");
const statfs = @import("../platform/statfs.zig");
@@ -89,43 +90,49 @@ pub const Monitor = struct {
/// evidence that the disk filled — and a failed size scan leaves that one
/// gauge at its previous reading. Every failure increments
/// `sample_failures` and logs one line at `warn`.
pub fn sample(self: *Monitor, io: std.Io) void {
const free = statfs.freeBytes(self.data_path) catch {
pub fn sample(self: *Monitor, io: std.Io, store: ?*events.Store, now_s: i64) void {
const free = statfs.freeBytes(self.data_path) catch |err| {
self.countFailure();
log.warn("statvfs on {s} failed", .{self.data_path});
probeFailed(store, io, now_s, "statvfs", "statvfs on the data path failed", err);
return;
};
self.free_bytes.store(free, .monotonic);
if (store) |s| s.resolve(io, now_s, .disk_probe, "statvfs");
if (sumDir(io, self.data_dir, isDatabaseFile)) |bytes| {
self.db_bytes.store(bytes, .monotonic);
if (store) |s| s.resolve(io, now_s, .disk_probe, "data_dir");
} else |err| {
self.countFailure();
log.warn("sizing the data directory failed: {s}", .{@errorName(err)});
probeFailed(store, io, now_s, "data_dir", "sizing the data directory failed", err);
}
if (self.log_dir_path) |path| {
if (self.sumLogDir(io, path)) |bytes| {
self.log_bytes.store(bytes, .monotonic);
if (store) |s| s.resolve(io, now_s, .disk_probe, "log_dir");
} else |err| {
self.countFailure();
log.warn("sizing {s} failed: {s}", .{ path, @errorName(err) });
probeFailed(store, io, now_s, "log_dir", "sizing the log directory failed", err);
}
}
self.publish(classify(free, self.cfg), free);
self.publish(io, store, now_s, classify(free, self.cfg), free);
}
/// Sample first, then sleep: a process that starts on a full disk must not
/// serve a whole interval believing the state is `.ok`. `.boot` so a
/// suspended box still sees the interval elapse.
pub fn run(self: *Monitor, io: std.Io) std.Io.Cancelable!void {
pub fn run(self: *Monitor, io: std.Io, store: ?*events.Store) std.Io.Cancelable!void {
const interval: std.Io.Clock.Duration = .{
.raw = .fromSeconds(sample_interval_s),
.clock = .boot,
};
while (true) {
self.sample(io);
self.sample(io, store, std.Io.Clock.real.now(io).toSeconds());
try interval.sleep(io);
}
}
@@ -136,7 +143,14 @@ pub const Monitor = struct {
/// Logs on transitions only. A disk that sits at `.warn` for a week
/// produces one line, not ten thousand.
fn publish(self: *Monitor, next: State, free: u64) void {
fn publish(
self: *Monitor,
io: std.Io,
store: ?*events.Store,
now_s: i64,
next: State,
free: u64,
) void {
const previous: State = @enumFromInt(self.state_raw.swap(@intFromEnum(next), .monotonic));
if (previous == next) return;
log.warn("disk state {t} -> {t}: {d} bytes free on {s}", .{
@@ -145,6 +159,24 @@ pub const Monitor = struct {
free,
self.data_path,
});
const s = store orelse return;
if (next == .ok) {
s.resolve(io, now_s, .disk_space, disk_space_key);
return;
}
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "disk state {t} -> {t}: {d} bytes free on {s}", .{
previous,
next,
free,
self.data_path,
}) catch buf[0..];
s.report(io, now_s, .disk_space, disk_space_key, "data directory", switch (next) {
.warn => .warning,
.critical => .@"error",
.ok => unreachable,
}, detail);
}
fn sumLogDir(self: *Monitor, io: std.Io, path: [:0]const u8) !u64 {
@@ -155,6 +187,26 @@ pub const Monitor = struct {
}
};
/// The one subject `disk.space` ever has: this box has exactly one data
/// directory, and its filesystem is what the thresholds classify.
const disk_space_key = "data";
/// Every probe failure is a warning, not an error: an unreadable filesystem is
/// a gap in what the monitor can see, and the state it published last stands.
fn probeFailed(
store: ?*events.Store,
io: std.Io,
now_s: i64,
operation: []const u8,
message: []const u8,
err: anyerror,
) void {
const s = store orelse return;
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "{s}: {s}", .{ message, @errorName(err) }) catch buf[0..];
s.report(io, now_s, .disk_probe, operation, operation, .warning, detail);
}
fn everyFile(_: []const u8) bool {
return true;
}
@@ -190,6 +242,7 @@ fn sumDir(io: std.Io, dir: std.Io.Dir, accept: *const fn ([]const u8) bool) !u64
// tests
// ---------------------------------------------------------------------------
const events_fixture = @import("events_fixture.zig");
const testing = std.testing;
const mb = 1024 * 1024;
@@ -280,7 +333,7 @@ test "a sample sizes the databases and ignores every other file" {
try tmp.dir.writeFile(io, .{ .sub_path = "notes.txt", .data = &[_]u8{'d'} ** 4096 });
var monitor: Monitor = .init(.{ .min_free_mb = 0, .warn_free_mb = 0 }, tmp.dir, ".", null);
monitor.sample(io);
monitor.sample(io, null, 0);
const g = monitor.gauges();
try testing.expectEqual(@as(u64, 160), g.db_bytes);
@@ -308,7 +361,7 @@ test "a sample sizes every file in the log directory" {
const log_path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/logs", .{tmp.sub_path});
var monitor: Monitor = .init(.{ .min_free_mb = 0, .warn_free_mb = 0 }, tmp.dir, ".", log_path);
monitor.sample(io);
monitor.sample(io, null, 0);
try testing.expectEqual(@as(u64, 1000), monitor.gauges().log_bytes);
try testing.expectEqual(@as(u64, 0), monitor.sample_failures.load(.monotonic));
@@ -321,7 +374,7 @@ test "a failed statvfs counts and keeps the previous state" {
var monitor: Monitor = .init(.{}, std.Io.Dir.cwd(), "./nxdns-no-such-path-7c21", null);
monitor.state_raw.store(@intFromEnum(State.warn), .monotonic);
monitor.sample(io);
monitor.sample(io, null, 0);
try testing.expectEqual(State.warn, monitor.state());
try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
@@ -342,7 +395,7 @@ test "an unreadable log directory counts a failure but still publishes a state"
".",
"./nxdns-no-such-dir-4f8a",
);
monitor.sample(io);
monitor.sample(io, null, 0);
try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
try testing.expectEqual(State.ok, monitor.state());
@@ -386,7 +439,7 @@ test "an unreadable data directory fails the scan and keeps the previous gauge"
// The handle keeps its read permission from open time, so `iterate` still
// lists the file, but path resolution under the directory now fails.
try tmp.dir.setPermissions(io, .fromMode(0o600));
monitor.sample(io);
monitor.sample(io, null, 0);
try tmp.dir.setPermissions(io, .fromMode(0o700));
try testing.expectEqual(@as(u64, 4096), monitor.gauges().db_bytes);
@@ -409,13 +462,115 @@ test "a threshold above the real free space drives the state to critical" {
".",
null,
);
monitor.sample(io);
monitor.sample(io, null, 0);
try testing.expectEqual(State.critical, monitor.state());
try testing.expect(!monitor.writesAllowed());
monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
monitor.sample(io);
monitor.sample(io, null, 0);
try testing.expectEqual(State.ok, monitor.state());
try testing.expect(monitor.writesAllowed());
try testing.expectEqual(@as(u64, 0), monitor.sample_failures.load(.monotonic));
}
test "a disk transition records an episode per severity and closes it on recovery" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var tmp = testing.tmpDir(.{ .iterate = true });
defer tmp.cleanup();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
const unreachable_mb = std.math.maxInt(u32);
var monitor: Monitor = .init(
.{ .min_free_mb = unreachable_mb, .warn_free_mb = unreachable_mb },
tmp.dir,
".",
null,
);
monitor.sample(io, &fx.store, 1000);
try testing.expectEqual(State.critical, monitor.state());
try testing.expectEqualStrings("disk.space", try fx.text(
"SELECT code FROM operational_events WHERE resolved_at IS NULL",
));
try testing.expectEqualStrings("error", try fx.text(
"SELECT severity FROM operational_events WHERE resolved_at IS NULL",
));
try testing.expectEqualStrings("data", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
// A second critical sample is the same episode, not a second row: `publish`
// only reports on a transition.
monitor.sample(io, &fx.store, 1060);
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
monitor.sample(io, &fx.store, 1120);
try testing.expectEqual(State.ok, monitor.state());
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
try testing.expectEqual(@as(i64, 1120), try fx.count("SELECT resolved_at FROM operational_events"));
}
test "a failed probe opens an episode the next clean pass closes" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var tmp = testing.tmpDir(.{ .iterate = true });
defer tmp.cleanup();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
var monitor: Monitor = .init(
.{ .min_free_mb = 0, .warn_free_mb = 0 },
tmp.dir,
".",
"./nxdns-no-such-dir-4f8a",
);
monitor.sample(io, &fx.store, 1000);
try testing.expectEqualStrings("disk.probe", try fx.text(
"SELECT code FROM operational_events WHERE resolved_at IS NULL",
));
try testing.expectEqualStrings("log_dir", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
try testing.expectEqualStrings("warning", try fx.text(
"SELECT severity FROM operational_events WHERE resolved_at IS NULL",
));
try tmp.dir.createDirPath(io, "logs");
var path_buf: [256]u8 = undefined;
monitor.log_dir_path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/logs", .{tmp.sub_path});
monitor.sample(io, &fx.store, 1100);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
test "every emit site is inert when the store is absent" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var monitor: Monitor = .init(
.{ .min_free_mb = std.math.maxInt(u32), .warn_free_mb = std.math.maxInt(u32) },
std.Io.Dir.cwd(),
"./nxdns-no-such-path-7c21",
"./nxdns-no-such-dir-4f8a",
);
monitor.sample(io, null, 0);
try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
}
src/storage/events.zig
+1427
View File
File diff suppressed because it is too large Load Diff
src/storage/events_fixture.zig
+47
View File
@@ -0,0 +1,47 @@
//! Test support: a migrated in-memory `config.db` with an `events.Store` over
//! it, for the emitter tests that live beside their own subsystem.
//!
//! Every emitter takes `?*events.Store` and must work with `null`; the tests
//! that prove an emitter *does* record something need a real store, and nine
//! subsystems needing the same six lines is what this file removes.
//!
//! Built in place rather than returned by value: a `Store` holds a `*db.Db`, so
//! a fixture that moved after `Store.init` would leave that pointer behind.
const std = @import("std");
const db = @import("db.zig");
const events = @import("events.zig");
const migrations = @import("migrations.zig");
pub const Fixture = struct {
database: db.Db = undefined,
store: events.Store = undefined,
text_buf: [1024]u8 = undefined,
pub fn init(self: *Fixture, io: std.Io, now_s: i64) !void {
self.database = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer self.database.close();
try db.applyPragmas(&self.database, .{});
_ = try migrations.migrate(&self.database);
self.store = try events.Store.init(io, &self.database, now_s);
}
pub fn deinit(self: *Fixture) void {
self.database.close();
}
pub fn count(self: *Fixture, sql: []const u8) !i64 {
return self.database.queryInt(sql);
}
/// The one column a test names most often, for the newest row of `code`.
pub fn text(self: *Fixture, sql: []const u8) ![]const u8 {
var stmt = try self.database.prepare(sql);
defer stmt.deinit();
if (!try stmt.step()) return error.NoRow;
const value = stmt.columnText(0);
@memcpy(self.text_buf[0..value.len], value);
return self.text_buf[0..value.len];
}
};
src/storage/logger.zig
+118
View File
@@ -23,6 +23,7 @@ const std = @import("std");
const db = @import("db.zig");
const disk_monitor = @import("disk_monitor.zig");
const events = @import("events.zig");
const model = @import("../config/model.zig");
const queries_repo = @import("repositories/queries_repo.zig");
@@ -177,6 +178,9 @@ pub const Logger = struct {
/// closed and every entry counts as dropped from that point, so a caller
/// that sees this must not expect rows.
writer_failed: std.atomic.Value(bool),
/// Wired by the composition root after `init`, following the
/// `gate: ?*disk_monitor.Monitor` idiom. Null in every unit test here.
diagnostics: ?*events.Store = null,
/// `queue_buf.len` is the backpressure cap — the composition root
/// (`app.zig:311`) allocates `cfg.logging.query_log_buffer_max` entries,
@@ -254,6 +258,9 @@ pub const Logger = struct {
// Without a writer there is no consumer, so leaving the queue open
// would silently swallow every later entry.
self.writer_failed.store(true, .release);
// No recovery path claims this episode: the writer is gone for the
// life of the process, so the row stays active, which is the truth.
self.reportWrite(io, "writer", "preparing the batch statements failed", @errorName(err), 0);
self.queue.close(io);
self.dropRemaining(io);
return;
@@ -400,9 +407,41 @@ pub const Logger = struct {
writer.writeBatch(rows[0..entries.len]) catch |err| {
scope.warn("query log batch of {d} rows dropped: {s}", .{ entries.len, @errorName(err) });
self.countDropped(entries.len);
self.reportWrite(io, "batch", "a query log batch was dropped", @errorName(err), entries.len);
return;
};
_ = self.rows_written.fetchAdd(entries.len, .monotonic);
if (self.diagnostics) |store| {
store.resolve(io, std.Io.Clock.real.now(io).toSeconds(), .query_log_write, "batch");
}
}
/// An error, not a warning: dropped query rows are gone, and a writer that
/// never started means every later row is gone too.
fn reportWrite(
self: *Logger,
io: std.Io,
operation: []const u8,
message: []const u8,
error_name: []const u8,
rows: usize,
) void {
const store = self.diagnostics orelse return;
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "{s}: {s} ({d} rows)", .{
message,
error_name,
rows,
}) catch buf[0..];
store.report(
io,
std.Io.Clock.real.now(io).toSeconds(),
.query_log_write,
operation,
operation,
.@"error",
detail,
);
}
fn countDropped(self: *Logger, n: usize) void {
@@ -441,6 +480,7 @@ fn outcomeEntry(outcome: Outcome) ?Entry {
// tests
// ---------------------------------------------------------------------------
const events_fixture = @import("events_fixture.zig");
const querylog_schema = @import("querylog_schema.zig");
const testing = std.testing;
@@ -881,3 +921,81 @@ test "an empty batch touches neither the database nor the counters" {
try testing.expectEqual(@as(u64, 0), logger.rows_written.load(.monotonic));
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
}
test "a dropped batch opens an error episode the next good batch closes" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
try database.exec(
\\CREATE TRIGGER refuse_boom BEFORE INSERT ON query_log
\\WHEN new.client_ip = 'boom'
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
);
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
var writer = try queries_repo.BatchWriter.init(&database);
defer writer.deinit();
var buf: [4]Entry = undefined;
var logger: Logger = .init(.{}, &buf);
logger.diagnostics = &fx.store;
var doomed = sampleEntry(10, "poison.example");
doomed.setClientIp("boom");
const bad = [_]Entry{doomed};
try logger.flush(io, &writer, &bad, null);
try testing.expectEqualStrings("query_log.write", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("batch", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqualStrings("error", try fx.text("SELECT severity FROM operational_events"));
const good = [_]Entry{sampleEntry(11, "next.example")};
try logger.flush(io, &writer, &good, null);
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
test "a writer that cannot prepare leaves an episode no recovery path claims" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
// No schema: `BatchWriter.init` cannot prepare against a missing table.
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
var buf: [8]Entry = undefined;
var logger: Logger = .init(.{}, &buf);
logger.diagnostics = &fx.store;
try logger.runWriter(io, &database, null);
try testing.expectEqualStrings("writer", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
try testing.expectEqualStrings("error", try fx.text(
"SELECT severity FROM operational_events WHERE resolved_at IS NULL",
));
// The writer returned, so nothing can ever close this. A second run finds
// the queue closed and adds no second episode.
try logger.runWriter(io, &database, null);
try testing.expectEqual(
@as(i64, 1),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
src/storage/migrations.zig
+113 -4
View File
@@ -55,7 +55,24 @@ fn assertOrdered(list: []const Step) void {
/// foreign_keys` is a no-op inside a transaction, so applying it afterwards
/// would silently leave referential integrity off.
pub fn migrate(database: *db.Db) Error!u32 {
return migrateSteps(database, &steps);
const version = try migrateSteps(database, &steps);
try bridgeOperationalEvents(database);
return version;
}
/// **Removable when the v0.1 adoption gate lands.**
///
/// `operational_events` joined `config_schema.ddl_v1` after databases stamped
/// version 1 already existed, and a stamped database never runs step 1 again —
/// so on those installs nothing would ever create the table, silently, and the
/// diagnostics store would fail to open forever. This runs after the stamp,
/// with every statement conditional, and touches no other table.
///
/// It belongs here and not in `cli.openConfigDb`: that runs *before* migration
/// everywhere (`app.zig`, `cli.zig`), so creating the table there would make a
/// fresh database's unconditional `CREATE TABLE` in `ddl_v1` fail.
fn bridgeOperationalEvents(database: *db.Db) db.Error!void {
return database.exec(config_schema.operational_events_bridge);
}
/// Same logic against an injected step list. The seam exists for the rollback
@@ -159,7 +176,7 @@ test "migrate on a fresh database creates every table and seeds the default grou
const expected = [_][]const u8{
"schema_version", "groups", "clients", "client_prefixes",
"upstreams", "rules", "local_records", "forward_zones",
"blocklist_sources", "group_sources", "settings",
"blocklist_sources", "group_sources", "settings", "operational_events",
};
for (expected) |name| {
try testing.expect(try tableExists(&database, name));
@@ -373,9 +390,101 @@ test "delete_order and table_names name exactly the tables the schema creates" {
for (config_schema.table_names) |name| {
try testing.expect(try tableExists(&database, name));
}
// delete_order covers every table except `schema_version`.
// delete_order covers every table except two: `schema_version`, which is
// the migration's own, and `operational_events`, which is runtime state an
// import must never wipe.
try testing.expect(!try tableExists(&database, "no_such_table"));
try testing.expect(try tableExists(&database, "operational_events"));
try testing.expectEqual(
@as(i64, config_schema.delete_order.len + 1),
@as(i64, config_schema.delete_order.len + 2),
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
);
}
test "a version-1 database created without operational_events gains exactly it" {
// The silent divergence the bridge exists for: this is what the Pi's
// `config.db` looks like — stamped 1, so step 1 never runs again.
var database = try openMigrated();
defer database.close();
try database.exec(config_schema.ddl_v1);
try database.exec("DROP TABLE operational_events;");
try database.exec("INSERT INTO schema_version (version) VALUES (1);");
try testing.expect(!try tableExists(&database, "operational_events"));
const tables_before = try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'");
const groups_before = try database.queryInt("SELECT count(*) FROM groups");
try testing.expectEqual(@as(u32, 1), try migrate(&database));
try testing.expect(try tableExists(&database, "operational_events"));
try testing.expectEqual(
tables_before + 1,
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
);
// Both indexes came with it, and no other table moved.
try testing.expectEqual(
@as(i64, 2),
try database.queryInt(
"SELECT count(*) FROM sqlite_schema WHERE type='index' AND tbl_name='operational_events'",
),
);
try testing.expectEqual(groups_before, try database.queryInt("SELECT count(*) FROM groups"));
}
test "the bridge does not double-create on a fresh database or on a second run" {
var database = try openMigrated();
defer database.close();
// `ddl_v1` creates the table unconditionally, so a bridge that ran as part
// of the step list would fail here rather than be a no-op.
try testing.expectEqual(@as(u32, 1), try migrate(&database));
const schema_rows = try database.queryInt("SELECT count(*) FROM sqlite_schema");
try database.exec(
\\INSERT INTO operational_events
\\ (code, subject_key, subject_label, severity, first_seen, last_seen, occurrences)
\\VALUES ('disk.space', 'data', 'data', 'warning', 100, 100, 1);
);
try testing.expectEqual(@as(u32, 1), try migrate(&database));
try testing.expectEqual(schema_rows, try database.queryInt("SELECT count(*) FROM sqlite_schema"));
// A `CREATE TABLE IF NOT EXISTS` that had somehow replaced the table would
// show up as a lost row, not as a schema difference.
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM operational_events"));
}
test "the partial unique index allows one active row per key and any number of resolved ones" {
var database = try openMigrated();
defer database.close();
_ = try migrate(&database);
const insert =
\\INSERT INTO operational_events
\\ (code, subject_key, subject_label, severity, first_seen, last_seen, occurrences, resolved_at)
\\VALUES ('blocklist.refresh', 'https://a.example', 'a', 'warning', 100, 100, 1, ?1)
;
{
var stmt = try database.prepare(insert);
defer stmt.deinit();
try stmt.bindNull(1);
try stmt.exec();
}
{
// A second active row for the same (code, subject_key) is what
// `report`'s overflow probe exists to avoid, and the index proves it.
// Its own statement: `Stmt.reset` re-reports the code of a failed step,
// so a reused one would answer `error.Constraint` a second time.
var stmt = try database.prepare(insert);
defer stmt.deinit();
try stmt.bindNull(1);
try testing.expectError(error.Constraint, stmt.step());
}
var resolved = try database.prepare(insert);
defer resolved.deinit();
for ([_]i64{ 200, 300 }) |resolved_at| {
try resolved.reset();
try resolved.bindInt(1, resolved_at);
try resolved.exec();
}
try testing.expectEqual(@as(i64, 3), try database.queryInt("SELECT count(*) FROM operational_events"));
}
src/storage/phase6_integration_test.zig
+3 -3
View File
@@ -360,7 +360,7 @@ test "S8 case 5: a retention pass prunes the old rows and truncates the write-ah
try testing.expect(try f.sizeOf("querylog.db-wal") > 0);
var pass: retention.Retention = .init(.{ .retention_days = 30 });
pass.runOnce(io, log_db.database(), null);
pass.runOnce(io, log_db.database(), null, null);
try testing.expectEqual(@as(u64, 1), pass.snapshotStats().passes);
try testing.expectEqual(@as(u64, 2), pass.snapshotStats().rows_pruned);
@@ -396,7 +396,7 @@ test "S8 case 6: a critical disk gates the flushes and recovery releases them" {
data_path,
null,
);
monitor.sample(io);
monitor.sample(io, null, 0);
try testing.expectEqual(disk_monitor.State.critical, monitor.state());
try testing.expect(!monitor.writesAllowed());
try testing.expect(monitor.gauges().free_bytes > 0);
@@ -418,7 +418,7 @@ test "S8 case 6: a critical disk gates the flushes and recovery releases them" {
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(log_db.database()));
monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
monitor.sample(io);
monitor.sample(io, null, 0);
try testing.expectEqual(disk_monitor.State.ok, monitor.state());
try testing.expect(monitor.writesAllowed());
src/storage/querylog_schema.zig
+68 -1
View File
@@ -88,6 +88,18 @@ pub const OpenResult = struct {
database: db.Db,
/// Non-null feeds a counter and the `/api/health` rollup.
recreated: ?RecreateReason,
/// The path the previous file was kept as, by value. It existed only in a
/// stack buffer inside `open` before the diagnostics event needed it, and a
/// slice of that buffer would dangle the moment `open` returned.
///
/// Empty when nothing was renamed aside, which `.missing` and a clean open
/// both are.
aside_buf: [path_buf_len]u8 = undefined,
aside_len: u16 = 0,
pub fn aside(self: *const OpenResult) []const u8 {
return self.aside_buf[0..self.aside_len];
}
};
pub const Error = db.Error || error{AsideNameCollision} ||
@@ -156,7 +168,12 @@ pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
aside.?,
});
}
return .{ .database = fresh, .recreated = cause };
var result: OpenResult = .{ .database = fresh, .recreated = cause };
if (aside) |name| {
result.aside_len = @intCast(name.len);
@memcpy(result.aside_buf[0..name.len], name);
}
return result;
}
/// The whitelist. `null` means "propagate, do not touch the file".
@@ -321,3 +338,53 @@ test "recreatable selects exactly two of db.Error's members" {
}
try testing.expectEqual(@as(usize, 2), whitelisted);
}
test "a recreate returns the aside name by value and a fresh create returns none" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var tmp = testing.tmpDir(.{ .iterate = true });
defer tmp.cleanup();
var path_buf: [path_buf_len]u8 = undefined;
const path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/querylog.db", .{tmp.sub_path});
// First open: the file is missing, so nothing is renamed aside. The
// one-shot event is deliberately not emitted for this case.
var created = try open(io, std.Io.Dir.cwd(), path);
created.database.close();
try testing.expectEqual(RecreateReason.missing, created.recreated.?);
try testing.expectEqualStrings("", created.aside());
try tmp.dir.writeFile(io, .{ .sub_path = "querylog.db", .data = "not a database at all" });
var recreated = try open(io, std.Io.Dir.cwd(), path);
recreated.database.close();
try testing.expectEqual(RecreateReason.not_a_database, recreated.recreated.?);
try testing.expect(recreated.aside().len != 0);
// The name is a real file, which is the whole reason it travels out.
const kept = std.fs.path.basename(recreated.aside());
try tmp.dir.access(io, kept, .{});
}
test "a clean reopen reports no recreate and no aside" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var tmp = testing.tmpDir(.{ .iterate = true });
defer tmp.cleanup();
var path_buf: [path_buf_len]u8 = undefined;
const path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/querylog.db", .{tmp.sub_path});
var first = try open(io, std.Io.Dir.cwd(), path);
first.database.close();
var second = try open(io, std.Io.Dir.cwd(), path);
second.database.close();
try testing.expectEqual(@as(?RecreateReason, null), second.recreated);
try testing.expectEqualStrings("", second.aside());
}
src/storage/repositories/events_repo.zig
+1177
View File
File diff suppressed because it is too large Load Diff
src/storage/retention.zig
+149 -19
View File
@@ -13,6 +13,7 @@ const std = @import("std");
const db = @import("db.zig");
const disk_monitor = @import("disk_monitor.zig");
const events = @import("events.zig");
const model = @import("../config/model.zig");
const queries_repo = @import("repositories/queries_repo.zig");
const upstream_history_repo = @import("repositories/upstream_history_repo.zig");
@@ -102,15 +103,23 @@ pub const Retention = struct {
io: std.Io,
database: *db.Db,
monitor: ?*disk_monitor.Monitor,
store: ?*events.Store,
) void {
add(&self.counters.passes, 1);
const now = std.Io.Clock.real.now(io).toSeconds();
const cutoff = now - model.retentionSeconds(self.cfg);
// Diagnostics retention rides this pass rather than a schedule of its
// own: one daily housekeeping task, and a box restarted every night
// still prunes through `Store.init`.
if (store) |s| s.prune(io, now);
if (queries_repo.pruneOlderThan(database, cutoff)) |deleted| {
add(&self.counters.rows_pruned, @intCast(deleted));
maintenance(store, io, now, "prune", null);
} else |err| {
log.warn("retention prune before {d} failed: {s}", .{ cutoff, @errorName(err) });
maintenance(store, io, now, "prune", @errorName(err));
}
// Before the vacuum-cadence logic below, which returns early on six
@@ -123,14 +132,18 @@ pub const Retention = struct {
const history_cutoff = now - upstream_history_repo.retention_window_s;
if (upstream_history_repo.pruneOlderThan(database, history_cutoff)) |deleted| {
add(&self.counters.upstream_rows_pruned, @intCast(deleted));
maintenance(store, io, now, "history_prune", null);
} else |err| {
log.warn("upstream history prune before {d} failed: {s}", .{ history_cutoff, @errorName(err) });
maintenance(store, io, now, "history_prune", @errorName(err));
}
if (queries_repo.checkpointTruncate(database)) {
add(&self.counters.checkpoints, 1);
maintenance(store, io, now, "checkpoint", null);
} else |err| {
log.warn("retention checkpoint failed: {s}", .{@errorName(err)});
maintenance(store, io, now, "checkpoint", @errorName(err));
}
self.passes_since_vacuum += 1;
@@ -141,17 +154,37 @@ pub const Retention = struct {
if (monitor) |m| if (!m.writesAllowed()) {
add(&self.counters.vacuums_gated, 1);
log.warn("retention vacuum skipped: the disk monitor refuses writes", .{});
maintenance(store, io, now, "vacuum", "the disk monitor refuses writes");
return;
};
if (queries_repo.vacuum(database)) {
add(&self.counters.vacuums, 1);
self.passes_since_vacuum = 0;
maintenance(store, io, now, "vacuum", null);
} else |err| {
log.warn("retention vacuum failed: {s}", .{@errorName(err)});
maintenance(store, io, now, "vacuum", @errorName(err));
}
}
/// One step's outcome. `reason` null is the success branch of that same
/// step in that same pass, which is what closes its episode; a gated vacuum
/// is a failure of the step, because the work it owes is still owed.
fn maintenance(
store: ?*events.Store,
io: std.Io,
now_s: i64,
operation: []const u8,
reason: ?[]const u8,
) void {
const s = store orelse return;
const text = reason orelse return s.resolve(io, now_s, .query_log_maintenance, operation);
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "retention {s} failed: {s}", .{ operation, text }) catch buf[0..];
s.report(io, now_s, .query_log_maintenance, operation, operation, .warning, detail);
}
fn add(counter: *std.atomic.Value(u64), delta: u64) void {
_ = counter.fetchAdd(delta, .monotonic);
}
@@ -182,13 +215,14 @@ pub const Retention = struct {
io: std.Io,
database: *db.Db,
monitor: ?*disk_monitor.Monitor,
store: ?*events.Store,
) std.Io.Cancelable!void {
const interval: std.Io.Clock.Duration = .{
.raw = .fromSeconds(pass_interval_s),
.clock = .boot,
};
while (true) {
self.runOnce(io, database, monitor);
self.runOnce(io, database, monitor, store);
try interval.sleep(io);
}
}
@@ -198,6 +232,7 @@ pub const Retention = struct {
// tests
// ---------------------------------------------------------------------------
const events_fixture = @import("events_fixture.zig");
const querylog_schema = @import("querylog_schema.zig");
const testing = std.testing;
@@ -243,7 +278,7 @@ test "a pass prunes the rows past the retention window and keeps the rest" {
try writeRows(&database, &.{ now - 40 * day, now - 31 * day, now - 29 * day, now - 60 });
var retention: Retention = .init(.{ .retention_days = 30 });
retention.runOnce(io, &database, null);
retention.runOnce(io, &database, null, null);
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
@@ -267,12 +302,12 @@ test "the cutoff follows retention_days" {
try writeRows(&database, &.{now - 3 * day});
var keeps: Retention = .init(.{ .retention_days = 7 });
keeps.runOnce(io, &database, null);
keeps.runOnce(io, &database, null, null);
try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 0), keeps.snapshotStats().rows_pruned);
var prunes: Retention = .init(.{ .retention_days = 1 });
prunes.runOnce(io, &database, null);
prunes.runOnce(io, &database, null, null);
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 1), prunes.snapshotStats().rows_pruned);
}
@@ -287,16 +322,16 @@ test "the seventh pass vacuums and the six before it do not" {
var retention: Retention = .init(.{});
for (0..6) |_| {
retention.runOnce(io, &database, null);
retention.runOnce(io, &database, null, null);
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums);
}
retention.runOnce(io, &database, null);
retention.runOnce(io, &database, null, null);
try testing.expectEqual(@as(u64, 7), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
try testing.expectEqual(@as(u64, 7), retention.snapshotStats().checkpoints);
for (0..7) |_| retention.runOnce(io, &database, null);
for (0..7) |_| retention.runOnce(io, &database, null, null);
try testing.expectEqual(@as(u64, 14), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().vacuums);
}
@@ -315,7 +350,7 @@ test "a gated pass skips the vacuum, counts it, and vacuums on the next pass" {
monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
var gated: Retention = .init(.{});
for (0..vacuum_every_passes) |_| gated.runOnce(io, &database, &monitor);
for (0..vacuum_every_passes) |_| gated.runOnce(io, &database, &monitor, null);
// Prune and checkpoint ran on every pass; only the vacuum was refused.
try testing.expectEqual(@as(u64, vacuum_every_passes), gated.snapshotStats().passes);
@@ -325,12 +360,12 @@ test "a gated pass skips the vacuum, counts it, and vacuums on the next pass" {
// The vacuum is due again immediately, not seven passes later.
monitor.state_raw.store(@intFromEnum(disk_monitor.State.ok), .monotonic);
gated.runOnce(io, &database, &monitor);
gated.runOnce(io, &database, &monitor, null);
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums);
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums_gated);
// And the counter reset, so the next six passes vacuum nothing.
for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor);
for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor, null);
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums);
}
@@ -346,7 +381,7 @@ test "a warn state still allows the vacuum" {
monitor.state_raw.store(@intFromEnum(disk_monitor.State.warn), .monotonic);
var retention: Retention = .init(.{});
for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor);
for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor, null);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums_gated);
@@ -361,7 +396,7 @@ test "a pass over an empty database still counts" {
defer database.close();
var retention: Retention = .init(.{});
retention.runOnce(io, &database, null);
retention.runOnce(io, &database, null, null);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().rows_pruned);
@@ -385,7 +420,7 @@ test "a failing prune counts the pass and leaves the rows alone" {
);
var retention: Retention = .init(.{ .retention_days = 30 });
retention.runOnce(io, &database, null);
retention.runOnce(io, &database, null, null);
try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
@@ -412,7 +447,7 @@ test "the upstream-history window is its own, and a one-day query log does not s
// A query log kept for one day, and 30 days of upstream minutes beside it.
var retention: Retention = .init(.{ .retention_days = 1 });
retention.runOnce(io, &database, null);
retention.runOnce(io, &database, null, null);
const stats = retention.snapshotStats();
// One query-log row is older than one day; one minute row is older than the
@@ -450,7 +485,7 @@ test "the history prune runs on the passes where the vacuum logic returns early"
.{ .url = "https://a.example", .minute_ts = old_minute, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
var early: Retention = .init(.{});
early.runOnce(io, &database, null);
early.runOnce(io, &database, null, null);
try testing.expectEqual(@as(u64, 1), early.snapshotStats().upstream_rows_pruned);
try testing.expectEqual(@as(i64, 0), try upstream_history_repo.countMinutes(&database));
@@ -460,11 +495,11 @@ test "the history prune runs on the passes where the vacuum logic returns early"
monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
var gated: Retention = .init(.{});
for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor);
for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor, null);
try upstream_history_repo.flush(&database, &.{
.{ .url = "https://b.example", .minute_ts = old_minute, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
gated.runOnce(io, &database, &monitor);
gated.runOnce(io, &database, &monitor, null);
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums_gated);
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().upstream_rows_pruned);
@@ -487,13 +522,108 @@ test "the next pass retries what the failed one could not do" {
);
var retention: Retention = .init(.{ .retention_days = 30 });
retention.runOnce(io, &database, null);
retention.runOnce(io, &database, null, null);
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
try database.exec("DROP TRIGGER refuse_delete;");
retention.runOnce(io, &database, null);
retention.runOnce(io, &database, null, null);
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().rows_pruned);
}
test "a failing prune opens a maintenance episode the next clean pass closes" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
const now = std.Io.Clock.real.now(io).toSeconds();
try writeRows(&database, &.{now - 40 * 86_400});
try database.exec(
\\CREATE TRIGGER refuse_delete BEFORE DELETE ON query_log
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
);
var retention: Retention = .init(.{});
retention.runOnce(io, &database, null, &fx.store);
// Only the prune failed; checkpoint and history prune succeeded, and a
// success writes no row of its own.
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqualStrings("query_log.maintenance", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("prune", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
try database.exec("DROP TRIGGER refuse_delete;");
retention.runOnce(io, &database, null, &fx.store);
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
test "a gated vacuum is a maintenance failure the next ungated pass closes" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null);
monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
var retention: Retention = .init(.{});
for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor, &fx.store);
try testing.expectEqualStrings("vacuum", try fx.text(
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
));
monitor.state_raw.store(@intFromEnum(disk_monitor.State.ok), .monotonic);
retention.runOnce(io, &database, &monitor, &fx.store);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
test "a pass prunes the diagnostics store once" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
// Resolved further back than the retention window, so the pass must drop it.
const now = std.Io.Clock.real.now(io).toSeconds();
const stale = now - events.Store.resolved_retention_s - 86_400;
fx.store.reportResolved(io, stale, .query_log_recreated, "one-shot", "one-shot", .warning, "aside kept");
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
var retention: Retention = .init(.{});
retention.runOnce(io, &database, null, &fx.store);
try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
}
src/tests.zig
+4
View File
@@ -76,6 +76,9 @@ comptime {
_ = @import("server/rate_limiter.zig");
_ = @import("storage/repositories/queries_repo.zig");
_ = @import("storage/repositories/upstream_history_repo.zig");
_ = @import("storage/repositories/events_repo.zig");
_ = @import("storage/events.zig");
_ = @import("storage/events_fixture.zig");
_ = @import("upstream/history.zig");
_ = @import("storage/logger.zig");
_ = @import("platform/statfs.zig");
@@ -100,6 +103,7 @@ comptime {
_ = @import("web/metrics.zig");
_ = @import("web/handlers/stats.zig");
_ = @import("web/handlers/queries.zig");
_ = @import("web/handlers/diagnostics.zig");
_ = @import("web/handlers/lookup.zig");
_ = @import("web/handlers/upstream_health.zig");
_ = @import("web/handlers/health.zig");
src/upstream/history.zig
+82
View File
@@ -27,6 +27,7 @@
const std = @import("std");
const db = @import("../storage/db.zig");
const events = @import("../storage/events.zig");
const health = @import("health.zig");
const upstream_history_repo = @import("../storage/repositories/upstream_history_repo.zig");
@@ -114,6 +115,10 @@ pub const Accumulator = struct {
flush_cells: [max_pending]Cell,
flush_rows: [max_pending]upstream_history_repo.FlushRow,
flush_count: u32,
/// Set once by the composition root after `init`, following the
/// `gate: ?*disk_monitor.Monitor` idiom. Null everywhere else, and every
/// emit site below is inert when it is.
diagnostics: ?*events.Store = null,
/// `cells` and `flush_cells` are `undefined`: a cell is always written
/// before it is read, and `count` is what says which ones exist.
@@ -255,6 +260,9 @@ pub const Accumulator = struct {
if (write(database, rows)) {
_ = self.counters.flushes.fetchAdd(1, .monotonic);
if (self.diagnostics) |store| {
store.resolve(io, std.Io.Clock.real.now(io).toSeconds(), .upstream_history_write, flush_key);
}
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
self.last_flush_failed = false;
@@ -262,6 +270,22 @@ pub const Accumulator = struct {
} else |err| {
_ = self.counters.flush_failures.fetchAdd(1, .monotonic);
log.warn("flushing {d} upstream history rows failed: {s}", .{ rows.len, @errorName(err) });
if (self.diagnostics) |store| {
var buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&buf, "flushing {d} upstream history rows failed: {s}", .{
rows.len,
@errorName(err),
}) catch buf[0..];
store.report(
io,
std.Io.Clock.real.now(io).toSeconds(),
.upstream_history_write,
flush_key,
"upstream history flush",
.warning,
detail,
);
}
self.mergeBack(io);
}
}
@@ -296,6 +320,10 @@ pub const Accumulator = struct {
}
};
/// One accumulator, one flush task, one table: the subject of every
/// `upstream_history.write` episode is that single writer.
const flush_key = "flush";
/// Max-wins, matching the SQL upsert exactly: the newest failure in the minute
/// is the one whose name the cell keeps.
fn noteFailure(cell: *Accumulator.Cell, at: i64, error_name: []const u8) void {
@@ -312,6 +340,7 @@ fn noteFailure(cell: *Accumulator.Cell, at: i64, error_name: []const u8) void {
// tests
// ---------------------------------------------------------------------------
const events_fixture = @import("../storage/events_fixture.zig");
const querylog_schema = @import("../storage/querylog_schema.zig");
const testing = std.testing;
@@ -632,3 +661,56 @@ test "a flush against the real repository writes the minute rows" {
try testing.expectEqual(@as(i64, 2), try upstream_history_repo.countMinutes(&database));
try testing.expectEqual(@as(u64, 2), acc.snapshotStats(io).flushes);
}
test "a failed flush opens one episode and the next successful flush closes it" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
const acc = try newAccumulator();
defer testing.allocator.destroy(acc);
acc.diagnostics = &fx.store;
acc.recordFailure(io, "https://a.example", 1_700_000_000, "Timeout");
acc.flushOnce(io, &database, failingWrite);
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqualStrings("upstream_history.write", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("flush", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
try testing.expectEqual(
@as(i64, 1),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
acc.flushOnce(io, &database, upstream_history_repo.flush);
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
}
test "a flush with no store attached records nothing and still flushes" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const acc = try newAccumulator();
defer testing.allocator.destroy(acc);
acc.recordFailure(io, "https://a.example", 1_700_000_000, "Timeout");
acc.flushOnce(io, &database, failingWrite);
acc.flushOnce(io, &database, upstream_history_repo.flush);
try testing.expectEqual(@as(u64, 1), acc.counters.flushes.load(.monotonic));
}
src/upstream/pool.zig
+97
View File
@@ -47,6 +47,7 @@
const std = @import("std");
const events = @import("../storage/events.zig");
const health = @import("health.zig");
const history_mod = @import("history.zig");
const safe_url = @import("../safe_url.zig");
@@ -136,6 +137,9 @@ pub const Pool = struct {
/// pool is fully usable without it — `nxdns check` and every unit test here
/// run with no history at all.
history: ?*history_mod.Accumulator = null,
/// The diagnostics store, wired the same way and for the same reason as
/// `history`. Every emit here sits outside `mutex`; see `recordHistory`.
diagnostics: ?*events.Store = null,
pub fn init(
entries: []Entry,
@@ -327,6 +331,7 @@ pub const Pool = struct {
// constraint: the accumulator takes a mutex of its own, and no task may
// hold one of the two while it takes the other.
self.recordHistory(io, entry, .success);
self.recordDiagnostics(io, entry, .success);
}
fn recordFailure(
@@ -344,6 +349,7 @@ pub const Pool = struct {
// After the pool mutex is released, for the reason `recordSuccess`
// states.
self.recordHistory(io, entry, .{ .failure = @errorName(err) });
self.recordDiagnostics(io, entry, .{ .failure = @errorName(err) });
}
const Outcome = union(enum) { success, failure: []const u8 };
@@ -359,9 +365,34 @@ pub const Pool = struct {
.failure => |name| history.recordFailure(io, entry.endpoint.url, wall_s, name),
}
}
/// The same placement discipline as `recordHistory`: the store takes a
/// mutex of its own, so this runs after the pool's is released.
///
/// A success is the steady state of the whole program, so `resolve` is
/// built to issue no SQL when nothing is open (`storage/events.zig`).
fn recordDiagnostics(self: *Pool, io: std.Io, entry: *Entry, outcome: Outcome) void {
const store = self.diagnostics orelse return;
const url = entry.endpoint.url;
const now_s = std.Io.Clock.real.now(io).toSeconds();
switch (outcome) {
.success => store.resolve(io, now_s, .upstream_exchange, url),
.failure => |name| {
var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
const label = std.fmt.bufPrint(&label_buf, "{f}", .{safe_url.redact(url)}) catch &label_buf;
var detail_buf: [events.Store.max_detail_len]u8 = undefined;
const detail = std.fmt.bufPrint(&detail_buf, "upstream {f} failed: {s}", .{
safe_url.redactQuoted(url),
name,
}) catch &detail_buf;
store.report(io, now_s, .upstream_exchange, url, label, .warning, detail);
},
}
}
};
const db = @import("../storage/db.zig");
const events_fixture = @import("../storage/events_fixture.zig");
const querylog_schema = @import("../storage/querylog_schema.zig");
const upstream_history_repo = @import("../storage/repositories/upstream_history_repo.zig");
@@ -948,3 +979,69 @@ test "an entry that enters backoff while a task waits on it is not attempted" {
try testing.expectEqual(@as(u64, 1), entries[0].health.total_failures);
try testing.expect(entries[0].health.backoff_until != null);
}
test "a successful exchange with nothing open costs the store no statement" {
if (@FieldType(events.Store, "statements") != u64) return error.SkipZigTest;
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{testEntry("https://good.example/dns-query", &good, 10)};
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
pool.diagnostics = &fx.store;
var buf: [512]u8 = undefined;
const before = fx.store.statements;
for (0..20) |_| _ = try pool.exchange(io, query_bytes, &buf);
try testing.expectEqual(before, fx.store.statements);
try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
}
test "a failing then recovering upstream leaves exactly one resolved episode" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var flaky: Fake = .{ .behavior = .{ .fail = error.Timeout } };
var standby: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://flaky.example/dns-query", &flaky, 10),
testEntry("https://standby.example/dns-query", &standby, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
pool.diagnostics = &fx.store;
var buf: [512]u8 = undefined;
_ = try pool.exchange(io, query_bytes, &buf);
// Backoff would park the failing entry, so the second failure is driven
// through `recordFailure` itself rather than through another exchange.
pool.recordFailure(io, &entries[0], std.Io.Clock.awake.now(io), error.ConnectFailed);
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
try testing.expectEqualStrings("upstream.exchange", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings(
"https://flaky.example/dns-query",
try fx.text("SELECT subject_key FROM operational_events"),
);
flaky.behavior = .{ .reply = response_bytes };
pool.recordSuccess(io, &entries[0], std.Io.Clock.awake.now(io));
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqual(
@as(i64, 0),
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
);
try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
}
src/web/handlers/diagnostics.zig
+448
View File
@@ -0,0 +1,448 @@
//! `GET /api/diagnostics` and `GET /api/diagnostics/{id}` — the operational
//! event log, newest first.
//!
//! Keyset pagination and the same page envelope as `/api/queries`, for the same
//! reason: the table is append-only at the head, so `id < before` is one index
//! seek however deep a client has scrolled, and rows arriving between two pages
//! cannot shift the window and duplicate one.
//!
//! Filter parsing is separated from fetching, because parsing is where PLAN
//! §19's input validation lives and it is worth testing on its own. Every value
//! is length-capped here and bound as a SQL parameter by the repository;
//! nothing this file reads is ever concatenated into a statement.
//!
//! No SQL and no connection of its own: `events.Store` owns the one diagnostics
//! connection and locks its mutex around every read, so this file cannot race
//! the emitters writing through it.
const std = @import("std");
const db = @import("../../storage/db.zig");
const events = @import("../../storage/events.zig");
const http_util = @import("../http_util.zig");
const server = @import("../server.zig");
const log = std.log.scoped(.web_diagnostics);
pub const default_limit: u32 = 100;
pub const max_limit: u32 = events.max_limit;
/// Wide enough for every component `events.component` can produce, and for a
/// mistyped one to still be reported as a bad filter rather than a truncated
/// match.
pub const max_component_len = 64;
/// Where the string filters are copied to. The parsed filter borrows them, so
/// it must not outlive the buffers — in the handler both live in the same stack
/// frame.
pub const Buffers = struct {
state: [16]u8 = undefined,
severity: [16]u8 = undefined,
component: [max_component_len]u8 = undefined,
};
pub const FilterError = error{
BadState,
BadSeverity,
BadComponent,
BadSince,
BadUntil,
BadLimit,
BadBefore,
};
/// An absent parameter drops the filter; a malformed one is a 400 rather than a
/// filter silently left off, which would answer a question the client did not
/// ask.
pub fn parseFilter(query: []const u8, buffers: *Buffers) FilterError!events.Filter {
var filter: events.Filter = .{ .limit = default_limit };
if (http_util.queryValue(query, "state", &buffers.state) catch return error.BadState) |text| {
filter.state = std.meta.stringToEnum(events.State, text) orelse return error.BadState;
}
if (http_util.queryValue(query, "severity", &buffers.severity) catch return error.BadSeverity) |text| {
// Bound as text by the repository, so it is normalised to one of the
// two stored spellings here rather than passed through.
const severity = std.meta.stringToEnum(events.Severity, text) orelse return error.BadSeverity;
filter.severity = severity.text();
}
if (http_util.queryValue(query, "component", &buffers.component) catch return error.BadComponent) |text| {
if (text.len != 0) filter.component = text;
}
filter.since = http_util.queryInt(i64, query, "since") catch return error.BadSince;
filter.until = http_util.queryInt(i64, query, "until") catch return error.BadUntil;
if (http_util.queryInt(u32, query, "limit") catch return error.BadLimit) |limit| {
if (limit == 0 or limit > max_limit) return error.BadLimit;
filter.limit = limit;
}
if (http_util.queryInt(i64, query, "before") catch return error.BadBefore) |before| {
// Row ids are positive, so a non-positive cursor is a client bug, not
// an empty page.
if (before <= 0) return error.BadBefore;
filter.before = before;
}
return filter;
}
pub fn message(err: FilterError) []const u8 {
return switch (err) {
error.BadState => "state must be active, resolved or all",
error.BadSeverity => "severity must be warning or error",
error.BadComponent => "component is not a valid filter",
error.BadSince => "since must be a unix timestamp in seconds",
error.BadUntil => "until must be a unix timestamp in seconds",
error.BadLimit => "limit must be between 1 and 1000",
error.BadBefore => "before must be a positive row id",
};
}
const unavailable_message = "diagnostics unavailable";
pub fn list(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
var buffers: Buffers = .{};
const filter = parseFilter(request.query, &buffers) catch |err| {
return http_util.respondError(request, .bad_request, message(err));
};
const store = state.events orelse
return http_util.respondError(request, .service_unavailable, unavailable_message);
const page = store.selectEvents(io, request.arena, filter) catch |err| {
// The one thing this handler logs: a database fault is a property of
// the box, not of the request, and the client is told nothing about it.
log.warn("diagnostics read failed: {s}", .{@errorName(err)});
return http_util.respondError(request, .internal_server_error, "internal error");
};
return http_util.respondJson(request, .ok, page, &.{});
}
pub fn get(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const store = state.events orelse
return http_util.respondError(request, .service_unavailable, unavailable_message);
const row = store.selectOne(io, request.arena, request.id.?) catch |err| {
log.warn("diagnostics read failed: {s}", .{@errorName(err)});
return http_util.respondError(request, .internal_server_error, "internal error");
};
// An id retention has removed and one that never existed are the same
// answer, and the API does not pretend to tell them apart.
const event = row orelse return http_util.respondError(request, .not_found, "not found");
return http_util.respondJson(request, .ok, event, &.{});
}
/// An open episode is the current state of the box, so it is not history to
/// throw away — and the message says what would make it purgeable.
const still_active_message = "the event is still active; it can be purged once it resolves";
/// `DELETE /api/diagnostics` — how many resolved events went.
pub const PurgeResult = struct {
purged: i64,
};
/// `DELETE /api/diagnostics/{id}`. Resolution stays automatic; this is only
/// about when the history disappears, which is the operator's call.
///
/// Classified `runtime_action` in the route table, not `config_write`: the
/// event log is runtime state that no configuration file declares, so file
/// authority has nothing to say about it and the router lets this through in
/// both modes.
pub fn purge(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const store = state.events orelse
return http_util.respondError(request, .service_unavailable, unavailable_message);
// No log here: the store already latches and logs the false→true
// transition, and a warning per retried request would spam.
const outcome = store.purge(io, request.id.?) catch
return http_util.respondError(request, .internal_server_error, "internal error");
return switch (outcome) {
.deleted => http_util.respondEmpty(request, .no_content),
.active => http_util.respondError(request, .conflict, still_active_message),
.absent => http_util.respondError(request, .not_found, "not found"),
};
}
/// `DELETE /api/diagnostics` — the whole resolved history at once. Active
/// episodes are never touched, so an operator clearing the page cannot lose the
/// events that are still true.
pub fn purgeAll(
state: *server.WebState,
io: std.Io,
request: *http_util.Request,
) http_util.HandlerError!void {
const store = state.events orelse
return http_util.respondError(request, .service_unavailable, unavailable_message);
const purged = store.purgeAll(io) catch
return http_util.respondError(request, .internal_server_error, "internal error");
return http_util.respondJson(request, .ok, PurgeResult{ .purged = purged }, &.{});
}
// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------
const migrations = @import("../../storage/migrations.zig");
const testing = std.testing;
test "an empty query string is the default page over everything" {
var buffers: Buffers = .{};
const filter = try parseFilter("", &buffers);
try testing.expectEqual(default_limit, filter.limit);
try testing.expectEqual(events.State.all, filter.state);
try testing.expectEqual(@as(?[]const u8, null), filter.severity);
try testing.expectEqual(@as(?[]const u8, null), filter.component);
try testing.expectEqual(@as(?i64, null), filter.before);
try testing.expectEqual(@as(?i64, null), filter.since);
}
test "every filter reaches the store untouched" {
var buffers: Buffers = .{};
const filter = try parseFilter(
"state=resolved&severity=error&component=query_log&since=100&until=200&limit=250&before=900",
&buffers,
);
try testing.expectEqual(events.State.resolved, filter.state);
try testing.expectEqualStrings("error", filter.severity.?);
try testing.expectEqualStrings("query_log", filter.component.?);
try testing.expectEqual(@as(?i64, 100), filter.since);
try testing.expectEqual(@as(?i64, 200), filter.until);
try testing.expectEqual(@as(u32, 250), filter.limit);
try testing.expectEqual(@as(?i64, 900), filter.before);
}
test "an empty component is no filter at all" {
var buffers: Buffers = .{};
try testing.expectEqual(@as(?[]const u8, null), (try parseFilter("component=", &buffers)).component);
}
test "each malformed parameter names itself in a 400" {
var buffers: Buffers = .{};
try testing.expectError(error.BadState, parseFilter("state=open", &buffers));
try testing.expectError(error.BadState, parseFilter("state=", &buffers));
try testing.expectError(error.BadSeverity, parseFilter("severity=info", &buffers));
try testing.expectError(error.BadSeverity, parseFilter("severity=WARNING", &buffers));
try testing.expectError(error.BadSince, parseFilter("since=yesterday", &buffers));
try testing.expectError(error.BadUntil, parseFilter("until=", &buffers));
try testing.expectError(error.BadLimit, parseFilter("limit=0", &buffers));
try testing.expectError(error.BadLimit, parseFilter("limit=1001", &buffers));
try testing.expectError(error.BadLimit, parseFilter("limit=ten", &buffers));
try testing.expectError(error.BadBefore, parseFilter("before=0", &buffers));
try testing.expectError(error.BadBefore, parseFilter("before=-4", &buffers));
try testing.expectError(error.BadComponent, parseFilter("component=%zz", &buffers));
var long: [max_component_len + 8]u8 = @splat('a');
var text: std.ArrayList(u8) = .empty;
defer text.deinit(testing.allocator);
try text.appendSlice(testing.allocator, "component=");
try text.appendSlice(testing.allocator, &long);
try testing.expectError(error.BadComponent, parseFilter(text.items, &buffers));
// Every member of the error set has its own wording, and none of them is
// the empty string.
inline for (comptime std.meta.fieldNames(FilterError)) |name| {
try testing.expect(message(@field(FilterError, name)).len != 0);
}
}
test "the limit cap is the store's" {
var buffers: Buffers = .{};
try testing.expectEqual(max_limit, (try parseFilter("limit=1000", &buffers)).limit);
try testing.expectEqual(@as(u32, 1000), events.max_limit);
}
/// A store over a migrated in-memory `config.db`, built in place: a `Store`
/// holds a `*db.Db`, so a fixture that moved after `init` would leave that
/// pointer behind.
const Fixture = struct {
threaded: std.Io.Threaded = undefined,
io: std.Io = undefined,
database: db.Db = undefined,
store: events.Store = undefined,
state: server.WebState = undefined,
fn init(self: *Fixture) !void {
self.threaded = .init(testing.allocator, .{});
errdefer self.threaded.deinit();
self.io = self.threaded.io();
self.database = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer self.database.close();
try db.applyPragmas(&self.database, .{});
_ = try migrations.migrate(&self.database);
self.store = try events.Store.init(self.io, &self.database, 1000);
self.state = .{ .gpa = testing.allocator, .events = &self.store };
}
fn deinit(self: *Fixture) void {
self.database.close();
self.threaded.deinit();
}
fn seed(self: *Fixture) void {
const store = &self.store;
store.report(self.io, 1000, .blocklist_refresh, "https://a.example", "StevenBlack", .warning, "ConnectionTimedOut");
store.report(self.io, 1100, .blocklist_refresh, "https://a.example", "StevenBlack", .warning, "ConnectionTimedOut");
store.report(self.io, 1200, .listener_start, "doh", "doh", .@"error", "AddressInUse");
store.report(self.io, 1300, .query_log_write, "batch", "batch", .@"error", "Busy");
store.resolve(self.io, 1400, .query_log_write, "batch");
}
};
test "a page carries the events, the cursor and the active counts" {
var fx: Fixture = .{};
try fx.init();
defer fx.deinit();
fx.seed();
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const page = try fx.store.selectEvents(fx.io, arena.allocator(), .{ .limit = 100 });
try testing.expectEqual(@as(usize, 3), page.events.len);
try testing.expectEqual(@as(?i64, null), page.next_before);
try testing.expectEqual(events.Counts{ .warnings = 1, .errors = 1 }, page.active);
// Newest first, and the episode that repeated counts rather than repeats.
try testing.expectEqualStrings("query_log.write", page.events[0].code);
try testing.expectEqualStrings("blocklist.refresh", page.events[2].code);
try testing.expectEqual(@as(i64, 2), page.events[2].occurrences);
try testing.expectEqualStrings("StevenBlack", page.events[2].subject);
}
test "a full page carries a cursor and the last page does not" {
var fx: Fixture = .{};
try fx.init();
defer fx.deinit();
fx.seed();
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const gpa = arena.allocator();
const first = try fx.store.selectEvents(fx.io, gpa, .{ .limit = 2 });
try testing.expectEqual(@as(usize, 2), first.events.len);
try testing.expectEqual(first.events[1].id, first.next_before.?);
const second = try fx.store.selectEvents(fx.io, gpa, .{ .limit = 2, .before = first.next_before });
try testing.expectEqual(@as(usize, 1), second.events.len);
try testing.expectEqual(@as(?i64, null), second.next_before);
}
test "the parsed filters narrow the page the store answers with" {
var fx: Fixture = .{};
try fx.init();
defer fx.deinit();
fx.seed();
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const gpa = arena.allocator();
var buffers: Buffers = .{};
const active = try fx.store.selectEvents(fx.io, gpa, try parseFilter("state=active", &buffers));
try testing.expectEqual(@as(usize, 2), active.events.len);
const resolved = try fx.store.selectEvents(fx.io, gpa, try parseFilter("state=resolved", &buffers));
try testing.expectEqual(@as(usize, 1), resolved.events.len);
try testing.expectEqual(@as(?i64, 1400), resolved.events[0].resolved_at);
const errors = try fx.store.selectEvents(fx.io, gpa, try parseFilter("severity=error", &buffers));
try testing.expectEqual(@as(usize, 2), errors.events.len);
const blocklist = try fx.store.selectEvents(fx.io, gpa, try parseFilter("component=blocklist", &buffers));
try testing.expectEqual(@as(usize, 1), blocklist.events.len);
try testing.expectEqualStrings("blocklist", blocklist.events[0].component);
}
test "the wire object carries the label and never the subject key" {
var fx: Fixture = .{};
try fx.init();
defer fx.deinit();
// A key that would be unmistakable on the wire if it ever leaked: the
// upstream urls this store keys on can carry a token.
fx.store.report(fx.io, 1000, .upstream_exchange, "https://dns.example/secret-token", "dns.example", .warning, "Timeout");
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const page = try fx.store.selectEvents(fx.io, arena.allocator(), .{});
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
defer allocating.deinit();
try std.json.Stringify.value(page, .{}, &allocating.writer);
const text = allocating.written();
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "secret-token"));
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "subject_key"));
try testing.expect(std.mem.startsWith(u8, text, "{\"events\":["));
for ([_][]const u8{
"\"id\":", "\"code\":\"upstream.exchange\"",
"\"component\":\"upstream\"", "\"subject\":\"dns.example\"",
"\"severity\":\"warning\"", "\"first_seen\":",
"\"last_seen\":", "\"occurrences\":",
"\"resolved_at\":null", "\"detail\":\"Timeout\"",
"\"next_before\":null", "\"active\":{\"warnings\":1,\"errors\":0}",
}) |field| {
errdefer std.debug.print("missing {s} in {s}\n", .{ field, text });
try testing.expect(std.mem.containsAtLeast(u8, text, 1, field));
}
}
test "a detail page answers by id and reports an unknown one as absent" {
var fx: Fixture = .{};
try fx.init();
defer fx.deinit();
fx.seed();
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const found = (try fx.store.selectOne(fx.io, arena.allocator(), 1)).?;
try testing.expectEqualStrings("blocklist.refresh", found.code);
// The 404 the handler answers with is this null.
try testing.expect((try fx.store.selectOne(fx.io, arena.allocator(), 9999)) == null);
}
test "the purge-all body is the count and nothing else" {
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
defer allocating.deinit();
try std.json.Stringify.value(PurgeResult{ .purged = 3 }, .{}, &allocating.writer);
try testing.expectEqualStrings("{\"purged\":3}", allocating.written());
// The 409 says what would make the event purgeable, so it is not the
// generic conflict wording.
try testing.expect(still_active_message.len != 0);
try testing.expect(std.mem.containsAtLeast(u8, still_active_message, 1, "resolves"));
}
test "a state with no store answers 503 rather than an empty page" {
// The handler's only branch that does not need a request: a `WebState`
// whose store failed to open reports the endpoint unavailable, and never
// an empty list that would read as "nothing is wrong".
const state: server.WebState = .{ .gpa = testing.allocator };
try testing.expectEqual(@as(?*events.Store, null), state.events);
try testing.expect(unavailable_message.len != 0);
}
src/web/handlers/health.zig
+114 -1
View File
@@ -25,6 +25,15 @@ pub const Disk = struct {
sample_failures: u64,
};
/// The diagnostics store's own state, not a summary of what it holds: `state`
/// answers "is the operational log recording", and the two counts answer "what
/// is open right now".
pub const Diagnostics = struct {
state: []const u8,
active_warnings: u32,
active_errors: u32,
};
pub const Upstreams = struct {
available: u32,
total: u32,
@@ -34,6 +43,7 @@ pub const Body = struct {
status: []const u8,
disk: Disk,
upstreams: Upstreams,
diagnostics: Diagnostics,
queries_dropped: u64,
writer_failed: bool,
refreshes_gated: u64,
@@ -61,6 +71,16 @@ pub const Input = struct {
/// one overflow. Drops surface through the metric and through the API's
/// per-window `complete` instead.
history_flush_failing: bool = false,
/// The diagnostics store exists. The benign default matches every other
/// field here — a half-wired `Input` reports a box with nothing wrong — but
/// `collect` must assign it explicitly, because in a serving process an
/// absent store means `Store.init` failed.
diagnostics_present: bool = true,
/// The last diagnostics write failed. Current state, cleared by the next
/// write that succeeds, like `history_flush_failing`.
diagnostics_write_failed: bool = false,
diagnostics_active_warnings: u32 = 0,
diagnostics_active_errors: u32 = 0,
refreshes_gated: u64 = 0,
snapshot_generation: ?u64 = null,
};
@@ -68,6 +88,16 @@ pub const Input = struct {
pub const status_ok = "ok";
pub const status_degraded = "degraded";
pub const diagnostics_recording = "recording";
pub const diagnostics_unavailable = "unavailable";
/// The operational log is not recording — either the store never opened or its
/// writes are failing. Both mean the same thing to an operator: the record of
/// what went wrong is not being kept.
pub fn diagnosticsUnavailable(input: Input) bool {
return !input.diagnostics_present or input.diagnostics_write_failed;
}
/// Conditions an operator must act on, and every one of them is a fact about
/// now rather than a count of the past: a disk that is filling stops the query
/// log, a pool with nothing available stops resolution, a failed writer means
@@ -76,7 +106,7 @@ pub const status_degraded = "degraded";
/// the underlying condition does.
pub fn degraded(input: Input) bool {
return input.disk_state != .ok or input.upstreams_available == 0 or
input.writer_failed or input.history_flush_failing;
input.writer_failed or input.history_flush_failing or diagnosticsUnavailable(input);
}
pub fn rollup(input: Input) Body {
@@ -90,6 +120,11 @@ pub fn rollup(input: Input) Body {
.sample_failures = input.disk_sample_failures,
},
.upstreams = .{ .available = input.upstreams_available, .total = input.upstreams_total },
.diagnostics = .{
.state = if (diagnosticsUnavailable(input)) diagnostics_unavailable else diagnostics_recording,
.active_warnings = input.diagnostics_active_warnings,
.active_errors = input.diagnostics_active_errors,
},
.queries_dropped = input.queries_dropped,
.writer_failed = input.writer_failed,
.refreshes_gated = input.refreshes_gated,
@@ -128,6 +163,17 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
input.writer_failed = logger.writer_failed.load(.monotonic);
}
// Assigned before the `if`, not inside it: the field's benign default is
// `true`, so the natural `if (state.events) |store|` shape would report an
// absent store as recording — the one case that must degrade.
input.diagnostics_present = state.events != null;
if (state.events) |store| {
input.diagnostics_write_failed = store.writeFailed();
const counts = store.activeCounts(io);
input.diagnostics_active_warnings = counts.warnings;
input.diagnostics_active_errors = counts.errors;
}
if (state.history) |history| {
input.history_flush_failing = history.snapshotStats(io).last_flush_failed;
}
@@ -148,6 +194,8 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
// ---------------------------------------------------------------------------
const db = @import("../../storage/db.zig");
const events_mod = @import("../../storage/events.zig");
const migrations = @import("../../storage/migrations.zig");
const history_mod = @import("../../upstream/history.zig");
const logger_mod = @import("../../storage/logger.zig");
const upstream_history_repo = @import("../../storage/repositories/upstream_history_repo.zig");
@@ -172,6 +220,11 @@ test "the degraded matrix covers disk state, availability and the writer" {
// right now, and it recovers on its own the moment a flush succeeds.
.{ .input = withHistoryFailing(healthy, true), .degraded = true },
.{ .input = withHistoryFailing(healthy, false), .degraded = false },
// The operational log not recording is itself a fault an operator must
// act on: whatever fails next will leave no record of having failed.
.{ .input = withDiagnostics(healthy, false, false), .degraded = true },
.{ .input = withDiagnostics(healthy, true, true), .degraded = true },
.{ .input = withDiagnostics(healthy, true, false), .degraded = false },
// Two faults at once still report one status.
.{ .input = withWriterFailed(withDisk(healthy, .critical)), .degraded = true },
// Some upstreams down is not degraded while one still answers.
@@ -212,6 +265,66 @@ fn withHistoryFailing(input: Input, failing: bool) Input {
return out;
}
fn withDiagnostics(input: Input, present: bool, write_failed: bool) Input {
var out = input;
out.diagnostics_present = present;
out.diagnostics_write_failed = write_failed;
return out;
}
test "the diagnostics block reports the state and the open counts" {
const recording = rollup(.{
.upstreams_available = 1,
.diagnostics_active_warnings = 3,
.diagnostics_active_errors = 1,
});
try testing.expectEqualStrings(diagnostics_recording, recording.diagnostics.state);
try testing.expectEqual(@as(u32, 3), recording.diagnostics.active_warnings);
try testing.expectEqual(@as(u32, 1), recording.diagnostics.active_errors);
// Open episodes are what the box is doing, not a fault of the log: they do
// not degrade on their own.
try testing.expectEqualStrings(status_ok, recording.status);
// Failing writes: the counts are whatever was last read, and the state is
// the honest one.
const failing = rollup(.{ .upstreams_available = 1, .diagnostics_write_failed = true });
try testing.expectEqualStrings(diagnostics_unavailable, failing.diagnostics.state);
try testing.expectEqualStrings(status_degraded, failing.status);
const absent = rollup(.{ .upstreams_available = 1, .diagnostics_present = false });
try testing.expectEqualStrings(diagnostics_unavailable, absent.diagnostics.state);
try testing.expectEqualStrings(status_degraded, absent.status);
}
test "collect reports an absent store as unavailable rather than as recording" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
// `diagnostics_present` defaults to true like every other benign default,
// so an assignment `collect` forgot would read as a healthy log here.
var state: server.WebState = .{ .gpa = testing.allocator };
const absent = collect(&state, io);
try testing.expect(!absent.diagnostics_present);
try testing.expectEqualStrings(diagnostics_unavailable, rollup(absent).diagnostics.state);
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var store = try events_mod.Store.init(io, &database, 1000);
store.report(io, 1000, .disk_space, "data", "data", .warning, "low");
store.report(io, 1000, .listener_start, "doh", "doh", .@"error", "AddressInUse");
state.events = &store;
const present = collect(&state, io);
try testing.expect(present.diagnostics_present);
try testing.expect(!present.diagnostics_write_failed);
try testing.expectEqual(@as(u32, 1), present.diagnostics_active_warnings);
try testing.expectEqual(@as(u32, 1), present.diagnostics_active_errors);
try testing.expectEqualStrings(diagnostics_recording, rollup(present).diagnostics.state);
}
test "a history overflow that already happened does not degrade the rollup" {
// `rows_dropped` is cumulative and the rollup is stateless, so the only
// thing it could do with a drop count is latch on it. The accumulator's
src/web/metrics.zig
+96
View File
@@ -30,6 +30,7 @@ const dns_cache = @import("../cache/dns_cache.zig");
const dns_handler = @import("../server/handler.zig");
const disk_monitor = @import("../storage/disk_monitor.zig");
const dot_server = @import("../server/dot_server.zig");
const events_mod = @import("../storage/events.zig");
const history_mod = @import("../upstream/history.zig");
const http_util = @import("http_util.zig");
const logging = @import("../platform/logging.zig");
@@ -115,6 +116,15 @@ pub const UpstreamSample = struct {
success_rate: f32,
};
/// The diagnostics store, as one scrape sees it. Two gauges and a counter,
/// written out rather than reflected over a stats struct because they are not
/// all the same kind of number.
pub const DiagnosticsSample = struct {
active_warnings: u32,
active_errors: u32,
write_failures: u64,
};
/// Everything one scrape reports. A null section is a collaborator the state
/// does not have.
pub const Sample = struct {
@@ -129,6 +139,11 @@ pub const Sample = struct {
/// The upstream-history flush loop's counters (m26 ruling 7). Absent while
/// no accumulator is wired, like every other collaborator.
history: ?history_mod.Accumulator.Stats = null,
/// The diagnostics store's open episodes and its failed writes. Absent
/// while no store is wired, like every other collaborator — an operator
/// distinguishes "no series" from "zero episodes" through `/api/health`,
/// which says which of the two it is.
diagnostics: ?DiagnosticsSample = null,
blocklist: ?BlocklistSample = null,
disk: ?DiskSample = null,
/// One entry per enabled TLS endpoint (milestone-10 ruling 10). Rendered
@@ -205,6 +220,15 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
if (state.history) |history| sample.history = history.snapshotStats(io);
if (state.events) |store| {
const counts = store.activeCounts(io);
sample.diagnostics = .{
.active_warnings = counts.warnings,
.active_errors = counts.errors,
.write_failures = store.writeFailures(),
};
}
if (state.manager) |manager| {
const generation: ?u64 = if (manager.acquire(io)) |acquired| gen: {
defer acquired.release(io);
@@ -388,6 +412,27 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
);
}
if (sample.diagnostics) |diagnostics| {
try gauge(
w,
"nxdns_diagnostics_active_warnings",
"Operational event episodes open right now at warning severity.",
diagnostics.active_warnings,
);
try gauge(
w,
"nxdns_diagnostics_active_errors",
"Operational event episodes open right now at error severity.",
diagnostics.active_errors,
);
try counter(
w,
"nxdns_diagnostics_write_failures_total",
"Operational events dropped because the diagnostics database refused the write.",
diagnostics.write_failures,
);
}
if (sample.blocklist) |blocklist| {
try counter(
w,
@@ -638,8 +683,10 @@ fn writeLabelValue(w: *std.Io.Writer, value: []const u8) std.Io.Writer.Error!voi
// tests
// ---------------------------------------------------------------------------
const db = @import("../storage/db.zig");
const local_tables = @import("../server/local_tables.zig");
const logger_mod = @import("../storage/logger.zig");
const migrations = @import("../storage/migrations.zig");
const testing = std.testing;
/// A handler with no upstream reachable: every test here reads counters and
@@ -781,6 +828,55 @@ test "the upstream-history family renders three counters and one gauge" {
try testing.expect(!std.mem.containsAtLeast(u8, bare, 1, "nxdns_upstream_history_"));
}
test "the diagnostics family renders two gauges and one counter" {
const text = try renderToString(testing.allocator, .{
.diagnostics = .{ .active_warnings = 3, .active_errors = 1, .write_failures = 7 },
});
defer testing.allocator.free(text);
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_diagnostics_active_warnings gauge\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_diagnostics_active_warnings 3\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_diagnostics_active_errors gauge\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_diagnostics_active_errors 1\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_diagnostics_write_failures_total counter\n"));
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_diagnostics_write_failures_total 7\n"));
// No store is an absent family, not a family of zeros: "no episodes open"
// and "nothing is recording them" must not render the same.
const bare = try renderToString(testing.allocator, .{});
defer testing.allocator.free(bare);
try testing.expect(!std.mem.containsAtLeast(u8, bare, 1, "nxdns_diagnostics_"));
}
test "collect reads the diagnostics store's open episodes and failed writes" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try db.applyPragmas(&database, .{});
_ = try migrations.migrate(&database);
var store = try events_mod.Store.init(io, &database, 1000);
store.report(io, 1000, .disk_space, "data", "data", .warning, "low");
var state: server.WebState = .{ .gpa = testing.allocator, .events = &store };
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
defer arena.deinit();
const sample = try collect(&state, io, arena.allocator());
try testing.expectEqual(@as(u32, 1), sample.diagnostics.?.active_warnings);
try testing.expectEqual(@as(u32, 0), sample.diagnostics.?.active_errors);
try testing.expectEqual(@as(u64, 0), sample.diagnostics.?.write_failures);
// A write that cannot land is counted here, through the production path
// rather than by poking the field.
try database.exec("DROP TABLE operational_events;");
store.report(io, 1100, .disk_space, "data", "data", .warning, "low");
const after = try collect(&state, io, arena.allocator());
try testing.expect(after.diagnostics.?.write_failures >= 1);
}
test "every HELP line has a TYPE line and a sample, and every sample a name" {
const text = try renderToString(testing.allocator, .{});
defer testing.allocator.free(text);
src/web/openapi.yaml
+222 -1
View File
@@ -256,6 +256,131 @@ paths:
"503":
$ref: "#/components/responses/Unavailable"
/api/diagnostics:
get:
summary: Operational event log
description: |
Failure episodes, newest first. One event is one subject failing
continuously: it opens on the first failure, counts repeats in
`occurrences`, and gets a `resolved_at` when the subject recovers. A
subject that fails again opens a new event rather than reopening the
old one. Keyset pagination — follow `next_before` until it is null.
parameters:
- name: state
in: query
schema: { type: string, enum: [active, resolved, all], default: all }
- name: severity
in: query
schema: { type: string, enum: [warning, error] }
- name: component
in: query
description: Matches the part of `code` before the dot, exactly.
schema: { type: string, maxLength: 64 }
- name: since
in: query
description: |
Unix seconds. With `until`, selects episodes overlapping the
window; an episode resolved exactly at `since` does not overlap.
schema: { type: integer }
- name: until
in: query
description: Unix seconds, exclusive.
schema: { type: integer }
- name: limit
in: query
schema: { type: integer, minimum: 1, maximum: 1000, default: 100 }
- name: before
in: query
description: Return events with id strictly below this cursor.
schema: { type: integer, minimum: 1 }
responses:
"200":
description: One page.
content:
application/json:
schema:
$ref: "#/components/schemas/DiagnosticsPage"
"400":
$ref: "#/components/responses/BadRequest"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
delete:
summary: Purge every resolved event
description: |
Deletes the resolved history and answers with how many rows went.
Active events are never touched, so clearing the page cannot lose an
episode that is still failing. Resolution stays automatic; this only
decides when the history disappears. A runtime action, served in file
mode too — the event log is not configuration.
responses:
"200":
description: How many resolved events were removed.
content:
application/json:
schema:
$ref: "#/components/schemas/DiagnosticsPurge"
"401":
$ref: "#/components/responses/Unauthorized"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/diagnostics/{id}:
parameters:
- $ref: "#/components/parameters/RowId"
get:
summary: One operational event
description: |
404 for an id that never existed and for one retention has removed —
the API does not distinguish them.
responses:
"200":
description: The event.
content:
application/json:
schema:
$ref: "#/components/schemas/DiagnosticEvent"
"401":
$ref: "#/components/responses/Unauthorized"
"404":
$ref: "#/components/responses/NotFound"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
delete:
summary: Purge one resolved event
description: |
Deletes a resolved event. An event that is still active answers 409 —
an open episode is the current state of the box, not history — and an
id no row holds answers 404. A runtime action, served in file mode too.
responses:
"204":
description: Purged.
"401":
$ref: "#/components/responses/Unauthorized"
"404":
$ref: "#/components/responses/NotFound"
"409":
$ref: "#/components/responses/Conflict"
"429":
$ref: "#/components/responses/RateLimited"
"500":
$ref: "#/components/responses/Internal"
"503":
$ref: "#/components/responses/Unavailable"
/api/stats:
get:
summary: Totals for a period
@@ -1664,11 +1789,21 @@ components:
Health:
type: object
required: [status, disk, upstreams, queries_dropped, writer_failed, refreshes_gated, snapshot_generation]
required: [status, disk, upstreams, diagnostics, queries_dropped, writer_failed, refreshes_gated, snapshot_generation]
properties:
status:
type: string
enum: [ok, degraded]
diagnostics:
type: object
required: [state, active_warnings, active_errors]
properties:
state:
type: string
enum: [recording, unavailable]
description: unavailable when the event store failed to open or its writes are failing; either state degrades health.
active_warnings: { type: integer }
active_errors: { type: integer }
disk:
type: object
required: [state, free_bytes, db_bytes, log_bytes, sample_failures]
@@ -1760,6 +1895,92 @@ components:
nullable: true
description: Cursor for the next page; null on the last page.
DiagnosticEvent:
type: object
required: [id, code, component, subject, severity, first_seen, last_seen, occurrences, resolved_at, detail]
properties:
id: { type: integer }
code:
type: string
description: |
The failure kind, as `component.name`. One of a fixed set of
fifteen; new codes are added with new releases.
enum:
- disk.space
- disk.probe
- blocklist.refresh
- blocklist.snapshot
- blocklist.storage
- certificate.reload
- query_log.write
- query_log.maintenance
- query_log.recreated
- upstream_history.write
- upstream.exchange
- client_names.storage
- clients.storage
- listener.start
- configuration.load
component:
type: string
description: The part of `code` before the dot, repeated for filtering.
subject:
type: string
description: |
What failed, as a display name: a blocklist source name, an
endpoint, an operation. Redacted where it derives from a url; the
store's internal identity for the subject is never exposed.
severity:
type: string
enum: [warning, error]
first_seen:
type: integer
description: When this episode opened, unix seconds.
last_seen:
type: integer
description: The most recent failure of this episode, unix seconds.
occurrences:
type: integer
description: How many failures this episode has held; at least 1.
resolved_at:
type: integer
nullable: true
description: |
When the subject recovered, unix seconds. Null while the episode is
still open. A subject that fails again opens a new event rather than
reopening this one.
detail:
type: string
description: The last error of this episode, truncated to 512 bytes.
DiagnosticsPage:
type: object
required: [events, next_before, active]
properties:
events:
type: array
items:
$ref: "#/components/schemas/DiagnosticEvent"
next_before:
type: integer
nullable: true
description: Cursor for the next page; null on the last page.
active:
type: object
required: [warnings, errors]
description: Episodes open right now, whatever this page filtered to.
properties:
warnings: { type: integer }
errors: { type: integer }
DiagnosticsPurge:
type: object
required: [purged]
properties:
purged:
type: integer
description: How many resolved events the purge removed; zero when there were none.
StatsTotals:
type: object
required: [period, since, until, queries, blocked, cached, clients, avg_response_time_us]
src/web/routes.zig
+12 -1
View File
@@ -36,6 +36,7 @@ const auth = @import("handlers/auth.zig");
const blocklists = @import("handlers/blocklists.zig");
const certs = @import("handlers/certs.zig");
const clients = @import("handlers/clients.zig");
const diagnostics = @import("handlers/diagnostics.zig");
const groups = @import("handlers/groups.zig");
const health = @import("handlers/health.zig");
const live = @import("handlers/live.zig");
@@ -71,6 +72,14 @@ pub const table: []const router.RouteInfo = &.{
.{ .method = .GET, .pattern = "/api/lookup", .auth = .session, .policy = .read, .handler = lookup.handle },
.{ .method = .GET, .pattern = "/api/upstream/health", .auth = .session, .policy = .read, .handler = upstream_health.handle },
// Diagnostics: the operational event log (milestone 27). The two purges are
// `runtime_action` — the event log is runtime state no configuration file
// declares, so file authority has nothing to say about deleting from it.
.{ .method = .GET, .pattern = "/api/diagnostics", .auth = .session, .policy = .read, .handler = diagnostics.list },
.{ .method = .DELETE, .pattern = "/api/diagnostics", .auth = .session, .policy = .runtime_action, .handler = diagnostics.purgeAll },
.{ .method = .GET, .pattern = "/api/diagnostics/{id}", .auth = .session, .policy = .read, .handler = diagnostics.get },
.{ .method = .DELETE, .pattern = "/api/diagnostics/{id}", .auth = .session, .policy = .runtime_action, .handler = diagnostics.purge },
// Groups.
.{ .method = .GET, .pattern = "/api/groups", .auth = .session, .policy = .read, .handler = groups.list },
.{ .method = .POST, .pattern = "/api/groups", .auth = .session, .policy = .config_write, .handler = groups.create },
@@ -143,7 +152,7 @@ const std = @import("std");
const testing = std.testing;
test "the table carries every endpoint of the milestone" {
try testing.expectEqual(@as(usize, 56), table.len);
try testing.expectEqual(@as(usize, 60), table.len);
}
test "no two entries claim the same method and pattern" {
@@ -231,6 +240,8 @@ test "the runtime actions are exactly ruling 7's list" {
"POST /api/auth/login",
"POST /api/auth/logout",
"POST /api/blocklists/update",
"DELETE /api/diagnostics",
"DELETE /api/diagnostics/{id}",
"DELETE /api/clients/{id}",
"POST /api/pause",
"POST /api/certs/reload",
src/web/server.zig
+6
View File
@@ -32,6 +32,7 @@ const disk_monitor = @import("../storage/disk_monitor.zig");
const dns_handler = @import("../server/handler.zig");
const doh_server = @import("../server/doh_server.zig");
const dot_server = @import("../server/dot_server.zig");
const events_mod = @import("../storage/events.zig");
const http_util = @import("http_util.zig");
const listener_core = @import("../server/listener.zig");
const local_tables_mod = @import("../server/local_tables.zig");
@@ -183,6 +184,11 @@ pub const WebState = struct {
/// concurrent writes would misread each other's row counts.
config_lock: std.Io.Mutex = .init,
querylog_db: ?*db.Db = null,
/// The diagnostics event store, which owns a third connection of its own
/// and serializes every access — read and write — through its mutex. Null
/// when `Store.init` failed, which `/api/health` reports as `unavailable`
/// and treats as degraded.
events: ?*events_mod.Store = null,
version: []const u8 = "",
/// The `--admin-dev` asset directory, read by the dev-mode fallback. Empty
src/web/web_integration_test.zig
+199
View File
@@ -31,6 +31,7 @@ const auth = @import("auth.zig");
const clients_repo = @import("../storage/repositories/clients_repo.zig");
const db = @import("../storage/db.zig");
const dns_handler = @import("../server/handler.zig");
const events_mod = @import("../storage/events.zig");
const fetcher = @import("../filter/fetcher.zig");
const groups_repo = @import("../storage/repositories/groups_repo.zig");
const header = @import("../dns/header.zig");
@@ -58,6 +59,7 @@ const upstreams_repo = @import("../storage/repositories/upstreams_repo.zig");
const handlers_blocklists = @import("handlers/blocklists.zig");
const handlers_certs = @import("handlers/certs.zig");
const handlers_diagnostics = @import("handlers/diagnostics.zig");
const handlers_health = @import("handlers/health.zig");
const handlers_live = @import("handlers/live.zig");
const handlers_lookup = @import("handlers/lookup.zig");
@@ -277,6 +279,10 @@ const Env = struct {
tmp: testing.TmpDir,
config_db: db.Db,
querylog_db: db.Db,
/// The diagnostics store's own connection, as in production: the store
/// serializes every access through its mutex and shares it with nobody.
events_db: db.Db,
events_store: events_mod.Store,
http_client: std.http.Client,
transfer_buf: [fetcher.min_transfer_buf]u8,
redirect_buf: [fetcher.redirect_buffer_len]u8,
@@ -317,6 +323,13 @@ const Env = struct {
try self.querylog_db.exec(querylog_schema.ddl);
try seedQueryLog(&self.querylog_db);
self.events_db = try db.Db.open(":memory:", .{ .mode = .memory });
errdefer self.events_db.close();
try db.applyPragmas(&self.events_db, .{});
_ = try migrations.migrate(&self.events_db);
self.events_store = try events_mod.Store.init(ioh, &self.events_db, seeded_now);
seedEvents(ioh, &self.events_store);
// Real fetcher wiring; nothing in this suite downloads (the one
// refreshAll in the contract walk runs with zero source rows).
self.http_client = .{ .allocator = gpa, .io = ioh };
@@ -388,6 +401,7 @@ const Env = struct {
.hub = self.hub,
.config_db = &self.config_db,
.querylog_db = &self.querylog_db,
.events = &self.events_store,
.version = "w10-test",
.started_unix = std.Io.Clock.real.now(ioh).toSeconds(),
.fallback = options.fallback,
@@ -419,6 +433,7 @@ const Env = struct {
self.limiter.deinit();
self.mgr.deinit(ioh);
self.http_client.deinit();
self.events_db.close();
self.querylog_db.close();
self.config_db.close();
self.tmp.cleanup();
@@ -480,6 +495,21 @@ fn seedQueryLog(database: *db.Db) !void {
}
}
/// A fixed instant, like every other seeded timestamp here: the contract
/// samples are byte-compared, so nothing the walk writes may come from a clock.
const seeded_now: i64 = 1_787_118_000;
/// One active episode and one resolved one, so `/api/diagnostics` answers with
/// both states and the committed contract sample describes a real page rather
/// than an empty one.
fn seedEvents(io: std.Io, store: *events_mod.Store) void {
store.report(io, seeded_now, .blocklist_refresh, "https://lists.example/ads.txt", "StevenBlack", .warning, "download failed: ConnectionTimedOut");
store.report(io, seeded_now + 300, .blocklist_refresh, "https://lists.example/ads.txt", "StevenBlack", .warning, "download failed: ConnectionTimedOut");
store.report(io, seeded_now + 60, .upstream_history_write, "history", "history", .warning, "Busy");
store.resolve(io, seeded_now + 120, .upstream_history_write, "history");
}
// ---------------------------------------------------------------------------
// the contract table (ruling 23)
// ---------------------------------------------------------------------------
@@ -627,6 +657,15 @@ const contract = [_]Contract{
.{ .method = .GET, .pattern = "/api/stats/timeseries", .auth = .session, .policy = .read, .target = "/api/stats/timeseries?period=1h", .status = 200, .check = jsonShape(handlers_stats.TimeseriesBody) },
.{ .method = .GET, .pattern = "/api/upstream/health", .auth = .session, .policy = .read, .target = "/api/upstream/health", .status = 200, .check = jsonShape(handlers_upstream_health.Body) },
// Diagnostics. The seeded store holds one active episode (id 1) and one
// resolved one, so both the page and the detail answer with real rows.
.{ .method = .GET, .pattern = "/api/diagnostics", .auth = .session, .policy = .read, .target = "/api/diagnostics?limit=10", .status = 200, .check = jsonShape(events_mod.EventsPage) },
.{ .method = .GET, .pattern = "/api/diagnostics/{id}", .auth = .session, .policy = .read, .target = "/api/diagnostics/1", .status = 200, .check = jsonShape(events_mod.Event) },
// The purges follow the reads: id 2 is the seeded resolved episode, and the
// sweep after it takes whatever resolved history is left (none).
.{ .method = .DELETE, .pattern = "/api/diagnostics/{id}", .auth = .session, .policy = .runtime_action, .target = "/api/diagnostics/2", .status = 204, .kind = .none },
.{ .method = .DELETE, .pattern = "/api/diagnostics", .auth = .session, .policy = .runtime_action, .target = "/api/diagnostics", .status = 200, .check = jsonShape(handlers_diagnostics.PurgeResult) },
// Groups. The migrated schema seeds `default` as id 1; the POST creates
// id 2, which the delete at the end of the walk removes.
.{ .method = .GET, .pattern = "/api/groups", .auth = .session, .policy = .read, .target = "/api/groups", .status = 200, .check = jsonShape(GroupsList) },
@@ -1000,6 +1039,157 @@ fn fileModeClasses(io: std.Io, env: *Env) anyerror!void {
try conn.request("POST", "/api/certs/reload", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), response.status);
// Diagnostics are runtime state, not configuration: purging resolved
// history is served under file authority like any other runtime action.
try conn.request("DELETE", "/api/diagnostics", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), response.status);
try testing.expectEqualStrings("{\"purged\":1}", response.body);
try conn.request("DELETE", "/api/diagnostics/1", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 409), response.status);
}
fn diagnosticsRejections(io: std.Io, env: *Env) anyerror!void {
var body_buf: [8192]u8 = undefined;
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
// Every bad parameter is a 400 whose message names the parameter, rather
// than a filter silently dropped — which would answer a question the client
// did not ask.
const bad = [_]struct { target: []const u8, needle: []const u8 }{
.{ .target = "/api/diagnostics?state=open", .needle = "state" },
.{ .target = "/api/diagnostics?severity=info", .needle = "severity" },
.{ .target = "/api/diagnostics?since=yesterday", .needle = "since" },
.{ .target = "/api/diagnostics?until=", .needle = "until" },
.{ .target = "/api/diagnostics?limit=0", .needle = "limit" },
.{ .target = "/api/diagnostics?limit=1001", .needle = "limit" },
.{ .target = "/api/diagnostics?before=0", .needle = "before" },
};
for (bad) |case| {
try conn.request("GET", case.target, null, null);
const response = try conn.receive(&body_buf);
errdefer std.debug.print("{s}: {d} {s}\n", .{ case.target, response.status, response.body });
try testing.expectEqual(@as(u16, 400), response.status);
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, case.needle));
}
// The resolved episode the seed left behind is reachable by id, and an id
// nothing holds is a 404 rather than an empty object.
try conn.request("GET", "/api/diagnostics?state=resolved", null, null);
var response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), response.status);
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "upstream_history.write"));
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "\"active\":{\"warnings\":1,\"errors\":0}"));
try conn.request("GET", "/api/diagnostics/999999", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 404), response.status);
}
fn diagnosticsPurge(io: std.Io, env: *Env) anyerror!void {
var body_buf: [8192]u8 = undefined;
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
// Row 1 is the seeded active episode: still the state of the box, so the
// purge is refused with a message that says what would change that.
try conn.request("DELETE", "/api/diagnostics/1", null, null);
var response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 409), response.status);
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "still active"));
try conn.request("DELETE", "/api/diagnostics/999999", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 404), response.status);
// Row 2 is the seeded resolved episode.
try conn.request("DELETE", "/api/diagnostics/2", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 204), response.status);
try testing.expectEqualStrings("", response.body);
// Gone is a different answer from still open, even for a row that existed a
// moment ago.
try conn.request("DELETE", "/api/diagnostics/2", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 404), response.status);
// Nothing resolved is left, and the sweep says so rather than failing.
try conn.request("DELETE", "/api/diagnostics", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), response.status);
try testing.expectEqualStrings("{\"purged\":0}", response.body);
// The active episode survived every one of those, counts included.
try conn.request("GET", "/api/diagnostics", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), response.status);
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "blocklist.refresh"));
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "\"active\":{\"warnings\":1,\"errors\":0}"));
try testing.expect(!std.mem.containsAtLeast(u8, response.body, 1, "upstream_history.write"));
}
test "W10 milestone 27: a purge takes resolved events only, and says which of the three answers it gave" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{});
defer env.destroy();
try bounded(env.io(), default_budget, diagnosticsPurge, .{ env.io(), env });
}
fn diagnosticsPurgeAll(io: std.Io, env: *Env) anyerror!void {
var body_buf: [8192]u8 = undefined;
var conn: Conn = undefined;
try conn.connect(io, env.addr);
defer conn.close(io);
// A second resolved episode, so the count the sweep reports is a number it
// had to compute rather than the one row the seed leaves.
env.events_store.reportResolved(io, seeded_now, .query_log_recreated, "one-shot", "corrupt", .warning, "aside");
try conn.request("DELETE", "/api/diagnostics", null, null);
var response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), response.status);
try testing.expectEqualStrings("{\"purged\":2}", response.body);
try conn.request("GET", "/api/diagnostics?state=resolved", null, null);
response = try conn.receive(&body_buf);
try testing.expectEqual(@as(u16, 200), response.status);
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "\"events\":[]"));
// And the episode that is still failing is untouched: the operator clearing
// the page cannot lose what is still true.
try conn.request("GET", "/api/diagnostics?state=active", null, null);
response = try conn.receive(&body_buf);
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "blocklist.refresh"));
}
test "W10 milestone 27: purging all resolved events counts them and leaves the active ones" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{});
defer env.destroy();
try bounded(env.io(), default_budget, diagnosticsPurgeAll, .{ env.io(), env });
}
test "W10 milestone 27: every diagnostics filter names itself in a 400, and an unknown id is a 404" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var env = try Env.create(gpa, .{});
defer env.destroy();
try bounded(env.io(), default_budget, diagnosticsRejections, .{ env.io(), env });
}
test "W10 milestone 20: file authority rejects configuration writes and spares the rest" {
@@ -2197,6 +2387,15 @@ const contract_sample_walk = [_]ContractSample{
.{ .name = "login", .ts_type = "LoginResponse", .method = "POST", .target = "/api/auth/login", .body = "{\"password\":\"\"}", .status = 200 },
.{ .name = "logout", .ts_type = "LogoutResponse", .method = "POST", .target = "/api/auth/logout", .body = "{}", .status = 200 },
// Diagnostics, ahead of every write below: the seeded store holds one
// active episode (id 1) and one resolved one, and a later pass that
// reported an event of its own would move the page under the golden.
.{ .name = "get_diagnostics", .ts_type = "DiagnosticsPage", .method = "GET", .target = "/api/diagnostics?limit=10", .status = 200 },
.{ .name = "get_diagnostic", .ts_type = "DiagnosticEvent", .method = "GET", .target = "/api/diagnostics/1", .status = 200 },
// The sweep runs after both reads and takes the seeded resolved episode;
// the per-id purge answers 204, which has no body to sample.
.{ .name = "purge_diagnostics", .ts_type = "DiagnosticsPurge", .method = "DELETE", .target = "/api/diagnostics", .status = 200 },
// Blocklists. The row is created disabled so the refresh below has a status
// to report and still downloads nothing.
.{ .name = "create_blocklist", .ts_type = "BlocklistEcho", .method = "POST", .target = "/api/blocklists", .body = "{\"url\":\"https://lists.example/ads.txt\",\"name\":\"ads\",\"enabled\":false}", .status = 201 },