milestone 16: behavioral fixes for silent failures, locks, counters and the query log

src/app.zig

@@ -534,6 +534,35 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
     };
     defer if (tcp4) |*s| s.deinit(gpa, io);
 
+    // Ruling 13: `/metrics` sums each transport's listeners into one family, so
+    // the web state carries pointers to whichever of the four came up. The
+    // arrays are declared here rather than beside `web_state` because a
+    // listener that failed to bind is not in them; `group.cancel` below runs
+    // before this frame is released, so no web task can outlive them.
+    var udp_listeners: [2]*udp_server.UdpServer = undefined;
+    var udp_count: usize = 0;
+    if (udp6) |*s| {
+        udp_listeners[udp_count] = s;
+        udp_count += 1;
+    }
+    if (udp4) |*s| {
+        udp_listeners[udp_count] = s;
+        udp_count += 1;
+    }
+    web_state.udp_listeners = udp_listeners[0..udp_count];
+
+    var tcp_listeners: [2]*tcp_server.TcpServer = undefined;
+    var tcp_count: usize = 0;
+    if (tcp6) |*s| {
+        tcp_listeners[tcp_count] = s;
+        tcp_count += 1;
+    }
+    if (tcp4) |*s| {
+        tcp_listeners[tcp_count] = s;
+        tcp_count += 1;
+    }
+    web_state.tcp_listeners = tcp_listeners[0..tcp_count];
+
     // -----------------------------------------------------------------------
     // run
     // -----------------------------------------------------------------------
@@ -558,11 +587,11 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
 
     const gate: ?*disk_monitor.Monitor = &monitor;
     try group.concurrent(io, logger_mod.Logger.runWriter, .{ &query_logger, io, &querylog_writer_db, gate });
-    try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db });
+    try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db, gate });
     try group.concurrent(io, disk_monitor.Monitor.run, .{ &monitor, io });
     try group.concurrent(io, manager_mod.Manager.runScheduler, .{ &manager, io });
     try group.concurrent(io, clients.Tracker.run, .{ &tracker, io, &tracker_db, gate });
-    try group.concurrent(io, runMaintenance, .{ &h, io });
+    try group.concurrent(io, runMaintenance, .{ &h, if (web_limiter) |*l| l else null, io });
 
     // Started last (ruling 26), canceled by the same `group.cancel`; its inner
     // connection group is canceled, not awaited (ruling 4), so an idle
@@ -699,37 +728,57 @@ fn bindDot(
 // background maintenance
 // ---------------------------------------------------------------------------
 
-/// Sweeps the cache and the rate-limiter table. Both are guarded by mutexes the
-/// handler owns, because the handler is what contends for them; the sweeps live
-/// here because walking a whole table is not work a query should pay for.
+/// Sweeps the cache, the DNS rate-limiter table and the API rate-limiter table.
+/// The first two are guarded by mutexes the handler owns, because the handler is
+/// what contends for them; the sweeps live here because walking a whole table is
+/// not work a query should pay for. The API limiter needs the same schedule for
+/// the same reason: its table holds 4096 addresses, and once it is full every
+/// unknown address pays an eviction scan.
 ///
 /// The locks are taken cancelably: unlike `handle`, this loop has an error
 /// union to carry `error.Canceled` out of, and a shutdown that arrives while
 /// the query path holds a lock should not wait for it.
-fn runMaintenance(h: *handler.Handler, io: std.Io) std.Io.Cancelable!void {
+fn runMaintenance(
+    h: *handler.Handler,
+    api: ?*api_limiter.ApiLimiter,
+    io: std.Io,
+) std.Io.Cancelable!void {
     const interval: std.Io.Clock.Duration = .{
         .raw = .fromSeconds(maintenance_interval_s),
         .clock = .boot,
     };
     while (true) {
         try interval.sleep(io);
-
-        if (h.cache) |cache| {
-            const now_s = std.Io.Clock.real.now(io).toSeconds();
-            try h.cache_mutex.lock(io);
-            _ = cache.sweep(now_s);
-            h.cache_mutex.unlock(io);
-        }
-
-        if (h.limiter) |limiter| {
-            const now = std.Io.Clock.awake.now(io);
-            try h.limiter_mutex.lock(io);
-            _ = limiter.sweep(now);
-            h.limiter_mutex.unlock(io);
-        }
+        try maintenanceOnce(h, api, io);
     }
 }
 
+/// One sweep of each table. Separate from the loop so a test can run a pass
+/// without waiting out `maintenance_interval_s`.
+fn maintenanceOnce(
+    h: *handler.Handler,
+    api: ?*api_limiter.ApiLimiter,
+    io: std.Io,
+) std.Io.Cancelable!void {
+    if (h.cache) |cache| {
+        const now_s = std.Io.Clock.real.now(io).toSeconds();
+        try h.cache_mutex.lock(io);
+        _ = cache.sweep(now_s);
+        h.cache_mutex.unlock(io);
+    }
+
+    if (h.limiter) |limiter| {
+        const now = std.Io.Clock.awake.now(io);
+        try h.limiter_mutex.lock(io);
+        _ = limiter.sweep(now);
+        h.limiter_mutex.unlock(io);
+    }
+
+    // The API limiter takes its own mutex, unlike the two above, which are the
+    // handler's. Nothing here holds a lock across the call.
+    if (api) |limiter| _ = limiter.sweep(io, std.Io.Clock.awake.now(io));
+}
+
 // ---------------------------------------------------------------------------
 // upstreams
 // ---------------------------------------------------------------------------
@@ -1278,3 +1327,42 @@ fn appendBind(w: *Writer, which: []const u8, addr: ?net.IpAddress) void {
     const value = addr orelse return;
     w.print(" {s} {f}", .{ which, value }) catch {};
 }
+
+const test_address = @import("platform/address.zig");
+
+test "one maintenance pass drops the api limiter's stale buckets" {
+    var threaded: std.Io.Threaded = .init(std.testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var limiter = try api_limiter.ApiLimiter.init(std.testing.allocator, .{
+        .rate_per_min = 60,
+        .localhost_exempt = false,
+        .sse_max_per_ip = 3,
+    });
+    defer limiter.deinit();
+
+    // A bucket last touched a full window ago has refilled to capacity, so a
+    // fresh bucket would answer identically and the sweep may drop it. The
+    // pass reads the real `.awake` clock, so the bucket is aged by dating the
+    // request rather than by waiting.
+    const now = std.Io.Clock.awake.now(io);
+    const window_ns = @as(i96, api_limiter.window_seconds) * std.time.ns_per_s;
+    const client: test_address.NetAddress = .{ .ip4 = .{ 192, 168, 1, 10 } };
+    _ = limiter.check(io, .{ .nanoseconds = now.nanoseconds - 2 * window_ns }, client);
+    try std.testing.expectEqual(@as(u32, 1), limiter.trackedClients(io));
+
+    var h: handler.Handler = .{
+        .upstream = .{ .ptr = undefined, .exchangeFn = undefined },
+        .blocking = .{ .mode = .zero, .ttl = 5 },
+        .forward_read_timeout = .{ .raw = .fromMilliseconds(50), .clock = .awake },
+    };
+    try maintenanceOnce(&h, &limiter, io);
+
+    // Before ruling 12 the sweep had no production caller, so the table kept
+    // this bucket until the process restarted.
+    try std.testing.expectEqual(@as(u32, 0), limiter.trackedClients(io));
+
+    // A limiter the app did not build is not a reason for the pass to fail.
+    try maintenanceOnce(&h, null, io);
+}