metrics: peak gauges for in-flight queries, stream connections and the query log queue

src/server/doh_server.zig

@@ -148,6 +148,10 @@ pub const DohServer = struct {
         return self.core.boundAddress();
     }
 
+    pub fn connectionsPeak(self: *const DohServer) u64 {
+        return self.core.connectionsPeak();
+    }
+
     pub fn snapshotStats(self: *const DohServer) Snapshot {
         const core = &self.core.stats;
         return .{

src/server/dot_server.zig

@@ -125,6 +125,10 @@ pub const DotServer = struct {
         self.* = undefined;
     }
 
+    pub fn connectionsPeak(self: *const DotServer) u64 {
+        return self.core.connectionsPeak();
+    }
+
     pub fn snapshotStats(self: *const DotServer) StatsSnapshot {
         const core = &self.core.stats;
         return .{

src/server/handler.zig

@@ -130,6 +130,11 @@ pub const Handler = struct {
     pause: ?*pause.Pause = null,
     tracker: ?*clients.Tracker = null,
     stats: Stats = .{},
+    /// Queries inside `handle` right now, across every listener, and the most
+    /// there have been at once (specs/memory-budget.md item 0). Gauges, so they
+    /// stay out of `Stats`, which `/metrics` exports field by field as counters.
+    in_flight: std.atomic.Value(u64) = .init(0),
+    in_flight_peak: std.atomic.Value(u64) = .init(0),
 
     pub const Stats = struct {
         /// Queries answered by the upstream pool.
@@ -229,6 +234,21 @@ pub const Handler = struct {
         query: []const u8,
         response_buf: []u8,
         scratch: *Scratch,
+    ) Outcome {
+        const now_in_flight = self.in_flight.fetchAdd(1, .monotonic) + 1;
+        _ = self.in_flight_peak.fetchMax(now_in_flight, .monotonic);
+        defer _ = self.in_flight.fetchSub(1, .monotonic);
+        return self.answer(io, which, from, query, response_buf, scratch);
+    }
+
+    fn answer(
+        self: *Handler,
+        io: std.Io,
+        which: Transport,
+        from: address.NetAddress,
+        query: []const u8,
+        response_buf: []u8,
+        scratch: *Scratch,
     ) Outcome {
         std.debug.assert(response_buf.len >= udp_limit_min);
         std.debug.assert(response_buf.len <= transport.max_message_len);
@@ -1459,6 +1479,24 @@ test "a response arriving on the listener port is dropped" {
     try testing.expectEqual(@as(usize, 0), fake.calls);
 }
 
+test "every query counts in flight for exactly its own handling" {
+    var t: TestIo = .init();
+    defer t.deinit();
+
+    var fake: FakeUpstream = .{ .reply = response_bytes };
+    var h_owner: upstream_owner.Borrowed = .{};
+    var h = bare(h_owner.client(fake.client()));
+    // Two queries other listeners are still handling.
+    h.in_flight.store(2, .monotonic);
+
+    var buf: [udp_limit_min]u8 = undefined;
+    try testing.expect(udp(&h, t.io(), query_bytes, &buf) == .reply);
+    try testing.expectEqual(Handler.Outcome.drop, udp(&h, t.io(), response_bytes, &buf));
+
+    try testing.expectEqual(@as(u64, 2), h.in_flight.load(.monotonic));
+    try testing.expectEqual(@as(u64, 3), h.in_flight_peak.load(.monotonic));
+}
+
 test "a query shorter than a header is dropped" {
     var t: TestIo = .init();
     defer t.deinit();

src/server/listener.zig

@@ -83,6 +83,10 @@ pub const CoreStats = struct {
     accept_errors: std.atomic.Value(u64) = .init(0),
     connection_errors: std.atomic.Value(u64) = .init(0),
     idle_timeouts: std.atomic.Value(u64) = .init(0),
+    /// The most slots that have been taken at once (specs/memory-budget.md
+    /// item 0). A gauge, so no listener's `Snapshot` carries it: those export
+    /// every field as a counter. Written only under `Core.mutex`.
+    connections_peak: std.atomic.Value(u64) = .init(0),
 };
 
 // ---------------------------------------------------------------------------
@@ -255,6 +259,9 @@ pub fn Core(comptime Cfg: type) type {
         /// that shuts the active connections down, so a stream that arrives
         /// after that scan can never claim a slot the scan will not visit again.
         shutdown_begun: bool,
+        /// Guarded by `mutex`. Slots that are not `.free`: a slot is taken from
+        /// `claim` until `finish` frees it, closing included.
+        taken: u64,
         stats: CoreStats,
         run_state: std.atomic.Value(State),
         stopped: std.Io.Event,
@@ -299,6 +306,7 @@ pub fn Core(comptime Cfg: type) type {
                 .conns = conns,
                 .mutex = .init,
                 .shutdown_begun = false,
+                .taken = 0,
                 .stats = .{},
                 .run_state = .init(.idle),
                 .stopped = .unset,
@@ -336,6 +344,10 @@ pub fn Core(comptime Cfg: type) type {
             return self.listener.socket.address;
         }
 
+        pub fn connectionsPeak(self: *const Self) u64 {
+            return self.stats.connections_peak.load(.monotonic);
+        }
+
         /// Accept loop. Returns when the task is canceled or `deinit` stops it.
         pub fn serve(self: *Self, io: std.Io) void {
             if (self.run_state.cmpxchgStrong(.idle, .serving, .acq_rel, .acquire) != null) return;
@@ -443,6 +455,10 @@ pub fn Core(comptime Cfg: type) type {
                     self.conns[index].stream = stream;
                     self.conns[index].peer = stream.socket.address;
                     self.conns[index].state = .active;
+                    self.taken += 1;
+                    if (self.taken > self.stats.connections_peak.load(.monotonic)) {
+                        self.stats.connections_peak.store(self.taken, .monotonic);
+                    }
                 },
                 .at_capacity, .shutting_down => {},
             }
@@ -464,6 +480,7 @@ pub fn Core(comptime Cfg: type) type {
 
             self.mutex.lockUncancelable(io);
             conn.state = .free;
+            self.taken -= 1;
             self.mutex.unlock(io);
         }
 

src/server/tcp_server.zig

@@ -99,6 +99,10 @@ pub const TcpServer = struct {
         return self.core.boundAddress();
     }
 
+    pub fn connectionsPeak(self: *const TcpServer) u64 {
+        return self.core.connectionsPeak();
+    }
+
     /// The counters, read one at a time. A scrape that lands mid-accept can see
     /// a connection counted before its outcome is; a lock would buy a
     /// consistency no consumer needs.

src/server/tcp_server_integration_test.zig

@@ -329,6 +329,52 @@ test "a canceled serve does not wait for a live connection" {
     try client_result;
 }
 
+test "the connections peak is the most slots held at once" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    var threaded: std.Io.Threaded = .init(gpa, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var fake: FakeUpstream = .{ .reply = response_bytes };
+    var h_owner: upstream_owner.Borrowed = .{};
+    var h = bareHandler(h_owner.client(fake.client()));
+
+    const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
+    var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{
+        .max_connections = 3,
+        .idle_timeout = .{ .raw = .fromSeconds(600), .clock = .awake },
+    });
+    const server_address = server.boundAddress();
+
+    var serving: std.Io.Group = .init;
+    try serving.concurrent(io, tcp_server.TcpServer.serve, .{ &server, io });
+
+    var answered: [2]std.Io.Event = @splat(.unset);
+    var client_results: [2]anyerror!void = @splat({});
+    var clients_group: std.Io.Group = .init;
+    for (&answered, &client_results) |*event, *result| {
+        try clients_group.concurrent(io, holdConnection, .{ io, server_address, event, result });
+    }
+    for (&answered) |*event| event.waitUncancelable(io);
+
+    // Both replies prove both slots are claimed, and the listener has room for
+    // a third that never arrives.
+    const held_open = server.connectionsPeak();
+
+    server.deinit(io);
+    serving.await(io) catch |err| switch (err) {
+        error.Canceled => unreachable,
+    };
+    clients_group.await(io) catch |err| switch (err) {
+        error.Canceled => unreachable,
+    };
+
+    for (client_results) |result| try result;
+    try testing.expectEqual(@as(u64, 2), held_open);
+}
+
 /// The first slot the server would hand out, read after `serve` has returned so
 /// nothing can be writing it.
 fn firstFreeSlot(server: *const tcp_server.TcpServer) ?usize {

src/storage/logger.zig

@@ -320,7 +320,68 @@ fn emptyAsNull(value: []const u8) ?[]const u8 {
     return if (value.len == 0) null else value;
 }
 
-const EntryQueue = std.Io.Queue(Entry);
+/// `std.Io.Queue` plus the occupancy high-water that sizes
+/// `logging.query_log_buffer_max` (specs/memory-budget.md item 0).
+///
+/// The queue keeps its length behind its own mutex with no accessor, so the
+/// count lives beside it. Raised before an insert and lowered after a removal,
+/// it never reads below the true occupancy; it overstates it by at most the
+/// producers inside `tryPut` plus the entries a getter has taken and not yet
+/// subtracted. An upper bound is the safe side for sizing a buffer.
+const EntryQueue = struct {
+    inner: std.Io.Queue(Entry),
+    held: std.atomic.Value(u64) = .init(0),
+    peak: std.atomic.Value(u64) = .init(0),
+
+    pub fn init(buffer: []Entry) EntryQueue {
+        return .{ .inner = .init(buffer) };
+    }
+
+    pub fn capacity(q: *const EntryQueue) usize {
+        return q.inner.capacity();
+    }
+
+    pub fn close(q: *EntryQueue, io: std.Io) void {
+        q.inner.close(io);
+    }
+
+    pub fn peakOccupancy(q: *const EntryQueue) u64 {
+        return q.peak.load(.monotonic);
+    }
+
+    /// Takes `entry` if there is room or a waiting getter, without blocking.
+    pub fn tryPut(q: *EntryQueue, io: std.Io, entry: Entry) (std.Io.QueueClosedError || std.Io.Cancelable)!bool {
+        const held = q.held.fetchAdd(1, .monotonic) + 1;
+        const put = q.inner.put(io, &.{entry}, 0) catch |err| {
+            _ = q.held.fetchSub(1, .monotonic);
+            return err;
+        };
+        if (put == 0) {
+            _ = q.held.fetchSub(1, .monotonic);
+            return false;
+        }
+        _ = q.peak.fetchMax(held, .monotonic);
+        return true;
+    }
+
+    pub fn get(q: *EntryQueue, io: std.Io, buffer: []Entry, min: usize) (std.Io.QueueClosedError || std.Io.Cancelable)!usize {
+        const n = try q.inner.get(io, buffer, min);
+        _ = q.held.fetchSub(n, .monotonic);
+        return n;
+    }
+
+    pub fn getUncancelable(q: *EntryQueue, io: std.Io, buffer: []Entry, min: usize) std.Io.QueueClosedError!usize {
+        const n = try q.inner.getUncancelable(io, buffer, min);
+        _ = q.held.fetchSub(n, .monotonic);
+        return n;
+    }
+
+    pub fn getOne(q: *EntryQueue, io: std.Io) (std.Io.QueueClosedError || std.Io.Cancelable)!Entry {
+        const entry = try q.inner.getOne(io);
+        _ = q.held.fetchSub(1, .monotonic);
+        return entry;
+    }
+};
 
 /// A database failure is not in here: a batch the database refuses is dropped,
 /// counted and reported where it happens, and the writer carries on.
@@ -525,8 +586,7 @@ pub const Logger = struct {
         while (true) {
             // A closed queue or a canceled task means shutdown is underway;
             // both leave this entry unwritten, which is what the counter says.
-            const put = self.queue.put(io, &.{entry}, 0) catch break;
-            if (put == 1) return;
+            if (self.queue.tryPut(io, entry) catch break) return;
 
             // A zero-capacity queue holds nothing to drop: the put above was
             // this entry's one chance at a waiting getter.
@@ -1442,6 +1502,34 @@ test "a zero-capacity queue drops every entry exactly once" {
     try testing.expectEqual(@as(u64, 5), logger.queries_dropped.load(.monotonic));
 }
 
+test "the queue peak is the most entries held at once, never the arrivals" {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var buf: [4]Entry = undefined;
+    var logger: Logger = .init(.{}, &buf);
+    try testing.expectEqual(@as(u64, 0), logger.queue.peakOccupancy());
+
+    logger.log(io, sampleEntry(1, "one.example"));
+    logger.log(io, sampleEntry(2, "two.example"));
+    logger.log(io, sampleEntry(3, "three.example"));
+    _ = try logger.queue.getOne(io);
+    _ = try logger.queue.getOne(io);
+    logger.log(io, sampleEntry(4, "four.example"));
+    try testing.expectEqual(@as(u64, 3), logger.queue.peakOccupancy());
+
+    // A full queue evicts to make room, so the peak stops at the capacity.
+    for (5..10) |i| logger.log(io, sampleEntry(@intCast(i), "more.example"));
+    try testing.expectEqual(@as(u64, 4), logger.queue.peakOccupancy());
+    try testing.expectEqual(@as(u64, 4), logger.queue.held.load(.monotonic));
+
+    var drained: [4]Entry = undefined;
+    try testing.expectEqual(@as(usize, 4), try logger.queue.get(io, &drained, 0));
+    try testing.expectEqual(@as(u64, 0), logger.queue.held.load(.monotonic));
+    try testing.expectEqual(@as(u64, 4), logger.queue.peakOccupancy());
+}
+
 test "log after shutdown drops instead of blocking" {
     var threaded: std.Io.Threaded = .init(testing.allocator, .{});
     defer threaded.deinit();

src/storage/logger_controller.zig

@@ -63,12 +63,14 @@ const Totals = struct {
     rows_written: u64 = 0,
     batches_gated: u64 = 0,
     last_drop_s: i64 = 0,
+    queue_peak: u64 = 0,
 
     fn fold(self: *Totals, lg: *const logger.Logger) void {
         self.queries_dropped += lg.queries_dropped.load(.monotonic);
         self.rows_written += lg.rows_written.load(.monotonic);
         self.batches_gated += lg.batches_gated.load(.monotonic);
         self.last_drop_s = @max(self.last_drop_s, lg.last_drop_s.load(.monotonic));
+        self.queue_peak = @max(self.queue_peak, lg.queue.peakOccupancy());
     }
 };
 
@@ -80,6 +82,8 @@ pub const Sample = struct {
     batches_gated: u64,
     /// Null while nothing has been dropped, exactly as `Logger.lastDropSeconds`.
     last_drop_s: ?i64,
+    /// The most entries any generation's queue has held at once.
+    queue_peak: u64,
     /// The LIVE writer's state. A retired writer's failures are in the counters.
     writer_failed: bool,
     /// The worst episode any outstanding generation is in. A retired writer
@@ -376,6 +380,7 @@ pub const Controller = struct {
             .rows_written = totals.rows_written,
             .batches_gated = totals.batches_gated,
             .last_drop_s = if (totals.last_drop_s == 0) null else totals.last_drop_s,
+            .queue_peak = totals.queue_peak,
             .writer_failed = self.live.logger.writer_failed.load(.acquire),
             .gate_episode = episode,
         };

src/web/metrics.zig

@@ -130,6 +130,20 @@ pub const DiagnosticsSample = struct {
     write_failures: u64,
 };
 
+/// The most of each kind of concurrent work there has been at once since start
+/// (specs/memory-budget.md item 0): what the admission budget, the listener
+/// caps and the query-log queue are sized from. Gauges, so none of them rides
+/// in a counter group. A null field is a collaborator the state does not have.
+pub const PeakSample = struct {
+    dns_in_flight: ?u64 = null,
+    querylog_queue: ?u64 = null,
+    /// The larger of the two address families' peaks. Each family has its own
+    /// slot pool, so a cap is sized per listener, never from their sum.
+    tcp_connections: ?u64 = null,
+    dot_connections: ?u64 = null,
+    doh_connections: ?u64 = null,
+};
+
 /// Everything one scrape reports. A null section is a collaborator the state
 /// does not have.
 pub const Sample = struct {
@@ -172,6 +186,7 @@ pub const Sample = struct {
     /// label and cannot live in `UpstreamSample`. Absent while no pool is
     /// wired, like every other collaborator.
     upstream_budget_exhausted_total: ?u64 = null,
+    peaks: PeakSample = .{},
 };
 
 pub fn handle(
@@ -193,6 +208,7 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
 
     if (state.handler) |handler| {
         sample.dns = dnsCounters(&handler.stats);
+        sample.peaks.dns_in_flight = handler.in_flight_peak.load(.monotonic);
 
         // The pointer is loaded under the same mutex a query loads it under,
         // never before it: a live `cache.size` or `dns.rate_limit` change
@@ -223,6 +239,7 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
             .rows_written = reading.rows_written,
             .batches_gated = reading.batches_gated,
         };
+        sample.peaks.querylog_queue = reading.queue_peak;
     }
 
     if (state.tracker) |tracker| sample.tracker = .{
@@ -270,9 +287,12 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
         };
     }
     if (state.dot_listener) |listener| sample.dot_listener = listener.snapshotStats();
+    if (state.doh_listener) |listener| sample.peaks.doh_connections = listener.connectionsPeak();
+    if (state.dot_listener) |listener| sample.peaks.dot_connections = listener.connectionsPeak();
 
     sample.udp_listener = sumListeners(udp_server.Snapshot, udp_server.UdpServer, state.udp_listeners);
     sample.tcp_listener = sumListeners(tcp_server.Snapshot, tcp_server.TcpServer, state.tcp_listeners);
+    sample.peaks.tcp_connections = largestConnectionsPeak(state.tcp_listeners);
 
     if (state.upstreams) |owner| {
         const generation = owner.acquire(io);
@@ -308,6 +328,13 @@ fn sumListeners(comptime Snapshot: type, comptime Server: type, listeners: []con
     return total;
 }
 
+/// Null for an empty slice, like `sumListeners`.
+fn largestConnectionsPeak(listeners: []const *tcp_server.TcpServer) ?u64 {
+    var largest: ?u64 = null;
+    for (listeners) |listener| largest = @max(largest orelse 0, listener.connectionsPeak());
+    return largest;
+}
+
 fn dnsCounters(stats: *const dns_handler.Handler.Stats) DnsCounters {
     var out: DnsCounters = undefined;
     inline for (dns_stat_fields, 0..) |field, i| {
@@ -471,6 +498,8 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
         try counterGroup(w, "nxdns_dot_server_", "DoT listener counter", listener);
     }
 
+    try renderPeaks(w, sample.peaks);
+
     if (sample.doh_certs != null or sample.dot_certs != null) try renderCerts(w, sample);
 
     if (sample.upstreams.len != 0) try renderUpstreams(w, sample.upstreams);
@@ -485,6 +514,39 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
     }
 }
 
+fn renderPeaks(w: *std.Io.Writer, peaks: PeakSample) std.Io.Writer.Error!void {
+    if (peaks.dns_in_flight) |value| try gauge(
+        w,
+        "nxdns_dns_in_flight_peak",
+        "Most DNS queries handled at once since start, across every listener.",
+        value,
+    );
+    if (peaks.querylog_queue) |value| try gauge(
+        w,
+        "nxdns_querylog_queue_peak",
+        "Upper bound on the most entries the query log queue has held at once since start.",
+        value,
+    );
+    if (peaks.tcp_connections) |value| try gauge(
+        w,
+        "nxdns_tcp_server_connections_peak",
+        "Most connections one TCP/53 listener has held at once since start.",
+        value,
+    );
+    if (peaks.dot_connections) |value| try gauge(
+        w,
+        "nxdns_dot_server_connections_peak",
+        "Most connections the DoT listener has held at once since start.",
+        value,
+    );
+    if (peaks.doh_connections) |value| try gauge(
+        w,
+        "nxdns_doh_server_connections_peak",
+        "Most connections the DoH listener has held at once since start.",
+        value,
+    );
+}
+
 fn renderCerts(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
     try labeledHead(w, "nxdns_cert_reloads_total", "Certificate reloads that published a new context.", "counter");
     if (sample.doh_certs) |stats| try endpointValue(w, "nxdns_cert_reloads_total", "doh", stats.reloads);
@@ -979,6 +1041,35 @@ test "an unwired collaborator omits its family rather than reporting zeros" {
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_dot_server_"));
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_udp_server_"));
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_tcp_server_"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "_peak"));
+}
+
+test "each peak is a gauge, and an unwired one is absent" {
+    const text = try renderToString(testing.allocator, .{ .peaks = .{
+        .dns_in_flight = 7,
+        .querylog_queue = 120,
+        .tcp_connections = 3,
+    } });
+    defer testing.allocator.free(text);
+
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_dns_in_flight_peak gauge\nnxdns_dns_in_flight_peak 7\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_querylog_queue_peak gauge\nnxdns_querylog_queue_peak 120\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_tcp_server_connections_peak gauge\nnxdns_tcp_server_connections_peak 3\n"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_dot_server_connections_peak"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_doh_server_connections_peak"));
+}
+
+test "the TCP peak is the larger listener's, never the sum" {
+    var tcp6: tcp_server.TcpServer = undefined;
+    tcp6.core.stats = .{};
+    tcp6.core.stats.connections_peak.store(4, .monotonic);
+
+    var tcp4: tcp_server.TcpServer = undefined;
+    tcp4.core.stats = .{};
+    tcp4.core.stats.connections_peak.store(6, .monotonic);
+
+    try testing.expectEqual(@as(?u64, 6), largestConnectionsPeak(&.{ &tcp6, &tcp4 }));
+    try testing.expectEqual(@as(?u64, null), largestConnectionsPeak(&.{}));
 }
 
 test "the plain-DNS listener families carry every counter of both listeners" {