metrics: peak gauges for in-flight queries, stream connections and the query log queue

src/server/doh_server.zig

@@ -148,6 +148,10 @@ pub const DohServer = struct {
         return self.core.boundAddress();
     }
 
+    pub fn connectionsPeak(self: *const DohServer) u64 {
+        return self.core.connectionsPeak();
+    }
+
     pub fn snapshotStats(self: *const DohServer) Snapshot {
         const core = &self.core.stats;
         return .{

src/server/dot_server.zig

@@ -125,6 +125,10 @@ pub const DotServer = struct {
         self.* = undefined;
     }
 
+    pub fn connectionsPeak(self: *const DotServer) u64 {
+        return self.core.connectionsPeak();
+    }
+
     pub fn snapshotStats(self: *const DotServer) StatsSnapshot {
         const core = &self.core.stats;
         return .{

src/server/handler.zig

@@ -130,6 +130,11 @@ pub const Handler = struct {
     pause: ?*pause.Pause = null,
     tracker: ?*clients.Tracker = null,
     stats: Stats = .{},
+    /// Queries inside `handle` right now, across every listener, and the most
+    /// there have been at once (specs/memory-budget.md item 0). Gauges, so they
+    /// stay out of `Stats`, which `/metrics` exports field by field as counters.
+    in_flight: std.atomic.Value(u64) = .init(0),
+    in_flight_peak: std.atomic.Value(u64) = .init(0),
 
     pub const Stats = struct {
         /// Queries answered by the upstream pool.
@@ -229,6 +234,21 @@ pub const Handler = struct {
         query: []const u8,
         response_buf: []u8,
         scratch: *Scratch,
+    ) Outcome {
+        const now_in_flight = self.in_flight.fetchAdd(1, .monotonic) + 1;
+        _ = self.in_flight_peak.fetchMax(now_in_flight, .monotonic);
+        defer _ = self.in_flight.fetchSub(1, .monotonic);
+        return self.answer(io, which, from, query, response_buf, scratch);
+    }
+
+    fn answer(
+        self: *Handler,
+        io: std.Io,
+        which: Transport,
+        from: address.NetAddress,
+        query: []const u8,
+        response_buf: []u8,
+        scratch: *Scratch,
     ) Outcome {
         std.debug.assert(response_buf.len >= udp_limit_min);
         std.debug.assert(response_buf.len <= transport.max_message_len);
@@ -1459,6 +1479,24 @@ test "a response arriving on the listener port is dropped" {
     try testing.expectEqual(@as(usize, 0), fake.calls);
 }
 
+test "every query counts in flight for exactly its own handling" {
+    var t: TestIo = .init();
+    defer t.deinit();
+
+    var fake: FakeUpstream = .{ .reply = response_bytes };
+    var h_owner: upstream_owner.Borrowed = .{};
+    var h = bare(h_owner.client(fake.client()));
+    // Two queries other listeners are still handling.
+    h.in_flight.store(2, .monotonic);
+
+    var buf: [udp_limit_min]u8 = undefined;
+    try testing.expect(udp(&h, t.io(), query_bytes, &buf) == .reply);
+    try testing.expectEqual(Handler.Outcome.drop, udp(&h, t.io(), response_bytes, &buf));
+
+    try testing.expectEqual(@as(u64, 2), h.in_flight.load(.monotonic));
+    try testing.expectEqual(@as(u64, 3), h.in_flight_peak.load(.monotonic));
+}
+
 test "a query shorter than a header is dropped" {
     var t: TestIo = .init();
     defer t.deinit();

src/server/listener.zig

@@ -83,6 +83,10 @@ pub const CoreStats = struct {
     accept_errors: std.atomic.Value(u64) = .init(0),
     connection_errors: std.atomic.Value(u64) = .init(0),
     idle_timeouts: std.atomic.Value(u64) = .init(0),
+    /// The most slots that have been taken at once (specs/memory-budget.md
+    /// item 0). A gauge, so no listener's `Snapshot` carries it: those export
+    /// every field as a counter. Written only under `Core.mutex`.
+    connections_peak: std.atomic.Value(u64) = .init(0),
 };
 
 // ---------------------------------------------------------------------------
@@ -255,6 +259,9 @@ pub fn Core(comptime Cfg: type) type {
         /// that shuts the active connections down, so a stream that arrives
         /// after that scan can never claim a slot the scan will not visit again.
         shutdown_begun: bool,
+        /// Guarded by `mutex`. Slots that are not `.free`: a slot is taken from
+        /// `claim` until `finish` frees it, closing included.
+        taken: u64,
         stats: CoreStats,
         run_state: std.atomic.Value(State),
         stopped: std.Io.Event,
@@ -299,6 +306,7 @@ pub fn Core(comptime Cfg: type) type {
                 .conns = conns,
                 .mutex = .init,
                 .shutdown_begun = false,
+                .taken = 0,
                 .stats = .{},
                 .run_state = .init(.idle),
                 .stopped = .unset,
@@ -336,6 +344,10 @@ pub fn Core(comptime Cfg: type) type {
             return self.listener.socket.address;
         }
 
+        pub fn connectionsPeak(self: *const Self) u64 {
+            return self.stats.connections_peak.load(.monotonic);
+        }
+
         /// Accept loop. Returns when the task is canceled or `deinit` stops it.
         pub fn serve(self: *Self, io: std.Io) void {
             if (self.run_state.cmpxchgStrong(.idle, .serving, .acq_rel, .acquire) != null) return;
@@ -443,6 +455,10 @@ pub fn Core(comptime Cfg: type) type {
                     self.conns[index].stream = stream;
                     self.conns[index].peer = stream.socket.address;
                     self.conns[index].state = .active;
+                    self.taken += 1;
+                    if (self.taken > self.stats.connections_peak.load(.monotonic)) {
+                        self.stats.connections_peak.store(self.taken, .monotonic);
+                    }
                 },
                 .at_capacity, .shutting_down => {},
             }
@@ -464,6 +480,7 @@ pub fn Core(comptime Cfg: type) type {
 
             self.mutex.lockUncancelable(io);
             conn.state = .free;
+            self.taken -= 1;
             self.mutex.unlock(io);
         }
 

src/server/tcp_server.zig

@@ -99,6 +99,10 @@ pub const TcpServer = struct {
         return self.core.boundAddress();
     }
 
+    pub fn connectionsPeak(self: *const TcpServer) u64 {
+        return self.core.connectionsPeak();
+    }
+
     /// The counters, read one at a time. A scrape that lands mid-accept can see
     /// a connection counted before its outcome is; a lock would buy a
     /// consistency no consumer needs.

src/server/tcp_server_integration_test.zig

@@ -329,6 +329,52 @@ test "a canceled serve does not wait for a live connection" {
     try client_result;
 }
 
+test "the connections peak is the most slots held at once" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    var threaded: std.Io.Threaded = .init(gpa, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var fake: FakeUpstream = .{ .reply = response_bytes };
+    var h_owner: upstream_owner.Borrowed = .{};
+    var h = bareHandler(h_owner.client(fake.client()));
+
+    const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
+    var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{
+        .max_connections = 3,
+        .idle_timeout = .{ .raw = .fromSeconds(600), .clock = .awake },
+    });
+    const server_address = server.boundAddress();
+
+    var serving: std.Io.Group = .init;
+    try serving.concurrent(io, tcp_server.TcpServer.serve, .{ &server, io });
+
+    var answered: [2]std.Io.Event = @splat(.unset);
+    var client_results: [2]anyerror!void = @splat({});
+    var clients_group: std.Io.Group = .init;
+    for (&answered, &client_results) |*event, *result| {
+        try clients_group.concurrent(io, holdConnection, .{ io, server_address, event, result });
+    }
+    for (&answered) |*event| event.waitUncancelable(io);
+
+    // Both replies prove both slots are claimed, and the listener has room for
+    // a third that never arrives.
+    const held_open = server.connectionsPeak();
+
+    server.deinit(io);
+    serving.await(io) catch |err| switch (err) {
+        error.Canceled => unreachable,
+    };
+    clients_group.await(io) catch |err| switch (err) {
+        error.Canceled => unreachable,
+    };
+
+    for (client_results) |result| try result;
+    try testing.expectEqual(@as(u64, 2), held_open);
+}
+
 /// The first slot the server would hand out, read after `serve` has returned so
 /// nothing can be writing it.
 fn firstFreeSlot(server: *const tcp_server.TcpServer) ?usize {