metrics: peak gauges for in-flight queries, stream connections and the query log queue

src/server/handler.zig

@@ -130,6 +130,11 @@ pub const Handler = struct {
     pause: ?*pause.Pause = null,
     tracker: ?*clients.Tracker = null,
     stats: Stats = .{},
+    /// Queries inside `handle` right now, across every listener, and the most
+    /// there have been at once (specs/memory-budget.md item 0). Gauges, so they
+    /// stay out of `Stats`, which `/metrics` exports field by field as counters.
+    in_flight: std.atomic.Value(u64) = .init(0),
+    in_flight_peak: std.atomic.Value(u64) = .init(0),
 
     pub const Stats = struct {
         /// Queries answered by the upstream pool.
@@ -229,6 +234,21 @@ pub const Handler = struct {
         query: []const u8,
         response_buf: []u8,
         scratch: *Scratch,
+    ) Outcome {
+        const now_in_flight = self.in_flight.fetchAdd(1, .monotonic) + 1;
+        _ = self.in_flight_peak.fetchMax(now_in_flight, .monotonic);
+        defer _ = self.in_flight.fetchSub(1, .monotonic);
+        return self.answer(io, which, from, query, response_buf, scratch);
+    }
+
+    fn answer(
+        self: *Handler,
+        io: std.Io,
+        which: Transport,
+        from: address.NetAddress,
+        query: []const u8,
+        response_buf: []u8,
+        scratch: *Scratch,
     ) Outcome {
         std.debug.assert(response_buf.len >= udp_limit_min);
         std.debug.assert(response_buf.len <= transport.max_message_len);
@@ -1459,6 +1479,24 @@ test "a response arriving on the listener port is dropped" {
     try testing.expectEqual(@as(usize, 0), fake.calls);
 }
 
+test "every query counts in flight for exactly its own handling" {
+    var t: TestIo = .init();
+    defer t.deinit();
+
+    var fake: FakeUpstream = .{ .reply = response_bytes };
+    var h_owner: upstream_owner.Borrowed = .{};
+    var h = bare(h_owner.client(fake.client()));
+    // Two queries other listeners are still handling.
+    h.in_flight.store(2, .monotonic);
+
+    var buf: [udp_limit_min]u8 = undefined;
+    try testing.expect(udp(&h, t.io(), query_bytes, &buf) == .reply);
+    try testing.expectEqual(Handler.Outcome.drop, udp(&h, t.io(), response_bytes, &buf));
+
+    try testing.expectEqual(@as(u64, 2), h.in_flight.load(.monotonic));
+    try testing.expectEqual(@as(u64, 3), h.in_flight_peak.load(.monotonic));
+}
+
 test "a query shorter than a header is dropped" {
     var t: TestIo = .init();
     defer t.deinit();