metrics: peak gauges for in-flight queries, stream connections and the query log queue

src/web/metrics.zig

@@ -130,6 +130,20 @@ pub const DiagnosticsSample = struct {
     write_failures: u64,
 };
 
+/// The most of each kind of concurrent work there has been at once since start
+/// (specs/memory-budget.md item 0): what the admission budget, the listener
+/// caps and the query-log queue are sized from. Gauges, so none of them rides
+/// in a counter group. A null field is a collaborator the state does not have.
+pub const PeakSample = struct {
+    dns_in_flight: ?u64 = null,
+    querylog_queue: ?u64 = null,
+    /// The larger of the two address families' peaks. Each family has its own
+    /// slot pool, so a cap is sized per listener, never from their sum.
+    tcp_connections: ?u64 = null,
+    dot_connections: ?u64 = null,
+    doh_connections: ?u64 = null,
+};
+
 /// Everything one scrape reports. A null section is a collaborator the state
 /// does not have.
 pub const Sample = struct {
@@ -172,6 +186,7 @@ pub const Sample = struct {
     /// label and cannot live in `UpstreamSample`. Absent while no pool is
     /// wired, like every other collaborator.
     upstream_budget_exhausted_total: ?u64 = null,
+    peaks: PeakSample = .{},
 };
 
 pub fn handle(
@@ -193,6 +208,7 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
 
     if (state.handler) |handler| {
         sample.dns = dnsCounters(&handler.stats);
+        sample.peaks.dns_in_flight = handler.in_flight_peak.load(.monotonic);
 
         // The pointer is loaded under the same mutex a query loads it under,
         // never before it: a live `cache.size` or `dns.rate_limit` change
@@ -223,6 +239,7 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
             .rows_written = reading.rows_written,
             .batches_gated = reading.batches_gated,
         };
+        sample.peaks.querylog_queue = reading.queue_peak;
     }
 
     if (state.tracker) |tracker| sample.tracker = .{
@@ -270,9 +287,12 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
         };
     }
     if (state.dot_listener) |listener| sample.dot_listener = listener.snapshotStats();
+    if (state.doh_listener) |listener| sample.peaks.doh_connections = listener.connectionsPeak();
+    if (state.dot_listener) |listener| sample.peaks.dot_connections = listener.connectionsPeak();
 
     sample.udp_listener = sumListeners(udp_server.Snapshot, udp_server.UdpServer, state.udp_listeners);
     sample.tcp_listener = sumListeners(tcp_server.Snapshot, tcp_server.TcpServer, state.tcp_listeners);
+    sample.peaks.tcp_connections = largestConnectionsPeak(state.tcp_listeners);
 
     if (state.upstreams) |owner| {
         const generation = owner.acquire(io);
@@ -308,6 +328,13 @@ fn sumListeners(comptime Snapshot: type, comptime Server: type, listeners: []con
     return total;
 }
 
+/// Null for an empty slice, like `sumListeners`.
+fn largestConnectionsPeak(listeners: []const *tcp_server.TcpServer) ?u64 {
+    var largest: ?u64 = null;
+    for (listeners) |listener| largest = @max(largest orelse 0, listener.connectionsPeak());
+    return largest;
+}
+
 fn dnsCounters(stats: *const dns_handler.Handler.Stats) DnsCounters {
     var out: DnsCounters = undefined;
     inline for (dns_stat_fields, 0..) |field, i| {
@@ -471,6 +498,8 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
         try counterGroup(w, "nxdns_dot_server_", "DoT listener counter", listener);
     }
 
+    try renderPeaks(w, sample.peaks);
+
     if (sample.doh_certs != null or sample.dot_certs != null) try renderCerts(w, sample);
 
     if (sample.upstreams.len != 0) try renderUpstreams(w, sample.upstreams);
@@ -485,6 +514,39 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
     }
 }
 
+fn renderPeaks(w: *std.Io.Writer, peaks: PeakSample) std.Io.Writer.Error!void {
+    if (peaks.dns_in_flight) |value| try gauge(
+        w,
+        "nxdns_dns_in_flight_peak",
+        "Most DNS queries handled at once since start, across every listener.",
+        value,
+    );
+    if (peaks.querylog_queue) |value| try gauge(
+        w,
+        "nxdns_querylog_queue_peak",
+        "Upper bound on the most entries the query log queue has held at once since start.",
+        value,
+    );
+    if (peaks.tcp_connections) |value| try gauge(
+        w,
+        "nxdns_tcp_server_connections_peak",
+        "Most connections one TCP/53 listener has held at once since start.",
+        value,
+    );
+    if (peaks.dot_connections) |value| try gauge(
+        w,
+        "nxdns_dot_server_connections_peak",
+        "Most connections the DoT listener has held at once since start.",
+        value,
+    );
+    if (peaks.doh_connections) |value| try gauge(
+        w,
+        "nxdns_doh_server_connections_peak",
+        "Most connections the DoH listener has held at once since start.",
+        value,
+    );
+}
+
 fn renderCerts(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
     try labeledHead(w, "nxdns_cert_reloads_total", "Certificate reloads that published a new context.", "counter");
     if (sample.doh_certs) |stats| try endpointValue(w, "nxdns_cert_reloads_total", "doh", stats.reloads);
@@ -979,6 +1041,35 @@ test "an unwired collaborator omits its family rather than reporting zeros" {
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_dot_server_"));
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_udp_server_"));
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_tcp_server_"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "_peak"));
+}
+
+test "each peak is a gauge, and an unwired one is absent" {
+    const text = try renderToString(testing.allocator, .{ .peaks = .{
+        .dns_in_flight = 7,
+        .querylog_queue = 120,
+        .tcp_connections = 3,
+    } });
+    defer testing.allocator.free(text);
+
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_dns_in_flight_peak gauge\nnxdns_dns_in_flight_peak 7\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_querylog_queue_peak gauge\nnxdns_querylog_queue_peak 120\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_tcp_server_connections_peak gauge\nnxdns_tcp_server_connections_peak 3\n"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_dot_server_connections_peak"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_doh_server_connections_peak"));
+}
+
+test "the TCP peak is the larger listener's, never the sum" {
+    var tcp6: tcp_server.TcpServer = undefined;
+    tcp6.core.stats = .{};
+    tcp6.core.stats.connections_peak.store(4, .monotonic);
+
+    var tcp4: tcp_server.TcpServer = undefined;
+    tcp4.core.stats = .{};
+    tcp4.core.stats.connections_peak.store(6, .monotonic);
+
+    try testing.expectEqual(@as(?u64, 6), largestConnectionsPeak(&.{ &tcp6, &tcp4 }));
+    try testing.expectEqual(@as(?u64, null), largestConnectionsPeak(&.{}));
 }
 
 test "the plain-DNS listener families carry every counter of both listeners" {