milestone 31: concurrent upstream exchanges, dot session reuse, queue metrics

src/web/metrics.zig

@@ -113,6 +113,11 @@ pub const UpstreamSample = struct {
     total_successes: u64,
     total_failures: u64,
     success_rate: f32,
+    in_flight: u32,
+    slots: u32,
+    queued_total: u64,
+    queued_seconds_total: f64,
+    reuse_recoveries_total: u64,
 };
 
 /// The diagnostics store, as one scrape sees it. Two gauges and a counter,
@@ -318,6 +323,11 @@ fn upstreams(pool: *pool_mod.Pool, io: std.Io, arena: Allocator) Allocator.Error
             .total_successes = entry.total_successes,
             .total_failures = entry.total_failures,
             .success_rate = entry.success_rate,
+            .in_flight = entry.in_flight,
+            .slots = entry.slots,
+            .queued_total = entry.queued_total,
+            .queued_seconds_total = entry.queued_seconds_total,
+            .reuse_recoveries_total = entry.reuse_recoveries_total,
         };
     }
     return out;
@@ -510,6 +520,47 @@ fn renderUpstreams(w: *std.Io.Writer, list: []const UpstreamSample) std.Io.Write
     for (list, 0..) |entry, i| {
         try labeledValue(w, "nxdns_upstream_failures_total", i, entry.url, entry.total_failures);
     }
+
+    try labeledHead(w, "nxdns_upstream_in_flight", "Exchanges in flight against an upstream.", "gauge");
+    for (list, 0..) |entry, i| {
+        try labeledValue(w, "nxdns_upstream_in_flight", i, entry.url, entry.in_flight);
+    }
+
+    try labeledHead(w, "nxdns_upstream_slots", "Concurrent exchanges an upstream allows.", "gauge");
+    for (list, 0..) |entry, i| {
+        try labeledValue(w, "nxdns_upstream_slots", i, entry.url, entry.slots);
+    }
+
+    try labeledHead(
+        w,
+        "nxdns_upstream_queued_total",
+        "Exchanges that waited for a slot. Approximate; sampled at admission.",
+        "counter",
+    );
+    for (list, 0..) |entry, i| {
+        try labeledValue(w, "nxdns_upstream_queued_total", i, entry.url, entry.queued_total);
+    }
+
+    try labeledHead(
+        w,
+        "nxdns_upstream_queued_seconds_total",
+        "Seconds exchanges spent waiting for a slot. Approximate; sampled at admission.",
+        "counter",
+    );
+    for (list, 0..) |entry, i| {
+        try writeUpstreamLabels(w, "nxdns_upstream_queued_seconds_total", i, entry.url);
+        try w.print(" {d:.4}\n", .{entry.queued_seconds_total});
+    }
+
+    try labeledHead(
+        w,
+        "nxdns_upstream_reuse_recoveries_total",
+        "Stale reused DoT connections recovered by a redial.",
+        "counter",
+    );
+    for (list, 0..) |entry, i| {
+        try labeledValue(w, "nxdns_upstream_reuse_recoveries_total", i, entry.url, entry.reuse_recoveries_total);
+    }
 }
 
 /// Every field of a plain counter struct, under one prefix.
@@ -553,7 +604,7 @@ fn labeledValue(
 }
 
 /// The label set every upstream family shares, up to and including the closing
-/// brace. One definition, because six families have to agree on it exactly:
+/// brace. One definition, because eleven families have to agree on it exactly:
 /// Prometheus identifies a series by its name and its whole label set, so a
 /// family that labelled its samples differently would be a different series.
 ///
@@ -651,6 +702,7 @@ const db = @import("../storage/db.zig");
 const local_tables = @import("../server/local_tables.zig");
 const logger_mod = @import("../storage/logger.zig");
 const migrations = @import("../storage/migrations.zig");
+const transport = @import("../upstream/transport.zig");
 const testing = std.testing;
 
 /// A handler with no upstream reachable: every test here reads counters and
@@ -684,6 +736,11 @@ test "a full sample renders the whole exposition, byte for byte" {
             .total_successes = 9,
             .total_failures = 1,
             .success_rate = 0.9,
+            .in_flight = 2,
+            .slots = 8,
+            .queued_total = 3,
+            .queued_seconds_total = 0.25,
+            .reuse_recoveries_total = 1,
         },
     };
 
@@ -760,10 +817,16 @@ test "a full sample renders the whole exposition, byte for byte" {
         1,
         "nxdns_upstream_success_rate{index=\"0\",url=\"https://dns.example\"} 0.9000\n",
     ));
+    try testing.expect(std.mem.containsAtLeast(
+        u8,
+        text,
+        1,
+        "nxdns_upstream_failures_total{index=\"0\",url=\"https://dns.example\"} 1\n",
+    ));
     try testing.expect(std.mem.endsWith(
         u8,
         text,
-        "nxdns_upstream_failures_total{index=\"0\",url=\"https://dns.example\"} 1\n",
+        "nxdns_upstream_reuse_recoveries_total{index=\"0\",url=\"https://dns.example\"} 1\n",
     ));
 }
 
@@ -1115,12 +1178,18 @@ test "a label value escapes the characters the format reserves" {
             .total_successes = 0,
             .total_failures = 2,
             .success_rate = 0,
+            .in_flight = 1,
+            .slots = 8,
+            .queued_total = 4,
+            .queued_seconds_total = 1.5,
+            .reuse_recoveries_total = 2,
         }};
         const text = try renderToString(testing.allocator, .{ .upstreams = &upstream_list });
         defer testing.allocator.free(text);
 
         // The url wrote no line of its own, and lost none: every line is a
-        // comment or a sample, and the six families contribute six samples.
+        // comment or a sample, and the eleven families contribute eleven
+        // samples.
         var samples: usize = 0;
         var lines = std.mem.splitScalar(u8, text, '\n');
         while (lines.next()) |line| {
@@ -1131,7 +1200,7 @@ test "a label value escapes the characters the format reserves" {
             // Both labels are there, and both values close where they opened.
             try testing.expectEqual(@as(?usize, 2), labelPairs(line));
         }
-        try testing.expectEqual(@as(usize, 6), samples);
+        try testing.expectEqual(@as(usize, 11), samples);
     }
 }
 
@@ -1149,6 +1218,11 @@ test "two upstreams on one host stay two series" {
             .total_successes = 5,
             .total_failures = 0,
             .success_rate = 1,
+            .in_flight = 0,
+            .slots = 8,
+            .queued_total = 0,
+            .queued_seconds_total = 0,
+            .reuse_recoveries_total = 0,
         },
         .{
             .url = "https://dns.nextdns.io/efgh34",
@@ -1158,6 +1232,11 @@ test "two upstreams on one host stay two series" {
             .total_successes = 9,
             .total_failures = 3,
             .success_rate = 0.75,
+            .in_flight = 3,
+            .slots = 8,
+            .queued_total = 7,
+            .queued_seconds_total = 2,
+            .reuse_recoveries_total = 5,
         },
     };
     const text = try renderToString(testing.allocator, .{ .upstreams = &upstream_list });
@@ -1206,7 +1285,7 @@ test "two upstreams on one host stay two series" {
     try testing.expectEqualStrings("0", up_values[1]);
 
     // No two samples in the scrape share a series key, whatever the urls were.
-    var keys: [32][]const u8 = undefined;
+    var keys: [64][]const u8 = undefined;
     var count: usize = 0;
     var lines = std.mem.splitScalar(u8, text, '\n');
     while (lines.next()) |line| {
@@ -1216,7 +1295,7 @@ test "two upstreams on one host stay two series" {
         keys[count] = key;
         count += 1;
     }
-    try testing.expectEqual(@as(usize, 12), count);
+    try testing.expectEqual(@as(usize, 22), count);
 }
 
 test "an upstream url is redacted before it reaches an open endpoint's label" {
@@ -1233,6 +1312,11 @@ test "an upstream url is redacted before it reaches an open endpoint's label" {
             .total_successes = 3,
             .total_failures = 0,
             .success_rate = 1,
+            .in_flight = 0,
+            .slots = 8,
+            .queued_total = 0,
+            .queued_seconds_total = 0,
+            .reuse_recoveries_total = 0,
         },
         .{
             .url = "https://user:hunter2@dns.example:8443/dns-query?apikey=s3cr3t#frag",
@@ -1242,6 +1326,11 @@ test "an upstream url is redacted before it reaches an open endpoint's label" {
             .total_successes = 0,
             .total_failures = 4,
             .success_rate = 0,
+            .in_flight = 0,
+            .slots = 8,
+            .queued_total = 0,
+            .queued_seconds_total = 0,
+            .reuse_recoveries_total = 0,
         },
     };
     const text = try renderToString(testing.allocator, .{ .upstreams = &upstream_list });
@@ -1254,7 +1343,8 @@ test "an upstream url is redacted before it reaches an open endpoint's label" {
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "frag"));
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "dns-query"));
 
-    // Every family carries the label, so none of the six may keep the whole url.
+    // Every family carries the label, so none of the eleven may keep the whole
+    // url.
     try testing.expect(std.mem.containsAtLeast(
         u8,
         text,
@@ -1302,6 +1392,54 @@ test "an upstream url is redacted before it reaches an open endpoint's label" {
     ));
 }
 
+test "the queue families render with the label set every upstream family shares" {
+    // The surface the Pi burst defect is read from: whether an upstream is
+    // queueing (`queued_total`/`in_flight` against `slots`) and whether it is
+    // churning connections (`reuse_recoveries_total`).
+    const upstream_list = [_]UpstreamSample{.{
+        .url = "tls://dns.example:853",
+        .enabled = true,
+        .available = true,
+        .consecutive_failures = 0,
+        .total_successes = 30,
+        .total_failures = 0,
+        .success_rate = 1,
+        .in_flight = 6,
+        .slots = 8,
+        .queued_total = 22,
+        .queued_seconds_total = 1.5,
+        .reuse_recoveries_total = 3,
+    }};
+    const text = try renderToString(testing.allocator, .{ .upstreams = &upstream_list });
+    defer testing.allocator.free(text);
+
+    const label = "{index=\"0\",url=\"tls://dns.example:853\"}";
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_in_flight" ++ label ++ " 6\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_slots" ++ label ++ " 8\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_queued_total" ++ label ++ " 22\n"));
+    try testing.expect(std.mem.containsAtLeast(
+        u8,
+        text,
+        1,
+        "nxdns_upstream_queued_seconds_total" ++ label ++ " 1.5000\n",
+    ));
+    try testing.expect(std.mem.containsAtLeast(
+        u8,
+        text,
+        1,
+        "nxdns_upstream_reuse_recoveries_total" ++ label ++ " 3\n",
+    ));
+
+    // The types a scrape reads them as, and the approximation the two queue
+    // counters carry stated where an operator meets them.
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_upstream_in_flight gauge\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_upstream_slots gauge\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_upstream_queued_total counter\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_upstream_queued_seconds_total counter\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_upstream_reuse_recoveries_total counter\n"));
+    try testing.expectEqual(@as(usize, 2), std.mem.count(u8, text, "Approximate; sampled at admission."));
+}
+
 test "a url longer than the redaction bound cannot run past it" {
     const long_host = "h" ** (4 * safe_url.max_len);
     const upstream_list = [_]UpstreamSample{.{
@@ -1312,6 +1450,11 @@ test "a url longer than the redaction bound cannot run past it" {
         .total_successes = 0,
         .total_failures = 0,
         .success_rate = 1,
+        .in_flight = 0,
+        .slots = 8,
+        .queued_total = 0,
+        .queued_seconds_total = 0,
+        .reuse_recoveries_total = 0,
     }};
     const text = try renderToString(testing.allocator, .{ .upstreams = &upstream_list });
     defer testing.allocator.free(text);
@@ -1325,6 +1468,115 @@ test "a url longer than the redaction bound cannot run past it" {
     ));
 }
 
+/// A leaf client that answers, and counts that it was asked.
+///
+/// `pool.zig`'s own fake is private to that file and models failure modes this
+/// test has no use for; all this one has to do is let a real `Pool.exchange`
+/// complete, so the health half of the sample below is the pool's own
+/// bookkeeping rather than a literal.
+const AnsweringClient = struct {
+    calls: std.atomic.Value(u32) = .init(0),
+
+    const reply = "\x12\x34\x81\x80";
+
+    fn exchangeFn(
+        ptr: *anyopaque,
+        io: std.Io,
+        query: []const u8,
+        response_buf: []u8,
+        selected: *?[]const u8,
+    ) transport.ExchangeError![]u8 {
+        _ = io;
+        _ = query;
+        const self: *AnsweringClient = @ptrCast(@alignCast(ptr));
+        _ = self.calls.fetchAdd(1, .acq_rel);
+        selected.* = "fake://leaf";
+        @memcpy(response_buf[0..reply.len], reply);
+        return response_buf[0..reply.len];
+    }
+
+    fn client(self: *AnsweringClient) transport.Client {
+        return .{ .ptr = self, .exchangeFn = exchangeFn };
+    }
+};
+
+test "the queue families carry what a real pool recorded, through the real snapshot" {
+    // The test above renders a hand-built `UpstreamSample`, so it proves the
+    // exposition and nothing else. This one drives a real `Pool` and goes
+    // through `Pool.snapshot` and `upstreams`, which is where a crossed field
+    // would live. Every driven value is distinct for that reason: a swap
+    // anywhere along the path renders the wrong number rather than a match.
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var leaf: AnsweringClient = .{};
+    var slots = [_]pool_mod.Slot{
+        .{ .client = leaf.client() },
+        .{ .client = leaf.client() },
+    };
+    var recoveries: std.atomic.Value(u64) = .init(0);
+    var entries = [_]pool_mod.Entry{.{
+        .endpoint = try .parse("tls://dns.example:853"),
+        .slots = &slots,
+        .priority = 10,
+        .enabled = true,
+        .health = .init,
+        .sem = .{ .permits = slots.len },
+        .reuse_recoveries = &recoveries,
+    }};
+    var pool: pool_mod.Pool = .init(&entries, .{}, .{
+        .attempt = .{ .raw = .fromSeconds(10), .clock = .awake },
+        .total = .{ .raw = .fromSeconds(30), .clock = .awake },
+    }, 1);
+
+    var buf: [512]u8 = undefined;
+    var selected: ?[]const u8 = null;
+    _ = try pool.exchange(io, "\x12\x34\x01\x00", &buf, &selected);
+    try testing.expectEqual(@as(u32, 1), leaf.calls.load(.acquire));
+
+    // The counters a burst would move, written straight into the entry: this
+    // test is about what the snapshot path carries, not about reproducing a
+    // queue. The exchange above has already returned, so nothing else is
+    // touching them.
+    entries[0].in_flight.store(5, .release);
+    entries[0].queued_total.store(7, .release);
+    entries[0].queued_ns_total.store(1_500_000_000, .release);
+    recoveries.store(9, .release);
+
+    var arena: std.heap.ArenaAllocator = .init(testing.allocator);
+    defer arena.deinit();
+    const list = try upstreams(&pool, io, arena.allocator());
+    const text = try renderToString(testing.allocator, .{ .upstreams = list });
+    defer testing.allocator.free(text);
+
+    const label = "{index=\"0\",url=\"tls://dns.example:853\"}";
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_in_flight" ++ label ++ " 5\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_slots" ++ label ++ " 2\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_queued_total" ++ label ++ " 7\n"));
+    try testing.expect(std.mem.containsAtLeast(
+        u8,
+        text,
+        1,
+        "nxdns_upstream_queued_seconds_total" ++ label ++ " 1.5000\n",
+    ));
+    try testing.expect(std.mem.containsAtLeast(
+        u8,
+        text,
+        1,
+        "nxdns_upstream_reuse_recoveries_total" ++ label ++ " 9\n",
+    ));
+
+    // The one series the pool wrote by itself, so the five above are read
+    // against an entry the pool really did use.
+    try testing.expect(std.mem.containsAtLeast(
+        u8,
+        text,
+        1,
+        "nxdns_upstream_successes_total" ++ label ++ " 1\n",
+    ));
+}
+
 test "collect reads the live counters of the components it is given" {
     var threaded: std.Io.Threaded = .init(testing.allocator, .{});
     defer threaded.deinit();

src/web/web_integration_test.zig

@@ -313,6 +313,8 @@ const Env = struct {
     pauser: pause_mod.Pause,
     tables: local_tables_mod.LocalTables,
     pool_entries: [1]pool_mod.Entry,
+    pool_slots: [1]pool_mod.Slot,
+    pool_recoveries: std.atomic.Value(u64),
     pool: pool_mod.Pool,
     state: server.WebState,
     web: server.Server,
@@ -387,14 +389,18 @@ const Env = struct {
         self.pauser = .{};
         self.tables = .empty;
 
+        // Never exchanged with: the pool feeds `/metrics` and the
+        // `/api/health` upstream condition only.
+        self.pool_slots = .{.{ .client = .{ .ptr = undefined, .exchangeFn = undefined } }};
+        self.pool_recoveries = .init(0);
         self.pool_entries = .{.{
             .endpoint = transport.Endpoint.parse("https://dns.example/dns-query") catch unreachable,
-            // Never exchanged with: the pool feeds `/metrics` and the
-            // `/api/health` upstream condition only.
-            .client = .{ .ptr = undefined, .exchangeFn = undefined },
+            .slots = &self.pool_slots,
             .priority = 1,
             .enabled = true,
             .health = .init,
+            .sem = .{ .permits = self.pool_slots.len },
+            .reuse_recoveries = &self.pool_recoveries,
         }};
         self.pool = .init(&self.pool_entries, .{}, .{
             .attempt = .{ .raw = .fromMilliseconds(50), .clock = .awake },