milestone 31: concurrent upstream exchanges, dot session reuse, queue metrics

src/upstream/dot_client_integration_test.zig

@@ -0,0 +1,651 @@
+//! Hermetic loopback tests for DoT session reuse (`dot_client.zig`).
+//!
+//! This lives in its own file because it needs `@import("build_options")`,
+//! which only exists when the compilation is driven by build.zig, and because a
+//! DoT server is a whole fixture rather than a stub. `-Dintegration` gates it;
+//! nothing here leaves the machine and nothing here resolves a name.
+//!
+//! The peer is the same mbedTLS `platform/tls_server.zig` the nxdns listener
+//! uses, serving RFC 1035 §4.2.2 framed DNS on the plaintext side. Its
+//! certificate is the committed self-signed fixture, which is why every test
+//! preloads that certificate into its own `Certificate.Bundle`: `DotClient`
+//! hardcodes `.ca = .system` and there is deliberately no way to ask it to skip
+//! verification — a fixture-only trust anchor is a test's business, an
+//! insecure-verify switch in the production client would be a hole in it.
+//!
+//! Every test races its client work against a budget. The failures under test
+//! are "the client waits for a reply that never comes" shaped, and without a
+//! deadline those hang the suite instead of failing it.
+
+const std = @import("std");
+const build_options = @import("build_options");
+const tls = std.crypto.tls;
+const net = std.Io.net;
+const Certificate = std.crypto.Certificate;
+
+const dot_client = @import("dot_client.zig");
+const pool_mod = @import("pool.zig");
+const transport = @import("transport.zig");
+const tls_server = @import("../platform/tls_server.zig");
+const events_fixture = @import("../storage/events_fixture.zig");
+
+const testing = std.testing;
+
+/// Long enough that no handshake or exchange on loopback needs it, short enough
+/// that a hang ends the test rather than the suite.
+const budget: std.Io.Clock.Duration = .{ .raw = .fromSeconds(10), .clock = .awake };
+
+/// The fixture certificate carries `DNS:localhost`, and
+/// `Certificate.Parsed.verifyHostName` matches dNSName SANs only, so this is
+/// the only name a `DotClient` can verify it as. The dial target stays the
+/// loopback IP literal.
+const fixture_host = "localhost";
+
+/// A query for example.com A: id 0x1234, RD set, one question.
+const query_bytes =
+    "\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00" ++
+    "\x07example\x03com\x00\x00\x01\x00\x01";
+
+/// The matching response: the question echoed plus one A record.
+const response_bytes =
+    "\x12\x34\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00" ++
+    "\x07example\x03com\x00\x00\x01\x00\x01" ++
+    "\xc0\x0c\x00\x01\x00\x01\x00\x00\x01\x2c\x00\x04\x5d\xb8\xd8\x22";
+
+/// The same answer under a different transaction id. Well-framed and the length
+/// the prefix claims, so the client reads the whole of it and only
+/// `transport.validateResponse` can reject it — which is the point: a validation
+/// failure is not an I/O failure, and it has to close the session all the same.
+const mismatched_response_bytes =
+    "\x99\x99\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00" ++
+    "\x07example\x03com\x00\x00\x01\x00\x01" ++
+    "\xc0\x0c\x00\x01\x00\x01\x00\x00\x01\x2c\x00\x04\x5d\xb8\xd8\x22";
+
+// ---------------------------------------------------------------------------
+// Server side
+// ---------------------------------------------------------------------------
+
+/// The listening half: one mbedTLS context, one loopback listener, and the
+/// count of connections the scripts below have accepted. That count is what
+/// "one connection" and "no redial" are asserted on.
+const Server = struct {
+    ctx: tls_server.ServerContext,
+    listener: net.Server,
+    /// Every dial a script chose to `accept`. The listener stays up for the
+    /// whole of every test on purpose: a refused dial is invisible to this
+    /// counter, so a script that stopped listening could not tell "the client
+    /// did not dial again" from "it dialed and was refused". A dial beyond what
+    /// the script accepts sits un-handshaken in the backlog instead, stalling
+    /// the client into its budget — the test fails either way, but only an
+    /// accepted dial shows up in this count.
+    accepts: std.atomic.Value(u32) = .init(0),
+    /// Framed queries taken off the wire. A test that has to know the client is
+    /// blocked waiting for a reply waits on this rather than on a sleep: a
+    /// Debug-build ECDSA handshake on loopback is slow enough that any fixed
+    /// gap is either a flake or a stall.
+    queries: std.atomic.Value(u32) = .init(0),
+
+    fn init(self: *Server, gpa: std.mem.Allocator, io: std.Io) !void {
+        const fixtures = @import("test_fixtures");
+        self.* = .{
+            .ctx = try .init(gpa, fixtures.cert_pem, fixtures.key_pem, null),
+            .listener = try (net.IpAddress{ .ip4 = .loopback(0) }).listen(io, .{
+                .reuse_address = true,
+            }),
+        };
+    }
+
+    fn deinit(self: *Server, gpa: std.mem.Allocator, io: std.Io) void {
+        self.listener.deinit(io);
+        self.ctx.deinit(gpa);
+    }
+
+    fn address(self: *const Server) net.IpAddress {
+        return self.listener.socket.address;
+    }
+};
+
+/// One accepted connection. Pinned: `ServerStream` recovers its `std.Io`
+/// interfaces from their addresses inside itself and hands mbedTLS a pointer to
+/// itself as the BIO context, and its plaintext buffers live here too.
+const Accepted = struct {
+    server: *Server,
+    stream: net.Stream,
+    tls: tls_server.ServerStream,
+    read_buffer: [4096]u8,
+    write_buffer: [4096]u8,
+
+    fn accept(self: *Accepted, gpa: std.mem.Allocator, server: *Server, io: std.Io) !void {
+        self.server = server;
+        self.stream = try server.listener.accept(io);
+        _ = server.accepts.fetchAdd(1, .acq_rel);
+        errdefer self.stream.close(io);
+        try self.tls.accept(gpa, &server.ctx, io, &self.stream, &self.read_buffer, &self.write_buffer);
+    }
+
+    fn close(self: *Accepted, gpa: std.mem.Allocator, io: std.Io) void {
+        self.tls.close(gpa);
+        self.stream.close(io);
+    }
+
+    /// Reads one framed query and writes `response_bytes` back.
+    fn answerOnce(self: *Accepted) !void {
+        var query_buf: [2048]u8 = undefined;
+        _ = try self.readFramed(&query_buf);
+        try self.writeFramed(response_bytes);
+    }
+
+    fn readFramed(self: *Accepted, buf: []u8) ![]u8 {
+        var prefix: [transport.prefix_len]u8 = undefined;
+        try self.tls.reader().readSliceAll(&prefix);
+        const len = transport.parsePrefix(prefix);
+        if (len > buf.len) return error.QueryTooLarge;
+        try self.tls.reader().readSliceAll(buf[0..len]);
+        _ = self.server.queries.fetchAdd(1, .acq_rel);
+        return buf[0..len];
+    }
+
+    fn writeFramed(self: *Accepted, message: []const u8) !void {
+        const prefix = transport.framePrefix(@intCast(message.len));
+        try self.tls.writer().writeAll(&prefix);
+        try self.tls.writer().writeAll(message);
+        try self.tls.writer().flush();
+    }
+};
+
+/// Answers `count` queries on one connection and holds it open until the test
+/// tears it down.
+fn serveOnOneConnection(
+    gpa: std.mem.Allocator,
+    server: *Server,
+    io: std.Io,
+    count: usize,
+) anyerror!void {
+    var session: Accepted = undefined;
+    try session.accept(gpa, server, io);
+    defer session.close(gpa, io);
+    for (0..count) |_| try session.answerOnce();
+}
+
+/// Answers one query, drops the connection the way an upstream reaps an idle
+/// one, then accepts a second and answers one more.
+fn serveThenCloseThenServe(gpa: std.mem.Allocator, server: *Server, io: std.Io) anyerror!void {
+    for (0..2) |_| {
+        var session: Accepted = undefined;
+        try session.accept(gpa, server, io);
+        defer session.close(gpa, io);
+        try session.answerOnce();
+    }
+}
+
+/// Answers one query, then replies to the next with a single length-prefix byte
+/// and closes. The second reply has started, so the client may not redial.
+fn serveThenTruncate(gpa: std.mem.Allocator, server: *Server, io: std.Io) anyerror!void {
+    var session: Accepted = undefined;
+    try session.accept(gpa, server, io);
+    defer session.close(gpa, io);
+
+    try session.answerOnce();
+
+    var query_buf: [2048]u8 = undefined;
+    _ = try session.readFramed(&query_buf);
+    try session.tls.writer().writeAll(&[_]u8{0x00});
+    try session.tls.writer().flush();
+}
+
+/// Answers one query, then answers the next with a frame the client can read
+/// whole and `transport.validateResponse` must still reject.
+fn serveThenAnswerWithWrongId(gpa: std.mem.Allocator, server: *Server, io: std.Io) anyerror!void {
+    var session: Accepted = undefined;
+    try session.accept(gpa, server, io);
+    defer session.close(gpa, io);
+
+    try session.answerOnce();
+
+    var query_buf: [2048]u8 = undefined;
+    _ = try session.readFramed(&query_buf);
+    try session.writeFramed(mismatched_response_bytes);
+}
+
+/// Answers one query, drops the connection, then takes the redial and drops that
+/// one too — after reading its query and before answering it.
+///
+/// The redial *succeeds*, which is the point: the exchange that fails is the
+/// retry, running on a session this call dialed itself. `retryDecision` gives a
+/// fresh session no second chance, so the client owes exactly two dials.
+fn serveThenCloseThenFailTheRetry(gpa: std.mem.Allocator, server: *Server, io: std.Io) anyerror!void {
+    {
+        var session: Accepted = undefined;
+        try session.accept(gpa, server, io);
+        defer session.close(gpa, io);
+        try session.answerOnce();
+    }
+    {
+        var session: Accepted = undefined;
+        try session.accept(gpa, server, io);
+        defer session.close(gpa, io);
+        // Read the query before dropping the connection: the failure under test
+        // is the client's *receive*, and a peer that closed before taking the
+        // query could fail its send instead.
+        var query_buf: [2048]u8 = undefined;
+        _ = try session.readFramed(&query_buf);
+    }
+}
+
+/// Completes the handshake, takes the query and never answers it.
+fn serveSilently(gpa: std.mem.Allocator, server: *Server, io: std.Io) anyerror!void {
+    var session: Accepted = undefined;
+    try session.accept(gpa, server, io);
+    defer session.close(gpa, io);
+
+    var query_buf: [2048]u8 = undefined;
+    _ = try session.readFramed(&query_buf);
+    const forever: std.Io.Clock.Duration = .{ .raw = .fromSeconds(3600), .clock = .awake };
+    try forever.sleep(io);
+}
+
+// ---------------------------------------------------------------------------
+// Client side
+// ---------------------------------------------------------------------------
+
+/// Everything one `DotClient` borrows, plus the client itself.
+///
+/// Built in place: `DotClient` pins the TLS state of an open session inside
+/// itself, and its endpoint and buffers are borrowed from this struct.
+const ClientFixture = struct {
+    gpa: std.mem.Allocator,
+    bundle: Certificate.Bundle = .empty,
+    bundle_lock: std.Io.RwLock = .init,
+    buffers: [4 * tls.Client.min_buffer_len]u8 = undefined,
+    url_buf: [32]u8 = undefined,
+    recoveries: std.atomic.Value(u64) = .init(0),
+    dot: dot_client.DotClient = undefined,
+
+    fn init(self: *ClientFixture, gpa: std.mem.Allocator, io: std.Io, address: net.IpAddress) !void {
+        self.* = .{ .gpa = gpa };
+        errdefer self.bundle.deinit(gpa);
+        try self.preloadFixtureCert(io);
+
+        const url = try std.fmt.bufPrint(&self.url_buf, "tls://127.0.0.1:{d}", .{address.ip4.port});
+        const chunk = tls.Client.min_buffer_len;
+        self.dot = .init(
+            try .parse(url),
+            fixture_host,
+            gpa,
+            &self.bundle,
+            &self.bundle_lock,
+            &self.recoveries,
+            .{
+                .tls_read = self.buffers[0..chunk],
+                .tls_write = self.buffers[chunk .. 2 * chunk],
+                .stream_read = self.buffers[2 * chunk .. 3 * chunk],
+                .stream_write = self.buffers[3 * chunk ..],
+            },
+        );
+    }
+
+    fn deinit(self: *ClientFixture, io: std.Io) void {
+        self.dot.close(io);
+        self.bundle.deinit(self.gpa);
+    }
+
+    /// Makes the self-signed fixture certificate the one trust anchor this
+    /// client has.
+    ///
+    /// A non-empty bundle is what both `DotClient.ensureBundle` and
+    /// `TlsStream.init` check before they would scan the system store, so
+    /// preloading it also keeps the test off the host's CA directory entirely.
+    /// The decode mirrors `Certificate.Bundle.addCertsFromFile`, which is the
+    /// only PEM entry point the stdlib exposes and takes a file; writing the
+    /// committed fixture back out to disk to read it in again would be the
+    /// longer way round to the same three calls.
+    fn preloadFixtureCert(self: *ClientFixture, io: std.Io) !void {
+        const fixtures = @import("test_fixtures");
+        const begin_marker = "-----BEGIN CERTIFICATE-----";
+        const end_marker = "-----END CERTIFICATE-----";
+
+        const body_start = (std.mem.find(u8, fixtures.cert_pem, begin_marker) orelse
+            return error.MissingBeginCertificateMarker) + begin_marker.len;
+        const body_end = std.mem.findPos(u8, fixtures.cert_pem, body_start, end_marker) orelse
+            return error.MissingEndCertificateMarker;
+        const encoded = std.mem.trim(u8, fixtures.cert_pem[body_start..body_end], " \t\r\n");
+
+        const decoder = std.base64.standard.decoderWithIgnore(" \t\r\n");
+        try self.bundle.bytes.ensureUnusedCapacity(self.gpa, encoded.len / 4 * 3 + 3);
+        const decoded_start: u32 = @intCast(self.bundle.bytes.items.len);
+        const written = try decoder.decode(
+            self.bundle.bytes.allocatedSlice()[decoded_start..],
+            encoded,
+        );
+        self.bundle.bytes.items.len += written;
+        try self.bundle.parseCert(self.gpa, decoded_start, std.Io.Clock.real.now(io).toSeconds());
+        try testing.expect(self.bundle.map.count() == 1);
+    }
+};
+
+/// Blocks until the server has taken `count` framed queries off the wire.
+///
+/// Bounded by the same budget as everything else here: a server that died
+/// before it read one would otherwise hang the suite.
+fn awaitQueries(io: std.Io, server: *Server, count: u32) !void {
+    const step: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(10), .clock = .awake };
+    const steps = 1000;
+    for (0..steps) |_| {
+        if (server.queries.load(.acquire) >= count) return;
+        try step.sleep(io);
+    }
+    return error.DotFixtureTimedOut;
+}
+
+fn expire(io: std.Io, duration: std.Io.Clock.Duration) std.Io.Cancelable!void {
+    return duration.sleep(io);
+}
+
+const Outcome = union(enum) {
+    client: anyerror!void,
+    expiry: std.Io.Cancelable!void,
+};
+
+/// Runs `client_work` against the budget and tears the server task down either
+/// way: a client that fails before it connects would otherwise leave the server
+/// blocked in `accept` forever.
+fn runAgainstServer(
+    io: std.Io,
+    server_task: *std.Io.Future(anyerror!void),
+    comptime client_work: anytype,
+    args: anytype,
+) !void {
+    var outcomes: [2]Outcome = undefined;
+    var race: std.Io.Select(Outcome) = .init(io, &outcomes);
+    defer race.cancelDiscard();
+    try race.concurrent(.client, client_work, args);
+    try race.concurrent(.expiry, expire, .{ io, budget });
+
+    const client_result: anyerror!void = switch (try race.await()) {
+        .client => |result| result,
+        .expiry => |result| blk: {
+            try result;
+            break :blk error.DotFixtureTimedOut;
+        },
+    };
+
+    const server_result = if (client_result) |_|
+        server_task.await(io)
+    else |_|
+        server_task.cancel(io);
+
+    try client_result;
+    server_result catch |err| switch (err) {
+        // The scripts that hold a connection open past their last answer are
+        // torn down by the cancel above, which is the intended end for them.
+        error.Canceled => {},
+        else => return err,
+    };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+fn twoExchangesOverOneSession(io: std.Io, fixture: *ClientFixture) anyerror!void {
+    var buf: [512]u8 = undefined;
+
+    const first = try fixture.dot.exchange(io, query_bytes, &buf);
+    try testing.expectEqualSlices(u8, response_bytes, first);
+    // The point of the milestone: the connection outlives the exchange.
+    try testing.expect(fixture.dot.session != null);
+
+    const second = try fixture.dot.exchange(io, query_bytes, &buf);
+    try testing.expectEqualSlices(u8, response_bytes, second);
+    try testing.expect(fixture.dot.session != null);
+}
+
+test "two exchanges through one DoT client share one connection" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    var threaded: std.Io.Threaded = .init(gpa, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var server: Server = undefined;
+    try server.init(gpa, io);
+    defer server.deinit(gpa, io);
+
+    var fixture: ClientFixture = undefined;
+    try fixture.init(gpa, io, server.address());
+    defer fixture.deinit(io);
+
+    var server_task = try io.concurrent(serveOnOneConnection, .{ gpa, &server, io, @as(usize, 2) });
+    try runAgainstServer(io, &server_task, twoExchangesOverOneSession, .{ io, &fixture });
+
+    try testing.expectEqual(@as(u32, 1), server.accepts.load(.acquire));
+    try testing.expectEqual(@as(u64, 0), fixture.recoveries.load(.acquire));
+}
+
+fn twoPoolExchanges(io: std.Io, pool: *pool_mod.Pool) anyerror!void {
+    var buf: [512]u8 = undefined;
+    var selected: ?[]const u8 = null;
+    const first = try pool.exchange(io, query_bytes, &buf, &selected);
+    try testing.expectEqualSlices(u8, response_bytes, first);
+    const second = try pool.exchange(io, query_bytes, &buf, &selected);
+    try testing.expectEqualSlices(u8, response_bytes, second);
+}
+
+test "a session the upstream closed is recovered by one redial and counted, not blamed" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    var threaded: std.Io.Threaded = .init(gpa, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var server: Server = undefined;
+    try server.init(gpa, io);
+    defer server.deinit(gpa, io);
+
+    var fixture: ClientFixture = undefined;
+    try fixture.init(gpa, io, server.address());
+    defer fixture.deinit(io);
+
+    // Through a real pool entry, because the claim is about what the pool does
+    // *not* see: an upstream reaping an idle connection must not cost it health
+    // or raise an operational event.
+    var slots = [_]pool_mod.Slot{.{ .client = fixture.dot.client() }};
+    var entries = [_]pool_mod.Entry{.{
+        .endpoint = fixture.dot.endpoint,
+        .slots = &slots,
+        .priority = 10,
+        .enabled = true,
+        .health = .init,
+        .sem = .{ .permits = slots.len },
+        .reuse_recoveries = &fixture.recoveries,
+    }};
+    var pool: pool_mod.Pool = .init(&entries, .{
+        .failure_threshold = 2,
+        .base_backoff_ms = 60_000,
+        .max_backoff_ms = 60_000,
+    }, .{
+        .attempt = .{ .raw = .fromSeconds(10), .clock = .awake },
+        .total = .{ .raw = .fromSeconds(30), .clock = .awake },
+    }, 1);
+
+    var fx: events_fixture.Fixture = .{};
+    try fx.init(io, 1000);
+    defer fx.deinit();
+    pool.diagnostics = &fx.store;
+
+    var server_task = try io.concurrent(serveThenCloseThenServe, .{ gpa, &server, io });
+    try runAgainstServer(io, &server_task, twoPoolExchanges, .{ io, &pool });
+
+    try testing.expectEqual(@as(u32, 2), server.accepts.load(.acquire));
+    try testing.expectEqual(@as(u64, 1), fixture.recoveries.load(.acquire));
+    try testing.expectEqual(@as(u64, 2), entries[0].health.total_successes);
+    try testing.expectEqual(@as(u64, 0), entries[0].health.total_failures);
+    try testing.expectEqual(@as(u32, 0), entries[0].health.consecutive_failures);
+    try testing.expectEqual(@as(?std.Io.Timestamp, null), entries[0].health.backoff_until);
+    try testing.expectEqual(
+        @as(i64, 0),
+        try fx.count("SELECT count(*) FROM operational_events"),
+    );
+}
+
+fn exchangeThenReadTruncatedReply(io: std.Io, fixture: *ClientFixture) anyerror!void {
+    var buf: [512]u8 = undefined;
+    _ = try fixture.dot.exchange(io, query_bytes, &buf);
+
+    // One prefix byte arrived, so the reply had started: re-sending the query on
+    // a fresh connection would be a second question, not a recovery.
+    try testing.expectError(error.ReceiveFailed, fixture.dot.exchange(io, query_bytes, &buf));
+    try testing.expect(fixture.dot.session == null);
+}
+
+test "a reused session that failed after one response byte is not redialed" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    var threaded: std.Io.Threaded = .init(gpa, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var server: Server = undefined;
+    try server.init(gpa, io);
+    defer server.deinit(gpa, io);
+
+    var fixture: ClientFixture = undefined;
+    try fixture.init(gpa, io, server.address());
+    defer fixture.deinit(io);
+
+    var server_task = try io.concurrent(serveThenTruncate, .{ gpa, &server, io });
+    try runAgainstServer(io, &server_task, exchangeThenReadTruncatedReply, .{ io, &fixture });
+
+    // The whole claim: the client never came back for a second connection.
+    try testing.expectEqual(@as(u32, 1), server.accepts.load(.acquire));
+    try testing.expectEqual(@as(u64, 0), fixture.recoveries.load(.acquire));
+}
+
+fn exchangeThenReadWrongId(io: std.Io, fixture: *ClientFixture) anyerror!void {
+    var buf: [512]u8 = undefined;
+    _ = try fixture.dot.exchange(io, query_bytes, &buf);
+    try testing.expect(fixture.dot.session != null);
+
+    // The read succeeded; only the bytes are wrong. Nothing about that says the
+    // connection is stale, so it is final — but the stream position after a
+    // frame this client will not trust is unknowable, so the session goes.
+    try testing.expectError(error.ResponseMismatch, fixture.dot.exchange(io, query_bytes, &buf));
+    try testing.expect(fixture.dot.session == null);
+}
+
+test "a reply that fails validation is final and clears the session without redialing" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    var threaded: std.Io.Threaded = .init(gpa, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var server: Server = undefined;
+    try server.init(gpa, io);
+    defer server.deinit(gpa, io);
+
+    var fixture: ClientFixture = undefined;
+    try fixture.init(gpa, io, server.address());
+    defer fixture.deinit(io);
+
+    var server_task = try io.concurrent(serveThenAnswerWithWrongId, .{ gpa, &server, io });
+    try runAgainstServer(io, &server_task, exchangeThenReadWrongId, .{ io, &fixture });
+
+    // Both halves of the claim. One accept: the reused session read a whole
+    // frame, so nothing here is a lifecycle failure and no redial is owed. And
+    // nothing was recovered, so the counter stays where it was.
+    try testing.expectEqual(@as(u32, 1), server.accepts.load(.acquire));
+    try testing.expectEqual(@as(u64, 0), fixture.recoveries.load(.acquire));
+}
+
+fn exchangeThenFailTheRetry(io: std.Io, fixture: *ClientFixture) anyerror!void {
+    var buf: [512]u8 = undefined;
+    _ = try fixture.dot.exchange(io, query_bytes, &buf);
+    try testing.expect(fixture.dot.session != null);
+
+    // Stale session, no response byte, lifecycle cause: the one redial is owed,
+    // taken, and connected. The exchange on that fresh session then fails its
+    // read, and the retry's outcome is the exchange's outcome — no third dial,
+    // because a session this call dialed itself is never retried.
+    try testing.expectError(error.ReceiveFailed, fixture.dot.exchange(io, query_bytes, &buf));
+    try testing.expect(fixture.dot.session == null);
+
+    // What the `nxdns check` probe loop relies on: closing a client whose
+    // exchange already failed is a no-op, not a double close.
+    fixture.dot.close(io);
+    try testing.expect(fixture.dot.session == null);
+}
+
+test "a stale session whose retry fails is final and leaves no session behind" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    var threaded: std.Io.Threaded = .init(gpa, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var server: Server = undefined;
+    try server.init(gpa, io);
+    defer server.deinit(gpa, io);
+
+    var fixture: ClientFixture = undefined;
+    try fixture.init(gpa, io, server.address());
+    defer fixture.deinit(io);
+
+    var server_task = try io.concurrent(serveThenCloseThenFailTheRetry, .{ gpa, &server, io });
+    try runAgainstServer(io, &server_task, exchangeThenFailTheRetry, .{ io, &fixture });
+
+    // Exactly two: the first exchange and the one redial. A client that retried
+    // its retry would need a third dial; the script accepts no third connection,
+    // so that dial would stall un-handshaken until the client's budget failed
+    // the test — it cannot succeed silently.
+    try testing.expectEqual(@as(u32, 2), server.accepts.load(.acquire));
+    // The redial connected but the exchange on it failed, so nothing was
+    // recovered and nothing is counted.
+    try testing.expectEqual(@as(u64, 0), fixture.recoveries.load(.acquire));
+}
+
+fn oneExchange(io: std.Io, fixture: *ClientFixture) transport.ExchangeError!void {
+    var buf: [512]u8 = undefined;
+    _ = try fixture.dot.exchange(io, query_bytes, &buf);
+}
+
+test "a canceled exchange leaves no session open" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    var threaded: std.Io.Threaded = .init(gpa, .{});
+    defer threaded.deinit();
+    const io = threaded.io();
+
+    var server: Server = undefined;
+    try server.init(gpa, io);
+    defer server.deinit(gpa, io);
+
+    var fixture: ClientFixture = undefined;
+    try fixture.init(gpa, io, server.address());
+    defer fixture.deinit(io);
+
+    var server_task = try io.concurrent(serveSilently, .{ gpa, &server, io });
+    var client_task = try io.concurrent(oneExchange, .{ io, &fixture });
+
+    // The query is on the wire and no answer is coming, so the client is
+    // provably blocked in its read — the state a total-budget expiry cancels an
+    // exchange in.
+    try awaitQueries(io, &server, 1);
+    try testing.expectError(error.Canceled, client_task.cancel(io));
+
+    // Mid-frame state is unknowable after a cancellation, so the next exchange
+    // has to start from a fresh dial.
+    try testing.expect(fixture.dot.session == null);
+    try testing.expectEqual(@as(u64, 0), fixture.recoveries.load(.acquire));
+
+    server_task.cancel(io) catch |err| switch (err) {
+        error.Canceled => {},
+        else => return err,
+    };
+}