mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 21 Activity

resolver transport: udp/tcp servers, doh/dot clients, pool failover with health

Browse Source
This commit is contained in:
mokhtar
2026-08-01 12:36:50 +02:00
parent 346f2dc502
commit 17d0401f8a
15 changed files with 6062 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/upstream/pool.zig
+723
View File
@@ -0,0 +1,723 @@
//! Priority-ordered sequential failover across upstream endpoints, with a
//! per-attempt deadline, health tracking and backoff (PLAN §9).
//!
//! The pool is itself a `transport.Client`, so the handler above it sees one
//! interface and knows nothing about how many upstreams exist.
//!
//! Two passes, not one. Pass one walks the enabled entries that health says are
//! available. Pass two runs only when pass one attempted nothing, and skips the
//! backoff check: every endpoint being in backoff must not turn into SERVFAIL
//! for every client. A probe is better than a guaranteed failure, and the probe
//! is how backoff recovers.
//!
//! Concurrency (PLAN §9, spec S4.2): several handler tasks share one `Pool`,
//! and they hold two different locks at two different scopes.
//!
//! `Pool.mutex` guards health mutation, the jitter RNG and `snapshot`. It is
//! never held across an exchange, so a slow upstream cannot block a health read
//! or an attempt on some other entry.
//!
//! `Entry.busy` guards one entry's `client`. A task holds it for a whole
//! attempt against that entry — the exchange and the timeout race around it —
//! and drops it before the failover loop moves to the next candidate. This is
//! what makes the pool safe to share: `DohClient` owns a request buffer and a
//! transfer buffer, `DotClient` owns four TLS buffers and the stream state
//! built on them, so one client may be inside `exchange` only once at a time.
//!
//! Because that wait is serializing, pass one re-reads health after it takes
//! `busy`: the attempts a task queued behind can fail the entry into backoff
//! while it waits, and pass one must not attempt an entry that is unavailable by
//! the time it gets its turn. Lock order is always `busy` then `Pool.mutex`,
//! never the reverse.
//!
//! The guarantee is therefore: at most one in-flight exchange per entry, any
//! number of entries in flight at once, and bookkeeping that never waits on a
//! peer. Two concurrent queries that resolve to the same sole upstream do
//! serialize. At household scale that is the right trade — the alternative is a
//! client instance per listener task, which multiplies TLS buffers and
//! connections for a query rate that never needed them.
const std = @import("std");
const health = @import("health.zig");
const transport = @import("transport.zig");
const log = std.log.scoped(.upstream);
pub const Entry = struct {
endpoint: transport.Endpoint,
client: transport.Client,
/// Lower is tried first (PLAN §11.2 `upstreams.priority`).
priority: i32,
enabled: bool,
health: health.State,
/// Held for the whole of one attempt against this entry, so `client` is
/// never re-entered while it is using its own buffers. Defaulted because
/// `init` sorts `entries` by value, which may only copy unlocked mutexes,
/// and it runs before the pool is reachable by any task.
busy: std.Io.Mutex = .init,
};
/// A copy of one entry's health, taken under the mutex. Feeds
/// `GET /api/upstream/health` in Phase 8.
pub const Snapshot = struct {
url: []const u8,
enabled: bool,
available: bool,
consecutive_failures: u32,
total_successes: u64,
total_failures: u64,
success_rate: f32,
last_success_at: ?std.Io.Timestamp,
last_error_at: ?std.Io.Timestamp,
/// Borrowed from the entry; valid until that entry's next failure.
last_error: []const u8,
backoff_until: ?std.Io.Timestamp,
};
pub const Pool = struct {
/// Caller-owned, sorted ascending by priority in `init`.
entries: []Entry,
cfg: health.Config,
/// On the `.awake` clock, so a suspended Pi does not burn the budget.
attempt_timeout: std.Io.Clock.Duration,
mutex: std.Io.Mutex,
rng: std.Random.DefaultPrng,
pub fn init(
entries: []Entry,
cfg: health.Config,
attempt_timeout: std.Io.Clock.Duration,
seed: u64,
) Pool {
std.debug.assert(entries.len > 0);
// Stable, so entries sharing a priority keep their configured order.
std.mem.sort(Entry, entries, {}, byPriority);
return .{
.entries = entries,
.cfg = cfg,
.attempt_timeout = attempt_timeout,
.mutex = .init,
.rng = .init(seed),
};
}
fn byPriority(_: void, a: Entry, b: Entry) bool {
return a.priority < b.priority;
}
pub fn client(self: *Pool) transport.Client {
return .{ .ptr = self, .exchangeFn = exchangeErased };
}
fn exchangeErased(
ptr: *anyopaque,
io: std.Io,
query: []const u8,
response_buf: []u8,
) transport.ExchangeError![]u8 {
const self: *Pool = @ptrCast(@alignCast(ptr));
return self.exchange(io, query, response_buf);
}
/// `response_buf` is handed to each attempt in turn, so a failed attempt
/// may have written into it. The returned slice is only meaningful on
/// success; on error the buffer's contents are undefined.
pub fn exchange(
self: *Pool,
io: std.Io,
query: []const u8,
response_buf: []u8,
) transport.ExchangeError![]u8 {
const now = std.Io.Clock.awake.now(io);
var last_fault: ?transport.ExchangeError = null;
var attempted = false;
var pass: u8 = 0;
while (pass < 2) : (pass += 1) {
for (self.entries) |*entry| {
if (!entry.enabled) continue;
if (pass == 0 and !self.entryAvailable(io, entry, now)) continue;
// Cancelable, unlike the health-bookkeeping locks below: a task
// waiting its turn on a busy upstream has done nothing that a
// cancellation could corrupt, so it gives up here rather than
// queueing behind an exchange it will not use. The wait itself
// is bounded by the holder's `attempt_timeout`; the waiter's own
// budget only starts once it has the lock.
try entry.busy.lock(io);
defer entry.busy.unlock(io);
// The check above ran before the wait, and the attempts this one
// queued behind may have failed the entry into backoff while it
// waited. Pass one must not touch an entry that is unavailable
// now, so re-read health against a fresh `now` and move on if it
// is. Pass two skips this on purpose: it probes regardless of
// backoff, which is how backoff recovers.
if (pass == 0) {
const recheck = std.Io.Clock.awake.now(io);
if (!self.entryAvailable(io, entry, recheck)) continue;
}
attempted = true;
const result = self.attempt(io, entry.client, query, response_buf);
const completed_at = std.Io.Clock.awake.now(io);
const response = result catch |err| switch (transport.group(err)) {
.peer_fault => {
log.debug("upstream {s} failed: {t}", .{ entry.endpoint.url, err });
self.recordFailure(io, entry, completed_at, err);
last_fault = err;
continue;
},
// The next upstream would hit the same wall, and this says
// nothing about any peer, so it is never recorded.
.local_resource => return err,
.cancellation => return error.Canceled,
};
self.recordSuccess(io, entry, completed_at);
return response;
}
if (attempted) break;
}
// Guaranteed non-null whenever any entry is enabled: pass two attempts
// every enabled entry regardless of backoff.
if (last_fault) |err| return err;
return error.ConnectFailed;
}
/// Copies health into `out` in pool order; returns the number written.
pub fn snapshot(self: *Pool, io: std.Io, out: []Snapshot) std.Io.Cancelable!usize {
const now = std.Io.Clock.awake.now(io);
try self.mutex.lock(io);
defer self.mutex.unlock(io);
const count = @min(self.entries.len, out.len);
for (self.entries[0..count], out[0..count]) |*entry, *slot| {
slot.* = .{
.url = entry.endpoint.url,
.enabled = entry.enabled,
.available = entry.enabled and entry.health.available(now),
.consecutive_failures = entry.health.consecutive_failures,
.total_successes = entry.health.total_successes,
.total_failures = entry.health.total_failures,
.success_rate = entry.health.successRate(),
.last_success_at = entry.health.last_success_at,
.last_error_at = entry.health.last_error_at,
.last_error = entry.health.lastError(),
.backoff_until = entry.health.backoff_until,
};
}
return count;
}
/// One exchange raced against the per-attempt budget. No stream read or
/// write in 0.16.0 takes a timeout, so the budget is a second task and the
/// loser is canceled.
fn attempt(
self: *Pool,
io: std.Io,
entry_client: transport.Client,
query: []const u8,
response_buf: []u8,
) transport.ExchangeError![]u8 {
var outcomes: [2]Outcome = undefined;
var race: std.Io.Select(Outcome) = .init(io, &outcomes);
defer race.cancelDiscard();
race.concurrent(.exchange, transport.Client.exchange, .{
entry_client, io, query, response_buf,
}) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SystemResources,
};
race.concurrent(.expiry, expire, .{ io, self.attempt_timeout }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SystemResources,
};
switch (try race.await()) {
.exchange => |result| return result,
.expiry => |result| {
// A canceled sleep means this whole task is being torn down,
// not that the upstream is slow.
try result;
return error.Timeout;
},
}
}
fn entryAvailable(
self: *Pool,
io: std.Io,
entry: *const Entry,
now: std.Io.Timestamp,
) bool {
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
return entry.health.available(now);
}
fn recordSuccess(self: *Pool, io: std.Io, entry: *Entry, at: std.Io.Timestamp) void {
// Uncancelable: this section takes no Io and never blocks on a peer.
// Losing the bookkeeping for a completed exchange to a cancellation
// that arrives one instruction later would corrupt health for good.
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
entry.health.recordSuccess(at);
}
fn recordFailure(
self: *Pool,
io: std.Io,
entry: *Entry,
at: std.Io.Timestamp,
err: transport.ExchangeError,
) void {
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
entry.health.recordFailure(at, @errorName(err), self.cfg, self.rng.random().int(u32));
}
};
const Outcome = union(enum) {
exchange: transport.ExchangeError![]u8,
expiry: std.Io.Cancelable!void,
};
fn expire(io: std.Io, duration: std.Io.Clock.Duration) std.Io.Cancelable!void {
return duration.sleep(io);
}
const testing = std.testing;
/// A query for example.com A: id 0x1234, RD set, one question.
const query_bytes =
"\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00" ++
"\x07example\x03com\x00\x00\x01\x00\x01";
/// The matching response: the question echoed plus one A record.
const response_bytes =
"\x12\x34\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00" ++
"\x07example\x03com\x00\x00\x01\x00\x01" ++
"\xc0\x0c\x00\x01\x00\x01\x00\x00\x01\x2c\x00\x04\x5d\xb8\xd8\x22";
/// Stands in for a DoH or DoT client. Every behaviour the pool has to react to
/// is one variant, and every call is counted so a test can assert that an entry
/// in backoff was not touched.
const Fake = struct {
behavior: Behavior,
calls: usize = 0,
in_flight: std.atomic.Value(u32) = .init(0),
/// The most tasks ever inside `exchangeFn` at once. The per-entry lock is
/// only doing its job while this stays at 1.
peak_in_flight: std.atomic.Value(u32) = .init(0),
const Behavior = union(enum) {
/// Copy these bytes into the caller's buffer and return them.
reply: []const u8,
/// Fail with this error.
fail: transport.ExchangeError,
/// Sleep, then reply. Used to outrun the pool's attempt budget.
slow: struct { duration: std.Io.Clock.Duration, reply: []const u8 },
/// Sleep, then fail. Holds the entry's lock long enough for a second
/// task to queue on it before the failure opens a backoff window.
slow_fail: struct { duration: std.Io.Clock.Duration, err: transport.ExchangeError },
};
fn exchangeFn(
ptr: *anyopaque,
io: std.Io,
query: []const u8,
response_buf: []u8,
) transport.ExchangeError![]u8 {
_ = query;
const self: *Fake = @ptrCast(@alignCast(ptr));
const entrants = self.in_flight.fetchAdd(1, .acq_rel) + 1;
defer _ = self.in_flight.fetchSub(1, .acq_rel);
_ = self.peak_in_flight.fetchMax(entrants, .acq_rel);
self.calls += 1;
switch (self.behavior) {
.reply => |bytes| return copy(bytes, response_buf),
.fail => |err| return err,
.slow => |slow| {
try slow.duration.sleep(io);
return copy(slow.reply, response_buf);
},
.slow_fail => |slow| {
try slow.duration.sleep(io);
return slow.err;
},
}
}
fn copy(bytes: []const u8, response_buf: []u8) transport.ExchangeError![]u8 {
if (bytes.len > response_buf.len) return error.ResponseTooLarge;
@memcpy(response_buf[0..bytes.len], bytes);
return response_buf[0..bytes.len];
}
fn client(self: *Fake) transport.Client {
return .{ .ptr = self, .exchangeFn = exchangeFn };
}
};
fn testEntry(url: []const u8, fake: *Fake, priority: i32) Entry {
return .{
.endpoint = Endpoint.parse(url) catch unreachable,
.client = fake.client(),
.priority = priority,
.enabled = true,
.health = .init,
};
}
const Endpoint = transport.Endpoint;
/// Long enough that no test can outlive a backoff it just set, and short enough
/// that nothing waits on it.
const test_cfg: health.Config = .{
.failure_threshold = 2,
.base_backoff_ms = 60_000,
.max_backoff_ms = 60_000,
};
const test_timeout: std.Io.Clock.Duration = .{ .raw = .fromSeconds(10), .clock = .awake };
test "Pool satisfies the Client interface" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var fake: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{testEntry("https://a.example/dns-query", &fake, 10)};
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
var buf: [512]u8 = undefined;
const reply = try pool.client().exchange(io, query_bytes, &buf);
try testing.expectEqualSlices(u8, response_bytes, reply);
}
test "entries are tried in ascending priority order" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var low: Fake = .{ .behavior = .{ .reply = response_bytes } };
var high: Fake = .{ .behavior = .{ .reply = response_bytes } };
// Registered out of order: priority, not position, decides.
var entries = [_]Entry{
testEntry("https://high.example/dns-query", &high, 100),
testEntry("https://low.example/dns-query", &low, 10),
};
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
try testing.expectEqual(@as(i32, 10), entries[0].priority);
var buf: [512]u8 = undefined;
_ = try pool.exchange(io, query_bytes, &buf);
try testing.expectEqual(@as(usize, 1), low.calls);
try testing.expectEqual(@as(usize, 0), high.calls);
}
test "a peer fault fails over to the next entry and is recorded" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var bad: Fake = .{ .behavior = .{ .fail = error.Timeout } };
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://bad.example/dns-query", &bad, 10),
testEntry("https://good.example/dns-query", &good, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
var buf: [512]u8 = undefined;
const reply = try pool.exchange(io, query_bytes, &buf);
try testing.expectEqualSlices(u8, response_bytes, reply);
try testing.expectEqual(@as(u32, 1), entries[0].health.consecutive_failures);
try testing.expectEqual(@as(u64, 1), entries[0].health.total_failures);
try testing.expectEqualStrings("Timeout", entries[0].health.lastError());
try testing.expectEqual(@as(u64, 1), entries[1].health.total_successes);
}
test "an entry in backoff is skipped while another is available" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var bad: Fake = .{ .behavior = .{ .fail = error.ConnectFailed } };
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://bad.example/dns-query", &bad, 10),
testEntry("https://good.example/dns-query", &good, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
var buf: [512]u8 = undefined;
// Two failures reach `failure_threshold` and open a backoff window.
_ = try pool.exchange(io, query_bytes, &buf);
_ = try pool.exchange(io, query_bytes, &buf);
try testing.expectEqual(@as(usize, 2), bad.calls);
try testing.expect(entries[0].health.backoff_until != null);
_ = try pool.exchange(io, query_bytes, &buf);
try testing.expectEqual(@as(usize, 2), bad.calls);
try testing.expectEqual(@as(usize, 3), good.calls);
}
test "every entry in backoff is still probed" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var first: Fake = .{ .behavior = .{ .fail = error.ConnectFailed } };
var second: Fake = .{ .behavior = .{ .fail = error.BadResponse } };
var entries = [_]Entry{
testEntry("https://first.example/dns-query", &first, 10),
testEntry("https://second.example/dns-query", &second, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
var buf: [512]u8 = undefined;
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
try testing.expect(entries[0].health.backoff_until != null);
try testing.expect(entries[1].health.backoff_until != null);
// Pass one now has no candidate at all. Pass two probes both anyway.
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
try testing.expectEqual(@as(usize, 3), first.calls);
try testing.expectEqual(@as(usize, 3), second.calls);
}
test "a local resource error short-circuits and records nothing" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var broke: Fake = .{ .behavior = .{ .fail = error.OutOfMemory } };
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://broke.example/dns-query", &broke, 10),
testEntry("https://good.example/dns-query", &good, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
var buf: [512]u8 = undefined;
try testing.expectError(error.OutOfMemory, pool.exchange(io, query_bytes, &buf));
try testing.expectEqual(@as(usize, 0), good.calls);
try testing.expectEqual(@as(u64, 0), entries[0].health.total_failures);
try testing.expectEqual(@as(u32, 0), entries[0].health.consecutive_failures);
}
test "a cancellation short-circuits and records nothing" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var canceled: Fake = .{ .behavior = .{ .fail = error.Canceled } };
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://canceled.example/dns-query", &canceled, 10),
testEntry("https://good.example/dns-query", &good, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
var buf: [512]u8 = undefined;
try testing.expectError(error.Canceled, pool.exchange(io, query_bytes, &buf));
try testing.expectEqual(@as(usize, 0), good.calls);
try testing.expectEqual(@as(u64, 0), entries[0].health.total_failures);
}
test "an attempt that outruns the budget is a recorded Timeout" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var slow: Fake = .{ .behavior = .{ .slow = .{
.duration = .{ .raw = .fromSeconds(30), .clock = .awake },
.reply = response_bytes,
} } };
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://slow.example/dns-query", &slow, 10),
testEntry("https://good.example/dns-query", &good, 20),
};
const budget: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(20), .clock = .awake };
var pool: Pool = .init(&entries, test_cfg, budget, 1);
var buf: [512]u8 = undefined;
const reply = try pool.exchange(io, query_bytes, &buf);
try testing.expectEqualSlices(u8, response_bytes, reply);
try testing.expectEqual(@as(usize, 1), slow.calls);
try testing.expectEqual(@as(u32, 1), entries[0].health.consecutive_failures);
try testing.expectEqualStrings("Timeout", entries[0].health.lastError());
try testing.expectEqual(@as(u64, 1), entries[1].health.total_successes);
}
test "every entry disabled yields ConnectFailed" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var one: Fake = .{ .behavior = .{ .reply = response_bytes } };
var two: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://one.example/dns-query", &one, 10),
testEntry("https://two.example/dns-query", &two, 20),
};
for (&entries) |*entry| entry.enabled = false;
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
var buf: [512]u8 = undefined;
try testing.expectError(error.ConnectFailed, pool.exchange(io, query_bytes, &buf));
try testing.expectEqual(@as(usize, 0), one.calls);
try testing.expectEqual(@as(usize, 0), two.calls);
}
test "snapshot reports the counters in pool order" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var bad: Fake = .{ .behavior = .{ .fail = error.Timeout } };
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://bad.example/dns-query", &bad, 10),
testEntry("https://good.example/dns-query", &good, 20),
};
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
var buf: [512]u8 = undefined;
_ = try pool.exchange(io, query_bytes, &buf);
_ = try pool.exchange(io, query_bytes, &buf);
var out: [4]Snapshot = undefined;
const written = try pool.snapshot(io, &out);
try testing.expectEqual(@as(usize, 2), written);
try testing.expectEqualStrings("https://bad.example/dns-query", out[0].url);
try testing.expect(out[0].enabled);
try testing.expect(!out[0].available);
try testing.expectEqual(@as(u32, 2), out[0].consecutive_failures);
try testing.expectEqual(@as(u64, 2), out[0].total_failures);
try testing.expectEqual(@as(u64, 0), out[0].total_successes);
try testing.expectEqual(@as(f32, 0.0), out[0].success_rate);
try testing.expectEqualStrings("Timeout", out[0].last_error);
try testing.expect(out[0].last_error_at != null);
try testing.expect(out[0].backoff_until != null);
try testing.expectEqualStrings("https://good.example/dns-query", out[1].url);
try testing.expect(out[1].available);
try testing.expectEqual(@as(u64, 2), out[1].total_successes);
try testing.expectEqual(@as(f32, 1.0), out[1].success_rate);
try testing.expect(out[1].last_success_at != null);
try testing.expectEqual(@as(?std.Io.Timestamp, null), out[1].backoff_until);
// A short `out` truncates rather than overflowing.
var one: [1]Snapshot = undefined;
try testing.expectEqual(@as(usize, 1), try pool.snapshot(io, &one));
}
/// `Io.concurrent` stores the return value in the future, so the reply slice is
/// reduced to its length here and the bytes are read back out of the caller's
/// buffer. Returning a `usize` also lets the test discard a result with
/// `catch 0`.
fn exchangeLen(pool: *Pool, io: std.Io, buf: []u8) transport.ExchangeError!usize {
const reply = try pool.exchange(io, query_bytes, buf);
return reply.len;
}
test "concurrent exchanges through one entry do not overlap" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
// Slow enough that an unserialized second task would still be inside the
// fake when the first one is, and short enough to stay far under the
// 10-second attempt budget even when the two run back to back.
var fake: Fake = .{ .behavior = .{ .slow = .{
.duration = .{ .raw = .fromMilliseconds(50), .clock = .awake },
.reply = response_bytes,
} } };
var entries = [_]Entry{testEntry("https://only.example/dns-query", &fake, 10)};
var pool: Pool = .init(&entries, test_cfg, test_timeout, 1);
var buf_a: [512]u8 = undefined;
var buf_b: [512]u8 = undefined;
var first = io.concurrent(exchangeLen, .{ &pool, io, &buf_a }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = first.await(io) catch 0;
var second = io.concurrent(exchangeLen, .{ &pool, io, &buf_b }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = second.await(io) catch 0;
const len_a = try first.await(io);
const len_b = try second.await(io);
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..len_a]);
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..len_b]);
try testing.expectEqual(@as(usize, 2), fake.calls);
try testing.expectEqual(@as(u32, 1), fake.peak_in_flight.load(.acquire));
try testing.expectEqual(@as(u64, 2), entries[0].health.total_successes);
}
test "an entry that enters backoff while a task waits on it is not attempted" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
// One failure is enough to open the backoff window, so the first task's
// failure makes the entry unavailable to the second task, which by then is
// already past its own availability check and waiting on `busy`.
const cfg: health.Config = .{
.failure_threshold = 1,
.base_backoff_ms = 60_000,
.max_backoff_ms = 60_000,
};
var slow_bad: Fake = .{ .behavior = .{ .slow_fail = .{
.duration = .{ .raw = .fromMilliseconds(50), .clock = .awake },
.err = error.ConnectFailed,
} } };
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
var entries = [_]Entry{
testEntry("https://slow-bad.example/dns-query", &slow_bad, 10),
testEntry("https://good.example/dns-query", &good, 20),
};
var pool: Pool = .init(&entries, cfg, test_timeout, 1);
var buf_a: [512]u8 = undefined;
var buf_b: [512]u8 = undefined;
var first = io.concurrent(exchangeLen, .{ &pool, io, &buf_a }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = first.await(io) catch 0;
var second = io.concurrent(exchangeLen, .{ &pool, io, &buf_b }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return error.SkipZigTest,
};
defer _ = second.await(io) catch 0;
const len_a = try first.await(io);
const len_b = try second.await(io);
// Both tasks fail over to the healthy entry and get an answer.
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..len_a]);
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..len_b]);
try testing.expectEqual(@as(usize, 2), good.calls);
// The point of the test: the entry was attempted once, not twice. Without
// the re-check the second task would take the lock and attempt it anyway.
try testing.expectEqual(@as(usize, 1), slow_bad.calls);
try testing.expectEqual(@as(u64, 1), entries[0].health.total_failures);
try testing.expect(entries[0].health.backoff_until != null);
}