milestone 28: query provenance — every logged query is exactly explainable
Gates / frontend (push) Successful in 1m36s
Gates / test (push) Successful in 1m56s
Gates / test-aarch64 (push) Successful in 7m37s
Gates / package (push) Successful in 9m12s
Gates / container (push) Successful in 13s
CI / gates (push) Successful in 19m4s
Gates / frontend (push) Successful in 1m36s
Gates / test (push) Successful in 1m56s
Gates / test-aarch64 (push) Successful in 7m37s
Gates / package (push) Successful in 9m12s
Gates / container (push) Successful in 13s
CI / gates (push) Successful in 19m4s
query rows gain qclass, rcode, group, policy action and reason, the matched rule or list entry with its source, cname and safe-search targets, route kind, forward zone, and the resolver that actually answered — the pool and local markers die. servfails are logged and name the resolver that lost; post-parse protocol refusals become rows. a detail page at /queries/:id renders the ordered explanation, and coverage watermarks distinguish an empty history from a missing one. the schema fingerprint changes: existing query history is recreated with the old file kept aside and the reset filed as a resolved diagnostic. fixes an oversized udp reply being rebuilt as noerror, which handed clients a truncated nxdomain as success.
This commit is contained in:
+291
-46
@@ -172,9 +172,10 @@ pub const Pool = struct {
|
||||
io: std.Io,
|
||||
query: []const u8,
|
||||
response_buf: []u8,
|
||||
selected: *?[]const u8,
|
||||
) transport.ExchangeError![]u8 {
|
||||
const self: *Pool = @ptrCast(@alignCast(ptr));
|
||||
return self.exchange(io, query, response_buf);
|
||||
return self.exchange(io, query, response_buf, selected);
|
||||
}
|
||||
|
||||
/// `response_buf` is handed to each attempt in turn, so a failed attempt
|
||||
@@ -191,15 +192,16 @@ pub const Pool = struct {
|
||||
io: std.Io,
|
||||
query: []const u8,
|
||||
response_buf: []u8,
|
||||
selected: *?[]const u8,
|
||||
) transport.ExchangeError![]u8 {
|
||||
const len = try transport.raceWithin(io, self.timeouts.total, exchangeLoopLen, .{
|
||||
self, io, query, response_buf,
|
||||
self, io, query, response_buf, selected,
|
||||
});
|
||||
return response_buf[0..len];
|
||||
}
|
||||
|
||||
/// The two-pass failover loop, as a raceable task. It returns the reply's
|
||||
/// length rather than its slice for the reason `exchangeLen` in the tests
|
||||
/// length rather than its slice for the reason `Attributed` in the tests
|
||||
/// below does: `Io.concurrent` stores the future's return value, so the
|
||||
/// bytes are read back out of the caller's `response_buf` by `exchange`.
|
||||
fn exchangeLoopLen(
|
||||
@@ -207,6 +209,7 @@ pub const Pool = struct {
|
||||
io: std.Io,
|
||||
query: []const u8,
|
||||
response_buf: []u8,
|
||||
selected: *?[]const u8,
|
||||
) transport.ExchangeError!usize {
|
||||
const now = std.Io.Clock.awake.now(io);
|
||||
var last_fault: ?transport.ExchangeError = null;
|
||||
@@ -240,6 +243,13 @@ pub const Pool = struct {
|
||||
}
|
||||
attempted = true;
|
||||
|
||||
// Before the attempt, not after it: a failover reports whoever
|
||||
// answered, an all-failed exchange reports the last endpoint
|
||||
// tried, and a cancellation mid-flight reports the endpoint the
|
||||
// query was in. The url is owned by the Endpoint, which outlives
|
||||
// the pool, so the borrow stays valid past this loop.
|
||||
selected.* = entry.endpoint.url;
|
||||
|
||||
const result = self.attempt(io, entry.client, query, response_buf);
|
||||
const completed_at = std.Io.Clock.awake.now(io);
|
||||
|
||||
@@ -294,6 +304,12 @@ pub const Pool = struct {
|
||||
}
|
||||
|
||||
/// One exchange raced against the per-attempt budget.
|
||||
///
|
||||
/// The leaf client reports an identity of its own, which the pool discards:
|
||||
/// the entry's endpoint is the pool's own naming of the same resolver, and
|
||||
/// it is what the caller was handed. A leaf writing its identity into the
|
||||
/// caller's slot would let a test fake overwrite the endpoint that actually
|
||||
/// answered.
|
||||
fn attempt(
|
||||
self: *Pool,
|
||||
io: std.Io,
|
||||
@@ -301,8 +317,9 @@ pub const Pool = struct {
|
||||
query: []const u8,
|
||||
response_buf: []u8,
|
||||
) transport.ExchangeError![]u8 {
|
||||
var leaf_selected: ?[]const u8 = null;
|
||||
return transport.raceWithin(io, self.timeouts.attempt, transport.Client.exchange, .{
|
||||
entry_client, io, query, response_buf,
|
||||
entry_client, io, query, response_buf, &leaf_selected,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -409,11 +426,23 @@ const response_bytes =
|
||||
"\x07example\x03com\x00\x00\x01\x00\x01" ++
|
||||
"\xc0\x0c\x00\x01\x00\x01\x00\x00\x01\x2c\x00\x04\x5d\xb8\xd8\x22";
|
||||
|
||||
/// The same question answered differently, so a reply identifies the entry that
|
||||
/// produced it: one A record, a different address.
|
||||
const alt_response_bytes =
|
||||
"\x12\x34\x81\x80\x00\x01\x00\x01\x00\x00\x00\x00" ++
|
||||
"\x07example\x03com\x00\x00\x01\x00\x01" ++
|
||||
"\xc0\x0c\x00\x01\x00\x01\x00\x00\x01\x2c\x00\x04\x0a\x00\x00\x01";
|
||||
|
||||
/// Stands in for a DoH or DoT client. Every behaviour the pool has to react to
|
||||
/// is one variant, and every call is counted so a test can assert that an entry
|
||||
/// in backoff was not touched.
|
||||
const Fake = struct {
|
||||
behavior: Behavior,
|
||||
/// Replaces `behavior` after the first call. One entry that fails the task
|
||||
/// which reaches it first and answers the next is what makes two concurrent
|
||||
/// exchanges end on different entries; a single behaviour cannot say that.
|
||||
/// Mutated under the entry's `busy` lock, like `calls`.
|
||||
then: ?Behavior = null,
|
||||
calls: usize = 0,
|
||||
in_flight: std.atomic.Value(u32) = .init(0),
|
||||
/// The most tasks ever inside `exchangeFn` at once. The per-entry lock is
|
||||
@@ -437,15 +466,24 @@ const Fake = struct {
|
||||
io: std.Io,
|
||||
query: []const u8,
|
||||
response_buf: []u8,
|
||||
selected: *?[]const u8,
|
||||
) transport.ExchangeError![]u8 {
|
||||
_ = query;
|
||||
// Deliberately not the endpoint url: the pool must report its own
|
||||
// entry, so a test can tell the two apart.
|
||||
selected.* = "fake://leaf";
|
||||
const self: *Fake = @ptrCast(@alignCast(ptr));
|
||||
const entrants = self.in_flight.fetchAdd(1, .acq_rel) + 1;
|
||||
defer _ = self.in_flight.fetchSub(1, .acq_rel);
|
||||
_ = self.peak_in_flight.fetchMax(entrants, .acq_rel);
|
||||
|
||||
self.calls += 1;
|
||||
switch (self.behavior) {
|
||||
const behavior = self.behavior;
|
||||
if (self.then) |next| {
|
||||
self.behavior = next;
|
||||
self.then = null;
|
||||
}
|
||||
switch (behavior) {
|
||||
.reply => |bytes| return copy(bytes, response_buf),
|
||||
.fail => |err| return err,
|
||||
.slow => |slow| {
|
||||
@@ -539,10 +577,106 @@ test "Pool satisfies the Client interface" {
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
const reply = try pool.client().exchange(io, query_bytes, &buf);
|
||||
var selected: ?[]const u8 = null;
|
||||
const reply = try pool.client().exchange(io, query_bytes, &buf, &selected);
|
||||
try testing.expectEqualSlices(u8, response_bytes, reply);
|
||||
}
|
||||
|
||||
test "the pool reports the endpoint that answered, not the leaf client's own name" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var fake: Fake = .{ .behavior = .{ .reply = response_bytes } };
|
||||
var entries = [_]Entry{testEntry("https://a.example/dns-query", &fake, 10)};
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
var selected: ?[]const u8 = null;
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
try testing.expectEqualStrings("https://a.example/dns-query", selected.?);
|
||||
}
|
||||
|
||||
test "a failover reports the endpoint that answered, not the first one tried" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var bad: Fake = .{ .behavior = .{ .fail = error.Timeout } };
|
||||
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
|
||||
var entries = [_]Entry{
|
||||
testEntry("https://bad.example/dns-query", &bad, 10),
|
||||
testEntry("https://good.example/dns-query", &good, 20),
|
||||
};
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
var selected: ?[]const u8 = null;
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
try testing.expectEqualStrings("https://good.example/dns-query", selected.?);
|
||||
}
|
||||
|
||||
test "an all-failed exchange reports the last endpoint attempted" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var first: Fake = .{ .behavior = .{ .fail = error.ConnectFailed } };
|
||||
var last: Fake = .{ .behavior = .{ .fail = error.Timeout } };
|
||||
var entries = [_]Entry{
|
||||
testEntry("https://first.example/dns-query", &first, 10),
|
||||
testEntry("https://last.example/dns-query", &last, 20),
|
||||
};
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
var selected: ?[]const u8 = null;
|
||||
try testing.expectError(error.Timeout, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
try testing.expectEqualStrings("https://last.example/dns-query", selected.?);
|
||||
}
|
||||
|
||||
test "a timeout mid-flight reports the endpoint the query was in" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var stalling: Fake = .{ .behavior = .{ .slow = .{
|
||||
.duration = .{ .raw = .fromSeconds(30), .clock = .awake },
|
||||
.reply = response_bytes,
|
||||
} } };
|
||||
var untouched: Fake = .{ .behavior = .{ .reply = response_bytes } };
|
||||
var entries = [_]Entry{
|
||||
testEntry("https://stalling.example/dns-query", &stalling, 10),
|
||||
testEntry("https://untouched.example/dns-query", &untouched, 20),
|
||||
};
|
||||
var pool: Pool = .init(&entries, test_cfg, .{
|
||||
.attempt = .{ .raw = .fromMilliseconds(200), .clock = .awake },
|
||||
.total = .{ .raw = .fromMilliseconds(60), .clock = .awake },
|
||||
}, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
var selected: ?[]const u8 = null;
|
||||
try testing.expectError(error.Timeout, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
try testing.expectEqualStrings("https://stalling.example/dns-query", selected.?);
|
||||
try testing.expectEqual(@as(usize, 0), untouched.calls);
|
||||
}
|
||||
|
||||
test "an exchange that attempted nothing reports no endpoint" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var fake: Fake = .{ .behavior = .{ .reply = response_bytes } };
|
||||
var entries = [_]Entry{testEntry("https://a.example/dns-query", &fake, 10)};
|
||||
entries[0].enabled = false;
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
var selected: ?[]const u8 = null;
|
||||
try testing.expectError(error.ConnectFailed, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
try testing.expect(selected == null);
|
||||
}
|
||||
|
||||
test "entries are tried in ascending priority order" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
@@ -560,7 +694,8 @@ test "entries are tried in ascending priority order" {
|
||||
try testing.expectEqual(@as(i32, 10), entries[0].priority);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
_ = try pool.exchange(io, query_bytes, &buf);
|
||||
var selected: ?[]const u8 = null;
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
try testing.expectEqual(@as(usize, 1), low.calls);
|
||||
try testing.expectEqual(@as(usize, 0), high.calls);
|
||||
}
|
||||
@@ -579,7 +714,8 @@ test "a peer fault fails over to the next entry and is recorded" {
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
const reply = try pool.exchange(io, query_bytes, &buf);
|
||||
var selected: ?[]const u8 = null;
|
||||
const reply = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
try testing.expectEqualSlices(u8, response_bytes, reply);
|
||||
|
||||
try testing.expectEqual(@as(u32, 1), entries[0].health.consecutive_failures);
|
||||
@@ -603,12 +739,13 @@ test "an entry in backoff is skipped while another is available" {
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
// Two failures reach `failure_threshold` and open a backoff window.
|
||||
_ = try pool.exchange(io, query_bytes, &buf);
|
||||
_ = try pool.exchange(io, query_bytes, &buf);
|
||||
var selected: ?[]const u8 = null;
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
try testing.expectEqual(@as(usize, 2), bad.calls);
|
||||
try testing.expect(entries[0].health.backoff_until != null);
|
||||
|
||||
_ = try pool.exchange(io, query_bytes, &buf);
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
try testing.expectEqual(@as(usize, 2), bad.calls);
|
||||
try testing.expectEqual(@as(usize, 3), good.calls);
|
||||
}
|
||||
@@ -627,13 +764,14 @@ test "every entry in backoff is still probed" {
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
|
||||
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
|
||||
var selected: ?[]const u8 = null;
|
||||
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
try testing.expect(entries[0].health.backoff_until != null);
|
||||
try testing.expect(entries[1].health.backoff_until != null);
|
||||
|
||||
// Pass one now has no candidate at all. Pass two probes both anyway.
|
||||
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf));
|
||||
try testing.expectError(error.BadResponse, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
try testing.expectEqual(@as(usize, 3), first.calls);
|
||||
try testing.expectEqual(@as(usize, 3), second.calls);
|
||||
}
|
||||
@@ -652,7 +790,8 @@ test "a local resource error short-circuits and records nothing" {
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
try testing.expectError(error.OutOfMemory, pool.exchange(io, query_bytes, &buf));
|
||||
var selected: ?[]const u8 = null;
|
||||
try testing.expectError(error.OutOfMemory, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
try testing.expectEqual(@as(usize, 0), good.calls);
|
||||
try testing.expectEqual(@as(u64, 0), entries[0].health.total_failures);
|
||||
try testing.expectEqual(@as(u32, 0), entries[0].health.consecutive_failures);
|
||||
@@ -672,7 +811,8 @@ test "a cancellation short-circuits and records nothing" {
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
try testing.expectError(error.Canceled, pool.exchange(io, query_bytes, &buf));
|
||||
var selected: ?[]const u8 = null;
|
||||
try testing.expectError(error.Canceled, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
try testing.expectEqual(@as(usize, 0), good.calls);
|
||||
try testing.expectEqual(@as(u64, 0), entries[0].health.total_failures);
|
||||
}
|
||||
@@ -699,7 +839,8 @@ test "an attempt that outruns the budget is a recorded Timeout" {
|
||||
}, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
const reply = try pool.exchange(io, query_bytes, &buf);
|
||||
var selected: ?[]const u8 = null;
|
||||
const reply = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
try testing.expectEqualSlices(u8, response_bytes, reply);
|
||||
|
||||
try testing.expectEqual(@as(usize, 1), slow.calls);
|
||||
@@ -736,7 +877,8 @@ test "two stalling upstreams cost the total budget, not one budget each" {
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
const started = std.Io.Clock.awake.now(io);
|
||||
try testing.expectError(error.Timeout, pool.exchange(io, query_bytes, &buf));
|
||||
var selected: ?[]const u8 = null;
|
||||
try testing.expectError(error.Timeout, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
const elapsed_ns = std.Io.Clock.awake.now(io).nanoseconds - started.nanoseconds;
|
||||
|
||||
// Under one attempt budget, so the outer deadline is provably what fired.
|
||||
@@ -770,7 +912,8 @@ test "every entry disabled yields ConnectFailed without waiting out the total bu
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
const started = std.Io.Clock.awake.now(io);
|
||||
try testing.expectError(error.ConnectFailed, pool.exchange(io, query_bytes, &buf));
|
||||
var selected: ?[]const u8 = null;
|
||||
try testing.expectError(error.ConnectFailed, pool.exchange(io, query_bytes, &buf, &selected));
|
||||
const elapsed_ns = std.Io.Clock.awake.now(io).nanoseconds - started.nanoseconds;
|
||||
|
||||
try testing.expect(elapsed_ns < @as(i96, 5) * std.time.ns_per_s);
|
||||
@@ -799,7 +942,8 @@ test "a wired accumulator receives both outcomes the pool records" {
|
||||
var buf: [512]u8 = undefined;
|
||||
// One exchange: the first entry fails over into the second, so this drives
|
||||
// one failure and one success.
|
||||
_ = try pool.exchange(io, query_bytes, &buf);
|
||||
var selected: ?[]const u8 = null;
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
|
||||
// Two cells, one per url, in whatever minute the wall clock is in.
|
||||
try testing.expectEqual(@as(u32, 2), acc.snapshotStats(io).pending);
|
||||
@@ -852,8 +996,9 @@ test "snapshot reports the counters in pool order" {
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
_ = try pool.exchange(io, query_bytes, &buf);
|
||||
_ = try pool.exchange(io, query_bytes, &buf);
|
||||
var selected: ?[]const u8 = null;
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
|
||||
var out: [4]Snapshot = undefined;
|
||||
const written = try pool.snapshot(io, &out);
|
||||
@@ -882,13 +1027,30 @@ test "snapshot reports the counters in pool order" {
|
||||
try testing.expectEqual(@as(usize, 1), try pool.snapshot(io, &one));
|
||||
}
|
||||
|
||||
/// One concurrent call's whole result: how much of its buffer the reply filled,
|
||||
/// and which resolver the pool said answered it.
|
||||
///
|
||||
/// `Io.concurrent` stores the return value in the future, so the reply slice is
|
||||
/// reduced to its length here and the bytes are read back out of the caller's
|
||||
/// buffer. Returning a `usize` also lets the test discard a result with
|
||||
/// `catch 0`.
|
||||
fn exchangeLen(pool: *Pool, io: std.Io, buf: []u8) transport.ExchangeError!usize {
|
||||
const reply = try pool.exchange(io, query_bytes, buf);
|
||||
return reply.len;
|
||||
/// buffer. `selected` survives the trip because it borrows an `Endpoint.url`,
|
||||
/// which outlives the pool.
|
||||
const Attributed = struct {
|
||||
reply_len: usize,
|
||||
selected: ?[]const u8,
|
||||
|
||||
/// What a teardown `await` discards into once the assertions above it have
|
||||
/// already taken the value.
|
||||
const discarded: Attributed = .{ .reply_len = 0, .selected = null };
|
||||
};
|
||||
|
||||
/// Each call keeps its own `selected` out-value rather than dropping it: the
|
||||
/// pointer is written per call, on the stack of the task that made it, and a
|
||||
/// pool that hung it off `*Pool` instead would hand one call's identity to
|
||||
/// another. Nothing but a per-call capture can see that.
|
||||
fn exchangeAttributed(pool: *Pool, io: std.Io, buf: []u8) transport.ExchangeError!Attributed {
|
||||
var selected: ?[]const u8 = null;
|
||||
const reply = try pool.exchange(io, query_bytes, buf, &selected);
|
||||
return .{ .reply_len = reply.len, .selected = selected };
|
||||
}
|
||||
|
||||
test "concurrent exchanges through one entry do not overlap" {
|
||||
@@ -909,25 +1071,102 @@ test "concurrent exchanges through one entry do not overlap" {
|
||||
var buf_a: [512]u8 = undefined;
|
||||
var buf_b: [512]u8 = undefined;
|
||||
|
||||
var first = io.concurrent(exchangeLen, .{ &pool, io, &buf_a }) catch |err| switch (err) {
|
||||
var first = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_a }) catch |err| switch (err) {
|
||||
error.ConcurrencyUnavailable => return error.SkipZigTest,
|
||||
};
|
||||
defer _ = first.await(io) catch 0;
|
||||
var second = io.concurrent(exchangeLen, .{ &pool, io, &buf_b }) catch |err| switch (err) {
|
||||
defer _ = first.await(io) catch Attributed.discarded;
|
||||
var second = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_b }) catch |err| switch (err) {
|
||||
error.ConcurrencyUnavailable => return error.SkipZigTest,
|
||||
};
|
||||
defer _ = second.await(io) catch 0;
|
||||
defer _ = second.await(io) catch Attributed.discarded;
|
||||
|
||||
const len_a = try first.await(io);
|
||||
const len_b = try second.await(io);
|
||||
const result_a = try first.await(io);
|
||||
const result_b = try second.await(io);
|
||||
|
||||
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..len_a]);
|
||||
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..len_b]);
|
||||
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..result_a.reply_len]);
|
||||
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..result_b.reply_len]);
|
||||
try testing.expectEqualStrings("https://only.example/dns-query", result_a.selected.?);
|
||||
try testing.expectEqualStrings("https://only.example/dns-query", result_b.selected.?);
|
||||
try testing.expectEqual(@as(usize, 2), fake.calls);
|
||||
try testing.expectEqual(@as(u32, 1), fake.peak_in_flight.load(.acquire));
|
||||
try testing.expectEqual(@as(u64, 2), entries[0].health.total_successes);
|
||||
}
|
||||
|
||||
/// The two entries of the test below, each answering with bytes only it
|
||||
/// produces so a call's reply proves which entry served it independently of
|
||||
/// what the pool reported.
|
||||
const divergent_first_url = "https://first.example/dns-query";
|
||||
const divergent_second_url = "https://second.example/dns-query";
|
||||
|
||||
fn expectAnsweredByReporter(result: Attributed, buf: []const u8) !void {
|
||||
const url = result.selected orelse return error.TestExpectedSelectedResolver;
|
||||
const reply = if (std.mem.eql(u8, url, divergent_first_url))
|
||||
alt_response_bytes
|
||||
else if (std.mem.eql(u8, url, divergent_second_url))
|
||||
response_bytes
|
||||
else
|
||||
return error.TestUnexpectedResolver;
|
||||
try testing.expectEqualSlices(u8, reply, buf[0..result.reply_len]);
|
||||
}
|
||||
|
||||
test "overlapping exchanges each report the entry that answered that call" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
// The first entry fails the task that reaches it first, slowly enough that
|
||||
// the second task is queued behind it, then answers that second task. One
|
||||
// failure is under `test_cfg`'s threshold of two, so no backoff steers the
|
||||
// waiting task away and the two calls end on different entries.
|
||||
var first_entry: Fake = .{
|
||||
.behavior = .{ .slow_fail = .{
|
||||
.duration = .{ .raw = .fromMilliseconds(50), .clock = .awake },
|
||||
.err = error.ConnectFailed,
|
||||
} },
|
||||
.then = .{ .reply = alt_response_bytes },
|
||||
};
|
||||
// Slow too, so the failed-over call is still in flight while the other call
|
||||
// is being answered — a shared identity would be overwritten under it.
|
||||
var second_entry: Fake = .{ .behavior = .{ .slow = .{
|
||||
.duration = .{ .raw = .fromMilliseconds(50), .clock = .awake },
|
||||
.reply = response_bytes,
|
||||
} } };
|
||||
var entries = [_]Entry{
|
||||
testEntry(divergent_first_url, &first_entry, 10),
|
||||
testEntry(divergent_second_url, &second_entry, 20),
|
||||
};
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var buf_a: [512]u8 = undefined;
|
||||
var buf_b: [512]u8 = undefined;
|
||||
|
||||
var first = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_a }) catch |err| switch (err) {
|
||||
error.ConcurrencyUnavailable => return error.SkipZigTest,
|
||||
};
|
||||
defer _ = first.await(io) catch Attributed.discarded;
|
||||
var second = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_b }) catch |err| switch (err) {
|
||||
error.ConcurrencyUnavailable => return error.SkipZigTest,
|
||||
};
|
||||
defer _ = second.await(io) catch Attributed.discarded;
|
||||
|
||||
const result_a = try first.await(io);
|
||||
const result_b = try second.await(io);
|
||||
|
||||
// Which task lands where depends on the order they take the first entry's
|
||||
// lock, so the claim is over the pair: one identity each, and each one
|
||||
// matching the bytes that call received.
|
||||
try testing.expect(!std.mem.eql(u8, result_a.selected.?, result_b.selected.?));
|
||||
try expectAnsweredByReporter(result_a, &buf_a);
|
||||
try expectAnsweredByReporter(result_b, &buf_b);
|
||||
|
||||
try testing.expectEqual(@as(usize, 2), first_entry.calls);
|
||||
try testing.expectEqual(@as(usize, 1), second_entry.calls);
|
||||
try testing.expectEqual(@as(u32, 1), first_entry.peak_in_flight.load(.acquire));
|
||||
try testing.expectEqual(@as(u64, 1), entries[0].health.total_failures);
|
||||
try testing.expectEqual(@as(u64, 1), entries[0].health.total_successes);
|
||||
try testing.expectEqual(@as(u64, 1), entries[1].health.total_successes);
|
||||
}
|
||||
|
||||
test "an entry that enters backoff while a task waits on it is not attempted" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
@@ -956,21 +1195,25 @@ test "an entry that enters backoff while a task waits on it is not attempted" {
|
||||
var buf_a: [512]u8 = undefined;
|
||||
var buf_b: [512]u8 = undefined;
|
||||
|
||||
var first = io.concurrent(exchangeLen, .{ &pool, io, &buf_a }) catch |err| switch (err) {
|
||||
var first = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_a }) catch |err| switch (err) {
|
||||
error.ConcurrencyUnavailable => return error.SkipZigTest,
|
||||
};
|
||||
defer _ = first.await(io) catch 0;
|
||||
var second = io.concurrent(exchangeLen, .{ &pool, io, &buf_b }) catch |err| switch (err) {
|
||||
defer _ = first.await(io) catch Attributed.discarded;
|
||||
var second = io.concurrent(exchangeAttributed, .{ &pool, io, &buf_b }) catch |err| switch (err) {
|
||||
error.ConcurrencyUnavailable => return error.SkipZigTest,
|
||||
};
|
||||
defer _ = second.await(io) catch 0;
|
||||
defer _ = second.await(io) catch Attributed.discarded;
|
||||
|
||||
const len_a = try first.await(io);
|
||||
const len_b = try second.await(io);
|
||||
const result_a = try first.await(io);
|
||||
const result_b = try second.await(io);
|
||||
|
||||
// Both tasks fail over to the healthy entry and get an answer.
|
||||
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..len_a]);
|
||||
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..len_b]);
|
||||
// Both tasks fail over to the healthy entry and get an answer, and both
|
||||
// report it: the identity is the entry that answered, not the one that
|
||||
// failed on the way there.
|
||||
try testing.expectEqualSlices(u8, response_bytes, buf_a[0..result_a.reply_len]);
|
||||
try testing.expectEqualSlices(u8, response_bytes, buf_b[0..result_b.reply_len]);
|
||||
try testing.expectEqualStrings("https://good.example/dns-query", result_a.selected.?);
|
||||
try testing.expectEqualStrings("https://good.example/dns-query", result_b.selected.?);
|
||||
try testing.expectEqual(@as(usize, 2), good.calls);
|
||||
|
||||
// The point of the test: the entry was attempted once, not twice. Without
|
||||
@@ -998,7 +1241,8 @@ test "a successful exchange with nothing open costs the store no statement" {
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
const before = fx.store.statements;
|
||||
for (0..20) |_| _ = try pool.exchange(io, query_bytes, &buf);
|
||||
var selected: ?[]const u8 = null;
|
||||
for (0..20) |_| _ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
try testing.expectEqual(before, fx.store.statements);
|
||||
try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
}
|
||||
@@ -1022,7 +1266,8 @@ test "a failing then recovering upstream leaves exactly one resolved episode" {
|
||||
pool.diagnostics = &fx.store;
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
_ = try pool.exchange(io, query_bytes, &buf);
|
||||
var selected: ?[]const u8 = null;
|
||||
_ = try pool.exchange(io, query_bytes, &buf, &selected);
|
||||
// Backoff would park the failing entry, so the second failure is driven
|
||||
// through `recordFailure` itself rather than through another exchange.
|
||||
pool.recordFailure(io, &entries[0], std.Io.Clock.awake.now(io), error.ConnectFailed);
|
||||
|
||||
Reference in New Issue
Block a user