mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 25 Activity

server: every dns listener shares one budget of 32 queries, stream listeners take 16 connections, the task pool is capped from those bounds, and work it cannot start is counted by site

Browse Source
This commit is contained in:
mokhtar
2026-10-22 15:24:10 +02:00
parent e4e405f8e9
commit 63cc5f8dc3
23 changed files with 983 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/server/doh_server.zig
+36 -21
View File
@@ -11,7 +11,7 @@
//!
//! The handshake runs under the same race budget tcp_server applies to its
//! reads (ruling 3's rationale): a client that connects and never handshakes
//! must not pin one of the 64 slots forever.
//! must not pin a slot forever.
//!
//! Over capacity the raw TCP stream is closed and counted, with no response.
//! The web listener's 503 is not possible here: HTTP exists only on the far
@@ -19,6 +19,11 @@
//! certificate acquisition — exactly what an over-capacity listener does not
//! have to spend. Closing is the cheapest honest refusal.
//!
//! A query holds a permit from the shared query budget from its decoded body
//! to its sent response. With none free the connection is closed unanswered
//! and counted, the TCP/53 rule: a 503 would be the one DNS refusal that
//! differs by transport.
//!
//! Only `/dns-query` exists. This is not the admin API, so errors are plain
//! text, not JSON, and there are no cache-control headers (LAN, no
//! intermediaries).
@@ -34,6 +39,7 @@ const doh_client = @import("../upstream/doh_client.zig");
const handler = @import("handler.zig");
const listener = @import("listener.zig");
const model = @import("../config/model.zig");
const query_budget = @import("query_budget.zig");
const tls_server = @import("../platform/tls_server.zig");
const transport = @import("../upstream/transport.zig");
const upstream_owner = @import("../upstream/owner.zig");
@@ -52,12 +58,10 @@ pub const alpn_protocols: [*:null]const ?[*:0]const u8 = &.{"http/1.1"};
const recv_buffer_len = 8 * 1024;
const send_buffer_len = 4 * 1024;
pub const default_max_connections: u16 = 64;
const allow_header: http.Header = .{ .name = "allow", .value = "GET, POST" };
pub const Options = struct {
max_connections: u16 = default_max_connections,
max_connections: u16 = listener.dns_max_connections,
/// The budget for the TLS handshake and for the wait on the next request
/// head of a keep-alive connection (milestone-16 ruling 10). A request
/// already being served has no timeout, like the web listener: the port is
@@ -87,27 +91,23 @@ pub const Snapshot = struct {
idle_timeouts: u64,
connection_errors: u64,
bad_requests: u64,
dropped_no_permit: u64,
};
pub const DohServer = struct {
core: listener.Core(Config),
handler: *handler.Handler,
budget: *query_budget.QueryBudget,
certs: *cert_store.CertStore,
options: Options,
stats: Stats,
/// One slot is ~150 KiB, so the default 64 connections cost ~9.4 MiB. The
/// two message buffers cannot shrink: a POST body and the reply both go up
/// to the 65535 bytes a DNS message can be. The `ServerStream` plaintext
/// buffers belong to the core; its `read_buf` doubles as the HTTP head cap
/// (see `recv_buffer_len`).
/// The `ServerStream` plaintext buffers belong to the core; its `read_buf`
/// doubles as the HTTP head cap (see `recv_buffer_len`).
pub const Payload = struct {
/// The decoded query: a POST body or a GET `dns` parameter.
/// The decoded query: a POST body or a GET `dns` parameter, up to the
/// 65535 bytes a DNS message can be. Read before a permit is taken.
query: [transport.max_message_len]u8,
reply: [transport.max_message_len]u8,
/// The handler's per-query working memory. A connection is answered
/// serially, so one query uses it at a time.
scratch: handler.Scratch,
/// Valid between a successful `ServerStream.accept` and the
/// `close(gpa)` in `serveOne`'s defer.
tls: tls_server.ServerStream,
@@ -131,12 +131,14 @@ pub const DohServer = struct {
io: std.Io,
listen_address: net.IpAddress,
h: *handler.Handler,
budget: *query_budget.QueryBudget,
certs: *cert_store.CertStore,
options: Options,
) ListenError!DohServer {
return .{
.core = try listener.Core(Config).listen(gpa, io, listen_address, options.max_connections),
.handler = h,
.budget = budget,
.certs = certs,
.options = options,
.stats = .{},
@@ -163,6 +165,7 @@ pub const DohServer = struct {
.idle_timeouts = core.idle_timeouts.load(.monotonic),
.connection_errors = core.connection_errors.load(.monotonic),
.bad_requests = self.stats.bad_requests.load(.monotonic),
.dropped_no_permit = core.dropped_no_permit.load(.monotonic),
};
}
@@ -192,7 +195,7 @@ pub const DohServer = struct {
const stage: Handshake = .{ .conn = conn, .gpa = gpa, .ctx = &entry.ctx, .io = io };
switch (listener.handshakeStage(io, self.options.idle_timeout, stage)) {
.ok => {},
.canceled => return,
.canceled, .unavailable => return,
// Milestone-16 ruling 9: a stalled handshake is refused like a broken
// one, the DoT arrangement. `idle_timeouts` belongs to the keep-alive
// wait below, so the two listeners export the same names for the
@@ -220,9 +223,10 @@ pub const DohServer = struct {
listener.bump(&stats.idle_timeouts);
return;
},
// Cancellation is shutdown; `.failed` here is only the wrapper
// failing to start, which costs nothing and counts as nothing.
.canceled, .failed => return,
// Cancellation is shutdown, and `.unavailable` is counted by
// the race. `.failed` cannot happen: `receiveHeadInto` stores
// the head's error instead of returning it.
.canceled, .unavailable, .failed => return,
}
var request = head catch |err| switch (err) {
@@ -385,13 +389,19 @@ pub const DohServer = struct {
request: *http.Server.Request,
query: []const u8,
) error{ WriteFailed, HttpExpectationFailed }!Next {
const work = self.budget.tryAcquire(io) orelse {
listener.bump(&self.core.stats.dropped_no_permit);
return .close;
};
defer self.budget.release(io, work);
const outcome = self.handler.handle(
io,
.tcp,
address.NetAddress.fromIp(conn.peer),
query,
&conn.payload.reply,
&conn.payload.scratch,
&work.reply,
&work.scratch,
);
switch (outcome) {
.drop => return self.refuse(request, .bad_request, "bad request\n", &.{}, false),
@@ -626,6 +636,7 @@ const Harness = struct {
upstream: FailingUpstream,
upstream_owner: upstream_owner.Borrowed,
h: handler.Handler,
permits: query_budget.QueryBudget,
server: DohServer,
group: std.Io.Group,
@@ -659,8 +670,11 @@ const Harness = struct {
.local_tables = &hx.tables,
};
hx.permits = try .init(testing.allocator, 4);
errdefer hx.permits.deinit();
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
hx.server = try DohServer.listen(testing.allocator, hio, listen_address, &hx.h, &hx.store, options);
hx.server = try DohServer.listen(testing.allocator, hio, listen_address, &hx.h, &hx.permits, &hx.store, options);
errdefer hx.server.deinit(hio);
hx.group = .init;
@@ -673,6 +687,7 @@ const Harness = struct {
hx.group.await(hio) catch |err| switch (err) {
error.Canceled => unreachable,
};
hx.permits.deinit();
hx.store.deinit(hio);
hx.tables.deinit(testing.allocator);
hx.tmp.cleanup();
src/server/dot_server.zig
+44 -22
View File
@@ -18,6 +18,8 @@
//! TCP/53 each connection costs one allocation. The pool itself is still
//! fixed and pre-allocated.
//!
//! A query holds a permit from the shared query budget exactly as on TCP/53.
//!
//! ALPN belongs to the `CertStore`'s `ServerContext` (set at store init);
//! this listener only acquires entries.
@@ -26,17 +28,18 @@ const address = @import("../platform/address.zig");
const cert_store = @import("cert_store.zig");
const handler = @import("handler.zig");
const listener = @import("listener.zig");
const query_budget = @import("query_budget.zig");
const tls_server = @import("../platform/tls_server.zig");
const transport = @import("../upstream/transport.zig");
const upstream_owner = @import("../upstream/owner.zig");
/// Plaintext staging for `ServerStream`: the framing bytes and the decrypted
/// record tail pass through here, while whole messages go straight to
/// `Payload.query`/`Payload.reply`.
/// `Payload.query` and from the permit's reply.
const stream_buffer_len = 1024;
pub const Options = struct {
max_connections: u16 = 64,
max_connections: u16 = listener.dns_max_connections,
/// RFC 7766 §6.2.3 recommends a few seconds of idle tolerance, and the
/// handshake runs under the same budget.
idle_timeout: std.Io.Clock.Duration = .{ .raw = .fromSeconds(10), .clock = .awake },
@@ -53,28 +56,24 @@ pub const Stats = struct {
/// walks for the `nxdns_dot_server_*` families.
pub const StatsSnapshot = struct {
connections: u64,
rejected_at_capacity: u64,
tls_handshake_failures: u64,
idle_timeouts: u64,
connection_errors: u64,
dropped_no_permit: u64,
};
pub const DotServer = struct {
core: listener.Core(Config),
handler: *handler.Handler,
budget: *query_budget.QueryBudget,
certs: *cert_store.CertStore,
options: Options,
stats: Stats,
/// One slot is ~137 KiB — the same two message ceilings as TCP/53 plus the
/// `ServerStream` bookkeeping — so the default 64 connections stay inside
/// the PLAN §18 budget.
pub const Payload = struct {
/// Read before a permit is taken, as on TCP/53.
query: [transport.max_message_len]u8,
reply: [transport.max_message_len]u8,
/// The handler's per-query working memory. It belongs to the slot so
/// that answering a message allocates nothing, and a connection is
/// answered serially, so one query uses it at a time.
scratch: handler.Scratch,
/// Pinned once its `accept` succeeds: mbedTLS holds a pointer to it,
/// and the slot never moves.
tls: tls_server.ServerStream,
@@ -98,12 +97,14 @@ pub const DotServer = struct {
io: std.Io,
listen_address: std.Io.net.IpAddress,
h: *handler.Handler,
budget: *query_budget.QueryBudget,
certs: *cert_store.CertStore,
options: Options,
) ListenError!DotServer {
return .{
.core = try listener.Core(Config).listen(gpa, io, listen_address, options.max_connections),
.handler = h,
.budget = budget,
.certs = certs,
.options = options,
.stats = .{},
@@ -133,9 +134,11 @@ pub const DotServer = struct {
const core = &self.core.stats;
return .{
.connections = core.connections.load(.monotonic),
.rejected_at_capacity = core.rejected_at_capacity.load(.monotonic),
.tls_handshake_failures = self.stats.tls_handshake_failures.load(.monotonic),
.idle_timeouts = core.idle_timeouts.load(.monotonic),
.connection_errors = core.connection_errors.load(.monotonic),
.dropped_no_permit = core.dropped_no_permit.load(.monotonic),
};
}
@@ -157,7 +160,7 @@ pub const DotServer = struct {
const stage: Handshake = .{ .conn = conn, .gpa = gpa, .ctx = &entry.ctx, .io = io };
switch (listener.handshakeStage(io, budget, stage)) {
.ok => {},
.canceled => return,
.canceled, .unavailable => return,
// A stalled handshake is refused like a broken one: it must not
// pin a connection slot for longer than the idle budget.
.timed_out, .failed => {
@@ -181,7 +184,7 @@ pub const DotServer = struct {
listener.bump(&stats.idle_timeouts);
return;
},
.canceled => return,
.canceled, .unavailable => return,
// A transport EOF without close_notify lands here too: the
// stream reads it as a truncation, never as a clean end.
.failed => {
@@ -208,7 +211,7 @@ pub const DotServer = struct {
switch (listener.race(io, budget, listener.readBody, .{ reader, conn.payload.query[0..len] })) {
.ok => {},
.canceled => return,
.canceled, .unavailable => return,
// A half-sent message is a broken peer, not an idle one.
.timed_out, .failed => {
listener.bump(&stats.connection_errors);
@@ -216,13 +219,20 @@ pub const DotServer = struct {
},
}
// There is no framing for "no answer", so the connection ends.
const work = self.budget.tryAcquire(io) orelse {
listener.bump(&stats.dropped_no_permit);
return;
};
defer self.budget.release(io, work);
const outcome = self.handler.handle(
io,
.tcp,
address.NetAddress.fromIp(conn.peer),
conn.payload.query[0..len],
&conn.payload.reply,
&conn.payload.scratch,
&work.reply,
&work.scratch,
);
const bytes = switch (outcome) {
// There is no framing for "no answer", so the connection ends.
@@ -233,7 +243,7 @@ pub const DotServer = struct {
const out = transport.framePrefix(@intCast(bytes.len));
switch (listener.race(io, budget, listener.writeReply, .{ writer, &out, bytes })) {
.ok => {},
.canceled => return,
.canceled, .unavailable => return,
.timed_out, .failed => {
listener.bump(&stats.connection_errors);
return;
@@ -577,7 +587,9 @@ test "dot: two framed queries share one TLS connection" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: std.Io.net.IpAddress = try .parse("127.0.0.1", 0);
var server = try DotServer.listen(gpa, io, listen_address, &h, &env.store, .{ .max_connections = 2 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try DotServer.listen(gpa, io, listen_address, &h, &permits, &env.store, .{ .max_connections = 2 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
@@ -621,7 +633,9 @@ test "dot: a transport EOF without close_notify is a connection error, not a cra
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: std.Io.net.IpAddress = try .parse("127.0.0.1", 0);
var server = try DotServer.listen(gpa, io, listen_address, &h, &env.store, .{ .max_connections = 2 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try DotServer.listen(gpa, io, listen_address, &h, &permits, &env.store, .{ .max_connections = 2 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
@@ -661,7 +675,9 @@ test "dot: plain TCP bytes fail the handshake and are counted" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: std.Io.net.IpAddress = try .parse("127.0.0.1", 0);
var server = try DotServer.listen(gpa, io, listen_address, &h, &env.store, .{ .max_connections = 2 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try DotServer.listen(gpa, io, listen_address, &h, &permits, &env.store, .{ .max_connections = 2 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
@@ -749,7 +765,9 @@ test "dot: a reload serves new handshakes without breaking the old connection" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: std.Io.net.IpAddress = try .parse("127.0.0.1", 0);
var server = try DotServer.listen(gpa, io, listen_address, &h, &env.store, .{ .max_connections = 2 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try DotServer.listen(gpa, io, listen_address, &h, &permits, &env.store, .{ .max_connections = 2 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
@@ -841,7 +859,9 @@ test "dot: a cert path change serves the new certificate on the next handshake"
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: std.Io.net.IpAddress = try .parse("127.0.0.1", 0);
var server = try DotServer.listen(gpa, io, listen_address, &h, &env.store, .{ .max_connections = 2 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try DotServer.listen(gpa, io, listen_address, &h, &permits, &env.store, .{ .max_connections = 2 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
@@ -883,7 +903,9 @@ test "dot: an idle connection is closed with close_notify and counted" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: std.Io.net.IpAddress = try .parse("127.0.0.1", 0);
var server = try DotServer.listen(gpa, io, listen_address, &h, &env.store, .{
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try DotServer.listen(gpa, io, listen_address, &h, &permits, &env.store, .{
.max_connections = 2,
.idle_timeout = short_idle,
});
src/server/handler.zig
+1 -1
View File
@@ -468,7 +468,7 @@ pub const Handler = struct {
}
};
/// Per-query working memory, owned by the listener's slot so that no query path
/// Per-query working memory, owned by the query's permit (`query_budget.Work`) so that no query path
/// allocates. Every buffer is used by one query at a time.
pub const Scratch = struct {
normalize: [types.max_name_len]u8,
src/server/listener.zig
+39 -4
View File
@@ -28,8 +28,16 @@ const std = @import("std");
const net = std.Io.net;
const Allocator = std.mem.Allocator;
const concurrency = @import("../platform/concurrency.zig");
const transport = @import("../upstream/transport.zig");
/// The connection cap of each DNS stream listener: TCP per address family, DoT
/// and DoH. A connection holds memory and a task, never a query permit, while
/// it waits for a query. The Pi saw no concurrent TCP connection in 37 hours
/// and runs neither DoT nor DoH, so this is household policy, not a
/// measurement: a few stub resolvers retrying truncated answers, with room.
pub const dns_max_connections: u16 = 16;
/// Lifecycle of the accept loop. `serve` claims `.serving`, `deinit` publishes
/// `.closing`, and the two meet at `stopped` so no task touches a connection
/// slot after it is freed.
@@ -83,6 +91,10 @@ pub const CoreStats = struct {
accept_errors: std.atomic.Value(u64) = .init(0),
connection_errors: std.atomic.Value(u64) = .init(0),
idle_timeouts: std.atomic.Value(u64) = .init(0),
/// A DNS listener read a whole query and found every permit of the shared
/// query budget taken, so the connection ended unanswered. The web
/// listener answers no DNS and keeps this at zero.
dropped_no_permit: std.atomic.Value(u64) = .init(0),
/// The most slots that have been taken at once (specs/memory-budget.md
/// item 0). A gauge, so no listener's `Snapshot` carries it: those export
/// every field as a counter. Written only under `Core.mutex`.
@@ -98,7 +110,10 @@ pub const Outcome = union(enum) {
expiry: std.Io.Cancelable!void,
};
pub const Result = enum { ok, timed_out, failed, canceled };
/// `unavailable` is the race that could not start for want of a task. It is
/// counted here, under `concurrency.Site.listener_race`, so a caller ends the
/// connection without counting it as the peer's failure.
pub const Result = enum { ok, timed_out, failed, canceled, unavailable };
/// Runs one connection operation against a budget and cancels the loser. No
/// stream read or write in 0.16.0 accepts a timeout, so every wait on a peer
@@ -114,10 +129,16 @@ pub fn race(
defer select.cancelDiscard();
select.concurrent(.op, f, args) catch |err| switch (err) {
error.ConcurrencyUnavailable => return .failed,
error.ConcurrencyUnavailable => {
concurrency.unavailable(.listener_race);
return .unavailable;
},
};
select.concurrent(.expiry, expire, .{ io, budget }) catch |err| switch (err) {
error.ConcurrencyUnavailable => return .failed,
error.ConcurrencyUnavailable => {
concurrency.unavailable(.listener_race);
return .unavailable;
},
};
return switch (select.await() catch return .canceled) {
@@ -421,7 +442,7 @@ pub fn Core(comptime Cfg: type) type {
group.concurrent(io, runConn, .{ self, io, index }) catch |err| switch (err) {
error.ConcurrencyUnavailable => {
bump(&self.stats.rejected_at_capacity);
concurrency.unavailable(.connection);
self.finish(io, index);
continue;
},
@@ -524,6 +545,20 @@ fn testConns(count: usize) ![]TestSlot {
return conns;
}
fn neverCalled() anyerror!void {
unreachable;
}
test "a race that cannot start is unavailable and counted, not a failure" {
var threaded: std.Io.Threaded = .init(std.testing.allocator, .{ .concurrent_limit = .nothing });
defer threaded.deinit();
const before = concurrency.snapshot().get(.listener_race);
const budget: std.Io.Clock.Duration = .{ .raw = .fromSeconds(60), .clock = .awake };
try std.testing.expectEqual(Result.unavailable, race(threaded.io(), budget, neverCalled, .{}));
try std.testing.expectEqual(@as(u64, 1), concurrency.snapshot().get(.listener_race) - before);
}
test "the connection pool hands out every slot once" {
const conns = try testConns(3);
defer testing.allocator.free(conns);
src/server/phase7_integration_test.zig
+15 -3
View File
@@ -53,6 +53,7 @@ const shutdown = @import("shutdown.zig");
const transport = @import("../upstream/transport.zig");
const upstream_owner = @import("../upstream/owner.zig");
const types = @import("../dns/types.zig");
const query_budget = @import("query_budget.zig");
const udp_server = @import("udp_server.zig");
const testing = std.testing;
@@ -96,20 +97,29 @@ fn baseHandler(up: *upstream_owner.Owner) handler.Handler {
/// holds a pointer into it.
const Loop = struct {
server: udp_server.UdpServer,
/// On the heap because the server keeps a pointer to it and `bind` returns
/// the loop by value.
permits: *query_budget.QueryBudget,
group: std.Io.Group,
client: net.Socket,
server_address: net.IpAddress,
fn bind(gpa: Allocator, io: std.Io, h: *handler.Handler) !Loop {
const permits = try gpa.create(query_budget.QueryBudget);
errdefer gpa.destroy(permits);
permits.* = try .init(gpa, 4);
errdefer permits.deinit();
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try udp_server.UdpServer.bind(gpa, io, listen_address, h, .{ .max_in_flight = 4 });
errdefer server.deinit(gpa, io);
var server = try udp_server.UdpServer.bind(io, listen_address, h, permits);
errdefer server.deinit(io);
const client_address: net.IpAddress = try .parse("127.0.0.1", 0);
const client = try client_address.bind(io, .{ .mode = .dgram });
return .{
.server = server,
.permits = permits,
.group = .init,
.client = client,
.server_address = server.boundAddress(),
@@ -128,9 +138,11 @@ const Loop = struct {
}
fn stop(self: *Loop, gpa: Allocator, io: std.Io) void {
self.server.deinit(gpa, io);
self.server.deinit(io);
self.group.cancel(io);
self.client.close(io);
self.permits.deinit();
gpa.destroy(self.permits);
}
};
src/server/query_budget.zig
+151
View File
@@ -0,0 +1,151 @@
//! The DNS work budget every listener shares (specs/memory-budget.md item 3).
//!
//! A permit is the memory one query needs while it is answered: the reply
//! buffer, the handler's scratch and, for UDP, the datagram. Both address
//! families of UDP and TCP, DoT and DoH all draw from one pool, allocated once
//! at boot. A query takes a permit only once it has arrived whole and returns
//! it when its reply is written, so a keep-alive connection between requests
//! holds none.
//!
//! Nothing waits for a permit. A listener that finds the pool empty drops the
//! query and counts it, because a wait would only turn a visible drop into
//! invisible latency, and every permit is also the tasks its query may start,
//! which `app.concurrent_limit` adds up from `permits`.
const std = @import("std");
const Allocator = std.mem.Allocator;
const handler = @import("handler.zig");
const transport = @import("../upstream/transport.zig");
/// The most DNS queries answered at once, across every listener. The Pi's
/// highest reading of `nxdns_dns_in_flight_peak` was 27 over 37 hours of
/// household traffic, and a query beyond this is dropped and counted.
pub const permits = 32;
/// The EDNS ceiling the UDP listener advertises. A larger datagram arrives
/// truncated and is dropped rather than guessed at.
pub const max_datagram = 4096;
comptime {
// `handler.handle` asserts this range on the buffers it is given.
std.debug.assert(max_datagram >= handler.udp_limit_min);
std.debug.assert(max_datagram >= handler.udp_limit_max);
std.debug.assert(transport.max_message_len >= handler.udp_limit_min);
}
pub const Work = struct {
/// UDP's copy of the datagram, so its listener can receive the next one
/// while this query is answered. A stream query stays in its connection,
/// which reads it before taking a permit: a slow sender holds none.
datagram: [max_datagram]u8,
/// A whole DNS message, not a whole datagram. A UDP reply between
/// `max_datagram` and `transport.max_message_len` must reach the handler
/// intact, because only the handler can turn it into the TC=1 reply that
/// sends the client to TCP; the datagram it sends stays bounded by
/// `udpLimit`.
reply: [transport.max_message_len]u8,
scratch: handler.Scratch,
};
pub const QueryBudget = struct {
gpa: Allocator,
work: []Work,
mutex: std.Io.Mutex,
/// Guarded by `mutex`: the indices of the free permits are
/// `free[0..free_len]`.
free: []u16,
free_len: usize,
pub fn init(gpa: Allocator, count: u16) Allocator.Error!QueryBudget {
std.debug.assert(count > 0);
const work = try gpa.alloc(Work, count);
errdefer gpa.free(work);
const free = try gpa.alloc(u16, count);
for (free, 0..) |*slot, index| slot.* = @intCast(index);
return .{ .gpa = gpa, .work = work, .mutex = .init, .free = free, .free_len = count };
}
/// Every listener has stopped, so every permit is back.
pub fn deinit(self: *QueryBudget) void {
std.debug.assert(self.free_len == self.work.len);
self.gpa.free(self.free);
self.gpa.free(self.work);
self.* = undefined;
}
/// A permit, or null when every one is taken. Waits for the bookkeeping
/// mutex, which no holder keeps across any Io, but never for a permit:
/// treating a contended lock as an empty pool would drop queries the
/// budget has room for.
pub fn tryAcquire(self: *QueryBudget, io: std.Io) ?*Work {
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
if (self.free_len == 0) return null;
self.free_len -= 1;
return &self.work[self.free[self.free_len]];
}
pub fn release(self: *QueryBudget, io: std.Io, work: *Work) void {
const index = (@intFromPtr(work) - @intFromPtr(self.work.ptr)) / @sizeOf(Work);
std.debug.assert(&self.work[index] == work);
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
std.debug.assert(self.free_len < self.work.len);
self.free[self.free_len] = @intCast(index);
self.free_len += 1;
}
/// Permits held right now.
pub fn taken(self: *QueryBudget, io: std.Io) usize {
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
return self.work.len - self.free_len;
}
};
const testing = std.testing;
test "a budget hands out every permit once, then refuses" {
var budget: QueryBudget = try .init(testing.allocator, 3);
defer budget.deinit();
const io = testing.io;
var held: [3]*Work = undefined;
for (&held) |*slot| slot.* = budget.tryAcquire(io) orelse return error.TestUnexpectedResult;
try testing.expect(held[0] != held[1] and held[1] != held[2] and held[0] != held[2]);
try testing.expectEqual(@as(?*Work, null), budget.tryAcquire(io));
try testing.expectEqual(@as(usize, 3), budget.taken(io));
for (held) |work| budget.release(io, work);
try testing.expectEqual(@as(usize, 0), budget.taken(io));
}
test "a released permit is handed out again" {
var budget: QueryBudget = try .init(testing.allocator, 2);
defer budget.deinit();
const io = testing.io;
const first = budget.tryAcquire(io).?;
const second = budget.tryAcquire(io).?;
budget.release(io, first);
try testing.expectEqual(first, budget.tryAcquire(io).?);
budget.release(io, first);
budget.release(io, second);
}
test "init survives every allocation failure" {
try testing.checkAllAllocationFailures(testing.allocator, struct {
fn run(gpa: Allocator) !void {
var budget: QueryBudget = try .init(gpa, 2);
budget.deinit();
}
}.run, .{});
}
test "the default budget stays small" {
// 4096 + 65535 + scratch per permit: about 2.4 MiB for the whole process,
// where each UDP and TCP listener used to hold 4.8 and 8.7 MiB of its own.
try testing.expect(@sizeOf(Work) * permits < 3 * 1024 * 1024);
}
src/server/resolver_integration_test.zig
+6 -3
View File
@@ -16,6 +16,7 @@ const build_options = @import("build_options");
const net = std.Io.net;
const handler = @import("handler.zig");
const query_budget = @import("query_budget.zig");
const tcp_server = @import("tcp_server.zig");
const udp_server = @import("udp_server.zig");
const model = @import("../config/model.zig");
@@ -282,8 +283,10 @@ test "the whole resolver answers over udp and tcp and fails over to a healthy up
var h = bareHandler(h_owner.client(upstreams.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var udp = try udp_server.UdpServer.bind(gpa, io, listen_address, &h, .{ .max_in_flight = 4 });
var tcp = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{ .max_connections = 2 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var udp = try udp_server.UdpServer.bind(io, listen_address, &h, &permits);
var tcp = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{ .max_connections = 2 });
const udp_address = udp.boundAddress();
const tcp_address = tcp.boundAddress();
@@ -293,7 +296,7 @@ test "the whole resolver answers over udp and tcp and fails over to a healthy up
// Deferred so a failed check still stops both servers: a `serve` left in
// accept makes `threaded.deinit` wait for it forever.
defer {
udp.deinit(gpa, io);
udp.deinit(io);
tcp.deinit(io);
group.cancel(io);
}
src/server/tcp_server.zig
+28 -19
View File
@@ -9,7 +9,9 @@
//! `listener.Core`'s (milestone-18 ruling 1); this file is the per-connection
//! loop and nothing else. Connection slots are fixed and pre-allocated. Over
//! capacity the listener closes the new stream immediately and counts it; it
//! never queues, and it never allocates per connection.
//! never queues, and it never allocates per connection. Each query holds a
//! permit from the shared query budget from its last byte read to its reply's
//! last byte written; with none free the connection ends and is counted.
//!
//! No stream read or write in 0.16.0 accepts a timeout, so every per-connection
//! operation is raced against `Options.idle_timeout` through `listener.race`
@@ -19,20 +21,21 @@ const std = @import("std");
const address = @import("../platform/address.zig");
const handler = @import("handler.zig");
const listener = @import("listener.zig");
const query_budget = @import("query_budget.zig");
const transport = @import("../upstream/transport.zig");
/// The stream buffers only stage the framing bytes. A message longer than this
/// is read straight into `Conn.query` and written straight from `Conn.reply`,
/// so making them larger would buy nothing.
/// is read straight into `Payload.query` and written straight from the
/// permit's reply, so making them larger would buy nothing.
const stream_buffer_len = 1024;
pub const Options = struct {
max_connections: u16 = 64,
max_connections: u16 = listener.dns_max_connections,
/// RFC 7766 §6.2.3 recommends a few seconds of idle tolerance.
idle_timeout: std.Io.Clock.Duration = .{ .raw = .fromSeconds(10), .clock = .awake },
};
/// TCP/53 keeps no counter of its own: the shared six are exactly what it
/// TCP/53 keeps no counter of its own: the shared ones are exactly what it
/// counts.
pub const Stats = listener.CoreStats;
@@ -47,24 +50,20 @@ pub const Snapshot = struct {
accept_errors: u64,
connection_errors: u64,
idle_timeouts: u64,
dropped_no_permit: u64,
};
pub const TcpServer = struct {
core: listener.Core(Config),
handler: *handler.Handler,
budget: *query_budget.QueryBudget,
options: Options,
/// One slot is ~137 KiB, so the default 64 connections cost ~8.8 MiB, which
/// is inside the PLAN §18 budget. The two message buffers cannot be shared
/// or shrunk: the handler holds the query while the reply is built, and
/// both ceilings are the 65535 bytes the length prefix can express.
/// The query is read here, before a permit is taken, so a client that
/// sends slowly holds connection memory and no query permit. 65535 bytes,
/// the most the length prefix can express.
pub const Payload = struct {
query: [transport.max_message_len]u8,
reply: [transport.max_message_len]u8,
/// The handler's per-query working memory. It belongs to the slot so
/// that answering a message allocates nothing, and a connection is
/// answered serially, so one query uses it at a time.
scratch: handler.Scratch,
};
const Config = struct {
@@ -85,11 +84,13 @@ pub const TcpServer = struct {
io: std.Io,
listen_address: std.Io.net.IpAddress,
h: *handler.Handler,
budget: *query_budget.QueryBudget,
options: Options,
) ListenError!TcpServer {
return .{
.core = try listener.Core(Config).listen(gpa, io, listen_address, options.max_connections),
.handler = h,
.budget = budget,
.options = options,
};
}
@@ -115,6 +116,7 @@ pub const TcpServer = struct {
.accept_errors = stats.accept_errors.load(.monotonic),
.connection_errors = stats.connection_errors.load(.monotonic),
.idle_timeouts = stats.idle_timeouts.load(.monotonic),
.dropped_no_permit = stats.dropped_no_permit.load(.monotonic),
};
}
@@ -147,7 +149,7 @@ pub const TcpServer = struct {
listener.bump(&stats.idle_timeouts);
return;
},
.canceled => return,
.canceled, .unavailable => return,
.failed => {
listener.bump(&stats.connection_errors);
return;
@@ -172,7 +174,7 @@ pub const TcpServer = struct {
switch (listener.race(io, budget, listener.readBody, .{ &reader.interface, conn.payload.query[0..len] })) {
.ok => {},
.canceled => return,
.canceled, .unavailable => return,
// A half-sent message is a broken peer, not an idle one.
.timed_out, .failed => {
listener.bump(&stats.connection_errors);
@@ -180,13 +182,20 @@ pub const TcpServer = struct {
},
}
// There is no framing for "no answer", so the connection ends.
const work = self.budget.tryAcquire(io) orelse {
listener.bump(&stats.dropped_no_permit);
return;
};
defer self.budget.release(io, work);
const outcome = self.handler.handle(
io,
.tcp,
address.NetAddress.fromIp(conn.peer),
conn.payload.query[0..len],
&conn.payload.reply,
&conn.payload.scratch,
&work.reply,
&work.scratch,
);
const bytes = switch (outcome) {
// There is no framing for "no answer", so the connection ends.
@@ -197,7 +206,7 @@ pub const TcpServer = struct {
const out = transport.framePrefix(@intCast(bytes.len));
switch (listener.race(io, budget, listener.writeReply, .{ &writer.interface, &out, bytes })) {
.ok => {},
.canceled => return,
.canceled, .unavailable => return,
.timed_out, .failed => {
listener.bump(&stats.connection_errors);
return;
src/server/tcp_server_integration_test.zig
+187 -7
View File
@@ -13,7 +13,9 @@ const std = @import("std");
const build_options = @import("build_options");
const net = std.Io.net;
const concurrency = @import("../platform/concurrency.zig");
const handler = @import("handler.zig");
const query_budget = @import("query_budget.zig");
const tcp_server = @import("tcp_server.zig");
const model = @import("../config/model.zig");
const response = @import("../filter/response.zig");
@@ -179,7 +181,9 @@ test "two length-prefixed queries share one connection" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{ .max_connections = 2 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{ .max_connections = 2 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
@@ -213,7 +217,9 @@ test "the claimed slot records the connecting client" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{ .max_connections = 2 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{ .max_connections = 2 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
@@ -295,7 +301,9 @@ test "a canceled serve does not wait for a live connection" {
// The idle budget is the whole time a drain would have to wait out, so it
// is set far beyond any patience this test run has: if `serve` waits for
// the connection instead of canceling it, the wait never ends.
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{
.max_connections = 2,
.idle_timeout = .{ .raw = .fromSeconds(600), .clock = .awake },
});
@@ -346,7 +354,9 @@ test "the connections peak is the most slots held at once" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{
.max_connections = 3,
.idle_timeout = .{ .raw = .fromSeconds(600), .clock = .awake },
});
@@ -401,7 +411,9 @@ test "an idle connection is closed and counted" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{
.max_connections = 2,
.idle_timeout = short_idle,
});
@@ -436,7 +448,9 @@ test "a zero-length message is a connection error" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{
.max_connections = 2,
.idle_timeout = short_idle,
});
@@ -490,7 +504,9 @@ test "deinit ends a serve loop that is blocked on accept" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, .{ .max_connections = 2 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{ .max_connections = 2 });
var group: std.Io.Group = .init;
try group.concurrent(io, tcp_server.TcpServer.serve, .{ &server, io });
@@ -501,3 +517,167 @@ test "deinit ends a serve loop that is blocked on accept" {
error.Canceled => unreachable,
};
}
/// Writes one framed query and returns the frame the server answered with.
fn askOn(reader: *std.Io.Reader, writer: *std.Io.Writer) ![]const u8 {
try writer.writeAll(&transport.framePrefix(@intCast(query_bytes.len)));
try writer.writeAll(query_bytes);
try writer.flush();
const len = transport.parsePrefix((try reader.takeArray(transport.prefix_len)).*);
return reader.take(len);
}
/// One answered query, then the connection idles with every permit back in the
/// budget; a second query that finds none taken ends the connection unanswered.
fn keepAliveThenStarved(io: std.Io, address: net.IpAddress, permits: *query_budget.QueryBudget) anyerror!void {
var stream = try address.connect(io, .{ .mode = .stream });
defer stream.close(io);
var read_buf: [1024]u8 = undefined;
var write_buf: [1024]u8 = undefined;
var reader = stream.reader(io, &read_buf);
var writer = stream.writer(io, &write_buf);
try expectAnswersQuery(try askOn(&reader.interface, &writer.interface));
// The server returns the permit just after its reply's last byte, which can
// reach this side first. `bounded` caps the wait.
const tick: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(1), .clock = .awake };
while (permits.taken(io) != 0) try tick.sleep(io);
const held = permits.tryAcquire(io).?;
defer permits.release(io, held);
try writer.interface.writeAll(&transport.framePrefix(@intCast(query_bytes.len)));
try writer.interface.writeAll(query_bytes);
try writer.interface.flush();
var sink: [64]u8 = undefined;
if (try reader.interface.readSliceShort(&sink) != 0) return error.TestUnexpectedBytes;
}
test "a keep-alive connection holds no permit between queries, and a query without one ends it" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var threaded: std.Io.Threaded = .init(gpa, .{});
defer threaded.deinit();
const io = threaded.io();
var fake: FakeUpstream = .{ .reply = response_bytes };
var h_owner: upstream_owner.Borrowed = .{};
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var permits: query_budget.QueryBudget = try .init(gpa, 1);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{ .max_connections = 2 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
try group.concurrent(io, tcp_server.TcpServer.serve, .{ &server, io });
defer {
server.deinit(io);
group.await(io) catch |err| switch (err) {
error.Canceled => unreachable,
};
}
try bounded(io, keepAliveThenStarved, .{ io, server_address, &permits });
try testing.expectEqual(@as(u64, 1), server.core.stats.dropped_no_permit.load(.monotonic));
try testing.expectEqual(@as(u64, 0), server.core.stats.connection_errors.load(.monotonic));
try testing.expectEqual(@as(u64, 1), h.stats.queries.load(.monotonic));
}
/// Holds the only slot with an answered connection, then opens a second one,
/// which the server closes without reading.
fn oneHeldOneRefused(io: std.Io, address: net.IpAddress) anyerror!void {
var held = try address.connect(io, .{ .mode = .stream });
defer held.close(io);
var read_buf: [1024]u8 = undefined;
var write_buf: [1024]u8 = undefined;
var reader = held.reader(io, &read_buf);
var writer = held.writer(io, &write_buf);
try expectAnswersQuery(try askOn(&reader.interface, &writer.interface));
var refused = try address.connect(io, .{ .mode = .stream });
defer refused.close(io);
var refused_buf: [64]u8 = undefined;
var refused_reader = refused.reader(io, &refused_buf);
var sink: [64]u8 = undefined;
if (try refused_reader.interface.readSliceShort(&sink) != 0) return error.TestUnexpectedBytes;
}
test "a listener at its connection cap refuses and counts" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var threaded: std.Io.Threaded = .init(gpa, .{});
defer threaded.deinit();
const io = threaded.io();
var fake: FakeUpstream = .{ .reply = response_bytes };
var h_owner: upstream_owner.Borrowed = .{};
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, io, listen_address, &h, &permits, .{ .max_connections = 1 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
try group.concurrent(io, tcp_server.TcpServer.serve, .{ &server, io });
defer {
server.deinit(io);
group.await(io) catch |err| switch (err) {
error.Canceled => unreachable,
};
}
try bounded(io, oneHeldOneRefused, .{ io, server_address });
try testing.expectEqual(@as(u64, 1), server.core.stats.rejected_at_capacity.load(.monotonic));
try testing.expectEqual(@as(u64, 1), server.core.stats.connections.load(.monotonic));
}
test "a connection that gets no task is closed and counted, not refused at capacity" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
// The accept loop is the one task the server's pool allows. The client
// runs on a pool of its own, so `bounded` has the tasks it needs.
var server_threaded: std.Io.Threaded = .init(gpa, .{ .concurrent_limit = .limited(1) });
defer server_threaded.deinit();
const server_io = server_threaded.io();
var client_threaded: std.Io.Threaded = .init(gpa, .{});
defer client_threaded.deinit();
const io = client_threaded.io();
var fake: FakeUpstream = .{ .reply = response_bytes };
var h_owner: upstream_owner.Borrowed = .{};
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try tcp_server.TcpServer.listen(gpa, server_io, listen_address, &h, &permits, .{ .max_connections = 2 });
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
try group.concurrent(server_io, tcp_server.TcpServer.serve, .{ &server, server_io });
defer {
server.deinit(server_io);
group.await(server_io) catch |err| switch (err) {
error.Canceled => unreachable,
};
}
const before = concurrency.snapshot().get(.connection);
try bounded(io, waitForServerClose, .{ io, server_address });
try testing.expectEqual(@as(u64, 1), concurrency.snapshot().get(.connection) - before);
try testing.expectEqual(@as(u64, 0), server.core.stats.rejected_at_capacity.load(.monotonic));
try testing.expectEqual(@as(u64, 0), server.core.stats.connections.load(.monotonic));
}
src/server/udp_server.zig
+51 -183
View File
@@ -1,49 +1,39 @@
//! The UDP/53 listener.
//!
//! One receive loop feeds a bounded pool of in-flight tasks. A datagram that
//! finds no free slot is dropped and counted, never queued: an unbounded queue
//! only moves the failure from "visible drop" to "invisible latency", which is
//! the failure mode PLAN §1 names.
//! One receive loop hands each datagram to a task that holds a permit from the
//! shared query budget. A datagram that finds no permit is dropped and counted,
//! never queued: an unbounded queue only moves the failure from "visible drop"
//! to "invisible latency", which is the failure mode PLAN §1 names.
//!
//! Every slot carries its own query and reply buffer, so answering a datagram
//! allocates nothing. The pool is sized once in `bind` and never grows.
//! The permit carries the query, the reply and the handler's scratch, so
//! answering a datagram allocates nothing.
const std = @import("std");
const address = @import("../platform/address.zig");
const concurrency = @import("../platform/concurrency.zig");
const handler = @import("handler.zig");
const transport = @import("../upstream/transport.zig");
const query_budget = @import("query_budget.zig");
const log = std.log.scoped(.udp_server);
/// The EDNS ceiling this server advertises. A larger datagram arrives
/// truncated and is dropped rather than guessed at.
pub const max_datagram = 4096;
comptime {
// `handler.handle` asserts this range on the buffers it is given.
std.debug.assert(max_datagram >= handler.udp_limit_min);
std.debug.assert(max_datagram >= handler.udp_limit_max);
std.debug.assert(transport.max_message_len >= handler.udp_limit_min);
}
pub const max_datagram = query_budget.max_datagram;
/// How long a receive waits before the loop re-reads the stop flag. `deinit`
/// therefore returns within this much time, without depending on a blocked
/// `receive` being unblocked by a close — a guarantee POSIX does not make.
const poll_interval: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(100), .clock = .awake };
pub const Options = struct {
max_in_flight: u16 = 64,
};
pub const Stats = struct {
received: std.atomic.Value(u64) = .init(0),
/// The datagram arrived with `flags.trunc`: the kernel discarded its tail.
dropped_oversize: std.atomic.Value(u64) = .init(0),
/// Every permit of the shared query budget was taken.
dropped_no_slot: std.atomic.Value(u64) = .init(0),
/// The handler answered `.drop` — an upstream exchange canceled by
/// shutdown. The handler counts its own malformed drops; this counter is
/// what keeps the listener's "every dropped datagram is counted" rule true
/// for the drops it cannot see.
/// for the drops it cannot see. A datagram that got a permit but no task
/// is counted under `nxdns_concurrency_unavailable_total{site="udp_query"}`.
dropped_handler: std.atomic.Value(u64) = .init(0),
receive_errors: std.atomic.Value(u64) = .init(0),
send_errors: std.atomic.Value(u64) = .init(0),
@@ -63,69 +53,31 @@ pub const Snapshot = struct {
};
/// Lifecycle of the receive loop. `serve` claims `.serving`, `deinit` publishes
/// `.closing`, and the two meet at `stopped` so no task touches the slots after
/// they are freed.
/// `.closing`, and the two meet at `stopped` so no task touches the socket
/// after it is closed.
const State = enum(u32) { idle, serving, closing };
pub const UdpServer = struct {
socket: std.Io.net.Socket,
handler: *handler.Handler,
slots: []Slot,
mutex: std.Io.Mutex,
budget: *query_budget.QueryBudget,
stats: Stats,
state: std.atomic.Value(State),
stopped: std.Io.Event,
pub const Slot = struct {
query: [max_datagram]u8,
/// The buffer the handler receives the upstream answer into, so it
/// holds a whole DNS message, not a whole datagram. A reply between
/// `max_datagram` and `transport.max_message_len` must reach the
/// handler intact: only the handler can turn it into the TC=1 reply
/// that makes the client retry over TCP (S5.3). A 4096-byte buffer
/// would instead fail the exchange with `error.ResponseTooLarge` and
/// answer SERVFAIL, which is wrong.
///
/// The datagram actually sent stays bounded by `udpLimit` inside the
/// handler, so nothing larger than 4096 bytes leaves this socket.
///
/// Cost: 4096 + 65535 + the scratch below ≈ 74 KiB per slot, so the
/// default 64 slots hold ≈ 4.6 MiB. The PLAN §18 budget is 100 MiB with
/// ~1M blocked domains, so this pool takes about 5% of it.
reply: [transport.max_message_len]u8,
/// The handler's per-query working memory. It belongs to the slot so
/// that answering a datagram still allocates nothing, and one slot
/// serves one query at a time.
scratch: handler.Scratch,
from: std.Io.net.IpAddress,
len: usize,
/// Guarded by `UdpServer.mutex`.
in_use: bool,
};
pub const BindError = std.Io.net.IpAddress.BindError || error{OutOfMemory};
pub const BindError = std.Io.net.IpAddress.BindError;
pub fn bind(
gpa: std.mem.Allocator,
io: std.Io,
bind_address: std.Io.net.IpAddress,
h: *handler.Handler,
options: Options,
budget: *query_budget.QueryBudget,
) BindError!UdpServer {
std.debug.assert(options.max_in_flight > 0);
const slots = try gpa.alloc(Slot, options.max_in_flight);
errdefer gpa.free(slots);
for (slots) |*slot| slot.in_use = false;
const local = bind_address;
const socket = try local.bind(io, .{ .mode = .dgram });
const socket = try bind_address.bind(io, .{ .mode = .dgram });
return .{
.socket = socket,
.handler = h,
.slots = slots,
.mutex = .init,
.budget = budget,
.stats = .{},
.state = .init(.idle),
.stopped = .unset,
@@ -139,7 +91,7 @@ pub const UdpServer = struct {
/// The counters, read one at a time. A scrape that lands mid-datagram can
/// see a receive counted before its drop is; a lock would buy a consistency
/// no consumer needs, and the receive loop takes that lock per datagram.
/// no consumer needs.
pub fn snapshotStats(self: *const UdpServer) Snapshot {
return .{
.received = self.stats.received.load(.monotonic),
@@ -169,172 +121,88 @@ pub const UdpServer = struct {
self.stopped.set(io);
}
pub fn deinit(self: *UdpServer, gpa: std.mem.Allocator, io: std.Io) void {
pub fn deinit(self: *UdpServer, io: std.Io) void {
// `serve` reads this between receives, so it stops on its own and the
// socket is closed only once nothing can be reading from it.
if (self.state.swap(.closing, .acq_rel) == .serving) self.stopped.waitUncancelable(io);
self.socket.close(io);
gpa.free(self.slots);
self.* = undefined;
}
fn receiveLoop(self: *UdpServer, io: std.Io, group: *std.Io.Group) void {
// A datagram that finds no free slot still has to leave the socket
// buffer, or the loop would spin on the same undeliverable packet.
var overflow: [max_datagram]u8 = undefined;
// The datagram is received here, without a permit, so an idle listener
// holds none; a query that gets one is copied into it.
var datagram: [max_datagram]u8 = undefined;
while (self.state.load(.acquire) == .serving) {
const claimed = self.claim(io);
const buffer = if (claimed) |index| &self.slots[index].query else &overflow;
const msg = self.socket.receiveTimeout(io, buffer, .{ .duration = poll_interval }) catch |err| {
if (claimed) |index| self.release(io, index);
const msg = self.socket.receiveTimeout(io, &datagram, .{ .duration = poll_interval }) catch |err| {
switch (err) {
error.Canceled => return,
// The stop flag is re-read at the top of the loop.
error.Timeout => continue,
error.ConcurrencyUnavailable => concurrency.unavailable(.udp_receive),
else => {
bump(&self.stats.receive_errors);
log.debug("udp receive failed: {t}", .{err});
// One bad datagram must not kill the listener, and a
// persistent error must not turn it into a spin.
poll_interval.sleep(io) catch return;
continue;
},
}
// One bad datagram must not kill the listener, and a persistent
// error must not turn it into a spin.
poll_interval.sleep(io) catch return;
continue;
};
bump(&self.stats.received);
const index = claimed orelse {
bump(&self.stats.dropped_no_slot);
continue;
};
// The kernel already threw the tail away, so the message cannot be
// parsed and any answer would be a guess.
if (msg.flags.trunc) {
bump(&self.stats.dropped_oversize);
self.release(io, index);
continue;
}
const slot = &self.slots[index];
slot.from = msg.from;
slot.len = msg.data.len;
const work = self.budget.tryAcquire(io) orelse {
bump(&self.stats.dropped_no_slot);
continue;
};
@memcpy(work.datagram[0..msg.data.len], msg.data);
group.concurrent(io, respondOne, .{ self, io, index }) catch |err| switch (err) {
group.concurrent(io, respondOne, .{ self, io, work, msg.from, msg.data.len }) catch |err| switch (err) {
error.ConcurrencyUnavailable => {
bump(&self.stats.dropped_no_slot);
self.release(io, index);
concurrency.unavailable(.udp_query);
self.budget.release(io, work);
},
};
}
}
fn respondOne(self: *UdpServer, io: std.Io, index: usize) void {
const slot = &self.slots[index];
defer self.release(io, index);
fn respondOne(
self: *UdpServer,
io: std.Io,
work: *query_budget.Work,
from: std.Io.net.IpAddress,
len: usize,
) void {
defer self.budget.release(io, work);
const outcome = self.handler.handle(
io,
.udp,
address.NetAddress.fromIp(slot.from),
slot.query[0..slot.len],
&slot.reply,
&slot.scratch,
address.NetAddress.fromIp(from),
work.datagram[0..len],
&work.reply,
&work.scratch,
);
switch (outcome) {
.drop => bump(&self.stats.dropped_handler),
.reply => |bytes| self.socket.send(io, &slot.from, bytes) catch |err| {
.reply => |bytes| self.socket.send(io, &from, bytes) catch |err| {
bump(&self.stats.send_errors);
log.debug("udp send to a client failed: {t}", .{err});
},
}
}
fn claim(self: *UdpServer, io: std.Io) ?usize {
// Uncancelable: this section takes no Io and never blocks on a peer, so
// it cannot deadlock, and losing the lock mid-update would leak a slot.
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
const index = firstFree(self.slots) orelse return null;
self.slots[index].in_use = true;
return index;
}
fn release(self: *UdpServer, io: std.Io, index: usize) void {
self.mutex.lockUncancelable(io);
defer self.mutex.unlock(io);
self.slots[index].in_use = false;
}
};
/// The capacity rule, without the mutex, so it is testable without a backend.
fn firstFree(slots: []const UdpServer.Slot) ?usize {
for (slots, 0..) |*slot, index| {
if (!slot.in_use) return index;
}
return null;
}
fn bump(counter: *std.atomic.Value(u64)) void {
_ = counter.fetchAdd(1, .monotonic);
}
const testing = std.testing;
fn testSlots(count: usize) ![]UdpServer.Slot {
const slots = try testing.allocator.alloc(UdpServer.Slot, count);
for (slots) |*slot| slot.in_use = false;
return slots;
}
test "the slot pool hands out every index once" {
const slots = try testSlots(3);
defer testing.allocator.free(slots);
for (0..slots.len) |expected| {
const index = firstFree(slots) orelse return error.TestUnexpectedResult;
try testing.expectEqual(expected, index);
slots[index].in_use = true;
}
}
test "a full slot pool refuses instead of growing" {
const slots = try testSlots(2);
defer testing.allocator.free(slots);
for (slots) |*slot| slot.in_use = true;
try testing.expectEqual(@as(?usize, null), firstFree(slots));
}
test "a slot's reply buffer holds a whole DNS message" {
const slots = try testSlots(1);
defer testing.allocator.free(slots);
// The handler asserts both ends of this range on the buffer it is given,
// and an upstream reply of up to 65535 bytes has to fit so the handler can
// truncate it instead of the exchange failing.
try testing.expectEqual(@as(usize, transport.max_message_len), slots[0].reply.len);
try testing.expect(slots[0].reply.len >= handler.udp_limit_min);
try testing.expect(slots[0].reply.len > max_datagram);
}
test "the default slot pool stays inside the memory budget" {
// 4096 + 65535 + scratch ≈ 74 KiB per slot; 64 slots ≈ 4.6 MiB, against the
// 100 MiB of PLAN §18.
const options: Options = .{};
const pool_bytes = @sizeOf(UdpServer.Slot) * @as(usize, options.max_in_flight);
try testing.expect(pool_bytes < 8 * 1024 * 1024);
}
test "a released slot is handed out again" {
const slots = try testSlots(2);
defer testing.allocator.free(slots);
for (slots) |*slot| slot.in_use = true;
slots[1].in_use = false;
try testing.expectEqual(@as(?usize, 1), firstFree(slots));
}
src/server/udp_server_integration_test.zig
+103 -8
View File
@@ -12,7 +12,9 @@ const std = @import("std");
const build_options = @import("build_options");
const net = std.Io.net;
const concurrency = @import("../platform/concurrency.zig");
const handler = @import("handler.zig");
const query_budget = @import("query_budget.zig");
const udp_server = @import("udp_server.zig");
const model = @import("../config/model.zig");
const response = @import("../filter/response.zig");
@@ -116,7 +118,9 @@ test "a udp query is answered on the loopback" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try udp_server.UdpServer.bind(gpa, io, listen_address, &h, .{ .max_in_flight = 4 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try udp_server.UdpServer.bind(io, listen_address, &h, &permits);
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
@@ -124,7 +128,7 @@ test "a udp query is answered on the loopback" {
// Deferred so a failed check still stops the server: a `serve` left in
// accept makes `threaded.deinit` wait for it forever.
defer {
server.deinit(gpa, io);
server.deinit(io);
group.await(io) catch |err| switch (err) {
error.Canceled => unreachable,
};
@@ -158,13 +162,15 @@ test "a runt datagram is dropped and no reply is sent" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try udp_server.UdpServer.bind(gpa, io, listen_address, &h, .{ .max_in_flight = 4 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try udp_server.UdpServer.bind(io, listen_address, &h, &permits);
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
try group.concurrent(io, udp_server.UdpServer.serve, .{ &server, io });
defer {
server.deinit(gpa, io);
server.deinit(io);
group.await(io) catch |err| switch (err) {
error.Canceled => unreachable,
};
@@ -195,13 +201,15 @@ test "an oversize datagram arrives truncated and is dropped" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try udp_server.UdpServer.bind(gpa, io, listen_address, &h, .{ .max_in_flight = 4 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try udp_server.UdpServer.bind(io, listen_address, &h, &permits);
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
try group.concurrent(io, udp_server.UdpServer.serve, .{ &server, io });
defer {
server.deinit(gpa, io);
server.deinit(io);
group.await(io) catch |err| switch (err) {
error.Canceled => unreachable,
};
@@ -238,14 +246,101 @@ test "deinit ends a serve loop that is blocked on receive" {
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var server = try udp_server.UdpServer.bind(gpa, io, listen_address, &h, .{ .max_in_flight = 4 });
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try udp_server.UdpServer.bind(io, listen_address, &h, &permits);
var group: std.Io.Group = .init;
try group.concurrent(io, udp_server.UdpServer.serve, .{ &server, io });
// No datagram ever arrives, so `serve` is inside a receive when this runs.
server.deinit(gpa, io);
server.deinit(io);
group.await(io) catch |err| switch (err) {
error.Canceled => unreachable,
};
}
test "with every permit taken a udp query is dropped and counted" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
var threaded: std.Io.Threaded = .init(gpa, .{});
defer threaded.deinit();
const io = threaded.io();
var fake: FakeUpstream = .{ .reply = response_bytes };
var h_owner: upstream_owner.Borrowed = .{};
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var permits: query_budget.QueryBudget = try .init(gpa, 1);
defer permits.deinit();
var server = try udp_server.UdpServer.bind(io, listen_address, &h, &permits);
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
try group.concurrent(io, udp_server.UdpServer.serve, .{ &server, io });
defer {
server.deinit(io);
group.await(io) catch |err| switch (err) {
error.Canceled => unreachable,
};
}
// Another listener's query, as far as this one can tell.
const held = permits.tryAcquire(io).?;
defer permits.release(io, held);
const client_address: net.IpAddress = try .parse("127.0.0.1", 0);
var client = try client_address.bind(io, .{ .mode = .dgram });
defer client.close(io);
try client.send(io, &server_address, query_bytes);
var buf: [udp_server.max_datagram]u8 = undefined;
try testing.expectError(error.Timeout, client.receiveTimeout(io, &buf, silence));
try testing.expectEqual(@as(u64, 1), server.stats.dropped_no_slot.load(.monotonic));
try testing.expectEqual(@as(u64, 0), h.stats.queries.load(.monotonic));
}
test "a udp query with a permit but no task is dropped, counted, and returns its permit" {
if (!build_options.integration) return error.SkipZigTest;
const gpa = testing.allocator;
// The serve task is the one task this pool allows.
var threaded: std.Io.Threaded = .init(gpa, .{ .concurrent_limit = .limited(1) });
defer threaded.deinit();
const io = threaded.io();
var fake: FakeUpstream = .{ .reply = response_bytes };
var h_owner: upstream_owner.Borrowed = .{};
var h = bareHandler(h_owner.client(fake.client()));
const listen_address: net.IpAddress = try .parse("127.0.0.1", 0);
var permits: query_budget.QueryBudget = try .init(gpa, 4);
defer permits.deinit();
var server = try udp_server.UdpServer.bind(io, listen_address, &h, &permits);
const server_address = server.boundAddress();
var group: std.Io.Group = .init;
try group.concurrent(io, udp_server.UdpServer.serve, .{ &server, io });
defer {
server.deinit(io);
group.await(io) catch |err| switch (err) {
error.Canceled => unreachable,
};
}
const client_address: net.IpAddress = try .parse("127.0.0.1", 0);
var client = try client_address.bind(io, .{ .mode = .dgram });
defer client.close(io);
const before = concurrency.snapshot().get(.udp_query);
try client.send(io, &server_address, query_bytes);
var buf: [udp_server.max_datagram]u8 = undefined;
try testing.expectError(error.Timeout, client.receiveTimeout(io, &buf, silence));
try testing.expectEqual(@as(u64, 1), concurrency.snapshot().get(.udp_query) - before);
try testing.expectEqual(@as(usize, 0), permits.taken(io));
try testing.expectEqual(@as(u64, 0), server.stats.dropped_no_slot.load(.monotonic));
}