milestone 10: doh and dot listeners, cert store with hot reload and cert reload api

src/web/handlers/certs.zig

@@ -0,0 +1,136 @@
+//! `POST /api/certs/reload` — reload the DoH/DoT certificates from disk
+//! (milestone-10 ruling 8).
+//!
+//! Always answers 200: the per-endpoint outcome IS the payload. A failed
+//! reload is an outcome, not a server error — the store publishes nothing on
+//! failure (PLAN:297), so the old certificate keeps serving and nothing about
+//! the web server's own state is exceptional. A disabled endpoint has no
+//! store and reports `{enabled: false, reloaded: false, error: null}`.
+
+const std = @import("std");
+
+const cert_store = @import("../../server/cert_store.zig");
+const http_util = @import("../http_util.zig");
+const server = @import("../server.zig");
+
+const Request = http_util.Request;
+const HandlerError = http_util.HandlerError;
+
+pub const Outcome = struct {
+    enabled: bool,
+    reloaded: bool,
+    /// `cert_store.humanMessage` text; null on success and while disabled.
+    @"error": ?[]const u8,
+};
+
+pub const View = struct {
+    doh: Outcome,
+    dot: Outcome,
+};
+
+/// Reloads every enabled endpoint's store. A null store is a disabled
+/// endpoint: the composition root only constructs one for an enabled
+/// endpoint (milestone-10 ruling 11).
+pub fn applyReload(state: *server.WebState, io: std.Io) View {
+    return .{
+        .doh = outcome(state.doh_certs, io),
+        .dot = outcome(state.dot_certs, io),
+    };
+}
+
+fn outcome(store: ?*cert_store.CertStore, io: std.Io) Outcome {
+    const live = store orelse return .{ .enabled = false, .reloaded = false, .@"error" = null };
+    live.reload(io) catch |err| return .{
+        .enabled = true,
+        .reloaded = false,
+        .@"error" = cert_store.humanMessage(err),
+    };
+    return .{ .enabled = true, .reloaded = true, .@"error" = null };
+}
+
+pub fn post(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void {
+    return http_util.respondJson(request, .ok, applyReload(state, io), &.{});
+}
+
+// ---------------------------------------------------------------------------
+// tests
+// ---------------------------------------------------------------------------
+
+const fixtures = @import("test_fixtures");
+const testing = std.testing;
+
+test "both endpoints disabled report exactly the disabled outcome" {
+    var state: server.WebState = .{ .gpa = testing.allocator };
+    const view = applyReload(&state, undefined);
+    for ([_]Outcome{ view.doh, view.dot }) |per_endpoint| {
+        try testing.expect(!per_endpoint.enabled);
+        try testing.expect(!per_endpoint.reloaded);
+        try testing.expectEqual(@as(?[]const u8, null), per_endpoint.@"error");
+    }
+}
+
+/// Fixture PEMs in a tmp directory, addressed the way `app.zig` hands config
+/// paths to the store. Must not move after `init`: the path slices point into
+/// the buffers below.
+const TestEnv = struct {
+    threaded: std.Io.Threaded,
+    tmp: testing.TmpDir,
+    cert_path_buf: [128]u8,
+    key_path_buf: [128]u8,
+    cert_path: []const u8,
+    key_path: []const u8,
+
+    fn init(env: *TestEnv) !void {
+        env.threaded = .init(testing.allocator, .{});
+        errdefer env.threaded.deinit();
+        env.tmp = testing.tmpDir(.{});
+        errdefer env.tmp.cleanup();
+
+        const env_io = env.threaded.io();
+        try env.tmp.dir.writeFile(env_io, .{ .sub_path = "cert.pem", .data = fixtures.cert_pem });
+        try env.tmp.dir.writeFile(env_io, .{ .sub_path = "key.pem", .data = fixtures.key_pem });
+        env.cert_path = try std.fmt.bufPrint(&env.cert_path_buf, ".zig-cache/tmp/{s}/cert.pem", .{env.tmp.sub_path});
+        env.key_path = try std.fmt.bufPrint(&env.key_path_buf, ".zig-cache/tmp/{s}/key.pem", .{env.tmp.sub_path});
+    }
+
+    fn deinit(env: *TestEnv) void {
+        env.tmp.cleanup();
+        env.threaded.deinit();
+    }
+
+    fn io(env: *TestEnv) std.Io {
+        return env.threaded.io();
+    }
+};
+
+test "a wired store reloads; a broken one reports the message in the same payload" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try cert_store.CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    var state: server.WebState = .{ .gpa = testing.allocator, .doh_certs = &store };
+
+    const succeeded = applyReload(&state, io);
+    try testing.expect(succeeded.doh.enabled);
+    try testing.expect(succeeded.doh.reloaded);
+    try testing.expectEqual(@as(?[]const u8, null), succeeded.doh.@"error");
+    try testing.expect(!succeeded.dot.enabled);
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
+
+    // The certificate file vanishes: the outcome names the failure, the store
+    // keeps its old generation, and the dot half is untouched.
+    try env.tmp.dir.deleteFile(io, "cert.pem");
+    const failed = applyReload(&state, io);
+    try testing.expect(failed.doh.enabled);
+    try testing.expect(!failed.doh.reloaded);
+    try testing.expectEqualStrings(
+        cert_store.humanMessage(error.CertUnreadable),
+        failed.doh.@"error".?,
+    );
+    try testing.expect(!failed.dot.enabled);
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reload_failures);
+}

src/web/metrics.zig

@@ -21,10 +21,12 @@
 const std = @import("std");
 const Allocator = std.mem.Allocator;
 
+const cert_store = @import("../server/cert_store.zig");
 const clients = @import("../server/clients.zig");
 const dns_cache = @import("../cache/dns_cache.zig");
 const dns_handler = @import("../server/handler.zig");
 const disk_monitor = @import("../storage/disk_monitor.zig");
+const dot_server = @import("../server/dot_server.zig");
 const http_util = @import("http_util.zig");
 const logging = @import("../platform/logging.zig");
 const pool_mod = @import("../upstream/pool.zig");
@@ -80,6 +82,18 @@ pub const DiskSample = struct {
     sample_failures: u64,
 };
 
+/// The DoH listener counters this exposition exports (milestone-10 ruling 10):
+/// the four every TLS listener keeps, plus DoH's `bad_requests`. A subset of
+/// `doh_server.Snapshot` on purpose — the accept-side refusal counters stay
+/// internal, exactly as they do for the DoT listener and TCP/53.
+pub const DohListenerSample = struct {
+    connections: u64,
+    tls_handshake_failures: u64,
+    idle_timeouts: u64,
+    connection_errors: u64,
+    bad_requests: u64,
+};
+
 /// One upstream, with every string owned by the caller's arena.
 pub const UpstreamSample = struct {
     url: []const u8,
@@ -103,6 +117,16 @@ pub const Sample = struct {
     retention: ?retention_mod.Stats = null,
     blocklist: ?BlocklistSample = null,
     disk: ?DiskSample = null,
+    /// One entry per enabled TLS endpoint (milestone-10 ruling 10). Rendered
+    /// under an `endpoint` label so both share the two `nxdns_cert_*`
+    /// families. `last_reload_unix` is deliberately not exported: the reload
+    /// endpoint reports cert state on demand.
+    doh_certs: ?cert_store.CertStore.Stats = null,
+    dot_certs: ?cert_store.CertStore.Stats = null,
+    /// The listener families (ruling 10): absent while an endpoint is
+    /// disabled or its bind failed, like every other unwired collaborator.
+    doh_listener: ?DohListenerSample = null,
+    dot_listener: ?dot_server.StatsSnapshot = null,
     upstreams: []const UpstreamSample = &.{},
 };
 
@@ -169,6 +193,21 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
         .sample_failures = monitor.sample_failures.load(.monotonic),
     };
 
+    if (state.doh_certs) |store| sample.doh_certs = store.snapshotStats();
+    if (state.dot_certs) |store| sample.dot_certs = store.snapshotStats();
+
+    if (state.doh_listener) |listener| {
+        const snapshot = listener.snapshotStats();
+        sample.doh_listener = .{
+            .connections = snapshot.connections,
+            .tls_handshake_failures = snapshot.tls_handshake_failures,
+            .idle_timeouts = snapshot.idle_timeouts,
+            .connection_errors = snapshot.connection_errors,
+            .bad_requests = snapshot.bad_requests,
+        };
+    }
+    if (state.dot_listener) |listener| sample.dot_listener = listener.snapshotStats();
+
     if (state.pool) |pool| sample.upstreams = try upstreams(pool, io, arena);
 
     return sample;
@@ -294,9 +333,48 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
         );
     }
 
+    if (sample.doh_listener) |listener| {
+        try counterGroup(w, "nxdns_doh_server_", "DoH listener counter", listener);
+    }
+    if (sample.dot_listener) |listener| {
+        try counterGroup(w, "nxdns_dot_server_", "DoT listener counter", listener);
+    }
+
+    if (sample.doh_certs != null or sample.dot_certs != null) try renderCerts(w, sample);
+
     if (sample.upstreams.len != 0) try renderUpstreams(w, sample.upstreams);
 }
 
+fn renderCerts(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
+    try labeledHead(w, "nxdns_cert_reloads_total", "Certificate reloads that published a new context.", "counter");
+    if (sample.doh_certs) |stats| try endpointValue(w, "nxdns_cert_reloads_total", "doh", stats.reloads);
+    if (sample.dot_certs) |stats| try endpointValue(w, "nxdns_cert_reloads_total", "dot", stats.reloads);
+
+    try labeledHead(
+        w,
+        "nxdns_cert_reload_failures_total",
+        "Certificate reloads that failed; the old certificate keeps serving.",
+        "counter",
+    );
+    if (sample.doh_certs) |stats| {
+        try endpointValue(w, "nxdns_cert_reload_failures_total", "doh", stats.reload_failures);
+    }
+    if (sample.dot_certs) |stats| {
+        try endpointValue(w, "nxdns_cert_reload_failures_total", "dot", stats.reload_failures);
+    }
+}
+
+/// The endpoint names are ours ("doh"/"dot"), so unlike a url label there is
+/// nothing to escape.
+fn endpointValue(
+    w: *std.Io.Writer,
+    name: []const u8,
+    endpoint: []const u8,
+    value: u64,
+) std.Io.Writer.Error!void {
+    try w.print("{s}{{endpoint=\"{s}\"}} {d}\n", .{ name, endpoint, value });
+}
+
 fn renderUpstreams(w: *std.Io.Writer, list: []const UpstreamSample) std.Io.Writer.Error!void {
     try labeledHead(w, "nxdns_upstream_up", "1 while an upstream is enabled and healthy.", "gauge");
     for (list) |entry| try labeledValue(w, "nxdns_upstream_up", entry.url, @intFromBool(entry.available));
@@ -528,6 +606,69 @@ test "an unwired collaborator omits its family rather than reporting zeros" {
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_disk_"));
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_upstream_"));
     try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_blocklist_"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_cert_"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_doh_server_"));
+    try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "nxdns_dot_server_"));
+}
+
+test "listener counters render only for the wired servers" {
+    const doh_only = try renderToString(testing.allocator, .{
+        .doh_listener = .{
+            .connections = 9,
+            .tls_handshake_failures = 2,
+            .idle_timeouts = 1,
+            .connection_errors = 0,
+            .bad_requests = 4,
+        },
+    });
+    defer testing.allocator.free(doh_only);
+
+    try testing.expect(std.mem.containsAtLeast(u8, doh_only, 1, "nxdns_doh_server_connections_total 9\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, doh_only, 1, "nxdns_doh_server_tls_handshake_failures_total 2\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, doh_only, 1, "nxdns_doh_server_idle_timeouts_total 1\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, doh_only, 1, "nxdns_doh_server_connection_errors_total 0\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, doh_only, 1, "nxdns_doh_server_bad_requests_total 4\n"));
+    try testing.expect(!std.mem.containsAtLeast(u8, doh_only, 1, "nxdns_dot_server_"));
+
+    const dot_only = try renderToString(testing.allocator, .{
+        .dot_listener = .{
+            .connections = 5,
+            .tls_handshake_failures = 0,
+            .idle_timeouts = 3,
+            .connection_errors = 1,
+        },
+    });
+    defer testing.allocator.free(dot_only);
+
+    try testing.expect(std.mem.containsAtLeast(u8, dot_only, 1, "nxdns_dot_server_connections_total 5\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, dot_only, 1, "nxdns_dot_server_idle_timeouts_total 3\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, dot_only, 1, "nxdns_dot_server_connection_errors_total 1\n"));
+    // The DoT listener has no HTTP layer, so no bad_requests family.
+    try testing.expect(!std.mem.containsAtLeast(u8, dot_only, 1, "nxdns_dot_server_bad_requests_total"));
+    try testing.expect(!std.mem.containsAtLeast(u8, dot_only, 1, "nxdns_doh_server_"));
+}
+
+test "cert reload counters render per endpoint, only for the wired stores" {
+    const one = try renderToString(testing.allocator, .{
+        .doh_certs = .{ .reloads = 2, .reload_failures = 1, .last_reload_unix = 1_700_000_000 },
+    });
+    defer testing.allocator.free(one);
+
+    try testing.expect(std.mem.containsAtLeast(u8, one, 1, "nxdns_cert_reloads_total{endpoint=\"doh\"} 2\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, one, 1, "nxdns_cert_reload_failures_total{endpoint=\"doh\"} 1\n"));
+    try testing.expect(!std.mem.containsAtLeast(u8, one, 1, "endpoint=\"dot\""));
+    // The wall-clock second stays off the exposition.
+    try testing.expect(!std.mem.containsAtLeast(u8, one, 1, "last_reload"));
+
+    const both = try renderToString(testing.allocator, .{
+        .doh_certs = .{ .reloads = 0, .reload_failures = 0, .last_reload_unix = 0 },
+        .dot_certs = .{ .reloads = 3, .reload_failures = 0, .last_reload_unix = 0 },
+    });
+    defer testing.allocator.free(both);
+
+    try testing.expect(std.mem.containsAtLeast(u8, both, 1, "nxdns_cert_reloads_total{endpoint=\"doh\"} 0\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, both, 1, "nxdns_cert_reloads_total{endpoint=\"dot\"} 3\n"));
+    try testing.expect(std.mem.containsAtLeast(u8, both, 1, "nxdns_cert_reload_failures_total{endpoint=\"dot\"} 0\n"));
 }
 
 test "a label value escapes the characters the format reserves" {

src/web/openapi.yaml

@@ -1463,6 +1463,25 @@ paths:
         "503":
           $ref: "#/components/responses/Unavailable"
 
+  /api/certs/reload:
+    post:
+      summary: Reload the TLS certificates from disk
+      description: |
+        Reloads the certificate and key of every enabled DoH/DoT endpoint.
+        Always answers 200: the per-endpoint outcome is the payload, and a
+        failed reload leaves the previous certificate serving.
+      responses:
+        "200":
+          description: The outcome for each endpoint.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/CertsReload"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "429":
+          $ref: "#/components/responses/RateLimited"
+
 components:
   securitySchemes:
     sessionCookie:
@@ -2044,6 +2063,29 @@ components:
           maximum: 604800
           description: Only meaningful with `paused = true`; absent means indefinite.
 
+    CertReloadOutcome:
+      type: object
+      required: [enabled, reloaded, error]
+      properties:
+        enabled:
+          type: boolean
+          description: Whether the endpoint is enabled in the configuration.
+        reloaded:
+          type: boolean
+        error:
+          type: string
+          nullable: true
+          description: Why the reload failed; null on success and while disabled.
+
+    CertsReload:
+      type: object
+      required: [doh, dot]
+      properties:
+        doh:
+          $ref: "#/components/schemas/CertReloadOutcome"
+        dot:
+          $ref: "#/components/schemas/CertReloadOutcome"
+
     Settings:
       type: object
       required: [runtime, upstream, dns, blocking, cache, web, doh_server, dot_server, edns, logging, disk, blocklist_update]

src/web/openapi.zig

@@ -48,11 +48,6 @@ test "every served route appears textually in the document" {
     }
 }
 
-test "the document does not promise what phase 9 owns" {
-    // Ruling 2: certs/reload lands with the DoH/DoT server, whole.
-    try testing.expect(!std.mem.containsAtLeast(u8, yaml, 1, "certs/reload"));
-}
-
 test "the document names the contract's fixed points" {
     for ([_][]const u8{
         "openapi: 3.0.3",

src/web/routes.zig

@@ -23,6 +23,7 @@ const router = @import("router.zig");
 
 const auth = @import("handlers/auth.zig");
 const blocklists = @import("handlers/blocklists.zig");
+const certs = @import("handlers/certs.zig");
 const clients = @import("handlers/clients.zig");
 const groups = @import("handlers/groups.zig");
 const health = @import("handlers/health.zig");
@@ -118,6 +119,9 @@ pub const table: []const router.RouteInfo = &.{
     .{ .method = .POST, .pattern = "/api/pause", .auth = .session, .handler = pause.post },
     .{ .method = .GET, .pattern = "/api/settings", .auth = .session, .handler = settings.get },
     .{ .method = .PUT, .pattern = "/api/settings", .auth = .session, .handler = settings.put },
+
+    // Certificates (milestone-10 ruling 8).
+    .{ .method = .POST, .pattern = "/api/certs/reload", .auth = .session, .handler = certs.post },
 };
 
 // ---------------------------------------------------------------------------
@@ -128,7 +132,7 @@ const std = @import("std");
 const testing = std.testing;
 
 test "the table carries every endpoint of the milestone" {
-    try testing.expectEqual(@as(usize, 55), table.len);
+    try testing.expectEqual(@as(usize, 56), table.len);
 }
 
 test "no two entries claim the same method and pattern" {

src/web/server.zig

@@ -33,10 +33,13 @@ const Allocator = std.mem.Allocator;
 const address = @import("../platform/address.zig");
 const api_limiter = @import("api_limiter.zig");
 const auth = @import("auth.zig");
+const cert_store = @import("../server/cert_store.zig");
 const clients = @import("../server/clients.zig");
 const db = @import("../storage/db.zig");
 const disk_monitor = @import("../storage/disk_monitor.zig");
 const dns_handler = @import("../server/handler.zig");
+const doh_server = @import("../server/doh_server.zig");
+const dot_server = @import("../server/dot_server.zig");
 const http_util = @import("http_util.zig");
 const local_tables_mod = @import("../server/local_tables.zig");
 const logger_mod = @import("../storage/logger.zig");
@@ -138,6 +141,16 @@ pub const WebState = struct {
     /// live-query handler subscribes.
     hub: ?*sse.Hub = null,
     sink: ?*query_sink.QuerySink = null,
+    /// The DoH/DoT certificate stores; null while an endpoint is disabled.
+    /// `POST /api/certs/reload` reloads through these, and `/metrics` reads
+    /// their counters (milestone-10 rulings 8 and 10).
+    doh_certs: ?*cert_store.CertStore = null,
+    dot_certs: ?*cert_store.CertStore = null,
+    /// The DoH/DoT listeners themselves; null while an endpoint is disabled
+    /// or its bind failed. `/metrics` reads their connection counters
+    /// (milestone-10 ruling 10).
+    doh_listener: ?*doh_server.DohServer = null,
+    dot_listener: ?*dot_server.DotServer = null,
 
     /// The web task's own connections (m7 ruling 21) — never the DNS path's.
     config_db: ?*db.Db = null,

src/web/web_integration_test.zig

@@ -56,6 +56,7 @@ const types = @import("../dns/types.zig");
 const upstreams_repo = @import("../storage/repositories/upstreams_repo.zig");
 
 const handlers_blocklists = @import("handlers/blocklists.zig");
+const handlers_certs = @import("handlers/certs.zig");
 const handlers_health = @import("handlers/health.zig");
 const handlers_live = @import("handlers/live.zig");
 const handlers_lookup = @import("handlers/lookup.zig");
@@ -670,6 +671,11 @@ const contract = [_]Contract{
     .{ .method = .GET, .pattern = "/api/settings", .auth = .session, .target = "/api/settings", .status = 200, .check = jsonShape(SettingsView) },
     .{ .method = .PUT, .pattern = "/api/settings", .auth = .session, .target = "/api/settings", .body = "{\"dns\":{\"port\":5353}}", .status = 200, .check = jsonShape(SettingsView) },
 
+    // Certificates. The walk's environment wires no cert store, so both
+    // endpoints report disabled — and the reload still answers 200 (m10
+    // ruling 8: the outcome is the payload).
+    .{ .method = .POST, .pattern = "/api/certs/reload", .auth = .session, .target = "/api/certs/reload", .status = 200, .check = jsonShape(handlers_certs.View) },
+
     // The walk's last delete returns the groups table to its seeded shape.
     .{ .method = .DELETE, .pattern = "/api/groups/{id}", .auth = .session, .target = "/api/groups/2", .status = 204, .kind = .none },
 };
@@ -773,6 +779,36 @@ test "W10 contract: every route answers its documented status and shape" {
     try bounded(env.io(), default_budget, contractWalk, .{ env.io(), env });
 }
 
+// The wire shape of the certs reload payload, restated so the handler's own
+// `View` cannot vouch for itself. Runs in every suite: it needs no socket.
+const CertOutcomeShape = struct { enabled: bool, reloaded: bool, @"error": ?[]const u8 };
+const CertsReloadShape = struct { doh: CertOutcomeShape, dot: CertOutcomeShape };
+
+test "the certs reload payload with both endpoints disabled parses strictly" {
+    const gpa = testing.allocator;
+
+    var state: server.WebState = .{ .gpa = gpa };
+    const view = handlers_certs.applyReload(&state, undefined);
+
+    var out: std.Io.Writer.Allocating = .init(gpa);
+    defer out.deinit();
+    try std.json.Stringify.value(view, .{}, &out.writer);
+
+    var arena_state: std.heap.ArenaAllocator = .init(gpa);
+    defer arena_state.deinit();
+    const parsed = try std.json.parseFromSliceLeaky(
+        CertsReloadShape,
+        arena_state.allocator(),
+        out.written(),
+        .{ .ignore_unknown_fields = false },
+    );
+    for ([_]CertOutcomeShape{ parsed.doh, parsed.dot }) |per_endpoint| {
+        try testing.expect(!per_endpoint.enabled);
+        try testing.expect(!per_endpoint.reloaded);
+        try testing.expectEqual(@as(?[]const u8, null), per_endpoint.@"error");
+    }
+}
+
 // ---------------------------------------------------------------------------
 // auth on/off matrix (rulings 17, 18)
 // ---------------------------------------------------------------------------