milestone 10: doh and dot listeners, cert store with hot reload and cert reload api

src/server/cert_store.zig

@@ -0,0 +1,810 @@
+//! Refcounted certificate holder for the DoH/DoT listeners (PLAN §9,
+//! milestone-10 rulings 6-7). One `CertStore` owns the published
+//! `tls_server.ServerContext` generation for one endpoint; listeners `acquire`
+//! it per connection and `release` it when the connection ends, so a reload
+//! never frees a context while a handshake or stream still uses it.
+//!
+//! Reload is publish-nothing-on-failure (PLAN:297): both PEM files are read
+//! and a whole new context built before anything is swapped, and any failure
+//! leaves the old generation serving. The watcher polls `statFile` every
+//! `poll_interval_s` — there is no usable file-change notification inside
+//! `std.Io` — and compares mtime+size of BOTH files against the pair recorded
+//! at the last successful load, so a same-second atomic rename still registers
+//! through the size.
+
+const std = @import("std");
+const tls_server = @import("../platform/tls_server.zig");
+
+const log = std.log.scoped(.cert_store);
+
+pub const poll_interval_s = 30;
+
+/// Largest PEM file `reload` accepts, applied to the certificate chain and the
+/// key separately. Real chains are a few KiB; anything bigger is a wrong file.
+pub const max_pem_bytes = 64 * 1024;
+
+pub const ReloadError = error{
+    OutOfMemory,
+    /// The certificate file could not be opened, read, or stat'ed.
+    CertUnreadable,
+    CertTooLarge,
+    /// The private key file could not be opened, read, or stat'ed.
+    KeyUnreadable,
+    KeyTooLarge,
+    /// The certificate chain PEM could not be parsed.
+    CertParse,
+    /// The private key PEM could not be parsed.
+    KeyParse,
+    /// The private key does not belong to the leaf certificate.
+    KeyMismatch,
+    /// Seeding the CTR-DRBG from the platform entropy source failed.
+    EntropyFailed,
+    /// Mbed TLS rejected the server configuration.
+    ConfigFailed,
+};
+
+/// One line for the operator, used by `POST /api/certs/reload` as the `error`
+/// payload field and by the watcher's warning.
+pub fn humanMessage(err: ReloadError) []const u8 {
+    return switch (err) {
+        error.OutOfMemory => "out of memory",
+        error.CertUnreadable => "certificate file is not readable",
+        error.CertTooLarge => "certificate file exceeds 64 KiB",
+        error.KeyUnreadable => "private key file is not readable",
+        error.KeyTooLarge => "private key file exceeds 64 KiB",
+        error.CertParse => "certificate PEM could not be parsed",
+        error.KeyParse => "private key PEM could not be parsed",
+        error.KeyMismatch => "private key does not belong to the certificate",
+        error.EntropyFailed => "seeding the TLS random generator failed",
+        error.ConfigFailed => "building the TLS server configuration failed",
+    };
+}
+
+/// The identity of one file's content as far as the watcher can see it
+/// without reading: modification time and size together, because an atomic
+/// rename within one filesystem timestamp granule still changes the size in
+/// any realistic re-issue, and a truncate-and-rewrite changes the mtime.
+pub const FileSig = struct {
+    mtime_ns: i96,
+    size: u64,
+};
+
+/// The stat pair recorded at the last successful load.
+pub const Signature = struct {
+    cert: FileSig,
+    key: FileSig,
+};
+
+/// The watcher's whole decision, pure so a table can test it: reload exactly
+/// when either file's mtime or size differs from the loaded pair.
+pub fn changed(loaded: Signature, observed: Signature) bool {
+    return fileChanged(loaded.cert, observed.cert) or fileChanged(loaded.key, observed.key);
+}
+
+fn fileChanged(loaded: FileSig, observed: FileSig) bool {
+    return loaded.mtime_ns != observed.mtime_ns or loaded.size != observed.size;
+}
+
+/// One published generation. Freed only when `retired` and `refs == 0`; both
+/// transitions happen under the store's mutex, so the last releaser (or the
+/// reload that retires an idle entry) frees it exactly once.
+pub const Entry = struct {
+    ctx: tls_server.ServerContext,
+    refs: u32,
+    retired: bool,
+};
+
+/// See `CertStore.after_load_hook`.
+pub const ReloadHook = struct {
+    ctx: *anyopaque,
+    call: *const fn (ctx: *anyopaque, io: std.Io) void,
+};
+
+pub const CertStore = struct {
+    gpa: std.mem.Allocator,
+    /// Borrowed from the config; must outlive the store.
+    cert_path: []const u8,
+    /// Borrowed from the config; must outlive the store.
+    key_path: []const u8,
+    /// Passed through to every `ServerContext.init`; the same lifetime rule
+    /// applies — a comptime-constant NULL-terminated array, never memory that
+    /// can go away before the store.
+    alpn: ?[*:null]const ?[*:0]const u8,
+
+    /// Guards `current`, `loaded`, and every `Entry.refs`/`Entry.retired`.
+    /// `lockUncancelable` throughout: `acquire`/`release` sit on the
+    /// per-connection path with no error union for `error.Canceled`, and no
+    /// critical section here holds I/O.
+    mutex: std.Io.Mutex,
+    /// Serializes whole reloads (read files, build context, publish) so an
+    /// overlapping reload cannot publish an older read after a newer one:
+    /// whichever reload reads the files last publishes last. Held across file
+    /// I/O, which is fine — `acquire`/`release` never touch it, so connection
+    /// latency is unchanged. Lock order: `reload_mutex` is always taken BEFORE
+    /// `mutex` and never while holding it.
+    reload_mutex: std.Io.Mutex,
+    current: *Entry,
+    loaded: Signature,
+
+    /// Test seam: runs inside `reload` between a successful `load` and the
+    /// publish, i.e. inside `reload_mutex`. Lets a test occupy the window
+    /// where an unserialized reload could be overtaken. Must not call
+    /// `reload` synchronously (that would self-deadlock on `reload_mutex`).
+    after_load_hook: ?ReloadHook,
+
+    reloads: std.atomic.Value(u64),
+    reload_failures: std.atomic.Value(u64),
+    /// Wall-clock second of the last successful load, including the one in
+    /// `init`.
+    last_reload_unix: std.atomic.Value(i64),
+
+    pub const Stats = struct {
+        reloads: u64,
+        reload_failures: u64,
+        last_reload_unix: i64,
+    };
+
+    /// Performs the initial load. A failure here is the boot-time "bad cert"
+    /// case: nothing is constructed and the caller exits with
+    /// `humanMessage(err)` (milestone-10 ruling 11).
+    pub fn init(
+        gpa: std.mem.Allocator,
+        io: std.Io,
+        cert_path: []const u8,
+        key_path: []const u8,
+        alpn: ?[*:null]const ?[*:0]const u8,
+    ) ReloadError!CertStore {
+        const first = try load(gpa, io, cert_path, key_path, alpn);
+        return .{
+            .gpa = gpa,
+            .cert_path = cert_path,
+            .key_path = key_path,
+            .alpn = alpn,
+            .mutex = .init,
+            .reload_mutex = .init,
+            .current = first.entry,
+            .loaded = first.sig,
+            .after_load_hook = null,
+            .reloads = .init(0),
+            .reload_failures = .init(0),
+            .last_reload_unix = .init(std.Io.Clock.real.now(io).toSeconds()),
+        };
+    }
+
+    /// Every listener must have released its entries first: the current
+    /// generation must be idle, and a retired generation with refs would have
+    /// been freed by its last release already.
+    pub fn deinit(self: *CertStore, io: std.Io) void {
+        self.mutex.lockUncancelable(io);
+        const current = self.current;
+        std.debug.assert(current.refs == 0);
+        self.mutex.unlock(io);
+        self.destroyEntry(current);
+        self.* = undefined;
+    }
+
+    /// Pins the current generation for one connection. The returned entry
+    /// stays valid until the matching `release`, across any number of reloads.
+    pub fn acquire(self: *CertStore, io: std.Io) *Entry {
+        self.mutex.lockUncancelable(io);
+        defer self.mutex.unlock(io);
+        self.current.refs += 1;
+        return self.current;
+    }
+
+    pub fn release(self: *CertStore, io: std.Io, entry: *Entry) void {
+        self.mutex.lockUncancelable(io);
+        std.debug.assert(entry.refs > 0);
+        entry.refs -= 1;
+        const free_it = entry.retired and entry.refs == 0;
+        self.mutex.unlock(io);
+        if (free_it) self.destroyEntry(entry);
+    }
+
+    /// Builds a whole new generation from the files on disk, then swaps it in
+    /// and retires the old one. On ANY failure the store is untouched: the old
+    /// generation keeps serving and the loaded signature keeps its value, so
+    /// the watcher retries on its next poll.
+    ///
+    /// `reload_mutex` covers the whole read-build-publish sequence: without
+    /// it, a reload could read the files, stall in `ServerContext.init`, and
+    /// publish that stale read over a generation another reload already
+    /// published from newer bytes.
+    pub fn reload(self: *CertStore, io: std.Io) ReloadError!void {
+        self.reload_mutex.lockUncancelable(io);
+        defer self.reload_mutex.unlock(io);
+
+        const next = load(self.gpa, io, self.cert_path, self.key_path, self.alpn) catch |err| {
+            _ = self.reload_failures.fetchAdd(1, .monotonic);
+            return err;
+        };
+
+        if (self.after_load_hook) |hook| hook.call(hook.ctx, io);
+
+        self.mutex.lockUncancelable(io);
+        const old = self.current;
+        self.current = next.entry;
+        self.loaded = next.sig;
+        old.retired = true;
+        const free_old = old.refs == 0;
+        self.mutex.unlock(io);
+
+        if (free_old) self.destroyEntry(old);
+        _ = self.reloads.fetchAdd(1, .monotonic);
+        self.last_reload_unix.store(std.Io.Clock.real.now(io).toSeconds(), .monotonic);
+    }
+
+    /// Sleep first: `init` just loaded the files this poll would compare
+    /// against. `.boot` so a suspended box still sees the interval elapse.
+    pub fn watch(self: *CertStore, io: std.Io) std.Io.Cancelable!void {
+        const interval: std.Io.Clock.Duration = .{
+            .raw = .fromSeconds(poll_interval_s),
+            .clock = .boot,
+        };
+        while (true) {
+            try interval.sleep(io);
+            self.pollOnce(io);
+        }
+    }
+
+    /// One watcher pass. A file that cannot be stat'ed only warns — a
+    /// mid-rename window or a permission slip is not evidence the certificate
+    /// changed, and the old one keeps serving either way. A failed reload
+    /// warns and counts (`reload_failures`); the signature stays at the loaded
+    /// pair, so every subsequent poll retries until the files parse.
+    pub fn pollOnce(self: *CertStore, io: std.Io) void {
+        const cert_sig = statSig(io, self.cert_path) catch {
+            log.warn("stat {s} failed; keeping the loaded certificate", .{self.cert_path});
+            return;
+        };
+        const key_sig = statSig(io, self.key_path) catch {
+            log.warn("stat {s} failed; keeping the loaded certificate", .{self.key_path});
+            return;
+        };
+        const observed: Signature = .{ .cert = cert_sig, .key = key_sig };
+
+        self.mutex.lockUncancelable(io);
+        const loaded = self.loaded;
+        self.mutex.unlock(io);
+        if (!changed(loaded, observed)) return;
+
+        if (self.reload(io)) {
+            log.info("certificate reloaded from {s}", .{self.cert_path});
+        } else |err| {
+            log.warn("certificate reload from {s} failed ({s}); the old certificate keeps serving", .{
+                self.cert_path,
+                humanMessage(err),
+            });
+        }
+    }
+
+    pub fn snapshotStats(self: *const CertStore) Stats {
+        return .{
+            .reloads = self.reloads.load(.monotonic),
+            .reload_failures = self.reload_failures.load(.monotonic),
+            .last_reload_unix = self.last_reload_unix.load(.monotonic),
+        };
+    }
+
+    fn destroyEntry(self: *CertStore, entry: *Entry) void {
+        entry.ctx.deinit(self.gpa);
+        self.gpa.destroy(entry);
+    }
+};
+
+const Loaded = struct {
+    entry: *Entry,
+    sig: Signature,
+};
+
+/// Stat first, then read: a file replaced between the two makes the recorded
+/// signature stale, so the watcher's next poll sees the difference and loads
+/// again. The other order would record the new signature over the old bytes
+/// and miss the update.
+fn load(
+    gpa: std.mem.Allocator,
+    io: std.Io,
+    cert_path: []const u8,
+    key_path: []const u8,
+    alpn: ?[*:null]const ?[*:0]const u8,
+) ReloadError!Loaded {
+    const cert_sig = statSig(io, cert_path) catch return error.CertUnreadable;
+    const key_sig = statSig(io, key_path) catch return error.KeyUnreadable;
+
+    const cert_pem = readPem(gpa, io, cert_path) catch |err| return switch (err) {
+        error.OutOfMemory => error.OutOfMemory,
+        error.TooLarge => error.CertTooLarge,
+        error.Unreadable => error.CertUnreadable,
+    };
+    defer gpa.free(cert_pem);
+
+    const key_pem = readPem(gpa, io, key_path) catch |err| return switch (err) {
+        error.OutOfMemory => error.OutOfMemory,
+        error.TooLarge => error.KeyTooLarge,
+        error.Unreadable => error.KeyUnreadable,
+    };
+    defer gpa.free(key_pem);
+
+    const entry = try gpa.create(Entry);
+    errdefer gpa.destroy(entry);
+    entry.* = .{
+        .ctx = try tls_server.ServerContext.init(gpa, cert_pem, key_pem, alpn),
+        .refs = 0,
+        .retired = false,
+    };
+
+    return .{
+        .entry = entry,
+        .sig = .{ .cert = cert_sig, .key = key_sig },
+    };
+}
+
+/// Mbed TLS wants PEM with a terminating zero byte counted in the length, so
+/// the file lands in a sentinel-terminated allocation. The limit admits
+/// exactly `max_pem_bytes` and rejects the first byte beyond it.
+fn readPem(
+    gpa: std.mem.Allocator,
+    io: std.Io,
+    path: []const u8,
+) error{ OutOfMemory, TooLarge, Unreadable }![:0]u8 {
+    return std.Io.Dir.cwd().readFileAllocOptions(
+        io,
+        path,
+        gpa,
+        .limited(max_pem_bytes + 1),
+        .of(u8),
+        0,
+    ) catch |err| switch (err) {
+        error.OutOfMemory => error.OutOfMemory,
+        error.StreamTooLong => error.TooLarge,
+        else => error.Unreadable,
+    };
+}
+
+fn statSig(io: std.Io, path: []const u8) !FileSig {
+    const st = try std.Io.Dir.cwd().statFile(io, path, .{});
+    return .{ .mtime_ns = st.mtime.nanoseconds, .size = st.size };
+}
+
+// ---------------------------------------------------------------------------
+// tests
+// ---------------------------------------------------------------------------
+
+const fixtures = @import("test_fixtures");
+const testing = std.testing;
+
+test "the stat-compare decision table" {
+    const a: FileSig = .{ .mtime_ns = 1_000, .size = 100 };
+    const a_later: FileSig = .{ .mtime_ns = 2_000, .size = 100 };
+    const a_grown: FileSig = .{ .mtime_ns = 1_000, .size = 101 };
+    const b: FileSig = .{ .mtime_ns = 5_000, .size = 500 };
+    const b_later: FileSig = .{ .mtime_ns = 6_000, .size = 500 };
+    const b_grown: FileSig = .{ .mtime_ns = 5_000, .size = 501 };
+
+    const cases = [_]struct {
+        loaded: Signature,
+        observed: Signature,
+        want: bool,
+    }{
+        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a, .key = b }, .want = false },
+        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a_later, .key = b }, .want = true },
+        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a_grown, .key = b }, .want = true },
+        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a, .key = b_later }, .want = true },
+        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a, .key = b_grown }, .want = true },
+        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a_later, .key = b_grown }, .want = true },
+        // A same-second rewrite registers through the size alone.
+        .{
+            .loaded = .{ .cert = a, .key = b },
+            .observed = .{ .cert = .{ .mtime_ns = 1_000, .size = 99 }, .key = b },
+            .want = true,
+        },
+        // The pair swapped between the two files is a change, not a wash.
+        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = b, .key = a }, .want = true },
+    };
+
+    for (cases) |case| {
+        try testing.expectEqual(case.want, changed(case.loaded, case.observed));
+    }
+}
+
+test "every reload error carries a human message" {
+    inline for (comptime @typeInfo(ReloadError).error_set.?) |e| {
+        try testing.expect(humanMessage(@field(anyerror, e.name)).len > 0);
+    }
+}
+
+/// Fixture PEMs written into a tmp directory, addressed by cwd-relative paths
+/// the same way `app.zig` hands config paths to the store. Must not move after
+/// `init`: the path slices point into the buffers below.
+const TestEnv = struct {
+    threaded: std.Io.Threaded,
+    tmp: testing.TmpDir,
+    cert_path_buf: [128]u8,
+    key_path_buf: [128]u8,
+    cert_path: []const u8,
+    key_path: []const u8,
+
+    fn init(env: *TestEnv) !void {
+        env.threaded = .init(testing.allocator, .{});
+        errdefer env.threaded.deinit();
+        env.tmp = testing.tmpDir(.{});
+        errdefer env.tmp.cleanup();
+
+        const env_io = env.threaded.io();
+        try env.tmp.dir.writeFile(env_io, .{ .sub_path = "cert.pem", .data = fixtures.cert_pem });
+        try env.tmp.dir.writeFile(env_io, .{ .sub_path = "key.pem", .data = fixtures.key_pem });
+        env.cert_path = try std.fmt.bufPrint(&env.cert_path_buf, ".zig-cache/tmp/{s}/cert.pem", .{env.tmp.sub_path});
+        env.key_path = try std.fmt.bufPrint(&env.key_path_buf, ".zig-cache/tmp/{s}/key.pem", .{env.tmp.sub_path});
+    }
+
+    fn deinit(env: *TestEnv) void {
+        env.tmp.cleanup();
+        env.threaded.deinit();
+    }
+
+    fn io(env: *TestEnv) std.Io {
+        return env.threaded.io();
+    }
+};
+
+test "loaded PEM bytes are NUL-terminated and intact" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+
+    const pem = try readPem(testing.allocator, env.io(), env.cert_path);
+    defer testing.allocator.free(pem);
+
+    try testing.expectEqualStrings(fixtures.cert_pem, pem);
+    try testing.expectEqual(@as(u8, 0), pem[pem.len]);
+}
+
+test "the size cap admits 64 KiB and rejects one byte more" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    const at_cap = try testing.allocator.alloc(u8, max_pem_bytes);
+    defer testing.allocator.free(at_cap);
+    @memset(at_cap, 'a');
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = at_cap });
+
+    const loaded = try readPem(testing.allocator, io, env.cert_path);
+    testing.allocator.free(loaded);
+
+    const over = try testing.allocator.alloc(u8, max_pem_bytes + 1);
+    defer testing.allocator.free(over);
+    @memset(over, 'a');
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = over });
+
+    try testing.expectError(error.TooLarge, readPem(testing.allocator, io, env.cert_path));
+}
+
+test "init fails typed on a missing file and on garbage PEM" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    try testing.expectError(
+        error.CertUnreadable,
+        CertStore.init(testing.allocator, io, "./nxdns-no-such-cert-9b31.pem", env.key_path, null),
+    );
+    try testing.expectError(
+        error.KeyUnreadable,
+        CertStore.init(testing.allocator, io, env.cert_path, "./nxdns-no-such-key-9b31.pem", null),
+    );
+
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "not a certificate" });
+    try testing.expectError(
+        error.CertParse,
+        CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null),
+    );
+}
+
+test "init fails typed on an oversized certificate" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    const over = try testing.allocator.alloc(u8, max_pem_bytes + 1);
+    defer testing.allocator.free(over);
+    @memset(over, 'a');
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = over });
+
+    try testing.expectError(
+        error.CertTooLarge,
+        CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null),
+    );
+}
+
+test "acquire and release across a reload free the old entry after the last release" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    const old = store.acquire(io);
+    const old_again = store.acquire(io);
+    try testing.expectEqual(old, old_again);
+    try testing.expectEqual(@as(u32, 2), old.refs);
+    try testing.expect(!old.retired);
+
+    try store.reload(io);
+
+    // The old generation is retired but pinned; the store publishes a new one.
+    try testing.expect(old.retired);
+    try testing.expectEqual(@as(u32, 2), old.refs);
+    const fresh = store.acquire(io);
+    try testing.expect(fresh != old);
+    try testing.expect(!fresh.retired);
+
+    // The first release leaves the old entry alive for the second holder;
+    // the testing allocator proves the second release frees it exactly once.
+    store.release(io, old);
+    try testing.expectEqual(@as(u32, 1), old.refs);
+    store.release(io, old);
+
+    store.release(io, fresh);
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
+}
+
+test "a reload with no holders frees the old entry immediately" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    // No acquire in flight: the swap itself must free the old generation, or
+    // the testing allocator reports the leak at the end of this test.
+    try store.reload(io);
+    try store.reload(io);
+    try testing.expectEqual(@as(u64, 2), store.snapshotStats().reloads);
+}
+
+test "a failed reload publishes nothing" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    const before = store.acquire(io);
+    store.release(io, before);
+
+    // Bad PEM bytes: the file reads fine and fails in the parser.
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "garbage" });
+    try testing.expectError(error.CertParse, store.reload(io));
+
+    var held = store.acquire(io);
+    try testing.expectEqual(before, held);
+    try testing.expect(!held.retired);
+    store.release(io, held);
+
+    // Bad path: the key file vanishes.
+    try env.tmp.dir.deleteFile(io, "key.pem");
+    try testing.expectError(error.KeyUnreadable, store.reload(io));
+
+    held = store.acquire(io);
+    try testing.expectEqual(before, held);
+    store.release(io, held);
+
+    const stats = store.snapshotStats();
+    try testing.expectEqual(@as(u64, 0), stats.reloads);
+    try testing.expectEqual(@as(u64, 2), stats.reload_failures);
+}
+
+test "a mismatched key pair fails the reload and keeps the old pair" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    const before = store.acquire(io);
+    store.release(io, before);
+
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "key.pem", .data = fixtures.mismatched_key_pem });
+    try testing.expectError(error.KeyMismatch, store.reload(io));
+
+    const held = store.acquire(io);
+    try testing.expectEqual(before, held);
+    store.release(io, held);
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reload_failures);
+}
+
+test "pollOnce reloads on a changed stat pair and stays put on an unchanged one" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    store.pollOnce(io);
+    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
+
+    // Same certificate plus a trailing newline: the PEM still parses and the
+    // size differs even when the rewrite lands within one timestamp granule.
+    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
+    defer testing.allocator.free(grown);
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = grown });
+
+    const old = store.acquire(io);
+    store.release(io, old);
+    store.pollOnce(io);
+
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
+    const fresh = store.acquire(io);
+    try testing.expect(fresh != old);
+    store.release(io, fresh);
+
+    store.pollOnce(io);
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
+}
+
+test "pollOnce warns and keeps serving when a reload fails" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    const before = store.acquire(io);
+    store.release(io, before);
+
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
+    store.pollOnce(io);
+
+    const stats = store.snapshotStats();
+    try testing.expectEqual(@as(u64, 0), stats.reloads);
+    try testing.expectEqual(@as(u64, 1), stats.reload_failures);
+
+    const held = store.acquire(io);
+    try testing.expectEqual(before, held);
+    store.release(io, held);
+}
+
+test "pollOnce does nothing when a file cannot be stat'ed" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    try env.tmp.dir.deleteFile(io, "cert.pem");
+    store.pollOnce(io);
+
+    const stats = store.snapshotStats();
+    try testing.expectEqual(@as(u64, 0), stats.reloads);
+    try testing.expectEqual(@as(u64, 0), stats.reload_failures);
+}
+
+test "init records the load time and reload advances it" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    const at_init = store.snapshotStats();
+    try testing.expectEqual(@as(u64, 0), at_init.reloads);
+    try testing.expectEqual(@as(u64, 0), at_init.reload_failures);
+    try testing.expect(at_init.last_reload_unix > 0);
+
+    try store.reload(io);
+    const after = store.snapshotStats();
+    try testing.expectEqual(@as(u64, 1), after.reloads);
+    try testing.expect(after.last_reload_unix >= at_init.last_reload_unix);
+}
+
+test "init accepts a comptime ALPN list" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    const alpn: [1:null]?[*:0]const u8 = .{"http/1.1"};
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, &alpn);
+    defer store.deinit(io);
+
+    try store.reload(io);
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
+}
+
+test "the watch loop sleeps before polling and returns on cancel" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    var future = try io.concurrent(CertStore.watch, .{ &store, io });
+    // Cancelling immediately proves the loop starts by sleeping rather than
+    // by reloading.
+    try testing.expectError(error.Canceled, future.cancel(io));
+    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
+}
+
+// Pins the reload-ordering race via `after_load_hook`: the hook runs inside
+// reload A's load-to-publish window, where it (1) probes that `reload_mutex`
+// is held, the serialization this exists for (without it a second reload
+// could publish here and be overwritten by A's stale read), then (2) stages
+// newer cert bytes and starts an overlapping reload B. B can only read the
+// files after A publishes, so B both reads and publishes last and the newer
+// bytes win. This proves the lock spans the window and last-read-wins for
+// this schedule; it does not enumerate every interleaving.
+test "a reload overlapping another reload's window publishes last" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
+    defer testing.allocator.free(grown);
+
+    const Window = struct {
+        store: *CertStore,
+        env: *TestEnv,
+        grown: []const u8,
+        overlapping: ?std.Io.Future(ReloadError!void) = null,
+        lock_spans_window: bool = false,
+        staged: bool = false,
+
+        // Reload B re-enters this hook under `reload_mutex`, so its read of
+        // `overlapping` is ordered after A's write and the early return makes
+        // the window work run exactly once.
+        fn call(ctx: *anyopaque, hook_io: std.Io) void {
+            const w: *@This() = @ptrCast(@alignCast(ctx));
+            if (w.overlapping != null) return;
+            if (w.store.reload_mutex.tryLock()) {
+                w.store.reload_mutex.unlock(hook_io);
+            } else {
+                w.lock_spans_window = true;
+            }
+            w.env.tmp.dir.writeFile(hook_io, .{ .sub_path = "cert.pem", .data = w.grown }) catch return;
+            w.staged = true;
+            w.overlapping = hook_io.concurrent(CertStore.reload, .{ w.store, hook_io }) catch null;
+        }
+    };
+
+    var window: Window = .{ .store = &store, .env = &env, .grown = grown };
+    store.after_load_hook = .{ .ctx = &window, .call = Window.call };
+
+    try store.reload(io);
+
+    try testing.expect(window.lock_spans_window);
+    try testing.expect(window.staged);
+    var overlapping = window.overlapping orelse return error.ConcurrentUnavailable;
+    try overlapping.await(io);
+
+    try testing.expectEqual(@as(u64, 2), store.snapshotStats().reloads);
+    store.mutex.lockUncancelable(io);
+    const final = store.loaded;
+    store.mutex.unlock(io);
+    try testing.expectEqual(@as(u64, grown.len), final.cert.size);
+}