nxdns/src/server/cert_store.zig

//! Refcounted certificate holder for the DoH/DoT listeners (PLAN §9,
//! milestone-10 rulings 6-7). One `CertStore` owns the published
//! `tls_server.ServerContext` generation for one endpoint; listeners `acquire`
//! it per connection and `release` it when the connection ends, so a reload
//! never frees a context while a handshake or stream still uses it.
//!
//! Reload is publish-nothing-on-failure (PLAN:297): both PEM files are read
//! and a whole new context built before anything is swapped, and any failure
//! leaves the old generation serving. The watcher polls `statFile` every
//! `poll_interval_s` — there is no usable file-change notification inside
//! `std.Io` — and compares mtime+size of BOTH files against the pair recorded
//! at the last successful load, so a same-second atomic rename still registers
//! through the size.

const std = @import("std");

const events = @import("../storage/events.zig");
const tls_server = @import("../platform/tls_server.zig");

const log = std.log.scoped(.cert_store);

pub const poll_interval_s = 30;

/// Largest PEM file `reload` accepts, applied to the certificate chain and the
/// key separately. Real chains are a few KiB; anything bigger is a wrong file.
pub const max_pem_bytes = 64 * 1024;

pub const ReloadError = error{
    OutOfMemory,
    /// The certificate file could not be opened, read, or stat'ed.
    CertUnreadable,
    CertTooLarge,
    /// The private key file could not be opened, read, or stat'ed.
    KeyUnreadable,
    KeyTooLarge,
    /// The certificate chain PEM could not be parsed.
    CertParse,
    /// The private key PEM could not be parsed.
    KeyParse,
    /// The private key does not belong to the leaf certificate.
    KeyMismatch,
    /// Seeding the CTR-DRBG from the platform entropy source failed.
    EntropyFailed,
    /// Mbed TLS rejected the server configuration.
    ConfigFailed,
};

/// One line for the operator, used by `POST /api/certs/reload` as the `error`
/// payload field and by the watcher's warning.
pub fn humanMessage(err: ReloadError) []const u8 {
    return switch (err) {
        error.OutOfMemory => "out of memory",
        error.CertUnreadable => "certificate file is not readable",
        error.CertTooLarge => "certificate file exceeds 64 KiB",
        error.KeyUnreadable => "private key file is not readable",
        error.KeyTooLarge => "private key file exceeds 64 KiB",
        error.CertParse => "certificate PEM could not be parsed",
        error.KeyParse => "private key PEM could not be parsed",
        error.KeyMismatch => "private key does not belong to the certificate",
        error.EntropyFailed => "seeding the TLS random generator failed",
        error.ConfigFailed => "building the TLS server configuration failed",
    };
}

/// The identity of one file's content as far as the watcher can see it
/// without reading: modification time and size together, because an atomic
/// rename within one filesystem timestamp granule still changes the size in
/// any realistic re-issue, and a truncate-and-rewrite changes the mtime.
pub const FileSig = struct {
    mtime_ns: i96,
    size: u64,
};

/// The stat pair recorded at the last successful load.
pub const Signature = struct {
    cert: FileSig,
    key: FileSig,
};

/// The watcher's whole decision, pure so a table can test it: reload exactly
/// when either file's mtime or size differs from the loaded pair.
pub fn changed(loaded: Signature, observed: Signature) bool {
    return fileChanged(loaded.cert, observed.cert) or fileChanged(loaded.key, observed.key);
}

fn fileChanged(loaded: FileSig, observed: FileSig) bool {
    return loaded.mtime_ns != observed.mtime_ns or loaded.size != observed.size;
}

/// One published generation. Freed only when `retired` and `refs == 0`; both
/// transitions happen under the store's mutex, so the last releaser (or the
/// reload that retires an idle entry) frees it exactly once.
pub const Entry = struct {
    ctx: tls_server.ServerContext,
    refs: u32,
    retired: bool,
};

/// See `CertStore.after_load_hook`.
pub const ReloadHook = struct {
    ctx: *anyopaque,
    call: *const fn (ctx: *anyopaque, io: std.Io) void,
};

pub const CertStore = struct {
    /// Which endpoint's certificate this store holds. It is the subject of
    /// every `certificate.reload` event, and a box serving both DoH and DoT
    /// runs two stores over two file pairs.
    pub const Kind = enum { doh, dot };

    gpa: std.mem.Allocator,
    /// Owned. A settings apply replaces both paths, so a borrowed config slice
    /// would dangle the moment the row it came from went away. Read and
    /// written only under `reload_mutex`, which every apply and every reload
    /// holds for its whole read-build-publish sequence.
    cert_path: []u8,
    /// Owned; see `cert_path`.
    key_path: []u8,
    /// Passed through to every `ServerContext.init`; the same lifetime rule
    /// applies — a comptime-constant NULL-terminated array, never memory that
    /// can go away before the store.
    alpn: ?[*:null]const ?[*:0]const u8,

    /// Guards `current`, `loaded`, and every `Entry.refs`/`Entry.retired`.
    /// `lockUncancelable` throughout: `acquire`/`release` sit on the
    /// per-connection path with no error union for `error.Canceled`, and no
    /// critical section here holds I/O.
    mutex: std.Io.Mutex,
    /// Serializes whole reloads (read files, build context, publish) so an
    /// overlapping reload cannot publish an older read after a newer one:
    /// whichever reload reads the files last publishes last. Held across file
    /// I/O, which is fine — `acquire`/`release` never touch it, so connection
    /// latency is unchanged. Lock order: `reload_mutex` is always taken BEFORE
    /// `mutex` and never while holding it.
    reload_mutex: std.Io.Mutex,
    current: *Entry,
    loaded: Signature,

    /// Test seam: runs inside `reload` between a successful `load` and the
    /// publish, i.e. inside `reload_mutex`. Lets a test occupy the window
    /// where an unserialized reload could be overtaken. Must not call
    /// `reload`, `pollOnce` or `preparePathChange` synchronously — all four
    /// take `reload_mutex`, which is not reentrant.
    after_load_hook: ?ReloadHook,

    /// Set by the composition root right after `init`, with `diagnostics`.
    kind: Kind = .doh,
    /// Wired the same way and for the same reason as every other subsystem's:
    /// the store is fully usable without it, and `nxdns check` has none.
    diagnostics: ?*events.Store = null,

    reloads: std.atomic.Value(u64),
    reload_failures: std.atomic.Value(u64),
    /// Wall-clock second of the last successful load, including the one in
    /// `init`.
    last_reload_unix: std.atomic.Value(i64),

    pub const Stats = struct {
        reloads: u64,
        reload_failures: u64,
        last_reload_unix: i64,
    };

    /// Performs the initial load. A failure here is the boot-time "bad cert"
    /// case: nothing is constructed and the caller exits with
    /// `humanMessage(err)` (milestone-10 ruling 11).
    pub fn init(
        gpa: std.mem.Allocator,
        io: std.Io,
        cert_path: []const u8,
        key_path: []const u8,
        alpn: ?[*:null]const ?[*:0]const u8,
    ) ReloadError!CertStore {
        const first = try load(gpa, io, cert_path, key_path, alpn);
        errdefer {
            first.entry.ctx.deinit(gpa);
            gpa.destroy(first.entry);
        }
        const owned_cert = try gpa.dupe(u8, cert_path);
        errdefer gpa.free(owned_cert);
        const owned_key = try gpa.dupe(u8, key_path);
        return .{
            .gpa = gpa,
            .cert_path = owned_cert,
            .key_path = owned_key,
            .alpn = alpn,
            .mutex = .init,
            .reload_mutex = .init,
            .current = first.entry,
            .loaded = first.sig,
            .after_load_hook = null,
            .reloads = .init(0),
            .reload_failures = .init(0),
            .last_reload_unix = .init(std.Io.Clock.real.now(io).toSeconds()),
        };
    }

    /// Every listener must have released its entries first: the current
    /// generation must be idle, and a retired generation with refs would have
    /// been freed by its last release already.
    pub fn deinit(self: *CertStore, io: std.Io) void {
        self.mutex.lockUncancelable(io);
        const current = self.current;
        std.debug.assert(current.refs == 0);
        self.mutex.unlock(io);
        self.destroyEntry(current);
        self.gpa.free(self.cert_path);
        self.gpa.free(self.key_path);
        self.* = undefined;
    }

    /// Pins the current generation for one connection. The returned entry
    /// stays valid until the matching `release`, across any number of reloads.
    pub fn acquire(self: *CertStore, io: std.Io) *Entry {
        self.mutex.lockUncancelable(io);
        defer self.mutex.unlock(io);
        self.current.refs += 1;
        return self.current;
    }

    pub fn release(self: *CertStore, io: std.Io, entry: *Entry) void {
        self.mutex.lockUncancelable(io);
        std.debug.assert(entry.refs > 0);
        entry.refs -= 1;
        const free_it = entry.retired and entry.refs == 0;
        self.mutex.unlock(io);
        if (free_it) self.destroyEntry(entry);
    }

    /// Builds a whole new generation from the files on disk, then swaps it in
    /// and retires the old one. On ANY failure the store is untouched: the old
    /// generation keeps serving and the loaded signature keeps its value, so
    /// the watcher retries on its next poll.
    ///
    /// `reload_mutex` covers the whole read-build-publish sequence: without
    /// it, a reload could read the files, stall in `ServerContext.init`, and
    /// publish that stale read over a generation another reload already
    /// published from newer bytes.
    pub fn reload(self: *CertStore, io: std.Io) ReloadError!void {
        self.reload_mutex.lockUncancelable(io);
        defer self.reload_mutex.unlock(io);
        return self.reloadLocked(io);
    }

    /// `reload`'s body, for callers that already hold `reload_mutex` —
    /// `pollOnce` does, because it must read `cert_path`/`key_path` under the
    /// same lock an apply replaces them under. `std.Io.Mutex` is not
    /// reentrant, so this exists rather than a recursive `reload` call.
    fn reloadLocked(self: *CertStore, io: std.Io) ReloadError!void {
        const next = load(self.gpa, io, self.cert_path, self.key_path, self.alpn) catch |err| {
            _ = self.reload_failures.fetchAdd(1, .monotonic);
            return err;
        };

        if (self.after_load_hook) |hook| hook.call(hook.ctx, io);

        self.mutex.lockUncancelable(io);
        const old = self.current;
        self.current = next.entry;
        self.loaded = next.sig;
        old.retired = true;
        const free_old = old.refs == 0;
        self.mutex.unlock(io);

        if (free_old) self.destroyEntry(old);
        _ = self.reloads.fetchAdd(1, .monotonic);
        self.last_reload_unix.store(std.Io.Clock.real.now(io).toSeconds(), .monotonic);
    }

    /// A candidate certificate loaded from new paths, not yet published.
    /// Holding one means holding `reload_mutex`: exactly one of
    /// `publishPathChange` or `abortPathChange` must follow, and it releases
    /// the lock.
    pub const PreparedPaths = struct {
        cert_path: []u8,
        key_path: []u8,
        loaded: Loaded,
        /// Read at prepare so publish reads no clock: publish must touch
        /// nothing outside memory it already owns.
        loaded_at_unix: i64,
    };

    /// Prepare half of a `doh_server`/`dot_server` cert-path change: takes
    /// `reload_mutex` and loads the certificate and key from the NEW paths.
    /// Nothing is published, so a failure leaves the store exactly as it was —
    /// the old certificate keeps serving and the caller writes no DB row. The
    /// lock is released on failure and held on success, which is what makes
    /// the whole apply serialized against `reload` and `pollOnce`.
    pub fn preparePathChange(
        self: *CertStore,
        io: std.Io,
        cert_path: []const u8,
        key_path: []const u8,
    ) ReloadError!PreparedPaths {
        self.reload_mutex.lockUncancelable(io);
        errdefer self.reload_mutex.unlock(io);

        const owned_cert = try self.gpa.dupe(u8, cert_path);
        errdefer self.gpa.free(owned_cert);
        const owned_key = try self.gpa.dupe(u8, key_path);
        errdefer self.gpa.free(owned_key);

        const next = load(self.gpa, io, cert_path, key_path, self.alpn) catch |err| {
            _ = self.reload_failures.fetchAdd(1, .monotonic);
            return err;
        };
        return .{
            .cert_path = owned_cert,
            .key_path = owned_key,
            .loaded = next,
            .loaded_at_unix = std.Io.Clock.real.now(io).toSeconds(),
        };
    }

    /// Publish half: infallible and I/O-free. The paths and the generation
    /// they were loaded from are installed together — the generation `mutex`
    /// is taken only for that swap, inside `reload_mutex`, the same lock order
    /// `reload` uses. Releases `reload_mutex`.
    ///
    /// Connections that pinned the old generation finish on the old
    /// certificate; the old entry is freed once its last reader releases.
    pub fn publishPathChange(self: *CertStore, io: std.Io, prepared: PreparedPaths) void {
        const old_cert_path = self.cert_path;
        const old_key_path = self.key_path;

        self.mutex.lockUncancelable(io);
        const old = self.current;
        self.cert_path = prepared.cert_path;
        self.key_path = prepared.key_path;
        self.current = prepared.loaded.entry;
        self.loaded = prepared.loaded.sig;
        old.retired = true;
        const free_old = old.refs == 0;
        self.mutex.unlock(io);

        // The branch runs before the counter bump, never after: a `bool` still
        // live across an atomic read-modify-write is the zig 0.16.0 Debug
        // miscompile AGENTS.md documents.
        if (free_old) self.destroyEntry(old);
        self.gpa.free(old_cert_path);
        self.gpa.free(old_key_path);

        _ = self.reloads.fetchAdd(1, .monotonic);
        self.last_reload_unix.store(prepared.loaded_at_unix, .monotonic);
        self.reload_mutex.unlock(io);
    }

    /// Discards a prepared candidate — the commit that would have published it
    /// failed, or a sibling owner's prepare did. Releases `reload_mutex`.
    pub fn abortPathChange(self: *CertStore, io: std.Io, prepared: PreparedPaths) void {
        self.gpa.free(prepared.cert_path);
        self.gpa.free(prepared.key_path);
        self.destroyEntry(prepared.loaded.entry);
        self.reload_mutex.unlock(io);
    }

    /// Sleep first: `init` just loaded the files this poll would compare
    /// against. `.boot` so a suspended box still sees the interval elapse.
    pub fn watch(self: *CertStore, io: std.Io) std.Io.Cancelable!void {
        const interval: std.Io.Clock.Duration = .{
            .raw = .fromSeconds(poll_interval_s),
            .clock = .boot,
        };
        while (true) {
            try interval.sleep(io);
            self.pollOnce(io, std.Io.Clock.real.now(io).toSeconds());
        }
    }

    /// One watcher pass. A file that cannot be stat'ed only warns — a
    /// mid-rename window or a permission slip is not evidence the certificate
    /// changed, and the old one keeps serving either way. A failed reload
    /// warns and counts (`reload_failures`); the signature stays at the loaded
    /// pair, so every subsequent poll retries until the files parse.
    ///
    /// The whole pass runs under `reload_mutex`: the paths it stats are the
    /// ones an apply replaces, and a poll that read a path outside the lock
    /// could stat a freed slice or reload a pair that was never published
    /// together.
    pub fn pollOnce(self: *CertStore, io: std.Io, now_s: i64) void {
        self.reload_mutex.lockUncancelable(io);
        defer self.reload_mutex.unlock(io);

        const cert_sig = statSig(io, self.cert_path) catch |err| {
            log.warn("stat {s} failed; keeping the loaded certificate", .{self.cert_path});
            self.reportReload(io, now_s, "stat of the certificate failed", @errorName(err));
            return;
        };
        const key_sig = statSig(io, self.key_path) catch |err| {
            log.warn("stat {s} failed; keeping the loaded certificate", .{self.key_path});
            self.reportReload(io, now_s, "stat of the private key failed", @errorName(err));
            return;
        };
        const observed: Signature = .{ .cert = cert_sig, .key = key_sig };

        self.mutex.lockUncancelable(io);
        const loaded = self.loaded;
        self.mutex.unlock(io);
        if (!changed(loaded, observed)) {
            // A poll that stat'ed both files and found nothing to do is a
            // fully healthy pass, so it closes any episode a transient stat
            // failure opened. Without this, a file that never changes again
            // would leave that episode open forever.
            if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
            return;
        }

        if (self.reloadLocked(io)) {
            log.info("certificate reloaded from {s}", .{self.cert_path});
            if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
        } else |err| {
            log.warn("certificate reload from {s} failed ({s}); the old certificate keeps serving", .{
                self.cert_path,
                humanMessage(err),
            });
            self.reportReload(io, now_s, "certificate reload failed", humanMessage(err));
        }
    }

    /// A warning, never an error: a stat failure can be a rename window, and a
    /// failed reload leaves the loaded certificate serving. Nothing is down.
    fn reportReload(self: *CertStore, io: std.Io, now_s: i64, message: []const u8, reason: []const u8) void {
        const store = self.diagnostics orelse return;
        var buf: [events.Store.max_detail_len]u8 = undefined;
        const detail = std.fmt.bufPrint(&buf, "{s}: {s}", .{ message, reason }) catch buf[0..];
        store.report(io, now_s, .certificate_reload, @tagName(self.kind), @tagName(self.kind), .warning, detail);
    }

    pub fn snapshotStats(self: *const CertStore) Stats {
        return .{
            .reloads = self.reloads.load(.monotonic),
            .reload_failures = self.reload_failures.load(.monotonic),
            .last_reload_unix = self.last_reload_unix.load(.monotonic),
        };
    }

    fn destroyEntry(self: *CertStore, entry: *Entry) void {
        entry.ctx.deinit(self.gpa);
        self.gpa.destroy(entry);
    }
};

const Loaded = struct {
    entry: *Entry,
    sig: Signature,
};

/// Stat first, then read: a file replaced between the two makes the recorded
/// signature stale, so the watcher's next poll sees the difference and loads
/// again. The other order would record the new signature over the old bytes
/// and miss the update.
fn load(
    gpa: std.mem.Allocator,
    io: std.Io,
    cert_path: []const u8,
    key_path: []const u8,
    alpn: ?[*:null]const ?[*:0]const u8,
) ReloadError!Loaded {
    const cert_sig = statSig(io, cert_path) catch return error.CertUnreadable;
    const key_sig = statSig(io, key_path) catch return error.KeyUnreadable;

    const cert_pem = readPem(gpa, io, cert_path) catch |err| return switch (err) {
        error.OutOfMemory => error.OutOfMemory,
        error.TooLarge => error.CertTooLarge,
        error.Unreadable => error.CertUnreadable,
    };
    defer gpa.free(cert_pem);

    const key_pem = readKeyPem(gpa, io, key_path) catch |err| return switch (err) {
        error.OutOfMemory => error.OutOfMemory,
        error.TooLarge => error.KeyTooLarge,
        error.Unreadable => error.KeyUnreadable,
    };
    defer {
        std.crypto.secureZero(u8, key_pem);
        gpa.free(key_pem);
    }

    const entry = try gpa.create(Entry);
    errdefer gpa.destroy(entry);
    entry.* = .{
        .ctx = try tls_server.ServerContext.init(gpa, cert_pem, key_pem, alpn),
        .refs = 0,
        .retired = false,
    };

    return .{
        .entry = entry,
        .sig = .{ .cert = cert_sig, .key = key_sig },
    };
}

/// Mbed TLS wants PEM with a terminating zero byte counted in the length, so
/// the file lands in a sentinel-terminated allocation. The limit admits
/// exactly `max_pem_bytes` and rejects the first byte beyond it.
///
/// The certificate only. A private key goes through `readKeyPem`, which does
/// not leave copies behind.
fn readPem(
    gpa: std.mem.Allocator,
    io: std.Io,
    path: []const u8,
) error{ OutOfMemory, TooLarge, Unreadable }![:0]u8 {
    return std.Io.Dir.cwd().readFileAllocOptions(
        io,
        path,
        gpa,
        .limited(max_pem_bytes + 1),
        .of(u8),
        0,
    ) catch |err| switch (err) {
        error.OutOfMemory => error.OutOfMemory,
        error.StreamTooLong => error.TooLarge,
        else => error.Unreadable,
    };
}

/// `readPem` for the private key, with the same cap and the same
/// sentinel-terminated result. The difference is that no copy of the key
/// survives this function: `readFileAllocOptions` grows its buffer as it reads,
/// and every intermediate copy it abandons stays legible in freed pages that no
/// wipe at the call site can reach. One fixed staging buffer never grows, and it
/// is wiped before it goes back to the allocator. The caller wipes the returned
/// slice the same way before freeing it (the idiom is auth.zig's `secureZero`
/// defers).
fn readKeyPem(
    gpa: std.mem.Allocator,
    io: std.Io,
    path: []const u8,
) error{ OutOfMemory, TooLarge, Unreadable }![:0]u8 {
    const staging = try gpa.alloc(u8, max_pem_bytes + 1);
    defer {
        std.crypto.secureZero(u8, staging);
        gpa.free(staging);
    }

    const bytes = std.Io.Dir.cwd().readFile(io, path, staging) catch return error.Unreadable;
    // A read that filled the staging buffer is ambiguous — `readFile` cannot
    // say whether more followed — and one byte past `max_pem_bytes` is over the
    // cap either way.
    if (bytes.len == staging.len) return error.TooLarge;

    const key = try gpa.allocSentinel(u8, bytes.len, 0);
    @memcpy(key, bytes);
    return key;
}

fn statSig(io: std.Io, path: []const u8) !FileSig {
    const st = try std.Io.Dir.cwd().statFile(io, path, .{});
    return .{ .mtime_ns = st.mtime.nanoseconds, .size = st.size };
}

// ---------------------------------------------------------------------------
// tests
// ---------------------------------------------------------------------------

const events_fixture = @import("../storage/events_fixture.zig");
const fixtures = @import("test_fixtures");
const testing = std.testing;

test "the stat-compare decision table" {
    const a: FileSig = .{ .mtime_ns = 1_000, .size = 100 };
    const a_later: FileSig = .{ .mtime_ns = 2_000, .size = 100 };
    const a_grown: FileSig = .{ .mtime_ns = 1_000, .size = 101 };
    const b: FileSig = .{ .mtime_ns = 5_000, .size = 500 };
    const b_later: FileSig = .{ .mtime_ns = 6_000, .size = 500 };
    const b_grown: FileSig = .{ .mtime_ns = 5_000, .size = 501 };

    const cases = [_]struct {
        loaded: Signature,
        observed: Signature,
        want: bool,
    }{
        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a, .key = b }, .want = false },
        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a_later, .key = b }, .want = true },
        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a_grown, .key = b }, .want = true },
        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a, .key = b_later }, .want = true },
        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a, .key = b_grown }, .want = true },
        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = a_later, .key = b_grown }, .want = true },
        // A same-second rewrite registers through the size alone.
        .{
            .loaded = .{ .cert = a, .key = b },
            .observed = .{ .cert = .{ .mtime_ns = 1_000, .size = 99 }, .key = b },
            .want = true,
        },
        // The pair swapped between the two files is a change, not a wash.
        .{ .loaded = .{ .cert = a, .key = b }, .observed = .{ .cert = b, .key = a }, .want = true },
    };

    for (cases) |case| {
        try testing.expectEqual(case.want, changed(case.loaded, case.observed));
    }
}

test "every reload error carries a human message" {
    inline for (comptime @typeInfo(ReloadError).error_set.?) |e| {
        try testing.expect(humanMessage(@field(anyerror, e.name)).len > 0);
    }
}

/// Fixture PEMs written into a tmp directory, addressed by cwd-relative paths
/// the same way `app.zig` hands config paths to the store. Must not move after
/// `init`: the path slices point into the buffers below.
const TestEnv = struct {
    threaded: std.Io.Threaded,
    tmp: testing.TmpDir,
    cert_path_buf: [128]u8,
    key_path_buf: [128]u8,
    cert_path: []const u8,
    key_path: []const u8,

    fn init(env: *TestEnv) !void {
        env.threaded = .init(testing.allocator, .{});
        errdefer env.threaded.deinit();
        env.tmp = testing.tmpDir(.{});
        errdefer env.tmp.cleanup();

        const env_io = env.threaded.io();
        try env.tmp.dir.writeFile(env_io, .{ .sub_path = "cert.pem", .data = fixtures.cert_pem });
        try env.tmp.dir.writeFile(env_io, .{ .sub_path = "key.pem", .data = fixtures.key_pem });
        env.cert_path = try std.fmt.bufPrint(&env.cert_path_buf, ".zig-cache/tmp/{s}/cert.pem", .{env.tmp.sub_path});
        env.key_path = try std.fmt.bufPrint(&env.key_path_buf, ".zig-cache/tmp/{s}/key.pem", .{env.tmp.sub_path});
    }

    fn deinit(env: *TestEnv) void {
        env.tmp.cleanup();
        env.threaded.deinit();
    }

    fn io(env: *TestEnv) std.Io {
        return env.threaded.io();
    }
};

test "loaded PEM bytes are NUL-terminated and intact" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();

    const pem = try readPem(testing.allocator, env.io(), env.cert_path);
    defer testing.allocator.free(pem);

    try testing.expectEqualStrings(fixtures.cert_pem, pem);
    try testing.expectEqual(@as(u8, 0), pem[pem.len]);
}

test "the size cap admits 64 KiB and rejects one byte more" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    const at_cap = try testing.allocator.alloc(u8, max_pem_bytes);
    defer testing.allocator.free(at_cap);
    @memset(at_cap, 'a');
    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = at_cap });

    const loaded = try readPem(testing.allocator, io, env.cert_path);
    testing.allocator.free(loaded);

    const over = try testing.allocator.alloc(u8, max_pem_bytes + 1);
    defer testing.allocator.free(over);
    @memset(over, 'a');
    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = over });

    try testing.expectError(error.TooLarge, readPem(testing.allocator, io, env.cert_path));
}

test "the key PEM path keeps the shape and the cap of the certificate path" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    const key = try readKeyPem(testing.allocator, io, env.key_path);
    defer testing.allocator.free(key);
    try testing.expectEqualStrings(fixtures.key_pem, key);
    try testing.expectEqual(@as(u8, 0), key[key.len]);

    const at_cap = try testing.allocator.alloc(u8, max_pem_bytes);
    defer testing.allocator.free(at_cap);
    @memset(at_cap, 'a');
    try env.tmp.dir.writeFile(io, .{ .sub_path = "key.pem", .data = at_cap });
    testing.allocator.free(try readKeyPem(testing.allocator, io, env.key_path));

    const over = try testing.allocator.alloc(u8, max_pem_bytes + 1);
    defer testing.allocator.free(over);
    @memset(over, 'a');
    try env.tmp.dir.writeFile(io, .{ .sub_path = "key.pem", .data = over });
    try testing.expectError(error.TooLarge, readKeyPem(testing.allocator, io, env.key_path));

    try testing.expectError(
        error.Unreadable,
        readKeyPem(testing.allocator, io, "./nxdns-no-such-key-9b31.pem"),
    );
}

test "init fails typed on a missing file and on garbage PEM" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    try testing.expectError(
        error.CertUnreadable,
        CertStore.init(testing.allocator, io, "./nxdns-no-such-cert-9b31.pem", env.key_path, null),
    );
    try testing.expectError(
        error.KeyUnreadable,
        CertStore.init(testing.allocator, io, env.cert_path, "./nxdns-no-such-key-9b31.pem", null),
    );

    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "not a certificate" });
    try testing.expectError(
        error.CertParse,
        CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null),
    );
}

test "init fails typed on an oversized certificate" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    const over = try testing.allocator.alloc(u8, max_pem_bytes + 1);
    defer testing.allocator.free(over);
    @memset(over, 'a');
    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = over });

    try testing.expectError(
        error.CertTooLarge,
        CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null),
    );
}

test "acquire and release across a reload free the old entry after the last release" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    const old = store.acquire(io);
    const old_again = store.acquire(io);
    try testing.expectEqual(old, old_again);
    try testing.expectEqual(@as(u32, 2), old.refs);
    try testing.expect(!old.retired);

    try store.reload(io);

    // The old generation is retired but pinned; the store publishes a new one.
    try testing.expect(old.retired);
    try testing.expectEqual(@as(u32, 2), old.refs);
    const fresh = store.acquire(io);
    try testing.expect(fresh != old);
    try testing.expect(!fresh.retired);

    // The first release leaves the old entry alive for the second holder;
    // the testing allocator proves the second release frees it exactly once.
    store.release(io, old);
    try testing.expectEqual(@as(u32, 1), old.refs);
    store.release(io, old);

    store.release(io, fresh);
    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
}

test "a reload with no holders frees the old entry immediately" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    // No acquire in flight: the swap itself must free the old generation, or
    // the testing allocator reports the leak at the end of this test.
    try store.reload(io);
    try store.reload(io);
    try testing.expectEqual(@as(u64, 2), store.snapshotStats().reloads);
}

test "a failed reload publishes nothing" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    const before = store.acquire(io);
    store.release(io, before);

    // Bad PEM bytes: the file reads fine and fails in the parser.
    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "garbage" });
    try testing.expectError(error.CertParse, store.reload(io));

    var held = store.acquire(io);
    try testing.expectEqual(before, held);
    try testing.expect(!held.retired);
    store.release(io, held);

    // Bad path: the key file vanishes.
    try env.tmp.dir.deleteFile(io, "key.pem");
    try testing.expectError(error.KeyUnreadable, store.reload(io));

    held = store.acquire(io);
    try testing.expectEqual(before, held);
    store.release(io, held);

    const stats = store.snapshotStats();
    try testing.expectEqual(@as(u64, 0), stats.reloads);
    try testing.expectEqual(@as(u64, 2), stats.reload_failures);
}

test "a mismatched key pair fails the reload and keeps the old pair" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    const before = store.acquire(io);
    store.release(io, before);

    try env.tmp.dir.writeFile(io, .{ .sub_path = "key.pem", .data = fixtures.mismatched_key_pem });
    try testing.expectError(error.KeyMismatch, store.reload(io));

    const held = store.acquire(io);
    try testing.expectEqual(before, held);
    store.release(io, held);
    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reload_failures);
}

test "pollOnce reloads on a changed stat pair and stays put on an unchanged one" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    store.pollOnce(io, 1000);
    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);

    // Same certificate plus a trailing newline: the PEM still parses and the
    // size differs even when the rewrite lands within one timestamp granule.
    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
    defer testing.allocator.free(grown);
    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = grown });

    const old = store.acquire(io);
    store.release(io, old);
    store.pollOnce(io, 1000);

    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
    const fresh = store.acquire(io);
    try testing.expect(fresh != old);
    store.release(io, fresh);

    store.pollOnce(io, 1000);
    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
}

test "pollOnce warns and keeps serving when a reload fails" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    const before = store.acquire(io);
    store.release(io, before);

    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
    store.pollOnce(io, 1000);

    const stats = store.snapshotStats();
    try testing.expectEqual(@as(u64, 0), stats.reloads);
    try testing.expectEqual(@as(u64, 1), stats.reload_failures);

    const held = store.acquire(io);
    try testing.expectEqual(before, held);
    store.release(io, held);
}

test "pollOnce does nothing when a file cannot be stat'ed" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    try env.tmp.dir.deleteFile(io, "cert.pem");
    store.pollOnce(io, 1000);

    const stats = store.snapshotStats();
    try testing.expectEqual(@as(u64, 0), stats.reloads);
    try testing.expectEqual(@as(u64, 0), stats.reload_failures);
}

test "init records the load time and reload advances it" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    const at_init = store.snapshotStats();
    try testing.expectEqual(@as(u64, 0), at_init.reloads);
    try testing.expectEqual(@as(u64, 0), at_init.reload_failures);
    try testing.expect(at_init.last_reload_unix > 0);

    try store.reload(io);
    const after = store.snapshotStats();
    try testing.expectEqual(@as(u64, 1), after.reloads);
    try testing.expect(after.last_reload_unix >= at_init.last_reload_unix);
}

test "init accepts a comptime ALPN list" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    const alpn: [1:null]?[*:0]const u8 = .{"http/1.1"};
    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, &alpn);
    defer store.deinit(io);

    try store.reload(io);
    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
}

test "the watch loop sleeps before polling and returns on cancel" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    var future = try io.concurrent(CertStore.watch, .{ &store, io });
    // Cancelling immediately proves the loop starts by sleeping rather than
    // by reloading.
    try testing.expectError(error.Canceled, future.cancel(io));
    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
}

// Pins the reload-ordering race via `after_load_hook`: the hook runs inside
// reload A's load-to-publish window, where it (1) probes that `reload_mutex`
// is held, the serialization this exists for (without it a second reload
// could publish here and be overwritten by A's stale read), then (2) stages
// newer cert bytes and starts an overlapping reload B. B can only read the
// files after A publishes, so B both reads and publishes last and the newer
// bytes win. This proves the lock spans the window and last-read-wins for
// this schedule; it does not enumerate every interleaving.
test "a reload overlapping another reload's window publishes last" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
    defer testing.allocator.free(grown);

    const Window = struct {
        store: *CertStore,
        env: *TestEnv,
        grown: []const u8,
        overlapping: ?std.Io.Future(ReloadError!void) = null,
        lock_spans_window: bool = false,
        staged: bool = false,

        // Reload B re-enters this hook under `reload_mutex`, so its read of
        // `overlapping` is ordered after A's write and the early return makes
        // the window work run exactly once.
        fn call(ctx: *anyopaque, hook_io: std.Io) void {
            const w: *@This() = @ptrCast(@alignCast(ctx));
            if (w.overlapping != null) return;
            if (w.store.reload_mutex.tryLock()) {
                w.store.reload_mutex.unlock(hook_io);
            } else {
                w.lock_spans_window = true;
            }
            w.env.tmp.dir.writeFile(hook_io, .{ .sub_path = "cert.pem", .data = w.grown }) catch return;
            w.staged = true;
            w.overlapping = hook_io.concurrent(CertStore.reload, .{ w.store, hook_io }) catch null;
        }
    };

    var window: Window = .{ .store = &store, .env = &env, .grown = grown };
    store.after_load_hook = .{ .ctx = &window, .call = Window.call };

    try store.reload(io);

    try testing.expect(window.lock_spans_window);
    try testing.expect(window.staged);
    var overlapping = window.overlapping orelse return error.ConcurrentUnavailable;
    try overlapping.await(io);

    try testing.expectEqual(@as(u64, 2), store.snapshotStats().reloads);
    store.mutex.lockUncancelable(io);
    const final = store.loaded;
    store.mutex.unlock(io);
    try testing.expectEqual(@as(u64, grown.len), final.cert.size);
}

test "a failed reload opens an episode keyed by endpoint kind and a good one closes it" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var fx: events_fixture.Fixture = .{};
    try fx.init(io, 1000);
    defer fx.deinit();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);
    store.kind = .dot;
    store.diagnostics = &fx.store;

    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
    store.pollOnce(io, 1000);

    try testing.expectEqualStrings("certificate.reload", try fx.text("SELECT code FROM operational_events"));
    try testing.expectEqualStrings("dot", try fx.text("SELECT subject_key FROM operational_events"));
    try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));

    // The same certificate plus a newline: it parses, and the size differs even
    // within one timestamp granule.
    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
    defer testing.allocator.free(grown);
    try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = grown });
    store.pollOnce(io, 1100);

    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
    try testing.expectEqual(
        @as(i64, 0),
        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
    );
}

test "a failed stat opens the same episode a failed reload would" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var fx: events_fixture.Fixture = .{};
    try fx.init(io, 1000);
    defer fx.deinit();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);
    store.diagnostics = &fx.store;

    try env.tmp.dir.deleteFile(io, "cert.pem");
    store.pollOnce(io, 1000);

    try testing.expectEqualStrings("doh", try fx.text(
        "SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
    ));
    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reload_failures);
}

test "an unchanged poll closes the episode a transient stat failure opened" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var fx: events_fixture.Fixture = .{};
    try fx.init(io, 1000);
    defer fx.deinit();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);
    store.diagnostics = &fx.store;

    // What a stat failure in a rename window left open. The file it named is
    // back and unchanged, so no reload will ever close this episode.
    fx.store.report(io, 1000, .certificate_reload, "doh", "doh", .warning, "stat of the certificate failed: FileNotFound");

    store.pollOnce(io, 1100);

    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
    try testing.expectEqual(
        @as(i64, 0),
        try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
    );
}

// ---------------------------------------------------------------------------
// path apply (milestone-34 S3.4)
// ---------------------------------------------------------------------------

test "a bad candidate is refused at prepare and the store keeps serving" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    const before = store.acquire(io);
    store.release(io, before);
    const before_paths_cert = store.cert_path;

    try env.tmp.dir.writeFile(io, .{ .sub_path = "bad.pem", .data = "not a certificate" });
    var bad_buf: [128]u8 = undefined;
    const bad_path = try std.fmt.bufPrint(&bad_buf, ".zig-cache/tmp/{s}/bad.pem", .{env.tmp.sub_path});

    try testing.expectError(
        error.CertParse,
        store.preparePathChange(io, bad_path, env.key_path),
    );

    // Nothing published: same generation, same paths, and `reload_mutex` was
    // released — a second prepare would deadlock otherwise.
    const after = store.acquire(io);
    store.release(io, after);
    try testing.expectEqual(before, after);
    try testing.expectEqual(before_paths_cert.ptr, store.cert_path.ptr);
    try testing.expectEqualStrings(env.cert_path, store.cert_path);
    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reload_failures);

    // A missing candidate path is refused the same way.
    try testing.expectError(
        error.CertUnreadable,
        store.preparePathChange(io, "./nxdns-no-such-cert-4a11.pem", env.key_path),
    );
}

test "a published path change installs the new pair and retires the old generation" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    // A second, byte-different copy of the same valid pair under new names.
    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
    defer testing.allocator.free(grown);
    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-cert.pem", .data = grown });
    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-key.pem", .data = fixtures.key_pem });
    var cert_buf: [128]u8 = undefined;
    var key_buf: [128]u8 = undefined;
    const next_cert = try std.fmt.bufPrint(&cert_buf, ".zig-cache/tmp/{s}/next-cert.pem", .{env.tmp.sub_path});
    const next_key = try std.fmt.bufPrint(&key_buf, ".zig-cache/tmp/{s}/next-key.pem", .{env.tmp.sub_path});

    // A connection pinned to the old generation finishes on it.
    const pinned = store.acquire(io);

    const prepared = try store.preparePathChange(io, next_cert, next_key);
    store.publishPathChange(io, prepared);

    try testing.expectEqualStrings(next_cert, store.cert_path);
    try testing.expectEqualStrings(next_key, store.key_path);
    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
    try testing.expect(pinned.retired);

    const serving = store.acquire(io);
    try testing.expect(serving != pinned);
    store.release(io, serving);
    store.release(io, pinned);

    // The watcher now measures the new pair, so an untouched pair polls clean
    // and a rewritten one reloads.
    store.pollOnce(io, 1_000);
    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-cert.pem", .data = fixtures.cert_pem });
    store.pollOnce(io, 1_100);
    try testing.expectEqual(@as(u64, 2), store.snapshotStats().reloads);
}

test "an aborted path change frees the candidate and leaves the store untouched" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    const before = store.acquire(io);
    store.release(io, before);

    // The commit this candidate was built for failed; the testing allocator
    // proves the abort frees everything the prepare took.
    const prepared = try store.preparePathChange(io, env.cert_path, env.key_path);
    store.abortPathChange(io, prepared);

    const after = store.acquire(io);
    store.release(io, after);
    try testing.expectEqual(before, after);
    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);

    // `reload_mutex` came back, so the store still reloads.
    try store.reload(io);
}

test "a reload racing a path apply is serialized behind it" {
    var env: TestEnv = undefined;
    try env.init();
    defer env.deinit();
    const io = env.io();

    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
    defer store.deinit(io);

    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
    defer testing.allocator.free(grown);
    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-cert.pem", .data = grown });
    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-key.pem", .data = fixtures.key_pem });
    var cert_buf: [128]u8 = undefined;
    var key_buf: [128]u8 = undefined;
    const next_cert = try std.fmt.bufPrint(&cert_buf, ".zig-cache/tmp/{s}/next-cert.pem", .{env.tmp.sub_path});
    const next_key = try std.fmt.bufPrint(&key_buf, ".zig-cache/tmp/{s}/next-key.pem", .{env.tmp.sub_path});

    // Prepare holds `reload_mutex` across the whole apply.
    const prepared = try store.preparePathChange(io, next_cert, next_key);
    try testing.expect(!store.reload_mutex.tryLock());

    // The concurrent reload cannot start, so it cannot publish the OLD paths
    // over the new generation.
    var racing = try io.concurrent(CertStore.reload, .{ &store, io });
    store.publishPathChange(io, prepared);
    try racing.await(io);

    // Two publications, and the last word is the apply's pair: the racing
    // reload reread the paths the apply installed.
    try testing.expectEqual(@as(u64, 2), store.snapshotStats().reloads);
    try testing.expectEqualStrings(next_cert, store.cert_path);
    store.mutex.lockUncancelable(io);
    const final = store.loaded;
    store.mutex.unlock(io);
    try testing.expectEqual(@as(u64, grown.len), final.cert.size);
}