db-mode config changes apply live in-process

settings and upstream writes now follow a prepare, commit, publish, retire
contract: candidates are built and validated before the database transaction,
published as infallible pointer swaps, and old generations retire after their
readers drain. per-query policy values snapshot once per query; upstream pool,
cache, rate limiter, sessions, api limiter, log sink, blocklist scheduler and
the query-log queue each gained one named live operation. restart_required
shrinks from every scalar key to the bind keys and web.enabled; the admin ui
drops its restart notices for everything else. file mode is unchanged.

src/server/cert_store.zig

@@ -109,10 +109,13 @@ pub const CertStore = struct {
     pub const Kind = enum { doh, dot };
 
     gpa: std.mem.Allocator,
-    /// Borrowed from the config; must outlive the store.
-    cert_path: []const u8,
-    /// Borrowed from the config; must outlive the store.
-    key_path: []const u8,
+    /// Owned. A settings apply replaces both paths, so a borrowed config slice
+    /// would dangle the moment the row it came from went away. Read and
+    /// written only under `reload_mutex`, which every apply and every reload
+    /// holds for its whole read-build-publish sequence.
+    cert_path: []u8,
+    /// Owned; see `cert_path`.
+    key_path: []u8,
     /// Passed through to every `ServerContext.init`; the same lifetime rule
     /// applies — a comptime-constant NULL-terminated array, never memory that
     /// can go away before the store.
@@ -136,7 +139,8 @@ pub const CertStore = struct {
     /// Test seam: runs inside `reload` between a successful `load` and the
     /// publish, i.e. inside `reload_mutex`. Lets a test occupy the window
     /// where an unserialized reload could be overtaken. Must not call
-    /// `reload` synchronously (that would self-deadlock on `reload_mutex`).
+    /// `reload`, `pollOnce` or `preparePathChange` synchronously — all four
+    /// take `reload_mutex`, which is not reentrant.
     after_load_hook: ?ReloadHook,
 
     /// Set by the composition root right after `init`, with `diagnostics`.
@@ -168,10 +172,17 @@ pub const CertStore = struct {
         alpn: ?[*:null]const ?[*:0]const u8,
     ) ReloadError!CertStore {
         const first = try load(gpa, io, cert_path, key_path, alpn);
+        errdefer {
+            first.entry.ctx.deinit(gpa);
+            gpa.destroy(first.entry);
+        }
+        const owned_cert = try gpa.dupe(u8, cert_path);
+        errdefer gpa.free(owned_cert);
+        const owned_key = try gpa.dupe(u8, key_path);
         return .{
             .gpa = gpa,
-            .cert_path = cert_path,
-            .key_path = key_path,
+            .cert_path = owned_cert,
+            .key_path = owned_key,
             .alpn = alpn,
             .mutex = .init,
             .reload_mutex = .init,
@@ -193,6 +204,8 @@ pub const CertStore = struct {
         std.debug.assert(current.refs == 0);
         self.mutex.unlock(io);
         self.destroyEntry(current);
+        self.gpa.free(self.cert_path);
+        self.gpa.free(self.key_path);
         self.* = undefined;
     }
 
@@ -226,7 +239,14 @@ pub const CertStore = struct {
     pub fn reload(self: *CertStore, io: std.Io) ReloadError!void {
         self.reload_mutex.lockUncancelable(io);
         defer self.reload_mutex.unlock(io);
+        return self.reloadLocked(io);
+    }
 
+    /// `reload`'s body, for callers that already hold `reload_mutex` —
+    /// `pollOnce` does, because it must read `cert_path`/`key_path` under the
+    /// same lock an apply replaces them under. `std.Io.Mutex` is not
+    /// reentrant, so this exists rather than a recursive `reload` call.
+    fn reloadLocked(self: *CertStore, io: std.Io) ReloadError!void {
         const next = load(self.gpa, io, self.cert_path, self.key_path, self.alpn) catch |err| {
             _ = self.reload_failures.fetchAdd(1, .monotonic);
             return err;
@@ -247,6 +267,93 @@ pub const CertStore = struct {
         self.last_reload_unix.store(std.Io.Clock.real.now(io).toSeconds(), .monotonic);
     }
 
+    /// A candidate certificate loaded from new paths, not yet published.
+    /// Holding one means holding `reload_mutex`: exactly one of
+    /// `publishPathChange` or `abortPathChange` must follow, and it releases
+    /// the lock.
+    pub const PreparedPaths = struct {
+        cert_path: []u8,
+        key_path: []u8,
+        loaded: Loaded,
+        /// Read at prepare so publish reads no clock: publish must touch
+        /// nothing outside memory it already owns.
+        loaded_at_unix: i64,
+    };
+
+    /// Prepare half of a `doh_server`/`dot_server` cert-path change: takes
+    /// `reload_mutex` and loads the certificate and key from the NEW paths.
+    /// Nothing is published, so a failure leaves the store exactly as it was —
+    /// the old certificate keeps serving and the caller writes no DB row. The
+    /// lock is released on failure and held on success, which is what makes
+    /// the whole apply serialized against `reload` and `pollOnce`.
+    pub fn preparePathChange(
+        self: *CertStore,
+        io: std.Io,
+        cert_path: []const u8,
+        key_path: []const u8,
+    ) ReloadError!PreparedPaths {
+        self.reload_mutex.lockUncancelable(io);
+        errdefer self.reload_mutex.unlock(io);
+
+        const owned_cert = try self.gpa.dupe(u8, cert_path);
+        errdefer self.gpa.free(owned_cert);
+        const owned_key = try self.gpa.dupe(u8, key_path);
+        errdefer self.gpa.free(owned_key);
+
+        const next = load(self.gpa, io, cert_path, key_path, self.alpn) catch |err| {
+            _ = self.reload_failures.fetchAdd(1, .monotonic);
+            return err;
+        };
+        return .{
+            .cert_path = owned_cert,
+            .key_path = owned_key,
+            .loaded = next,
+            .loaded_at_unix = std.Io.Clock.real.now(io).toSeconds(),
+        };
+    }
+
+    /// Publish half: infallible and I/O-free. The paths and the generation
+    /// they were loaded from are installed together — the generation `mutex`
+    /// is taken only for that swap, inside `reload_mutex`, the same lock order
+    /// `reload` uses. Releases `reload_mutex`.
+    ///
+    /// Connections that pinned the old generation finish on the old
+    /// certificate; the old entry is freed once its last reader releases.
+    pub fn publishPathChange(self: *CertStore, io: std.Io, prepared: PreparedPaths) void {
+        const old_cert_path = self.cert_path;
+        const old_key_path = self.key_path;
+
+        self.mutex.lockUncancelable(io);
+        const old = self.current;
+        self.cert_path = prepared.cert_path;
+        self.key_path = prepared.key_path;
+        self.current = prepared.loaded.entry;
+        self.loaded = prepared.loaded.sig;
+        old.retired = true;
+        const free_old = old.refs == 0;
+        self.mutex.unlock(io);
+
+        // The branch runs before the counter bump, never after: a `bool` still
+        // live across an atomic read-modify-write is the zig 0.16.0 Debug
+        // miscompile AGENTS.md documents.
+        if (free_old) self.destroyEntry(old);
+        self.gpa.free(old_cert_path);
+        self.gpa.free(old_key_path);
+
+        _ = self.reloads.fetchAdd(1, .monotonic);
+        self.last_reload_unix.store(prepared.loaded_at_unix, .monotonic);
+        self.reload_mutex.unlock(io);
+    }
+
+    /// Discards a prepared candidate — the commit that would have published it
+    /// failed, or a sibling owner's prepare did. Releases `reload_mutex`.
+    pub fn abortPathChange(self: *CertStore, io: std.Io, prepared: PreparedPaths) void {
+        self.gpa.free(prepared.cert_path);
+        self.gpa.free(prepared.key_path);
+        self.destroyEntry(prepared.loaded.entry);
+        self.reload_mutex.unlock(io);
+    }
+
     /// Sleep first: `init` just loaded the files this poll would compare
     /// against. `.boot` so a suspended box still sees the interval elapse.
     pub fn watch(self: *CertStore, io: std.Io) std.Io.Cancelable!void {
@@ -265,7 +372,15 @@ pub const CertStore = struct {
     /// changed, and the old one keeps serving either way. A failed reload
     /// warns and counts (`reload_failures`); the signature stays at the loaded
     /// pair, so every subsequent poll retries until the files parse.
+    ///
+    /// The whole pass runs under `reload_mutex`: the paths it stats are the
+    /// ones an apply replaces, and a poll that read a path outside the lock
+    /// could stat a freed slice or reload a pair that was never published
+    /// together.
     pub fn pollOnce(self: *CertStore, io: std.Io, now_s: i64) void {
+        self.reload_mutex.lockUncancelable(io);
+        defer self.reload_mutex.unlock(io);
+
         const cert_sig = statSig(io, self.cert_path) catch |err| {
             log.warn("stat {s} failed; keeping the loaded certificate", .{self.cert_path});
             self.reportReload(io, now_s, "stat of the certificate failed", @errorName(err));
@@ -290,7 +405,7 @@ pub const CertStore = struct {
             return;
         }
 
-        if (self.reload(io)) {
+        if (self.reloadLocked(io)) {
             log.info("certificate reloaded from {s}", .{self.cert_path});
             if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
         } else |err| {
@@ -993,3 +1108,154 @@ test "an unchanged poll closes the episode a transient stat failure opened" {
         try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
     );
 }
+
+// ---------------------------------------------------------------------------
+// path apply (milestone-34 S3.4)
+// ---------------------------------------------------------------------------
+
+test "a bad candidate is refused at prepare and the store keeps serving" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    const before = store.acquire(io);
+    store.release(io, before);
+    const before_paths_cert = store.cert_path;
+
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "bad.pem", .data = "not a certificate" });
+    var bad_buf: [128]u8 = undefined;
+    const bad_path = try std.fmt.bufPrint(&bad_buf, ".zig-cache/tmp/{s}/bad.pem", .{env.tmp.sub_path});
+
+    try testing.expectError(
+        error.CertParse,
+        store.preparePathChange(io, bad_path, env.key_path),
+    );
+
+    // Nothing published: same generation, same paths, and `reload_mutex` was
+    // released — a second prepare would deadlock otherwise.
+    const after = store.acquire(io);
+    store.release(io, after);
+    try testing.expectEqual(before, after);
+    try testing.expectEqual(before_paths_cert.ptr, store.cert_path.ptr);
+    try testing.expectEqualStrings(env.cert_path, store.cert_path);
+    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reload_failures);
+
+    // A missing candidate path is refused the same way.
+    try testing.expectError(
+        error.CertUnreadable,
+        store.preparePathChange(io, "./nxdns-no-such-cert-4a11.pem", env.key_path),
+    );
+}
+
+test "a published path change installs the new pair and retires the old generation" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    // A second, byte-different copy of the same valid pair under new names.
+    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
+    defer testing.allocator.free(grown);
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-cert.pem", .data = grown });
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-key.pem", .data = fixtures.key_pem });
+    var cert_buf: [128]u8 = undefined;
+    var key_buf: [128]u8 = undefined;
+    const next_cert = try std.fmt.bufPrint(&cert_buf, ".zig-cache/tmp/{s}/next-cert.pem", .{env.tmp.sub_path});
+    const next_key = try std.fmt.bufPrint(&key_buf, ".zig-cache/tmp/{s}/next-key.pem", .{env.tmp.sub_path});
+
+    // A connection pinned to the old generation finishes on it.
+    const pinned = store.acquire(io);
+
+    const prepared = try store.preparePathChange(io, next_cert, next_key);
+    store.publishPathChange(io, prepared);
+
+    try testing.expectEqualStrings(next_cert, store.cert_path);
+    try testing.expectEqualStrings(next_key, store.key_path);
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
+    try testing.expect(pinned.retired);
+
+    const serving = store.acquire(io);
+    try testing.expect(serving != pinned);
+    store.release(io, serving);
+    store.release(io, pinned);
+
+    // The watcher now measures the new pair, so an untouched pair polls clean
+    // and a rewritten one reloads.
+    store.pollOnce(io, 1_000);
+    try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-cert.pem", .data = fixtures.cert_pem });
+    store.pollOnce(io, 1_100);
+    try testing.expectEqual(@as(u64, 2), store.snapshotStats().reloads);
+}
+
+test "an aborted path change frees the candidate and leaves the store untouched" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    const before = store.acquire(io);
+    store.release(io, before);
+
+    // The commit this candidate was built for failed; the testing allocator
+    // proves the abort frees everything the prepare took.
+    const prepared = try store.preparePathChange(io, env.cert_path, env.key_path);
+    store.abortPathChange(io, prepared);
+
+    const after = store.acquire(io);
+    store.release(io, after);
+    try testing.expectEqual(before, after);
+    try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
+
+    // `reload_mutex` came back, so the store still reloads.
+    try store.reload(io);
+}
+
+test "a reload racing a path apply is serialized behind it" {
+    var env: TestEnv = undefined;
+    try env.init();
+    defer env.deinit();
+    const io = env.io();
+
+    var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
+    defer store.deinit(io);
+
+    const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
+    defer testing.allocator.free(grown);
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-cert.pem", .data = grown });
+    try env.tmp.dir.writeFile(io, .{ .sub_path = "next-key.pem", .data = fixtures.key_pem });
+    var cert_buf: [128]u8 = undefined;
+    var key_buf: [128]u8 = undefined;
+    const next_cert = try std.fmt.bufPrint(&cert_buf, ".zig-cache/tmp/{s}/next-cert.pem", .{env.tmp.sub_path});
+    const next_key = try std.fmt.bufPrint(&key_buf, ".zig-cache/tmp/{s}/next-key.pem", .{env.tmp.sub_path});
+
+    // Prepare holds `reload_mutex` across the whole apply.
+    const prepared = try store.preparePathChange(io, next_cert, next_key);
+    try testing.expect(!store.reload_mutex.tryLock());
+
+    // The concurrent reload cannot start, so it cannot publish the OLD paths
+    // over the new generation.
+    var racing = try io.concurrent(CertStore.reload, .{ &store, io });
+    store.publishPathChange(io, prepared);
+    try racing.await(io);
+
+    // Two publications, and the last word is the apply's pair: the racing
+    // reload reread the paths the apply installed.
+    try testing.expectEqual(@as(u64, 2), store.snapshotStats().reloads);
+    try testing.expectEqualStrings(next_cert, store.cert_path);
+    store.mutex.lockUncancelable(io);
+    const final = store.loaded;
+    store.mutex.unlock(io);
+    try testing.expectEqual(@as(u64, grown.len), final.cert.size);
+}