db-mode config changes apply live in-process
Gates / frontend (push) Successful in 1m43s
Gates / test (push) Successful in 2m14s
Gates / test-aarch64 (push) Successful in 8m3s
Gates / package (push) Successful in 5m42s
Gates / container (push) Successful in 54s
CI / gates (push) Successful in 50m24s

settings and upstream writes now follow a prepare, commit, publish, retire
contract: candidates are built and validated before the database transaction,
published as infallible pointer swaps, and old generations retire after their
readers drain. per-query policy values snapshot once per query; upstream pool,
cache, rate limiter, sessions, api limiter, log sink, blocklist scheduler and
the query-log queue each gained one named live operation. restart_required
shrinks from every scalar key to the bind keys and web.enabled; the admin ui
drops its restart notices for everything else. file mode is unchanged.
This commit is contained in:
2026-08-24 00:04:28 +02:00
parent f7f4c8be09
commit ce143d1d87
47 changed files with 7698 additions and 926 deletions
+192 -6
View File
@@ -399,8 +399,23 @@ const discard_stall = if (builtin.is_test) struct {
}
};
/// The §11.4 privacy policy: one value, never two. A producer decides the
/// domain fields and the client field from ONE load of `privacy_packed`, so no
/// entry can leave `transformed` with the domain redacted and the client
/// exposed, or the reverse, because a `setPrivacy` landed between the two
/// decisions.
pub const Privacy = packed struct(u8) {
hide_domains: bool = false,
hide_client_ips: bool = false,
_reserved: u6 = 0,
};
pub const Logger = struct {
cfg: model.Logging,
/// Both privacy flags in one atomic byte, loaded once per entry.
privacy_packed: std.atomic.Value(u8),
/// Independent of the privacy policy: it governs when a batch commits, not
/// what a row contains, so nothing pairs the two.
flush_interval_s: std.atomic.Value(u16),
queue: EntryQueue,
queries_dropped: std.atomic.Value(u64),
/// When the newest drop happened, in unix seconds; 0 means none yet. Read
@@ -434,7 +449,11 @@ pub const Logger = struct {
/// touched it.
pub fn init(cfg: model.Logging, queue_buf: []Entry) Logger {
return .{
.cfg = cfg,
.privacy_packed = .init(@bitCast(Privacy{
.hide_domains = cfg.hide_domains,
.hide_client_ips = cfg.hide_client_ips,
})),
.flush_interval_s = .init(cfg.query_log_flush_interval_s),
.queue = .init(queue_buf),
.queries_dropped = .init(0),
.last_drop_s = .init(0),
@@ -464,8 +483,9 @@ pub const Logger = struct {
/// configuration labels the operator wrote, identical on every row that
/// hits them, and they say nothing about which name a client looked up.
pub fn transformed(self: *const Logger, entry: Entry) Entry {
const policy = self.privacy();
var out = entry;
if (self.cfg.hide_domains) {
if (policy.hide_domains) {
out.setDomain(hidden_marker);
// Only where there is something to hide: an empty field means the
// query had no such value, and writing a marker would claim it did.
@@ -473,10 +493,24 @@ pub const Logger = struct {
if (out.cname_len != 0) out.setCnameTarget(hidden_marker);
if (out.safe_search_len != 0) out.setSafeSearchTarget(hidden_marker);
}
if (self.cfg.hide_client_ips) out.setClientIp(hidden_marker);
if (policy.hide_client_ips) out.setClientIp(hidden_marker);
return out;
}
/// The live policy, from one load. Every producer decision about one entry
/// must come from a single call to this.
pub fn privacy(self: *const Logger) Privacy {
return @bitCast(self.privacy_packed.load(.monotonic));
}
pub fn setPrivacy(self: *Logger, p: Privacy) void {
self.privacy_packed.store(@bitCast(p), .monotonic);
}
pub fn setFlushInterval(self: *Logger, seconds: u16) void {
self.flush_interval_s.store(seconds, .monotonic);
}
/// `log` without the transforms, for a caller that already applied them.
pub fn logTransformed(self: *Logger, io: std.Io, entry: Entry) void {
self.enqueue(io, entry);
@@ -540,7 +574,22 @@ pub const Logger = struct {
return;
};
defer writer.deinit();
return self.runPrepared(io, &writer, monitor);
}
/// The writer loop over statements someone else prepared.
///
/// `runWriter` prepares and then calls this. A logger generation created by
/// a resize prepares separately, before anything is published, so that a
/// statement failure is refused at prepare time instead of silently killing
/// the writer of a queue producers are already filling
/// (`logger_controller.zig`).
pub fn runPrepared(
self: *Logger,
io: std.Io,
writer: *queries_repo.BatchWriter,
monitor: ?*disk_monitor.Monitor,
) std.Io.Cancelable!void {
var batch: [flush_batch]Entry = undefined;
while (true) {
// A closed queue hands over its buffered elements before it reports
@@ -564,7 +613,7 @@ pub const Logger = struct {
self.countDropped(io, n, at);
return err;
};
self.flush(io, &writer, batch[0..n], monitor) catch |err| switch (err) {
self.flush(io, writer, batch[0..n], monitor) catch |err| switch (err) {
error.Canceled => |e| {
self.countDropped(io, n, at);
return e;
@@ -606,7 +655,7 @@ pub const Logger = struct {
/// returns without waiting for anything.
fn flushDeadline(self: *const Logger, io: std.Io) std.Io.Clock.Timestamp {
return .fromNow(io, .{
.raw = .fromSeconds(self.cfg.query_log_flush_interval_s),
.raw = .fromSeconds(self.flush_interval_s.load(.monotonic)),
.clock = .boot,
});
}
@@ -644,6 +693,18 @@ pub const Logger = struct {
self.draining.store(true, .release);
}
/// `shutdown` without `draining`: this generation is being replaced, not
/// the process stopped.
///
/// The flag is what turns a gate-held batch into a counted loss
/// (`flush`'s `GatedAtShutdown`), and a retired writer must not take that
/// path — the disk can still recover, and the rows it is holding are still
/// going to be written when it does. Every producer of this generation
/// must have released it before the close, exactly as at shutdown.
pub fn retire(self: *Logger, io: std.Io) void {
self.queue.close(io);
}
/// Fills `batch` behind the entry already in slot 0, until it is full or
/// `deadline` passes. `n` counts the slots that hold an entry, and stays
/// accurate on the cancellation path so the caller can count what is lost.
@@ -1240,6 +1301,80 @@ test "log hides only the field its switch names" {
try testing.expectEqualStrings("192.0.2.10", untouched.clientIp());
}
/// The rendezvous that forces the flip to land BETWEEN two producer entries
/// rather than whenever the scheduler feels like it.
const PrivacyFlip = struct {
logger: *Logger,
/// Set by the producer once its pre-flip entry is enqueued.
before_done: std.Io.Event = .unset,
/// Set by the flipper once `setPrivacy` has returned.
flipped: std.Io.Event = .unset,
fn run(self: *PrivacyFlip, io: std.Io) void {
self.before_done.wait(io) catch return;
self.logger.setPrivacy(.{ .hide_domains = true, .hide_client_ips = true });
self.flipped.set(io);
}
};
test "a privacy flip redacts every entry after it and none before it" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var buf: [4]Entry = undefined;
var logger: Logger = .init(.{}, &buf);
var flip: PrivacyFlip = .{ .logger = &logger };
var future = try io.concurrent(PrivacyFlip.run, .{ &flip, io });
logger.log(io, sampleEntry(1, "before.example"));
flip.before_done.set(io);
try flip.flipped.wait(io);
logger.log(io, sampleEntry(2, "after.example"));
future.await(io);
const before = try logger.queue.getOne(io);
try testing.expectEqualStrings("before.example", before.domain());
try testing.expectEqualStrings("192.0.2.10", before.clientIp());
const after = try logger.queue.getOne(io);
try testing.expectEqualStrings(hidden_marker, after.domain());
try testing.expectEqualStrings(hidden_marker, after.clientIp());
}
fn flipPrivacyRepeatedly(logger: *Logger, rounds: usize) void {
for (0..rounds) |i| {
logger.setPrivacy(if (i % 2 == 0)
.{}
else
.{ .hide_domains = true, .hide_client_ips = true });
}
}
test "no producer observes a privacy policy that redacts one field and not the other" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var buf: [4]Entry = undefined;
var logger: Logger = .init(.{}, &buf);
const source = sampleEntry(1, "tracker.example");
const rounds = 20_000;
var flipper = try io.concurrent(flipPrivacyRepeatedly, .{ &logger, rounds });
// Recorded, not asserted, while the flipper runs: an assertion that
// returned here would leave `Threaded.deinit` joining a task nothing ends.
var mixed = false;
for (0..rounds) |_| {
const out = logger.transformed(source);
const domain_hidden = std.mem.eql(u8, out.domain(), hidden_marker);
const client_hidden = std.mem.eql(u8, out.clientIp(), hidden_marker);
if (domain_hidden != client_hidden) mixed = true;
}
flipper.await(io);
try testing.expect(!mixed);
}
test "the split halves reproduce log byte for byte" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
@@ -1507,6 +1642,57 @@ test "a full batch flushes without waiting for the interval" {
try testing.expectEqual(@as(i64, 150), try queries_repo.countRows(&database));
}
test "the writer's next cycle uses the interval set since its last one" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
var buf: [8]Entry = undefined;
// Zero: every cycle commits what it has and goes straight back to `getOne`,
// so the first entry proves the writer is running and parked between cycles.
var logger: Logger = .init(.{ .query_log_flush_interval_s = 0 }, &buf);
var future = try io.concurrent(Logger.runWriter, .{
&logger,
io,
&database,
@as(?*disk_monitor.Monitor, null),
});
// See the note in "shutdown writes the batch the writer holds": this must
// run before the deferred `database.close`.
defer {
logger.shutdown(io);
future.await(io) catch {};
}
logger.log(io, sampleEntry(1, "first.example"));
const poll: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(5), .clock = .awake };
var waited: usize = 0;
while (logger.rows_written.load(.monotonic) == 0 and waited < 400) : (waited += 1) {
try poll.sleep(io);
}
// An hour, installed while the writer is parked: the cycle the next entry
// starts must wait it out instead of committing at once.
logger.setFlushInterval(3600);
logger.log(io, sampleEntry(2, "second.example"));
const quarter: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(250), .clock = .awake };
try quarter.sleep(io);
const written_under_the_new_interval = logger.rows_written.load(.monotonic);
logger.shutdown(io);
try future.await(io);
try testing.expect(waited < 400);
// The first entry, and only the first: the second is still held.
try testing.expectEqual(@as(u64, 1), written_under_the_new_interval);
// The close releases it, which is what makes the hold a hold and not a loss.
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
}
test "a gated flush holds the batch until the disk recovers" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();