db-mode config changes apply live in-process
Gates / frontend (push) Successful in 1m43s
Gates / test (push) Successful in 2m14s
Gates / test-aarch64 (push) Successful in 8m3s
Gates / package (push) Successful in 5m42s
Gates / container (push) Successful in 54s
CI / gates (push) Successful in 50m24s
Gates / frontend (push) Successful in 1m43s
Gates / test (push) Successful in 2m14s
Gates / test-aarch64 (push) Successful in 8m3s
Gates / package (push) Successful in 5m42s
Gates / container (push) Successful in 54s
CI / gates (push) Successful in 50m24s
settings and upstream writes now follow a prepare, commit, publish, retire contract: candidates are built and validated before the database transaction, published as infallible pointer swaps, and old generations retire after their readers drain. per-query policy values snapshot once per query; upstream pool, cache, rate limiter, sessions, api limiter, log sink, blocklist scheduler and the query-log queue each gained one named live operation. restart_required shrinks from every scalar key to the bind keys and web.enabled; the admin ui drops its restart notices for everything else. file mode is unchanged.
This commit is contained in:
+192
-6
@@ -399,8 +399,23 @@ const discard_stall = if (builtin.is_test) struct {
|
||||
}
|
||||
};
|
||||
|
||||
/// The §11.4 privacy policy: one value, never two. A producer decides the
|
||||
/// domain fields and the client field from ONE load of `privacy_packed`, so no
|
||||
/// entry can leave `transformed` with the domain redacted and the client
|
||||
/// exposed, or the reverse, because a `setPrivacy` landed between the two
|
||||
/// decisions.
|
||||
pub const Privacy = packed struct(u8) {
|
||||
hide_domains: bool = false,
|
||||
hide_client_ips: bool = false,
|
||||
_reserved: u6 = 0,
|
||||
};
|
||||
|
||||
pub const Logger = struct {
|
||||
cfg: model.Logging,
|
||||
/// Both privacy flags in one atomic byte, loaded once per entry.
|
||||
privacy_packed: std.atomic.Value(u8),
|
||||
/// Independent of the privacy policy: it governs when a batch commits, not
|
||||
/// what a row contains, so nothing pairs the two.
|
||||
flush_interval_s: std.atomic.Value(u16),
|
||||
queue: EntryQueue,
|
||||
queries_dropped: std.atomic.Value(u64),
|
||||
/// When the newest drop happened, in unix seconds; 0 means none yet. Read
|
||||
@@ -434,7 +449,11 @@ pub const Logger = struct {
|
||||
/// touched it.
|
||||
pub fn init(cfg: model.Logging, queue_buf: []Entry) Logger {
|
||||
return .{
|
||||
.cfg = cfg,
|
||||
.privacy_packed = .init(@bitCast(Privacy{
|
||||
.hide_domains = cfg.hide_domains,
|
||||
.hide_client_ips = cfg.hide_client_ips,
|
||||
})),
|
||||
.flush_interval_s = .init(cfg.query_log_flush_interval_s),
|
||||
.queue = .init(queue_buf),
|
||||
.queries_dropped = .init(0),
|
||||
.last_drop_s = .init(0),
|
||||
@@ -464,8 +483,9 @@ pub const Logger = struct {
|
||||
/// configuration labels the operator wrote, identical on every row that
|
||||
/// hits them, and they say nothing about which name a client looked up.
|
||||
pub fn transformed(self: *const Logger, entry: Entry) Entry {
|
||||
const policy = self.privacy();
|
||||
var out = entry;
|
||||
if (self.cfg.hide_domains) {
|
||||
if (policy.hide_domains) {
|
||||
out.setDomain(hidden_marker);
|
||||
// Only where there is something to hide: an empty field means the
|
||||
// query had no such value, and writing a marker would claim it did.
|
||||
@@ -473,10 +493,24 @@ pub const Logger = struct {
|
||||
if (out.cname_len != 0) out.setCnameTarget(hidden_marker);
|
||||
if (out.safe_search_len != 0) out.setSafeSearchTarget(hidden_marker);
|
||||
}
|
||||
if (self.cfg.hide_client_ips) out.setClientIp(hidden_marker);
|
||||
if (policy.hide_client_ips) out.setClientIp(hidden_marker);
|
||||
return out;
|
||||
}
|
||||
|
||||
/// The live policy, from one load. Every producer decision about one entry
|
||||
/// must come from a single call to this.
|
||||
pub fn privacy(self: *const Logger) Privacy {
|
||||
return @bitCast(self.privacy_packed.load(.monotonic));
|
||||
}
|
||||
|
||||
pub fn setPrivacy(self: *Logger, p: Privacy) void {
|
||||
self.privacy_packed.store(@bitCast(p), .monotonic);
|
||||
}
|
||||
|
||||
pub fn setFlushInterval(self: *Logger, seconds: u16) void {
|
||||
self.flush_interval_s.store(seconds, .monotonic);
|
||||
}
|
||||
|
||||
/// `log` without the transforms, for a caller that already applied them.
|
||||
pub fn logTransformed(self: *Logger, io: std.Io, entry: Entry) void {
|
||||
self.enqueue(io, entry);
|
||||
@@ -540,7 +574,22 @@ pub const Logger = struct {
|
||||
return;
|
||||
};
|
||||
defer writer.deinit();
|
||||
return self.runPrepared(io, &writer, monitor);
|
||||
}
|
||||
|
||||
/// The writer loop over statements someone else prepared.
|
||||
///
|
||||
/// `runWriter` prepares and then calls this. A logger generation created by
|
||||
/// a resize prepares separately, before anything is published, so that a
|
||||
/// statement failure is refused at prepare time instead of silently killing
|
||||
/// the writer of a queue producers are already filling
|
||||
/// (`logger_controller.zig`).
|
||||
pub fn runPrepared(
|
||||
self: *Logger,
|
||||
io: std.Io,
|
||||
writer: *queries_repo.BatchWriter,
|
||||
monitor: ?*disk_monitor.Monitor,
|
||||
) std.Io.Cancelable!void {
|
||||
var batch: [flush_batch]Entry = undefined;
|
||||
while (true) {
|
||||
// A closed queue hands over its buffered elements before it reports
|
||||
@@ -564,7 +613,7 @@ pub const Logger = struct {
|
||||
self.countDropped(io, n, at);
|
||||
return err;
|
||||
};
|
||||
self.flush(io, &writer, batch[0..n], monitor) catch |err| switch (err) {
|
||||
self.flush(io, writer, batch[0..n], monitor) catch |err| switch (err) {
|
||||
error.Canceled => |e| {
|
||||
self.countDropped(io, n, at);
|
||||
return e;
|
||||
@@ -606,7 +655,7 @@ pub const Logger = struct {
|
||||
/// returns without waiting for anything.
|
||||
fn flushDeadline(self: *const Logger, io: std.Io) std.Io.Clock.Timestamp {
|
||||
return .fromNow(io, .{
|
||||
.raw = .fromSeconds(self.cfg.query_log_flush_interval_s),
|
||||
.raw = .fromSeconds(self.flush_interval_s.load(.monotonic)),
|
||||
.clock = .boot,
|
||||
});
|
||||
}
|
||||
@@ -644,6 +693,18 @@ pub const Logger = struct {
|
||||
self.draining.store(true, .release);
|
||||
}
|
||||
|
||||
/// `shutdown` without `draining`: this generation is being replaced, not
|
||||
/// the process stopped.
|
||||
///
|
||||
/// The flag is what turns a gate-held batch into a counted loss
|
||||
/// (`flush`'s `GatedAtShutdown`), and a retired writer must not take that
|
||||
/// path — the disk can still recover, and the rows it is holding are still
|
||||
/// going to be written when it does. Every producer of this generation
|
||||
/// must have released it before the close, exactly as at shutdown.
|
||||
pub fn retire(self: *Logger, io: std.Io) void {
|
||||
self.queue.close(io);
|
||||
}
|
||||
|
||||
/// Fills `batch` behind the entry already in slot 0, until it is full or
|
||||
/// `deadline` passes. `n` counts the slots that hold an entry, and stays
|
||||
/// accurate on the cancellation path so the caller can count what is lost.
|
||||
@@ -1240,6 +1301,80 @@ test "log hides only the field its switch names" {
|
||||
try testing.expectEqualStrings("192.0.2.10", untouched.clientIp());
|
||||
}
|
||||
|
||||
/// The rendezvous that forces the flip to land BETWEEN two producer entries
|
||||
/// rather than whenever the scheduler feels like it.
|
||||
const PrivacyFlip = struct {
|
||||
logger: *Logger,
|
||||
/// Set by the producer once its pre-flip entry is enqueued.
|
||||
before_done: std.Io.Event = .unset,
|
||||
/// Set by the flipper once `setPrivacy` has returned.
|
||||
flipped: std.Io.Event = .unset,
|
||||
|
||||
fn run(self: *PrivacyFlip, io: std.Io) void {
|
||||
self.before_done.wait(io) catch return;
|
||||
self.logger.setPrivacy(.{ .hide_domains = true, .hide_client_ips = true });
|
||||
self.flipped.set(io);
|
||||
}
|
||||
};
|
||||
|
||||
test "a privacy flip redacts every entry after it and none before it" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var buf: [4]Entry = undefined;
|
||||
var logger: Logger = .init(.{}, &buf);
|
||||
var flip: PrivacyFlip = .{ .logger = &logger };
|
||||
|
||||
var future = try io.concurrent(PrivacyFlip.run, .{ &flip, io });
|
||||
logger.log(io, sampleEntry(1, "before.example"));
|
||||
flip.before_done.set(io);
|
||||
try flip.flipped.wait(io);
|
||||
logger.log(io, sampleEntry(2, "after.example"));
|
||||
future.await(io);
|
||||
|
||||
const before = try logger.queue.getOne(io);
|
||||
try testing.expectEqualStrings("before.example", before.domain());
|
||||
try testing.expectEqualStrings("192.0.2.10", before.clientIp());
|
||||
|
||||
const after = try logger.queue.getOne(io);
|
||||
try testing.expectEqualStrings(hidden_marker, after.domain());
|
||||
try testing.expectEqualStrings(hidden_marker, after.clientIp());
|
||||
}
|
||||
|
||||
fn flipPrivacyRepeatedly(logger: *Logger, rounds: usize) void {
|
||||
for (0..rounds) |i| {
|
||||
logger.setPrivacy(if (i % 2 == 0)
|
||||
.{}
|
||||
else
|
||||
.{ .hide_domains = true, .hide_client_ips = true });
|
||||
}
|
||||
}
|
||||
|
||||
test "no producer observes a privacy policy that redacts one field and not the other" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var buf: [4]Entry = undefined;
|
||||
var logger: Logger = .init(.{}, &buf);
|
||||
const source = sampleEntry(1, "tracker.example");
|
||||
|
||||
const rounds = 20_000;
|
||||
var flipper = try io.concurrent(flipPrivacyRepeatedly, .{ &logger, rounds });
|
||||
// Recorded, not asserted, while the flipper runs: an assertion that
|
||||
// returned here would leave `Threaded.deinit` joining a task nothing ends.
|
||||
var mixed = false;
|
||||
for (0..rounds) |_| {
|
||||
const out = logger.transformed(source);
|
||||
const domain_hidden = std.mem.eql(u8, out.domain(), hidden_marker);
|
||||
const client_hidden = std.mem.eql(u8, out.clientIp(), hidden_marker);
|
||||
if (domain_hidden != client_hidden) mixed = true;
|
||||
}
|
||||
flipper.await(io);
|
||||
try testing.expect(!mixed);
|
||||
}
|
||||
|
||||
test "the split halves reproduce log byte for byte" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
@@ -1507,6 +1642,57 @@ test "a full batch flushes without waiting for the interval" {
|
||||
try testing.expectEqual(@as(i64, 150), try queries_repo.countRows(&database));
|
||||
}
|
||||
|
||||
test "the writer's next cycle uses the interval set since its last one" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var buf: [8]Entry = undefined;
|
||||
// Zero: every cycle commits what it has and goes straight back to `getOne`,
|
||||
// so the first entry proves the writer is running and parked between cycles.
|
||||
var logger: Logger = .init(.{ .query_log_flush_interval_s = 0 }, &buf);
|
||||
|
||||
var future = try io.concurrent(Logger.runWriter, .{
|
||||
&logger,
|
||||
io,
|
||||
&database,
|
||||
@as(?*disk_monitor.Monitor, null),
|
||||
});
|
||||
// See the note in "shutdown writes the batch the writer holds": this must
|
||||
// run before the deferred `database.close`.
|
||||
defer {
|
||||
logger.shutdown(io);
|
||||
future.await(io) catch {};
|
||||
}
|
||||
|
||||
logger.log(io, sampleEntry(1, "first.example"));
|
||||
const poll: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(5), .clock = .awake };
|
||||
var waited: usize = 0;
|
||||
while (logger.rows_written.load(.monotonic) == 0 and waited < 400) : (waited += 1) {
|
||||
try poll.sleep(io);
|
||||
}
|
||||
|
||||
// An hour, installed while the writer is parked: the cycle the next entry
|
||||
// starts must wait it out instead of committing at once.
|
||||
logger.setFlushInterval(3600);
|
||||
logger.log(io, sampleEntry(2, "second.example"));
|
||||
const quarter: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(250), .clock = .awake };
|
||||
try quarter.sleep(io);
|
||||
const written_under_the_new_interval = logger.rows_written.load(.monotonic);
|
||||
|
||||
logger.shutdown(io);
|
||||
try future.await(io);
|
||||
|
||||
try testing.expect(waited < 400);
|
||||
// The first entry, and only the first: the second is still held.
|
||||
try testing.expectEqual(@as(u64, 1), written_under_the_new_interval);
|
||||
// The close releases it, which is what makes the hold a hold and not a loss.
|
||||
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
|
||||
}
|
||||
|
||||
test "a gated flush holds the batch until the disk recovers" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
|
||||
Reference in New Issue
Block a user