db-mode config changes apply live in-process
Gates / frontend (push) Successful in 1m43s
Gates / test (push) Successful in 2m14s
Gates / test-aarch64 (push) Successful in 8m3s
Gates / package (push) Successful in 5m42s
Gates / container (push) Successful in 54s
CI / gates (push) Successful in 50m24s
Gates / frontend (push) Successful in 1m43s
Gates / test (push) Successful in 2m14s
Gates / test-aarch64 (push) Successful in 8m3s
Gates / package (push) Successful in 5m42s
Gates / container (push) Successful in 54s
CI / gates (push) Successful in 50m24s
settings and upstream writes now follow a prepare, commit, publish, retire contract: candidates are built and validated before the database transaction, published as infallible pointer swaps, and old generations retire after their readers drain. per-query policy values snapshot once per query; upstream pool, cache, rate limiter, sessions, api limiter, log sink, blocklist scheduler and the query-log queue each gained one named live operation. restart_required shrinks from every scalar key to the bind keys and web.enabled; the admin ui drops its restart notices for everything else. file mode is unchanged.
This commit is contained in:
@@ -33,11 +33,65 @@ pub fn classify(free_bytes: u64, cfg: model.Disk) State {
|
||||
return .ok;
|
||||
}
|
||||
|
||||
/// `min_free_mb` and `warn_free_mb` are one invariant pair — `classify` reads
|
||||
/// both and reports the more severe verdict — so they live in one atomic word
|
||||
/// and a reader unpacks a single load. Two atomics would let a sample land
|
||||
/// between the two stores and classify against half of one configuration and
|
||||
/// half of another.
|
||||
fn packThresholds(d: model.Disk) u64 {
|
||||
return (@as(u64, d.min_free_mb) << 32) | d.warn_free_mb;
|
||||
}
|
||||
|
||||
fn unpackThresholds(bits: u64) model.Disk {
|
||||
return .{
|
||||
.min_free_mb = @truncate(bits >> 32),
|
||||
.warn_free_mb = @truncate(bits),
|
||||
};
|
||||
}
|
||||
|
||||
/// Where the measured log directory comes from. `sample` borrows the path
|
||||
/// across a directory scan, so the path cannot simply be replaced under it: a
|
||||
/// reader pins a generation for the whole borrow and `setLogDir` retires the
|
||||
/// old one, which is freed by whichever of the two — the last reader or the
|
||||
/// setter — finds it retired with no refs.
|
||||
pub const LogDirSource = union(enum) {
|
||||
/// The path `init` was given, borrowed from the config. It outlives the
|
||||
/// process, so a reader still holding it after a swap is safe and it needs
|
||||
/// no pin. Null means logs do not go to a file.
|
||||
boot: ?[:0]const u8,
|
||||
/// Every generation `setLogDir` installs. Null means the same as above.
|
||||
installed: ?*LogDir,
|
||||
};
|
||||
|
||||
/// A reader's hold on the log directory for the length of one scan. `pinned`
|
||||
/// is null for the boot source, which nothing frees.
|
||||
pub const LogDirBorrow = struct {
|
||||
path: ?[:0]const u8,
|
||||
pinned: ?*LogDir,
|
||||
};
|
||||
|
||||
pub const LogDir = struct {
|
||||
path: [:0]const u8,
|
||||
refs: u32 = 0,
|
||||
retired: bool = false,
|
||||
/// Non-null exactly for heap generations, and the allocator that frees
|
||||
/// them.
|
||||
gpa: ?std.mem.Allocator = null,
|
||||
|
||||
fn destroy(self: *LogDir) void {
|
||||
const gpa = self.gpa orelse return;
|
||||
gpa.free(self.path);
|
||||
gpa.destroy(self);
|
||||
}
|
||||
};
|
||||
|
||||
pub const Monitor = struct {
|
||||
cfg: model.Disk,
|
||||
thresholds_packed: std.atomic.Value(u64),
|
||||
data_dir: std.Io.Dir,
|
||||
data_path: [:0]const u8,
|
||||
log_dir_path: ?[:0]const u8,
|
||||
/// Guards `log_dir` and every generation's `refs`/`retired`.
|
||||
log_dir_mutex: std.Io.Mutex,
|
||||
log_dir: LogDirSource,
|
||||
|
||||
state_raw: std.atomic.Value(u8),
|
||||
free_bytes: std.atomic.Value(u64),
|
||||
@@ -57,10 +111,11 @@ pub const Monitor = struct {
|
||||
log_dir_path: ?[:0]const u8,
|
||||
) Monitor {
|
||||
return .{
|
||||
.cfg = cfg,
|
||||
.thresholds_packed = .init(packThresholds(cfg)),
|
||||
.data_dir = data_dir,
|
||||
.data_path = data_path,
|
||||
.log_dir_path = log_dir_path,
|
||||
.log_dir_mutex = .init,
|
||||
.log_dir = .{ .boot = log_dir_path },
|
||||
.state_raw = .init(@intFromEnum(State.ok)),
|
||||
.free_bytes = .init(0),
|
||||
.db_bytes = .init(0),
|
||||
@@ -69,6 +124,90 @@ pub const Monitor = struct {
|
||||
};
|
||||
}
|
||||
|
||||
/// The live threshold pair, from one load: `warn >= min` holds for every
|
||||
/// value this ever returns, whatever a concurrent `setThresholds` does.
|
||||
pub fn thresholds(self: *const Monitor) model.Disk {
|
||||
return unpackThresholds(self.thresholds_packed.load(.monotonic));
|
||||
}
|
||||
|
||||
pub fn setThresholds(self: *Monitor, d: model.Disk) void {
|
||||
self.thresholds_packed.store(packThresholds(d), .monotonic);
|
||||
}
|
||||
|
||||
/// Pins the log directory for one scan. Every borrow is matched by a
|
||||
/// `releaseLogDir`, which is what lets `setLogDir` free a generation the
|
||||
/// moment no scan is reading its path.
|
||||
pub fn acquireLogDir(self: *Monitor, io: std.Io) LogDirBorrow {
|
||||
self.log_dir_mutex.lockUncancelable(io);
|
||||
defer self.log_dir_mutex.unlock(io);
|
||||
|
||||
switch (self.log_dir) {
|
||||
.boot => |path| return .{ .path = path, .pinned = null },
|
||||
.installed => |maybe| {
|
||||
const gen = maybe orelse return .{ .path = null, .pinned = null };
|
||||
gen.refs += 1;
|
||||
return .{ .path = gen.path, .pinned = gen };
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn releaseLogDir(self: *Monitor, io: std.Io, borrow: LogDirBorrow) void {
|
||||
const gen = borrow.pinned orelse return;
|
||||
|
||||
self.log_dir_mutex.lockUncancelable(io);
|
||||
std.debug.assert(gen.refs > 0);
|
||||
gen.refs -= 1;
|
||||
const free_it = gen.retired and gen.refs == 0;
|
||||
self.log_dir_mutex.unlock(io);
|
||||
|
||||
if (free_it) gen.destroy();
|
||||
}
|
||||
|
||||
/// Prepare half of a log-directory change: allocates the owned path and
|
||||
/// its generation node before any commit, so publish cannot fail. `path`
|
||||
/// null means logs no longer go to a file and nothing is measured.
|
||||
pub fn prepareLogDir(
|
||||
gpa: std.mem.Allocator,
|
||||
path: ?[]const u8,
|
||||
) std.mem.Allocator.Error!?*LogDir {
|
||||
const p = path orelse return null;
|
||||
const owned = try gpa.dupeZ(u8, p);
|
||||
errdefer gpa.free(owned);
|
||||
const gen = try gpa.create(LogDir);
|
||||
gen.* = .{ .path = owned, .gpa = gpa };
|
||||
return gen;
|
||||
}
|
||||
|
||||
/// Discards a generation `prepareLogDir` built that will not be published.
|
||||
pub fn destroyPreparedLogDir(prepared: ?*LogDir) void {
|
||||
if (prepared) |gen| gen.destroy();
|
||||
}
|
||||
|
||||
/// Publish half: infallible and I/O-free. The old generation is retired
|
||||
/// and freed here when no scan holds it, or by the last release otherwise.
|
||||
pub fn setLogDir(self: *Monitor, io: std.Io, prepared: ?*LogDir) void {
|
||||
self.log_dir_mutex.lockUncancelable(io);
|
||||
const old: ?*LogDir = switch (self.log_dir) {
|
||||
.boot => null,
|
||||
.installed => |maybe| maybe,
|
||||
};
|
||||
self.log_dir = .{ .installed = prepared };
|
||||
var free_old = false;
|
||||
if (old) |gen| {
|
||||
gen.retired = true;
|
||||
free_old = gen.refs == 0;
|
||||
}
|
||||
self.log_dir_mutex.unlock(io);
|
||||
|
||||
if (free_old) old.?.destroy();
|
||||
}
|
||||
|
||||
/// Frees any installed log-directory generation. Every scan must have
|
||||
/// released first, which shutdown ordering guarantees.
|
||||
pub fn deinit(self: *Monitor, io: std.Io) void {
|
||||
self.setLogDir(io, null);
|
||||
}
|
||||
|
||||
pub fn state(self: *const Monitor) State {
|
||||
return @enumFromInt(self.state_raw.load(.monotonic));
|
||||
}
|
||||
@@ -109,7 +248,9 @@ pub const Monitor = struct {
|
||||
probeFailed(store, io, now_s, "data_dir", "sizing the data directory failed", err);
|
||||
}
|
||||
|
||||
if (self.log_dir_path) |path| {
|
||||
const borrow = self.acquireLogDir(io);
|
||||
defer self.releaseLogDir(io, borrow);
|
||||
if (borrow.path) |path| {
|
||||
if (self.sumLogDir(io, path)) |bytes| {
|
||||
self.log_bytes.store(bytes, .monotonic);
|
||||
if (store) |s| s.resolve(io, now_s, .disk_probe, "log_dir");
|
||||
@@ -120,7 +261,7 @@ pub const Monitor = struct {
|
||||
}
|
||||
}
|
||||
|
||||
self.publish(io, store, now_s, classify(free, self.cfg), free);
|
||||
self.publish(io, store, now_s, classify(free, self.thresholds()), free);
|
||||
}
|
||||
|
||||
/// Sample first, then sleep: a process that starts on a full disk must not
|
||||
@@ -466,7 +607,7 @@ test "a threshold above the real free space drives the state to critical" {
|
||||
try testing.expectEqual(State.critical, monitor.state());
|
||||
try testing.expect(!monitor.writesAllowed());
|
||||
|
||||
monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
|
||||
monitor.setThresholds(.{ .min_free_mb = 0, .warn_free_mb = 0 });
|
||||
monitor.sample(io, null, 0);
|
||||
try testing.expectEqual(State.ok, monitor.state());
|
||||
try testing.expect(monitor.writesAllowed());
|
||||
@@ -509,7 +650,7 @@ test "a disk transition records an episode per severity and closes it on recover
|
||||
monitor.sample(io, &fx.store, 1060);
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
|
||||
monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
|
||||
monitor.setThresholds(.{ .min_free_mb = 0, .warn_free_mb = 0 });
|
||||
monitor.sample(io, &fx.store, 1120);
|
||||
try testing.expectEqual(State.ok, monitor.state());
|
||||
try testing.expectEqual(
|
||||
@@ -551,7 +692,9 @@ test "a failed probe opens an episode the next clean pass closes" {
|
||||
|
||||
try tmp.dir.createDirPath(io, "logs");
|
||||
var path_buf: [256]u8 = undefined;
|
||||
monitor.log_dir_path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/logs", .{tmp.sub_path});
|
||||
const good_dir = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/logs", .{tmp.sub_path});
|
||||
monitor.setLogDir(io, try Monitor.prepareLogDir(testing.allocator, good_dir));
|
||||
defer monitor.deinit(io);
|
||||
monitor.sample(io, &fx.store, 1100);
|
||||
|
||||
try testing.expectEqual(
|
||||
@@ -560,6 +703,37 @@ test "a failed probe opens an episode the next clean pass closes" {
|
||||
);
|
||||
}
|
||||
|
||||
/// Alternates between two pairs that each satisfy `warn >= min`, so any
|
||||
/// observed pair violating it can only have been torn out of two stores.
|
||||
fn storeThresholdPairs(monitor: *Monitor, rounds: usize) void {
|
||||
for (0..rounds) |i| {
|
||||
monitor.setThresholds(if (i % 2 == 0)
|
||||
.{ .min_free_mb = 1, .warn_free_mb = 2 }
|
||||
else
|
||||
.{ .min_free_mb = 3_000_000, .warn_free_mb = 4_000_000 });
|
||||
}
|
||||
}
|
||||
|
||||
test "a threshold reader never observes a pair from two different stores" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var monitor: Monitor = .init(.{ .min_free_mb = 1, .warn_free_mb = 2 }, std.Io.Dir.cwd(), ".", null);
|
||||
|
||||
const rounds = 20_000;
|
||||
var writer = try io.concurrent(storeThresholdPairs, .{ &monitor, rounds });
|
||||
// Recorded, not asserted, while the writer runs: an assertion that returned
|
||||
// here would leave `Threaded.deinit` joining a task nothing ends.
|
||||
var torn = false;
|
||||
for (0..rounds) |_| {
|
||||
const pair = monitor.thresholds();
|
||||
if (pair.warn_free_mb < pair.min_free_mb) torn = true;
|
||||
}
|
||||
writer.await(io);
|
||||
try testing.expect(!torn);
|
||||
}
|
||||
|
||||
test "every emit site is inert when the store is absent" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
@@ -574,3 +748,89 @@ test "every emit site is inert when the store is absent" {
|
||||
monitor.sample(io, null, 0);
|
||||
try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// setLogDir (milestone-34 S3.5)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test "setLogDir re-points the measurement and frees the retired generation" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var tmp = testing.tmpDir(.{ .iterate = true });
|
||||
defer tmp.cleanup();
|
||||
try tmp.dir.createDirPath(io, "first");
|
||||
try tmp.dir.createDirPath(io, "second");
|
||||
try tmp.dir.writeFile(io, .{ .sub_path = "first/nxdns.log", .data = "aaaa" });
|
||||
try tmp.dir.writeFile(io, .{ .sub_path = "second/nxdns.log", .data = "bbbbbbbb" });
|
||||
|
||||
var first_buf: [160]u8 = undefined;
|
||||
var second_buf: [160]u8 = undefined;
|
||||
const first = try std.fmt.bufPrint(&first_buf, ".zig-cache/tmp/{s}/first", .{tmp.sub_path});
|
||||
const second = try std.fmt.bufPrint(&second_buf, ".zig-cache/tmp/{s}/second", .{tmp.sub_path});
|
||||
|
||||
var monitor: Monitor = .init(.{ .min_free_mb = 0, .warn_free_mb = 0 }, tmp.dir, ".", null);
|
||||
defer monitor.deinit(io);
|
||||
|
||||
// Boot measures nothing.
|
||||
monitor.sample(io, null, 1_000);
|
||||
try testing.expectEqual(@as(u64, 0), monitor.gauges().log_bytes);
|
||||
|
||||
monitor.setLogDir(io, try Monitor.prepareLogDir(testing.allocator, first));
|
||||
monitor.sample(io, null, 1_100);
|
||||
try testing.expectEqual(@as(u64, 4), monitor.gauges().log_bytes);
|
||||
|
||||
// The retired generation is freed here; the testing allocator says so.
|
||||
monitor.setLogDir(io, try Monitor.prepareLogDir(testing.allocator, second));
|
||||
monitor.sample(io, null, 1_200);
|
||||
try testing.expectEqual(@as(u64, 8), monitor.gauges().log_bytes);
|
||||
|
||||
// Output moved away from file: nothing is measured, and the gauge keeps
|
||||
// its last reading rather than claiming zero bytes of logs.
|
||||
monitor.setLogDir(io, null);
|
||||
monitor.sample(io, null, 1_300);
|
||||
try testing.expectEqual(@as(u64, 8), monitor.gauges().log_bytes);
|
||||
}
|
||||
|
||||
test "a prepared log directory that is never published is freed by the caller" {
|
||||
const prepared = try Monitor.prepareLogDir(testing.allocator, "/var/log/nxdns");
|
||||
Monitor.destroyPreparedLogDir(prepared);
|
||||
try testing.expectEqual(@as(?*LogDir, null), try Monitor.prepareLogDir(testing.allocator, null));
|
||||
}
|
||||
|
||||
test "a sample borrowing a log directory survives a concurrent setLogDir" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var tmp = testing.tmpDir(.{ .iterate = true });
|
||||
defer tmp.cleanup();
|
||||
try tmp.dir.createDirPath(io, "logs");
|
||||
try tmp.dir.writeFile(io, .{ .sub_path = "logs/nxdns.log", .data = "aaaa" });
|
||||
|
||||
var path_buf: [160]u8 = undefined;
|
||||
const logs = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/logs", .{tmp.sub_path});
|
||||
|
||||
var monitor: Monitor = .init(.{ .min_free_mb = 0, .warn_free_mb = 0 }, tmp.dir, ".", null);
|
||||
defer monitor.deinit(io);
|
||||
monitor.setLogDir(io, try Monitor.prepareLogDir(testing.allocator, logs));
|
||||
|
||||
const Racer = struct {
|
||||
fn sample(m: *Monitor, sio: std.Io) void {
|
||||
for (0..200) |i| m.sample(sio, null, @intCast(1_000 + i));
|
||||
}
|
||||
fn repoint(m: *Monitor, sio: std.Io, p: []const u8) void {
|
||||
for (0..200) |i| {
|
||||
const prepared = Monitor.prepareLogDir(testing.allocator, if (i % 2 == 0) p else null) catch return;
|
||||
m.setLogDir(sio, prepared);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var group: std.Io.Group = .init;
|
||||
defer group.cancel(io);
|
||||
try group.concurrent(io, Racer.sample, .{ &monitor, io });
|
||||
try group.concurrent(io, Racer.repoint, .{ &monitor, io, logs });
|
||||
try group.await(io);
|
||||
}
|
||||
|
||||
+192
-6
@@ -399,8 +399,23 @@ const discard_stall = if (builtin.is_test) struct {
|
||||
}
|
||||
};
|
||||
|
||||
/// The §11.4 privacy policy: one value, never two. A producer decides the
|
||||
/// domain fields and the client field from ONE load of `privacy_packed`, so no
|
||||
/// entry can leave `transformed` with the domain redacted and the client
|
||||
/// exposed, or the reverse, because a `setPrivacy` landed between the two
|
||||
/// decisions.
|
||||
pub const Privacy = packed struct(u8) {
|
||||
hide_domains: bool = false,
|
||||
hide_client_ips: bool = false,
|
||||
_reserved: u6 = 0,
|
||||
};
|
||||
|
||||
pub const Logger = struct {
|
||||
cfg: model.Logging,
|
||||
/// Both privacy flags in one atomic byte, loaded once per entry.
|
||||
privacy_packed: std.atomic.Value(u8),
|
||||
/// Independent of the privacy policy: it governs when a batch commits, not
|
||||
/// what a row contains, so nothing pairs the two.
|
||||
flush_interval_s: std.atomic.Value(u16),
|
||||
queue: EntryQueue,
|
||||
queries_dropped: std.atomic.Value(u64),
|
||||
/// When the newest drop happened, in unix seconds; 0 means none yet. Read
|
||||
@@ -434,7 +449,11 @@ pub const Logger = struct {
|
||||
/// touched it.
|
||||
pub fn init(cfg: model.Logging, queue_buf: []Entry) Logger {
|
||||
return .{
|
||||
.cfg = cfg,
|
||||
.privacy_packed = .init(@bitCast(Privacy{
|
||||
.hide_domains = cfg.hide_domains,
|
||||
.hide_client_ips = cfg.hide_client_ips,
|
||||
})),
|
||||
.flush_interval_s = .init(cfg.query_log_flush_interval_s),
|
||||
.queue = .init(queue_buf),
|
||||
.queries_dropped = .init(0),
|
||||
.last_drop_s = .init(0),
|
||||
@@ -464,8 +483,9 @@ pub const Logger = struct {
|
||||
/// configuration labels the operator wrote, identical on every row that
|
||||
/// hits them, and they say nothing about which name a client looked up.
|
||||
pub fn transformed(self: *const Logger, entry: Entry) Entry {
|
||||
const policy = self.privacy();
|
||||
var out = entry;
|
||||
if (self.cfg.hide_domains) {
|
||||
if (policy.hide_domains) {
|
||||
out.setDomain(hidden_marker);
|
||||
// Only where there is something to hide: an empty field means the
|
||||
// query had no such value, and writing a marker would claim it did.
|
||||
@@ -473,10 +493,24 @@ pub const Logger = struct {
|
||||
if (out.cname_len != 0) out.setCnameTarget(hidden_marker);
|
||||
if (out.safe_search_len != 0) out.setSafeSearchTarget(hidden_marker);
|
||||
}
|
||||
if (self.cfg.hide_client_ips) out.setClientIp(hidden_marker);
|
||||
if (policy.hide_client_ips) out.setClientIp(hidden_marker);
|
||||
return out;
|
||||
}
|
||||
|
||||
/// The live policy, from one load. Every producer decision about one entry
|
||||
/// must come from a single call to this.
|
||||
pub fn privacy(self: *const Logger) Privacy {
|
||||
return @bitCast(self.privacy_packed.load(.monotonic));
|
||||
}
|
||||
|
||||
pub fn setPrivacy(self: *Logger, p: Privacy) void {
|
||||
self.privacy_packed.store(@bitCast(p), .monotonic);
|
||||
}
|
||||
|
||||
pub fn setFlushInterval(self: *Logger, seconds: u16) void {
|
||||
self.flush_interval_s.store(seconds, .monotonic);
|
||||
}
|
||||
|
||||
/// `log` without the transforms, for a caller that already applied them.
|
||||
pub fn logTransformed(self: *Logger, io: std.Io, entry: Entry) void {
|
||||
self.enqueue(io, entry);
|
||||
@@ -540,7 +574,22 @@ pub const Logger = struct {
|
||||
return;
|
||||
};
|
||||
defer writer.deinit();
|
||||
return self.runPrepared(io, &writer, monitor);
|
||||
}
|
||||
|
||||
/// The writer loop over statements someone else prepared.
|
||||
///
|
||||
/// `runWriter` prepares and then calls this. A logger generation created by
|
||||
/// a resize prepares separately, before anything is published, so that a
|
||||
/// statement failure is refused at prepare time instead of silently killing
|
||||
/// the writer of a queue producers are already filling
|
||||
/// (`logger_controller.zig`).
|
||||
pub fn runPrepared(
|
||||
self: *Logger,
|
||||
io: std.Io,
|
||||
writer: *queries_repo.BatchWriter,
|
||||
monitor: ?*disk_monitor.Monitor,
|
||||
) std.Io.Cancelable!void {
|
||||
var batch: [flush_batch]Entry = undefined;
|
||||
while (true) {
|
||||
// A closed queue hands over its buffered elements before it reports
|
||||
@@ -564,7 +613,7 @@ pub const Logger = struct {
|
||||
self.countDropped(io, n, at);
|
||||
return err;
|
||||
};
|
||||
self.flush(io, &writer, batch[0..n], monitor) catch |err| switch (err) {
|
||||
self.flush(io, writer, batch[0..n], monitor) catch |err| switch (err) {
|
||||
error.Canceled => |e| {
|
||||
self.countDropped(io, n, at);
|
||||
return e;
|
||||
@@ -606,7 +655,7 @@ pub const Logger = struct {
|
||||
/// returns without waiting for anything.
|
||||
fn flushDeadline(self: *const Logger, io: std.Io) std.Io.Clock.Timestamp {
|
||||
return .fromNow(io, .{
|
||||
.raw = .fromSeconds(self.cfg.query_log_flush_interval_s),
|
||||
.raw = .fromSeconds(self.flush_interval_s.load(.monotonic)),
|
||||
.clock = .boot,
|
||||
});
|
||||
}
|
||||
@@ -644,6 +693,18 @@ pub const Logger = struct {
|
||||
self.draining.store(true, .release);
|
||||
}
|
||||
|
||||
/// `shutdown` without `draining`: this generation is being replaced, not
|
||||
/// the process stopped.
|
||||
///
|
||||
/// The flag is what turns a gate-held batch into a counted loss
|
||||
/// (`flush`'s `GatedAtShutdown`), and a retired writer must not take that
|
||||
/// path — the disk can still recover, and the rows it is holding are still
|
||||
/// going to be written when it does. Every producer of this generation
|
||||
/// must have released it before the close, exactly as at shutdown.
|
||||
pub fn retire(self: *Logger, io: std.Io) void {
|
||||
self.queue.close(io);
|
||||
}
|
||||
|
||||
/// Fills `batch` behind the entry already in slot 0, until it is full or
|
||||
/// `deadline` passes. `n` counts the slots that hold an entry, and stays
|
||||
/// accurate on the cancellation path so the caller can count what is lost.
|
||||
@@ -1240,6 +1301,80 @@ test "log hides only the field its switch names" {
|
||||
try testing.expectEqualStrings("192.0.2.10", untouched.clientIp());
|
||||
}
|
||||
|
||||
/// The rendezvous that forces the flip to land BETWEEN two producer entries
|
||||
/// rather than whenever the scheduler feels like it.
|
||||
const PrivacyFlip = struct {
|
||||
logger: *Logger,
|
||||
/// Set by the producer once its pre-flip entry is enqueued.
|
||||
before_done: std.Io.Event = .unset,
|
||||
/// Set by the flipper once `setPrivacy` has returned.
|
||||
flipped: std.Io.Event = .unset,
|
||||
|
||||
fn run(self: *PrivacyFlip, io: std.Io) void {
|
||||
self.before_done.wait(io) catch return;
|
||||
self.logger.setPrivacy(.{ .hide_domains = true, .hide_client_ips = true });
|
||||
self.flipped.set(io);
|
||||
}
|
||||
};
|
||||
|
||||
test "a privacy flip redacts every entry after it and none before it" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var buf: [4]Entry = undefined;
|
||||
var logger: Logger = .init(.{}, &buf);
|
||||
var flip: PrivacyFlip = .{ .logger = &logger };
|
||||
|
||||
var future = try io.concurrent(PrivacyFlip.run, .{ &flip, io });
|
||||
logger.log(io, sampleEntry(1, "before.example"));
|
||||
flip.before_done.set(io);
|
||||
try flip.flipped.wait(io);
|
||||
logger.log(io, sampleEntry(2, "after.example"));
|
||||
future.await(io);
|
||||
|
||||
const before = try logger.queue.getOne(io);
|
||||
try testing.expectEqualStrings("before.example", before.domain());
|
||||
try testing.expectEqualStrings("192.0.2.10", before.clientIp());
|
||||
|
||||
const after = try logger.queue.getOne(io);
|
||||
try testing.expectEqualStrings(hidden_marker, after.domain());
|
||||
try testing.expectEqualStrings(hidden_marker, after.clientIp());
|
||||
}
|
||||
|
||||
fn flipPrivacyRepeatedly(logger: *Logger, rounds: usize) void {
|
||||
for (0..rounds) |i| {
|
||||
logger.setPrivacy(if (i % 2 == 0)
|
||||
.{}
|
||||
else
|
||||
.{ .hide_domains = true, .hide_client_ips = true });
|
||||
}
|
||||
}
|
||||
|
||||
test "no producer observes a privacy policy that redacts one field and not the other" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var buf: [4]Entry = undefined;
|
||||
var logger: Logger = .init(.{}, &buf);
|
||||
const source = sampleEntry(1, "tracker.example");
|
||||
|
||||
const rounds = 20_000;
|
||||
var flipper = try io.concurrent(flipPrivacyRepeatedly, .{ &logger, rounds });
|
||||
// Recorded, not asserted, while the flipper runs: an assertion that
|
||||
// returned here would leave `Threaded.deinit` joining a task nothing ends.
|
||||
var mixed = false;
|
||||
for (0..rounds) |_| {
|
||||
const out = logger.transformed(source);
|
||||
const domain_hidden = std.mem.eql(u8, out.domain(), hidden_marker);
|
||||
const client_hidden = std.mem.eql(u8, out.clientIp(), hidden_marker);
|
||||
if (domain_hidden != client_hidden) mixed = true;
|
||||
}
|
||||
flipper.await(io);
|
||||
try testing.expect(!mixed);
|
||||
}
|
||||
|
||||
test "the split halves reproduce log byte for byte" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
@@ -1507,6 +1642,57 @@ test "a full batch flushes without waiting for the interval" {
|
||||
try testing.expectEqual(@as(i64, 150), try queries_repo.countRows(&database));
|
||||
}
|
||||
|
||||
test "the writer's next cycle uses the interval set since its last one" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var buf: [8]Entry = undefined;
|
||||
// Zero: every cycle commits what it has and goes straight back to `getOne`,
|
||||
// so the first entry proves the writer is running and parked between cycles.
|
||||
var logger: Logger = .init(.{ .query_log_flush_interval_s = 0 }, &buf);
|
||||
|
||||
var future = try io.concurrent(Logger.runWriter, .{
|
||||
&logger,
|
||||
io,
|
||||
&database,
|
||||
@as(?*disk_monitor.Monitor, null),
|
||||
});
|
||||
// See the note in "shutdown writes the batch the writer holds": this must
|
||||
// run before the deferred `database.close`.
|
||||
defer {
|
||||
logger.shutdown(io);
|
||||
future.await(io) catch {};
|
||||
}
|
||||
|
||||
logger.log(io, sampleEntry(1, "first.example"));
|
||||
const poll: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(5), .clock = .awake };
|
||||
var waited: usize = 0;
|
||||
while (logger.rows_written.load(.monotonic) == 0 and waited < 400) : (waited += 1) {
|
||||
try poll.sleep(io);
|
||||
}
|
||||
|
||||
// An hour, installed while the writer is parked: the cycle the next entry
|
||||
// starts must wait it out instead of committing at once.
|
||||
logger.setFlushInterval(3600);
|
||||
logger.log(io, sampleEntry(2, "second.example"));
|
||||
const quarter: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(250), .clock = .awake };
|
||||
try quarter.sleep(io);
|
||||
const written_under_the_new_interval = logger.rows_written.load(.monotonic);
|
||||
|
||||
logger.shutdown(io);
|
||||
try future.await(io);
|
||||
|
||||
try testing.expect(waited < 400);
|
||||
// The first entry, and only the first: the second is still held.
|
||||
try testing.expectEqual(@as(u64, 1), written_under_the_new_interval);
|
||||
// The close releases it, which is what makes the hold a hold and not a loss.
|
||||
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
|
||||
}
|
||||
|
||||
test "a gated flush holds the batch until the disk recovers" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -406,7 +406,8 @@ test "S8 case 5: a retention pass prunes the old rows and truncates the write-ah
|
||||
try testing.expectEqual(@as(i64, 4), try queries_repo.countRows(log_db.database()));
|
||||
try testing.expect(try f.sizeOf("querylog.db-wal") > 0);
|
||||
|
||||
var pass: retention.Retention = .init(.{ .retention_days = 30 });
|
||||
var pass_days: retention.RetentionDays = .init(30);
|
||||
var pass: retention.Retention = .init(&pass_days);
|
||||
pass.runOnce(io, log_db.database(), null, null);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), pass.snapshotStats().passes);
|
||||
@@ -473,7 +474,7 @@ test "S8 case 6: a critical disk gates the flushes and recovery releases them" {
|
||||
try testing.expectEqual(@as(u64, 0), query_log.rows_written.load(.monotonic));
|
||||
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(log_db.database()));
|
||||
|
||||
monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
|
||||
monitor.setThresholds(.{ .min_free_mb = 0, .warn_free_mb = 0 });
|
||||
monitor.sample(io, null, 0);
|
||||
try testing.expectEqual(disk_monitor.State.ok, monitor.state());
|
||||
try testing.expect(monitor.writesAllowed());
|
||||
|
||||
@@ -199,6 +199,29 @@ pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
|
||||
return result;
|
||||
}
|
||||
|
||||
/// An additional connection to a `querylog.db` that `open` has already
|
||||
/// established, with the pragmas every connection to the file needs.
|
||||
///
|
||||
/// The canonical opener for every background connection: the log writer, the
|
||||
/// retention pass and the web task each own one (`retention.zig`'s contract),
|
||||
/// and a logger generation opens one per writer for that writer's whole life
|
||||
/// (`logger_controller.zig`) — two writers must never share a handle.
|
||||
///
|
||||
/// `dir` and `path` follow `open`'s resolution rule, and `dir` participates in
|
||||
/// it the same way: the caller passes either an absolute path with `dir` open
|
||||
/// on its parent, or `std.Io.Dir.cwd()` with a cwd-relative path. Nothing here
|
||||
/// touches the directory itself — the file already exists by contract — so the
|
||||
/// handle is present to make the pairing explicit at every call site rather
|
||||
/// than to be dereferenced.
|
||||
pub fn reopen(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) db.Error!db.Db {
|
||||
_ = io;
|
||||
_ = dir;
|
||||
var database = try db.Db.open(path, .{ .mode = .read_write_existing });
|
||||
errdefer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
return database;
|
||||
}
|
||||
|
||||
/// The whitelist. `null` means "propagate, do not touch the file".
|
||||
fn recreatable(e: db.Error) ?RecreateReason {
|
||||
return switch (e) {
|
||||
|
||||
+77
-17
@@ -14,7 +14,6 @@ const std = @import("std");
|
||||
const db = @import("db.zig");
|
||||
const disk_monitor = @import("disk_monitor.zig");
|
||||
const events = @import("events.zig");
|
||||
const model = @import("../config/model.zig");
|
||||
const queries_repo = @import("repositories/queries_repo.zig");
|
||||
|
||||
const log = std.log.scoped(.retention);
|
||||
@@ -50,15 +49,42 @@ const Counters = struct {
|
||||
vacuums_gated: std.atomic.Value(u64) = .init(0),
|
||||
};
|
||||
|
||||
/// `logging.retention_days`, shared by its two consumers — this pass and
|
||||
/// `server/clients.zig`'s stale-client prune. One cell rather than a copy in
|
||||
/// each: the two must never prune to different cutoffs, and a settings apply
|
||||
/// stores once. Owned by app-level state and outlives both readers.
|
||||
///
|
||||
/// `.monotonic` is enough: the value stands alone and orders nothing else, and
|
||||
/// each consumer reads it once per pass.
|
||||
pub const RetentionDays = struct {
|
||||
value: std.atomic.Value(u32),
|
||||
|
||||
pub fn init(days: u16) RetentionDays {
|
||||
return .{ .value = .init(days) };
|
||||
}
|
||||
|
||||
pub fn get(self: *const RetentionDays) u32 {
|
||||
return self.value.load(.monotonic);
|
||||
}
|
||||
|
||||
pub fn setRetentionDays(self: *RetentionDays, days: u16) void {
|
||||
self.value.store(days, .monotonic);
|
||||
}
|
||||
|
||||
pub fn seconds(self: *const RetentionDays) i64 {
|
||||
return @as(i64, self.get()) * std.time.s_per_day;
|
||||
}
|
||||
};
|
||||
|
||||
pub const Retention = struct {
|
||||
cfg: model.Logging,
|
||||
days: *const RetentionDays,
|
||||
counters: Counters,
|
||||
/// Passes since the last vacuum that succeeded. Plain rather than atomic:
|
||||
/// only the retention task reads or writes it, and no consumer reports it.
|
||||
passes_since_vacuum: u32,
|
||||
|
||||
pub fn init(cfg: model.Logging) Retention {
|
||||
return .{ .cfg = cfg, .counters = .{}, .passes_since_vacuum = 0 };
|
||||
pub fn init(days: *const RetentionDays) Retention {
|
||||
return .{ .days = days, .counters = .{}, .passes_since_vacuum = 0 };
|
||||
}
|
||||
|
||||
/// The counters, read one at a time. A scrape that lands mid-pass can see a
|
||||
@@ -100,7 +126,7 @@ pub const Retention = struct {
|
||||
) void {
|
||||
add(&self.counters.passes, 1);
|
||||
const now = std.Io.Clock.real.now(io).toSeconds();
|
||||
const cutoff = now - model.retentionSeconds(self.cfg);
|
||||
const cutoff = now - self.days.seconds();
|
||||
|
||||
// Diagnostics retention rides this pass rather than a schedule of its
|
||||
// own: one daily housekeeping task, and a box restarted every night
|
||||
@@ -268,7 +294,8 @@ test "a pass prunes the rows past the retention window and keeps the rest" {
|
||||
const day = 86_400;
|
||||
try writeRows(&database, &.{ now - 40 * day, now - 31 * day, now - 29 * day, now - 60 });
|
||||
|
||||
var retention: Retention = .init(.{ .retention_days = 30 });
|
||||
var retention_days: RetentionDays = .init(30);
|
||||
var retention: Retention = .init(&retention_days);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
|
||||
@@ -292,17 +319,41 @@ test "the cutoff follows retention_days" {
|
||||
// window of the other.
|
||||
try writeRows(&database, &.{now - 3 * day});
|
||||
|
||||
var keeps: Retention = .init(.{ .retention_days = 7 });
|
||||
var keeps_days: RetentionDays = .init(7);
|
||||
var keeps: Retention = .init(&keeps_days);
|
||||
keeps.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
|
||||
try testing.expectEqual(@as(u64, 0), keeps.snapshotStats().rows_pruned);
|
||||
|
||||
var prunes: Retention = .init(.{ .retention_days = 1 });
|
||||
var prunes_days: RetentionDays = .init(1);
|
||||
var prunes: Retention = .init(&prunes_days);
|
||||
prunes.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
|
||||
try testing.expectEqual(@as(u64, 1), prunes.snapshotStats().rows_pruned);
|
||||
}
|
||||
|
||||
test "setRetentionDays changes the cutoff the next pass prunes by" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
const now = std.Io.Clock.real.now(io).toSeconds();
|
||||
try writeRows(&database, &.{now - 3 * 86_400});
|
||||
|
||||
var days: RetentionDays = .init(7);
|
||||
var pass: Retention = .init(&days);
|
||||
pass.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
|
||||
|
||||
days.setRetentionDays(1);
|
||||
pass.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
|
||||
try testing.expectEqual(@as(u64, 1), pass.snapshotStats().rows_pruned);
|
||||
}
|
||||
|
||||
test "the seventh pass vacuums and the six before it do not" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
@@ -311,7 +362,8 @@ test "the seventh pass vacuums and the six before it do not" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
var retention_days: RetentionDays = .init(30);
|
||||
var retention: Retention = .init(&retention_days);
|
||||
for (0..6) |_| {
|
||||
retention.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums);
|
||||
@@ -340,7 +392,8 @@ test "a gated pass skips the vacuum, counts it, and vacuums on the next pass" {
|
||||
var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null);
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
|
||||
|
||||
var gated: Retention = .init(.{});
|
||||
var gated_days: RetentionDays = .init(30);
|
||||
var gated: Retention = .init(&gated_days);
|
||||
for (0..vacuum_every_passes) |_| gated.runOnce(io, &database, &monitor, null);
|
||||
|
||||
// Prune and checkpoint ran on every pass; only the vacuum was refused.
|
||||
@@ -371,7 +424,8 @@ test "a warn state still allows the vacuum" {
|
||||
var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null);
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.warn), .monotonic);
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
var retention_days: RetentionDays = .init(30);
|
||||
var retention: Retention = .init(&retention_days);
|
||||
for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor, null);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
|
||||
@@ -386,7 +440,8 @@ test "a pass over an empty database still counts" {
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
var retention_days: RetentionDays = .init(30);
|
||||
var retention: Retention = .init(&retention_days);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
|
||||
@@ -410,7 +465,8 @@ test "a failing prune counts the pass and leaves the rows alone" {
|
||||
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
||||
);
|
||||
|
||||
var retention: Retention = .init(.{ .retention_days = 30 });
|
||||
var retention_days: RetentionDays = .init(30);
|
||||
var retention: Retention = .init(&retention_days);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
|
||||
@@ -435,7 +491,8 @@ test "the next pass retries what the failed one could not do" {
|
||||
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
||||
);
|
||||
|
||||
var retention: Retention = .init(.{ .retention_days = 30 });
|
||||
var retention_days: RetentionDays = .init(30);
|
||||
var retention: Retention = .init(&retention_days);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
|
||||
|
||||
@@ -466,7 +523,8 @@ test "a failing prune opens a maintenance episode the next clean pass closes" {
|
||||
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
||||
);
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
var retention_days: RetentionDays = .init(30);
|
||||
var retention: Retention = .init(&retention_days);
|
||||
retention.runOnce(io, &database, null, &fx.store);
|
||||
|
||||
// Only the prune failed; the checkpoint succeeded, and a success writes no
|
||||
@@ -501,7 +559,8 @@ test "a gated vacuum is a maintenance failure the next ungated pass closes" {
|
||||
var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null);
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
var retention_days: RetentionDays = .init(30);
|
||||
var retention: Retention = .init(&retention_days);
|
||||
for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor, &fx.store);
|
||||
|
||||
try testing.expectEqualStrings("vacuum", try fx.text(
|
||||
@@ -536,7 +595,8 @@ test "a pass prunes the diagnostics store once" {
|
||||
fx.store.reportResolved(io, stale, .query_log_recreated, "one-shot", "one-shot", .warning, "aside kept");
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
var retention_days: RetentionDays = .init(30);
|
||||
var retention: Retention = .init(&retention_days);
|
||||
retention.runOnce(io, &database, null, &fx.store);
|
||||
|
||||
try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
|
||||
Reference in New Issue
Block a user