mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 13 Activity

milestone 8: web server, rest api, sse, auth, metrics and static assets

Browse Source
This commit is contained in:
mokhtar
2026-08-02 00:54:13 +02:00
parent a8092bb1b9
commit 5253c47303
59 changed files with 19640 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/storage/retention.zig
+57 -27
View File
@@ -26,6 +26,7 @@ pub const vacuum_every_passes = 7;
/// expresses, so a finer schedule would prune nothing new.
pub const pass_interval_s = 86_400;
/// A consistent copy of the counters, for `/metrics` and the health rollup.
pub const Stats = struct {
passes: u64 = 0,
rows_pruned: u64 = 0,
@@ -33,12 +34,34 @@ pub const Stats = struct {
vacuums: u64 = 0,
};
/// The live counters. Atomic because the retention task writes them and the web
/// task reads them, on different threads, with no lock between the two — the
/// same shape the query logger uses for its own counters.
const Counters = struct {
passes: std.atomic.Value(u64) = .init(0),
rows_pruned: std.atomic.Value(u64) = .init(0),
checkpoints: std.atomic.Value(u64) = .init(0),
vacuums: std.atomic.Value(u64) = .init(0),
};
pub const Retention = struct {
cfg: model.Logging,
stats: Stats,
counters: Counters,
pub fn init(cfg: model.Logging) Retention {
return .{ .cfg = cfg, .stats = .{} };
return .{ .cfg = cfg, .counters = .{} };
}
/// The four counters, read one at a time. A scrape that lands mid-pass can
/// see a pass counted before the rows it pruned are; the alternative is a
/// lock on the pass itself, which buys a consistency no consumer needs.
pub fn snapshotStats(self: *const Retention) Stats {
return .{
.passes = self.counters.passes.load(.monotonic),
.rows_pruned = self.counters.rows_pruned.load(.monotonic),
.checkpoints = self.counters.checkpoints.load(.monotonic),
.vacuums = self.counters.vacuums.load(.monotonic),
};
}
/// One pass: prune, checkpoint, and on every seventh pass vacuum.
@@ -53,29 +76,36 @@ pub const Retention = struct {
///
/// `database` must be a connection no other task uses; see `run`.
pub fn runOnce(self: *Retention, io: std.Io, database: *db.Db) void {
self.stats.passes += 1;
const pass = add(&self.counters.passes, 1) + 1;
const cutoff = std.Io.Clock.real.now(io).toSeconds() - model.retentionSeconds(self.cfg);
if (queries_repo.pruneOlderThan(database, cutoff)) |deleted| {
self.stats.rows_pruned += @intCast(deleted);
_ = add(&self.counters.rows_pruned, @intCast(deleted));
} else |err| {
log.warn("retention prune before {d} failed: {s}", .{ cutoff, @errorName(err) });
}
if (queries_repo.checkpointTruncate(database)) {
self.stats.checkpoints += 1;
_ = add(&self.counters.checkpoints, 1);
} else |err| {
log.warn("retention checkpoint failed: {s}", .{@errorName(err)});
}
if (self.stats.passes % vacuum_every_passes != 0) return;
if (pass % vacuum_every_passes != 0) return;
if (queries_repo.vacuum(database)) {
self.stats.vacuums += 1;
_ = add(&self.counters.vacuums, 1);
} else |err| {
log.warn("retention vacuum failed: {s}", .{@errorName(err)});
}
}
/// Returns the value before the addition, which is what the pass counter
/// needs: only this task increments it, so `previous + 1` is this pass's
/// number.
fn add(counter: *std.atomic.Value(u64), delta: u64) u64 {
return counter.fetchAdd(delta, .monotonic);
}
/// Daily loop, first pass immediately. Phase 7 starts it.
///
/// `boot` rather than `awake`: a box that suspends overnight must still see
@@ -159,10 +189,10 @@ test "a pass prunes the rows past the retention window and keeps the rest" {
retention.runOnce(io, &database);
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 1), retention.stats.passes);
try testing.expectEqual(@as(u64, 2), retention.stats.rows_pruned);
try testing.expectEqual(@as(u64, 1), retention.stats.checkpoints);
try testing.expectEqual(@as(u64, 0), retention.stats.vacuums);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().rows_pruned);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().checkpoints);
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums);
}
test "the cutoff follows retention_days" {
@@ -182,12 +212,12 @@ test "the cutoff follows retention_days" {
var keeps: Retention = .init(.{ .retention_days = 7 });
keeps.runOnce(io, &database);
try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 0), keeps.stats.rows_pruned);
try testing.expectEqual(@as(u64, 0), keeps.snapshotStats().rows_pruned);
var prunes: Retention = .init(.{ .retention_days = 1 });
prunes.runOnce(io, &database);
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 1), prunes.stats.rows_pruned);
try testing.expectEqual(@as(u64, 1), prunes.snapshotStats().rows_pruned);
}
test "the seventh pass vacuums and the six before it do not" {
@@ -201,17 +231,17 @@ test "the seventh pass vacuums and the six before it do not" {
var retention: Retention = .init(.{});
for (0..6) |_| {
retention.runOnce(io, &database);
try testing.expectEqual(@as(u64, 0), retention.stats.vacuums);
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums);
}
retention.runOnce(io, &database);
try testing.expectEqual(@as(u64, 7), retention.stats.passes);
try testing.expectEqual(@as(u64, 1), retention.stats.vacuums);
try testing.expectEqual(@as(u64, 7), retention.stats.checkpoints);
try testing.expectEqual(@as(u64, 7), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
try testing.expectEqual(@as(u64, 7), retention.snapshotStats().checkpoints);
for (0..7) |_| retention.runOnce(io, &database);
try testing.expectEqual(@as(u64, 14), retention.stats.passes);
try testing.expectEqual(@as(u64, 2), retention.stats.vacuums);
try testing.expectEqual(@as(u64, 14), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().vacuums);
}
test "a pass over an empty database still counts" {
@@ -225,9 +255,9 @@ test "a pass over an empty database still counts" {
var retention: Retention = .init(.{});
retention.runOnce(io, &database);
try testing.expectEqual(@as(u64, 1), retention.stats.passes);
try testing.expectEqual(@as(u64, 0), retention.stats.rows_pruned);
try testing.expectEqual(@as(u64, 1), retention.stats.checkpoints);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().rows_pruned);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().checkpoints);
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
}
@@ -250,10 +280,10 @@ test "a failing prune counts the pass and leaves the rows alone" {
retention.runOnce(io, &database);
try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 1), retention.stats.passes);
try testing.expectEqual(@as(u64, 0), retention.stats.rows_pruned);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().rows_pruned);
// The checkpoint runs whether or not the prune did.
try testing.expectEqual(@as(u64, 1), retention.stats.checkpoints);
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().checkpoints);
}
test "the next pass retries what the failed one could not do" {
@@ -279,6 +309,6 @@ test "the next pass retries what the failed one could not do" {
retention.runOnce(io, &database);
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
try testing.expectEqual(@as(u64, 2), retention.stats.passes);
try testing.expectEqual(@as(u64, 2), retention.stats.rows_pruned);
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().passes);
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().rows_pruned);
}