mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 13 Activity

milestone 30: overview as a dashboard, explicit health contract, period aggregations
Gates / frontend (push) Successful in 1m32s
Details
Gates / test (push) Successful in 1m54s
Details
Gates / package (push) Successful in 5m28s
Details
Gates / container (push) Successful in 14s
Details
Gates / test-aarch64 (push) Failing after 3h10m0s
Details
CI / gates (push) Failing after 3h11m55s
Details

Browse Source
This commit is contained in:
mokhtar
2026-08-22 16:45:15 +02:00
parent 17422fac21
commit 648d9b4496
89 changed files with 7222 additions and 4239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/storage/retention.zig
+2 -102
View File
@@ -16,7 +16,6 @@ const disk_monitor = @import("disk_monitor.zig");
const events = @import("events.zig");
const model = @import("../config/model.zig");
const queries_repo = @import("repositories/queries_repo.zig");
const upstream_history_repo = @import("repositories/upstream_history_repo.zig");
const log = std.log.scoped(.retention);
@@ -33,10 +32,6 @@ pub const pass_interval_s = 86_400;
pub const Stats = struct {
passes: u64 = 0,
rows_pruned: u64 = 0,
/// Upstream-history minute rows, counted apart from `rows_pruned`: that
/// counter is the query log's, and an operator watching it must not see it
/// move because a different table was tidied.
upstream_rows_pruned: u64 = 0,
checkpoints: u64 = 0,
vacuums: u64 = 0,
/// Vacuums the disk monitor refused. The pass still pruned and
@@ -50,7 +45,6 @@ pub const Stats = struct {
const Counters = struct {
passes: std.atomic.Value(u64) = .init(0),
rows_pruned: std.atomic.Value(u64) = .init(0),
upstream_rows_pruned: std.atomic.Value(u64) = .init(0),
checkpoints: std.atomic.Value(u64) = .init(0),
vacuums: std.atomic.Value(u64) = .init(0),
vacuums_gated: std.atomic.Value(u64) = .init(0),
@@ -74,7 +68,6 @@ pub const Retention = struct {
return .{
.passes = self.counters.passes.load(.monotonic),
.rows_pruned = self.counters.rows_pruned.load(.monotonic),
.upstream_rows_pruned = self.counters.upstream_rows_pruned.load(.monotonic),
.checkpoints = self.counters.checkpoints.load(.monotonic),
.vacuums = self.counters.vacuums.load(.monotonic),
.vacuums_gated = self.counters.vacuums_gated.load(.monotonic),
@@ -124,22 +117,6 @@ pub const Retention = struct {
maintenance(store, io, now, "prune", @errorName(err));
}
// Before the vacuum-cadence logic below, which returns early on six
// passes out of seven and again whenever the monitor refuses the
// vacuum. A step placed after it would almost never run.
//
// `logging.retention_days` is not the window here: these are per-minute
// aggregates whose whole purpose is to outlive the per-query rows, so
// the window is the repository's own constant.
const history_cutoff = now - upstream_history_repo.retention_window_s;
if (upstream_history_repo.pruneOlderThan(database, history_cutoff)) |deleted| {
add(&self.counters.upstream_rows_pruned, @intCast(deleted));
maintenance(store, io, now, "history_prune", null);
} else |err| {
log.warn("upstream history prune before {d} failed: {s}", .{ history_cutoff, @errorName(err) });
maintenance(store, io, now, "history_prune", @errorName(err));
}
if (queries_repo.checkpointTruncate(database)) {
add(&self.counters.checkpoints, 1);
maintenance(store, io, now, "checkpoint", null);
@@ -443,83 +420,6 @@ test "a failing prune counts the pass and leaves the rows alone" {
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().checkpoints);
}
test "the upstream-history window is its own, and a one-day query log does not shrink it" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const now = std.Io.Clock.real.now(io).toSeconds();
const day = 86_400;
try writeRows(&database, &.{ now - 2 * day, now - 60 });
try upstream_history_repo.flush(&database, &.{
.{ .url = "https://gone.example", .minute_ts = now - 32 * day, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
.{ .url = "https://kept.example", .minute_ts = now - 29 * day, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
// A query log kept for one day, and 30 days of upstream minutes beside it.
var retention: Retention = .init(.{ .retention_days = 1 });
retention.runOnce(io, &database, null, null);
const stats = retention.snapshotStats();
// One query-log row is older than one day; one minute row is older than the
// fixed 31-day upstream window. Each counter moved by its own amount.
try testing.expectEqual(@as(u64, 1), stats.rows_pruned);
try testing.expectEqual(@as(u64, 1), stats.upstream_rows_pruned);
try testing.expectEqual(@as(i64, 1), try upstream_history_repo.countMinutes(&database));
// The day-29 row is exactly what a 30-day dashboard window asks for.
const kept = try upstream_history_repo.windowStats(
&database,
"https://kept.example",
now - 30 * day,
now,
);
try testing.expectEqual(@as(u64, 1), kept.attempts);
// And the emptied target went with its rows.
try testing.expectEqual(@as(i64, 1), try upstream_history_repo.countTargets(&database));
}
test "the history prune runs on the passes where the vacuum logic returns early" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
var database = try openLog();
defer database.close();
const now = std.Io.Clock.real.now(io).toSeconds();
const old_minute = now - 40 * 86_400;
// The first pass: `passes_since_vacuum` is 1, so the vacuum block returns
// before it does anything. A prune placed after that block would never run
// on six passes out of seven.
try upstream_history_repo.flush(&database, &.{
.{ .url = "https://a.example", .minute_ts = old_minute, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
var early: Retention = .init(.{});
early.runOnce(io, &database, null, null);
try testing.expectEqual(@as(u64, 1), early.snapshotStats().upstream_rows_pruned);
try testing.expectEqual(@as(i64, 0), try upstream_history_repo.countMinutes(&database));
// The gated pass: the disk monitor refuses the vacuum and that branch
// returns too, and the prune still has to have happened before it.
var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null);
monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
var gated: Retention = .init(.{});
for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor, null);
try upstream_history_repo.flush(&database, &.{
.{ .url = "https://b.example", .minute_ts = old_minute, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
});
gated.runOnce(io, &database, &monitor, null);
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums_gated);
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().upstream_rows_pruned);
try testing.expectEqual(@as(i64, 0), try upstream_history_repo.countMinutes(&database));
}
test "the next pass retries what the failed one could not do" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
@@ -569,8 +469,8 @@ test "a failing prune opens a maintenance episode the next clean pass closes" {
var retention: Retention = .init(.{});
retention.runOnce(io, &database, null, &fx.store);
// Only the prune failed; checkpoint and history prune succeeded, and a
// success writes no row of its own.
// Only the prune failed; the checkpoint succeeded, and a success writes no
// row of its own.
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
try testing.expectEqualStrings("query_log.maintenance", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("prune", try fx.text("SELECT subject_key FROM operational_events"));