mokhtar/nxdns
1
0
Fork 0
Code Pull Requests Actions Packages Releases 25 Activity

storage: version querylog.db and migrate it in place, never reset a healthy file

Browse Source 
querylog.db carries a schema version; migrations run at startup as one transaction after a vacuumed 0600 backup, and every failure refuses startup (exit 2, no systemd restart loop) instead of starting empty. corruption is the only automatic recreate left. the cut gate now requires a fixture-proven migration or an explicit versioned break with restore instructions, and locks shipped migration files and fixtures byte-for-byte.
This commit is contained in:
mokhtar
2026-08-28 17:56:19 +02:00
parent dd5a9f6ab8
commit 272655f60c
25 changed files with 4312 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app.zig
+59 -63
View File
@@ -58,6 +58,7 @@ const model = @import("config/model.zig");
const pause = @import("server/pause.zig");
const queries_repo = @import("storage/repositories/queries_repo.zig");
const query_sink = @import("server/query_sink.zig");
const querylog_migrations = @import("storage/querylog_migrations.zig");
const querylog_schema = @import("storage/querylog_schema.zig");
const rate_limiter = @import("server/rate_limiter.zig");
const reconcile = @import("config/reconcile.zig");
@@ -1316,9 +1317,9 @@ test "the recreated detail names the aside and the new coverage start" {
var buf: [events.Store.max_detail_len]u8 = undefined;
try std.testing.expectEqualStrings(
"previous file kept as 'querylog.db.schema-changed-1700000000'; " ++
"previous file kept as 'querylog.db.quick-check-failed-1700000000'; " ++
"query history is available from 1700000001",
recreatedDetail(&buf, "querylog.db.schema-changed-1700000000", 1700000001),
recreatedDetail(&buf, "querylog.db.quick-check-failed-1700000000", 1700000001),
);
// A fresh file that will not answer is a separate failure; the line still
@@ -1437,7 +1438,7 @@ const m29_ddl: [:0]const u8 =
\\VALUES (1, unixepoch(), unixepoch() + 1);
;
test "an m29 query log is set aside and recreated without the upstream-history tables" {
test "an m29 query log refuses the startup and is left exactly as it is" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
@@ -1448,10 +1449,6 @@ test "an m29 query log is set aside and recreated without the upstream-history t
var path_buf: [256]u8 = undefined;
const path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/querylog.db", .{tmp.sub_path});
var fx: events_fixture.Fixture = .{};
try fx.init(io, 1000);
defer fx.deinit();
// The fixture is only worth anything while it is still a *different*
// schema from this build's, and one that carries the deleted tables.
try testing.expect(!std.mem.eql(u8, m29_ddl, querylog_schema.ddl));
@@ -1461,9 +1458,8 @@ test "an m29 query log is set aside and recreated without the upstream-history t
// edit to the literal cannot satisfy by changing what it is compared to.
try testing.expectEqual(m29_fingerprint, @as(i32, @bitCast(std.hash.Crc32.hash(m29_ddl))));
// A healthy m29 file, stamped with the fingerprint m29's own DDL produced
// and backdated so its coverage promise is visibly the older one.
const m29_coverage = blk: {
// A healthy m29 file, stamped with the fingerprint m29's own DDL produced.
{
var m29 = try db.Db.open(path, .{ .mode = .read_write_create });
defer m29.close();
try db.applyPragmas(&m29, .{});
@@ -1481,43 +1477,47 @@ test "an m29 query log is set aside and recreated without the upstream-history t
"PRAGMA user_version = {d};",
.{m29_fingerprint},
));
break :blk try m29.queryInt("SELECT available_since FROM querylog_meta");
};
try testing.expectEqual(m29_available_since, m29_coverage);
var opened = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer opened.database.close();
// Set aside under the name that says the file was healthy and this build
// moved, and still on disk for an operator who wants it.
try testing.expectEqual(querylog_schema.RecreateReason.fingerprint_mismatch, opened.recreated.?);
try testing.expect(std.mem.indexOf(u8, opened.aside(), ".schema-changed-") != null);
try tmp.dir.access(io, std.fs.path.basename(opened.aside()), .{});
// The two tables are gone from the file this process will write to.
for ([_][]const u8{ "upstream_targets", "upstream_minute", "idx_upstream_minute_ts" }) |name| {
var stmt = try opened.database.prepare("SELECT count(*) FROM sqlite_schema WHERE name = ?1");
defer stmt.deinit();
try stmt.bindText(1, name);
try testing.expect(try stmt.step());
try testing.expectEqual(@as(i64, 0), stmt.columnInt(0));
}
// Coverage restarts: the new file does not inherit the replaced one's
// promise about what it can answer. Strictly newer, not merely not-older —
// a recreation that copied the watermark across would pass the weaker test.
const coverage = try queries_repo.availableSince(&opened.database);
try testing.expect(coverage > m29_coverage);
// m29 predates the version stamp entirely: its `user_version` is a CRC of a
// schema no migration chain starts from, so the only honest answer is to
// refuse and say so. The pre-0.0.12 contract — set it aside and start empty
// — is gone.
querylog_migrations.expected_failures.begin();
defer querylog_migrations.expected_failures.end();
try testing.expectError(
error.SchemaUnsupported,
querylog_schema.open(io, std.Io.Dir.cwd(), path),
);
reportQuerylogRecreated(&fx.store, io, 2000, &opened, &opened.database);
try testing.expectEqualStrings("query_log.recreated", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings(
"fingerprint_mismatch",
try fx.text("SELECT subject_key FROM operational_events"),
// Nothing was renamed, nothing was created, and the file still answers for
// itself: the operator can downgrade and keep the history.
var entries: usize = 0;
var it = tmp.dir.iterate();
while (try it.next(io)) |entry| {
try testing.expect(std.mem.startsWith(u8, entry.name, "querylog.db"));
try testing.expect(std.mem.indexOfScalar(u8, entry.name[10..], '.') == null);
entries += 1;
}
try testing.expect(entries >= 1);
var reopened = try db.Db.open(path, .{ .mode = .read_write_existing });
defer reopened.close();
try testing.expectEqual(
@as(i64, m29_fingerprint),
try reopened.queryInt("PRAGMA user_version"),
);
try testing.expectEqual(
m29_available_since,
try reopened.queryInt("SELECT available_since FROM querylog_meta"),
);
try testing.expectEqual(
@as(i64, 1),
try reopened.queryInt("SELECT count(*) FROM upstream_targets"),
);
}
test "a fingerprint recreate files a resolved event naming the real aside and watermark" {
test "a recreate files a resolved event naming the real aside and watermark" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
@@ -1539,27 +1539,17 @@ test "a fingerprint recreate files a resolved event naming the real aside and wa
created.database.close();
try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
// A healthy file this build's DDL no longer matches, which is what an
// upgrade that edits the schema produces.
{
var stamped = try db.Db.open(path, .{ .mode = .read_write_existing });
defer stamped.close();
var sql_buf: [64]u8 = undefined;
try stamped.exec(try std.fmt.bufPrintZ(
&sql_buf,
"PRAGMA user_version = {d};",
.{querylog_schema.fingerprint +% 1},
));
}
// Real damage: corruption is the only thing that recreates now.
try tmp.dir.writeFile(io, .{ .sub_path = "querylog.db", .data = "not a database at all" });
var recreated = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer recreated.database.close();
try testing.expectEqual(querylog_schema.RecreateReason.fingerprint_mismatch, recreated.recreated.?);
try testing.expectEqual(querylog_schema.RecreateReason.not_a_database, recreated.recreated.?);
reportQuerylogRecreated(&fx.store, io, 2000, &recreated, &recreated.database);
try testing.expectEqualStrings("query_log.recreated", try fx.text("SELECT code FROM operational_events"));
try testing.expectEqualStrings("fingerprint_mismatch", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqualStrings("not_a_database", try fx.text("SELECT subject_key FROM operational_events"));
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
// One-shot: already over when it is filed, so it never becomes an open
// episode `/api/health` counts.
@@ -1612,18 +1602,17 @@ test "a recreate under a long data directory keeps the watermark and a usable na
{
var created = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer created.database.close();
var sql_buf: [64]u8 = undefined;
try created.database.exec(try std.fmt.bufPrintZ(
&sql_buf,
"PRAGMA user_version = {d};",
.{querylog_schema.fingerprint +% 1},
));
created.database.close();
}
{
var deep_dir = try tmp.dir.openDir(io, nested, .{});
defer deep_dir.close(io);
try deep_dir.writeFile(io, .{ .sub_path = "querylog.db", .data = "not a database at all" });
}
var recreated = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer recreated.database.close();
try testing.expectEqual(querylog_schema.RecreateReason.fingerprint_mismatch, recreated.recreated.?);
try testing.expectEqual(querylog_schema.RecreateReason.not_a_database, recreated.recreated.?);
const line_overhead = "previous file kept as ''; query history is available from ".len;
try testing.expect(recreated.aside().len + line_overhead > events.Store.max_detail_len);
@@ -1682,6 +1671,13 @@ test "run maps a rejected configuration to exit 2 and everything else to exit 1"
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.ParseZon));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.NoUsableUpstreams));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.BadCertificate));
// The query-log schema refusals: `run` is the only command that reaches
// them, and exit 1 would put a deliberate refusal under the unit's
// `Restart=on-failure`.
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.SchemaTooNew));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.SchemaUnsupported));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.MigrationFailed));
try std.testing.expectEqual(cli.exit_check, failureExitCode(error.MigrationBackupFailed));
try std.testing.expectEqual(cli.exit_runtime, failureExitCode(error.AccessDenied));
try std.testing.expectEqual(cli.exit_runtime, failureExitCode(error.OutOfMemory));
}