storage: version querylog.db and migrate it in place, never reset a healthy file
Gates / frontend (push) Successful in 2m8s
Gates / test (push) Successful in 2m46s
Gates / test-aarch64 (push) Successful in 8m38s
Gates / package (push) Successful in 4m39s
Gates / container (push) Successful in 15s
CI / gates (push) Successful in 31m58s

querylog.db carries a schema version; migrations run at startup as one transaction after a vacuumed 0600 backup, and every failure refuses startup (exit 2, no systemd restart loop) instead of starting empty. corruption is the only automatic recreate left. the cut gate now requires a fixture-proven migration or an explicit versioned break with restore instructions, and locks shipped migration files and fixtures byte-for-byte.
This commit is contained in:
2026-08-28 17:56:19 +02:00
parent c9701fae85
commit 0f01c2fbd7
25 changed files with 4312 additions and 176 deletions
+183
View File
@@ -54,9 +54,24 @@ pub const c = struct {
pub extern fn sqlite3_column_count(stmt: *c.Stmt) c_int;
pub extern fn sqlite3_column_type(stmt: *c.Stmt, col: c_int) c_int;
pub extern fn sqlite3_column_int64(stmt: *c.Stmt, col: c_int) i64;
pub extern fn sqlite3_column_double(stmt: *c.Stmt, col: c_int) f64;
pub extern fn sqlite3_column_text(stmt: *c.Stmt, col: c_int) ?[*]const u8;
pub extern fn sqlite3_column_bytes(stmt: *c.Stmt, col: c_int) c_int;
pub extern fn sqlite3_set_authorizer(db: *Sqlite3, xAuth: ?*const AuthCallback, user_data: ?*anyopaque) c_int;
pub extern fn sqlite3_get_autocommit(db: *Sqlite3) c_int;
pub extern fn sqlite3_last_insert_rowid(db: *Sqlite3) i64;
/// `int (*)(void*, int, const char*, const char*, const char*, const char*)`.
/// The four name arguments are NULL for actions that do not use them, which
/// `SQLITE_TRANSACTION` is except for its operation name.
pub const AuthCallback = fn (
user_data: ?*anyopaque,
action: c_int,
arg1: ?[*:0]const u8,
arg2: ?[*:0]const u8,
database: ?[*:0]const u8,
trigger_or_view: ?[*:0]const u8,
) callconv(.c) c_int;
pub extern fn sqlite3_changes(db: *Sqlite3) c_int;
pub extern fn sqlite3_total_changes(db: *Sqlite3) c_int;
};
@@ -105,6 +120,20 @@ pub const open_flag = struct {
pub const exrescode: c_int = 0x2000000;
};
/// The authorizer verdicts and the one action code nxdns denies, from the
/// vendored `sqlite3.h` (3.53.4). `SQLITE_DENY` fails the *prepare* with
/// `SQLITE_AUTH`, which is what makes it a real guard: the statement never
/// runs at all.
pub const auth = struct {
pub const deny: c_int = 1;
pub const transaction: c_int = 22;
/// `SAVEPOINT`, `RELEASE` and `ROLLBACK TO` report under this code, not
/// under `transaction`. A savepoint at the outermost level opens a real
/// transaction and `RELEASE` commits it, so a step using one splits the
/// migration exactly as a bare `COMMIT` would.
pub const savepoint: c_int = 32;
};
/// Column type codes returned by `sqlite3_column_type`.
pub const column_type = struct {
pub const integer: c_int = 1;
@@ -367,6 +396,7 @@ pub const Db = struct {
/// message is read back through `sqlite3_errmsg`, so there is no
/// `sqlite3_free` obligation.
pub fn exec(self: *Db, sql: [:0]const u8) Error!void {
if (execFaultTripped(sql)) return error.Internal;
return check(c.sqlite3_exec(self.handle, sql.ptr, null, null, null));
}
@@ -424,6 +454,38 @@ pub const Db = struct {
return value;
}
/// False while a transaction is open on this connection. The migration
/// runner's belt check: a step that somehow ended the runner's transaction
/// must not be allowed to look like a success.
pub fn inTransaction(self: *Db) bool {
return c.sqlite3_get_autocommit(self.handle) == 0;
}
/// Denies every transaction statement — `BEGIN`, `COMMIT`, `ROLLBACK`,
/// `SAVEPOINT`, `RELEASE` — until `clearAuthorizer` runs.
///
/// `guard` must outlive the installed window: SQLite keeps the pointer.
/// Install and clear are a scoped pair; the migration runner clears on
/// every exit path, because a leaked authorizer would go on denying the
/// ROLLBACK that cleans up after the very statement it rejected.
pub fn denyTransactions(self: *Db, guard: *TransactionGuard) Error!void {
guard.* = .{};
return check(c.sqlite3_set_authorizer(self.handle, transactionAuthorizer, guard));
}
/// Never fails in a way the caller can act on: passing a null callback only
/// clears state SQLite already holds. A failure is logged and swallowed so
/// this stays usable in `defer`.
pub fn clearAuthorizer(self: *Db) void {
const rc = c.sqlite3_set_authorizer(self.handle, null, null);
if (rc != result.ok) {
log.err("sqlite3_set_authorizer(null) returned {s} (code {d})", .{
std.mem.span(c.sqlite3_errstr(rc)),
rc,
});
}
}
pub fn lastInsertRowid(self: *Db) i64 {
return c.sqlite3_last_insert_rowid(self.handle);
}
@@ -442,6 +504,34 @@ pub const Db = struct {
}
};
/// Records whether the authorizer installed by `Db.denyTransactions` actually
/// rejected anything. The rejection reaches the caller as `error.Auth`, which
/// is indistinguishable from any other authorization failure; this flag is what
/// lets the migration runner name the real cause.
pub const TransactionGuard = struct {
denied: bool = false,
};
fn transactionAuthorizer(
user_data: ?*anyopaque,
action: c_int,
arg1: ?[*:0]const u8,
arg2: ?[*:0]const u8,
database: ?[*:0]const u8,
trigger_or_view: ?[*:0]const u8,
) callconv(.c) c_int {
_ = arg2;
_ = database;
_ = trigger_or_view;
if (action != auth.transaction and action != auth.savepoint) return result.ok;
const guard: *TransactionGuard = @ptrCast(@alignCast(user_data.?));
guard.denied = true;
log.warn("migration step attempted a transaction statement: {s}", .{
if (arg1) |op| std.mem.span(op) else "(unnamed)",
});
return auth.deny;
}
fn openHandle(filename: [:0]const u8, flags: c_int) Error!*c.Sqlite3 {
var handle: ?*c.Sqlite3 = null;
const rc = c.sqlite3_open_v2(filename.ptr, &handle, flags, null);
@@ -963,6 +1053,45 @@ pub const read_tx_faults = if (builtin.is_test) struct {
}
} else struct {};
/// Fails one chosen `Db.exec` so a test can drive a failure SQLite itself will
/// not produce on demand. Test builds only, same shape as `read_tx_seam`.
///
/// The migration paths this exists for — the legacy restamp and the
/// post-commit pragma restore — run statements that always succeed against a
/// healthy file, and their recovery behaviour is the whole point of the
/// milestone. Matching on the SQL text rather than counting calls keeps a test
/// naming the statement it means.
const exec_seam = if (builtin.is_test) struct {
var fail_matching: ?[]const u8 = null;
} else struct {};
fn execFaultTripped(sql: []const u8) bool {
if (!builtin.is_test) return false;
const needle = exec_seam.fail_matching orelse return false;
if (std.mem.indexOf(u8, sql, needle) == null) return false;
exec_seam.fail_matching = null;
return true;
}
/// The seam's controls, for tests in this file and in the storage layer.
pub const exec_faults = if (builtin.is_test) struct {
/// Arms the next `Db.exec` whose SQL contains `needle` to fail with
/// `error.Internal` before the statement reaches SQLite. One shot: it
/// disarms itself when it trips. Pair it with `defer disarm()` so a test
/// that never trips the fault cannot leak it into the next one.
pub fn failNextMatching(needle: []const u8) void {
exec_seam.fail_matching = needle;
}
pub fn disarm() void {
exec_seam.fail_matching = null;
}
pub fn armed() bool {
return exec_seam.fail_matching != null;
}
} else struct {};
const testing = std.testing;
fn openMemory() Error!Db {
@@ -1479,3 +1608,57 @@ test "a duplicate insert into a UNIQUE column returns error.Constraint" {
try stmt.bindText(1, "only");
try testing.expectError(error.Constraint, stmt.step());
}
test "the transaction authorizer denies transaction statements and clears cleanly" {
var db = try openMemory();
defer db.close();
try db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY);");
try db.exec("BEGIN IMMEDIATE;");
try testing.expect(db.inTransaction());
var guard: TransactionGuard = .{};
try db.denyTransactions(&guard);
// Ordinary work still runs: only the transaction statements are refused.
try db.exec("INSERT INTO t (id) VALUES (1);");
try testing.expect(!guard.denied);
try testing.expectError(error.Auth, db.exec("COMMIT;"));
try testing.expect(guard.denied);
// The deny happens at prepare, so the transaction is still open.
try testing.expect(db.inTransaction());
// SAVEPOINT and its RELEASE report under a different action code, and they
// are the same bypass: at the outermost level they are a transaction under
// another name, and inside one they can still discard the migration's work.
guard.denied = false;
try testing.expectError(error.Auth, db.exec("SAVEPOINT half_a_migration;"));
try testing.expect(guard.denied);
guard.denied = false;
try testing.expectError(error.Auth, db.exec("RELEASE half_a_migration;"));
try testing.expect(guard.denied);
try testing.expect(db.inTransaction());
db.clearAuthorizer();
// The same connection is usable again — a leaked authorizer would strand it
// inside the transaction by denying this too.
try db.exec("ROLLBACK;");
try testing.expect(!db.inTransaction());
try testing.expectEqual(@as(i64, 0), try db.queryInt("SELECT count(*) FROM t"));
}
test "the exec fault seam fires once, on the statement it names" {
var db = try openMemory();
defer db.close();
try db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY);");
exec_faults.failNextMatching("COMMIT");
defer exec_faults.disarm();
try db.exec("BEGIN IMMEDIATE;");
try db.exec("INSERT INTO t (id) VALUES (1);");
try testing.expectError(error.Internal, db.exec("COMMIT;"));
try testing.expect(!exec_faults.armed());
// Disarmed: the retry is a real COMMIT.
try db.exec("COMMIT;");
try testing.expectEqual(@as(i64, 1), try db.queryInt("SELECT count(*) FROM t"));
}
+2 -1
View File
@@ -27,6 +27,7 @@ const disk_monitor = @import("disk_monitor.zig");
const logger = @import("logger.zig");
const queries_repo = @import("repositories/queries_repo.zig");
const querylog_schema = @import("querylog_schema.zig");
const querylog_versions = @import("querylog_versions.zig");
const retention = @import("retention.zig");
const testing = std.testing;
@@ -233,7 +234,7 @@ test "S8 case 1: the logger writes a real querylog.db end to end" {
try testing.expectEqual(@as(i64, 250), try queries_repo.countRows(log_db.database()));
try testing.expectEqual(@as(i64, 10), try queries_repo.countDomains(log_db.database()));
try testing.expectEqual(
@as(i64, querylog_schema.fingerprint),
@as(i64, querylog_versions.current_version),
try log_db.database().queryInt("PRAGMA user_version"),
);
}
+535
View File
@@ -0,0 +1,535 @@
//! The shipped `querylog.db` fixtures, and the proof that every supported
//! schema version reaches the current one with the operator's rows intact.
//!
//! A file of its own, not a section of `querylog_migrations.zig`, because of the
//! link contract that split `querylog_versions.zig` out in the first place. The
//! assertions here need `repositories/queries_repo.zig`'s projection-coherence
//! oracle, and that file reaches across `src/` for the config and filter types;
//! importing it from `querylog_migrations.zig` would pull all of it into the
//! module `tools/cut.zig` builds `querylog_schema.zig` as, where those paths lie
//! outside the module root and do not compile.
//!
//! The fixtures themselves — `testdata/querylog-v<N>-{schema,data}.sql` — are
//! immutable once released. Every version in `[minimum_supported_version,
//! current_version]` has a pair: the current version's pair is the next
//! migration's starting point, and an explicit break ships the new baseline.
const std = @import("std");
const db = @import("db.zig");
const migrations = @import("querylog_migrations.zig");
const queries_repo = @import("repositories/queries_repo.zig");
const querylog_schema = @import("querylog_schema.zig");
const versions = @import("querylog_versions.zig");
const testing = std.testing;
/// A temporary directory and the `querylog.db` path inside it. Deliberately not
/// `querylog_migrations.zig`'s test harness: that one builds synthetic schemas,
/// while everything here starts from the shipped fixture files.
const Harness = struct {
threaded: std.Io.Threaded,
tmp: std.testing.TmpDir,
buf: [256]u8 = undefined,
fn init() Harness {
return .{
.threaded = .init(testing.allocator, .{}),
.tmp = testing.tmpDir(.{ .iterate = true }),
};
}
fn deinit(self: *Harness) void {
self.tmp.cleanup();
self.threaded.deinit();
}
fn io(self: *Harness) std.Io {
return self.threaded.io();
}
fn path(self: *Harness) [:0]const u8 {
return std.fmt.bufPrintZ(&self.buf, ".zig-cache/tmp/{s}/querylog.db", .{self.tmp.sub_path}) catch
unreachable;
}
fn openLive(self: *Harness) !db.Db {
var database = try db.Db.open(self.path(), .{ .mode = .read_write_existing });
errdefer database.close();
try db.applyPragmas(&database, .{});
return database;
}
};
/// One shipped schema version's frozen pair. Both halves are immutable once
/// released — the release cut byte-compares them against the previous tag — and
/// every version in `[minimum_supported_version, current_version]` must have a
/// pair, which the cut gate also enforces.
const Fixture = struct {
version: i32,
schema: [:0]const u8,
data: [:0]const u8,
};
const fixtures = [_]Fixture{
.{
.version = 1,
.schema = @embedFile("testdata/querylog-v1-schema.sql"),
.data = @embedFile("testdata/querylog-v1-data.sql"),
},
};
fn fixtureFor(version: i32) ?Fixture {
for (fixtures) |fixture| {
if (fixture.version == version) return fixture;
}
return null;
}
/// Writes a fixture pair to `path` and stamps it. `stamp` is a parameter rather
/// than `fixture.version` because the legacy-fingerprint file is the same
/// version-1 bytes under a different stamp.
fn writeFixture(path: [:0]const u8, fixture: Fixture, stamp: i32) !void {
var database = try db.Db.open(path, .{ .mode = .read_write_create });
defer database.close();
try db.applyPragmas(&database, .{});
try database.exec(fixture.schema);
try database.exec(fixture.data);
var buf: [64]u8 = undefined;
const sql = std.fmt.bufPrintZ(&buf, "PRAGMA user_version = {d};", .{stamp}) catch unreachable;
try database.exec(sql);
}
/// A second path in the harness's directory, for the reference databases the
/// assertions below compare against.
fn sidePath(h: *Harness, buf: []u8, name: []const u8) [:0]const u8 {
return std.fmt.bufPrintZ(buf, ".zig-cache/tmp/{s}/{s}", .{ h.tmp.sub_path, name }) catch unreachable;
}
/// A database holding nothing but the current `ddl`, which is what a file
/// created by this build is.
fn openFreshCurrent(h: *Harness, buf: []u8) !db.Db {
var database = try db.Db.open(sidePath(h, buf, "fresh.db"), .{ .mode = .read_write_create });
errdefer database.close();
try db.applyPragmas(&database, .{});
try database.exec(querylog_schema.ddl);
return database;
}
/// An untouched load of `fixture`, to compare a migrated or opened file against
/// rather than restating the fixture's contents in the assertions.
fn openPristine(h: *Harness, buf: []u8, fixture: Fixture) !db.Db {
const path = sidePath(h, buf, "pristine.db");
try writeFixture(path, fixture, fixture.version);
var database = try db.Db.open(path, .{ .mode = .read_write_existing });
errdefer database.close();
try db.applyPragmas(&database, .{});
return database;
}
/// Every row of every table, as one canonical text. Exact equality is only the
/// right question for a file no migration has reshaped; a migrated file is
/// checked by the counts and the watermark instead.
fn dumpContent(gpa: std.mem.Allocator, database: *db.Db, out: *std.ArrayList(u8)) !void {
var tables: std.ArrayList([]u8) = .empty;
defer freeOwned(gpa, &tables);
{
var stmt = try database.prepare(
\\SELECT name FROM sqlite_schema
\\WHERE type = 'table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\'
\\ORDER BY name
);
defer stmt.deinit();
// Copied out before the per-table statements step: a borrowed
// `columnText` would not survive them.
while (try stmt.step()) try tables.append(gpa, try stmt.columnTextAlloc(gpa, 0));
}
// The lines are sorted here rather than by the query: the four `bucket_*`
// tables are WITHOUT ROWID, so `ORDER BY rowid` is not available to all of
// them and no single column list is.
var lines: std.ArrayList([]u8) = .empty;
defer freeOwned(gpa, &lines);
for (tables.items) |table| {
var sql_buf: [256]u8 = undefined;
const sql = std.fmt.bufPrint(&sql_buf, "SELECT * FROM \"{s}\"", .{table}) catch unreachable;
var stmt = try database.prepare(sql);
defer stmt.deinit();
while (try stmt.step()) {
var line: std.ArrayList(u8) = .empty;
errdefer line.deinit(gpa);
try line.print(gpa, "R|{s}", .{table});
var col: c_int = 0;
const columns: c_int = db.c.sqlite3_column_count(stmt.handle);
while (col < columns) : (col += 1) {
try line.print(gpa, "|{s}", .{stmt.columnTextOrNull(col) orelse "<null>"});
}
try lines.append(gpa, try line.toOwnedSlice(gpa));
}
}
std.mem.sortUnstable([]u8, lines.items, {}, struct {
fn lessThan(_: void, a: []u8, b: []u8) bool {
return std.mem.lessThan(u8, a, b);
}
}.lessThan);
for (lines.items) |line| {
try out.appendSlice(gpa, line);
try out.append(gpa, '\n');
}
}
/// One value, encoded so that no two different values can produce the same
/// bytes: a type tag, the byte length, then the bytes themselves.
///
/// Nothing here is a sentinel and nothing is escaped, which is the point. A
/// serialization that wrote NULL as `<null>` cannot tell a NULL apart from the
/// six-character string of the same name, and one that separated values with
/// `|` cannot tell `a|b` in one column from `a` and `b` in two — so a migration
/// that turned a NULL `upstream` into text, or shifted a value from one column
/// into its neighbour, would compare EQUAL to the original. The length prefix
/// makes the stream uniquely decodable, so equal encodings mean equal rows.
///
/// A float travels as its bit pattern rather than as printed digits: the
/// question here is whether the value survived, not whether it rounds the same.
fn writeValue(gpa: std.mem.Allocator, stmt: *db.Stmt, col: c_int, out: *std.ArrayList(u8)) !void {
switch (db.c.sqlite3_column_type(stmt.handle, col)) {
db.column_type.null_value => try out.print(gpa, "n0:", .{}),
db.column_type.integer => {
var buf: [24]u8 = undefined;
const text = std.fmt.bufPrint(&buf, "{d}", .{stmt.columnInt(col)}) catch unreachable;
try out.print(gpa, "i{d}:{s}", .{ text.len, text });
},
db.column_type.float => {
const bits: u64 = @bitCast(db.c.sqlite3_column_double(stmt.handle, col));
var buf: [24]u8 = undefined;
const text = std.fmt.bufPrint(&buf, "{d}", .{bits}) catch unreachable;
try out.print(gpa, "f{d}:{s}", .{ text.len, text });
},
db.column_type.blob => {
// `columnText` on a blob hands back the same bytes SQLite stores,
// which is what this compares; it is not read as text.
const bytes = stmt.columnText(col);
try out.print(gpa, "b{d}:{s}", .{ bytes.len, bytes });
},
else => {
const bytes = stmt.columnText(col);
try out.print(gpa, "t{d}:{s}", .{ bytes.len, bytes });
},
}
}
/// One query's rows, in the order the query returns them, tagged with `label` so
/// that a difference names the relation it came from. The column count is part
/// of each row for the same reason the lengths are part of each value.
fn dumpQuery(
gpa: std.mem.Allocator,
database: *db.Db,
label: []const u8,
sql: [:0]const u8,
out: *std.ArrayList(u8),
) !void {
var stmt = try database.prepare(sql);
defer stmt.deinit();
while (try stmt.step()) {
const columns: c_int = db.c.sqlite3_column_count(stmt.handle);
try out.print(gpa, "{s}:{d}:", .{ label, columns });
var col: c_int = 0;
while (col < columns) : (col += 1) {
try writeValue(gpa, &stmt, col, out);
}
try out.append(gpa, '\n');
}
}
/// The operator's data as the application means it, in an order a migration
/// cannot permute.
///
/// `q.*` rather than a column list on purpose: a migration that adds a column
/// must show that column here, and a hand-written list would quietly stop
/// covering the table the day it grows. `d.domain` rides along so that the text
/// a row names is compared, not only the id it happens to hold.
const logical_relations = [_]struct { label: []const u8, sql: [:0]const u8 }{
.{
.label = "query_log",
.sql =
\\SELECT q.*, d.domain FROM query_log q
\\JOIN domains d ON d.id = q.domain_id
\\ORDER BY q.id
,
},
.{ .label = "domains", .sql = "SELECT * FROM domains ORDER BY id" },
.{ .label = "querylog_meta", .sql = "SELECT * FROM querylog_meta ORDER BY id" },
};
fn dumpLogical(gpa: std.mem.Allocator, database: *db.Db, out: *std.ArrayList(u8)) !void {
for (logical_relations) |relation| {
try dumpQuery(gpa, database, relation.label, relation.sql, out);
}
}
fn freeOwned(gpa: std.mem.Allocator, list: *std.ArrayList([]u8)) void {
for (list.items) |item| gpa.free(item);
list.deinit(gpa);
}
fn expectSameContent(a: *db.Db, b: *db.Db) !void {
var text_a: std.ArrayList(u8) = .empty;
defer text_a.deinit(testing.allocator);
var text_b: std.ArrayList(u8) = .empty;
defer text_b.deinit(testing.allocator);
try dumpContent(testing.allocator, a, &text_a);
try dumpContent(testing.allocator, b, &text_b);
try testing.expectEqualStrings(text_a.items, text_b.items);
}
fn rowCount(database: *db.Db, table: []const u8) !i64 {
var buf: [128]u8 = undefined;
return database.queryInt(std.fmt.bufPrint(&buf, "SELECT count(*) FROM \"{s}\"", .{table}) catch unreachable);
}
/// What must hold after a fixture has come through `openVersioned`, whatever
/// path it took: every row the operator had is still there with the same
/// CONTENT, and the projections still agree with the raw rows.
///
/// Counting rows and checking one watermark is what this used to do, and a
/// migration that shifted a timestamp, dropped a `qtype` or crossed two rows'
/// `client_ip` values passed it. The comparison is therefore the full logical
/// content of the three operator tables, `available_since` included as one
/// column of `querylog_meta` among the rest. The counts stay because a count
/// difference is the failure worth naming plainly.
///
/// The relations are named rather than derived because they are the ones holding
/// operator data; the `bucket_*` projections are derived from them, which
/// `expectProjectionsMatchRecompute` is the right check for. A future migration
/// that renames a table amends this alongside the step that does it. So does one
/// that renumbers `id` values: this asserts they survive, which every rebuild
/// written to the A.2 rule does.
fn expectFixtureSurvived(opened: *db.Db, pristine: *db.Db) !void {
for ([_][]const u8{ "query_log", "domains", "querylog_meta" }) |table| {
try testing.expectEqual(try rowCount(pristine, table), try rowCount(opened, table));
}
var opened_text: std.ArrayList(u8) = .empty;
defer opened_text.deinit(testing.allocator);
var pristine_text: std.ArrayList(u8) = .empty;
defer pristine_text.deinit(testing.allocator);
try dumpLogical(testing.allocator, opened, &opened_text);
try dumpLogical(testing.allocator, pristine, &pristine_text);
try testing.expectEqualStrings(pristine_text.items, opened_text.items);
try queries_repo.expectProjectionsMatchRecompute(opened);
}
test "the survival comparison tells a NULL apart from text that looks like one" {
// The mutation a count-and-watermark check misses entirely, and a
// sentinel-string serialization misses just as completely: one column of one
// row stops being NULL and becomes the very text the sentinel used. Every
// count, the watermark and the projections all still agree.
const fixture = fixtureFor(1) orelse return error.MissingFixture;
var h: Harness = .init();
defer h.deinit();
try writeFixture(h.path(), fixture, fixture.version);
var side: [256]u8 = undefined;
var pristine = try openPristine(&h, &side, fixture);
defer pristine.close();
var corrupted = try h.openLive();
defer corrupted.close();
try testing.expectEqual(@as(i64, 0), corrupted.changes());
try corrupted.exec(
\\UPDATE query_log SET upstream = '<null>'
\\WHERE id = (SELECT min(id) FROM query_log WHERE upstream IS NULL)
);
// The fixture has to actually carry a NULL `upstream` for this to be a test
// of anything.
try testing.expectEqual(@as(i64, 1), corrupted.changes());
// Everything the old check looked at still agrees, which is why it passed.
for ([_][]const u8{ "query_log", "domains", "querylog_meta" }) |table| {
try testing.expectEqual(try rowCount(&pristine, table), try rowCount(&corrupted, table));
}
try testing.expectEqual(
try pristine.queryInt("SELECT available_since FROM querylog_meta WHERE id = 1"),
try corrupted.queryInt("SELECT available_since FROM querylog_meta WHERE id = 1"),
);
try queries_repo.expectProjectionsMatchRecompute(&corrupted);
// The dumps are compared here rather than through `expectFixtureSurvived`
// so that a PASSING run stays silent: `expectEqualStrings` prints the whole
// diff before it returns its error, and this is the comparison that function
// makes.
var corrupted_text: std.ArrayList(u8) = .empty;
defer corrupted_text.deinit(testing.allocator);
var pristine_text: std.ArrayList(u8) = .empty;
defer pristine_text.deinit(testing.allocator);
try dumpLogical(testing.allocator, &corrupted, &corrupted_text);
try dumpLogical(testing.allocator, &pristine, &pristine_text);
try testing.expect(!std.mem.eql(u8, corrupted_text.items, pristine_text.items));
}
test "every supported version ships a fixture pair" {
// The cut gate enforces this against a release; the suite enforces it
// against a commit, so a version bump that forgot its fixtures fails here
// long before anyone reaches for `zig build cut`.
var version = versions.minimum_supported_version;
while (version <= versions.current_version) : (version += 1) {
try testing.expect(fixtureFor(version) != null);
}
}
test "each shipped fixture pair is coherent before any migration touches it" {
for (fixtures) |fixture| {
var h: Harness = .init();
defer h.deinit();
try writeFixture(h.path(), fixture, fixture.version);
var database = try h.openLive();
defer database.close();
try testing.expectEqual(
@as(i64, fixture.version),
try database.queryInt("PRAGMA user_version"),
);
// An incoherent fixture has to fail as a fixture, not later as a
// migration that appears to have corrupted the projections.
try queries_repo.expectProjectionsMatchRecompute(&database);
try testing.expectEqual(@as(i64, 0), try database.queryInt("SELECT count(*) FROM pragma_foreign_key_check"));
}
}
test "every fixture below the current version migrates to it through the shipped chain" {
// Empty while the chain is: `minimum_supported_version == current_version`
// today. It is written as the loop so that the day a step ships, the
// fixture it starts from is proved through the REAL production plan with no
// edit to this test.
var version = versions.minimum_supported_version;
while (version < versions.current_version) : (version += 1) {
const fixture = fixtureFor(version) orelse return error.MissingFixture;
var h: Harness = .init();
defer h.deinit();
try writeFixture(h.path(), fixture, version);
var side: [256]u8 = undefined;
var pristine = try openPristine(&h, &side, fixture);
defer pristine.close();
var result = try querylog_schema.open(h.io(), std.Io.Dir.cwd(), h.path());
defer result.database.close();
try testing.expectEqual(@as(?querylog_schema.RecreateReason, null), result.recreated);
try testing.expectEqual(
@as(i64, versions.current_version),
try result.database.queryInt("PRAGMA user_version"),
);
var fresh_buf: [256]u8 = undefined;
var fresh = try openFreshCurrent(&h, &fresh_buf);
defer fresh.close();
try testing.expect(try migrations.schemaEquivalent(testing.allocator, &result.database, &fresh));
try expectFixtureSurvived(&result.database, &pristine);
}
}
test "the current version's fixture pair opens on the current lane unchanged" {
// The loop above never reaches this pair, and Gate 2 of the cut requires it
// to exist. This is what proves it is a real, coherent file rather than one
// shipped to satisfy a gate.
const fixture = fixtureFor(versions.current_version) orelse return error.MissingFixture;
var h: Harness = .init();
defer h.deinit();
try writeFixture(h.path(), fixture, versions.current_version);
var side: [256]u8 = undefined;
var pristine = try openPristine(&h, &side, fixture);
defer pristine.close();
var result = try querylog_schema.open(h.io(), std.Io.Dir.cwd(), h.path());
defer result.database.close();
try testing.expectEqual(@as(?querylog_schema.RecreateReason, null), result.recreated);
try testing.expectEqual(
@as(i64, versions.current_version),
try result.database.queryInt("PRAGMA user_version"),
);
var fresh_buf: [256]u8 = undefined;
var fresh = try openFreshCurrent(&h, &fresh_buf);
defer fresh.close();
try testing.expect(try migrations.schemaEquivalent(testing.allocator, &result.database, &fresh));
try expectFixtureSurvived(&result.database, &pristine);
// No migration ran, so nothing reshaped anything: byte-for-byte the rows
// that were loaded.
try expectSameContent(&result.database, &pristine);
}
test "a fixture carrying the 0.0.12 fingerprint restamps, and refuses once the minimum rises" {
const fixture = fixtureFor(1) orelse return error.MissingFixture;
// Version 1 is the legacy fingerprint's logical version, so the pair only
// has anything to say while 1 is still supported.
if (versions.minimum_supported_version <= 1 and versions.current_version >= 1) {
var h: Harness = .init();
defer h.deinit();
try writeFixture(h.path(), fixture, versions.legacy_fingerprint);
var side: [256]u8 = undefined;
var pristine = try openPristine(&h, &side, fixture);
defer pristine.close();
var result = try querylog_schema.open(h.io(), std.Io.Dir.cwd(), h.path());
defer result.database.close();
try testing.expectEqual(@as(?querylog_schema.RecreateReason, null), result.recreated);
try testing.expectEqual(
@as(i64, versions.current_version),
try result.database.queryInt("PRAGMA user_version"),
);
var fresh_buf: [256]u8 = undefined;
var fresh = try openFreshCurrent(&h, &fresh_buf);
defer fresh.close();
try testing.expect(try migrations.schemaEquivalent(testing.allocator, &result.database, &fresh));
try expectFixtureSurvived(&result.database, &pristine);
}
// The companion, already in the suite for the day a break raises the
// minimum above 1: the same bytes under the same stamp are then a file this
// build cannot reach, and it is refused without being touched.
var h: Harness = .init();
defer h.deinit();
try writeFixture(h.path(), fixture, versions.legacy_fingerprint);
const after_break: querylog_schema.Plan = .{
.minimum = 2,
.current = 2,
.legacy_fingerprint = versions.legacy_fingerprint,
.step_sql = &.{},
};
try testing.expect(querylog_schema.classify(versions.legacy_fingerprint, after_break).action ==
.refuse_unsupported);
try testing.expect(!querylog_schema.classify(versions.legacy_fingerprint, after_break).restamp);
var handle: ?db.Db = try h.openLive();
defer if (handle) |*open_db| open_db.close();
migrations.expected_failures.begin();
defer migrations.expected_failures.end();
try testing.expectError(
error.SchemaUnsupported,
querylog_schema.openVersioned(h.io(), std.Io.Dir.cwd(), h.path(), &handle, after_break),
);
var check = try h.openLive();
defer check.close();
try testing.expectEqual(
@as(i64, versions.legacy_fingerprint),
try check.queryInt("PRAGMA user_version"),
);
var pristine_buf: [256]u8 = undefined;
var pristine = try openPristine(&h, &pristine_buf, fixture);
defer pristine.close();
try expectSameContent(&check, &pristine);
}
File diff suppressed because it is too large Load Diff
+722 -40
View File
@@ -1,14 +1,15 @@
//! The `querylog.db` schema and its open-or-recreate policy.
//! The `querylog.db` schema and its open policy.
//!
//! `querylog.db` is never migrated (PLAN §3.7). It holds expendable log rows,
//! so a schema change replaces the file instead of upgrading it. The
//! replacement trigger is a fingerprint derived from the DDL text itself, so
//! editing the schema below automatically invalidates every existing file — the
//! policy cannot drift out of sync with the SQL.
//! `querylog.db` carries a logical schema version in `PRAGMA user_version`
//! (`querylog_versions.zig`), and a file stamped below the current version is
//! MIGRATED in place. A healthy file is never replaced and never set aside: a
//! version this build cannot reach refuses the startup with instructions
//! instead, because the operator's query history is not this program's to
//! discard.
//!
//! **Recreating is destructive, so the predicate is a positive whitelist.** Only
//! a missing file, `error.Corrupt`, `error.NotADb`, a failed `PRAGMA
//! quick_check` and a fingerprint mismatch recreate. Every other error
//! a missing file, `error.Corrupt`, `error.NotADb` and a failed `PRAGMA
//! quick_check` recreate — genuine corruption, nothing else. Every other error
//! propagates and the file on disk is not touched. `error.Busy` / `error.Locked`
//! mean another process holds the write lock — waiting is right, deleting is
//! catastrophic. `error.OutOfMemory` is this process's problem. `error.CantOpen`
@@ -19,9 +20,21 @@
const std = @import("std");
const db = @import("db.zig");
const migrations = @import("querylog_migrations.zig");
const versions = @import("querylog_versions.zig");
const log = std.log.scoped(.querylog_schema);
/// See `querylog_migrations.fail`: `err`, unless a test has said it is causing
/// this refusal on purpose.
fn fail(comptime fmt: []const u8, args: anytype) void {
if (migrations.expected_failures.capturing()) {
log.warn(fmt, args);
} else {
log.err(fmt, args);
}
}
/// PLAN §11.3, plus the coverage watermark of milestone 28. Multi-statement
/// text — it goes through `db.Db.exec`, never through `prepare`.
///
@@ -139,14 +152,21 @@ pub const fingerprint: i32 = blk: {
break :blk fingerprintOf(ddl);
};
const set_user_version = std.fmt.comptimePrint("PRAGMA user_version = {d};", .{fingerprint});
/// What a fresh file is stamped with. The logical version, not the fingerprint:
/// from this release on, `user_version` is a version number.
const set_user_version = std.fmt.comptimePrint(
"PRAGMA user_version = {d};",
.{versions.current_version},
);
/// Long enough for any path this program will be handed, plus the aside suffix.
/// A longer path is `error.NameTooLong`, which is what the filesystem calls
/// would have returned anyway.
const path_buf_len = 4096 + 64;
pub const RecreateReason = enum { missing, corrupt, not_a_database, quick_check_failed, fingerprint_mismatch };
/// Corruption, and nothing else. A healthy file with a version this build does
/// not handle refuses the startup; it is never recreated and never set aside.
pub const RecreateReason = enum { missing, corrupt, not_a_database, quick_check_failed };
pub const OpenResult = struct {
database: db.Db,
@@ -166,10 +186,75 @@ pub const OpenResult = struct {
}
};
pub const Error = db.Error || error{AsideNameCollision} ||
std.Io.Dir.RenamePreserveError || std.Io.Dir.DeleteFileError || std.Io.Dir.AccessError;
pub const Error = db.Error || error{
AsideNameCollision,
/// The file's version is above this build's. A downgrade, almost always.
SchemaTooNew,
/// The file's version is one this build cannot migrate from: older than
/// `minimum_supported_version`, or not a stamp nxdns ever wrote.
SchemaUnsupported,
MigrationFailed,
MigrationBackupFailed,
NameTooLong,
} || std.Io.Dir.RenamePreserveError || std.Io.Dir.DeleteFileError || std.Io.Dir.AccessError;
/// Opens `path`, recreating it if and only if it is genuinely unusable.
/// The version metadata `openVersioned` works against. Production passes
/// `production_plan`; tests inject synthetic chains, which is what makes
/// migration, the post-commit branch and the ownership of the handle testable
/// through the real open path while the shipped chain is still empty.
pub const Plan = struct {
minimum: i32,
current: i32,
legacy_fingerprint: i32,
step_sql: []const [:0]const u8,
};
pub const production_plan: Plan = .{
.minimum = versions.minimum_supported_version,
.current = versions.current_version,
.legacy_fingerprint = versions.legacy_fingerprint,
.step_sql = versions.step_sql,
};
/// What a stamped `user_version` means. A pure function of the stamp and the
/// plan's three numbers — no file, no clock, no mutation.
pub const Action = enum { open_current, migrate, refuse_too_new, refuse_unsupported };
pub const Classification = struct {
/// The stamp mapped onto the version line. Equal to the stamp except for
/// the legacy fingerprint, which IS version 1.
logical: i32,
action: Action,
/// The legacy fingerprint must be replaced by its logical number before the
/// file is used — but only on a lane that accepts the file. An unsupported
/// file is never modified.
restamp: bool,
};
/// The classification table. The order is load-bearing: the legacy fingerprint
/// becomes version 1 FIRST, and only then is version 1 judged against the
/// plan's range. After a future explicit break raises the minimum above 1, a
/// legacy-stamped file therefore classifies as below-minimum and refuses
/// without ever being restamped.
pub fn classify(stamped: i32, plan: Plan) Classification {
const legacy = stamped == plan.legacy_fingerprint;
const logical: i32 = if (legacy) 1 else stamped;
const action: Action = if (logical == plan.current)
.open_current
else if (logical >= plan.minimum and logical < plan.current)
.migrate
else if (logical > plan.current and logical <= versions.version_floor_guard)
.refuse_too_new
else
.refuse_unsupported;
const accepted = action == .open_current or action == .migrate;
return .{ .logical = logical, .action = action, .restamp = legacy and accepted };
}
/// Opens `path`, recreating it if and only if it is genuinely unusable, and
/// migrating it if and only if it carries an older supported version.
///
/// `path` is resolved twice by two different mechanisms: `dir`-relative for the
/// filesystem calls, and process-cwd-relative by SQLite's VFS, which knows
@@ -180,7 +265,7 @@ pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
var handle: ?db.Db = null;
errdefer if (handle) |*h| h.close();
const reason: ?RecreateReason = probe: {
const cause: RecreateReason = probe: {
dir.access(io, path, .{}) catch |e| switch (e) {
error.FileNotFound => break :probe .missing,
else => |other| return other,
@@ -197,15 +282,10 @@ pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
break :probe recreatable(e) orelse return e;
if (!healthy) break :probe .quick_check_failed;
const stamped = opened.queryInt("PRAGMA user_version") catch |e|
break :probe recreatable(e) orelse return e;
if (stamped != fingerprint) break :probe .fingerprint_mismatch;
break :probe null;
try openVersioned(io, dir, path, &handle, production_plan);
return .{ .database = handle.?, .recreated = null };
};
const cause = reason orelse return .{ .database = handle.?, .recreated = null };
// Close first, so SQLite checkpoints and drops `-wal`/`-shm` where it can.
if (handle) |*h| h.close();
handle = null;
@@ -240,6 +320,131 @@ pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
return result;
}
/// The version half of `open`, against an injectable `plan`.
///
/// `handle` is the slot holding the healthy, pragma-applied connection to
/// `path`. On success the connection stays in it, at `plan.current`. On every
/// failure this function closes the connection and sets the slot to null, so
/// the caller's own error-path close cannot double-close it — including the
/// committed-but-unclean path, which is the one place the handle must be
/// dropped even though the file on disk is fine.
///
/// **nxdns owns `path` exclusively.** It opens `querylog.db` once at startup,
/// before it serves anything, and no second process shares a data directory —
/// the standing deployment contract. The backup-then-lock sequence in
/// `querylog_migrations.runMigration` relies on it: between the `VACUUM INTO`
/// and the `BEGIN IMMEDIATE` there is no lock, and the re-read of
/// `user_version` under the lock is what turns a violation of that contract
/// into a refusal instead of a corrupted migration.
pub fn openVersioned(
io: std.Io,
dir: std.Io.Dir,
path: [:0]const u8,
handle: *?db.Db,
plan: Plan,
) Error!void {
const database = &(handle.*.?);
errdefer closeSlot(handle);
const stamped64 = try database.queryInt("PRAGMA user_version");
const stamped = std.math.cast(i32, stamped64) orelse {
// `user_version` is a signed 32-bit field, so this cannot come from
// SQLite. Refusing is the same answer any other foreign stamp gets.
refusalLog(path, stamped64, plan, "SchemaUnsupported");
return error.SchemaUnsupported;
};
const verdict = classify(stamped, plan);
switch (verdict.action) {
.refuse_too_new => {
refusalLog(path, stamped64, plan, "SchemaTooNew");
return error.SchemaTooNew;
},
.refuse_unsupported => {
refusalLog(path, stamped64, plan, "SchemaUnsupported");
return error.SchemaUnsupported;
},
.open_current, .migrate => {},
}
if (verdict.restamp) try restampLegacy(database, path, verdict.logical);
switch (verdict.action) {
.migrate => {
const first = @as(usize, @intCast(verdict.logical - plan.minimum));
migrations.runMigration(
io,
dir,
path,
database,
plan.step_sql[first..],
verdict.logical,
plan.current,
) catch |e| switch (e) {
// Two different states of the FILE — migrated and kept with its
// backup, or logically untouched — and one shared state of the
// CONNECTION: its pragmas are not what `applyPragmas`
// guarantees, so it must not serve. Closing it here is the
// single close either path gets. After a commit the next start
// opens the migrated file on the current-version lane; after a
// failure it retries the migration from the top.
error.MigrationCommittedButUnclean, error.MigrationFailedUnclean => {
closeSlot(handle);
return error.MigrationFailed;
},
error.MigrationBackupFailed => return error.MigrationBackupFailed,
error.MigrationFailed, error.NameTooLong => return error.MigrationFailed,
else => |other| return other,
};
},
.open_current => migrations.pruneBackupsConservative(io, dir, path),
else => unreachable,
}
}
/// Replaces the 0.0.12/0.0.13 fingerprint stamp with the logical version it
/// stands for. This is the milestone's only real mutation of operator data, so
/// it runs in its own transaction and any failure leaves the legacy stamp and
/// every row exactly as they were — a refusal, never a recreate.
fn restampLegacy(database: *db.Db, path: []const u8, logical: i32) Error!void {
var stamp_buf: [64]u8 = undefined;
const stamp = std.fmt.bufPrintZ(&stamp_buf, "PRAGMA user_version = {d};", .{logical}) catch
unreachable; // an i32 and a fixed prefix cannot overrun 64 bytes
restamp: {
var tx = db.Tx.begin(database) catch break :restamp;
database.exec(stamp) catch {
tx.rollback();
break :restamp;
};
tx.commit() catch {
tx.rollback();
break :restamp;
};
log.info("querylog database '{s}' carried the 0.0.12 schema fingerprint; " ++
"restamped as schema version {d}", .{ path, logical });
return;
}
var buf: [256]u8 = undefined;
fail("cannot restamp querylog database '{s}' as schema version {d}: {s}; " ++
"the file is unchanged", .{ path, logical, database.lastError(&buf) });
return error.MigrationFailed;
}
fn closeSlot(handle: *?db.Db) void {
if (handle.*) |*h| h.close();
handle.* = null;
}
fn refusalLog(path: []const u8, stamped: i64, plan: Plan, name: []const u8) void {
fail("refusing to open querylog database '{s}': it is stamped {d}, and this build " ++
"supports schema versions {d} to {d} ({s}). The file is left exactly as it is; " ++
"see docs/how-to/troubleshoot.md, \"The server refuses to start over querylog.db\"", .{
path, stamped, plan.minimum, plan.current, name,
});
}
/// An additional connection to a `querylog.db` that `open` has already
/// established, with the pragmas every connection to the file needs.
///
@@ -285,16 +490,15 @@ fn quickCheck(database: *db.Db) db.Error!bool {
/// What the aside file's name calls the reason it was set aside.
///
/// The name is the only account of the reason an operator gets: the log line
/// naming it scrolls away, the file stays for months. `fingerprint_mismatch` is
/// a database with nothing wrong with it — this build's DDL moved — so calling
/// its file "corrupt" invites the operator to delete evidence of a healthy file.
/// naming it scrolls away, the file stays for months. Every tag here names real
/// damage, which is the whole set of reasons left — a healthy file whose
/// version this build cannot handle refuses the startup and is not renamed.
fn asideTag(reason: RecreateReason) []const u8 {
return switch (reason) {
.missing => unreachable, // there is no file to rename
.corrupt => "corrupt",
.not_a_database => "not-a-database",
.quick_check_failed => "quick-check-failed",
.fingerprint_mismatch => "schema-changed",
};
}
@@ -455,11 +659,14 @@ test "querylog_meta is seeded with one row the schema will not let a second join
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM querylog_meta"));
}
test "the user_version statement stamps the fingerprint" {
test "the user_version statement stamps the current schema version" {
var database = try db.Db.open(":memory:", .{ .mode = .memory });
defer database.close();
try database.exec(set_user_version);
try testing.expectEqual(@as(i64, fingerprint), try database.queryInt("PRAGMA user_version"));
try testing.expectEqual(
@as(i64, versions.current_version),
try database.queryInt("PRAGMA user_version"),
);
}
// The behaviour these two cases describe — a resource error leaves the file on
@@ -488,7 +695,6 @@ test "the aside name says why, and a healthy file is never called corrupt" {
try testing.expectEqualStrings("corrupt", asideTag(.corrupt));
try testing.expectEqualStrings("not-a-database", asideTag(.not_a_database));
try testing.expectEqualStrings("quick-check-failed", asideTag(.quick_check_failed));
try testing.expectEqualStrings("schema-changed", asideTag(.fingerprint_mismatch));
}
test "recreatable selects exactly two of db.Error's members" {
@@ -532,7 +738,7 @@ test "a recreate returns the aside name by value and a fresh create returns none
try tmp.dir.access(io, kept, .{});
}
test "a recreate resets coverage to the new file and keeps the old one aside" {
test "a corrupt file is recreated, coverage restarts, and the old one is kept aside" {
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
defer threaded.deinit();
const io = threaded.io();
@@ -549,21 +755,14 @@ test "a recreate resets coverage to the new file and keeps the old one aside" {
try created.database.exec("INSERT INTO domains (domain) VALUES ('old.example');");
created.database.close();
// A healthy file this build's DDL no longer matches — the case milestone
// 28's own schema edit produces on every upgrade.
{
var stamped = try db.Db.open(path, .{ .mode = .read_write_existing });
defer stamped.close();
var sql_buf: [64]u8 = undefined;
try stamped.exec(try std.fmt.bufPrintZ(&sql_buf, "PRAGMA user_version = {d};", .{fingerprint +% 1}));
}
// Real damage, which is now the only thing that recreates.
try tmp.dir.writeFile(io, .{ .sub_path = "querylog.db", .data = "not a database at all" });
var recreated = try open(io, std.Io.Dir.cwd(), path);
defer recreated.database.close();
try testing.expectEqual(RecreateReason.fingerprint_mismatch, recreated.recreated.?);
// The name says the file was healthy and this build moved, not that it rotted.
try testing.expect(std.mem.indexOf(u8, recreated.aside(), ".schema-changed-") != null);
try testing.expectEqual(RecreateReason.not_a_database, recreated.recreated.?);
try testing.expect(std.mem.indexOf(u8, recreated.aside(), ".not-a-database-") != null);
try tmp.dir.access(io, std.fs.path.basename(recreated.aside()), .{});
// Exactly one meta row, and coverage starts at the recreate rather than
@@ -602,3 +801,486 @@ test "a clean reopen reports no recreate and no aside" {
try testing.expectEqual(@as(?RecreateReason, null), second.recreated);
try testing.expectEqualStrings("", second.aside());
}
test "classification is a pure function of the stamp and the plan's three numbers" {
const plan: Plan = .{
.minimum = 1,
.current = 3,
.legacy_fingerprint = versions.legacy_fingerprint,
.step_sql = &.{},
};
try testing.expectEqual(Action.open_current, classify(3, plan).action);
try testing.expectEqual(Action.migrate, classify(1, plan).action);
try testing.expectEqual(Action.migrate, classify(2, plan).action);
try testing.expectEqual(Action.refuse_too_new, classify(4, plan).action);
try testing.expectEqual(Action.refuse_too_new, classify(versions.version_floor_guard, plan).action);
// Above the floor guard is not a version this project ever wrote.
try testing.expectEqual(
Action.refuse_unsupported,
classify(versions.version_floor_guard + 1, plan).action,
);
for ([_]i32{ 0, -1, -1_000_000, 603440875 }) |foreign| {
try testing.expectEqual(Action.refuse_unsupported, classify(foreign, plan).action);
try testing.expect(!classify(foreign, plan).restamp);
}
// The legacy fingerprint IS version 1, and being version 1 is what decides
// its lane.
const legacy = classify(versions.legacy_fingerprint, plan);
try testing.expectEqual(@as(i32, 1), legacy.logical);
try testing.expectEqual(Action.migrate, legacy.action);
try testing.expect(legacy.restamp);
}
test "a legacy stamp below a raised minimum refuses without a restamp" {
// What a future explicit break looks like from this side: the minimum has
// moved past 1, so the 0.0.12 file is no longer reachable. The ORDER is the
// point — mapping to 1 first and judging second is what stops the restamp
// from mutating a file this build will refuse anyway.
const after_break: Plan = .{
.minimum = 3,
.current = 3,
.legacy_fingerprint = versions.legacy_fingerprint,
.step_sql = &.{},
};
const legacy = classify(versions.legacy_fingerprint, after_break);
try testing.expectEqual(@as(i32, 1), legacy.logical);
try testing.expectEqual(Action.refuse_unsupported, legacy.action);
try testing.expect(!legacy.restamp);
// And versions 1 and 2, which the break dropped, refuse the same way.
try testing.expectEqual(Action.refuse_unsupported, classify(1, after_break).action);
try testing.expectEqual(Action.refuse_unsupported, classify(2, after_break).action);
try testing.expectEqual(Action.open_current, classify(3, after_break).action);
}
test "the production plan classifies a fresh stamp as current" {
try testing.expectEqual(
Action.open_current,
classify(versions.current_version, production_plan).action,
);
try testing.expect(classify(versions.legacy_fingerprint, production_plan).restamp);
}
/// The five lines every file-backed test below opens with.
const Fixture = struct {
threaded: std.Io.Threaded,
tmp: std.testing.TmpDir,
buf: [256]u8 = undefined,
fn init() Fixture {
return .{
.threaded = .init(testing.allocator, .{}),
.tmp = testing.tmpDir(.{ .iterate = true }),
};
}
fn deinit(self: *Fixture) void {
self.tmp.cleanup();
self.threaded.deinit();
}
fn io(self: *Fixture) std.Io {
return self.threaded.io();
}
fn path(self: *Fixture) [:0]const u8 {
return std.fmt.bufPrintZ(&self.buf, ".zig-cache/tmp/{s}/querylog.db", .{self.tmp.sub_path}) catch
unreachable;
}
fn stamp(self: *Fixture, value: i32) !void {
var database = try db.Db.open(self.path(), .{ .mode = .read_write_existing });
defer database.close();
var sql: [64]u8 = undefined;
try database.exec(try std.fmt.bufPrintZ(&sql, "PRAGMA user_version = {d};", .{value}));
}
fn liveHandle(self: *Fixture) !?db.Db {
var database = try db.Db.open(self.path(), .{ .mode = .read_write_existing });
errdefer database.close();
try db.applyPragmas(&database, .{});
return database;
}
fn countMatching(self: *Fixture, prefix: []const u8) !usize {
var found: usize = 0;
var it = self.tmp.dir.iterate();
while (try it.next(self.io())) |entry| {
if (std.mem.startsWith(u8, entry.name, prefix)) found += 1;
}
return found;
}
fn expectModeOfOnlyMatch(self: *Fixture, prefix: []const u8, expected: std.posix.mode_t) !void {
var it = self.tmp.dir.iterate();
while (try it.next(self.io())) |entry| {
if (!std.mem.startsWith(u8, entry.name, prefix)) continue;
const stat = try self.tmp.dir.statFile(self.io(), entry.name, .{});
const mode = stat.permissions.toMode() & 0o777;
if (mode != expected) {
std.debug.print("mode of '{s}' is {o}, expected {o}\n", .{ entry.name, mode, expected });
return error.TestUnexpectedResult;
}
return;
}
std.debug.print("no file starting with '{s}'\n", .{prefix});
return error.TestUnexpectedResult;
}
};
test "a fresh file is stamped with the current schema version" {
var f: Fixture = .init();
defer f.deinit();
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer created.database.close();
try testing.expectEqual(RecreateReason.missing, created.recreated.?);
try testing.expectEqual(
@as(i64, versions.current_version),
try created.database.queryInt("PRAGMA user_version"),
);
}
test "a 0.0.13 file is restamped as version 1 and keeps every row" {
var f: Fixture = .init();
defer f.deinit();
{
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer created.database.close();
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
}
// Exactly what 0.0.12 and 0.0.13 wrote: the CRC of their DDL, which is this
// build's DDL unchanged.
try f.stamp(versions.legacy_fingerprint);
try testing.expectEqual(versions.legacy_fingerprint, fingerprint);
{
var upgraded = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer upgraded.database.close();
try testing.expectEqual(@as(?RecreateReason, null), upgraded.recreated);
try testing.expectEqual(@as(i64, 1), try upgraded.database.queryInt("PRAGMA user_version"));
try testing.expectEqual(
@as(i64, 1),
try upgraded.database.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
);
}
// Nothing was set aside on the way, and the second start is an ordinary
// current-version open.
try testing.expectEqual(@as(usize, 0), try f.countMatching("querylog.db.schema"));
var again = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer again.database.close();
try testing.expectEqual(@as(?RecreateReason, null), again.recreated);
try testing.expectEqual(@as(i64, 1), try again.database.queryInt("PRAGMA user_version"));
}
test "a version this build cannot handle refuses and leaves the file alone" {
var f: Fixture = .init();
defer f.deinit();
{
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer created.database.close();
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
}
const watermark = blk: {
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
defer probe.close();
break :blk try probe.queryInt("SELECT available_since FROM querylog_meta");
};
migrations.expected_failures.begin();
defer migrations.expected_failures.end();
const lanes = [_]struct { stamp: i32, expected: anyerror }{
.{ .stamp = versions.current_version + 1, .expected = error.SchemaTooNew },
.{ .stamp = versions.version_floor_guard, .expected = error.SchemaTooNew },
.{ .stamp = 0, .expected = error.SchemaUnsupported },
.{ .stamp = -3, .expected = error.SchemaUnsupported },
.{ .stamp = 603440875, .expected = error.SchemaUnsupported },
};
for (lanes) |lane| {
try f.stamp(lane.stamp);
try testing.expectError(lane.expected, open(f.io(), std.Io.Dir.cwd(), f.path()));
// Schema, rows, watermark and stamp all as they were, and nothing new
// beside the file.
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
defer probe.close();
try testing.expectEqual(@as(i64, lane.stamp), try probe.queryInt("PRAGMA user_version"));
try testing.expectEqual(
@as(i64, 1),
try probe.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
);
try testing.expectEqual(
watermark,
try probe.queryInt("SELECT available_since FROM querylog_meta"),
);
try testing.expectEqual(@as(usize, 0), try f.countMatching("querylog.db."));
}
}
test "a restamp that fails at the statement or at the commit refuses without loss" {
for ([_][]const u8{ "PRAGMA user_version = 1;", "COMMIT;" }) |failing| {
var f: Fixture = .init();
defer f.deinit();
{
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer created.database.close();
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
}
try f.stamp(versions.legacy_fingerprint);
migrations.expected_failures.begin();
defer migrations.expected_failures.end();
db.exec_faults.failNextMatching(failing);
defer db.exec_faults.disarm();
try testing.expectError(
error.MigrationFailed,
open(f.io(), std.Io.Dir.cwd(), f.path()),
);
try testing.expect(!db.exec_faults.armed());
// The legacy stamp and every row are exactly as they were: this is a
// refusal, and a refusal never costs the operator anything.
{
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
defer probe.close();
try testing.expectEqual(
@as(i64, versions.legacy_fingerprint),
try probe.queryInt("PRAGMA user_version"),
);
}
// And the next start, with nothing injected, does the restamp properly.
var recovered = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer recovered.database.close();
try testing.expectEqual(@as(i64, 1), try recovered.database.queryInt("PRAGMA user_version"));
try testing.expectEqual(
@as(i64, 1),
try recovered.database.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
);
}
}
/// A one-step chain from the real current version to one above it. Nothing in
/// the shipped chain can exercise migration while `step_sql` is empty, so the
/// open path's migration lanes are driven through this instead.
const synthetic_plan: Plan = .{
.minimum = versions.current_version,
.current = versions.current_version + 1,
.legacy_fingerprint = versions.legacy_fingerprint,
.step_sql = &.{"CREATE TABLE migration_marker (id INTEGER PRIMARY KEY);"},
};
test "a post-commit failure keeps the migration, keeps the backup, and refuses once" {
var f: Fixture = .init();
defer f.deinit();
{
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer created.database.close();
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
}
{
var handle = try f.liveHandle();
errdefer if (handle) |*h| h.close();
migrations.expected_failures.begin();
defer migrations.expected_failures.end();
// The first statement of the pragma restore, which runs only after a
// successful COMMIT.
db.exec_faults.failNextMatching("legacy_alter_table = OFF");
defer db.exec_faults.disarm();
try testing.expectError(
error.MigrationFailed,
openVersioned(f.io(), std.Io.Dir.cwd(), f.path(), &handle, synthetic_plan),
);
// Closed exactly once, by the open path: the slot it was handed is
// empty, so no caller can close it again.
try testing.expect(handle == null);
}
// The file IS migrated. The log said so, and this is what it meant.
{
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
defer probe.close();
try testing.expectEqual(
@as(i64, synthetic_plan.current),
try probe.queryInt("PRAGMA user_version"),
);
try testing.expectEqual(
@as(i64, 1),
try probe.queryInt("SELECT count(*) FROM sqlite_schema WHERE name = 'migration_marker'"),
);
try testing.expectEqual(
@as(i64, 1),
try probe.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
);
}
try testing.expectEqual(@as(usize, 1), try f.countMatching("querylog.db.pre-migrate-"));
// The backup is the whole query history at the moment of the migration, so
// it carries the live file's mode and not SQLite's `0644 & ~umask`.
try f.expectModeOfOnlyMatch("querylog.db.pre-migrate-", 0o600);
// The next start is ordinary: the current-version lane, no second
// migration, and the one backup still there for the operator.
var handle = try f.liveHandle();
defer if (handle) |*h| h.close();
try openVersioned(f.io(), std.Io.Dir.cwd(), f.path(), &handle, synthetic_plan);
try testing.expect(handle != null);
try testing.expectEqual(
@as(i64, synthetic_plan.current),
try handle.?.queryInt("PRAGMA user_version"),
);
try testing.expectEqual(@as(usize, 1), try f.countMatching("querylog.db.pre-migrate-"));
}
/// The same shape as `synthetic_plan`, with a step SQLite refuses to prepare. It
/// drives the pre-commit failure path without any fault seam, leaving the seam
/// free for the pragma restore.
const failing_plan: Plan = .{
.minimum = versions.current_version,
.current = versions.current_version + 1,
.legacy_fingerprint = versions.legacy_fingerprint,
.step_sql = &.{"CREATE TABLE migration_marker (id INTEGER PRIMARY KEY) NOT A STATEMENT;"},
};
test "a pre-commit failure whose pragma restore also fails closes the connection" {
var f: Fixture = .init();
defer f.deinit();
{
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer created.database.close();
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
}
// The runner's own answer first: a failed restore is a DIFFERENT error from
// a failed migration, because the two leave the connection in different
// states even though they leave the file in the same one.
{
var handle = try f.liveHandle();
defer if (handle) |*h| h.close();
migrations.expected_failures.begin();
defer migrations.expected_failures.end();
db.exec_faults.failNextMatching("legacy_alter_table = OFF");
defer db.exec_faults.disarm();
try testing.expectError(error.MigrationFailedUnclean, migrations.runMigration(
f.io(),
std.Io.Dir.cwd(),
f.path(),
&handle.?,
failing_plan.step_sql,
failing_plan.minimum,
failing_plan.current,
));
try testing.expect(!db.exec_faults.armed());
}
// And the open path's answer: the handle is closed, exactly as it is after a
// post-commit restore failure. A connection that may still hold
// `foreign_keys = OFF` never reaches the server.
{
var handle = try f.liveHandle();
errdefer if (handle) |*h| h.close();
migrations.expected_failures.begin();
defer migrations.expected_failures.end();
db.exec_faults.failNextMatching("legacy_alter_table = OFF");
defer db.exec_faults.disarm();
try testing.expectError(
error.MigrationFailed,
openVersioned(f.io(), std.Io.Dir.cwd(), f.path(), &handle, failing_plan),
);
try testing.expect(handle == null);
}
// The contrast that makes the rule visible, back at the runner, where the
// connection survives to be inspected: the same failing step with the
// restore working is a plain `MigrationFailed`, and that error promises the
// pragmas `applyPragmas` guarantees.
{
var handle = try f.liveHandle();
defer if (handle) |*h| h.close();
migrations.expected_failures.begin();
defer migrations.expected_failures.end();
try testing.expectError(error.MigrationFailed, migrations.runMigration(
f.io(),
std.Io.Dir.cwd(),
f.path(),
&handle.?,
failing_plan.step_sql,
failing_plan.minimum,
failing_plan.current,
));
try testing.expectEqual(@as(i64, 1), try handle.?.queryInt("PRAGMA foreign_keys"));
try testing.expectEqual(@as(i64, 0), try handle.?.queryInt("PRAGMA legacy_alter_table"));
}
// No path committed anything, and every run deleted its own backup.
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
defer probe.close();
try testing.expectEqual(
@as(i64, failing_plan.minimum),
try probe.queryInt("PRAGMA user_version"),
);
try testing.expectEqual(
@as(i64, 1),
try probe.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
);
try testing.expectEqual(@as(usize, 0), try f.countMatching("querylog.db.pre-migrate-"));
}
test "a plain open retries a retention cleanup that once failed" {
var f: Fixture = .init();
defer f.deinit();
{
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
created.database.close();
}
// What a migration whose step-4 cleanup failed leaves behind: an older
// epoch, the newest epoch, and a same-second collision name tied with it.
const seeded = [_][]const u8{
"querylog.db.pre-migrate-1600000000",
"querylog.db.pre-migrate-1700000000",
"querylog.db.pre-migrate-1700000000-2",
"querylog.db.pre-migrate-handwritten",
};
for (seeded) |name| {
try f.tmp.dir.writeFile(f.io(), .{ .sub_path = name, .data = "x" });
}
var opened = try open(f.io(), std.Io.Dir.cwd(), f.path());
defer opened.database.close();
try testing.expectEqual(@as(?RecreateReason, null), opened.recreated);
// Only the strictly older epoch goes: both files tied at the newest epoch
// survive, because this pass cannot tell which of them a migration made,
// and a name it did not write is never its to delete.
try testing.expect(!try exists(&f, seeded[0]));
for (seeded[1..]) |name| try testing.expect(try exists(&f, name));
}
fn exists(f: *Fixture, name: []const u8) !bool {
f.tmp.dir.access(f.io(), name, .{}) catch |e| switch (e) {
error.FileNotFound => return false,
else => return e,
};
return true;
}
+89
View File
@@ -0,0 +1,89 @@
//! The `querylog.db` schema version chain: comptime metadata and nothing else.
//!
//! Separate from `querylog_migrations.zig` so `tools/cut.zig` can import it
//! without linking SQLite. Nothing in this file may reach for `db.zig`, for a
//! C symbol, or for an allocator — the release gate reads these constants at
//! build time, and a dependency here would drag the whole storage layer into
//! the cut tool.
//!
//! **There are no migration hooks.** A step is a SQL file, period. Every
//! shipped step is therefore byte-comparable against the previous tag, which is
//! what lets the cut gate prove a released migration was never edited. A future
//! change that genuinely cannot be expressed in SQL must amend this design in
//! its own spec rather than adding a code path here.
const std = @import("std");
/// The version a file created by this build carries in `PRAGMA user_version`.
pub const current_version: i32 = 1;
/// The oldest stamped version this build can reach `current_version` from.
/// A file stamped below this refuses to open.
///
/// An EXPLICIT BREAK in a future release is expressed here and only here: bump
/// `current_version`, set `minimum_supported_version = current_version`, and
/// ship no step. The chain then cannot reach the new version from below the
/// minimum, so `open` refuses the old file by the ordinary rules. A break is
/// always versioned, always refused at runtime, and never silent.
pub const minimum_supported_version: i32 = 1;
/// The literal `user_version` the 0.0.12 and 0.0.13 binaries stamped: the CRC32
/// of their DDL text, under the pre-migration policy where a stamp mismatch
/// meant "replace the file".
///
/// FROZEN. It is derived from nothing at build time on purpose — recomputing it
/// from today's DDL would silently stop recognising the files it exists to
/// recognise the moment the schema moves. Editing it strands every 0.0.12 and
/// 0.0.13 file that has not yet been opened by a migration-aware build, which
/// is why the cut gate fails on any change to this line.
pub const legacy_fingerprint: i32 = 1975011655;
/// Logical versions live far below any plausible CRC32 stamp. A value above
/// this is not a version this project ever wrote, so it classifies as
/// unsupported rather than as a from-the-future schema.
pub const version_floor_guard: i32 = 1_000_000;
/// One entry per shipped step: `step_sql[i]` migrates version
/// `minimum_supported_version + i` to `minimum_supported_version + i + 1`.
/// Each entry is `@embedFile("migrations/v<from>.sql")`, and each such file is
/// immutable once released.
///
/// **Step-authoring rules** (the runner enforces the first, the equivalence
/// oracle catches violations of the rest):
///
/// - A step contains no transaction statement. No `BEGIN`, no `COMMIT`, no
/// `ROLLBACK`, no `SAVEPOINT`: the runner wraps the whole chain in one
/// transaction and installs an authorizer that denies them outright.
/// - A step that changes a table's shape must REBUILD it, so that the CREATE
/// text SQLite stores ends up byte-identical to the fresh DDL's:
/// `DROP` every view over `<t>` first; `ALTER TABLE <t> RENAME TO <t>_old`;
/// `CREATE TABLE <t> ...` pasted verbatim from `querylog_schema.ddl`;
/// `INSERT INTO <t> SELECT ... FROM <t>_old`; `DROP TABLE <t>_old`; recreate
/// every index and trigger of `<t>` verbatim; recreate the dropped views
/// verbatim last.
/// - `ALTER TABLE ... ADD COLUMN` and `ALTER TABLE ... RENAME COLUMN` on a kept
/// table are forbidden. SQLite rewrites the stored CREATE text under them,
/// and the oracle's exact-text layer would rightly call the result unequal.
pub const step_sql: []const [:0]const u8 = &.{};
comptime {
std.debug.assert(minimum_supported_version >= 1);
std.debug.assert(minimum_supported_version <= current_version);
std.debug.assert(current_version <= version_floor_guard);
std.debug.assert(legacy_fingerprint < 0 or legacy_fingerprint > version_floor_guard);
std.debug.assert(step_sql.len == @as(usize, @intCast(current_version - minimum_supported_version)));
}
test "the chain covers exactly the supported range" {
try std.testing.expectEqual(
@as(usize, @intCast(current_version - minimum_supported_version)),
step_sql.len,
);
}
test "the legacy anchor is the literal 0.0.12 stamp" {
// Not `fingerprintOf(ddl)`. The number is a historical fact about released
// binaries, so a test that recomputed it would move with the schema and
// prove nothing.
try std.testing.expectEqual(@as(i32, 1975011655), legacy_fingerprint);
}
+4 -1
View File
@@ -2518,7 +2518,10 @@ const recompute_checks = [_]struct { projection: []const u8, recompute: []const
},
};
fn expectProjectionsMatchRecompute(database: *db.Db) !void {
/// Exported for `querylog_migrations.zig`'s fixture and migration tests: the
/// authority on projection coherence is this file, and a second copy of the
/// recompute SQL there would be free to drift from the writer it checks.
pub fn expectProjectionsMatchRecompute(database: *db.Db) !void {
for (recompute_checks) |check| {
var buf: [4096]u8 = undefined;
const sql = try std.fmt.bufPrint(
+40 -25
View File
@@ -28,7 +28,9 @@ const model = @import("../config/model.zig");
const validate = @import("../config/validate.zig");
const db = @import("db.zig");
const migrations = @import("migrations.zig");
const querylog_migrations = @import("querylog_migrations.zig");
const querylog_schema = @import("querylog_schema.zig");
const querylog_versions = @import("querylog_versions.zig");
const testing = std.testing;
@@ -354,7 +356,7 @@ test "S7 case 1: querylog open on a fresh directory creates the schema" {
try testing.expectEqual(querylog_schema.RecreateReason.missing, result.recreated.?);
try testing.expectEqual(
@as(i64, querylog_schema.fingerprint),
@as(i64, querylog_versions.current_version),
try result.database.queryInt("PRAGMA user_version"),
);
try testing.expectEqual(
@@ -387,7 +389,7 @@ test "S7 case 2: reopening a healthy querylog recreates nothing" {
try testing.expectEqual(@as(usize, 0), asides.items.items.len);
}
test "S7 case 3: a wrong user_version recreates and keeps the old file aside" {
test "S7 case 3: a version this build cannot handle refuses and touches nothing" {
if (!build_options.integration) return error.SkipZigTest;
var f: Fixture = .init();
@@ -396,29 +398,42 @@ test "S7 case 3: a wrong user_version recreates and keeps the old file aside" {
var buf: [path_buf_len]u8 = undefined;
const path = try f.pathZ(&buf, "querylog.db");
try stampUserVersion(path, querylog_schema.fingerprint +% 1);
const original = try f.read("querylog.db");
defer testing.allocator.free(original);
querylog_migrations.expected_failures.begin();
defer querylog_migrations.expected_failures.end();
var result = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer result.database.close();
try testing.expectEqual(
querylog_schema.RecreateReason.fingerprint_mismatch,
result.recreated.?,
);
// Both refusal lanes, against a checkpointed file with no sidecars beside
// it: a stamp above this build's version (a downgrade) and a stamp that is
// not a version at all (a pre-0.0.12 fingerprint, or a foreign file).
const refusals = [_]struct { stamp: i32, expected: anyerror }{
.{ .stamp = querylog_versions.current_version + 1, .expected = error.SchemaTooNew },
.{ .stamp = 0, .expected = error.SchemaUnsupported },
.{ .stamp = -7, .expected = error.SchemaUnsupported },
.{ .stamp = 603440875, .expected = error.SchemaUnsupported },
};
var asides = try collectAsides(&f);
defer asides.deinit();
try testing.expectEqual(@as(usize, 1), asides.items.items.len);
for (refusals) |lane| {
try stampUserVersion(path, lane.stamp);
try testing.expect(!try f.exists("querylog.db-wal"));
// The file was healthy: this build's schema moved, the database did not rot.
// An operator who reads "corrupt" here deletes a file that was never broken.
try testing.expect(std.mem.startsWith(u8, asides.items.items[0], "querylog.db.schema-changed-"));
const original = try f.read("querylog.db");
defer testing.allocator.free(original);
const kept = try f.read(asides.items.items[0]);
defer testing.allocator.free(kept);
try testing.expectEqualSlices(u8, original, kept);
try testing.expectError(
lane.expected,
querylog_schema.open(io, std.Io.Dir.cwd(), path),
);
// Byte-identical, not merely "still readable": nothing was rewritten,
// no aside was made, and no fresh database was created beside it.
const after = try f.read("querylog.db");
defer testing.allocator.free(after);
try testing.expectEqualSlices(u8, original, after);
var asides = try collectAsides(&f);
defer asides.deinit();
try testing.expectEqual(@as(usize, 0), asides.items.items.len);
}
}
test "S7 case 4: a garbage file recreates and the garbage is preserved" {
@@ -468,11 +483,11 @@ test "S7 case 5: two recreates in the same second produce two distinct aside fil
var round: usize = 0;
while (round < 2) : (round += 1) {
try stampUserVersion(path, querylog_schema.fingerprint +% 1);
try f.write("querylog.db", "not a database at all");
var result = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer result.database.close();
try testing.expectEqual(
querylog_schema.RecreateReason.fingerprint_mismatch,
querylog_schema.RecreateReason.not_a_database,
result.recreated.?,
);
}
@@ -492,7 +507,7 @@ test "S7 case 6: a stale write-ahead log is removed before the fresh database is
var buf: [path_buf_len]u8 = undefined;
const path = try f.pathZ(&buf, "querylog.db");
try stampUserVersion(path, querylog_schema.fingerprint +% 1);
try f.write("querylog.db", "not a database at all");
// Existence alone proves nothing: the fresh database turns WAL on again and
// writes its own `-wal`. The marker is what distinguishes the stale file
@@ -503,7 +518,7 @@ test "S7 case 6: a stale write-ahead log is removed before the fresh database is
var result = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
defer result.database.close();
try testing.expectEqual(
querylog_schema.RecreateReason.fingerprint_mismatch,
querylog_schema.RecreateReason.not_a_database,
result.recreated.?,
);
@@ -591,7 +606,7 @@ test "S7 case 23: a locked querylog propagates Busy and is never destroyed" {
defer reopened.database.close();
try testing.expectEqual(@as(?querylog_schema.RecreateReason, null), reopened.recreated);
try testing.expectEqual(
@as(i64, querylog_schema.fingerprint),
@as(i64, querylog_versions.current_version),
try reopened.database.queryInt("PRAGMA user_version"),
);
+88
View File
@@ -0,0 +1,88 @@
-- FROZEN FIXTURE. Representative content for a querylog.db at schema version 1,
-- loaded on top of `querylog-v1-schema.sql`.
--
-- IMMUTABLE once released, for the same reason as its schema half: the release
-- gate byte-compares it against the previous tag. A future schema version ships
-- a NEW pair rather than editing this one.
--
-- The rows are chosen to be hard on a migration rather than realistic: every
-- `route_kind`, the NULL variants of `qtype`, `cache_hit`, `response_time_us`,
-- `upstream` and `forward_zone`, a non-IN qclass with a non-zero rcode, the
-- group and source id/name pairs, `cname_target`/`safe_search_target`, an
-- `available_since` that has been advanced away from its DDL default, and
-- timestamps that straddle two 30-minute projection buckets.
--
-- The `bucket_*` rows below are the recomputation of the raw rows, transcribed
-- from the same SQL `queries_repo`'s coherence oracle recomputes with. A
-- fixture-validity test runs that oracle over this file BEFORE any migration,
-- so an incoherent transcription fails on its own rather than as a migration
-- bug.
UPDATE querylog_meta SET created_at = 1699998000, available_since = 1699998600 WHERE id = 1;
INSERT INTO domains (id, domain) VALUES
(1, 'ads.example'),
(2, 'news.example'),
(3, 'chat.example'),
(4, 'printer.lan'),
(5, 'nas.lan'),
(6, 'cdn.example');
INSERT INTO query_log (
id, timestamp, domain_id, client_ip, qtype, blocked, response_time_us,
cache_hit, upstream, qclass, rcode, group_id, group_name, policy_action,
policy_reason, matched, source_id, source_name, cname_target,
safe_search_target, route_kind, forward_zone
) VALUES
(1, 1699999260, 1, '10.0.0.1', 1, 1, NULL, 0, NULL, 1, 0, 7, 'kids',
'block', 'blocklist_domain', 'ads.example', 3, 'stevenblack', NULL, NULL,
'blocked', NULL),
(2, 1699999320, 2, '10.0.0.1', 28, 0, 1500, 0, '9.9.9.9:853', 1, 0, NULL,
NULL, 'allow', 'no_match', NULL, NULL, NULL, NULL, NULL, 'upstream', NULL),
(3, 1699999380, 2, '10.0.0.2', 1, 0, 90, 1, NULL, 1, 0, NULL, NULL,
'allow', 'no_match', NULL, NULL, NULL, NULL, NULL, 'cache', NULL),
(4, 1699999440, 3, '10.0.0.2', NULL, 0, NULL, NULL, NULL, 3, 4, NULL, NULL,
'not_evaluated', 'non_in_class', NULL, NULL, NULL, NULL, NULL, 'rejected',
NULL),
(5, 1699999500, 4, '10.0.0.3', 1, 0, 200, 0, NULL, 1, 0, NULL, NULL,
'not_evaluated', 'local_record', NULL, NULL, NULL, NULL, NULL, 'local',
NULL),
(6, 1700000700, 5, '10.0.0.3', 15, 0, 3400, 0, NULL, 1, 0, NULL, NULL,
'not_evaluated', 'forward_zone', NULL, NULL, NULL, NULL, NULL,
'forward_zone', 'lan.example'),
(7, 1700001060, 1, '10.0.0.1', 1, 1, NULL, 0, NULL, 1, 0, 7, 'kids',
'block', 'blocklist_wildcard', '*.ads.example', 3, 'stevenblack', NULL,
NULL, 'blocked', NULL),
(8, 1700001120, 6, '10.0.0.4', 65, 0, 2500, 1, '1.1.1.1:853', 1, 0, NULL,
NULL, 'allow', 'no_match', NULL, NULL, NULL, 'edge.cdn.example',
'forcesafesearch.example', 'upstream', NULL);
INSERT INTO bucket_totals (bucket, queries, blocked, cached, rt_sum, rt_count) VALUES
(1699999200, 6, 1, 1, 5190, 4),
(1700001000, 2, 1, 1, 2500, 1);
INSERT INTO bucket_clients (bucket, client_ip, queries) VALUES
(1699999200, '10.0.0.1', 2),
(1699999200, '10.0.0.2', 2),
(1699999200, '10.0.0.3', 2),
(1700001000, '10.0.0.1', 1),
(1700001000, '10.0.0.4', 1);
-- qtype -1 is the lossless encoding of the NULL qtype on row 4.
INSERT INTO bucket_types (bucket, qtype, count) VALUES
(1699999200, -1, 1),
(1699999200, 1, 3),
(1699999200, 15, 1),
(1699999200, 28, 1),
(1700001000, 1, 1),
(1700001000, 65, 1);
INSERT INTO bucket_routes (bucket, route_kind, source_present, source_text, count) VALUES
(1699999200, 'blocked', 0, '', 1),
(1699999200, 'cache', 0, '', 1),
(1699999200, 'forward_zone', 1, 'lan.example', 1),
(1699999200, 'local', 0, '', 1),
(1699999200, 'rejected', 0, '', 1),
(1699999200, 'upstream', 1, '9.9.9.9:853', 1),
(1700001000, 'blocked', 0, '', 1),
(1700001000, 'upstream', 1, '1.1.1.1:853', 1);
+88
View File
@@ -0,0 +1,88 @@
-- FROZEN FIXTURE. querylog.db schema version 1: byte-for-byte the `ddl` text of
-- `src/storage/querylog_schema.zig`, which is the schema the 0.0.12 and 0.0.13
-- binaries created and the one version 1 names.
--
-- IMMUTABLE once released. The release gate byte-compares this file against the
-- previous tag and fails the cut on any edit, because it is the starting point
-- every future migration is proved against: editing it would prove a migration
-- against a file no operator ever had. A new schema version ships a NEW pair.
--
-- The `PRAGMA user_version` stamp is deliberately NOT part of this file. The
-- loader applies it, which is what lets one fixture serve both the version-1
-- stamp and the 0.0.12/0.0.13 legacy fingerprint.
CREATE TABLE domains (
id INTEGER PRIMARY KEY,
domain TEXT NOT NULL UNIQUE
);
CREATE TABLE query_log (
id INTEGER PRIMARY KEY,
timestamp INTEGER NOT NULL,
domain_id INTEGER NOT NULL REFERENCES domains(id),
client_ip TEXT NOT NULL, -- text, not a FK: log rows are immutable facts
qtype INTEGER,
blocked INTEGER NOT NULL,
response_time_us INTEGER,
cache_hit INTEGER,
upstream TEXT,
qclass INTEGER NOT NULL,
rcode INTEGER NOT NULL,
group_id INTEGER, -- text/id pairs, not FKs: a renamed
group_name TEXT, -- group must not rewrite history
policy_action TEXT NOT NULL,
policy_reason TEXT NOT NULL,
matched TEXT,
source_id INTEGER,
source_name TEXT,
cname_target TEXT,
safe_search_target TEXT,
route_kind TEXT NOT NULL,
forward_zone TEXT,
CHECK (rcode BETWEEN 0 AND 4095) -- twelve bits (RFC 6891 6.1.3)
);
CREATE INDEX idx_query_log_ts ON query_log(timestamp);
CREATE INDEX idx_query_log_client ON query_log(client_ip);
CREATE INDEX idx_query_log_domain ON query_log(domain_id);
CREATE TABLE querylog_meta (
id INTEGER PRIMARY KEY CHECK (id = 1), -- one row, enforced by the schema
created_at INTEGER NOT NULL,
available_since INTEGER NOT NULL
);
INSERT INTO querylog_meta (id, created_at, available_since)
VALUES (1, unixepoch(), unixepoch() + 1);
CREATE TABLE bucket_totals (
bucket INTEGER PRIMARY KEY,
queries INTEGER NOT NULL,
blocked INTEGER NOT NULL,
cached INTEGER NOT NULL,
rt_sum INTEGER NOT NULL, -- sum(response_time_us) over timed rows
rt_count INTEGER NOT NULL -- count(response_time_us)
) WITHOUT ROWID;
CREATE TABLE bucket_clients (
bucket INTEGER NOT NULL,
client_ip TEXT NOT NULL,
queries INTEGER NOT NULL,
PRIMARY KEY (bucket, client_ip)
) WITHOUT ROWID;
CREATE TABLE bucket_types (
bucket INTEGER NOT NULL,
qtype INTEGER NOT NULL, -- -1 encodes a NULL qtype, losslessly
count INTEGER NOT NULL,
PRIMARY KEY (bucket, qtype)
) WITHOUT ROWID;
CREATE TABLE bucket_routes (
bucket INTEGER NOT NULL,
route_kind TEXT NOT NULL,
source_present INTEGER NOT NULL, -- 0: source NULL; 1: source = source_text
source_text TEXT NOT NULL, -- '' when source_present = 0
count INTEGER NOT NULL,
PRIMARY KEY (bucket, route_kind, source_present, source_text),
CHECK (source_present IN (0, 1)),
CHECK (source_present = 1 OR source_text = '')
) WITHOUT ROWID;