storage: version querylog.db and migrate it in place, never reset a healthy file
Gates / frontend (push) Successful in 2m8s
Gates / test (push) Successful in 2m46s
Gates / test-aarch64 (push) Successful in 8m38s
Gates / package (push) Successful in 4m39s
Gates / container (push) Successful in 15s
CI / gates (push) Successful in 31m58s
Gates / frontend (push) Successful in 2m8s
Gates / test (push) Successful in 2m46s
Gates / test-aarch64 (push) Successful in 8m38s
Gates / package (push) Successful in 4m39s
Gates / container (push) Successful in 15s
CI / gates (push) Successful in 31m58s
querylog.db carries a schema version; migrations run at startup as one transaction after a vacuumed 0600 backup, and every failure refuses startup (exit 2, no systemd restart loop) instead of starting empty. corruption is the only automatic recreate left. the cut gate now requires a fixture-proven migration or an explicit versioned break with restore instructions, and locks shipped migration files and fixtures byte-for-byte.
This commit is contained in:
@@ -54,9 +54,24 @@ pub const c = struct {
|
||||
pub extern fn sqlite3_column_count(stmt: *c.Stmt) c_int;
|
||||
pub extern fn sqlite3_column_type(stmt: *c.Stmt, col: c_int) c_int;
|
||||
pub extern fn sqlite3_column_int64(stmt: *c.Stmt, col: c_int) i64;
|
||||
pub extern fn sqlite3_column_double(stmt: *c.Stmt, col: c_int) f64;
|
||||
pub extern fn sqlite3_column_text(stmt: *c.Stmt, col: c_int) ?[*]const u8;
|
||||
pub extern fn sqlite3_column_bytes(stmt: *c.Stmt, col: c_int) c_int;
|
||||
pub extern fn sqlite3_set_authorizer(db: *Sqlite3, xAuth: ?*const AuthCallback, user_data: ?*anyopaque) c_int;
|
||||
pub extern fn sqlite3_get_autocommit(db: *Sqlite3) c_int;
|
||||
pub extern fn sqlite3_last_insert_rowid(db: *Sqlite3) i64;
|
||||
|
||||
/// `int (*)(void*, int, const char*, const char*, const char*, const char*)`.
|
||||
/// The four name arguments are NULL for actions that do not use them, which
|
||||
/// `SQLITE_TRANSACTION` is except for its operation name.
|
||||
pub const AuthCallback = fn (
|
||||
user_data: ?*anyopaque,
|
||||
action: c_int,
|
||||
arg1: ?[*:0]const u8,
|
||||
arg2: ?[*:0]const u8,
|
||||
database: ?[*:0]const u8,
|
||||
trigger_or_view: ?[*:0]const u8,
|
||||
) callconv(.c) c_int;
|
||||
pub extern fn sqlite3_changes(db: *Sqlite3) c_int;
|
||||
pub extern fn sqlite3_total_changes(db: *Sqlite3) c_int;
|
||||
};
|
||||
@@ -105,6 +120,20 @@ pub const open_flag = struct {
|
||||
pub const exrescode: c_int = 0x2000000;
|
||||
};
|
||||
|
||||
/// The authorizer verdicts and the one action code nxdns denies, from the
|
||||
/// vendored `sqlite3.h` (3.53.4). `SQLITE_DENY` fails the *prepare* with
|
||||
/// `SQLITE_AUTH`, which is what makes it a real guard: the statement never
|
||||
/// runs at all.
|
||||
pub const auth = struct {
|
||||
pub const deny: c_int = 1;
|
||||
pub const transaction: c_int = 22;
|
||||
/// `SAVEPOINT`, `RELEASE` and `ROLLBACK TO` report under this code, not
|
||||
/// under `transaction`. A savepoint at the outermost level opens a real
|
||||
/// transaction and `RELEASE` commits it, so a step using one splits the
|
||||
/// migration exactly as a bare `COMMIT` would.
|
||||
pub const savepoint: c_int = 32;
|
||||
};
|
||||
|
||||
/// Column type codes returned by `sqlite3_column_type`.
|
||||
pub const column_type = struct {
|
||||
pub const integer: c_int = 1;
|
||||
@@ -367,6 +396,7 @@ pub const Db = struct {
|
||||
/// message is read back through `sqlite3_errmsg`, so there is no
|
||||
/// `sqlite3_free` obligation.
|
||||
pub fn exec(self: *Db, sql: [:0]const u8) Error!void {
|
||||
if (execFaultTripped(sql)) return error.Internal;
|
||||
return check(c.sqlite3_exec(self.handle, sql.ptr, null, null, null));
|
||||
}
|
||||
|
||||
@@ -424,6 +454,38 @@ pub const Db = struct {
|
||||
return value;
|
||||
}
|
||||
|
||||
/// False while a transaction is open on this connection. The migration
|
||||
/// runner's belt check: a step that somehow ended the runner's transaction
|
||||
/// must not be allowed to look like a success.
|
||||
pub fn inTransaction(self: *Db) bool {
|
||||
return c.sqlite3_get_autocommit(self.handle) == 0;
|
||||
}
|
||||
|
||||
/// Denies every transaction statement — `BEGIN`, `COMMIT`, `ROLLBACK`,
|
||||
/// `SAVEPOINT`, `RELEASE` — until `clearAuthorizer` runs.
|
||||
///
|
||||
/// `guard` must outlive the installed window: SQLite keeps the pointer.
|
||||
/// Install and clear are a scoped pair; the migration runner clears on
|
||||
/// every exit path, because a leaked authorizer would go on denying the
|
||||
/// ROLLBACK that cleans up after the very statement it rejected.
|
||||
pub fn denyTransactions(self: *Db, guard: *TransactionGuard) Error!void {
|
||||
guard.* = .{};
|
||||
return check(c.sqlite3_set_authorizer(self.handle, transactionAuthorizer, guard));
|
||||
}
|
||||
|
||||
/// Never fails in a way the caller can act on: passing a null callback only
|
||||
/// clears state SQLite already holds. A failure is logged and swallowed so
|
||||
/// this stays usable in `defer`.
|
||||
pub fn clearAuthorizer(self: *Db) void {
|
||||
const rc = c.sqlite3_set_authorizer(self.handle, null, null);
|
||||
if (rc != result.ok) {
|
||||
log.err("sqlite3_set_authorizer(null) returned {s} (code {d})", .{
|
||||
std.mem.span(c.sqlite3_errstr(rc)),
|
||||
rc,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn lastInsertRowid(self: *Db) i64 {
|
||||
return c.sqlite3_last_insert_rowid(self.handle);
|
||||
}
|
||||
@@ -442,6 +504,34 @@ pub const Db = struct {
|
||||
}
|
||||
};
|
||||
|
||||
/// Records whether the authorizer installed by `Db.denyTransactions` actually
|
||||
/// rejected anything. The rejection reaches the caller as `error.Auth`, which
|
||||
/// is indistinguishable from any other authorization failure; this flag is what
|
||||
/// lets the migration runner name the real cause.
|
||||
pub const TransactionGuard = struct {
|
||||
denied: bool = false,
|
||||
};
|
||||
|
||||
fn transactionAuthorizer(
|
||||
user_data: ?*anyopaque,
|
||||
action: c_int,
|
||||
arg1: ?[*:0]const u8,
|
||||
arg2: ?[*:0]const u8,
|
||||
database: ?[*:0]const u8,
|
||||
trigger_or_view: ?[*:0]const u8,
|
||||
) callconv(.c) c_int {
|
||||
_ = arg2;
|
||||
_ = database;
|
||||
_ = trigger_or_view;
|
||||
if (action != auth.transaction and action != auth.savepoint) return result.ok;
|
||||
const guard: *TransactionGuard = @ptrCast(@alignCast(user_data.?));
|
||||
guard.denied = true;
|
||||
log.warn("migration step attempted a transaction statement: {s}", .{
|
||||
if (arg1) |op| std.mem.span(op) else "(unnamed)",
|
||||
});
|
||||
return auth.deny;
|
||||
}
|
||||
|
||||
fn openHandle(filename: [:0]const u8, flags: c_int) Error!*c.Sqlite3 {
|
||||
var handle: ?*c.Sqlite3 = null;
|
||||
const rc = c.sqlite3_open_v2(filename.ptr, &handle, flags, null);
|
||||
@@ -963,6 +1053,45 @@ pub const read_tx_faults = if (builtin.is_test) struct {
|
||||
}
|
||||
} else struct {};
|
||||
|
||||
/// Fails one chosen `Db.exec` so a test can drive a failure SQLite itself will
|
||||
/// not produce on demand. Test builds only, same shape as `read_tx_seam`.
|
||||
///
|
||||
/// The migration paths this exists for — the legacy restamp and the
|
||||
/// post-commit pragma restore — run statements that always succeed against a
|
||||
/// healthy file, and their recovery behaviour is the whole point of the
|
||||
/// milestone. Matching on the SQL text rather than counting calls keeps a test
|
||||
/// naming the statement it means.
|
||||
const exec_seam = if (builtin.is_test) struct {
|
||||
var fail_matching: ?[]const u8 = null;
|
||||
} else struct {};
|
||||
|
||||
fn execFaultTripped(sql: []const u8) bool {
|
||||
if (!builtin.is_test) return false;
|
||||
const needle = exec_seam.fail_matching orelse return false;
|
||||
if (std.mem.indexOf(u8, sql, needle) == null) return false;
|
||||
exec_seam.fail_matching = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// The seam's controls, for tests in this file and in the storage layer.
|
||||
pub const exec_faults = if (builtin.is_test) struct {
|
||||
/// Arms the next `Db.exec` whose SQL contains `needle` to fail with
|
||||
/// `error.Internal` before the statement reaches SQLite. One shot: it
|
||||
/// disarms itself when it trips. Pair it with `defer disarm()` so a test
|
||||
/// that never trips the fault cannot leak it into the next one.
|
||||
pub fn failNextMatching(needle: []const u8) void {
|
||||
exec_seam.fail_matching = needle;
|
||||
}
|
||||
|
||||
pub fn disarm() void {
|
||||
exec_seam.fail_matching = null;
|
||||
}
|
||||
|
||||
pub fn armed() bool {
|
||||
return exec_seam.fail_matching != null;
|
||||
}
|
||||
} else struct {};
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
fn openMemory() Error!Db {
|
||||
@@ -1479,3 +1608,57 @@ test "a duplicate insert into a UNIQUE column returns error.Constraint" {
|
||||
try stmt.bindText(1, "only");
|
||||
try testing.expectError(error.Constraint, stmt.step());
|
||||
}
|
||||
|
||||
test "the transaction authorizer denies transaction statements and clears cleanly" {
|
||||
var db = try openMemory();
|
||||
defer db.close();
|
||||
try db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY);");
|
||||
|
||||
try db.exec("BEGIN IMMEDIATE;");
|
||||
try testing.expect(db.inTransaction());
|
||||
|
||||
var guard: TransactionGuard = .{};
|
||||
try db.denyTransactions(&guard);
|
||||
// Ordinary work still runs: only the transaction statements are refused.
|
||||
try db.exec("INSERT INTO t (id) VALUES (1);");
|
||||
try testing.expect(!guard.denied);
|
||||
try testing.expectError(error.Auth, db.exec("COMMIT;"));
|
||||
try testing.expect(guard.denied);
|
||||
// The deny happens at prepare, so the transaction is still open.
|
||||
try testing.expect(db.inTransaction());
|
||||
|
||||
// SAVEPOINT and its RELEASE report under a different action code, and they
|
||||
// are the same bypass: at the outermost level they are a transaction under
|
||||
// another name, and inside one they can still discard the migration's work.
|
||||
guard.denied = false;
|
||||
try testing.expectError(error.Auth, db.exec("SAVEPOINT half_a_migration;"));
|
||||
try testing.expect(guard.denied);
|
||||
guard.denied = false;
|
||||
try testing.expectError(error.Auth, db.exec("RELEASE half_a_migration;"));
|
||||
try testing.expect(guard.denied);
|
||||
try testing.expect(db.inTransaction());
|
||||
|
||||
db.clearAuthorizer();
|
||||
// The same connection is usable again — a leaked authorizer would strand it
|
||||
// inside the transaction by denying this too.
|
||||
try db.exec("ROLLBACK;");
|
||||
try testing.expect(!db.inTransaction());
|
||||
try testing.expectEqual(@as(i64, 0), try db.queryInt("SELECT count(*) FROM t"));
|
||||
}
|
||||
|
||||
test "the exec fault seam fires once, on the statement it names" {
|
||||
var db = try openMemory();
|
||||
defer db.close();
|
||||
try db.exec("CREATE TABLE t (id INTEGER PRIMARY KEY);");
|
||||
|
||||
exec_faults.failNextMatching("COMMIT");
|
||||
defer exec_faults.disarm();
|
||||
|
||||
try db.exec("BEGIN IMMEDIATE;");
|
||||
try db.exec("INSERT INTO t (id) VALUES (1);");
|
||||
try testing.expectError(error.Internal, db.exec("COMMIT;"));
|
||||
try testing.expect(!exec_faults.armed());
|
||||
// Disarmed: the retry is a real COMMIT.
|
||||
try db.exec("COMMIT;");
|
||||
try testing.expectEqual(@as(i64, 1), try db.queryInt("SELECT count(*) FROM t"));
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ const disk_monitor = @import("disk_monitor.zig");
|
||||
const logger = @import("logger.zig");
|
||||
const queries_repo = @import("repositories/queries_repo.zig");
|
||||
const querylog_schema = @import("querylog_schema.zig");
|
||||
const querylog_versions = @import("querylog_versions.zig");
|
||||
const retention = @import("retention.zig");
|
||||
|
||||
const testing = std.testing;
|
||||
@@ -233,7 +234,7 @@ test "S8 case 1: the logger writes a real querylog.db end to end" {
|
||||
try testing.expectEqual(@as(i64, 250), try queries_repo.countRows(log_db.database()));
|
||||
try testing.expectEqual(@as(i64, 10), try queries_repo.countDomains(log_db.database()));
|
||||
try testing.expectEqual(
|
||||
@as(i64, querylog_schema.fingerprint),
|
||||
@as(i64, querylog_versions.current_version),
|
||||
try log_db.database().queryInt("PRAGMA user_version"),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,535 @@
|
||||
//! The shipped `querylog.db` fixtures, and the proof that every supported
|
||||
//! schema version reaches the current one with the operator's rows intact.
|
||||
//!
|
||||
//! A file of its own, not a section of `querylog_migrations.zig`, because of the
|
||||
//! link contract that split `querylog_versions.zig` out in the first place. The
|
||||
//! assertions here need `repositories/queries_repo.zig`'s projection-coherence
|
||||
//! oracle, and that file reaches across `src/` for the config and filter types;
|
||||
//! importing it from `querylog_migrations.zig` would pull all of it into the
|
||||
//! module `tools/cut.zig` builds `querylog_schema.zig` as, where those paths lie
|
||||
//! outside the module root and do not compile.
|
||||
//!
|
||||
//! The fixtures themselves — `testdata/querylog-v<N>-{schema,data}.sql` — are
|
||||
//! immutable once released. Every version in `[minimum_supported_version,
|
||||
//! current_version]` has a pair: the current version's pair is the next
|
||||
//! migration's starting point, and an explicit break ships the new baseline.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const db = @import("db.zig");
|
||||
const migrations = @import("querylog_migrations.zig");
|
||||
const queries_repo = @import("repositories/queries_repo.zig");
|
||||
const querylog_schema = @import("querylog_schema.zig");
|
||||
const versions = @import("querylog_versions.zig");
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
/// A temporary directory and the `querylog.db` path inside it. Deliberately not
|
||||
/// `querylog_migrations.zig`'s test harness: that one builds synthetic schemas,
|
||||
/// while everything here starts from the shipped fixture files.
|
||||
const Harness = struct {
|
||||
threaded: std.Io.Threaded,
|
||||
tmp: std.testing.TmpDir,
|
||||
buf: [256]u8 = undefined,
|
||||
|
||||
fn init() Harness {
|
||||
return .{
|
||||
.threaded = .init(testing.allocator, .{}),
|
||||
.tmp = testing.tmpDir(.{ .iterate = true }),
|
||||
};
|
||||
}
|
||||
|
||||
fn deinit(self: *Harness) void {
|
||||
self.tmp.cleanup();
|
||||
self.threaded.deinit();
|
||||
}
|
||||
|
||||
fn io(self: *Harness) std.Io {
|
||||
return self.threaded.io();
|
||||
}
|
||||
|
||||
fn path(self: *Harness) [:0]const u8 {
|
||||
return std.fmt.bufPrintZ(&self.buf, ".zig-cache/tmp/{s}/querylog.db", .{self.tmp.sub_path}) catch
|
||||
unreachable;
|
||||
}
|
||||
|
||||
fn openLive(self: *Harness) !db.Db {
|
||||
var database = try db.Db.open(self.path(), .{ .mode = .read_write_existing });
|
||||
errdefer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
return database;
|
||||
}
|
||||
};
|
||||
|
||||
/// One shipped schema version's frozen pair. Both halves are immutable once
|
||||
/// released — the release cut byte-compares them against the previous tag — and
|
||||
/// every version in `[minimum_supported_version, current_version]` must have a
|
||||
/// pair, which the cut gate also enforces.
|
||||
const Fixture = struct {
|
||||
version: i32,
|
||||
schema: [:0]const u8,
|
||||
data: [:0]const u8,
|
||||
};
|
||||
|
||||
const fixtures = [_]Fixture{
|
||||
.{
|
||||
.version = 1,
|
||||
.schema = @embedFile("testdata/querylog-v1-schema.sql"),
|
||||
.data = @embedFile("testdata/querylog-v1-data.sql"),
|
||||
},
|
||||
};
|
||||
|
||||
fn fixtureFor(version: i32) ?Fixture {
|
||||
for (fixtures) |fixture| {
|
||||
if (fixture.version == version) return fixture;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Writes a fixture pair to `path` and stamps it. `stamp` is a parameter rather
|
||||
/// than `fixture.version` because the legacy-fingerprint file is the same
|
||||
/// version-1 bytes under a different stamp.
|
||||
fn writeFixture(path: [:0]const u8, fixture: Fixture, stamp: i32) !void {
|
||||
var database = try db.Db.open(path, .{ .mode = .read_write_create });
|
||||
defer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
try database.exec(fixture.schema);
|
||||
try database.exec(fixture.data);
|
||||
|
||||
var buf: [64]u8 = undefined;
|
||||
const sql = std.fmt.bufPrintZ(&buf, "PRAGMA user_version = {d};", .{stamp}) catch unreachable;
|
||||
try database.exec(sql);
|
||||
}
|
||||
|
||||
/// A second path in the harness's directory, for the reference databases the
|
||||
/// assertions below compare against.
|
||||
fn sidePath(h: *Harness, buf: []u8, name: []const u8) [:0]const u8 {
|
||||
return std.fmt.bufPrintZ(buf, ".zig-cache/tmp/{s}/{s}", .{ h.tmp.sub_path, name }) catch unreachable;
|
||||
}
|
||||
|
||||
/// A database holding nothing but the current `ddl`, which is what a file
|
||||
/// created by this build is.
|
||||
fn openFreshCurrent(h: *Harness, buf: []u8) !db.Db {
|
||||
var database = try db.Db.open(sidePath(h, buf, "fresh.db"), .{ .mode = .read_write_create });
|
||||
errdefer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
try database.exec(querylog_schema.ddl);
|
||||
return database;
|
||||
}
|
||||
|
||||
/// An untouched load of `fixture`, to compare a migrated or opened file against
|
||||
/// rather than restating the fixture's contents in the assertions.
|
||||
fn openPristine(h: *Harness, buf: []u8, fixture: Fixture) !db.Db {
|
||||
const path = sidePath(h, buf, "pristine.db");
|
||||
try writeFixture(path, fixture, fixture.version);
|
||||
var database = try db.Db.open(path, .{ .mode = .read_write_existing });
|
||||
errdefer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
return database;
|
||||
}
|
||||
|
||||
/// Every row of every table, as one canonical text. Exact equality is only the
|
||||
/// right question for a file no migration has reshaped; a migrated file is
|
||||
/// checked by the counts and the watermark instead.
|
||||
fn dumpContent(gpa: std.mem.Allocator, database: *db.Db, out: *std.ArrayList(u8)) !void {
|
||||
var tables: std.ArrayList([]u8) = .empty;
|
||||
defer freeOwned(gpa, &tables);
|
||||
{
|
||||
var stmt = try database.prepare(
|
||||
\\SELECT name FROM sqlite_schema
|
||||
\\WHERE type = 'table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\'
|
||||
\\ORDER BY name
|
||||
);
|
||||
defer stmt.deinit();
|
||||
// Copied out before the per-table statements step: a borrowed
|
||||
// `columnText` would not survive them.
|
||||
while (try stmt.step()) try tables.append(gpa, try stmt.columnTextAlloc(gpa, 0));
|
||||
}
|
||||
|
||||
// The lines are sorted here rather than by the query: the four `bucket_*`
|
||||
// tables are WITHOUT ROWID, so `ORDER BY rowid` is not available to all of
|
||||
// them and no single column list is.
|
||||
var lines: std.ArrayList([]u8) = .empty;
|
||||
defer freeOwned(gpa, &lines);
|
||||
|
||||
for (tables.items) |table| {
|
||||
var sql_buf: [256]u8 = undefined;
|
||||
const sql = std.fmt.bufPrint(&sql_buf, "SELECT * FROM \"{s}\"", .{table}) catch unreachable;
|
||||
var stmt = try database.prepare(sql);
|
||||
defer stmt.deinit();
|
||||
while (try stmt.step()) {
|
||||
var line: std.ArrayList(u8) = .empty;
|
||||
errdefer line.deinit(gpa);
|
||||
try line.print(gpa, "R|{s}", .{table});
|
||||
var col: c_int = 0;
|
||||
const columns: c_int = db.c.sqlite3_column_count(stmt.handle);
|
||||
while (col < columns) : (col += 1) {
|
||||
try line.print(gpa, "|{s}", .{stmt.columnTextOrNull(col) orelse "<null>"});
|
||||
}
|
||||
try lines.append(gpa, try line.toOwnedSlice(gpa));
|
||||
}
|
||||
}
|
||||
|
||||
std.mem.sortUnstable([]u8, lines.items, {}, struct {
|
||||
fn lessThan(_: void, a: []u8, b: []u8) bool {
|
||||
return std.mem.lessThan(u8, a, b);
|
||||
}
|
||||
}.lessThan);
|
||||
for (lines.items) |line| {
|
||||
try out.appendSlice(gpa, line);
|
||||
try out.append(gpa, '\n');
|
||||
}
|
||||
}
|
||||
|
||||
/// One value, encoded so that no two different values can produce the same
|
||||
/// bytes: a type tag, the byte length, then the bytes themselves.
|
||||
///
|
||||
/// Nothing here is a sentinel and nothing is escaped, which is the point. A
|
||||
/// serialization that wrote NULL as `<null>` cannot tell a NULL apart from the
|
||||
/// six-character string of the same name, and one that separated values with
|
||||
/// `|` cannot tell `a|b` in one column from `a` and `b` in two — so a migration
|
||||
/// that turned a NULL `upstream` into text, or shifted a value from one column
|
||||
/// into its neighbour, would compare EQUAL to the original. The length prefix
|
||||
/// makes the stream uniquely decodable, so equal encodings mean equal rows.
|
||||
///
|
||||
/// A float travels as its bit pattern rather than as printed digits: the
|
||||
/// question here is whether the value survived, not whether it rounds the same.
|
||||
fn writeValue(gpa: std.mem.Allocator, stmt: *db.Stmt, col: c_int, out: *std.ArrayList(u8)) !void {
|
||||
switch (db.c.sqlite3_column_type(stmt.handle, col)) {
|
||||
db.column_type.null_value => try out.print(gpa, "n0:", .{}),
|
||||
db.column_type.integer => {
|
||||
var buf: [24]u8 = undefined;
|
||||
const text = std.fmt.bufPrint(&buf, "{d}", .{stmt.columnInt(col)}) catch unreachable;
|
||||
try out.print(gpa, "i{d}:{s}", .{ text.len, text });
|
||||
},
|
||||
db.column_type.float => {
|
||||
const bits: u64 = @bitCast(db.c.sqlite3_column_double(stmt.handle, col));
|
||||
var buf: [24]u8 = undefined;
|
||||
const text = std.fmt.bufPrint(&buf, "{d}", .{bits}) catch unreachable;
|
||||
try out.print(gpa, "f{d}:{s}", .{ text.len, text });
|
||||
},
|
||||
db.column_type.blob => {
|
||||
// `columnText` on a blob hands back the same bytes SQLite stores,
|
||||
// which is what this compares; it is not read as text.
|
||||
const bytes = stmt.columnText(col);
|
||||
try out.print(gpa, "b{d}:{s}", .{ bytes.len, bytes });
|
||||
},
|
||||
else => {
|
||||
const bytes = stmt.columnText(col);
|
||||
try out.print(gpa, "t{d}:{s}", .{ bytes.len, bytes });
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// One query's rows, in the order the query returns them, tagged with `label` so
|
||||
/// that a difference names the relation it came from. The column count is part
|
||||
/// of each row for the same reason the lengths are part of each value.
|
||||
fn dumpQuery(
|
||||
gpa: std.mem.Allocator,
|
||||
database: *db.Db,
|
||||
label: []const u8,
|
||||
sql: [:0]const u8,
|
||||
out: *std.ArrayList(u8),
|
||||
) !void {
|
||||
var stmt = try database.prepare(sql);
|
||||
defer stmt.deinit();
|
||||
while (try stmt.step()) {
|
||||
const columns: c_int = db.c.sqlite3_column_count(stmt.handle);
|
||||
try out.print(gpa, "{s}:{d}:", .{ label, columns });
|
||||
var col: c_int = 0;
|
||||
while (col < columns) : (col += 1) {
|
||||
try writeValue(gpa, &stmt, col, out);
|
||||
}
|
||||
try out.append(gpa, '\n');
|
||||
}
|
||||
}
|
||||
|
||||
/// The operator's data as the application means it, in an order a migration
|
||||
/// cannot permute.
|
||||
///
|
||||
/// `q.*` rather than a column list on purpose: a migration that adds a column
|
||||
/// must show that column here, and a hand-written list would quietly stop
|
||||
/// covering the table the day it grows. `d.domain` rides along so that the text
|
||||
/// a row names is compared, not only the id it happens to hold.
|
||||
const logical_relations = [_]struct { label: []const u8, sql: [:0]const u8 }{
|
||||
.{
|
||||
.label = "query_log",
|
||||
.sql =
|
||||
\\SELECT q.*, d.domain FROM query_log q
|
||||
\\JOIN domains d ON d.id = q.domain_id
|
||||
\\ORDER BY q.id
|
||||
,
|
||||
},
|
||||
.{ .label = "domains", .sql = "SELECT * FROM domains ORDER BY id" },
|
||||
.{ .label = "querylog_meta", .sql = "SELECT * FROM querylog_meta ORDER BY id" },
|
||||
};
|
||||
|
||||
fn dumpLogical(gpa: std.mem.Allocator, database: *db.Db, out: *std.ArrayList(u8)) !void {
|
||||
for (logical_relations) |relation| {
|
||||
try dumpQuery(gpa, database, relation.label, relation.sql, out);
|
||||
}
|
||||
}
|
||||
|
||||
fn freeOwned(gpa: std.mem.Allocator, list: *std.ArrayList([]u8)) void {
|
||||
for (list.items) |item| gpa.free(item);
|
||||
list.deinit(gpa);
|
||||
}
|
||||
|
||||
fn expectSameContent(a: *db.Db, b: *db.Db) !void {
|
||||
var text_a: std.ArrayList(u8) = .empty;
|
||||
defer text_a.deinit(testing.allocator);
|
||||
var text_b: std.ArrayList(u8) = .empty;
|
||||
defer text_b.deinit(testing.allocator);
|
||||
try dumpContent(testing.allocator, a, &text_a);
|
||||
try dumpContent(testing.allocator, b, &text_b);
|
||||
try testing.expectEqualStrings(text_a.items, text_b.items);
|
||||
}
|
||||
|
||||
fn rowCount(database: *db.Db, table: []const u8) !i64 {
|
||||
var buf: [128]u8 = undefined;
|
||||
return database.queryInt(std.fmt.bufPrint(&buf, "SELECT count(*) FROM \"{s}\"", .{table}) catch unreachable);
|
||||
}
|
||||
|
||||
/// What must hold after a fixture has come through `openVersioned`, whatever
|
||||
/// path it took: every row the operator had is still there with the same
|
||||
/// CONTENT, and the projections still agree with the raw rows.
|
||||
///
|
||||
/// Counting rows and checking one watermark is what this used to do, and a
|
||||
/// migration that shifted a timestamp, dropped a `qtype` or crossed two rows'
|
||||
/// `client_ip` values passed it. The comparison is therefore the full logical
|
||||
/// content of the three operator tables, `available_since` included as one
|
||||
/// column of `querylog_meta` among the rest. The counts stay because a count
|
||||
/// difference is the failure worth naming plainly.
|
||||
///
|
||||
/// The relations are named rather than derived because they are the ones holding
|
||||
/// operator data; the `bucket_*` projections are derived from them, which
|
||||
/// `expectProjectionsMatchRecompute` is the right check for. A future migration
|
||||
/// that renames a table amends this alongside the step that does it. So does one
|
||||
/// that renumbers `id` values: this asserts they survive, which every rebuild
|
||||
/// written to the A.2 rule does.
|
||||
fn expectFixtureSurvived(opened: *db.Db, pristine: *db.Db) !void {
|
||||
for ([_][]const u8{ "query_log", "domains", "querylog_meta" }) |table| {
|
||||
try testing.expectEqual(try rowCount(pristine, table), try rowCount(opened, table));
|
||||
}
|
||||
|
||||
var opened_text: std.ArrayList(u8) = .empty;
|
||||
defer opened_text.deinit(testing.allocator);
|
||||
var pristine_text: std.ArrayList(u8) = .empty;
|
||||
defer pristine_text.deinit(testing.allocator);
|
||||
try dumpLogical(testing.allocator, opened, &opened_text);
|
||||
try dumpLogical(testing.allocator, pristine, &pristine_text);
|
||||
try testing.expectEqualStrings(pristine_text.items, opened_text.items);
|
||||
|
||||
try queries_repo.expectProjectionsMatchRecompute(opened);
|
||||
}
|
||||
|
||||
test "the survival comparison tells a NULL apart from text that looks like one" {
|
||||
// The mutation a count-and-watermark check misses entirely, and a
|
||||
// sentinel-string serialization misses just as completely: one column of one
|
||||
// row stops being NULL and becomes the very text the sentinel used. Every
|
||||
// count, the watermark and the projections all still agree.
|
||||
const fixture = fixtureFor(1) orelse return error.MissingFixture;
|
||||
|
||||
var h: Harness = .init();
|
||||
defer h.deinit();
|
||||
try writeFixture(h.path(), fixture, fixture.version);
|
||||
|
||||
var side: [256]u8 = undefined;
|
||||
var pristine = try openPristine(&h, &side, fixture);
|
||||
defer pristine.close();
|
||||
|
||||
var corrupted = try h.openLive();
|
||||
defer corrupted.close();
|
||||
try testing.expectEqual(@as(i64, 0), corrupted.changes());
|
||||
try corrupted.exec(
|
||||
\\UPDATE query_log SET upstream = '<null>'
|
||||
\\WHERE id = (SELECT min(id) FROM query_log WHERE upstream IS NULL)
|
||||
);
|
||||
// The fixture has to actually carry a NULL `upstream` for this to be a test
|
||||
// of anything.
|
||||
try testing.expectEqual(@as(i64, 1), corrupted.changes());
|
||||
|
||||
// Everything the old check looked at still agrees, which is why it passed.
|
||||
for ([_][]const u8{ "query_log", "domains", "querylog_meta" }) |table| {
|
||||
try testing.expectEqual(try rowCount(&pristine, table), try rowCount(&corrupted, table));
|
||||
}
|
||||
try testing.expectEqual(
|
||||
try pristine.queryInt("SELECT available_since FROM querylog_meta WHERE id = 1"),
|
||||
try corrupted.queryInt("SELECT available_since FROM querylog_meta WHERE id = 1"),
|
||||
);
|
||||
try queries_repo.expectProjectionsMatchRecompute(&corrupted);
|
||||
|
||||
// The dumps are compared here rather than through `expectFixtureSurvived`
|
||||
// so that a PASSING run stays silent: `expectEqualStrings` prints the whole
|
||||
// diff before it returns its error, and this is the comparison that function
|
||||
// makes.
|
||||
var corrupted_text: std.ArrayList(u8) = .empty;
|
||||
defer corrupted_text.deinit(testing.allocator);
|
||||
var pristine_text: std.ArrayList(u8) = .empty;
|
||||
defer pristine_text.deinit(testing.allocator);
|
||||
try dumpLogical(testing.allocator, &corrupted, &corrupted_text);
|
||||
try dumpLogical(testing.allocator, &pristine, &pristine_text);
|
||||
try testing.expect(!std.mem.eql(u8, corrupted_text.items, pristine_text.items));
|
||||
}
|
||||
|
||||
test "every supported version ships a fixture pair" {
|
||||
// The cut gate enforces this against a release; the suite enforces it
|
||||
// against a commit, so a version bump that forgot its fixtures fails here
|
||||
// long before anyone reaches for `zig build cut`.
|
||||
var version = versions.minimum_supported_version;
|
||||
while (version <= versions.current_version) : (version += 1) {
|
||||
try testing.expect(fixtureFor(version) != null);
|
||||
}
|
||||
}
|
||||
|
||||
test "each shipped fixture pair is coherent before any migration touches it" {
|
||||
for (fixtures) |fixture| {
|
||||
var h: Harness = .init();
|
||||
defer h.deinit();
|
||||
try writeFixture(h.path(), fixture, fixture.version);
|
||||
|
||||
var database = try h.openLive();
|
||||
defer database.close();
|
||||
try testing.expectEqual(
|
||||
@as(i64, fixture.version),
|
||||
try database.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
// An incoherent fixture has to fail as a fixture, not later as a
|
||||
// migration that appears to have corrupted the projections.
|
||||
try queries_repo.expectProjectionsMatchRecompute(&database);
|
||||
try testing.expectEqual(@as(i64, 0), try database.queryInt("SELECT count(*) FROM pragma_foreign_key_check"));
|
||||
}
|
||||
}
|
||||
|
||||
test "every fixture below the current version migrates to it through the shipped chain" {
|
||||
// Empty while the chain is: `minimum_supported_version == current_version`
|
||||
// today. It is written as the loop so that the day a step ships, the
|
||||
// fixture it starts from is proved through the REAL production plan with no
|
||||
// edit to this test.
|
||||
var version = versions.minimum_supported_version;
|
||||
while (version < versions.current_version) : (version += 1) {
|
||||
const fixture = fixtureFor(version) orelse return error.MissingFixture;
|
||||
|
||||
var h: Harness = .init();
|
||||
defer h.deinit();
|
||||
try writeFixture(h.path(), fixture, version);
|
||||
|
||||
var side: [256]u8 = undefined;
|
||||
var pristine = try openPristine(&h, &side, fixture);
|
||||
defer pristine.close();
|
||||
|
||||
var result = try querylog_schema.open(h.io(), std.Io.Dir.cwd(), h.path());
|
||||
defer result.database.close();
|
||||
try testing.expectEqual(@as(?querylog_schema.RecreateReason, null), result.recreated);
|
||||
try testing.expectEqual(
|
||||
@as(i64, versions.current_version),
|
||||
try result.database.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
|
||||
var fresh_buf: [256]u8 = undefined;
|
||||
var fresh = try openFreshCurrent(&h, &fresh_buf);
|
||||
defer fresh.close();
|
||||
try testing.expect(try migrations.schemaEquivalent(testing.allocator, &result.database, &fresh));
|
||||
try expectFixtureSurvived(&result.database, &pristine);
|
||||
}
|
||||
}
|
||||
|
||||
test "the current version's fixture pair opens on the current lane unchanged" {
|
||||
// The loop above never reaches this pair, and Gate 2 of the cut requires it
|
||||
// to exist. This is what proves it is a real, coherent file rather than one
|
||||
// shipped to satisfy a gate.
|
||||
const fixture = fixtureFor(versions.current_version) orelse return error.MissingFixture;
|
||||
|
||||
var h: Harness = .init();
|
||||
defer h.deinit();
|
||||
try writeFixture(h.path(), fixture, versions.current_version);
|
||||
|
||||
var side: [256]u8 = undefined;
|
||||
var pristine = try openPristine(&h, &side, fixture);
|
||||
defer pristine.close();
|
||||
|
||||
var result = try querylog_schema.open(h.io(), std.Io.Dir.cwd(), h.path());
|
||||
defer result.database.close();
|
||||
try testing.expectEqual(@as(?querylog_schema.RecreateReason, null), result.recreated);
|
||||
try testing.expectEqual(
|
||||
@as(i64, versions.current_version),
|
||||
try result.database.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
|
||||
var fresh_buf: [256]u8 = undefined;
|
||||
var fresh = try openFreshCurrent(&h, &fresh_buf);
|
||||
defer fresh.close();
|
||||
try testing.expect(try migrations.schemaEquivalent(testing.allocator, &result.database, &fresh));
|
||||
try expectFixtureSurvived(&result.database, &pristine);
|
||||
// No migration ran, so nothing reshaped anything: byte-for-byte the rows
|
||||
// that were loaded.
|
||||
try expectSameContent(&result.database, &pristine);
|
||||
}
|
||||
|
||||
test "a fixture carrying the 0.0.12 fingerprint restamps, and refuses once the minimum rises" {
|
||||
const fixture = fixtureFor(1) orelse return error.MissingFixture;
|
||||
|
||||
// Version 1 is the legacy fingerprint's logical version, so the pair only
|
||||
// has anything to say while 1 is still supported.
|
||||
if (versions.minimum_supported_version <= 1 and versions.current_version >= 1) {
|
||||
var h: Harness = .init();
|
||||
defer h.deinit();
|
||||
try writeFixture(h.path(), fixture, versions.legacy_fingerprint);
|
||||
|
||||
var side: [256]u8 = undefined;
|
||||
var pristine = try openPristine(&h, &side, fixture);
|
||||
defer pristine.close();
|
||||
|
||||
var result = try querylog_schema.open(h.io(), std.Io.Dir.cwd(), h.path());
|
||||
defer result.database.close();
|
||||
try testing.expectEqual(@as(?querylog_schema.RecreateReason, null), result.recreated);
|
||||
try testing.expectEqual(
|
||||
@as(i64, versions.current_version),
|
||||
try result.database.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
|
||||
var fresh_buf: [256]u8 = undefined;
|
||||
var fresh = try openFreshCurrent(&h, &fresh_buf);
|
||||
defer fresh.close();
|
||||
try testing.expect(try migrations.schemaEquivalent(testing.allocator, &result.database, &fresh));
|
||||
try expectFixtureSurvived(&result.database, &pristine);
|
||||
}
|
||||
|
||||
// The companion, already in the suite for the day a break raises the
|
||||
// minimum above 1: the same bytes under the same stamp are then a file this
|
||||
// build cannot reach, and it is refused without being touched.
|
||||
var h: Harness = .init();
|
||||
defer h.deinit();
|
||||
try writeFixture(h.path(), fixture, versions.legacy_fingerprint);
|
||||
|
||||
const after_break: querylog_schema.Plan = .{
|
||||
.minimum = 2,
|
||||
.current = 2,
|
||||
.legacy_fingerprint = versions.legacy_fingerprint,
|
||||
.step_sql = &.{},
|
||||
};
|
||||
try testing.expect(querylog_schema.classify(versions.legacy_fingerprint, after_break).action ==
|
||||
.refuse_unsupported);
|
||||
try testing.expect(!querylog_schema.classify(versions.legacy_fingerprint, after_break).restamp);
|
||||
|
||||
var handle: ?db.Db = try h.openLive();
|
||||
defer if (handle) |*open_db| open_db.close();
|
||||
|
||||
migrations.expected_failures.begin();
|
||||
defer migrations.expected_failures.end();
|
||||
try testing.expectError(
|
||||
error.SchemaUnsupported,
|
||||
querylog_schema.openVersioned(h.io(), std.Io.Dir.cwd(), h.path(), &handle, after_break),
|
||||
);
|
||||
|
||||
var check = try h.openLive();
|
||||
defer check.close();
|
||||
try testing.expectEqual(
|
||||
@as(i64, versions.legacy_fingerprint),
|
||||
try check.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
var pristine_buf: [256]u8 = undefined;
|
||||
var pristine = try openPristine(&h, &pristine_buf, fixture);
|
||||
defer pristine.close();
|
||||
try expectSameContent(&check, &pristine);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+722
-40
@@ -1,14 +1,15 @@
|
||||
//! The `querylog.db` schema and its open-or-recreate policy.
|
||||
//! The `querylog.db` schema and its open policy.
|
||||
//!
|
||||
//! `querylog.db` is never migrated (PLAN §3.7). It holds expendable log rows,
|
||||
//! so a schema change replaces the file instead of upgrading it. The
|
||||
//! replacement trigger is a fingerprint derived from the DDL text itself, so
|
||||
//! editing the schema below automatically invalidates every existing file — the
|
||||
//! policy cannot drift out of sync with the SQL.
|
||||
//! `querylog.db` carries a logical schema version in `PRAGMA user_version`
|
||||
//! (`querylog_versions.zig`), and a file stamped below the current version is
|
||||
//! MIGRATED in place. A healthy file is never replaced and never set aside: a
|
||||
//! version this build cannot reach refuses the startup with instructions
|
||||
//! instead, because the operator's query history is not this program's to
|
||||
//! discard.
|
||||
//!
|
||||
//! **Recreating is destructive, so the predicate is a positive whitelist.** Only
|
||||
//! a missing file, `error.Corrupt`, `error.NotADb`, a failed `PRAGMA
|
||||
//! quick_check` and a fingerprint mismatch recreate. Every other error
|
||||
//! a missing file, `error.Corrupt`, `error.NotADb` and a failed `PRAGMA
|
||||
//! quick_check` recreate — genuine corruption, nothing else. Every other error
|
||||
//! propagates and the file on disk is not touched. `error.Busy` / `error.Locked`
|
||||
//! mean another process holds the write lock — waiting is right, deleting is
|
||||
//! catastrophic. `error.OutOfMemory` is this process's problem. `error.CantOpen`
|
||||
@@ -19,9 +20,21 @@
|
||||
const std = @import("std");
|
||||
|
||||
const db = @import("db.zig");
|
||||
const migrations = @import("querylog_migrations.zig");
|
||||
const versions = @import("querylog_versions.zig");
|
||||
|
||||
const log = std.log.scoped(.querylog_schema);
|
||||
|
||||
/// See `querylog_migrations.fail`: `err`, unless a test has said it is causing
|
||||
/// this refusal on purpose.
|
||||
fn fail(comptime fmt: []const u8, args: anytype) void {
|
||||
if (migrations.expected_failures.capturing()) {
|
||||
log.warn(fmt, args);
|
||||
} else {
|
||||
log.err(fmt, args);
|
||||
}
|
||||
}
|
||||
|
||||
/// PLAN §11.3, plus the coverage watermark of milestone 28. Multi-statement
|
||||
/// text — it goes through `db.Db.exec`, never through `prepare`.
|
||||
///
|
||||
@@ -139,14 +152,21 @@ pub const fingerprint: i32 = blk: {
|
||||
break :blk fingerprintOf(ddl);
|
||||
};
|
||||
|
||||
const set_user_version = std.fmt.comptimePrint("PRAGMA user_version = {d};", .{fingerprint});
|
||||
/// What a fresh file is stamped with. The logical version, not the fingerprint:
|
||||
/// from this release on, `user_version` is a version number.
|
||||
const set_user_version = std.fmt.comptimePrint(
|
||||
"PRAGMA user_version = {d};",
|
||||
.{versions.current_version},
|
||||
);
|
||||
|
||||
/// Long enough for any path this program will be handed, plus the aside suffix.
|
||||
/// A longer path is `error.NameTooLong`, which is what the filesystem calls
|
||||
/// would have returned anyway.
|
||||
const path_buf_len = 4096 + 64;
|
||||
|
||||
pub const RecreateReason = enum { missing, corrupt, not_a_database, quick_check_failed, fingerprint_mismatch };
|
||||
/// Corruption, and nothing else. A healthy file with a version this build does
|
||||
/// not handle refuses the startup; it is never recreated and never set aside.
|
||||
pub const RecreateReason = enum { missing, corrupt, not_a_database, quick_check_failed };
|
||||
|
||||
pub const OpenResult = struct {
|
||||
database: db.Db,
|
||||
@@ -166,10 +186,75 @@ pub const OpenResult = struct {
|
||||
}
|
||||
};
|
||||
|
||||
pub const Error = db.Error || error{AsideNameCollision} ||
|
||||
std.Io.Dir.RenamePreserveError || std.Io.Dir.DeleteFileError || std.Io.Dir.AccessError;
|
||||
pub const Error = db.Error || error{
|
||||
AsideNameCollision,
|
||||
/// The file's version is above this build's. A downgrade, almost always.
|
||||
SchemaTooNew,
|
||||
/// The file's version is one this build cannot migrate from: older than
|
||||
/// `minimum_supported_version`, or not a stamp nxdns ever wrote.
|
||||
SchemaUnsupported,
|
||||
MigrationFailed,
|
||||
MigrationBackupFailed,
|
||||
NameTooLong,
|
||||
} || std.Io.Dir.RenamePreserveError || std.Io.Dir.DeleteFileError || std.Io.Dir.AccessError;
|
||||
|
||||
/// Opens `path`, recreating it if and only if it is genuinely unusable.
|
||||
/// The version metadata `openVersioned` works against. Production passes
|
||||
/// `production_plan`; tests inject synthetic chains, which is what makes
|
||||
/// migration, the post-commit branch and the ownership of the handle testable
|
||||
/// through the real open path while the shipped chain is still empty.
|
||||
pub const Plan = struct {
|
||||
minimum: i32,
|
||||
current: i32,
|
||||
legacy_fingerprint: i32,
|
||||
step_sql: []const [:0]const u8,
|
||||
};
|
||||
|
||||
pub const production_plan: Plan = .{
|
||||
.minimum = versions.minimum_supported_version,
|
||||
.current = versions.current_version,
|
||||
.legacy_fingerprint = versions.legacy_fingerprint,
|
||||
.step_sql = versions.step_sql,
|
||||
};
|
||||
|
||||
/// What a stamped `user_version` means. A pure function of the stamp and the
|
||||
/// plan's three numbers — no file, no clock, no mutation.
|
||||
pub const Action = enum { open_current, migrate, refuse_too_new, refuse_unsupported };
|
||||
|
||||
pub const Classification = struct {
|
||||
/// The stamp mapped onto the version line. Equal to the stamp except for
|
||||
/// the legacy fingerprint, which IS version 1.
|
||||
logical: i32,
|
||||
action: Action,
|
||||
/// The legacy fingerprint must be replaced by its logical number before the
|
||||
/// file is used — but only on a lane that accepts the file. An unsupported
|
||||
/// file is never modified.
|
||||
restamp: bool,
|
||||
};
|
||||
|
||||
/// The classification table. The order is load-bearing: the legacy fingerprint
|
||||
/// becomes version 1 FIRST, and only then is version 1 judged against the
|
||||
/// plan's range. After a future explicit break raises the minimum above 1, a
|
||||
/// legacy-stamped file therefore classifies as below-minimum and refuses
|
||||
/// without ever being restamped.
|
||||
pub fn classify(stamped: i32, plan: Plan) Classification {
|
||||
const legacy = stamped == plan.legacy_fingerprint;
|
||||
const logical: i32 = if (legacy) 1 else stamped;
|
||||
|
||||
const action: Action = if (logical == plan.current)
|
||||
.open_current
|
||||
else if (logical >= plan.minimum and logical < plan.current)
|
||||
.migrate
|
||||
else if (logical > plan.current and logical <= versions.version_floor_guard)
|
||||
.refuse_too_new
|
||||
else
|
||||
.refuse_unsupported;
|
||||
|
||||
const accepted = action == .open_current or action == .migrate;
|
||||
return .{ .logical = logical, .action = action, .restamp = legacy and accepted };
|
||||
}
|
||||
|
||||
/// Opens `path`, recreating it if and only if it is genuinely unusable, and
|
||||
/// migrating it if and only if it carries an older supported version.
|
||||
///
|
||||
/// `path` is resolved twice by two different mechanisms: `dir`-relative for the
|
||||
/// filesystem calls, and process-cwd-relative by SQLite's VFS, which knows
|
||||
@@ -180,7 +265,7 @@ pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
|
||||
var handle: ?db.Db = null;
|
||||
errdefer if (handle) |*h| h.close();
|
||||
|
||||
const reason: ?RecreateReason = probe: {
|
||||
const cause: RecreateReason = probe: {
|
||||
dir.access(io, path, .{}) catch |e| switch (e) {
|
||||
error.FileNotFound => break :probe .missing,
|
||||
else => |other| return other,
|
||||
@@ -197,15 +282,10 @@ pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
|
||||
break :probe recreatable(e) orelse return e;
|
||||
if (!healthy) break :probe .quick_check_failed;
|
||||
|
||||
const stamped = opened.queryInt("PRAGMA user_version") catch |e|
|
||||
break :probe recreatable(e) orelse return e;
|
||||
if (stamped != fingerprint) break :probe .fingerprint_mismatch;
|
||||
|
||||
break :probe null;
|
||||
try openVersioned(io, dir, path, &handle, production_plan);
|
||||
return .{ .database = handle.?, .recreated = null };
|
||||
};
|
||||
|
||||
const cause = reason orelse return .{ .database = handle.?, .recreated = null };
|
||||
|
||||
// Close first, so SQLite checkpoints and drops `-wal`/`-shm` where it can.
|
||||
if (handle) |*h| h.close();
|
||||
handle = null;
|
||||
@@ -240,6 +320,131 @@ pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
|
||||
return result;
|
||||
}
|
||||
|
||||
/// The version half of `open`, against an injectable `plan`.
|
||||
///
|
||||
/// `handle` is the slot holding the healthy, pragma-applied connection to
|
||||
/// `path`. On success the connection stays in it, at `plan.current`. On every
|
||||
/// failure this function closes the connection and sets the slot to null, so
|
||||
/// the caller's own error-path close cannot double-close it — including the
|
||||
/// committed-but-unclean path, which is the one place the handle must be
|
||||
/// dropped even though the file on disk is fine.
|
||||
///
|
||||
/// **nxdns owns `path` exclusively.** It opens `querylog.db` once at startup,
|
||||
/// before it serves anything, and no second process shares a data directory —
|
||||
/// the standing deployment contract. The backup-then-lock sequence in
|
||||
/// `querylog_migrations.runMigration` relies on it: between the `VACUUM INTO`
|
||||
/// and the `BEGIN IMMEDIATE` there is no lock, and the re-read of
|
||||
/// `user_version` under the lock is what turns a violation of that contract
|
||||
/// into a refusal instead of a corrupted migration.
|
||||
pub fn openVersioned(
|
||||
io: std.Io,
|
||||
dir: std.Io.Dir,
|
||||
path: [:0]const u8,
|
||||
handle: *?db.Db,
|
||||
plan: Plan,
|
||||
) Error!void {
|
||||
const database = &(handle.*.?);
|
||||
errdefer closeSlot(handle);
|
||||
|
||||
const stamped64 = try database.queryInt("PRAGMA user_version");
|
||||
const stamped = std.math.cast(i32, stamped64) orelse {
|
||||
// `user_version` is a signed 32-bit field, so this cannot come from
|
||||
// SQLite. Refusing is the same answer any other foreign stamp gets.
|
||||
refusalLog(path, stamped64, plan, "SchemaUnsupported");
|
||||
return error.SchemaUnsupported;
|
||||
};
|
||||
|
||||
const verdict = classify(stamped, plan);
|
||||
switch (verdict.action) {
|
||||
.refuse_too_new => {
|
||||
refusalLog(path, stamped64, plan, "SchemaTooNew");
|
||||
return error.SchemaTooNew;
|
||||
},
|
||||
.refuse_unsupported => {
|
||||
refusalLog(path, stamped64, plan, "SchemaUnsupported");
|
||||
return error.SchemaUnsupported;
|
||||
},
|
||||
.open_current, .migrate => {},
|
||||
}
|
||||
|
||||
if (verdict.restamp) try restampLegacy(database, path, verdict.logical);
|
||||
|
||||
switch (verdict.action) {
|
||||
.migrate => {
|
||||
const first = @as(usize, @intCast(verdict.logical - plan.minimum));
|
||||
migrations.runMigration(
|
||||
io,
|
||||
dir,
|
||||
path,
|
||||
database,
|
||||
plan.step_sql[first..],
|
||||
verdict.logical,
|
||||
plan.current,
|
||||
) catch |e| switch (e) {
|
||||
// Two different states of the FILE — migrated and kept with its
|
||||
// backup, or logically untouched — and one shared state of the
|
||||
// CONNECTION: its pragmas are not what `applyPragmas`
|
||||
// guarantees, so it must not serve. Closing it here is the
|
||||
// single close either path gets. After a commit the next start
|
||||
// opens the migrated file on the current-version lane; after a
|
||||
// failure it retries the migration from the top.
|
||||
error.MigrationCommittedButUnclean, error.MigrationFailedUnclean => {
|
||||
closeSlot(handle);
|
||||
return error.MigrationFailed;
|
||||
},
|
||||
error.MigrationBackupFailed => return error.MigrationBackupFailed,
|
||||
error.MigrationFailed, error.NameTooLong => return error.MigrationFailed,
|
||||
else => |other| return other,
|
||||
};
|
||||
},
|
||||
.open_current => migrations.pruneBackupsConservative(io, dir, path),
|
||||
else => unreachable,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces the 0.0.12/0.0.13 fingerprint stamp with the logical version it
|
||||
/// stands for. This is the milestone's only real mutation of operator data, so
|
||||
/// it runs in its own transaction and any failure leaves the legacy stamp and
|
||||
/// every row exactly as they were — a refusal, never a recreate.
|
||||
fn restampLegacy(database: *db.Db, path: []const u8, logical: i32) Error!void {
|
||||
var stamp_buf: [64]u8 = undefined;
|
||||
const stamp = std.fmt.bufPrintZ(&stamp_buf, "PRAGMA user_version = {d};", .{logical}) catch
|
||||
unreachable; // an i32 and a fixed prefix cannot overrun 64 bytes
|
||||
|
||||
restamp: {
|
||||
var tx = db.Tx.begin(database) catch break :restamp;
|
||||
database.exec(stamp) catch {
|
||||
tx.rollback();
|
||||
break :restamp;
|
||||
};
|
||||
tx.commit() catch {
|
||||
tx.rollback();
|
||||
break :restamp;
|
||||
};
|
||||
log.info("querylog database '{s}' carried the 0.0.12 schema fingerprint; " ++
|
||||
"restamped as schema version {d}", .{ path, logical });
|
||||
return;
|
||||
}
|
||||
|
||||
var buf: [256]u8 = undefined;
|
||||
fail("cannot restamp querylog database '{s}' as schema version {d}: {s}; " ++
|
||||
"the file is unchanged", .{ path, logical, database.lastError(&buf) });
|
||||
return error.MigrationFailed;
|
||||
}
|
||||
|
||||
fn closeSlot(handle: *?db.Db) void {
|
||||
if (handle.*) |*h| h.close();
|
||||
handle.* = null;
|
||||
}
|
||||
|
||||
fn refusalLog(path: []const u8, stamped: i64, plan: Plan, name: []const u8) void {
|
||||
fail("refusing to open querylog database '{s}': it is stamped {d}, and this build " ++
|
||||
"supports schema versions {d} to {d} ({s}). The file is left exactly as it is; " ++
|
||||
"see docs/how-to/troubleshoot.md, \"The server refuses to start over querylog.db\"", .{
|
||||
path, stamped, plan.minimum, plan.current, name,
|
||||
});
|
||||
}
|
||||
|
||||
/// An additional connection to a `querylog.db` that `open` has already
|
||||
/// established, with the pragmas every connection to the file needs.
|
||||
///
|
||||
@@ -285,16 +490,15 @@ fn quickCheck(database: *db.Db) db.Error!bool {
|
||||
/// What the aside file's name calls the reason it was set aside.
|
||||
///
|
||||
/// The name is the only account of the reason an operator gets: the log line
|
||||
/// naming it scrolls away, the file stays for months. `fingerprint_mismatch` is
|
||||
/// a database with nothing wrong with it — this build's DDL moved — so calling
|
||||
/// its file "corrupt" invites the operator to delete evidence of a healthy file.
|
||||
/// naming it scrolls away, the file stays for months. Every tag here names real
|
||||
/// damage, which is the whole set of reasons left — a healthy file whose
|
||||
/// version this build cannot handle refuses the startup and is not renamed.
|
||||
fn asideTag(reason: RecreateReason) []const u8 {
|
||||
return switch (reason) {
|
||||
.missing => unreachable, // there is no file to rename
|
||||
.corrupt => "corrupt",
|
||||
.not_a_database => "not-a-database",
|
||||
.quick_check_failed => "quick-check-failed",
|
||||
.fingerprint_mismatch => "schema-changed",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -455,11 +659,14 @@ test "querylog_meta is seeded with one row the schema will not let a second join
|
||||
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM querylog_meta"));
|
||||
}
|
||||
|
||||
test "the user_version statement stamps the fingerprint" {
|
||||
test "the user_version statement stamps the current schema version" {
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
defer database.close();
|
||||
try database.exec(set_user_version);
|
||||
try testing.expectEqual(@as(i64, fingerprint), try database.queryInt("PRAGMA user_version"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, versions.current_version),
|
||||
try database.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
}
|
||||
|
||||
// The behaviour these two cases describe — a resource error leaves the file on
|
||||
@@ -488,7 +695,6 @@ test "the aside name says why, and a healthy file is never called corrupt" {
|
||||
try testing.expectEqualStrings("corrupt", asideTag(.corrupt));
|
||||
try testing.expectEqualStrings("not-a-database", asideTag(.not_a_database));
|
||||
try testing.expectEqualStrings("quick-check-failed", asideTag(.quick_check_failed));
|
||||
try testing.expectEqualStrings("schema-changed", asideTag(.fingerprint_mismatch));
|
||||
}
|
||||
|
||||
test "recreatable selects exactly two of db.Error's members" {
|
||||
@@ -532,7 +738,7 @@ test "a recreate returns the aside name by value and a fresh create returns none
|
||||
try tmp.dir.access(io, kept, .{});
|
||||
}
|
||||
|
||||
test "a recreate resets coverage to the new file and keeps the old one aside" {
|
||||
test "a corrupt file is recreated, coverage restarts, and the old one is kept aside" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
@@ -549,21 +755,14 @@ test "a recreate resets coverage to the new file and keeps the old one aside" {
|
||||
try created.database.exec("INSERT INTO domains (domain) VALUES ('old.example');");
|
||||
created.database.close();
|
||||
|
||||
// A healthy file this build's DDL no longer matches — the case milestone
|
||||
// 28's own schema edit produces on every upgrade.
|
||||
{
|
||||
var stamped = try db.Db.open(path, .{ .mode = .read_write_existing });
|
||||
defer stamped.close();
|
||||
var sql_buf: [64]u8 = undefined;
|
||||
try stamped.exec(try std.fmt.bufPrintZ(&sql_buf, "PRAGMA user_version = {d};", .{fingerprint +% 1}));
|
||||
}
|
||||
// Real damage, which is now the only thing that recreates.
|
||||
try tmp.dir.writeFile(io, .{ .sub_path = "querylog.db", .data = "not a database at all" });
|
||||
|
||||
var recreated = try open(io, std.Io.Dir.cwd(), path);
|
||||
defer recreated.database.close();
|
||||
|
||||
try testing.expectEqual(RecreateReason.fingerprint_mismatch, recreated.recreated.?);
|
||||
// The name says the file was healthy and this build moved, not that it rotted.
|
||||
try testing.expect(std.mem.indexOf(u8, recreated.aside(), ".schema-changed-") != null);
|
||||
try testing.expectEqual(RecreateReason.not_a_database, recreated.recreated.?);
|
||||
try testing.expect(std.mem.indexOf(u8, recreated.aside(), ".not-a-database-") != null);
|
||||
try tmp.dir.access(io, std.fs.path.basename(recreated.aside()), .{});
|
||||
|
||||
// Exactly one meta row, and coverage starts at the recreate rather than
|
||||
@@ -602,3 +801,486 @@ test "a clean reopen reports no recreate and no aside" {
|
||||
try testing.expectEqual(@as(?RecreateReason, null), second.recreated);
|
||||
try testing.expectEqualStrings("", second.aside());
|
||||
}
|
||||
|
||||
test "classification is a pure function of the stamp and the plan's three numbers" {
|
||||
const plan: Plan = .{
|
||||
.minimum = 1,
|
||||
.current = 3,
|
||||
.legacy_fingerprint = versions.legacy_fingerprint,
|
||||
.step_sql = &.{},
|
||||
};
|
||||
|
||||
try testing.expectEqual(Action.open_current, classify(3, plan).action);
|
||||
try testing.expectEqual(Action.migrate, classify(1, plan).action);
|
||||
try testing.expectEqual(Action.migrate, classify(2, plan).action);
|
||||
try testing.expectEqual(Action.refuse_too_new, classify(4, plan).action);
|
||||
try testing.expectEqual(Action.refuse_too_new, classify(versions.version_floor_guard, plan).action);
|
||||
// Above the floor guard is not a version this project ever wrote.
|
||||
try testing.expectEqual(
|
||||
Action.refuse_unsupported,
|
||||
classify(versions.version_floor_guard + 1, plan).action,
|
||||
);
|
||||
for ([_]i32{ 0, -1, -1_000_000, 603440875 }) |foreign| {
|
||||
try testing.expectEqual(Action.refuse_unsupported, classify(foreign, plan).action);
|
||||
try testing.expect(!classify(foreign, plan).restamp);
|
||||
}
|
||||
|
||||
// The legacy fingerprint IS version 1, and being version 1 is what decides
|
||||
// its lane.
|
||||
const legacy = classify(versions.legacy_fingerprint, plan);
|
||||
try testing.expectEqual(@as(i32, 1), legacy.logical);
|
||||
try testing.expectEqual(Action.migrate, legacy.action);
|
||||
try testing.expect(legacy.restamp);
|
||||
}
|
||||
|
||||
test "a legacy stamp below a raised minimum refuses without a restamp" {
|
||||
// What a future explicit break looks like from this side: the minimum has
|
||||
// moved past 1, so the 0.0.12 file is no longer reachable. The ORDER is the
|
||||
// point — mapping to 1 first and judging second is what stops the restamp
|
||||
// from mutating a file this build will refuse anyway.
|
||||
const after_break: Plan = .{
|
||||
.minimum = 3,
|
||||
.current = 3,
|
||||
.legacy_fingerprint = versions.legacy_fingerprint,
|
||||
.step_sql = &.{},
|
||||
};
|
||||
|
||||
const legacy = classify(versions.legacy_fingerprint, after_break);
|
||||
try testing.expectEqual(@as(i32, 1), legacy.logical);
|
||||
try testing.expectEqual(Action.refuse_unsupported, legacy.action);
|
||||
try testing.expect(!legacy.restamp);
|
||||
|
||||
// And versions 1 and 2, which the break dropped, refuse the same way.
|
||||
try testing.expectEqual(Action.refuse_unsupported, classify(1, after_break).action);
|
||||
try testing.expectEqual(Action.refuse_unsupported, classify(2, after_break).action);
|
||||
try testing.expectEqual(Action.open_current, classify(3, after_break).action);
|
||||
}
|
||||
|
||||
test "the production plan classifies a fresh stamp as current" {
|
||||
try testing.expectEqual(
|
||||
Action.open_current,
|
||||
classify(versions.current_version, production_plan).action,
|
||||
);
|
||||
try testing.expect(classify(versions.legacy_fingerprint, production_plan).restamp);
|
||||
}
|
||||
|
||||
/// The five lines every file-backed test below opens with.
|
||||
const Fixture = struct {
|
||||
threaded: std.Io.Threaded,
|
||||
tmp: std.testing.TmpDir,
|
||||
buf: [256]u8 = undefined,
|
||||
|
||||
fn init() Fixture {
|
||||
return .{
|
||||
.threaded = .init(testing.allocator, .{}),
|
||||
.tmp = testing.tmpDir(.{ .iterate = true }),
|
||||
};
|
||||
}
|
||||
|
||||
fn deinit(self: *Fixture) void {
|
||||
self.tmp.cleanup();
|
||||
self.threaded.deinit();
|
||||
}
|
||||
|
||||
fn io(self: *Fixture) std.Io {
|
||||
return self.threaded.io();
|
||||
}
|
||||
|
||||
fn path(self: *Fixture) [:0]const u8 {
|
||||
return std.fmt.bufPrintZ(&self.buf, ".zig-cache/tmp/{s}/querylog.db", .{self.tmp.sub_path}) catch
|
||||
unreachable;
|
||||
}
|
||||
|
||||
fn stamp(self: *Fixture, value: i32) !void {
|
||||
var database = try db.Db.open(self.path(), .{ .mode = .read_write_existing });
|
||||
defer database.close();
|
||||
var sql: [64]u8 = undefined;
|
||||
try database.exec(try std.fmt.bufPrintZ(&sql, "PRAGMA user_version = {d};", .{value}));
|
||||
}
|
||||
|
||||
fn liveHandle(self: *Fixture) !?db.Db {
|
||||
var database = try db.Db.open(self.path(), .{ .mode = .read_write_existing });
|
||||
errdefer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
return database;
|
||||
}
|
||||
|
||||
fn countMatching(self: *Fixture, prefix: []const u8) !usize {
|
||||
var found: usize = 0;
|
||||
var it = self.tmp.dir.iterate();
|
||||
while (try it.next(self.io())) |entry| {
|
||||
if (std.mem.startsWith(u8, entry.name, prefix)) found += 1;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
fn expectModeOfOnlyMatch(self: *Fixture, prefix: []const u8, expected: std.posix.mode_t) !void {
|
||||
var it = self.tmp.dir.iterate();
|
||||
while (try it.next(self.io())) |entry| {
|
||||
if (!std.mem.startsWith(u8, entry.name, prefix)) continue;
|
||||
const stat = try self.tmp.dir.statFile(self.io(), entry.name, .{});
|
||||
const mode = stat.permissions.toMode() & 0o777;
|
||||
if (mode != expected) {
|
||||
std.debug.print("mode of '{s}' is {o}, expected {o}\n", .{ entry.name, mode, expected });
|
||||
return error.TestUnexpectedResult;
|
||||
}
|
||||
return;
|
||||
}
|
||||
std.debug.print("no file starting with '{s}'\n", .{prefix});
|
||||
return error.TestUnexpectedResult;
|
||||
}
|
||||
};
|
||||
|
||||
test "a fresh file is stamped with the current schema version" {
|
||||
var f: Fixture = .init();
|
||||
defer f.deinit();
|
||||
|
||||
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer created.database.close();
|
||||
try testing.expectEqual(RecreateReason.missing, created.recreated.?);
|
||||
try testing.expectEqual(
|
||||
@as(i64, versions.current_version),
|
||||
try created.database.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a 0.0.13 file is restamped as version 1 and keeps every row" {
|
||||
var f: Fixture = .init();
|
||||
defer f.deinit();
|
||||
|
||||
{
|
||||
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer created.database.close();
|
||||
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
|
||||
}
|
||||
// Exactly what 0.0.12 and 0.0.13 wrote: the CRC of their DDL, which is this
|
||||
// build's DDL unchanged.
|
||||
try f.stamp(versions.legacy_fingerprint);
|
||||
try testing.expectEqual(versions.legacy_fingerprint, fingerprint);
|
||||
|
||||
{
|
||||
var upgraded = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer upgraded.database.close();
|
||||
try testing.expectEqual(@as(?RecreateReason, null), upgraded.recreated);
|
||||
try testing.expectEqual(@as(i64, 1), try upgraded.database.queryInt("PRAGMA user_version"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try upgraded.database.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
|
||||
);
|
||||
}
|
||||
|
||||
// Nothing was set aside on the way, and the second start is an ordinary
|
||||
// current-version open.
|
||||
try testing.expectEqual(@as(usize, 0), try f.countMatching("querylog.db.schema"));
|
||||
var again = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer again.database.close();
|
||||
try testing.expectEqual(@as(?RecreateReason, null), again.recreated);
|
||||
try testing.expectEqual(@as(i64, 1), try again.database.queryInt("PRAGMA user_version"));
|
||||
}
|
||||
|
||||
test "a version this build cannot handle refuses and leaves the file alone" {
|
||||
var f: Fixture = .init();
|
||||
defer f.deinit();
|
||||
|
||||
{
|
||||
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer created.database.close();
|
||||
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
|
||||
}
|
||||
const watermark = blk: {
|
||||
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
|
||||
defer probe.close();
|
||||
break :blk try probe.queryInt("SELECT available_since FROM querylog_meta");
|
||||
};
|
||||
|
||||
migrations.expected_failures.begin();
|
||||
defer migrations.expected_failures.end();
|
||||
|
||||
const lanes = [_]struct { stamp: i32, expected: anyerror }{
|
||||
.{ .stamp = versions.current_version + 1, .expected = error.SchemaTooNew },
|
||||
.{ .stamp = versions.version_floor_guard, .expected = error.SchemaTooNew },
|
||||
.{ .stamp = 0, .expected = error.SchemaUnsupported },
|
||||
.{ .stamp = -3, .expected = error.SchemaUnsupported },
|
||||
.{ .stamp = 603440875, .expected = error.SchemaUnsupported },
|
||||
};
|
||||
|
||||
for (lanes) |lane| {
|
||||
try f.stamp(lane.stamp);
|
||||
try testing.expectError(lane.expected, open(f.io(), std.Io.Dir.cwd(), f.path()));
|
||||
|
||||
// Schema, rows, watermark and stamp all as they were, and nothing new
|
||||
// beside the file.
|
||||
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
|
||||
defer probe.close();
|
||||
try testing.expectEqual(@as(i64, lane.stamp), try probe.queryInt("PRAGMA user_version"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try probe.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
|
||||
);
|
||||
try testing.expectEqual(
|
||||
watermark,
|
||||
try probe.queryInt("SELECT available_since FROM querylog_meta"),
|
||||
);
|
||||
try testing.expectEqual(@as(usize, 0), try f.countMatching("querylog.db."));
|
||||
}
|
||||
}
|
||||
|
||||
test "a restamp that fails at the statement or at the commit refuses without loss" {
|
||||
for ([_][]const u8{ "PRAGMA user_version = 1;", "COMMIT;" }) |failing| {
|
||||
var f: Fixture = .init();
|
||||
defer f.deinit();
|
||||
|
||||
{
|
||||
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer created.database.close();
|
||||
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
|
||||
}
|
||||
try f.stamp(versions.legacy_fingerprint);
|
||||
|
||||
migrations.expected_failures.begin();
|
||||
defer migrations.expected_failures.end();
|
||||
db.exec_faults.failNextMatching(failing);
|
||||
defer db.exec_faults.disarm();
|
||||
|
||||
try testing.expectError(
|
||||
error.MigrationFailed,
|
||||
open(f.io(), std.Io.Dir.cwd(), f.path()),
|
||||
);
|
||||
try testing.expect(!db.exec_faults.armed());
|
||||
|
||||
// The legacy stamp and every row are exactly as they were: this is a
|
||||
// refusal, and a refusal never costs the operator anything.
|
||||
{
|
||||
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
|
||||
defer probe.close();
|
||||
try testing.expectEqual(
|
||||
@as(i64, versions.legacy_fingerprint),
|
||||
try probe.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
}
|
||||
|
||||
// And the next start, with nothing injected, does the restamp properly.
|
||||
var recovered = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer recovered.database.close();
|
||||
try testing.expectEqual(@as(i64, 1), try recovered.database.queryInt("PRAGMA user_version"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try recovered.database.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A one-step chain from the real current version to one above it. Nothing in
|
||||
/// the shipped chain can exercise migration while `step_sql` is empty, so the
|
||||
/// open path's migration lanes are driven through this instead.
|
||||
const synthetic_plan: Plan = .{
|
||||
.minimum = versions.current_version,
|
||||
.current = versions.current_version + 1,
|
||||
.legacy_fingerprint = versions.legacy_fingerprint,
|
||||
.step_sql = &.{"CREATE TABLE migration_marker (id INTEGER PRIMARY KEY);"},
|
||||
};
|
||||
|
||||
test "a post-commit failure keeps the migration, keeps the backup, and refuses once" {
|
||||
var f: Fixture = .init();
|
||||
defer f.deinit();
|
||||
|
||||
{
|
||||
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer created.database.close();
|
||||
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
|
||||
}
|
||||
|
||||
{
|
||||
var handle = try f.liveHandle();
|
||||
errdefer if (handle) |*h| h.close();
|
||||
|
||||
migrations.expected_failures.begin();
|
||||
defer migrations.expected_failures.end();
|
||||
// The first statement of the pragma restore, which runs only after a
|
||||
// successful COMMIT.
|
||||
db.exec_faults.failNextMatching("legacy_alter_table = OFF");
|
||||
defer db.exec_faults.disarm();
|
||||
|
||||
try testing.expectError(
|
||||
error.MigrationFailed,
|
||||
openVersioned(f.io(), std.Io.Dir.cwd(), f.path(), &handle, synthetic_plan),
|
||||
);
|
||||
// Closed exactly once, by the open path: the slot it was handed is
|
||||
// empty, so no caller can close it again.
|
||||
try testing.expect(handle == null);
|
||||
}
|
||||
|
||||
// The file IS migrated. The log said so, and this is what it meant.
|
||||
{
|
||||
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
|
||||
defer probe.close();
|
||||
try testing.expectEqual(
|
||||
@as(i64, synthetic_plan.current),
|
||||
try probe.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try probe.queryInt("SELECT count(*) FROM sqlite_schema WHERE name = 'migration_marker'"),
|
||||
);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try probe.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
|
||||
);
|
||||
}
|
||||
try testing.expectEqual(@as(usize, 1), try f.countMatching("querylog.db.pre-migrate-"));
|
||||
// The backup is the whole query history at the moment of the migration, so
|
||||
// it carries the live file's mode and not SQLite's `0644 & ~umask`.
|
||||
try f.expectModeOfOnlyMatch("querylog.db.pre-migrate-", 0o600);
|
||||
|
||||
// The next start is ordinary: the current-version lane, no second
|
||||
// migration, and the one backup still there for the operator.
|
||||
var handle = try f.liveHandle();
|
||||
defer if (handle) |*h| h.close();
|
||||
try openVersioned(f.io(), std.Io.Dir.cwd(), f.path(), &handle, synthetic_plan);
|
||||
try testing.expect(handle != null);
|
||||
try testing.expectEqual(
|
||||
@as(i64, synthetic_plan.current),
|
||||
try handle.?.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
try testing.expectEqual(@as(usize, 1), try f.countMatching("querylog.db.pre-migrate-"));
|
||||
}
|
||||
|
||||
/// The same shape as `synthetic_plan`, with a step SQLite refuses to prepare. It
|
||||
/// drives the pre-commit failure path without any fault seam, leaving the seam
|
||||
/// free for the pragma restore.
|
||||
const failing_plan: Plan = .{
|
||||
.minimum = versions.current_version,
|
||||
.current = versions.current_version + 1,
|
||||
.legacy_fingerprint = versions.legacy_fingerprint,
|
||||
.step_sql = &.{"CREATE TABLE migration_marker (id INTEGER PRIMARY KEY) NOT A STATEMENT;"},
|
||||
};
|
||||
|
||||
test "a pre-commit failure whose pragma restore also fails closes the connection" {
|
||||
var f: Fixture = .init();
|
||||
defer f.deinit();
|
||||
|
||||
{
|
||||
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer created.database.close();
|
||||
try created.database.exec("INSERT INTO domains (domain) VALUES ('kept.example');");
|
||||
}
|
||||
|
||||
// The runner's own answer first: a failed restore is a DIFFERENT error from
|
||||
// a failed migration, because the two leave the connection in different
|
||||
// states even though they leave the file in the same one.
|
||||
{
|
||||
var handle = try f.liveHandle();
|
||||
defer if (handle) |*h| h.close();
|
||||
|
||||
migrations.expected_failures.begin();
|
||||
defer migrations.expected_failures.end();
|
||||
db.exec_faults.failNextMatching("legacy_alter_table = OFF");
|
||||
defer db.exec_faults.disarm();
|
||||
|
||||
try testing.expectError(error.MigrationFailedUnclean, migrations.runMigration(
|
||||
f.io(),
|
||||
std.Io.Dir.cwd(),
|
||||
f.path(),
|
||||
&handle.?,
|
||||
failing_plan.step_sql,
|
||||
failing_plan.minimum,
|
||||
failing_plan.current,
|
||||
));
|
||||
try testing.expect(!db.exec_faults.armed());
|
||||
}
|
||||
|
||||
// And the open path's answer: the handle is closed, exactly as it is after a
|
||||
// post-commit restore failure. A connection that may still hold
|
||||
// `foreign_keys = OFF` never reaches the server.
|
||||
{
|
||||
var handle = try f.liveHandle();
|
||||
errdefer if (handle) |*h| h.close();
|
||||
|
||||
migrations.expected_failures.begin();
|
||||
defer migrations.expected_failures.end();
|
||||
db.exec_faults.failNextMatching("legacy_alter_table = OFF");
|
||||
defer db.exec_faults.disarm();
|
||||
|
||||
try testing.expectError(
|
||||
error.MigrationFailed,
|
||||
openVersioned(f.io(), std.Io.Dir.cwd(), f.path(), &handle, failing_plan),
|
||||
);
|
||||
try testing.expect(handle == null);
|
||||
}
|
||||
|
||||
// The contrast that makes the rule visible, back at the runner, where the
|
||||
// connection survives to be inspected: the same failing step with the
|
||||
// restore working is a plain `MigrationFailed`, and that error promises the
|
||||
// pragmas `applyPragmas` guarantees.
|
||||
{
|
||||
var handle = try f.liveHandle();
|
||||
defer if (handle) |*h| h.close();
|
||||
|
||||
migrations.expected_failures.begin();
|
||||
defer migrations.expected_failures.end();
|
||||
try testing.expectError(error.MigrationFailed, migrations.runMigration(
|
||||
f.io(),
|
||||
std.Io.Dir.cwd(),
|
||||
f.path(),
|
||||
&handle.?,
|
||||
failing_plan.step_sql,
|
||||
failing_plan.minimum,
|
||||
failing_plan.current,
|
||||
));
|
||||
try testing.expectEqual(@as(i64, 1), try handle.?.queryInt("PRAGMA foreign_keys"));
|
||||
try testing.expectEqual(@as(i64, 0), try handle.?.queryInt("PRAGMA legacy_alter_table"));
|
||||
}
|
||||
|
||||
// No path committed anything, and every run deleted its own backup.
|
||||
var probe = try db.Db.open(f.path(), .{ .mode = .read_write_existing });
|
||||
defer probe.close();
|
||||
try testing.expectEqual(
|
||||
@as(i64, failing_plan.minimum),
|
||||
try probe.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try probe.queryInt("SELECT count(*) FROM domains WHERE domain = 'kept.example'"),
|
||||
);
|
||||
try testing.expectEqual(@as(usize, 0), try f.countMatching("querylog.db.pre-migrate-"));
|
||||
}
|
||||
|
||||
test "a plain open retries a retention cleanup that once failed" {
|
||||
var f: Fixture = .init();
|
||||
defer f.deinit();
|
||||
|
||||
{
|
||||
var created = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
created.database.close();
|
||||
}
|
||||
|
||||
// What a migration whose step-4 cleanup failed leaves behind: an older
|
||||
// epoch, the newest epoch, and a same-second collision name tied with it.
|
||||
const seeded = [_][]const u8{
|
||||
"querylog.db.pre-migrate-1600000000",
|
||||
"querylog.db.pre-migrate-1700000000",
|
||||
"querylog.db.pre-migrate-1700000000-2",
|
||||
"querylog.db.pre-migrate-handwritten",
|
||||
};
|
||||
for (seeded) |name| {
|
||||
try f.tmp.dir.writeFile(f.io(), .{ .sub_path = name, .data = "x" });
|
||||
}
|
||||
|
||||
var opened = try open(f.io(), std.Io.Dir.cwd(), f.path());
|
||||
defer opened.database.close();
|
||||
try testing.expectEqual(@as(?RecreateReason, null), opened.recreated);
|
||||
|
||||
// Only the strictly older epoch goes: both files tied at the newest epoch
|
||||
// survive, because this pass cannot tell which of them a migration made,
|
||||
// and a name it did not write is never its to delete.
|
||||
try testing.expect(!try exists(&f, seeded[0]));
|
||||
for (seeded[1..]) |name| try testing.expect(try exists(&f, name));
|
||||
}
|
||||
|
||||
fn exists(f: *Fixture, name: []const u8) !bool {
|
||||
f.tmp.dir.access(f.io(), name, .{}) catch |e| switch (e) {
|
||||
error.FileNotFound => return false,
|
||||
else => return e,
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
//! The `querylog.db` schema version chain: comptime metadata and nothing else.
|
||||
//!
|
||||
//! Separate from `querylog_migrations.zig` so `tools/cut.zig` can import it
|
||||
//! without linking SQLite. Nothing in this file may reach for `db.zig`, for a
|
||||
//! C symbol, or for an allocator — the release gate reads these constants at
|
||||
//! build time, and a dependency here would drag the whole storage layer into
|
||||
//! the cut tool.
|
||||
//!
|
||||
//! **There are no migration hooks.** A step is a SQL file, period. Every
|
||||
//! shipped step is therefore byte-comparable against the previous tag, which is
|
||||
//! what lets the cut gate prove a released migration was never edited. A future
|
||||
//! change that genuinely cannot be expressed in SQL must amend this design in
|
||||
//! its own spec rather than adding a code path here.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
/// The version a file created by this build carries in `PRAGMA user_version`.
|
||||
pub const current_version: i32 = 1;
|
||||
|
||||
/// The oldest stamped version this build can reach `current_version` from.
|
||||
/// A file stamped below this refuses to open.
|
||||
///
|
||||
/// An EXPLICIT BREAK in a future release is expressed here and only here: bump
|
||||
/// `current_version`, set `minimum_supported_version = current_version`, and
|
||||
/// ship no step. The chain then cannot reach the new version from below the
|
||||
/// minimum, so `open` refuses the old file by the ordinary rules. A break is
|
||||
/// always versioned, always refused at runtime, and never silent.
|
||||
pub const minimum_supported_version: i32 = 1;
|
||||
|
||||
/// The literal `user_version` the 0.0.12 and 0.0.13 binaries stamped: the CRC32
|
||||
/// of their DDL text, under the pre-migration policy where a stamp mismatch
|
||||
/// meant "replace the file".
|
||||
///
|
||||
/// FROZEN. It is derived from nothing at build time on purpose — recomputing it
|
||||
/// from today's DDL would silently stop recognising the files it exists to
|
||||
/// recognise the moment the schema moves. Editing it strands every 0.0.12 and
|
||||
/// 0.0.13 file that has not yet been opened by a migration-aware build, which
|
||||
/// is why the cut gate fails on any change to this line.
|
||||
pub const legacy_fingerprint: i32 = 1975011655;
|
||||
|
||||
/// Logical versions live far below any plausible CRC32 stamp. A value above
|
||||
/// this is not a version this project ever wrote, so it classifies as
|
||||
/// unsupported rather than as a from-the-future schema.
|
||||
pub const version_floor_guard: i32 = 1_000_000;
|
||||
|
||||
/// One entry per shipped step: `step_sql[i]` migrates version
|
||||
/// `minimum_supported_version + i` to `minimum_supported_version + i + 1`.
|
||||
/// Each entry is `@embedFile("migrations/v<from>.sql")`, and each such file is
|
||||
/// immutable once released.
|
||||
///
|
||||
/// **Step-authoring rules** (the runner enforces the first, the equivalence
|
||||
/// oracle catches violations of the rest):
|
||||
///
|
||||
/// - A step contains no transaction statement. No `BEGIN`, no `COMMIT`, no
|
||||
/// `ROLLBACK`, no `SAVEPOINT`: the runner wraps the whole chain in one
|
||||
/// transaction and installs an authorizer that denies them outright.
|
||||
/// - A step that changes a table's shape must REBUILD it, so that the CREATE
|
||||
/// text SQLite stores ends up byte-identical to the fresh DDL's:
|
||||
/// `DROP` every view over `<t>` first; `ALTER TABLE <t> RENAME TO <t>_old`;
|
||||
/// `CREATE TABLE <t> ...` pasted verbatim from `querylog_schema.ddl`;
|
||||
/// `INSERT INTO <t> SELECT ... FROM <t>_old`; `DROP TABLE <t>_old`; recreate
|
||||
/// every index and trigger of `<t>` verbatim; recreate the dropped views
|
||||
/// verbatim last.
|
||||
/// - `ALTER TABLE ... ADD COLUMN` and `ALTER TABLE ... RENAME COLUMN` on a kept
|
||||
/// table are forbidden. SQLite rewrites the stored CREATE text under them,
|
||||
/// and the oracle's exact-text layer would rightly call the result unequal.
|
||||
pub const step_sql: []const [:0]const u8 = &.{};
|
||||
|
||||
comptime {
|
||||
std.debug.assert(minimum_supported_version >= 1);
|
||||
std.debug.assert(minimum_supported_version <= current_version);
|
||||
std.debug.assert(current_version <= version_floor_guard);
|
||||
std.debug.assert(legacy_fingerprint < 0 or legacy_fingerprint > version_floor_guard);
|
||||
std.debug.assert(step_sql.len == @as(usize, @intCast(current_version - minimum_supported_version)));
|
||||
}
|
||||
|
||||
test "the chain covers exactly the supported range" {
|
||||
try std.testing.expectEqual(
|
||||
@as(usize, @intCast(current_version - minimum_supported_version)),
|
||||
step_sql.len,
|
||||
);
|
||||
}
|
||||
|
||||
test "the legacy anchor is the literal 0.0.12 stamp" {
|
||||
// Not `fingerprintOf(ddl)`. The number is a historical fact about released
|
||||
// binaries, so a test that recomputed it would move with the schema and
|
||||
// prove nothing.
|
||||
try std.testing.expectEqual(@as(i32, 1975011655), legacy_fingerprint);
|
||||
}
|
||||
@@ -2518,7 +2518,10 @@ const recompute_checks = [_]struct { projection: []const u8, recompute: []const
|
||||
},
|
||||
};
|
||||
|
||||
fn expectProjectionsMatchRecompute(database: *db.Db) !void {
|
||||
/// Exported for `querylog_migrations.zig`'s fixture and migration tests: the
|
||||
/// authority on projection coherence is this file, and a second copy of the
|
||||
/// recompute SQL there would be free to drift from the writer it checks.
|
||||
pub fn expectProjectionsMatchRecompute(database: *db.Db) !void {
|
||||
for (recompute_checks) |check| {
|
||||
var buf: [4096]u8 = undefined;
|
||||
const sql = try std.fmt.bufPrint(
|
||||
|
||||
@@ -28,7 +28,9 @@ const model = @import("../config/model.zig");
|
||||
const validate = @import("../config/validate.zig");
|
||||
const db = @import("db.zig");
|
||||
const migrations = @import("migrations.zig");
|
||||
const querylog_migrations = @import("querylog_migrations.zig");
|
||||
const querylog_schema = @import("querylog_schema.zig");
|
||||
const querylog_versions = @import("querylog_versions.zig");
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
@@ -354,7 +356,7 @@ test "S7 case 1: querylog open on a fresh directory creates the schema" {
|
||||
|
||||
try testing.expectEqual(querylog_schema.RecreateReason.missing, result.recreated.?);
|
||||
try testing.expectEqual(
|
||||
@as(i64, querylog_schema.fingerprint),
|
||||
@as(i64, querylog_versions.current_version),
|
||||
try result.database.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
try testing.expectEqual(
|
||||
@@ -387,7 +389,7 @@ test "S7 case 2: reopening a healthy querylog recreates nothing" {
|
||||
try testing.expectEqual(@as(usize, 0), asides.items.items.len);
|
||||
}
|
||||
|
||||
test "S7 case 3: a wrong user_version recreates and keeps the old file aside" {
|
||||
test "S7 case 3: a version this build cannot handle refuses and touches nothing" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
var f: Fixture = .init();
|
||||
@@ -396,29 +398,42 @@ test "S7 case 3: a wrong user_version recreates and keeps the old file aside" {
|
||||
|
||||
var buf: [path_buf_len]u8 = undefined;
|
||||
const path = try f.pathZ(&buf, "querylog.db");
|
||||
try stampUserVersion(path, querylog_schema.fingerprint +% 1);
|
||||
|
||||
const original = try f.read("querylog.db");
|
||||
defer testing.allocator.free(original);
|
||||
querylog_migrations.expected_failures.begin();
|
||||
defer querylog_migrations.expected_failures.end();
|
||||
|
||||
var result = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
|
||||
defer result.database.close();
|
||||
try testing.expectEqual(
|
||||
querylog_schema.RecreateReason.fingerprint_mismatch,
|
||||
result.recreated.?,
|
||||
);
|
||||
// Both refusal lanes, against a checkpointed file with no sidecars beside
|
||||
// it: a stamp above this build's version (a downgrade) and a stamp that is
|
||||
// not a version at all (a pre-0.0.12 fingerprint, or a foreign file).
|
||||
const refusals = [_]struct { stamp: i32, expected: anyerror }{
|
||||
.{ .stamp = querylog_versions.current_version + 1, .expected = error.SchemaTooNew },
|
||||
.{ .stamp = 0, .expected = error.SchemaUnsupported },
|
||||
.{ .stamp = -7, .expected = error.SchemaUnsupported },
|
||||
.{ .stamp = 603440875, .expected = error.SchemaUnsupported },
|
||||
};
|
||||
|
||||
var asides = try collectAsides(&f);
|
||||
defer asides.deinit();
|
||||
try testing.expectEqual(@as(usize, 1), asides.items.items.len);
|
||||
for (refusals) |lane| {
|
||||
try stampUserVersion(path, lane.stamp);
|
||||
try testing.expect(!try f.exists("querylog.db-wal"));
|
||||
|
||||
// The file was healthy: this build's schema moved, the database did not rot.
|
||||
// An operator who reads "corrupt" here deletes a file that was never broken.
|
||||
try testing.expect(std.mem.startsWith(u8, asides.items.items[0], "querylog.db.schema-changed-"));
|
||||
const original = try f.read("querylog.db");
|
||||
defer testing.allocator.free(original);
|
||||
|
||||
const kept = try f.read(asides.items.items[0]);
|
||||
defer testing.allocator.free(kept);
|
||||
try testing.expectEqualSlices(u8, original, kept);
|
||||
try testing.expectError(
|
||||
lane.expected,
|
||||
querylog_schema.open(io, std.Io.Dir.cwd(), path),
|
||||
);
|
||||
|
||||
// Byte-identical, not merely "still readable": nothing was rewritten,
|
||||
// no aside was made, and no fresh database was created beside it.
|
||||
const after = try f.read("querylog.db");
|
||||
defer testing.allocator.free(after);
|
||||
try testing.expectEqualSlices(u8, original, after);
|
||||
|
||||
var asides = try collectAsides(&f);
|
||||
defer asides.deinit();
|
||||
try testing.expectEqual(@as(usize, 0), asides.items.items.len);
|
||||
}
|
||||
}
|
||||
|
||||
test "S7 case 4: a garbage file recreates and the garbage is preserved" {
|
||||
@@ -468,11 +483,11 @@ test "S7 case 5: two recreates in the same second produce two distinct aside fil
|
||||
|
||||
var round: usize = 0;
|
||||
while (round < 2) : (round += 1) {
|
||||
try stampUserVersion(path, querylog_schema.fingerprint +% 1);
|
||||
try f.write("querylog.db", "not a database at all");
|
||||
var result = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
|
||||
defer result.database.close();
|
||||
try testing.expectEqual(
|
||||
querylog_schema.RecreateReason.fingerprint_mismatch,
|
||||
querylog_schema.RecreateReason.not_a_database,
|
||||
result.recreated.?,
|
||||
);
|
||||
}
|
||||
@@ -492,7 +507,7 @@ test "S7 case 6: a stale write-ahead log is removed before the fresh database is
|
||||
|
||||
var buf: [path_buf_len]u8 = undefined;
|
||||
const path = try f.pathZ(&buf, "querylog.db");
|
||||
try stampUserVersion(path, querylog_schema.fingerprint +% 1);
|
||||
try f.write("querylog.db", "not a database at all");
|
||||
|
||||
// Existence alone proves nothing: the fresh database turns WAL on again and
|
||||
// writes its own `-wal`. The marker is what distinguishes the stale file
|
||||
@@ -503,7 +518,7 @@ test "S7 case 6: a stale write-ahead log is removed before the fresh database is
|
||||
var result = try querylog_schema.open(io, std.Io.Dir.cwd(), path);
|
||||
defer result.database.close();
|
||||
try testing.expectEqual(
|
||||
querylog_schema.RecreateReason.fingerprint_mismatch,
|
||||
querylog_schema.RecreateReason.not_a_database,
|
||||
result.recreated.?,
|
||||
);
|
||||
|
||||
@@ -591,7 +606,7 @@ test "S7 case 23: a locked querylog propagates Busy and is never destroyed" {
|
||||
defer reopened.database.close();
|
||||
try testing.expectEqual(@as(?querylog_schema.RecreateReason, null), reopened.recreated);
|
||||
try testing.expectEqual(
|
||||
@as(i64, querylog_schema.fingerprint),
|
||||
@as(i64, querylog_versions.current_version),
|
||||
try reopened.database.queryInt("PRAGMA user_version"),
|
||||
);
|
||||
|
||||
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
-- FROZEN FIXTURE. Representative content for a querylog.db at schema version 1,
|
||||
-- loaded on top of `querylog-v1-schema.sql`.
|
||||
--
|
||||
-- IMMUTABLE once released, for the same reason as its schema half: the release
|
||||
-- gate byte-compares it against the previous tag. A future schema version ships
|
||||
-- a NEW pair rather than editing this one.
|
||||
--
|
||||
-- The rows are chosen to be hard on a migration rather than realistic: every
|
||||
-- `route_kind`, the NULL variants of `qtype`, `cache_hit`, `response_time_us`,
|
||||
-- `upstream` and `forward_zone`, a non-IN qclass with a non-zero rcode, the
|
||||
-- group and source id/name pairs, `cname_target`/`safe_search_target`, an
|
||||
-- `available_since` that has been advanced away from its DDL default, and
|
||||
-- timestamps that straddle two 30-minute projection buckets.
|
||||
--
|
||||
-- The `bucket_*` rows below are the recomputation of the raw rows, transcribed
|
||||
-- from the same SQL `queries_repo`'s coherence oracle recomputes with. A
|
||||
-- fixture-validity test runs that oracle over this file BEFORE any migration,
|
||||
-- so an incoherent transcription fails on its own rather than as a migration
|
||||
-- bug.
|
||||
|
||||
UPDATE querylog_meta SET created_at = 1699998000, available_since = 1699998600 WHERE id = 1;
|
||||
|
||||
INSERT INTO domains (id, domain) VALUES
|
||||
(1, 'ads.example'),
|
||||
(2, 'news.example'),
|
||||
(3, 'chat.example'),
|
||||
(4, 'printer.lan'),
|
||||
(5, 'nas.lan'),
|
||||
(6, 'cdn.example');
|
||||
|
||||
INSERT INTO query_log (
|
||||
id, timestamp, domain_id, client_ip, qtype, blocked, response_time_us,
|
||||
cache_hit, upstream, qclass, rcode, group_id, group_name, policy_action,
|
||||
policy_reason, matched, source_id, source_name, cname_target,
|
||||
safe_search_target, route_kind, forward_zone
|
||||
) VALUES
|
||||
(1, 1699999260, 1, '10.0.0.1', 1, 1, NULL, 0, NULL, 1, 0, 7, 'kids',
|
||||
'block', 'blocklist_domain', 'ads.example', 3, 'stevenblack', NULL, NULL,
|
||||
'blocked', NULL),
|
||||
(2, 1699999320, 2, '10.0.0.1', 28, 0, 1500, 0, '9.9.9.9:853', 1, 0, NULL,
|
||||
NULL, 'allow', 'no_match', NULL, NULL, NULL, NULL, NULL, 'upstream', NULL),
|
||||
(3, 1699999380, 2, '10.0.0.2', 1, 0, 90, 1, NULL, 1, 0, NULL, NULL,
|
||||
'allow', 'no_match', NULL, NULL, NULL, NULL, NULL, 'cache', NULL),
|
||||
(4, 1699999440, 3, '10.0.0.2', NULL, 0, NULL, NULL, NULL, 3, 4, NULL, NULL,
|
||||
'not_evaluated', 'non_in_class', NULL, NULL, NULL, NULL, NULL, 'rejected',
|
||||
NULL),
|
||||
(5, 1699999500, 4, '10.0.0.3', 1, 0, 200, 0, NULL, 1, 0, NULL, NULL,
|
||||
'not_evaluated', 'local_record', NULL, NULL, NULL, NULL, NULL, 'local',
|
||||
NULL),
|
||||
(6, 1700000700, 5, '10.0.0.3', 15, 0, 3400, 0, NULL, 1, 0, NULL, NULL,
|
||||
'not_evaluated', 'forward_zone', NULL, NULL, NULL, NULL, NULL,
|
||||
'forward_zone', 'lan.example'),
|
||||
(7, 1700001060, 1, '10.0.0.1', 1, 1, NULL, 0, NULL, 1, 0, 7, 'kids',
|
||||
'block', 'blocklist_wildcard', '*.ads.example', 3, 'stevenblack', NULL,
|
||||
NULL, 'blocked', NULL),
|
||||
(8, 1700001120, 6, '10.0.0.4', 65, 0, 2500, 1, '1.1.1.1:853', 1, 0, NULL,
|
||||
NULL, 'allow', 'no_match', NULL, NULL, NULL, 'edge.cdn.example',
|
||||
'forcesafesearch.example', 'upstream', NULL);
|
||||
|
||||
INSERT INTO bucket_totals (bucket, queries, blocked, cached, rt_sum, rt_count) VALUES
|
||||
(1699999200, 6, 1, 1, 5190, 4),
|
||||
(1700001000, 2, 1, 1, 2500, 1);
|
||||
|
||||
INSERT INTO bucket_clients (bucket, client_ip, queries) VALUES
|
||||
(1699999200, '10.0.0.1', 2),
|
||||
(1699999200, '10.0.0.2', 2),
|
||||
(1699999200, '10.0.0.3', 2),
|
||||
(1700001000, '10.0.0.1', 1),
|
||||
(1700001000, '10.0.0.4', 1);
|
||||
|
||||
-- qtype -1 is the lossless encoding of the NULL qtype on row 4.
|
||||
INSERT INTO bucket_types (bucket, qtype, count) VALUES
|
||||
(1699999200, -1, 1),
|
||||
(1699999200, 1, 3),
|
||||
(1699999200, 15, 1),
|
||||
(1699999200, 28, 1),
|
||||
(1700001000, 1, 1),
|
||||
(1700001000, 65, 1);
|
||||
|
||||
INSERT INTO bucket_routes (bucket, route_kind, source_present, source_text, count) VALUES
|
||||
(1699999200, 'blocked', 0, '', 1),
|
||||
(1699999200, 'cache', 0, '', 1),
|
||||
(1699999200, 'forward_zone', 1, 'lan.example', 1),
|
||||
(1699999200, 'local', 0, '', 1),
|
||||
(1699999200, 'rejected', 0, '', 1),
|
||||
(1699999200, 'upstream', 1, '9.9.9.9:853', 1),
|
||||
(1700001000, 'blocked', 0, '', 1),
|
||||
(1700001000, 'upstream', 1, '1.1.1.1:853', 1);
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
-- FROZEN FIXTURE. querylog.db schema version 1: byte-for-byte the `ddl` text of
|
||||
-- `src/storage/querylog_schema.zig`, which is the schema the 0.0.12 and 0.0.13
|
||||
-- binaries created and the one version 1 names.
|
||||
--
|
||||
-- IMMUTABLE once released. The release gate byte-compares this file against the
|
||||
-- previous tag and fails the cut on any edit, because it is the starting point
|
||||
-- every future migration is proved against: editing it would prove a migration
|
||||
-- against a file no operator ever had. A new schema version ships a NEW pair.
|
||||
--
|
||||
-- The `PRAGMA user_version` stamp is deliberately NOT part of this file. The
|
||||
-- loader applies it, which is what lets one fixture serve both the version-1
|
||||
-- stamp and the 0.0.12/0.0.13 legacy fingerprint.
|
||||
|
||||
CREATE TABLE domains (
|
||||
id INTEGER PRIMARY KEY,
|
||||
domain TEXT NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE query_log (
|
||||
id INTEGER PRIMARY KEY,
|
||||
timestamp INTEGER NOT NULL,
|
||||
domain_id INTEGER NOT NULL REFERENCES domains(id),
|
||||
client_ip TEXT NOT NULL, -- text, not a FK: log rows are immutable facts
|
||||
qtype INTEGER,
|
||||
blocked INTEGER NOT NULL,
|
||||
response_time_us INTEGER,
|
||||
cache_hit INTEGER,
|
||||
upstream TEXT,
|
||||
qclass INTEGER NOT NULL,
|
||||
rcode INTEGER NOT NULL,
|
||||
group_id INTEGER, -- text/id pairs, not FKs: a renamed
|
||||
group_name TEXT, -- group must not rewrite history
|
||||
policy_action TEXT NOT NULL,
|
||||
policy_reason TEXT NOT NULL,
|
||||
matched TEXT,
|
||||
source_id INTEGER,
|
||||
source_name TEXT,
|
||||
cname_target TEXT,
|
||||
safe_search_target TEXT,
|
||||
route_kind TEXT NOT NULL,
|
||||
forward_zone TEXT,
|
||||
CHECK (rcode BETWEEN 0 AND 4095) -- twelve bits (RFC 6891 6.1.3)
|
||||
);
|
||||
CREATE INDEX idx_query_log_ts ON query_log(timestamp);
|
||||
CREATE INDEX idx_query_log_client ON query_log(client_ip);
|
||||
CREATE INDEX idx_query_log_domain ON query_log(domain_id);
|
||||
|
||||
CREATE TABLE querylog_meta (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1), -- one row, enforced by the schema
|
||||
created_at INTEGER NOT NULL,
|
||||
available_since INTEGER NOT NULL
|
||||
);
|
||||
INSERT INTO querylog_meta (id, created_at, available_since)
|
||||
VALUES (1, unixepoch(), unixepoch() + 1);
|
||||
|
||||
CREATE TABLE bucket_totals (
|
||||
bucket INTEGER PRIMARY KEY,
|
||||
queries INTEGER NOT NULL,
|
||||
blocked INTEGER NOT NULL,
|
||||
cached INTEGER NOT NULL,
|
||||
rt_sum INTEGER NOT NULL, -- sum(response_time_us) over timed rows
|
||||
rt_count INTEGER NOT NULL -- count(response_time_us)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE bucket_clients (
|
||||
bucket INTEGER NOT NULL,
|
||||
client_ip TEXT NOT NULL,
|
||||
queries INTEGER NOT NULL,
|
||||
PRIMARY KEY (bucket, client_ip)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE bucket_types (
|
||||
bucket INTEGER NOT NULL,
|
||||
qtype INTEGER NOT NULL, -- -1 encodes a NULL qtype, losslessly
|
||||
count INTEGER NOT NULL,
|
||||
PRIMARY KEY (bucket, qtype)
|
||||
) WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE bucket_routes (
|
||||
bucket INTEGER NOT NULL,
|
||||
route_kind TEXT NOT NULL,
|
||||
source_present INTEGER NOT NULL, -- 0: source NULL; 1: source = source_text
|
||||
source_text TEXT NOT NULL, -- '' when source_present = 0
|
||||
count INTEGER NOT NULL,
|
||||
PRIMARY KEY (bucket, route_kind, source_present, source_text),
|
||||
CHECK (source_present IN (0, 1)),
|
||||
CHECK (source_present = 1 OR source_text = '')
|
||||
) WITHOUT ROWID;
|
||||
Reference in New Issue
Block a user