//! The `querylog.db` schema and its open-or-recreate policy. //! //! `querylog.db` is never migrated (PLAN §3.7). It holds expendable log rows, //! so a schema change replaces the file instead of upgrading it. The //! replacement trigger is a fingerprint derived from the DDL text itself, so //! editing the schema below automatically invalidates every existing file — the //! policy cannot drift out of sync with the SQL. //! //! **Recreating is destructive, so the predicate is a positive whitelist.** Only //! a missing file, `error.Corrupt`, `error.NotADb`, a failed `PRAGMA //! quick_check` and a fingerprint mismatch recreate. Every other error //! propagates and the file on disk is not touched. `error.Busy` / `error.Locked` //! mean another process holds the write lock — waiting is right, deleting is //! catastrophic. `error.OutOfMemory` is this process's problem. `error.CantOpen` //! is usually a permission or missing-directory problem that recreating would //! mask rather than fix. Same for `error.ReadOnly`, `error.IoErr`, `error.Full`, //! `error.Perm`, `error.Auth` and `error.Canceled`. const std = @import("std"); const db = @import("db.zig"); const log = std.log.scoped(.querylog_schema); /// Verbatim from PLAN §11.3. Multi-statement text — it goes through /// `db.Db.exec`, never through `prepare`. pub const ddl: [:0]const u8 = \\CREATE TABLE domains ( \\ id INTEGER PRIMARY KEY, \\ domain TEXT NOT NULL UNIQUE \\); \\ \\CREATE TABLE query_log ( \\ id INTEGER PRIMARY KEY, \\ timestamp INTEGER NOT NULL, \\ domain_id INTEGER NOT NULL REFERENCES domains(id), \\ client_ip TEXT NOT NULL, -- text, not a FK: log rows are immutable facts \\ qtype INTEGER, \\ blocked INTEGER NOT NULL, \\ block_reason TEXT, \\ response_time_us INTEGER, \\ cache_hit INTEGER, \\ upstream TEXT \\); \\CREATE INDEX idx_query_log_ts ON query_log(timestamp); \\CREATE INDEX idx_query_log_client ON query_log(client_ip); \\CREATE INDEX idx_query_log_domain ON query_log(domain_id); \\ \\CREATE TABLE upstream_targets ( \\ id INTEGER PRIMARY KEY, \\ url TEXT NOT NULL UNIQUE -- the historical identity: config.db ids cannot cross database files \\); \\ \\CREATE TABLE upstream_minute ( \\ upstream_id INTEGER NOT NULL REFERENCES upstream_targets(id), \\ minute_ts INTEGER NOT NULL, \\ successes INTEGER NOT NULL, \\ failures INTEGER NOT NULL, \\ last_failure_ts INTEGER, \\ last_error TEXT, \\ PRIMARY KEY (upstream_id, minute_ts), \\ CHECK (successes >= 0), \\ CHECK (failures >= 0) \\) WITHOUT ROWID; \\CREATE INDEX idx_upstream_minute_ts ON upstream_minute(minute_ts); ; /// `PRAGMA user_version` is a signed 32-bit field. Deriving the fingerprint from /// the DDL means editing the schema automatically invalidates every existing /// file — which is exactly the policy. pub const fingerprint: i32 = blk: { // Covers the CRC lookup-table generation in std.hash.crc, which evaluates // under this scope's quota and overflows the 1000 default (and 100k). @setEvalBranchQuota(2_000_000); break :blk @bitCast(std.hash.Crc32.hash(ddl)); }; const set_user_version = std.fmt.comptimePrint("PRAGMA user_version = {d};", .{fingerprint}); /// Long enough for any path this program will be handed, plus the aside suffix. /// A longer path is `error.NameTooLong`, which is what the filesystem calls /// would have returned anyway. const path_buf_len = 4096 + 64; pub const RecreateReason = enum { missing, corrupt, not_a_database, quick_check_failed, fingerprint_mismatch }; pub const OpenResult = struct { database: db.Db, /// Non-null feeds a counter and the `/api/health` rollup. recreated: ?RecreateReason, }; pub const Error = db.Error || error{AsideNameCollision} || std.Io.Dir.RenamePreserveError || std.Io.Dir.DeleteFileError || std.Io.Dir.AccessError; /// Opens `path`, recreating it if and only if it is genuinely unusable. /// /// `path` is resolved twice by two different mechanisms: `dir`-relative for the /// filesystem calls, and process-cwd-relative by SQLite's VFS, which knows /// nothing about `dir`. The caller must therefore pass either an absolute path /// with `dir` open on its parent, or `std.Io.Dir.cwd()` with a cwd-relative /// path. pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult { var handle: ?db.Db = null; errdefer if (handle) |*h| h.close(); const reason: ?RecreateReason = probe: { dir.access(io, path, .{}) catch |e| switch (e) { error.FileNotFound => break :probe .missing, else => |other| return other, }; handle = db.Db.open(path, .{ .mode = .read_write_existing }) catch |e| break :probe recreatable(e) orelse return e; const opened = &handle.?; db.applyPragmas(opened, .{}) catch |e| break :probe recreatable(e) orelse return e; const healthy = quickCheck(opened) catch |e| break :probe recreatable(e) orelse return e; if (!healthy) break :probe .quick_check_failed; const stamped = opened.queryInt("PRAGMA user_version") catch |e| break :probe recreatable(e) orelse return e; if (stamped != fingerprint) break :probe .fingerprint_mismatch; break :probe null; }; const cause = reason orelse return .{ .database = handle.?, .recreated = null }; // Close first, so SQLite checkpoints and drops `-wal`/`-shm` where it can. if (handle) |*h| h.close(); handle = null; var aside_buf: [path_buf_len]u8 = undefined; const aside: ?[]const u8 = if (cause == .missing) null else try renameAside(io, dir, path, &aside_buf); // Not optional: a stale WAL left beside the renamed database would be // replayed into the freshly created file and corrupt it immediately. Any // failure other than "already gone" propagates rather than building the new // database on a half-cleaned state. try deleteSidecars(io, dir, path); const fresh = try createFresh(path); if (cause == .missing) { log.info("created querylog database '{s}'", .{path}); } else { log.warn("recreated querylog database '{s}': {s}; previous file kept as '{s}'", .{ path, @tagName(cause), aside.?, }); } return .{ .database = fresh, .recreated = cause }; } /// The whitelist. `null` means "propagate, do not touch the file". fn recreatable(e: db.Error) ?RecreateReason { return switch (e) { error.Corrupt => .corrupt, error.NotADb => .not_a_database, else => null, }; } /// `PRAGMA quick_check` rather than `integrity_check`: it skips the expensive /// index-vs-table cross-check while still catching structural damage, and a /// damaged index on an expendable log is not worth a multi-second startup scan. fn quickCheck(database: *db.Db) db.Error!bool { var stmt = try database.prepare("PRAGMA quick_check"); defer stmt.deinit(); if (!try stmt.step()) return false; return std.ascii.eqlIgnoreCase(stmt.columnText(0), "ok"); } /// Renames the unusable file out of the way and returns the name it now has. /// /// `renamePreserve` is `RENAME_NOREPLACE`: it returns `error.PathAlreadyExists` /// instead of overwriting. A previously saved corrupt file must never be /// destroyed by the next recreate, and two recreates in the same second are not /// hypothetical on a boot loop — hence the uniquifying retries. fn renameAside(io: std.Io, dir: std.Io.Dir, path: []const u8, buf: []u8) Error![]const u8 { const seconds = std.Io.Clock.real.now(io).toSeconds(); var attempt: u32 = 0; while (attempt < 100) : (attempt += 1) { const aside = if (attempt == 0) std.fmt.bufPrint(buf, "{s}.corrupt-{d}", .{ path, seconds }) catch return error.NameTooLong else std.fmt.bufPrint(buf, "{s}.corrupt-{d}-{d}", .{ path, seconds, attempt }) catch return error.NameTooLong; dir.renamePreserve(path, dir, aside, io) catch |e| switch (e) { error.PathAlreadyExists => continue, else => |other| return other, }; return aside; } return error.AsideNameCollision; } fn deleteSidecars(io: std.Io, dir: std.Io.Dir, path: []const u8) Error!void { var buf: [path_buf_len]u8 = undefined; for ([_][]const u8{ "-wal", "-shm" }) |suffix| { const sidecar = std.fmt.bufPrint(&buf, "{s}{s}", .{ path, suffix }) catch return error.NameTooLong; dir.deleteFile(io, sidecar) catch |e| switch (e) { error.FileNotFound => {}, else => |other| return other, }; } } fn createFresh(path: [:0]const u8) db.Error!db.Db { var database = try db.Db.open(path, .{ .mode = .read_write_create }); errdefer database.close(); try db.applyPragmas(&database, .{}); var tx = try db.Tx.begin(&database); errdefer tx.rollback(); try database.exec(ddl); try database.exec(set_user_version); try tx.commit(); return database; } const testing = std.testing; test "fingerprint matches a fresh hash of the DDL" { try testing.expectEqual(fingerprint, @as(i32, @bitCast(std.hash.Crc32.hash(ddl)))); } test "ddl creates the query-log tables, the upstream-history tables and every index" { var database = try db.Db.open(":memory:", .{ .mode = .memory }); defer database.close(); try db.applyPragmas(&database, .{}); try database.exec(ddl); try testing.expectEqual( @as(i64, 4), try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"), ); const objects = [_][]const u8{ "domains", "query_log", "idx_query_log_ts", "idx_query_log_client", "idx_query_log_domain", "upstream_targets", "upstream_minute", "idx_upstream_minute_ts", }; for (objects) |name| { var stmt = try database.prepare("SELECT count(*) FROM sqlite_schema WHERE name = ?1"); defer stmt.deinit(); try stmt.bindText(1, name); try testing.expect(try stmt.step()); try testing.expectEqual(@as(i64, 1), stmt.columnInt(0)); } } test "the user_version statement stamps the fingerprint" { var database = try db.Db.open(":memory:", .{ .mode = .memory }); defer database.close(); try database.exec(set_user_version); try testing.expectEqual(@as(i64, fingerprint), try database.queryInt("PRAGMA user_version")); } // The behaviour these two cases describe — a resource error leaves the file on // disk alone — is proven against a real file by "S7 case 23" in // `storage_integration_test.zig`, which locks a healthy `querylog.db` from a // second connection and asserts `open` returns `error.Busy` with the bytes, the // file and the absence of an aside all intact. `error.OutOfMemory` has no such // case: `open` takes no allocator, and SQLite allocates through its own global // allocator, so there is no seam to inject a failure through. The two tests // below are what covers it. test "recreatable is a whitelist and never selects a resource error" { try testing.expectEqual(RecreateReason.corrupt, recreatable(error.Corrupt).?); try testing.expectEqual(RecreateReason.not_a_database, recreatable(error.NotADb).?); const propagating = [_]db.Error{ error.Busy, error.Locked, error.OutOfMemory, error.CantOpen, error.ReadOnly, error.IoErr, error.Full, error.Perm, error.Auth, error.Misuse, error.Constraint, error.SqliteError, error.Unexpected, }; for (propagating) |e| { try testing.expect(recreatable(e) == null); } } test "recreatable selects exactly two of db.Error's members" { // Exhaustive over the whole set, so a variant added to `db.Error` later // defaults to propagate. The list above only proves the named errors are // safe today; this proves nothing else can join the whitelist unnoticed. var whitelisted: usize = 0; inline for (@typeInfo(db.Error).error_set.?) |member| { if (recreatable(@field(db.Error, member.name)) != null) whitelisted += 1; } try testing.expectEqual(@as(usize, 2), whitelisted); }