storage and config: sqlite wrapper, migrations, querylog policy, repositories, zon config with import/export/check cli
This commit is contained in:
@@ -0,0 +1,280 @@
|
||||
//! The `querylog.db` schema and its open-or-recreate policy.
|
||||
//!
|
||||
//! `querylog.db` is never migrated (PLAN §3.7). It holds expendable log rows,
|
||||
//! so a schema change replaces the file instead of upgrading it. The
|
||||
//! replacement trigger is a fingerprint derived from the DDL text itself, so
|
||||
//! editing the schema below automatically invalidates every existing file — the
|
||||
//! policy cannot drift out of sync with the SQL.
|
||||
//!
|
||||
//! **Recreating is destructive, so the predicate is a positive whitelist.** Only
|
||||
//! a missing file, `error.Corrupt`, `error.NotADb`, a failed `PRAGMA
|
||||
//! quick_check` and a fingerprint mismatch recreate. Every other error
|
||||
//! propagates and the file on disk is not touched. `error.Busy` / `error.Locked`
|
||||
//! mean another process holds the write lock — waiting is right, deleting is
|
||||
//! catastrophic. `error.OutOfMemory` is this process's problem. `error.CantOpen`
|
||||
//! is usually a permission or missing-directory problem that recreating would
|
||||
//! mask rather than fix. Same for `error.ReadOnly`, `error.IoErr`, `error.Full`,
|
||||
//! `error.Perm`, `error.Auth` and `error.Canceled`.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const db = @import("db.zig");
|
||||
|
||||
const log = std.log.scoped(.querylog_schema);
|
||||
|
||||
/// Verbatim from PLAN §11.3. Multi-statement text — it goes through
|
||||
/// `db.Db.exec`, never through `prepare`.
|
||||
pub const ddl: [:0]const u8 =
|
||||
\\CREATE TABLE domains (
|
||||
\\ id INTEGER PRIMARY KEY,
|
||||
\\ domain TEXT NOT NULL UNIQUE
|
||||
\\);
|
||||
\\
|
||||
\\CREATE TABLE query_log (
|
||||
\\ id INTEGER PRIMARY KEY,
|
||||
\\ timestamp INTEGER NOT NULL,
|
||||
\\ domain_id INTEGER NOT NULL REFERENCES domains(id),
|
||||
\\ client_ip TEXT NOT NULL, -- text, not a FK: log rows are immutable facts
|
||||
\\ qtype INTEGER,
|
||||
\\ blocked INTEGER NOT NULL,
|
||||
\\ block_reason TEXT,
|
||||
\\ response_time_us INTEGER,
|
||||
\\ cache_hit INTEGER,
|
||||
\\ upstream TEXT
|
||||
\\);
|
||||
\\CREATE INDEX idx_query_log_ts ON query_log(timestamp);
|
||||
\\CREATE INDEX idx_query_log_client ON query_log(client_ip);
|
||||
\\CREATE INDEX idx_query_log_domain ON query_log(domain_id);
|
||||
;
|
||||
|
||||
/// `PRAGMA user_version` is a signed 32-bit field. Deriving the fingerprint from
|
||||
/// the DDL means editing the schema automatically invalidates every existing
|
||||
/// file — which is exactly the policy.
|
||||
pub const fingerprint: i32 = blk: {
|
||||
// Covers the CRC lookup-table generation in std.hash.crc, which evaluates
|
||||
// under this scope's quota and overflows the 1000 default (and 100k).
|
||||
@setEvalBranchQuota(2_000_000);
|
||||
break :blk @bitCast(std.hash.Crc32.hash(ddl));
|
||||
};
|
||||
|
||||
const set_user_version = std.fmt.comptimePrint("PRAGMA user_version = {d};", .{fingerprint});
|
||||
|
||||
/// Long enough for any path this program will be handed, plus the aside suffix.
|
||||
/// A longer path is `error.NameTooLong`, which is what the filesystem calls
|
||||
/// would have returned anyway.
|
||||
const path_buf_len = 4096 + 64;
|
||||
|
||||
pub const RecreateReason = enum { missing, corrupt, not_a_database, quick_check_failed, fingerprint_mismatch };
|
||||
|
||||
pub const OpenResult = struct {
|
||||
database: db.Db,
|
||||
/// Non-null feeds a counter and the `/api/health` rollup in Phase 8.
|
||||
recreated: ?RecreateReason,
|
||||
};
|
||||
|
||||
pub const Error = db.Error || error{AsideNameCollision} ||
|
||||
std.Io.Dir.RenamePreserveError || std.Io.Dir.DeleteFileError || std.Io.Dir.AccessError;
|
||||
|
||||
/// Opens `path`, recreating it if and only if it is genuinely unusable.
|
||||
///
|
||||
/// `path` is resolved twice by two different mechanisms: `dir`-relative for the
|
||||
/// filesystem calls, and process-cwd-relative by SQLite's VFS, which knows
|
||||
/// nothing about `dir`. The caller must therefore pass either an absolute path
|
||||
/// with `dir` open on its parent, or `std.Io.Dir.cwd()` with a cwd-relative
|
||||
/// path.
|
||||
pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
|
||||
var handle: ?db.Db = null;
|
||||
errdefer if (handle) |*h| h.close();
|
||||
|
||||
const reason: ?RecreateReason = probe: {
|
||||
dir.access(io, path, .{}) catch |e| switch (e) {
|
||||
error.FileNotFound => break :probe .missing,
|
||||
else => |other| return other,
|
||||
};
|
||||
|
||||
handle = db.Db.open(path, .{ .mode = .read_write_existing }) catch |e|
|
||||
break :probe recreatable(e) orelse return e;
|
||||
const opened = &handle.?;
|
||||
|
||||
db.applyPragmas(opened, .{}) catch |e|
|
||||
break :probe recreatable(e) orelse return e;
|
||||
|
||||
const healthy = quickCheck(opened) catch |e|
|
||||
break :probe recreatable(e) orelse return e;
|
||||
if (!healthy) break :probe .quick_check_failed;
|
||||
|
||||
const stamped = opened.queryInt("PRAGMA user_version") catch |e|
|
||||
break :probe recreatable(e) orelse return e;
|
||||
if (stamped != fingerprint) break :probe .fingerprint_mismatch;
|
||||
|
||||
break :probe null;
|
||||
};
|
||||
|
||||
const cause = reason orelse return .{ .database = handle.?, .recreated = null };
|
||||
|
||||
// Close first, so SQLite checkpoints and drops `-wal`/`-shm` where it can.
|
||||
if (handle) |*h| h.close();
|
||||
handle = null;
|
||||
|
||||
var aside_buf: [path_buf_len]u8 = undefined;
|
||||
const aside: ?[]const u8 = if (cause == .missing)
|
||||
null
|
||||
else
|
||||
try renameAside(io, dir, path, &aside_buf);
|
||||
|
||||
// Not optional: a stale WAL left beside the renamed database would be
|
||||
// replayed into the freshly created file and corrupt it immediately. Any
|
||||
// failure other than "already gone" propagates rather than building the new
|
||||
// database on a half-cleaned state.
|
||||
try deleteSidecars(io, dir, path);
|
||||
|
||||
const fresh = try createFresh(path);
|
||||
if (cause == .missing) {
|
||||
log.info("created querylog database '{s}'", .{path});
|
||||
} else {
|
||||
log.warn("recreated querylog database '{s}': {s}; previous file kept as '{s}'", .{
|
||||
path,
|
||||
@tagName(cause),
|
||||
aside.?,
|
||||
});
|
||||
}
|
||||
return .{ .database = fresh, .recreated = cause };
|
||||
}
|
||||
|
||||
/// The whitelist. `null` means "propagate, do not touch the file".
|
||||
fn recreatable(e: db.Error) ?RecreateReason {
|
||||
return switch (e) {
|
||||
error.Corrupt => .corrupt,
|
||||
error.NotADb => .not_a_database,
|
||||
else => null,
|
||||
};
|
||||
}
|
||||
|
||||
/// `PRAGMA quick_check` rather than `integrity_check`: it skips the expensive
|
||||
/// index-vs-table cross-check while still catching structural damage, and a
|
||||
/// damaged index on an expendable log is not worth a multi-second startup scan.
|
||||
fn quickCheck(database: *db.Db) db.Error!bool {
|
||||
var stmt = try database.prepare("PRAGMA quick_check");
|
||||
defer stmt.deinit();
|
||||
if (!try stmt.step()) return false;
|
||||
return std.ascii.eqlIgnoreCase(stmt.columnText(0), "ok");
|
||||
}
|
||||
|
||||
/// Renames the unusable file out of the way and returns the name it now has.
|
||||
///
|
||||
/// `renamePreserve` is `RENAME_NOREPLACE`: it returns `error.PathAlreadyExists`
|
||||
/// instead of overwriting. A previously saved corrupt file must never be
|
||||
/// destroyed by the next recreate, and two recreates in the same second are not
|
||||
/// hypothetical on a boot loop — hence the uniquifying retries.
|
||||
fn renameAside(io: std.Io, dir: std.Io.Dir, path: []const u8, buf: []u8) Error![]const u8 {
|
||||
const seconds = std.Io.Clock.real.now(io).toSeconds();
|
||||
var attempt: u32 = 0;
|
||||
while (attempt < 100) : (attempt += 1) {
|
||||
const aside = if (attempt == 0)
|
||||
std.fmt.bufPrint(buf, "{s}.corrupt-{d}", .{ path, seconds }) catch return error.NameTooLong
|
||||
else
|
||||
std.fmt.bufPrint(buf, "{s}.corrupt-{d}-{d}", .{ path, seconds, attempt }) catch return error.NameTooLong;
|
||||
|
||||
dir.renamePreserve(path, dir, aside, io) catch |e| switch (e) {
|
||||
error.PathAlreadyExists => continue,
|
||||
else => |other| return other,
|
||||
};
|
||||
return aside;
|
||||
}
|
||||
return error.AsideNameCollision;
|
||||
}
|
||||
|
||||
fn deleteSidecars(io: std.Io, dir: std.Io.Dir, path: []const u8) Error!void {
|
||||
var buf: [path_buf_len]u8 = undefined;
|
||||
for ([_][]const u8{ "-wal", "-shm" }) |suffix| {
|
||||
const sidecar = std.fmt.bufPrint(&buf, "{s}{s}", .{ path, suffix }) catch return error.NameTooLong;
|
||||
dir.deleteFile(io, sidecar) catch |e| switch (e) {
|
||||
error.FileNotFound => {},
|
||||
else => |other| return other,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
fn createFresh(path: [:0]const u8) db.Error!db.Db {
|
||||
var database = try db.Db.open(path, .{ .mode = .read_write_create });
|
||||
errdefer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
|
||||
var tx = try db.Tx.begin(&database);
|
||||
errdefer tx.rollback();
|
||||
try database.exec(ddl);
|
||||
try database.exec(set_user_version);
|
||||
try tx.commit();
|
||||
|
||||
return database;
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "fingerprint matches a fresh hash of the DDL" {
|
||||
try testing.expectEqual(fingerprint, @as(i32, @bitCast(std.hash.Crc32.hash(ddl))));
|
||||
}
|
||||
|
||||
test "ddl creates domains, query_log and the three indexes" {
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
defer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
try database.exec(ddl);
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 2),
|
||||
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
|
||||
);
|
||||
const objects = [_][]const u8{
|
||||
"domains", "query_log",
|
||||
"idx_query_log_ts", "idx_query_log_client",
|
||||
"idx_query_log_domain",
|
||||
};
|
||||
for (objects) |name| {
|
||||
var stmt = try database.prepare("SELECT count(*) FROM sqlite_schema WHERE name = ?1");
|
||||
defer stmt.deinit();
|
||||
try stmt.bindText(1, name);
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqual(@as(i64, 1), stmt.columnInt(0));
|
||||
}
|
||||
}
|
||||
|
||||
test "the user_version statement stamps the fingerprint" {
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
defer database.close();
|
||||
try database.exec(set_user_version);
|
||||
try testing.expectEqual(@as(i64, fingerprint), try database.queryInt("PRAGMA user_version"));
|
||||
}
|
||||
|
||||
// The behaviour these two cases describe — a resource error leaves the file on
|
||||
// disk alone — is proven against a real file by "S7 case 23" in
|
||||
// `storage_integration_test.zig`, which locks a healthy `querylog.db` from a
|
||||
// second connection and asserts `open` returns `error.Busy` with the bytes, the
|
||||
// file and the absence of an aside all intact. `error.OutOfMemory` has no such
|
||||
// case: `open` takes no allocator, and SQLite allocates through its own global
|
||||
// allocator, so there is no seam to inject a failure through. The two tests
|
||||
// below are what covers it.
|
||||
test "recreatable is a whitelist and never selects a resource error" {
|
||||
try testing.expectEqual(RecreateReason.corrupt, recreatable(error.Corrupt).?);
|
||||
try testing.expectEqual(RecreateReason.not_a_database, recreatable(error.NotADb).?);
|
||||
const propagating = [_]db.Error{
|
||||
error.Busy, error.Locked, error.OutOfMemory, error.CantOpen,
|
||||
error.ReadOnly, error.IoErr, error.Full, error.Perm,
|
||||
error.Auth, error.Misuse, error.Constraint, error.SqliteError,
|
||||
error.Unexpected,
|
||||
};
|
||||
for (propagating) |e| {
|
||||
try testing.expect(recreatable(e) == null);
|
||||
}
|
||||
}
|
||||
|
||||
test "recreatable selects exactly two of db.Error's members" {
|
||||
// Exhaustive over the whole set, so a variant added to `db.Error` later
|
||||
// defaults to propagate. The list above only proves the named errors are
|
||||
// safe today; this proves nothing else can join the whitelist unnoticed.
|
||||
var whitelisted: usize = 0;
|
||||
inline for (@typeInfo(db.Error).error_set.?) |member| {
|
||||
if (recreatable(@field(db.Error, member.name)) != null) whitelisted += 1;
|
||||
}
|
||||
try testing.expectEqual(@as(usize, 2), whitelisted);
|
||||
}
|
||||
Reference in New Issue
Block a user