storage and config: sqlite wrapper, migrations, querylog policy, repositories, zon config with import/export/check cli
This commit is contained in:
@@ -0,0 +1,255 @@
|
||||
//! The `config.db` migration runner.
|
||||
//!
|
||||
//! Steps are compiled into the binary in ascending order and applied inside
|
||||
//! **one** transaction, then the reached version is stamped. SQLite runs DDL
|
||||
//! transactionally, so a step that fails leaves the file exactly as it was.
|
||||
//!
|
||||
//! A database stamped *newer* than this binary is never silently accepted and
|
||||
//! never downgraded: it is `error.SchemaTooNew`, distinct from every other
|
||||
//! error, so the CLI can tell the operator to install a newer nxdns.
|
||||
|
||||
const std = @import("std");
|
||||
const assert = std.debug.assert;
|
||||
|
||||
const db = @import("db.zig");
|
||||
const config_schema = @import("config_schema.zig");
|
||||
|
||||
const log = std.log.scoped(.migrations);
|
||||
|
||||
pub const Step = struct { version: u32, sql: [:0]const u8 };
|
||||
|
||||
pub const steps = [_]Step{
|
||||
.{ .version = 1, .sql = config_schema.ddl_v1 },
|
||||
};
|
||||
|
||||
pub const target_version: u32 = steps[steps.len - 1].version;
|
||||
|
||||
comptime {
|
||||
assertOrdered(&steps);
|
||||
}
|
||||
|
||||
pub const Error = db.Error || error{ SchemaTooNew, SchemaCorrupt };
|
||||
|
||||
/// Versions must be `1, 2, 3, …` with no gaps. A gap would make "apply every
|
||||
/// step newer than the stamped version" ambiguous about what the stamp means.
|
||||
fn assertOrdered(list: []const Step) void {
|
||||
assert(list.len > 0);
|
||||
for (list, 0..) |step, i| assert(@as(usize, step.version) == i + 1);
|
||||
}
|
||||
|
||||
/// Reads the stamped version, applies every newer step in one transaction and
|
||||
/// stamps the result. Returns the version now in the file.
|
||||
///
|
||||
/// `database` must already have had `db.applyPragmas` called: `PRAGMA
|
||||
/// foreign_keys` is a no-op inside a transaction, so applying it afterwards
|
||||
/// would silently leave referential integrity off.
|
||||
pub fn migrate(database: *db.Db) Error!u32 {
|
||||
return migrateSteps(database, &steps);
|
||||
}
|
||||
|
||||
/// Same logic against an injected step list. The seam exists for the rollback
|
||||
/// and stepwise-upgrade tests, which need a second step that `steps` does not
|
||||
/// yet have.
|
||||
pub fn migrateSteps(database: *db.Db, list: []const Step) Error!u32 {
|
||||
assertOrdered(list);
|
||||
const target = list[list.len - 1].version;
|
||||
|
||||
const current = try readVersion(database);
|
||||
if (current > target) {
|
||||
log.warn("config.db is at schema version {d}; this nxdns binary supports {d}", .{ current, target });
|
||||
return error.SchemaTooNew;
|
||||
}
|
||||
if (current == target) return current;
|
||||
|
||||
var tx = try db.Tx.begin(database);
|
||||
errdefer tx.rollback();
|
||||
|
||||
// Re-read under BEGIN IMMEDIATE. Two processes starting at the same moment
|
||||
// both saw `current` above; the one that loses the write lock arrives here
|
||||
// after the other committed and finds nothing to do.
|
||||
const stamped = try readVersion(database);
|
||||
if (stamped > target) {
|
||||
log.warn("config.db is at schema version {d}; this nxdns binary supports {d}", .{ stamped, target });
|
||||
return error.SchemaTooNew;
|
||||
}
|
||||
if (stamped == target) {
|
||||
try tx.commit();
|
||||
return stamped;
|
||||
}
|
||||
|
||||
for (list) |step| {
|
||||
if (step.version <= stamped) continue;
|
||||
try database.exec(step.sql);
|
||||
}
|
||||
|
||||
try database.exec("DELETE FROM schema_version;");
|
||||
var stmt = try database.prepare("INSERT INTO schema_version (version) VALUES (?1)");
|
||||
defer stmt.deinit();
|
||||
try stmt.bindInt(1, target);
|
||||
try stmt.exec();
|
||||
|
||||
try tx.commit();
|
||||
log.info("config.db migrated from schema version {d} to {d}", .{ stamped, target });
|
||||
return target;
|
||||
}
|
||||
|
||||
/// `0` when `schema_version` does not exist yet. Zero rows or more than one row
|
||||
/// is `error.SchemaCorrupt` — the version of a database is never guessed.
|
||||
fn readVersion(database: *db.Db) Error!u32 {
|
||||
const present = try database.queryInt(
|
||||
"SELECT count(*) FROM sqlite_schema WHERE type='table' AND name='schema_version'",
|
||||
);
|
||||
if (present == 0) return 0;
|
||||
|
||||
const rows = try database.queryInt("SELECT count(*) FROM schema_version");
|
||||
if (rows != 1) {
|
||||
log.warn("schema_version holds {d} rows; exactly one is required", .{rows});
|
||||
return error.SchemaCorrupt;
|
||||
}
|
||||
|
||||
const version = try database.queryInt("SELECT version FROM schema_version");
|
||||
if (version < 0 or version > std.math.maxInt(u32)) {
|
||||
log.warn("schema_version holds an out-of-range version {d}", .{version});
|
||||
return error.SchemaCorrupt;
|
||||
}
|
||||
return @intCast(version);
|
||||
}
|
||||
|
||||
fn tableExists(database: *db.Db, name: []const u8) db.Error!bool {
|
||||
var stmt = try database.prepare("SELECT count(*) FROM sqlite_schema WHERE type='table' AND name = ?1");
|
||||
defer stmt.deinit();
|
||||
try stmt.bindText(1, name);
|
||||
if (!try stmt.step()) return error.SqliteError;
|
||||
return stmt.columnInt(0) != 0;
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
fn openMigrated() !db.Db {
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
errdefer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
return database;
|
||||
}
|
||||
|
||||
test "migrate on a fresh database creates every table and seeds the default group" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
try testing.expectEqual(target_version, try migrate(&database));
|
||||
|
||||
const expected = [_][]const u8{
|
||||
"schema_version", "groups", "clients", "client_prefixes",
|
||||
"upstreams", "rules", "local_records", "forward_zones",
|
||||
"blocklist_sources", "group_sources", "settings",
|
||||
};
|
||||
for (expected) |name| {
|
||||
try testing.expect(try tableExists(&database, name));
|
||||
}
|
||||
try testing.expectEqual(
|
||||
@as(i64, expected.len),
|
||||
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
|
||||
);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM groups"));
|
||||
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT id FROM groups"));
|
||||
var stmt = try database.prepare("SELECT name, safe_search FROM groups");
|
||||
defer stmt.deinit();
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqualStrings("default", stmt.columnText(0));
|
||||
try testing.expect(!stmt.columnBool(1));
|
||||
}
|
||||
|
||||
test "migrate is idempotent" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
try testing.expectEqual(target_version, try migrate(&database));
|
||||
const before = try database.queryInt("SELECT count(*) FROM sqlite_schema");
|
||||
const rowid_before = database.lastInsertRowid();
|
||||
|
||||
try testing.expectEqual(target_version, try migrate(&database));
|
||||
try testing.expectEqual(before, try database.queryInt("SELECT count(*) FROM sqlite_schema"));
|
||||
try testing.expectEqual(rowid_before, database.lastInsertRowid());
|
||||
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM schema_version"));
|
||||
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM groups"));
|
||||
}
|
||||
|
||||
test "a database stamped newer than the binary is error.SchemaTooNew" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
_ = try migrate(&database);
|
||||
|
||||
const future: i64 = @as(i64, target_version) + 1;
|
||||
var stmt = try database.prepare("UPDATE schema_version SET version = ?1");
|
||||
defer stmt.deinit();
|
||||
try stmt.bindInt(1, future);
|
||||
try stmt.exec();
|
||||
|
||||
try testing.expectError(error.SchemaTooNew, migrate(&database));
|
||||
try testing.expectEqual(future, try database.queryInt("SELECT version FROM schema_version"));
|
||||
}
|
||||
|
||||
test "schema_version holding two rows is error.SchemaCorrupt" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
_ = try migrate(&database);
|
||||
|
||||
try database.exec("INSERT INTO schema_version (version) VALUES (1);");
|
||||
try testing.expectError(error.SchemaCorrupt, migrate(&database));
|
||||
}
|
||||
|
||||
test "a failing step rolls the whole migration back" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
const broken = [_]Step{
|
||||
.{ .version = 1, .sql = config_schema.ddl_v1 },
|
||||
.{ .version = 2, .sql = "CREATE TABLE second (" },
|
||||
};
|
||||
// db.zig maps SQLITE_ERROR — the generic "SQL error" — to error.Unexpected.
|
||||
try testing.expectError(error.Unexpected, migrateSteps(&database, &broken));
|
||||
|
||||
try testing.expect(!try tableExists(&database, "schema_version"));
|
||||
try testing.expect(!try tableExists(&database, "groups"));
|
||||
try testing.expect(!try tableExists(&database, "second"));
|
||||
try testing.expectEqual(@as(u32, 0), try readVersion(&database));
|
||||
}
|
||||
|
||||
test "a stepwise upgrade applies only the new steps" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
|
||||
try testing.expectEqual(@as(u32, 1), try migrateSteps(&database, &first));
|
||||
try testing.expect(try tableExists(&database, "groups"));
|
||||
try testing.expect(!try tableExists(&database, "extra"));
|
||||
|
||||
const second = [_]Step{
|
||||
.{ .version = 1, .sql = config_schema.ddl_v1 },
|
||||
.{ .version = 2, .sql = "CREATE TABLE extra (id INTEGER PRIMARY KEY);" },
|
||||
};
|
||||
try testing.expectEqual(@as(u32, 2), try migrateSteps(&database, &second));
|
||||
try testing.expect(try tableExists(&database, "extra"));
|
||||
try testing.expectEqual(@as(u32, 2), try readVersion(&database));
|
||||
// Step 1 did not run a second time: `groups` still holds one seeded row.
|
||||
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM groups"));
|
||||
}
|
||||
|
||||
test "delete_order and content_tables name exactly the tables the schema creates" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
_ = try migrate(&database);
|
||||
|
||||
for (config_schema.delete_order) |name| {
|
||||
try testing.expect(try tableExists(&database, name));
|
||||
}
|
||||
for (config_schema.content_tables) |name| {
|
||||
try testing.expect(try tableExists(&database, name));
|
||||
}
|
||||
// delete_order covers every table except `schema_version`.
|
||||
try testing.expectEqual(
|
||||
@as(i64, config_schema.delete_order.len + 1),
|
||||
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user