From c2eac8d486653490a594d842d8024f89b1c0ae20 Mon Sep 17 00:00:00 2001 From: m5r Date: Wed, 21 Oct 2026 21:40:05 +0200 Subject: [PATCH] storage: each runtime database connection keeps a page cache sized for its role, read back at open --- CHANGELOG.md | 1 + src/app.zig | 15 +++-- src/cli.zig | 27 ++++++++ src/storage/db.zig | 79 ++++++++++++++++++++++++ src/storage/logger_controller.zig | 18 ++++++ src/storage/storage_integration_test.zig | 29 +++++++++ 6 files changed, 161 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6e26da..844fb8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Sections are written by hand. Nothing here is generated from commit messages: th ### Changed +- **Each database connection keeps a page cache sized for its job** instead of SQLite's 2 MiB default on all seven: 256 KiB for the configuration database and for retention, 1 MiB for the query-history writer and reader. - **The documentation is written for people running nxdns.** It starts from Get started, which downloads a release instead of building from source, and drops the API reference, the troubleshooting page and the performance pages. It is published at , and `INSTALL.md` in the tarball links there. ## [0.0.24] - 2026-10-21 diff --git a/src/app.zig b/src/app.zig index 4e519d9..a74fabd 100644 --- a/src/app.zig +++ b/src/app.zig @@ -397,14 +397,13 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 { var data = try cli.DataDir.open(io, gpa, paths.data_dir, true); defer data.close(io, gpa); - var config_db = try data.openConfigDb(io); + var config_db = try data.openMigratedConfigDb(io); defer config_db.close(); - _ = try migrations.migrate(&config_db); // Opened unconditionally, not gated on `cfg.web.enabled`: diagnostics record // what went wrong whether or not anyone is running the UI, and the store // owns this connection outright (`storage/events.zig`). - var events_db = try data.openConfigDb(io); + var events_db = try data.openRuntimeDb(io, .config); defer events_db.close(); const boot_now_s = std.Io.Clock.real.now(io).toSeconds(); var event_store_storage: ?events.Store = events.Store.init(io, &events_db, boot_now_s) catch |err| blk: { @@ -656,21 +655,21 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 { defer log_controller.deinit(io); var sink: query_sink.QuerySink = .init(&log_controller, hub); - var querylog_retention_db = try data.reopenQuerylogDb(io); + var querylog_retention_db = try data.openRuntimeDb(io, .querylog_retention); defer querylog_retention_db.close(); - var tracker_db = try data.openConfigDb(io); + var tracker_db = try data.openRuntimeDb(io, .config); defer tracker_db.close(); // The web task's own two connections (ruling 26; m7 ruling 21: one SQLite // connection per task), opened only when the web interface is (ruling 6). - // `reopenQuerylogDb` requires the file `openQuerylogDb` established above. + // A query-log role requires the file `openQuerylogDb` established above. var web_config_db: ?db.Db = null; defer if (web_config_db) |*database| database.close(); var web_querylog_db: ?db.Db = null; defer if (web_querylog_db) |*database| database.close(); if (cfg.web.enabled) { - web_config_db = try data.openConfigDb(io); - web_querylog_db = try data.reopenQuerylogDb(io); + web_config_db = try data.openRuntimeDb(io, .config); + web_querylog_db = try data.openRuntimeDb(io, .querylog_reader); } var sessions: ?auth.Sessions = if (cfg.web.enabled) .init(cfg.web.session_ttl_hours) else null; diff --git a/src/cli.zig b/src/cli.zig index 7a6214a..6b14048 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -335,10 +335,14 @@ pub const DataDir = struct { /// Returns the whole `OpenResult`, recreate reason and aside name included: /// the composition root records a recreate as a one-shot diagnostics event, /// and the aside name is what tells an operator where the old file went. + /// + /// The connection is the query-log writer's, made a runtime connection + /// once the schema step is done. pub fn openQuerylogDb(self: *const DataDir, io: std.Io) !querylog_schema.OpenResult { var opened = try querylog_schema.open(io, std.Io.Dir.cwd(), self.querylog_db_path); errdefer opened.database.close(); try self.restrictQuerylogPermissions(io); + try db.initRuntimeConnection(&opened.database, .querylog_writer); return opened; } @@ -352,6 +356,29 @@ pub const DataDir = struct { return querylog_schema.reopen(io, std.Io.Dir.cwd(), self.querylog_db_path); } + /// The server's own `config.db` connection: migrated to the current schema, + /// then made a runtime connection. CLI commands migrate through + /// `openConfigDb` and keep SQLite's default cache. + pub fn openMigratedConfigDb(self: *const DataDir, io: std.Io) !db.Db { + var database = try self.openConfigDb(io); + errdefer database.close(); + _ = try migrations.migrate(&database); + try db.initRuntimeConnection(&database, .config); + return database; + } + + /// A further connection for the running server, its page cache sized for + /// `role`. A query-log role requires the file `openQuerylogDb` established. + pub fn openRuntimeDb(self: *const DataDir, io: std.Io, role: db.Role) !db.Db { + var database = switch (role) { + .config => try self.openConfigDb(io), + .querylog_writer, .querylog_retention, .querylog_reader => try self.reopenQuerylogDb(io), + }; + errdefer database.close(); + try db.initRuntimeConnection(&database, role); + return database; + } + /// A sidecar that does not exist yet is not a failure: `-wal` and `-shm` /// appear when SQLite first writes, and the next call catches them. fn restrictQuerylogPermissions(self: *const DataDir, io: std.Io) !void { diff --git a/src/storage/db.zig b/src/storage/db.zig index a319a24..f298bca 100644 --- a/src/storage/db.zig +++ b/src/storage/db.zig @@ -878,6 +878,69 @@ pub fn applyPragmas(self: *Db, p: Pragmas) Error!void { } } +/// What a runtime connection is for, which decides how much page cache it +/// may keep. +/// +/// SQLite's default is 2000 KiB per connection, and every page it caches is a +/// heap block the process keeps once the cache has filled. A page that falls +/// out of this cache is read again from the kernel's page cache, not from the +/// disk, so each role keeps only the pages its own access pattern reuses. The +/// sizes below are in 4 KiB pages, SQLite's default page size, against the +/// Pi's files: `config.db` 136 KiB, `querylog.db` 107 MiB. +pub const Role = enum { + /// 64 pages. The Pi's whole `config.db` is 34. Its one growing table, the + /// resolved diagnostics events, is capped at `events.Store.max_resolved_rows` + /// and is read only when the diagnostics page asks; the configuration and + /// the active events every request reads stay within the cache. + config, + /// 256 pages. One insert walks ten b-trees: the query log, its three + /// indexes, the domains table and its unique index, and the four bucket + /// projections. On a 107 MiB log their interior pages number about a + /// hundred; the rest holds the rightmost leaves the appends land on and + /// the leaves of the domains asked for most. + querylog_writer, + /// 64 pages. A pass runs once a day: one statement deletes the day that + /// aged out, the four projections drop their old buckets and recompute + /// the one the cutoff straddles, and every seventh pass a `VACUUM` streams + /// the whole file through the pager. The pages a pass would reuse, such as + /// the domain-index leaves its deleted rows share, come back from the + /// kernel's cache as copies, and a transaction with more dirty pages than + /// fit spills them to the WAL early. A larger cache would stay allocated + /// through the 24 hours between passes to save the pass a few copies. + querylog_retention, + /// 256 pages. The web API's history page walks the timestamp index and + /// its rows; the statistics read the bucket projections, which every + /// dashboard refresh asks for again. + querylog_reader, + + pub fn cacheKib(role: Role) u32 { + return switch (role) { + .config => 256, + .querylog_writer => 1024, + .querylog_retention => 256, + .querylog_reader => 1024, + }; + } +}; + +/// The step that makes a connection a runtime connection: sets its page cache +/// for `role` and reads it back. Called after the schema is in place, by the +/// code that opens a connection for the running server; migrations and CLI +/// commands keep SQLite's default. +pub fn initRuntimeConnection(self: *Db, role: Role) Error!void { + const kib = role.cacheKib(); + var buf: [48]u8 = undefined; + // A negative `cache_size` is a size in KiB; a positive one counts pages, + // whose size depends on the database file. + const sql = std.fmt.bufPrintZ(&buf, "PRAGMA cache_size = -{d};", .{kib}) catch unreachable; + try self.exec(sql); + const set = try self.queryInt("PRAGMA cache_size"); + if (set != -@as(i64, kib)) { + log.warn("PRAGMA cache_size for a {t} connection reads {d}, expected -{d}", .{ role, set, kib }); + return error.SqliteError; + } +} + /// A write transaction. /// /// Usage contract, followed everywhere in this milestone: @@ -1168,6 +1231,22 @@ test "applyPragmas succeeds and foreign_keys reads back as 1" { try testing.expectEqual(@as(i64, 1), try db.queryInt("PRAGMA foreign_keys")); } +test "a runtime connection reports its role's cache size, a migration connection the default" { + var migrating = try openMemory(); + defer migrating.close(); + try applyPragmas(&migrating, .{}); + // SQLite's compiled-in default: 2000 KiB. + try testing.expectEqual(@as(i64, -2000), try migrating.queryInt("PRAGMA cache_size")); + + for (std.enums.values(Role)) |role| { + var runtime = try openMemory(); + defer runtime.close(); + try applyPragmas(&runtime, .{}); + try initRuntimeConnection(&runtime, role); + try testing.expectEqual(-@as(i64, role.cacheKib()), try runtime.queryInt("PRAGMA cache_size")); + } +} + test "open on a directory path returns error.CantOpen and leaks no handle" { var i: usize = 0; while (i < 1000) : (i += 1) { diff --git a/src/storage/logger_controller.zig b/src/storage/logger_controller.zig index 1eb5879..01d4291 100644 --- a/src/storage/logger_controller.zig +++ b/src/storage/logger_controller.zig @@ -433,6 +433,10 @@ pub const Controller = struct { if (self.retired != null) return error.PreviousResizeDraining; } var database = try querylog_schema.reopen(io, source.dir, source.path); + db.initRuntimeConnection(&database, .querylog_writer) catch |err| { + database.close(); + return err; + }; const generation = build(self.gpa, database, cfg, self.diagnostics) catch |err| { database.close(); return err; @@ -851,6 +855,20 @@ test "a candidate whose writer cannot prepare its statements leaves the running try testing.expectEqual(@as(u64, 8), h.controller.sample(test_io).rows_written); } +test "a resized generation's connection is a runtime writer connection" { + var h: Harness = .{}; + try h.start(.{ .query_log_buffer_max = 16, .query_log_flush_interval_s = 0 }); + defer h.stop(); + + const prepared = try h.prepareResize(64); + defer h.controller.abandon(test_io, prepared); + const database = &prepared.generation.owned.?.database; + try testing.expectEqual( + -@as(i64, db.Role.querylog_writer.cacheKib()), + try database.queryInt("PRAGMA cache_size"), + ); +} + test "a prepared candidate the caller abandons closes everything it opened" { var h: Harness = .{}; try h.start(.{ .query_log_buffer_max = 16, .query_log_flush_interval_s = 0 }); diff --git a/src/storage/storage_integration_test.zig b/src/storage/storage_integration_test.zig index e766553..7bb1453 100644 --- a/src/storage/storage_integration_test.zig +++ b/src/storage/storage_integration_test.zig @@ -678,6 +678,35 @@ test "S7 case 10: a config.db stamped one version ahead is refused and left alon try testing.expectEqualSlices(u8, before, after); } +test "every server connection reports its role's cache size, and a migrating connection the default" { + if (!build_options.integration) return error.SkipZigTest; + + var f: Fixture = .init(); + defer f.deinit(); + + // The CLI's path: migrations run on SQLite's default. + var data = try openMigrated(&f, "data"); + defer data.deinit(); + try testing.expectEqual(@as(i64, -2000), try data.database.queryInt("PRAGMA cache_size")); + + // The server's two schema connections, then every further one. + var config = try data.dir.openMigratedConfigDb(io); + defer config.close(); + try testing.expectEqual(-@as(i64, db.Role.config.cacheKib()), try config.queryInt("PRAGMA cache_size")); + var writer = try data.dir.openQuerylogDb(io); + defer writer.database.close(); + try testing.expectEqual( + -@as(i64, db.Role.querylog_writer.cacheKib()), + try writer.database.queryInt("PRAGMA cache_size"), + ); + + for (std.enums.values(db.Role)) |role| { + var runtime = try data.dir.openRuntimeDb(io, role); + defer runtime.close(); + try testing.expectEqual(-@as(i64, role.cacheKib()), try runtime.queryInt("PRAGMA cache_size")); + } +} + // --------------------------------------------------------------------------- // case 11-19: export and import on real files // ---------------------------------------------------------------------------