milestone 20: declarative configuration for iac
This commit is contained in:
@@ -22,6 +22,7 @@ const build_options = @import("build_options");
|
||||
const net = std.Io.net;
|
||||
|
||||
const model = @import("../config/model.zig");
|
||||
const reconcile = @import("../config/reconcile.zig");
|
||||
const db = @import("../storage/db.zig");
|
||||
const migrations = @import("../storage/migrations.zig");
|
||||
const context = @import("../storage/repositories/context.zig");
|
||||
@@ -400,6 +401,10 @@ const HttpFixture = struct {
|
||||
server: net.Server,
|
||||
body: []const u8,
|
||||
route: std.atomic.Value(u8),
|
||||
/// Connections accepted, whatever came over them. A test that claims a pass
|
||||
/// downloaded nothing reads this rather than the route counters: a refetch
|
||||
/// that failed on the wire is still a refetch, and this counts it.
|
||||
accepted: std.atomic.Value(u32),
|
||||
/// How many parts the `chunked` route has flushed. The test reads it to
|
||||
/// prove the reply really left this server in pieces, because a `Writer`
|
||||
/// reports a buffered part as written and would otherwise hide a fixture
|
||||
@@ -422,6 +427,7 @@ const HttpFixture = struct {
|
||||
.server = try local.listen(io, .{ .reuse_address = true }),
|
||||
.body = body,
|
||||
.route = .init(@intFromEnum(Route.body)),
|
||||
.accepted = .init(0),
|
||||
.flushed_parts = .init(0),
|
||||
.stall_reached = .unset,
|
||||
.stall_release = .unset,
|
||||
@@ -449,6 +455,7 @@ const HttpFixture = struct {
|
||||
while (true) {
|
||||
var stream = self.server.accept(io) catch return;
|
||||
defer stream.close(io);
|
||||
_ = self.accepted.fetchAdd(1, .monotonic);
|
||||
|
||||
var read_buf: [8192]u8 = undefined;
|
||||
var write_buf: [8192]u8 = undefined;
|
||||
@@ -1372,6 +1379,112 @@ test "10d: a source deleted mid-refresh does not take the refresh's temporary fi
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 10e: the restart invariant, across the config engine and the filter layer
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test "10e: a reconcile then a restart reuses the compiled files and downloads nothing" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fixture = try HttpFixture.init(io, http_body);
|
||||
defer fixture.deinit(io);
|
||||
var group: std.Io.Group = .init;
|
||||
defer group.cancel(io);
|
||||
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||
|
||||
var url_buf: [64]u8 = undefined;
|
||||
const url = try fixture.url(&url_buf);
|
||||
const id = try seedSource(&env.database, url);
|
||||
|
||||
// One real download, so the compiled artifacts exist and are named after
|
||||
// the row id the rest of this test is about.
|
||||
try testing.expect(try refreshOnce(env, url));
|
||||
try testing.expectEqual(@as(u32, 1), fixture.accepted.load(.monotonic));
|
||||
|
||||
var dir = try env.blocklistDir();
|
||||
defer dir.close(io);
|
||||
var list_buf: [64]u8 = undefined;
|
||||
var wild_buf: [64]u8 = undefined;
|
||||
const list_name = try std.fmt.bufPrint(&list_buf, "{d}.list", .{id});
|
||||
const wild_name = try std.fmt.bufPrint(&wild_buf, "{d}.wild", .{id});
|
||||
const list_before = try dir.statFile(io, list_name, .{});
|
||||
const wild_before = try dir.statFile(io, wild_name, .{});
|
||||
|
||||
// File mode, declaring exactly what the database already holds. The engine
|
||||
// has to recognise the source by its url and leave the row where it is:
|
||||
// the compiled files are named after that id, and the manager looks for
|
||||
// them under the same number.
|
||||
const cfg: model.Config = .{
|
||||
.groups = &.{.{ .name = "default" }},
|
||||
.blocklist_sources = &.{.{ .url = url, .name = source_name }},
|
||||
.group_sources = &.{.{ .group = "default", .source_url = url }},
|
||||
.upstreams = &.{.{ .url = "https://dns.example/dns-query" }},
|
||||
};
|
||||
var pass = try reconcile.begin(
|
||||
io,
|
||||
gpa,
|
||||
&env.database,
|
||||
cfg,
|
||||
std.Io.Clock.real.now(io).toSeconds(),
|
||||
.{},
|
||||
);
|
||||
errdefer pass.rollback();
|
||||
// Committed before the manager comes up, which is the ordering the invariant
|
||||
// rests on: a manager that read the table mid-transaction could see either
|
||||
// half of a source it is about to look for on disk.
|
||||
try pass.commit();
|
||||
|
||||
// The restart. The old manager is gone and a new one comes up over the same
|
||||
// directory and the same database with nothing carried across in memory.
|
||||
// `runScheduler` is the boot sequence the server runs — the orphan sweep,
|
||||
// then the startup pass — and a disabled update makes it return rather than
|
||||
// wait out an interval.
|
||||
env.mgr.deinit(io);
|
||||
env.mgr = try manager.Manager.init(
|
||||
gpa,
|
||||
&env.database,
|
||||
.{ .dir = env.tmp.dir },
|
||||
&env.f,
|
||||
.{ .enabled = false },
|
||||
budget,
|
||||
);
|
||||
try env.mgr.runScheduler(io);
|
||||
|
||||
// Nothing was downloaded. The server is still listening, so this is a
|
||||
// decision the pass made rather than a connection it could not have opened.
|
||||
try testing.expectEqual(@as(u32, 1), fixture.accepted.load(.monotonic));
|
||||
|
||||
// The same two files: not recompiled, and not swept as orphans and written
|
||||
// back.
|
||||
const list_after = try dir.statFile(io, list_name, .{});
|
||||
const wild_after = try dir.statFile(io, wild_name, .{});
|
||||
try testing.expectEqual(list_before.inode, list_after.inode);
|
||||
try testing.expectEqual(list_before.mtime, list_after.mtime);
|
||||
try testing.expectEqual(wild_before.inode, wild_after.inode);
|
||||
try testing.expectEqual(wild_before.mtime, wild_after.mtime);
|
||||
|
||||
// The row kept the id those files are named after, and the snapshot the
|
||||
// restart published is the one compiled from them.
|
||||
var rows = try listRows(&env.database);
|
||||
defer rows.deinit();
|
||||
try testing.expectEqual(id, (try rows.byUrl(url)).id);
|
||||
try testing.expectEqual(manager.State.ok, (try env.status(id)).state);
|
||||
|
||||
// Asserted last, after the behaviour it explains: the engine wrote no row
|
||||
// at all, which is why the id above survived and why the restart above had
|
||||
// files to find.
|
||||
try testing.expectEqual(@as(u32, 0), pass.summary.sources.total());
|
||||
|
||||
const decision, _ = try env.evaluate("ads.example.com");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expectEqual(matcher.Reason.blocklist_domain, decision.reason);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 11–12: local records, from the database to the wire
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1199,6 +1199,14 @@ pub const Manager = struct {
|
||||
|
||||
if (state != .ok) return true;
|
||||
const last = row.last_updated orelse return true;
|
||||
// The Pi has no RTC, so a fetch stamped while the clock ran ahead of
|
||||
// real time (a pre-NTP boot, a restored image) leaves a `last_updated`
|
||||
// in the future. Plain interval arithmetic would then suspend every
|
||||
// refresh until real time caught up with the poison stamp, and the
|
||||
// reconcile engine preserves runtime columns faithfully, so nothing
|
||||
// else would ever clear it. A stamp from the future is not evidence of
|
||||
// a recent fetch.
|
||||
if (last > now) return true;
|
||||
return now - last >= model.updateIntervalSeconds(self.update);
|
||||
}
|
||||
|
||||
@@ -1674,6 +1682,40 @@ test "acquire before any reload returns null and holds no lock" {
|
||||
manager.lock.unlock(io);
|
||||
}
|
||||
|
||||
test "needsRefresh treats a last_updated in the future as due" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
var f: fetcher.Fetcher = undefined;
|
||||
var manager = try testManager(&database, &f);
|
||||
defer manager.deinit(io);
|
||||
|
||||
// `.ok` is the only state that consults the clock at all; every other one
|
||||
// is already due, so the arithmetic below would be unreachable without it.
|
||||
var statuses = [_]SourceStatus{.{ .id = 1, .state = .ok }};
|
||||
manager.statuses = &statuses;
|
||||
defer manager.statuses = &.{};
|
||||
|
||||
const row = testRow(1, true);
|
||||
const stamp = row.last_updated.?;
|
||||
const interval = model.updateIntervalSeconds(manager.update);
|
||||
|
||||
// The ordinary cases still hold: fresh is not due, stale is.
|
||||
try testing.expect(!manager.needsRefresh(io, row, stamp + 1));
|
||||
try testing.expect(manager.needsRefresh(io, row, stamp + interval));
|
||||
|
||||
// The Pi has no RTC. A fetch stamped while the clock ran ahead of real
|
||||
// time leaves `now - last` negative, which reads as "fetched moments ago"
|
||||
// and suspends every refresh until real time catches the poison stamp —
|
||||
// for a whole day here, and for as long as the clock was wrong in general.
|
||||
try testing.expect(manager.needsRefresh(io, row, stamp - 1));
|
||||
try testing.expect(manager.needsRefresh(io, row, stamp - 86_400));
|
||||
}
|
||||
|
||||
test "the disk gate skips a scheduled refresh only while writes are critical" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
|
||||
Reference in New Issue
Block a user