milestone 15: make a green run mean a real pass

src/cli.zig

@@ -73,6 +73,29 @@ pub const Command = union(enum) {
     help,
 };
 
+/// The argv spelling of a command paired with its tag. The two differ for
+/// `export` and `import`, whose tags carry a trailing underscore because both
+/// words are Zig keywords.
+pub const CommandName = struct { name: []const u8, tag: std.meta.Tag(Command) };
+
+/// The one list of subcommands. `parseArgs` matches the command word against
+/// it, `docs_drift_test.zig` derives its reference-heading needles from it, and
+/// a test below holds `usage_text` to it.
+pub const command_names = [_]CommandName{
+    .{ .name = "run", .tag = .run },
+    .{ .name = "check", .tag = .check },
+    .{ .name = "export", .tag = .export_ },
+    .{ .name = "import", .tag = .import_ },
+    .{ .name = "version", .tag = .version },
+    .{ .name = "help", .tag = .help },
+};
+
+comptime {
+    // A tag added to `Command` without an entry here fails the compile rather
+    // than silently dropping out of the parser and the doc guard.
+    std.debug.assert(command_names.len == @typeInfo(Command).@"union".fields.len);
+}
+
 pub const ParseError = error{
     UnknownCommand,
     UnknownFlag,
@@ -88,18 +111,30 @@ pub fn parseArgs(argv: []const []const u8) ParseError!Command {
     const command = argv[0];
     const rest = argv[1..];
 
-    if (eql(command, "version")) {
-        if (rest.len != 0) return error.TooManyArguments;
-        return .version;
-    }
-    if (eql(command, "help") or eql(command, "--help") or eql(command, "-h")) {
+    // The two flag spellings of `help` are not subcommands, so they are not in
+    // `command_names` and are matched before it.
+    if (eql(command, "--help") or eql(command, "-h")) {
         if (rest.len != 0) return error.TooManyArguments;
         return .help;
     }
-    if (eql(command, "run")) return .{ .run = try parseRunArgs(rest) };
-    if (eql(command, "check")) return .{ .check = try parseCheckArgs(rest) };
-    if (eql(command, "export")) return .{ .export_ = try parseExportArgs(rest) };
-    if (eql(command, "import")) return .{ .import_ = try parseImportArgs(rest) };
+
+    for (command_names) |entry| {
+        if (!eql(command, entry.name)) continue;
+        switch (entry.tag) {
+            .run => return .{ .run = try parseRunArgs(rest) },
+            .check => return .{ .check = try parseCheckArgs(rest) },
+            .export_ => return .{ .export_ = try parseExportArgs(rest) },
+            .import_ => return .{ .import_ = try parseImportArgs(rest) },
+            .version => {
+                if (rest.len != 0) return error.TooManyArguments;
+                return .version;
+            },
+            .help => {
+                if (rest.len != 0) return error.TooManyArguments;
+                return .help;
+            },
+        }
+    }
     return error.UnknownCommand;
 }
 
@@ -1076,6 +1111,19 @@ test "parseArgs rejects an extra positional argument" {
     try testing.expectError(error.TooManyArguments, parseArgs(&.{ "-h", "extra" }));
 }
 
+test "usage_text lists every command in command_names" {
+    // The commands block indents each entry by two spaces, so a command that
+    // survives only as a word inside an option description does not count.
+    for (command_names) |entry| {
+        var needle_buf: [32]u8 = undefined;
+        const needle = try std.fmt.bufPrint(&needle_buf, "\n  {s} ", .{entry.name});
+        if (std.mem.indexOf(u8, usage_text, needle) == null) {
+            std.debug.print("command missing from usage_text: {s}\n", .{entry.name});
+            return error.CommandMissingFromUsage;
+        }
+    }
+}
+
 test "usage writes non-empty text" {
     var out: Writer.Allocating = .init(testing.allocator);
     defer out.deinit();

src/docs_drift_test.zig

@@ -5,6 +5,7 @@
 
 const std = @import("std");
 const docs = @import("docs_files");
+const cli = @import("cli.zig");
 const routes = @import("web/routes.zig");
 const model = @import("config/model.zig");
 
@@ -48,12 +49,11 @@ test "every settings key appears in docs/reference/configuration.md" {
 
 test "every cli subcommand has its own reference heading in docs/reference/cli.md" {
     const gpa = std.testing.allocator;
-    const subcommands = [_][]const u8{ "run", "check", "export", "import", "version", "help" };
-    for (subcommands) |name| {
+    for (cli.command_names) |entry| {
         // Anchors on the reference-section heading ("## `import FILE`" starts
         // with "## `import"), so prose mentions elsewhere cannot mask a removed
         // command section.
-        const needle = try std.fmt.allocPrint(gpa, "## `{s}", .{name});
+        const needle = try std.fmt.allocPrint(gpa, "## `{s}", .{entry.name});
         defer gpa.free(needle);
         if (std.mem.indexOf(u8, docs.reference_cli_md, needle) == null) {
             std.debug.print("subcommand heading missing from docs/reference/cli.md: {s}\n", .{needle});

src/filter/compiler.zig

@@ -466,6 +466,32 @@ test "an over-long line is skipped when the reader buffer is large" {
     try testing.expectEqualStrings("a.example.com\nb.example.com\n", c.list());
 }
 
+test "an over-long final line with no newline ends the stream inside the discard" {
+    const gpa = testing.allocator;
+    var text: std.ArrayList(u8) = .empty;
+    defer text.deinit(gpa);
+
+    try text.appendSlice(gpa, "a.example.com\n");
+    try text.appendNTimes(gpa, 'x', 5_000);
+
+    // A reader buffer smaller than the trailing line makes `takeDelimiter`
+    // report `error.StreamTooLong`, and the discard that follows then runs out
+    // of input because nothing terminates that line. That is the `EndOfStream`
+    // arm: the loop must count the line and stop, not treat the exhausted
+    // stream as a read failure.
+    var backing: std.Io.Reader = .fixed(text.items);
+    var buf: [max_line_len]u8 = undefined;
+    var limited = backing.limited(.unlimited, &buf);
+
+    var c = try compileReader(gpa, &limited.interface, .domains);
+    defer c.deinit();
+
+    try testing.expectEqual(@as(u32, 1), c.result.counts.long_lines);
+    try testing.expectEqual(@as(u32, 1), c.result.counts.domains);
+    try testing.expectEqualStrings("a.example.com\n", c.list());
+    try testing.expect(std.mem.indexOf(u8, c.list(), "xxxx") == null);
+}
+
 test "carriage returns are stripped" {
     var c = try compileText(testing.allocator, "b.example.com\r\na.example.com\r\n", .domains);
     defer c.deinit();

src/filter/filter_integration_test.zig

@@ -351,10 +351,37 @@ const Env = struct {
 // fixtures: the loopback http server
 // ---------------------------------------------------------------------------
 
-const Route = enum(u8) { body, redirect, not_found, oversize };
+const Route = enum(u8) { body, redirect, not_found, oversize, chunked };
 
 const redirect_path = "/redirected.txt";
 
+const chunked_domains = 1_500;
+
+/// The body the `chunked` route serves (milestone-15 ruling 7). Two thresholds
+/// matter here and they are different: a body over `fetcher.min_transfer_buf`
+/// (16 KiB) makes the fetcher's `pumpBody` loop iterate instead of finishing in
+/// one read, and a body over the fixture connection's 8192-byte write buffer
+/// makes the reply reach the wire in parts. 1500 lines of 31 bytes clears both
+/// with room to spare.
+fn chunkedFixtureBody(gpa: std.mem.Allocator) ![]u8 {
+    var out: std.Io.Writer.Allocating = .init(gpa);
+    errdefer out.deinit();
+    for (0..chunked_domains) |i| {
+        try out.writer.print("0.0.0.0 chunk{d:0>5}.example.com\n", .{i});
+    }
+    return out.toOwnedSlice();
+}
+
+/// `body` cut into three equal parts, deliberately not at line boundaries: a
+/// domain name then straddles every part boundary, so a byte the transfer loses
+/// or repeats there corrupts a name and moves the compiled count, instead of
+/// being absorbed by a spare newline.
+fn thirds(body: []const u8) [3][]const u8 {
+    const first = body.len / 3;
+    const second = 2 * (body.len / 3);
+    return .{ body[0..first], body[first..second], body[second..] };
+}
+
 /// Past `fetcher.max_body_bytes`. The cap is a constant, so the only way to
 /// reach it in a test is a declared length: the fetcher refuses on the response
 /// head, before a byte of body streams.
@@ -364,6 +391,11 @@ const HttpFixture = struct {
     server: net.Server,
     body: []const u8,
     route: std.atomic.Value(u8),
+    /// How many parts the `chunked` route has flushed. The test reads it to
+    /// prove the reply really left this server in pieces, because a `Writer`
+    /// reports a buffered part as written and would otherwise hide a fixture
+    /// that sent everything in one go.
+    flushed_parts: std.atomic.Value(u32),
 
     fn init(io: std.Io, body: []const u8) !HttpFixture {
         const local: net.IpAddress = try .parse("127.0.0.1", 0);
@@ -371,6 +403,7 @@ const HttpFixture = struct {
             .server = try local.listen(io, .{ .reuse_address = true }),
             .body = body,
             .route = .init(@intFromEnum(Route.body)),
+            .flushed_parts = .init(0),
         };
     }
 
@@ -429,8 +462,27 @@ const HttpFixture = struct {
                 .transfer_encoding = .none,
                 .extra_headers = &.{.{ .name = "content-length", .value = oversize_length }},
             }),
+            .chunked => try self.respondChunked(request),
         }
     }
+
+    /// Streams the body in three flushed parts instead of one `respond`
+    /// (milestone-15 ruling 7). Every other arm sends ~130 bytes in a single
+    /// write, which the fetcher consumes in one read: the loop that 35f2324
+    /// killed the process in was never driven end to end by this suite.
+    fn respondChunked(self: *HttpFixture, request: *std.http.Server.Request) !void {
+        var send_buf: [4096]u8 = undefined;
+        var stream = try request.respondStreaming(&send_buf, .{
+            .respond_options = .{ .keep_alive = false },
+        });
+        for (thirds(self.body)) |part| {
+            try stream.writer.writeAll(part);
+            // Without this the parts sit in `send_buf` and leave as one write.
+            try stream.flush();
+            _ = self.flushed_parts.fetchAdd(1, .release);
+        }
+        try stream.end();
+    }
 };
 
 // ---------------------------------------------------------------------------
@@ -1478,3 +1530,64 @@ test "17: each blocking mode synthesizes the documented blocked reply" {
         try testing.expect(packet.findOptRecord(nx_packet) != null);
     }
 }
+
+// ---------------------------------------------------------------------------
+// 18: the multi-read download path
+// ---------------------------------------------------------------------------
+
+test "18: a body streamed in flushed parts survives the fetcher's multi-read pump" {
+    if (!build_options.integration) return error.SkipZigTest;
+
+    const gpa = testing.allocator;
+    const env = try Env.create(gpa);
+    defer env.destroy();
+    const io = env.io();
+
+    const body = try chunkedFixtureBody(gpa);
+    defer gpa.free(body);
+    try testing.expect(body.len >= 24 * 1024);
+    try testing.expect(body.len > fetcher.min_transfer_buf);
+
+    var fixture = try HttpFixture.init(io, body);
+    defer fixture.deinit(io);
+    fixture.setRoute(.chunked);
+    var group: std.Io.Group = .init;
+    defer group.cancel(io);
+    try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
+
+    var url_buf: [64]u8 = undefined;
+    const url = try fixture.url(&url_buf);
+    const id = try seedSource(&env.database, url);
+
+    try testing.expect(try refreshOnce(env, url));
+    try env.mgr.reload(io);
+
+    try testing.expect(fixture.flushed_parts.load(.acquire) >= 3);
+
+    var rows = try listRows(&env.database);
+    defer rows.deinit();
+    const row = try rows.byUrl(url);
+    try testing.expectEqual(@as(i64, chunked_domains), row.domain_count);
+    try testing.expectEqual(@as(i64, 0), row.wildcard_count);
+    try testing.expectEqual(manager.State.ok, (try env.status(id)).state);
+
+    // Every name arrived intact, including the two that straddled a flush
+    // boundary: a byte lost or repeated at a boundary corrupts a name, which
+    // moves the compiled count away from the number of lines sent.
+    var dir = try env.blocklistDir();
+    defer dir.close(io);
+    var bodies = try Bodies.read(gpa, io, dir, "1");
+    defer bodies.deinit(gpa);
+    try testing.expectEqual(
+        @as(usize, chunked_domains),
+        std.mem.count(u8, manager.stripHeader(bodies.list), "\n"),
+    );
+
+    var buf: [64]u8 = undefined;
+    for ([_]usize{ 0, chunked_domains / 3, 2 * chunked_domains / 3, chunked_domains - 1 }) |i| {
+        const domain = try std.fmt.bufPrint(&buf, "chunk{d:0>5}.example.com", .{i});
+        const decision, _ = try env.evaluate(domain);
+        try testing.expect(decision.blocked);
+        try testing.expectEqual(matcher.Reason.blocklist_domain, decision.reason);
+    }
+}

src/platform/logging.zig

@@ -26,6 +26,7 @@
 //! and a log line emitted from the sink would recurse.
 
 const std = @import("std");
+const builtin = @import("builtin");
 const model = @import("../config/model.zig");
 
 /// Two upstream failures with the same key inside this window produce one line.
@@ -585,9 +586,25 @@ fn rotateStepsLocked() RotateError!void {
     try renameLocked(dir, p, first);
 }
 
+const RotateFault = enum { none, fail_delete, fail_rename };
+
+/// The two rotation steps fail only when the filesystem does, which no unit
+/// test can arrange on demand, so the failure paths are driven through this
+/// seam instead. The storage exists in a test build only, and
+/// `rotateFaultTripped` reduces to `false` everywhere else.
+const rotate_fault_seam = if (builtin.is_test) struct {
+    var fault: RotateFault = .none;
+} else struct {};
+
+fn rotateFaultTripped(comptime which: RotateFault) bool {
+    if (!builtin.is_test) return false;
+    return rotate_fault_seam.fault == which;
+}
+
 /// A generation that does not exist yet is not a failure: the first rotations
 /// of a fresh log directory find nothing to delete.
 fn deleteLocked(dir: std.Io.Dir, p: []const u8) RotateError!void {
+    if (rotateFaultTripped(.fail_delete)) return error.RotateFailed;
     dir.deleteFile(state.io, p) catch |err| switch (err) {
         error.FileNotFound => {},
         else => return error.RotateFailed,
@@ -595,6 +612,7 @@ fn deleteLocked(dir: std.Io.Dir, p: []const u8) RotateError!void {
 }
 
 fn renameLocked(dir: std.Io.Dir, from: []const u8, to: []const u8) RotateError!void {
+    if (rotateFaultTripped(.fail_rename)) return error.RotateFailed;
     dir.rename(from, dir, to, state.io) catch |err| switch (err) {
         error.FileNotFound => {},
         else => return error.RotateFailed,
@@ -605,10 +623,9 @@ fn renameLocked(dir: std.Io.Dir, from: []const u8, to: []const u8) RotateError!v
 // Tests
 //
 // The pure pieces only. `logFn` is never exercised: installing a sink under the
-// test runner would eat the harness's own output. File behaviour is S8's, and
-// the rotation failure paths (`rotateLocked` returning false, the pending
-// rotation that keeps the oversized file closed) stay uncovered: they need a
-// filesystem that fails a delete or a rename on demand.
+// test runner would eat the harness's own output. File behaviour is S8's. The
+// rotation failure paths need a filesystem that fails a delete or a rename on
+// demand, which `rotate_fault_seam` supplies.
 // ---------------------------------------------------------------------------
 
 const testing = std.testing;
@@ -908,6 +925,72 @@ test "a failed open counts exactly one sink error" {
     try testing.expectEqual(@as(u64, 0), state.stats.lines_written);
 }
 
+/// The shared body of the two rotation-failure tests: they differ only in which
+/// step is made to fail and in how many `max_files` it takes to reach it.
+///
+/// `state.io` is normally installed by `install`, which no test calls, so a real
+/// one is put in place for the duration: `rotateLocked` swaps cancel protection
+/// on it before the first step runs, and the `fail_rename` case deletes the
+/// oldest generation for real before it reaches the rename. That delete is why
+/// the path points into a fresh tmp directory rather than at any real log.
+fn expectRotationFailureCounted(fault: RotateFault, max_files: u8) !void {
+    var threaded: std.Io.Threaded = .init(testing.allocator, .{});
+    defer threaded.deinit();
+
+    var tmp = testing.tmpDir(.{});
+    defer tmp.cleanup();
+    var path_buf: [128]u8 = undefined;
+    const p = try std.fmt.bufPrint(&path_buf, ".zig-cache/tmp/{s}/nxdns.log", .{tmp.sub_path});
+
+    var stderr_buf: [64]u8 = undefined;
+    _ = std.debug.lockStderr(&stderr_buf);
+    const saved_stats = state.stats;
+    const saved_path_len = state.path_len;
+    const saved_max_files = state.max_files;
+    const saved_file = state.file;
+    const saved_pending = state.rotate_pending;
+    const saved_io = state.io;
+    defer {
+        rotate_fault_seam.fault = .none;
+        state.stats = saved_stats;
+        state.path_len = saved_path_len;
+        state.max_files = saved_max_files;
+        state.file = saved_file;
+        state.rotate_pending = saved_pending;
+        state.io = saved_io;
+        std.debug.unlockStderr();
+    }
+
+    state.stats = .{};
+    state.io = threaded.io();
+    state.path_len = p.len;
+    @memcpy(state.path_buf[0..p.len], p);
+    state.max_files = max_files;
+    state.file = null;
+    state.rotate_pending = true;
+    rotate_fault_seam.fault = fault;
+
+    try testing.expect(!prepareFileLocked(64));
+    try testing.expectEqual(@as(u64, 1), state.stats.sink_errors);
+    try testing.expectEqual(@as(u64, 0), state.stats.rotations);
+    // The oversized file stays closed and the rotation stays owed, so the next
+    // line retries the rotation instead of appending past `max_bytes`.
+    try testing.expect(state.rotate_pending);
+    try testing.expectEqual(@as(?std.Io.File, null), state.file);
+}
+
+test "a failed rotation delete counts exactly one sink error" {
+    // `max_files` below 2 keeps no generations, so the whole rotation is the one
+    // delete of the live path.
+    try expectRotationFailureCounted(.fail_delete, 1);
+}
+
+test "a failed rotation rename counts exactly one sink error" {
+    // `max_files` of 2 keeps generation 1, so the steps are one delete of the
+    // oldest generation followed by the rename of the live path onto it.
+    try expectRotationFailureCounted(.fail_rename, 2);
+}
+
 test "rotatedName appends the generation" {
     var buf: [max_rotated_path_bytes]u8 = undefined;
     try testing.expectEqualStrings(

src/tests.zig

@@ -12,6 +12,7 @@ comptime {
     _ = @import("dns/record.zig");
     _ = @import("dns/edns.zig");
     _ = @import("dns/packet.zig");
+    _ = @import("dns/dns.zig");
     _ = @import("platform/address.zig");
     _ = @import("platform/tls_client.zig");
     _ = @import("platform/tls_client_integration_test.zig");