milestone 5: blocklist filtering, local records and conditional forwarding
This commit is contained in:
@@ -0,0 +1,500 @@
|
||||
//! Compiles a downloaded blocklist into the two bodies nxdns stores on disk:
|
||||
//! a `.list` body of exact names and a `.wild` body of suffixes.
|
||||
//!
|
||||
//! Pure over reader/writer interfaces: an allocator, a `*std.Io.Reader` and two
|
||||
//! `*std.Io.Writer`. No `std.Io` value, no file, no clock. A compiled body is a
|
||||
//! pure function of (bytes, format), which is what makes two runs — and two
|
||||
//! permutations of the same input — byte-identical.
|
||||
//!
|
||||
//! Nothing but the sorted, deduplicated names is written: no header, no
|
||||
//! timestamp, no counts. The header belongs to the caller, and the checksum
|
||||
//! covers the two bodies only.
|
||||
|
||||
const std = @import("std");
|
||||
const parsers = @import("parsers.zig");
|
||||
const name = @import("../dns/name.zig");
|
||||
const types = @import("../dns/types.zig");
|
||||
|
||||
const Sha256 = std.crypto.hash.sha2.Sha256;
|
||||
|
||||
pub const max_domains: u32 = 2_000_000;
|
||||
pub const max_line_len: usize = 4096;
|
||||
|
||||
pub const Counts = struct {
|
||||
domains: u32 = 0,
|
||||
wildcards: u32 = 0,
|
||||
skipped_regex: u32 = 0,
|
||||
skipped_unsupported: u32 = 0,
|
||||
/// Not a valid domain name (`dns.name.fromText` rejected it, a non-ASCII
|
||||
/// byte, or fewer than two labels).
|
||||
invalid: u32 = 0,
|
||||
/// Lines longer than `max_line_len`, skipped whole.
|
||||
long_lines: u32 = 0,
|
||||
/// Duplicates removed by the sort/unique pass.
|
||||
duplicates: u32 = 0,
|
||||
};
|
||||
|
||||
pub const Result = struct {
|
||||
counts: Counts,
|
||||
/// Lowercase hex sha256 over the `.list` body followed by the `.wild` body.
|
||||
checksum: [64]u8,
|
||||
};
|
||||
|
||||
pub const Error = error{ OutOfMemory, TooManyDomains, ReadFailed, WriteFailed };
|
||||
|
||||
/// Reads `r` to end of stream and writes the two compiled bodies.
|
||||
///
|
||||
/// `counts.domains` and `counts.wildcards` are the written, deduplicated
|
||||
/// counts: they are what `blocklist_sources.domain_count` and `wildcard_count`
|
||||
/// store and what the UI shows.
|
||||
pub fn compile(
|
||||
gpa: std.mem.Allocator,
|
||||
r: *std.Io.Reader,
|
||||
format: parsers.Format,
|
||||
list_w: *std.Io.Writer,
|
||||
wild_w: *std.Io.Writer,
|
||||
) Error!Result {
|
||||
var counts: Counts = .{};
|
||||
|
||||
var list: Entries = .{};
|
||||
defer list.deinit(gpa);
|
||||
var wild: Entries = .{};
|
||||
defer wild.deinit(gpa);
|
||||
|
||||
while (true) {
|
||||
const raw = r.takeDelimiter('\n') catch |err| switch (err) {
|
||||
error.ReadFailed => return error.ReadFailed,
|
||||
// `takeDelimiter` leaves the stream unmodified on `StreamTooLong`
|
||||
// (Reader.zig:885). Without this discard the loop re-reads the same
|
||||
// bytes forever.
|
||||
error.StreamTooLong => {
|
||||
counts.long_lines += 1;
|
||||
_ = r.discardDelimiterInclusive('\n') catch |discard_err| switch (discard_err) {
|
||||
error.EndOfStream => break,
|
||||
error.ReadFailed => return error.ReadFailed,
|
||||
};
|
||||
continue;
|
||||
},
|
||||
} orelse break;
|
||||
|
||||
var line = raw;
|
||||
if (line.len != 0 and line[line.len - 1] == '\r') line = line[0 .. line.len - 1];
|
||||
// A reader whose buffer is larger than `max_line_len` reports the
|
||||
// over-long line here instead of through `error.StreamTooLong`.
|
||||
if (line.len > max_line_len) {
|
||||
counts.long_lines += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const parsed = parsers.parseLine(format, line);
|
||||
switch (parsed.kind) {
|
||||
.ignore => {},
|
||||
.regex => counts.skipped_regex += 1,
|
||||
.unsupported => counts.skipped_unsupported += 1,
|
||||
.domain => {
|
||||
var fields = std.mem.tokenizeAny(u8, parsed.text, &std.ascii.whitespace);
|
||||
while (fields.next()) |field| {
|
||||
try addCandidate(gpa, field, false, false, &list, &wild, &counts);
|
||||
}
|
||||
},
|
||||
.wildcard => try addCandidate(
|
||||
gpa,
|
||||
parsed.text,
|
||||
true,
|
||||
parsed.covers_apex,
|
||||
&list,
|
||||
&wild,
|
||||
&counts,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
var hasher = Sha256.init(.{});
|
||||
counts.domains = try emit(&list, list_w, &hasher, &counts.duplicates);
|
||||
counts.wildcards = try emit(&wild, wild_w, &hasher, &counts.duplicates);
|
||||
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
return .{ .counts = counts, .checksum = std.fmt.bytesToHex(digest, .lower) };
|
||||
}
|
||||
|
||||
/// Normalizes one whitespace-separated candidate and files it under `.list`,
|
||||
/// `.wild`, or neither.
|
||||
fn addCandidate(
|
||||
gpa: std.mem.Allocator,
|
||||
field: []const u8,
|
||||
from_wildcard_line: bool,
|
||||
covers_apex: bool,
|
||||
list: *Entries,
|
||||
wild: *Entries,
|
||||
counts: *Counts,
|
||||
) Error!void {
|
||||
var candidate = field;
|
||||
var is_wildcard = from_wildcard_line;
|
||||
if (std.mem.startsWith(u8, candidate, "*.")) {
|
||||
is_wildcard = true;
|
||||
candidate = candidate[2..];
|
||||
}
|
||||
// A '*' anywhere else makes this a pattern, and patterns belong to the
|
||||
// `rules` table; a blocklist entry is a name or a suffix.
|
||||
if (std.mem.indexOfScalar(u8, candidate, '*') != null) {
|
||||
counts.invalid += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (candidate.len != 0 and candidate[candidate.len - 1] == '.') {
|
||||
candidate = candidate[0 .. candidate.len - 1];
|
||||
}
|
||||
if (candidate.len == 0 or candidate.len > types.max_name_len) {
|
||||
counts.invalid += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
var buf: [types.max_name_len]u8 = undefined;
|
||||
for (candidate, 0..) |c, i| {
|
||||
if (c >= 0x80 or std.ascii.isControl(c)) {
|
||||
counts.invalid += 1;
|
||||
return;
|
||||
}
|
||||
buf[i] = std.ascii.toLower(c);
|
||||
}
|
||||
const normalized = buf[0..candidate.len];
|
||||
|
||||
const parsed_name = name.fromText(normalized) catch {
|
||||
counts.invalid += 1;
|
||||
return;
|
||||
};
|
||||
// The two-label minimum is what keeps `localhost`, `local`, `broadcasthost`
|
||||
// and the `ip6-*` names that every hosts list carries from black-holing the
|
||||
// loopback names of every client on the LAN.
|
||||
if (parsed_name.labelCount() < 2) {
|
||||
counts.invalid += 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (is_wildcard) {
|
||||
try wild.append(gpa, normalized);
|
||||
// An ABP `||x^` rule covers `x` itself as well as its subdomains.
|
||||
if (covers_apex) try list.append(gpa, normalized);
|
||||
} else {
|
||||
try list.append(gpa, normalized);
|
||||
}
|
||||
}
|
||||
|
||||
/// Sorts, deduplicates, writes and hashes one body. Returns the written count.
|
||||
fn emit(
|
||||
entries: *Entries,
|
||||
w: *std.Io.Writer,
|
||||
hasher: *Sha256,
|
||||
duplicates: *u32,
|
||||
) Error!u32 {
|
||||
const offsets = entries.offsets.items;
|
||||
if (offsets.len > 1) std.mem.sortUnstableContext(0, offsets.len, SortContext{ .entries = entries });
|
||||
|
||||
var written: u32 = 0;
|
||||
var prev: []const u8 = "";
|
||||
var first = true;
|
||||
for (0..offsets.len) |i| {
|
||||
const text = entries.get(i);
|
||||
if (!first and std.mem.eql(u8, prev, text)) {
|
||||
duplicates.* += 1;
|
||||
continue;
|
||||
}
|
||||
try w.writeAll(text);
|
||||
try w.writeByte('\n');
|
||||
hasher.update(text);
|
||||
hasher.update("\n");
|
||||
prev = text;
|
||||
first = false;
|
||||
written += 1;
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
/// Length-prefixed candidate bytes plus the offsets that index them. Sorting
|
||||
/// permutes the offsets, so the bytes never move.
|
||||
const Entries = struct {
|
||||
bytes: std.ArrayList(u8) = .empty,
|
||||
offsets: std.ArrayList(u32) = .empty,
|
||||
|
||||
fn deinit(self: *Entries, gpa: std.mem.Allocator) void {
|
||||
self.bytes.deinit(gpa);
|
||||
self.offsets.deinit(gpa);
|
||||
}
|
||||
|
||||
fn append(self: *Entries, gpa: std.mem.Allocator, text: []const u8) Error!void {
|
||||
if (self.offsets.items.len >= max_domains) return error.TooManyDomains;
|
||||
const offset = std.math.cast(u32, self.bytes.items.len) orelse return error.TooManyDomains;
|
||||
try self.bytes.append(gpa, @intCast(text.len));
|
||||
try self.bytes.appendSlice(gpa, text);
|
||||
try self.offsets.append(gpa, offset);
|
||||
}
|
||||
|
||||
fn get(self: *const Entries, i: usize) []const u8 {
|
||||
const offset = self.offsets.items[i];
|
||||
const len = self.bytes.items[offset];
|
||||
return self.bytes.items[offset + 1 ..][0..len];
|
||||
}
|
||||
};
|
||||
|
||||
const SortContext = struct {
|
||||
entries: *Entries,
|
||||
|
||||
pub fn lessThan(self: SortContext, a: usize, b: usize) bool {
|
||||
return std.mem.order(u8, self.entries.get(a), self.entries.get(b)) == .lt;
|
||||
}
|
||||
|
||||
pub fn swap(self: SortContext, a: usize, b: usize) void {
|
||||
const offsets = self.entries.offsets.items;
|
||||
std.mem.swap(u32, &offsets[a], &offsets[b]);
|
||||
}
|
||||
};
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
const Compiled = struct {
|
||||
result: Result,
|
||||
list_w: std.Io.Writer.Allocating,
|
||||
wild_w: std.Io.Writer.Allocating,
|
||||
|
||||
fn deinit(self: *Compiled) void {
|
||||
self.list_w.deinit();
|
||||
self.wild_w.deinit();
|
||||
}
|
||||
|
||||
fn list(self: *Compiled) []const u8 {
|
||||
return self.list_w.written();
|
||||
}
|
||||
|
||||
fn wild(self: *Compiled) []const u8 {
|
||||
return self.wild_w.written();
|
||||
}
|
||||
};
|
||||
|
||||
fn compileText(gpa: std.mem.Allocator, text: []const u8, format: parsers.Format) Error!Compiled {
|
||||
var r: std.Io.Reader = .fixed(text);
|
||||
return compileReader(gpa, &r, format);
|
||||
}
|
||||
|
||||
fn compileReader(gpa: std.mem.Allocator, r: *std.Io.Reader, format: parsers.Format) Error!Compiled {
|
||||
var list_w: std.Io.Writer.Allocating = .init(gpa);
|
||||
errdefer list_w.deinit();
|
||||
var wild_w: std.Io.Writer.Allocating = .init(gpa);
|
||||
errdefer wild_w.deinit();
|
||||
const result = try compile(gpa, r, format, &list_w.writer, &wild_w.writer);
|
||||
return .{ .result = result, .list_w = list_w, .wild_w = wild_w };
|
||||
}
|
||||
|
||||
const hosts_fixture =
|
||||
"# a comment\n" ++
|
||||
"0.0.0.0 ads.example.com\n" ++
|
||||
"0.0.0.0 ads.example.com\n" ++
|
||||
"127.0.0.1 localhost\n" ++
|
||||
"0.0.0.0 tracker.example.org # tracker\n" ++
|
||||
"/ads\\d+/\n" ++
|
||||
"\n" ++
|
||||
"0.0.0.0 EXAMPLE.com.\n";
|
||||
|
||||
test "hosts fixture compiles to a sorted deduplicated body" {
|
||||
var c = try compileText(testing.allocator, hosts_fixture, .hosts);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqualStrings(
|
||||
"ads.example.com\nexample.com\ntracker.example.org\n",
|
||||
c.list(),
|
||||
);
|
||||
try testing.expectEqualStrings("", c.wild());
|
||||
try testing.expectEqual(@as(u32, 3), c.result.counts.domains);
|
||||
try testing.expectEqual(@as(u32, 0), c.result.counts.wildcards);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_regex);
|
||||
try testing.expectEqual(@as(u32, 0), c.result.counts.skipped_unsupported);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.invalid);
|
||||
try testing.expectEqual(@as(u32, 0), c.result.counts.long_lines);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.duplicates);
|
||||
}
|
||||
|
||||
test "domains fixture compiles" {
|
||||
const fixture =
|
||||
"# a comment\n" ++
|
||||
"b.example.com\n" ++
|
||||
"! a bang comment\n" ++
|
||||
"a.example.com\n" ++
|
||||
"/re/\n" ++
|
||||
"0.0.0.0 two.fields.com\n" ++
|
||||
"localhost\n";
|
||||
|
||||
var c = try compileText(testing.allocator, fixture, .domains);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqualStrings("a.example.com\nb.example.com\n", c.list());
|
||||
try testing.expectEqualStrings("", c.wild());
|
||||
try testing.expectEqual(@as(u32, 2), c.result.counts.domains);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_regex);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_unsupported);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.invalid);
|
||||
}
|
||||
|
||||
test "abp apex rule lands in both bodies" {
|
||||
const fixture =
|
||||
"! Title: test\n" ++
|
||||
"||x.com^\n" ++
|
||||
"||y.com^$third-party\n" ++
|
||||
"@@||z.com^\n" ++
|
||||
"/re/\n" ++
|
||||
"bare.com\n";
|
||||
|
||||
var c = try compileText(testing.allocator, fixture, .abp);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqualStrings("bare.com\nx.com\n", c.list());
|
||||
try testing.expectEqualStrings("x.com\n", c.wild());
|
||||
try testing.expectEqual(@as(u32, 2), c.result.counts.domains);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.wildcards);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_regex);
|
||||
try testing.expectEqual(@as(u32, 2), c.result.counts.skipped_unsupported);
|
||||
}
|
||||
|
||||
test "two runs of the same input are byte-identical" {
|
||||
var a = try compileText(testing.allocator, hosts_fixture, .hosts);
|
||||
defer a.deinit();
|
||||
var b = try compileText(testing.allocator, hosts_fixture, .hosts);
|
||||
defer b.deinit();
|
||||
|
||||
try testing.expectEqualStrings(a.list(), b.list());
|
||||
try testing.expectEqualStrings(a.wild(), b.wild());
|
||||
try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum);
|
||||
}
|
||||
|
||||
test "a permutation of the input compiles to the same bodies" {
|
||||
const shuffled =
|
||||
"0.0.0.0 EXAMPLE.com.\n" ++
|
||||
"/ads\\d+/\n" ++
|
||||
"0.0.0.0 tracker.example.org # tracker\n" ++
|
||||
"\n" ++
|
||||
"0.0.0.0 ads.example.com\n" ++
|
||||
"127.0.0.1 localhost\n" ++
|
||||
"# a comment\n" ++
|
||||
"0.0.0.0 ads.example.com\n";
|
||||
|
||||
var a = try compileText(testing.allocator, hosts_fixture, .hosts);
|
||||
defer a.deinit();
|
||||
var b = try compileText(testing.allocator, shuffled, .hosts);
|
||||
defer b.deinit();
|
||||
|
||||
try testing.expectEqualStrings(a.list(), b.list());
|
||||
try testing.expectEqualStrings(a.wild(), b.wild());
|
||||
try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum);
|
||||
}
|
||||
|
||||
test "uppercase and trailing dot normalize" {
|
||||
var c = try compileText(testing.allocator, "AdS.Example.COM.\n", .domains);
|
||||
defer c.deinit();
|
||||
try testing.expectEqualStrings("ads.example.com\n", c.list());
|
||||
}
|
||||
|
||||
test "invalid candidates are counted and written nowhere" {
|
||||
const fixture =
|
||||
"caf\xc3\xa9.example.com\n" ++
|
||||
"localhost\n" ++
|
||||
"a*b.com\n";
|
||||
|
||||
var c = try compileText(testing.allocator, fixture, .domains);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqual(@as(u32, 3), c.result.counts.invalid);
|
||||
try testing.expectEqual(@as(u32, 0), c.result.counts.domains);
|
||||
try testing.expectEqualStrings("", c.list());
|
||||
try testing.expectEqualStrings("", c.wild());
|
||||
}
|
||||
|
||||
test "a leading star label becomes a wildcard entry" {
|
||||
var c = try compileText(testing.allocator, "*.ads.example.com\n", .domains);
|
||||
defer c.deinit();
|
||||
try testing.expectEqualStrings("", c.list());
|
||||
try testing.expectEqualStrings("ads.example.com\n", c.wild());
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.wildcards);
|
||||
}
|
||||
|
||||
test "an over-long line is skipped whole" {
|
||||
const gpa = testing.allocator;
|
||||
var text: std.ArrayList(u8) = .empty;
|
||||
defer text.deinit(gpa);
|
||||
|
||||
var line_buf: [64]u8 = undefined;
|
||||
var i: usize = 0;
|
||||
while (i < 3_000) : (i += 1) {
|
||||
if (i == 1_500) {
|
||||
try text.appendNTimes(gpa, 'x', 5_000);
|
||||
try text.append(gpa, '\n');
|
||||
}
|
||||
try text.appendSlice(gpa, try std.fmt.bufPrint(&line_buf, "d{d:0>5}.example.com\n", .{i}));
|
||||
}
|
||||
|
||||
// A reader buffer smaller than the long line makes `takeDelimiter` report
|
||||
// `error.StreamTooLong`, which is the path that loops forever without the
|
||||
// discard.
|
||||
var backing: std.Io.Reader = .fixed(text.items);
|
||||
var buf: [max_line_len]u8 = undefined;
|
||||
var limited = backing.limited(.unlimited, &buf);
|
||||
|
||||
var c = try compileReader(gpa, &limited.interface, .domains);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.long_lines);
|
||||
try testing.expectEqual(@as(u32, 3_000), c.result.counts.domains);
|
||||
try testing.expect(std.mem.startsWith(u8, c.list(), "d00000.example.com\n"));
|
||||
try testing.expect(std.mem.endsWith(u8, c.list(), "d02999.example.com\n"));
|
||||
try testing.expect(std.mem.indexOf(u8, c.list(), "d01499.example.com\n") != null);
|
||||
try testing.expect(std.mem.indexOf(u8, c.list(), "d01500.example.com\n") != null);
|
||||
try testing.expect(std.mem.indexOf(u8, c.list(), "xxxx") == null);
|
||||
}
|
||||
|
||||
test "an over-long line is skipped when the reader buffer is large" {
|
||||
const gpa = testing.allocator;
|
||||
var text: std.ArrayList(u8) = .empty;
|
||||
defer text.deinit(gpa);
|
||||
|
||||
try text.appendSlice(gpa, "a.example.com\n");
|
||||
try text.appendNTimes(gpa, 'x', 5_000);
|
||||
try text.append(gpa, '\n');
|
||||
try text.appendSlice(gpa, "b.example.com\n");
|
||||
|
||||
var c = try compileText(gpa, text.items, .domains);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.long_lines);
|
||||
try testing.expectEqualStrings("a.example.com\nb.example.com\n", c.list());
|
||||
}
|
||||
|
||||
test "carriage returns are stripped" {
|
||||
var c = try compileText(testing.allocator, "b.example.com\r\na.example.com\r\n", .domains);
|
||||
defer c.deinit();
|
||||
try testing.expectEqualStrings("a.example.com\nb.example.com\n", c.list());
|
||||
}
|
||||
|
||||
test "empty input produces empty bodies and the sha256 of the empty string" {
|
||||
var c = try compileText(testing.allocator, "", .domains);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqualStrings("", c.list());
|
||||
try testing.expectEqualStrings("", c.wild());
|
||||
try testing.expectEqualStrings(
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
&c.result.checksum,
|
||||
);
|
||||
}
|
||||
|
||||
fn compileUnderFailure(gpa: std.mem.Allocator) !void {
|
||||
var c = compileText(gpa, hosts_fixture, .hosts) catch |err| switch (err) {
|
||||
// `Writer.Allocating` reports an exhausted allocator as `WriteFailed`;
|
||||
// it has no other failure mode.
|
||||
error.WriteFailed => return error.OutOfMemory,
|
||||
else => |e| return e,
|
||||
};
|
||||
defer c.deinit();
|
||||
try testing.expectEqual(@as(u32, 3), c.result.counts.domains);
|
||||
}
|
||||
|
||||
test "compile under allocation failure" {
|
||||
try testing.checkAllAllocationFailures(testing.allocator, compileUnderFailure, .{});
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
//! An immutable exact-match set of domain names, built from a compiled
|
||||
//! blocklist body.
|
||||
//!
|
||||
//! The layout is a flat arena of length-prefixed names plus an open-addressed
|
||||
//! table of `u32` offsets into it. No Bloom filter and no hash-only key set: a
|
||||
//! false positive in a DNS sinkhole blocks a real domain for a real household
|
||||
//! and is undebuggable from the outside, so every probe compares full bytes.
|
||||
//!
|
||||
//! Pure. Takes an allocator and bytes; no `std.Io`, no clock, no entropy
|
||||
//! source — the hash seed arrives as a parameter.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
pub const DomainSet = struct {
|
||||
/// Length-prefixed lowercase names, back to back: [len: u8][bytes]…
|
||||
arena: []const u8,
|
||||
/// Open-addressed table of offsets into `arena`; `empty_slot` marks a hole.
|
||||
/// Length is always a power of two.
|
||||
index: []const u32,
|
||||
count: u32,
|
||||
seed: u64,
|
||||
|
||||
pub const empty_slot: u32 = std.math.maxInt(u32);
|
||||
pub const max_count: u32 = 4_000_000;
|
||||
pub const max_arena_bytes: usize = 1 << 31;
|
||||
|
||||
pub const Error = error{ OutOfMemory, TooManyDomains, SetTooLarge, NotSorted, NotLowercase };
|
||||
|
||||
/// An empty set that owns nothing. `contains` on it is always false.
|
||||
pub const empty: DomainSet = .{
|
||||
.arena = &.{},
|
||||
.index = &.{},
|
||||
.count = 0,
|
||||
.seed = 0,
|
||||
};
|
||||
|
||||
/// Builds from a compiled body: LF-separated, lowercase, sorted ascending,
|
||||
/// deduplicated, every line 1–255 bytes.
|
||||
///
|
||||
/// The sortedness is verified, not assumed: a hand-edited or truncated file
|
||||
/// must fail loudly rather than produce a set that silently misses entries.
|
||||
/// Verification is one comparison per line. Because a valid line is at least
|
||||
/// one byte, comparing against an initial empty `prev` also rejects an empty
|
||||
/// line and a duplicate line — both are `error.NotSorted`, since neither is
|
||||
/// strictly ascending.
|
||||
///
|
||||
/// A line longer than 255 bytes is `error.SetTooLarge`: the arena stores a
|
||||
/// `u8` length prefix and must not silently truncate, for the same reason
|
||||
/// the `u32` offsets bound `max_arena_bytes`.
|
||||
///
|
||||
/// `seed` randomizes the hash. Query names are attacker-supplied, so a fixed
|
||||
/// seed would make probe-chain flooding computable offline.
|
||||
pub fn build(gpa: std.mem.Allocator, body: []const u8, seed: u64) Error!DomainSet {
|
||||
if (body.len == 0) return empty;
|
||||
if (body.len > max_arena_bytes) return error.SetTooLarge;
|
||||
|
||||
var count: u32 = 0;
|
||||
var arena_len: usize = 0;
|
||||
{
|
||||
var prev: []const u8 = "";
|
||||
var it: LineIterator = .{ .rest = body };
|
||||
while (it.next()) |line| {
|
||||
for (line) |c| if (c >= 'A' and c <= 'Z') return error.NotLowercase;
|
||||
if (line.len > std.math.maxInt(u8)) return error.SetTooLarge;
|
||||
if (std.mem.order(u8, prev, line) != .lt) return error.NotSorted;
|
||||
if (count == max_count) return error.TooManyDomains;
|
||||
prev = line;
|
||||
count += 1;
|
||||
arena_len += 1 + line.len;
|
||||
}
|
||||
}
|
||||
|
||||
const capacity = capacityFor(count);
|
||||
|
||||
const arena = try gpa.alloc(u8, arena_len);
|
||||
errdefer gpa.free(arena);
|
||||
const index = try gpa.alloc(u32, capacity);
|
||||
@memset(index, empty_slot);
|
||||
|
||||
const mask = capacity - 1;
|
||||
var write_at: usize = 0;
|
||||
var it: LineIterator = .{ .rest = body };
|
||||
while (it.next()) |line| {
|
||||
const offset: u32 = @intCast(write_at);
|
||||
arena[write_at] = @intCast(line.len);
|
||||
@memcpy(arena[write_at + 1 ..][0..line.len], line);
|
||||
write_at += 1 + line.len;
|
||||
|
||||
var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(seed, line))) & mask;
|
||||
while (index[slot] != empty_slot) slot = (slot + 1) & mask;
|
||||
index[slot] = offset;
|
||||
}
|
||||
|
||||
return .{ .arena = arena, .index = index, .count = count, .seed = seed };
|
||||
}
|
||||
|
||||
pub fn deinit(self: *DomainSet, gpa: std.mem.Allocator) void {
|
||||
gpa.free(self.arena);
|
||||
gpa.free(self.index);
|
||||
self.* = empty;
|
||||
}
|
||||
|
||||
/// `domain` must be normalized (lowercase, no trailing dot). Allocation-free.
|
||||
pub fn contains(self: *const DomainSet, domain: []const u8) bool {
|
||||
if (self.index.len == 0) return false;
|
||||
const mask = self.index.len - 1;
|
||||
var slot: usize = @as(usize, @truncate(std.hash.Wyhash.hash(self.seed, domain))) & mask;
|
||||
while (true) {
|
||||
const offset = self.index[slot];
|
||||
if (offset == empty_slot) return false;
|
||||
const len = self.arena[offset];
|
||||
if (len == domain.len and std.mem.eql(u8, self.arena[offset + 1 ..][0..len], domain)) {
|
||||
return true;
|
||||
}
|
||||
slot = (slot + 1) & mask;
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes held, for the memory report in `Snapshot.memoryBytes`.
|
||||
pub fn memoryBytes(self: *const DomainSet) usize {
|
||||
return self.arena.len + self.index.len * @sizeOf(u32);
|
||||
}
|
||||
};
|
||||
|
||||
/// Smallest power of two at least `count * 4 / 3`, minimum 16. The load factor
|
||||
/// stays at or under 0.75, which keeps at least one hole and therefore
|
||||
/// terminates the probe loop in `contains`.
|
||||
fn capacityFor(count: u32) usize {
|
||||
const wanted = (@as(u64, count) * 4 + 2) / 3;
|
||||
var capacity: usize = 16;
|
||||
while (capacity < wanted) capacity *= 2;
|
||||
return capacity;
|
||||
}
|
||||
|
||||
/// Yields LF-separated lines. A final line without a trailing LF is yielded;
|
||||
/// a trailing LF does not yield a final empty line.
|
||||
const LineIterator = struct {
|
||||
rest: []const u8,
|
||||
|
||||
fn next(self: *LineIterator) ?[]const u8 {
|
||||
if (self.rest.len == 0) return null;
|
||||
if (std.mem.indexOfScalar(u8, self.rest, '\n')) |nl| {
|
||||
defer self.rest = self.rest[nl + 1 ..];
|
||||
return self.rest[0..nl];
|
||||
}
|
||||
defer self.rest = self.rest[self.rest.len..];
|
||||
return self.rest;
|
||||
}
|
||||
};
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
const small_body =
|
||||
"ads.example.com\n" ++
|
||||
"example.com\n" ++
|
||||
"tracker.example.org\n" ++
|
||||
"zzz.example.net\n";
|
||||
|
||||
test "build and contains over a small body" {
|
||||
var set = try DomainSet.build(testing.allocator, small_body, 0x1234);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(u32, 4), set.count);
|
||||
try testing.expect(set.contains("ads.example.com"));
|
||||
try testing.expect(set.contains("example.com"));
|
||||
try testing.expect(set.contains("tracker.example.org"));
|
||||
try testing.expect(set.contains("zzz.example.net"));
|
||||
|
||||
// A prefix, a suffix, an uppercase spelling and the empty string.
|
||||
try testing.expect(!set.contains("example.co"));
|
||||
try testing.expect(!set.contains("example.com.evil.net"));
|
||||
try testing.expect(!set.contains("Example.com"));
|
||||
try testing.expect(!set.contains(""));
|
||||
}
|
||||
|
||||
test "build rejects an unsorted body" {
|
||||
try testing.expectError(
|
||||
error.NotSorted,
|
||||
DomainSet.build(testing.allocator, "b.example.com\na.example.com\n", 0),
|
||||
);
|
||||
}
|
||||
|
||||
test "build rejects an uppercase byte" {
|
||||
try testing.expectError(
|
||||
error.NotLowercase,
|
||||
DomainSet.build(testing.allocator, "a.example.com\nB.example.com\n", 0),
|
||||
);
|
||||
}
|
||||
|
||||
test "build rejects a duplicate line" {
|
||||
try testing.expectError(
|
||||
error.NotSorted,
|
||||
DomainSet.build(testing.allocator, "a.example.com\na.example.com\n", 0),
|
||||
);
|
||||
}
|
||||
|
||||
test "build rejects an over-long line" {
|
||||
var body: [300]u8 = undefined;
|
||||
@memset(&body, 'a');
|
||||
body[299] = '\n';
|
||||
try testing.expectError(error.SetTooLarge, DomainSet.build(testing.allocator, &body, 0));
|
||||
}
|
||||
|
||||
test "contains does not depend on the seed" {
|
||||
var a = try DomainSet.build(testing.allocator, small_body, 0);
|
||||
defer a.deinit(testing.allocator);
|
||||
var b = try DomainSet.build(testing.allocator, small_body, 0xdead_beef_cafe_f00d);
|
||||
defer b.deinit(testing.allocator);
|
||||
|
||||
var buf: [64]u8 = undefined;
|
||||
var i: usize = 0;
|
||||
while (i < 50) : (i += 1) {
|
||||
const probe = try std.fmt.bufPrint(&buf, "n{d}.example.com", .{i});
|
||||
try testing.expectEqual(a.contains(probe), b.contains(probe));
|
||||
}
|
||||
for ([_][]const u8{ "ads.example.com", "example.com", "zzz.example.net", "nope.test" }) |probe| {
|
||||
try testing.expectEqual(a.contains(probe), b.contains(probe));
|
||||
}
|
||||
}
|
||||
|
||||
test "ten thousand names round-trip" {
|
||||
const gpa = testing.allocator;
|
||||
var body: std.ArrayList(u8) = .empty;
|
||||
defer body.deinit(gpa);
|
||||
|
||||
// Fixed-width zero padding makes the generated order the sorted order.
|
||||
var line_buf: [64]u8 = undefined;
|
||||
var i: usize = 0;
|
||||
while (i < 10_000) : (i += 1) {
|
||||
try body.appendSlice(gpa, try std.fmt.bufPrint(&line_buf, "d{d:0>5}.example.com\n", .{i}));
|
||||
}
|
||||
|
||||
var set = try DomainSet.build(gpa, body.items, 0x5eed);
|
||||
defer set.deinit(gpa);
|
||||
try testing.expectEqual(@as(u32, 10_000), set.count);
|
||||
|
||||
var buf: [64]u8 = undefined;
|
||||
i = 0;
|
||||
while (i < 10_000) : (i += 1) {
|
||||
const nameStr = try std.fmt.bufPrint(&buf, "d{d:0>5}.example.com", .{i});
|
||||
try testing.expect(set.contains(nameStr));
|
||||
}
|
||||
try testing.expect(!set.contains("d10000.example.com"));
|
||||
try testing.expect(set.memoryBytes() < 2 * body.items.len);
|
||||
}
|
||||
|
||||
fn buildUnderFailure(gpa: std.mem.Allocator) !void {
|
||||
var set = try DomainSet.build(gpa, small_body, 0x1234);
|
||||
defer set.deinit(gpa);
|
||||
try testing.expect(set.contains("example.com"));
|
||||
}
|
||||
|
||||
test "build under allocation failure" {
|
||||
try testing.checkAllAllocationFailures(testing.allocator, buildUnderFailure, .{});
|
||||
}
|
||||
|
||||
test "the empty set owns nothing" {
|
||||
var set: DomainSet = .empty;
|
||||
try testing.expect(!set.contains("x"));
|
||||
try testing.expect(!set.contains(""));
|
||||
try testing.expectEqual(@as(usize, 0), set.memoryBytes());
|
||||
set.deinit(testing.allocator);
|
||||
try testing.expect(!set.contains("x"));
|
||||
}
|
||||
|
||||
test "an empty body builds the empty set" {
|
||||
var set = try DomainSet.build(testing.allocator, "", 7);
|
||||
defer set.deinit(testing.allocator);
|
||||
try testing.expectEqual(@as(u32, 0), set.count);
|
||||
try testing.expect(!set.contains("x"));
|
||||
}
|
||||
|
||||
test "a final line without a trailing newline is kept" {
|
||||
var set = try DomainSet.build(testing.allocator, "a.example.com\nb.example.com", 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
try testing.expectEqual(@as(u32, 2), set.count);
|
||||
try testing.expect(set.contains("b.example.com"));
|
||||
}
|
||||
|
||||
test "capacityFor keeps the load factor at or under three quarters" {
|
||||
try testing.expectEqual(@as(usize, 16), capacityFor(0));
|
||||
try testing.expectEqual(@as(usize, 16), capacityFor(12));
|
||||
try testing.expectEqual(@as(usize, 32), capacityFor(13));
|
||||
try testing.expectEqual(@as(usize, 2 << 20), capacityFor(1_000_000));
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
//! Blocklist download over HTTP/1.1.
|
||||
//!
|
||||
//! One `Fetcher` wraps a caller-owned `std.http.Client`, which owns the
|
||||
//! connection pool and the CA bundle, exactly as `upstream/doh_client.zig`
|
||||
//! does. This file knows nothing about parsing, files or the database: it GETs
|
||||
//! a URL and streams the bytes into a writer the caller supplies.
|
||||
//!
|
||||
//! The body is never held whole. A blocklist can reach `max_body_bytes`, and
|
||||
//! the caller writes into a temporary file anyway, so nothing here allocates.
|
||||
//!
|
||||
//! There is no timeout parameter and no sleep. `std.http.Client` has no
|
||||
//! per-request deadline, so the caller runs `fetch` under `io.concurrent` and
|
||||
//! cancels the future; this file only propagates `error.Canceled`.
|
||||
|
||||
const std = @import("std");
|
||||
const transport = @import("../upstream/transport.zig");
|
||||
|
||||
pub const max_body_bytes: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// RFC 9110 recommends at least 8000 bytes for the redirect buffer
|
||||
/// (`std.http.Client` doc comment, Client.zig:1128).
|
||||
pub const redirect_buffer_len: usize = 8192;
|
||||
|
||||
pub const min_transfer_buf: usize = 16 * 1024;
|
||||
|
||||
pub const Error = error{
|
||||
BadUrl,
|
||||
ConnectFailed,
|
||||
TlsFailed,
|
||||
SendFailed,
|
||||
ReceiveFailed,
|
||||
HttpStatus,
|
||||
BodyTooLarge,
|
||||
Timeout,
|
||||
Canceled,
|
||||
OutOfMemory,
|
||||
SystemResources,
|
||||
Unexpected,
|
||||
};
|
||||
|
||||
pub const Result = struct {
|
||||
bytes_read: u64,
|
||||
status: std.http.Status,
|
||||
};
|
||||
|
||||
pub const Fetcher = struct {
|
||||
/// Caller-owned; shared across sources, pools connections.
|
||||
http: *std.http.Client,
|
||||
/// Caller-owned HTTP body transfer buffer, at least `min_transfer_buf`.
|
||||
transfer_buf: []u8,
|
||||
/// Caller-owned. `receiveHead` follows redirects itself and needs this to
|
||||
/// outlive `Request.uri`. At least `redirect_buffer_len`.
|
||||
redirect_buf: []u8,
|
||||
/// The status of the most recent response head, or null before the first
|
||||
/// one. `error.HttpStatus` carries no `Result`, and the operator's message
|
||||
/// needs the number, so it is readable here after a failed `fetch`.
|
||||
last_status: ?std.http.Status = null,
|
||||
|
||||
/// GETs `url` and streams the body into `w`.
|
||||
pub fn fetch(
|
||||
self: *Fetcher,
|
||||
io: std.Io,
|
||||
url: []const u8,
|
||||
w: *std.Io.Writer,
|
||||
) Error!Result {
|
||||
// `std.http.Client` carries the `std.Io` it was constructed with and
|
||||
// takes none per request. The parameter stays in the signature because
|
||||
// the manager drives every fetch through one `std.Io`.
|
||||
_ = io;
|
||||
|
||||
std.debug.assert(self.transfer_buf.len >= min_transfer_buf);
|
||||
std.debug.assert(self.redirect_buf.len >= redirect_buffer_len);
|
||||
|
||||
const uri = try parseUrl(url);
|
||||
|
||||
self.last_status = null;
|
||||
|
||||
var req = self.http.request(.GET, uri, .{
|
||||
.keep_alive = true,
|
||||
.headers = .{
|
||||
// Identity only: a compressed transfer encoding would need
|
||||
// `Response.readerDecompressing`, a decompression buffer and a
|
||||
// second failure surface, for a download that runs once a day.
|
||||
.accept_encoding = .{ .override = "identity" },
|
||||
},
|
||||
}) catch |err| return mapError(err, .connect);
|
||||
defer req.deinit();
|
||||
|
||||
req.sendBodiless() catch |err| return mapError(err, .send);
|
||||
|
||||
var resp = req.receiveHead(self.redirect_buf) catch |err| return mapError(err, .receive);
|
||||
|
||||
self.last_status = resp.head.status;
|
||||
if (resp.head.status != .ok) return error.HttpStatus;
|
||||
|
||||
// `content-type` is deliberately not checked: blocklists are served as
|
||||
// text/plain, application/octet-stream and text/html alike, and the
|
||||
// compiler's invalid-line counters are the honest signal about content.
|
||||
if (resp.head.content_length) |declared| {
|
||||
if (declared > max_body_bytes) return error.BodyTooLarge;
|
||||
}
|
||||
|
||||
const body = resp.reader(self.transfer_buf);
|
||||
var total: u64 = 0;
|
||||
while (true) {
|
||||
const n = body.readSliceShort(self.transfer_buf) catch |err|
|
||||
return mapError(err, .receive);
|
||||
if (n == 0) break;
|
||||
total += n;
|
||||
if (total > max_body_bytes) return error.BodyTooLarge;
|
||||
// The caller owns `w` and can read the concrete failure from its
|
||||
// own writer; this taxonomy has no member for a failing sink.
|
||||
w.writeAll(self.transfer_buf[0..n]) catch return error.Unexpected;
|
||||
}
|
||||
|
||||
return .{ .bytes_read = total, .status = resp.head.status };
|
||||
}
|
||||
};
|
||||
|
||||
/// A scheme other than `http`/`https`, an unparseable URL and a URL with no
|
||||
/// host are one fault to the operator: the source row is unusable.
|
||||
fn parseUrl(url: []const u8) Error!std.Uri {
|
||||
const uri = std.Uri.parse(url) catch return error.BadUrl;
|
||||
if (!std.mem.eql(u8, uri.scheme, "http") and
|
||||
!std.mem.eql(u8, uri.scheme, "https")) return error.BadUrl;
|
||||
const host = uri.host orelse return error.BadUrl;
|
||||
if (host.isEmpty()) return error.BadUrl;
|
||||
return uri;
|
||||
}
|
||||
|
||||
/// Which call failed. The phase is what decides the classification, and only
|
||||
/// the call site knows it — guessing it from an error name would be wrong the
|
||||
/// first time two phases shared an error.
|
||||
const Phase = enum { connect, send, receive };
|
||||
|
||||
fn mapError(err: anyerror, phase: Phase) Error {
|
||||
if (transport.mapLocal(err)) |local| return narrowLocal(local);
|
||||
switch (err) {
|
||||
error.Timeout => return error.Timeout,
|
||||
// Both are ruled out by `parseUrl` before the client is touched.
|
||||
error.UnsupportedUriScheme, error.UriMissingHost => return error.BadUrl,
|
||||
error.TooManyHttpRedirects => return error.HttpStatus,
|
||||
else => {},
|
||||
}
|
||||
const err_name = @errorName(err);
|
||||
if (std.mem.startsWith(u8, err_name, "Tls") or
|
||||
std.mem.startsWith(u8, err_name, "Certificate")) return error.TlsFailed;
|
||||
return switch (phase) {
|
||||
.connect => error.ConnectFailed,
|
||||
.send => error.SendFailed,
|
||||
.receive => error.ReceiveFailed,
|
||||
};
|
||||
}
|
||||
|
||||
/// `transport.mapLocal` answers in `transport.ExchangeError`, which is wider
|
||||
/// than this file's taxonomy. Both file-descriptor quotas are the same
|
||||
/// exhaustion to a downloader, and `error.BufferTooSmall` cannot occur because
|
||||
/// this file hands the client no undersized buffer.
|
||||
fn narrowLocal(local: transport.ExchangeError) Error {
|
||||
return switch (local) {
|
||||
error.OutOfMemory => error.OutOfMemory,
|
||||
error.SystemResources,
|
||||
error.ProcessFdQuotaExceeded,
|
||||
error.SystemFdQuotaExceeded,
|
||||
=> error.SystemResources,
|
||||
error.Canceled => error.Canceled,
|
||||
else => error.Unexpected,
|
||||
};
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
fn undefinedFetcher(transfer_buf: []u8, redirect_buf: []u8) Fetcher {
|
||||
// `http` is never driven: every test below asserts a rejection that
|
||||
// happens before the first client call.
|
||||
const http: *std.http.Client = undefined;
|
||||
return .{ .http = http, .transfer_buf = transfer_buf, .redirect_buf = redirect_buf };
|
||||
}
|
||||
|
||||
test "fetch rejects a non-http scheme before any client use" {
|
||||
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||
var sink_buf: [0]u8 = .{};
|
||||
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||
try testing.expectError(
|
||||
error.BadUrl,
|
||||
f.fetch(undefined, "ftp://example.com/list.txt", &discarding.writer),
|
||||
);
|
||||
}
|
||||
|
||||
test "fetch rejects a url with no scheme before any client use" {
|
||||
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||
var sink_buf: [0]u8 = .{};
|
||||
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||
try testing.expectError(error.BadUrl, f.fetch(undefined, "x", &discarding.writer));
|
||||
}
|
||||
|
||||
test "fetch rejects a url with no host before any client use" {
|
||||
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||
var sink_buf: [0]u8 = .{};
|
||||
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||
try testing.expectError(error.BadUrl, f.fetch(undefined, "https://", &discarding.writer));
|
||||
}
|
||||
|
||||
test "parseUrl accepts http and https urls" {
|
||||
const plain = try parseUrl("http://example.com/hosts.txt");
|
||||
try testing.expectEqualStrings("http", plain.scheme);
|
||||
const secure = try parseUrl("https://example.com:8443/hosts.txt");
|
||||
try testing.expectEqualStrings("https", secure.scheme);
|
||||
try testing.expectEqual(@as(?u16, 8443), secure.port);
|
||||
}
|
||||
|
||||
test "parseUrl rejects the url forms the source table can hold" {
|
||||
try testing.expectError(error.BadUrl, parseUrl("ftp://example.com/list"));
|
||||
try testing.expectError(error.BadUrl, parseUrl("file:///etc/hosts"));
|
||||
try testing.expectError(error.BadUrl, parseUrl("x"));
|
||||
try testing.expectError(error.BadUrl, parseUrl(""));
|
||||
try testing.expectError(error.BadUrl, parseUrl("https://"));
|
||||
}
|
||||
|
||||
test "mapError maps local errors before phase errors" {
|
||||
try testing.expectEqual(error.OutOfMemory, mapError(error.OutOfMemory, .connect));
|
||||
try testing.expectEqual(error.Canceled, mapError(error.Canceled, .receive));
|
||||
try testing.expectEqual(error.Unexpected, mapError(error.Unexpected, .send));
|
||||
try testing.expectEqual(error.SystemResources, mapError(error.SystemResources, .connect));
|
||||
try testing.expectEqual(
|
||||
error.SystemResources,
|
||||
mapError(error.ProcessFdQuotaExceeded, .connect),
|
||||
);
|
||||
try testing.expectEqual(
|
||||
error.SystemResources,
|
||||
mapError(error.SystemFdQuotaExceeded, .connect),
|
||||
);
|
||||
}
|
||||
|
||||
test "mapError maps tls errors regardless of phase" {
|
||||
try testing.expectEqual(error.TlsFailed, mapError(error.TlsInitializationFailed, .connect));
|
||||
try testing.expectEqual(error.TlsFailed, mapError(error.TlsAlert, .receive));
|
||||
try testing.expectEqual(error.TlsFailed, mapError(error.CertificateExpired, .connect));
|
||||
}
|
||||
|
||||
test "mapError maps a redirect overrun to HttpStatus" {
|
||||
try testing.expectEqual(error.HttpStatus, mapError(error.TooManyHttpRedirects, .receive));
|
||||
}
|
||||
|
||||
test "mapError maps a connect timeout to Timeout" {
|
||||
try testing.expectEqual(error.Timeout, mapError(error.Timeout, .connect));
|
||||
}
|
||||
|
||||
test "mapError maps remaining errors by phase" {
|
||||
try testing.expectEqual(error.ConnectFailed, mapError(error.ConnectionRefused, .connect));
|
||||
try testing.expectEqual(error.SendFailed, mapError(error.WriteFailed, .send));
|
||||
try testing.expectEqual(error.ReceiveFailed, mapError(error.ReadFailed, .receive));
|
||||
try testing.expectEqual(error.ReceiveFailed, mapError(error.HttpHeadersInvalid, .receive));
|
||||
}
|
||||
|
||||
test "caps are the values the memory budget was sized against" {
|
||||
try testing.expectEqual(@as(usize, 64 * 1024 * 1024), max_body_bytes);
|
||||
try testing.expectEqual(@as(usize, 8192), redirect_buffer_len);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,102 @@
|
||||
//! The Adblock Plus filter syntax, restricted to what a DNS sinkhole can
|
||||
//! honour: domain anchors and bare names. Pure, `std` only.
|
||||
//!
|
||||
//! Exception rules (`@@`) are `.unsupported` rather than an allow entry. The
|
||||
//! allow surface is the `rules` table, and a downloaded list that could quietly
|
||||
//! allow a domain across every group is a policy hole the operator did not open.
|
||||
|
||||
const std = @import("std");
|
||||
const parsers = @import("parsers.zig");
|
||||
|
||||
/// Tokens that a candidate name may never contain. `^` is a separator token in
|
||||
/// this syntax and only a trailing one is meaningful for a domain rule.
|
||||
const rule_tokens = "*^|/$";
|
||||
|
||||
pub fn parseLine(line: []const u8) parsers.Line {
|
||||
const text = std.mem.trim(u8, line, &std.ascii.whitespace);
|
||||
if (text.len == 0) return .{ .kind = .ignore };
|
||||
if (text[0] == '!') return .{ .kind = .ignore };
|
||||
if (text[0] == '[') return .{ .kind = .ignore };
|
||||
if (parsers.isElementHiding(text)) return .{ .kind = .unsupported };
|
||||
if (text[0] == '#') return .{ .kind = .ignore };
|
||||
if (std.mem.startsWith(u8, text, "@@")) return .{ .kind = .unsupported };
|
||||
if (text[0] == '/') return .{ .kind = .regex };
|
||||
if (std.mem.findScalar(u8, text, '$') != null) return .{ .kind = .unsupported };
|
||||
|
||||
if (std.mem.startsWith(u8, text, "||")) {
|
||||
var candidate = text[2..];
|
||||
if (std.mem.endsWith(u8, candidate, "^")) candidate = candidate[0 .. candidate.len - 1];
|
||||
if (candidate.len == 0) return .{ .kind = .unsupported };
|
||||
if (std.mem.findAny(u8, candidate, rule_tokens) != null) return .{ .kind = .unsupported };
|
||||
// A domain anchor covers the domain itself as well as its subdomains,
|
||||
// so the compiler emits an apex entry beside the wildcard one.
|
||||
return .{ .kind = .wildcard, .text = candidate, .covers_apex = true };
|
||||
}
|
||||
|
||||
if (std.mem.findAny(u8, text, rule_tokens) != null) return .{ .kind = .unsupported };
|
||||
return .{ .kind = .domain, .text = text };
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "a bang comment is ignored" {
|
||||
try testing.expectEqual(parsers.Kind.ignore, parseLine("! comment").kind);
|
||||
}
|
||||
|
||||
test "a list header is ignored" {
|
||||
try testing.expectEqual(parsers.Kind.ignore, parseLine("[Adblock Plus 2.0]").kind);
|
||||
}
|
||||
|
||||
test "an empty line is ignored" {
|
||||
try testing.expectEqual(parsers.Kind.ignore, parseLine("").kind);
|
||||
}
|
||||
|
||||
test "a domain anchor is a wildcard that covers its apex" {
|
||||
const line = parseLine("||example.com^");
|
||||
try testing.expectEqual(parsers.Kind.wildcard, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
try testing.expect(line.covers_apex);
|
||||
}
|
||||
|
||||
test "a domain anchor without a separator is still a wildcard" {
|
||||
const line = parseLine("||example.com");
|
||||
try testing.expectEqual(parsers.Kind.wildcard, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
try testing.expect(line.covers_apex);
|
||||
}
|
||||
|
||||
test "a modifier list is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("||example.com^$third-party").kind);
|
||||
}
|
||||
|
||||
test "an exception rule is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("@@||example.com^").kind);
|
||||
}
|
||||
|
||||
test "element hiding is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("##.ad-banner").kind);
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com#@#.ad").kind);
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com#?#.ad").kind);
|
||||
}
|
||||
|
||||
test "a scheme anchor is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("|http://example.com").kind);
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("|https://example.com").kind);
|
||||
}
|
||||
|
||||
test "a regex rule is recognized" {
|
||||
try testing.expectEqual(parsers.Kind.regex, parseLine("/ads[0-9]+/").kind);
|
||||
}
|
||||
|
||||
test "a bare name is a domain" {
|
||||
const line = parseLine("example.com");
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
}
|
||||
|
||||
test "a rule token outside the supported forms is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("ads*.example.com").kind);
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com^").kind);
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com|").kind);
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("||example.com/path^").kind);
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
//! The plain domain-list format: one candidate name per line. Pure, `std` only.
|
||||
|
||||
const std = @import("std");
|
||||
const parsers = @import("parsers.zig");
|
||||
|
||||
pub fn parseLine(line: []const u8) parsers.Line {
|
||||
// Both comment markers appear in domain lists in the wild.
|
||||
const marker = std.mem.findAny(u8, line, "#!") orelse line.len;
|
||||
const text = std.mem.trim(u8, line[0..marker], &std.ascii.whitespace);
|
||||
if (text.len == 0) return .{ .kind = .ignore };
|
||||
if (text[0] == '/') return .{ .kind = .regex };
|
||||
// A second field means the list is a mis-detected hosts file. Counting that
|
||||
// is honest; guessing which field is the name is not.
|
||||
if (std.mem.findAny(u8, text, &std.ascii.whitespace) != null) return .{ .kind = .unsupported };
|
||||
return .{ .kind = .domain, .text = text };
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "a bare name" {
|
||||
const line = parseLine("ads.example.com");
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings("ads.example.com", line.text);
|
||||
}
|
||||
|
||||
test "a bang comment line is ignored" {
|
||||
try testing.expectEqual(parsers.Kind.ignore, parseLine("! comment").kind);
|
||||
}
|
||||
|
||||
test "a hash comment line is ignored" {
|
||||
try testing.expectEqual(parsers.Kind.ignore, parseLine("# comment").kind);
|
||||
}
|
||||
|
||||
test "an inline comment is removed" {
|
||||
const line = parseLine("example.com # x");
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
}
|
||||
|
||||
test "a regex line is recognized" {
|
||||
try testing.expectEqual(parsers.Kind.regex, parseLine("/re/").kind);
|
||||
}
|
||||
|
||||
test "a hosts line in a domains list is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("0.0.0.0 example.com").kind);
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
//! The `hosts(5)` blocklist format: an optional sink address followed by one or
|
||||
//! more names. Pure, `std` only.
|
||||
//!
|
||||
//! The sink address is not checked against a list of "blocking" addresses: a
|
||||
//! list that maps to `127.0.0.1`, `0.0.0.0` or `::` carries the same
|
||||
//! instruction, and a list that maps to a real address is still a set of names
|
||||
//! the operator asked to block.
|
||||
|
||||
const std = @import("std");
|
||||
const parsers = @import("parsers.zig");
|
||||
|
||||
pub fn parseLine(line: []const u8) parsers.Line {
|
||||
const uncommented = if (std.mem.findScalar(u8, line, '#')) |at| line[0..at] else line;
|
||||
const text = std.mem.trim(u8, uncommented, &std.ascii.whitespace);
|
||||
if (text.len == 0) return .{ .kind = .ignore };
|
||||
if (text[0] == '/') return .{ .kind = .regex };
|
||||
|
||||
const split = std.mem.findAny(u8, text, &std.ascii.whitespace) orelse {
|
||||
// A bare address with no name is a hosts line that names nothing to
|
||||
// block; it is counted rather than compiled into an entry.
|
||||
if (parsers.looksLikeIpLiteral(text)) return .{ .kind = .unsupported };
|
||||
return .{ .kind = .domain, .text = text };
|
||||
};
|
||||
|
||||
// Lists that are bare name lists with a hosts extension are common, so a
|
||||
// first field that is not an address is a name like any other.
|
||||
if (!parsers.looksLikeIpLiteral(text[0..split])) return .{ .kind = .domain, .text = text };
|
||||
|
||||
const names = std.mem.trimStart(u8, text[split..], &std.ascii.whitespace);
|
||||
return .{ .kind = .domain, .text = names };
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "a sink address and one name" {
|
||||
const line = parseLine("0.0.0.0 ads.example.com");
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings("ads.example.com", line.text);
|
||||
}
|
||||
|
||||
test "a sink address and several names" {
|
||||
const line = parseLine("127.0.0.1 a.example.com b.example.com");
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings("a.example.com b.example.com", line.text);
|
||||
}
|
||||
|
||||
test "single-label names survive the parser" {
|
||||
const line = parseLine("::1 ip6-localhost ip6-loopback");
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings("ip6-localhost ip6-loopback", line.text);
|
||||
}
|
||||
|
||||
test "a trailing comment is removed" {
|
||||
const line = parseLine("0.0.0.0 ads.example.com # tracker");
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings("ads.example.com", line.text);
|
||||
}
|
||||
|
||||
test "a comment line is ignored" {
|
||||
try testing.expectEqual(parsers.Kind.ignore, parseLine("# whole line").kind);
|
||||
}
|
||||
|
||||
test "an empty line is ignored" {
|
||||
try testing.expectEqual(parsers.Kind.ignore, parseLine("").kind);
|
||||
}
|
||||
|
||||
test "a blank line is ignored" {
|
||||
try testing.expectEqual(parsers.Kind.ignore, parseLine(" ").kind);
|
||||
}
|
||||
|
||||
test "a regex line is recognized" {
|
||||
try testing.expectEqual(parsers.Kind.regex, parseLine("/ads\\d+/").kind);
|
||||
}
|
||||
|
||||
test "a bare name without a sink address" {
|
||||
const line = parseLine("example.com");
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
}
|
||||
|
||||
test "a bare sink address names nothing" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("0.0.0.0").kind);
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
//! Blocklist line parsers: the shared vocabulary and the format sniffer.
|
||||
//!
|
||||
//! These files decide **format**, not validity. Whether a candidate is a usable
|
||||
//! domain name is the compiler's decision, taken through `dns.name.fromText`.
|
||||
//! `std` is the only import here and in every sibling parser: this file is the
|
||||
//! root of a separate fuzz module, and a module root cannot import across its
|
||||
//! own directory boundary.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
pub const hosts = @import("parser_hosts.zig");
|
||||
pub const domains = @import("parser_domains.zig");
|
||||
pub const abp = @import("parser_abp.zig");
|
||||
pub const wildcard = @import("wildcard.zig");
|
||||
|
||||
pub const Format = enum { hosts, domains, abp };
|
||||
|
||||
pub const Kind = enum {
|
||||
/// Nothing on the line, or only a comment.
|
||||
ignore,
|
||||
/// `text` holds one or more whitespace-separated candidate names.
|
||||
domain,
|
||||
/// `text` holds one candidate suffix; every proper subdomain of it matches.
|
||||
wildcard,
|
||||
/// A regex rule. Counted, skipped, never compiled (PLAN §2.2).
|
||||
regex,
|
||||
/// Syntactically a rule of this format, but one nxdns cannot honour:
|
||||
/// an ABP modifier list, an exception rule, element hiding, a scheme anchor.
|
||||
unsupported,
|
||||
};
|
||||
|
||||
pub const Line = struct {
|
||||
kind: Kind,
|
||||
/// Borrowed from the caller's line. Not lowercased, not validated.
|
||||
text: []const u8 = "",
|
||||
/// `.wildcard` only. ABP `||x^` covers `x` itself as well as its subdomains,
|
||||
/// so the compiler emits an additional `.list` entry when this is set.
|
||||
covers_apex: bool = false,
|
||||
};
|
||||
|
||||
/// Dispatches to the format's parser. The line must not contain '\n' or '\r';
|
||||
/// the caller strips them.
|
||||
pub fn parseLine(format: Format, line: []const u8) Line {
|
||||
return switch (format) {
|
||||
.hosts => hosts.parseLine(line),
|
||||
.domains => domains.parseLine(line),
|
||||
.abp => abp.parseLine(line),
|
||||
};
|
||||
}
|
||||
|
||||
pub const sample_lines = 64;
|
||||
|
||||
/// Picks a format from the first `sample_lines` lines that are not blank and
|
||||
/// not comments: an ABP marker (`||`, `@@`, `##`, `$`) wins `.abp`; otherwise a
|
||||
/// majority of lines whose first field looks like an IP literal wins `.hosts`;
|
||||
/// otherwise `.domains`.
|
||||
pub fn detectFormat(sample: []const u8) Format {
|
||||
var considered: usize = 0;
|
||||
var ip_first: usize = 0;
|
||||
|
||||
var it = std.mem.splitScalar(u8, sample, '\n');
|
||||
while (it.next()) |raw| {
|
||||
if (considered == sample_lines) break;
|
||||
const line = std.mem.trim(u8, raw, &std.ascii.whitespace);
|
||||
if (line.len == 0) continue;
|
||||
if (hasAbpMarker(line)) return .abp;
|
||||
if (isComment(line)) continue;
|
||||
considered += 1;
|
||||
if (looksLikeIpLiteral(firstField(line))) ip_first += 1;
|
||||
}
|
||||
|
||||
if (ip_first * 2 > considered) return .hosts;
|
||||
return .domains;
|
||||
}
|
||||
|
||||
/// `!` is the ABP comment marker and `#` the hosts one; both appear in every
|
||||
/// format in the wild. `##`, `#@#` and `#?#` are element-hiding rules, not
|
||||
/// comments, so they stay visible to `hasAbpMarker`.
|
||||
pub fn isComment(line: []const u8) bool {
|
||||
if (line.len == 0) return false;
|
||||
if (line[0] == '!') return true;
|
||||
if (line[0] != '#') return false;
|
||||
return !isElementHiding(line);
|
||||
}
|
||||
|
||||
/// The element-hiding separators, which may also follow a domain list
|
||||
/// (`example.com##.ad-banner`).
|
||||
pub fn isElementHiding(line: []const u8) bool {
|
||||
for ([_][]const u8{ "##", "#@#", "#?#", "#$#", "#%#" }) |marker| {
|
||||
if (std.mem.find(u8, line, marker) != null) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
fn hasAbpMarker(line: []const u8) bool {
|
||||
if (std.mem.startsWith(u8, line, "||")) return true;
|
||||
if (std.mem.startsWith(u8, line, "@@")) return true;
|
||||
if (isElementHiding(line)) return true;
|
||||
// A '$' modifier list only counts on a rule line: a hosts file whose
|
||||
// comments mention a price must not be sniffed as ABP.
|
||||
if (!isComment(line) and std.mem.findScalar(u8, line, '$') != null) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// The line up to the first ASCII whitespace byte.
|
||||
pub fn firstField(line: []const u8) []const u8 {
|
||||
const end = std.mem.findAny(u8, line, &std.ascii.whitespace) orelse line.len;
|
||||
return line[0..end];
|
||||
}
|
||||
|
||||
/// A sniffing heuristic, not a parser: it recognizes dotted-quad IPv4 and any
|
||||
/// hex-and-colon IPv6 spelling. `platform/address.zig` holds the real parser and
|
||||
/// importing it would break this file's std-only constraint.
|
||||
pub fn looksLikeIpLiteral(field: []const u8) bool {
|
||||
if (field.len == 0) return false;
|
||||
|
||||
if (std.mem.findScalar(u8, field, ':') != null) {
|
||||
for (field) |c| {
|
||||
if (c != ':' and c != '.' and !std.ascii.isHex(c)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
var parts: usize = 0;
|
||||
var it = std.mem.splitScalar(u8, field, '.');
|
||||
while (it.next()) |part| {
|
||||
parts += 1;
|
||||
if (part.len == 0 or part.len > 3) return false;
|
||||
for (part) |c| {
|
||||
if (!std.ascii.isDigit(c)) return false;
|
||||
}
|
||||
}
|
||||
return parts == 4;
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "detectFormat recognizes a hosts file" {
|
||||
const sample =
|
||||
\\# Title: example
|
||||
\\0.0.0.0 ads.example.com
|
||||
\\0.0.0.0 track.example.net
|
||||
\\127.0.0.1 metrics.example.org
|
||||
\\
|
||||
;
|
||||
try testing.expectEqual(Format.hosts, detectFormat(sample));
|
||||
}
|
||||
|
||||
test "detectFormat recognizes a domains file" {
|
||||
const sample =
|
||||
\\# Title: example
|
||||
\\ads.example.com
|
||||
\\track.example.net
|
||||
\\metrics.example.org
|
||||
\\
|
||||
;
|
||||
try testing.expectEqual(Format.domains, detectFormat(sample));
|
||||
}
|
||||
|
||||
test "detectFormat recognizes an abp file" {
|
||||
const sample =
|
||||
\\[Adblock Plus 2.0]
|
||||
\\! Title: example
|
||||
\\||ads.example.com^
|
||||
\\||track.example.net^
|
||||
\\
|
||||
;
|
||||
try testing.expectEqual(Format.abp, detectFormat(sample));
|
||||
}
|
||||
|
||||
test "detectFormat falls back to domains on an all-comment sample" {
|
||||
var buffer: [64 * 16]u8 = undefined;
|
||||
var w: usize = 0;
|
||||
for (0..64) |_| {
|
||||
@memcpy(buffer[w..][0..14], "# a comment.\n\n");
|
||||
w += 14;
|
||||
}
|
||||
try testing.expectEqual(Format.domains, detectFormat(buffer[0..w]));
|
||||
}
|
||||
|
||||
test "detectFormat is not fooled by a dollar sign in a comment" {
|
||||
const sample =
|
||||
\\# donations welcome, $5 covers a month
|
||||
\\0.0.0.0 ads.example.com
|
||||
\\0.0.0.0 track.example.net
|
||||
\\
|
||||
;
|
||||
try testing.expectEqual(Format.hosts, detectFormat(sample));
|
||||
}
|
||||
|
||||
test "parseLine dispatches to the hosts parser" {
|
||||
const line = parseLine(.hosts, "0.0.0.0 ads.example.com");
|
||||
try testing.expectEqual(Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings("ads.example.com", line.text);
|
||||
}
|
||||
|
||||
test "parseLine dispatches to the domains parser" {
|
||||
const line = parseLine(.domains, "0.0.0.0 ads.example.com");
|
||||
try testing.expectEqual(Kind.unsupported, line.kind);
|
||||
}
|
||||
|
||||
test "parseLine dispatches to the abp parser" {
|
||||
const line = parseLine(.abp, "||ads.example.com^");
|
||||
try testing.expectEqual(Kind.wildcard, line.kind);
|
||||
try testing.expectEqualStrings("ads.example.com", line.text);
|
||||
try testing.expect(line.covers_apex);
|
||||
}
|
||||
|
||||
test "looksLikeIpLiteral separates addresses from names" {
|
||||
try testing.expect(looksLikeIpLiteral("0.0.0.0"));
|
||||
try testing.expect(looksLikeIpLiteral("127.0.0.1"));
|
||||
try testing.expect(looksLikeIpLiteral("::1"));
|
||||
try testing.expect(looksLikeIpLiteral("fd00::dead:beef"));
|
||||
try testing.expect(!looksLikeIpLiteral("example.com"));
|
||||
try testing.expect(!looksLikeIpLiteral("add.face.cafe"));
|
||||
try testing.expect(!looksLikeIpLiteral("1.2.3"));
|
||||
try testing.expect(!looksLikeIpLiteral(""));
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
//! Blocked-response synthesis (PLAN §6.2). Pure: no allocation, no `std.Io`,
|
||||
//! no clock. The caller supplies the buffer and gets back a prefix of it.
|
||||
//!
|
||||
//! No SOA is placed in the authority section. nxdns is not authoritative for a
|
||||
//! blocked name, and a synthesized SOA would hand resolvers a negative-caching
|
||||
//! TTL nxdns cannot honour: the operator can unblock the name at any moment,
|
||||
//! and a client that cached the negative answer for the SOA's MINIMUM would
|
||||
//! keep failing long after the block was lifted.
|
||||
|
||||
const std = @import("std");
|
||||
const types = @import("../dns/types.zig");
|
||||
const header = @import("../dns/header.zig");
|
||||
const question = @import("../dns/question.zig");
|
||||
const edns = @import("../dns/edns.zig");
|
||||
const packet = @import("../dns/packet.zig");
|
||||
const model = @import("../config/model.zig");
|
||||
|
||||
pub const Options = struct {
|
||||
mode: model.BlockResponse,
|
||||
ttl: u32,
|
||||
};
|
||||
|
||||
pub const Error = packet.ResponseBuilder.Error;
|
||||
|
||||
const zero_a = [_]u8{0} ** 4;
|
||||
const zero_aaaa = [_]u8{0} ** 16;
|
||||
|
||||
/// Writes a blocked reply for `q` into `buf` and returns a prefix of it.
|
||||
///
|
||||
/// `.zero`: A → 0.0.0.0, AAAA → ::, every other qtype → NOERROR with no answer
|
||||
/// (NODATA). No address exists to synthesize for a qtype that carries none,
|
||||
/// and answering NXDOMAIN for, say, an MX query would tell the client the
|
||||
/// name does not exist while an A query for the same name says it does.
|
||||
/// `.nxdomain`: RCODE = NXDOMAIN, no answer, for every qtype.
|
||||
///
|
||||
/// Only class `IN` is answered with addresses; any other class takes the
|
||||
/// NODATA path, because `0.0.0.0` is an IN-class address and means nothing in
|
||||
/// CH or HS.
|
||||
///
|
||||
/// `request_opt` echoes EDNS exactly as `handler.zig` does: a query that
|
||||
/// carried an OPT record gets a reply carrying one with the same payload size
|
||||
/// and the DO bit passed through.
|
||||
pub fn writeBlocked(
|
||||
buf: []u8,
|
||||
request: header.Header,
|
||||
q: question.Question,
|
||||
request_opt: ?edns.OptRecord,
|
||||
do_bit: bool,
|
||||
options: Options,
|
||||
) Error![]u8 {
|
||||
var b = try packet.ResponseBuilder.init(buf, request, q);
|
||||
|
||||
switch (options.mode) {
|
||||
.nxdomain => b.setRcode(.nx_domain),
|
||||
.zero => if (q.qclass == .in) switch (q.qtype) {
|
||||
.a => try b.addAnswer(q.name, .a, .in, options.ttl, &zero_a),
|
||||
.aaaa => try b.addAnswer(q.name, .aaaa, .in, options.ttl, &zero_aaaa),
|
||||
else => {},
|
||||
},
|
||||
}
|
||||
|
||||
if (request_opt) |opt| try b.addOptEcho(opt, do_bit);
|
||||
return b.finish();
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
const name = @import("../dns/name.zig");
|
||||
const record = @import("../dns/record.zig");
|
||||
|
||||
/// A query for example.com A with an EDNS(0) OPT record advertising 4096
|
||||
/// bytes: id 0x1234, RD set, one question, one additional.
|
||||
const query_bytes =
|
||||
"\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x01" ++
|
||||
"\x07example\x03com\x00\x00\x01\x00\x01" ++
|
||||
"\x00\x00\x29\x10\x00\x00\x00\x00\x00\x00\x00";
|
||||
|
||||
const blocked_name = "ads.example.com";
|
||||
const ttl: u32 = 5;
|
||||
|
||||
fn requestHeader() header.Header {
|
||||
return (packet.parse(query_bytes) catch unreachable).header;
|
||||
}
|
||||
|
||||
fn requestOpt() edns.OptRecord {
|
||||
const p = packet.parse(query_bytes) catch unreachable;
|
||||
return edns.parseOpt(query_bytes, packet.findOptRecord(p).?) catch unreachable;
|
||||
}
|
||||
|
||||
fn blockedQuestion(qtype: types.Type, qclass: types.Class) !question.Question {
|
||||
return .{ .name = try name.fromText(blocked_name), .qtype = qtype, .qclass = qclass };
|
||||
}
|
||||
|
||||
/// Builds a blocked reply and re-parses it, asserting the parts every case
|
||||
/// shares: the echoed id, the QR and RA flags, the echoed question, an empty
|
||||
/// authority section, and an additional section that holds the OPT record only
|
||||
/// when the query carried one.
|
||||
fn expectBlocked(
|
||||
buf: []u8,
|
||||
mode: model.BlockResponse,
|
||||
qtype: types.Type,
|
||||
qclass: types.Class,
|
||||
with_opt: bool,
|
||||
) !packet.Packet {
|
||||
const q = try blockedQuestion(qtype, qclass);
|
||||
const bytes = try writeBlocked(
|
||||
buf,
|
||||
requestHeader(),
|
||||
q,
|
||||
if (with_opt) requestOpt() else null,
|
||||
false,
|
||||
.{ .mode = mode, .ttl = ttl },
|
||||
);
|
||||
|
||||
const p = try packet.parse(bytes);
|
||||
try testing.expectEqual(@as(u16, 0x1234), p.header.id);
|
||||
try testing.expect(p.header.flags.qr);
|
||||
try testing.expect(p.header.flags.ra);
|
||||
try testing.expect(p.header.flags.rd);
|
||||
try testing.expectEqual(@as(u16, 1), p.header.qdcount);
|
||||
try testing.expectEqual(@as(u16, 0), p.header.nscount);
|
||||
try testing.expectEqual(@as(u16, if (with_opt) 1 else 0), p.header.arcount);
|
||||
|
||||
const echoed = packet.firstQuestion(p).?;
|
||||
try testing.expectEqualSlices(u8, q.name.wire(), echoed.name.wire());
|
||||
try testing.expectEqual(qtype, echoed.qtype);
|
||||
try testing.expectEqual(qclass, echoed.qclass);
|
||||
|
||||
if (with_opt) {
|
||||
const opt = try edns.parseOpt(bytes, packet.findOptRecord(p).?);
|
||||
try testing.expectEqual(@as(u16, 4096), opt.udp_payload_size);
|
||||
try testing.expectEqual(false, opt.do_bit);
|
||||
} else {
|
||||
try testing.expect(packet.findOptRecord(p) == null);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
fn expectNodata(mode: model.BlockResponse, qtype: types.Type, qclass: types.Class) !void {
|
||||
for ([_]bool{ false, true }) |with_opt| {
|
||||
var buf: [512]u8 = undefined;
|
||||
const p = try expectBlocked(&buf, mode, qtype, qclass, with_opt);
|
||||
try testing.expectEqual(types.Rcode.no_error, p.header.flags.rcode);
|
||||
try testing.expectEqual(@as(u16, 0), p.header.ancount);
|
||||
}
|
||||
}
|
||||
|
||||
fn expectNxdomain(qtype: types.Type) !void {
|
||||
for ([_]bool{ false, true }) |with_opt| {
|
||||
var buf: [512]u8 = undefined;
|
||||
const p = try expectBlocked(&buf, .nxdomain, qtype, .in, with_opt);
|
||||
try testing.expectEqual(types.Rcode.nx_domain, p.header.flags.rcode);
|
||||
try testing.expectEqual(@as(u16, 0), p.header.ancount);
|
||||
}
|
||||
}
|
||||
|
||||
test "zero mode answers A with 0.0.0.0" {
|
||||
for ([_]bool{ false, true }) |with_opt| {
|
||||
var buf: [512]u8 = undefined;
|
||||
const p = try expectBlocked(&buf, .zero, .a, .in, with_opt);
|
||||
try testing.expectEqual(types.Rcode.no_error, p.header.flags.rcode);
|
||||
try testing.expectEqual(@as(u16, 1), p.header.ancount);
|
||||
|
||||
var it = packet.answers(p);
|
||||
const answer = (try it.next()).?;
|
||||
try testing.expectEqual(types.Type.a, answer.rtype);
|
||||
try testing.expectEqual(@as(u16, @intFromEnum(types.Class.in)), answer.class);
|
||||
try testing.expectEqual(ttl, answer.ttl);
|
||||
try testing.expectEqualSlices(
|
||||
u8,
|
||||
(try name.fromText(blocked_name)).wire(),
|
||||
answer.name.wire(),
|
||||
);
|
||||
try testing.expectEqual([4]u8{ 0, 0, 0, 0 }, try record.rdataA(p.bytes, answer));
|
||||
try testing.expect((try it.next()) == null);
|
||||
}
|
||||
}
|
||||
|
||||
test "zero mode answers AAAA with ::" {
|
||||
for ([_]bool{ false, true }) |with_opt| {
|
||||
var buf: [512]u8 = undefined;
|
||||
const p = try expectBlocked(&buf, .zero, .aaaa, .in, with_opt);
|
||||
try testing.expectEqual(types.Rcode.no_error, p.header.flags.rcode);
|
||||
try testing.expectEqual(@as(u16, 1), p.header.ancount);
|
||||
|
||||
var it = packet.answers(p);
|
||||
const answer = (try it.next()).?;
|
||||
try testing.expectEqual(types.Type.aaaa, answer.rtype);
|
||||
try testing.expectEqual(ttl, answer.ttl);
|
||||
try testing.expectEqual(zero_aaaa, try record.rdataAaaa(p.bytes, answer));
|
||||
try testing.expect((try it.next()) == null);
|
||||
}
|
||||
}
|
||||
|
||||
test "zero mode answers MX with NODATA" {
|
||||
try expectNodata(.zero, .mx, .in);
|
||||
}
|
||||
|
||||
test "zero mode answers HTTPS with NODATA" {
|
||||
try expectNodata(.zero, .https, .in);
|
||||
}
|
||||
|
||||
test "zero mode answers a non-IN class with NODATA" {
|
||||
try expectNodata(.zero, .a, .ch);
|
||||
try expectNodata(.zero, .aaaa, .any);
|
||||
}
|
||||
|
||||
test "nxdomain mode answers A with NXDOMAIN" {
|
||||
try expectNxdomain(.a);
|
||||
}
|
||||
|
||||
test "nxdomain mode answers AAAA with NXDOMAIN" {
|
||||
try expectNxdomain(.aaaa);
|
||||
}
|
||||
|
||||
test "nxdomain mode answers MX with NXDOMAIN" {
|
||||
try expectNxdomain(.mx);
|
||||
}
|
||||
|
||||
test "nxdomain mode answers HTTPS with NXDOMAIN" {
|
||||
try expectNxdomain(.https);
|
||||
}
|
||||
|
||||
test "the DO bit passes through" {
|
||||
for ([_]bool{ false, true }) |do_bit| {
|
||||
var buf: [512]u8 = undefined;
|
||||
const bytes = try writeBlocked(
|
||||
&buf,
|
||||
requestHeader(),
|
||||
try blockedQuestion(.a, .in),
|
||||
requestOpt(),
|
||||
do_bit,
|
||||
.{ .mode = .zero, .ttl = ttl },
|
||||
);
|
||||
const p = try packet.parse(bytes);
|
||||
const opt = try edns.parseOpt(bytes, packet.findOptRecord(p).?);
|
||||
try testing.expectEqual(do_bit, opt.do_bit);
|
||||
}
|
||||
}
|
||||
|
||||
test "a ttl of zero survives the round trip" {
|
||||
var buf: [512]u8 = undefined;
|
||||
const bytes = try writeBlocked(
|
||||
&buf,
|
||||
requestHeader(),
|
||||
try blockedQuestion(.a, .in),
|
||||
null,
|
||||
false,
|
||||
.{ .mode = .zero, .ttl = 0 },
|
||||
);
|
||||
const p = try packet.parse(bytes);
|
||||
var it = packet.answers(p);
|
||||
try testing.expectEqual(@as(u32, 0), (try it.next()).?.ttl);
|
||||
}
|
||||
|
||||
test "a buffer too small reports a write failure instead of truncating" {
|
||||
const q = try blockedQuestion(.a, .in);
|
||||
const options: Options = .{ .mode = .zero, .ttl = ttl };
|
||||
|
||||
// Room for the header and the question, but not for the answer record.
|
||||
var no_room_for_answer: [40]u8 = undefined;
|
||||
try testing.expectError(
|
||||
error.WriteFailed,
|
||||
writeBlocked(&no_room_for_answer, requestHeader(), q, null, false, options),
|
||||
);
|
||||
|
||||
// Room for the header and the question and the answer, but not the OPT.
|
||||
var no_room_for_opt: [72]u8 = undefined;
|
||||
try testing.expectError(
|
||||
error.WriteFailed,
|
||||
writeBlocked(&no_room_for_opt, requestHeader(), q, requestOpt(), false, options),
|
||||
);
|
||||
|
||||
// Not even room for the header.
|
||||
var tiny: [8]u8 = undefined;
|
||||
try testing.expectError(
|
||||
error.WriteFailed,
|
||||
writeBlocked(&tiny, requestHeader(), q, null, false, options),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
//! One group's explicit rules (PLAN §3.10 levels 1–4), compiled once into an
|
||||
//! immutable form the query path can read without allocating.
|
||||
//!
|
||||
//! Exact patterns go into a `DomainSet`; wildcard patterns stay a flat, sorted
|
||||
//! array of strings and are scanned linearly. Operator-authored wildcards are
|
||||
//! few — `max_wildcards_per_group` caps them at 4096 — and a linear scan over
|
||||
//! that many short patterns is cheaper than an index that would have to be
|
||||
//! rebuilt on every snapshot swap.
|
||||
//!
|
||||
//! Pure: an allocator and plain values, no `std.Io`, no clock, no entropy
|
||||
//! source. The hash seed arrives as a parameter.
|
||||
|
||||
const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
const model = @import("../config/model.zig");
|
||||
const name = @import("../dns/name.zig");
|
||||
const types = @import("../dns/types.zig");
|
||||
const domain_set = @import("domain_set.zig");
|
||||
const wildcard = @import("wildcard.zig");
|
||||
|
||||
pub const Error = error{ OutOfMemory, BadPattern, TooManyWildcards } || domain_set.DomainSet.Error;
|
||||
|
||||
/// Both wildcard lists of one group together. The cap exists so a rules table
|
||||
/// edited into the millions cannot turn every query into a linear scan.
|
||||
pub const max_wildcards_per_group: usize = 4096;
|
||||
|
||||
pub const RuleSet = struct {
|
||||
exact_allow: domain_set.DomainSet = .empty,
|
||||
exact_block: domain_set.DomainSet = .empty,
|
||||
/// Normalized and sorted, so a rebuild of the same rows produces the same
|
||||
/// order and the same first match.
|
||||
wildcard_allow: []const []const u8 = &.{},
|
||||
wildcard_block: []const []const u8 = &.{},
|
||||
/// One block holding the bytes of both wildcard lists; freed as a unit.
|
||||
wildcard_bytes: []const u8 = &.{},
|
||||
|
||||
pub const empty: RuleSet = .{};
|
||||
|
||||
/// `rows` are one group's rules only; splitting `listRules` output by group
|
||||
/// belongs to the caller, which is the only holder of the group table.
|
||||
///
|
||||
/// Patterns are normalized (lowercase over ASCII, one trailing dot
|
||||
/// stripped) and validated: `.exact` through `dns.name.fromText`,
|
||||
/// `.wildcard` through `wildcard.validate`. An invalid pattern is
|
||||
/// `error.BadPattern`, not a skipped row — every pattern passed
|
||||
/// `config/validate.zig` on the way in, so an invalid one here means the
|
||||
/// rows were edited underneath nxdns and a silently dropped allow rule
|
||||
/// would block a domain the operator unblocked.
|
||||
pub fn build(gpa: Allocator, rows: []const model.Rule, seed: u64) Error!RuleSet {
|
||||
if (rows.len == 0) return .empty;
|
||||
|
||||
var scratch: std.ArrayList(u8) = .empty;
|
||||
defer scratch.deinit(gpa);
|
||||
var spans: [4]std.ArrayList(Span) = .{ .empty, .empty, .empty, .empty };
|
||||
defer for (&spans) |*bucket| bucket.deinit(gpa);
|
||||
|
||||
var wildcards: usize = 0;
|
||||
var buf: [types.max_name_len]u8 = undefined;
|
||||
for (rows) |row| {
|
||||
const pattern = normalize(row.pattern, &buf) catch return error.BadPattern;
|
||||
switch (row.kind) {
|
||||
.exact => _ = name.fromText(pattern) catch return error.BadPattern,
|
||||
.wildcard => {
|
||||
wildcard.validate(pattern) catch return error.BadPattern;
|
||||
wildcards += 1;
|
||||
if (wildcards > max_wildcards_per_group) return error.TooManyWildcards;
|
||||
},
|
||||
}
|
||||
const bucket = &spans[bucketOf(row.kind, row.action)];
|
||||
try bucket.append(gpa, .{ .offset = scratch.items.len, .len = pattern.len });
|
||||
try scratch.appendSlice(gpa, pattern);
|
||||
}
|
||||
|
||||
// `scratch` stops growing here, so spans can become slices of it.
|
||||
var sorted: [4]std.ArrayList([]const u8) = .{ .empty, .empty, .empty, .empty };
|
||||
defer for (&sorted) |*bucket| bucket.deinit(gpa);
|
||||
for (&spans, &sorted) |*bucket, *out| {
|
||||
try out.ensureTotalCapacityPrecise(gpa, bucket.items.len);
|
||||
for (bucket.items) |span| {
|
||||
out.appendAssumeCapacity(scratch.items[span.offset..][0..span.len]);
|
||||
}
|
||||
std.mem.sort([]const u8, out.items, {}, lessThanBytes);
|
||||
dedupSorted(out);
|
||||
}
|
||||
|
||||
var self: RuleSet = .empty;
|
||||
errdefer self.deinit(gpa);
|
||||
|
||||
self.exact_allow = try buildSet(gpa, sorted[bucketOf(.exact, .allow)].items, seed);
|
||||
self.exact_block = try buildSet(gpa, sorted[bucketOf(.exact, .block)].items, seed);
|
||||
|
||||
const allow = sorted[bucketOf(.wildcard, .allow)].items;
|
||||
const block = sorted[bucketOf(.wildcard, .block)].items;
|
||||
var total: usize = 0;
|
||||
for (allow) |pattern| total += pattern.len;
|
||||
for (block) |pattern| total += pattern.len;
|
||||
|
||||
const bytes = try gpa.alloc(u8, total);
|
||||
self.wildcard_bytes = bytes;
|
||||
var at: usize = 0;
|
||||
self.wildcard_allow = try copyPatterns(gpa, allow, bytes, &at);
|
||||
self.wildcard_block = try copyPatterns(gpa, block, bytes, &at);
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
pub fn deinit(self: *RuleSet, gpa: Allocator) void {
|
||||
self.exact_allow.deinit(gpa);
|
||||
self.exact_block.deinit(gpa);
|
||||
gpa.free(self.wildcard_allow);
|
||||
gpa.free(self.wildcard_block);
|
||||
gpa.free(self.wildcard_bytes);
|
||||
self.* = .empty;
|
||||
}
|
||||
|
||||
pub fn memoryBytes(self: *const RuleSet) usize {
|
||||
return self.exact_allow.memoryBytes() +
|
||||
self.exact_block.memoryBytes() +
|
||||
self.wildcard_bytes.len +
|
||||
(self.wildcard_allow.len + self.wildcard_block.len) * @sizeOf([]const u8);
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internals
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Patterns are recorded as offsets because `scratch` reallocates while it
|
||||
/// grows; they become slices only after the collecting pass ends.
|
||||
const Span = struct { offset: usize, len: usize };
|
||||
|
||||
fn bucketOf(kind: model.RuleKind, action: model.RuleAction) usize {
|
||||
const kind_bit: usize = switch (kind) {
|
||||
.exact => 0,
|
||||
.wildcard => 2,
|
||||
};
|
||||
const action_bit: usize = switch (action) {
|
||||
.allow => 0,
|
||||
.block => 1,
|
||||
};
|
||||
return kind_bit + action_bit;
|
||||
}
|
||||
|
||||
fn lessThanBytes(_: void, a: []const u8, b: []const u8) bool {
|
||||
return std.mem.order(u8, a, b) == .lt;
|
||||
}
|
||||
|
||||
/// Duplicate rows are removed rather than rejected: two identical block rules
|
||||
/// are not a corrupt database, and `DomainSet.build` requires a strictly
|
||||
/// ascending body.
|
||||
fn dedupSorted(list: *std.ArrayList([]const u8)) void {
|
||||
var kept: usize = 0;
|
||||
for (list.items) |item| {
|
||||
if (kept > 0 and std.mem.eql(u8, list.items[kept - 1], item)) continue;
|
||||
list.items[kept] = item;
|
||||
kept += 1;
|
||||
}
|
||||
list.shrinkRetainingCapacity(kept);
|
||||
}
|
||||
|
||||
fn buildSet(gpa: Allocator, patterns: []const []const u8, seed: u64) Error!domain_set.DomainSet {
|
||||
if (patterns.len == 0) return .empty;
|
||||
|
||||
var body: std.ArrayList(u8) = .empty;
|
||||
defer body.deinit(gpa);
|
||||
for (patterns) |pattern| {
|
||||
try body.appendSlice(gpa, pattern);
|
||||
try body.append(gpa, '\n');
|
||||
}
|
||||
return domain_set.DomainSet.build(gpa, body.items, seed);
|
||||
}
|
||||
|
||||
fn copyPatterns(
|
||||
gpa: Allocator,
|
||||
patterns: []const []const u8,
|
||||
bytes: []u8,
|
||||
at: *usize,
|
||||
) Error![]const []const u8 {
|
||||
if (patterns.len == 0) return &.{};
|
||||
const out = try gpa.alloc([]const u8, patterns.len);
|
||||
for (out, patterns) |*slot, pattern| {
|
||||
@memcpy(bytes[at.*..][0..pattern.len], pattern);
|
||||
slot.* = bytes[at.*..][0..pattern.len];
|
||||
at.* += pattern.len;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const NameError = error{BadName};
|
||||
|
||||
/// Lowercases over ASCII and strips one trailing dot. A byte ≥ 0x80 is
|
||||
/// rejected: query names reach the matcher ASCII-lowercased, so a pattern
|
||||
/// carrying a high byte could never match anything.
|
||||
fn normalize(text: []const u8, buf: *[types.max_name_len]u8) NameError![]const u8 {
|
||||
var rest = text;
|
||||
if (rest.len > 0 and rest[rest.len - 1] == '.') rest = rest[0 .. rest.len - 1];
|
||||
if (rest.len == 0 or rest.len > types.max_name_len) return error.BadName;
|
||||
|
||||
for (rest, 0..) |byte, i| {
|
||||
if (byte >= 0x80 or byte < 0x21) return error.BadName;
|
||||
buf[i] = std.ascii.toLower(byte);
|
||||
}
|
||||
return buf[0..rest.len];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
fn rule(pattern: []const u8, kind: model.RuleKind, action: model.RuleAction) model.Rule {
|
||||
return .{ .group = "default", .pattern = pattern, .kind = kind, .action = action };
|
||||
}
|
||||
|
||||
test "exact rules land in the matching set" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("ads.example.com", .exact, .block),
|
||||
rule("good.example.com", .exact, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0x5eed);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(set.exact_block.contains("ads.example.com"));
|
||||
try testing.expect(!set.exact_block.contains("good.example.com"));
|
||||
try testing.expect(set.exact_allow.contains("good.example.com"));
|
||||
try testing.expectEqual(@as(usize, 0), set.wildcard_allow.len);
|
||||
try testing.expectEqual(@as(usize, 0), set.wildcard_block.len);
|
||||
}
|
||||
|
||||
test "wildcard rules land in the matching list, sorted" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("*.z.example.com", .wildcard, .block),
|
||||
rule("*.a.example.com", .wildcard, .block),
|
||||
rule("*.allowed.example.com", .wildcard, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0x5eed);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(usize, 2), set.wildcard_block.len);
|
||||
try testing.expectEqualStrings("*.a.example.com", set.wildcard_block[0]);
|
||||
try testing.expectEqualStrings("*.z.example.com", set.wildcard_block[1]);
|
||||
try testing.expectEqual(@as(usize, 1), set.wildcard_allow.len);
|
||||
try testing.expectEqualStrings("*.allowed.example.com", set.wildcard_allow[0]);
|
||||
}
|
||||
|
||||
test "patterns are normalized to lowercase without a trailing dot" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("ADS.Example.COM.", .exact, .block),
|
||||
rule("*.Tracker.NET.", .wildcard, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(set.exact_block.contains("ads.example.com"));
|
||||
try testing.expectEqualStrings("*.tracker.net", set.wildcard_block[0]);
|
||||
}
|
||||
|
||||
test "duplicate rows collapse to one entry" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("ads.example.com", .exact, .block),
|
||||
rule("ads.example.com.", .exact, .block),
|
||||
rule("*.x.example.com", .wildcard, .block),
|
||||
rule("*.x.example.com", .wildcard, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(u32, 1), set.exact_block.count);
|
||||
try testing.expectEqual(@as(usize, 1), set.wildcard_block.len);
|
||||
}
|
||||
|
||||
test "an invalid exact pattern is an error" {
|
||||
for ([_][]const u8{ "", ".", "a..b", "ads example.com", "ads\u{00e9}.example.com" }) |pattern| {
|
||||
const rows = [_]model.Rule{rule(pattern, .exact, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0));
|
||||
}
|
||||
}
|
||||
|
||||
test "an invalid wildcard pattern is an error" {
|
||||
for ([_][]const u8{ "example.com", "ad*.example.com", "*..com" }) |pattern| {
|
||||
const rows = [_]model.Rule{rule(pattern, .wildcard, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0));
|
||||
}
|
||||
}
|
||||
|
||||
test "too many wildcards is an error" {
|
||||
const gpa = testing.allocator;
|
||||
const rows = try gpa.alloc(model.Rule, max_wildcards_per_group + 1);
|
||||
defer gpa.free(rows);
|
||||
|
||||
var patterns: std.ArrayList([]u8) = .empty;
|
||||
defer {
|
||||
for (patterns.items) |p| gpa.free(p);
|
||||
patterns.deinit(gpa);
|
||||
}
|
||||
for (rows, 0..) |*row, i| {
|
||||
const pattern = try std.fmt.allocPrint(gpa, "*.n{d}.example.com", .{i});
|
||||
try patterns.append(gpa, pattern);
|
||||
row.* = rule(pattern, .wildcard, .block);
|
||||
}
|
||||
|
||||
try testing.expectError(error.TooManyWildcards, RuleSet.build(gpa, rows, 0));
|
||||
}
|
||||
|
||||
test "an empty rule list builds the empty set" {
|
||||
var set = try RuleSet.build(testing.allocator, &[_]model.Rule{}, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(!set.exact_block.contains("ads.example.com"));
|
||||
try testing.expectEqual(@as(usize, 0), set.memoryBytes());
|
||||
}
|
||||
|
||||
test "the empty rule set owns nothing" {
|
||||
var set: RuleSet = .empty;
|
||||
try testing.expect(!set.exact_allow.contains("x.example.com"));
|
||||
try testing.expectEqual(@as(usize, 0), set.memoryBytes());
|
||||
set.deinit(testing.allocator);
|
||||
}
|
||||
|
||||
test "memoryBytes counts every part" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("ads.example.com", .exact, .block),
|
||||
rule("*.tracker.net", .wildcard, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(set.memoryBytes() > set.exact_block.memoryBytes());
|
||||
try testing.expect(set.memoryBytes() >= "*.tracker.net".len);
|
||||
}
|
||||
|
||||
fn buildUnderFailure(gpa: Allocator) !void {
|
||||
const rows = [_]model.Rule{
|
||||
rule("ads.example.com", .exact, .block),
|
||||
rule("good.example.com", .exact, .allow),
|
||||
rule("*.tracker.net", .wildcard, .block),
|
||||
rule("*.ok.tracker.net", .wildcard, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(gpa, &rows, 0x5eed);
|
||||
defer set.deinit(gpa);
|
||||
try testing.expect(set.exact_block.contains("ads.example.com"));
|
||||
try testing.expectEqualStrings("*.tracker.net", set.wildcard_block[0]);
|
||||
}
|
||||
|
||||
test "build leaks nothing under allocation failure" {
|
||||
try testing.checkAllAllocationFailures(testing.allocator, buildUnderFailure, .{});
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//! Per-group safe search (PLAN §7.4): the table of search and video domains
|
||||
//! whose queries are rewritten to a provider-operated restricted hostname.
|
||||
//! Pure: no allocation, no `std.Io`, no clock.
|
||||
//!
|
||||
//! Every target is a name the operator's upstream resolves. nxdns never
|
||||
//! hardcodes an address for one: the providers move these hosts, and a stale
|
||||
//! literal would send a household's search traffic to somebody else's server.
|
||||
//!
|
||||
//! Google's country domains (`google.de`, `google.co.uk`, …) are deliberately
|
||||
//! not enumerated. The list is unbounded, it goes stale, and
|
||||
//! `forcesafesearch.google.com` is the documented target for every one of them.
|
||||
//! An operator who needs a country domain adds a rule.
|
||||
|
||||
const std = @import("std");
|
||||
const name = @import("../dns/name.zig");
|
||||
|
||||
pub const Entry = struct { domain: []const u8, target: []const u8 };
|
||||
|
||||
/// Sorted ascending by `domain`, so the table is searchable and diffable. The
|
||||
/// order is a correctness precondition of `lookup`, so it is asserted below at
|
||||
/// comptime rather than left to review.
|
||||
pub const table = [_]Entry{
|
||||
.{ .domain = "bing.com", .target = "strict.bing.com" },
|
||||
.{ .domain = "duckduckgo.com", .target = "safe.duckduckgo.com" },
|
||||
.{ .domain = "google.com", .target = "forcesafesearch.google.com" },
|
||||
.{ .domain = "m.youtube.com", .target = "restrictmoderate.youtube.com" },
|
||||
.{ .domain = "pixabay.com", .target = "safesearch.pixabay.com" },
|
||||
.{ .domain = "www.bing.com", .target = "strict.bing.com" },
|
||||
.{ .domain = "www.duckduckgo.com", .target = "safe.duckduckgo.com" },
|
||||
.{ .domain = "www.google.com", .target = "forcesafesearch.google.com" },
|
||||
.{ .domain = "www.youtube-nocookie.com", .target = "restrictmoderate.youtube.com" },
|
||||
.{ .domain = "www.youtube.com", .target = "restrictmoderate.youtube.com" },
|
||||
.{ .domain = "youtube.com", .target = "restrictmoderate.youtube.com" },
|
||||
.{ .domain = "youtube.googleapis.com", .target = "restrictmoderate.youtube.com" },
|
||||
.{ .domain = "youtubei.googleapis.com", .target = "restrictmoderate.youtube.com" },
|
||||
};
|
||||
|
||||
comptime {
|
||||
for (table[1..], table[0 .. table.len - 1]) |entry, previous| {
|
||||
if (std.mem.order(u8, previous.domain, entry.domain) != .lt) {
|
||||
@compileError("safesearch.table must be sorted ascending by domain and duplicate-free");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn orderDomain(query: []const u8, entry: Entry) std.math.Order {
|
||||
return std.mem.order(u8, query, entry.domain);
|
||||
}
|
||||
|
||||
/// Exact match. `domain` must be normalized: lowercase, no trailing dot.
|
||||
pub fn lookup(domain: []const u8) ?[]const u8 {
|
||||
const index = std.sort.binarySearch(Entry, &table, domain, orderDomain) orelse return null;
|
||||
return table[index].target;
|
||||
}
|
||||
|
||||
/// The rewritten question name for a matched query. Applying it — sending the
|
||||
/// rewritten question upstream and prefixing the answer with a CNAME from the
|
||||
/// original name to the target — is the handler's job (Phase 7). Nothing here
|
||||
/// builds a response.
|
||||
pub fn rewrite(domain: []const u8) ?name.Name {
|
||||
const target = lookup(domain) orelse return null;
|
||||
// Every target in `table` is a syntactically valid name, which the
|
||||
// "every table entry is a valid domain name" test below asserts.
|
||||
return name.fromText(target) catch unreachable;
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "every table entry is a valid domain name" {
|
||||
for (table) |entry| {
|
||||
_ = try name.fromText(entry.domain);
|
||||
_ = try name.fromText(entry.target);
|
||||
}
|
||||
}
|
||||
|
||||
test "lookup finds every table entry" {
|
||||
for (table) |entry| {
|
||||
try testing.expectEqualStrings(entry.target, lookup(entry.domain).?);
|
||||
}
|
||||
}
|
||||
|
||||
test "lookup matches exactly and nothing else" {
|
||||
try testing.expect(lookup("example.com") == null);
|
||||
try testing.expect(lookup("") == null);
|
||||
try testing.expect(lookup("com") == null);
|
||||
// A subdomain of a listed name is not listed: the table is exact.
|
||||
try testing.expect(lookup("images.google.com") == null);
|
||||
// Nor is a parent, nor a name a listed one is a prefix of.
|
||||
try testing.expect(lookup("google.com.evil.net") == null);
|
||||
// The caller normalizes; an uppercase spelling is a miss, not a hit.
|
||||
try testing.expect(lookup("GOOGLE.COM") == null);
|
||||
// A trailing dot is not stripped here either.
|
||||
try testing.expect(lookup("google.com.") == null);
|
||||
// A country domain is deliberately absent.
|
||||
try testing.expect(lookup("google.de") == null);
|
||||
}
|
||||
|
||||
test "lookup maps the youtube family to one target" {
|
||||
const youtube = "restrictmoderate.youtube.com";
|
||||
for ([_][]const u8{
|
||||
"m.youtube.com",
|
||||
"www.youtube-nocookie.com",
|
||||
"www.youtube.com",
|
||||
"youtube.com",
|
||||
"youtube.googleapis.com",
|
||||
"youtubei.googleapis.com",
|
||||
}) |domain| {
|
||||
try testing.expectEqualStrings(youtube, lookup(domain).?);
|
||||
}
|
||||
}
|
||||
|
||||
test "rewrite returns the target as a wire name" {
|
||||
const rewritten = rewrite("www.google.com").?;
|
||||
try testing.expectEqualSlices(
|
||||
u8,
|
||||
(try name.fromText("forcesafesearch.google.com")).wire(),
|
||||
rewritten.wire(),
|
||||
);
|
||||
try testing.expect(rewrite("example.com") == null);
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
//! Label-pattern wildcards for filtering rules (PLAN §3.9). Pure: no
|
||||
//! allocation, no `std.Io`, no recursion.
|
||||
//!
|
||||
//! A pattern is a domain name in which one or more labels are exactly `*`.
|
||||
//! Each `*` label matches one or more labels of the queried name. Partial-label
|
||||
//! globbing (`ad*.example.com`) is deliberately absent: it is regex by another
|
||||
//! name, which PLAN §2.2 rules out.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
pub const max_labels = 128;
|
||||
|
||||
/// The longest domain name is 255 wire bytes, which is 253 bytes of text.
|
||||
const max_pattern_len = 253;
|
||||
const max_label_len = 63;
|
||||
|
||||
pub const PatternError = error{
|
||||
/// No label is exactly "*".
|
||||
NoWildcard,
|
||||
/// A label contains '*' but is not exactly "*". Partial-label globbing
|
||||
/// (`ad*.example.com`) is out of scope: it is regex by another name, and
|
||||
/// PLAN §3.9 defines the wildcard as a label pattern.
|
||||
PartialWildcardLabel,
|
||||
EmptyLabel,
|
||||
LabelTooLong,
|
||||
PatternTooLong,
|
||||
TooManyLabels,
|
||||
};
|
||||
|
||||
/// Syntax only. A valid pattern has at least one label that is exactly "*",
|
||||
/// every other label is 1–63 bytes with no '*' inside it, and the whole
|
||||
/// pattern is at most 253 bytes over at most `max_labels` labels.
|
||||
pub fn validate(pattern: []const u8) PatternError!void {
|
||||
// The label count is checked before the byte length so that both bounds
|
||||
// stay individually reportable: any pattern with more than `max_labels`
|
||||
// labels also exceeds `max_pattern_len`.
|
||||
if (std.mem.count(u8, pattern, ".") + 1 > max_labels) return error.TooManyLabels;
|
||||
if (pattern.len > max_pattern_len) return error.PatternTooLong;
|
||||
|
||||
var star = false;
|
||||
var it = std.mem.splitScalar(u8, pattern, '.');
|
||||
while (it.next()) |label| {
|
||||
if (label.len == 0) return error.EmptyLabel;
|
||||
if (label.len > max_label_len) return error.LabelTooLong;
|
||||
if (std.mem.eql(u8, label, "*")) {
|
||||
star = true;
|
||||
} else if (std.mem.findScalar(u8, label, '*') != null) {
|
||||
return error.PartialWildcardLabel;
|
||||
}
|
||||
}
|
||||
if (!star) return error.NoWildcard;
|
||||
}
|
||||
|
||||
/// `domain` is already normalized: lowercase, no trailing dot. `pattern` is
|
||||
/// lowercase. Each "*" label matches ONE OR MORE labels.
|
||||
/// Allocation-free; the backtracking is bounded by `max_labels` on both sides.
|
||||
pub fn matches(pattern: []const u8, domain: []const u8) bool {
|
||||
var pattern_labels: [max_labels][]const u8 = undefined;
|
||||
var domain_labels: [max_labels][]const u8 = undefined;
|
||||
|
||||
// `validate` rejects a pattern above the label bound and the 253-byte name
|
||||
// limit bounds the domain, so neither overflow can reach here from the
|
||||
// matcher. Both are re-checked so that unvalidated input still terminates.
|
||||
const pattern_len = split(pattern, &pattern_labels) orelse return false;
|
||||
const domain_len = split(domain, &domain_labels) orelse return false;
|
||||
|
||||
var d: usize = 0;
|
||||
var p: usize = 0;
|
||||
var star: ?usize = null;
|
||||
var star_end: usize = 0;
|
||||
|
||||
while (d < domain_len) {
|
||||
if (p < pattern_len and isStar(pattern_labels[p])) {
|
||||
// A '*' takes one label now and grows by one on each backtrack.
|
||||
star = p;
|
||||
p += 1;
|
||||
d += 1;
|
||||
star_end = d;
|
||||
} else if (p < pattern_len and std.mem.eql(u8, pattern_labels[p], domain_labels[d])) {
|
||||
p += 1;
|
||||
d += 1;
|
||||
} else if (star) |s| {
|
||||
p = s + 1;
|
||||
star_end += 1;
|
||||
d = star_end;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// A trailing '*' has already consumed its label; nothing may be left over.
|
||||
return p == pattern_len;
|
||||
}
|
||||
|
||||
fn isStar(label: []const u8) bool {
|
||||
return label.len == 1 and label[0] == '*';
|
||||
}
|
||||
|
||||
/// Null when `text` holds more than `max_labels` labels.
|
||||
fn split(text: []const u8, out: *[max_labels][]const u8) ?usize {
|
||||
var n: usize = 0;
|
||||
var it = std.mem.splitScalar(u8, text, '.');
|
||||
while (it.next()) |label| {
|
||||
if (n == max_labels) return null;
|
||||
out[n] = label;
|
||||
n += 1;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "validate accepts a leading wildcard label" {
|
||||
try validate("*.doubleclick.net");
|
||||
}
|
||||
|
||||
test "validate accepts an interior wildcard label" {
|
||||
try validate("ads.*.example.com");
|
||||
}
|
||||
|
||||
test "validate rejects a pattern with no wildcard label" {
|
||||
try testing.expectError(error.NoWildcard, validate("example.com"));
|
||||
}
|
||||
|
||||
test "validate rejects a partial wildcard label" {
|
||||
try testing.expectError(error.PartialWildcardLabel, validate("a*b.com"));
|
||||
}
|
||||
|
||||
test "validate rejects an empty label" {
|
||||
try testing.expectError(error.EmptyLabel, validate("a..b"));
|
||||
}
|
||||
|
||||
test "validate rejects an oversize label" {
|
||||
const pattern = "*." ++ ("a" ** 64);
|
||||
try testing.expectError(error.LabelTooLong, validate(pattern));
|
||||
}
|
||||
|
||||
test "validate rejects an oversize pattern" {
|
||||
const label = "a" ** 60;
|
||||
const pattern = "*." ++ label ++ "." ++ label ++ "." ++ label ++ "." ++ label ++ "." ++ label;
|
||||
try testing.expect(pattern.len > 253);
|
||||
try testing.expectError(error.PatternTooLong, validate(pattern));
|
||||
}
|
||||
|
||||
test "validate rejects too many labels" {
|
||||
const pattern = "*." ++ ("a." ** 199) ++ "com";
|
||||
try testing.expectError(error.TooManyLabels, validate(pattern));
|
||||
}
|
||||
|
||||
test "matches one label under a leading wildcard" {
|
||||
try testing.expect(matches("*.doubleclick.net", "a.doubleclick.net"));
|
||||
}
|
||||
|
||||
test "matches several labels under a leading wildcard" {
|
||||
try testing.expect(matches("*.doubleclick.net", "a.b.doubleclick.net"));
|
||||
}
|
||||
|
||||
test "a leading wildcard does not match the apex" {
|
||||
try testing.expect(!matches("*.doubleclick.net", "doubleclick.net"));
|
||||
}
|
||||
|
||||
test "matches one label at an interior wildcard" {
|
||||
try testing.expect(matches("ads.*.example.com", "ads.eu.example.com"));
|
||||
}
|
||||
|
||||
test "matches several labels at an interior wildcard" {
|
||||
try testing.expect(matches("ads.*.example.com", "ads.eu.west.example.com"));
|
||||
}
|
||||
|
||||
test "an interior wildcard requires at least one label" {
|
||||
try testing.expect(!matches("ads.*.example.com", "ads.example.com"));
|
||||
}
|
||||
|
||||
test "a pattern does not match a name that only contains it" {
|
||||
try testing.expect(!matches("*.example.com", "example.com.evil.net"));
|
||||
}
|
||||
|
||||
test "a pathological pattern terminates" {
|
||||
const pattern = ("*." ** 8) ++ "example.com";
|
||||
const domain = ("a." ** 98) ++ "example.net";
|
||||
try testing.expect(!matches(pattern, domain));
|
||||
}
|
||||
Reference in New Issue
Block a user