milestone 28: query provenance — every logged query is exactly explainable
Gates / frontend (push) Successful in 1m36s
Gates / test (push) Successful in 1m56s
Gates / test-aarch64 (push) Successful in 7m37s
Gates / package (push) Successful in 9m12s
Gates / container (push) Successful in 13s
CI / gates (push) Successful in 19m4s
Gates / frontend (push) Successful in 1m36s
Gates / test (push) Successful in 1m56s
Gates / test-aarch64 (push) Successful in 7m37s
Gates / package (push) Successful in 9m12s
Gates / container (push) Successful in 13s
CI / gates (push) Successful in 19m4s
query rows gain qclass, rcode, group, policy action and reason, the matched rule or list entry with its source, cname and safe-search targets, route kind, forward zone, and the resolver that actually answered — the pool and local markers die. servfails are logged and name the resolver that lost; post-parse protocol refusals become rows. a detail page at /queries/:id renders the ordered explanation, and coverage watermarks distinguish an empty history from a missing one. the schema fingerprint changes: existing query history is recreated with the old file kept aside and the reset filed as a resolved diagnostic. fixes an oversized udp reply being rebuilt as noerror, which handed clients a truncated nxdomain as success.
This commit is contained in:
+416
-53
@@ -34,8 +34,12 @@ const std = @import("std");
|
||||
const db = @import("db.zig");
|
||||
const disk_monitor = @import("disk_monitor.zig");
|
||||
const events = @import("events.zig");
|
||||
const limits = @import("../config/limits.zig");
|
||||
const model = @import("../config/model.zig");
|
||||
const provenance = @import("provenance.zig");
|
||||
const queries_repo = @import("repositories/queries_repo.zig");
|
||||
const regex = @import("../filter/regex.zig");
|
||||
const safe_url = @import("../safe_url.zig");
|
||||
|
||||
/// Named `scope` rather than `log`: `Logger.log` is the enqueue entry point,
|
||||
/// and the two names collide inside the struct.
|
||||
@@ -58,10 +62,32 @@ pub const gate_retry_s = 1;
|
||||
pub const max_domain_len = 253;
|
||||
/// RFC 5952 text of any IPv6 address, zone identifier included.
|
||||
pub const max_client_len = 45;
|
||||
pub const max_reason_len = 32;
|
||||
pub const max_upstream_len = 64;
|
||||
|
||||
/// `matched` holds the rule that decided the query, and the widest rule the
|
||||
/// configuration accepts is a regex pattern at `regex.max_pattern_len`. It does
|
||||
/// not fit a `u8` length, which is why this one field carries a `u16`.
|
||||
pub const max_matched_len = regex.max_pattern_len;
|
||||
|
||||
/// The redacted resolver identity of the exchange that actually happened. Two
|
||||
/// bounds apply and the buffer takes the larger, so neither producer truncates:
|
||||
/// the longest well-formed `scheme://host:port` with a maximal host, and
|
||||
/// `safe_url.redact`'s own output bound (`max_len` plus the `...` it appends
|
||||
/// when it truncates).
|
||||
pub const max_upstream_len = @max(
|
||||
"https://".len + max_domain_len + ":65535".len,
|
||||
safe_url.max_len + 3,
|
||||
);
|
||||
|
||||
/// `cname_target`, `safe_search_target` and `forward_zone` each hold a domain
|
||||
/// name, so they are all the same width as `domain`.
|
||||
const max_name_len = max_domain_len;
|
||||
|
||||
/// One row on its way to `query_log`, carrying its own bytes.
|
||||
///
|
||||
/// Every field is by value: `Io.Queue` copies elements as raw bytes, so nothing
|
||||
/// here may borrow from the query that produced it. The buffer widths above are
|
||||
/// therefore the row's real storage cost, multiplied by the queue capacity —
|
||||
/// see `query_log_buffer_max`.
|
||||
pub const Entry = struct {
|
||||
timestamp: i64,
|
||||
domain_buf: [max_domain_len]u8,
|
||||
@@ -69,28 +95,68 @@ pub const Entry = struct {
|
||||
client_buf: [max_client_len]u8,
|
||||
client_len: u8,
|
||||
qtype: ?u16,
|
||||
qclass: u16,
|
||||
/// The client-visible RCODE. Twelve bits, not four: an EDNS extended code
|
||||
/// carries eight more bits in the OPT record than the header's four. The
|
||||
/// type is the enforcement — the column's `CHECK` in `querylog_schema.ddl`
|
||||
/// bounds the same value against every other writer of the file.
|
||||
rcode: u12,
|
||||
blocked: bool,
|
||||
reason_buf: [max_reason_len]u8,
|
||||
reason_len: u8,
|
||||
response_time_us: ?i64,
|
||||
cache_hit: ?bool,
|
||||
upstream_buf: [max_upstream_len]u8,
|
||||
upstream_len: u8,
|
||||
upstream_len: u16,
|
||||
|
||||
group_id: ?i64,
|
||||
group_buf: [limits.max_group_name_len]u8,
|
||||
group_len: u8,
|
||||
policy_action: provenance.PolicyAction,
|
||||
policy_reason: provenance.PolicyReason,
|
||||
matched_buf: [max_matched_len]u8,
|
||||
matched_len: u16,
|
||||
source_id: ?i64,
|
||||
source_buf: [limits.max_source_name_len]u8,
|
||||
source_len: u8,
|
||||
cname_buf: [max_name_len]u8,
|
||||
cname_len: u8,
|
||||
safe_search_buf: [max_name_len]u8,
|
||||
safe_search_len: u8,
|
||||
route_kind: provenance.RouteKind,
|
||||
forward_zone_buf: [max_name_len]u8,
|
||||
forward_zone_len: u8,
|
||||
|
||||
/// The borrowed shape of an entry. `init` copies out of it, so a caller can
|
||||
/// build one from slices that die with the query.
|
||||
///
|
||||
/// Every text field defaults to `""`, which reaches a nullable column as
|
||||
/// NULL. The three fields with no sensible empty value — the two enums and
|
||||
/// the route — default to what a query the pipeline has not yet explained
|
||||
/// would honestly say about itself.
|
||||
pub const Fields = struct {
|
||||
timestamp: i64,
|
||||
domain: []const u8,
|
||||
client_ip: []const u8,
|
||||
qtype: ?u16 = null,
|
||||
qclass: u16 = 0,
|
||||
rcode: u12 = 0,
|
||||
blocked: bool = false,
|
||||
/// Empty means "no reason", which reaches the database as NULL.
|
||||
block_reason: []const u8 = "",
|
||||
response_time_us: ?i64 = null,
|
||||
cache_hit: ?bool = null,
|
||||
/// Empty means "no upstream", which reaches the database as NULL.
|
||||
/// Empty means "no upstream was attempted", which reaches the database
|
||||
/// as NULL. Already redacted by the caller.
|
||||
upstream: []const u8 = "",
|
||||
|
||||
group_id: ?i64 = null,
|
||||
group_name: []const u8 = "",
|
||||
policy_action: provenance.PolicyAction = .not_evaluated,
|
||||
policy_reason: provenance.PolicyReason = .no_match,
|
||||
matched: []const u8 = "",
|
||||
source_id: ?i64 = null,
|
||||
source_name: []const u8 = "",
|
||||
cname_target: []const u8 = "",
|
||||
safe_search_target: []const u8 = "",
|
||||
route_kind: provenance.RouteKind = .upstream,
|
||||
forward_zone: []const u8 = "",
|
||||
};
|
||||
|
||||
/// Copies each string in, truncated to what its buffer holds. A name longer
|
||||
@@ -104,27 +170,61 @@ pub const Entry = struct {
|
||||
.client_buf = undefined,
|
||||
.client_len = 0,
|
||||
.qtype = f.qtype,
|
||||
.qclass = f.qclass,
|
||||
.rcode = f.rcode,
|
||||
.blocked = f.blocked,
|
||||
.reason_buf = undefined,
|
||||
.reason_len = 0,
|
||||
.response_time_us = f.response_time_us,
|
||||
.cache_hit = f.cache_hit,
|
||||
.upstream_buf = undefined,
|
||||
.upstream_len = 0,
|
||||
.group_id = f.group_id,
|
||||
.group_buf = undefined,
|
||||
.group_len = 0,
|
||||
.policy_action = f.policy_action,
|
||||
.policy_reason = f.policy_reason,
|
||||
.matched_buf = undefined,
|
||||
.matched_len = 0,
|
||||
.source_id = f.source_id,
|
||||
.source_buf = undefined,
|
||||
.source_len = 0,
|
||||
.cname_buf = undefined,
|
||||
.cname_len = 0,
|
||||
.safe_search_buf = undefined,
|
||||
.safe_search_len = 0,
|
||||
.route_kind = f.route_kind,
|
||||
.forward_zone_buf = undefined,
|
||||
.forward_zone_len = 0,
|
||||
};
|
||||
entry.setDomain(f.domain);
|
||||
entry.setClientIp(f.client_ip);
|
||||
entry.reason_len = copyInto(&entry.reason_buf, f.block_reason);
|
||||
entry.upstream_len = copyInto(&entry.upstream_buf, f.upstream);
|
||||
copyInto(&entry.upstream_buf, &entry.upstream_len, f.upstream);
|
||||
copyInto(&entry.group_buf, &entry.group_len, f.group_name);
|
||||
entry.setMatched(f.matched);
|
||||
copyInto(&entry.source_buf, &entry.source_len, f.source_name);
|
||||
entry.setCnameTarget(f.cname_target);
|
||||
entry.setSafeSearchTarget(f.safe_search_target);
|
||||
copyInto(&entry.forward_zone_buf, &entry.forward_zone_len, f.forward_zone);
|
||||
return entry;
|
||||
}
|
||||
|
||||
pub fn setDomain(self: *Entry, value: []const u8) void {
|
||||
self.domain_len = copyInto(&self.domain_buf, value);
|
||||
copyInto(&self.domain_buf, &self.domain_len, value);
|
||||
}
|
||||
|
||||
pub fn setClientIp(self: *Entry, value: []const u8) void {
|
||||
self.client_len = copyInto(&self.client_buf, value);
|
||||
copyInto(&self.client_buf, &self.client_len, value);
|
||||
}
|
||||
|
||||
pub fn setMatched(self: *Entry, value: []const u8) void {
|
||||
copyInto(&self.matched_buf, &self.matched_len, value);
|
||||
}
|
||||
|
||||
pub fn setCnameTarget(self: *Entry, value: []const u8) void {
|
||||
copyInto(&self.cname_buf, &self.cname_len, value);
|
||||
}
|
||||
|
||||
pub fn setSafeSearchTarget(self: *Entry, value: []const u8) void {
|
||||
copyInto(&self.safe_search_buf, &self.safe_search_len, value);
|
||||
}
|
||||
|
||||
pub fn domain(self: *const Entry) []const u8 {
|
||||
@@ -135,19 +235,56 @@ pub const Entry = struct {
|
||||
return self.client_buf[0..self.client_len];
|
||||
}
|
||||
|
||||
pub fn blockReason(self: *const Entry) []const u8 {
|
||||
return self.reason_buf[0..self.reason_len];
|
||||
}
|
||||
|
||||
pub fn upstream(self: *const Entry) []const u8 {
|
||||
return self.upstream_buf[0..self.upstream_len];
|
||||
}
|
||||
|
||||
pub fn groupName(self: *const Entry) []const u8 {
|
||||
return self.group_buf[0..self.group_len];
|
||||
}
|
||||
|
||||
pub fn matched(self: *const Entry) []const u8 {
|
||||
return self.matched_buf[0..self.matched_len];
|
||||
}
|
||||
|
||||
pub fn sourceName(self: *const Entry) []const u8 {
|
||||
return self.source_buf[0..self.source_len];
|
||||
}
|
||||
|
||||
pub fn cnameTarget(self: *const Entry) []const u8 {
|
||||
return self.cname_buf[0..self.cname_len];
|
||||
}
|
||||
|
||||
pub fn safeSearchTarget(self: *const Entry) []const u8 {
|
||||
return self.safe_search_buf[0..self.safe_search_len];
|
||||
}
|
||||
|
||||
pub fn forwardZone(self: *const Entry) []const u8 {
|
||||
return self.forward_zone_buf[0..self.forward_zone_len];
|
||||
}
|
||||
};
|
||||
|
||||
fn copyInto(buf: []u8, value: []const u8) u8 {
|
||||
/// The memory budget the queue is allowed to occupy. `Entry` travels by value,
|
||||
/// so the composition root allocates `query_log_buffer_max` of them in full at
|
||||
/// boot (`app.zig`) and the SSE hub embeds a ring of them per subscriber.
|
||||
const queue_budget_bytes = 64 * 1024 * 1024;
|
||||
|
||||
/// The ceiling `config/validate.zig` enforces on `logging.query_log_buffer_max`,
|
||||
/// derived from the width of `Entry` rather than picked.
|
||||
///
|
||||
/// The provenance columns of milestone 28 roughly tripled `Entry`, so the bound
|
||||
/// that matters is bytes, not entries: an operator who asks for a million
|
||||
/// entries is asking for well over a gigabyte of queue. This is a sanity bound,
|
||||
/// not a memory-fit guarantee — what actually fits depends on the box.
|
||||
pub const query_log_buffer_max: u32 = @intCast(queue_budget_bytes / @sizeOf(Entry));
|
||||
|
||||
/// Copies as much of `value` as `buf` holds, and stores the length through
|
||||
/// `len`. `len`'s type never bounds anything — `buf.len` does — so the same
|
||||
/// helper serves the `u8` fields and the `u16` ones.
|
||||
fn copyInto(buf: []u8, len: anytype, value: []const u8) void {
|
||||
const n = @min(buf.len, value.len);
|
||||
@memcpy(buf[0..n], value[0..n]);
|
||||
return @intCast(n);
|
||||
len.* = @intCast(n);
|
||||
}
|
||||
|
||||
/// The row borrows from `entry`, which must outlive the `writeBatch` call.
|
||||
@@ -157,11 +294,23 @@ fn toRow(entry: *const Entry) queries_repo.Row {
|
||||
.domain = entry.domain(),
|
||||
.client_ip = entry.clientIp(),
|
||||
.qtype = entry.qtype,
|
||||
.qclass = entry.qclass,
|
||||
.rcode = entry.rcode,
|
||||
.blocked = entry.blocked,
|
||||
.block_reason = emptyAsNull(entry.blockReason()),
|
||||
.response_time_us = entry.response_time_us,
|
||||
.cache_hit = entry.cache_hit,
|
||||
.upstream = emptyAsNull(entry.upstream()),
|
||||
.group_id = entry.group_id,
|
||||
.group_name = emptyAsNull(entry.groupName()),
|
||||
.policy_action = entry.policy_action,
|
||||
.policy_reason = entry.policy_reason,
|
||||
.matched = emptyAsNull(entry.matched()),
|
||||
.source_id = entry.source_id,
|
||||
.source_name = emptyAsNull(entry.sourceName()),
|
||||
.cname_target = emptyAsNull(entry.cnameTarget()),
|
||||
.safe_search_target = emptyAsNull(entry.safeSearchTarget()),
|
||||
.route_kind = entry.route_kind,
|
||||
.forward_zone = emptyAsNull(entry.forwardZone()),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -231,9 +380,23 @@ pub const Logger = struct {
|
||||
/// and hands the result to every consumer, so nothing downstream — the
|
||||
/// database or the event stream — can observe a value the operator asked
|
||||
/// to hide.
|
||||
/// `hide_domains` covers every field derived from the query name, not just
|
||||
/// `domain`: a matched wildcard, a CNAME target and a safe-search target
|
||||
/// each name the very thing the operator asked to keep out of the log.
|
||||
///
|
||||
/// `forward_zone`, `group_name` and `source_name` stay visible. They are
|
||||
/// configuration labels the operator wrote, identical on every row that
|
||||
/// hits them, and they say nothing about which name a client looked up.
|
||||
pub fn transformed(self: *const Logger, entry: Entry) Entry {
|
||||
var out = entry;
|
||||
if (self.cfg.hide_domains) out.setDomain(hidden_marker);
|
||||
if (self.cfg.hide_domains) {
|
||||
out.setDomain(hidden_marker);
|
||||
// Only where there is something to hide: an empty field means the
|
||||
// query had no such value, and writing a marker would claim it did.
|
||||
if (out.matched_len != 0) out.setMatched(hidden_marker);
|
||||
if (out.cname_len != 0) out.setCnameTarget(hidden_marker);
|
||||
if (out.safe_search_len != 0) out.setSafeSearchTarget(hidden_marker);
|
||||
}
|
||||
if (self.cfg.hide_client_ips) out.setClientIp(hidden_marker);
|
||||
return out;
|
||||
}
|
||||
@@ -574,6 +737,34 @@ fn sampleEntry(timestamp: i64, domain: []const u8) Entry {
|
||||
});
|
||||
}
|
||||
|
||||
/// Every provenance field set to a distinct recognisable value, so a test that
|
||||
/// loses one loses it visibly.
|
||||
fn fullFields(timestamp: i64) Entry.Fields {
|
||||
return .{
|
||||
.timestamp = timestamp,
|
||||
.domain = "ads.example.com",
|
||||
.client_ip = "2001:db8::1",
|
||||
.qtype = 28,
|
||||
.qclass = 1,
|
||||
.rcode = 3,
|
||||
.blocked = true,
|
||||
.response_time_us = 42,
|
||||
.cache_hit = true,
|
||||
.upstream = "https://dns.example/dns-query",
|
||||
.group_id = 7,
|
||||
.group_name = "kids",
|
||||
.policy_action = .block,
|
||||
.policy_reason = .blocklist_wildcard,
|
||||
.matched = "*.ads.example",
|
||||
.source_id = 3,
|
||||
.source_name = "steven black",
|
||||
.cname_target = "tracker.cdn.example",
|
||||
.safe_search_target = "forcesafesearch.google.com",
|
||||
.route_kind = .blocked,
|
||||
.forward_zone = "home.arpa",
|
||||
};
|
||||
}
|
||||
|
||||
fn openLog() !db.Db {
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
errdefer database.close();
|
||||
@@ -583,44 +774,72 @@ fn openLog() !db.Db {
|
||||
}
|
||||
|
||||
test "an entry carries its own bytes and reads them back" {
|
||||
const entry: Entry = .init(.{
|
||||
.timestamp = 1700000000,
|
||||
.domain = "ads.example.com",
|
||||
.client_ip = "2001:db8::1",
|
||||
.qtype = 28,
|
||||
.blocked = true,
|
||||
.block_reason = "blocklist",
|
||||
.response_time_us = 42,
|
||||
.cache_hit = true,
|
||||
.upstream = "dns.example",
|
||||
});
|
||||
const entry: Entry = .init(fullFields(1700000000));
|
||||
|
||||
try testing.expectEqualStrings("ads.example.com", entry.domain());
|
||||
try testing.expectEqualStrings("2001:db8::1", entry.clientIp());
|
||||
try testing.expectEqualStrings("blocklist", entry.blockReason());
|
||||
try testing.expectEqualStrings("dns.example", entry.upstream());
|
||||
try testing.expectEqualStrings("https://dns.example/dns-query", entry.upstream());
|
||||
try testing.expectEqual(@as(?u16, 28), entry.qtype);
|
||||
try testing.expectEqual(@as(u16, 1), entry.qclass);
|
||||
try testing.expectEqual(@as(u12, 3), entry.rcode);
|
||||
try testing.expect(entry.blocked);
|
||||
try testing.expectEqual(@as(?i64, 42), entry.response_time_us);
|
||||
try testing.expectEqual(@as(?bool, true), entry.cache_hit);
|
||||
|
||||
try testing.expectEqual(@as(?i64, 7), entry.group_id);
|
||||
try testing.expectEqualStrings("kids", entry.groupName());
|
||||
try testing.expectEqual(provenance.PolicyAction.block, entry.policy_action);
|
||||
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, entry.policy_reason);
|
||||
try testing.expectEqualStrings("*.ads.example", entry.matched());
|
||||
try testing.expectEqual(@as(?i64, 3), entry.source_id);
|
||||
try testing.expectEqualStrings("steven black", entry.sourceName());
|
||||
try testing.expectEqualStrings("tracker.cdn.example", entry.cnameTarget());
|
||||
try testing.expectEqualStrings("forcesafesearch.google.com", entry.safeSearchTarget());
|
||||
try testing.expectEqual(provenance.RouteKind.blocked, entry.route_kind);
|
||||
try testing.expectEqualStrings("home.arpa", entry.forwardZone());
|
||||
}
|
||||
|
||||
test "an oversize string is truncated to what its buffer holds" {
|
||||
const long_domain = "a" ** 400;
|
||||
const entry: Entry = .init(.{
|
||||
.timestamp = 1,
|
||||
.domain = long_domain,
|
||||
.client_ip = "192.0.2.1",
|
||||
.block_reason = "r" ** 64,
|
||||
.upstream = "u" ** 128,
|
||||
.domain = "a" ** 400,
|
||||
.client_ip = "c" ** 80,
|
||||
.upstream = "u" ** 600,
|
||||
.group_name = "g" ** 200,
|
||||
.matched = "m" ** 600,
|
||||
.source_name = "s" ** 200,
|
||||
.cname_target = "n" ** 400,
|
||||
.safe_search_target = "f" ** 400,
|
||||
.forward_zone = "z" ** 400,
|
||||
});
|
||||
|
||||
try testing.expectEqual(@as(usize, max_domain_len), entry.domain().len);
|
||||
try testing.expectEqual(@as(usize, max_reason_len), entry.blockReason().len);
|
||||
try testing.expectEqual(@as(usize, max_client_len), entry.clientIp().len);
|
||||
try testing.expectEqual(@as(usize, max_upstream_len), entry.upstream().len);
|
||||
try testing.expectEqual(@as(usize, limits.max_group_name_len), entry.groupName().len);
|
||||
try testing.expectEqual(@as(usize, max_matched_len), entry.matched().len);
|
||||
try testing.expectEqual(@as(usize, limits.max_source_name_len), entry.sourceName().len);
|
||||
try testing.expectEqual(@as(usize, max_name_len), entry.cnameTarget().len);
|
||||
try testing.expectEqual(@as(usize, max_name_len), entry.safeSearchTarget().len);
|
||||
try testing.expectEqual(@as(usize, max_name_len), entry.forwardZone().len);
|
||||
try testing.expectEqualStrings("a" ** max_domain_len, entry.domain());
|
||||
}
|
||||
|
||||
test "a 256-byte matched pattern is stored whole" {
|
||||
// The widest rule the configuration accepts is a regex at
|
||||
// `regex.max_pattern_len`, and it does not fit a `u8` length — which is the
|
||||
// whole reason `matched_len` is a `u16`.
|
||||
const widest = "p" ** regex.max_pattern_len;
|
||||
const entry: Entry = .init(.{
|
||||
.timestamp = 1,
|
||||
.domain = "example.com",
|
||||
.client_ip = "192.0.2.1",
|
||||
.matched = widest,
|
||||
});
|
||||
try testing.expectEqualStrings(widest, entry.matched());
|
||||
try testing.expectEqual(@as(u16, regex.max_pattern_len), entry.matched_len);
|
||||
}
|
||||
|
||||
test "toRow maps the empty strings to null and passes the rest through" {
|
||||
const bare: Entry = .init(.{
|
||||
.timestamp = 7,
|
||||
@@ -631,23 +850,85 @@ test "toRow maps the empty strings to null and passes the rest through" {
|
||||
try testing.expectEqual(@as(i64, 7), bare_row.timestamp);
|
||||
try testing.expectEqualStrings("example.com", bare_row.domain);
|
||||
try testing.expectEqualStrings("192.0.2.5", bare_row.client_ip);
|
||||
try testing.expectEqual(@as(?[]const u8, null), bare_row.block_reason);
|
||||
try testing.expectEqual(@as(?[]const u8, null), bare_row.upstream);
|
||||
try testing.expectEqual(@as(?u16, null), bare_row.qtype);
|
||||
try testing.expectEqual(@as(?bool, null), bare_row.cache_hit);
|
||||
// Every optional text field of an entry nothing filled in reaches its
|
||||
// column as NULL rather than as an empty string.
|
||||
try testing.expectEqual(@as(?[]const u8, null), bare_row.upstream);
|
||||
try testing.expectEqual(@as(?[]const u8, null), bare_row.group_name);
|
||||
try testing.expectEqual(@as(?[]const u8, null), bare_row.matched);
|
||||
try testing.expectEqual(@as(?[]const u8, null), bare_row.source_name);
|
||||
try testing.expectEqual(@as(?[]const u8, null), bare_row.cname_target);
|
||||
try testing.expectEqual(@as(?[]const u8, null), bare_row.safe_search_target);
|
||||
try testing.expectEqual(@as(?[]const u8, null), bare_row.forward_zone);
|
||||
|
||||
const full: Entry = .init(.{
|
||||
.timestamp = 8,
|
||||
.domain = "blocked.example",
|
||||
.client_ip = "192.0.2.6",
|
||||
.blocked = true,
|
||||
.block_reason = "blocklist",
|
||||
.upstream = "9.9.9.9",
|
||||
});
|
||||
const full: Entry = .init(fullFields(8));
|
||||
const full_row = toRow(&full);
|
||||
try testing.expect(full_row.blocked);
|
||||
try testing.expectEqualStrings("blocklist", full_row.block_reason.?);
|
||||
try testing.expectEqualStrings("9.9.9.9", full_row.upstream.?);
|
||||
try testing.expectEqual(@as(u16, 1), full_row.qclass);
|
||||
try testing.expectEqual(@as(u12, 3), full_row.rcode);
|
||||
try testing.expectEqualStrings("https://dns.example/dns-query", full_row.upstream.?);
|
||||
try testing.expectEqual(@as(?i64, 7), full_row.group_id);
|
||||
try testing.expectEqualStrings("kids", full_row.group_name.?);
|
||||
try testing.expectEqual(provenance.PolicyAction.block, full_row.policy_action);
|
||||
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, full_row.policy_reason);
|
||||
try testing.expectEqualStrings("*.ads.example", full_row.matched.?);
|
||||
try testing.expectEqual(@as(?i64, 3), full_row.source_id);
|
||||
try testing.expectEqualStrings("steven black", full_row.source_name.?);
|
||||
try testing.expectEqualStrings("tracker.cdn.example", full_row.cname_target.?);
|
||||
try testing.expectEqualStrings("forcesafesearch.google.com", full_row.safe_search_target.?);
|
||||
try testing.expectEqual(provenance.RouteKind.blocked, full_row.route_kind);
|
||||
try testing.expectEqualStrings("home.arpa", full_row.forward_zone.?);
|
||||
}
|
||||
|
||||
test "an entry with every provenance field set survives the queue, toRow, insert and detailById" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
var writer = try queries_repo.BatchWriter.init(&database);
|
||||
defer writer.deinit();
|
||||
|
||||
var buf: [4]Entry = undefined;
|
||||
var logger: Logger = .init(.{}, &buf);
|
||||
|
||||
// The widest `matched` the configuration accepts, carried the whole way:
|
||||
// 256 bytes does not fit the `u8` length every other text field uses.
|
||||
const widest_matched = "p" ** max_matched_len;
|
||||
var fields = fullFields(1234);
|
||||
fields.matched = widest_matched;
|
||||
logger.log(io, .init(fields));
|
||||
|
||||
const queued = try logger.queue.getOne(io);
|
||||
try logger.flush(io, &writer, &.{queued}, null);
|
||||
|
||||
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const stored = (try queries_repo.detailById(&database, arena_state.allocator(), 1)).?;
|
||||
|
||||
try testing.expectEqual(@as(i64, 1234), stored.ts);
|
||||
try testing.expectEqualStrings("ads.example.com", stored.domain);
|
||||
try testing.expectEqualStrings("2001:db8::1", stored.client_ip);
|
||||
try testing.expectEqual(@as(?u16, 28), stored.qtype);
|
||||
try testing.expectEqual(@as(u16, 1), stored.qclass);
|
||||
try testing.expectEqual(@as(u12, 3), stored.rcode);
|
||||
try testing.expect(stored.blocked);
|
||||
try testing.expectEqual(@as(?i64, 42), stored.response_time_us);
|
||||
try testing.expectEqual(@as(?bool, true), stored.cache_hit);
|
||||
try testing.expectEqualStrings("https://dns.example/dns-query", stored.upstream);
|
||||
try testing.expectEqual(@as(?i64, 7), stored.group_id);
|
||||
try testing.expectEqualStrings("kids", stored.group_name);
|
||||
try testing.expectEqual(provenance.PolicyAction.block, stored.policy_action);
|
||||
try testing.expectEqual(provenance.PolicyReason.blocklist_wildcard, stored.policy_reason);
|
||||
try testing.expectEqualStrings(widest_matched, stored.matched);
|
||||
try testing.expectEqual(@as(?i64, 3), stored.source_id);
|
||||
try testing.expectEqualStrings("steven black", stored.source_name);
|
||||
try testing.expectEqualStrings("tracker.cdn.example", stored.cname_target);
|
||||
try testing.expectEqualStrings("forcesafesearch.google.com", stored.safe_search_target);
|
||||
try testing.expectEqual(provenance.RouteKind.blocked, stored.route_kind);
|
||||
try testing.expectEqualStrings("home.arpa", stored.forward_zone);
|
||||
}
|
||||
|
||||
test "log applies both privacy transforms before the entry reaches the queue" {
|
||||
@@ -667,6 +948,88 @@ test "log applies both privacy transforms before the entry reaches the queue" {
|
||||
try testing.expectEqual(@as(u64, 0), logger.queries_dropped.load(.monotonic));
|
||||
}
|
||||
|
||||
test "hide_domains hides every query-derived name and leaves the labels alone" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var buf: [4]Entry = undefined;
|
||||
var logger: Logger = .init(.{ .hide_domains = true }, &buf);
|
||||
logger.log(io, .init(fullFields(1)));
|
||||
const hidden = try logger.queue.getOne(io);
|
||||
|
||||
// Every field derived from the name the client asked for.
|
||||
try testing.expectEqualStrings(hidden_marker, hidden.domain());
|
||||
try testing.expectEqualStrings(hidden_marker, hidden.matched());
|
||||
try testing.expectEqualStrings(hidden_marker, hidden.cnameTarget());
|
||||
try testing.expectEqualStrings(hidden_marker, hidden.safeSearchTarget());
|
||||
|
||||
// The client is governed by `hide_client_ips`, not by this flag.
|
||||
try testing.expectEqualStrings("2001:db8::1", hidden.clientIp());
|
||||
|
||||
// Configuration labels the operator wrote. They are identical on every row
|
||||
// that hits them and say nothing about which name a client looked up.
|
||||
try testing.expectEqualStrings("kids", hidden.groupName());
|
||||
try testing.expectEqualStrings("steven black", hidden.sourceName());
|
||||
try testing.expectEqualStrings("home.arpa", hidden.forwardZone());
|
||||
try testing.expectEqualStrings("https://dns.example/dns-query", hidden.upstream());
|
||||
}
|
||||
|
||||
test "hide_client_ips hides the client and nothing else" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var buf: [4]Entry = undefined;
|
||||
var logger: Logger = .init(.{ .hide_client_ips = true }, &buf);
|
||||
logger.log(io, .init(fullFields(1)));
|
||||
const hidden = try logger.queue.getOne(io);
|
||||
|
||||
try testing.expectEqualStrings(hidden_marker, hidden.clientIp());
|
||||
try testing.expectEqualStrings("ads.example.com", hidden.domain());
|
||||
try testing.expectEqualStrings("*.ads.example", hidden.matched());
|
||||
try testing.expectEqualStrings("tracker.cdn.example", hidden.cnameTarget());
|
||||
try testing.expectEqualStrings("forcesafesearch.google.com", hidden.safeSearchTarget());
|
||||
try testing.expectEqualStrings("kids", hidden.groupName());
|
||||
try testing.expectEqualStrings("steven black", hidden.sourceName());
|
||||
try testing.expectEqualStrings("home.arpa", hidden.forwardZone());
|
||||
}
|
||||
|
||||
test "hide_domains writes no marker into a field the query never had" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var buf: [4]Entry = undefined;
|
||||
var logger: Logger = .init(.{ .hide_domains = true }, &buf);
|
||||
// An ordinary allowed query: no rule matched, no CNAME was uncloaked, no
|
||||
// safe-search rewrite happened. Marking those "hidden" would claim the
|
||||
// query had values it did not.
|
||||
logger.log(io, sampleEntry(1, "plain.example"));
|
||||
const hidden = try logger.queue.getOne(io);
|
||||
|
||||
try testing.expectEqualStrings(hidden_marker, hidden.domain());
|
||||
try testing.expectEqualStrings("", hidden.matched());
|
||||
try testing.expectEqualStrings("", hidden.cnameTarget());
|
||||
try testing.expectEqualStrings("", hidden.safeSearchTarget());
|
||||
}
|
||||
|
||||
test "the entry queue's worst case stays inside its byte budget" {
|
||||
// The bound `config/validate.zig` enforces is derived from this, so the
|
||||
// budget is what a maximal configuration can actually cost.
|
||||
try testing.expect(@as(usize, query_log_buffer_max) * @sizeOf(Entry) <= queue_budget_bytes);
|
||||
// One more entry than the ceiling would exceed it, so the ceiling is the
|
||||
// largest value that fits rather than a round number under it.
|
||||
try testing.expect((@as(usize, query_log_buffer_max) + 1) * @sizeOf(Entry) > queue_budget_bytes);
|
||||
|
||||
// The shipped default has to be comfortably inside the budget, or the
|
||||
// out-of-the-box configuration is the one that spends it. At the widths
|
||||
// above it costs about 17 MiB, roughly a quarter of the ceiling.
|
||||
const default_max: usize = (model.Logging{}).query_log_buffer_max;
|
||||
try testing.expect(default_max <= query_log_buffer_max);
|
||||
try testing.expect(default_max * @sizeOf(Entry) <= queue_budget_bytes / 2);
|
||||
}
|
||||
|
||||
test "log hides only the field its switch names" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
|
||||
Reference in New Issue
Block a user