Compare commits
9
Commits
b340521716
...
v0.0.3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
efbe355070
|
||
|
|
fc60214b3e
|
||
|
|
3c794b645b
|
||
|
|
c428bc2398
|
||
|
|
c7c1e21267
|
||
|
|
21c5ce1f36
|
||
|
|
1bce81eea0
|
||
|
|
21571e448e
|
||
|
|
2ab7c1f1de
|
@@ -9,6 +9,12 @@ on:
|
||||
branches: [master]
|
||||
pull_request:
|
||||
branches: [master]
|
||||
# A runner pod replaced mid-job kills the job it is running: the step's log
|
||||
# stops dead, no epilogue is written, and Gitea marks the orphan failed.
|
||||
# Nothing in the tree is wrong when that happens and nothing in the tree can
|
||||
# fix it, so the gate set must be re-runnable without an empty commit. This
|
||||
# adds a trigger, not a check — the job list below stays the gate set alone.
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
gates:
|
||||
|
||||
+75
-184
@@ -48,8 +48,9 @@ env:
|
||||
# version to exist in the tag and in build.zig.zon and nowhere else, and
|
||||
# ruling 5 makes verify-dist fail when the version under build disagrees with
|
||||
# build.zig.zon. An invented CI string such as "0.0.0-ci" therefore cannot
|
||||
# pass this file's own packaging gate. The package and container jobs read
|
||||
# the version out of build.zig.zon instead.
|
||||
# pass this file's own packaging gate. The package job reads the version out
|
||||
# of build.zig.zon instead, and the container job takes it from that job's
|
||||
# output.
|
||||
|
||||
jobs:
|
||||
test:
|
||||
@@ -179,6 +180,11 @@ jobs:
|
||||
needs: [frontend]
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
# The container job needs the version and installs no Zig, so it cannot read
|
||||
# build.zig.zon the way this job does.
|
||||
outputs:
|
||||
version: ${{ steps.zon-version.outputs.version }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
|
||||
@@ -215,12 +221,17 @@ jobs:
|
||||
# Ruling 2: build.zig.zon is the only place besides the tag that carries
|
||||
# the version, and ruling 5 makes verify-dist assert the two agree. The
|
||||
# gate builds the version the repository declares.
|
||||
#
|
||||
# This is the only job that reads it. The container job used to run its own
|
||||
# `sed` over the same file; it now receives this step's output, so the two
|
||||
# jobs cannot disagree about what the repository declares. The parse itself
|
||||
# matches verify-dist's, through the zon grammar rather than a regex.
|
||||
- name: Build the container gate tool
|
||||
run: zig build container-check-tool
|
||||
|
||||
- name: Read the version from build.zig.zon
|
||||
run: |
|
||||
set -euo pipefail
|
||||
version=$(sed -n 's/^[[:space:]]*\.version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' build.zig.zon | head -1)
|
||||
test -n "$version"
|
||||
echo "CI_VERSION=$version" >> "$GITHUB_ENV"
|
||||
id: zon-version
|
||||
run: ./zig-out/bin/container_check version
|
||||
|
||||
- name: Build the release artifacts
|
||||
run: |
|
||||
@@ -257,7 +268,13 @@ jobs:
|
||||
# The zip round-trip drops the executable bit. That is survivable only
|
||||
# because the Dockerfile chmods the binary itself and the contents
|
||||
# assertion compares sha256 of file contents, never modes. The archive
|
||||
# modes are asserted by verify-dist, above, on the originals.
|
||||
# modes are asserted by verify-dist, above, on the originals. The gate tool
|
||||
# rides along in the same artifact and the container job chmods it back.
|
||||
#
|
||||
# Adding zig-out/bin/container_check moves the artifact's common root from
|
||||
# zig-out/dist up to zig-out, which is why the container job restores into
|
||||
# zig-out rather than zig-out/dist. The Dockerfile's COPY paths still
|
||||
# resolve; they are relative to the repository root either way.
|
||||
- name: Upload the staged payload for the container job
|
||||
uses: actions/upload-artifact@c24449f33cd45d4826c6702db7e49f7cdb9b551d # v3.2.1-node20
|
||||
with:
|
||||
@@ -265,201 +282,69 @@ jobs:
|
||||
path: |
|
||||
zig-out/dist/bin
|
||||
zig-out/dist/stage
|
||||
zig-out/bin/container_check
|
||||
if-no-files-found: error
|
||||
|
||||
container:
|
||||
needs: [package]
|
||||
runs-on: ubuntu-24.04
|
||||
|
||||
env:
|
||||
# Ruling 2 and ruling 5: one parse of build.zig.zon per run, done in the
|
||||
# package job. This job installs no Zig and cannot repeat it.
|
||||
CI_VERSION: ${{ needs.package.outputs.version }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
|
||||
# Restored under zig-out/dist because that is where the Dockerfile's
|
||||
# COPY lines look, with the repository root as the build context. This
|
||||
# job compiles nothing and bundles nothing: the payload is the one the
|
||||
# package job already built and verify-dist already checked, which is
|
||||
# also the point — an image built from a second, independent `dist` run
|
||||
# would prove nothing about the artifacts the release publishes.
|
||||
# Restored under zig-out because that is where the Dockerfile's COPY lines
|
||||
# look for dist/, with the repository root as the build context, and
|
||||
# because the payload also carries bin/container_check. This job compiles
|
||||
# nothing and bundles nothing: the payload is the one the package job
|
||||
# already built and verify-dist already checked, which is also the point —
|
||||
# an image built from a second, independent `dist` run would prove nothing
|
||||
# about the artifacts the release publishes.
|
||||
- name: Download the staged payload built by the package job
|
||||
uses: actions/download-artifact@ad191675b41f6a5b46da9a048cb6893812da158b # v3.1.0-node20
|
||||
with:
|
||||
name: dist-payload
|
||||
path: zig-out/dist
|
||||
path: zig-out
|
||||
|
||||
# Same single source of truth as the package job (rulings 2 and 5). This
|
||||
# job still needs the version for the stage directory name it hashes
|
||||
# against the image and for the VERSION build arg.
|
||||
- name: Read the version from build.zig.zon
|
||||
run: |
|
||||
set -euo pipefail
|
||||
version=$(sed -n 's/^[[:space:]]*\.version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' build.zig.zon | head -1)
|
||||
test -n "$version"
|
||||
echo "CI_VERSION=$version" >> "$GITHUB_ENV"
|
||||
# The artifact zip carries no modes.
|
||||
- name: Restore the gate tool's executable bit
|
||||
run: chmod +x zig-out/bin/container_check
|
||||
|
||||
# Image tags, container names and published host ports are all
|
||||
# daemon-global. This workflow is called by both ci.yml and release.yml
|
||||
# and the self-hosted runners share one docker daemon, so the fixed
|
||||
# `nxdns:ci` and `nxdns-smoke` made two concurrent runs collide: the
|
||||
# second `docker create --name` fails outright, and worse, whichever run
|
||||
# finishes first deletes the other's container mid-test. The names below
|
||||
# carry the run identity plus four random bytes — the run id alone is not
|
||||
# enough, because two *repositories* on the same daemon can issue the
|
||||
# same one. The published port becomes ephemeral for the same reason.
|
||||
- name: Name this run's docker objects
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rand=$(head -c 4 /dev/urandom | od -An -tx1 | tr -d ' \n')
|
||||
suffix="${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}-$rand"
|
||||
{
|
||||
echo "CI_IMAGE=nxdns:ci-$suffix"
|
||||
echo "SMOKE_NAME=nxdns-smoke-$suffix"
|
||||
} >> "$GITHUB_ENV"
|
||||
echo "image nxdns:ci-$suffix, container nxdns-smoke-$suffix"
|
||||
# Everything this gate decides — this run's docker object names, the
|
||||
# ownership label, the image build, the contents assertion against the
|
||||
# staged payload, and the smoke test with its topology detection and its
|
||||
# absolute probe deadline — lives in tools/container_check.zig, along with
|
||||
# the rationale comments that used to sit in this file. That is the same
|
||||
# move release.yml made (milestone-14 deviation 24): logic in a `run:`
|
||||
# block cannot be type-checked, run on a laptop, or covered by a test.
|
||||
- name: Run the container acceptance gate
|
||||
run: ./zig-out/bin/container_check gate
|
||||
|
||||
# The build args carry the OCI label values (ruling 6); release.yml
|
||||
# passes the same three and then asserts the resulting
|
||||
# org.opencontainers.image.version label. BuildKit is not optional here:
|
||||
# the builder stage is pinned to $BUILDPLATFORM, which the classic
|
||||
# builder does not define, so DOCKER_BUILDKIT=0 fails at the first FROM.
|
||||
- name: Build the image
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker build -t "$CI_IMAGE" -f deploy/docker/Dockerfile \
|
||||
--build-arg VERSION="$CI_VERSION" \
|
||||
--build-arg REVISION="$GITHUB_SHA" \
|
||||
--build-arg CREATED="1970-01-01T00:00:00Z" \
|
||||
.
|
||||
|
||||
# Ruling 6: the binary in the image must be the binary in the tarball.
|
||||
# Ruling 3: distributing the image is distribution, so /LICENSE and
|
||||
# /THIRD-PARTY-NOTICES must be in it and must be the same files the
|
||||
# tarball carries — that is an acceptance criterion and nothing checked
|
||||
# it. Comparing against the staged payload rather than merely asserting
|
||||
# the paths exist costs nothing and catches a stale or empty copy.
|
||||
# The gate removes its own container and image on every exit path it
|
||||
# survives. This step is the exit path for a cancelled job and for a step
|
||||
# that died before the gate's own defers ran — on a long-lived self-hosted
|
||||
# daemon those accumulate one layer set per run. The name pair only exists
|
||||
# if the gate got as far as writing $GITHUB_ENV, so the label sweep covers
|
||||
# the rest of this run.
|
||||
#
|
||||
# Native triple only: this job builds a single-architecture image.
|
||||
# release.yml covers both platforms against the pushed multi-arch index.
|
||||
- name: Assert the image contents match the packaged artifacts
|
||||
run: |
|
||||
set -euo pipefail
|
||||
stage="zig-out/dist/stage/nxdns-$CI_VERSION-x86_64-linux-musl"
|
||||
test -d "$stage"
|
||||
|
||||
out=$(mktemp -d)
|
||||
cid=$(docker create "$CI_IMAGE")
|
||||
trap 'docker rm -f "$cid" >/dev/null 2>&1 || true; rm -rf "$out"' EXIT
|
||||
|
||||
rc=0
|
||||
for member in nxdns LICENSE THIRD-PARTY-NOTICES; do
|
||||
docker cp "$cid:/$member" "$out/$member"
|
||||
want=$(sha256sum "$stage/$member" | cut -d' ' -f1)
|
||||
got=$(sha256sum "$out/$member" | cut -d' ' -f1)
|
||||
if [ "$want" = "$got" ]; then
|
||||
echo "/$member matches ($got)"
|
||||
else
|
||||
echo "/$member DIFFERS: image $got, packaged $want"
|
||||
rc=1
|
||||
fi
|
||||
done
|
||||
test "$rc" -eq 0
|
||||
|
||||
- name: Smoke test the container
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
docker run --rm "$CI_IMAGE" version
|
||||
|
||||
mkdir -p etc-nxdns
|
||||
cat > etc-nxdns/config.zon <<'EOF'
|
||||
.{
|
||||
.groups = .{ .{ .name = "default" } },
|
||||
.upstreams = .{ .{ .url = "https://cloudflare-dns.com/dns-query" } },
|
||||
}
|
||||
EOF
|
||||
|
||||
# No bind mount: the runner talks to the daemon over a mounted
|
||||
# socket, so a -v path would resolve on the docker host (where the
|
||||
# workspace does not exist) and mount an empty directory over
|
||||
# /etc/nxdns. docker cp streams the file through the socket instead.
|
||||
#
|
||||
# Networking: this job itself runs in a container on the runner's
|
||||
# per-job network. A published port binds on the daemon's host, not
|
||||
# here, and docker does not route between the default bridge and
|
||||
# that network — a bridge-IP curl hangs to its connect timeout. So
|
||||
# the smoke container joins the job's own network, where its name
|
||||
# resolves and its port is reachable. On a host runner the inspect
|
||||
# finds no container and the published-port path covers it.
|
||||
#
|
||||
# `-p 127.0.0.1::8080` takes an ephemeral host port instead of a
|
||||
# fixed 18080, which two concurrent runs on this daemon cannot both
|
||||
# bind. The actual port is read back with `docker port`.
|
||||
#
|
||||
# The command and the sysctl mirror deploy/docker/compose.yaml,
|
||||
# because that is the invocation this gate exists to prove. The
|
||||
# invocation is the sole configuration authority (milestone-20 ruling
|
||||
# 1): the image's bare `run` grades the database, and a fresh
|
||||
# /var/lib/nxdns volume holds no upstream, so it exits 2 with
|
||||
# NoUsableUpstreams before it ever binds a port.
|
||||
net=$(docker inspect "$(hostname)" \
|
||||
-f '{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}' \
|
||||
2>/dev/null || true)
|
||||
cid=$(docker create --name "$SMOKE_NAME" \
|
||||
${net:+--network "$net"} \
|
||||
-p 127.0.0.1::8080 \
|
||||
--sysctl net.ipv4.ip_unprivileged_port_start=0 \
|
||||
"$CI_IMAGE" run --config=/etc/nxdns/config.zon)
|
||||
trap 'docker rm -f "$SMOKE_NAME" >/dev/null 2>&1 || true' EXIT
|
||||
docker cp etc-nxdns/config.zon "$SMOKE_NAME:/etc/nxdns/config.zon"
|
||||
docker start "$SMOKE_NAME"
|
||||
|
||||
# Before anything that assumes a live container. `docker port` fails
|
||||
# on one that already exited, and under `set -e` that failure is the
|
||||
# whole diagnosis the log gets — the container's own stderr never
|
||||
# reaches CI.
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$cid")" != "true" ]; then
|
||||
echo "container exited during startup"
|
||||
docker logs "$cid" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
hostport=$(docker port "$SMOKE_NAME" 8080/tcp | head -1 | awk -F: '{ print $NF }')
|
||||
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$cid")
|
||||
echo "published host port: ${hostport:-none}, container ip: ${ip:-none}"
|
||||
healthy=""
|
||||
for _ in $(seq 1 30); do
|
||||
if [ "$(docker inspect -f '{{.State.Running}}' "$cid")" != "true" ]; then
|
||||
echo "container exited during startup"
|
||||
docker logs "$cid" || true
|
||||
exit 1
|
||||
fi
|
||||
if curl -fsS --connect-timeout 2 "http://$SMOKE_NAME:8080/api/health" \
|
||||
|| { [ -n "$hostport" ] && curl -fsS --connect-timeout 2 "http://127.0.0.1:$hostport/api/health"; } \
|
||||
|| { [ -n "$ip" ] && curl -fsS --connect-timeout 2 "http://$ip:8080/api/health"; }; then
|
||||
healthy=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ -z "$healthy" ]; then
|
||||
echo "no /api/health response within 30 seconds"
|
||||
docker logs "$cid" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker stop -t 30 "$SMOKE_NAME"
|
||||
exit_code=$(docker inspect -f '{{.State.ExitCode}}' "$SMOKE_NAME")
|
||||
echo "exit code after SIGTERM: $exit_code"
|
||||
docker logs "$SMOKE_NAME" || true
|
||||
test "$exit_code" -eq 0
|
||||
|
||||
# The per-step traps only cover the step that set them. This is the exit
|
||||
# path for a cancelled job, a step that died before its trap was
|
||||
# installed, and the image itself, which no trap ever removed — on a
|
||||
# long-lived self-hosted daemon those accumulate one layer set per run.
|
||||
# It covers no more than that. `always()` does not run when the runner or
|
||||
# the pod itself dies, and the filter below names THIS attempt's label
|
||||
# value exactly — deliberately, since a concurrent run of another
|
||||
# repository must not be swept, but that also means a later attempt cannot
|
||||
# collect an earlier one's leak. What the label buys for those cases is
|
||||
# discovery, not recovery: `docker ps -a --filter
|
||||
# label=net.mial.nxdns.ci` and the matching `docker images` list every
|
||||
# object this workflow has ever left behind, with the repository, run and
|
||||
# attempt that owns each one. Reclaiming them is a manual sweep today, and
|
||||
# the hook a janitor job would use later.
|
||||
- name: Remove this run's docker objects
|
||||
if: always()
|
||||
env:
|
||||
CI_LABEL: net.mial.nxdns.ci=${{ github.repository }}/${{ github.run_id }}/${{ github.run_attempt }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -n "${SMOKE_NAME:-}" ]; then
|
||||
@@ -468,4 +353,10 @@ jobs:
|
||||
if [ -n "${CI_IMAGE:-}" ]; then
|
||||
docker image rm -f "$CI_IMAGE" >/dev/null 2>&1 || true
|
||||
fi
|
||||
for cid in $(docker ps -aq --filter "label=$CI_LABEL"); do
|
||||
docker rm -f "$cid" >/dev/null 2>&1 || true
|
||||
done
|
||||
for iid in $(docker images -q --filter "label=$CI_LABEL"); do
|
||||
docker image rm -f "$iid" >/dev/null 2>&1 || true
|
||||
done
|
||||
exit 0
|
||||
|
||||
@@ -10,6 +10,55 @@ subject rarely does.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.0.3] - 2026-08-15
|
||||
|
||||
Devices name themselves: the clients table asks the router over reverse DNS
|
||||
instead of waiting for the operator to type every name. The CI container gate
|
||||
also moved from workflow shell into a compiled, tested tool, which fixed a
|
||||
latent temp-directory bug shared with the release tool.
|
||||
|
||||
### Added
|
||||
|
||||
- **Client names learned over reverse DNS.** A client row that carries no
|
||||
hand-typed name gets one from the network: each tracker flush pass takes up
|
||||
to 16 unnamed rows, builds each address's reverse name, matches it against
|
||||
the declared `forward_zones`, and on a match sends one PTR query to that
|
||||
zone's resolver, storing the answer as a *learned* name. This requires a
|
||||
conditional forward zone covering the LAN's reverse space — for example
|
||||
`168.192.in-addr.arpa` pointed at the router; without one, nothing is sent
|
||||
anywhere. A hand-typed name always wins, learned names never appear in
|
||||
`nxdns export` and are never set by `nxdns import`, and each row refreshes
|
||||
once a day (an hour after a failure), so a rename can show stale for up to
|
||||
24 hours. The API's `Client` object gains a `learned_name` field and the
|
||||
clients page shows it.
|
||||
|
||||
### Changed
|
||||
|
||||
- The container CI gate — image build, image-contents assertion against the
|
||||
packaged artifacts, and the startup/shutdown smoke test — moved from
|
||||
workflow shell into `tools/container_check.zig`, compiled and unit-tested by
|
||||
`zig build test` and runnable on a laptop against a local docker daemon.
|
||||
The health probe now runs under a real 60-second deadline (the shell loop's
|
||||
"30 seconds" could stretch past three minutes), and the gate's docker
|
||||
objects carry an ownership label so anything a dead runner leaks is
|
||||
discoverable. The version in CI is parsed from `build.zig.zon` through the
|
||||
zon grammar, once, instead of by two copies of a `sed` regex.
|
||||
|
||||
### Fixed
|
||||
|
||||
- The release tool's temporary-directory claim was not exclusive: the
|
||||
"create" it relied on succeeds on a directory that already exists, so a
|
||||
stale or concurrent directory could be silently adopted, written into, and
|
||||
deleted on exit. Both the release tool and the new container gate now claim
|
||||
their directories exclusively and retry on collision.
|
||||
|
||||
## [0.0.2] - 2026-08-14
|
||||
|
||||
Configuration can now be a file that every boot converges to, filtering gains
|
||||
regex rules and honors blocklist exception lines, and two refresh bugs that
|
||||
silently kept stale state are fixed. Note the three breaking changes below if
|
||||
you script against `nxdns import` or run with `--config`.
|
||||
|
||||
### Added
|
||||
|
||||
- **Declarative configuration for IaC.** `nxdns run --config=<file>` makes the
|
||||
@@ -24,6 +73,26 @@ subject rarely does.
|
||||
configuration rows, names the tables and counts, and applies it only with
|
||||
the new `--allow-delete` flag. Additive and edit-in-place imports need no
|
||||
flag.
|
||||
- **Regex rules.** Rules gain a third kind, `regex`, beside `exact` and
|
||||
`wildcard`, for per-group allow and block patterns such as `^ad[0-9]+-`. The
|
||||
engine is homegrown and linear-time by construction, so no pattern can make
|
||||
matching blow up; backreferences and lookaround do not exist, and a bad
|
||||
pattern is refused at insert time with the limit it hit. Matches appear in
|
||||
`/api/lookup` and the query log as `rule_allow_regex` / `rule_block_regex`.
|
||||
Regex still comes only from you: regex lines in downloaded lists stay
|
||||
counted and skipped.
|
||||
- **Blocklist exception lines are honored.** An Adblock-Plus `@@||name^` line
|
||||
in a downloaded list now lifts that name — and its subdomains — out of what
|
||||
the attached lists block. Exceptions sit below every rule you wrote: a
|
||||
downloaded list can reopen only a hole another downloaded list dug, never
|
||||
override an operator decision. Each source reports how many it carried.
|
||||
- **Browser-only lines are counted where you can see them.** Every source now
|
||||
reports how many of its lines nxdns skipped as syntax with no DNS meaning —
|
||||
cosmetic filters, `$`-modifier rules — beside the existing skipped-regex
|
||||
count. Both blocklist tables show the number and the UI explains the
|
||||
difference: a list whose skipped-unsupported count dwarfs its domain count
|
||||
is written for browser extensions, and its DNS or hosts variant will block
|
||||
more. Previously such a list compiled to almost nothing and looked clean.
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -49,6 +118,29 @@ subject rarely does.
|
||||
- `nxdns import --force` is renamed `--allow-delete`.
|
||||
- A fresh install no longer seeds from `/etc/nxdns/config.zon` by presence.
|
||||
Use `nxdns import` once, or run in file mode with `--config`.
|
||||
- The admin UI's internals moved to TypeScript 7 and replaced Tailwind with
|
||||
StyleX and React Aria. The visible change is small: selects are real
|
||||
widgets with working keyboard focus; everything else renders as before.
|
||||
- The `config.db` schema is a single baseline definition again; numbered
|
||||
migration steps start accumulating at v0.1.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A list switching a name between its exact and wildcard forms never took
|
||||
effect.** The compiled-list checksum hashed the exact and wildcard bodies as
|
||||
one unseparated byte stream, so a list carrying `a.example` and the same
|
||||
list carrying `*.a.example` produced the same digest, and the refresh kept
|
||||
the old compiled files. The checksum now separates the bodies. Every source
|
||||
recompiles once on its first refresh after the upgrade; no re-download of
|
||||
unchanged content is forced beyond the refresh's normal fetch.
|
||||
- **A refresh could store stale skip counts.** When a refresh found the list
|
||||
content unchanged, it wrote the previously stored skip counters back to the
|
||||
database while showing the fresh ones in the UI, and the next restart
|
||||
reverted the numbers to the stale copy. All counters now persist from the
|
||||
fresh compile.
|
||||
- An Adblock-Plus entry with embedded whitespace
|
||||
(`||good.example bad.example^`) compiled into an entry no query could ever
|
||||
match. Such lines are now counted as unsupported instead.
|
||||
|
||||
## [0.0.1] - 2026-08-09
|
||||
|
||||
|
||||
+6
-3
@@ -61,9 +61,12 @@ chown root:nxdns /etc/nxdns/config.zon
|
||||
chmod 0640 /etc/nxdns/config.zon
|
||||
```
|
||||
|
||||
0640 with group `nxdns` rather than 0600: the service runs as `nxdns` and has to
|
||||
read this file on the first start, and systemd leaves `/etc/nxdns` owned by
|
||||
root.
|
||||
0640 with group `nxdns` rather than 0600: the service runs as `nxdns`, and
|
||||
systemd leaves `/etc/nxdns` owned by root. Keep that group read bit for good.
|
||||
Under `run --config` the service reads this file on **every** start, not once,
|
||||
so tightening the mode after the first boot breaks the next restart. Under
|
||||
database authority it is `nxdns import` that reads the file, as whoever runs
|
||||
that command, and a bare `nxdns run` never reads it at all.
|
||||
|
||||
Check it before starting the service:
|
||||
|
||||
|
||||
@@ -23,17 +23,19 @@ Serves a household LAN (≈2–20 devices). Portfolio-grade public repo with ext
|
||||
- DNS server for LAN clients: UDP/53, TCP/53, DoH server, DoT server.
|
||||
- Upstream resolution: DoH (HTTP/1.1), DoT.
|
||||
- Local DNS records (A/AAAA/CNAME) + conditional forwarding (zone → designated resolver, plain UDP/TCP allowed).
|
||||
- Domain filtering: blocklists (hosts/domains/ABP), custom rules (allow/block; exact, parent-walk, wildcard), CNAME uncloaking (depth 8), per-group safe-search rewrite.
|
||||
- Domain filtering: blocklists (hosts/domains/ABP, including `@@||name^` exception lines), custom rules (allow/block; exact, parent-walk, wildcard, regex), CNAME uncloaking (depth 8), per-group safe-search rewrite.
|
||||
- DNS caching: positive + negative, in-memory only.
|
||||
- Client/group model: IPv4 + IPv6 parity, per-client group assignment, per-group source assignments.
|
||||
- Query logging + analytics: async batched writes to SQLite (WAL), retention cleanup, dashboard + time buckets, live SSE stream.
|
||||
- Web app + REST API: LAN/Tailscale admin UI, optional password auth, OpenAPI schema + CI contract tests.
|
||||
- Observability: upstream health API + UI, disk monitor with UI banner, bounded log rotation, Prometheus `/metrics`.
|
||||
- Ops: DB-as-truth config, `nxdns export`/`import` (ZON), scheduled + manual blocklist updates, TLS cert watcher + reload, auto-migration on upgrade, systemd service + Dockerfile + compose.
|
||||
- Ops: config authority chosen by the invocation (database, or a file named by `--config`), `nxdns export`/`import` (ZON), scheduled + manual blocklist updates, TLS cert watcher + reload, auto-migration on upgrade, systemd service + Dockerfile + compose.
|
||||
|
||||
### 2.2 Out of Scope (permanent scope decisions, not deferrals)
|
||||
|
||||
- Regex rules. Wildcards + parent-walk cover the real use cases; regex on the DNS hot path means ReDoS exposure plus an immature dependency or a homegrown engine. Regex lines in blocklists are counted, skipped, and the skip count is surfaced in the UI.
|
||||
- Regex from downloaded blocklists. A list is other people's code running on the household's DNS, and the engine exists for rules the operator wrote. Regex lines in blocklists stay counted and skipped, and the skip count stays surfaced in the UI. (Operator regex rules themselves are **in** scope as of milestone 21: the ReDoS objection that once ruled them out is answered by `filter/regex.zig`, a homegrown Pike VM whose running time is bounded by program length × name length by construction, with no dependency and no backtracking. It is reached only after every hash and wildcard level has missed, and only on a cache miss.)
|
||||
- ABP syntax beyond domain anchors and `@@||name^` exceptions. `$dnstype`, `$dnsrewrite`, `$client`, `$denyallow` and every browser modifier stay unsupported and counted; the one tolerated modifier is a `$important` suffix on an exception line, which changes nothing about where that exception lands.
|
||||
- Partial-segment wildcards (`ads*.example.com`) as a rule kind. The regex kind covers the need without a second globbing dialect.
|
||||
- DHCP server.
|
||||
- DNSSEC validation (DO bit passthrough only).
|
||||
- DoQ (QUIC), HTTP/2 upstream transport.
|
||||
@@ -74,8 +76,8 @@ Verified: 0.16.0 ships `std.crypto.tls.Client` only. There is no server-side TLS
|
||||
|
||||
### 3.5 Config Format + Truth Model (Decision F)
|
||||
|
||||
- **DB is truth. Config file format is ZON** (`std.zon` parse + stringify — typed parsing into config structs, exact round-trip, stdlib-maintained, comments supported). No TOML: a third-party parser plus a hand-written serializer is two failure surfaces in the correctness-critical bootstrap/round-trip path, bought for syntax familiarity.
|
||||
- First start: if DB empty and `/etc/nxdns/config.zon` exists, validate → seed DB. Subsequent starts ignore the file.
|
||||
- **The invocation picks truth (see the next point). Config file format is ZON** (`std.zon` parse + stringify — typed parsing into config structs, exact round-trip, stdlib-maintained, comments supported). No TOML: a third-party parser plus a hand-written serializer is two failure surfaces in the correctness-critical round-trip path, bought for syntax familiarity.
|
||||
- The invocation picks the authority, and nothing else does (m20): `nxdns run` makes the database the configuration, `nxdns run --config FILE` makes the file the sole source and refuses the API routes that would edit configuration. There is no seeding and no first-start special case.
|
||||
- `nxdns export [--out file.zon]` dumps DB state as canonical ZON. `nxdns import <file.zon>` validates + replaces DB contents (`--force` if the DB holds configuration; client rows materialised from traffic do not count, and survive the replacement. First-seen/last-seen are runtime state, not configuration: they follow the address, so an import never restamps a device the DB already knew). Export/import = backup + host migration, **not** upgrades (§3.7).
|
||||
- No file watcher, no auto-regeneration.
|
||||
|
||||
@@ -89,21 +91,21 @@ Two SQLite files with opposite write profiles, isolated from each other:
|
||||
|
||||
### 3.7 Upgrades: Auto-Migration (Decision J)
|
||||
|
||||
- `config.db`: numbered, sequential SQL migration steps compiled into the binary. At startup: read schema version row, apply newer steps inside a transaction, continue. Operator upgrade = install binary, restart.
|
||||
- `config.db`: numbered, sequential SQL migration steps compiled into the binary. At startup: read schema version row, apply newer steps inside a transaction, continue. Operator upgrade = install binary, restart. Before v0.1 the list holds one step — the baseline of §11.2, edited in place — because nxdns has no installs and a step exists only to reconcile a database somebody already has.
|
||||
- `querylog.db`: **no migrations.** On schema mismatch: rename aside, recreate fresh.
|
||||
|
||||
### 3.8 Blocklist Storage (Decision A)
|
||||
|
||||
Blocklist domains are **not** stored in SQLite — they are a cache of re-downloadable remote artifacts, not config or state:
|
||||
|
||||
- Each source compiles to `/var/lib/nxdns/blocklists/<source_id>.list`: normalized, one domain per line, small header (source URL, fetch time, count, checksum). Wildcard/regex-flavored lines: wildcards go to `<source_id>.wild`; regex lines are counted + skipped (count in metadata → UI).
|
||||
- Each source compiles to `/var/lib/nxdns/blocklists/<source_id>.list`: normalized, one domain per line, small header (source URL, fetch time, counts, checksum). Wildcard/regex/exception-flavored lines: wildcards go to `<source_id>.wild`, ABP exceptions (`@@||name^`) to `<source_id>.allow`; regex lines and browser-syntax lines nxdns cannot translate into a DNS decision are counted + skipped, both counts in metadata → UI. The checksum covers the three bodies in that order, each followed by a separator byte so that moving a name between bodies — an upstream switching `a.example` to `*.a.example` — changes the digest and forces a republish.
|
||||
- `config.db` keeps source **metadata only** (`blocklist_sources`).
|
||||
- Startup + post-update: parse files into the immutable in-memory matcher (RCU swap, §9.5).
|
||||
- Corruption recovery is per-file: checksum mismatch → re-download one list.
|
||||
|
||||
### 3.9 Rule Model (Decision B)
|
||||
|
||||
Rule kinds: `exact`, parent-walk (implicit via candidate chain), `wildcard` (`*` segment patterns, e.g. `*.doubleclick.net`, `ads.*.example.com`). Actions: `allow` | `block`. Kind is an explicit column — the model is extensible without breakage, but regex stays out of scope (§2.2).
|
||||
Rule kinds: `exact`, parent-walk (implicit via candidate chain), `wildcard` (`*` segment patterns, e.g. `*.doubleclick.net`, `ads.*.example.com`), `regex` (the linear-time engine of `filter/regex.zig`, matched unanchored against the whole normalized name). Actions: `allow` | `block`. Kind is an explicit column, so a fourth kind widens one `CHECK` and touches no other table. A regex pattern is stored exactly as written — it is not a name, so it is never lowercased or dot-stripped — and is compiled at both edges: `config/validate.zig` refuses a bad one with the limit it hit, and `filter/rules.zig` compiles it once per snapshot.
|
||||
|
||||
### 3.10 Filtering Precedence
|
||||
|
||||
@@ -111,11 +113,25 @@ Rule kinds: `exact`, parent-walk (implicit via candidate chain), `wildcard` (`*`
|
||||
2. Exact/parent **block** rules
|
||||
3. Wildcard allow rules
|
||||
4. Wildcard block rules
|
||||
5. Blocklist domains
|
||||
6. Blocklist wildcards
|
||||
5. Regex allow rules
|
||||
6. Regex block rules
|
||||
7. Blocklist exceptions (`@@||name^`)
|
||||
8. Blocklist domains
|
||||
9. Blocklist wildcards
|
||||
|
||||
Tie-break at same specificity: **allow wins**.
|
||||
|
||||
Each level is checked against the whole candidate chain before the next level is
|
||||
checked against any, which is what makes an allow rule on a parent beat a block
|
||||
rule on the child.
|
||||
|
||||
Two positions carry an argument rather than a preference. The regex levels come
|
||||
last among the operator rules because they are the only ones that are not a set
|
||||
lookup or a label walk: a regex runs only once every cheaper level has missed.
|
||||
Blocklist exceptions come below **every** operator level because a downloaded
|
||||
list may cancel what another list blocked and must never cancel what the
|
||||
operator decided — no list can open an allow hole the operator did not open.
|
||||
|
||||
### 3.11 Network Posture
|
||||
|
||||
- Default web bind: LAN/Tailscale-friendly (non-loopback allowed).
|
||||
@@ -128,9 +144,9 @@ IPv4 + IPv6 full parity for: client identity, rate limiting, logging, group assi
|
||||
|
||||
### 3.13 Filesystem Layout (FHS)
|
||||
|
||||
- `/etc/nxdns/config.zon` — bootstrap (first start only).
|
||||
- `/etc/nxdns/config.zon` — the declarative source, read only when `run --config` names it. A file no flag names changes nothing.
|
||||
- `/var/lib/nxdns/config.db`, `/var/lib/nxdns/querylog.db`
|
||||
- `/var/lib/nxdns/blocklists/*.list|*.wild` (plus `*.raw.tmp|*.list.tmp|*.wild.tmp` during a refresh)
|
||||
- `/var/lib/nxdns/blocklists/*.list|*.wild|*.allow` (plus `*.raw.tmp|*.list.tmp|*.wild.tmp|*.allow.tmp` during a refresh)
|
||||
- `/var/log/nxdns/nxdns.log` — only in file output mode; default is stderr → journald.
|
||||
|
||||
### 3.14 Frontend Stack (Decision I)
|
||||
@@ -173,7 +189,7 @@ Client DNS Query
|
||||
Response to client
|
||||
```
|
||||
|
||||
Cross-cutting: `ConfigManager` (bootstrap/import/export/settings), `BlocklistManager` (fetch/compile/swap), `Storage` (two SQLite DBs), `Cache`, `UpstreamHealth`, `DiskMonitor`, `Auth`, `Web API` (REST + SSE + metrics).
|
||||
Cross-cutting: `ConfigManager` (load/reconcile/import/export/settings), `BlocklistManager` (fetch/compile/swap), `Storage` (two SQLite DBs), `Cache`, `UpstreamHealth`, `DiskMonitor`, `Auth`, `Web API` (REST + SSE + metrics).
|
||||
|
||||
---
|
||||
|
||||
@@ -203,7 +219,8 @@ src/
|
||||
filter/ # pure
|
||||
matcher.zig rules.zig wildcard.zig
|
||||
parser_hosts.zig parser_domains.zig parser_abp.zig
|
||||
fetcher.zig compiler.zig # list download -> compiled .list/.wild files
|
||||
fetcher.zig compiler.zig # list download -> compiled .list/.wild/.allow files
|
||||
regex.zig # linear-time Pike VM for operator regex rules
|
||||
safesearch.zig
|
||||
|
||||
local/ # pure
|
||||
@@ -223,7 +240,8 @@ src/
|
||||
logger.zig retention.zig disk_monitor.zig
|
||||
|
||||
config/
|
||||
model.zig bootstrap.zig import.zig export.zig validate.zig # all ZON via std.zon
|
||||
model.zig loader.zig reconcile.zig import.zig export.zig # all ZON via std.zon
|
||||
validate.zig faults.zig
|
||||
|
||||
web/
|
||||
server.zig router.zig auth.zig sse.zig static.zig metrics.zig openapi.zig
|
||||
@@ -276,18 +294,22 @@ For `{domain, group_id}` (the qtype travels with the query for logging and respo
|
||||
for matching):
|
||||
1. Normalize: lowercase, trim trailing dot.
|
||||
2. Build candidate chain (full, parent1, parent2, …).
|
||||
3. Explicit rules per §3.10 precedence, evaluated against every candidate in the chain.
|
||||
4. Group's blocklist domains (hash set over compiled lists), matched against the query name only.
|
||||
5. Group's blocklist wildcards, matched against every proper parent of the query name.
|
||||
6. No match → allow.
|
||||
3. Explicit rules per §3.10 precedence, allow before block at each level: exact rules against every candidate in the chain, then wildcard patterns and then regex patterns against the whole name (both kinds express their own reach, so neither walks the chain).
|
||||
4. Group's blocklist exceptions (`@@` entries), against every candidate in the chain. They cancel a block a list made and never one a rule made.
|
||||
5. Group's blocklist domains (hash set over compiled lists), matched against the query name only.
|
||||
6. Group's blocklist wildcards, matched against every proper parent of the query name.
|
||||
7. No match → allow.
|
||||
|
||||
Blocklist entries do not parent-walk; only rules do (§3.9). ABP `||x.y^` emits both a domain entry
|
||||
`x.y` and a wildcard entry `x.y`, which together give domain-and-subdomains semantics.
|
||||
Blocklist *domain* entries do not parent-walk: they are matched against the query name alone. Wildcard
|
||||
entries match every proper parent, and exception entries walk the candidate chain the way rules do
|
||||
(§3.9), so `@@||good.ads.example^` also lifts `y.good.ads.example`. ABP `||x.y^` emits both a domain
|
||||
entry `x.y` and a wildcard entry `x.y`, which together give domain-and-subdomains semantics.
|
||||
|
||||
### 7.2 Group Assignment
|
||||
|
||||
- Per source IP: (1) exact match in `clients`, (2) longest-prefix match in `client_prefixes` (ties: longer prefix, then priority), (3) `default` group.
|
||||
- Auto-materialization: first query from an unseen IP inserts a `clients` row (`hand_edited=0`, `first_seen=now`) for UI visibility and stable group assignment. The query log does **not** FK to it (§3.6).
|
||||
- Materialized clients name themselves: the tracker's flush pass sends one PTR query per unnamed row through the declared forward zones (§6.5), and the answer is runtime state in `learned_name`, never configuration.
|
||||
- Retention drops `hand_edited=0` clients with no queries in `retention_days`.
|
||||
|
||||
### 7.3 Reload
|
||||
@@ -341,7 +363,16 @@ Per-group boolean. Rewrites known engine domains to their safe-search CNAME targ
|
||||
|
||||
`journal_mode=WAL`, `synchronous=NORMAL`, `foreign_keys=ON`, `busy_timeout` set.
|
||||
|
||||
### 11.2 config.db Schema
|
||||
### 11.2 config.db Schema (v1 baseline)
|
||||
|
||||
The DDL below is the live schema, kept byte-identical to
|
||||
`src/storage/config_schema.zig`. `src/storage/migrations.zig` carries it as its
|
||||
one and only step, so a database is at **version 1** or it does not exist.
|
||||
|
||||
Until nxdns reaches v0.1 this baseline is **editable**: a schema change edits
|
||||
this section and `config_schema.zig` together and adds no migration step. nxdns
|
||||
has no installs, so there is no database for a step to reconcile. At v0.1 the
|
||||
baseline freezes and every later change becomes an append-only step.
|
||||
|
||||
```sql
|
||||
CREATE TABLE schema_version (version INTEGER NOT NULL);
|
||||
@@ -357,6 +388,8 @@ CREATE TABLE clients (
|
||||
id INTEGER PRIMARY KEY,
|
||||
ip TEXT NOT NULL UNIQUE, -- canonical text form (v4 dotted / v6 RFC 5952)
|
||||
name TEXT,
|
||||
learned_name TEXT,
|
||||
name_attempt_after INTEGER NOT NULL DEFAULT 0,
|
||||
group_id INTEGER NOT NULL REFERENCES groups(id),
|
||||
hand_edited INTEGER NOT NULL DEFAULT 0,
|
||||
first_seen INTEGER NOT NULL,
|
||||
@@ -374,7 +407,8 @@ CREATE TABLE upstreams (
|
||||
id INTEGER PRIMARY KEY,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
priority INTEGER NOT NULL DEFAULT 100,
|
||||
enabled INTEGER NOT NULL DEFAULT 1
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
tls_name TEXT NOT NULL DEFAULT '' -- DoT verification name; empty verifies against the url host
|
||||
);
|
||||
|
||||
CREATE TABLE blocklist_sources (
|
||||
@@ -386,7 +420,9 @@ CREATE TABLE blocklist_sources (
|
||||
last_updated INTEGER,
|
||||
domain_count INTEGER NOT NULL DEFAULT 0,
|
||||
wildcard_count INTEGER NOT NULL DEFAULT 0,
|
||||
exception_count INTEGER NOT NULL DEFAULT 0,
|
||||
skipped_regex_count INTEGER NOT NULL DEFAULT 0,
|
||||
skipped_unsupported_count INTEGER NOT NULL DEFAULT 0,
|
||||
checksum TEXT
|
||||
);
|
||||
|
||||
@@ -400,7 +436,7 @@ CREATE TABLE rules (
|
||||
id INTEGER PRIMARY KEY,
|
||||
group_id INTEGER NOT NULL REFERENCES groups(id) ON DELETE CASCADE,
|
||||
pattern TEXT NOT NULL,
|
||||
kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard')),
|
||||
kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard','regex')),
|
||||
action TEXT NOT NULL CHECK(action IN ('allow','block')),
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
@@ -471,52 +507,24 @@ Periodic delete of rows older than `retention_days`; scheduled checkpoint/VACUUM
|
||||
|
||||
## 12. Configuration
|
||||
|
||||
### 12.1 Bootstrap ZON Shape
|
||||
### 12.1 Config ZON Shape
|
||||
|
||||
The canonical shape is not duplicated here. It lives in
|
||||
[docs/reference/configuration.md](docs/reference/configuration.md), which is
|
||||
handwritten against `config/model.zig` and only partly guarded (the drift test
|
||||
covers settings-key rows, not the whole shape, so a new collection can go
|
||||
undocumented while the guard stays green), and `nxdns export` emits it. A copy in
|
||||
this document is how §12.1 came to describe an `.upstream.servers` field that
|
||||
never existed and to omit the required `.groups` and `.upstreams` — a sample
|
||||
nobody could load. The skeleton, for orientation only:
|
||||
|
||||
```zon
|
||||
.{
|
||||
.upstream = .{
|
||||
.servers = .{ "https://cloudflare-dns.com/dns-query", "tls://dns.google:853" },
|
||||
.read_timeout_ms = 3000,
|
||||
.groups = .{ .{ .name = "default" } },
|
||||
.upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
|
||||
.rules = .{
|
||||
.{ .group = "default", .pattern = "^ad[0-9]+-", .kind = .regex, .action = .block },
|
||||
},
|
||||
.dns = .{
|
||||
.bind_ipv4 = "0.0.0.0",
|
||||
.bind_ipv6 = "::",
|
||||
.port = 53,
|
||||
.rate_limit = 1000,
|
||||
.rate_window_seconds = 60,
|
||||
},
|
||||
.blocking = .{ .response = .zero, .ttl = 5 }, // .zero | .nxdomain
|
||||
.cache = .{ .size = 10000, .negative_ttl_max = 3600 },
|
||||
.web = .{
|
||||
.enabled = true,
|
||||
.bind = "0.0.0.0",
|
||||
.port = 8080,
|
||||
.password = "", // empty => auth disabled
|
||||
.session_ttl_hours = 24,
|
||||
.api_rate_limit_per_min = 300,
|
||||
.sse_max_connections_per_ip = 3,
|
||||
},
|
||||
.doh_server = .{ .enabled = false, .bind = "0.0.0.0", .port = 443,
|
||||
.cert_path = "/etc/nxdns/cert.pem", .key_path = "/etc/nxdns/key.pem" },
|
||||
.dot_server = .{ .enabled = false, .bind = "0.0.0.0", .port = 853,
|
||||
.cert_path = "/etc/nxdns/cert.pem", .key_path = "/etc/nxdns/key.pem" },
|
||||
.edns = .{ .ecs_mode = .strip }, // .strip | .forward
|
||||
.local_records = .{ .{ .name = "nas.lan", .rtype = .A, .value = "192.168.1.10" } },
|
||||
.forward_zones = .{ .{ .zone = "lan.home", .resolver = "udp://192.168.1.1:53" } },
|
||||
.logging = .{
|
||||
.level = .info,
|
||||
.retention_days = 30,
|
||||
.query_log_buffer_max = 10000,
|
||||
.hide_domains = false,
|
||||
.hide_client_ips = false,
|
||||
.output = .stderr, // .stderr | .syslog | .file
|
||||
.file_path = "/var/log/nxdns/nxdns.log",
|
||||
.max_size_mb = 50,
|
||||
.max_files = 5,
|
||||
},
|
||||
.disk = .{ .min_free_mb = 200, .warn_free_mb = 500 },
|
||||
.blocklist_update = .{ .enabled = true, .interval_hours = 24 },
|
||||
}
|
||||
```
|
||||
|
||||
@@ -592,11 +600,11 @@ UDP server, TCP server, DoH + DoT upstream clients, pool + failover/backoff + he
|
||||
Exit: A/AAAA forwarding over UDP + TCP; health populated.
|
||||
|
||||
### Phase 4 — Storage + Config
|
||||
SQLite wrapper; config.db schema + migration runner; querylog.db schema + recreate-on-mismatch; repositories; ZON bootstrap + import/export; `nxdns check`.
|
||||
Exit: first start seeds DB from ZON; export → import round-trips byte-stable.
|
||||
SQLite wrapper; config.db schema + migration runner; querylog.db schema + recreate-on-mismatch; repositories; ZON loading + import/export; `nxdns check`.
|
||||
Exit: export → import round-trips byte-stable. (The ZON bootstrap this phase shipped was replaced in m20 by the two authority modes above.)
|
||||
|
||||
### Phase 5 — Filtering + Local DNS
|
||||
Rule matcher (exact/parent/wildcard); blocklist parsers → compiled file format; fetcher + scheduled update; RCU swap; per-group safe-search; local records; forward zones.
|
||||
Rule matcher (exact/parent/wildcard; `regex` added in m21); blocklist parsers → compiled file format; fetcher + scheduled update; RCU swap; per-group safe-search; local records; forward zones.
|
||||
Exit: precedence table validated by tests; local zone answers + conditional forwards work.
|
||||
|
||||
### Phase 6 — Cache + Rate Limit + Logging + Disk Monitor
|
||||
@@ -623,7 +631,7 @@ Exit: documented deployment works end-to-end on the Pi 5.
|
||||
|
||||
## 17. Testing Strategy
|
||||
|
||||
- **Unit**: DNS encode/decode; rule precedence + wildcard matcher; cache put/get/TTL rewrite; ZON bootstrap + export round-trip; rate limiter; migration runner (fresh + stepwise upgrade).
|
||||
- **Unit**: DNS encode/decode; rule precedence + wildcard matcher; cache put/get/TTL rewrite; ZON loading + export round-trip; rate limiter; migration runner (fresh + stepwise upgrade).
|
||||
- **Fuzz**: DNS parser malformed-packet fuzzing; blocklist parser fuzzing.
|
||||
- **Integration**: UDP/TCP query path; blocked path; allow-over-block; wildcard precedence; CNAME uncloaking block; local records + forward zones; upstream failover/backoff/health; disk-full degradation; querylog.db corruption recovery; API CRUD; auth on/off; SSE; contract tests.
|
||||
- **Manual**: `dig @pi example.com` / blocked domain / local record; DoH/DoT client checks; dashboard + live log.
|
||||
@@ -697,7 +705,7 @@ longer the only path.
|
||||
| # | Decision |
|
||||
|---|----------|
|
||||
| A | Blocklists compile to flat files under `/var/lib/nxdns/blocklists/`; DB stores source metadata only |
|
||||
| B | Rule kinds: exact, parent-walk, wildcard. Regex permanently out of scope |
|
||||
| B | Rule kinds: exact, parent-walk, wildcard, regex (m21, own linear-time engine). Regex *from downloaded lists* stays out of scope |
|
||||
| C | In scope: local DoH/DoT server, local records, conditional forwarding. Out: HTTP/2, DoQ, DHCP, DNSSEC, clustering |
|
||||
| D | mbedTLS (vendored) terminates server TLS; stdlib TLS for upstream client |
|
||||
| E | `std.Io` injected everywhere; `Threaded` backend (io_uring flag dropped in m11 — Evented networking is stubbed at 0.16.0); no custom thread pool |
|
||||
|
||||
@@ -17,8 +17,9 @@ UI, the REST API and `/metrics` read.
|
||||
|
||||
## Features
|
||||
|
||||
- Blocklist filtering: subscribe to hosts/domain lists, plus your own allow
|
||||
and block rules with wildcard support (`*.example.com`)
|
||||
- Blocklist filtering: subscribe to hosts, domain and Adblock Plus lists —
|
||||
whose `@@` exception lines are honoured — plus your own allow and block rules,
|
||||
exact, wildcard (`*.example.com`) or regular expression
|
||||
- Two configuration modes: a database the web UI edits, or a ZON file you keep
|
||||
in git and converge onto at every start
|
||||
- Per-client policy groups: different filtering for the kids' tablet and
|
||||
|
||||
@@ -157,6 +157,15 @@ pub fn build(b: *std.Build) void {
|
||||
.import_module = sourceModule(b, target, optimize, "src/web/http_util.zig"),
|
||||
});
|
||||
|
||||
// `src/filter/regex.zig` imports only std (milestone-21 ruling 5), so its
|
||||
// fuzz module roots directly at the file as well.
|
||||
addFuzzSuite(b, target, optimize, fuzz, test_step, .{
|
||||
.name = "regex-fuzz",
|
||||
.root = "tests/fuzz/regex_fuzz.zig",
|
||||
.import_name = "regex",
|
||||
.import_module = sourceModule(b, target, optimize, "src/filter/regex.zig"),
|
||||
});
|
||||
|
||||
// The bench harness (milestone-12 ruling 1). The measured roots
|
||||
// (matcher.zig, dns_cache.zig, compiler.zig) share files in their relative
|
||||
// import closures (model.zig, types.zig, ...), and a file may belong to
|
||||
@@ -239,6 +248,27 @@ pub fn build(b: *std.Build) void {
|
||||
});
|
||||
test_step.dependOn(&b.addRunArtifact(release_tests).step);
|
||||
|
||||
// The container acceptance gate. Installed rather than run from the build
|
||||
// graph for the same reason as the release tool: it needs a live docker
|
||||
// daemon and the workflow's environment, neither of which a Run step in this
|
||||
// graph can supply.
|
||||
const container_check_tool = hostTool(b, "container_check");
|
||||
b.step("container-check-tool", "Install the container gate tool into zig-out/bin")
|
||||
.dependOn(&b.addInstallArtifact(container_check_tool, .{}).step);
|
||||
|
||||
// Its pure decisions — object naming, the ownership label, the inspect-JSON
|
||||
// topology read, the probe deadline, the build.zig.zon version parse — are
|
||||
// what the shell it replaced could never be tested on.
|
||||
const container_check_tests = b.addTest(.{
|
||||
.name = "container-check-tool",
|
||||
.root_module = b.createModule(.{
|
||||
.root_source_file = b.path("tools/container_check.zig"),
|
||||
.target = b.graph.host,
|
||||
.optimize = optimize,
|
||||
}),
|
||||
});
|
||||
test_step.dependOn(&b.addRunArtifact(container_check_tests).step);
|
||||
|
||||
addDist(b, options, web_assets, .{
|
||||
.version = version_option,
|
||||
.version_string = version_string,
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
.{
|
||||
.name = .nxdns,
|
||||
.version = "0.0.1",
|
||||
.version = "0.0.3",
|
||||
.minimum_zig_version = "0.16.0",
|
||||
.paths = .{""},
|
||||
.fingerprint = 0x3307b311dded1d91,
|
||||
|
||||
@@ -29,7 +29,7 @@ Directories:
|
||||
| Directory | Role |
|
||||
|---|---|
|
||||
| `src/dns/` | Pure DNS wire format: header, names, questions, records, whole packets, EDNS(0)/ECS (`edns.zig`), enums and limits (`types.zig`). No allocation, no `std.Io` beyond writing to a caller's writer. |
|
||||
| `src/filter/` | Blocklist pipeline: line parsers (hosts, domains, ABP), the compiler that turns a downloaded list into `.list`/`.wild` bodies, `domain_set.zig` (exact-match set, no Bloom filter), `matcher.zig` (the immutable snapshot every query evaluates against), per-group `rules.zig`, `wildcard.zig`, `safesearch.zig`, blocked-response synthesis (`response.zig`). Two I/O edges live here too: `fetcher.zig` (HTTP download) and `manager.zig` (files + DB + snapshot swap). |
|
||||
| `src/filter/` | Blocklist pipeline: line parsers (hosts, domains, ABP), the compiler that turns a downloaded list into `.list`/`.wild`/`.allow` bodies, `domain_set.zig` (exact-match set, no Bloom filter), `matcher.zig` (the immutable snapshot every query evaluates against), per-group `rules.zig`, `wildcard.zig`, `regex.zig` (a Pike VM for the operator's regex rules, linear-time by construction), `safesearch.zig`, blocked-response synthesis (`response.zig`). Two I/O edges live here too: `fetcher.zig` (HTTP download) and `manager.zig` (files + DB + snapshot swap). |
|
||||
| `src/local/` | Local DNS records and conditional forward zones: immutable lookup tables built once from DB rows (`records.zig`, `forward_zones.zig`), plus the plain UDP/TCP client for LAN resolvers (`forward_client.zig`). |
|
||||
| `src/cache/` | `dns_cache.zig`: bounded in-memory TTL cache of whole response messages, keyed by the question. The clock arrives as a parameter. |
|
||||
| `src/upstream/` | Upstream resolution: shared vocabulary and the `Client` interface (`transport.zig`), DoH client (RFC 8484), DoT client (RFC 7858), per-endpoint health and backoff (`health.zig`), and `pool.zig` — priority-ordered failover that is itself a `transport.Client`, so the handler sees one interface. |
|
||||
|
||||
@@ -99,7 +99,7 @@ nxdns import /tmp/nxdns-lab/backup.zon --data-dir /tmp/nxdns-lab/data-restored
|
||||
```
|
||||
|
||||
```
|
||||
info(migrations): config.db migrated from schema version 0 to 2
|
||||
info(migrations): config.db migrated from schema version 0 to 1
|
||||
imported /tmp/nxdns-lab/backup.zon
|
||||
```
|
||||
|
||||
|
||||
@@ -211,7 +211,7 @@ why the container is `docker-nxdns-1`.
|
||||
A healthy first start logs the reconcile, the authority and the bound sockets:
|
||||
|
||||
```
|
||||
info(migrations): config.db migrated from schema version 0 to 2
|
||||
info(migrations): config.db migrated from schema version 0 to 1
|
||||
reconciled '/etc/nxdns/config.zon': upstreams +1 ~0 -0; settings +45 ~0 -0;
|
||||
settings keys changed: dns.bind_ipv4 dns.bind_ipv6 dns.port web.bind web.port …
|
||||
web authentication is now enabled
|
||||
|
||||
@@ -184,8 +184,13 @@ was written before the first start.
|
||||
|
||||
0640 with group `nxdns` rather than 0600: `/etc/nxdns` is a
|
||||
`ConfigurationDirectory`, which systemd leaves owned by root, and the service
|
||||
runs as `nxdns` and has to read this file on the first start. A root-owned 0600
|
||||
file would be unreadable to it.
|
||||
runs as `nxdns`. A root-owned 0600 file would be unreadable to it.
|
||||
|
||||
Keep that group read bit for good, not just for the first boot. Under
|
||||
`run --config` the service reads this file on **every** start, so tightening
|
||||
the mode later breaks the next restart. Under database authority it is
|
||||
`nxdns import` that reads the file, as whoever runs that command, and a bare
|
||||
`nxdns run` never reads it at all.
|
||||
|
||||
Do not expect `nxdns check` to catch a permissive mode here. Its only
|
||||
permission warning is for a TLS private key
|
||||
@@ -220,7 +225,7 @@ nxdns import /etc/nxdns/config.zon
|
||||
```
|
||||
|
||||
```
|
||||
info(migrations): config.db migrated from schema version 0 to 2
|
||||
info(migrations): config.db migrated from schema version 0 to 1
|
||||
imported /etc/nxdns/config.zon
|
||||
```
|
||||
|
||||
|
||||
@@ -41,8 +41,8 @@ zig build bench -Doptimize=ReleaseFast -- filter --domains=100000 --iters=20000
|
||||
nxdns bench suite=filter domains=100000 iters=20000 seed=0x5eed optimize=ReleaseFast
|
||||
|
||||
suite ops p50(us) p95(us) p99(us) max(us)
|
||||
filter 20000 0.14 0.25 0.27 0.51
|
||||
blocked 6670/20000, Snapshot.memoryBytes 3.0 MiB, VmRSS 5.4 MiB
|
||||
filter 20000 2.38 2.76 2.88 20.32
|
||||
blocked 6670/20000, 32 regex rules, Snapshot.memoryBytes 3.0 MiB, VmRSS 5.6 MiB
|
||||
target p95 < 1ms: PASS
|
||||
target VmRSS < 100 MiB: PASS
|
||||
```
|
||||
@@ -60,8 +60,8 @@ zig build bench -Doptimize=ReleaseFast -- cache --iters=20000
|
||||
|
||||
```
|
||||
suite ops p50(us) p95(us) p99(us) max(us)
|
||||
cache 20000 0.12 0.21 0.23 0.54
|
||||
hits 10000/20000, DnsCache.memoryBytes 4.3 MiB, VmRSS 6.2 MiB
|
||||
cache 20000 0.14 0.18 0.20 5.14
|
||||
hits 10000/20000, DnsCache.memoryBytes 4.3 MiB, VmRSS 6.3 MiB
|
||||
target p95 < 5ms: PASS
|
||||
```
|
||||
|
||||
@@ -71,7 +71,7 @@ zig build bench -Doptimize=ReleaseFast -- compile --domains=100000
|
||||
|
||||
```
|
||||
suite ops p50(us) p95(us) p99(us) max(us)
|
||||
compile 100000 wall 11.512ms, 8686215 lines/s, 100000 domains kept (informational)
|
||||
compile 100000 wall 15.623ms, 6400464 lines/s, 100000 domains kept (informational)
|
||||
```
|
||||
|
||||
`--seed=N` changes the generated domains and the query order; the default is
|
||||
@@ -94,6 +94,9 @@ usage: zig build bench -Doptimize=ReleaseFast -- [filter|cache|compile|all] [--d
|
||||
is building the key, getting the entry and stamping the response id.
|
||||
- `blocked N/M` and `hits N/M` are sanity counters. The harness aborts if either
|
||||
is zero — a suite that never hits its own path measures nothing.
|
||||
- `32 regex rules` on the `filter` line is the rule set the suite loads. No
|
||||
generated query matches any of them, so every operation runs all 32 programs
|
||||
to their end, which is the costly case and the one worth measuring.
|
||||
- Two memory figures appear on purpose. `Snapshot.memoryBytes` and
|
||||
`DnsCache.memoryBytes` are the in-repo accounting of those structures; `VmRSS`
|
||||
is what the kernel holds resident for the whole process, allocator slack and
|
||||
@@ -121,8 +124,8 @@ zig build bench -Doptimize=ReleaseFast -- filter --domains=100000 --iters=20000
|
||||
```
|
||||
|
||||
```
|
||||
filter 20000 0.14 0.26 0.27 2.42
|
||||
blocked 6670/20000, Snapshot.memoryBytes 3.0 MiB, VmRSS 5.4 MiB
|
||||
filter 20000 2.34 2.71 2.85 15.06
|
||||
blocked 6670/20000, 32 regex rules, Snapshot.memoryBytes 3.0 MiB, VmRSS 5.6 MiB
|
||||
target p95 < 1ms: PASS
|
||||
target VmRSS < 100 MiB: PASS
|
||||
```
|
||||
|
||||
@@ -401,12 +401,30 @@ snapshot loaded but has no sources in it. The line
|
||||
a source in the admin interface, or a `blocklist_sources` entry to the
|
||||
configuration file with a `group_sources` link naming a group.
|
||||
|
||||
One name resolving while its neighbours are blocked is a third case, and
|
||||
`/api/lookup` answers it directly: it reports which level of the filtering
|
||||
ladder decided, and against what.
|
||||
|
||||
```sh
|
||||
curl -s 'http://127.0.0.1:8080/api/lookup?domain=api.ads.tvb.com'
|
||||
```
|
||||
|
||||
```json
|
||||
{"domain":"api.ads.tvb.com","group_id":1,"local_records":false,"forward_zone":null,"blocked":false,"reason":"blocklist_exception","matched":"api.ads.tvb.com","source_url":"https://adguardteam.github.io/HostlistsRegistry/assets/filter_1.txt","safe_search_rewrite":null}
|
||||
```
|
||||
|
||||
`blocklist_exception` means a downloaded list lifted that name with an `@@`
|
||||
line, and `source_url` names the list that did it. Nothing is broken, and the
|
||||
list is not overruling you: an exception cancels only what another list blocks.
|
||||
Your own rule wins over it. Adding an exact block rule for the same name and
|
||||
asking again reports `rule_block_exact`, `blocked` true and a null `source_url`.
|
||||
|
||||
## A database stamped by a newer binary
|
||||
|
||||
**Symptom.** After putting an older binary back, it will not start:
|
||||
|
||||
```
|
||||
warning(migrations): config.db is at schema version 99; this nxdns binary supports 2
|
||||
warning(migrations): config.db is at schema version 99; this nxdns binary supports 1
|
||||
nxdns run failed: SchemaTooNew
|
||||
```
|
||||
|
||||
|
||||
+9
-10
@@ -307,7 +307,7 @@ opens the database immutable and never migrates, so on a database still one
|
||||
version behind it reports the mismatch and exits 2 rather than fixing it:
|
||||
|
||||
```
|
||||
FAIL /var/lib/nxdns/config.db: schema version 0, this nxdns expects 2; `nxdns run` migrates it, `check` will not
|
||||
FAIL /var/lib/nxdns/config.db: schema version 0, this nxdns expects 1; `nxdns run` migrates it, `check` will not
|
||||
```
|
||||
|
||||
That line was reproduced here against a database stamped at version 0; the path
|
||||
@@ -317,29 +317,28 @@ A fresh database is created at the current schema version; an older one is
|
||||
stepped up to it. The log line names both versions:
|
||||
|
||||
```
|
||||
info(migrations): config.db migrated from schema version 0 to 2
|
||||
info(migrations): config.db migrated from schema version 0 to 1
|
||||
```
|
||||
|
||||
> Verified on this host: that exact line is what `nxdns import` printed when it
|
||||
> created the scratch database used throughout this page. An empty data
|
||||
> directory is schema version 0, which is why a first run reports a migration
|
||||
> rather than nothing. The step from a populated older schema to 2 was not
|
||||
> reproduced here — it needs a database written by an older binary, which this
|
||||
> host does not have.
|
||||
> rather than nothing. Version 1 is the only schema nxdns has published, so an
|
||||
> upgrade from a populated older one is not a case that exists yet.
|
||||
|
||||
Rolling back is the case that has no answer. A database stamped by a newer
|
||||
binary refuses to open, so an older binary against an upgraded data directory
|
||||
fails to start:
|
||||
|
||||
```
|
||||
warning(migrations): config.db is at schema version 99; this nxdns binary supports 2
|
||||
warning(migrations): config.db is at schema version 99; this nxdns binary supports 1
|
||||
nxdns run failed: SchemaTooNew
|
||||
```
|
||||
|
||||
> Not reproduced on this host: the same missing ingredient as above, a
|
||||
> database at a schema version this binary does not support. The two lines
|
||||
> are the messages `src/storage/migrations.zig` emits, not a run captured
|
||||
> here.
|
||||
> Reproduced on this host, with one substitution: no binary from the future was
|
||||
> available, so the scratch database's `schema_version` row was set to 99 by
|
||||
> hand and `nxdns run` was pointed at it. The two lines above are that run's
|
||||
> output.
|
||||
|
||||
That run exits 1. Recovering means importing the export you took in step 1 into
|
||||
a fresh data directory with the older binary.
|
||||
|
||||
@@ -277,3 +277,31 @@ included. The password then lives where the rest of the configuration lives: set
|
||||
Request and response schemas for every operation live in the OpenAPI document:
|
||||
`src/web/openapi.yaml` in the repository, or `GET /api/openapi.yaml` from a
|
||||
running server.
|
||||
|
||||
### Block reasons
|
||||
|
||||
Three places carry the same tag: `block_reason` on a `GET /api/queries` row,
|
||||
`block_reason` on a live-stream frame, and `reason` on a `GET /api/lookup`
|
||||
answer. The tag names the level that decided the query, and the levels are
|
||||
listed here in the order they are consulted — the first one that matches wins,
|
||||
so a rule always outranks a list.
|
||||
|
||||
| Tag | Decided by |
|
||||
| --- | --- |
|
||||
| `rule_allow_exact` | An `exact` rule with action `allow` |
|
||||
| `rule_block_exact` | An `exact` rule with action `block` |
|
||||
| `rule_allow_wildcard` | A `wildcard` rule with action `allow` |
|
||||
| `rule_block_wildcard` | A `wildcard` rule with action `block` |
|
||||
| `rule_allow_regex` | A `regex` rule with action `allow` |
|
||||
| `rule_block_regex` | A `regex` rule with action `block` |
|
||||
| `blocklist_exception` | An `@@` exception line in a downloaded list |
|
||||
| `blocklist_domain` | A plain name in a downloaded list |
|
||||
| `blocklist_wildcard` | A domain anchor (`||name^`) in a downloaded list |
|
||||
|
||||
`/api/lookup` also answers `none` when nothing matched. A query row never
|
||||
carries `none`: `block_reason` is null unless the query was blocked.
|
||||
|
||||
A `cname:` prefix means the decision landed on a CNAME target rather than on
|
||||
the name the client asked for, so `cname:blocklist_domain` reads as "the list
|
||||
blocks a name this answer redirects to". Only `/api/queries` and the live
|
||||
stream show the prefix; `/api/lookup` does not follow CNAMEs.
|
||||
|
||||
@@ -62,8 +62,8 @@ reconciled '/etc/nxdns/config.zon': no changes
|
||||
|
||||
Blocklist state is not declarative and survives every reconcile: a source whose
|
||||
URL the file still names keeps its row id, its checksum, its counters and its
|
||||
compiled `<id>.list` and `<id>.wild`, so a restart in file mode downloads
|
||||
nothing. Editing a source's URL is a new identity — a new row, a new id, and a
|
||||
compiled `<id>.list`, `<id>.wild` and `<id>.allow`, so a restart in file mode
|
||||
downloads nothing. Editing a source's URL is a new identity — a new row, a new id, and a
|
||||
fresh download.
|
||||
|
||||
### Failing to start in file mode
|
||||
|
||||
@@ -251,6 +251,34 @@ Known clients with a fixed group assignment.
|
||||
Consumed by the filter engine's exact address-to-group lookup
|
||||
(`src/filter/matcher.zig`).
|
||||
|
||||
#### Learned names
|
||||
|
||||
A client row that carries no name of its own can still show one. Every flush
|
||||
pass the client tracker takes at most 16 unnamed rows that are due, builds each
|
||||
address's reverse name (`192.168.1.10` becomes `10.1.168.192.in-addr.arpa`), and
|
||||
matches it against the declared [`forward_zones`](#forward_zones). On a match it
|
||||
sends one PTR query to that zone's resolver and stores the answer as the row's
|
||||
*learned* name. A row becomes due again 24 hours after the resolver answered or
|
||||
returned NXDOMAIN, and 1 hour after any other outcome — no covering zone, a
|
||||
transport failure, or a reply nxdns rejected — so declaring the missing zone or
|
||||
fixing the resolver shows names within the hour. This
|
||||
requires a conditional forward zone that covers the LAN's reverse space — for
|
||||
example `168.192.in-addr.arpa` pointed at the router. Without such a zone nxdns
|
||||
sends the reverse name to nobody, and the row stays unnamed.
|
||||
|
||||
The PTR query goes to the resolver of the declared zone, wherever the operator
|
||||
pointed it; nxdns does not second-guess that declaration, and it cannot stop a
|
||||
LAN resolver from forwarding the query onward.
|
||||
|
||||
A learned name is runtime state, like `last_seen`:
|
||||
|
||||
- A hand-typed `name` always wins, and a named row is never asked about again.
|
||||
- Learned names never appear in `nxdns export`, and `nxdns import` never sets
|
||||
one.
|
||||
- Each row refreshes once a day. A device rename or a DHCP lease change can
|
||||
therefore display a stale name for up to 24 hours, until the next refresh
|
||||
overwrites it or the router reports no name and nxdns clears it.
|
||||
|
||||
### client_prefixes
|
||||
|
||||
Group assignment by CIDR prefix, for clients without an exact entry.
|
||||
@@ -279,6 +307,29 @@ Consumed by the blocklist manager (`src/filter/manager.zig`): downloaded by the
|
||||
fetcher and compiled into domain sets. A disabled source is neither downloaded
|
||||
nor loaded.
|
||||
|
||||
A list in Adblock Plus syntax may also carry exception lines, `@@||name^` and
|
||||
`@@||name`, either of which may end in `$important`. Those become allow entries
|
||||
that cancel what any attached list blocks, for the name and its subdomains. They
|
||||
cancel nothing an operator decided: every rule of the table above is checked
|
||||
first, so a downloaded list can reopen only a hole another downloaded list dug.
|
||||
Each source reports how many it carried as `exceptions`; there is no way to write
|
||||
one by hand, and no reason to want one — write an allow rule instead.
|
||||
|
||||
Two counters report what a compile skipped, and they are different facts.
|
||||
`skipped_regex` counts regex lines: nxdns has a regex engine, but it takes
|
||||
patterns only from the operator, so a regex line in a downloaded list is counted,
|
||||
skipped and surfaced — adopt the ones you trust as `regex` rules.
|
||||
`skipped_unsupported` counts lines nxdns cannot safely translate into a DNS
|
||||
decision: cosmetic element hiding (`##`, `#@#`, `#?#`), rules carrying a `$`
|
||||
modifier (except `$important` on an exception line, tolerated above), scheme
|
||||
anchors, non-anchored `@@` forms — and, in a `domains`-format
|
||||
list, a line holding more than one field before its inline comment, which usually
|
||||
means the list is really a hosts file that was declared as `domains`. Neither is
|
||||
an error, and the two are never one number. A large `skipped_unsupported` beside
|
||||
a small `domain_count` usually means the list is written for browser extensions,
|
||||
and its DNS or hosts variant will block more here. Both appear per source in the
|
||||
blocklists UI and on `/api/blocklists`.
|
||||
|
||||
### group_sources
|
||||
|
||||
Which groups consult which blocklist sources.
|
||||
@@ -299,15 +350,48 @@ Per-group allow and block overrides, checked before the blocklists.
|
||||
|---|---|---|---|
|
||||
| `group` | string | required | must name a declared group |
|
||||
| `pattern` | string | required | see below |
|
||||
| `kind` | enum `.exact` \| `.wildcard` | required | — |
|
||||
| `kind` | enum `.exact` \| `.wildcard` \| `.regex` | required | — |
|
||||
| `action` | enum `.allow` \| `.block` | required | — |
|
||||
|
||||
Pattern rules: an `.exact` pattern is a plain domain name and may not contain
|
||||
`*`. A `.wildcard` pattern must contain at least one label that is exactly `*`
|
||||
(`*.tracker.example`, or `*` alone), and every other label must be a legal DNS
|
||||
label. `ads*.example` is not a valid wildcard.
|
||||
label. `ads*.example` is not a valid wildcard; a partial label is what the
|
||||
`.regex` kind is for.
|
||||
|
||||
Consumed by the filter engine's rule sets (`src/filter/rules.zig`).
|
||||
A `.regex` pattern is a regular expression matched against the whole normalized
|
||||
lowercase name, unanchored unless you write `^` or `$` — the POSIX-grep
|
||||
convention. It is stored exactly as you typed it, which the other two kinds are
|
||||
not: lowercasing would turn `\D` into `\d`, and trimming a trailing `.` would
|
||||
delete an any-byte atom. The engine (`src/filter/regex.zig`) accepts literal
|
||||
bytes, `.` for any byte, character classes `[a-z0-9]` with a leading `^` for
|
||||
negation, the escapes `\d` and `\w` plus `\` before any other ASCII punctuation
|
||||
to make it a literal, the repetitions `*` `+` `?` `{n}` `{n,m}` `{n,}`,
|
||||
alternation `|`, grouping `(...)`, and the anchors `^` and `$`.
|
||||
|
||||
Everything else is refused at the edge rather than approximated, so a pattern
|
||||
written for another engine fails where you can read the diagnostic instead of
|
||||
silently matching names you did not mean:
|
||||
|
||||
- backreferences, lookaround, captures, named groups, Unicode classes and the
|
||||
`(?…)` prefix they share;
|
||||
- any alphanumeric escape the list above omits — `\s`, `\b`, `\1`, `\D`;
|
||||
- a `]` inside a class, unless written `\]`;
|
||||
- an empty pattern, and an empty branch: `ads|` is refused rather than read as a
|
||||
pattern that matches every name;
|
||||
- a quantifier applied straight to another quantifier: `a+?` is refused rather
|
||||
than read as `(a+)?`, which matches every name. Write `(a+)?` to mean that.
|
||||
|
||||
A pattern is at most 256 bytes and compiles to at most 1024 instructions, each
|
||||
limit with its own diagnostic, and one group holds at most 256 regex rules.
|
||||
Groups do not capture, and the engine simulates every alternative in lockstep,
|
||||
so a pattern costs at most its compiled length times the length of the name —
|
||||
`(a+)+b` is as cheap here as it is expensive in a backtracking engine.
|
||||
|
||||
Consumed by the filter engine's rule sets (`src/filter/rules.zig`), which checks
|
||||
the three kinds in the order they are listed above, allow before block within
|
||||
each. Regex is checked last of the three because it is the only kind that costs
|
||||
more than a hash lookup or a label walk.
|
||||
|
||||
### local_records
|
||||
|
||||
@@ -339,6 +423,16 @@ name would be a bootstrap problem. Matching is longest suffix
|
||||
(`src/local/forward_client.zig`) with `upstream.read_timeout_ms` as the read
|
||||
deadline.
|
||||
|
||||
Reverse zones are declared the same way, and one is the prerequisite for
|
||||
[learned client names](#learned-names):
|
||||
|
||||
```zig
|
||||
.forward_zones = .{
|
||||
.{ .zone = "lan", .resolver = "udp://192.168.1.1:53" },
|
||||
.{ .zone = "168.192.in-addr.arpa", .resolver = "udp://192.168.1.1:53" },
|
||||
},
|
||||
```
|
||||
|
||||
## Password and hash
|
||||
|
||||
Both fields are optional, and the difference between *absent* and *empty* is the
|
||||
@@ -548,10 +642,12 @@ upstream. Everything else keeps its default.
|
||||
.{ .group = "kids", .source_url = "https://lists.example/ads.txt" },
|
||||
},
|
||||
|
||||
// Overrides beat blocklists. Wildcards need a label that is exactly "*".
|
||||
// Overrides beat blocklists. Wildcards need a label that is exactly "*";
|
||||
// a partial label takes a regex, which is unanchored unless you say "^".
|
||||
.rules = .{
|
||||
.{ .group = "default", .pattern = "allowed.example", .kind = .exact, .action = .allow },
|
||||
.{ .group = "kids", .pattern = "*.tracker.example", .kind = .wildcard, .action = .block },
|
||||
.{ .group = "kids", .pattern = "^ad[0-9]+-", .kind = .regex, .action = .block },
|
||||
},
|
||||
|
||||
// Local names, answered without any upstream.
|
||||
|
||||
@@ -46,19 +46,21 @@ older ones from the main file. That is the "uncheckpointed changes" failure in
|
||||
| `blocklists/` | Compiled blocklist snapshots, one subdirectory of the data directory. | 0700 |
|
||||
| `blocklists/<id>.list` | Exact domains for blocklist source `<id>`, one per line, behind a header. | 0600 |
|
||||
| `blocklists/<id>.wild` | Wildcard entries for the same source. | 0600 |
|
||||
| `blocklists/<id>.raw.tmp`, `<id>.list.tmp`, `<id>.wild.tmp` | Transient refresh state: the downloaded body and the two compile outputs before they are published by rename. | 0600 |
|
||||
| `blocklists/<id>.allow` | Exception entries for the same source: the names its `@@` lines lift. Absent on a source compiled before exceptions were honoured, which reads as empty. | 0600 |
|
||||
| `blocklists/<id>.raw.tmp`, `<id>.list.tmp`, `<id>.wild.tmp`, `<id>.allow.tmp` | Transient refresh state: the downloaded body and the three compile outputs before they are published by rename. | 0600 |
|
||||
|
||||
`<id>` is the `blocklist_sources` row id.
|
||||
|
||||
### The orphan sweep
|
||||
|
||||
The sweep decides by id, not by suffix. It matches all five names above and
|
||||
The sweep decides by id, not by suffix. It matches all seven names above and
|
||||
deletes those whose `<id>` is no longer a `blocklist_sources` row, so the
|
||||
compiled `.list` and `.wild` of a removed source go, and so do a `.raw.tmp`,
|
||||
`.list.tmp` or `.wild.tmp` left behind by a refresh that was killed before it
|
||||
could clean up. Files belonging to a source that still has a row are never
|
||||
touched, whatever state they are in: the sweep holds the same lock every refresh
|
||||
takes, so it never reads the directory while a refresh is part-way through.
|
||||
compiled `.list`, `.wild` and `.allow` of a removed source go, and so do a
|
||||
`.raw.tmp`, `.list.tmp`, `.wild.tmp` or `.allow.tmp` left behind by a refresh
|
||||
that was killed before it could clean up. Files belonging to a source that still
|
||||
has a row are never touched, whatever state they are in: the sweep holds the same
|
||||
lock every refresh takes, so it never reads the directory while a refresh is
|
||||
part-way through.
|
||||
|
||||
It runs at three moments:
|
||||
|
||||
@@ -86,7 +88,7 @@ losing the refresh pass behind them, let alone the server.
|
||||
|
||||
The temporaries of a source that still exists are cleaned by the refresh that
|
||||
owns them rather than by the sweep: each refresh deletes its own `.raw.tmp`,
|
||||
`.list.tmp` and `.wild.tmp` as it finishes, successfully or not.
|
||||
`.list.tmp`, `.wild.tmp` and `.allow.tmp` as it finishes, successfully or not.
|
||||
|
||||
A `querylog.db` is moved aside when it is missing nothing but usability:
|
||||
SQLite reports it corrupt or not a database, `PRAGMA quick_check` does not
|
||||
|
||||
@@ -27,7 +27,7 @@ the asset-free figure is a real measurement rather than an estimate.
|
||||
|
||||
## Measured: x86_64 development host
|
||||
|
||||
Date: 2026-08-02. Hardware and build: Intel Core i7-14700K, Linux 6.18,
|
||||
Date: 2026-08-13. Hardware and build: Intel Core i7-14700K, Linux 6.18,
|
||||
Zig 0.16.0, `-Doptimize=ReleaseFast`, harness defaults (1,000,000 domains,
|
||||
200,000 iterations per suite, seed 0x5eed).
|
||||
|
||||
@@ -37,18 +37,26 @@ baseline for regressions on the machine development happens on.
|
||||
|
||||
```
|
||||
suite ops p50(us) p95(us) p99(us) max(us)
|
||||
filter 200000 0.11 0.18 0.27 16.41
|
||||
blocked 66699/200000, Snapshot.memoryBytes 28.0 MiB, VmRSS 31.8 MiB
|
||||
filter 200000 2.41 2.80 2.97 22.16
|
||||
blocked 66699/200000, 32 regex rules, Snapshot.memoryBytes 28.0 MiB, VmRSS 32.0 MiB
|
||||
target p95 < 1ms: PASS
|
||||
target VmRSS < 100 MiB: PASS
|
||||
cache 200000 0.10 0.14 0.17 3.53
|
||||
hits 100000/200000, DnsCache.memoryBytes 4.3 MiB, VmRSS 7.6 MiB
|
||||
cache 200000 0.11 0.17 0.22 5.40
|
||||
hits 100000/200000, DnsCache.memoryBytes 4.3 MiB, VmRSS 7.7 MiB
|
||||
target p95 < 5ms: PASS
|
||||
compile 1000000 wall 96.025ms, 10413949 lines/s, 1000000 domains kept (informational)
|
||||
compile 1000000 wall 98.597ms, 10142224 lines/s, 1000000 domains kept (informational)
|
||||
```
|
||||
|
||||
Every in-process §18 target passes on this host: the two latency targets by
|
||||
three to four orders of magnitude, the memory target by about 3x.
|
||||
Every in-process §18 target passes on this host: the filter target by about
|
||||
360x, the cache target by about four orders of magnitude, the memory target by
|
||||
about 3x.
|
||||
|
||||
The filter suite loads 32 regex rules that no query in the mix matches, which is
|
||||
the expensive case rather than the cheap one: the regex levels sit below every
|
||||
hash and wildcard level, so a name no pattern matches is the name that runs all
|
||||
32 programs to their end. Every op pays that, which is what moved the filter p95
|
||||
from 0.18 µs before regex rules existed to the 2.80 µs above. The margin against
|
||||
the 1 ms target is what makes paying it on every miss an acceptable price.
|
||||
|
||||
### The two memory figures
|
||||
|
||||
|
||||
+45
-25
@@ -9,13 +9,13 @@ delete the directory.
|
||||
Follow the steps in order. Each one says what it did.
|
||||
|
||||
Every command below was executed on x86_64 Linux with Zig 0.16.0, Node.js
|
||||
24.14.1, dig 9.20.26 and curl 8.21.0. Steps 4 to 11, 13 and 14 were re-run end
|
||||
to end for this revision, and the transcripts are that run's output with the
|
||||
24.14.1, dig 9.20.26 and curl 8.21.0. Steps 2, 4 to 11, 13 and 14 were re-run
|
||||
end to end for this revision, and the transcripts are that run's output with the
|
||||
tutorial directory substituted. Two things were not re-run: the browser page in
|
||||
step 12 — its endpoints were exercised, the page itself was not opened — and the
|
||||
two build commands in steps 1 and 2, which had already produced the binary under
|
||||
test. The ZON block at the end of step 14 was checked with `nxdns check
|
||||
--config` rather than started.
|
||||
`npm` build in step 1, whose `web/dist` was already on disk and is the one the
|
||||
binary under test embeds. The ZON block at the end of step 14 was checked with
|
||||
`nxdns check --config` rather than started.
|
||||
|
||||
## What you need
|
||||
|
||||
@@ -109,7 +109,7 @@ zig-out/bin/nxdns import ~/nxdns-tutorial/config.zon --data-dir ~/nxdns-tutorial
|
||||
```
|
||||
|
||||
```
|
||||
info(migrations): config.db migrated from schema version 0 to 2
|
||||
info(migrations): config.db migrated from schema version 0 to 1
|
||||
imported /home/you/nxdns-tutorial/config.zon
|
||||
```
|
||||
|
||||
@@ -127,7 +127,7 @@ zig-out/bin/nxdns run --data-dir ~/nxdns-tutorial/data
|
||||
|
||||
```
|
||||
info(querylog_schema): created querylog database '/home/you/nxdns-tutorial/data/querylog.db'
|
||||
info(blocklist_manager): blocklist snapshot generation 1: 0 of 0 sources loaded, 199 bytes
|
||||
info(blocklist_manager): blocklist snapshot generation 1: 0 of 0 sources loaded, 231 bytes
|
||||
info(nxdns): authority: database
|
||||
info(nxdns): nxdns <version> serving on udp [::1]:15353 udp 127.0.0.1:15353 tcp [::1]:15353 tcp 127.0.0.1:15353; 1 upstream(s); blocklist generation 1
|
||||
info(web_server): web interface listening on 127.0.0.1:8080
|
||||
@@ -146,8 +146,8 @@ dig @127.0.0.1 -p 15353 example.com A +noall +answer
|
||||
```
|
||||
|
||||
```
|
||||
example.com. 90 IN A 172.66.147.243
|
||||
example.com. 90 IN A 104.20.23.154
|
||||
example.com. 229 IN A 172.66.147.243
|
||||
example.com. 229 IN A 104.20.23.154
|
||||
```
|
||||
|
||||
nxdns had no answer cached, so it forwarded the query to
|
||||
@@ -220,20 +220,30 @@ curl -s -X POST http://127.0.0.1:8080/api/blocklists/update
|
||||
```
|
||||
|
||||
```json
|
||||
{"sources":[{"id":1,"state":"ok","loaded":true,"last_attempt":1786473715,"last_success":1786473715,"url":"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts","last_error":"","domains":99559,"wildcards":0,"skipped_regex":0}]}
|
||||
{"sources":[{"id":1,"state":"ok","loaded":true,"last_attempt":1786629237,"last_success":1786629238,"url":"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts","last_error":"","domains":97648,"wildcards":0,"exceptions":0,"skipped_regex":0,"skipped_unsupported":0}]}
|
||||
```
|
||||
|
||||
The four zeros describe this particular download, not the hosts format.
|
||||
`wildcards` counts entries covering a name and its subdomains, `exceptions`
|
||||
counts the `@@` lines an Adblock Plus list uses to lift a name another list
|
||||
blocks, and `skipped_regex` counts the regex lines nxdns declines to take from a
|
||||
downloaded list. `skipped_unsupported` counts lines nxdns cannot translate into a
|
||||
DNS decision, and a large value next to a small `domains` means the list targets
|
||||
browsers rather than DNS. Only `exceptions` is Adblock-Plus-only: a hosts list
|
||||
can carry regex lines, `*.`-prefixed wildcards, and bare sink addresses that
|
||||
count as unsupported. This one carries none of them.
|
||||
|
||||
The download is about 3 MB and takes a few seconds. Watch the first terminal
|
||||
until this appears:
|
||||
|
||||
```
|
||||
info(blocklist_manager): blocklist snapshot generation 6: 1 of 1 sources loaded, 3096006 bytes
|
||||
info(blocklist_manager): blocklist snapshot generation 6: 1 of 1 sources loaded, 2523973 bytes
|
||||
```
|
||||
|
||||
`1 of 1 sources loaded` is the line to wait for. nxdns builds each blocklist
|
||||
snapshot in full and swaps it in atomically, so queries keep being answered from
|
||||
the previous snapshot the whole time the new one is being built. After this, the
|
||||
domain count in the JSON above — 99559 on the day this was run — is live.
|
||||
domain count in the JSON above — 97648 on the day this was run — is live.
|
||||
|
||||
From here on, the list is on disk under `~/nxdns-tutorial/data/blocklists`.
|
||||
Restarting nxdns does not re-download it.
|
||||
@@ -259,16 +269,18 @@ dig @127.0.0.1 -p 15353 wikipedia.org A +noall +answer
|
||||
```
|
||||
|
||||
```
|
||||
wikipedia.org. 17 IN A 185.15.58.224
|
||||
wikipedia.org. 130 IN A 185.15.58.224
|
||||
```
|
||||
|
||||
One thing to know before you try other names: a blocklist entry blocks exactly
|
||||
the name it names. `doubleclick.net` on the list does not block
|
||||
`ads.doubleclick.net`; that name is blocked because the list happens to contain
|
||||
it too. Blocklist entries do not walk up the parent chain — only rules you write
|
||||
yourself can, with a wildcard pattern such as `*.doubleclick.net`. So when you
|
||||
pick a domain to test, pick one that is literally in the file.
|
||||
`www.google-analytics.com` is another that is.
|
||||
One thing to know before you try other names: an entry in a hosts list blocks
|
||||
exactly the name it names. `doubleclick.net` is on this list, and so are
|
||||
`ad.doubleclick.net` and `www.google-analytics.com`. `ads.doubleclick.net` is
|
||||
not on it, and nxdns does not block it — the entry for the parent says nothing
|
||||
about the child. Two things do walk up the parent chain, and neither is in play
|
||||
here: a rule you write yourself, with a wildcard pattern such as
|
||||
`*.doubleclick.net`, and an Adblock Plus list's `||doubleclick.net^`, which
|
||||
covers the name and everything under it. This list is a hosts file, so when you
|
||||
pick a domain to test against it, pick one that is literally in the file.
|
||||
|
||||
## 12. Open the web interface
|
||||
|
||||
@@ -291,7 +303,7 @@ nxdns catches SIGINT and SIGTERM, stops serving and exits 0.
|
||||
Start it again with the same command as in step 6 and read the first log lines:
|
||||
|
||||
```
|
||||
info(blocklist_manager): blocklist snapshot generation 1: 1 of 1 sources loaded, 3096006 bytes
|
||||
info(blocklist_manager): blocklist snapshot generation 1: 1 of 1 sources loaded, 2523973 bytes
|
||||
info(nxdns): authority: database
|
||||
```
|
||||
|
||||
@@ -314,8 +326,12 @@ zig-out/bin/nxdns run --data-dir ~/nxdns-tutorial/data --config ~/nxdns-tutorial
|
||||
|
||||
```
|
||||
reconciled '/home/you/nxdns-tutorial/config.zon': sources +0 ~0 -1; group_sources +0 ~0 -1;
|
||||
info(blocklist_manager): blocklist snapshot generation 1: 0 of 0 sources loaded, 199 bytes
|
||||
info(blocklist_manager): blocklist snapshot generation 1: 0 of 0 sources loaded, 231 bytes
|
||||
info(nxdns): authority: file (/home/you/nxdns-tutorial/config.zon)
|
||||
info(nxdns): nxdns <version> serving on udp [::1]:15353 udp 127.0.0.1:15353 tcp [::1]:15353 tcp 127.0.0.1:15353; 1 upstream(s); blocklist generation 1
|
||||
info(blocklist_manager): pruned orphaned blocklist file 1.allow
|
||||
info(blocklist_manager): pruned orphaned blocklist file 1.wild
|
||||
info(blocklist_manager): pruned orphaned blocklist file 1.list
|
||||
```
|
||||
|
||||
**Read that first line.** The blocklist source is gone. That is not a bug — it is
|
||||
@@ -324,8 +340,12 @@ source, and in file mode the file is the complete statement of what the
|
||||
configuration is, so anything the database holds that the file does not name is
|
||||
removed at every start. The reconcile said so in one line before doing it.
|
||||
|
||||
The compiled list is still on disk and the query log is untouched; what changed
|
||||
is the configuration, and it now matches the file exactly.
|
||||
The three `pruned` lines are the rest of that removal: with the row gone, the
|
||||
compiled files it owned belong to nobody, so the sweep that runs at every start
|
||||
deletes them. The three names are the three bodies one source compiles into —
|
||||
exact domains, wildcards, and the exceptions an Adblock Plus list can lift. The
|
||||
query log is untouched; what changed is the configuration, and it now matches
|
||||
the file exactly.
|
||||
|
||||
Neither mode is the "advanced" one. Database mode suits a box someone
|
||||
administers through the web interface. File mode suits a file kept in git and
|
||||
@@ -345,7 +365,7 @@ Ctrl-C to stop it.
|
||||
|
||||
## What you have now
|
||||
|
||||
A resolver that answers real queries, a real blocklist of about 99000 domains
|
||||
A resolver that answers real queries, a real blocklist of about 98000 domains
|
||||
attached to the default group, a query log, and a web interface — all inside one
|
||||
directory you can delete:
|
||||
|
||||
|
||||
+338
-35
@@ -71,6 +71,11 @@ order preserved; the on-disk header (manager.zig:221-241) gains
|
||||
`# exceptions {d}` after the `# wildcards` line and the pinning test at
|
||||
manager.zig:1914-1946 is extended, not weakened.
|
||||
|
||||
The "checksum over the `.list` body followed by the `.wild` body" sentence
|
||||
exists in THREE places, not the one this ruling first named: `compiler.zig:38-39`,
|
||||
`manager.zig:218` and `sources_repo.zig:100`. All three move together, or the
|
||||
next reader trusts a stale one.
|
||||
|
||||
### 4. Exception counts persist and surface
|
||||
|
||||
Migration step 3 (`ddl_v3`, appended at `src/storage/migrations.zig:23-26`):
|
||||
@@ -82,8 +87,13 @@ checksum doc line at sources_repo.zig:100 is updated alongside
|
||||
`SourceStatus` rehydration (`manager.zig:1458-1502`) restores it alongside the
|
||||
existing three counts; `StatusView` (`src/web/handlers/blocklists.zig:52-78`)
|
||||
gains `exceptions: u32`; the blocklists UI shows it where `skipped_regex`
|
||||
already shows (`web/src/features/blocklists/SourceStatusSection.tsx`,
|
||||
`web/src/lib/types.ts:163,192`).
|
||||
already shows.
|
||||
|
||||
`skipped_regex` shows in TWO tables, and `exceptions` follows it into both:
|
||||
`SourceStatus.skipped_regex` (`web/src/lib/types.ts:192`) renders at
|
||||
`SourceStatusSection.tsx:112`, and `Blocklist.skipped_regex_count`
|
||||
(`types.ts:163`) renders at `BlocklistsPage.tsx:188`. S1 therefore owns
|
||||
`BlocklistsPage.tsx` as well.
|
||||
|
||||
### 5. The regex engine is a Pike VM, linear-time by construction, `std` only
|
||||
|
||||
@@ -91,7 +101,25 @@ New file `src/filter/regex.zig`. Syntax: literal bytes, `.`, character
|
||||
classes `[...]` with ranges and leading-`^` negation, escapes
|
||||
`\. \\ \- \d \w`, repetition `* + ? {n} {n,m}`, alternation `|`,
|
||||
non-capturing grouping `(...)`, anchors `^` and `$`. No backreferences, no
|
||||
lookaround, no captures. Matching is unanchored unless anchors are written
|
||||
lookaround, no captures.
|
||||
|
||||
**Two syntax amendments from S2's review**, both widening what is accepted:
|
||||
|
||||
- `{n,}` is legal. It lowers to `{n}` followed by `*`, stays linear, and an
|
||||
over-large `n` still reports `PatternTooComplex`. Operators write this form;
|
||||
rejecting it buys no safety.
|
||||
- `\` before any ASCII punctuation yields that literal, not only the five
|
||||
escapes listed above — `\*`, `\/` and `\+` occur in Pi-hole-style patterns.
|
||||
This can only narrow a pattern to a literal, never silently change its
|
||||
meaning. Escapes that WOULD change meaning stay rejected: any alphanumeric
|
||||
escape outside `\d` and `\w` (`\s`, `\b`, `\1`, `\D`, `\p{L}`) is
|
||||
`BadPattern`.
|
||||
|
||||
**One rejection the review added:** a quantifier applied directly to another
|
||||
quantifier is `BadPattern`. `a+?` previously compiled as `(a+)?`, which
|
||||
matches every name — a block rule written in conventional lazy syntax would
|
||||
have sinkholed the whole LAN instead of being refused. Parenthesised forms
|
||||
such as `(a+)?` stay legal and keep their meaning. Matching is unanchored unless anchors are written
|
||||
(POSIX-grep convention, matching Pi-hole user expectations). Input is the
|
||||
normalized lowercase name, ≤ `types.max_name_len` bytes. Hard limits, each a
|
||||
distinct error: pattern ≤ 256 bytes (`PatternTooLong`), compiled program
|
||||
@@ -106,8 +134,19 @@ pub fn matches(prog: *const Program, input: []const u8) bool;
|
||||
|
||||
`matches` is a Pike VM: two thread lists, each program counter admitted at
|
||||
most once per input position, worst case O(program × input) with zero
|
||||
allocation at match time (thread lists sized from the program at compile
|
||||
time). The engine is a fuzz-module root like parsers.zig and imports only
|
||||
allocation at match time.
|
||||
|
||||
**Amended in S2.** This ruling first said the thread lists live in the
|
||||
`Program`, sized at compile time. They do not, and must not: the runtime is
|
||||
`std.Io.Threaded`, so several query threads evaluate one shared snapshot at
|
||||
once. Scratch inside a shared `Program` is a data race, and reaching it
|
||||
through `*const Program` would need a `@constCast` that is undefined
|
||||
behaviour on a genuinely const program. All VM scratch — both thread lists,
|
||||
the admission marks and the closure stack — is instead a fixed array on the
|
||||
caller's stack, sized by the compile-time `max_program_len` constant. Zero
|
||||
allocation at match time is preserved, the published signature is unchanged,
|
||||
and a `Program` becomes safe to share across threads, which the original
|
||||
wording would have prevented. The engine is a fuzz-module root like parsers.zig and imports only
|
||||
`std`.
|
||||
|
||||
### 6. `regex` is a third rule kind, validated at the edge, memoized by the cache
|
||||
@@ -125,7 +164,14 @@ set, and a rename would fail both. `model.RuleKind`
|
||||
validate.zig:891-902) and `src/filter/rules.zig:62-68` extend. `RuleSet`
|
||||
(`rules.zig:28-36`) grows `regex_allow` and `regex_block` slices holding
|
||||
compiled `Program`s plus their pattern texts (for `Decision.matched`);
|
||||
`bucketOf` (rules.zig:133-143) becomes a six-bucket layout;
|
||||
`bucketOf` (rules.zig:133-143) becomes a six-bucket layout, and the fixed
|
||||
`var spans: [4]std.ArrayList(Span)` at rules.zig:55 widens with it;
|
||||
`patternIsValid` (`validate.zig:1099-1103`) is declared
|
||||
`error{OutOfMemory}!bool`, so a regex compile's `BadPattern`,
|
||||
`PatternTooLong` and `PatternTooComplex` must either fold into `false` or
|
||||
widen that error set together with its caller at validate.zig:893 — the
|
||||
diagnostic text itself needs no edit, because validate.zig:898 already
|
||||
interpolates `rule.kind.toDb()` into "{f} is not a valid {s} pattern";
|
||||
`max_regex_per_group: usize = 256` with `TooManyRegexRules` mirroring
|
||||
`max_wildcards_per_group` (rules.zig:26). A pattern that fails to compile is
|
||||
`error.BadPattern` at snapshot build, never skipped (rules.zig:41-49 doc
|
||||
@@ -144,7 +190,14 @@ rules.zig:42 becomes `"kind must be 'exact', 'wildcard' or 'regex'"`.
|
||||
`src/web/openapi.yaml:1948,1962,1976`: all three `enum: [exact, wildcard]` become
|
||||
`[exact, wildcard, regex]`. `web/src/lib/types.ts:195`:
|
||||
`RuleKind = "exact" | "wildcard" | "regex"`; the rules page kind selector
|
||||
gains the option. Contract samples regenerated. `nxdns export` / `import`
|
||||
gains the option. Milestone 23 replaced the native `<select>` with a React
|
||||
Aria wrapper, so that is now a data edit, not JSX: append to `KIND_OPTIONS`
|
||||
at `RulesPage.tsx:15-18`, and extend the option-list assertion at
|
||||
`RulesPage.test.tsx:119`. Any new test that opens the selector depends on the
|
||||
`CSS.escape` polyfill in `web/vitest.setup.ts`. Contract samples are
|
||||
regenerated with the AGENTS.md command
|
||||
(`zig build test -Dintegration -Dcontract-samples-out=...`); no npm script
|
||||
generates them. `nxdns export` / `import`
|
||||
round-trip the new kind with no extra work once `RuleKind.toDb/fromDb` extend
|
||||
— the enum round-trip test at model.zig:782 is extended to prove it.
|
||||
|
||||
@@ -156,7 +209,11 @@ counted and skipped; `$` modifiers (except the `$important` suffix of ruling
|
||||
1), partial-segment wildcards, and browser-syntax honoring stay permanently
|
||||
out. PLAN.md:26 (§2.1 filtering sentence), PLAN.md:106 (§3.9), PLAN.md:108-117
|
||||
(§3.10 precedence) and PLAN.md:700 (decision B) are updated to match rulings
|
||||
2 and 6. In-code echoes of the old §2.2 move with it:
|
||||
2 and 6. PLAN.md:99 (§3.8) documents exactly two compiled bodies and names
|
||||
`<source_id>.wild` as the second; ruling 3's third body makes it stale, so S1
|
||||
updates that line when it lands the `.allow` body. PLAN.md:73 ("No
|
||||
TOML/regex/HTTP packages needed") stays true and stays as written — a
|
||||
homegrown engine adds no package. In-code echoes of the old §2.2 move with it:
|
||||
`src/filter/wildcard.zig:6-7,20-23`, `src/filter/parsers.zig:25`,
|
||||
`src/filter/parser_abp.zig:5-6`, `src/filter/compiler.zig:129-131`.
|
||||
|
||||
@@ -205,8 +262,10 @@ Owns: `src/filter/parsers.zig`, `src/filter/parser_abp.zig`,
|
||||
(step 3 only), `src/storage/repositories/sources_repo.zig`,
|
||||
`src/web/handlers/blocklists.zig`, `src/web/handlers/lookup.zig` (doc
|
||||
sentence only), `src/web/openapi.yaml` (StatusView shape only),
|
||||
`web/src/features/blocklists/*`, `web/src/lib/types.ts` (source-stat fields
|
||||
only), `web/src/lib/contractSamples.gen.ts`,
|
||||
`web/src/features/blocklists/*` (which includes `BlocklistsPage.tsx` per
|
||||
ruling 4), `web/src/lib/types.ts` (source-stat fields
|
||||
only), `web/src/lib/contractSamples.gen.ts`, `PLAN.md` (line 99 only, per
|
||||
ruling 8 — S3 owns every other PLAN edit),
|
||||
`tests/fuzz/blocklist_fuzz.zig`, `tests/fuzz/compiler_fuzz.zig`, and the
|
||||
dump-golden assertions in `src/config/import.zig` and
|
||||
`src/config/reconcile.zig` test suites only (ruling 9's `exception_count`
|
||||
@@ -222,7 +281,9 @@ fallout from `ddl_v3`; S1 touches no reconcile logic).
|
||||
exceptions as loadable, not rejected.
|
||||
- S1.4 matcher: `SourceSets.exceptions`, `Reason.blocklist_exception`,
|
||||
the new evaluate level per ruling 2, `memoryBytes` includes the new sets.
|
||||
- S1.5 storage + API + UI per ruling 4.
|
||||
- S1.5 storage + API + UI per ruling 4. `migrations.zig:349` asserts
|
||||
`target_version == 2`; S1 moves it to 3 (S3 moves it to 4). The spec did
|
||||
not name that test; it fails otherwise.
|
||||
- S1.6 tests: parser cases (`@@||x^`, `@@||x`, `@@||x^$important`,
|
||||
`@@||x^$third-party` → unsupported, `@@x` → unsupported); compiler
|
||||
three-body + checksum-compat cases (empty allow body reproduces the old
|
||||
@@ -233,15 +294,21 @@ fallout from `ddl_v3`; S1 touches no reconcile logic).
|
||||
`filter_integration_test.zig` with a real ABP fixture carrying `@@` lines.
|
||||
|
||||
Acceptance (S1):
|
||||
- [ ] `zig build test` passes; fuzz targets build and run.
|
||||
- [ ] A fixture list with `||ads.example^` and `@@||good.ads.example^`
|
||||
- [x] `zig build test` passes; fuzz targets build and run.
|
||||
- [x] A fixture list with `||ads.example^` and `@@||good.ads.example^`
|
||||
compiled and loaded blocks `ads.example` and `x.ads.example`, does not
|
||||
block `good.ads.example` or `y.good.ads.example`, and
|
||||
`/api/lookup` reports `blocklist_exception` with the source id for the
|
||||
latter two.
|
||||
- [ ] A pre-milestone data directory (no `.allow` files, old checksums)
|
||||
loads with zero checksum mismatches.
|
||||
- [ ] `POST /api/blocklists/update` response rows carry `exceptions`.
|
||||
latter two. Also proven live against the AdGuard DNS filter, which
|
||||
carries `||ads.tvb.com^` beside `@@||api.ads.tvb.com^`: `dig` sinkholed
|
||||
`ads.tvb.com` and `x.ads.tvb.com` to 0.0.0.0 and resolved
|
||||
`api.ads.tvb.com` normally.
|
||||
- [x] A pre-milestone data directory (no `.allow` files, old checksums)
|
||||
loads with zero checksum mismatches. Proven live by deleting a loaded
|
||||
source's `.allow` and reloading.
|
||||
- [x] `POST /api/blocklists/update` response rows carry `exceptions`
|
||||
(live: `"domains":154667,"wildcards":154666,"exceptions":11,
|
||||
"skipped_regex":21`).
|
||||
|
||||
### Session S2: the regex engine (tier 2, engine only)
|
||||
|
||||
@@ -257,9 +324,13 @@ target only).
|
||||
- S2.3 the fuzz target per ruling 10.
|
||||
|
||||
Acceptance (S2):
|
||||
- [ ] `zig build test` passes with the new file in `src/tests.zig`.
|
||||
- [ ] The step-bound property holds under the fuzz corpus.
|
||||
- [ ] `regex.zig` imports nothing but `std`.
|
||||
- [x] `zig build test` passes with the new file in `src/tests.zig`.
|
||||
- [x] The step-bound property holds under the fuzz corpus. `expectLinear`
|
||||
(`tests/fuzz/regex_fuzz.zig:101`) asserts
|
||||
`steps <= program_len * (input_len + 1)` over the corpus that
|
||||
`zig build test` replays. An interactive `--fuzz` session could not be
|
||||
used as additional evidence — see the deviation below.
|
||||
- [x] `regex.zig` imports nothing but `std`.
|
||||
|
||||
### Session S3: the regex rule kind, wired through (needs S1 + S2)
|
||||
|
||||
@@ -280,21 +351,32 @@ ruling 9), `PLAN.md`, `src/filter/wildcard.zig` (comments),
|
||||
- S3.4 web + UI + openapi + samples per ruling 7.
|
||||
- S3.5 PLAN and comment amendments per ruling 8.
|
||||
- S3.6 bench: the filter suite in `tools/bench.zig` gains a variant with 32
|
||||
regex rules loaded; the existing p95 < 1 ms assertion covers it.
|
||||
regex rules loaded; the existing p95 < 1 ms assertion covers it. The line to
|
||||
change is `tools/bench.zig:158`, which currently reads `.rules = &.{}` — the
|
||||
filter bench loads no rules at all today, so the variant is new coverage
|
||||
rather than an edit to an existing rule set. S3 also moves
|
||||
`migrations.zig:349` from 3 to 4.
|
||||
- S3.7 tests: rule CRUD with kind `regex` through the API including the 400
|
||||
for a bad pattern at insert time; precedence cases regex-allow over
|
||||
regex-block, wildcard over regex, regex over list entries; export/import
|
||||
round trip; a migration test upgrading a v3 database.
|
||||
|
||||
Acceptance (S3):
|
||||
- [ ] `zig build test` and `cd web && npm test` pass.
|
||||
- [ ] `POST /api/rules` with `{"kind":"regex","pattern":"^ad[0-9]+-"}`
|
||||
returns 201; with `"pattern":"("` returns 400 naming the pattern.
|
||||
- [ ] A regex block rule blocks a matching name; `/api/lookup` reports
|
||||
`rule_block_regex` and `matched` carries the pattern text.
|
||||
- [ ] `zig build bench -Doptimize=ReleaseFast -- filter` passes its targets
|
||||
with the regex variant present.
|
||||
- [ ] PLAN §2.2 no longer forbids operator regex; all listed echoes updated.
|
||||
- [x] `zig build test` and `cd web && npm test` pass.
|
||||
- [x] `POST /api/rules` with `{"kind":"regex","pattern":"^ad[0-9]+-"}`
|
||||
returns 201; with `"pattern":"("` returns 400 naming the pattern
|
||||
(live: `{"error":"rules[0].pattern: '(' is not a valid regex pattern"}`).
|
||||
- [x] A regex block rule blocks a matching name; `/api/lookup` reports
|
||||
`rule_block_regex` and `matched` carries the pattern text (live:
|
||||
`ad42-tracker.example.com` blocked, `matched` `^ad[0-9]+-`).
|
||||
- [x] `zig build bench -Doptimize=ReleaseFast -- filter` passes its targets
|
||||
with the regex variant present (32 regex rules; p95 2.89 µs against a
|
||||
1 ms target; VmRSS 32.0 MiB against a 100 MiB target).
|
||||
- [x] PLAN §2.2 no longer forbids operator regex; all listed echoes updated.
|
||||
Ruling 8's list turned out to be incomplete: §7.1's evaluation sequence,
|
||||
the §5 module tree and the Phase 5 summary also spoke of a two-body,
|
||||
three-kind, no-exception world. All are corrected, and the sweep that
|
||||
found them also caught milestone-20 drift the ruling never covered.
|
||||
|
||||
### Orchestrator
|
||||
|
||||
@@ -314,15 +396,236 @@ Deleted surface: none.
|
||||
|
||||
## Acceptance (milestone complete)
|
||||
|
||||
- [ ] All session acceptance boxes above.
|
||||
- [ ] Schema at version 4; a v2 database migrates cleanly with data intact.
|
||||
- [ ] A pre-milestone blocklist data directory loads without refetch.
|
||||
- [ ] The six-level operator precedence plus three list levels behave per
|
||||
- [x] All session acceptance boxes above.
|
||||
- [x] Schema at version 4; a v2 database migrates cleanly with data intact
|
||||
(`migrations.zig:344`, "a version 2 database upgrades and keeps its
|
||||
sources at exception_count 0"; the live smoke migrated `0 to 4`).
|
||||
- [x] A pre-milestone blocklist data directory loads without refetch.
|
||||
- [x] The six-level operator precedence plus three list levels behave per
|
||||
ruling 2, proven by matcher tests that enumerate adjacent-level pairs.
|
||||
- [ ] No `src/filter/` fuzz-root file imports anything but `std`.
|
||||
- [ ] Contract samples, openapi.yaml and `web/src/lib/types.ts` agree with
|
||||
Every boundary on the nine-level ladder has a test in which one query
|
||||
matches **both** levels, so reversing either order fails the suite. The
|
||||
milestone added four: "both wildcard levels beat an allow regex that
|
||||
matches", "an operator block rule beats a list exception", "a list
|
||||
exception beats a list domain entry on the same name", and "a list
|
||||
domain entry beats a list wildcard entry". The last two were added after
|
||||
the second review pass found the earlier claim overstated — the existing
|
||||
exception test used a name absent from `.list`, so it pinned
|
||||
exception-versus-wildcard rather than exception-versus-domain.
|
||||
- [x] No `src/filter/` fuzz-root file imports anything but `std`
|
||||
(`regex.zig` imports `std` alone).
|
||||
- [x] Contract samples, openapi.yaml and `web/src/lib/types.ts` agree with
|
||||
the server (the drift guards pass).
|
||||
|
||||
## Recorded (anchor re-verification, 2026-08-12)
|
||||
|
||||
The spec was written against `ffc3ca6` and verified against `a8e0fe4`. It was
|
||||
re-verified a third time after milestones 22 and 23 landed (TypeScript 7,
|
||||
Tailwind removed, StyleX and React Aria). Findings, all folded into the
|
||||
rulings above:
|
||||
|
||||
- **No Zig file changed** between `a8e0fe4` and this re-verification. Every
|
||||
Zig anchor holds; six ranges are off by a line or two but still contain what
|
||||
the spec names. The schema is still at version 2, so migration steps 3 and 4
|
||||
are genuinely new.
|
||||
- The rules-page kind selector is no longer a `<select>`. It is a
|
||||
`KIND_OPTIONS` array feeding `web/src/ui/Select.tsx`, a React Aria wrapper
|
||||
(ruling 7 rewritten).
|
||||
- `migrations.zig:349` pins `target_version` and is unnamed by the original
|
||||
spec. S1 moves it to 3, S3 to 4.
|
||||
- `exception_count` has two UI sites, not one, so S1 owns `BlocklistsPage.tsx`
|
||||
(ruling 4 rewritten).
|
||||
- The checksum sentence has three copies, not one (ruling 3 rewritten).
|
||||
- `patternIsValid`'s error set cannot carry the engine's three error tags as
|
||||
written (ruling 6 rewritten).
|
||||
- `PLAN.md:99` documents two compiled bodies and goes stale with ruling 3
|
||||
(ruling 8 rewritten).
|
||||
- `tools/bench.zig:158` reads `.rules = &.{}`, so the filter bench loads no
|
||||
rules today.
|
||||
- `web/vitest.setup.ts` polyfills `CSS.escape`; without it, a test that opens
|
||||
the React Aria selector throws under jsdom.
|
||||
- `npm run build` gained `scripts/assert-css-layers.mjs`, which fails on any
|
||||
rule outside a cascade layer. A pure StyleX change cannot trip it.
|
||||
|
||||
## Recorded (implementation)
|
||||
|
||||
Deviations and findings from the build, the Codex review rounds and the live
|
||||
smoke. Everything here is folded into the code; nothing is outstanding.
|
||||
|
||||
- **Regex scratch is on the caller's stack, not in `Program`.** `std.Io.Threaded`
|
||||
means several query threads share one snapshot, so the Pike VM's two thread
|
||||
lists cannot live in the compiled program. `matches` takes its scratch from
|
||||
the caller's frame, which keeps `Program` immutable and shareable and keeps
|
||||
the match path allocation-free.
|
||||
- **`{n,}` is legal and `\` + any ASCII punctuation is legal.** Ruling 5 named
|
||||
neither. Quantifier-on-quantifier (`a+?` read as `(a+)?`) is `BadPattern`:
|
||||
the first Codex round found `a+?` compiling to something that matched
|
||||
everything.
|
||||
- **Embedded whitespace was accepted on the anchored ABP forms.** A line such
|
||||
as `||good.example bad.example^` reached the compiler, which lowercases and
|
||||
length-checks but does not reject a space, and wrote an entry only a query
|
||||
carrying the same space could match. `compiler.zig` passes `.wildcard` and
|
||||
`.exception` text to `addCandidate` whole, which is why the space survived
|
||||
there. `parser_abp.isNameCandidate` now refuses whitespace and control bytes
|
||||
on those two paths.
|
||||
|
||||
The bare-name path deliberately does **not** use it. `compiler.zig:93-98`
|
||||
tokenizes `.domain` text on whitespace and adds each field separately, so a
|
||||
bare line carrying a space was never broken — it produced two valid entries.
|
||||
Since `detectFormat` assigns one format per source, a mostly-ABP list that
|
||||
also carries hosts-style lines depends on exactly that tokenizer to keep
|
||||
them working. The first attempt at this fix applied the helper to all three
|
||||
paths and silently dropped that fallback; the second Codex pass caught it.
|
||||
Two tests now pin it: a parser test that the bare form stays `.domain`, and
|
||||
a compiler test that an ABP-classified list carrying `0.0.0.0 ads.example`
|
||||
still emits `ads.example`. The third pass pointed out that the parser test
|
||||
alone would pass even if the compiler stopped tokenizing, which is the
|
||||
behaviour the fallback actually depends on.
|
||||
- **The `.allow` body left stale enumerations behind it.** Adding a third
|
||||
compiled body — and with it a fourth temporary, `.allow.tmp` — updated the
|
||||
production code but not every place that lists the file names. The third review pass found four: two `manager.zig` tests that
|
||||
spell the names by hand, the orphan-sweep fixture in
|
||||
`filter_integration_test.zig`, and `PLAN.md` §3.13 and §5. The
|
||||
`manager.zig` table-driven test was worse than stale — it iterates
|
||||
`source_file_suffixes` itself, so deleting an entry changes the code and the
|
||||
test's expectations together and everything still passes. The whole repo was
|
||||
then swept for the pattern rather than the four instances patched, which
|
||||
turned up seven more — including the reload-cancellation test, whose fixture
|
||||
wrote no `.allow` file at all, so the third of `loadSource`'s three read
|
||||
sites was never exercised. A fourth pass then found test 10e comparing only
|
||||
`.list` and `.wild` across a restart, so a restart that rewrote the exception
|
||||
body alone would have stayed green. A `comptime` assertion on
|
||||
`source_file_suffixes.len` now breaks the build when a suffix is added or
|
||||
removed without updating the hand-written tests.
|
||||
- **A pre-existing flake in the required suite.** `src/cli.zig:1538` asserted
|
||||
`std.mem.count(u8, text, "OK") == 0` over output that embeds the temporary
|
||||
directory path. `std.testing.tmpDir` names that directory with base64 over
|
||||
12 random bytes, whose 64-symbol alphabet includes uppercase: 15 adjacent
|
||||
positions each carry `OK` with probability 1/4096, so about one run in 273
|
||||
fails a test that has nothing to do with naming. It surfaced during
|
||||
this milestone's watched-fail injections. The assertion now checks that no
|
||||
line *starts* with a verdict, which covers both `OK:` and
|
||||
`OK upstreams[...]` and cannot match a path segment. Reproduced and fixed
|
||||
outside the milestone's scope because a randomly failing required gate
|
||||
devalues every green run after it.
|
||||
- **`PLAN.md` still described the seed-once config model.** Seven sites said or
|
||||
implied that the first start seeds the database from `/etc/nxdns/config.zon`,
|
||||
which milestone 20 replaced with the two authority modes selected by the
|
||||
presence of `--config`. One of them listed a `config/bootstrap.zig` that does
|
||||
not exist — the module is `loader.zig` plus `reconcile.zig`. The claim was
|
||||
checked against `src/app.zig:339`, not just against the docs. This is
|
||||
milestone-20 drift found while fixing the milestone-21 echoes in the same
|
||||
document, and corrected because PLAN is the source of truth a later session
|
||||
builds from.
|
||||
- **`PLAN.md` §12.1 held a config sample nobody could load.** It described an
|
||||
`.upstream.servers` field that never existed and omitted the required
|
||||
`.groups` and `.upstreams`. The section now points at
|
||||
`docs/reference/configuration.md` and `nxdns export` and keeps only a
|
||||
skeleton: a second copy of the schema is what produced the drift, so the
|
||||
copy is gone rather than corrected.
|
||||
- **`PLAN.md` §7.1 claimed blocklist entries never parent-walk.** Two of the
|
||||
three list levels do: wildcard entries match every proper parent, and
|
||||
exception entries walk the candidate chain, which is why
|
||||
`@@||good.ads.example^` also lifts `y.good.ads.example`. Only domain entries
|
||||
match the query name alone.
|
||||
- **`src/config/model.zig`'s header described the retired bootstrap** and
|
||||
omitted `exception_count` from its list of runtime columns. The first attempt
|
||||
at correcting it introduced a new error — it called import a wholesale
|
||||
replacement set against reconciliation — which the sixth pass caught.
|
||||
`import.zig:3` is explicit that import has been a thin wrapper over
|
||||
`reconcile.zig` since milestone 20, so there is one declarative write path,
|
||||
not two, and it preserves the runtime state of every row the input still
|
||||
names (`reconcile.zig:1139`). A seventh pass then corrected two more claims
|
||||
in the same header: `Config` is the whole shape of a config file but not the
|
||||
only shape the repositories accept (the API writes through `RuleInput`,
|
||||
`ClientInput`, `ClientEdit`), and compiled-body reuse turns on the preserved
|
||||
source id and checksum rather than the counters — `loadSource` names the
|
||||
files after the id and accepts them only against the stored checksum.
|
||||
- **`PLAN.md` §11.2 presented the v1 DDL as the live schema.** It predates
|
||||
three migrations, so it lacks `upstreams.tls_name` and
|
||||
`blocklist_sources.exception_count` and its `kind` CHECK admits only
|
||||
`exact` and `wildcard` — implementing against it would produce a database
|
||||
that rejects every regex rule this milestone added. The section is now
|
||||
labelled the v1 baseline and points at `config_schema.zig` and
|
||||
`migrations.zig`, with the three steps named.
|
||||
- **`INSTALL.md` said the service reads `config.zon` "on the first start".**
|
||||
Neither authority mode behaves that way: under `run --config` the service
|
||||
reads it on every start, and under database authority `nxdns import` reads it
|
||||
while a bare `nxdns run` never does. The sentence justified a file mode, so
|
||||
an operator tightening permissions after first boot would have broken the
|
||||
next file-mode restart. More milestone-20 drift. The claim had a second copy
|
||||
in `docs/how-to/install-with-systemd.md`, found only because the seventh pass
|
||||
looked for it after the first copy was fixed.
|
||||
- **`PLAN.md` §7.1 and the Phase 5 summary missed this milestone's own
|
||||
additions.** The evaluation sequence went straight from operator rules to
|
||||
blocklist domains, omitting the exception level, and the matcher was still
|
||||
enumerated as exact/parent/wildcard with no regex. Ruling 8 required these
|
||||
echoes and S3 did not reach them.
|
||||
- **The UI trimmed regex patterns.** `RulesPage.onSubmit` applied
|
||||
`pattern.trim()` to every kind, so a regex created in the UI did not store
|
||||
the bytes an identical `POST /api/rules` would. It now trims only `exact`
|
||||
and `wildcard`, where the server normalizes anyway. No client-side
|
||||
whitespace rejection was added: `" foo|bar"` still has a live `bar` branch,
|
||||
so refusing it would over-reject, and the server stays the authority on
|
||||
pattern validity.
|
||||
- **The rule pattern field opted into mobile autocapitalization.** An
|
||||
autocapitalized regex validates and then silently never matches, because
|
||||
query names are lowercase and a regex is never normalized. The field now
|
||||
sets `autoCapitalize="none"`, `autoCorrect="off"`, `spellCheck={false}`.
|
||||
- **`RuleSet.build` received the snapshot arena for its temporaries.** An
|
||||
arena reclaims only its most recent allocation, so every `defer …deinit`
|
||||
inside `build` was a silent no-op and the scratch survived until snapshot
|
||||
teardown, uncounted by `memoryBytes()`. `build` now takes a permanent and a
|
||||
scratch allocator, and `matcher.zig` passes `arena` and `gpa` respectively.
|
||||
Regex programs compile into scratch and are copied across by a new
|
||||
`Program.clone`, which keeps `regex.zig` a single-allocator engine and
|
||||
leaves `compile`'s signature — and therefore `config/validate.zig` and the
|
||||
fuzz target — untouched.
|
||||
|
||||
Measured on 16 groups each holding 256 regex rules of 254 bytes,
|
||||
4096 wildcards and 2048 exact rules, with no blocklist sources:
|
||||
arena capacity fell from 135,662,440 to 12,686,214 bytes against an
|
||||
unchanged `memoryBytes()` of 11,058,791, so the ratio of real to reported
|
||||
went from 12.27× to 1.15×. The hidden footprint per snapshot fell from
|
||||
118.83 MiB to 1.55 MiB, so 117.28 MiB went away. A reload holds two
|
||||
snapshots, so it was carrying twice that.
|
||||
`memoryBytes()` needed no change: the formula was always right about what
|
||||
the `RuleSet` retains, and the divergence was arena capacity the formula
|
||||
does not claim to describe. The residual 1.15× is arena node headers and
|
||||
page rounding, which `memoryBytes` documents itself as excluding.
|
||||
|
||||
The guard is `the build's temporaries stay out of the permanent allocator`,
|
||||
which asserts `arena.queryCapacity() < 2 * set.memoryBytes()` and passes
|
||||
`scratch` as `testing.allocator`, so a permanent allocation wrongly taken
|
||||
from scratch also fails as a leak. It was watched failing with the split
|
||||
reverted.
|
||||
- **An interactive `--fuzz` session cannot run on zig 0.16.0.** Building any
|
||||
fuzz target with `-ffuzz` fails inside the stock
|
||||
`/usr/lib/zig/compiler/test_runner.zig:566`, which passes a
|
||||
`*builtin.StackTrace` to `debug.writeStackTrace` where a
|
||||
`*const debug.StackTrace` is wanted — two distinct struct declarations. The
|
||||
failure is entirely inside the toolchain and reproduces on `compiler-fuzz`,
|
||||
a target this milestone did not touch. Corpus replay under `zig build test`
|
||||
is unaffected and remains the evidence for the step-bound property.
|
||||
- **`api.md` gained a block-reason table.** The milestone added three reason
|
||||
tags and no doc page enumerated any of them. The table lists all nine in
|
||||
evaluation order plus `none` and the `cname:` prefix.
|
||||
- **Two comment sites still counted three temporaries.** `manager.zig` line 40
|
||||
(the `refresh_lock` invariant) and line 1293 (`pruneOrphans`) both omitted
|
||||
`.allow.tmp`. Each presents itself as exhaustive, so a maintainer could have
|
||||
added an `.allow.tmp` writer outside `refresh_lock` and let the orphan sweep
|
||||
delete it mid-compile. `sourceFileId` and `source_file_suffixes` already
|
||||
matched all four.
|
||||
- **The rule pattern placeholder named only two kinds.** Ruling 7 requires the
|
||||
web contract to name the third kind everywhere it names the first two, and
|
||||
the placeholder read `ads.example.com or *.example.com`. It now carries a
|
||||
regex example too.
|
||||
- **Two pre-existing tutorial errors surfaced during the docs sweep.**
|
||||
`tutorial/first-run.md` step 14 claimed the compiled list stays on disk when
|
||||
it is in fact pruned (real output: `pruned orphaned blocklist file 1.allow`,
|
||||
`1.wild`, `1.list`), and step 11 named `ads.doubleclick.net`, which
|
||||
StevenBlack no longer carries.
|
||||
|
||||
## Anti-requirements
|
||||
|
||||
- No `$` modifier support beyond tolerating `$important` on exception lines.
|
||||
|
||||
@@ -0,0 +1,581 @@
|
||||
# Milestone 24: `skipped_unsupported` is persisted, surfaced and explained
|
||||
|
||||
Goal: close the compile-pipeline silent drop that misleads the operator. `Counts.
|
||||
skipped_unsupported` (`src/filter/compiler.zig:34`) is counted on every compile
|
||||
(compiler.zig:92) and then discarded on the happy path — it reaches the
|
||||
compiled-file header and the `NoValidEntries` error text, but no database
|
||||
column, no API field and no UI cell. Its sibling `skipped_regex` reaches all
|
||||
three. A blocklist made almost entirely of cosmetic browser filters therefore
|
||||
compiles to almost nothing and looks, in the UI, exactly like a clean list.
|
||||
AGENTS.md forbids exactly this ("no silent drops"). The milestone persists the
|
||||
count, surfaces it everywhere `skipped_regex` is surfaced, and explains to the
|
||||
operator why the two skip counters are different facts.
|
||||
|
||||
Design written 2026-08-13 against HEAD `21571e4`, revised after a Codex review
|
||||
of the first draft.
|
||||
|
||||
## Implementation contract (read first)
|
||||
|
||||
- Read `AGENTS.md`, then this spec whole, before session work starts.
|
||||
- **The v1 baseline is editable and there is no migration step.** As of commit
|
||||
`21571e4`, `src/storage/migrations.zig` holds exactly one step and
|
||||
`target_version` is 1 (migrations.zig:29-36). nxdns has zero installs; the
|
||||
baseline freezes at v0.1 (`config_schema.zig:6-12`, PLAN §3.7, §11.2). The
|
||||
new column is one line edited into `config_schema.ddl_v1` plus the identical
|
||||
line in PLAN §11.2, which is kept byte-identical to it. A diff that adds a
|
||||
`ddl_v2` or a second `Step` is wrong and must be reverted, not merged.
|
||||
- After the API shape change, regenerate the contract samples
|
||||
(`web/src/lib/contractSamples.gen.ts`; procedure in AGENTS.md — the
|
||||
`zig build test -Dintegration -Dcontract-samples-out=...` command, never a
|
||||
hand edit) and update `web/src/lib/types.ts` to match.
|
||||
- No new `src/**.zig` files, so `src/tests.zig` and `build.zig` are untouched.
|
||||
|
||||
## Rulings (binding)
|
||||
|
||||
### 1. The column is `skipped_unsupported_count`, one edited line, no step
|
||||
|
||||
`skipped_regex_count` (config_schema.zig:63) sets the convention; the new
|
||||
column follows it. In `config_schema.ddl_v1`, directly after the
|
||||
`skipped_regex_count` line inside `CREATE TABLE blocklist_sources`:
|
||||
|
||||
```sql
|
||||
skipped_unsupported_count INTEGER NOT NULL DEFAULT 0,
|
||||
```
|
||||
|
||||
The identical line goes into PLAN §11.2 after PLAN.md:421 — that section is
|
||||
kept byte-identical to `ddl_v1` and currently is (verified: the fenced SQL
|
||||
matches the Zig string literal line for line).
|
||||
|
||||
Consequence to accept, not to fix: a development database stamped version 1
|
||||
before this edit will fail the first `SELECT` naming the column with "no such
|
||||
column", because the stamped version equals `target_version` and the step never
|
||||
reruns. That is the documented pre-v0.1 contract (`config_schema.zig:6-12`) —
|
||||
delete the scratch database. Do not add fallback SQL, `PRAGMA table_info`
|
||||
probing, or a migration step to paper over it.
|
||||
|
||||
The baseline test "a fresh database reaches the baseline with every v1 column
|
||||
and rule kind" (migrations.zig:266-283) gains
|
||||
`try testing.expect(try columnExists(&database, "blocklist_sources", "skipped_unsupported_count"));`
|
||||
beside the existing `exception_count` probe.
|
||||
|
||||
### 2. The repo persists it beside `skipped_regex_count`
|
||||
|
||||
`src/storage/repositories/sources_repo.zig`, mirroring `skipped_regex_count`
|
||||
exactly (no default on either struct field, so every construction site fails
|
||||
to compile until it names the count — that is the visibility this milestone is
|
||||
about):
|
||||
|
||||
- `SourceRow` (sources_repo.zig:80-97) gains `skipped_unsupported_count: i64`
|
||||
after `skipped_regex_count`.
|
||||
- `SourceStats` (sources_repo.zig:99-110) gains the same field.
|
||||
- `row_columns_sql` (sources_repo.zig:112-117) appends
|
||||
`skipped_unsupported_count` to the SELECT list (index 11);
|
||||
`readSourceRow` (sources_repo.zig:128-148) reads it with
|
||||
`stmt.columnInt(11)`.
|
||||
- `update_stats_sql` (sources_repo.zig:159-164) adds
|
||||
`skipped_unsupported_count = ?8`; `updateSourceStats`
|
||||
(sources_repo.zig:168-179) binds it.
|
||||
- The module doc comment (sources_repo.zig:3-5) adds the column to its list of
|
||||
server-produced facts, and so does the `src/config/model.zig:14` doc comment
|
||||
that repeats that list.
|
||||
|
||||
**The sum at sources_repo.zig:318 includes the new column.** That sum lives in
|
||||
the test "insertBlocklistSource leaves the runtime columns at their defaults";
|
||||
it exists to prove an insert leaves every runtime counter at 0, and the new
|
||||
column is a runtime counter. This is a deliberate decision, not an oversight:
|
||||
no production query sums these columns into a "total entries" figure, and none
|
||||
may start to — `skipped_regex_count` and `skipped_unsupported_count` count
|
||||
lines *not* written, unlike `domain_count`, `wildcard_count` and
|
||||
`exception_count`, so any future entries total must exclude both. The test sum
|
||||
asserts defaults, which is the one context where adding them is correct.
|
||||
|
||||
Existing `SourceStats` / assertion sites in this file (the literals at
|
||||
sources_repo.zig:386-393, 420-427, 483-493 and the zero-default loop at
|
||||
sources_repo.zig:365-374) carry the new field with distinct non-zero values
|
||||
where their siblings have them, and the round-trip assertions extend to it.
|
||||
|
||||
### 3. The manager writes it, and rehydration restores it
|
||||
|
||||
`src/filter/manager.zig`. The header already prints
|
||||
`# skipped_unsupported {d}` (manager.zig:243) and `SourceStatus.counts` is a
|
||||
full `compiler.Counts` (manager.zig:178), so within one process the count
|
||||
already reaches the status table. What is missing is the database, which is
|
||||
the only thing a restart reads — rehydration never reparses headers.
|
||||
|
||||
- The fresh-publish path (manager.zig:854-861) passes
|
||||
`.skipped_unsupported_count = compiled.result.counts.skipped_unsupported` to
|
||||
`updateSourceStats`.
|
||||
- **The checksum-unchanged path (manager.zig:821-836) carries a bug this
|
||||
milestone fixes.** It writes the *stored* `row.skipped_regex_count` back to
|
||||
the database (manager.zig:830) while handing the *fresh*
|
||||
`compiled.result.counts` to the in-memory status (manager.zig:833). The
|
||||
checksum covers the `.list`, `.wild` and `.allow` bodies only, and a skipped
|
||||
line lands in none of them — so a list that changes only its regex or
|
||||
browser-syntax lines keeps its checksum, the running server shows the new
|
||||
number, the database keeps the old one, and the next restart silently reverts
|
||||
what the operator saw. That is milestone 21's column, wrong today.
|
||||
|
||||
Both skip counters take `compiled.result.counts` on this path:
|
||||
`.skipped_regex_count = compiled.result.counts.skipped_regex` and
|
||||
`.skipped_unsupported_count = compiled.result.counts.skipped_unsupported`.
|
||||
~~`domain_count`, `wildcard_count` and `exception_count` keep reading from
|
||||
`row` — they count written entries, so an unchanged checksum does mean an
|
||||
unchanged value for them.~~ **Corrected post-commit:** that claim was false.
|
||||
The unframed digest could not distinguish an entry in `.list` from the same
|
||||
entry in `.wild`, so an unchanged checksum did not vouch for the entry
|
||||
counts either. The addendum below frames the hash and makes all five stat
|
||||
fields read fresh on this path.
|
||||
|
||||
This needs a test the author has watched fail with the fix reverted: two
|
||||
compiles of bodies whose written entries are identical but whose skipped
|
||||
lines differ, asserting the same checksum, then asserting the database holds
|
||||
the second compile's counts, then rehydrating through `applyLoadOutcomes` to
|
||||
prove the restored status carries them. Report the observed failure output.
|
||||
- `applyLoadOutcomes` rehydration (manager.zig:1545-1550) adds
|
||||
`.skipped_unsupported = countOf(row.skipped_unsupported_count)` to the
|
||||
`Counts` it rebuilds, so a restarted server reports what the last compile
|
||||
skipped instead of 0.
|
||||
- Test literals name the field: one `SourceStats` (manager.zig:1928) and two
|
||||
`SourceRow` (manager.zig:2051, 2225). The rehydration test that feeds a row
|
||||
through
|
||||
`applyLoadOutcomes` asserts a non-zero `skipped_unsupported` lands in
|
||||
`status.counts`. The header pin test (manager.zig:2030-2031) already covers
|
||||
the `# skipped_unsupported` line and is extended only if its fixture counts
|
||||
change.
|
||||
- `rejectedWithoutEntries` (manager.zig:1640-1643) and `failNoValidEntries`
|
||||
(manager.zig:1127-1128) already read the count and are unchanged.
|
||||
|
||||
`src/filter/filter_integration_test.zig` names `.skipped_regex_count` in six
|
||||
`SourceStats` literals (:826, :858, :1075, :1145, :1257, :2158) — each gains
|
||||
the sibling field. The stats round-trip assertion at :915 gains a sibling
|
||||
assertion for the new column.
|
||||
|
||||
**Do not add an unsupported line to the existing fixture.** The fixture at
|
||||
filter_integration_test.zig:112 is hosts-shaped, and `detectFormat`
|
||||
(`src/filter/parsers.zig:118`) assigns one format to a whole source. A single
|
||||
`##.ad-banner` or `$`-modifier line flips it to ABP, which re-parses every
|
||||
existing line: the `*.wild` entry becomes unsupported and address fields
|
||||
tokenize as domains (`src/filter/parser_abp.zig:68`). Every expected count in
|
||||
that test would move, for a reason unrelated to this milestone.
|
||||
|
||||
Add a **separate ABP-format fixture and test** carrying `##.ad-banner` and
|
||||
`||ads.example^$third-party` beside two blockable names, and assert
|
||||
`skipped_unsupported_count = 2` through the compile-persist-read round trip
|
||||
there.
|
||||
|
||||
`src/config/reconcile.zig`: the runtime-column preservation test seeds stats
|
||||
at reconcile.zig:1096-1110 and asserts survival at reconcile.zig:1173. The
|
||||
seed gains `.skipped_unsupported_count` with a distinct value and the
|
||||
assertion block gains its expectation. No reconcile logic changes: the engine
|
||||
updates declarative columns by name and never touches runtime columns, so the
|
||||
new column survives with zero code change — the test is the proof.
|
||||
|
||||
### 4. The API speaks it in both shapes
|
||||
|
||||
Two shapes carry blocklist counters, and the new count joins both under the
|
||||
names its siblings set:
|
||||
|
||||
- **`Blocklist`** (rows of `GET /api/blocklists`, serialized straight from
|
||||
`SourceRow`): the field arrives automatically once `SourceRow` has it, as
|
||||
`skipped_unsupported_count`. `src/web/openapi.yaml:1877-1895` adds it to
|
||||
`required` and `properties`.
|
||||
- **`SourceStatus`** (rows of `POST /api/blocklists/update`): `StatusView`
|
||||
(`src/web/handlers/blocklists.zig:52-80`) gains
|
||||
`skipped_unsupported: u32` after `skipped_regex`, mapped from
|
||||
`status.counts.skipped_unsupported` in `from`.
|
||||
`src/web/openapi.yaml:1920-1938` adds it to `required` and `properties`.
|
||||
The handler test literals at blocklists.zig:323 (`SourceStats`) and :386
|
||||
(`counts`) carry the field with non-zero values and the response assertions
|
||||
extend to it.
|
||||
|
||||
Contract fallout, all in the same session (ruling 6 explains why):
|
||||
|
||||
- Regenerate `web/src/lib/contractSamples.gen.ts` with the AGENTS.md command.
|
||||
- `web/src/lib/types.ts`: `Blocklist` gains
|
||||
`skipped_unsupported_count: number` (types.ts:154-166); `SourceStatus`
|
||||
gains `skipped_unsupported: number` (types.ts:183-195).
|
||||
- The web test mocks are **not** typed against these interfaces, so `tsc` will
|
||||
not force them. `BLOCKLISTS` in
|
||||
`web/src/features/blocklists/BlocklistsPage.test.tsx` is an inferred object
|
||||
literal handed to a `Record<string, unknown>` (BlocklistsPage.test.tsx:40),
|
||||
and the same untyped-fetch pattern holds in
|
||||
`web/src/features/groups/GroupsPage.test.tsx:16` and
|
||||
`web/src/features/settings/authority.test.tsx:64` — both of which already
|
||||
omit `exception_count` without failing. Updating a mock here is a semantic
|
||||
fixture change, not a typecheck fix, and the implementer must not expect a
|
||||
compiler error to point at them.
|
||||
|
||||
So: the BlocklistsPage mocks (:21, :34 blocklist rows; :104, :155, :168
|
||||
status rows) gain the field because S2's rendering assertions read it. The
|
||||
groups and settings mocks are left alone — they render no counter column,
|
||||
and widening them buys nothing. S1 picks values no other cell in the same
|
||||
table already shows (`3` and `7` are taken), e.g.
|
||||
`skipped_unsupported_count: 21` and `skipped_unsupported: 17`.
|
||||
|
||||
### 5. The UI shows it always, in both tables, and says what it means
|
||||
|
||||
Both tables gain a `Skipped unsupported` column directly after
|
||||
`Skipped regex`, rendered unconditionally:
|
||||
|
||||
- `web/src/features/blocklists/BlocklistsPage.tsx`: header after :157, cell
|
||||
`{b.skipped_unsupported_count}` after :190, same
|
||||
`shared.td, shared.tabularNums` props as its neighbours.
|
||||
- `web/src/features/blocklists/SourceStatusSection.tsx`: header after :91,
|
||||
cell `{source.skipped_unsupported}` after :114.
|
||||
|
||||
Always-shown is a decision, not a default: every other counter column here is
|
||||
unconditional, including `Skipped regex` and `Exceptions`, which are 0 for
|
||||
every plain hosts list, and the tutorial already explains those zeros. A
|
||||
column that appears only when non-zero would make two lists' tables disagree
|
||||
in shape, would hide the header that gives the number its meaning, and would
|
||||
special-case exactly the counter this milestone exists to make visible. The
|
||||
zero cell is not noise; it states that nothing in this list was classified as
|
||||
unsupported — which is narrower than "clean", since `invalid` and `long_lines`
|
||||
stay unsurfaced (ruling 7).
|
||||
|
||||
The two counters mean different things and the page must say so once. A muted
|
||||
paragraph (the existing `styles.empty`-style muted text, matching
|
||||
`SourceStatusSection`'s `note` treatment) rendered under the sources table in
|
||||
`BlocklistsPage.tsx`, inside the same `else` branch as the table — the note
|
||||
describes the two skip columns, so it appears exactly when they do. "Always
|
||||
visible" above means unconditional on values, never hidden at 0; it does not
|
||||
mean the empty state (`blocklists.length === 0`) carries a paragraph about
|
||||
columns that are not on screen. The empty state stays one instruction. Exact
|
||||
copy:
|
||||
|
||||
> Both “Skipped” columns count lines nxdns read and did not take. Skipped
|
||||
> regex lines are patterns nxdns accepts only from you — adopt one you trust
|
||||
> as a regex rule. Skipped unsupported lines are syntax nxdns cannot translate
|
||||
> into a DNS decision: cosmetic element hiding, browser-only modifiers. A
|
||||
> skipped unsupported count that dwarfs the domain count usually means the
|
||||
> list is written for a browser extension, and its DNS or hosts variant will
|
||||
> block more here.
|
||||
|
||||
`BlocklistsPage.test.tsx` asserts: both new headers render, the mock values
|
||||
(`21`, and `17` after an update snapshot) render, and the note text is
|
||||
present. No threshold logic, no badge, no coloring by magnitude (see
|
||||
anti-requirements).
|
||||
|
||||
### 6. The docs explain both counters without contradicting what stands
|
||||
|
||||
- `docs/reference/configuration.md`, section `### blocklist_sources`, after
|
||||
the exceptions paragraph (configuration.md:282-288), a new paragraph:
|
||||
|
||||
> Two counters report what a compile skipped, and they are different facts.
|
||||
> `skipped_regex` counts regex lines: nxdns has a regex engine, but it takes
|
||||
> patterns only from the operator, so a regex line in a downloaded list is
|
||||
> counted, skipped and surfaced — adopt the ones you trust as `regex` rules.
|
||||
> `skipped_unsupported` counts lines nxdns cannot safely translate into a DNS
|
||||
> decision: cosmetic element hiding (`##`, `#@#`, `#?#`), rules carrying a
|
||||
> `$` modifier (except `$important` on an exception line, tolerated above),
|
||||
> scheme anchors, non-anchored `@@` forms — and, in a
|
||||
> `domains`-format list, a line holding more than one field before its
|
||||
> inline comment, which usually means the list is really a hosts file that
|
||||
> was declared as `domains`. Neither is an error, and the two are never one
|
||||
> number. A large `skipped_unsupported` beside a small `domain_count`
|
||||
> usually means the list is written for browser extensions, and its DNS or
|
||||
> hosts variant will block more here. Both appear per source in the
|
||||
> blocklists UI and on `/api/blocklists`.
|
||||
|
||||
- `docs/tutorial/first-run.md`: the recorded JSON at first-run.md:223 gains
|
||||
`"skipped_unsupported":0` after `"skipped_regex":0` (the endpoint now
|
||||
returns it; the recorded values themselves stand). The paragraph at
|
||||
first-run.md:226-231 becomes four zeros, keeps its `skipped_regex` sentence
|
||||
as written, and **drops the "parts of this list that a hosts file cannot
|
||||
have" framing** — a deliberate small widening ruled during implementation
|
||||
review. The framing was false before this milestone touched it:
|
||||
`parser_hosts.zig` recognizes regex lines (:16), reports a bare sink
|
||||
address as unsupported (:21), and a `*.`-prefixed name compiles to a
|
||||
wildcard in any format. Only `exceptions` is Adblock-Plus-only. The
|
||||
paragraph now presents the zeros as facts about this particular download
|
||||
and says so.
|
||||
- The `$` modifier is named as skipped **with its one exception**: `$important`
|
||||
on an anchored `@@` exception line is accepted and lands in
|
||||
`exception_count` (`parser_abp.zig`, PLAN §2.2). The configuration.md
|
||||
paragraph above, and the `SourceRow.skipped_unsupported_count` doc comment
|
||||
in `sources_repo.zig` that mirrors it, both carry the qualifier — an
|
||||
unqualified "rules carrying a `$` modifier" contradicts the parser.
|
||||
- PLAN.md:101 (§3.8) currently reads "regex lines are counted + skipped
|
||||
(counts in metadata → UI)". It becomes: regex lines *and* browser-syntax
|
||||
lines are counted and skipped, both counts in metadata → UI. One sentence;
|
||||
§2.2 (PLAN.md:36-37) already says unsupported forms "stay unsupported and
|
||||
counted" and needs no edit.
|
||||
- `docs/reference/files-and-directories.md` is untouched: the compiled-file
|
||||
header already carried `# skipped_unsupported` before this milestone and
|
||||
the file table there does not enumerate header lines.
|
||||
|
||||
### 7. The other three counters stay out, and the reasons differ per counter
|
||||
|
||||
The first draft claimed `invalid`, `long_lines` and `duplicates` were already
|
||||
"header-and-error-path only". That is false:
|
||||
|
||||
- `invalid` reaches the compiled-file header (manager.zig:244) and the
|
||||
`NoValidEntries` text (manager.zig:1127).
|
||||
- `long_lines` reaches the `NoValidEntries` text only — not the header.
|
||||
- `duplicates` reaches **nothing**: counted at compile, discarded on every
|
||||
path.
|
||||
|
||||
They stay out of scope, and not for one shared reason — which is why the goal
|
||||
above says "the silent drop that misleads the operator" rather than "the one
|
||||
silent drop left":
|
||||
|
||||
- `duplicates` hides no failure. A deduplicated name still blocks; the count
|
||||
measures input redundancy, not lost coverage. Discarding it discards a
|
||||
curiosity, so "no silent drops" does not reach it — there is no drop.
|
||||
- `invalid` and `long_lines` do measure dropped lines, and they remain only
|
||||
partially surfaced. The catastrophic form — a download that is all rejects —
|
||||
already fails loudly (`rejectedWithoutEntries`, state `no_valid_entries`);
|
||||
the residual is a list that loses some lines and still loads, visible in
|
||||
the header file and nowhere the UI reaches. That residual is real, it is
|
||||
recorded here deliberately, and it is not this milestone: the two skip
|
||||
counters name an action the operator can take (adopt the patterns; fetch
|
||||
the DNS variant), while these two say only "the list is malformed", which
|
||||
no column makes more actionable.
|
||||
|
||||
The decision is reversible; a later milestone can widen the table. What this
|
||||
spec may not do is claim the three are visible when `duplicates` is not.
|
||||
|
||||
### 8. Export, import and reconcile change nothing
|
||||
|
||||
`nxdns export` / `import` carry configuration; the compile counters are facts
|
||||
a running server produces. `model.BlocklistSource` holds only the four
|
||||
configuration columns, the insert leaves runtime columns at their defaults
|
||||
(sources_repo.zig:1-9, :48-61), and the dump helpers used by the import and
|
||||
reconcile suites are `SELECT *` compared dump-to-dump
|
||||
(`src/config/import.zig:167-186`, `src/config/reconcile.zig:988`), so no
|
||||
golden text names columns and the new column appears in both sides of every
|
||||
comparison. Ruling 3's reconcile-test extension is the only touch in
|
||||
`src/config/`, plus the model.zig:14 comment from ruling 2.
|
||||
|
||||
## Sessions
|
||||
|
||||
Two sessions, strictly sequential: S1 then S2. No parallelism — deliberately.
|
||||
The regenerated `contractSamples.gen.ts` typechecks only against a `types.ts`
|
||||
that already carries the new fields, and the samples are produced by a Zig
|
||||
integration run, so the generator and the interface it must satisfy sit on
|
||||
opposite sides of the language boundary and cannot land independently. S2's
|
||||
rendering assertions then read fields that only exist once S1 has landed both.
|
||||
|
||||
(The mocks are *not* part of this argument: they are untyped, so widening
|
||||
`types.ts` does not break them. The first draft claimed otherwise.)
|
||||
|
||||
### Session S1: column, persistence, API, contract
|
||||
|
||||
Owns: `src/storage/config_schema.zig`, `src/storage/migrations.zig` (test
|
||||
only), `src/storage/repositories/sources_repo.zig`, `src/filter/manager.zig`,
|
||||
`src/filter/filter_integration_test.zig`, `src/config/reconcile.zig` (test
|
||||
only), `src/config/model.zig` (comment only),
|
||||
`src/web/handlers/blocklists.zig`, `src/web/openapi.yaml`,
|
||||
`web/src/lib/types.ts`, `web/src/lib/contractSamples.gen.ts` (regenerated,
|
||||
never hand-edited), `web/src/features/blocklists/BlocklistsPage.test.tsx`
|
||||
(mock fields only — no rendering assertions),
|
||||
`PLAN.md` (the §11.2 line and
|
||||
the §3.8 sentence).
|
||||
|
||||
- S1.1 rulings 1 and 2: the DDL line in both copies, the repo columns, the
|
||||
migrations and repo tests.
|
||||
- S1.2 ruling 3: both `updateSourceStats` call sites, rehydration, test
|
||||
literals, the reconcile preservation assertion.
|
||||
- S1.3 ruling 4: `StatusView`, openapi.yaml, sample regeneration, `types.ts`,
|
||||
mock fields.
|
||||
|
||||
Acceptance (S1):
|
||||
- [ ] `zig build test` passes; the migrations baseline test proves
|
||||
`blocklist_sources.skipped_unsupported_count` exists.
|
||||
- [ ] A compile of the new **ABP-format** fixture carrying `##.ad-banner` and
|
||||
`||ads.example^$third-party` persists `skipped_unsupported_count = 2`
|
||||
through `updateSourceStats` and reads it back through `listSourceRows`.
|
||||
The existing hosts-shaped fixture is unchanged, and every count it
|
||||
already asserts still holds.
|
||||
- [ ] A second compile whose written entries are byte-identical but whose
|
||||
skipped lines differ produces the same checksum and still updates both
|
||||
skip counters in the database; the test was watched failing with the
|
||||
manager.zig:830 fix reverted, and the failure output is recorded.
|
||||
- [ ] `applyLoadOutcomes` fed a row with `skipped_unsupported_count = 5` and
|
||||
no live refresh yields `status.counts.skipped_unsupported == 5`.
|
||||
- [ ] `zig build test -Dintegration` passes, and the regenerated
|
||||
`contractSamples.gen.ts` carries `skipped_unsupported_count` in the
|
||||
Blocklist sample and `skipped_unsupported` in the SourceStatus sample.
|
||||
- [ ] `cd web && npm run typecheck && npm test` pass with the widened types.
|
||||
|
||||
### Session S2: UI and docs (needs S1)
|
||||
|
||||
Owns: `web/src/features/blocklists/BlocklistsPage.tsx`,
|
||||
`web/src/features/blocklists/SourceStatusSection.tsx`,
|
||||
`web/src/features/blocklists/BlocklistsPage.test.tsx` (rendering assertions),
|
||||
`docs/reference/configuration.md`, `docs/tutorial/first-run.md`.
|
||||
|
||||
- S2.1 ruling 5: both columns, the note paragraph, the rendering assertions.
|
||||
- S2.2 ruling 6: the two doc edits.
|
||||
|
||||
Acceptance (S2):
|
||||
- [ ] `cd web && npm run typecheck && npm test && npm run lint` pass; the new
|
||||
tests assert both `Skipped unsupported` headers, the mock values and the
|
||||
note text.
|
||||
- [ ] `npm run build` passes (`assert-css-layers.mjs` runs inside it).
|
||||
- [ ] `docs/tutorial/first-run.md` no longer says "three zeros", its JSON
|
||||
sample carries `skipped_unsupported`, and its `skipped_regex` sentence
|
||||
is unchanged.
|
||||
|
||||
### Orchestrator
|
||||
|
||||
Verify S1 acceptance before starting S2. After S2, run the full gate set
|
||||
(`zig build test`, `zig build test -Dintegration`, `test-aarch64` if qemu is
|
||||
present, `cd web && npm test`, `npm run assert-bundled`), then a live smoke
|
||||
against a scratch server with two real sources:
|
||||
|
||||
- `https://easylist.to/easylist/easylist.txt` — a browser-targeted list;
|
||||
expect a `skipped_unsupported` several times its `domains` (do not assert an
|
||||
exact number; assert the ratio and non-zero).
|
||||
- `https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts` — expect
|
||||
`skipped_unsupported` 0.
|
||||
|
||||
Verify the numbers appear in the `POST /api/blocklists/update` response, in
|
||||
`GET /api/blocklists`, and in both UI tables. Then restart the server and
|
||||
verify the counts survive into the status table without a refresh — that is
|
||||
ruling 3's rehydration working against the real database. Record deviations
|
||||
in `## Recorded (implementation)`.
|
||||
|
||||
## Module layout
|
||||
|
||||
New files: none. Deleted surface: none.
|
||||
|
||||
## File ownership
|
||||
|
||||
| File | Session |
|
||||
| --- | --- |
|
||||
| `src/storage/config_schema.zig` | S1 |
|
||||
| `src/storage/migrations.zig` | S1 |
|
||||
| `src/storage/repositories/sources_repo.zig` | S1 |
|
||||
| `src/filter/manager.zig` | S1 |
|
||||
| `src/filter/filter_integration_test.zig` | S1 |
|
||||
| `src/config/reconcile.zig` (test), `src/config/model.zig` (comment) | S1 |
|
||||
| `src/web/handlers/blocklists.zig`, `src/web/openapi.yaml` | S1 |
|
||||
| `web/src/lib/types.ts`, `web/src/lib/contractSamples.gen.ts` | S1 |
|
||||
| `PLAN.md` (§11.2 line, §3.8 sentence) | S1 |
|
||||
| `web/src/features/blocklists/BlocklistsPage.test.tsx` | S1 (mock fields), then S2 (assertions) — sequential, never concurrent |
|
||||
| `web/src/features/blocklists/BlocklistsPage.tsx`, `web/src/features/blocklists/SourceStatusSection.tsx` | S2 |
|
||||
| `docs/reference/configuration.md`, `docs/tutorial/first-run.md` | S2 |
|
||||
|
||||
## Acceptance (milestone complete)
|
||||
|
||||
- [ ] All session acceptance boxes above.
|
||||
- [ ] `config_schema.ddl_v1` and PLAN §11.2 are byte-identical, both carrying
|
||||
the new line; `migrations.steps` still holds exactly one step and
|
||||
`target_version` is still 1.
|
||||
- [ ] The live smoke: EasyList shows a large `skipped_unsupported` and
|
||||
StevenBlack shows 0, in the API and in both UI tables, and both values
|
||||
survive a server restart.
|
||||
- [ ] `nxdns export` output is byte-identical before and after a refresh that
|
||||
wrote the new column (runtime columns stay out of exports).
|
||||
- [ ] The regenerated contract samples typecheck against `web/src/lib/types.ts`
|
||||
— that is what the sample mechanism proves, and it covers server-to-
|
||||
TypeScript agreement only.
|
||||
- [ ] `src/web/openapi.yaml` is reviewed **by hand** against the two changed
|
||||
response shapes, and the reviewer says so in `## Recorded`. No automated
|
||||
guard covers this: `web_integration_test.zig:2089` checks that routes and
|
||||
methods exist and `:2575` counts operations, but nothing compares a
|
||||
component schema to a real response. An openapi.yaml that omits the new
|
||||
field will pass every gate in this repo.
|
||||
|
||||
## Anti-requirements
|
||||
|
||||
- No migration step, no `ddl_v2`, no runtime schema probing. The baseline is
|
||||
edited in place per §3.7; a pre-edit scratch database is deleted, not
|
||||
reconciled.
|
||||
- No merging of the two skip counters into one number, anywhere — not in the
|
||||
API, not in a UI total, not in prose. They are different facts.
|
||||
- No "this list targets browsers" heuristic: no threshold, badge, warning
|
||||
color or ratio computation in the UI. The number plus the note paragraph is
|
||||
the surface; a cutoff would be an invented policy.
|
||||
- No conditional rendering of the new column. It shows at 0 like every other
|
||||
counter column.
|
||||
- No change to `nxdns export` / `import` ZON: compile statistics are not
|
||||
configuration.
|
||||
- No per-line diagnostics, no sample of skipped lines in logs, API or UI —
|
||||
counters and health surfaces, not log spam (AGENTS.md).
|
||||
- No change to `rejectedWithoutEntries` or `failNoValidEntries` semantics.
|
||||
- No hand edits to `contractSamples.gen.ts`.
|
||||
- No new columns beyond the one. See ruling 7 for what that leaves open and
|
||||
why — the reason is a scope decision, not a claim that the other counters
|
||||
are already visible.
|
||||
|
||||
## Addendum (post-`1bce81e`): the body checksum is framed
|
||||
|
||||
A filtering correctness bug found by the implementation review, predating this
|
||||
milestone. Fixed as a follow-up commit; this addendum is its design record —
|
||||
the user ruled it a follow-up, not a milestone 25.
|
||||
|
||||
### The defect
|
||||
|
||||
`bodyChecksum` (manager.zig:1660) and the compiler's incremental hashing
|
||||
(compiler.zig:105-108) both digest the unframed concatenation
|
||||
`list ++ wild ++ allow`. The compiler strips `*.` from a wildcard candidate
|
||||
(compiler.zig:130-133), so upstream `a.example` (list `a.example\n`, wild
|
||||
empty) and upstream `*.a.example` (list empty, wild `a.example\n`) hash the
|
||||
same bytes. `diskBodiesMatch` (manager.zig:1085) recomputes with the same
|
||||
function, so the refresh takes the unchanged-checksum branch and a list that
|
||||
switches an exact block to a wildcard block never takes effect. Ruling 3's
|
||||
entry-count argument rested on the digest distinguishing bodies; it does not,
|
||||
and the strikethrough above records that.
|
||||
|
||||
### The fix: a `0x00` separator after each body
|
||||
|
||||
The digest becomes `SHA-256(list ‖ 00 ‖ wild ‖ 00 ‖ allow ‖ 00)` — one zero
|
||||
byte fed to the hasher **after each of the three bodies**, same order as
|
||||
today. Soundness: a compiled body holds only validated name bytes and `\n`;
|
||||
`addCandidate` rejects any byte ≥ `0x80` or control byte
|
||||
(compiler.zig:151-155), so `0x00` cannot occur in a body and the three
|
||||
boundaries are unambiguous. Two distinct `(list, wild, allow)` triples cannot
|
||||
produce one digest short of SHA-256 itself.
|
||||
|
||||
A separator, not a length prefix, because the compiler hashes while it emits
|
||||
and does not know a body's length up front; a trailing byte needs no pre-pass.
|
||||
|
||||
Both producers move together or every refresh republishes forever:
|
||||
`compiler.compile` feeds the byte after each `emit` call, and
|
||||
`manager.bodyChecksum` feeds it after each body slice. Export the separator as
|
||||
a `pub const` from `compiler.zig` and have `bodyChecksum` use it — two literal
|
||||
`0`s in two files is how the next drift starts.
|
||||
|
||||
### Consequences, accepted
|
||||
|
||||
- Every stored checksum changes once. Zero installs; a development data
|
||||
directory fails its startup checksum verification and the startup refresh
|
||||
pass re-downloads and repairs it (or delete the data directory — the ruling
|
||||
1 stance).
|
||||
- Checksum compatibility with pre-exception digests — milestone 21 ruling 3's
|
||||
"empty allow body reproduces the old digest" property — is dead, and its
|
||||
rationale comments go with it: `Result.checksum` (compiler.zig:44-53), the
|
||||
`Header` doc and `bodyChecksum` doc (manager.zig:218-225, 1645-1648),
|
||||
`SourceStats.checksum` (sources_repo.zig), and the PLAN §3.8 sentence
|
||||
"keeps the digest it had when only two existed" (PLAN.md:101). All are
|
||||
rewritten to state the framed digest. The body order stays list, wild,
|
||||
allow.
|
||||
|
||||
### The branch reads all five fresh
|
||||
|
||||
On the checksum-unchanged path, all five stat fields —
|
||||
`domain_count`, `wildcard_count`, `exception_count`, `skipped_regex_count`,
|
||||
`skipped_unsupported_count` — now read from `compiled.result.counts`;
|
||||
`checksum` keeps passing `stored`. With framing, fresh and stored entry counts
|
||||
are provably equal, so this is not a correctness requirement — it removes the
|
||||
per-field vouching argument from the code entirely, and any future digest
|
||||
weakness then degrades to consistent stats rather than a split between
|
||||
database and status table.
|
||||
|
||||
### Regression tests (each watched failing with the framing reverted)
|
||||
|
||||
- [ ] Compiler: compiling `a.example` and compiling `*.a.example` produce
|
||||
**different** checksums. This is the collision itself.
|
||||
- [ ] Manager: publish a source whose body is `a.example`; refresh it with
|
||||
upstream bytes `*.a.example`. Assert the unchanged-checksum branch is
|
||||
**not** taken: the on-disk `.wild` stripped body is `a.example\n`, the
|
||||
`.list` body is empty, and the database row reads `domain_count = 0`,
|
||||
`wildcard_count = 1`.
|
||||
- [ ] Agreement: `bodyChecksum` over the three stripped on-disk bodies equals
|
||||
`compile`'s reported checksum for a fixture where **all three bodies are
|
||||
non-empty** (extend the existing agreement coverage if it exists; the
|
||||
empty-allow case no longer exercises the third frame).
|
||||
|
||||
Report the observed failure output for each, per the ruling 3 convention.
|
||||
@@ -0,0 +1,642 @@
|
||||
# Milestone 25: client names learned over reverse DNS
|
||||
|
||||
Goal: the clients table names devices by itself. The tracker already
|
||||
materialises every querying address into a `clients` row (PLAN §7.2,
|
||||
`src/server/clients.zig`), but `name` stays NULL until the operator types one —
|
||||
which defeats the point of auto-materialisation on a LAN whose router already
|
||||
knows every DHCP hostname. This milestone asks the router: for each unnamed
|
||||
client, nxdns builds the address's reverse name, matches it against the
|
||||
operator's conditional forward zones (PLAN §6.5), sends one PTR query to the
|
||||
zone's resolver, and stores the answer as a *learned* name — runtime state
|
||||
beside `last_seen`, never configuration. The dashboard shows it; a hand-typed
|
||||
name always wins; a reverse name no forward zone covers is never sent anywhere.
|
||||
|
||||
Design written 2026-08-14 against HEAD `c428bc2`, revised after a Codex review
|
||||
of the first draft (all 19 findings accepted; the review's fingerprints are
|
||||
called out inline where a first-draft claim was wrong).
|
||||
|
||||
## Implementation contract (read first)
|
||||
|
||||
- Read `AGENTS.md`, then this spec whole, before session work starts.
|
||||
- **The v1 baseline is editable and there is no migration step** (milestone-24
|
||||
ruling 1 stands unchanged). The two new columns are lines edited into
|
||||
`config_schema.ddl_v1` plus the identical lines in PLAN §11.2, kept
|
||||
byte-identical. A `ddl_v2` or a second `Step` is wrong and must be reverted.
|
||||
A development database stamped version 1 before the edit fails its first
|
||||
`SELECT` naming the columns; delete the scratch database.
|
||||
- The new `src/local/reverse_name.zig` is pure: bytes in, bytes out, no
|
||||
`std.Io`, no clock, no sockets. (The blanket claim "src/local/ is pure" from
|
||||
the first draft was false — `forward_client.zig` does socket I/O by design;
|
||||
the purity requirement here binds the *new* module and the lookup-table
|
||||
modules it sits beside.) Everything that touches the network or the database
|
||||
lives in `src/server/`.
|
||||
- After the API shape change, regenerate
|
||||
`web/src/lib/contractSamples.gen.ts` with the AGENTS.md command (never a
|
||||
hand edit) and update `web/src/lib/types.ts` to match, in the same session.
|
||||
- Verify stdlib claims against `../zig` at tag 0.16.0. `specs/research/` holds
|
||||
verified notes. One verified here: randomness is on the `Io` interface —
|
||||
`io.random(buffer)` (`std/Io.zig:2468`); `std.crypto.random` does not exist
|
||||
in 0.16.0.
|
||||
- No `std.log.err` in new code. PTR answers are untrusted bytes: never let one
|
||||
reach a log line, a metric label, or the UI without passing `acceptHostname`
|
||||
(ruling 4).
|
||||
|
||||
## Rulings (binding)
|
||||
|
||||
### 1. When: the tracker's flush pass resolves, bounded, serial, last
|
||||
|
||||
Resolution is a step appended to the tracker's existing flush pass
|
||||
(`Tracker.flushOnce`, `src/server/clients.zig:164`), not a new loop and not a
|
||||
query-path hook. Reasons: the pass already runs every `flush_interval_s` (60 s)
|
||||
on a dedicated task with a dedicated `config.db` connection (app.zig:523-524,
|
||||
:754), it already respects the disk-monitor gate, and a name can only be
|
||||
learned for a row that exists — which the flush itself just wrote. A DNS query
|
||||
must never wait on naming, and with this placement it structurally cannot.
|
||||
|
||||
**Order within one pass is fixed: drain-and-write, then prune (when due), then
|
||||
resolve.** One `now_s` is read at the top of the pass and used by all three
|
||||
steps. Resolving before pruning could spend PTR queries on rows the same pass
|
||||
deletes; the fixed order makes that impossible, and a test proves a row past
|
||||
the prune cutoff on a due pass causes no exchange.
|
||||
|
||||
Per pass, the resolver:
|
||||
|
||||
- selects at most `max_per_pass = 16` candidate rows (ruling 3 defines a
|
||||
candidate), ordered by `name_attempt_after` ascending then ip;
|
||||
- resolves them **serially** — in-flight is exactly 1. At 16 per 60 s pass the
|
||||
LAN resolver sees at most ~0.27 queries/second from naming.
|
||||
|
||||
**Naming has its own read timeout**, `ptr_read_timeout` = 2 s on the `.awake`
|
||||
clock, a `pub const` on the resolver — it does *not* inherit the handler's
|
||||
`forward_read_timeout`, which the operator may have set as high as the
|
||||
upstream budget. **The budget bounds the whole attempt, not one transport
|
||||
leg**: `ForwardClient` hands its duration to the UDP receive and then again
|
||||
to the TCP fallback, so a resolver that answers UDP late with TC=1 and then
|
||||
stalls TCP would spend up to 2× the budget per attempt — 64 s per pass, not
|
||||
32. The default `exchangeFn` therefore wraps the entire `ForwardClient`
|
||||
exchange in `transport.raceWithin(io, ptr_read_timeout, ...)` (the same
|
||||
mechanism `exchangeTcp` itself uses, forward_client.zig:195), so UDP,
|
||||
fallback and all share one deadline. The worst case is 16 × 2 s = 32 s of
|
||||
naming per pass, and that number is a bound, not an estimate. That is acceptable because the drain already ran first: a slow pass
|
||||
delays the *next* pass's start (the run loop sleeps after `flushOnce`
|
||||
returns), and one extra interval against a 512-slot pending table on a
|
||||
household LAN drops nothing. The acceptance test for this: with a stub client
|
||||
that times out on every exchange, the pass still drains its pending batch
|
||||
before any exchange is attempted, and attempts stop at `max_per_pass`.
|
||||
|
||||
Retry cadence is split by outcome (first draft used one 24 h cadence for
|
||||
everything, which left a freshly-declared zone, a recovered router, or a fixed
|
||||
resolver unnamed for a day while forward-zone changes publish live):
|
||||
|
||||
- **definitive** outcomes (`answered`, `nxdomain`): next attempt after
|
||||
`refresh_after_s = 86_400` — the daily refresh that tracks DHCP renames;
|
||||
- **non-definitive** outcomes (`no_zone`, `failed`, `invalid`): next attempt
|
||||
after `retry_after_s = 3_600` — an operator who declares the missing zone
|
||||
or fixes the resolver sees names within the hour, and a dead resolver costs
|
||||
at most 16 timeouts per hour, not per minute.
|
||||
|
||||
Both constants are `pub const` on the resolver; no config knob (household
|
||||
scale, AGENTS.md: no generality nobody asked for).
|
||||
|
||||
### 2. Where: only through a declared forward zone, never the pool
|
||||
|
||||
The reverse name (`10.1.168.192.in-addr.arpa`, or the 32-nibble `ip6.arpa`
|
||||
form) is matched against the live forward-zones table — the same
|
||||
`forward_zones.Zones.match` the query path uses, bracketed by
|
||||
`LocalTables.acquire`/`release` (`src/server/local_tables.zig:48`), so naming
|
||||
always sees the generation the operator last published.
|
||||
|
||||
**The zone's `resolver` is copied by value under the shared lock, and the
|
||||
handle is released before any socket work.** `match` returns a pointer into
|
||||
the current table generation, and `swap` frees the old generation as soon as
|
||||
it holds the exclusive lock — a handle held across a PTR exchange, or a
|
||||
released handle whose `*const Zone` is still dereferenced, is a use-after-free
|
||||
against a config reload. `validate.Resolver` is a plain value; the copy is the
|
||||
whole fix. A regression test must place the `swap` **inside the hazard
|
||||
window** — after the handle release, before the copied resolver is used. A
|
||||
test that swaps before `runPass` evaluates its arguments exercises nothing
|
||||
(the first implementation's test had exactly this defect: the stub's exchange
|
||||
performed the swap, but the resolver had already been copied into the call's
|
||||
arguments, so the test passed regardless of what production retained). The
|
||||
required shape: two candidates in one pass, two generations whose zones
|
||||
cover both but carry **different resolver ports**; the stub `exchangeFn`
|
||||
performs the swap during the *first* attempt (freeing the old generation
|
||||
under `testing.allocator`) and records each attempt's resolver. Assertions:
|
||||
the first attempt's resolver still reads the old generation's port after the
|
||||
swap (the copy, not freed memory), and the second attempt's resolver reads
|
||||
the *new* generation's port (each attempt re-acquires; no table pointer or
|
||||
handle is cached across attempts). With a production `runPass` that retains
|
||||
`*const Zone` or the handle across the release, the first assertion reads
|
||||
freed memory and the second reads the stale generation — either fails.
|
||||
|
||||
- Match found: one PTR query to that zone's resolver through
|
||||
`forward_client.ForwardClient`, built per attempt as `Context.viaForwardZone`
|
||||
builds one (handler.zig:411), with `ptr_read_timeout` (ruling 1).
|
||||
- No match: **no query is sent to anyone** — not the pool, not any resolver.
|
||||
The attempt counts under `no_zone` and retries per ruling 1.
|
||||
|
||||
Scope of the promise, stated precisely (the first draft's "never to a public
|
||||
resolver" claimed more than the code enforces): what this milestone
|
||||
guarantees is that a reverse name reaches **only the resolver of a zone the
|
||||
operator declared**, and never the upstream pool. `parseResolver`
|
||||
(validate.zig:274) accepts any IP literal, so an operator who declares
|
||||
`168.192.in-addr.arpa → udp://8.8.8.8:53` has pointed their private reverse
|
||||
space at a public resolver — that is their declaration, and nxdns does not
|
||||
second-guess it, any more than it second-guesses the same zone for forward
|
||||
queries. Likewise nxdns cannot stop a declared LAN resolver forwarding
|
||||
onward. The docs (ruling 11) say this in one sentence.
|
||||
|
||||
The DNS cache is not consulted and not written: one PTR per client per day is
|
||||
not worth a cache entry, and bypassing the cache keeps the resolver a plain
|
||||
consumer of the exchange seam.
|
||||
|
||||
### 3. Precedence: the operator's name always wins; NXDOMAIN clears
|
||||
|
||||
Schema (ruling 5) keeps learned names in their own column, so precedence is a
|
||||
display rule, not a write conflict — `name` is never written by this feature
|
||||
and `learned_name` is never written by the operator.
|
||||
|
||||
A **candidate** row satisfies both of:
|
||||
|
||||
- `name IS NULL OR name = ''` — a row whose displayed name would come from
|
||||
learning. `hand_edited` does not appear in the predicate (the first draft
|
||||
had it; display precedence never consults it, so candidacy must not
|
||||
either): a hand-edited row the operator grouped but did not name still
|
||||
benefits, and a named row is skipped whatever its flag — its learned name
|
||||
would never be shown.
|
||||
- `name_attempt_after <= now_s` (ruling 6 gives the exact SQL).
|
||||
|
||||
Outcomes of an attempt, written through one repo call (ruling 6), each
|
||||
setting `name_attempt_after` per ruling 1's cadence:
|
||||
|
||||
- **Answer with a valid PTR target**: store it in `learned_name`
|
||||
(lowercased, no trailing dot), overwriting whatever learned name was there —
|
||||
the router is the authority on its own zone, and a changed answer is a
|
||||
renamed device, not a conflict. A test covers the overwrite: two attempts,
|
||||
two different valid targets, second wins.
|
||||
- **NXDOMAIN, or NOERROR with no matching PTR record** (NODATA): clear
|
||||
`learned_name` to NULL. The router affirmatively says the address has no
|
||||
name; keeping a stale one would show a device under its previous owner's
|
||||
hostname after a lease change.
|
||||
- **Everything else** — transport error, any RCODE other than NOERROR and
|
||||
NXDOMAIN (REFUSED, FORMERR, SERVFAIL, unknown values), malformed reply,
|
||||
invalid hostname (ruling 4): keep the stored `learned_name` as it stands,
|
||||
count under `failed` (or `invalid`). An outage must not strip names from
|
||||
the whole dashboard.
|
||||
|
||||
**"RCODE" here is the full 12-bit value**: the header's 4 bits extended by
|
||||
the OPT record's upper 8 bits when the response carries one
|
||||
(`packet.findOptRecord` locates it; `dns/edns.zig` decodes it). A response
|
||||
whose header says NOERROR under a nonzero extended RCODE is a failure, not
|
||||
an answer and not NODATA — classifying on the header nibble alone would
|
||||
store or clear a name on what the resolver called an error. An OPT record
|
||||
that fails to parse makes the whole reply malformed (`failed`, keep). The
|
||||
first implementation had exactly this bug; the regression test is a
|
||||
NOERROR header + OPT with a nonzero extended RCODE, asserting `failed`
|
||||
counts and the stored name survives — it must fail with the classification
|
||||
reverted to header-only.
|
||||
|
||||
**Staleness bound, stated honestly:** an address reassigned to a new device
|
||||
immediately after a successful lookup shows the previous device's hostname for
|
||||
up to `refresh_after_s` (24 h) — the next refresh then overwrites or clears
|
||||
it. That bound is accepted: it matches the prune cadence's granularity, and
|
||||
tightening it means more PTR traffic for a cosmetic lag. The docs state the
|
||||
bound.
|
||||
|
||||
`upsertSeen`, `updateClient`, `pruneStale` and the reconcile engine are
|
||||
untouched: a pruned row takes its learned name with it, a re-materialised
|
||||
device is re-learned within a pass, and `updateClient` writing a name removes
|
||||
the row from candidacy (the name predicate) without touching `learned_name`.
|
||||
|
||||
### 4. PTR answers are untrusted input: `acceptHostname` gates everything
|
||||
|
||||
The answer bytes come from whatever box the operator pointed a zone at, and
|
||||
they land in a UI. Validation has three layers, all required:
|
||||
|
||||
1. **Transport**: `transport.validateResponse` (ID and question echo — that is
|
||||
*all* it checks; the first draft leaned on it for more).
|
||||
2. **Record selection**: walk `packet.answers`; the accepted record is the
|
||||
**first** whose `rtype` is `.ptr`, whose class is `.in`, and whose owner
|
||||
name equals the queried reverse name case-insensitively
|
||||
(`name.eqlIgnoreCase`, name.zig:185). Records failing any of these are
|
||||
skipped, not errors; an answer section with no accepted record under
|
||||
RCODE NOERROR is NODATA (ruling 3 clears). Extra accepted records after
|
||||
the first are ignored.
|
||||
3. **Hostname shape**: the PTR target decodes via `record.rdataCname`
|
||||
(record.zig:101 handles PTR) and formats via `name.formatText`; the text is
|
||||
accepted only if every byte is in `[a-z0-9._-]` after ASCII-lowercasing
|
||||
`A-Z`, labels are 1–63 bytes, the whole name is ≤ 253 bytes, no label
|
||||
starts or ends with `-`, and there is no empty label (which also rejects a
|
||||
trailing dot — `formatText` emits none, so one appearing is malformed).
|
||||
Underscore is included because real DHCP hostnames carry it; nothing else
|
||||
is. A failing target counts under `invalid` and keeps the stored name.
|
||||
|
||||
`acceptHostname` is a pure function in `src/local/reverse_name.zig`, tested
|
||||
against: the empty string, a 254-byte name, a 64-byte label, `a b`, `a\x00b`,
|
||||
`héllo`, `-x`, `x-.y`, `a..b`, `.a`, `a.` (trailing dot), and accepting
|
||||
`nas-1.lan`, `my_printer.home`, `x`.
|
||||
|
||||
### 5. Schema: two runtime columns, not a reuse of `name`
|
||||
|
||||
`clients` gains, in `config_schema.ddl_v1` directly after `name`:
|
||||
|
||||
```sql
|
||||
learned_name TEXT,
|
||||
name_attempt_after INTEGER NOT NULL DEFAULT 0,
|
||||
```
|
||||
|
||||
with the identical lines in PLAN §11.2. `name_attempt_after` is the epoch
|
||||
second before which the resolver will not attempt this row again; 0 (the
|
||||
default, and every pre-existing row) means "eligible now". Storing the *next*
|
||||
attempt rather than the *last* one keeps the candidate predicate a plain
|
||||
comparison (no addition that could overflow on a corrupt value, no special
|
||||
case for "never attempted") and lets one column carry ruling 1's two cadences.
|
||||
|
||||
Reusing `name` with a provenance marker was rejected: `name` feeds
|
||||
`listClients`, which feeds `nxdns export`, and a learned name in an export
|
||||
would turn runtime state into configuration — it would churn export diffs as
|
||||
leases move, and under file authority the next reconcile would fight it. With
|
||||
separate columns:
|
||||
|
||||
- **Export/import untouched.** `listClients` selects `name` only
|
||||
(clients_repo.zig:33-38); no code change, and a test proves an export before
|
||||
and after a learned name lands is byte-identical.
|
||||
- **Reconcile untouched.** The engine never touches runtime columns; the
|
||||
learned columns ride the row exactly as `last_seen` does. Promotion
|
||||
(declaring an observed address) keeps them; deletion of a declared row drops
|
||||
them with the row, correctly.
|
||||
- **File-authority mode needs no write rule.** Naming is a runtime write like
|
||||
`upsertSeen`, permitted in both authority modes for the same reason. In
|
||||
file mode the operator's *declared* name wins through the ordinary
|
||||
precedence (a non-empty `name` removes the row from candidacy after the
|
||||
reconcile writes it), and the learned name stays display-only runtime
|
||||
state.
|
||||
|
||||
The baseline test in `src/storage/migrations.zig` ("a fresh database reaches
|
||||
the baseline...") gains `columnExists` probes for both columns.
|
||||
|
||||
### 6. Repo surface: two calls, owned by the resolver
|
||||
|
||||
`src/storage/repositories/clients_repo.zig`, in the runtime section beside
|
||||
`upsertSeen` / `pruneStale`:
|
||||
|
||||
- `resolveCandidates(database, out, now_s)` — fills a caller-supplied
|
||||
fixed-capacity buffer (capacity `max_per_pass`, each slot
|
||||
`logger.max_client_len` bytes — the bound every address text in this
|
||||
program is sized by) and returns the filled slice or count. **No
|
||||
allocation**: the candidate set is small by construction and the resolver
|
||||
runs on the tracker's task, which owns no allocator today. The WHERE
|
||||
clause, exactly:
|
||||
|
||||
```sql
|
||||
(name IS NULL OR name = '') AND name_attempt_after <= ?1
|
||||
ORDER BY name_attempt_after, ip LIMIT 16
|
||||
```
|
||||
|
||||
Tests cover: `now_s` smaller than any cadence constant still selects
|
||||
never-attempted rows (`DEFAULT 0`); the boundary `name_attempt_after =
|
||||
now_s` selects; `now_s + 1` does not; an extreme stored value
|
||||
(`i64` max) never selects and never traps.
|
||||
- `noteNameOutcome(database, ip, outcome)` where the outcome carries
|
||||
`attempt_after: i64` and one of: store `text`, clear, keep. One UPDATE:
|
||||
always sets `name_attempt_after`; sets `learned_name` to the text on
|
||||
store, to NULL on clear, leaves it alone on keep. **A row deleted between
|
||||
selection and this call makes the UPDATE touch zero rows; that is a
|
||||
no-op by design, not an error and not a counter** — the device left, and
|
||||
nothing was learned about nothing.
|
||||
|
||||
`ClientRow` gains `learned_name: []const u8` (NULL reads as `""`, like
|
||||
`name`); `readClientRow` and both SELECTs carry it. `name_attempt_after` is
|
||||
**not** exposed on `ClientRow` or the API — it is scheduling state with no
|
||||
operator meaning, and keeping it out keeps the contract surface one field.
|
||||
`listClients` (export) is not widened — that is the point of ruling 5.
|
||||
|
||||
### 7. The resolver module: `src/server/client_names.zig`
|
||||
|
||||
New file. A `Resolver` struct owning:
|
||||
|
||||
- `stats: Stats` with the six **outcome** counters `attempted`, `answered`,
|
||||
`nxdomain`, `no_zone`, `invalid`, `failed`, plus two counters *outside* the
|
||||
outcome sum: `read_failures` (the candidate SELECT failed; the pass skips
|
||||
naming) and `write_failures` (`noteNameOutcome` failed; the outcome was
|
||||
still counted). Guarded by the tracker's mutex pattern, with a
|
||||
`snapshotStats`. Invariant, asserted in a test: `attempted` equals
|
||||
`answered + nxdomain + no_zone + invalid + failed`. Database failures log
|
||||
at most one `warn` per pass each for reads and writes, tracker-style
|
||||
(clients.zig:181-186); network failures are the counters' job and log
|
||||
nothing new (the `forward_client` module's existing `debug`-level
|
||||
diagnostics are exempt from this milestone's no-log rule — they predate it,
|
||||
are debug-level, and suppressing them per-caller would fork the client).
|
||||
- an **exchange seam**: production code cannot inject into `ForwardClient`
|
||||
(it is a concrete struct the resolver constructs), so the seam sits above
|
||||
it — a field
|
||||
`exchangeFn: *const fn (io, validate.Resolver, query, response_buf) transport.ExchangeError![]u8`
|
||||
defaulting to a function that builds a stack `ForwardClient` with
|
||||
`ptr_read_timeout` and calls `exchange`. Tests replace the pointer; the
|
||||
production default is itself covered by one test against a loopback UDP
|
||||
socket if one already exists in the suite's patterns, otherwise by the live
|
||||
smoke. The no-zone test runs the **production** pass code with a counting
|
||||
stub and asserts the count stays 0.
|
||||
- a `runPass(io, database, tables, now_s)`: select candidates (ruling 6), and
|
||||
for each — build the reverse name (pure, ruling 8), acquire/match/**copy
|
||||
resolver**/release (ruling 2), build the PTR query: header with a random id
|
||||
filled from `io.random` (`std/Io.zig:2468` — *not* `std.crypto.random`,
|
||||
which 0.16.0 does not have), RD set, one question, qtype `.ptr`, qclass
|
||||
`.in`, encoded with `dns/header` + `dns/question.encode`; exchange through
|
||||
the seam, classify per rulings 3–4, write per ruling 6. The id is
|
||||
test-visible through the seam (the stub sees the query bytes), so no
|
||||
determinism hook is needed beyond the seam itself.
|
||||
|
||||
Wiring: `Tracker.flushOnce` gains an optional `*client_names.Resolver`
|
||||
parameter, invoked **after** the drain/write and prune steps (ruling 1's
|
||||
order); `app.zig` constructs the Resolver beside the tracker (holding the
|
||||
`*LocalTables` pointer) and passes it through `Tracker.run`'s arguments
|
||||
(app.zig:754). The resolver runs on the tracker's task against `tracker_db` —
|
||||
the dedicated-connection rule (clients.zig:127-131) is satisfied because it
|
||||
is the *same* task, serial with the flush. A gated pass (disk critical) skips
|
||||
naming too: naming writes.
|
||||
|
||||
### 8. The pure part: `src/local/reverse_name.zig`
|
||||
|
||||
New file, pure (no Io, no clock). Two functions plus their tests:
|
||||
|
||||
- `reverseName(addr: address.NetAddress, buf: *[max_reverse_len]u8) []const u8`
|
||||
— `d.c.b.a.in-addr.arpa` for v4, the 32-nibble lowercase `ip6.arpa` form for
|
||||
v6. `max_reverse_len` is a comptime bound (v6 form: 32 nibbles · 2 + 8 for
|
||||
`ip6.arpa` = 72; compute, don't hand-wave). An IPv4-mapped v6 address was
|
||||
already canonicalised to `ip4` by `NetAddress` (clients.zig test at :305
|
||||
proves the tracker's addresses are canonical) — assert, don't re-handle.
|
||||
- `acceptHostname(text: []const u8) bool` — ruling 4's gate, with ruling 4's
|
||||
test table.
|
||||
|
||||
`src/platform/address.zig` types are plain values; importing them into
|
||||
`local/` keeps the module pure. Register the file in `src/tests.zig` the way
|
||||
its siblings are registered.
|
||||
|
||||
### 9. Failure visibility: one metrics group
|
||||
|
||||
The resolver must be reachable from `metrics.collect`, which reads
|
||||
`server.WebState` — so `src/web/server.zig`'s `WebState` gains an optional
|
||||
`client_names: ?*client_names.Resolver` collaborator, populated by `app.zig`
|
||||
beside `tracker` (the first draft wired the sample and forgot the state; S2
|
||||
owns `src/web/server.zig` for this field). `src/web/metrics.zig`: `Sample`
|
||||
gains `client_names: ?client_names.Resolver.Stats`, collected beside the
|
||||
tracker's (metrics.zig:191-193) and rendered as
|
||||
`counterGroup(w, "nxdns_client_names_", ...)` beside `nxdns_clients_`
|
||||
(metrics.zig:333-334). `counterGroup` renders every struct field, so all
|
||||
eight counters (six outcomes + `read_failures` + `write_failures`) surface
|
||||
without per-field code. The render test pins one line (e.g.
|
||||
`nxdns_client_names_no_zone_total`).
|
||||
|
||||
### 10. API and UI: learned is visible and visibly different
|
||||
|
||||
- `GET /api/clients` rows carry `learned_name`
|
||||
(`src/web/openapi.yaml` `required` + `properties` for the client schema;
|
||||
contract samples regenerated; `web/src/lib/types.ts` client row type gains
|
||||
`learned_name: string`). `name_attempt_after` stays internal (ruling 6).
|
||||
- `web/src/features/clients/ClientsPage.tsx`: the name cell shows `name` when
|
||||
non-empty; otherwise `learned_name` in the page's existing muted text style
|
||||
with a visually-distinct treatment that marks it as learned (exact
|
||||
affordance is the implementer's, but: no new colour system, reuse the muted
|
||||
style the page already has, and the distinction must survive a screen
|
||||
reader — an `aria-label` or visible suffix, not colour alone). An empty
|
||||
both shows what it shows today.
|
||||
- `ClientEditDialog.tsx` may pre-fill its name field with `learned_name` when
|
||||
`name` is empty — adopting the learned name as a typed name is the natural
|
||||
gesture — but saving still goes through the ordinary PUT and sets
|
||||
`hand_edited` server-side as today. This applies in database-authority mode
|
||||
only; in file mode client edits answer 403 as they do today, and the
|
||||
file-mode story is ruling 5's (declared name wins via candidacy).
|
||||
- Mocks in `ClientsPage.test.tsx` gain the field with distinct values;
|
||||
rendering tests assert: a named row shows `name` and not `learned_name`; an
|
||||
unnamed row shows `learned_name` with the learned affordance.
|
||||
|
||||
### 11. Docs
|
||||
|
||||
- `docs/reference/configuration.md`, clients section: a paragraph stating the
|
||||
learned-name mechanism; that it requires a conditional forward zone
|
||||
covering the LAN's reverse space (with the `168.192.in-addr.arpa` example);
|
||||
that hand-typed names win; that learned names never appear in exports; that
|
||||
a rename or lease change can display stale for up to a day (ruling 3's
|
||||
bound). The cadence must be described as it is, not rounded up: each pass
|
||||
attempts **at most 16 due rows**, and a row is due per ruling 1's two
|
||||
cadences — "every unnamed device, once a minute" overstates both coverage
|
||||
and rate and must not appear; and one sentence that PTR queries go to the declared zone's
|
||||
resolver, wherever the operator pointed it (ruling 2's scope).
|
||||
- The how-to/tutorial page that documents conditional forward zones gains the
|
||||
reverse-zone example if it lacks one (locate it; do not guess its path).
|
||||
- PLAN §7.2 gains one sentence: materialised clients are named by PTR through
|
||||
the declared forward zones; learned names are runtime state.
|
||||
- `docs/reference/api.md` only if it enumerates client row fields (check).
|
||||
|
||||
## Sessions
|
||||
|
||||
Three, strictly sequential: S1 → S2 → S3. S2 needs S1's columns and repo
|
||||
calls; S2 owns the contract regeneration (the API shape changes when S1 widens
|
||||
`ClientRow`, but the sample regen runs the integration suite, which S2's
|
||||
wiring changes — one regen at the end of S2 avoids regenerating twice). S3
|
||||
reads the fields S2's regenerated types carry.
|
||||
|
||||
### Session S1: pure module, schema, repo
|
||||
|
||||
Owns: `src/local/reverse_name.zig` (new), `src/tests.zig` (registration),
|
||||
`src/storage/config_schema.zig`, `src/storage/migrations.zig` (test),
|
||||
`src/storage/repositories/clients_repo.zig`, `PLAN.md` (§11.2 lines only).
|
||||
|
||||
- S1.1 ruling 8: `reverse_name.zig` with exhaustive tests (v4, v6, bounds;
|
||||
`acceptHostname` accept/reject table from ruling 4).
|
||||
- S1.2 ruling 5: the two DDL lines in both copies, baseline column probes.
|
||||
- S1.3 ruling 6: repo calls, `ClientRow` widening, tests — including: the
|
||||
candidate SQL's boundary and extreme-value cases; the ordering; NULL
|
||||
round-trips as `""`; export is byte-stable across a `noteNameOutcome`;
|
||||
`updateClient` leaves `learned_name` alone; the zero-row UPDATE no-op.
|
||||
|
||||
Acceptance (S1):
|
||||
- [ ] `zig build test` passes; baseline test proves both columns exist.
|
||||
- [ ] `reverseName` output for `192.168.1.10` is
|
||||
`10.1.168.192.in-addr.arpa` and for `fd00::1` is the full 32-nibble
|
||||
lowercase form; both are matched by a `forward_zones.Zones` built over
|
||||
`168.192.in-addr.arpa` / a covering `ip6.arpa` zone in a test that ties
|
||||
the two modules together.
|
||||
- [ ] `acceptHostname` passes ruling 4's full accept/reject table, including
|
||||
`a..b`, `.a` and the trailing-dot case.
|
||||
- [ ] An export taken before and after `noteNameOutcome` on an observed row is
|
||||
byte-identical.
|
||||
|
||||
### Session S2: resolver, wiring, metrics, API contract (needs S1)
|
||||
|
||||
Owns: `src/server/client_names.zig` (new), `src/tests.zig` (registration),
|
||||
`src/server/clients.zig`, `src/app.zig`, `src/web/server.zig` (the
|
||||
`client_names` field), `src/web/metrics.zig`, `src/web/openapi.yaml`,
|
||||
`web/src/lib/types.ts`, `web/src/lib/contractSamples.gen.ts` (regenerated),
|
||||
`web/src/features/clients/ClientsPage.test.tsx` (mock fields only).
|
||||
|
||||
- S2.1 ruling 7: the Resolver, the exchange seam, `flushOnce` hook (ruling 1
|
||||
order), app and WebState wiring.
|
||||
- S2.2 rulings 1–4: classification tests against a stub exchange. Tests must
|
||||
cover: answered stores and lowercases; a second answered overwrites;
|
||||
NXDOMAIN clears; NODATA clears; REFUSED and an unknown RCODE keep and count
|
||||
`failed`; timeout keeps; invalid hostname keeps and counts `invalid`; a
|
||||
record with wrong type, class, or owner name is skipped; no-zone sends
|
||||
nothing (the stub's call count is the assertion) and counts;
|
||||
`name_attempt_after` lands at `now + refresh_after_s` for definitive and
|
||||
`now + retry_after_s` for non-definitive outcomes; the candidate cap holds;
|
||||
a named row is never attempted; a gated pass attempts nothing; drain
|
||||
precedes any exchange when every exchange times out (ruling 1); a row past
|
||||
the prune cutoff on a due pass causes no exchange (ruling 1); the
|
||||
two-generation swap test per ruling 2's exact shape (swap inside the
|
||||
hazard window, both assertions); a NOERROR header under a nonzero OPT
|
||||
extended RCODE counts `failed` and keeps the stored name (ruling 3); the
|
||||
default `exchangeFn` bounds UDP plus the TCP fallback under one
|
||||
`ptr_read_timeout` deadline (ruling 1) — a loopback resolver that answers
|
||||
UDP with TC=1 late in the budget and then stalls TCP must fail the attempt
|
||||
within one budget, not two (assert elapsed with margin; inject a shortened
|
||||
budget locally if the default seam needs one).
|
||||
- S2.3 ruling 9: WebState field, metrics sample, render pin.
|
||||
- S2.4 ruling 10's API half: openapi.yaml, regen samples, types.ts, mock
|
||||
fields.
|
||||
|
||||
Acceptance (S2):
|
||||
- [ ] `zig build test` and `zig build test -Dintegration` pass; the
|
||||
regenerated samples carry `learned_name` in the client row sample and
|
||||
do not carry `name_attempt_after`.
|
||||
- [ ] `cd web && npm run typecheck && npm test` pass with the widened types.
|
||||
- [ ] The stats-sum invariant test passes (`attempted` = sum of the five
|
||||
outcomes; `read_failures`/`write_failures` outside it).
|
||||
- [ ] The no-zone test proves zero exchanges for an uncovered reverse name
|
||||
through the production pass code.
|
||||
- [ ] The concurrent-swap regression test passes (ruling 2).
|
||||
|
||||
### Session S3: UI and docs (needs S2)
|
||||
|
||||
Owns: `web/src/features/clients/ClientsPage.tsx`,
|
||||
`web/src/features/clients/ClientEditDialog.tsx`,
|
||||
`web/src/features/clients/ClientsPage.test.tsx` (rendering assertions),
|
||||
`docs/reference/configuration.md`, `docs/reference/api.md` (if applicable),
|
||||
the conditional-forwarding doc page, `PLAN.md` (§7.2 sentence only).
|
||||
|
||||
- S3.1 ruling 10's UI half.
|
||||
- S3.2 ruling 11.
|
||||
|
||||
Acceptance (S3):
|
||||
- [ ] `cd web && npm run typecheck && npm test && npm run lint && npm run
|
||||
build` pass; rendering tests cover named-wins and learned-affordance.
|
||||
- [ ] The docs name the reverse-zone prerequisite with a concrete
|
||||
`in-addr.arpa` example, the one-day staleness bound, and the
|
||||
resolver-is-the-operator's-choice sentence.
|
||||
|
||||
### Orchestrator
|
||||
|
||||
Verify each session's acceptance before starting the next. After S3, the full
|
||||
gate set (`zig build test`, `zig build test -Dintegration`, `test-aarch64` if
|
||||
qemu is present, `cd web && npm test`, `npm run assert-bundled`), then a live
|
||||
smoke per the verify-against-the-real-network rule: a scratch server with a
|
||||
forward zone `168.192.in-addr.arpa → udp://<router>:53` (or a local stub
|
||||
resolver serving PTR), one real query from a LAN client, then wait one flush
|
||||
interval and confirm the learned name in `GET /api/clients`, in the UI, and in
|
||||
`nxdns_client_names_answered_total`. Then delete the zone and confirm the next
|
||||
attempt counts `no_zone` with no upstream traffic (tcpdump or the pool
|
||||
counters — pool queries must not move); with the hourly non-definitive
|
||||
cadence, age `name_attempt_after` by SQL in the scratch database rather than
|
||||
waiting. In database mode, hand-edit the name and confirm it wins and the row
|
||||
stops being attempted. In file mode, confirm a file-declared name wins after
|
||||
reconcile and the learned name remains display-only. Record deviations in
|
||||
`## Recorded (implementation)`.
|
||||
|
||||
## Module layout
|
||||
|
||||
New files: `src/local/reverse_name.zig`, `src/server/client_names.zig`.
|
||||
Deleted surface: none.
|
||||
|
||||
## File ownership
|
||||
|
||||
| File | Session |
|
||||
| --- | --- |
|
||||
| `src/local/reverse_name.zig` (new) | S1 |
|
||||
| `src/storage/config_schema.zig`, `src/storage/migrations.zig` (test) | S1 |
|
||||
| `src/storage/repositories/clients_repo.zig` | S1 |
|
||||
| `PLAN.md` | S1 (§11.2 lines), then S3 (§7.2 sentence) — sequential, never concurrent |
|
||||
| `src/tests.zig` | S1, then S2 — sequential |
|
||||
| `src/server/client_names.zig` (new) | S2 |
|
||||
| `src/server/clients.zig`, `src/app.zig`, `src/web/server.zig`, `src/web/metrics.zig` | S2 |
|
||||
| `src/web/openapi.yaml`, `web/src/lib/types.ts`, `web/src/lib/contractSamples.gen.ts` | S2 |
|
||||
| `web/src/features/clients/ClientsPage.test.tsx` | S2 (mock fields), then S3 (assertions) — sequential |
|
||||
| `web/src/features/clients/ClientsPage.tsx`, `ClientEditDialog.tsx` | S3 |
|
||||
| `docs/reference/configuration.md`, `docs/reference/api.md`, forwarding doc page | S3 |
|
||||
|
||||
## Acceptance (milestone complete)
|
||||
|
||||
- [ ] All session acceptance boxes.
|
||||
- [ ] `config_schema.ddl_v1` and PLAN §11.2 byte-identical, both carrying the
|
||||
two lines; `migrations.steps` still one step, `target_version` still 1.
|
||||
- [ ] The live smoke: a real client's hostname appears without any operator
|
||||
edit; removing the zone stops queries entirely (`no_zone` moves, pool
|
||||
counters do not).
|
||||
- [ ] `nxdns export` output is byte-identical before and after names are
|
||||
learned.
|
||||
- [ ] Database mode: hand-editing a name wins over the learned name
|
||||
immediately and permanently (the row leaves candidacy). File mode: a
|
||||
file-declared name wins after reconcile; the learned name is
|
||||
display-only runtime state.
|
||||
- [ ] `src/web/openapi.yaml` reviewed by hand against the changed client row
|
||||
shape, reviewer says so in `## Recorded` (no automated guard covers
|
||||
schema-to-response agreement — milestone-24 finding, still true).
|
||||
|
||||
## Anti-requirements
|
||||
|
||||
- No migration step, no `ddl_v2`, no runtime schema probing.
|
||||
- No PTR query to the upstream pool, and none for a reverse name no declared
|
||||
forward zone covers — under any fallback, ever. (The declared resolver
|
||||
itself is the operator's choice; ruling 2 scopes the promise.)
|
||||
- No reuse of `clients.name` for learned names, and no learned data in
|
||||
`nxdns export` / `import` ZON or in reconcile semantics.
|
||||
- No mDNS, NetBIOS, DHCP-lease-file parsing, or any second naming source.
|
||||
PTR through declared zones is the mechanism; a router that serves no
|
||||
reverse zone yields `no_zone` counts and an unnamed row, visibly.
|
||||
- No per-query or query-path resolution: naming rides the flush pass only.
|
||||
- No config knobs for cadence, caps, or timeouts; constants per ruling 1.
|
||||
- No DNS-cache participation for PTR probes.
|
||||
- No unbounded anything: candidates per pass, in-flight, per-attempt timeout,
|
||||
name length, and retry cadence are all bounded above by rulings 1 and 4.
|
||||
- No storing or displaying a PTR target that fails `acceptHostname` — not
|
||||
even truncated or escaped.
|
||||
- No new per-attempt log lines from this milestone's code; counters and the
|
||||
metrics group are the surface. `forward_client.zig`'s existing debug
|
||||
diagnostics stand as they are.
|
||||
- No allocation on the naming path: candidate storage is fixed-capacity
|
||||
(ruling 6).
|
||||
|
||||
## Recorded (implementation)
|
||||
|
||||
- `src/server/phase7_integration_test.zig` sits outside the ownership table
|
||||
but gained a mechanical `, null` argument at every `flushOnce`/`run` call
|
||||
site, following the new `names: ?*client_names.Resolver` parameter.
|
||||
- Review: one external round found four defects (extended RCODE ignored the
|
||||
OPT upper bits; `exchangeWithin` did not bound the UDP+TCP pair under one
|
||||
deadline; the swap test asserted nothing observable; a docs cadence
|
||||
sentence). All four were fixed; the re-review of
|
||||
`src/server/client_names.zig` returned no findings.
|
||||
- Every fix shipped with a test the author watched fail with the fix
|
||||
reverted. For the swap test the hazard was injected (a pass-level resolver
|
||||
cache that skips re-acquisition) and the second-port assertion failed; the
|
||||
one-deadline test's bound is `budget + late/2` (550 ms) because the
|
||||
unbounded path takes ~700 ms and a 2×-budget bound would pass it.
|
||||
- `src/web/openapi.yaml` was reviewed by hand against `ClientRow`
|
||||
(clients_repo.zig:312): the nine required fields match one-to-one and
|
||||
`name_attempt_after` is absent from the response, as intended.
|
||||
- Live smoke (scratch server, file mode, zone `127.in-addr.arpa` at a local
|
||||
stub PTR resolver): one real query materialised the client; the flush pass
|
||||
learned `smoke-host.lan` with no operator edit, visible in
|
||||
`GET /api/clients` and `nxdns_client_names_answered_total`. Removing the
|
||||
zone and re-arming `name_attempt_after` produced `no_zone` with zero
|
||||
packets to the stub and no pool movement. The learned name survived a
|
||||
restart's reconcile. A file-declared `name` won in the API after reconcile
|
||||
with the learned name still present as display-only state, and the named
|
||||
row left candidacy (`attempted` stayed 0 over a full flush interval). The
|
||||
database-mode hand-edit path runs the same candidacy SQL
|
||||
(`name IS NULL OR name = ''`) and is covered by the repo unit tests rather
|
||||
than a second live run. The UI half is covered by the ClientsPage
|
||||
rendering tests, not a live browser check.
|
||||
- The first smoke attempt polled a stale pre-milestone binary out of
|
||||
`zig-out/bin` — `zig build test` does not refresh the install step. Rebuild
|
||||
before any live check.
|
||||
+6
-1
@@ -34,6 +34,7 @@ const api_limiter = @import("web/api_limiter.zig");
|
||||
const auth = @import("web/auth.zig");
|
||||
const cert_store = @import("server/cert_store.zig");
|
||||
const cli = @import("cli.zig");
|
||||
const client_names = @import("server/client_names.zig");
|
||||
const clients = @import("server/clients.zig");
|
||||
const config_export = @import("config/export.zig");
|
||||
const db = @import("storage/db.zig");
|
||||
@@ -462,6 +463,9 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
|
||||
var paused: pause.Pause = .{};
|
||||
var tracker: clients.Tracker = .init(cfg.logging.retention_days);
|
||||
// Naming rides the tracker's pass, on the tracker's task and connection
|
||||
// (milestone-25 ruling 1), and reads the live forward zones.
|
||||
var client_names_resolver: client_names.Resolver = .init(&tables);
|
||||
|
||||
// The queue holds waiting tasks in intrusive lists, so neither the buffer
|
||||
// nor the `Logger` may move once a task has touched either.
|
||||
@@ -635,6 +639,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
.handler = &h,
|
||||
.pause = &paused,
|
||||
.tracker = &tracker,
|
||||
.client_names = &client_names_resolver,
|
||||
.manager = &manager,
|
||||
.pool = &pool,
|
||||
.monitor = &monitor,
|
||||
@@ -751,7 +756,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db, gate });
|
||||
try group.concurrent(io, disk_monitor.Monitor.run, .{ &monitor, io });
|
||||
try group.concurrent(io, manager_mod.Manager.runScheduler, .{ &manager, io });
|
||||
try group.concurrent(io, clients.Tracker.run, .{ &tracker, io, &tracker_db, gate });
|
||||
try group.concurrent(io, clients.Tracker.run, .{ &tracker, io, &tracker_db, gate, &client_names_resolver });
|
||||
try group.concurrent(io, runMaintenance, .{ &h, if (web_limiter) |*l| l else null, io });
|
||||
|
||||
// Started last (ruling 26), canceled by the same `group.cancel`; its inner
|
||||
|
||||
+10
-1
@@ -1527,7 +1527,16 @@ test "bare check with no config database exits 2 and says how to make one" {
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "no config database at "));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, config_db_name));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, db_source_hint));
|
||||
try testing.expectEqual(@as(usize, 0), std.mem.count(u8, text, "OK"));
|
||||
// Not a bare count of "OK" over the whole text: this message embeds the
|
||||
// temporary directory path, and `std.testing.tmpDir` names that directory
|
||||
// with base64 over random bytes, so a run whose name happens to carry those
|
||||
// two letters would fail a test that has nothing to do with naming. Every
|
||||
// verdict this command prints — `OK:` and `OK upstreams[...]` alike — opens
|
||||
// a line, so that is what to assert on.
|
||||
var lines = std.mem.splitScalar(u8, text, '\n');
|
||||
while (lines.next()) |line| {
|
||||
try testing.expect(!std.mem.startsWith(u8, line, "OK"));
|
||||
}
|
||||
try testing.expectEqualStrings("", captured.err.written());
|
||||
}
|
||||
|
||||
|
||||
+32
-8
@@ -1,6 +1,9 @@
|
||||
//! The one configuration model. Bootstrap, import, export, the repositories and
|
||||
//! the running server all speak this struct; nothing else describes nxdns
|
||||
//! configuration.
|
||||
//! The one *declarative* configuration model: loading, reconciliation, import,
|
||||
//! export and the running server all speak this struct, and it is the whole
|
||||
//! shape of a config file. It is not the only shape the repositories accept —
|
||||
//! the API edits rows one at a time through narrower inputs such as
|
||||
//! `RuleInput`, `ClientInput` and `ClientEdit`, so a field added here does not
|
||||
//! reach those paths by itself.
|
||||
//!
|
||||
//! Pure: no `std.Io` value is a parameter anywhere, no SQLite, no clock. The
|
||||
//! only `std.Io` types that appear are `std.Io.Duration` as a conversion result.
|
||||
@@ -8,11 +11,21 @@
|
||||
//! Runtime columns are deliberately absent. `clients.first_seen`,
|
||||
//! `clients.last_seen`, `rules.created_at` and
|
||||
//! `blocklist_sources.{last_updated, domain_count, wildcard_count,
|
||||
//! skipped_regex_count, checksum}` are facts a running server produces, not
|
||||
//! configuration. Including them would make two exports taken minutes apart
|
||||
//! differ, which would make the byte-stable round trip untestable against a
|
||||
//! live server. Import sets the timestamps to the import time and leaves the
|
||||
//! counters at their column defaults.
|
||||
//! exception_count, skipped_regex_count, skipped_unsupported_count, checksum}`
|
||||
//! are facts a running server produces, not configuration. Including them would make two exports taken
|
||||
//! minutes apart differ, which would make the byte-stable round trip untestable
|
||||
//! against a live server.
|
||||
//!
|
||||
//! Declarative configuration reaches the database through exactly one path:
|
||||
//! `config/reconcile.zig`. `nxdns import` is a thin wrapper over it, and so is
|
||||
//! `run --config`. Reconciliation asks what changed rather than replacing
|
||||
//! wholesale, so a row the input still names keeps the runtime state attached
|
||||
//! to it: a source matched by url keeps its id, checksum and counters, a rule
|
||||
//! keeps its `created_at`, and a client keeps its first-seen and last-seen
|
||||
//! stamps. The source id and checksum are the two that decide whether a
|
||||
//! file-mode restart reuses the compiled bodies or downloads them again:
|
||||
//! `loadSource` names the files after the id and accepts them only against the
|
||||
//! stored checksum. The counters ride along as reported state.
|
||||
|
||||
const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
@@ -256,20 +269,25 @@ pub const BlocklistSource = struct {
|
||||
|
||||
pub const GroupSource = struct { group: []const u8, source_url: []const u8 };
|
||||
|
||||
/// The three spellings `CHECK(kind IN ('exact','wildcard','regex'))` admits
|
||||
/// after migration step 4.
|
||||
pub const RuleKind = enum {
|
||||
exact,
|
||||
wildcard,
|
||||
regex,
|
||||
|
||||
pub fn toDb(self: RuleKind) []const u8 {
|
||||
return switch (self) {
|
||||
.exact => "exact",
|
||||
.wildcard => "wildcard",
|
||||
.regex => "regex",
|
||||
};
|
||||
}
|
||||
|
||||
pub fn fromDb(text: []const u8) ?RuleKind {
|
||||
if (std.mem.eql(u8, text, "exact")) return .exact;
|
||||
if (std.mem.eql(u8, text, "wildcard")) return .wildcard;
|
||||
if (std.mem.eql(u8, text, "regex")) return .regex;
|
||||
return null;
|
||||
}
|
||||
};
|
||||
@@ -789,6 +807,12 @@ test "every toDb and fromDb enum pair round-trips over all tags" {
|
||||
try expectEnumRoundTrip(RecordType);
|
||||
}
|
||||
|
||||
test "RuleKind carries the third kind through export and import" {
|
||||
try testing.expectEqualStrings("regex", RuleKind.regex.toDb());
|
||||
try testing.expectEqual(RuleKind.regex, RuleKind.fromDb("regex").?);
|
||||
try testing.expect(RuleKind.fromDb("Regex") == null);
|
||||
}
|
||||
|
||||
test "RecordType stores the uppercase DDL spelling" {
|
||||
try testing.expectEqualStrings("A", RecordType.a.toDb());
|
||||
try testing.expectEqualStrings("AAAA", RecordType.aaaa.toDb());
|
||||
|
||||
@@ -3,10 +3,10 @@
|
||||
//!
|
||||
//! The defect this module exists to fix: `import.applyToDb` deletes and
|
||||
//! reinserts every row, `blocklist_sources` included, and the compiled
|
||||
//! blocklists are named after the source row id (`<id>.list` / `<id>.wild`). A
|
||||
//! configuration re-applied on every boot would therefore hand every source a
|
||||
//! new id, orphan every compiled file, and re-download every blocklist on every
|
||||
//! restart.
|
||||
//! blocklists are named after the source row id (`<id>.list` / `<id>.wild` /
|
||||
//! `<id>.allow`). A configuration re-applied on every boot would therefore hand
|
||||
//! every source a new id, orphan every compiled file, and re-download every
|
||||
//! blocklist on every restart.
|
||||
//!
|
||||
//! So nothing is wiped. Every table has an identity; a row the file and the
|
||||
//! database agree on is **updated in place**, keeping its row id and every
|
||||
@@ -1100,7 +1100,9 @@ fn seedSourceStats(database: *db.Db, id: i64) !void {
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 4321,
|
||||
.wildcard_count = 21,
|
||||
.exception_count = 9,
|
||||
.skipped_regex_count = 7,
|
||||
.skipped_unsupported_count = 33,
|
||||
.checksum = "a" ** 64,
|
||||
});
|
||||
}
|
||||
@@ -1169,6 +1171,9 @@ test "a source keeps its id, its checksum and its counters across a reconcile" {
|
||||
try testing.expectEqualStrings("advertising", row.name);
|
||||
try testing.expectEqual(@as(?i64, 1_700_000_000), row.last_updated);
|
||||
try testing.expectEqual(@as(i64, 4321), row.domain_count);
|
||||
try testing.expectEqual(@as(i64, 9), row.exception_count);
|
||||
try testing.expectEqual(@as(i64, 7), row.skipped_regex_count);
|
||||
try testing.expectEqual(@as(i64, 33), row.skipped_unsupported_count);
|
||||
try testing.expectEqualStrings("a" ** 64, row.checksum.?);
|
||||
}
|
||||
|
||||
@@ -1550,6 +1555,50 @@ test "rules keep created_at across a reconcile, duplicates included" {
|
||||
}
|
||||
}
|
||||
|
||||
test "a regex rule declared in the file converges into the table and back out" {
|
||||
var bench: Bench = undefined;
|
||||
try bench.init();
|
||||
defer bench.deinit();
|
||||
|
||||
// Under `.managed_file` authority the API refuses rule writes, so this is
|
||||
// the only way a regex rule reaches the table in that mode. `reconcileRules`
|
||||
// compares the whole tuple and needs no code of its own for the new kind.
|
||||
const with_regex: [:0]const u8 =
|
||||
\\.{
|
||||
\\ .groups = .{ .{ .name = "default" } },
|
||||
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
|
||||
\\ .rules = .{
|
||||
\\ .{ .group = "default", .pattern = "^ad[0-9]+-", .kind = .regex, .action = .block },
|
||||
\\ },
|
||||
\\}
|
||||
;
|
||||
const first = try bench.apply(with_regex, 1_700_000_000);
|
||||
try testing.expectEqual(@as(u32, 1), first.rules.inserted);
|
||||
|
||||
const gpa = testing.allocator;
|
||||
var rows = try rules_repo.listRuleRows(&bench.database, gpa);
|
||||
defer rows.deinit(gpa);
|
||||
defer rules_repo.freeRuleRows(gpa, rows.items);
|
||||
try testing.expectEqual(@as(usize, 1), rows.items.len);
|
||||
try testing.expectEqual(model.RuleKind.regex, rows.items[0].kind);
|
||||
try testing.expectEqualStrings("^ad[0-9]+-", rows.items[0].pattern);
|
||||
|
||||
// Idempotent: the tuple matches itself, so a second pass writes nothing.
|
||||
const second = try bench.apply(with_regex, 1_800_000_000);
|
||||
try testing.expectEqual(@as(u32, 0), second.rules.total());
|
||||
|
||||
// And a file that stops declaring it takes the row with it.
|
||||
const without: [:0]const u8 =
|
||||
\\.{
|
||||
\\ .groups = .{ .{ .name = "default" } },
|
||||
\\ .upstreams = .{ .{ .url = "https://dns.example/dns-query" } },
|
||||
\\}
|
||||
;
|
||||
const third = try bench.apply(without, 1_900_000_000);
|
||||
try testing.expectEqual(@as(u32, 1), third.rules.deleted);
|
||||
try testing.expectEqual(@as(i64, 0), try rules_repo.countRules(&bench.database));
|
||||
}
|
||||
|
||||
test "dropping one of two identical rules removes exactly one row" {
|
||||
var bench: Bench = undefined;
|
||||
try bench.init();
|
||||
|
||||
+111
-4
@@ -44,6 +44,7 @@ const Writer = std.Io.Writer;
|
||||
const model = @import("model.zig");
|
||||
const address = @import("../platform/address.zig");
|
||||
const dns_name = @import("../dns/name.zig");
|
||||
const regex = @import("../filter/regex.zig");
|
||||
const safe_url = @import("../safe_url.zig");
|
||||
const transport = @import("../upstream/transport.zig");
|
||||
|
||||
@@ -890,13 +891,31 @@ fn checkCollections(cfg: Config, diags: *Diagnostics, scratch: Allocator) error{
|
||||
|
||||
for (cfg.rules, 0..) |rule, i| {
|
||||
try checkGroupRef(diags, &group_names, rule.group, "rules[{d}].group", .{i});
|
||||
if (!try patternIsValid(scratch, rule.pattern, rule.kind)) {
|
||||
// `null` is a good pattern; anything else is the sentence fragment that
|
||||
// says which of the regex engine's limits refused it. The empty string
|
||||
// is a plain syntax refusal, which is the only verdict the exact and
|
||||
// wildcard kinds can reach.
|
||||
const detail: ?[]const u8 = if (patternIsValid(scratch, rule.pattern, rule.kind)) |valid|
|
||||
(if (valid) null else "")
|
||||
else |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
error.BadPattern => "",
|
||||
error.PatternTooLong => std.fmt.comptimePrint(
|
||||
" (over {d} bytes)",
|
||||
.{regex.max_pattern_len},
|
||||
),
|
||||
error.PatternTooComplex => std.fmt.comptimePrint(
|
||||
" (over {d} compiled instructions)",
|
||||
.{regex.max_program_len},
|
||||
),
|
||||
};
|
||||
if (detail) |suffix| {
|
||||
try diags.add(
|
||||
error.BadRulePattern,
|
||||
"rules[{d}].pattern",
|
||||
.{i},
|
||||
"{f} is not a valid {s} pattern",
|
||||
.{ safe_url.quoteText(rule.pattern), rule.kind.toDb() },
|
||||
"{f} is not a valid {s} pattern{s}",
|
||||
.{ safe_url.quoteText(rule.pattern), rule.kind.toDb(), suffix },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1096,11 +1115,18 @@ fn sourceUrlIsValid(url: []const u8) bool {
|
||||
/// Syntax only. Matching semantics are Phase 5's: an exact pattern carries no
|
||||
/// `*` at all, a wildcard pattern carries at least one label that is exactly
|
||||
/// `*`, and every remaining label must survive `dns.name.fromText`.
|
||||
///
|
||||
/// A regex pattern is validated by compiling it, and its three refusals arrive
|
||||
/// as errors rather than as `false` so the caller can name the one that fired.
|
||||
/// The distinction is the operator's, not the compiler's: "not a valid regex
|
||||
/// pattern" sends someone hunting for a typo in a pattern whose only fault is
|
||||
/// that it is longer than `regex.max_pattern_len` or wider than
|
||||
/// `regex.max_program_len`, and neither limit is visible in the pattern text.
|
||||
fn patternIsValid(
|
||||
scratch: Allocator,
|
||||
pattern: []const u8,
|
||||
kind: model.RuleKind,
|
||||
) error{OutOfMemory}!bool {
|
||||
) regex.Error!bool {
|
||||
switch (kind) {
|
||||
.exact => {
|
||||
if (std.mem.findScalar(u8, pattern, '*') != null) return false;
|
||||
@@ -1126,6 +1152,11 @@ fn patternIsValid(
|
||||
_ = dns_name.fromText(substituted.items) catch return false;
|
||||
return true;
|
||||
},
|
||||
.regex => {
|
||||
var program = try regex.compile(scratch, pattern);
|
||||
program.deinit(scratch);
|
||||
return true;
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2105,6 +2136,82 @@ test "rule patterns accept wildcards only when the kind says so" {
|
||||
try expectProblem(bad_label, error.BadRulePattern, "rules[0].pattern");
|
||||
}
|
||||
|
||||
/// The message of the first failure, so a test can assert the sentence an
|
||||
/// operator reads and not only the error tag.
|
||||
fn expectMessage(cfg: Config, expected: ValidateError, expected_message: []const u8) !void {
|
||||
var diags: Diagnostics = .init(testing.allocator);
|
||||
defer diags.deinit();
|
||||
try testing.expectError(expected, validate(cfg, &diags));
|
||||
const failure = diags.firstFailure() orelse return error.TestExpectedFailure;
|
||||
try testing.expectEqualStrings(expected_message, failure.message);
|
||||
}
|
||||
|
||||
/// The tail of the first failure's message. `quoteText` truncates the value it
|
||||
/// quotes at `safe_url.max_len`, so a diagnostic about an over-long pattern
|
||||
/// cannot be matched whole.
|
||||
fn expectMessageSuffix(cfg: Config, expected: ValidateError, expected_suffix: []const u8) !void {
|
||||
var diags: Diagnostics = .init(testing.allocator);
|
||||
defer diags.deinit();
|
||||
try testing.expectError(expected, validate(cfg, &diags));
|
||||
const failure = diags.firstFailure() orelse return error.TestExpectedFailure;
|
||||
if (!std.mem.endsWith(u8, failure.message, expected_suffix)) {
|
||||
std.debug.print("message {s} does not end with {s}\n", .{ failure.message, expected_suffix });
|
||||
return error.TestExpectedEqual;
|
||||
}
|
||||
}
|
||||
|
||||
fn regexRule(pattern: []const u8) [1]model.Rule {
|
||||
return .{.{ .group = "default", .pattern = pattern, .kind = .regex, .action = .block }};
|
||||
}
|
||||
|
||||
test "a regex rule is validated by compiling it" {
|
||||
var cfg = baseConfig();
|
||||
const good = regexRule("^ad[0-9]+-\\.(example|test)\\.com$");
|
||||
cfg.rules = &good;
|
||||
try expectClean(cfg);
|
||||
|
||||
// A regex is not a name: the wildcard and exact kinds reject `*`, and this
|
||||
// one has to accept the characters that make a pattern a pattern.
|
||||
const starred = regexRule("ads.*\\.example");
|
||||
cfg.rules = &starred;
|
||||
try expectClean(cfg);
|
||||
}
|
||||
|
||||
test "each of the regex engine's three refusals names itself in the diagnostic" {
|
||||
var cfg = baseConfig();
|
||||
|
||||
const unclosed = regexRule("(");
|
||||
cfg.rules = &unclosed;
|
||||
try expectProblem(cfg, error.BadRulePattern, "rules[0].pattern");
|
||||
try expectMessage(cfg, error.BadRulePattern, "'(' is not a valid regex pattern");
|
||||
|
||||
// Too long and too complex are the two an operator cannot see by reading
|
||||
// the pattern, so the message has to carry the limit that fired.
|
||||
const too_long = regexRule("a" ** (regex.max_pattern_len + 1));
|
||||
cfg.rules = &too_long;
|
||||
try expectMessageSuffix(
|
||||
cfg,
|
||||
error.BadRulePattern,
|
||||
"is not a valid regex pattern (over 256 bytes)",
|
||||
);
|
||||
|
||||
// Well inside 256 bytes of pattern, well past 1024 instructions of program.
|
||||
const too_complex = regexRule("(abcdefghij){200}");
|
||||
cfg.rules = &too_complex;
|
||||
try expectMessage(
|
||||
cfg,
|
||||
error.BadRulePattern,
|
||||
"'(abcdefghij){200}' is not a valid regex pattern (over 1024 compiled instructions)",
|
||||
);
|
||||
}
|
||||
|
||||
test "an empty regex pattern is refused rather than matching every name" {
|
||||
var cfg = baseConfig();
|
||||
const empty = regexRule("");
|
||||
cfg.rules = ∅
|
||||
try expectProblem(cfg, error.BadRulePattern, "rules[0].pattern");
|
||||
}
|
||||
|
||||
test "parseResolver accepts udp and tcp with an IP literal and a port" {
|
||||
const udp4 = try parseResolver("udp://192.168.1.1:53");
|
||||
try testing.expectEqual(ResolverScheme.udp, udp4.scheme);
|
||||
|
||||
+199
-44
@@ -1,14 +1,15 @@
|
||||
//! Compiles a downloaded blocklist into the two bodies nxdns stores on disk:
|
||||
//! a `.list` body of exact names and a `.wild` body of suffixes.
|
||||
//! Compiles a downloaded blocklist into the three bodies nxdns stores on disk:
|
||||
//! a `.list` body of exact names, a `.wild` body of suffixes and an `.allow`
|
||||
//! body of the names the list's `@@` exceptions lift.
|
||||
//!
|
||||
//! Pure over reader/writer interfaces: an allocator, a `*std.Io.Reader` and two
|
||||
//! `*std.Io.Writer`. No `std.Io` value, no file, no clock. A compiled body is a
|
||||
//! pure function of (bytes, format), which is what makes two runs — and two
|
||||
//! Pure over reader/writer interfaces: an allocator, a `*std.Io.Reader` and
|
||||
//! three `*std.Io.Writer`. No `std.Io` value, no file, no clock. A compiled body
|
||||
//! is a pure function of (bytes, format), which is what makes two runs — and two
|
||||
//! permutations of the same input — byte-identical.
|
||||
//!
|
||||
//! Nothing but the sorted, deduplicated names is written: no header, no
|
||||
//! timestamp, no counts. The header belongs to the caller, and the checksum
|
||||
//! covers the two bodies only.
|
||||
//! covers the three bodies only.
|
||||
|
||||
const std = @import("std");
|
||||
const parsers = @import("parsers.zig");
|
||||
@@ -23,6 +24,12 @@ pub const max_line_len: usize = 4096;
|
||||
pub const Counts = struct {
|
||||
domains: u32 = 0,
|
||||
wildcards: u32 = 0,
|
||||
/// Written, deduplicated `.allow` entries: the names this list's `@@`
|
||||
/// exceptions lift out of what other lists block.
|
||||
exceptions: u32 = 0,
|
||||
/// Regex lines this list carried, counted and skipped. nxdns has an engine
|
||||
/// for them now, but it stays reserved for operator rules: a downloaded list
|
||||
/// is other people's patterns, and PLAN §2.2 keeps them out.
|
||||
skipped_regex: u32 = 0,
|
||||
skipped_unsupported: u32 = 0,
|
||||
/// Not a valid domain name (`dns.name.fromText` rejected it, a non-ASCII
|
||||
@@ -36,30 +43,49 @@ pub const Counts = struct {
|
||||
|
||||
pub const Result = struct {
|
||||
counts: Counts,
|
||||
/// Lowercase hex sha256 over the `.list` body followed by the `.wild` body.
|
||||
/// Lowercase hex sha256 over the `.list` body, the `.wild` body and the
|
||||
/// `.allow` body in that order, each followed by `body_separator`.
|
||||
///
|
||||
/// The separator is what makes the digest identify a compile. Without it
|
||||
/// the three bodies concatenate ambiguously: a wildcard is stored with its
|
||||
/// `*.` stripped, so an upstream that changes `a.example` to `*.a.example`
|
||||
/// moves the same bytes from the `.list` body to the `.wild` body and
|
||||
/// hashes to the same digest. `Manager.diskBodiesMatch` would then accept
|
||||
/// the stale files, the refresh would keep them, and the wildcard would
|
||||
/// never take effect.
|
||||
checksum: [64]u8,
|
||||
};
|
||||
|
||||
/// Fed to the checksum hasher after each of the three bodies, so the digest
|
||||
/// reads them as three fields rather than one run of bytes.
|
||||
///
|
||||
/// `0x00` is sound as a separator because it can never occur inside a body:
|
||||
/// `addCandidate` rejects every control byte and every byte at or above `0x80`,
|
||||
/// so a body holds none. Any producer of this digest must use this constant —
|
||||
/// `compiler.compile` hashes while it emits and `Manager.bodyChecksum` hashes
|
||||
/// three finished buffers, and a one-byte disagreement between them would make
|
||||
/// every refresh republish for ever.
|
||||
pub const body_separator = "\x00";
|
||||
|
||||
pub const Error = error{ OutOfMemory, TooManyDomains, ReadFailed, WriteFailed };
|
||||
|
||||
/// Reads `r` to end of stream and writes the two compiled bodies.
|
||||
/// Reads `r` to end of stream and writes the three compiled bodies.
|
||||
///
|
||||
/// `counts.domains` and `counts.wildcards` are the written, deduplicated
|
||||
/// counts: they are what `blocklist_sources.domain_count` and `wildcard_count`
|
||||
/// store and what the UI shows.
|
||||
/// `counts.domains`, `counts.wildcards` and `counts.exceptions` are the written,
|
||||
/// deduplicated counts: they are what `blocklist_sources.domain_count`,
|
||||
/// `wildcard_count` and `exception_count` store and what the UI shows.
|
||||
pub fn compile(
|
||||
gpa: std.mem.Allocator,
|
||||
r: *std.Io.Reader,
|
||||
format: parsers.Format,
|
||||
list_w: *std.Io.Writer,
|
||||
wild_w: *std.Io.Writer,
|
||||
allow_w: *std.Io.Writer,
|
||||
) Error!Result {
|
||||
var counts: Counts = .{};
|
||||
|
||||
var list: Entries = .{};
|
||||
defer list.deinit(gpa);
|
||||
var wild: Entries = .{};
|
||||
defer wild.deinit(gpa);
|
||||
var bodies: Bodies = .{};
|
||||
defer bodies.deinit(gpa);
|
||||
|
||||
while (try parsers.nextBoundedLine(r, max_line_len)) |event| {
|
||||
const raw = switch (event) {
|
||||
@@ -81,32 +107,30 @@ pub fn compile(
|
||||
.domain => {
|
||||
var fields = std.mem.tokenizeAny(u8, parsed.text, &std.ascii.whitespace);
|
||||
while (fields.next()) |field| {
|
||||
try addCandidate(gpa, field, false, false, &list, &wild, &counts);
|
||||
try addCandidate(gpa, field, parsed, &bodies, &counts);
|
||||
}
|
||||
},
|
||||
.wildcard => try addCandidate(
|
||||
gpa,
|
||||
parsed.text,
|
||||
true,
|
||||
parsed.covers_apex,
|
||||
&list,
|
||||
&wild,
|
||||
&counts,
|
||||
),
|
||||
.wildcard, .exception => try addCandidate(gpa, parsed.text, parsed, &bodies, &counts),
|
||||
}
|
||||
}
|
||||
|
||||
// Each body is followed by `body_separator`, which is what keeps the digest
|
||||
// from confusing a name in one body with the same name in another.
|
||||
var hasher = Sha256.init(.{});
|
||||
counts.domains = try emit(&list, list_w, &hasher, &counts.duplicates);
|
||||
counts.wildcards = try emit(&wild, wild_w, &hasher, &counts.duplicates);
|
||||
counts.domains = try emit(&bodies.list, list_w, &hasher, &counts.duplicates);
|
||||
hasher.update(body_separator);
|
||||
counts.wildcards = try emit(&bodies.wild, wild_w, &hasher, &counts.duplicates);
|
||||
hasher.update(body_separator);
|
||||
counts.exceptions = try emit(&bodies.allow, allow_w, &hasher, &counts.duplicates);
|
||||
hasher.update(body_separator);
|
||||
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
return .{ .counts = counts, .checksum = std.fmt.bytesToHex(digest, .lower) };
|
||||
}
|
||||
|
||||
/// Normalizes one whitespace-separated candidate and files it under `.list`,
|
||||
/// `.wild`, or neither.
|
||||
/// Normalizes one whitespace-separated candidate of `line` and files it under
|
||||
/// `.list`, `.wild`, `.allow`, or nowhere.
|
||||
///
|
||||
/// The normalization below is deliberately not `dns.name.normalizeText`: this
|
||||
/// one adds the two-label minimum, rejects control bytes, and reports every
|
||||
@@ -114,20 +138,19 @@ pub fn compile(
|
||||
fn addCandidate(
|
||||
gpa: std.mem.Allocator,
|
||||
field: []const u8,
|
||||
from_wildcard_line: bool,
|
||||
covers_apex: bool,
|
||||
list: *Entries,
|
||||
wild: *Entries,
|
||||
line: parsers.Line,
|
||||
bodies: *Bodies,
|
||||
counts: *Counts,
|
||||
) Error!void {
|
||||
var candidate = field;
|
||||
var is_wildcard = from_wildcard_line;
|
||||
var is_wildcard = line.kind == .wildcard;
|
||||
if (std.mem.startsWith(u8, candidate, "*.")) {
|
||||
is_wildcard = true;
|
||||
candidate = candidate[2..];
|
||||
}
|
||||
// A '*' anywhere else makes this a pattern, and patterns belong to the
|
||||
// `rules` table; a blocklist entry is a name or a suffix.
|
||||
// `rules` table, where the operator writes them as a `.wildcard` or a
|
||||
// `.regex`; a blocklist entry is a name or a suffix.
|
||||
if (std.mem.indexOfScalar(u8, candidate, '*') != null) {
|
||||
counts.invalid += 1;
|
||||
return;
|
||||
@@ -163,12 +186,17 @@ fn addCandidate(
|
||||
return;
|
||||
}
|
||||
|
||||
if (is_wildcard) {
|
||||
try wild.append(gpa, normalized);
|
||||
// An exception needs no apex entry beside its suffix entry: the matcher
|
||||
// walks the `.allow` set over the full name and every parent, so one entry
|
||||
// lifts `x` and every subdomain of it at once.
|
||||
if (line.kind == .exception) {
|
||||
try bodies.allow.append(gpa, normalized);
|
||||
} else if (is_wildcard) {
|
||||
try bodies.wild.append(gpa, normalized);
|
||||
// An ABP `||x^` rule covers `x` itself as well as its subdomains.
|
||||
if (covers_apex) try list.append(gpa, normalized);
|
||||
if (line.covers_apex) try bodies.list.append(gpa, normalized);
|
||||
} else {
|
||||
try list.append(gpa, normalized);
|
||||
try bodies.list.append(gpa, normalized);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,6 +230,20 @@ fn emit(
|
||||
return written;
|
||||
}
|
||||
|
||||
/// The three bodies under construction, in the order they are written and
|
||||
/// hashed.
|
||||
const Bodies = struct {
|
||||
list: Entries = .{},
|
||||
wild: Entries = .{},
|
||||
allow: Entries = .{},
|
||||
|
||||
fn deinit(self: *Bodies, gpa: std.mem.Allocator) void {
|
||||
self.list.deinit(gpa);
|
||||
self.wild.deinit(gpa);
|
||||
self.allow.deinit(gpa);
|
||||
}
|
||||
};
|
||||
|
||||
/// Length-prefixed candidate bytes plus the offsets that index them. Sorting
|
||||
/// permutes the offsets, so the bytes never move.
|
||||
const Entries = struct {
|
||||
@@ -247,10 +289,12 @@ const Compiled = struct {
|
||||
result: Result,
|
||||
list_w: std.Io.Writer.Allocating,
|
||||
wild_w: std.Io.Writer.Allocating,
|
||||
allow_w: std.Io.Writer.Allocating,
|
||||
|
||||
fn deinit(self: *Compiled) void {
|
||||
self.list_w.deinit();
|
||||
self.wild_w.deinit();
|
||||
self.allow_w.deinit();
|
||||
}
|
||||
|
||||
fn list(self: *Compiled) []const u8 {
|
||||
@@ -260,6 +304,10 @@ const Compiled = struct {
|
||||
fn wild(self: *Compiled) []const u8 {
|
||||
return self.wild_w.written();
|
||||
}
|
||||
|
||||
fn allow(self: *Compiled) []const u8 {
|
||||
return self.allow_w.written();
|
||||
}
|
||||
};
|
||||
|
||||
fn compileText(gpa: std.mem.Allocator, text: []const u8, format: parsers.Format) Error!Compiled {
|
||||
@@ -272,8 +320,10 @@ fn compileReader(gpa: std.mem.Allocator, r: *std.Io.Reader, format: parsers.Form
|
||||
errdefer list_w.deinit();
|
||||
var wild_w: std.Io.Writer.Allocating = .init(gpa);
|
||||
errdefer wild_w.deinit();
|
||||
const result = try compile(gpa, r, format, &list_w.writer, &wild_w.writer);
|
||||
return .{ .result = result, .list_w = list_w, .wild_w = wild_w };
|
||||
var allow_w: std.Io.Writer.Allocating = .init(gpa);
|
||||
errdefer allow_w.deinit();
|
||||
const result = try compile(gpa, r, format, &list_w.writer, &wild_w.writer, &allow_w.writer);
|
||||
return .{ .result = result, .list_w = list_w, .wild_w = wild_w, .allow_w = allow_w };
|
||||
}
|
||||
|
||||
const hosts_fixture =
|
||||
@@ -339,10 +389,110 @@ test "abp apex rule lands in both bodies" {
|
||||
|
||||
try testing.expectEqualStrings("bare.com\nx.com\n", c.list());
|
||||
try testing.expectEqualStrings("x.com\n", c.wild());
|
||||
try testing.expectEqualStrings("z.com\n", c.allow());
|
||||
try testing.expectEqual(@as(u32, 2), c.result.counts.domains);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.wildcards);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.exceptions);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_regex);
|
||||
try testing.expectEqual(@as(u32, 2), c.result.counts.skipped_unsupported);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_unsupported);
|
||||
}
|
||||
|
||||
test "an abp list's hosts-style lines reach the domain body through the split" {
|
||||
// The case `parser_abp` defers here: it hands a whitespace-carrying bare
|
||||
// candidate over whole, and only the tokenization in `compile` files the
|
||||
// name out of it. A mixed list — `!` header and `||` rules, so `detectFormat`
|
||||
// calls the whole file `abp`, plus the hosts lines such lists carry — reaches
|
||||
// a compiled body no other way, and no parser test can see it happen.
|
||||
const fixture =
|
||||
"! Title: mixed\n" ++
|
||||
"||blocked.example^\n" ++
|
||||
"0.0.0.0 ads.example\n" ++
|
||||
"127.0.0.1 localhost\n";
|
||||
|
||||
var c = try compileText(testing.allocator, fixture, .abp);
|
||||
defer c.deinit();
|
||||
|
||||
// The address field is filed as a name of its own: abp lines have no hosts
|
||||
// framing, so the compiler cannot know which field is the address. `0.0.0.0`
|
||||
// and `127.0.0.1` are names nobody resolves, which is why the split is worth
|
||||
// more than the two spurious entries cost.
|
||||
try testing.expectEqualStrings(
|
||||
"0.0.0.0\n127.0.0.1\nads.example\nblocked.example\n",
|
||||
c.list(),
|
||||
);
|
||||
try testing.expectEqualStrings("blocked.example\n", c.wild());
|
||||
try testing.expectEqual(@as(u32, 4), c.result.counts.domains);
|
||||
// `localhost` is the one label the two-label minimum drops.
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.invalid);
|
||||
}
|
||||
|
||||
test "the allow body is sorted, deduplicated and normalized like the others" {
|
||||
const fixture =
|
||||
"@@||GOOD.ads.example^\n" ++
|
||||
"@@||a.ads.example^$important\n" ++
|
||||
"@@||good.ads.example\n" ++
|
||||
"@@||localhost^\n" ++
|
||||
"@@||bad*.ads.example^\n" ++
|
||||
"||ads.example^\n";
|
||||
|
||||
var c = try compileText(testing.allocator, fixture, .abp);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqualStrings("a.ads.example\ngood.ads.example\n", c.allow());
|
||||
try testing.expectEqual(@as(u32, 2), c.result.counts.exceptions);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.duplicates);
|
||||
// `localhost` is one label, and the starred name is not a name at all.
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.invalid);
|
||||
try testing.expectEqual(@as(u32, 1), c.result.counts.skipped_unsupported);
|
||||
|
||||
// The exceptions changed neither block body.
|
||||
try testing.expectEqualStrings("ads.example\n", c.list());
|
||||
try testing.expectEqualStrings("ads.example\n", c.wild());
|
||||
}
|
||||
|
||||
test "moving a name between bodies changes the checksum" {
|
||||
// The collision the separator exists to prevent. A wildcard is stored with
|
||||
// its `*.` stripped, so both compiles write the bytes `a.example\n` — one
|
||||
// into the `.list` body, one into the `.wild` body. Unframed, the two hash
|
||||
// identically, `diskBodiesMatch` accepts the stale files, and an upstream
|
||||
// that switched a name to a wildcard never takes effect.
|
||||
var exact = try compileText(testing.allocator, "a.example\n", .domains);
|
||||
defer exact.deinit();
|
||||
var wild = try compileText(testing.allocator, "*.a.example\n", .domains);
|
||||
defer wild.deinit();
|
||||
|
||||
try testing.expectEqualStrings("a.example\n", exact.list());
|
||||
try testing.expectEqualStrings("", exact.wild());
|
||||
try testing.expectEqualStrings("", wild.list());
|
||||
try testing.expectEqualStrings("a.example\n", wild.wild());
|
||||
|
||||
try testing.expect(!std.mem.eql(u8, &exact.result.checksum, &wild.result.checksum));
|
||||
}
|
||||
|
||||
test "the checksum of a source with exceptions covers all three bodies in order" {
|
||||
const fixture =
|
||||
"||ads.example^\n" ++
|
||||
"@@||good.ads.example^\n";
|
||||
|
||||
var c = try compileText(testing.allocator, fixture, .abp);
|
||||
defer c.deinit();
|
||||
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(c.list());
|
||||
hasher.update(body_separator);
|
||||
hasher.update(c.wild());
|
||||
hasher.update(body_separator);
|
||||
hasher.update(c.allow());
|
||||
hasher.update(body_separator);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &c.result.checksum);
|
||||
|
||||
// A non-empty allow body does move the digest, so a list that gains an
|
||||
// exception is recompiled rather than silently kept.
|
||||
var without = try compileText(testing.allocator, "||ads.example^\n", .abp);
|
||||
defer without.deinit();
|
||||
try testing.expect(!std.mem.eql(u8, &c.result.checksum, &without.result.checksum));
|
||||
}
|
||||
|
||||
test "two runs of the same input are byte-identical" {
|
||||
@@ -353,6 +503,7 @@ test "two runs of the same input are byte-identical" {
|
||||
|
||||
try testing.expectEqualStrings(a.list(), b.list());
|
||||
try testing.expectEqualStrings(a.wild(), b.wild());
|
||||
try testing.expectEqualStrings(a.allow(), b.allow());
|
||||
try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum);
|
||||
}
|
||||
|
||||
@@ -374,6 +525,7 @@ test "a permutation of the input compiles to the same bodies" {
|
||||
|
||||
try testing.expectEqualStrings(a.list(), b.list());
|
||||
try testing.expectEqualStrings(a.wild(), b.wild());
|
||||
try testing.expectEqualStrings(a.allow(), b.allow());
|
||||
try testing.expectEqualSlices(u8, &a.result.checksum, &b.result.checksum);
|
||||
}
|
||||
|
||||
@@ -489,14 +641,17 @@ test "carriage returns are stripped" {
|
||||
try testing.expectEqualStrings("a.example.com\nb.example.com\n", c.list());
|
||||
}
|
||||
|
||||
test "empty input produces empty bodies and the sha256 of the empty string" {
|
||||
test "empty input produces empty bodies and the digest of three separators" {
|
||||
var c = try compileText(testing.allocator, "", .domains);
|
||||
defer c.deinit();
|
||||
|
||||
try testing.expectEqualStrings("", c.list());
|
||||
try testing.expectEqualStrings("", c.wild());
|
||||
try testing.expectEqualStrings("", c.allow());
|
||||
// Three empty bodies still hash their three separators, so an empty compile
|
||||
// has a digest of its own rather than the sha256 of the empty string.
|
||||
try testing.expectEqualStrings(
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"709e80c88487a2411e1ee4dfb9f22a861492d20c4765150c0c794abd70f8147c",
|
||||
&c.result.checksum,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -33,10 +33,13 @@ const sources_repo = @import("../storage/repositories/sources_repo.zig");
|
||||
|
||||
const compiler = @import("compiler.zig");
|
||||
const fetcher = @import("fetcher.zig");
|
||||
const parsers = @import("parsers.zig");
|
||||
const manager = @import("manager.zig");
|
||||
const matcher = @import("matcher.zig");
|
||||
const response = @import("response.zig");
|
||||
|
||||
const lookup = @import("../web/handlers/lookup.zig");
|
||||
|
||||
const forward_client = @import("../local/forward_client.zig");
|
||||
const forward_zones = @import("../local/forward_zones.zig");
|
||||
const records = @import("../local/records.zig");
|
||||
@@ -60,6 +63,11 @@ const budget: std.Io.Clock.Duration = .{ .raw = .fromSeconds(10), .clock = .awak
|
||||
/// The forward-zone read timeout. Case 15 asserts a silent resolver gives up
|
||||
/// inside twice this, so it has to be short enough to keep the run quick and
|
||||
/// long enough that a loopback answer always beats it.
|
||||
/// `/api/lookup` reports the local tables beside the filter decision; this
|
||||
/// suite's cases are about the filter half, so both are empty here.
|
||||
const empty_records: records.Records = .empty;
|
||||
const empty_zones: forward_zones.Zones = .empty;
|
||||
|
||||
const read_timeout: std.Io.Clock.Duration = .{ .raw = .fromMilliseconds(200), .clock = .awake };
|
||||
|
||||
const file_limit: std.Io.Limit = .limited(8 * 1024 * 1024);
|
||||
@@ -111,49 +119,88 @@ const http_body =
|
||||
|
||||
const http_domains: i64 = 2;
|
||||
const http_wildcards: i64 = 1;
|
||||
const http_exceptions: i64 = 0;
|
||||
const http_regex: i64 = 1;
|
||||
/// Zero, and asserted rather than assumed: a hosts list carries no line a DNS
|
||||
/// sinkhole cannot translate, which is what makes the abp fixture below a
|
||||
/// separate source instead of two more lines in this one.
|
||||
const http_unsupported: i64 = 0;
|
||||
|
||||
/// Compiles `text` into `<base>.list` and `<base>.wild` under `dir`, exactly as
|
||||
/// the manager's compile stage does, and returns the compiler's own result.
|
||||
/// An ABP-format list: one element-hiding rule and one `$`-modifier rule that
|
||||
/// nxdns counts and skips, beside two names it blocks. `detectFormat` assigns
|
||||
/// one format to a whole source, so these lines cannot join `http_body` — a
|
||||
/// single `##` there would re-parse every hosts line as ABP.
|
||||
const abp_body =
|
||||
"! small abp list\n" ++
|
||||
"##.ad-banner\n" ++
|
||||
"||ads.example^$third-party\n" ++
|
||||
"||blocked.example^\n" ++
|
||||
"tracker.example\n";
|
||||
|
||||
/// `||blocked.example^` covers its own apex, so it writes one `.list` entry
|
||||
/// beside its `.wild` one; `tracker.example` writes the second.
|
||||
const abp_domains: i64 = 2;
|
||||
const abp_wildcards: i64 = 1;
|
||||
const abp_unsupported: i64 = 2;
|
||||
|
||||
/// Compiles `text` into `<base>.list`, `<base>.wild` and `<base>.allow` under
|
||||
/// `dir`, exactly as the manager's compile stage does, and returns the
|
||||
/// compiler's own result.
|
||||
fn compileToFiles(
|
||||
gpa: std.mem.Allocator,
|
||||
io: std.Io,
|
||||
dir: std.Io.Dir,
|
||||
base: []const u8,
|
||||
text: []const u8,
|
||||
format: parsers.Format,
|
||||
) !compiler.Result {
|
||||
var list_name_buf: [64]u8 = undefined;
|
||||
var wild_name_buf: [64]u8 = undefined;
|
||||
var allow_name_buf: [64]u8 = undefined;
|
||||
const list_name = try std.fmt.bufPrint(&list_name_buf, "{s}.list", .{base});
|
||||
const wild_name = try std.fmt.bufPrint(&wild_name_buf, "{s}.wild", .{base});
|
||||
const allow_name = try std.fmt.bufPrint(&allow_name_buf, "{s}.allow", .{base});
|
||||
|
||||
const list_file = try dir.createFile(io, list_name, .{ .permissions = .fromMode(0o600) });
|
||||
defer list_file.close(io);
|
||||
const wild_file = try dir.createFile(io, wild_name, .{ .permissions = .fromMode(0o600) });
|
||||
defer wild_file.close(io);
|
||||
const allow_file = try dir.createFile(io, allow_name, .{ .permissions = .fromMode(0o600) });
|
||||
defer allow_file.close(io);
|
||||
|
||||
const buffers = try gpa.alloc(u8, 2 * 16 * 1024);
|
||||
const buffers = try gpa.alloc(u8, 3 * 16 * 1024);
|
||||
defer gpa.free(buffers);
|
||||
|
||||
var r: std.Io.Reader = .fixed(text);
|
||||
var list_w = list_file.writer(io, buffers[0 .. 16 * 1024]);
|
||||
var wild_w = wild_file.writer(io, buffers[16 * 1024 ..]);
|
||||
var wild_w = wild_file.writer(io, buffers[16 * 1024 .. 32 * 1024]);
|
||||
var allow_w = allow_file.writer(io, buffers[32 * 1024 ..]);
|
||||
|
||||
const result = try compiler.compile(gpa, &r, .hosts, &list_w.interface, &wild_w.interface);
|
||||
const result = try compiler.compile(
|
||||
gpa,
|
||||
&r,
|
||||
format,
|
||||
&list_w.interface,
|
||||
&wild_w.interface,
|
||||
&allow_w.interface,
|
||||
);
|
||||
try list_w.interface.flush();
|
||||
try wild_w.interface.flush();
|
||||
try allow_w.interface.flush();
|
||||
return result;
|
||||
}
|
||||
|
||||
/// The two compiled bodies of one source, read back from disk with their
|
||||
/// The three compiled bodies of one source, read back from disk with their
|
||||
/// headers stripped, exactly as `Manager.reload` reads them.
|
||||
const Bodies = struct {
|
||||
list: []u8,
|
||||
wild: []u8,
|
||||
allow: []u8,
|
||||
|
||||
fn read(gpa: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, base: []const u8) !Bodies {
|
||||
var list_name_buf: [64]u8 = undefined;
|
||||
var wild_name_buf: [64]u8 = undefined;
|
||||
var allow_name_buf: [64]u8 = undefined;
|
||||
const list = try dir.readFileAlloc(
|
||||
io,
|
||||
try std.fmt.bufPrint(&list_name_buf, "{s}.list", .{base}),
|
||||
@@ -167,21 +214,34 @@ const Bodies = struct {
|
||||
gpa,
|
||||
file_limit,
|
||||
);
|
||||
return .{ .list = list, .wild = wild };
|
||||
errdefer gpa.free(wild);
|
||||
const allow = try dir.readFileAlloc(
|
||||
io,
|
||||
try std.fmt.bufPrint(&allow_name_buf, "{s}.allow", .{base}),
|
||||
gpa,
|
||||
file_limit,
|
||||
);
|
||||
return .{ .list = list, .wild = wild, .allow = allow };
|
||||
}
|
||||
|
||||
fn deinit(self: *Bodies, gpa: std.mem.Allocator) void {
|
||||
gpa.free(self.list);
|
||||
gpa.free(self.wild);
|
||||
gpa.free(self.allow);
|
||||
self.* = undefined;
|
||||
}
|
||||
};
|
||||
|
||||
/// A one-group, one-source snapshot over two compiled bodies.
|
||||
fn snapshotOver(gpa: std.mem.Allocator, list_body: []const u8, wild_body: []const u8) !matcher.Snapshot {
|
||||
/// A one-group, one-source snapshot over the three compiled bodies.
|
||||
fn snapshotOver(
|
||||
gpa: std.mem.Allocator,
|
||||
list_body: []const u8,
|
||||
wild_body: []const u8,
|
||||
allow_body: []const u8,
|
||||
) !matcher.Snapshot {
|
||||
const sources = [_]model.BlocklistSource{.{ .url = source_url, .name = source_name }};
|
||||
const compiled = [_]?matcher.Snapshot.Compiled{
|
||||
.{ .list_body = list_body, .wild_body = wild_body },
|
||||
.{ .list_body = list_body, .wild_body = wild_body, .allow_body = allow_body },
|
||||
};
|
||||
return matcher.Snapshot.build(gpa, .{
|
||||
.groups = &.{.{ .name = "default" }},
|
||||
@@ -198,10 +258,17 @@ fn snapshotOver(gpa: std.mem.Allocator, list_body: []const u8, wild_body: []cons
|
||||
});
|
||||
}
|
||||
|
||||
fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 {
|
||||
/// The third producer of this digest, beside `compiler.compile` and
|
||||
/// `Manager.bodyChecksum`. All three must frame the bodies the same way or the
|
||||
/// manager reads its own files as damaged.
|
||||
fn bodyChecksum(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) [64]u8 {
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(list_body);
|
||||
hasher.update(compiler.body_separator);
|
||||
hasher.update(wild_body);
|
||||
hasher.update(compiler.body_separator);
|
||||
hasher.update(allow_body);
|
||||
hasher.update(compiler.body_separator);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
return std.fmt.bytesToHex(digest, .lower);
|
||||
@@ -353,7 +420,7 @@ const Env = struct {
|
||||
// fixtures: the loopback http server
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Route = enum(u8) { body, redirect, not_found, oversize, chunked, stall };
|
||||
const Route = enum(u8) { body, redirect, not_found, oversize, chunked, stall, changed };
|
||||
|
||||
/// How long the `stall` route holds a reply open when nothing releases it.
|
||||
///
|
||||
@@ -400,6 +467,11 @@ const oversize_length = "104857600";
|
||||
const HttpFixture = struct {
|
||||
server: net.Server,
|
||||
body: []const u8,
|
||||
/// What the `changed` route serves: the same list after its author edited
|
||||
/// it. A test sets this before the serving task starts and reaches it by
|
||||
/// switching the route, so the two bodies are read through the atomic that
|
||||
/// selects them and never written beside a request in flight.
|
||||
changed_body: []const u8,
|
||||
route: std.atomic.Value(u8),
|
||||
/// Connections accepted, whatever came over them. A test that claims a pass
|
||||
/// downloaded nothing reads this rather than the route counters: a refetch
|
||||
@@ -426,6 +498,7 @@ const HttpFixture = struct {
|
||||
return .{
|
||||
.server = try local.listen(io, .{ .reuse_address = true }),
|
||||
.body = body,
|
||||
.changed_body = "",
|
||||
.route = .init(@intFromEnum(Route.body)),
|
||||
.accepted = .init(0),
|
||||
.flushed_parts = .init(0),
|
||||
@@ -474,6 +547,7 @@ const HttpFixture = struct {
|
||||
fn respond(self: *HttpFixture, io: std.Io, request: *std.http.Server.Request) !void {
|
||||
switch (@as(Route, @enumFromInt(self.route.load(.acquire)))) {
|
||||
.body => try request.respond(self.body, .{ .keep_alive = false }),
|
||||
.changed => try request.respond(self.changed_body, .{ .keep_alive = false }),
|
||||
.redirect => if (std.mem.eql(u8, request.head.target, redirect_path))
|
||||
try request.respond(self.body, .{ .keep_alive = false })
|
||||
else
|
||||
@@ -679,7 +753,7 @@ test "1: a compiled hosts fixture loads into a snapshot that blocks its domains"
|
||||
const text = try hostsFixture(gpa);
|
||||
defer gpa.free(text);
|
||||
|
||||
const result = try compileToFiles(gpa, io, tmp.dir, "1", text);
|
||||
const result = try compileToFiles(gpa, io, tmp.dir, "1", text, .hosts);
|
||||
try testing.expectEqual(@as(u32, fixture_domains), result.counts.domains);
|
||||
try testing.expectEqual(@as(u32, 1), result.counts.skipped_regex);
|
||||
// The three single-label names are the only invalid candidates here.
|
||||
@@ -692,7 +766,7 @@ test "1: a compiled hosts fixture loads into a snapshot that blocks its domains"
|
||||
try testing.expect(std.mem.find(u8, bodies.list, bare) == null);
|
||||
}
|
||||
|
||||
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild);
|
||||
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild, bodies.allow);
|
||||
defer snapshot.deinit();
|
||||
const group = snapshot.groupIndexByName("default").?;
|
||||
|
||||
@@ -732,8 +806,8 @@ test "2: recompiling the same fixture produces byte-identical files and checksum
|
||||
var second_dir = try tmp.dir.createDirPathOpen(io, "second", .{});
|
||||
defer second_dir.close(io);
|
||||
|
||||
const first = try compileToFiles(gpa, io, first_dir, "1", text);
|
||||
const second = try compileToFiles(gpa, io, second_dir, "1", text);
|
||||
const first = try compileToFiles(gpa, io, first_dir, "1", text, .hosts);
|
||||
const second = try compileToFiles(gpa, io, second_dir, "1", text, .hosts);
|
||||
|
||||
try testing.expectEqualStrings(&first.checksum, &second.checksum);
|
||||
try testing.expectEqual(first.counts, second.counts);
|
||||
@@ -745,11 +819,12 @@ test "2: recompiling the same fixture produces byte-identical files and checksum
|
||||
|
||||
try testing.expectEqualSlices(u8, first_bodies.list, second_bodies.list);
|
||||
try testing.expectEqualSlices(u8, first_bodies.wild, second_bodies.wild);
|
||||
try testing.expectEqualSlices(u8, first_bodies.allow, second_bodies.allow);
|
||||
|
||||
// The checksum the compiler reported is the one over the two bodies it
|
||||
// The checksum the compiler reported is the one over the three bodies it
|
||||
// wrote, which is what the manager stores and compares against.
|
||||
try testing.expectEqualStrings(
|
||||
&bodyChecksum(first_bodies.list, first_bodies.wild),
|
||||
&bodyChecksum(first_bodies.list, first_bodies.wild, first_bodies.allow),
|
||||
&first.checksum,
|
||||
);
|
||||
}
|
||||
@@ -781,8 +856,10 @@ test "3: a damaged compiled file never replaces a serving snapshot with a worse
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 2,
|
||||
.wildcard_count = 1,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(good_list, good_wild),
|
||||
.skipped_unsupported_count = 0,
|
||||
.checksum = &bodyChecksum(good_list, good_wild, ""),
|
||||
});
|
||||
|
||||
try env.mgr.reload(io);
|
||||
@@ -812,8 +889,10 @@ test "3: a damaged compiled file never replaces a serving snapshot with a worse
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 2,
|
||||
.wildcard_count = 1,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(unsorted_list, good_wild),
|
||||
.skipped_unsupported_count = 0,
|
||||
.checksum = &bodyChecksum(unsorted_list, good_wild, ""),
|
||||
});
|
||||
|
||||
try testing.expectError(error.NotSorted, env.mgr.reload(io));
|
||||
@@ -870,6 +949,7 @@ test "4: a 200 response is fetched, compiled, recorded and served" {
|
||||
try testing.expectEqual(http_domains, row.domain_count);
|
||||
try testing.expectEqual(http_wildcards, row.wildcard_count);
|
||||
try testing.expectEqual(http_regex, row.skipped_regex_count);
|
||||
try testing.expectEqual(http_unsupported, row.skipped_unsupported_count);
|
||||
try testing.expectEqual(@as(usize, 64), (row.checksum orelse return error.TestNoChecksum).len);
|
||||
try testing.expect(row.last_updated != null);
|
||||
|
||||
@@ -1028,7 +1108,9 @@ test "8: refetching identical content skips the rewrite and still moves last_upd
|
||||
.last_updated = 1_000,
|
||||
.domain_count = http_domains,
|
||||
.wildcard_count = http_wildcards,
|
||||
.exception_count = http_exceptions,
|
||||
.skipped_regex_count = http_regex,
|
||||
.skipped_unsupported_count = http_unsupported,
|
||||
.checksum = blk: {
|
||||
var rows = try listRows(&env.database);
|
||||
defer rows.deinit();
|
||||
@@ -1069,6 +1151,9 @@ const ReloadTask = struct {
|
||||
/// Writes one source's compiled files and records their checksum, without any
|
||||
/// network: the swap and the orphan sweep care about files and rows, not about
|
||||
/// where the bytes came from.
|
||||
///
|
||||
/// All three files, including an empty `.allow`, because that is what a publish
|
||||
/// leaves: `publishOne` runs once per body and never skips the empty one.
|
||||
fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []const u8) !void {
|
||||
const io = env.io();
|
||||
var dir = try env.blocklistDir();
|
||||
@@ -1076,6 +1161,7 @@ fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []c
|
||||
|
||||
var list_name_buf: [64]u8 = undefined;
|
||||
var wild_name_buf: [64]u8 = undefined;
|
||||
var allow_name_buf: [64]u8 = undefined;
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&list_name_buf, "{d}.list", .{id}),
|
||||
.data = list_body,
|
||||
@@ -1084,13 +1170,19 @@ fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []c
|
||||
.sub_path = try std.fmt.bufPrint(&wild_name_buf, "{d}.wild", .{id}),
|
||||
.data = wild_body,
|
||||
});
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&allow_name_buf, "{d}.allow", .{id}),
|
||||
.data = "",
|
||||
});
|
||||
|
||||
try sources_repo.updateSourceStats(&env.database, id, .{
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 1,
|
||||
.wildcard_count = 0,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, wild_body),
|
||||
.skipped_unsupported_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, wild_body, ""),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1146,14 +1238,21 @@ test "10: pruneOrphans deletes the files of a deleted source and leaves live one
|
||||
|
||||
var dir = try env.blocklistDir();
|
||||
defer dir.close(io);
|
||||
// Every name a source can own, spelled out: this fixture is what decides
|
||||
// whether the sweep covers the whole set, so it enumerates
|
||||
// `manager.source_file_suffixes` by hand rather than sharing it. A suffix
|
||||
// added to the manager and not added here is swept by nothing and asserted
|
||||
// by nothing.
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.allow", .data = "lifted.example.com\n" });
|
||||
// Id 9999 has no `blocklist_sources` row, so no refresh can be writing for
|
||||
// it: these temporaries are what a refresh that died mid-write leaves
|
||||
// behind, and the sweep is the only thing that will ever remove them.
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.list.tmp", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.wild.tmp", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.allow.tmp", .data = "" });
|
||||
|
||||
// The live source does have a row, so its temporary is a refresh in
|
||||
// progress and must survive a sweep that runs beside it.
|
||||
@@ -1165,12 +1264,15 @@ test "10: pruneOrphans deletes the files of a deleted source and leaves live one
|
||||
|
||||
var live_buf: [64]u8 = undefined;
|
||||
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.list", .{id}), .{});
|
||||
try dir.access(io, try std.fmt.bufPrint(&live_buf, "{d}.allow", .{id}), .{});
|
||||
try dir.access(io, live_tmp, .{});
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list.tmp", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild.tmp", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow.tmp", .{}));
|
||||
}
|
||||
|
||||
test "10b: the scheduler sweeps orphans on its own, with no operator call" {
|
||||
@@ -1190,8 +1292,10 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
|
||||
.last_updated = std.Io.Clock.real.now(io).toSeconds(),
|
||||
.domain_count = 1,
|
||||
.wildcard_count = 0,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, ""),
|
||||
.skipped_unsupported_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, "", ""),
|
||||
});
|
||||
|
||||
var dir = try env.blocklistDir();
|
||||
@@ -1201,7 +1305,9 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
|
||||
// either.
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.list", .data = "gone.example.com\n" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.wild", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.allow", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.raw.tmp", .data = "" });
|
||||
try dir.writeFile(io, .{ .sub_path = "9999.allow.tmp", .data = "" });
|
||||
|
||||
// `runScheduler` is the entry point `app.zig` hands to `Io.Group`, and the
|
||||
// only one the server ever calls. A disabled update stops it after the
|
||||
@@ -1212,7 +1318,9 @@ test "10b: the scheduler sweeps orphans on its own, with no operator call" {
|
||||
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.list", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.wild", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.raw.tmp", .{}));
|
||||
try testing.expectError(error.FileNotFound, dir.access(io, "9999.allow.tmp", .{}));
|
||||
|
||||
// The live source kept its files and is still filtering: the sweep did not
|
||||
// take the snapshot the same pass had just published.
|
||||
@@ -1376,6 +1484,7 @@ test "10d: a source deleted mid-refresh does not take the refresh's temporary fi
|
||||
try testing.expect(!std.mem.endsWith(u8, entry.name, ".tmp"));
|
||||
try testing.expect(!std.mem.endsWith(u8, entry.name, ".list"));
|
||||
try testing.expect(!std.mem.endsWith(u8, entry.name, ".wild"));
|
||||
try testing.expect(!std.mem.endsWith(u8, entry.name, ".allow"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1410,10 +1519,13 @@ test "10e: a reconcile then a restart reuses the compiled files and downloads no
|
||||
defer dir.close(io);
|
||||
var list_buf: [64]u8 = undefined;
|
||||
var wild_buf: [64]u8 = undefined;
|
||||
var allow_buf: [64]u8 = undefined;
|
||||
const list_name = try std.fmt.bufPrint(&list_buf, "{d}.list", .{id});
|
||||
const wild_name = try std.fmt.bufPrint(&wild_buf, "{d}.wild", .{id});
|
||||
const allow_name = try std.fmt.bufPrint(&allow_buf, "{d}.allow", .{id});
|
||||
const list_before = try dir.statFile(io, list_name, .{});
|
||||
const wild_before = try dir.statFile(io, wild_name, .{});
|
||||
const allow_before = try dir.statFile(io, allow_name, .{});
|
||||
|
||||
// File mode, declaring exactly what the database already holds. The engine
|
||||
// has to recognise the source by its url and leave the row where it is:
|
||||
@@ -1459,14 +1571,19 @@ test "10e: a reconcile then a restart reuses the compiled files and downloads no
|
||||
// decision the pass made rather than a connection it could not have opened.
|
||||
try testing.expectEqual(@as(u32, 1), fixture.accepted.load(.monotonic));
|
||||
|
||||
// The same two files: not recompiled, and not swept as orphans and written
|
||||
// back.
|
||||
// The same three files: not recompiled, and not swept as orphans and
|
||||
// written back. `.allow` is asserted with the other two because a restart
|
||||
// that rewrote only the exception body would otherwise leave this test
|
||||
// green while changing what the snapshot lets through.
|
||||
const list_after = try dir.statFile(io, list_name, .{});
|
||||
const wild_after = try dir.statFile(io, wild_name, .{});
|
||||
const allow_after = try dir.statFile(io, allow_name, .{});
|
||||
try testing.expectEqual(list_before.inode, list_after.inode);
|
||||
try testing.expectEqual(list_before.mtime, list_after.mtime);
|
||||
try testing.expectEqual(wild_before.inode, wild_after.inode);
|
||||
try testing.expectEqual(wild_before.mtime, wild_after.mtime);
|
||||
try testing.expectEqual(allow_before.inode, allow_after.inode);
|
||||
try testing.expectEqual(allow_before.mtime, allow_after.mtime);
|
||||
|
||||
// The row kept the id those files are named after, and the snapshot the
|
||||
// restart published is the one compiled from them.
|
||||
@@ -1788,12 +1905,12 @@ test "17: each blocking mode synthesizes the documented blocked reply" {
|
||||
|
||||
// The reply is synthesized for a name the compiled files actually block, so
|
||||
// this case covers the decision and the response together.
|
||||
const result = try compileToFiles(gpa, io, tmp.dir, "1", "0.0.0.0 ads.example.com\n");
|
||||
const result = try compileToFiles(gpa, io, tmp.dir, "1", "0.0.0.0 ads.example.com\n", .hosts);
|
||||
try testing.expectEqual(@as(u32, 1), result.counts.domains);
|
||||
|
||||
var bodies = try Bodies.read(gpa, io, tmp.dir, "1");
|
||||
defer bodies.deinit(gpa);
|
||||
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild);
|
||||
var snapshot = try snapshotOver(gpa, bodies.list, bodies.wild, bodies.allow);
|
||||
defer snapshot.deinit();
|
||||
|
||||
const group = snapshot.groupIndexByName("default").?;
|
||||
@@ -1909,3 +2026,382 @@ test "18: a body streamed in flushed parts survives the fetcher's multi-read pum
|
||||
try testing.expectEqual(matcher.Reason.blocklist_domain, decision.reason);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 19–20: list exceptions (milestone 21)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A real ABP list: one domain anchor, the exception that lifts one subtree out
|
||||
/// of it in each of the two accepted spellings, and two `@@` forms nxdns does
|
||||
/// not honour.
|
||||
const abp_exception_body =
|
||||
"[Adblock Plus 2.0]\n" ++
|
||||
"! Title: exceptions\n" ++
|
||||
"||ads.example^\n" ++
|
||||
"||tracker.example^\n" ++
|
||||
"@@||good.ads.example^\n" ++
|
||||
"@@||fine.tracker.example$important\n" ++
|
||||
"@@||paid.ads.example^$third-party\n" ++
|
||||
"@@partial.ads.example\n";
|
||||
|
||||
const abp_exception_domains: i64 = 2;
|
||||
const abp_exception_wildcards: i64 = 2;
|
||||
const abp_exception_exceptions: i64 = 2;
|
||||
|
||||
test "19: a downloaded list's exceptions lift its own blocks and nothing else" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fixture = try HttpFixture.init(io, abp_exception_body);
|
||||
defer fixture.deinit(io);
|
||||
var group: std.Io.Group = .init;
|
||||
defer group.cancel(io);
|
||||
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||
|
||||
var url_buf: [64]u8 = undefined;
|
||||
const url = try fixture.url(&url_buf);
|
||||
const id = try seedSource(&env.database, url);
|
||||
|
||||
try testing.expect(try refreshOnce(env, url));
|
||||
try env.mgr.reload(io);
|
||||
|
||||
// The `.allow` body is a third file beside the two, and the row and the
|
||||
// status entry both carry its count — which is what a
|
||||
// `POST /api/blocklists/update` response row reports as `exceptions`.
|
||||
var dir = try env.blocklistDir();
|
||||
defer dir.close(io);
|
||||
var bodies = try Bodies.read(gpa, io, dir, "1");
|
||||
defer bodies.deinit(gpa);
|
||||
try testing.expectEqualStrings(
|
||||
"fine.tracker.example\ngood.ads.example\n",
|
||||
manager.stripHeader(bodies.allow),
|
||||
);
|
||||
|
||||
var rows = try listRows(&env.database);
|
||||
defer rows.deinit();
|
||||
const row = try rows.byUrl(url);
|
||||
try testing.expectEqual(abp_exception_domains, row.domain_count);
|
||||
try testing.expectEqual(abp_exception_wildcards, row.wildcard_count);
|
||||
try testing.expectEqual(abp_exception_exceptions, row.exception_count);
|
||||
|
||||
const status = try env.status(id);
|
||||
try testing.expectEqual(manager.State.ok, status.state);
|
||||
try testing.expect(status.loaded);
|
||||
try testing.expectEqual(@as(u32, @intCast(abp_exception_exceptions)), status.counts.exceptions);
|
||||
|
||||
// The blocks the list makes still land, apex and subdomain alike.
|
||||
for ([_][]const u8{ "ads.example", "x.ads.example", "paid.ads.example", "partial.ads.example" }) |blocked| {
|
||||
const decision, _ = try env.evaluate(blocked);
|
||||
try testing.expect(decision.blocked);
|
||||
}
|
||||
|
||||
// The two exceptions lift the excepted name and everything under it.
|
||||
for ([_][]const u8{
|
||||
"good.ads.example",
|
||||
"y.good.ads.example",
|
||||
"fine.tracker.example",
|
||||
"z.fine.tracker.example",
|
||||
}) |lifted| {
|
||||
const decision, _ = try env.evaluate(lifted);
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(matcher.Reason.blocklist_exception, decision.reason);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
// What `/api/lookup` answers, through the same function the handler calls:
|
||||
// the reason names the exception and the source id names the list.
|
||||
{
|
||||
const handle = env.mgr.acquire(io) orelse return error.TestNoSnapshot;
|
||||
defer handle.release(io);
|
||||
const group_index = handle.snapshot.groupIndexByName("default") orelse
|
||||
return error.TestGroupMissing;
|
||||
|
||||
const result = lookup.evaluate(
|
||||
handle.snapshot,
|
||||
group_index,
|
||||
"y.good.ads.example",
|
||||
&empty_records,
|
||||
&empty_zones,
|
||||
);
|
||||
try testing.expect(!result.blocked);
|
||||
try testing.expectEqual(matcher.Reason.blocklist_exception, result.reason);
|
||||
try testing.expectEqualStrings("good.ads.example", result.matched);
|
||||
try testing.expectEqual(@as(?i64, id), result.source_id);
|
||||
|
||||
const rendered = lookup.body("y.good.ads.example", result, url);
|
||||
try testing.expectEqualStrings("blocklist_exception", rendered.reason);
|
||||
try testing.expectEqualStrings(url, rendered.source_url.?);
|
||||
}
|
||||
|
||||
// An operator block rule outranks the list's exception: a downloaded list
|
||||
// may cancel what a list decided and never what the operator decided.
|
||||
const group_id = (try groups_repo.groupId(&env.database, "default")) orelse
|
||||
return error.TestGroupMissing;
|
||||
_ = try rules_repo.insertRuleRow(&env.database, .{
|
||||
.group_id = group_id,
|
||||
.pattern = "good.ads.example",
|
||||
.kind = .exact,
|
||||
.action = .block,
|
||||
}, 1_700_000_000);
|
||||
try env.mgr.reload(io);
|
||||
{
|
||||
const decision, _ = try env.evaluate("good.ads.example");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expectEqual(matcher.Reason.rule_block_exact, decision.reason);
|
||||
}
|
||||
}
|
||||
|
||||
test "20: a data directory with no .allow file loads, and an unframed checksum does not" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
const id = try seedSource(&env.database, source_url);
|
||||
|
||||
// Two compiled files and no `.allow` file, which is what a source that
|
||||
// published before exceptions existed left on disk.
|
||||
const list_body = "aaa.example.com\nbbb.example.com\n";
|
||||
const wild_body = "ccc.example.com\n";
|
||||
|
||||
var dir = try env.blocklistDir();
|
||||
defer dir.close(io);
|
||||
var list_name_buf: [64]u8 = undefined;
|
||||
var wild_name_buf: [64]u8 = undefined;
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&list_name_buf, "{d}.list", .{id}),
|
||||
.data = list_body,
|
||||
});
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&wild_name_buf, "{d}.wild", .{id}),
|
||||
.data = wild_body,
|
||||
});
|
||||
|
||||
var stats: sources_repo.SourceStats = .{
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 2,
|
||||
.wildcard_count = 1,
|
||||
.exception_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.skipped_unsupported_count = 0,
|
||||
.checksum = undefined,
|
||||
};
|
||||
|
||||
// The digest those two files carried before the bodies were framed. It is
|
||||
// not the digest of any body layout the compiler produces now, so the load
|
||||
// must report the mismatch instead of serving the files.
|
||||
{
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(list_body);
|
||||
hasher.update(wild_body);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
const unframed = std.fmt.bytesToHex(digest, .lower);
|
||||
|
||||
stats.checksum = &unframed;
|
||||
try sources_repo.updateSourceStats(&env.database, id, stats);
|
||||
try env.mgr.reload(io);
|
||||
|
||||
const status = try env.status(id);
|
||||
try testing.expectEqual(manager.State.load_failed, status.state);
|
||||
try testing.expect(!status.loaded);
|
||||
try testing.expectEqualStrings("ChecksumMismatch", status.errorText());
|
||||
}
|
||||
|
||||
// The same two files under the framed digest of three bodies, the third of
|
||||
// them empty. The absent `.allow` file is that empty body, so the source
|
||||
// loads and both entries filter.
|
||||
{
|
||||
const framed = bodyChecksum(list_body, wild_body, "");
|
||||
stats.checksum = &framed;
|
||||
try sources_repo.updateSourceStats(&env.database, id, stats);
|
||||
try env.mgr.reload(io);
|
||||
|
||||
const status = try env.status(id);
|
||||
try testing.expectEqual(manager.State.ok, status.state);
|
||||
try testing.expect(status.loaded);
|
||||
try testing.expectEqualStrings("", status.errorText());
|
||||
|
||||
const decision, _ = try env.evaluate("aaa.example.com");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expect((try env.evaluate("x.ccc.example.com"))[0].blocked);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 21–22: the unsupported count, from the compile to the database and back
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test "21: an abp list's unsupported lines are counted, persisted and read back" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fixture = try HttpFixture.init(io, abp_body);
|
||||
defer fixture.deinit(io);
|
||||
var group: std.Io.Group = .init;
|
||||
defer group.cancel(io);
|
||||
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||
|
||||
var url_buf: [64]u8 = undefined;
|
||||
const url = try fixture.url(&url_buf);
|
||||
const id = try seedSource(&env.database, url);
|
||||
|
||||
try testing.expect(try refreshOnce(env, url));
|
||||
try env.mgr.reload(io);
|
||||
|
||||
var rows = try listRows(&env.database);
|
||||
defer rows.deinit();
|
||||
const row = try rows.byUrl(url);
|
||||
try testing.expectEqual(abp_domains, row.domain_count);
|
||||
try testing.expectEqual(abp_wildcards, row.wildcard_count);
|
||||
try testing.expectEqual(@as(i64, 0), row.skipped_regex_count);
|
||||
try testing.expectEqual(abp_unsupported, row.skipped_unsupported_count);
|
||||
|
||||
const status = try env.status(id);
|
||||
try testing.expectEqual(manager.State.ok, status.state);
|
||||
try testing.expectEqual(@as(u32, @intCast(abp_unsupported)), status.counts.skipped_unsupported);
|
||||
|
||||
// What the number costs the operator: the `$`-modifier rule named a domain
|
||||
// and blocked nothing, while the two lines nxdns could translate did block.
|
||||
try testing.expect(!(try env.evaluate("ads.example"))[0].blocked);
|
||||
try testing.expect((try env.evaluate("blocked.example"))[0].blocked);
|
||||
try testing.expect((try env.evaluate("tracker.example"))[0].blocked);
|
||||
}
|
||||
|
||||
/// The same list before and after its author edited only lines nxdns skips.
|
||||
/// The written entries are identical in both, so the two compiles produce one
|
||||
/// checksum and the refresh takes the unchanged-checksum path.
|
||||
const churn_before =
|
||||
"! churn fixture\n" ++
|
||||
"##.ad-one\n" ++
|
||||
"||blocked.example^\n" ++
|
||||
"tracker.example\n";
|
||||
|
||||
const churn_after =
|
||||
"! churn fixture\n" ++
|
||||
"##.ad-one\n" ++
|
||||
"##.ad-two\n" ++
|
||||
"/ads[0-9]+/\n" ++
|
||||
"||blocked.example^\n" ++
|
||||
"tracker.example\n";
|
||||
|
||||
const boundary_before = "a.example\n";
|
||||
const boundary_after = "*.a.example\n";
|
||||
|
||||
test "23: a name moving from the list body to the wild body forces a republish" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fixture = try HttpFixture.init(io, boundary_before);
|
||||
fixture.changed_body = boundary_after;
|
||||
defer fixture.deinit(io);
|
||||
var group: std.Io.Group = .init;
|
||||
defer group.cancel(io);
|
||||
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||
|
||||
var url_buf: [64]u8 = undefined;
|
||||
const url = try fixture.url(&url_buf);
|
||||
_ = try seedSource(&env.database, url);
|
||||
|
||||
try testing.expect(try refreshOnce(env, url));
|
||||
|
||||
var first_checksum: [64]u8 = undefined;
|
||||
{
|
||||
var rows = try listRows(&env.database);
|
||||
defer rows.deinit();
|
||||
const row = try rows.byUrl(url);
|
||||
try testing.expectEqual(@as(i64, 1), row.domain_count);
|
||||
try testing.expectEqual(@as(i64, 0), row.wildcard_count);
|
||||
@memcpy(&first_checksum, row.checksum orelse return error.TestNoChecksum);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
test "22: a list that changed only its skipped lines still updates both skip counters" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fixture = try HttpFixture.init(io, churn_before);
|
||||
fixture.changed_body = churn_after;
|
||||
defer fixture.deinit(io);
|
||||
var group: std.Io.Group = .init;
|
||||
defer group.cancel(io);
|
||||
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||
|
||||
var url_buf: [64]u8 = undefined;
|
||||
const url = try fixture.url(&url_buf);
|
||||
const id = try seedSource(&env.database, url);
|
||||
|
||||
try testing.expect(try refreshOnce(env, url));
|
||||
|
||||
var first_checksum: [64]u8 = undefined;
|
||||
{
|
||||
var rows = try listRows(&env.database);
|
||||
defer rows.deinit();
|
||||
const row = try rows.byUrl(url);
|
||||
try testing.expectEqual(@as(i64, 0), row.skipped_regex_count);
|
||||
try testing.expectEqual(@as(i64, 1), row.skipped_unsupported_count);
|
||||
@memcpy(&first_checksum, row.checksum orelse return error.TestNoChecksum);
|
||||
}
|
||||
|
||||
// The edited list. Two more skipped lines and not one written entry moved,
|
||||
// so the refresh finds its stored checksum and rewrites nothing on disk.
|
||||
fixture.setRoute(.changed);
|
||||
try testing.expect(!try refreshOnce(env, url));
|
||||
|
||||
{
|
||||
var rows = try listRows(&env.database);
|
||||
defer rows.deinit();
|
||||
const row = try rows.byUrl(url);
|
||||
try testing.expectEqualStrings(&first_checksum, row.checksum orelse return error.TestNoChecksum);
|
||||
// The three written counts are the ones an unchanged checksum vouches
|
||||
// for; the two skip counts are the ones it says nothing about.
|
||||
try testing.expectEqual(abp_domains, row.domain_count);
|
||||
try testing.expectEqual(abp_wildcards, row.wildcard_count);
|
||||
try testing.expectEqual(@as(i64, 1), row.skipped_regex_count);
|
||||
try testing.expectEqual(@as(i64, 2), row.skipped_unsupported_count);
|
||||
}
|
||||
|
||||
const live = try env.status(id);
|
||||
try testing.expectEqual(@as(u32, 1), live.counts.skipped_regex);
|
||||
try testing.expectEqual(@as(u32, 2), live.counts.skipped_unsupported);
|
||||
|
||||
// The restart. A new manager over the same database and the same files
|
||||
// carries nothing across in memory, so the status it publishes is what
|
||||
// rehydration read out of the row — which is the only reason writing the
|
||||
// fresh counts above matters.
|
||||
env.mgr.deinit(io);
|
||||
env.mgr = try manager.Manager.init(
|
||||
gpa,
|
||||
&env.database,
|
||||
.{ .dir = env.tmp.dir },
|
||||
&env.f,
|
||||
.{ .enabled = false },
|
||||
budget,
|
||||
);
|
||||
try env.mgr.reload(io);
|
||||
|
||||
const restored = try env.status(id);
|
||||
try testing.expectEqual(manager.State.ok, restored.state);
|
||||
try testing.expect(restored.loaded);
|
||||
try testing.expectEqual(@as(u32, 1), restored.counts.skipped_regex);
|
||||
try testing.expectEqual(@as(u32, 2), restored.counts.skipped_unsupported);
|
||||
}
|
||||
|
||||
+257
-73
@@ -38,9 +38,10 @@
|
||||
//! publish: the download of one source, at up to 300 s each, and the compile
|
||||
//! that follows it. It also covers blocklist-directory maintenance, because
|
||||
//! those stages are the only writers of `.raw.tmp` / `.list.tmp` /
|
||||
//! `.wild.tmp` and `pruneOrphans` must not sweep the temporaries of a refresh
|
||||
//! that is still running. Two concurrent refreshes would share the fetcher's
|
||||
//! buffers and, for one source, the same temporary paths.
|
||||
//! `.wild.tmp` / `.allow.tmp` and `pruneOrphans` must not sweep the
|
||||
//! temporaries of a refresh that is still running. Two concurrent refreshes
|
||||
//! would share the fetcher's buffers and, for one source, the same temporary
|
||||
//! paths.
|
||||
//!
|
||||
//! **Lock ordering: `refresh_lock` is never acquired while `writer_lock` is
|
||||
//! held.** A path that needs both takes `refresh_lock` first. The public entry
|
||||
@@ -94,7 +95,7 @@ const io_buf_len: usize = 64 * 1024;
|
||||
/// would then be decided by almost no data.
|
||||
const sample_buf_len: usize = parsers.sample_lines * (compiler.max_line_len + 1);
|
||||
|
||||
/// `<id>` is at most 20 characters and the longest suffix is `.list.tmp`.
|
||||
/// `<id>` is at most 20 characters and the longest suffix is `.allow.tmp`.
|
||||
const name_buf_len: usize = 48;
|
||||
|
||||
/// How one blocklist source is named in a log line: by its row id and its name,
|
||||
@@ -215,9 +216,12 @@ pub const SourceStatus = struct {
|
||||
};
|
||||
|
||||
/// The header every compiled file carries, ahead of the body. The `sha256`
|
||||
/// covers the `.list` body followed by the `.wild` body and **not** the header,
|
||||
/// so it stays stable across a refetch of unchanged content while
|
||||
/// `fetched_at` moves.
|
||||
/// covers the `.list` body, then the `.wild` body, then the `.allow` body, and
|
||||
/// **not** the header, so it stays stable across a refetch of unchanged content
|
||||
/// while `fetched_at` moves.
|
||||
///
|
||||
/// Each body is followed by a separator byte, so the digest identifies which
|
||||
/// body a name sits in rather than only which names were written.
|
||||
pub const Header = struct {
|
||||
url: []const u8,
|
||||
format: parsers.Format,
|
||||
@@ -233,6 +237,7 @@ pub const Header = struct {
|
||||
try w.print("# fetched_at {d}\n", .{self.fetched_at});
|
||||
try w.print("# domains {d}\n", .{self.counts.domains});
|
||||
try w.print("# wildcards {d}\n", .{self.counts.wildcards});
|
||||
try w.print("# exceptions {d}\n", .{self.counts.exceptions});
|
||||
try w.print("# skipped_regex {d}\n", .{self.counts.skipped_regex});
|
||||
try w.print("# skipped_unsupported {d}\n", .{self.counts.skipped_unsupported});
|
||||
try w.print("# invalid {d}\n", .{self.counts.invalid});
|
||||
@@ -557,12 +562,14 @@ pub const Manager = struct {
|
||||
|
||||
var list_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_buf: [name_buf_len]u8 = undefined;
|
||||
var allow_buf: [name_buf_len]u8 = undefined;
|
||||
const list_name = compiledName(&list_buf, row.id, ".list");
|
||||
const wild_name = compiledName(&wild_buf, row.id, ".wild");
|
||||
const allow_name = compiledName(&allow_buf, row.id, ".allow");
|
||||
|
||||
// Reserved before the reads, so neither buffer can be orphaned by a
|
||||
// failing append: `bodies` owns each one from the moment it is read.
|
||||
try bodies.ensureUnusedCapacity(self.gpa, 2);
|
||||
// Reserved before the reads, so no buffer can be orphaned by a failing
|
||||
// append: `bodies` owns each one from the moment it is read.
|
||||
try bodies.ensureUnusedCapacity(self.gpa, 3);
|
||||
|
||||
// `error.Canceled` is the one-shot signal that this task is being torn
|
||||
// down, and it is consumed by whoever catches it. Recording it as a
|
||||
@@ -583,18 +590,38 @@ pub const Manager = struct {
|
||||
};
|
||||
bodies.appendAssumeCapacity(wild_bytes);
|
||||
|
||||
// A missing `.allow` file is an empty allow body, not a failure. The
|
||||
// digest still covers three bodies, the third of them empty, so a
|
||||
// source whose list carries no `@@` line matches whether its empty
|
||||
// `.allow` file survived or not.
|
||||
const allow_bytes: []const u8 = blk: {
|
||||
const read = dir.readFileAlloc(io, allow_name, self.gpa, .limited(max_compiled_bytes)) catch |err| {
|
||||
if (err == error.OutOfMemory) return error.OutOfMemory;
|
||||
if (err == error.Canceled) return error.Canceled;
|
||||
if (err == error.FileNotFound) break :blk "";
|
||||
return loadFailure(row, allow_name, err);
|
||||
};
|
||||
bodies.appendAssumeCapacity(read);
|
||||
break :blk read;
|
||||
};
|
||||
|
||||
const list_body = stripHeader(list_bytes);
|
||||
const wild_body = stripHeader(wild_bytes);
|
||||
const allow_body = stripHeader(allow_bytes);
|
||||
|
||||
// The checksum covers both bodies together, so a crash between the two
|
||||
// The checksum covers the three bodies together, so a crash between the
|
||||
// `replace` calls — a new `.list` beside an old `.wild` — is caught
|
||||
// here and refreshed, not served as a half-updated list.
|
||||
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body))) {
|
||||
if (!std.mem.eql(u8, stored, &bodyChecksum(list_body, wild_body, allow_body))) {
|
||||
log.warn("blocklist {f}: compiled files do not match the stored checksum", .{SourceLabel.of(row)});
|
||||
return .{ .failed = .{ .state = .load_failed, .text = "ChecksumMismatch" } };
|
||||
}
|
||||
|
||||
return .{ .loaded = .{ .list_body = list_body, .wild_body = wild_body } };
|
||||
return .{ .loaded = .{
|
||||
.list_body = list_body,
|
||||
.wild_body = wild_body,
|
||||
.allow_body = allow_body,
|
||||
} };
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -635,23 +662,28 @@ pub const Manager = struct {
|
||||
var raw_buf: [name_buf_len]u8 = undefined;
|
||||
var list_tmp_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_tmp_buf: [name_buf_len]u8 = undefined;
|
||||
var allow_tmp_buf: [name_buf_len]u8 = undefined;
|
||||
const raw_name = compiledName(&raw_buf, row.id, ".raw.tmp");
|
||||
const list_tmp = compiledName(&list_tmp_buf, row.id, ".list.tmp");
|
||||
const wild_tmp = compiledName(&wild_tmp_buf, row.id, ".wild.tmp");
|
||||
const tmp: TempNames = .{
|
||||
.list = compiledName(&list_tmp_buf, row.id, ".list.tmp"),
|
||||
.wild = compiledName(&wild_tmp_buf, row.id, ".wild.tmp"),
|
||||
.allow = compiledName(&allow_tmp_buf, row.id, ".allow.tmp"),
|
||||
};
|
||||
|
||||
// Installed before the calls that create these files, not after: an
|
||||
// `error.Canceled` or `error.OutOfMemory` returned straight out of
|
||||
// `download` or `compileTo` would outrun a later `defer` and leave a
|
||||
// temporary behind. Deleting a name that was never created is a no-op.
|
||||
defer self.deleteQuietly(io, dir, raw_name);
|
||||
defer self.deleteQuietly(io, dir, list_tmp);
|
||||
defer self.deleteQuietly(io, dir, wild_tmp);
|
||||
defer self.deleteQuietly(io, dir, tmp.list);
|
||||
defer self.deleteQuietly(io, dir, tmp.wild);
|
||||
defer self.deleteQuietly(io, dir, tmp.allow);
|
||||
|
||||
// The half that takes the time: one download of up to `total_budget`
|
||||
// and one compile of everything it returned. `refresh_lock` alone is
|
||||
// held here, so a rule save, a settings change or any other web
|
||||
// mutation that ends in `reload` runs beside it instead of behind it.
|
||||
const prepared = try self.prepareRefresh(io, dir, row, &status, raw_name, list_tmp, wild_tmp);
|
||||
const prepared = try self.prepareRefresh(io, dir, row, &status, raw_name, tmp);
|
||||
|
||||
// The half that publishes. The compiled files, the runtime columns and
|
||||
// the status entry land under one `writer_lock`, so a reload never
|
||||
@@ -659,7 +691,7 @@ pub const Manager = struct {
|
||||
self.writer_lock.lockUncancelable(io);
|
||||
defer self.writer_lock.unlock(io);
|
||||
|
||||
const replaced = try self.publishRefresh(io, dir, row, &status, prepared, list_tmp, wild_tmp);
|
||||
const replaced = try self.publishRefresh(io, dir, row, &status, prepared, tmp);
|
||||
self.commitStatus(io, status);
|
||||
return replaced;
|
||||
}
|
||||
@@ -690,6 +722,14 @@ pub const Manager = struct {
|
||||
return self.reload(io);
|
||||
}
|
||||
|
||||
/// The three temporary files one refresh compiles into, before the header
|
||||
/// is prepended and each is renamed over the file it replaces.
|
||||
const TempNames = struct {
|
||||
list: []const u8,
|
||||
wild: []const u8,
|
||||
allow: []const u8,
|
||||
};
|
||||
|
||||
/// What the fetch-and-compile half of a refresh produced. `.failed` needs
|
||||
/// no publish and has already recorded why in the status entry.
|
||||
const Prepared = union(enum) {
|
||||
@@ -712,8 +752,7 @@ pub const Manager = struct {
|
||||
row: sources_repo.SourceRow,
|
||||
status: *SourceStatus,
|
||||
raw_name: []const u8,
|
||||
list_tmp: []const u8,
|
||||
wild_tmp: []const u8,
|
||||
tmp: TempNames,
|
||||
) Error!Prepared {
|
||||
self.download(io, dir, raw_name, row) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
@@ -733,7 +772,7 @@ pub const Manager = struct {
|
||||
},
|
||||
};
|
||||
|
||||
const result = self.compileTo(io, dir, raw_name, format, list_tmp, wild_tmp) catch |err| switch (err) {
|
||||
const result = self.compileTo(io, dir, raw_name, format, tmp) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
error.Canceled => return error.Canceled,
|
||||
else => {
|
||||
@@ -762,8 +801,7 @@ pub const Manager = struct {
|
||||
row: sources_repo.SourceRow,
|
||||
status: *SourceStatus,
|
||||
prepared: Prepared,
|
||||
list_tmp: []const u8,
|
||||
wild_tmp: []const u8,
|
||||
tmp: TempNames,
|
||||
) Error!bool {
|
||||
const compiled = switch (prepared) {
|
||||
.failed => return false,
|
||||
@@ -782,11 +820,21 @@ pub const Manager = struct {
|
||||
if (std.mem.eql(u8, stored, &compiled.result.checksum) and
|
||||
self.diskBodiesMatch(io, dir, row.id, stored))
|
||||
{
|
||||
// Every count comes from the compile that just ran, not from
|
||||
// the row. The two skip counts have to: a skipped line lands in
|
||||
// no body, so a list that changed only its regex or
|
||||
// browser-syntax lines reaches here with a stale row. The three
|
||||
// written counts equal the row's anyway once the digest is
|
||||
// framed, so reading them from the compile costs nothing and
|
||||
// leaves no field whose freshness rests on an argument about
|
||||
// what the checksum covers.
|
||||
try sources_repo.updateSourceStats(self.database, row.id, .{
|
||||
.last_updated = now,
|
||||
.domain_count = row.domain_count,
|
||||
.wildcard_count = row.wildcard_count,
|
||||
.skipped_regex_count = row.skipped_regex_count,
|
||||
.domain_count = compiled.result.counts.domains,
|
||||
.wildcard_count = compiled.result.counts.wildcards,
|
||||
.exception_count = compiled.result.counts.exceptions,
|
||||
.skipped_regex_count = compiled.result.counts.skipped_regex,
|
||||
.skipped_unsupported_count = compiled.result.counts.skipped_unsupported,
|
||||
.checksum = stored,
|
||||
});
|
||||
status.succeed(now, compiled.result.counts);
|
||||
@@ -801,7 +849,7 @@ pub const Manager = struct {
|
||||
.counts = compiled.result.counts,
|
||||
.checksum = &compiled.result.checksum,
|
||||
};
|
||||
self.publish(io, dir, row.id, header, list_tmp, wild_tmp) catch |err| switch (err) {
|
||||
self.publish(io, dir, row.id, header, tmp) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
error.Canceled => return error.Canceled,
|
||||
else => {
|
||||
@@ -814,7 +862,9 @@ pub const Manager = struct {
|
||||
.last_updated = now,
|
||||
.domain_count = compiled.result.counts.domains,
|
||||
.wildcard_count = compiled.result.counts.wildcards,
|
||||
.exception_count = compiled.result.counts.exceptions,
|
||||
.skipped_regex_count = compiled.result.counts.skipped_regex,
|
||||
.skipped_unsupported_count = compiled.result.counts.skipped_unsupported,
|
||||
.checksum = &compiled.result.checksum,
|
||||
});
|
||||
status.succeed(now, compiled.result.counts);
|
||||
@@ -913,7 +963,7 @@ pub const Manager = struct {
|
||||
return parsers.detectFormat(sample.buffered());
|
||||
}
|
||||
|
||||
/// Compiles into two plain temporary files. The compiled bodies cannot go
|
||||
/// Compiles into three plain temporary files. The compiled bodies cannot go
|
||||
/// straight into the final files: the header carries counts that only exist
|
||||
/// once the whole input has been compiled, and the loader requires the
|
||||
/// header first.
|
||||
@@ -923,22 +973,24 @@ pub const Manager = struct {
|
||||
dir: std.Io.Dir,
|
||||
raw_name: []const u8,
|
||||
format: parsers.Format,
|
||||
list_tmp: []const u8,
|
||||
wild_tmp: []const u8,
|
||||
tmp: TempNames,
|
||||
) !compiler.Result {
|
||||
const raw = try dir.openFile(io, raw_name, .{});
|
||||
defer raw.close(io);
|
||||
const list_file = try dir.createFile(io, list_tmp, .{ .permissions = .fromMode(0o600) });
|
||||
const list_file = try dir.createFile(io, tmp.list, .{ .permissions = .fromMode(0o600) });
|
||||
defer list_file.close(io);
|
||||
const wild_file = try dir.createFile(io, wild_tmp, .{ .permissions = .fromMode(0o600) });
|
||||
const wild_file = try dir.createFile(io, tmp.wild, .{ .permissions = .fromMode(0o600) });
|
||||
defer wild_file.close(io);
|
||||
const allow_file = try dir.createFile(io, tmp.allow, .{ .permissions = .fromMode(0o600) });
|
||||
defer allow_file.close(io);
|
||||
|
||||
const buffers = try self.gpa.alloc(u8, 3 * io_buf_len);
|
||||
const buffers = try self.gpa.alloc(u8, 4 * io_buf_len);
|
||||
defer self.gpa.free(buffers);
|
||||
|
||||
var fr = raw.reader(io, buffers[0..io_buf_len]);
|
||||
var list_w = list_file.writer(io, buffers[io_buf_len .. 2 * io_buf_len]);
|
||||
var wild_w = wild_file.writer(io, buffers[2 * io_buf_len ..]);
|
||||
var wild_w = wild_file.writer(io, buffers[2 * io_buf_len .. 3 * io_buf_len]);
|
||||
var allow_w = allow_file.writer(io, buffers[3 * io_buf_len ..]);
|
||||
|
||||
const result = compiler.compile(
|
||||
self.gpa,
|
||||
@@ -946,18 +998,22 @@ pub const Manager = struct {
|
||||
format,
|
||||
&list_w.interface,
|
||||
&wild_w.interface,
|
||||
&allow_w.interface,
|
||||
) catch |err| switch (err) {
|
||||
// `compiler.Error` names the direction; the concrete cause is on
|
||||
// the stream that failed.
|
||||
error.ReadFailed => return fr.err orelse err,
|
||||
error.WriteFailed => return list_w.err orelse (wild_w.err orelse err),
|
||||
error.WriteFailed => return list_w.err orelse
|
||||
(wild_w.err orelse (allow_w.err orelse err)),
|
||||
else => return err,
|
||||
};
|
||||
|
||||
try list_w.interface.flush();
|
||||
try wild_w.interface.flush();
|
||||
try allow_w.interface.flush();
|
||||
try list_file.sync(io);
|
||||
try wild_file.sync(io);
|
||||
try allow_file.sync(io);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -970,16 +1026,17 @@ pub const Manager = struct {
|
||||
dir: std.Io.Dir,
|
||||
id: i64,
|
||||
header: Header,
|
||||
list_tmp: []const u8,
|
||||
wild_tmp: []const u8,
|
||||
tmp: TempNames,
|
||||
) !void {
|
||||
const buffers = try self.gpa.alloc(u8, 2 * io_buf_len);
|
||||
defer self.gpa.free(buffers);
|
||||
|
||||
var list_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_buf: [name_buf_len]u8 = undefined;
|
||||
try publishOne(io, dir, compiledName(&list_buf, id, ".list"), list_tmp, header, buffers);
|
||||
try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), wild_tmp, header, buffers);
|
||||
var allow_buf: [name_buf_len]u8 = undefined;
|
||||
try publishOne(io, dir, compiledName(&list_buf, id, ".list"), tmp.list, header, buffers);
|
||||
try publishOne(io, dir, compiledName(&wild_buf, id, ".wild"), tmp.wild, header, buffers);
|
||||
try publishOne(io, dir, compiledName(&allow_buf, id, ".allow"), tmp.allow, header, buffers);
|
||||
}
|
||||
|
||||
fn publishOne(
|
||||
@@ -1014,12 +1071,18 @@ pub const Manager = struct {
|
||||
try af.replace(io);
|
||||
}
|
||||
|
||||
/// Whether the two compiled files on disk hash to `expected`. A missing,
|
||||
/// Whether the compiled files on disk hash to `expected`. A missing,
|
||||
/// unreadable or corrupt file answers false, which sends the caller down
|
||||
/// the rewrite path — the only path that can repair it.
|
||||
///
|
||||
/// A missing `.allow` file is the one exception, and it is the same one
|
||||
/// `loadSource` makes: it reads as an empty allow body, which is what a list
|
||||
/// with no `@@` line compiles to anyway. Answering false there would rewrite
|
||||
/// such a list on every refresh for no change in content.
|
||||
fn diskBodiesMatch(self: *Manager, io: std.Io, dir: std.Io.Dir, id: i64, expected: []const u8) bool {
|
||||
var list_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_buf: [name_buf_len]u8 = undefined;
|
||||
var allow_buf: [name_buf_len]u8 = undefined;
|
||||
const limit: std.Io.Limit = .limited(max_compiled_bytes);
|
||||
|
||||
const list_bytes = dir.readFileAlloc(io, compiledName(&list_buf, id, ".list"), self.gpa, limit) catch
|
||||
@@ -1029,7 +1092,11 @@ pub const Manager = struct {
|
||||
return false;
|
||||
defer self.gpa.free(wild_bytes);
|
||||
|
||||
return compiledBodiesMatch(list_bytes, wild_bytes, expected);
|
||||
const allow_bytes = dir.readFileAlloc(io, compiledName(&allow_buf, id, ".allow"), self.gpa, limit) catch |err|
|
||||
if (err == error.FileNotFound) @as([]u8, &.{}) else return false;
|
||||
defer self.gpa.free(allow_bytes);
|
||||
|
||||
return compiledBodiesMatch(list_bytes, wild_bytes, allow_bytes, expected);
|
||||
}
|
||||
|
||||
fn reportFetchFailure(
|
||||
@@ -1231,12 +1298,12 @@ pub const Manager = struct {
|
||||
/// sweeps to nothing.
|
||||
pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
|
||||
// `refresh_lock` first, and for the reason it exists: the download and
|
||||
// the compile are the only writers of `.raw.tmp`, `.list.tmp` and
|
||||
// `.wild.tmp`, and they hold it for as long as they run. Without it
|
||||
// here, a source deleted through the API would sweep the temporaries of
|
||||
// a refresh still writing them — the row is gone, so nothing else in
|
||||
// this function would spare them — and the pass would fail on a raw
|
||||
// file that vanished under it.
|
||||
// the compile are the only writers of `.raw.tmp`, `.list.tmp`,
|
||||
// `.wild.tmp` and `.allow.tmp`, and they hold it for as long as they
|
||||
// run. Without it here, a source deleted through the API would sweep
|
||||
// the temporaries of a refresh still writing them — the row is gone, so
|
||||
// nothing else in this function would spare them — and the pass would
|
||||
// fail on a raw file that vanished under it.
|
||||
//
|
||||
// `writer_lock` second, in the one order this file ever takes them,
|
||||
// because the rows this reads and the compiled files it deletes are
|
||||
@@ -1477,7 +1544,7 @@ fn applyLoadOutcomes(
|
||||
entry.loaded = true;
|
||||
// Two states survive a successful load. `.ok`, because a
|
||||
// refresh in this process already filled the counters the
|
||||
// compile produced and the three database columns are a subset
|
||||
// compile produced and the five database columns are a subset
|
||||
// of them. And any refresh failure, because the files that just
|
||||
// loaded are exactly the ones the failed refresh could not
|
||||
// replace, so the operator must still see why.
|
||||
@@ -1485,7 +1552,9 @@ fn applyLoadOutcomes(
|
||||
entry.succeed(row.last_updated orelse 0, .{
|
||||
.domains = countOf(row.domain_count),
|
||||
.wildcards = countOf(row.wildcard_count),
|
||||
.exceptions = countOf(row.exception_count),
|
||||
.skipped_regex = countOf(row.skipped_regex_count),
|
||||
.skipped_unsupported = countOf(row.skipped_unsupported_count),
|
||||
});
|
||||
},
|
||||
.failed => |reason| {
|
||||
@@ -1555,43 +1624,67 @@ fn collectSample(r: *std.Io.Reader, w: *std.Io.Writer) error{ ReadFailed, WriteF
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether two compiled files carry the bodies `expected` was taken over.
|
||||
fn compiledBodiesMatch(list_bytes: []const u8, wild_bytes: []const u8, expected: []const u8) bool {
|
||||
return std.mem.eql(u8, expected, &bodyChecksum(stripHeader(list_bytes), stripHeader(wild_bytes)));
|
||||
/// Whether three compiled files carry the bodies `expected` was taken over.
|
||||
fn compiledBodiesMatch(
|
||||
list_bytes: []const u8,
|
||||
wild_bytes: []const u8,
|
||||
allow_bytes: []const u8,
|
||||
expected: []const u8,
|
||||
) bool {
|
||||
return std.mem.eql(u8, expected, &bodyChecksum(
|
||||
stripHeader(list_bytes),
|
||||
stripHeader(wild_bytes),
|
||||
stripHeader(allow_bytes),
|
||||
));
|
||||
}
|
||||
|
||||
/// A compile that produced no entry at all while rejecting lines is an error
|
||||
/// page, a compressed body or a format the sniff got wrong — not a blocklist.
|
||||
/// Publishing it would replace a working list with nothing and report `ok`. An
|
||||
/// input that rejected nothing is an empty list, which is legal.
|
||||
///
|
||||
/// A list of nothing but exceptions is loadable: an allow-only list published
|
||||
/// beside a blocking one is a shape operators use, and it produces entries.
|
||||
fn rejectedWithoutEntries(counts: compiler.Counts) bool {
|
||||
if (counts.domains != 0 or counts.wildcards != 0) return false;
|
||||
if (counts.domains != 0 or counts.wildcards != 0 or counts.exceptions != 0) return false;
|
||||
return counts.invalid != 0 or counts.skipped_unsupported != 0 or counts.long_lines != 0;
|
||||
}
|
||||
|
||||
fn bodyChecksum(list_body: []const u8, wild_body: []const u8) [64]u8 {
|
||||
/// The digest the `.list`, `.wild` and `.allow` bodies share, in that order,
|
||||
/// each followed by `compiler.body_separator`.
|
||||
///
|
||||
/// Must stay byte-for-byte what `compiler.compile` produces, separators
|
||||
/// included: this is the other half of the same digest, and the two are
|
||||
/// compared against each other on every refresh.
|
||||
fn bodyChecksum(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) [64]u8 {
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(list_body);
|
||||
hasher.update(compiler.body_separator);
|
||||
hasher.update(wild_body);
|
||||
hasher.update(compiler.body_separator);
|
||||
hasher.update(allow_body);
|
||||
hasher.update(compiler.body_separator);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
return std.fmt.bytesToHex(digest, .lower);
|
||||
}
|
||||
|
||||
fn compiledName(buf: *[name_buf_len]u8, id: i64, suffix: []const u8) []const u8 {
|
||||
// An `i64` prints in at most 20 characters and the longest suffix is nine,
|
||||
// An `i64` prints in at most 20 characters and the longest suffix is ten,
|
||||
// so `name_buf_len` cannot be exceeded.
|
||||
return std.fmt.bufPrint(buf, "{d}{s}", .{ id, suffix }) catch unreachable;
|
||||
}
|
||||
|
||||
/// Every name `compiledName` can produce, longest suffix first so `.list.tmp`
|
||||
/// is never read as `.list`.
|
||||
const source_file_suffixes = [_][]const u8{ ".list.tmp", ".wild.tmp", ".raw.tmp", ".list", ".wild" };
|
||||
const source_file_suffixes = [_][]const u8{
|
||||
".allow.tmp", ".list.tmp", ".wild.tmp", ".raw.tmp", ".allow", ".list", ".wild",
|
||||
};
|
||||
|
||||
/// The source id a file under the blocklist directory belongs to, or null when
|
||||
/// the name is not one of ours.
|
||||
///
|
||||
/// The three temporaries count. A refresh that dies between writing one and
|
||||
/// The four temporaries count. A refresh that dies between writing one and
|
||||
/// renaming it leaves a file no later refresh reuses and no `defer` reaches, so
|
||||
/// excluding them from the sweep means nothing ever removes them. Matching them
|
||||
/// is safe because `pruneOrphans` holds `refresh_lock` for its whole body:
|
||||
@@ -1830,18 +1923,25 @@ test "a canceled compiled-file read cancels the reload instead of recording it"
|
||||
|
||||
const list_body = "aaa.example.com\n";
|
||||
const wild_body = "";
|
||||
const allow_body = "";
|
||||
var dir = try tmp.dir.createDirPathOpen(io, "blocklists", .{});
|
||||
defer dir.close(io);
|
||||
var list_buf: [name_buf_len]u8 = undefined;
|
||||
var wild_buf: [name_buf_len]u8 = undefined;
|
||||
var allow_buf: [name_buf_len]u8 = undefined;
|
||||
try dir.writeFile(io, .{ .sub_path = compiledName(&list_buf, id, ".list"), .data = list_body });
|
||||
try dir.writeFile(io, .{ .sub_path = compiledName(&wild_buf, id, ".wild"), .data = wild_body });
|
||||
// Present rather than absent, so the third read is a real one: `loadSource`
|
||||
// treats a missing `.allow` as an empty body and would never open it.
|
||||
try dir.writeFile(io, .{ .sub_path = compiledName(&allow_buf, id, ".allow"), .data = allow_body });
|
||||
try sources_repo.updateSourceStats(&database, id, .{
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 1,
|
||||
.wildcard_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, wild_body),
|
||||
.skipped_unsupported_count = 0,
|
||||
.exception_count = 0,
|
||||
.checksum = &bodyChecksum(list_body, wild_body, allow_body),
|
||||
});
|
||||
|
||||
// The baseline every assertion below is against: one clean reload, one
|
||||
@@ -1853,11 +1953,12 @@ test "a canceled compiled-file read cancels the reload instead of recording it"
|
||||
try testing.expect(out[0].loaded);
|
||||
const published = mgr.generation;
|
||||
|
||||
// Both catch sites, in the order `loadSource` reads the two files. A
|
||||
// Every catch site, in the order `loadSource` reads the three files. A
|
||||
// cancellation is consumed by whoever catches it, so folding it into a load
|
||||
// failure would spend the shutdown signal and leave a status row reading
|
||||
// "Canceled" behind.
|
||||
for ([_][]const u8{ ".list", ".wild" }) |suffix| {
|
||||
// "Canceled" behind. The `.allow` read is the one that can get this wrong
|
||||
// twice over: it also has to keep `FileNotFound` apart from a cancellation.
|
||||
for ([_][]const u8{ ".list", ".wild", ".allow" }) |suffix| {
|
||||
var vtable: std.Io.VTable = undefined;
|
||||
const canceling = cancelingIo(io, suffix, &vtable);
|
||||
try testing.expectError(error.Canceled, mgr.reload(canceling));
|
||||
@@ -1921,6 +2022,7 @@ test "the header writer produces the documented text" {
|
||||
.counts = .{
|
||||
.domains = 12,
|
||||
.wildcards = 3,
|
||||
.exceptions = 7,
|
||||
.skipped_regex = 2,
|
||||
.skipped_unsupported = 1,
|
||||
.invalid = 5,
|
||||
@@ -1938,6 +2040,7 @@ test "the header writer produces the documented text" {
|
||||
\\# fetched_at 1700000000
|
||||
\\# domains 12
|
||||
\\# wildcards 3
|
||||
\\# exceptions 7
|
||||
\\# skipped_regex 2
|
||||
\\# skipped_unsupported 1
|
||||
\\# invalid 5
|
||||
@@ -1960,6 +2063,7 @@ test "the log label names a source without printing what its url carries" {
|
||||
.domain_count = 0,
|
||||
.wildcard_count = 0,
|
||||
.skipped_regex_count = 0,
|
||||
.skipped_unsupported_count = 0,
|
||||
.checksum = null,
|
||||
};
|
||||
const printed = try std.fmt.bufPrint(&buf, "blocklist {f}: download failed: {s}", .{
|
||||
@@ -2042,32 +2146,51 @@ test "a success clears the recorded error" {
|
||||
try testing.expectEqualStrings("", status.errorText());
|
||||
}
|
||||
|
||||
test "compiledName spells the four file names of a source" {
|
||||
comptime {
|
||||
// The two tests below spell every suffix out instead of looping over
|
||||
// `source_file_suffixes`: a test that reads the table moves with it, so a
|
||||
// name dropped from the table would take the assertion that covers it along.
|
||||
// An eighth suffix breaks the build here until both are extended.
|
||||
std.debug.assert(source_file_suffixes.len == 7);
|
||||
}
|
||||
|
||||
test "compiledName spells every file name of a source" {
|
||||
var buf: [name_buf_len]u8 = undefined;
|
||||
try testing.expectEqualStrings("42.list", compiledName(&buf, 42, ".list"));
|
||||
try testing.expectEqualStrings("42.wild", compiledName(&buf, 42, ".wild"));
|
||||
try testing.expectEqualStrings("42.allow", compiledName(&buf, 42, ".allow"));
|
||||
try testing.expectEqualStrings("42.raw.tmp", compiledName(&buf, 42, ".raw.tmp"));
|
||||
try testing.expectEqualStrings("42.list.tmp", compiledName(&buf, 42, ".list.tmp"));
|
||||
try testing.expectEqualStrings("42.wild.tmp", compiledName(&buf, 42, ".wild.tmp"));
|
||||
try testing.expectEqualStrings("42.allow.tmp", compiledName(&buf, 42, ".allow.tmp"));
|
||||
}
|
||||
|
||||
test "sourceFileId matches every name a refresh writes, including the temporaries" {
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.allow"));
|
||||
|
||||
// A temporary left by a refresh that died belongs to its source id, so the
|
||||
// sweep can tell whether that source still has a row.
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.raw.tmp"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.list.tmp"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.wild.tmp"));
|
||||
try testing.expectEqual(@as(?i64, 7), sourceFileId("7.allow.tmp"));
|
||||
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.list"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.allow"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.raw.tmp"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("notes.allow.tmp"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("7.tmp"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("7.raw"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("7.allowed"));
|
||||
try testing.expectEqual(@as(?i64, null), sourceFileId("README"));
|
||||
}
|
||||
|
||||
test "every name compiledName writes is a name the sweep can attribute" {
|
||||
// A round-trip over the table, not a coverage check: this loop reads the
|
||||
// same array the code reads, so it cannot notice a missing entry. The two
|
||||
// tests above are what pins the set.
|
||||
var buf: [name_buf_len]u8 = undefined;
|
||||
for (source_file_suffixes) |suffix| {
|
||||
try testing.expectEqual(@as(?i64, 42), sourceFileId(compiledName(&buf, 42, suffix)));
|
||||
@@ -2113,7 +2236,9 @@ fn testRow(id: i64, enabled: bool) sources_repo.SourceRow {
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 9,
|
||||
.wildcard_count = 4,
|
||||
.exception_count = 2,
|
||||
.skipped_regex_count = 1,
|
||||
.skipped_unsupported_count = 5,
|
||||
.checksum = "0" ** 64,
|
||||
};
|
||||
}
|
||||
@@ -2224,6 +2349,9 @@ test "a load of a source this process never refreshed takes the row counters" {
|
||||
try testing.expectEqual(@as(u32, 9), statuses[0].counts.domains);
|
||||
try testing.expectEqual(@as(u32, 4), statuses[0].counts.wildcards);
|
||||
try testing.expectEqual(@as(u32, 1), statuses[0].counts.skipped_regex);
|
||||
// Rehydration: a restart reads this from the row and nowhere else, because
|
||||
// no path reparses a compiled file's header.
|
||||
try testing.expectEqual(@as(u32, 5), statuses[0].counts.skipped_unsupported);
|
||||
}
|
||||
|
||||
test "a status borrows nothing, so a copy outlives the table it came from" {
|
||||
@@ -2260,17 +2388,59 @@ test "SourceStatus truncates a long url at max_url_len" {
|
||||
test "compiledBodiesMatch verifies the bodies, not the presence of the files" {
|
||||
const list_body = "a.example.com\nb.example.com\n";
|
||||
const wild_body = "c.example.com\n";
|
||||
const expected = bodyChecksum(list_body, wild_body);
|
||||
const allow_body = "d.example.com\n";
|
||||
const expected = bodyChecksum(list_body, wild_body, allow_body);
|
||||
|
||||
const header =
|
||||
"# nxdns blocklist\n" ++
|
||||
"# url https://lists.example/hosts.txt\n";
|
||||
try testing.expect(compiledBodiesMatch(header ++ list_body, header ++ wild_body, &expected));
|
||||
try testing.expect(compiledBodiesMatch(
|
||||
header ++ list_body,
|
||||
header ++ wild_body,
|
||||
header ++ allow_body,
|
||||
&expected,
|
||||
));
|
||||
|
||||
// The corruption a reload reports as `ChecksumMismatch`: the file is there,
|
||||
// its body is not what the checksum was taken over.
|
||||
try testing.expect(!compiledBodiesMatch(header ++ "a.example.com\nb.exa", header ++ wild_body, &expected));
|
||||
try testing.expect(!compiledBodiesMatch("", "", &expected));
|
||||
// its body is not what the checksum was taken over. An allow body that lost
|
||||
// its entry counts, because a dropped exception silently restores a block.
|
||||
try testing.expect(!compiledBodiesMatch(
|
||||
header ++ "a.example.com\nb.exa",
|
||||
header ++ wild_body,
|
||||
header ++ allow_body,
|
||||
&expected,
|
||||
));
|
||||
try testing.expect(!compiledBodiesMatch(header ++ list_body, header ++ wild_body, "", &expected));
|
||||
try testing.expect(!compiledBodiesMatch("", "", "", &expected));
|
||||
}
|
||||
|
||||
test "bodyChecksum separates the three bodies" {
|
||||
const list_body = "a.example.com\nb.example.com\n";
|
||||
const wild_body = "c.example.com\n";
|
||||
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update(list_body);
|
||||
hasher.update(compiler.body_separator);
|
||||
hasher.update(wild_body);
|
||||
hasher.update(compiler.body_separator);
|
||||
hasher.update(compiler.body_separator);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
const expected = std.fmt.bytesToHex(digest, .lower);
|
||||
|
||||
try testing.expectEqualStrings(&expected, &bodyChecksum(list_body, wild_body, ""));
|
||||
// No `.allow` file: what `loadSource` and `diskBodiesMatch` pass for one.
|
||||
// It is an empty body, and an empty body still gets its separator.
|
||||
try testing.expect(compiledBodiesMatch(list_body, wild_body, "", &expected));
|
||||
|
||||
// The framing itself: the same bytes in a different body is a different
|
||||
// digest. Unframed these two are equal, and a stale `.list` survives an
|
||||
// upstream that switched the name to a wildcard.
|
||||
try testing.expect(!std.mem.eql(
|
||||
u8,
|
||||
&bodyChecksum("a.example\n", "", ""),
|
||||
&bodyChecksum("", "a.example\n", ""),
|
||||
));
|
||||
}
|
||||
|
||||
test "rejectedWithoutEntries fails a compile that produced nothing usable" {
|
||||
@@ -2391,14 +2561,28 @@ test "collectSample steps over a line that does not fit the reader buffer" {
|
||||
try testing.expectEqualStrings("ads.example.com\n", w.buffered());
|
||||
}
|
||||
|
||||
test "bodyChecksum covers the list body followed by the wild body" {
|
||||
const both = bodyChecksum("a.example.com\n", "b.example.com\n");
|
||||
test "bodyChecksum covers the list body, then the wild body, then the allow body" {
|
||||
const all = bodyChecksum("a.example.com\n", "b.example.com\n", "c.example.com\n");
|
||||
var hasher = Sha256.init(.{});
|
||||
hasher.update("a.example.com\nb.example.com\n");
|
||||
hasher.update("a.example.com\n");
|
||||
hasher.update(compiler.body_separator);
|
||||
hasher.update("b.example.com\n");
|
||||
hasher.update(compiler.body_separator);
|
||||
hasher.update("c.example.com\n");
|
||||
hasher.update(compiler.body_separator);
|
||||
var digest: [Sha256.digest_length]u8 = undefined;
|
||||
hasher.final(&digest);
|
||||
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &both);
|
||||
try testing.expectEqualStrings(&std.fmt.bytesToHex(digest, .lower), &all);
|
||||
|
||||
// Order matters: the two halves are not interchangeable.
|
||||
try testing.expect(!std.mem.eql(u8, &both, &bodyChecksum("b.example.com\n", "a.example.com\n")));
|
||||
// Order matters: the three parts are not interchangeable.
|
||||
try testing.expect(!std.mem.eql(
|
||||
u8,
|
||||
&all,
|
||||
&bodyChecksum("b.example.com\n", "a.example.com\n", "c.example.com\n"),
|
||||
));
|
||||
try testing.expect(!std.mem.eql(
|
||||
u8,
|
||||
&all,
|
||||
&bodyChecksum("a.example.com\n", "c.example.com\n", "b.example.com\n"),
|
||||
));
|
||||
}
|
||||
|
||||
+348
-11
@@ -27,6 +27,11 @@ pub const Reason = enum {
|
||||
rule_block_exact,
|
||||
rule_allow_wildcard,
|
||||
rule_block_wildcard,
|
||||
rule_allow_regex,
|
||||
rule_block_regex,
|
||||
/// An `@@` exception from a downloaded list. It cancels what another list
|
||||
/// blocks and never what a rule decides — see `evaluate`.
|
||||
blocklist_exception,
|
||||
blocklist_domain,
|
||||
blocklist_wildcard,
|
||||
};
|
||||
@@ -34,12 +39,14 @@ pub const Reason = enum {
|
||||
pub const Decision = struct {
|
||||
blocked: bool,
|
||||
reason: Reason,
|
||||
/// The candidate (for the exact and blocklist levels) or the pattern (for
|
||||
/// the wildcard levels) that decided it. Borrowed from the caller's
|
||||
/// normalized buffer or from the snapshot. "" when `reason == .none`.
|
||||
/// The candidate (for the exact, exception and blocklist levels) or the
|
||||
/// pattern (for the wildcard and regex levels) that decided it. Borrowed
|
||||
/// from the caller's normalized buffer or from the snapshot. "" when
|
||||
/// `reason == .none`.
|
||||
matched: []const u8,
|
||||
/// `.blocklist_*` only: index into `Snapshot.sources`, so the query log and
|
||||
/// the UI can name the list that blocked the query.
|
||||
/// the UI can name the list that blocked the query — or, for
|
||||
/// `.blocklist_exception`, the list that lifted it.
|
||||
source: ?u32 = null,
|
||||
};
|
||||
|
||||
@@ -95,6 +102,10 @@ pub const SourceSets = struct {
|
||||
name: []const u8,
|
||||
domains: domain_set.DomainSet,
|
||||
wildcards: domain_set.DomainSet,
|
||||
/// The names this source's `@@` exceptions lift. One entry covers the name
|
||||
/// and every subdomain of it, because `evaluate` walks the full name and
|
||||
/// each parent against this set.
|
||||
exceptions: domain_set.DomainSet,
|
||||
};
|
||||
|
||||
pub const Group = struct {
|
||||
@@ -122,7 +133,15 @@ pub const Snapshot = struct {
|
||||
/// can tell which generation answered a query.
|
||||
generation: u64,
|
||||
|
||||
pub const Compiled = struct { list_body: []const u8, wild_body: []const u8 };
|
||||
/// The compiled bodies of one source. `allow_body` defaults to empty
|
||||
/// because a source compiled before exceptions were honoured has no
|
||||
/// `.allow` file at all. A source refreshed since then always has one,
|
||||
/// empty when its list carries no `@@` line.
|
||||
pub const Compiled = struct {
|
||||
list_body: []const u8,
|
||||
wild_body: []const u8,
|
||||
allow_body: []const u8 = "",
|
||||
};
|
||||
|
||||
pub const Input = struct {
|
||||
groups: []const model.Group,
|
||||
@@ -192,6 +211,7 @@ pub const Snapshot = struct {
|
||||
.name = try arena.dupe(u8, row.name),
|
||||
.domains = try domain_set.DomainSet.build(arena, bodies.list_body, input.seed),
|
||||
.wildcards = try domain_set.DomainSet.build(arena, bodies.wild_body, input.seed),
|
||||
.exceptions = try domain_set.DomainSet.build(arena, bodies.allow_body, input.seed),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -221,7 +241,11 @@ pub const Snapshot = struct {
|
||||
.id = id,
|
||||
.name = try arena.dupe(u8, row.name),
|
||||
.safe_search = row.safe_search,
|
||||
.rules = try rules.RuleSet.build(arena, group_rules.items, input.seed),
|
||||
// `arena` retains, `gpa` scratches: an arena reclaims only its
|
||||
// most recent allocation, so a rule build's temporaries taken
|
||||
// from it would outlive the build and go unreported by
|
||||
// `memoryBytes`.
|
||||
.rules = try rules.RuleSet.build(arena, gpa, group_rules.items, input.seed),
|
||||
.sources = try arena.dupe(u32, dedupSorted(group_sources.items)),
|
||||
};
|
||||
}
|
||||
@@ -269,7 +293,9 @@ pub const Snapshot = struct {
|
||||
/// PLAN §3.10 precedence, allow winning at equal specificity:
|
||||
/// 1. exact/parent allow rules 2. exact/parent block rules
|
||||
/// 3. wildcard allow rules 4. wildcard block rules
|
||||
/// 5. blocklist domains 6. blocklist wildcards
|
||||
/// 5. regex allow rules 6. regex block rules
|
||||
/// 7. blocklist exceptions
|
||||
/// 8. blocklist domains 9. blocklist wildcards
|
||||
///
|
||||
/// The order is level-by-level over the whole candidate chain, not
|
||||
/// candidate-by-candidate over the levels: level 1 is checked against every
|
||||
@@ -277,10 +303,24 @@ pub const Snapshot = struct {
|
||||
/// allow rule on the parent beat a block rule on the child, which is the
|
||||
/// behaviour an allow list is written for.
|
||||
///
|
||||
/// Level 5 tests only the full name and level 6 tests only proper parents:
|
||||
/// Levels 5 and 6 are last among the operator rules because they are the
|
||||
/// only ones that cost more than a hash lookup or a label walk: a regex is
|
||||
/// reached only once every set-shaped level has missed. They are matched
|
||||
/// against the full name alone — a pattern that should cover subdomains
|
||||
/// says so, which is what an unanchored regex already does.
|
||||
///
|
||||
/// Level 7 is where a downloaded list's `@@` exceptions are honoured, and
|
||||
/// its position is the whole safety argument: every operator rule has
|
||||
/// already returned by the time it runs, so an exception can cancel a block
|
||||
/// levels 8 and 9 would have made and nothing else. No downloaded list can
|
||||
/// open an allow hole the operator did not open. It walks the full name and
|
||||
/// every parent, because one `@@||x^` entry lifts `x` together with its
|
||||
/// subdomains.
|
||||
///
|
||||
/// Level 8 tests only the full name and level 9 tests only proper parents:
|
||||
/// a `.list` entry is the domain itself, a `.wild` entry is what `*.x.y`
|
||||
/// means. Both walk the group's sources in ascending index order, so the
|
||||
/// reported source is stable for a given snapshot.
|
||||
/// means. All three list levels walk the group's sources in ascending index
|
||||
/// order, so the reported source is stable for a given snapshot.
|
||||
///
|
||||
/// `domain` is normalized (`normalize`). No allocation, no lock, no clock.
|
||||
pub fn evaluate(self: *const Snapshot, group: u32, domain: []const u8) Decision {
|
||||
@@ -307,6 +347,27 @@ pub const Snapshot = struct {
|
||||
return .{ .blocked = true, .reason = .rule_block_wildcard, .matched = pattern };
|
||||
}
|
||||
|
||||
if (rules.matchRegex(g.rules.regex_allow, domain)) |pattern| {
|
||||
return .{ .blocked = false, .reason = .rule_allow_regex, .matched = pattern };
|
||||
}
|
||||
if (rules.matchRegex(g.rules.regex_block, domain)) |pattern| {
|
||||
return .{ .blocked = true, .reason = .rule_block_regex, .matched = pattern };
|
||||
}
|
||||
|
||||
var exceptions: Candidates = .init(domain);
|
||||
while (exceptions.next()) |candidate| {
|
||||
for (g.sources) |index| {
|
||||
if (self.sources[index].exceptions.contains(candidate)) {
|
||||
return .{
|
||||
.blocked = false,
|
||||
.reason = .blocklist_exception,
|
||||
.matched = candidate,
|
||||
.source = index,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (g.sources) |index| {
|
||||
if (self.sources[index].domains.contains(domain)) {
|
||||
return .{
|
||||
@@ -373,7 +434,8 @@ pub const Snapshot = struct {
|
||||
var total: usize = 0;
|
||||
for (self.sources) |*source| {
|
||||
total += @sizeOf(SourceSets) + source.name.len +
|
||||
source.domains.memoryBytes() + source.wildcards.memoryBytes();
|
||||
source.domains.memoryBytes() + source.wildcards.memoryBytes() +
|
||||
source.exceptions.memoryBytes();
|
||||
}
|
||||
for (self.groups) |*group| {
|
||||
total += @sizeOf(Group) + group.name.len +
|
||||
@@ -586,6 +648,71 @@ test "precedence: a block wildcard with no allow blocks" {
|
||||
try testing.expectEqualStrings("*.example.com", decision.matched);
|
||||
}
|
||||
|
||||
test "precedence: a block regex with no allow blocks, and reports its pattern" {
|
||||
const rows = [_]model.Rule{rule("^ad[0-9]+-", .regex, .block)};
|
||||
var snapshot = try build(testing.allocator, .{ .rules = &rows });
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "ad42-tracker.example.com");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_block_regex, decision.reason);
|
||||
// `matched` is the pattern the operator wrote, which is what the query log
|
||||
// has to name for the block to be explicable.
|
||||
try testing.expectEqualStrings("^ad[0-9]+-", decision.matched);
|
||||
|
||||
// Unanchored at the tail, anchored at the head: the digits must lead.
|
||||
try testing.expect(!snapshot.evaluate(0, "x.ad42-tracker.example.com").blocked);
|
||||
try testing.expect(!snapshot.evaluate(0, "ads.example.com").blocked);
|
||||
}
|
||||
|
||||
test "precedence: an allow regex beats a block regex that matches the same name" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("tracker", .regex, .block),
|
||||
rule("^good\\.", .regex, .allow),
|
||||
};
|
||||
var snapshot = try build(testing.allocator, .{ .rules = &rows });
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.tracker.example.com");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_allow_regex, decision.reason);
|
||||
try testing.expectEqualStrings("^good\\.", decision.matched);
|
||||
|
||||
try testing.expect(snapshot.evaluate(0, "bad.tracker.example.com").blocked);
|
||||
}
|
||||
|
||||
test "precedence: both wildcard levels beat an allow regex that matches" {
|
||||
// The adjacent pair either side of the wildcard/regex boundary. A regex is
|
||||
// the most expensive level and therefore the last operator level, so a
|
||||
// wildcard decides first whichever way it decides.
|
||||
for ([_]model.Rule{
|
||||
rule("*.example.com", .wildcard, .allow),
|
||||
rule("*.example.com", .wildcard, .block),
|
||||
}) |wild| {
|
||||
const rows = [_]model.Rule{ wild, rule("example", .regex, .allow) };
|
||||
var snapshot = try build(testing.allocator, .{ .rules = &rows });
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "a.example.com");
|
||||
try testing.expectEqual(wild.action == .block, decision.blocked);
|
||||
try testing.expect(decision.reason == .rule_allow_wildcard or
|
||||
decision.reason == .rule_block_wildcard);
|
||||
}
|
||||
}
|
||||
|
||||
test "precedence: an exact allow rule beats a block regex" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("tracker", .regex, .block),
|
||||
rule("good.tracker.example.com", .exact, .allow),
|
||||
};
|
||||
var snapshot = try build(testing.allocator, .{ .rules = &rows });
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.tracker.example.com");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_allow_exact, decision.reason);
|
||||
}
|
||||
|
||||
const one_source = [_]model.BlocklistSource{.{ .url = "https://lists.test/a", .name = "list a" }};
|
||||
const one_source_id = [_]i64{11};
|
||||
const one_link = [_]model.GroupSource{
|
||||
@@ -601,6 +728,14 @@ const Lists = struct {
|
||||
return .{ .compiled = .{.{ .list_body = list_body, .wild_body = wild_body }} };
|
||||
}
|
||||
|
||||
fn initWithExceptions(list_body: []const u8, wild_body: []const u8, allow_body: []const u8) Lists {
|
||||
return .{ .compiled = .{.{
|
||||
.list_body = list_body,
|
||||
.wild_body = wild_body,
|
||||
.allow_body = allow_body,
|
||||
}} };
|
||||
}
|
||||
|
||||
fn fixture(self: *const Lists) Fixture {
|
||||
return .{
|
||||
.sources = &one_source,
|
||||
@@ -669,6 +804,208 @@ test "precedence: an allow rule beats a wild entry" {
|
||||
try testing.expectEqual(Reason.rule_allow_exact, decision.reason);
|
||||
}
|
||||
|
||||
// --- list exceptions (milestone 21 ruling 2) --------------------------------
|
||||
|
||||
/// The fixture ruling 2 is written against: one list that blocks `ads.example`
|
||||
/// and its subdomains, and lifts `good.ads.example` back out.
|
||||
const exception_lists: Lists = .initWithExceptions(
|
||||
"ads.example\n",
|
||||
"ads.example\n",
|
||||
"good.ads.example\n",
|
||||
);
|
||||
|
||||
test "precedence: a list exception beats a list domain entry" {
|
||||
var snapshot = try build(testing.allocator, exception_lists.fixture());
|
||||
defer snapshot.deinit();
|
||||
|
||||
// The apex is blocked by the `.list` entry; the excepted name is not, even
|
||||
// though the same source blocks it through `.wild`.
|
||||
try testing.expect(snapshot.evaluate(0, "ads.example").blocked);
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.ads.example");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
|
||||
try testing.expectEqualStrings("good.ads.example", decision.matched);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: a list exception beats a list domain entry on the same name" {
|
||||
// The boundary the fixture above cannot pin: `good.ads.example` is not in
|
||||
// its `.list` body, so that test compares the exception against the
|
||||
// wildcard level. Here one name is carried by both `.allow` and `.list`,
|
||||
// which is the only way level 7 and level 8 are reached by one query.
|
||||
const lists: Lists = .initWithExceptions(
|
||||
"good.ads.example\n",
|
||||
"",
|
||||
"good.ads.example\n",
|
||||
);
|
||||
var snapshot = try build(testing.allocator, lists.fixture());
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.ads.example");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
|
||||
try testing.expectEqualStrings("good.ads.example", decision.matched);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: a list domain entry beats a list wildcard entry" {
|
||||
// Level 8 over level 9: the name is its own `.list` entry and a subdomain
|
||||
// of a `.wild` entry, so both would block and the reported reason is what
|
||||
// separates them.
|
||||
const lists: Lists = .init("x.ads.example\n", "ads.example\n");
|
||||
var snapshot = try build(testing.allocator, lists.fixture());
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "x.ads.example");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_domain, decision.reason);
|
||||
try testing.expectEqualStrings("x.ads.example", decision.matched);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: a list exception beats a list wildcard entry" {
|
||||
var snapshot = try build(testing.allocator, exception_lists.fixture());
|
||||
defer snapshot.deinit();
|
||||
|
||||
try testing.expect(snapshot.evaluate(0, "x.ads.example").blocked);
|
||||
|
||||
// The parent walk: one `@@||good.ads.example^` entry covers the subdomains
|
||||
// of the excepted name as well as the name itself.
|
||||
const decision = snapshot.evaluate(0, "y.good.ads.example");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
|
||||
try testing.expectEqualStrings("good.ads.example", decision.matched);
|
||||
try testing.expectEqual(@as(?u32, 0), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: an operator block rule beats a list exception" {
|
||||
// The property the exception level's position exists for: a downloaded list
|
||||
// may cancel what another list blocks and may never cancel what the
|
||||
// operator decided. All three operator block levels are checked, because
|
||||
// all three sit above the exception level.
|
||||
for ([_]model.Rule{
|
||||
rule("good.ads.example", .exact, .block),
|
||||
rule("*.ads.example", .wildcard, .block),
|
||||
rule("^good\\.ads\\.example$", .regex, .block),
|
||||
}) |blocking| {
|
||||
var fixture = exception_lists.fixture();
|
||||
const rows = [_]model.Rule{blocking};
|
||||
fixture.rules = &rows;
|
||||
|
||||
var snapshot = try build(testing.allocator, fixture);
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.ads.example");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expect(decision.reason == .rule_block_exact or
|
||||
decision.reason == .rule_block_wildcard or
|
||||
decision.reason == .rule_block_regex);
|
||||
}
|
||||
}
|
||||
|
||||
test "the regex reasons render as the wire strings the API and the query log carry" {
|
||||
// `web/handlers/lookup.zig` renders a reason as `@tagName`, and
|
||||
// `storage/logger.zig` stores one in a 32-byte `max_reason_len` buffer.
|
||||
// Neither can be reached from this file — pure core imports no web and no
|
||||
// storage — so the tag names and their length are pinned here.
|
||||
try testing.expectEqualStrings("rule_allow_regex", @tagName(Reason.rule_allow_regex));
|
||||
try testing.expectEqualStrings("rule_block_regex", @tagName(Reason.rule_block_regex));
|
||||
inline for (@typeInfo(Reason).@"enum".fields) |field| {
|
||||
try testing.expect(field.name.len <= 32);
|
||||
}
|
||||
}
|
||||
|
||||
test "precedence: an allow regex beats every list level" {
|
||||
// The other side of the same boundary: an operator allow rule lifts a list
|
||||
// block, whichever of the three list levels made it.
|
||||
const rows = [_]model.Rule{rule("ads\\.example$", .regex, .allow)};
|
||||
var fixture = exception_lists.fixture();
|
||||
fixture.rules = &rows;
|
||||
|
||||
var snapshot = try build(testing.allocator, fixture);
|
||||
defer snapshot.deinit();
|
||||
|
||||
// `.list` blocks the apex and `.wild` blocks the subdomains; the regex
|
||||
// covers both, and answers before either is consulted.
|
||||
for ([_][]const u8{ "ads.example", "x.ads.example" }) |domain| {
|
||||
const decision = snapshot.evaluate(0, domain);
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_allow_regex, decision.reason);
|
||||
try testing.expectEqual(@as(?u32, null), decision.source);
|
||||
}
|
||||
}
|
||||
|
||||
test "precedence: a block regex blocks a name no list carries" {
|
||||
const rows = [_]model.Rule{rule("^ad[0-9]+-", .regex, .block)};
|
||||
var fixture = exception_lists.fixture();
|
||||
fixture.rules = &rows;
|
||||
|
||||
var snapshot = try build(testing.allocator, fixture);
|
||||
defer snapshot.deinit();
|
||||
|
||||
const decision = snapshot.evaluate(0, "ad7-cdn.other.example");
|
||||
try testing.expect(decision.blocked);
|
||||
try testing.expectEqual(Reason.rule_block_regex, decision.reason);
|
||||
try testing.expectEqual(@as(?u32, null), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: a list exception is scoped to the groups the source is in" {
|
||||
const groups = [_]model.Group{ .{ .name = "default" }, .{ .name = "kids" } };
|
||||
const ids = [_]i64{ 1, 2 };
|
||||
var snapshot = try build(testing.allocator, .{
|
||||
.groups = &groups,
|
||||
.group_ids = &ids,
|
||||
.sources = &one_source,
|
||||
.source_ids = &one_source_id,
|
||||
.group_sources = &one_link,
|
||||
.compiled = &exception_lists.compiled,
|
||||
});
|
||||
defer snapshot.deinit();
|
||||
|
||||
const kids = snapshot.groupIndexByName("kids").?;
|
||||
try testing.expectEqual(
|
||||
Reason.blocklist_exception,
|
||||
snapshot.evaluate(snapshot.default_group, "good.ads.example").reason,
|
||||
);
|
||||
// `kids` is linked to no source, so neither the block nor the exception
|
||||
// reaches it.
|
||||
try testing.expectEqual(Reason.none, snapshot.evaluate(kids, "good.ads.example").reason);
|
||||
}
|
||||
|
||||
test "precedence: an exception in one list lifts the block another list made" {
|
||||
const sources = [_]model.BlocklistSource{
|
||||
.{ .url = "https://lists.test/a", .name = "list a" },
|
||||
.{ .url = "https://lists.test/b", .name = "list b" },
|
||||
};
|
||||
const source_ids = [_]i64{ 11, 12 };
|
||||
const links = [_]model.GroupSource{
|
||||
.{ .group = "default", .source_url = "https://lists.test/a" },
|
||||
.{ .group = "default", .source_url = "https://lists.test/b" },
|
||||
};
|
||||
const compiled = [_]?Snapshot.Compiled{
|
||||
.{ .list_body = "ads.example\n", .wild_body = "ads.example\n" },
|
||||
.{ .list_body = "", .wild_body = "", .allow_body = "good.ads.example\n" },
|
||||
};
|
||||
|
||||
var snapshot = try build(testing.allocator, .{
|
||||
.sources = &sources,
|
||||
.source_ids = &source_ids,
|
||||
.group_sources = &links,
|
||||
.compiled = &compiled,
|
||||
});
|
||||
defer snapshot.deinit();
|
||||
|
||||
try testing.expect(snapshot.evaluate(0, "ads.example").blocked);
|
||||
|
||||
const decision = snapshot.evaluate(0, "good.ads.example");
|
||||
try testing.expect(!decision.blocked);
|
||||
try testing.expectEqual(Reason.blocklist_exception, decision.reason);
|
||||
// The source reported is the one that lifted the block, not the one that
|
||||
// made it.
|
||||
try testing.expectEqual(@as(?u32, 1), decision.source);
|
||||
}
|
||||
|
||||
test "precedence: nothing configured allows with reason none" {
|
||||
var snapshot = try build(testing.allocator, .{});
|
||||
defer snapshot.deinit();
|
||||
|
||||
+132
-8
@@ -1,9 +1,17 @@
|
||||
//! The Adblock Plus filter syntax, restricted to what a DNS sinkhole can
|
||||
//! honour: domain anchors and bare names. Pure, `std` only.
|
||||
//!
|
||||
//! Exception rules (`@@`) are `.unsupported` rather than an allow entry. The
|
||||
//! allow surface is the `rules` table, and a downloaded list that could quietly
|
||||
//! allow a domain across every group is a policy hole the operator did not open.
|
||||
//! An exception rule is `.exception` in exactly two spellings, `@@||name^` and
|
||||
//! `@@||name`, each of which may carry the literal `$important` behind it. Every
|
||||
//! other `@@` form stays `.unsupported`: a bare `@@name`, a path, a scheme, any
|
||||
//! other modifier.
|
||||
//!
|
||||
//! What makes honouring them safe is where they land, not what they say. A list
|
||||
//! exception is evaluated below every operator rule (PLAN §3.10), so it can
|
||||
//! cancel a block another list made and nothing else. No downloaded list can
|
||||
//! open an allow hole the operator did not open, which is why the allow surface
|
||||
//! stays the `rules` table — including its `.regex` kind, which is the one
|
||||
//! regex dialect nxdns evaluates and which no downloaded list can reach.
|
||||
|
||||
const std = @import("std");
|
||||
const parsers = @import("parsers.zig");
|
||||
@@ -12,6 +20,31 @@ const parsers = @import("parsers.zig");
|
||||
/// this syntax and only a trailing one is meaningful for a domain rule.
|
||||
const rule_tokens = "*^|/$";
|
||||
|
||||
/// The one modifier an exception line may carry. AdGuard-authored lists write it
|
||||
/// on most of their `@@` rules and it changes nothing here: these exceptions
|
||||
/// already sit below every operator rule, so "important" cannot raise one above
|
||||
/// the decisions it is not allowed to reach.
|
||||
const important_modifier = "$important";
|
||||
|
||||
/// A candidate the compiler could turn into a name, for the two anchored forms
|
||||
/// only. `rule_tokens` covers the syntax characters; this also refuses
|
||||
/// whitespace, which is neither a rule token nor a control byte and so used to
|
||||
/// survive into a compiled body as an entry only a query carrying the same
|
||||
/// space could match. That is true of `||name` and `@@||name` because
|
||||
/// `compiler.zig` hands a `.wildcard` or `.exception` text to `addCandidate`
|
||||
/// whole. A `.domain` text is tokenized on whitespace first and each field
|
||||
/// filed separately, so the bare form does not come through here: refusing a
|
||||
/// space there would drop the hosts-style lines that a mixed list classified
|
||||
/// `abp` by `detectFormat` still contributes.
|
||||
fn isNameCandidate(candidate: []const u8) bool {
|
||||
if (candidate.len == 0) return false;
|
||||
if (std.mem.findAny(u8, candidate, rule_tokens) != null) return false;
|
||||
for (candidate) |c| {
|
||||
if (std.ascii.isWhitespace(c) or std.ascii.isControl(c)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
pub fn parseLine(line: []const u8) parsers.Line {
|
||||
const text = std.mem.trim(u8, line, &std.ascii.whitespace);
|
||||
if (text.len == 0) return .{ .kind = .ignore };
|
||||
@@ -19,15 +52,14 @@ pub fn parseLine(line: []const u8) parsers.Line {
|
||||
if (text[0] == '[') return .{ .kind = .ignore };
|
||||
if (parsers.isElementHiding(text)) return .{ .kind = .unsupported };
|
||||
if (text[0] == '#') return .{ .kind = .ignore };
|
||||
if (std.mem.startsWith(u8, text, "@@")) return .{ .kind = .unsupported };
|
||||
if (std.mem.startsWith(u8, text, "@@")) return parseException(text[2..]);
|
||||
if (text[0] == '/') return .{ .kind = .regex };
|
||||
if (std.mem.findScalar(u8, text, '$') != null) return .{ .kind = .unsupported };
|
||||
|
||||
if (std.mem.startsWith(u8, text, "||")) {
|
||||
var candidate = text[2..];
|
||||
if (std.mem.endsWith(u8, candidate, "^")) candidate = candidate[0 .. candidate.len - 1];
|
||||
if (candidate.len == 0) return .{ .kind = .unsupported };
|
||||
if (std.mem.findAny(u8, candidate, rule_tokens) != null) return .{ .kind = .unsupported };
|
||||
if (!isNameCandidate(candidate)) return .{ .kind = .unsupported };
|
||||
// A domain anchor covers the domain itself as well as its subdomains,
|
||||
// so the compiler emits an apex entry beside the wildcard one.
|
||||
return .{ .kind = .wildcard, .text = candidate, .covers_apex = true };
|
||||
@@ -37,6 +69,24 @@ pub fn parseLine(line: []const u8) parsers.Line {
|
||||
return .{ .kind = .domain, .text = text };
|
||||
}
|
||||
|
||||
/// One exception line, past its `@@`. The domain anchor and the trailing `^` get
|
||||
/// the same treatment they get on a block rule, so `@@||x^` and `||x^` accept
|
||||
/// and reject the same names.
|
||||
///
|
||||
/// `$important` is stripped before the anchor is read, because the `$` would
|
||||
/// otherwise be a rule token and refuse the whole line.
|
||||
fn parseException(rest: []const u8) parsers.Line {
|
||||
if (!std.mem.startsWith(u8, rest, "||")) return .{ .kind = .unsupported };
|
||||
|
||||
var candidate = rest[2..];
|
||||
if (std.mem.endsWith(u8, candidate, important_modifier)) {
|
||||
candidate = candidate[0 .. candidate.len - important_modifier.len];
|
||||
}
|
||||
if (std.mem.endsWith(u8, candidate, "^")) candidate = candidate[0 .. candidate.len - 1];
|
||||
if (!isNameCandidate(candidate)) return .{ .kind = .unsupported };
|
||||
return .{ .kind = .exception, .text = candidate, .covers_apex = true };
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "a bang comment is ignored" {
|
||||
@@ -69,8 +119,46 @@ test "a modifier list is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("||example.com^$third-party").kind);
|
||||
}
|
||||
|
||||
test "an exception rule is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("@@||example.com^").kind);
|
||||
test "an exception rule is an exception that covers its apex" {
|
||||
const line = parseLine("@@||example.com^");
|
||||
try testing.expectEqual(parsers.Kind.exception, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
try testing.expect(line.covers_apex);
|
||||
}
|
||||
|
||||
test "an exception rule without a separator is still an exception" {
|
||||
const line = parseLine("@@||example.com");
|
||||
try testing.expectEqual(parsers.Kind.exception, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
try testing.expect(line.covers_apex);
|
||||
}
|
||||
|
||||
test "an exception rule tolerates the important modifier" {
|
||||
for ([_][]const u8{ "@@||example.com^$important", "@@||example.com$important" }) |text| {
|
||||
const line = parseLine(text);
|
||||
try testing.expectEqual(parsers.Kind.exception, line.kind);
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
try testing.expect(line.covers_apex);
|
||||
}
|
||||
}
|
||||
|
||||
test "every exception form outside the two anchored ones is unsupported" {
|
||||
for ([_][]const u8{
|
||||
// No domain anchor: this is a substring rule in browser syntax, and
|
||||
// reading it as a name would allow far more than it says.
|
||||
"@@example.com",
|
||||
"@@|http://example.com",
|
||||
"@@||example.com/path^",
|
||||
"@@||example.com^$third-party",
|
||||
"@@||example.com^$important$third-party",
|
||||
"@@||example.com^$dnstype=A",
|
||||
"@@||^",
|
||||
"@@||$important",
|
||||
"@@",
|
||||
"@@||ads*.example.com^",
|
||||
}) |text| {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine(text).kind);
|
||||
}
|
||||
}
|
||||
|
||||
test "element hiding is unsupported" {
|
||||
@@ -94,6 +182,42 @@ test "a bare name is a domain" {
|
||||
try testing.expectEqualStrings("example.com", line.text);
|
||||
}
|
||||
|
||||
test "a candidate carrying whitespace is unsupported in the anchored forms" {
|
||||
// A space is not a rule token and it is not a control byte, so it used to
|
||||
// reach the compiler, which lowercases and length-checks but does not
|
||||
// reject it. An anchored form's text is filed whole, so the entry it wrote
|
||||
// could only ever match a query name carrying the same space.
|
||||
for ([_][]const u8{
|
||||
"||good.example bad.example^",
|
||||
"@@||good.example bad.example^",
|
||||
"||good.example\tbad.example",
|
||||
"@@||good.example\tbad.example",
|
||||
}) |text| {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine(text).kind);
|
||||
}
|
||||
}
|
||||
|
||||
test "a bare candidate carrying whitespace stays a domain" {
|
||||
// Not the same case: `compiler.zig` tokenizes a `.domain` text on
|
||||
// whitespace and files each field. Refusing it here would drop the
|
||||
// hosts-style lines of a mixed list, which `detectFormat` classifies `abp`
|
||||
// as a whole and which only reach a compiled body through that split.
|
||||
//
|
||||
// What this test pins is the parser half — the kind and the untouched text.
|
||||
// The split itself belongs to the compiler and is asserted there, by
|
||||
// "an abp list's hosts-style lines reach the domain body through the split":
|
||||
// a tokenizer removed from `compile` would leave every assertion below true.
|
||||
for ([_][]const u8{
|
||||
"good.example bad.example",
|
||||
"0.0.0.0 ads.example",
|
||||
"good.example\tbad.example",
|
||||
}) |text| {
|
||||
const line = parseLine(text);
|
||||
try testing.expectEqual(parsers.Kind.domain, line.kind);
|
||||
try testing.expectEqualStrings(text, line.text);
|
||||
}
|
||||
}
|
||||
|
||||
test "a rule token outside the supported forms is unsupported" {
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("ads*.example.com").kind);
|
||||
try testing.expectEqual(parsers.Kind.unsupported, parseLine("example.com^").kind);
|
||||
|
||||
+12
-4
@@ -22,10 +22,15 @@ pub const Kind = enum {
|
||||
domain,
|
||||
/// `text` holds one candidate suffix; every proper subdomain of it matches.
|
||||
wildcard,
|
||||
/// A regex rule. Counted, skipped, never compiled (PLAN §2.2).
|
||||
/// A regex line in a downloaded list. Counted, skipped, never compiled: the
|
||||
/// engine exists for rules the operator wrote, not for lists (PLAN §2.2).
|
||||
regex,
|
||||
/// `text` holds one candidate name an ABP exception rule (`@@||x^`) lifts:
|
||||
/// the name itself and every subdomain of it. Only the ABP parser emits it.
|
||||
exception,
|
||||
/// Syntactically a rule of this format, but one nxdns cannot honour:
|
||||
/// an ABP modifier list, an exception rule, element hiding, a scheme anchor.
|
||||
/// an ABP modifier list, an exception form outside `@@||x^`, element
|
||||
/// hiding, a scheme anchor.
|
||||
unsupported,
|
||||
};
|
||||
|
||||
@@ -33,8 +38,11 @@ pub const Line = struct {
|
||||
kind: Kind,
|
||||
/// Borrowed from the caller's line. Not lowercased, not validated.
|
||||
text: []const u8 = "",
|
||||
/// `.wildcard` only. ABP `||x^` covers `x` itself as well as its subdomains,
|
||||
/// so the compiler emits an additional `.list` entry when this is set.
|
||||
/// `.wildcard` and `.exception` only: the rule covers the anchored name
|
||||
/// itself as well as its subdomains, which is what ABP `||x^` and `@@||x^`
|
||||
/// mean. The compiler acts on it for a `.wildcard` line, by emitting an
|
||||
/// additional `.list` entry; an `.exception` line needs no second entry,
|
||||
/// because the allow walk tests the full name as well as its parents.
|
||||
covers_apex: bool = false,
|
||||
};
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+339
-62
@@ -1,4 +1,4 @@
|
||||
//! One group's explicit rules (PLAN §3.10 levels 1–4), compiled once into an
|
||||
//! One group's explicit rules (PLAN §3.10 levels 1–6), compiled once into an
|
||||
//! immutable form the query path can read without allocating.
|
||||
//!
|
||||
//! Exact patterns go into a `DomainSet`; wildcard patterns stay a flat, sorted
|
||||
@@ -7,7 +7,13 @@
|
||||
//! that many short patterns is cheaper than an index that would have to be
|
||||
//! rebuilt on every snapshot swap.
|
||||
//!
|
||||
//! Pure: an allocator and plain values, no `std.Io`, no clock, no entropy
|
||||
//! Regex patterns are compiled here, once per snapshot, into the linear-time
|
||||
//! programs of `regex.zig` and scanned the same way. `max_regex_per_group` caps
|
||||
//! them far lower, at 256: a regex costs a whole VM run where a wildcard costs a
|
||||
//! label comparison, and the matcher reaches them only after every hash and
|
||||
//! wildcard level has missed.
|
||||
//!
|
||||
//! Pure: allocators and plain values, no `std.Io`, no clock, no entropy
|
||||
//! source. The hash seed arrives as a parameter.
|
||||
|
||||
const std = @import("std");
|
||||
@@ -17,14 +23,40 @@ const model = @import("../config/model.zig");
|
||||
const name = @import("../dns/name.zig");
|
||||
const types = @import("../dns/types.zig");
|
||||
const domain_set = @import("domain_set.zig");
|
||||
const regex = @import("regex.zig");
|
||||
const wildcard = @import("wildcard.zig");
|
||||
|
||||
pub const Error = error{ OutOfMemory, BadPattern, TooManyWildcards } || domain_set.DomainSet.Error;
|
||||
pub const Error = error{
|
||||
OutOfMemory,
|
||||
BadPattern,
|
||||
TooManyWildcards,
|
||||
TooManyRegexRules,
|
||||
} || domain_set.DomainSet.Error;
|
||||
|
||||
/// Both wildcard lists of one group together. The cap exists so a rules table
|
||||
/// edited into the millions cannot turn every query into a linear scan.
|
||||
pub const max_wildcards_per_group: usize = 4096;
|
||||
|
||||
/// Both regex lists of one group together, capped well below the wildcards: a
|
||||
/// miss at this level runs every program to its end.
|
||||
pub const max_regex_per_group: usize = 256;
|
||||
|
||||
/// A compiled operator regex beside the text it was written as. The text is what
|
||||
/// `Decision.matched` reports, so the query log names the rule the operator
|
||||
/// wrote rather than an instruction count.
|
||||
pub const RegexRule = struct {
|
||||
pattern: []const u8,
|
||||
program: regex.Program,
|
||||
|
||||
/// Frees through a copy of the program: the slices holding these rules are
|
||||
/// `const`, and `Program.deinit` wants a mutable pointer only to blank the
|
||||
/// struct it is finished with.
|
||||
fn free(self: RegexRule, gpa: Allocator) void {
|
||||
var program = self.program;
|
||||
program.deinit(gpa);
|
||||
}
|
||||
};
|
||||
|
||||
pub const RuleSet = struct {
|
||||
exact_allow: domain_set.DomainSet = .empty,
|
||||
exact_block: domain_set.DomainSet = .empty,
|
||||
@@ -32,75 +64,115 @@ pub const RuleSet = struct {
|
||||
/// order and the same first match.
|
||||
wildcard_allow: []const []const u8 = &.{},
|
||||
wildcard_block: []const []const u8 = &.{},
|
||||
/// One block holding the bytes of both wildcard lists; freed as a unit.
|
||||
wildcard_bytes: []const u8 = &.{},
|
||||
/// Sorted and deduplicated like the wildcards, so the first regex to match a
|
||||
/// name is the same one on every rebuild of the same rows.
|
||||
regex_allow: []const RegexRule = &.{},
|
||||
regex_block: []const RegexRule = &.{},
|
||||
/// One block holding the pattern bytes of all four lists; freed as a unit.
|
||||
pattern_bytes: []const u8 = &.{},
|
||||
|
||||
pub const empty: RuleSet = .{};
|
||||
|
||||
/// `rows` are one group's rules only; splitting `listRules` output by group
|
||||
/// belongs to the caller, which is the only holder of the group table.
|
||||
///
|
||||
/// Patterns are normalized (lowercase over ASCII, one trailing dot
|
||||
/// Name patterns are normalized (lowercase over ASCII, one trailing dot
|
||||
/// stripped) and validated: `.exact` through `dns.name.fromText`,
|
||||
/// `.wildcard` through `wildcard.validate`. An invalid pattern is
|
||||
/// `error.BadPattern`, not a skipped row — every pattern passed
|
||||
/// `config/validate.zig` on the way in, so an invalid one here means the
|
||||
/// rows were edited underneath nxdns and a silently dropped allow rule
|
||||
/// `.wildcard` through `wildcard.validate`, `.regex` by compiling it. An
|
||||
/// invalid pattern is `error.BadPattern`, not a skipped row — every pattern
|
||||
/// passed `config/validate.zig` on the way in, so an invalid one here means
|
||||
/// the rows were edited underneath nxdns and a silently dropped allow rule
|
||||
/// would block a domain the operator unblocked.
|
||||
pub fn build(gpa: Allocator, rows: []const model.Rule, seed: u64) Error!RuleSet {
|
||||
///
|
||||
/// Two allocators, because the caller's `perm` is a snapshot arena: an
|
||||
/// arena reclaims only its most recent allocation, so every temporary taken
|
||||
/// from it would live as long as the snapshot and go unreported by
|
||||
/// `memoryBytes`. `perm` owns what the returned set retains and is what
|
||||
/// `deinit` frees; `scratch` owns the build's working storage, which is
|
||||
/// released by the time `build` returns. Passing one allocator as both is
|
||||
/// correct wherever freeing works normally.
|
||||
pub fn build(
|
||||
perm: Allocator,
|
||||
scratch: Allocator,
|
||||
rows: []const model.Rule,
|
||||
seed: u64,
|
||||
) Error!RuleSet {
|
||||
if (rows.len == 0) return .empty;
|
||||
|
||||
var scratch: std.ArrayList(u8) = .empty;
|
||||
defer scratch.deinit(gpa);
|
||||
var spans: [4]std.ArrayList(Span) = .{ .empty, .empty, .empty, .empty };
|
||||
defer for (&spans) |*bucket| bucket.deinit(gpa);
|
||||
var joined: std.ArrayList(u8) = .empty;
|
||||
defer joined.deinit(scratch);
|
||||
var spans: [6]std.ArrayList(Span) = @splat(.empty);
|
||||
defer for (&spans) |*bucket| bucket.deinit(scratch);
|
||||
|
||||
var wildcards: usize = 0;
|
||||
var regexes: usize = 0;
|
||||
var buf: [types.max_name_len]u8 = undefined;
|
||||
for (rows) |row| {
|
||||
const pattern = normalize(row.pattern, &buf) catch return error.BadPattern;
|
||||
switch (row.kind) {
|
||||
.exact => _ = name.fromText(pattern) catch return error.BadPattern,
|
||||
.wildcard => {
|
||||
wildcard.validate(pattern) catch return error.BadPattern;
|
||||
const pattern = switch (row.kind) {
|
||||
.exact => blk: {
|
||||
const text = normalize(row.pattern, &buf) catch return error.BadPattern;
|
||||
_ = name.fromText(text) catch return error.BadPattern;
|
||||
break :blk text;
|
||||
},
|
||||
.wildcard => blk: {
|
||||
const text = normalize(row.pattern, &buf) catch return error.BadPattern;
|
||||
wildcard.validate(text) catch return error.BadPattern;
|
||||
wildcards += 1;
|
||||
if (wildcards > max_wildcards_per_group) return error.TooManyWildcards;
|
||||
break :blk text;
|
||||
},
|
||||
}
|
||||
// A regex is not a name, so `normalize` must not touch it: it
|
||||
// strips a trailing `.`, which here is the any-byte atom, and it
|
||||
// lowercases, which turns the rejected `\D` into the accepted
|
||||
// `\d`. Either would silently change what the rule matches. The
|
||||
// bytes stay as the operator wrote them — the same bytes
|
||||
// `config/validate.zig` compiled at the edge. Compiling waits
|
||||
// until after the sort, so a duplicate is compiled once.
|
||||
.regex => blk: {
|
||||
regexes += 1;
|
||||
if (regexes > max_regex_per_group) return error.TooManyRegexRules;
|
||||
break :blk row.pattern;
|
||||
},
|
||||
};
|
||||
const bucket = &spans[bucketOf(row.kind, row.action)];
|
||||
try bucket.append(gpa, .{ .offset = scratch.items.len, .len = pattern.len });
|
||||
try scratch.appendSlice(gpa, pattern);
|
||||
try bucket.append(scratch, .{ .offset = joined.items.len, .len = pattern.len });
|
||||
try joined.appendSlice(scratch, pattern);
|
||||
}
|
||||
|
||||
// `scratch` stops growing here, so spans can become slices of it.
|
||||
var sorted: [4]std.ArrayList([]const u8) = .{ .empty, .empty, .empty, .empty };
|
||||
defer for (&sorted) |*bucket| bucket.deinit(gpa);
|
||||
// `joined` stops growing here, so spans can become slices of it.
|
||||
var sorted: [6]std.ArrayList([]const u8) = @splat(.empty);
|
||||
defer for (&sorted) |*bucket| bucket.deinit(scratch);
|
||||
for (&spans, &sorted) |*bucket, *out| {
|
||||
try out.ensureTotalCapacityPrecise(gpa, bucket.items.len);
|
||||
try out.ensureTotalCapacityPrecise(scratch, bucket.items.len);
|
||||
for (bucket.items) |span| {
|
||||
out.appendAssumeCapacity(scratch.items[span.offset..][0..span.len]);
|
||||
out.appendAssumeCapacity(joined.items[span.offset..][0..span.len]);
|
||||
}
|
||||
std.mem.sort([]const u8, out.items, {}, lessThanBytes);
|
||||
dedupSorted(out);
|
||||
}
|
||||
|
||||
var self: RuleSet = .empty;
|
||||
errdefer self.deinit(gpa);
|
||||
errdefer self.deinit(perm);
|
||||
|
||||
self.exact_allow = try buildSet(gpa, sorted[bucketOf(.exact, .allow)].items, seed);
|
||||
self.exact_block = try buildSet(gpa, sorted[bucketOf(.exact, .block)].items, seed);
|
||||
self.exact_allow = try buildSet(perm, scratch, sorted[bucketOf(.exact, .allow)].items, seed);
|
||||
self.exact_block = try buildSet(perm, scratch, sorted[bucketOf(.exact, .block)].items, seed);
|
||||
|
||||
const allow = sorted[bucketOf(.wildcard, .allow)].items;
|
||||
const block = sorted[bucketOf(.wildcard, .block)].items;
|
||||
const wild_allow = sorted[bucketOf(.wildcard, .allow)].items;
|
||||
const wild_block = sorted[bucketOf(.wildcard, .block)].items;
|
||||
const re_allow = sorted[bucketOf(.regex, .allow)].items;
|
||||
const re_block = sorted[bucketOf(.regex, .block)].items;
|
||||
var total: usize = 0;
|
||||
for (allow) |pattern| total += pattern.len;
|
||||
for (block) |pattern| total += pattern.len;
|
||||
for ([_][]const []const u8{ wild_allow, wild_block, re_allow, re_block }) |list| {
|
||||
for (list) |pattern| total += pattern.len;
|
||||
}
|
||||
|
||||
const bytes = try gpa.alloc(u8, total);
|
||||
self.wildcard_bytes = bytes;
|
||||
const bytes = try perm.alloc(u8, total);
|
||||
self.pattern_bytes = bytes;
|
||||
var at: usize = 0;
|
||||
self.wildcard_allow = try copyPatterns(gpa, allow, bytes, &at);
|
||||
self.wildcard_block = try copyPatterns(gpa, block, bytes, &at);
|
||||
self.wildcard_allow = try copyPatterns(perm, wild_allow, bytes, &at);
|
||||
self.wildcard_block = try copyPatterns(perm, wild_block, bytes, &at);
|
||||
self.regex_allow = try compilePatterns(perm, scratch, re_allow, bytes, &at);
|
||||
self.regex_block = try compilePatterns(perm, scratch, re_block, bytes, &at);
|
||||
|
||||
return self;
|
||||
}
|
||||
@@ -110,18 +182,35 @@ pub const RuleSet = struct {
|
||||
self.exact_block.deinit(gpa);
|
||||
gpa.free(self.wildcard_allow);
|
||||
gpa.free(self.wildcard_block);
|
||||
gpa.free(self.wildcard_bytes);
|
||||
freeRules(gpa, self.regex_allow);
|
||||
freeRules(gpa, self.regex_block);
|
||||
gpa.free(self.pattern_bytes);
|
||||
self.* = .empty;
|
||||
}
|
||||
|
||||
pub fn memoryBytes(self: *const RuleSet) usize {
|
||||
var programs: usize = 0;
|
||||
for (self.regex_allow) |item| programs += item.program.memoryBytes();
|
||||
for (self.regex_block) |item| programs += item.program.memoryBytes();
|
||||
return self.exact_allow.memoryBytes() +
|
||||
self.exact_block.memoryBytes() +
|
||||
self.wildcard_bytes.len +
|
||||
(self.wildcard_allow.len + self.wildcard_block.len) * @sizeOf([]const u8);
|
||||
self.pattern_bytes.len +
|
||||
programs +
|
||||
(self.wildcard_allow.len + self.wildcard_block.len) * @sizeOf([]const u8) +
|
||||
(self.regex_allow.len + self.regex_block.len) * @sizeOf(RegexRule);
|
||||
}
|
||||
};
|
||||
|
||||
/// The first regex of `list` that matches `domain`, or null. `list` is sorted,
|
||||
/// so "first" is stable across rebuilds of the same rows. The caller checks the
|
||||
/// allow list before the block list, as it does for wildcards.
|
||||
pub fn matchRegex(list: []const RegexRule, domain: []const u8) ?[]const u8 {
|
||||
for (list) |item| {
|
||||
if (regex.matches(&item.program, domain)) return item.pattern;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internals
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -131,15 +220,16 @@ pub const RuleSet = struct {
|
||||
const Span = struct { offset: usize, len: usize };
|
||||
|
||||
fn bucketOf(kind: model.RuleKind, action: model.RuleAction) usize {
|
||||
const kind_bit: usize = switch (kind) {
|
||||
const kind_base: usize = switch (kind) {
|
||||
.exact => 0,
|
||||
.wildcard => 2,
|
||||
.regex => 4,
|
||||
};
|
||||
const action_bit: usize = switch (action) {
|
||||
const action_offset: usize = switch (action) {
|
||||
.allow => 0,
|
||||
.block => 1,
|
||||
};
|
||||
return kind_bit + action_bit;
|
||||
return kind_base + action_offset;
|
||||
}
|
||||
|
||||
fn lessThanBytes(_: void, a: []const u8, b: []const u8) bool {
|
||||
@@ -159,26 +249,31 @@ fn dedupSorted(list: *std.ArrayList([]const u8)) void {
|
||||
list.shrinkRetainingCapacity(kept);
|
||||
}
|
||||
|
||||
fn buildSet(gpa: Allocator, patterns: []const []const u8, seed: u64) Error!domain_set.DomainSet {
|
||||
fn buildSet(
|
||||
perm: Allocator,
|
||||
scratch: Allocator,
|
||||
patterns: []const []const u8,
|
||||
seed: u64,
|
||||
) Error!domain_set.DomainSet {
|
||||
if (patterns.len == 0) return .empty;
|
||||
|
||||
var body: std.ArrayList(u8) = .empty;
|
||||
defer body.deinit(gpa);
|
||||
defer body.deinit(scratch);
|
||||
for (patterns) |pattern| {
|
||||
try body.appendSlice(gpa, pattern);
|
||||
try body.append(gpa, '\n');
|
||||
try body.appendSlice(scratch, pattern);
|
||||
try body.append(scratch, '\n');
|
||||
}
|
||||
return domain_set.DomainSet.build(gpa, body.items, seed);
|
||||
return domain_set.DomainSet.build(perm, body.items, seed);
|
||||
}
|
||||
|
||||
fn copyPatterns(
|
||||
gpa: Allocator,
|
||||
perm: Allocator,
|
||||
patterns: []const []const u8,
|
||||
bytes: []u8,
|
||||
at: *usize,
|
||||
) Error![]const []const u8 {
|
||||
if (patterns.len == 0) return &.{};
|
||||
const out = try gpa.alloc([]const u8, patterns.len);
|
||||
const out = try perm.alloc([]const u8, patterns.len);
|
||||
for (out, patterns) |*slot, pattern| {
|
||||
@memcpy(bytes[at.*..][0..pattern.len], pattern);
|
||||
slot.* = bytes[at.*..][0..pattern.len];
|
||||
@@ -187,6 +282,49 @@ fn copyPatterns(
|
||||
return out;
|
||||
}
|
||||
|
||||
/// Copies the pattern texts into `bytes` like `copyPatterns` and compiles each
|
||||
/// one. A compile failure is `error.BadPattern` whichever of the engine's three
|
||||
/// refusals fired: the pattern already passed `config/validate.zig`, which names
|
||||
/// the limit, so a row that fails here was written around that check.
|
||||
///
|
||||
/// Compiling into `scratch` and cloning across is what keeps a parse-time AST
|
||||
/// out of `perm`: `regex.compile` builds the AST, the child lists and the
|
||||
/// growing instruction buffer through the allocator it returns the program on.
|
||||
fn compilePatterns(
|
||||
perm: Allocator,
|
||||
scratch: Allocator,
|
||||
patterns: []const []const u8,
|
||||
bytes: []u8,
|
||||
at: *usize,
|
||||
) Error![]const RegexRule {
|
||||
if (patterns.len == 0) return &.{};
|
||||
const out = try perm.alloc(RegexRule, patterns.len);
|
||||
var built: usize = 0;
|
||||
errdefer {
|
||||
for (out[0..built]) |item| item.free(perm);
|
||||
perm.free(out);
|
||||
}
|
||||
for (out, patterns) |*slot, pattern| {
|
||||
var compiled = regex.compile(scratch, pattern) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
else => return error.BadPattern,
|
||||
};
|
||||
defer compiled.deinit(scratch);
|
||||
const program = try compiled.clone(perm);
|
||||
|
||||
@memcpy(bytes[at.*..][0..pattern.len], pattern);
|
||||
slot.* = .{ .pattern = bytes[at.*..][0..pattern.len], .program = program };
|
||||
at.* += pattern.len;
|
||||
built += 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
fn freeRules(gpa: Allocator, list: []const RegexRule) void {
|
||||
for (list) |item| item.free(gpa);
|
||||
gpa.free(list);
|
||||
}
|
||||
|
||||
const NameError = error{BadName};
|
||||
|
||||
/// Lowercases over ASCII and strips one trailing dot. A byte ≥ 0x80 is
|
||||
@@ -223,7 +361,7 @@ test "exact rules land in the matching set" {
|
||||
rule("ads.example.com", .exact, .block),
|
||||
rule("good.example.com", .exact, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0x5eed);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0x5eed);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(set.exact_block.contains("ads.example.com"));
|
||||
@@ -239,7 +377,7 @@ test "wildcard rules land in the matching list, sorted" {
|
||||
rule("*.a.example.com", .wildcard, .block),
|
||||
rule("*.allowed.example.com", .wildcard, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0x5eed);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0x5eed);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(usize, 2), set.wildcard_block.len);
|
||||
@@ -254,7 +392,7 @@ test "patterns are normalized to lowercase without a trailing dot" {
|
||||
rule("ADS.Example.COM.", .exact, .block),
|
||||
rule("*.Tracker.NET.", .wildcard, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(set.exact_block.contains("ads.example.com"));
|
||||
@@ -268,7 +406,7 @@ test "duplicate rows collapse to one entry" {
|
||||
rule("*.x.example.com", .wildcard, .block),
|
||||
rule("*.x.example.com", .wildcard, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(u32, 1), set.exact_block.count);
|
||||
@@ -278,14 +416,98 @@ test "duplicate rows collapse to one entry" {
|
||||
test "an invalid exact pattern is an error" {
|
||||
for ([_][]const u8{ "", ".", "a..b", "ads example.com", "ads\u{00e9}.example.com" }) |pattern| {
|
||||
const rows = [_]model.Rule{rule(pattern, .exact, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0));
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &rows, 0));
|
||||
}
|
||||
}
|
||||
|
||||
test "regex rules land in their own buckets, compiled and sorted" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("^zz", .regex, .block),
|
||||
rule("^aa", .regex, .block),
|
||||
rule("ok$", .regex, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0x5eed);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(usize, 2), set.regex_block.len);
|
||||
try testing.expectEqualStrings("^aa", set.regex_block[0].pattern);
|
||||
try testing.expectEqualStrings("^zz", set.regex_block[1].pattern);
|
||||
try testing.expectEqual(@as(usize, 1), set.regex_allow.len);
|
||||
try testing.expectEqualStrings("ok$", set.regex_allow[0].pattern);
|
||||
|
||||
try testing.expectEqualStrings("^aa", matchRegex(set.regex_block, "aabb.example").?);
|
||||
try testing.expect(matchRegex(set.regex_block, "bbaa.example") == null);
|
||||
try testing.expectEqualStrings("ok$", matchRegex(set.regex_allow, "example.ok").?);
|
||||
}
|
||||
|
||||
test "a regex pattern keeps the bytes the operator wrote" {
|
||||
// `normalize` would strip the trailing dot and lowercase the escape, and
|
||||
// either edit would change what the pattern matches. The exact and wildcard
|
||||
// kinds still normalize; only this one is exempt.
|
||||
const rows = [_]model.Rule{
|
||||
rule("ADS\\.Example\\.", .regex, .block),
|
||||
rule("ADS.Example.", .exact, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqualStrings("ADS\\.Example\\.", set.regex_block[0].pattern);
|
||||
try testing.expect(set.exact_block.contains("ads.example"));
|
||||
}
|
||||
|
||||
test "duplicate regex rows collapse to one compiled program" {
|
||||
const rows = [_]model.Rule{
|
||||
rule("^ad[0-9]+-", .regex, .block),
|
||||
rule("^ad[0-9]+-", .regex, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expectEqual(@as(usize, 1), set.regex_block.len);
|
||||
}
|
||||
|
||||
test "a regex pattern the engine refuses is an error, not a skipped row" {
|
||||
for ([_][]const u8{ "(", "", "a+?", "[z-a]", "\\s" }) |pattern| {
|
||||
const rows = [_]model.Rule{rule(pattern, .regex, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &rows, 0));
|
||||
}
|
||||
|
||||
// The size limits arrive as `BadPattern` too: which one fired is
|
||||
// `config/validate.zig`'s to report, and by here the row is simply wrong.
|
||||
const long = [_]model.Rule{rule("a" ** 300, .regex, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &long, 0));
|
||||
const complex = [_]model.Rule{rule("(abcdefghij){200}", .regex, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &complex, 0));
|
||||
}
|
||||
|
||||
test "too many regex rules is an error" {
|
||||
const gpa = testing.allocator;
|
||||
const rows = try gpa.alloc(model.Rule, max_regex_per_group + 1);
|
||||
defer gpa.free(rows);
|
||||
|
||||
var patterns: std.ArrayList([]u8) = .empty;
|
||||
defer {
|
||||
for (patterns.items) |p| gpa.free(p);
|
||||
patterns.deinit(gpa);
|
||||
}
|
||||
for (rows, 0..) |*row, i| {
|
||||
const pattern = try std.fmt.allocPrint(gpa, "^n{d}-", .{i});
|
||||
try patterns.append(gpa, pattern);
|
||||
row.* = rule(pattern, .regex, .block);
|
||||
}
|
||||
|
||||
try testing.expectError(error.TooManyRegexRules, RuleSet.build(gpa, gpa, rows, 0));
|
||||
|
||||
// The cap counts both actions together, like the wildcard one.
|
||||
rows[0].action = .allow;
|
||||
try testing.expectError(error.TooManyRegexRules, RuleSet.build(gpa, gpa, rows, 0));
|
||||
try testing.expectEqual(@as(usize, 256), max_regex_per_group);
|
||||
}
|
||||
|
||||
test "an invalid wildcard pattern is an error" {
|
||||
for ([_][]const u8{ "example.com", "ad*.example.com", "*..com" }) |pattern| {
|
||||
const rows = [_]model.Rule{rule(pattern, .wildcard, .block)};
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, &rows, 0));
|
||||
try testing.expectError(error.BadPattern, RuleSet.build(testing.allocator, testing.allocator, &rows, 0));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -305,11 +527,11 @@ test "too many wildcards is an error" {
|
||||
row.* = rule(pattern, .wildcard, .block);
|
||||
}
|
||||
|
||||
try testing.expectError(error.TooManyWildcards, RuleSet.build(gpa, rows, 0));
|
||||
try testing.expectError(error.TooManyWildcards, RuleSet.build(gpa, gpa, rows, 0));
|
||||
}
|
||||
|
||||
test "an empty rule list builds the empty set" {
|
||||
var set = try RuleSet.build(testing.allocator, &[_]model.Rule{}, 0);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &[_]model.Rule{}, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(!set.exact_block.contains("ads.example.com"));
|
||||
@@ -328,11 +550,63 @@ test "memoryBytes counts every part" {
|
||||
rule("ads.example.com", .exact, .block),
|
||||
rule("*.tracker.net", .wildcard, .block),
|
||||
};
|
||||
var set = try RuleSet.build(testing.allocator, &rows, 0);
|
||||
var set = try RuleSet.build(testing.allocator, testing.allocator, &rows, 0);
|
||||
defer set.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(set.memoryBytes() > set.exact_block.memoryBytes());
|
||||
try testing.expect(set.memoryBytes() >= "*.tracker.net".len);
|
||||
|
||||
// A compiled program is the largest thing a rule set holds, so leaving it
|
||||
// out would make the snapshot's memory report a fiction.
|
||||
const with_regex = [_]model.Rule{ rows[0], rows[1], rule("^ad[0-9]+-", .regex, .block) };
|
||||
var wider = try RuleSet.build(testing.allocator, testing.allocator, &with_regex, 0);
|
||||
defer wider.deinit(testing.allocator);
|
||||
|
||||
try testing.expect(wider.memoryBytes() > set.memoryBytes() + "^ad[0-9]+-".len);
|
||||
try testing.expect(wider.memoryBytes() >= wider.regex_block[0].program.memoryBytes());
|
||||
}
|
||||
|
||||
test "the build's temporaries stay out of the permanent allocator" {
|
||||
// The property the two-allocator split exists for. An arena reclaims only
|
||||
// its most recent allocation, so a temporary taken from `perm` would live
|
||||
// as long as the arena and be invisible to `memoryBytes`. Two checks, one
|
||||
// per direction: `testing.allocator` fails the test if anything the set
|
||||
// retains was taken from `scratch`, and the arena's capacity fails it if
|
||||
// the build's working storage was taken from `perm`.
|
||||
const gpa = testing.allocator;
|
||||
var patterns: std.ArrayList([]u8) = .empty;
|
||||
defer {
|
||||
for (patterns.items) |p| gpa.free(p);
|
||||
patterns.deinit(gpa);
|
||||
}
|
||||
var rows: std.ArrayList(model.Rule) = .empty;
|
||||
defer rows.deinit(gpa);
|
||||
|
||||
var i: usize = 0;
|
||||
while (i < 64) : (i += 1) {
|
||||
const regex_pattern = try std.fmt.allocPrint(gpa, "^r{d}-[0-9]+\\.ads\\.invalid$", .{i});
|
||||
try patterns.append(gpa, regex_pattern);
|
||||
try rows.append(gpa, rule(regex_pattern, .regex, .block));
|
||||
|
||||
const wild = try std.fmt.allocPrint(gpa, "*.w{d:0>5}.example.com", .{i});
|
||||
try patterns.append(gpa, wild);
|
||||
try rows.append(gpa, rule(wild, .wildcard, .block));
|
||||
|
||||
const exact = try std.fmt.allocPrint(gpa, "e{d:0>5}.example.com", .{i});
|
||||
try patterns.append(gpa, exact);
|
||||
try rows.append(gpa, rule(exact, .exact, .block));
|
||||
}
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(gpa);
|
||||
defer arena.deinit();
|
||||
const set = try RuleSet.build(arena.allocator(), gpa, rows.items, 0x5eed);
|
||||
|
||||
try testing.expectEqual(@as(usize, 64), set.regex_block.len);
|
||||
try testing.expect(set.exact_block.contains("e00007.example.com"));
|
||||
// Whole-arena capacity against what the set says it holds. The slack is the
|
||||
// allocator's page rounding; the defect this guards against was a factor of
|
||||
// twelve.
|
||||
try testing.expect(arena.queryCapacity() < 2 * set.memoryBytes());
|
||||
}
|
||||
|
||||
fn buildUnderFailure(gpa: Allocator) !void {
|
||||
@@ -341,11 +615,14 @@ fn buildUnderFailure(gpa: Allocator) !void {
|
||||
rule("good.example.com", .exact, .allow),
|
||||
rule("*.tracker.net", .wildcard, .block),
|
||||
rule("*.ok.tracker.net", .wildcard, .allow),
|
||||
rule("^ad[0-9]+-", .regex, .block),
|
||||
rule("\\.ok\\.", .regex, .allow),
|
||||
};
|
||||
var set = try RuleSet.build(gpa, &rows, 0x5eed);
|
||||
var set = try RuleSet.build(gpa, gpa, &rows, 0x5eed);
|
||||
defer set.deinit(gpa);
|
||||
try testing.expect(set.exact_block.contains("ads.example.com"));
|
||||
try testing.expectEqualStrings("*.tracker.net", set.wildcard_block[0]);
|
||||
try testing.expectEqualStrings("^ad[0-9]+-", set.regex_block[0].pattern);
|
||||
}
|
||||
|
||||
test "build leaks nothing under allocation failure" {
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
//! A pattern is a domain name in which one or more labels are exactly `*`.
|
||||
//! Each `*` label matches one or more labels of the queried name. Partial-label
|
||||
//! globbing (`ad*.example.com`) is deliberately absent: it is regex by another
|
||||
//! name, which PLAN §2.2 rules out.
|
||||
//! name, and PLAN §2.2 keeps one regex dialect rather than two. An operator who
|
||||
//! needs one writes a `.regex` rule, which `filter/regex.zig` compiles.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
@@ -18,8 +19,9 @@ pub const PatternError = error{
|
||||
/// No label is exactly "*".
|
||||
NoWildcard,
|
||||
/// A label contains '*' but is not exactly "*". Partial-label globbing
|
||||
/// (`ad*.example.com`) is out of scope: it is regex by another name, and
|
||||
/// PLAN §3.9 defines the wildcard as a label pattern.
|
||||
/// (`ad*.example.com`) is out of scope: PLAN §3.9 defines the wildcard as a
|
||||
/// label pattern, and the `.regex` kind covers what partial globbing was
|
||||
/// wanted for.
|
||||
PartialWildcardLabel,
|
||||
EmptyLabel,
|
||||
LabelTooLong,
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
//! Reverse DNS names and the hostname gate for learned client names
|
||||
//! (milestone 25). Pure: bytes in, bytes out — no `std.Io`, no clock, no
|
||||
//! sockets. Everything that queries a resolver or writes a row lives in
|
||||
//! `src/server/`.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const address = @import("../platform/address.zig");
|
||||
|
||||
const v4_suffix = "in-addr.arpa";
|
||||
const v6_suffix = "ip6.arpa";
|
||||
|
||||
/// The v6 form is the longest: 32 nibbles, each followed by a dot, then
|
||||
/// `ip6.arpa`.
|
||||
pub const max_reverse_len: usize = 32 * 2 + v6_suffix.len;
|
||||
|
||||
/// A PTR owner name is one label per byte (v4) or per nibble (v6), least
|
||||
/// significant first, under `in-addr.arpa` / `ip6.arpa`. Lowercase hex for v6,
|
||||
/// no trailing dot — the form `forward_zones.Zones.match` expects.
|
||||
///
|
||||
/// The tracker canonicalises an IPv4-mapped v6 address to `.ip4` before a
|
||||
/// `NetAddress` reaches here, so this function never sees one.
|
||||
pub fn reverseName(addr: address.NetAddress, buf: *[max_reverse_len]u8) []const u8 {
|
||||
var w = std.Io.Writer.fixed(buf);
|
||||
switch (addr) {
|
||||
.ip4 => |b| {
|
||||
w.print("{d}.{d}.{d}.{d}.{s}", .{ b[3], b[2], b[1], b[0], v4_suffix }) catch unreachable;
|
||||
},
|
||||
.ip6 => |b| {
|
||||
std.debug.assert(!isIp4Mapped(b));
|
||||
var i: usize = b.len;
|
||||
while (i > 0) {
|
||||
i -= 1;
|
||||
const byte = b[i];
|
||||
w.writeByte(hex_digits[byte & 0x0f]) catch unreachable;
|
||||
w.writeByte('.') catch unreachable;
|
||||
w.writeByte(hex_digits[byte >> 4]) catch unreachable;
|
||||
w.writeByte('.') catch unreachable;
|
||||
}
|
||||
w.writeAll(v6_suffix) catch unreachable;
|
||||
},
|
||||
}
|
||||
return w.buffered();
|
||||
}
|
||||
|
||||
const hex_digits = "0123456789abcdef";
|
||||
|
||||
fn isIp4Mapped(b: [16]u8) bool {
|
||||
return std.mem.eql(u8, b[0..10], &[_]u8{0} ** 10) and b[10] == 0xff and b[11] == 0xff;
|
||||
}
|
||||
|
||||
/// The gate every PTR target passes before it is stored, logged or displayed.
|
||||
/// The bytes come from whatever box the operator pointed a forward zone at, so
|
||||
/// nothing weaker is enough.
|
||||
///
|
||||
/// Accepts only `[a-z0-9._-]` after ASCII-lowercasing `A-Z`; labels are 1–63
|
||||
/// bytes and the whole name is at most 253; no label starts or ends with `-`.
|
||||
/// An empty label is rejected, which also rejects a trailing dot — `formatText`
|
||||
/// emits none, so one appearing means the reply was malformed.
|
||||
///
|
||||
/// Underscore is accepted because real DHCP hostnames carry it. Nothing else
|
||||
/// outside the set is.
|
||||
pub fn acceptHostname(text: []const u8) bool {
|
||||
if (text.len == 0 or text.len > 253) return false;
|
||||
|
||||
var label_len: usize = 0;
|
||||
var prev: u8 = 0;
|
||||
for (text) |raw| {
|
||||
const ch = std.ascii.toLower(raw);
|
||||
if (ch == '.') {
|
||||
if (label_len == 0) return false;
|
||||
if (prev == '-') return false;
|
||||
label_len = 0;
|
||||
prev = ch;
|
||||
continue;
|
||||
}
|
||||
if (label_len == 0 and ch == '-') return false;
|
||||
if (!isHostByte(ch)) return false;
|
||||
label_len += 1;
|
||||
if (label_len > 63) return false;
|
||||
prev = ch;
|
||||
}
|
||||
if (label_len == 0) return false;
|
||||
if (prev == '-') return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
fn isHostByte(ch: u8) bool {
|
||||
return (ch >= 'a' and ch <= 'z') or (ch >= '0' and ch <= '9') or ch == '_' or ch == '-';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "reverseName reverses the octets of a v4 address" {
|
||||
var buf: [max_reverse_len]u8 = undefined;
|
||||
try testing.expectEqualStrings(
|
||||
"10.1.168.192.in-addr.arpa",
|
||||
reverseName(try address.NetAddress.parse("192.168.1.10"), &buf),
|
||||
);
|
||||
try testing.expectEqualStrings(
|
||||
"0.0.0.0.in-addr.arpa",
|
||||
reverseName(try address.NetAddress.parse("0.0.0.0"), &buf),
|
||||
);
|
||||
try testing.expectEqualStrings(
|
||||
"255.255.255.255.in-addr.arpa",
|
||||
reverseName(try address.NetAddress.parse("255.255.255.255"), &buf),
|
||||
);
|
||||
}
|
||||
|
||||
test "reverseName writes the 32-nibble lowercase ip6.arpa form" {
|
||||
var buf: [max_reverse_len]u8 = undefined;
|
||||
try testing.expectEqualStrings(
|
||||
"1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.d.f.ip6.arpa",
|
||||
reverseName(try address.NetAddress.parse("fd00::1"), &buf),
|
||||
);
|
||||
// Every nibble distinct, so a swapped high/low half would show.
|
||||
try testing.expectEqualStrings(
|
||||
"b.a.9.8.7.6.5.4.3.2.1.0.f.e.d.c.b.a.9.8.7.6.5.4.3.2.1.0.f.e.d.c.ip6.arpa",
|
||||
reverseName(try address.NetAddress.parse("cdef:0123:4567:89ab:cdef:0123:4567:89ab"), &buf),
|
||||
);
|
||||
}
|
||||
|
||||
test "the v6 form is exactly max_reverse_len bytes and the v4 form is shorter" {
|
||||
var buf: [max_reverse_len]u8 = undefined;
|
||||
const v6 = reverseName(try address.NetAddress.parse("cdef:0123:4567:89ab:cdef:0123:4567:89ab"), &buf);
|
||||
try testing.expectEqual(max_reverse_len, v6.len);
|
||||
const v4 = reverseName(try address.NetAddress.parse("255.255.255.255"), &buf);
|
||||
try testing.expect(v4.len < max_reverse_len);
|
||||
}
|
||||
|
||||
test "an IPv4-mapped v6 literal is already canonical, so reverseName sees v4" {
|
||||
var buf: [max_reverse_len]u8 = undefined;
|
||||
try testing.expectEqualStrings(
|
||||
"10.1.168.192.in-addr.arpa",
|
||||
reverseName(try address.NetAddress.parse("::ffff:192.168.1.10"), &buf),
|
||||
);
|
||||
}
|
||||
|
||||
test "acceptHostname accepts and rejects ruling 4's table" {
|
||||
// 254 bytes, every label within 63: only the total length rejects it.
|
||||
const long_name = "a" ** 63 ++ "." ++ "a" ** 63 ++ "." ++ "a" ** 63 ++ "." ++ "a" ** 62;
|
||||
try testing.expectEqual(@as(usize, 254), long_name.len);
|
||||
const long_label = "a" ** 64;
|
||||
|
||||
const cases = [_]struct { text: []const u8, want: bool }{
|
||||
.{ .text = "", .want = false },
|
||||
.{ .text = long_name, .want = false },
|
||||
.{ .text = long_label, .want = false },
|
||||
.{ .text = "a b", .want = false },
|
||||
.{ .text = "a\x00b", .want = false },
|
||||
.{ .text = "héllo", .want = false },
|
||||
.{ .text = "-x", .want = false },
|
||||
.{ .text = "x-.y", .want = false },
|
||||
.{ .text = "a..b", .want = false },
|
||||
.{ .text = ".a", .want = false },
|
||||
.{ .text = "a.", .want = false },
|
||||
.{ .text = "nas-1.lan", .want = true },
|
||||
.{ .text = "my_printer.home", .want = true },
|
||||
.{ .text = "x", .want = true },
|
||||
};
|
||||
for (cases) |case| {
|
||||
testing.expectEqual(case.want, acceptHostname(case.text)) catch |err| {
|
||||
std.debug.print("acceptHostname(\"{s}\")\n", .{case.text});
|
||||
return err;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
test "a reverse name matches the forward zone declared over its reverse space" {
|
||||
const forward_zones = @import("forward_zones.zig");
|
||||
|
||||
var zones = try forward_zones.Zones.build(testing.allocator, &.{
|
||||
.{ .zone = "168.192.in-addr.arpa", .resolver = "udp://192.168.1.1:53" },
|
||||
.{ .zone = "0.0.d.f.ip6.arpa", .resolver = "udp://[fd00::1]:53" },
|
||||
});
|
||||
defer zones.deinit(testing.allocator);
|
||||
|
||||
var buf: [max_reverse_len]u8 = undefined;
|
||||
const v4 = reverseName(try address.NetAddress.parse("192.168.1.10"), &buf);
|
||||
try testing.expectEqualStrings("168.192.in-addr.arpa", zones.match(v4).?.zone);
|
||||
|
||||
var buf6: [max_reverse_len]u8 = undefined;
|
||||
const v6 = reverseName(try address.NetAddress.parse("fd00::1"), &buf6);
|
||||
try testing.expectEqualStrings("0.0.d.f.ip6.arpa", zones.match(v6).?.zone);
|
||||
|
||||
// An address outside both declared reverse zones matches nothing, which is
|
||||
// the `no_zone` outcome: no query is sent to anyone.
|
||||
const outside = reverseName(try address.NetAddress.parse("10.0.0.1"), &buf);
|
||||
try testing.expectEqual(@as(?*const forward_zones.Zone, null), zones.match(outside));
|
||||
}
|
||||
|
||||
test "acceptHostname takes the boundary lengths and mixed case" {
|
||||
try testing.expect(acceptHostname("a" ** 63));
|
||||
// 253 bytes, the longest name accepted.
|
||||
try testing.expect(acceptHostname("a" ** 63 ++ "." ++ "a" ** 63 ++ "." ++ "a" ** 63 ++ "." ++ "a" ** 61));
|
||||
try testing.expect(acceptHostname("NAS-1.LAN"));
|
||||
try testing.expect(!acceptHostname("x-"));
|
||||
try testing.expect(!acceptHostname("a.-b"));
|
||||
try testing.expect(!acceptHostname("a.b-.c"));
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+167
-26
@@ -21,6 +21,7 @@
|
||||
const std = @import("std");
|
||||
|
||||
const address = @import("../platform/address.zig");
|
||||
const client_names = @import("client_names.zig");
|
||||
const clients_repo = @import("../storage/repositories/clients_repo.zig");
|
||||
const db = @import("../storage/db.zig");
|
||||
const disk_monitor = @import("../storage/disk_monitor.zig");
|
||||
@@ -135,6 +136,7 @@ pub const Tracker = struct {
|
||||
io: std.Io,
|
||||
database: *db.Db,
|
||||
monitor: ?*disk_monitor.Monitor,
|
||||
names: ?*client_names.Resolver,
|
||||
) std.Io.Cancelable!void {
|
||||
const interval: std.Io.Clock.Duration = .{
|
||||
.raw = .fromSeconds(flush_interval_s),
|
||||
@@ -143,12 +145,18 @@ pub const Tracker = struct {
|
||||
while (true) {
|
||||
try interval.sleep(io);
|
||||
const writes_allowed = if (monitor) |m| m.writesAllowed() else true;
|
||||
self.flushOnce(io, database, writes_allowed);
|
||||
self.flushOnce(io, database, writes_allowed, names);
|
||||
}
|
||||
}
|
||||
|
||||
/// One pass: drain the table, write a row per client, and prune on every
|
||||
/// `prune_every_passes`-th pass.
|
||||
/// One pass: drain the table, write a row per client, prune on every
|
||||
/// `prune_every_passes`-th pass, and then learn names for the rows that
|
||||
/// have none (milestone-25 ruling 1).
|
||||
///
|
||||
/// The order of the three steps is fixed and one `now_s` serves all three.
|
||||
/// Resolving before pruning would spend PTR queries on rows the same pass
|
||||
/// deletes; resolving before the drain would make a slow resolver delay the
|
||||
/// writes the pass exists for.
|
||||
///
|
||||
/// A gated pass does nothing at all, not even count: the work it skipped is
|
||||
/// still owed, and the pending addresses it leaves behind are re-tracked by
|
||||
@@ -161,9 +169,17 @@ pub const Tracker = struct {
|
||||
/// Only `run` may call this concurrently with itself — the drain buffer is
|
||||
/// this call's stack, but the pass counter and the prune schedule assume a
|
||||
/// single caller.
|
||||
pub fn flushOnce(self: *Tracker, io: std.Io, database: *db.Db, writes_allowed: bool) void {
|
||||
pub fn flushOnce(
|
||||
self: *Tracker,
|
||||
io: std.Io,
|
||||
database: *db.Db,
|
||||
writes_allowed: bool,
|
||||
names: ?*client_names.Resolver,
|
||||
) void {
|
||||
// A gated pass skips naming too: naming writes.
|
||||
if (!writes_allowed) return;
|
||||
|
||||
const now_s = std.Io.Clock.real.now(io).toSeconds();
|
||||
var drained: [max_pending]Pending = undefined;
|
||||
const batch = self.drain(io, &drained);
|
||||
|
||||
@@ -193,18 +209,21 @@ pub const Tracker = struct {
|
||||
const due = self.passes % prune_every_passes == 0;
|
||||
self.mutex.unlock(io);
|
||||
|
||||
if (!due) return;
|
||||
const cutoff = std.Io.Clock.real.now(io).toSeconds() - @as(i64, self.retention_days) * 86_400;
|
||||
if (clients_repo.pruneStale(database, cutoff)) |deleted| {
|
||||
self.mutex.lockUncancelable(io);
|
||||
self.stats.pruned += deleted;
|
||||
self.mutex.unlock(io);
|
||||
} else |err| {
|
||||
log.warn("pruning clients before {d} failed: {s}", .{ cutoff, @errorName(err) });
|
||||
self.mutex.lockUncancelable(io);
|
||||
self.stats.flush_failures += 1;
|
||||
self.mutex.unlock(io);
|
||||
if (due) {
|
||||
const cutoff = now_s - @as(i64, self.retention_days) * 86_400;
|
||||
if (clients_repo.pruneStale(database, cutoff)) |deleted| {
|
||||
self.mutex.lockUncancelable(io);
|
||||
self.stats.pruned += deleted;
|
||||
self.mutex.unlock(io);
|
||||
} else |err| {
|
||||
log.warn("pruning clients before {d} failed: {s}", .{ cutoff, @errorName(err) });
|
||||
self.mutex.lockUncancelable(io);
|
||||
self.stats.flush_failures += 1;
|
||||
self.mutex.unlock(io);
|
||||
}
|
||||
}
|
||||
|
||||
if (names) |resolver| resolver.runPass(io, database, now_s);
|
||||
}
|
||||
|
||||
pub fn snapshotStats(self: *Tracker, io: std.Io) Stats {
|
||||
@@ -277,7 +296,7 @@ test "a client tracked twice before a flush yields one row at the later time" {
|
||||
try testing.expectEqual(@as(u64, 0), tracker.trackAt(io, parsed("192.168.1.10"), 1700000030));
|
||||
try testing.expectEqual(@as(u32, 1), tracker.pendingClients(io));
|
||||
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(i64, 1700000030), try lastSeen(&database, "192.168.1.10"));
|
||||
@@ -305,7 +324,7 @@ test "distinct clients each get a row and ipv6 text is canonical" {
|
||||
_ = tracker.trackAt(io, address.NetAddress.fromIp(try std.Io.net.IpAddress.parse("::ffff:192.168.1.10", 53)), 1700000003);
|
||||
try testing.expectEqual(@as(u32, 3), tracker.pendingClients(io));
|
||||
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 3), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(i64, 1700000003), try lastSeen(&database, "192.168.1.10"));
|
||||
@@ -343,7 +362,7 @@ test "a full table drops further clients and counts them" {
|
||||
// A tracked client still refreshes while the table is full, and the flush
|
||||
// makes room for the next newcomer.
|
||||
_ = tracker.trackAt(io, .{ .ip4 = .{ 0, 0, 0, 0 } }, 1700000060);
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
try testing.expectEqual(@as(i64, Tracker.max_pending), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(i64, 1700000060), try lastSeen(&database, "0.0.0.0"));
|
||||
|
||||
@@ -366,7 +385,7 @@ test "a flush touches a hand-edited row without changing what the operator set"
|
||||
|
||||
var tracker: Tracker = .init(30);
|
||||
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000000);
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(i64, 1700000000), try lastSeen(&database, "192.168.1.10"));
|
||||
@@ -394,7 +413,7 @@ test "a gated pass writes nothing and keeps the pending clients" {
|
||||
|
||||
var tracker: Tracker = .init(30);
|
||||
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000000);
|
||||
tracker.flushOnce(io, &database, monitor.writesAllowed());
|
||||
tracker.flushOnce(io, &database, monitor.writesAllowed(), null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 0), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(u32, 1), tracker.pendingClients(io));
|
||||
@@ -403,7 +422,7 @@ test "a gated pass writes nothing and keeps the pending clients" {
|
||||
|
||||
// Free space recovers and the same pending client lands.
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.warn), .monotonic);
|
||||
tracker.flushOnce(io, &database, monitor.writesAllowed());
|
||||
tracker.flushOnce(io, &database, monitor.writesAllowed(), null);
|
||||
try testing.expectEqual(@as(i64, 1), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(u64, 1), tracker.passes);
|
||||
}
|
||||
@@ -423,7 +442,7 @@ test "a failing upsert counts and leaves the client to be tracked again" {
|
||||
var tracker: Tracker = .init(30);
|
||||
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000000);
|
||||
_ = tracker.trackAt(io, parsed("192.168.1.11"), 1700000000);
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 0), try clients_repo.countClients(&database));
|
||||
const stats = tracker.snapshotStats(io);
|
||||
@@ -433,7 +452,7 @@ test "a failing upsert counts and leaves the client to be tracked again" {
|
||||
|
||||
try database.exec("DROP TRIGGER refuse_insert;");
|
||||
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000060);
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
try testing.expectEqual(@as(i64, 1), try clients_repo.countClients(&database));
|
||||
}
|
||||
|
||||
@@ -453,12 +472,12 @@ test "the pass that comes due prunes the clients that went quiet" {
|
||||
var tracker: Tracker = .init(30);
|
||||
// Every pass before the due one leaves both rows alone.
|
||||
for (0..Tracker.prune_every_passes - 1) |_| {
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
}
|
||||
try testing.expectEqual(@as(i64, 2), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(u64, 0), tracker.snapshotStats(io).pruned);
|
||||
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(i64, now - 29 * day), try lastSeen(&database, "10.0.0.2"));
|
||||
@@ -479,7 +498,7 @@ test "a shorter retention prunes what the default keeps" {
|
||||
|
||||
var tracker: Tracker = .init(1);
|
||||
tracker.passes = Tracker.prune_every_passes - 1;
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 0), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(u64, 1), tracker.snapshotStats(io).pruned);
|
||||
@@ -501,6 +520,7 @@ test "the run loop flushes on its interval and returns on cancel" {
|
||||
io,
|
||||
&database,
|
||||
@as(?*disk_monitor.Monitor, null),
|
||||
@as(?*client_names.Resolver, null),
|
||||
});
|
||||
// The first flush is one interval away, so cancelling immediately proves the
|
||||
// loop starts by sleeping rather than by writing.
|
||||
@@ -508,3 +528,124 @@ test "the run loop flushes on its interval and returns on cancel" {
|
||||
try testing.expectEqual(@as(i64, 0), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(u32, 1), tracker.pendingClients(io));
|
||||
}
|
||||
|
||||
// --- the naming step's place in the pass (milestone-25 ruling 1) ------------
|
||||
|
||||
const forward_zones = @import("../local/forward_zones.zig");
|
||||
const local_tables = @import("local_tables.zig");
|
||||
const validate = @import("../config/validate.zig");
|
||||
|
||||
/// Counts exchanges and times out on every one, and records how many client
|
||||
/// rows existed when the first exchange was attempted.
|
||||
const CountingExchange = struct {
|
||||
var calls: usize = 0;
|
||||
var database: ?*db.Db = null;
|
||||
var rows_at_first_call: i64 = -1;
|
||||
|
||||
fn reset(target: *db.Db) void {
|
||||
calls = 0;
|
||||
database = target;
|
||||
rows_at_first_call = -1;
|
||||
}
|
||||
|
||||
fn exchange(
|
||||
_: std.Io,
|
||||
_: validate.Resolver,
|
||||
_: []const u8,
|
||||
_: []u8,
|
||||
) @import("../upstream/transport.zig").ExchangeError![]u8 {
|
||||
if (calls == 0) {
|
||||
rows_at_first_call = clients_repo.countClients(database.?) catch -1;
|
||||
}
|
||||
calls += 1;
|
||||
return error.Timeout;
|
||||
}
|
||||
};
|
||||
|
||||
fn namingTables(io: std.Io, tables: *local_tables.LocalTables) !void {
|
||||
tables.swap(io, testing.allocator, .empty, try forward_zones.Zones.build(
|
||||
testing.allocator,
|
||||
&.{.{ .zone = "168.192.in-addr.arpa", .resolver = "udp://192.168.1.1:53" }},
|
||||
));
|
||||
}
|
||||
|
||||
test "the drain lands before any exchange, and attempts stop at the cap" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
var tables: local_tables.LocalTables = .empty;
|
||||
defer tables.deinit(testing.allocator);
|
||||
try namingTables(io, &tables);
|
||||
|
||||
var names: client_names.Resolver = .init(&tables);
|
||||
names.exchange_fn = CountingExchange.exchange;
|
||||
CountingExchange.reset(&database);
|
||||
|
||||
var tracker: Tracker = .init(30);
|
||||
const pending = clients_repo.max_per_pass + 4;
|
||||
for (0..pending) |i| {
|
||||
_ = tracker.trackAt(io, .{ .ip4 = .{ 192, 168, 2, @intCast(i) } }, 1700000000);
|
||||
}
|
||||
|
||||
tracker.flushOnce(io, &database, true, &names);
|
||||
|
||||
// Every pending row was written before the first exchange went out, which
|
||||
// is what keeps a slow resolver off the drain.
|
||||
try testing.expectEqual(@as(i64, @intCast(pending)), CountingExchange.rows_at_first_call);
|
||||
try testing.expectEqual(clients_repo.max_per_pass, CountingExchange.calls);
|
||||
try testing.expectEqual(@as(u64, @intCast(pending)), tracker.snapshotStats(io).flushed);
|
||||
try testing.expectEqual(@as(u64, clients_repo.max_per_pass), names.snapshotStats(io).failed);
|
||||
}
|
||||
|
||||
test "a row the due pass prunes is never asked about" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
var tables: local_tables.LocalTables = .empty;
|
||||
defer tables.deinit(testing.allocator);
|
||||
try namingTables(io, &tables);
|
||||
|
||||
var names: client_names.Resolver = .init(&tables);
|
||||
names.exchange_fn = CountingExchange.exchange;
|
||||
CountingExchange.reset(&database);
|
||||
|
||||
const now = std.Io.Clock.real.now(io).toSeconds();
|
||||
try clients_repo.upsertSeen(&database, "192.168.1.10", now - 40 * 86_400);
|
||||
|
||||
var tracker: Tracker = .init(30);
|
||||
tracker.passes = Tracker.prune_every_passes - 1;
|
||||
tracker.flushOnce(io, &database, true, &names);
|
||||
|
||||
try testing.expectEqual(@as(i64, 0), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(usize, 0), CountingExchange.calls);
|
||||
try testing.expectEqual(@as(u64, 0), names.snapshotStats(io).attempted);
|
||||
}
|
||||
|
||||
test "a gated pass attempts no naming either" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
var tables: local_tables.LocalTables = .empty;
|
||||
defer tables.deinit(testing.allocator);
|
||||
try namingTables(io, &tables);
|
||||
|
||||
var names: client_names.Resolver = .init(&tables);
|
||||
names.exchange_fn = CountingExchange.exchange;
|
||||
CountingExchange.reset(&database);
|
||||
|
||||
try clients_repo.upsertSeen(&database, "192.168.1.10", 1700000000);
|
||||
var tracker: Tracker = .init(30);
|
||||
tracker.flushOnce(io, &database, false, &names);
|
||||
|
||||
try testing.expectEqual(@as(usize, 0), CountingExchange.calls);
|
||||
try testing.expectEqual(@as(u64, 0), names.snapshotStats(io).attempted);
|
||||
}
|
||||
|
||||
@@ -844,7 +844,7 @@ test "S7 case 10: the querying client is materialised as a row" {
|
||||
// Two queries from one client are one pending entry, and the forced pass
|
||||
// stands in for the 60-second flush interval (S4 As-built seam).
|
||||
try testing.expectEqual(@as(u32, 1), tracker.pendingClients(io));
|
||||
tracker.flushOnce(io, &database, true);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try clients_repo.countClients(&database));
|
||||
try testing.expectEqual(@as(u32, 0), tracker.pendingClients(io));
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
//! The `config.db` schema, verbatim from PLAN §11.2, plus the table lists every
|
||||
//! other storage session needs.
|
||||
//!
|
||||
//! The DDL text is data, not code: `migrations.zig` carries it as step 1 and
|
||||
//! never edits it in place. A schema change is a *new* step with new DDL, so
|
||||
//! this string stays byte-identical to PLAN §11.2 forever.
|
||||
//! The DDL text is data, not code: `migrations.zig` carries it as step 1.
|
||||
//!
|
||||
//! Until nxdns reaches v0.1 this baseline is **editable**. nxdns has no
|
||||
//! installs, so a schema change edits this string and PLAN §11.2 together — it
|
||||
//! does not append a migration step. A step exists to reconcile a database
|
||||
//! somebody already has, and nobody has one.
|
||||
//!
|
||||
//! At v0.1 this string freezes and every later change becomes an append-only
|
||||
//! step. That is a deliberate act, not a rule the code already lives under.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
@@ -23,6 +29,8 @@ pub const ddl_v1: [:0]const u8 =
|
||||
\\ id INTEGER PRIMARY KEY,
|
||||
\\ ip TEXT NOT NULL UNIQUE, -- canonical text form (v4 dotted / v6 RFC 5952)
|
||||
\\ name TEXT,
|
||||
\\ learned_name TEXT,
|
||||
\\ name_attempt_after INTEGER NOT NULL DEFAULT 0,
|
||||
\\ group_id INTEGER NOT NULL REFERENCES groups(id),
|
||||
\\ hand_edited INTEGER NOT NULL DEFAULT 0,
|
||||
\\ first_seen INTEGER NOT NULL,
|
||||
@@ -40,7 +48,8 @@ pub const ddl_v1: [:0]const u8 =
|
||||
\\ id INTEGER PRIMARY KEY,
|
||||
\\ url TEXT NOT NULL UNIQUE,
|
||||
\\ priority INTEGER NOT NULL DEFAULT 100,
|
||||
\\ enabled INTEGER NOT NULL DEFAULT 1
|
||||
\\ enabled INTEGER NOT NULL DEFAULT 1,
|
||||
\\ tls_name TEXT NOT NULL DEFAULT '' -- DoT verification name; empty verifies against the url host
|
||||
\\);
|
||||
\\
|
||||
\\CREATE TABLE blocklist_sources (
|
||||
@@ -52,7 +61,9 @@ pub const ddl_v1: [:0]const u8 =
|
||||
\\ last_updated INTEGER,
|
||||
\\ domain_count INTEGER NOT NULL DEFAULT 0,
|
||||
\\ wildcard_count INTEGER NOT NULL DEFAULT 0,
|
||||
\\ exception_count INTEGER NOT NULL DEFAULT 0,
|
||||
\\ skipped_regex_count INTEGER NOT NULL DEFAULT 0,
|
||||
\\ skipped_unsupported_count INTEGER NOT NULL DEFAULT 0,
|
||||
\\ checksum TEXT
|
||||
\\);
|
||||
\\
|
||||
@@ -66,7 +77,7 @@ pub const ddl_v1: [:0]const u8 =
|
||||
\\ id INTEGER PRIMARY KEY,
|
||||
\\ group_id INTEGER NOT NULL REFERENCES groups(id) ON DELETE CASCADE,
|
||||
\\ pattern TEXT NOT NULL,
|
||||
\\ kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard')),
|
||||
\\ kind TEXT NOT NULL CHECK(kind IN ('exact','wildcard','regex')),
|
||||
\\ action TEXT NOT NULL CHECK(action IN ('allow','block')),
|
||||
\\ created_at INTEGER NOT NULL
|
||||
\\);
|
||||
|
||||
+57
-48
@@ -18,19 +18,18 @@ const log = std.log.scoped(.migrations);
|
||||
|
||||
pub const Step = struct { version: u32, sql: [:0]const u8 };
|
||||
|
||||
/// Append only. Editing a released step — or `config_schema.ddl_v1` — would make
|
||||
/// a fresh database and an upgraded one disagree, and nothing would detect it.
|
||||
/// One baseline, no steps. Until v0.1 a schema change edits
|
||||
/// `config_schema.ddl_v1` in place, because nxdns has no installs and there is
|
||||
/// no database in the world for a step to reconcile.
|
||||
///
|
||||
/// At v0.1 the baseline freezes and this list becomes append-only: editing a
|
||||
/// released step would make a fresh database and an upgraded one disagree, and
|
||||
/// nothing would detect it. `migrateSteps` already implements that discipline
|
||||
/// and its tests already pin it against injected step lists.
|
||||
pub const steps = [_]Step{
|
||||
.{ .version = 1, .sql = config_schema.ddl_v1 },
|
||||
.{ .version = 2, .sql = ddl_v2 },
|
||||
};
|
||||
|
||||
/// The DoT verification name (`upstreams.tls_name`). Empty keeps the pre-step-2
|
||||
/// behavior: verify the certificate against the url host.
|
||||
const ddl_v2: [:0]const u8 =
|
||||
\\ALTER TABLE upstreams ADD COLUMN tls_name TEXT NOT NULL DEFAULT '';
|
||||
;
|
||||
|
||||
/// The schema version this binary expects. A database `readVersion` reports
|
||||
/// below this needs `nxdns run` to migrate it; above it is `error.SchemaTooNew`
|
||||
/// and needs a newer nxdns.
|
||||
@@ -112,7 +111,7 @@ pub fn migrateSteps(database: *db.Db, list: []const Step) Error!u32 {
|
||||
///
|
||||
/// Reads only, so it works on a connection opened `.read_only` or
|
||||
/// `.immutable`. That is what it is public for: `nxdns check` may not migrate
|
||||
/// (ruling F-c), and "at version 1, this binary expects 2" tells an operator
|
||||
/// (ruling F-c), and "at version 0, this binary expects 1" tells an operator
|
||||
/// what to do where a bare SQLite error message does not.
|
||||
pub fn readVersion(database: *db.Db) Error!u32 {
|
||||
const present = try database.queryInt(
|
||||
@@ -264,52 +263,64 @@ fn columnExists(database: *db.Db, table: []const u8, column: []const u8) !bool {
|
||||
return stmt.columnInt(0) != 0;
|
||||
}
|
||||
|
||||
test "a fresh database reaches version 2 with the tls_name column" {
|
||||
test "a fresh database reaches the baseline with every v1 column and rule kind" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
try testing.expectEqual(@as(u32, 2), try migrate(&database));
|
||||
try testing.expectEqual(@as(u32, 2), target_version);
|
||||
try testing.expectEqual(@as(u32, 1), try migrate(&database));
|
||||
try testing.expectEqual(@as(u32, 1), target_version);
|
||||
try testing.expect(try columnExists(&database, "clients", "learned_name"));
|
||||
try testing.expect(try columnExists(&database, "clients", "name_attempt_after"));
|
||||
try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
|
||||
try testing.expect(try columnExists(&database, "blocklist_sources", "exception_count"));
|
||||
try testing.expect(try columnExists(&database, "blocklist_sources", "skipped_unsupported_count"));
|
||||
|
||||
try database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, '^ad[0-9]+-', 'regex', 'block', 100);
|
||||
);
|
||||
try testing.expectError(error.Constraint, database.exec(
|
||||
\\INSERT INTO rules (group_id, pattern, kind, action, created_at)
|
||||
\\VALUES (1, 'x', 'glob', 'block', 100);
|
||||
));
|
||||
}
|
||||
|
||||
test "a version 1 database upgrades to 2 and keeps its rows with an empty tls_name" {
|
||||
test "the baseline rules table cascades from its group" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
_ = try migrate(&database);
|
||||
|
||||
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
|
||||
try testing.expectEqual(@as(u32, 1), try migrateSteps(&database, &first));
|
||||
try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
|
||||
try database.exec(
|
||||
\\INSERT INTO groups (id, name) VALUES (2, 'kids');
|
||||
\\INSERT INTO rules (id, group_id, pattern, kind, action, created_at) VALUES
|
||||
\\ (9, 2, '*.tracker.net', 'wildcard', 'allow', 2000);
|
||||
);
|
||||
try database.exec("DELETE FROM groups WHERE id = 2;");
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try database.queryInt("SELECT count(*) FROM rules WHERE group_id = 2"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a failing step rolls back an upgrade of a populated database" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
_ = try migrate(&database);
|
||||
try database.exec("INSERT INTO upstreams (url, priority, enabled) VALUES ('tls://1.1.1.1:853', 10, 1);");
|
||||
|
||||
try testing.expectEqual(@as(u32, 2), try migrate(&database));
|
||||
try testing.expectEqual(@as(u32, 2), try readVersion(&database));
|
||||
try testing.expect(try columnExists(&database, "upstreams", "tls_name"));
|
||||
|
||||
var stmt = try database.prepare("SELECT url, tls_name FROM upstreams");
|
||||
defer stmt.deinit();
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqualStrings("tls://1.1.1.1:853", stmt.columnText(0));
|
||||
try testing.expectEqualStrings("", stmt.columnText(1));
|
||||
}
|
||||
|
||||
test "a failing step after step 2 rolls back the whole upgrade from version 1" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
|
||||
_ = try migrateSteps(&database, &first);
|
||||
|
||||
const broken = [_]Step{
|
||||
steps[0],
|
||||
steps[1],
|
||||
.{ .version = 3, .sql = "CREATE TABLE third (" },
|
||||
// Rollback of an *upgrade* is a different case from rollback of the initial
|
||||
// creation ("a failing step rolls the whole migration back"): here a
|
||||
// populated database must come back untouched, not cease to exist.
|
||||
const broken = steps ++ [_]Step{
|
||||
.{ .version = target_version + 1, .sql = "CREATE TABLE second (id INTEGER PRIMARY KEY);" },
|
||||
.{ .version = target_version + 2, .sql = "CREATE TABLE third (" },
|
||||
};
|
||||
try testing.expectError(error.Unexpected, migrateSteps(&database, &broken));
|
||||
|
||||
// One transaction: the ALTER TABLE of step 2 went back with step 3.
|
||||
try testing.expect(!try columnExists(&database, "upstreams", "tls_name"));
|
||||
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
|
||||
// One transaction: the step that did succeed went back with the one that did not.
|
||||
try testing.expect(!try tableExists(&database, "second"));
|
||||
try testing.expectEqual(target_version, try readVersion(&database));
|
||||
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
|
||||
}
|
||||
|
||||
test "readVersion reports 0 before a migration and target_version after it" {
|
||||
@@ -333,20 +344,18 @@ test "readVersion reads a file database through an immutable open, writing nothi
|
||||
var path_buf: [tmp_prefix.len + sub_path_len + 32]u8 = undefined;
|
||||
const path = try std.fmt.bufPrintZ(&path_buf, "{s}{s}/config.db", .{ tmp_prefix, &tmp.sub_path });
|
||||
|
||||
// A database an older nxdns left at version 1. `check` must report that, not
|
||||
// migrate it (ruling F-c).
|
||||
// `check` reads the stamped version without migrating (ruling F-c), so the
|
||||
// read has to work through a connection that cannot write at all.
|
||||
{
|
||||
var database = try db.Db.open(path, .{ .mode = .read_write_create });
|
||||
defer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
const first = [_]Step{.{ .version = 1, .sql = config_schema.ddl_v1 }};
|
||||
try testing.expectEqual(@as(u32, 1), try migrateSteps(&database, &first));
|
||||
try testing.expectEqual(target_version, try migrate(&database));
|
||||
}
|
||||
|
||||
var database = try db.Db.open(path, .{ .mode = .{ .immutable = testing.io } });
|
||||
defer database.close();
|
||||
try testing.expectEqual(@as(u32, 1), try readVersion(&database));
|
||||
try testing.expectEqual(@as(u32, 2), target_version);
|
||||
try testing.expectEqual(target_version, try readVersion(&database));
|
||||
|
||||
// A write through this connection is refused by SQLite, not by convention.
|
||||
try testing.expectError(error.ReadOnly, database.exec("DELETE FROM schema_version;"));
|
||||
|
||||
@@ -18,6 +18,7 @@ const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
const db = @import("../db.zig");
|
||||
const logger = @import("../logger.zig");
|
||||
const migrations = @import("../migrations.zig");
|
||||
const model = @import("../../config/model.zig");
|
||||
const context = @import("context.zig");
|
||||
@@ -122,6 +123,123 @@ pub fn pruneStale(database: *db.Db, cutoff_s: i64) db.Error!u32 {
|
||||
return @intCast(@min(deleted, std.math.maxInt(u32)));
|
||||
}
|
||||
|
||||
// --- learned names (milestone 25) ------------------------------------------
|
||||
//
|
||||
// `learned_name` and `name_attempt_after` are runtime state beside `last_seen`,
|
||||
// never configuration: `listClients` does not select them, so an export cannot
|
||||
// carry them, and the reconcile engine never writes them. The operator's `name`
|
||||
// is never touched here, and the operator never writes `learned_name` — so
|
||||
// precedence is a display rule, not a write conflict.
|
||||
|
||||
/// How many rows one naming pass may take. The candidate buffer is sized by
|
||||
/// this, and the SQL's LIMIT repeats it as a literal.
|
||||
pub const max_per_pass: usize = 16;
|
||||
|
||||
/// One selected address. Fixed-size storage because the naming path allocates
|
||||
/// nothing: `logger.max_client_len` is the bound every address text in this
|
||||
/// program is sized by.
|
||||
pub const Candidate = struct {
|
||||
buf: [logger.max_client_len]u8 = undefined,
|
||||
len: usize = 0,
|
||||
|
||||
pub fn ip(self: *const Candidate) []const u8 {
|
||||
return self.buf[0..self.len];
|
||||
}
|
||||
};
|
||||
|
||||
pub const CandidateBuf = [max_per_pass]Candidate;
|
||||
|
||||
const resolve_candidates_sql =
|
||||
\\SELECT ip FROM clients
|
||||
\\ WHERE (name IS NULL OR name = '') AND name_attempt_after <= ?1
|
||||
\\ ORDER BY name_attempt_after, ip LIMIT 16
|
||||
;
|
||||
|
||||
/// Fills `out` with the rows whose displayed name would come from learning and
|
||||
/// whose next attempt is due, and returns how many slots it filled.
|
||||
///
|
||||
/// `hand_edited` is deliberately absent from the predicate: display precedence
|
||||
/// never consults it, so candidacy must not either. A row with a name is
|
||||
/// skipped whatever its flag, because its learned name would never be shown.
|
||||
///
|
||||
/// `name_attempt_after` holds the epoch second before which the row is not
|
||||
/// attempted again, so the predicate is a plain comparison: 0 (the column
|
||||
/// default) means eligible now, and no arithmetic can overflow on a corrupt
|
||||
/// value.
|
||||
///
|
||||
/// `error.Mismatch`: a stored `ip` longer than `logger.max_client_len`, which
|
||||
/// means something other than nxdns wrote the row.
|
||||
pub fn resolveCandidates(database: *db.Db, out: *CandidateBuf, now_s: i64) db.Error!usize {
|
||||
var stmt = try database.prepare(resolve_candidates_sql);
|
||||
defer stmt.deinit();
|
||||
try stmt.bindInt(1, now_s);
|
||||
|
||||
var count: usize = 0;
|
||||
while (try stmt.step()) : (count += 1) {
|
||||
const ip = stmt.columnText(0);
|
||||
if (ip.len > logger.max_client_len) return error.Mismatch;
|
||||
@memcpy(out[count].buf[0..ip.len], ip);
|
||||
out[count].len = ip.len;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/// What one naming attempt decided about a row's learned name.
|
||||
pub const LearnedName = union(enum) {
|
||||
/// The resolver answered a target that passed `acceptHostname`.
|
||||
store: []const u8,
|
||||
/// The resolver said the address has no name (NXDOMAIN or NODATA).
|
||||
clear,
|
||||
/// Anything else — an outage must not strip names from the dashboard.
|
||||
keep,
|
||||
};
|
||||
|
||||
pub const NameOutcome = struct {
|
||||
/// The epoch second before which this row is not attempted again.
|
||||
attempt_after: i64,
|
||||
learned: LearnedName,
|
||||
};
|
||||
|
||||
const store_learned_sql =
|
||||
"UPDATE clients SET learned_name = ?2, name_attempt_after = ?3 WHERE ip = ?1";
|
||||
const clear_learned_sql =
|
||||
"UPDATE clients SET learned_name = NULL, name_attempt_after = ?2 WHERE ip = ?1";
|
||||
const keep_learned_sql =
|
||||
"UPDATE clients SET name_attempt_after = ?2 WHERE ip = ?1";
|
||||
|
||||
/// Records one attempt: always the next attempt time, and the learned name only
|
||||
/// when the outcome decided one.
|
||||
///
|
||||
/// A row deleted between selection and this call makes the UPDATE touch zero
|
||||
/// rows. That is a no-op by design — the device left, and nothing was learned
|
||||
/// about nothing — so it is neither an error nor a counted failure.
|
||||
pub fn noteNameOutcome(database: *db.Db, ip: []const u8, outcome: NameOutcome) db.Error!void {
|
||||
switch (outcome.learned) {
|
||||
.store => |text| {
|
||||
var stmt = try database.prepare(store_learned_sql);
|
||||
defer stmt.deinit();
|
||||
try stmt.bindText(1, ip);
|
||||
try stmt.bindText(2, text);
|
||||
try stmt.bindInt(3, outcome.attempt_after);
|
||||
try stmt.exec();
|
||||
},
|
||||
.clear => {
|
||||
var stmt = try database.prepare(clear_learned_sql);
|
||||
defer stmt.deinit();
|
||||
try stmt.bindText(1, ip);
|
||||
try stmt.bindInt(2, outcome.attempt_after);
|
||||
try stmt.exec();
|
||||
},
|
||||
.keep => {
|
||||
var stmt = try database.prepare(keep_learned_sql);
|
||||
defer stmt.deinit();
|
||||
try stmt.bindText(1, ip);
|
||||
try stmt.bindInt(2, outcome.attempt_after);
|
||||
try stmt.exec();
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn deleteAllClients(database: *db.Db) db.Error!void {
|
||||
return database.exec("DELETE FROM clients;");
|
||||
}
|
||||
@@ -197,6 +315,11 @@ pub const ClientRow = struct {
|
||||
/// `clients.name` is nullable; a NULL reads as `""`, as it does on the
|
||||
/// import path.
|
||||
name: []const u8,
|
||||
/// The name learned over reverse DNS, or `""` when nothing was learned.
|
||||
/// Display-only runtime state: `name` wins whenever it is non-empty, and
|
||||
/// this never reaches an export. `name_attempt_after` is deliberately not
|
||||
/// here — it is scheduling state with no operator meaning.
|
||||
learned_name: []const u8,
|
||||
group_id: i64,
|
||||
group: []const u8,
|
||||
hand_edited: bool,
|
||||
@@ -221,14 +344,16 @@ pub const ClientEdit = struct {
|
||||
};
|
||||
|
||||
const list_client_rows_sql =
|
||||
\\SELECT c.id, c.ip, c.name, c.group_id, g.name, c.hand_edited, c.first_seen, c.last_seen
|
||||
\\SELECT c.id, c.ip, c.name, c.group_id, g.name, c.hand_edited, c.first_seen, c.last_seen,
|
||||
\\ c.learned_name
|
||||
\\ FROM clients c
|
||||
\\ JOIN groups g ON g.id = c.group_id
|
||||
\\ ORDER BY c.ip
|
||||
;
|
||||
|
||||
const get_client_sql =
|
||||
\\SELECT c.id, c.ip, c.name, c.group_id, g.name, c.hand_edited, c.first_seen, c.last_seen
|
||||
\\SELECT c.id, c.ip, c.name, c.group_id, g.name, c.hand_edited, c.first_seen, c.last_seen,
|
||||
\\ c.learned_name
|
||||
\\ FROM clients c
|
||||
\\ JOIN groups g ON g.id = c.group_id
|
||||
\\ WHERE c.id = ?1
|
||||
@@ -263,10 +388,14 @@ fn readClientRow(stmt: *db.Stmt, gpa: Allocator) db.Error!ClientRow {
|
||||
errdefer gpa.free(name);
|
||||
const group = try stmt.columnTextAlloc(gpa, 4);
|
||||
errdefer gpa.free(group);
|
||||
// `clients.learned_name` is nullable; NULL reads as "", like `name`.
|
||||
const learned_name = try stmt.columnTextAlloc(gpa, 8);
|
||||
errdefer gpa.free(learned_name);
|
||||
return .{
|
||||
.id = stmt.columnInt(0),
|
||||
.ip = ip,
|
||||
.name = name,
|
||||
.learned_name = learned_name,
|
||||
.group_id = stmt.columnInt(3),
|
||||
.group = group,
|
||||
.hand_edited = stmt.columnBool(5),
|
||||
@@ -721,6 +850,202 @@ test "pruneStale spares hand-edited rows however stale" {
|
||||
try testing.expectEqual(@as(i64, 3), try countClients(&database));
|
||||
}
|
||||
|
||||
// --- learned names ---------------------------------------------------------
|
||||
|
||||
/// `columnText` is borrowed until the statement dies, so the value is copied
|
||||
/// into the caller's buffer.
|
||||
fn learnedName(database: *db.Db, ip: []const u8, buf: []u8) !?[]const u8 {
|
||||
var stmt = try database.prepare("SELECT learned_name FROM clients WHERE ip = ?1");
|
||||
defer stmt.deinit();
|
||||
try stmt.bindText(1, ip);
|
||||
try testing.expect(try stmt.step());
|
||||
if (stmt.isNull(0)) return null;
|
||||
const text = stmt.columnText(0);
|
||||
@memcpy(buf[0..text.len], text);
|
||||
return buf[0..text.len];
|
||||
}
|
||||
|
||||
fn attemptAfter(database: *db.Db, ip: []const u8) !i64 {
|
||||
var stmt = try database.prepare("SELECT name_attempt_after FROM clients WHERE ip = ?1");
|
||||
defer stmt.deinit();
|
||||
try stmt.bindText(1, ip);
|
||||
try testing.expect(try stmt.step());
|
||||
return stmt.columnInt(0);
|
||||
}
|
||||
|
||||
test "resolveCandidates selects unnamed rows in attempt-then-ip order" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
try upsertSeen(&database, "192.168.1.30", 1700000000);
|
||||
try upsertSeen(&database, "192.168.1.10", 1700000000);
|
||||
try upsertSeen(&database, "192.168.1.20", 1700000000);
|
||||
// Attempted already, and due later than the other two.
|
||||
try noteNameOutcome(&database, "192.168.1.10", .{ .attempt_after = 500, .learned = .keep });
|
||||
|
||||
var buf: CandidateBuf = undefined;
|
||||
const count = try resolveCandidates(&database, &buf, 1000);
|
||||
try testing.expectEqual(@as(usize, 3), count);
|
||||
try testing.expectEqualStrings("192.168.1.20", buf[0].ip());
|
||||
try testing.expectEqualStrings("192.168.1.30", buf[1].ip());
|
||||
try testing.expectEqualStrings("192.168.1.10", buf[2].ip());
|
||||
}
|
||||
|
||||
test "resolveCandidates skips named rows whatever their hand_edited flag" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
_ = try insertClientRow(&database, .{ .ip = "192.168.1.7", .name = "printer", .group_id = 1 }, 1);
|
||||
// Hand-edited but unnamed: grouped, not named, so it still benefits.
|
||||
_ = try insertClientRow(&database, .{ .ip = "192.168.1.8", .group_id = 1 }, 1);
|
||||
try upsertSeen(&database, "192.168.1.9", 1);
|
||||
// An empty name is as unnamed as NULL.
|
||||
try database.exec("UPDATE clients SET name = '' WHERE ip = '192.168.1.9';");
|
||||
|
||||
var buf: CandidateBuf = undefined;
|
||||
const count = try resolveCandidates(&database, &buf, 1000);
|
||||
try testing.expectEqual(@as(usize, 2), count);
|
||||
try testing.expectEqualStrings("192.168.1.8", buf[0].ip());
|
||||
try testing.expectEqualStrings("192.168.1.9", buf[1].ip());
|
||||
}
|
||||
|
||||
test "a never-attempted row is due at any now_s, and the cutoff is inclusive" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
var buf: CandidateBuf = undefined;
|
||||
|
||||
try upsertSeen(&database, "192.168.1.10", 1700000000);
|
||||
// `now_s` smaller than any cadence constant still selects the DEFAULT 0 row.
|
||||
try testing.expectEqual(@as(usize, 1), try resolveCandidates(&database, &buf, 0));
|
||||
|
||||
try noteNameOutcome(&database, "192.168.1.10", .{ .attempt_after = 1700086400, .learned = .keep });
|
||||
try testing.expectEqual(@as(usize, 1), try resolveCandidates(&database, &buf, 1700086400));
|
||||
try testing.expectEqual(@as(usize, 0), try resolveCandidates(&database, &buf, 1700086399));
|
||||
}
|
||||
|
||||
test "an extreme name_attempt_after never selects and never traps" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
var buf: CandidateBuf = undefined;
|
||||
|
||||
try upsertSeen(&database, "192.168.1.10", 1700000000);
|
||||
try noteNameOutcome(&database, "192.168.1.10", .{
|
||||
.attempt_after = std.math.maxInt(i64),
|
||||
.learned = .keep,
|
||||
});
|
||||
|
||||
try testing.expectEqual(@as(usize, 0), try resolveCandidates(&database, &buf, std.math.maxInt(i64) - 1));
|
||||
try testing.expectEqual(@as(usize, 1), try resolveCandidates(&database, &buf, std.math.maxInt(i64)));
|
||||
}
|
||||
|
||||
test "resolveCandidates stops at max_per_pass" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
for (0..max_per_pass + 4) |i| {
|
||||
var ip_buf: [logger.max_client_len]u8 = undefined;
|
||||
const ip = try std.fmt.bufPrint(&ip_buf, "192.168.2.{d}", .{i});
|
||||
try upsertSeen(&database, ip, 1700000000);
|
||||
}
|
||||
|
||||
var buf: CandidateBuf = undefined;
|
||||
try testing.expectEqual(max_per_pass, try resolveCandidates(&database, &buf, 1700000000));
|
||||
}
|
||||
|
||||
test "noteNameOutcome stores, overwrites, clears and keeps" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
var name_buf: [logger.max_client_len]u8 = undefined;
|
||||
try upsertSeen(&database, "192.168.1.10", 1700000000);
|
||||
|
||||
try noteNameOutcome(&database, "192.168.1.10", .{ .attempt_after = 10, .learned = .{ .store = "nas.lan" } });
|
||||
try testing.expectEqualStrings("nas.lan", (try learnedName(&database, "192.168.1.10", &name_buf)).?);
|
||||
try testing.expectEqual(@as(i64, 10), try attemptAfter(&database, "192.168.1.10"));
|
||||
|
||||
// The router is the authority on its own zone: a changed answer wins.
|
||||
try noteNameOutcome(&database, "192.168.1.10", .{ .attempt_after = 20, .learned = .{ .store = "tv.lan" } });
|
||||
try testing.expectEqualStrings("tv.lan", (try learnedName(&database, "192.168.1.10", &name_buf)).?);
|
||||
|
||||
// Keep leaves the stored name and only moves the schedule.
|
||||
try noteNameOutcome(&database, "192.168.1.10", .{ .attempt_after = 30, .learned = .keep });
|
||||
try testing.expectEqualStrings("tv.lan", (try learnedName(&database, "192.168.1.10", &name_buf)).?);
|
||||
try testing.expectEqual(@as(i64, 30), try attemptAfter(&database, "192.168.1.10"));
|
||||
|
||||
try noteNameOutcome(&database, "192.168.1.10", .{ .attempt_after = 40, .learned = .clear });
|
||||
try testing.expectEqual(@as(?[]const u8, null), try learnedName(&database, "192.168.1.10", &name_buf));
|
||||
try testing.expectEqual(@as(i64, 40), try attemptAfter(&database, "192.168.1.10"));
|
||||
}
|
||||
|
||||
test "noteNameOutcome on a row that no longer exists is a no-op" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
try noteNameOutcome(&database, "192.168.1.99", .{ .attempt_after = 10, .learned = .{ .store = "gone.lan" } });
|
||||
try noteNameOutcome(&database, "192.168.1.99", .{ .attempt_after = 10, .learned = .clear });
|
||||
try noteNameOutcome(&database, "192.168.1.99", .{ .attempt_after = 10, .learned = .keep });
|
||||
try testing.expectEqual(@as(i64, 0), try countClients(&database));
|
||||
}
|
||||
|
||||
test "a learned name reads back on ClientRow and a NULL reads as the empty string" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
try upsertSeen(&database, "192.168.1.10", 1700000000);
|
||||
try upsertSeen(&database, "192.168.1.11", 1700000000);
|
||||
try noteNameOutcome(&database, "192.168.1.10", .{ .attempt_after = 10, .learned = .{ .store = "nas.lan" } });
|
||||
|
||||
var rows = try listClientRows(&database, testing.allocator);
|
||||
defer rows.deinit(testing.allocator);
|
||||
defer freeClientRows(testing.allocator, rows.items);
|
||||
try testing.expectEqualStrings("nas.lan", rows.items[0].learned_name);
|
||||
try testing.expectEqualStrings("", rows.items[1].learned_name);
|
||||
|
||||
const fetched = (try getClient(&database, testing.allocator, rows.items[0].id)).?;
|
||||
defer freeClientRow(testing.allocator, fetched);
|
||||
try testing.expectEqualStrings("nas.lan", fetched.learned_name);
|
||||
}
|
||||
|
||||
test "updateClient leaves learned_name alone and removes the row from candidacy" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
var name_buf: [logger.max_client_len]u8 = undefined;
|
||||
try upsertSeen(&database, "192.168.1.10", 1700000000);
|
||||
try noteNameOutcome(&database, "192.168.1.10", .{ .attempt_after = 0, .learned = .{ .store = "nas.lan" } });
|
||||
const id = try database.queryInt("SELECT id FROM clients WHERE ip = '192.168.1.10'");
|
||||
try updateClient(&database, id, .{ .name = "the nas", .group_id = 1 });
|
||||
|
||||
try testing.expectEqualStrings("nas.lan", (try learnedName(&database, "192.168.1.10", &name_buf)).?);
|
||||
var buf: CandidateBuf = undefined;
|
||||
try testing.expectEqual(@as(usize, 0), try resolveCandidates(&database, &buf, 1700000000));
|
||||
}
|
||||
|
||||
test "an export is byte-identical across a learned name landing" {
|
||||
const export_mod = @import("../../config/export.zig");
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
var ids = try seedGroups(&database);
|
||||
defer ids.deinit(testing.allocator);
|
||||
try seedClients(&database, &ids);
|
||||
try upsertSeen(&database, "192.168.1.99", 1700000000);
|
||||
|
||||
var before: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer before.deinit();
|
||||
try export_mod.writeToWriter(testing.allocator, &database, &before.writer);
|
||||
|
||||
try noteNameOutcome(&database, "192.168.1.99", .{
|
||||
.attempt_after = 1700086400,
|
||||
.learned = .{ .store = "phone.lan" },
|
||||
});
|
||||
|
||||
var after: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer after.deinit();
|
||||
try export_mod.writeToWriter(testing.allocator, &database, &after.writer);
|
||||
|
||||
try testing.expectEqualStrings(before.written(), after.written());
|
||||
}
|
||||
|
||||
test "client_prefixes round-trip in prefix order with group names resolved" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
@@ -281,6 +281,31 @@ test "rules round-trip in group, kind, action, pattern, id order" {
|
||||
);
|
||||
}
|
||||
|
||||
test "a regex rule round-trips through the table with its pattern untouched" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
var ids = try seedGroupIds();
|
||||
defer ids.deinit(testing.allocator);
|
||||
|
||||
// Uppercase, a trailing metacharacter and a backslash escape: everything
|
||||
// the name-shaped kinds normalize away and this one must not.
|
||||
const pattern = "^AD[0-9]+-\\.example\\.";
|
||||
try insertRule(&database, .{
|
||||
.group = "default",
|
||||
.pattern = pattern,
|
||||
.kind = .regex,
|
||||
.action = .block,
|
||||
}, .{ .now = 1700000000, .group_ids = &ids });
|
||||
|
||||
var items = try listRules(&database, testing.allocator);
|
||||
defer items.deinit(testing.allocator);
|
||||
defer freeRules(testing.allocator, items.items);
|
||||
|
||||
try testing.expectEqual(@as(usize, 1), items.items.len);
|
||||
try testing.expectEqual(model.RuleKind.regex, items.items[0].kind);
|
||||
try testing.expectEqualStrings(pattern, items.items[0].pattern);
|
||||
}
|
||||
|
||||
test "a duplicate rule is accepted and stays deterministically ordered by id" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
//! `blocklist_sources`.
|
||||
//!
|
||||
//! Only the four configuration columns are read and written. `last_updated`,
|
||||
//! `domain_count`, `wildcard_count`, `skipped_regex_count` and `checksum` are
|
||||
//! facts a running server produces; an insert leaves them at their column
|
||||
//! defaults so two exports taken minutes apart stay identical.
|
||||
//! `domain_count`, `wildcard_count`, `exception_count`, `skipped_regex_count`,
|
||||
//! `skipped_unsupported_count` and `checksum` are facts a running server
|
||||
//! produces; an insert leaves them at their column defaults so two exports taken
|
||||
//! minutes apart stay identical.
|
||||
//!
|
||||
//! The import path is list / insert / deleteAll / count; the runtime columns and
|
||||
//! the REST surface follow it, both keyed by row id.
|
||||
@@ -88,7 +89,17 @@ pub const SourceRow = struct {
|
||||
last_updated: ?i64,
|
||||
domain_count: i64,
|
||||
wildcard_count: i64,
|
||||
/// Written `.allow` entries: the `@@||name^` exceptions the list carries.
|
||||
/// Defaulted for the same reason `is_suggested` is — the blocklist manager
|
||||
/// builds `SourceRow` values from the refresh columns alone.
|
||||
exception_count: i64 = 0,
|
||||
skipped_regex_count: i64,
|
||||
/// Lines the compiler read and could not translate into a DNS decision:
|
||||
/// cosmetic element hiding, `$`-modifier rules (save the tolerated
|
||||
/// `$important` exception suffix, which lands in `exception_count`), scheme
|
||||
/// anchors. Counted and not written, like `skipped_regex_count` and unlike
|
||||
/// the three counts above.
|
||||
skipped_unsupported_count: i64,
|
||||
checksum: ?[]const u8,
|
||||
};
|
||||
|
||||
@@ -96,15 +107,19 @@ pub const SourceStats = struct {
|
||||
last_updated: i64,
|
||||
domain_count: i64,
|
||||
wildcard_count: i64,
|
||||
exception_count: i64,
|
||||
skipped_regex_count: i64,
|
||||
/// Lowercase hex sha256 over the `.list` body followed by the `.wild` body.
|
||||
skipped_unsupported_count: i64,
|
||||
/// Lowercase hex sha256 over the `.list` body, then the `.wild` body, then
|
||||
/// the `.allow` body, each followed by `compiler.body_separator` so the
|
||||
/// digest cannot confuse a name in one body with the same name in another.
|
||||
checksum: []const u8,
|
||||
};
|
||||
|
||||
const row_columns_sql =
|
||||
\\SELECT id, url, name, enabled, last_updated,
|
||||
\\ domain_count, wildcard_count, skipped_regex_count, checksum,
|
||||
\\ is_suggested
|
||||
\\ is_suggested, exception_count, skipped_unsupported_count
|
||||
\\ FROM blocklist_sources
|
||||
;
|
||||
|
||||
@@ -133,7 +148,9 @@ fn readSourceRow(stmt: *db.Stmt, gpa: Allocator) db.Error!SourceRow {
|
||||
.last_updated = if (stmt.isNull(4)) null else stmt.columnInt(4),
|
||||
.domain_count = stmt.columnInt(5),
|
||||
.wildcard_count = stmt.columnInt(6),
|
||||
.exception_count = stmt.columnInt(10),
|
||||
.skipped_regex_count = stmt.columnInt(7),
|
||||
.skipped_unsupported_count = stmt.columnInt(11),
|
||||
.checksum = checksum,
|
||||
};
|
||||
}
|
||||
@@ -150,7 +167,8 @@ pub fn freeSourceRows(gpa: Allocator, items: []const SourceRow) void {
|
||||
const update_stats_sql =
|
||||
\\UPDATE blocklist_sources
|
||||
\\ SET last_updated = ?2, domain_count = ?3, wildcard_count = ?4,
|
||||
\\ skipped_regex_count = ?5, checksum = ?6
|
||||
\\ skipped_regex_count = ?5, checksum = ?6, exception_count = ?7,
|
||||
\\ skipped_unsupported_count = ?8
|
||||
\\ WHERE id = ?1
|
||||
;
|
||||
|
||||
@@ -165,6 +183,8 @@ pub fn updateSourceStats(database: *db.Db, id: i64, stats: SourceStats) db.Error
|
||||
try stmt.bindInt(4, stats.wildcard_count);
|
||||
try stmt.bindInt(5, stats.skipped_regex_count);
|
||||
try stmt.bindText(6, stats.checksum);
|
||||
try stmt.bindInt(7, stats.exception_count);
|
||||
try stmt.bindInt(8, stats.skipped_unsupported_count);
|
||||
try stmt.exec();
|
||||
}
|
||||
|
||||
@@ -304,7 +324,13 @@ test "insertBlocklistSource leaves the runtime columns at their defaults" {
|
||||
);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try database.queryInt("SELECT sum(domain_count + wildcard_count + skipped_regex_count) FROM blocklist_sources"),
|
||||
try database.queryInt(
|
||||
// Every runtime counter, summed only because this asserts they are
|
||||
// all 0. No production query may add the two skip counters to the
|
||||
// three written ones: a skipped line was never written.
|
||||
"SELECT sum(domain_count + wildcard_count + exception_count + skipped_regex_count" ++
|
||||
" + skipped_unsupported_count) FROM blocklist_sources",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -355,7 +381,9 @@ test "listSourceRows returns row ids and the runtime columns in url order" {
|
||||
try testing.expectEqual(@as(?[]const u8, null), row.checksum);
|
||||
try testing.expectEqual(@as(i64, 0), row.domain_count);
|
||||
try testing.expectEqual(@as(i64, 0), row.wildcard_count);
|
||||
try testing.expectEqual(@as(i64, 0), row.exception_count);
|
||||
try testing.expectEqual(@as(i64, 0), row.skipped_regex_count);
|
||||
try testing.expectEqual(@as(i64, 0), row.skipped_unsupported_count);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -373,7 +401,9 @@ test "updateSourceStats writes the runtime columns of one source only" {
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 4321,
|
||||
.wildcard_count = 21,
|
||||
.exception_count = 9,
|
||||
.skipped_regex_count = 7,
|
||||
.skipped_unsupported_count = 15,
|
||||
.checksum = "a" ** 64,
|
||||
});
|
||||
|
||||
@@ -389,7 +419,9 @@ test "updateSourceStats writes the runtime columns of one source only" {
|
||||
try testing.expectEqual(@as(?i64, 1_700_000_000), updated.last_updated);
|
||||
try testing.expectEqual(@as(i64, 4321), updated.domain_count);
|
||||
try testing.expectEqual(@as(i64, 21), updated.wildcard_count);
|
||||
try testing.expectEqual(@as(i64, 9), updated.exception_count);
|
||||
try testing.expectEqual(@as(i64, 7), updated.skipped_regex_count);
|
||||
try testing.expectEqual(@as(i64, 15), updated.skipped_unsupported_count);
|
||||
try testing.expectEqualStrings("a" ** 64, updated.checksum.?);
|
||||
|
||||
// The two untouched rows kept their defaults.
|
||||
@@ -405,7 +437,9 @@ fn listSourceRowsUnderFailure(gpa: Allocator) !void {
|
||||
.last_updated = 1,
|
||||
.domain_count = 2,
|
||||
.wildcard_count = 3,
|
||||
.exception_count = 5,
|
||||
.skipped_regex_count = 4,
|
||||
.skipped_unsupported_count = 6,
|
||||
.checksum = "b" ** 64,
|
||||
});
|
||||
|
||||
@@ -470,7 +504,9 @@ test "updateSource leaves the runtime columns where the refresh path left them"
|
||||
.last_updated = 1_700_000_000,
|
||||
.domain_count = 12,
|
||||
.wildcard_count = 3,
|
||||
.exception_count = 2,
|
||||
.skipped_regex_count = 1,
|
||||
.skipped_unsupported_count = 4,
|
||||
.checksum = "c" ** 64,
|
||||
});
|
||||
|
||||
|
||||
@@ -58,6 +58,7 @@ comptime {
|
||||
_ = @import("filter/parser_domains.zig");
|
||||
_ = @import("filter/parser_abp.zig");
|
||||
_ = @import("filter/wildcard.zig");
|
||||
_ = @import("filter/regex.zig");
|
||||
_ = @import("filter/domain_set.zig");
|
||||
_ = @import("filter/rules.zig");
|
||||
_ = @import("filter/matcher.zig");
|
||||
@@ -70,6 +71,7 @@ comptime {
|
||||
_ = @import("local/records.zig");
|
||||
_ = @import("local/forward_zones.zig");
|
||||
_ = @import("local/forward_client.zig");
|
||||
_ = @import("local/reverse_name.zig");
|
||||
_ = @import("cache/dns_cache.zig");
|
||||
_ = @import("server/rate_limiter.zig");
|
||||
_ = @import("storage/repositories/queries_repo.zig");
|
||||
@@ -80,6 +82,7 @@ comptime {
|
||||
_ = @import("storage/retention.zig");
|
||||
_ = @import("storage/phase6_integration_test.zig");
|
||||
_ = @import("server/pause.zig");
|
||||
_ = @import("server/client_names.zig");
|
||||
_ = @import("server/clients.zig");
|
||||
_ = @import("server/shutdown.zig");
|
||||
_ = @import("server/phase7_integration_test.zig");
|
||||
|
||||
@@ -59,7 +59,9 @@ pub const StatusView = struct {
|
||||
last_error: []const u8,
|
||||
domains: u32,
|
||||
wildcards: u32,
|
||||
exceptions: u32,
|
||||
skipped_regex: u32,
|
||||
skipped_unsupported: u32,
|
||||
|
||||
pub fn from(status: *const manager_mod.SourceStatus) StatusView {
|
||||
return .{
|
||||
@@ -72,7 +74,9 @@ pub const StatusView = struct {
|
||||
.last_error = status.errorText(),
|
||||
.domains = status.counts.domains,
|
||||
.wildcards = status.counts.wildcards,
|
||||
.exceptions = status.counts.exceptions,
|
||||
.skipped_regex = status.counts.skipped_regex,
|
||||
.skipped_unsupported = status.counts.skipped_unsupported,
|
||||
};
|
||||
}
|
||||
};
|
||||
@@ -317,7 +321,9 @@ test "editing a blocklist keeps the counters the refresh wrote" {
|
||||
.last_updated = 1700,
|
||||
.domain_count = 42,
|
||||
.wildcard_count = 3,
|
||||
.exception_count = 2,
|
||||
.skipped_regex_count = 1,
|
||||
.skipped_unsupported_count = 8,
|
||||
.checksum = "abc",
|
||||
});
|
||||
|
||||
@@ -332,6 +338,8 @@ test "editing a blocklist keeps the counters the refresh wrote" {
|
||||
try testing.expectEqualStrings("renamed", row.name);
|
||||
try testing.expect(!row.enabled);
|
||||
try testing.expectEqual(@as(i64, 42), row.domain_count);
|
||||
try testing.expectEqual(@as(i64, 1), row.skipped_regex_count);
|
||||
try testing.expectEqual(@as(i64, 8), row.skipped_unsupported_count);
|
||||
try testing.expectEqual(@as(usize, 2), bench.reloads);
|
||||
}
|
||||
|
||||
@@ -380,7 +388,13 @@ test "a status becomes the flat shape the API answers with" {
|
||||
const message = "connection refused";
|
||||
@memcpy(status.last_error[0..message.len], message);
|
||||
status.last_error_len = message.len;
|
||||
status.counts = .{ .domains = 10, .wildcards = 2, .skipped_regex = 1 };
|
||||
status.counts = .{
|
||||
.domains = 10,
|
||||
.wildcards = 2,
|
||||
.exceptions = 4,
|
||||
.skipped_regex = 1,
|
||||
.skipped_unsupported = 6,
|
||||
};
|
||||
|
||||
const view: StatusView = .from(&status);
|
||||
try testing.expectEqual(@as(i64, 7), view.id);
|
||||
@@ -389,4 +403,7 @@ test "a status becomes the flat shape the API answers with" {
|
||||
try testing.expectEqualStrings(url, view.url);
|
||||
try testing.expectEqualStrings(message, view.last_error);
|
||||
try testing.expectEqual(@as(u32, 10), view.domains);
|
||||
try testing.expectEqual(@as(u32, 4), view.exceptions);
|
||||
try testing.expectEqual(@as(u32, 1), view.skipped_regex);
|
||||
try testing.expectEqual(@as(u32, 6), view.skipped_unsupported);
|
||||
}
|
||||
|
||||
@@ -38,6 +38,8 @@ pub const Body = struct {
|
||||
reason: []const u8,
|
||||
/// The rule or list entry that decided it; "" when nothing matched.
|
||||
matched: []const u8,
|
||||
/// The list that decided it, which for `blocklist_exception` is the list
|
||||
/// whose `@@` rule lifted the block rather than one that made it.
|
||||
source_url: ?[]const u8,
|
||||
safe_search_rewrite: ?[]const u8,
|
||||
};
|
||||
@@ -50,7 +52,8 @@ pub const Result = struct {
|
||||
blocked: bool,
|
||||
reason: matcher.Reason,
|
||||
matched: []const u8,
|
||||
/// `blocklist_sources` row id of the list that matched.
|
||||
/// `blocklist_sources` row id of the list that matched, whether it blocked
|
||||
/// the name or lifted it through an `@@` exception.
|
||||
source_id: ?i64,
|
||||
safe_search_rewrite: ?[]const u8,
|
||||
};
|
||||
|
||||
@@ -39,7 +39,7 @@ const Created = union(enum) { id: i64, fail: Failure };
|
||||
/// not understood.
|
||||
fn toInput(body: Body) union(enum) { input: rules_repo.RuleInput, fail: Failure } {
|
||||
const kind = model.RuleKind.fromDb(body.kind) orelse
|
||||
return .{ .fail = .{ .invalid = "kind must be 'exact' or 'wildcard'" } };
|
||||
return .{ .fail = .{ .invalid = "kind must be 'exact', 'wildcard' or 'regex'" } };
|
||||
const action = model.RuleAction.fromDb(body.action) orelse
|
||||
return .{ .fail = .{ .invalid = "action must be 'allow' or 'block'" } };
|
||||
return .{ .input = .{
|
||||
@@ -308,7 +308,7 @@ test "an unknown kind or action is a 400 before anything is written" {
|
||||
try testing.expect(toInput(.{
|
||||
.group_id = 1,
|
||||
.pattern = "ads.example",
|
||||
.kind = "regex",
|
||||
.kind = "glob",
|
||||
.action = "block",
|
||||
}).fail == .invalid);
|
||||
|
||||
@@ -327,4 +327,46 @@ test "an unknown kind or action is a 400 before anything is written" {
|
||||
});
|
||||
try testing.expectEqual(model.RuleKind.wildcard, good.input.kind);
|
||||
try testing.expectEqual(model.RuleAction.allow, good.input.action);
|
||||
|
||||
const third = toInput(.{
|
||||
.group_id = 1,
|
||||
.pattern = "^ad[0-9]+-",
|
||||
.kind = "regex",
|
||||
.action = "block",
|
||||
});
|
||||
try testing.expectEqual(model.RuleKind.regex, third.input.kind);
|
||||
}
|
||||
|
||||
test "a regex rule is stored, and a pattern the engine refuses is a 400 that names it" {
|
||||
var bench: mutations.Bench = undefined;
|
||||
try bench.init(testing.allocator);
|
||||
defer bench.deinit(testing.allocator);
|
||||
|
||||
const created = try applyCreate(&bench.state, bench.io(), bench.arena(), .{
|
||||
.group_id = 1,
|
||||
.pattern = "^ad[0-9]+-",
|
||||
.kind = .regex,
|
||||
.action = .block,
|
||||
});
|
||||
const row = (try rules_repo.getRule(&bench.database, bench.arena(), created.id)).?;
|
||||
try testing.expectEqual(model.RuleKind.regex, row.kind);
|
||||
// Stored verbatim: a regex is not a name, so nothing lowercases or
|
||||
// dot-strips it on the way to the table.
|
||||
try testing.expectEqualStrings("^ad[0-9]+-", row.pattern);
|
||||
|
||||
const unclosed = try applyCreate(&bench.state, bench.io(), bench.arena(), .{
|
||||
.group_id = 1,
|
||||
.pattern = "(",
|
||||
.kind = .regex,
|
||||
.action = .block,
|
||||
});
|
||||
try testing.expectEqualStrings(
|
||||
"rules[0].pattern: '(' is not a valid regex pattern",
|
||||
unclosed.fail.invalid,
|
||||
);
|
||||
|
||||
// The refused pattern reached no table, and the good one is still the only
|
||||
// row: a 400 costs no write and no reload.
|
||||
try testing.expectEqual(@as(i64, 1), try bench.queryInt("SELECT count(*) FROM rules"));
|
||||
try testing.expectEqual(@as(usize, 1), bench.reloads);
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
const cert_store = @import("../server/cert_store.zig");
|
||||
const client_names = @import("../server/client_names.zig");
|
||||
const clients = @import("../server/clients.zig");
|
||||
const dns_cache = @import("../cache/dns_cache.zig");
|
||||
const dns_handler = @import("../server/handler.zig");
|
||||
@@ -122,6 +123,7 @@ pub const Sample = struct {
|
||||
cache: ?CacheSample = null,
|
||||
limiter: ?LimiterSample = null,
|
||||
tracker: ?TrackerSample = null,
|
||||
client_names: ?client_names.Resolver.Stats = null,
|
||||
retention: ?retention_mod.Stats = null,
|
||||
blocklist: ?BlocklistSample = null,
|
||||
disk: ?DiskSample = null,
|
||||
@@ -193,6 +195,8 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
|
||||
.pending_clients = tracker.pendingClients(io),
|
||||
};
|
||||
|
||||
if (state.client_names) |names| sample.client_names = names.snapshotStats(io);
|
||||
|
||||
if (state.retention) |retention| sample.retention = retention.snapshotStats();
|
||||
|
||||
if (state.manager) |manager| {
|
||||
@@ -340,6 +344,10 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
|
||||
);
|
||||
}
|
||||
|
||||
if (sample.client_names) |names| {
|
||||
try counterGroup(w, "nxdns_client_names_", "Learned client name counter", names);
|
||||
}
|
||||
|
||||
if (sample.retention) |retention| {
|
||||
try counterGroup(w, "nxdns_retention_", "Query log retention counter", retention);
|
||||
}
|
||||
@@ -594,6 +602,7 @@ fn writeLabelValue(w: *std.Io.Writer, value: []const u8) std.Io.Writer.Error!voi
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const local_tables = @import("../server/local_tables.zig");
|
||||
const logger_mod = @import("../storage/logger.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
@@ -645,6 +654,7 @@ test "a full sample renders the whole exposition, byte for byte" {
|
||||
.stats = .{ .tracked = 4, .flushed = 3, .dropped_full = 0, .pruned = 1, .flush_failures = 0 },
|
||||
.pending_clients = 2,
|
||||
},
|
||||
.client_names = .{ .attempted = 6, .answered = 3, .nxdomain = 1, .no_zone = 2, .invalid = 0, .failed = 0, .read_failures = 0, .write_failures = 1 },
|
||||
.retention = .{ .passes = 7, .rows_pruned = 100, .checkpoints = 7, .vacuums = 1 },
|
||||
.blocklist = .{ .refreshes_gated = 2, .generation = 4 },
|
||||
.disk = .{
|
||||
@@ -681,6 +691,9 @@ test "a full sample renders the whole exposition, byte for byte" {
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_dns_rate_limit_tracked_clients 3\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_clients_dropped_full_total 0\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_clients_pending 2\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_client_names_no_zone_total 2\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_client_names_answered_total 3\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_client_names_write_failures_total 1\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_retention_rows_pruned_total 100\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_blocklist_refreshes_gated_total 2\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_blocklist_generation 4\n"));
|
||||
@@ -1242,12 +1255,17 @@ test "collect reads the live counters of the components it is given" {
|
||||
|
||||
var tracker: clients.Tracker = .init(30);
|
||||
var retention: retention_mod.Retention = .init(.{});
|
||||
var tables: local_tables.LocalTables = .empty;
|
||||
var names: client_names.Resolver = .init(&tables);
|
||||
names.stats.no_zone = 4;
|
||||
names.stats.attempted = 4;
|
||||
|
||||
var state: server.WebState = .{
|
||||
.gpa = testing.allocator,
|
||||
.handler = &handler,
|
||||
.logger = &query_logger,
|
||||
.tracker = &tracker,
|
||||
.client_names = &names,
|
||||
.retention = &retention,
|
||||
};
|
||||
|
||||
@@ -1263,6 +1281,7 @@ test "collect reads the live counters of the components it is given" {
|
||||
try testing.expectEqual(@as(u64, 3), sample.limiter.?.stats.refused);
|
||||
try testing.expectEqual(@as(u64, 90), sample.logger.rows_written);
|
||||
try testing.expectEqual(@as(u64, 0), sample.tracker.?.pending_clients);
|
||||
try testing.expectEqual(@as(u64, 4), sample.client_names.?.no_zone);
|
||||
try testing.expectEqual(@as(u64, 0), sample.retention.?.passes);
|
||||
try testing.expectEqual(@as(?BlocklistSample, null), sample.blocklist);
|
||||
try testing.expectEqual(@as(usize, 0), sample.upstreams.len);
|
||||
|
||||
+17
-6
@@ -1876,7 +1876,7 @@ components:
|
||||
|
||||
Blocklist:
|
||||
type: object
|
||||
required: [id, url, name, enabled, is_suggested, last_updated, domain_count, wildcard_count, skipped_regex_count, checksum]
|
||||
required: [id, url, name, enabled, is_suggested, last_updated, domain_count, wildcard_count, exception_count, skipped_regex_count, skipped_unsupported_count, checksum]
|
||||
properties:
|
||||
id: { type: integer }
|
||||
url: { type: string }
|
||||
@@ -1888,7 +1888,9 @@ components:
|
||||
nullable: true
|
||||
domain_count: { type: integer }
|
||||
wildcard_count: { type: integer }
|
||||
exception_count: { type: integer }
|
||||
skipped_regex_count: { type: integer }
|
||||
skipped_unsupported_count: { type: integer }
|
||||
checksum:
|
||||
type: string
|
||||
nullable: true
|
||||
@@ -1918,7 +1920,7 @@ components:
|
||||
|
||||
SourceStatus:
|
||||
type: object
|
||||
required: [id, state, loaded, last_attempt, last_success, url, last_error, domains, wildcards, skipped_regex]
|
||||
required: [id, state, loaded, last_attempt, last_success, url, last_error, domains, wildcards, exceptions, skipped_regex, skipped_unsupported]
|
||||
properties:
|
||||
id: { type: integer }
|
||||
state:
|
||||
@@ -1933,7 +1935,9 @@ components:
|
||||
description: Empty when the last attempt succeeded.
|
||||
domains: { type: integer }
|
||||
wildcards: { type: integer }
|
||||
exceptions: { type: integer }
|
||||
skipped_regex: { type: integer }
|
||||
skipped_unsupported: { type: integer }
|
||||
|
||||
Rule:
|
||||
type: object
|
||||
@@ -1945,7 +1949,7 @@ components:
|
||||
pattern: { type: string }
|
||||
kind:
|
||||
type: string
|
||||
enum: [exact, wildcard]
|
||||
enum: [exact, wildcard, regex]
|
||||
action:
|
||||
type: string
|
||||
enum: [allow, block]
|
||||
@@ -1959,7 +1963,7 @@ components:
|
||||
pattern: { type: string }
|
||||
kind:
|
||||
type: string
|
||||
enum: [exact, wildcard]
|
||||
enum: [exact, wildcard, regex]
|
||||
action:
|
||||
type: string
|
||||
enum: [allow, block]
|
||||
@@ -1973,7 +1977,7 @@ components:
|
||||
pattern: { type: string }
|
||||
kind:
|
||||
type: string
|
||||
enum: [exact, wildcard]
|
||||
enum: [exact, wildcard, regex]
|
||||
action:
|
||||
type: string
|
||||
enum: [allow, block]
|
||||
@@ -2020,13 +2024,20 @@ components:
|
||||
|
||||
Client:
|
||||
type: object
|
||||
required: [id, ip, name, group_id, group, hand_edited, first_seen, last_seen]
|
||||
required: [id, ip, name, learned_name, group_id, group, hand_edited, first_seen, last_seen]
|
||||
properties:
|
||||
id: { type: integer }
|
||||
ip: { type: string }
|
||||
name:
|
||||
type: string
|
||||
description: Empty when the client was never named.
|
||||
learned_name:
|
||||
type: string
|
||||
description: |
|
||||
The name learned over reverse DNS, or empty when nothing was
|
||||
learned. Display-only runtime state: `name` wins whenever it is
|
||||
non-empty, and a learned name never appears in an export. The
|
||||
server writes it; a client cannot.
|
||||
group_id: { type: integer }
|
||||
group: { type: string }
|
||||
hand_edited: { type: boolean }
|
||||
|
||||
@@ -25,6 +25,7 @@ const address = @import("../platform/address.zig");
|
||||
const api_limiter = @import("api_limiter.zig");
|
||||
const auth = @import("auth.zig");
|
||||
const cert_store = @import("../server/cert_store.zig");
|
||||
const client_names = @import("../server/client_names.zig");
|
||||
const clients = @import("../server/clients.zig");
|
||||
const db = @import("../storage/db.zig");
|
||||
const disk_monitor = @import("../storage/disk_monitor.zig");
|
||||
@@ -130,6 +131,8 @@ pub const WebState = struct {
|
||||
handler: ?*dns_handler.Handler = null,
|
||||
pause: ?*pause_mod.Pause = null,
|
||||
tracker: ?*clients.Tracker = null,
|
||||
/// The learned-name resolver, for `metrics.collect` (milestone-25 ruling 9).
|
||||
client_names: ?*client_names.Resolver = null,
|
||||
manager: ?*manager_mod.Manager = null,
|
||||
pool: ?*pool_mod.Pool = null,
|
||||
monitor: ?*disk_monitor.Monitor = null,
|
||||
|
||||
@@ -1665,6 +1665,9 @@ fn createdId(body: []const u8) !i64 {
|
||||
return std.fmt.parseInt(i64, rest[0..end], 10);
|
||||
}
|
||||
|
||||
/// The three files a refresh publishes for one source. The `.allow` file is
|
||||
/// written here too: the delete path has to take every compiled body, and a
|
||||
/// sweep that missed one would leave an orphan this test could not see.
|
||||
fn writeCompiled(io: std.Io, dir: std.Io.Dir, id: i64, body: []const u8) !void {
|
||||
var buf: [64]u8 = undefined;
|
||||
try dir.writeFile(io, .{
|
||||
@@ -1675,6 +1678,10 @@ fn writeCompiled(io: std.Io, dir: std.Io.Dir, id: i64, body: []const u8) !void {
|
||||
.sub_path = try std.fmt.bufPrint(&buf, "{d}.wild", .{id}),
|
||||
.data = "",
|
||||
});
|
||||
try dir.writeFile(io, .{
|
||||
.sub_path = try std.fmt.bufPrint(&buf, "{d}.allow", .{id}),
|
||||
.data = "",
|
||||
});
|
||||
}
|
||||
|
||||
fn accessCompiled(io: std.Io, dir: std.Io.Dir, id: i64) !void {
|
||||
@@ -1726,6 +1733,11 @@ fn deleteSweepsCompiledFiles(io: std.Io, env: *Env) anyerror!void {
|
||||
try std.fmt.bufPrint(&name_buf, "{d}.wild", .{doomed}),
|
||||
.{},
|
||||
));
|
||||
try testing.expectError(error.FileNotFound, dir.access(
|
||||
io,
|
||||
try std.fmt.bufPrint(&name_buf, "{d}.allow", .{doomed}),
|
||||
.{},
|
||||
));
|
||||
try accessCompiled(io, dir, kept);
|
||||
}
|
||||
|
||||
|
||||
Vendored
+4
@@ -35,3 +35,7 @@ Regenerate with:
|
||||
```sh
|
||||
openssl ecparam -name prime256v1 -genkey -noout -out mismatched_key.pem
|
||||
```
|
||||
|
||||
`container-smoke.zon` is the smallest configuration that lets the daemon reach
|
||||
`serving`. `tools/container_check.zig` copies it into the smoke container, which
|
||||
is the only thing that reads it.
|
||||
|
||||
Vendored
+4
@@ -0,0 +1,4 @@
|
||||
.{
|
||||
.groups = .{ .{ .name = "default" } },
|
||||
.upstreams = .{ .{ .url = "https://cloudflare-dns.com/dns-query" } },
|
||||
}
|
||||
@@ -9,8 +9,10 @@
|
||||
//!
|
||||
//! - `Line.text` is always a slice of the caller's line, never a copy and
|
||||
//! never a dangling pointer into a temporary;
|
||||
//! - `covers_apex` is set only on a `.wildcard` line, because the compiler
|
||||
//! reads it only there;
|
||||
//! - `covers_apex` is set only on a `.wildcard` or an `.exception` line,
|
||||
//! because those are the two anchored forms it describes; the compiler acts
|
||||
//! on it for `.wildcard`, where it emits the apex entry beside the suffix
|
||||
//! one;
|
||||
//! - `wildcard.matches` terminates for any pattern, validated or not, and a
|
||||
//! match implies the domain has at least as many labels as the pattern,
|
||||
//! since every pattern label consumes at least one domain label.
|
||||
@@ -75,6 +77,11 @@ fn formatTarget(format: parsers.Format, smith: *Smith) anyerror!void {
|
||||
|
||||
const parsed = parsers.parseLine(format, line);
|
||||
try expectBorrowed(parsed, line);
|
||||
|
||||
// Exception syntax belongs to the ABP parser alone. A hosts or domains line
|
||||
// that produced one would open an allow hole in a format that has no way to
|
||||
// write one.
|
||||
if (format != .abp) try std.testing.expect(parsed.kind != .exception);
|
||||
}
|
||||
|
||||
/// The sniffer reads whole files, so this one keeps the line breaks.
|
||||
@@ -113,7 +120,12 @@ fn wildcardTarget(_: void, smith: *Smith) anyerror!void {
|
||||
/// The parser contract: `text` is a window into the caller's line, so the
|
||||
/// compiler may keep it for the length of that line and no longer.
|
||||
fn expectBorrowed(parsed: parsers.Line, line: []const u8) !void {
|
||||
if (parsed.covers_apex) try std.testing.expectEqual(parsers.Kind.wildcard, parsed.kind);
|
||||
if (parsed.covers_apex) {
|
||||
try std.testing.expect(parsed.kind == .wildcard or parsed.kind == .exception);
|
||||
}
|
||||
// An exception with no name would compile to an empty allow entry, which
|
||||
// `addCandidate` would then reject as invalid rather than honour.
|
||||
if (parsed.kind == .exception) try std.testing.expect(parsed.text.len != 0);
|
||||
if (parsed.text.len == 0) return;
|
||||
|
||||
const start = @intFromPtr(parsed.text.ptr);
|
||||
@@ -146,6 +158,12 @@ const hosts_line = "0.0.0.0 ads.example.com tracker.example.com # advertising";
|
||||
/// An ABP domain rule, which covers the apex as well as the subdomains.
|
||||
const abp_line = "||ads.example.net^";
|
||||
|
||||
/// The exception forms: the two anchored spellings, the one tolerated modifier,
|
||||
/// and a bare `@@` name, which stays unsupported.
|
||||
const abp_exception = "@@||good.ads.example.net^";
|
||||
const abp_exception_important = "@@||good.ads.example.net^$important";
|
||||
const abp_exception_unanchored = "@@good.ads.example.net";
|
||||
|
||||
/// A regex rule, which every parser counts and skips (PLAN §2.2).
|
||||
const regex_line = "/^ads[0-9]+\\.example\\.org$/";
|
||||
|
||||
@@ -161,6 +179,9 @@ const scheme_anchor = "|https://ads.example.com/track";
|
||||
const corpus = [_][]const u8{
|
||||
sliceInput(hosts_line),
|
||||
sliceInput(abp_line),
|
||||
sliceInput(abp_exception),
|
||||
sliceInput(abp_exception_important),
|
||||
sliceInput(abp_exception_unanchored),
|
||||
sliceInput(regex_line),
|
||||
sliceInput(long_line),
|
||||
sliceInput(element_hiding),
|
||||
|
||||
@@ -94,6 +94,8 @@ fn compileOnce(
|
||||
var list_w: std.Io.Writer.Discarding = .init(&list_sink);
|
||||
var wild_sink: [0]u8 = .{};
|
||||
var wild_w: std.Io.Writer.Discarding = .init(&wild_sink);
|
||||
var allow_sink: [0]u8 = .{};
|
||||
var allow_w: std.Io.Writer.Discarding = .init(&allow_sink);
|
||||
|
||||
return compiler.compile(
|
||||
std.testing.allocator,
|
||||
@@ -101,6 +103,7 @@ fn compileOnce(
|
||||
format,
|
||||
&list_w.writer,
|
||||
&wild_w.writer,
|
||||
&allow_w.writer,
|
||||
) catch |err| switch (err) {
|
||||
error.OutOfMemory,
|
||||
error.TooManyDomains,
|
||||
@@ -122,12 +125,13 @@ fn expectConsistent(counts: compiler.Counts, bytes: []const u8) !void {
|
||||
// A candidate is a non-empty whitespace-separated field or a whole wildcard
|
||||
// line, so every candidate consumes at least one byte of the input, and a
|
||||
// written name is a candidate that survived.
|
||||
const candidates = @as(u64, counts.domains) + counts.wildcards +
|
||||
const candidates = @as(u64, counts.domains) + counts.wildcards + counts.exceptions +
|
||||
counts.duplicates + counts.invalid;
|
||||
try std.testing.expect(candidates <= bytes.len + 1);
|
||||
|
||||
try std.testing.expect(counts.domains <= compiler.max_domains);
|
||||
try std.testing.expect(counts.wildcards <= compiler.max_domains);
|
||||
try std.testing.expect(counts.exceptions <= compiler.max_domains);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -154,6 +158,9 @@ const corpus = [_][]const u8{
|
||||
sliceInput(long_line_terminated),
|
||||
sliceInput("# a hosts list\n0.0.0.0 ads.example.com # advertising\n"),
|
||||
sliceInput("||ads.example.net^\n@@||allow.example.net^\n/re[0-9]+/\n"),
|
||||
// The three exception shapes: the two accepted spellings with the one
|
||||
// tolerated modifier, and a form that stays unsupported.
|
||||
sliceInput("@@||a.example.net^$important\n@@||b.example.net\n@@c.example.net\n"),
|
||||
sliceInput("*.wild.example.org\nlocalhost\nAdS.Example.COM.\n"),
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
//! Fuzz target for the regex engine (`src/filter/regex.zig`, milestone-21
|
||||
//! ruling 10).
|
||||
//!
|
||||
//! The contract: any byte string is a legal pattern, so `compile` may reject it
|
||||
//! however it likes but must return — never panic, never loop forever, never
|
||||
//! read out of bounds. Where it returns a program the target then checks what
|
||||
//! the filter is entitled to rely on:
|
||||
//!
|
||||
//! - the program obeys ruling 5's limits: it is non-empty, at most
|
||||
//! `max_program_len` instructions, and it came from a pattern of at most
|
||||
//! `max_pattern_len` bytes;
|
||||
//! - `matches` terminates on any input, and the VM's step count never exceeds
|
||||
//! program length × (input length + 1), which is the linearity claim the
|
||||
//! whole design rests on;
|
||||
//! - compile-then-match is deterministic: the same pattern compiled twice
|
||||
//! gives the same program length and the same verdict on the same input,
|
||||
//! and two runs of one program agree step for step.
|
||||
//!
|
||||
//! `regex.zig` imports only `std`, so this target's module roots directly at
|
||||
//! that file — no aggregator needed.
|
||||
//!
|
||||
//! Runner semantics: under a plain `zig build test` the target runs once per
|
||||
//! corpus entry plus once on empty input, which makes the corpus a regression
|
||||
//! suite. `zig build test --fuzz=<n>` gives it `n` generated inputs.
|
||||
|
||||
const std = @import("std");
|
||||
const regex = @import("regex");
|
||||
const smith_encode = @import("smith_encode.zig");
|
||||
|
||||
const sliceInput = smith_encode.sliceInput;
|
||||
const pairInput = smith_encode.pairInput;
|
||||
const Smith = std.testing.Smith;
|
||||
|
||||
/// Twice `regex.max_pattern_len`, so `error.PatternTooLong` is reachable rather
|
||||
/// than the only thing the target ever sees.
|
||||
const max_pattern = 2 * regex.max_pattern_len;
|
||||
|
||||
/// Past the 253 bytes of the longest text name, which is the longest input the
|
||||
/// filter ever hands the engine.
|
||||
const max_name = 512;
|
||||
|
||||
/// `Smith` entity ids: the pattern and the name it is matched against.
|
||||
const pattern_hash: u32 = 1;
|
||||
const name_hash: u32 = 2;
|
||||
|
||||
const fuzz_options: std.testing.FuzzInputOptions = .{ .corpus = &corpus };
|
||||
|
||||
test "fuzz regex.compile and regex.matches" {
|
||||
try std.testing.fuzz({}, regexTarget, fuzz_options);
|
||||
}
|
||||
|
||||
fn regexTarget(_: void, smith: *Smith) anyerror!void {
|
||||
var pattern_buf: [max_pattern]u8 = undefined;
|
||||
var name_buf: [max_name]u8 = undefined;
|
||||
const pattern = pattern_buf[0..smith.sliceWithHash(&pattern_buf, pattern_hash)];
|
||||
const input = name_buf[0..smith.sliceWithHash(&name_buf, name_hash)];
|
||||
|
||||
var prog = (try compileOnce(pattern)) orelse return;
|
||||
defer prog.deinit(std.testing.allocator);
|
||||
|
||||
// Ruling 5's limits, read off the program the compiler agreed to build.
|
||||
try std.testing.expect(pattern.len <= regex.max_pattern_len);
|
||||
try std.testing.expect(prog.insts.len > 0);
|
||||
try std.testing.expect(prog.insts.len <= regex.max_program_len);
|
||||
|
||||
const first = regex.run(&prog, input);
|
||||
try expectLinear(first, prog.insts.len, input.len);
|
||||
try std.testing.expectEqual(first.matched, regex.matches(&prog, input));
|
||||
|
||||
// The empty name is the cheapest way to reach the position-zero closure with
|
||||
// no consuming step behind it, so every pattern is run against it too.
|
||||
try expectLinear(regex.run(&prog, ""), prog.insts.len, 0);
|
||||
|
||||
const again = regex.run(&prog, input);
|
||||
try std.testing.expectEqual(first.matched, again.matched);
|
||||
try std.testing.expectEqual(first.steps, again.steps);
|
||||
|
||||
// Compiling is a pure function of the pattern bytes: the second program
|
||||
// matches the first instruction for instruction and answers the same.
|
||||
var second = (try compileOnce(pattern)) orelse return error.TestSecondCompileFailed;
|
||||
defer second.deinit(std.testing.allocator);
|
||||
try std.testing.expectEqual(prog.insts.len, second.insts.len);
|
||||
try std.testing.expectEqual(prog.classes.len, second.classes.len);
|
||||
try std.testing.expectEqual(first.matched, regex.matches(&second, input));
|
||||
}
|
||||
|
||||
/// One compile, or null when the engine rejected the pattern. Every member of
|
||||
/// `regex.Error` is a legitimate rejection: unparsable syntax, a pattern past
|
||||
/// the byte limit, a program past the instruction limit, and an allocator that
|
||||
/// ran out.
|
||||
fn compileOnce(pattern: []const u8) anyerror!?regex.Program {
|
||||
return regex.compile(std.testing.allocator, pattern) catch |err| switch (err) {
|
||||
error.OutOfMemory,
|
||||
error.BadPattern,
|
||||
error.PatternTooLong,
|
||||
error.PatternTooComplex,
|
||||
=> null,
|
||||
};
|
||||
}
|
||||
|
||||
fn expectLinear(result: regex.Run, program_len: usize, input_len: usize) !void {
|
||||
try std.testing.expect(result.steps <= program_len * (input_len + 1));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// corpus
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// `Smith` does not consume a corpus entry as raw input, so every entry below
|
||||
// goes through the `smith_encode.zig` encoders. An entry that carries only the
|
||||
// pattern leaves the name empty, which is the position-zero closure on its own.
|
||||
|
||||
/// The backtracker killers: exponential for a backtracking engine, linear here.
|
||||
const nested_plus = "(a+)+b";
|
||||
const nested_alternation = "^(a|aa)+$";
|
||||
const nested_star = "^(a*)*(b*)*$";
|
||||
|
||||
/// An epsilon cycle: the loop body consumes nothing, so only the VM's
|
||||
/// one-admission-per-position rule ends the walk.
|
||||
const empty_loop = "^((a*)*)*$";
|
||||
|
||||
/// A name long enough that a quadratic step count would show against the bound.
|
||||
const long_name = "a" ** 252 ++ "X";
|
||||
|
||||
/// Emits nothing at all, so only the emitter's visit budget ends the compile.
|
||||
const empty_body_blowup = "((((x{0}){900}){900}){900}){900}";
|
||||
|
||||
const corpus = [_][]const u8{
|
||||
pairInput(nested_plus, "a" ** 20 ++ "X"),
|
||||
pairInput(nested_alternation, long_name),
|
||||
pairInput(nested_star, long_name),
|
||||
pairInput(empty_loop, long_name),
|
||||
pairInput("^ad[0-9]+-", "ad42-serve.example.com"),
|
||||
pairInput("^(ads|track)\\.example\\.(com|net)$", "track.example.net"),
|
||||
pairInput("[^.]+\\.doubleclick\\.net$", "static.doubleclick.net"),
|
||||
pairInput("^\\w{1,8}\\.\\d{2}\\.example$", "ads_42.13.example"),
|
||||
// Every rejection path, so the corpus replays them rather than waiting on a
|
||||
// discovery: bad syntax, an over-long pattern, an over-large program, and a
|
||||
// compile that only the visit budget stops.
|
||||
sliceInput("(a"),
|
||||
sliceInput("[z-a]"),
|
||||
sliceInput("a{2,1}"),
|
||||
sliceInput("ads\\"),
|
||||
sliceInput("(?:ab)"),
|
||||
sliceInput("a+?"),
|
||||
sliceInput("a" ** (regex.max_pattern_len + 1)),
|
||||
sliceInput("(abcd){400}"),
|
||||
sliceInput(empty_body_blowup),
|
||||
};
|
||||
+40
-7
@@ -10,8 +10,9 @@
|
||||
//! What each suite measures:
|
||||
//! - `filter`: `matcher.normalize` + `Snapshot.evaluate` per op — the handler's
|
||||
//! filtering work — against a snapshot built from `--domains` generated exact
|
||||
//! entries plus a small wildcard body. Query mix cycles hit, miss and
|
||||
//! parent-walk. Target p95 < 1 ms; VmRSS < 100 MiB with the list loaded.
|
||||
//! entries, a small wildcard body and `bench_regex_rules` operator regex
|
||||
//! rules. Query mix cycles hit, miss and parent-walk. Target p95 < 1 ms;
|
||||
//! VmRSS < 100 MiB with the list loaded.
|
||||
//! - `cache`: `buildKey` + `DnsCache.get` + `packet.setId` — the handler's
|
||||
//! cache-hit path, TTL aging included — on a 10k-entry cache prefilled with a
|
||||
//! realistic response. Query mix alternates hit and miss. Target p95 < 5 ms.
|
||||
@@ -38,6 +39,11 @@ const rss_target_bytes: usize = 100 * 1024 * 1024;
|
||||
|
||||
const cache_entries: u32 = 10_000;
|
||||
|
||||
/// Operator regex rules the filter snapshot carries. A household writes a
|
||||
/// handful; 32 is the pessimistic end of plausible, and `max_regex_per_group`
|
||||
/// allows eight times as many.
|
||||
const bench_regex_rules: usize = 32;
|
||||
|
||||
/// Byte-for-byte copy of `response` in tests/fuzz/corpus.zig (a copy on
|
||||
/// purpose, same as the corpus itself: a bench input that changes whenever a
|
||||
/// test fixture is edited is a benchmark that silently shifts). A CNAME to
|
||||
@@ -147,6 +153,23 @@ fn runFilter(io: std.Io, gpa: Allocator, opts: Options, w: *Writer) !u32 {
|
||||
try body.appendSlice(gpa, text);
|
||||
}
|
||||
|
||||
// None of these matches the generated query mix, which is the expensive
|
||||
// case rather than the cheap one: the regex levels sit below every hash and
|
||||
// wildcard level, so a query that no regex matches is the query that runs
|
||||
// all of them to their end. Every op in this suite pays that.
|
||||
var patterns: std.ArrayList([]u8) = .empty;
|
||||
defer {
|
||||
for (patterns.items) |pattern| gpa.free(pattern);
|
||||
patterns.deinit(gpa);
|
||||
}
|
||||
var regex_rules: [bench_regex_rules]model.Rule = undefined;
|
||||
for (®ex_rules, 0..) |*row, i| {
|
||||
const pattern = try std.fmt.allocPrint(gpa, "^r{d}-[0-9]+\\.(ads|track)\\.invalid$", .{i});
|
||||
errdefer gpa.free(pattern);
|
||||
try patterns.append(gpa, pattern);
|
||||
row.* = .{ .group = "default", .pattern = pattern, .kind = .regex, .action = .block };
|
||||
}
|
||||
|
||||
const sources = [_]model.BlocklistSource{.{ .url = "bench://list", .name = "bench" }};
|
||||
const links = [_]model.GroupSource{.{ .group = "default", .source_url = "bench://list" }};
|
||||
var snapshot = try matcher.Snapshot.build(gpa, .{
|
||||
@@ -155,7 +178,7 @@ fn runFilter(io: std.Io, gpa: Allocator, opts: Options, w: *Writer) !u32 {
|
||||
.group_sources = &links,
|
||||
.sources = &sources,
|
||||
.source_ids = &.{1},
|
||||
.rules = &.{},
|
||||
.rules = ®ex_rules,
|
||||
.clients = &.{},
|
||||
.prefixes = &.{},
|
||||
.compiled = &.{.{ .list_body = body.items, .wild_body = wild_body }},
|
||||
@@ -205,9 +228,10 @@ fn runFilter(io: std.Io, gpa: Allocator, opts: Options, w: *Writer) !u32 {
|
||||
const pct = percentiles(samples);
|
||||
const rss = vmRssBytes(io);
|
||||
try printRow(w, "filter", opts.iters, pct);
|
||||
try w.print(" blocked {d}/{d}, Snapshot.memoryBytes {d:.1} MiB, VmRSS {d:.1} MiB\n", .{
|
||||
blocked, opts.iters, mib(snapshot.memoryBytes()), mib(rss),
|
||||
});
|
||||
try w.print(
|
||||
" blocked {d}/{d}, {d} regex rules, Snapshot.memoryBytes {d:.1} MiB, VmRSS {d:.1} MiB\n",
|
||||
.{ blocked, opts.iters, regex_rules.len, mib(snapshot.memoryBytes()), mib(rss) },
|
||||
);
|
||||
|
||||
var exceeded: u32 = 0;
|
||||
exceeded += try printTarget(w, "p95 < 1ms", pct.p95 < filter_p95_target_ns);
|
||||
@@ -287,11 +311,20 @@ fn runCompile(io: std.Io, gpa: Allocator, opts: Options, w: *Writer) !void {
|
||||
var reader = std.Io.Reader.fixed(body.items);
|
||||
var list_buf: [4096]u8 = undefined;
|
||||
var wild_buf: [4096]u8 = undefined;
|
||||
var allow_buf: [4096]u8 = undefined;
|
||||
var list_out: Writer.Discarding = .init(&list_buf);
|
||||
var wild_out: Writer.Discarding = .init(&wild_buf);
|
||||
var allow_out: Writer.Discarding = .init(&allow_buf);
|
||||
|
||||
const t0 = std.Io.Clock.awake.now(io);
|
||||
const result = compiler.compile(gpa, &reader, .hosts, &list_out.writer, &wild_out.writer) catch |err| {
|
||||
const result = compiler.compile(
|
||||
gpa,
|
||||
&reader,
|
||||
.hosts,
|
||||
&list_out.writer,
|
||||
&wild_out.writer,
|
||||
&allow_out.writer,
|
||||
) catch |err| {
|
||||
std.process.fatal("compiler.compile failed: {t}", .{err});
|
||||
};
|
||||
const t1 = std.Io.Clock.awake.now(io);
|
||||
|
||||
@@ -0,0 +1,997 @@
|
||||
//! The container acceptance gate for `.gitea/workflows/gates.yml`, and the one
|
||||
//! place that reads the version out of `build.zig.zon` for that workflow.
|
||||
//!
|
||||
//! It exists for the reason `tools/release.zig` exists (milestone-14 deviation
|
||||
//! 24): decision-bearing logic that only lives inside a YAML `run:` block cannot
|
||||
//! be type-checked, run on a laptop, or covered by a test. What moved in here
|
||||
//! was a `sed` parse of `build.zig.zon` duplicated across two jobs, the naming
|
||||
//! of this run's docker objects, the image-contents assertion, and a smoke test
|
||||
//! whose retry loop advertised a 30-second budget while actually allowing up to
|
||||
//! 210 seconds of connect timeouts.
|
||||
//!
|
||||
//! The workflow keeps what is genuinely the runner's: the job graph, SHA-pinned
|
||||
//! actions, artifact upload and download, and an `always()` cleanup backstop for
|
||||
//! the case where this program never runs at all.
|
||||
//!
|
||||
//! Usage:
|
||||
//!
|
||||
//! container_check version read build.zig.zon into $GITHUB_OUTPUT/$GITHUB_ENV
|
||||
//! container_check gate build the image and run the full acceptance
|
||||
//!
|
||||
//! Configuration comes from the environment, never from arguments, on the same
|
||||
//! grounds as release.zig: `argv` is world-readable through `/proc`.
|
||||
//!
|
||||
//! ## Why the plumbing below is a copy of release.zig's and not a shared module
|
||||
//!
|
||||
//! release.zig is deliberately one self-contained file, and so is this one. A
|
||||
//! shared `tools/ci.zig` would be the right move at three callers; at two it
|
||||
//! buys a coupling between the release path and the gate path that neither
|
||||
//! wants. Revisit when a third tool appears.
|
||||
|
||||
const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
const Io = std.Io;
|
||||
|
||||
const max_input_bytes = 1 << 30;
|
||||
|
||||
/// The members of the image that must equal the packaged copies. Ruling 6: the
|
||||
/// binary in the image must be the binary in the tarball. Ruling 3: distributing
|
||||
/// the image is distribution, so /LICENSE and /THIRD-PARTY-NOTICES must be in it
|
||||
/// and must be the same files the tarball carries — that is an acceptance
|
||||
/// criterion and, before this gate, nothing checked it. Comparing against the
|
||||
/// staged payload rather than merely asserting the paths exist costs nothing and
|
||||
/// catches a stale or empty copy.
|
||||
const image_members = [_][]const u8{ "nxdns", "LICENSE", "THIRD-PARTY-NOTICES" };
|
||||
|
||||
/// Native triple only: this gate builds a single-architecture image. release.yml
|
||||
/// covers both platforms against the pushed multi-arch index.
|
||||
const native_triple = "x86_64-linux-musl";
|
||||
|
||||
/// Leaked objects have to be discoverable by something other than a random name,
|
||||
/// because the name only exists in a `$GITHUB_ENV` file that dies with the
|
||||
/// runner pod. The label is written onto both the image and the container.
|
||||
const ownership_label = "net.mial.nxdns.ci";
|
||||
|
||||
/// The health probe's absolute wall-clock budget. The shell loop this replaces
|
||||
/// claimed 30 seconds and meant "30 iterations of up to three 2-second connect
|
||||
/// timeouts plus a 1-second sleep", which is a real ceiling near 210 seconds. A
|
||||
/// deadline is the honest shape: the container either answers within a minute of
|
||||
/// being started or the gate has found something.
|
||||
const probe_budget_ns: u64 = 60 * std.time.ns_per_s;
|
||||
|
||||
const probe_interval: Io.Clock.Duration = .{ .raw = .fromSeconds(1), .clock = .awake };
|
||||
|
||||
/// The ceiling on one curl attempt; see `attemptSeconds`. Ten seconds is long
|
||||
/// enough that a merely slow first response is not mistaken for a wedge, and
|
||||
/// short enough that the budget still buys several attempts.
|
||||
const max_attempt_seconds: u64 = 10;
|
||||
|
||||
/// The OCI `created` label value. Fixed rather than the current time: this image
|
||||
/// is never published, and a timestamp would be the only thing that changes
|
||||
/// between two builds of the same commit.
|
||||
const created_label = "1970-01-01T00:00:00Z";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Context
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Ctx = struct {
|
||||
arena: Allocator,
|
||||
gpa: Allocator,
|
||||
io: Io,
|
||||
env: *std.process.Environ.Map,
|
||||
out: *Io.Writer,
|
||||
failures: usize = 0,
|
||||
|
||||
fn pass(ctx: *Ctx, comptime check: []const u8, comptime template: []const u8, args: anytype) void {
|
||||
ctx.out.print("container-check: PASS " ++ check ++ ": " ++ template ++ "\n", args) catch {};
|
||||
ctx.out.flush() catch {};
|
||||
}
|
||||
|
||||
fn note(ctx: *Ctx, comptime template: []const u8, args: anytype) void {
|
||||
ctx.out.print("container-check: " ++ template ++ "\n", args) catch {};
|
||||
ctx.out.flush() catch {};
|
||||
}
|
||||
|
||||
/// Records a failure and keeps going. Used where reporting every member of a
|
||||
/// comparison is more useful than stopping at the first mismatch.
|
||||
fn soft(ctx: *Ctx, comptime check: []const u8, comptime template: []const u8, args: anytype) void {
|
||||
ctx.failures += 1;
|
||||
ctx.out.print("container-check: FAIL " ++ check ++ ": " ++ template ++ "\n", args) catch {};
|
||||
ctx.out.flush() catch {};
|
||||
}
|
||||
|
||||
/// Names the check and exits. Nothing that holds a live container calls
|
||||
/// this: `std.process.exit` does not run `defer`, so the cleanup paths
|
||||
/// unwind through `error.CheckFailed` instead.
|
||||
fn fatal(ctx: *Ctx, comptime check: []const u8, comptime template: []const u8, args: anytype) noreturn {
|
||||
ctx.out.print("container-check: FAIL " ++ check ++ ": " ++ template ++ "\n", args) catch {};
|
||||
ctx.out.flush() catch {};
|
||||
std.process.exit(1);
|
||||
}
|
||||
|
||||
fn get(ctx: *Ctx, name: []const u8) []const u8 {
|
||||
return ctx.env.get(name) orelse "";
|
||||
}
|
||||
|
||||
fn require(ctx: *Ctx, name: []const u8) []const u8 {
|
||||
const value = ctx.get(name);
|
||||
if (value.len == 0) ctx.fatal("environment", "{s} is empty or unset", .{name});
|
||||
return value;
|
||||
}
|
||||
|
||||
fn fmt(ctx: *Ctx, comptime template: []const u8, args: anytype) []const u8 {
|
||||
return std.fmt.allocPrint(ctx.arena, template, args) catch @panic("OOM");
|
||||
}
|
||||
};
|
||||
|
||||
/// A phase that already reported why it failed.
|
||||
const CheckFailed = error.CheckFailed;
|
||||
|
||||
pub fn main(init: std.process.Init) !u8 {
|
||||
const arena = init.arena.allocator();
|
||||
const argv = try init.minimal.args.toSlice(arena);
|
||||
|
||||
var out_buffer: [8192]u8 = undefined;
|
||||
var out = Io.File.stdout().writerStreaming(init.io, &out_buffer);
|
||||
|
||||
var ctx: Ctx = .{
|
||||
.arena = arena,
|
||||
.gpa = init.gpa,
|
||||
.io = init.io,
|
||||
.env = init.environ_map,
|
||||
.out = &out.interface,
|
||||
};
|
||||
|
||||
if (argv.len < 2) std.process.fatal("usage: container_check <subcommand>; see tools/container_check.zig", .{});
|
||||
const command = argv[1];
|
||||
|
||||
const result = dispatch(&ctx, command);
|
||||
ctx.out.flush() catch {};
|
||||
result catch |err| switch (err) {
|
||||
error.CheckFailed => return 1,
|
||||
else => return err,
|
||||
};
|
||||
return if (ctx.failures == 0) 0 else 1;
|
||||
}
|
||||
|
||||
fn dispatch(ctx: *Ctx, command: []const u8) !void {
|
||||
if (std.mem.eql(u8, command, "version")) return version(ctx);
|
||||
if (std.mem.eql(u8, command, "gate")) return gate(ctx);
|
||||
std.process.fatal("unknown subcommand '{s}'; see tools/container_check.zig", .{command});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pure helpers. Everything below this line that can be tested without a docker
|
||||
// daemon is tested at the foot of this file.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The suffix that separates this run's docker objects from every other run's.
|
||||
///
|
||||
/// Image tags, container names and published host ports are all daemon-global.
|
||||
/// gates.yml is called by both ci.yml and release.yml, and the self-hosted
|
||||
/// runners share one docker daemon, so the fixed `nxdns:ci` and `nxdns-smoke`
|
||||
/// this replaced made two concurrent runs collide: the second `docker create
|
||||
/// --name` fails outright, and worse, whichever run finishes first deletes the
|
||||
/// other's container mid-test. The run id alone is not enough either, because
|
||||
/// two *repositories* on the same daemon can issue the same one — hence the
|
||||
/// random bytes.
|
||||
fn objectSuffix(arena: Allocator, run_id: []const u8, attempt: []const u8, random_hex: []const u8) []const u8 {
|
||||
const id = if (run_id.len != 0) run_id else "0";
|
||||
const try_number = if (attempt.len != 0) attempt else "1";
|
||||
return std.fmt.allocPrint(arena, "{s}-{s}-{s}", .{ id, try_number, random_hex }) catch @panic("OOM");
|
||||
}
|
||||
|
||||
/// `<repo>/<run_id>/<attempt>`. Label only: the repository component never
|
||||
/// enters a docker object NAME, so nothing here needs sanitising for the
|
||||
/// stricter name grammar.
|
||||
fn labelValue(arena: Allocator, repository: []const u8, run_id: []const u8, attempt: []const u8) []const u8 {
|
||||
const id = if (run_id.len != 0) run_id else "0";
|
||||
const try_number = if (attempt.len != 0) attempt else "1";
|
||||
return std.fmt.allocPrint(arena, "{s}/{s}/{s}", .{ repository, id, try_number }) catch @panic("OOM");
|
||||
}
|
||||
|
||||
/// The staged directory the image is hashed against, as the package job's
|
||||
/// upload laid it out under `zig-out/dist`.
|
||||
fn stagePath(arena: Allocator, version_text: []const u8) []const u8 {
|
||||
return std.fmt.allocPrint(
|
||||
arena,
|
||||
"zig-out/dist/stage/nxdns-{s}-{s}",
|
||||
.{ version_text, native_triple },
|
||||
) catch @panic("OOM");
|
||||
}
|
||||
|
||||
/// The network names in a `docker inspect <id>` payload, which is a JSON array
|
||||
/// of one object. An empty result means this program is not running inside a
|
||||
/// container the daemon knows about, which is the host-runner topology.
|
||||
fn inspectNetworks(arena: Allocator, payload: []const u8) []const []const u8 {
|
||||
var list: std.ArrayList([]const u8) = .empty;
|
||||
const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena, payload, .{}) catch return list.items;
|
||||
const items = switch (parsed) {
|
||||
.array => |array| array.items,
|
||||
else => return list.items,
|
||||
};
|
||||
for (items) |item| {
|
||||
if (item != .object) continue;
|
||||
const settings = item.object.get("NetworkSettings") orelse continue;
|
||||
if (settings != .object) continue;
|
||||
const networks = settings.object.get("Networks") orelse continue;
|
||||
if (networks != .object) continue;
|
||||
var it = networks.object.iterator();
|
||||
while (it.next()) |entry| list.append(arena, entry.key_ptr.*) catch @panic("OOM");
|
||||
}
|
||||
return list.items;
|
||||
}
|
||||
|
||||
/// The container's address on the first network it joined. Diagnostic only —
|
||||
/// see `probeUrl` for why it is never a passing route.
|
||||
fn inspectAddress(arena: Allocator, payload: []const u8) []const u8 {
|
||||
const parsed = std.json.parseFromSliceLeaky(std.json.Value, arena, payload, .{}) catch return "";
|
||||
const items = switch (parsed) {
|
||||
.array => |array| array.items,
|
||||
else => return "",
|
||||
};
|
||||
for (items) |item| {
|
||||
if (item != .object) continue;
|
||||
const settings = item.object.get("NetworkSettings") orelse continue;
|
||||
if (settings != .object) continue;
|
||||
const networks = settings.object.get("Networks") orelse continue;
|
||||
if (networks != .object) continue;
|
||||
var it = networks.object.iterator();
|
||||
while (it.next()) |entry| {
|
||||
if (entry.value_ptr.* != .object) continue;
|
||||
const address = entry.value_ptr.object.get("IPAddress") orelse continue;
|
||||
if (address == .string and address.string.len != 0) return address.string;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/// `docker port <name> 8080/tcp` prints one `host:port` line per binding. The
|
||||
/// host half can be an IPv6 literal, so the port is what follows the LAST colon.
|
||||
fn parsePublishedPort(output: []const u8) []const u8 {
|
||||
var lines = std.mem.splitScalar(u8, output, '\n');
|
||||
while (lines.next()) |raw| {
|
||||
const line = std.mem.trim(u8, raw, " \t\r");
|
||||
if (line.len == 0) continue;
|
||||
const at = std.mem.lastIndexOfScalar(u8, line, ':') orelse continue;
|
||||
const port = line[at + 1 ..];
|
||||
if (port.len == 0) continue;
|
||||
return port;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/// The `--max-time` a single curl attempt gets, in whole seconds.
|
||||
///
|
||||
/// `--connect-timeout` bounds only the TCP connect. A handler that accepts the
|
||||
/// connection and then wedges — a deadlocked writer, a listener up before the
|
||||
/// database is — holds curl open forever, and the loop's own deadline never gets
|
||||
/// to run, because it is only consulted between attempts. So each attempt is
|
||||
/// capped at the lesser of `max_attempt_seconds` and whatever is left of the
|
||||
/// budget, and never below one second: a zero would mean "no limit" to curl,
|
||||
/// which is the exact failure being defended against.
|
||||
fn attemptSeconds(started_ns: i96, now_ns: i96, budget_ns: u64) u64 {
|
||||
const spent: u128 = if (now_ns <= started_ns) 0 else @intCast(now_ns - started_ns);
|
||||
const left: u128 = if (spent >= budget_ns) 0 else budget_ns - spent;
|
||||
const left_seconds: u64 = @intCast(left / std.time.ns_per_s);
|
||||
return @max(1, @min(max_attempt_seconds, left_seconds));
|
||||
}
|
||||
|
||||
/// Which of several joined networks the smoke container should join.
|
||||
///
|
||||
/// The runner's per-job network is always user-defined, so the three built-in
|
||||
/// names can never be it. Picking one of them would produce a container whose
|
||||
/// name does not resolve — docker's embedded DNS serves user-defined networks
|
||||
/// only — and the failure would read as "the daemon never came up" rather than
|
||||
/// as a wrong network. Returns null when nothing qualifies, which leaves the
|
||||
/// caller with the first entry and a note in the log.
|
||||
fn preferredNetwork(networks: []const []const u8) ?[]const u8 {
|
||||
const builtin = [_][]const u8{ "bridge", "host", "none" };
|
||||
for (networks) |candidate| {
|
||||
var is_builtin = false;
|
||||
for (builtin) |name| {
|
||||
if (std.mem.eql(u8, candidate, name)) is_builtin = true;
|
||||
}
|
||||
if (!is_builtin) return candidate;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn deadlineExpired(started_ns: i96, now_ns: i96, budget_ns: u64) bool {
|
||||
if (now_ns <= started_ns) return false;
|
||||
return @as(u128, @intCast(now_ns - started_ns)) >= budget_ns;
|
||||
}
|
||||
|
||||
fn elapsedSeconds(started_ns: i96, now_ns: i96) f64 {
|
||||
if (now_ns <= started_ns) return 0;
|
||||
const delta: f64 = @floatFromInt(@as(i64, @intCast(now_ns - started_ns)));
|
||||
return delta / @as(f64, std.time.ns_per_s);
|
||||
}
|
||||
|
||||
/// The version field of `build.zig.zon`, parsed exactly as
|
||||
/// `tools/verify_dist.zig`'s `checkZonVersion` parses it. Two readers of one
|
||||
/// file must not disagree about what it says, and the `sed` expression this
|
||||
/// replaces disagreed with the zon grammar in every case involving a comment.
|
||||
fn parseZonVersion(arena: Allocator, source: [:0]const u8) ![]const u8 {
|
||||
const Manifest = struct { version: []const u8 };
|
||||
const manifest = try std.zon.parse.fromSliceAlloc(Manifest, arena, source, null, .{
|
||||
.ignore_unknown_fields = true,
|
||||
.free_on_error = false,
|
||||
});
|
||||
return manifest.version;
|
||||
}
|
||||
|
||||
fn sha256Hex(bytes: []const u8) [64]u8 {
|
||||
var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined;
|
||||
std.crypto.hash.sha2.Sha256.hash(bytes, &digest, .{});
|
||||
return std.fmt.bytesToHex(digest, .lower);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Process and file plumbing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const Run = struct {
|
||||
code: u8,
|
||||
stdout: []const u8,
|
||||
stderr: []const u8,
|
||||
|
||||
fn ok(run: Run) bool {
|
||||
return run.code == 0;
|
||||
}
|
||||
|
||||
fn combined(run: Run, arena: Allocator) []const u8 {
|
||||
return std.mem.concat(arena, u8, &.{ run.stdout, run.stderr }) catch @panic("OOM");
|
||||
}
|
||||
|
||||
fn trimmedStdout(run: Run) []const u8 {
|
||||
return std.mem.trim(u8, run.stdout, " \t\r\n");
|
||||
}
|
||||
};
|
||||
|
||||
const RunOptions = struct {
|
||||
/// Extra environment for the child only. `DOCKER_BUILDKIT` travels this way
|
||||
/// so no sibling process inherits it.
|
||||
env: []const [2][]const u8 = &.{},
|
||||
cwd: ?[]const u8 = null,
|
||||
};
|
||||
|
||||
fn runCommand(ctx: *Ctx, argv: []const []const u8, options: RunOptions) !Run {
|
||||
var child_env: ?std.process.Environ.Map = null;
|
||||
defer if (child_env) |*map| map.deinit();
|
||||
if (options.env.len != 0) {
|
||||
var map = try ctx.env.clone(ctx.gpa);
|
||||
for (options.env) |pair| try map.put(pair[0], pair[1]);
|
||||
child_env = map;
|
||||
}
|
||||
|
||||
const cwd: std.process.Child.Cwd = if (options.cwd) |path| .{ .path = path } else .inherit;
|
||||
|
||||
const result = try std.process.run(ctx.gpa, ctx.io, .{
|
||||
.argv = argv,
|
||||
.cwd = cwd,
|
||||
.environ_map = if (child_env) |*map| map else null,
|
||||
.stdout_limit = .limited(max_input_bytes),
|
||||
.stderr_limit = .limited(max_input_bytes),
|
||||
});
|
||||
defer ctx.gpa.free(result.stdout);
|
||||
defer ctx.gpa.free(result.stderr);
|
||||
|
||||
return .{
|
||||
.code = termCode(result.term),
|
||||
.stdout = try ctx.arena.dupe(u8, result.stdout),
|
||||
.stderr = try ctx.arena.dupe(u8, result.stderr),
|
||||
};
|
||||
}
|
||||
|
||||
fn termCode(term: std.process.Child.Term) u8 {
|
||||
return switch (term) {
|
||||
.exited => |code| code,
|
||||
else => 255,
|
||||
};
|
||||
}
|
||||
|
||||
/// Runs a command and reports its output before failing. Used wherever a
|
||||
/// non-zero exit is a gate failure rather than information.
|
||||
fn mustRun(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8, options: RunOptions) ![]const u8 {
|
||||
const run = try runCommand(ctx, argv, options);
|
||||
if (!run.ok()) {
|
||||
ctx.soft(check, "`{s}` exited {d}: {s}", .{
|
||||
argv[0], run.code, std.mem.trimEnd(u8, run.combined(ctx.arena), "\n"),
|
||||
});
|
||||
return CheckFailed;
|
||||
}
|
||||
return run.stdout;
|
||||
}
|
||||
|
||||
fn readFile(ctx: *Ctx, path: []const u8) ![]const u8 {
|
||||
return Io.Dir.cwd().readFileAlloc(ctx.io, path, ctx.arena, .limited(max_input_bytes));
|
||||
}
|
||||
|
||||
fn writeFileMode(ctx: *Ctx, path: []const u8, bytes: []const u8, mode: std.posix.mode_t) !void {
|
||||
var handle = try Io.Dir.cwd().createFile(ctx.io, path, .{});
|
||||
defer handle.close(ctx.io);
|
||||
try handle.writeStreamingAll(ctx.io, bytes);
|
||||
// After the write, not through the creation mode, which `open(2)` masks
|
||||
// with the process umask.
|
||||
try handle.setPermissions(ctx.io, .fromMode(mode));
|
||||
}
|
||||
|
||||
/// `$GITHUB_ENV` and `$GITHUB_OUTPUT` are append-only files the runner reads
|
||||
/// after the step. There is no append mode on `Io.Dir`, and both files are small.
|
||||
fn appendLine(ctx: *Ctx, env_name: []const u8, line: []const u8) !void {
|
||||
const path = ctx.get(env_name);
|
||||
if (path.len == 0) {
|
||||
ctx.note("{s} is unset; not recording `{s}`", .{ env_name, line });
|
||||
return;
|
||||
}
|
||||
const existing = Io.Dir.cwd().readFileAlloc(ctx.io, path, ctx.arena, .limited(max_input_bytes)) catch "";
|
||||
const separator: []const u8 = if (existing.len == 0 or existing[existing.len - 1] == '\n') "" else "\n";
|
||||
const merged = try std.mem.concat(ctx.arena, u8, &.{ existing, separator, line, "\n" });
|
||||
try writeFileMode(ctx, path, merged, 0o644);
|
||||
}
|
||||
|
||||
fn runnerTemp(ctx: *Ctx) []const u8 {
|
||||
const temp = ctx.get("RUNNER_TEMP");
|
||||
return if (temp.len != 0) temp else "/tmp";
|
||||
}
|
||||
|
||||
/// A fresh directory under `RUNNER_TEMP`. The name is claimed by an exclusive
|
||||
/// create rather than by a random suffix: a collision is a retry, not a silent
|
||||
/// share.
|
||||
///
|
||||
/// `createDirPathStatus`, not `createDirPath`, because the latter has `mkdir -p`
|
||||
/// semantics — it succeeds on a directory that is already there and never
|
||||
/// reports `error.PathAlreadyExists`, which made the retry below dead code. This
|
||||
/// program then extracted the image into a stale or concurrently-held directory
|
||||
/// and deleted the whole tree on the way out.
|
||||
fn makeTempDir(ctx: *Ctx, prefix: []const u8) ![]const u8 {
|
||||
const base = runnerTemp(ctx);
|
||||
var attempt: usize = 0;
|
||||
while (attempt < 4096) : (attempt += 1) {
|
||||
const path = ctx.fmt("{s}/{s}.{s}-{d}", .{
|
||||
base, prefix, ctx.get("GITHUB_RUN_ID"), attempt,
|
||||
});
|
||||
const status = try Io.Dir.cwd().createDirPathStatus(ctx.io, path, .default_dir);
|
||||
if (status == .existed) continue;
|
||||
var dir = try Io.Dir.cwd().openDir(ctx.io, path, .{ .iterate = true });
|
||||
defer dir.close(ctx.io);
|
||||
try dir.setPermissions(ctx.io, .fromMode(0o700));
|
||||
return path;
|
||||
}
|
||||
ctx.fatal("temp-dir", "cannot create a {s}.* directory under {s}", .{ prefix, base });
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subcommands
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Ruling 2: build.zig.zon is the only place besides the tag that carries the
|
||||
/// version, and ruling 5 makes verify-dist assert the two agree. The gate builds
|
||||
/// the version the repository declares. This runs once, in the package job, and
|
||||
/// the container job receives the answer as a job output — the two `sed` blocks
|
||||
/// it replaced were two independent parses that could in principle disagree.
|
||||
fn version(ctx: *Ctx) !void {
|
||||
const source = Io.Dir.cwd().readFileAllocOptions(
|
||||
ctx.io,
|
||||
"build.zig.zon",
|
||||
ctx.arena,
|
||||
.limited(max_input_bytes),
|
||||
.of(u8),
|
||||
0,
|
||||
) catch |err| {
|
||||
ctx.soft("zon-version", "cannot read build.zig.zon: {t}", .{err});
|
||||
return CheckFailed;
|
||||
};
|
||||
const declared = parseZonVersion(ctx.arena, source) catch |err| {
|
||||
ctx.soft("zon-version", "cannot parse build.zig.zon: {t}", .{err});
|
||||
return CheckFailed;
|
||||
};
|
||||
if (declared.len == 0) {
|
||||
ctx.soft("zon-version", "build.zig.zon declares an empty version", .{});
|
||||
return CheckFailed;
|
||||
}
|
||||
|
||||
try appendLine(ctx, "GITHUB_OUTPUT", ctx.fmt("version={s}", .{declared}));
|
||||
try appendLine(ctx, "GITHUB_ENV", ctx.fmt("CI_VERSION={s}", .{declared}));
|
||||
ctx.pass("zon-version", "'{s}'", .{declared});
|
||||
}
|
||||
|
||||
/// The four bytes that keep two runs apart, from the `Io` interface's CSPRNG. A
|
||||
/// clock-seeded PRNG would defeat the purpose: two jobs starting in the same
|
||||
/// millisecond are exactly the collision these bytes exist to prevent.
|
||||
fn randomBytes(ctx: *Ctx) [4]u8 {
|
||||
var bytes: [4]u8 = undefined;
|
||||
ctx.io.random(&bytes);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
const Names = struct {
|
||||
image: []const u8,
|
||||
container: []const u8,
|
||||
label: []const u8,
|
||||
};
|
||||
|
||||
/// The identity of this run's docker objects, published to `$GITHUB_ENV` before
|
||||
/// anything is built. The order matters: the workflow's `always()` backstop can
|
||||
/// only remove what it can name, and a build that dies halfway still leaves
|
||||
/// layers behind.
|
||||
fn claimNames(ctx: *Ctx) !Names {
|
||||
const random_hex = std.fmt.bytesToHex(randomBytes(ctx), .lower);
|
||||
|
||||
const suffix = objectSuffix(
|
||||
ctx.arena,
|
||||
ctx.get("GITHUB_RUN_ID"),
|
||||
ctx.get("GITHUB_RUN_ATTEMPT"),
|
||||
&random_hex,
|
||||
);
|
||||
const names: Names = .{
|
||||
.image = ctx.fmt("nxdns:ci-{s}", .{suffix}),
|
||||
.container = ctx.fmt("nxdns-smoke-{s}", .{suffix}),
|
||||
.label = labelValue(
|
||||
ctx.arena,
|
||||
ctx.require("GITHUB_REPOSITORY"),
|
||||
ctx.get("GITHUB_RUN_ID"),
|
||||
ctx.get("GITHUB_RUN_ATTEMPT"),
|
||||
),
|
||||
};
|
||||
|
||||
try appendLine(ctx, "GITHUB_ENV", ctx.fmt("CI_IMAGE={s}", .{names.image}));
|
||||
try appendLine(ctx, "GITHUB_ENV", ctx.fmt("SMOKE_NAME={s}", .{names.container}));
|
||||
ctx.note("image {s}, container {s}, label {s}={s}", .{
|
||||
names.image, names.container, ownership_label, names.label,
|
||||
});
|
||||
return names;
|
||||
}
|
||||
|
||||
fn gate(ctx: *Ctx) !void {
|
||||
const version_text = ctx.require("CI_VERSION");
|
||||
const revision = ctx.require("GITHUB_SHA");
|
||||
const names = try claimNames(ctx);
|
||||
|
||||
try buildImage(ctx, names, version_text, revision);
|
||||
defer _ = runCommand(ctx, &.{ "docker", "image", "rm", "-f", names.image }, .{}) catch {};
|
||||
|
||||
_ = try mustRun(ctx, "image-version-command", &.{ "docker", "run", "--rm", names.image, "version" }, .{});
|
||||
ctx.pass("image-version-command", "`{s} version` exited 0", .{names.image});
|
||||
|
||||
try assertContents(ctx, names, version_text);
|
||||
try smoke(ctx, names);
|
||||
}
|
||||
|
||||
/// The build args carry the OCI label values (ruling 6); release.yml passes the
|
||||
/// same three and then asserts the resulting `org.opencontainers.image.version`
|
||||
/// label. BuildKit is not optional here: the builder stage is pinned to
|
||||
/// `$BUILDPLATFORM`, which the classic builder does not define, so
|
||||
/// `DOCKER_BUILDKIT=0` fails at the first `FROM`.
|
||||
fn buildImage(ctx: *Ctx, names: Names, version_text: []const u8, revision: []const u8) !void {
|
||||
const buildkit = [_][2][]const u8{.{ "DOCKER_BUILDKIT", "1" }};
|
||||
_ = try mustRun(ctx, "image-build", &.{
|
||||
"docker", "build",
|
||||
"-t", names.image,
|
||||
"-f", "deploy/docker/Dockerfile",
|
||||
"--label", ctx.fmt("{s}={s}", .{ ownership_label, names.label }),
|
||||
"--build-arg", ctx.fmt("VERSION={s}", .{version_text}),
|
||||
"--build-arg", ctx.fmt("REVISION={s}", .{revision}),
|
||||
"--build-arg", ctx.fmt("CREATED={s}", .{created_label}),
|
||||
".",
|
||||
}, .{ .env = &buildkit });
|
||||
ctx.pass("image-build", "{s} built from deploy/docker/Dockerfile", .{names.image});
|
||||
}
|
||||
|
||||
/// See `image_members` for what this asserts and why the licence files are in
|
||||
/// it. The comparison hashes file contents in this process rather than shelling
|
||||
/// out to `sha256sum`, and it never compares modes: the artifact zip round-trip
|
||||
/// drops the executable bit, which is survivable only because the Dockerfile
|
||||
/// chmods the binary itself and `verify-dist` already asserted the archive modes
|
||||
/// on the originals in the package job.
|
||||
fn assertContents(ctx: *Ctx, names: Names, version_text: []const u8) !void {
|
||||
const stage = stagePath(ctx.arena, version_text);
|
||||
var stage_dir = Io.Dir.cwd().openDir(ctx.io, stage, .{}) catch |err| {
|
||||
ctx.soft("image-contents", "the staged payload '{s}' is missing: {t}", .{ stage, err });
|
||||
return CheckFailed;
|
||||
};
|
||||
stage_dir.close(ctx.io);
|
||||
|
||||
const out = try makeTempDir(ctx, "container-check");
|
||||
defer Io.Dir.cwd().deleteTree(ctx.io, out) catch {};
|
||||
|
||||
// A stopped container, not a running one: `docker cp` reads the filesystem
|
||||
// of an image's container without ever starting its entrypoint.
|
||||
const created = try mustRun(ctx, "image-contents", &.{ "docker", "create", names.image }, .{});
|
||||
const cid = std.mem.trim(u8, created, " \t\r\n");
|
||||
defer _ = runCommand(ctx, &.{ "docker", "rm", "-f", cid }, .{}) catch {};
|
||||
|
||||
const before = ctx.failures;
|
||||
for (image_members) |member| {
|
||||
_ = try mustRun(ctx, "image-contents", &.{
|
||||
"docker", "cp", ctx.fmt("{s}:/{s}", .{ cid, member }), ctx.fmt("{s}/{s}", .{ out, member }),
|
||||
}, .{});
|
||||
const want = sha256Hex(try readFile(ctx, ctx.fmt("{s}/{s}", .{ stage, member })));
|
||||
const got = sha256Hex(try readFile(ctx, ctx.fmt("{s}/{s}", .{ out, member })));
|
||||
if (std.mem.eql(u8, &want, &got)) {
|
||||
ctx.pass("image-contents", "/{s} matches ({s})", .{ member, &got });
|
||||
} else {
|
||||
ctx.soft("image-contents", "/{s} DIFFERS: image {s}, packaged {s}", .{ member, &got, &want });
|
||||
}
|
||||
}
|
||||
if (ctx.failures != before) {
|
||||
ctx.note("the image does not carry the artifacts this run packaged", .{});
|
||||
return CheckFailed;
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the health probe connects, and why there is exactly one answer.
|
||||
///
|
||||
/// The gates job itself runs in a container on the runner's per-job network. A
|
||||
/// published port binds on the *daemon's* host, not in here, and docker does not
|
||||
/// route between the default bridge and that network — a bridge-IP curl hangs to
|
||||
/// its connect timeout. So when this program is containerised, the smoke
|
||||
/// container joins the job's own network, where its name resolves and its port
|
||||
/// is reachable, and that name is the sole passing route. On a host runner the
|
||||
/// inspect finds no container and the published port is the sole passing route.
|
||||
///
|
||||
/// The container's own bridge address used to be a third fallback. It is now
|
||||
/// diagnostic logging only: as a passing route it made the in-container topology
|
||||
/// silently accept a probe that had actually taken the path a real deployment
|
||||
/// never takes, which is the opposite of what a gate is for.
|
||||
const Topology = union(enum) {
|
||||
in_network: []const u8,
|
||||
published,
|
||||
};
|
||||
|
||||
fn detectTopology(ctx: *Ctx) Topology {
|
||||
var host_buffer: [std.posix.HOST_NAME_MAX]u8 = undefined;
|
||||
const hostname = std.posix.gethostname(&host_buffer) catch {
|
||||
ctx.note("no hostname; assuming a host runner and probing the published port", .{});
|
||||
return .published;
|
||||
};
|
||||
const run = runCommand(ctx, &.{ "docker", "inspect", hostname }, .{}) catch |err| {
|
||||
ctx.note("could not run `docker inspect {s}` ({t}); probing the published port", .{ hostname, err });
|
||||
return .published;
|
||||
};
|
||||
if (!run.ok()) {
|
||||
ctx.note("`docker inspect {s}` found no container; this is a host runner", .{hostname});
|
||||
return .published;
|
||||
}
|
||||
const networks = inspectNetworks(ctx.arena, run.stdout);
|
||||
if (networks.len == 0) {
|
||||
ctx.note("this job's container has joined no network; probing the published port", .{});
|
||||
return .published;
|
||||
}
|
||||
const chosen = preferredNetwork(networks) orelse networks[0];
|
||||
if (networks.len > 1) {
|
||||
// Still a guess once more than one network is user-defined, and a guess
|
||||
// that silently picks the wrong one fails as an unreachable name rather
|
||||
// than as anything diagnosable. Say so while it is cheap to read.
|
||||
ctx.note("this job's container is on {d} networks; joining '{s}'", .{ networks.len, chosen });
|
||||
}
|
||||
return .{ .in_network = chosen };
|
||||
}
|
||||
|
||||
fn smoke(ctx: *Ctx, names: Names) !void {
|
||||
const topology = detectTopology(ctx);
|
||||
|
||||
// No bind mount: the runner talks to the daemon over a mounted socket, so a
|
||||
// `-v` path would resolve on the docker host (where the workspace does not
|
||||
// exist) and mount an empty directory over /etc/nxdns. `docker cp` streams
|
||||
// the fixture through the socket instead.
|
||||
//
|
||||
// `-p 127.0.0.1::8080` takes an ephemeral host port instead of a fixed
|
||||
// 18080, which two concurrent runs on this daemon cannot both bind. The
|
||||
// actual port is read back with `docker port`.
|
||||
//
|
||||
// The command and the sysctl mirror deploy/docker/compose.yaml, because that
|
||||
// is the invocation this gate exists to prove. The invocation is the sole
|
||||
// configuration authority (milestone-20 ruling 1): the image's bare `run`
|
||||
// grades the database, and a fresh /var/lib/nxdns volume holds no upstream,
|
||||
// so it exits 2 with NoUsableUpstreams before it ever binds a port.
|
||||
var argv: std.ArrayList([]const u8) = .empty;
|
||||
try argv.appendSlice(ctx.arena, &.{ "docker", "create", "--name", names.container });
|
||||
try argv.appendSlice(ctx.arena, &.{ "--label", ctx.fmt("{s}={s}", .{ ownership_label, names.label }) });
|
||||
switch (topology) {
|
||||
.in_network => |network| try argv.appendSlice(ctx.arena, &.{ "--network", network }),
|
||||
.published => {},
|
||||
}
|
||||
try argv.appendSlice(ctx.arena, &.{ "-p", "127.0.0.1::8080" });
|
||||
try argv.appendSlice(ctx.arena, &.{ "--sysctl", "net.ipv4.ip_unprivileged_port_start=0" });
|
||||
try argv.appendSlice(ctx.arena, &.{ names.image, "run", "--config=/etc/nxdns/config.zon" });
|
||||
|
||||
_ = try mustRun(ctx, "smoke", argv.items, .{});
|
||||
defer _ = runCommand(ctx, &.{ "docker", "rm", "-f", names.container }, .{}) catch {};
|
||||
|
||||
_ = try mustRun(ctx, "smoke", &.{
|
||||
"docker", "cp", "tests/fixtures/container-smoke.zon",
|
||||
ctx.fmt("{s}:/etc/nxdns/config.zon", .{names.container}),
|
||||
}, .{});
|
||||
_ = try mustRun(ctx, "smoke", &.{ "docker", "start", names.container }, .{});
|
||||
|
||||
// Before anything that assumes a live container. `docker port` fails on one
|
||||
// that has already exited, and that failure would otherwise be the whole
|
||||
// diagnosis the log gets — the container's own stderr never reaches CI.
|
||||
if (!try isRunning(ctx, names.container)) {
|
||||
ctx.soft("smoke", "the container exited during startup", .{});
|
||||
dumpLogs(ctx, names.container);
|
||||
return CheckFailed;
|
||||
}
|
||||
|
||||
const url = try probeUrl(ctx, names, topology);
|
||||
try probeHealth(ctx, names, url);
|
||||
try stopGracefully(ctx, names.container);
|
||||
}
|
||||
|
||||
fn probeUrl(ctx: *Ctx, names: Names, topology: Topology) ![]const u8 {
|
||||
const inspected = try runCommand(ctx, &.{ "docker", "inspect", names.container }, .{});
|
||||
const address = if (inspected.ok()) inspectAddress(ctx.arena, inspected.stdout) else "";
|
||||
|
||||
switch (topology) {
|
||||
.in_network => {
|
||||
ctx.note("probing by container name over the job's network (container ip {s})", .{
|
||||
if (address.len != 0) address else "none",
|
||||
});
|
||||
return ctx.fmt("http://{s}:8080/api/health", .{names.container});
|
||||
},
|
||||
.published => {
|
||||
const ports = try mustRun(ctx, "smoke", &.{ "docker", "port", names.container, "8080/tcp" }, .{});
|
||||
const port = parsePublishedPort(ports);
|
||||
if (port.len == 0) {
|
||||
ctx.soft("smoke", "8080/tcp is not published on the host: '{s}'", .{
|
||||
std.mem.trimEnd(u8, ports, "\n"),
|
||||
});
|
||||
return CheckFailed;
|
||||
}
|
||||
ctx.note("probing the published loopback port {s} (container ip {s})", .{
|
||||
port, if (address.len != 0) address else "none",
|
||||
});
|
||||
return ctx.fmt("http://127.0.0.1:{s}/api/health", .{port});
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn probeHealth(ctx: *Ctx, names: Names, url: []const u8) !void {
|
||||
// curl rather than `std.http.Client`: zig 0.16's client has no connect
|
||||
// timeout that survives a black-holed route, and a probe that can hang
|
||||
// forever defeats the deadline this loop exists to enforce.
|
||||
const started = Io.Clock.awake.now(ctx.io);
|
||||
while (true) {
|
||||
// First, so no attempt is ever launched with the budget already spent.
|
||||
// Checking after the attempt instead let a round that started with less
|
||||
// than a second left run on the one-second floor `attemptSeconds`
|
||||
// applies, and report a pass past the deadline. A budget that expires
|
||||
// during the sleep now fails, which is what "absolute deadline" means.
|
||||
const now = Io.Clock.awake.now(ctx.io);
|
||||
if (deadlineExpired(started.nanoseconds, now.nanoseconds, probe_budget_ns)) {
|
||||
ctx.soft("smoke", "no response from {s} within {d:.1}s", .{
|
||||
url, elapsedSeconds(started.nanoseconds, now.nanoseconds),
|
||||
});
|
||||
dumpLogs(ctx, names.container);
|
||||
return CheckFailed;
|
||||
}
|
||||
|
||||
if (!try isRunning(ctx, names.container)) {
|
||||
ctx.soft("smoke", "the container exited after {d:.1}s, before answering {s}", .{
|
||||
elapsedSeconds(started.nanoseconds, Io.Clock.awake.now(ctx.io).nanoseconds), url,
|
||||
});
|
||||
dumpLogs(ctx, names.container);
|
||||
return CheckFailed;
|
||||
}
|
||||
|
||||
const max_time = attemptSeconds(
|
||||
started.nanoseconds,
|
||||
Io.Clock.awake.now(ctx.io).nanoseconds,
|
||||
probe_budget_ns,
|
||||
);
|
||||
const attempt = try runCommand(ctx, &.{
|
||||
"curl", "-fsS",
|
||||
"--connect-timeout", "2",
|
||||
"--max-time", ctx.fmt("{d}", .{max_time}),
|
||||
url,
|
||||
}, .{});
|
||||
if (attempt.ok()) {
|
||||
ctx.pass("smoke", "{s} answered after {d:.1}s: {s}", .{
|
||||
url,
|
||||
elapsedSeconds(started.nanoseconds, Io.Clock.awake.now(ctx.io).nanoseconds),
|
||||
std.mem.trim(u8, attempt.stdout, " \t\r\n"),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
probe_interval.sleep(ctx.io) catch |err| {
|
||||
ctx.soft("smoke", "the probe interval was interrupted: {t}", .{err});
|
||||
return CheckFailed;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// SIGTERM has to bring the daemon down cleanly, because that is what every
|
||||
/// container runtime sends and a non-zero code there is a shutdown bug.
|
||||
fn stopGracefully(ctx: *Ctx, container: []const u8) !void {
|
||||
_ = try mustRun(ctx, "smoke-stop", &.{ "docker", "stop", "-t", "30", container }, .{});
|
||||
const inspected = try mustRun(ctx, "smoke-stop", &.{
|
||||
"docker", "inspect", "-f", "{{.State.ExitCode}}", container,
|
||||
}, .{});
|
||||
const text = std.mem.trim(u8, inspected, " \t\r\n");
|
||||
dumpLogs(ctx, container);
|
||||
const code = std.fmt.parseInt(i32, text, 10) catch {
|
||||
ctx.soft("smoke-stop", "docker reported a non-numeric exit code '{s}'", .{text});
|
||||
return CheckFailed;
|
||||
};
|
||||
if (code != 0) {
|
||||
ctx.soft("smoke-stop", "the container exited {d} after SIGTERM, not 0", .{code});
|
||||
return CheckFailed;
|
||||
}
|
||||
ctx.pass("smoke-stop", "the container exited 0 after SIGTERM", .{});
|
||||
}
|
||||
|
||||
fn isRunning(ctx: *Ctx, container: []const u8) !bool {
|
||||
const run = try runCommand(ctx, &.{
|
||||
"docker", "inspect", "-f", "{{.State.Running}}", container,
|
||||
}, .{});
|
||||
if (!run.ok()) return false;
|
||||
return std.mem.eql(u8, run.trimmedStdout(), "true");
|
||||
}
|
||||
|
||||
fn dumpLogs(ctx: *Ctx, container: []const u8) void {
|
||||
const run = runCommand(ctx, &.{ "docker", "logs", container }, .{}) catch return;
|
||||
ctx.note("docker logs {s}:\n{s}", .{
|
||||
container, std.mem.trimEnd(u8, run.combined(ctx.arena), "\n"),
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "objectSuffix carries the run identity and the random bytes" {
|
||||
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
try testing.expectEqualStrings("42-2-deadbeef", objectSuffix(arena, "42", "2", "deadbeef"));
|
||||
// A workflow_dispatch on a runner that sets neither still has to produce a
|
||||
// usable name rather than `nxdns:ci--`.
|
||||
try testing.expectEqualStrings("0-1-cafe", objectSuffix(arena, "", "", "cafe"));
|
||||
}
|
||||
|
||||
test "labelValue is repo/run/attempt" {
|
||||
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
try testing.expectEqualStrings("m5r/nxdns/42/2", labelValue(arena, "m5r/nxdns", "42", "2"));
|
||||
try testing.expectEqualStrings("m5r/nxdns/0/1", labelValue(arena, "m5r/nxdns", "", ""));
|
||||
}
|
||||
|
||||
test "stagePath names the native staged directory" {
|
||||
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
|
||||
try testing.expectEqualStrings(
|
||||
"zig-out/dist/stage/nxdns-0.0.2-x86_64-linux-musl",
|
||||
stagePath(arena_state.allocator(), "0.0.2"),
|
||||
);
|
||||
}
|
||||
|
||||
test "inspectNetworks reads none, one and several networks" {
|
||||
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
try testing.expectEqual(@as(usize, 0), inspectNetworks(arena, "[]").len);
|
||||
try testing.expectEqual(@as(usize, 0), inspectNetworks(arena, "not json").len);
|
||||
// A daemon error body is an object, not an array, and must not read as a
|
||||
// network list.
|
||||
try testing.expectEqual(@as(usize, 0), inspectNetworks(arena, "{\"message\":\"no such object\"}").len);
|
||||
|
||||
const one =
|
||||
\\[{"NetworkSettings":{"Networks":{"job-net":{"IPAddress":"172.20.0.3"}}}}]
|
||||
;
|
||||
const single = inspectNetworks(arena, one);
|
||||
try testing.expectEqual(@as(usize, 1), single.len);
|
||||
try testing.expectEqualStrings("job-net", single[0]);
|
||||
try testing.expectEqualStrings("172.20.0.3", inspectAddress(arena, one));
|
||||
|
||||
const two =
|
||||
\\[{"NetworkSettings":{"Networks":{"a":{"IPAddress":"172.20.0.3"},"b":{"IPAddress":"172.21.0.4"}}}}]
|
||||
;
|
||||
try testing.expectEqual(@as(usize, 2), inspectNetworks(arena, two).len);
|
||||
try testing.expectEqualStrings("", inspectAddress(arena, "[]"));
|
||||
}
|
||||
|
||||
test "parsePublishedPort takes the port after the last colon" {
|
||||
try testing.expectEqualStrings("49153", parsePublishedPort("127.0.0.1:49153\n"));
|
||||
// An IPv6 binding carries colons in the host half.
|
||||
try testing.expectEqualStrings("49154", parsePublishedPort(":::49154\n"));
|
||||
try testing.expectEqualStrings("49153", parsePublishedPort("127.0.0.1:49153\n:::49154\n"));
|
||||
try testing.expectEqualStrings("", parsePublishedPort(""));
|
||||
try testing.expectEqualStrings("", parsePublishedPort("\n\n"));
|
||||
}
|
||||
|
||||
test "preferredNetwork skips the built-in networks" {
|
||||
try testing.expectEqualStrings(
|
||||
"job-net",
|
||||
preferredNetwork(&.{ "bridge", "job-net" }).?,
|
||||
);
|
||||
try testing.expectEqualStrings("job-net", preferredNetwork(&.{"job-net"}).?);
|
||||
// Nothing user-defined to pick, so the caller falls back and says so.
|
||||
try testing.expect(preferredNetwork(&.{ "bridge", "host", "none" }) == null);
|
||||
try testing.expect(preferredNetwork(&.{}) == null);
|
||||
// The first user-defined name wins, not the first name.
|
||||
try testing.expectEqualStrings("a", preferredNetwork(&.{ "host", "a", "b" }).?);
|
||||
}
|
||||
|
||||
test "each curl attempt is capped by what is left of the budget" {
|
||||
const start: i96 = 1_000_000_000;
|
||||
// Early on, the ceiling rather than the budget is what binds.
|
||||
try testing.expectEqual(@as(u64, 10), attemptSeconds(start, start, probe_budget_ns));
|
||||
try testing.expectEqual(
|
||||
@as(u64, 10),
|
||||
attemptSeconds(start, start + 40 * std.time.ns_per_s, probe_budget_ns),
|
||||
);
|
||||
// Near the deadline the remaining budget binds instead, so one attempt can
|
||||
// no longer outlive the loop that is supposed to bound it.
|
||||
try testing.expectEqual(
|
||||
@as(u64, 5),
|
||||
attemptSeconds(start, start + 55 * std.time.ns_per_s, probe_budget_ns),
|
||||
);
|
||||
// Never zero: curl reads `--max-time 0` as "no limit". `probeHealth` checks
|
||||
// the deadline before it computes this, so a spent budget no longer reaches
|
||||
// the floor through that loop; the floor stays because the helper must not
|
||||
// hand curl an unbounded attempt whatever the caller does.
|
||||
try testing.expectEqual(
|
||||
@as(u64, 1),
|
||||
attemptSeconds(start, start + 60 * std.time.ns_per_s, probe_budget_ns),
|
||||
);
|
||||
try testing.expectEqual(
|
||||
@as(u64, 1),
|
||||
attemptSeconds(start, start + 600 * std.time.ns_per_s, probe_budget_ns),
|
||||
);
|
||||
// A clock that reads backwards must not shorten the attempt either.
|
||||
try testing.expectEqual(
|
||||
@as(u64, 10),
|
||||
attemptSeconds(start, start - std.time.ns_per_s, probe_budget_ns),
|
||||
);
|
||||
}
|
||||
|
||||
test "the probe deadline is absolute wall clock" {
|
||||
const start: i96 = 1_000_000_000;
|
||||
try testing.expect(!deadlineExpired(start, start, probe_budget_ns));
|
||||
try testing.expect(!deadlineExpired(start, start + 59 * std.time.ns_per_s, probe_budget_ns));
|
||||
try testing.expect(deadlineExpired(start, start + 60 * std.time.ns_per_s, probe_budget_ns));
|
||||
// A clock that reads backwards must not end the loop instantly.
|
||||
try testing.expect(!deadlineExpired(start, start - std.time.ns_per_s, probe_budget_ns));
|
||||
try testing.expectApproxEqAbs(
|
||||
@as(f64, 1.5),
|
||||
elapsedSeconds(start, start + 1_500_000_000),
|
||||
0.001,
|
||||
);
|
||||
}
|
||||
|
||||
test "parseZonVersion reads the version through the zon grammar" {
|
||||
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
const source =
|
||||
\\.{
|
||||
\\ .name = .nxdns,
|
||||
\\ // .version = "9.9.9" is a comment, and sed did not know that
|
||||
\\ .version = "0.0.2",
|
||||
\\ .minimum_zig_version = "0.16.0",
|
||||
\\ .dependencies = .{},
|
||||
\\ .paths = .{""},
|
||||
\\}
|
||||
;
|
||||
try testing.expectEqualStrings("0.0.2", try parseZonVersion(arena, source));
|
||||
try testing.expectError(error.ParseZon, parseZonVersion(arena, ".{ .name = .nxdns }"));
|
||||
}
|
||||
|
||||
test "sha256Hex matches the known empty-input digest" {
|
||||
try testing.expectEqualStrings(
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
&sha256Hex(""),
|
||||
);
|
||||
}
|
||||
+10
-6
@@ -717,8 +717,14 @@ fn runnerTemp(ctx: *Ctx) []const u8 {
|
||||
}
|
||||
|
||||
/// A fresh directory under `RUNNER_TEMP`, named so `scrub` can find it. The name
|
||||
/// is claimed by an exclusive `makeDir` rather than by a random suffix: a
|
||||
/// collision is a retry, not a silent share.
|
||||
/// is claimed by an exclusive create rather than by a random suffix: a collision
|
||||
/// is a retry, not a silent share.
|
||||
///
|
||||
/// `createDirPathStatus`, not `createDirPath`, because the latter has `mkdir -p`
|
||||
/// semantics — it succeeds on a directory that is already there and never
|
||||
/// reports `error.PathAlreadyExists`, which made the retry below dead code. This
|
||||
/// program then adopted a stale or concurrently-held directory, wrote the
|
||||
/// signing material into it, and deleted the whole tree on the way out.
|
||||
fn makeTempDir(ctx: *Ctx, prefix: []const u8) ![]const u8 {
|
||||
const base = runnerTemp(ctx);
|
||||
var attempt: usize = 0;
|
||||
@@ -726,10 +732,8 @@ fn makeTempDir(ctx: *Ctx, prefix: []const u8) ![]const u8 {
|
||||
const path = ctx.fmt("{s}/{s}.{s}-{d}", .{
|
||||
base, prefix, ctx.get("GITHUB_RUN_ID"), attempt,
|
||||
});
|
||||
Io.Dir.cwd().createDirPath(ctx.io, path) catch |err| switch (err) {
|
||||
error.PathAlreadyExists => continue,
|
||||
else => return err,
|
||||
};
|
||||
const status = try Io.Dir.cwd().createDirPathStatus(ctx.io, path, .default_dir);
|
||||
if (status == .existed) continue;
|
||||
var dir = try Io.Dir.cwd().openDir(ctx.io, path, .{ .iterate = true });
|
||||
defer dir.close(ctx.io);
|
||||
try dir.setPermissions(ctx.io, .fromMode(0o700));
|
||||
|
||||
@@ -17,7 +17,9 @@ const BLOCKLISTS = {
|
||||
last_updated: 1700000000,
|
||||
domain_count: 1000,
|
||||
wildcard_count: 10,
|
||||
exception_count: 7,
|
||||
skipped_regex_count: 3,
|
||||
skipped_unsupported_count: 21,
|
||||
checksum: "abc",
|
||||
},
|
||||
{
|
||||
@@ -29,7 +31,9 @@ const BLOCKLISTS = {
|
||||
last_updated: null,
|
||||
domain_count: 0,
|
||||
wildcard_count: 0,
|
||||
exception_count: 0,
|
||||
skipped_regex_count: 0,
|
||||
skipped_unsupported_count: 0,
|
||||
checksum: null,
|
||||
},
|
||||
],
|
||||
@@ -98,7 +102,9 @@ const SNAPSHOT = {
|
||||
last_error: "",
|
||||
domains: 1200,
|
||||
wildcards: 12,
|
||||
exceptions: 9,
|
||||
skipped_regex: 4,
|
||||
skipped_unsupported: 17,
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -112,8 +118,15 @@ test("renders the source table and the status empty state", async () => {
|
||||
expect(screen.getByText("Suggested")).toBeTruthy();
|
||||
expect(screen.getByText("1000")).toBeTruthy();
|
||||
expect(screen.getByText("10")).toBeTruthy();
|
||||
expect(screen.getByText("7")).toBeTruthy();
|
||||
expect(screen.getByText("3")).toBeTruthy();
|
||||
expect(screen.getByText("21")).toBeTruthy();
|
||||
expect(screen.getByText("never")).toBeTruthy();
|
||||
expect(screen.getByRole("columnheader", { name: "Skipped regex" })).toBeTruthy();
|
||||
expect(screen.getByRole("columnheader", { name: "Skipped unsupported" })).toBeTruthy();
|
||||
expect(
|
||||
screen.getByText(/Skipped unsupported lines are syntax nxdns cannot translate into a DNS decision/),
|
||||
).toBeTruthy();
|
||||
|
||||
const enabledToggle = screen.getByLabelText("StevenBlack enabled") as HTMLInputElement;
|
||||
expect(enabledToggle.checked).toBe(true);
|
||||
@@ -147,7 +160,9 @@ test("update now disables the button, then replaces the status section from the
|
||||
last_error: "",
|
||||
domains: 1200,
|
||||
wildcards: 12,
|
||||
exceptions: 9,
|
||||
skipped_regex: 4,
|
||||
skipped_unsupported: 17,
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
@@ -159,7 +174,9 @@ test("update now disables the button, then replaces the status section from the
|
||||
last_error: "connect timed out",
|
||||
domains: 0,
|
||||
wildcards: 0,
|
||||
exceptions: 0,
|
||||
skipped_regex: 0,
|
||||
skipped_unsupported: 0,
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -172,7 +189,10 @@ test("update now disables the button, then replaces the status section from the
|
||||
expect(screen.getByText("connect timed out")).toBeTruthy();
|
||||
expect(screen.getByText("1200")).toBeTruthy();
|
||||
expect(screen.getByText("12")).toBeTruthy();
|
||||
expect(screen.getByText("9")).toBeTruthy();
|
||||
expect(screen.getByText("4")).toBeTruthy();
|
||||
expect(screen.getByText("17")).toBeTruthy();
|
||||
expect(screen.getAllByRole("columnheader", { name: "Skipped unsupported" })).toHaveLength(2);
|
||||
expect(screen.queryByText(/run .Update now. to fetch status/)).toBeNull();
|
||||
// The store notifies one flush before the mutation's success state lands.
|
||||
await screen.findByText(/Update completed/);
|
||||
|
||||
@@ -42,6 +42,10 @@ const styles = stylex.create({
|
||||
marginTop: "1rem",
|
||||
color: colors.textMuted,
|
||||
},
|
||||
note: {
|
||||
marginTop: "0.5rem",
|
||||
color: colors.textMuted,
|
||||
},
|
||||
table: {
|
||||
width: "100%",
|
||||
minWidth: "max-content",
|
||||
@@ -153,7 +157,9 @@ export default function BlocklistsPage() {
|
||||
<th {...stylex.props(shared.th)}>Enabled</th>
|
||||
<th {...stylex.props(shared.th)}>Domains</th>
|
||||
<th {...stylex.props(shared.th)}>Wildcards</th>
|
||||
<th {...stylex.props(shared.th)}>Exceptions</th>
|
||||
<th {...stylex.props(shared.th)}>Skipped regex</th>
|
||||
<th {...stylex.props(shared.th)}>Skipped unsupported</th>
|
||||
<th {...stylex.props(shared.th)}>Last updated</th>
|
||||
<th {...stylex.props(shared.th)}>
|
||||
<span {...stylex.props(shared.srOnly)}>Actions</span>
|
||||
@@ -185,7 +191,11 @@ export default function BlocklistsPage() {
|
||||
</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>{b.domain_count}</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>{b.wildcard_count}</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>{b.exception_count}</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>{b.skipped_regex_count}</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>
|
||||
{b.skipped_unsupported_count}
|
||||
</td>
|
||||
<td {...stylex.props(shared.td)}>
|
||||
{b.last_updated === null ? "never" : formatTime(b.last_updated)}
|
||||
</td>
|
||||
@@ -219,6 +229,13 @@ export default function BlocklistsPage() {
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
<p {...stylex.props(styles.note)}>
|
||||
Both “Skipped” columns count lines nxdns read and did not take. Skipped regex lines are patterns
|
||||
nxdns accepts only from you — adopt one you trust as a regex rule. Skipped unsupported lines are
|
||||
syntax nxdns cannot translate into a DNS decision: cosmetic element hiding, browser-only
|
||||
modifiers. A skipped unsupported count that dwarfs the domain count usually means the list is
|
||||
written for a browser extension, and its DNS or hosts variant will block more here.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
<InlineError error={tableError} />
|
||||
|
||||
@@ -87,7 +87,9 @@ export default function SourceStatusSection({ sources, namesById }: SourceStatus
|
||||
<th {...stylex.props(shared.th)}>Last success</th>
|
||||
<th {...stylex.props(shared.th)}>Domains</th>
|
||||
<th {...stylex.props(shared.th)}>Wildcards</th>
|
||||
<th {...stylex.props(shared.th)}>Exceptions</th>
|
||||
<th {...stylex.props(shared.th)}>Skipped regex</th>
|
||||
<th {...stylex.props(shared.th)}>Skipped unsupported</th>
|
||||
<th {...stylex.props(shared.th)}>Last error</th>
|
||||
</tr>
|
||||
</thead>
|
||||
@@ -109,7 +111,11 @@ export default function SourceStatusSection({ sources, namesById }: SourceStatus
|
||||
<td {...stylex.props(shared.td)}>{formatAttempt(source.last_success)}</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>{source.domains}</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>{source.wildcards}</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>{source.exceptions}</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>{source.skipped_regex}</td>
|
||||
<td {...stylex.props(shared.td, shared.tabularNums)}>
|
||||
{source.skipped_unsupported}
|
||||
</td>
|
||||
<td {...stylex.props(shared.td)}>
|
||||
{source.last_error === "" ? (
|
||||
<span {...stylex.props(styles.absent)}>—</span>
|
||||
|
||||
@@ -47,9 +47,12 @@ const styles = stylex.create({
|
||||
export default function ClientEditDialog({ client, groups, onClose }: Props) {
|
||||
const queryClient = useQueryClient();
|
||||
const mutation = useMutation(clientUpdateMutation(queryClient));
|
||||
const [name, setName] = useState(client.name);
|
||||
const [groupId, setGroupId] = useState(client.group_id);
|
||||
const readOnly = useReadOnlyConfig();
|
||||
// Adopting the learned name as a typed one is the natural gesture, but only
|
||||
// where the save can land: under file authority the PUT answers 403, and the
|
||||
// file's declared name is the one that wins.
|
||||
const [name, setName] = useState(client.name === "" && !readOnly ? client.learned_name : client.name);
|
||||
const [groupId, setGroupId] = useState(client.group_id);
|
||||
|
||||
return (
|
||||
<Dialog label={`Edit client ${client.ip}`} isOpen onClose={onClose}>
|
||||
|
||||
@@ -18,6 +18,7 @@ const CLIENTS = {
|
||||
id: 1,
|
||||
ip: "192.168.1.10",
|
||||
name: "laptop",
|
||||
learned_name: "laptop-1.lan",
|
||||
group_id: 1,
|
||||
group: "default",
|
||||
hand_edited: true,
|
||||
@@ -28,6 +29,7 @@ const CLIENTS = {
|
||||
id: 2,
|
||||
ip: "192.168.1.11",
|
||||
name: "",
|
||||
learned_name: "kids-tablet.lan",
|
||||
group_id: 2,
|
||||
group: "kids",
|
||||
hand_edited: false,
|
||||
@@ -95,6 +97,25 @@ test("renders the client table with group names and one hand-edited badge", asyn
|
||||
expect(screen.getAllByRole("cell", { name: "kids" })).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("a named row shows the typed name and hides the learned one", async () => {
|
||||
await renderClientsPage(BASE);
|
||||
|
||||
expect(screen.getByText("laptop")).toBeTruthy();
|
||||
expect(screen.queryByText("laptop-1.lan")).toBeNull();
|
||||
});
|
||||
|
||||
test("an unnamed row shows the learned name with the learned affordance", async () => {
|
||||
await renderClientsPage(BASE);
|
||||
|
||||
// The cell holds the learned name followed by the tag, so the match is on
|
||||
// the containing span rather than on a bare text node.
|
||||
const learned = screen.getByText(
|
||||
(content, element) => element?.tagName === "SPAN" && content.startsWith("kids-tablet.lan"),
|
||||
);
|
||||
// The affordance is text, not colour, so a screen reader announces it too.
|
||||
expect(within(learned).getByText("learned")).toBeTruthy();
|
||||
});
|
||||
|
||||
test("shows the DNS-activity empty state when there are no clients", async () => {
|
||||
await renderClientsPage({ ...BASE, "GET /api/clients": { clients: [] } });
|
||||
|
||||
|
||||
@@ -56,6 +56,23 @@ const styles = stylex.create({
|
||||
dash: {
|
||||
color: colors.textMuted,
|
||||
},
|
||||
/**
|
||||
* A learned name is runtime state, not something the operator typed, so it
|
||||
* reads muted and carries an outlined "learned" tag. The tag is real text —
|
||||
* a screen reader announces it — because colour alone is not an affordance.
|
||||
*/
|
||||
learnedTag: {
|
||||
marginLeft: "0.5rem",
|
||||
borderWidth: 1,
|
||||
borderStyle: "solid",
|
||||
borderColor: colors.border,
|
||||
borderRadius: "0.25rem",
|
||||
paddingInline: "0.375rem",
|
||||
paddingBlock: "0.125rem",
|
||||
fontSize: "0.75rem",
|
||||
lineHeight: "1rem",
|
||||
color: colors.textMuted,
|
||||
},
|
||||
badge: {
|
||||
marginLeft: "0.5rem",
|
||||
borderRadius: "0.25rem",
|
||||
@@ -133,10 +150,15 @@ export default function ClientsPage() {
|
||||
<tr key={client.id} {...stylex.props(styles.bodyRow)}>
|
||||
<td {...stylex.props(styles.cell, shared.mono)}>{client.ip}</td>
|
||||
<td {...stylex.props(styles.cell)}>
|
||||
{client.name === "" ? (
|
||||
<span {...stylex.props(styles.dash)}>—</span>
|
||||
) : (
|
||||
{client.name !== "" ? (
|
||||
client.name
|
||||
) : client.learned_name !== "" ? (
|
||||
<span {...stylex.props(styles.dash)}>
|
||||
{client.learned_name}
|
||||
<span {...stylex.props(styles.learnedTag)}>learned</span>
|
||||
</span>
|
||||
) : (
|
||||
<span {...stylex.props(styles.dash)}>—</span>
|
||||
)}
|
||||
{client.hand_edited && <span {...stylex.props(styles.badge)}>edited</span>}
|
||||
</td>
|
||||
|
||||
@@ -34,6 +34,7 @@ const RESPONSES: Record<string, unknown> = {
|
||||
// The API orders groups by name, so the id-1 default is not always first.
|
||||
let groups: { id: number; name: string; safe_search: boolean }[];
|
||||
let deleted: string[];
|
||||
let posted: { pattern: string; kind: string }[];
|
||||
|
||||
function deleteCalls(): string[] {
|
||||
return deleted;
|
||||
@@ -45,6 +46,7 @@ beforeEach(() => {
|
||||
{ id: 2, name: "Kids", safe_search: true },
|
||||
];
|
||||
deleted = [];
|
||||
posted = [];
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
@@ -54,6 +56,7 @@ beforeEach(() => {
|
||||
return new Response(null, { status: 204 });
|
||||
}
|
||||
if (url === "/api/rules" && init?.method === "POST") {
|
||||
posted.push(JSON.parse(String(init.body)) as { pattern: string; kind: string });
|
||||
return new Response(JSON.stringify({ error: "rate limited" }), {
|
||||
status: 429,
|
||||
headers: { "content-type": "application/json", "Retry-After": "5" },
|
||||
@@ -116,11 +119,61 @@ test("renders the rule table and the create form with contract enums", async ()
|
||||
expect(table.getByText("Kids")).toBeTruthy();
|
||||
expect(screen.getAllByRole("button", { name: "Delete" })).toHaveLength(2);
|
||||
|
||||
expect(await optionsOf("Kind")).toEqual(["exact", "wildcard"]);
|
||||
expect(await optionsOf("Kind")).toEqual(["exact", "wildcard", "regex"]);
|
||||
expect(await optionsOf("Action")).toEqual(["allow", "block"]);
|
||||
expect(await optionsOf("Group")).toEqual(["Default", "Kids"]);
|
||||
});
|
||||
|
||||
test("the kind selector can select the regex option, not only list it", async () => {
|
||||
renderRulesRoute();
|
||||
await screen.findByRole("heading", { name: "Rules" });
|
||||
|
||||
fireEvent.click(trigger("Kind"));
|
||||
const options = await screen.findAllByRole("option");
|
||||
const regex = options.find((option) => option.textContent === "regex");
|
||||
expect(regex).toBeTruthy();
|
||||
fireEvent.click(regex!);
|
||||
await waitFor(() => expect(screen.queryByRole("listbox")).toBeNull());
|
||||
|
||||
expect(trigger("Kind").textContent).toContain("regex");
|
||||
});
|
||||
|
||||
async function selectKind(label: string): Promise<void> {
|
||||
fireEvent.click(trigger("Kind"));
|
||||
const options = await screen.findAllByRole("option");
|
||||
fireEvent.click(options.find((option) => option.textContent === label)!);
|
||||
await waitFor(() => expect(screen.queryByRole("listbox")).toBeNull());
|
||||
}
|
||||
|
||||
// A regex is stored and matched byte for byte, so whitespace inside it is data,
|
||||
// not slop the UI may drop. Exact and wildcard are normalized server-side.
|
||||
test("a regex pattern is posted untrimmed, an exact pattern is trimmed", async () => {
|
||||
renderRulesRoute();
|
||||
await screen.findByRole("heading", { name: "Rules" });
|
||||
|
||||
await selectKind("regex");
|
||||
fireEvent.change(screen.getByLabelText("Pattern"), { target: { value: " foo|bar " } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Create rule" }));
|
||||
await waitFor(() => expect(posted).toHaveLength(1));
|
||||
expect(posted[0]).toMatchObject({ pattern: " foo|bar ", kind: "regex" });
|
||||
|
||||
await selectKind("exact");
|
||||
fireEvent.change(screen.getByLabelText("Pattern"), { target: { value: " ads.example.net " } });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Create rule" }));
|
||||
await waitFor(() => expect(posted).toHaveLength(2));
|
||||
expect(posted[1]).toMatchObject({ pattern: "ads.example.net", kind: "exact" });
|
||||
});
|
||||
|
||||
test("the pattern field opts out of mobile autocapitalize and autocorrect", async () => {
|
||||
renderRulesRoute();
|
||||
await screen.findByRole("heading", { name: "Rules" });
|
||||
|
||||
const input = screen.getByLabelText("Pattern");
|
||||
expect(input.getAttribute("autocapitalize")).toBe("none");
|
||||
expect(input.getAttribute("autocorrect")).toBe("off");
|
||||
expect(input.getAttribute("spellcheck")).toBe("false");
|
||||
});
|
||||
|
||||
test("rule create shows a countdown when rate limited with Retry-After", async () => {
|
||||
renderRulesRoute();
|
||||
await screen.findByRole("heading", { name: "Rules" });
|
||||
|
||||
@@ -15,6 +15,7 @@ import { READ_ONLY_HINT, useReadOnlyConfig } from "@/features/settings/authority
|
||||
const KIND_OPTIONS = [
|
||||
{ value: "exact", label: "exact" },
|
||||
{ value: "wildcard", label: "wildcard" },
|
||||
{ value: "regex", label: "regex" },
|
||||
];
|
||||
|
||||
const ACTION_OPTIONS = [
|
||||
@@ -97,10 +98,12 @@ export default function RulesPage() {
|
||||
|
||||
function onSubmit(event: FormEvent<HTMLFormElement>) {
|
||||
event.preventDefault();
|
||||
create.mutate(
|
||||
{ group_id: groupId, pattern: pattern.trim(), kind, action },
|
||||
{ onSuccess: () => setPattern("") },
|
||||
);
|
||||
// A regex pattern is stored and matched byte for byte, so the UI must not
|
||||
// edit it: trimming here would make a UI-created rule differ from the same
|
||||
// bytes posted to /api/rules. Name-shaped kinds are normalized server-side,
|
||||
// so trimming them only spares a pasted space a 400.
|
||||
const sent = kind === "regex" ? pattern : pattern.trim();
|
||||
create.mutate({ group_id: groupId, pattern: sent, kind, action }, { onSuccess: () => setPattern("") });
|
||||
}
|
||||
|
||||
function confirmDelete() {
|
||||
@@ -173,7 +176,13 @@ export default function RulesPage() {
|
||||
required
|
||||
value={pattern}
|
||||
onChange={(event) => setPattern(event.target.value)}
|
||||
placeholder="ads.example.com or *.example.com"
|
||||
placeholder="ads.example.com, *.example.com or ^ad[0-9]+-"
|
||||
// A phone keyboard capitalizing the first letter is silent for
|
||||
// exact and wildcard (normalized server-side) but fatal for a
|
||||
// regex, which matches the lowercase query name byte for byte.
|
||||
autoCapitalize="none"
|
||||
autoCorrect="off"
|
||||
spellCheck={false}
|
||||
{...stylex.props(shared.input, shared.focusRing)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -87,11 +87,13 @@ export const sample_list_blocklists: { blocklists: Blocklist[] } = {
|
||||
checksum: null,
|
||||
domain_count: 0,
|
||||
enabled: false,
|
||||
exception_count: 0,
|
||||
id: 0,
|
||||
is_suggested: false,
|
||||
last_updated: null,
|
||||
name: "ads",
|
||||
skipped_regex_count: 0,
|
||||
skipped_unsupported_count: 0,
|
||||
url: "https://lists.example/ads.txt",
|
||||
wildcard_count: 0,
|
||||
},
|
||||
@@ -110,12 +112,14 @@ export const sample_update_blocklists_now: { sources: SourceStatus[] } = {
|
||||
sources: [
|
||||
{
|
||||
domains: 0,
|
||||
exceptions: 0,
|
||||
id: 0,
|
||||
last_attempt: 0,
|
||||
last_error: "",
|
||||
last_success: 0,
|
||||
loaded: false,
|
||||
skipped_regex: 0,
|
||||
skipped_unsupported: 0,
|
||||
state: "never_fetched",
|
||||
url: "https://lists.example/ads.txt",
|
||||
wildcards: 0,
|
||||
@@ -255,6 +259,7 @@ export const sample_list_clients: { clients: Client[] } = {
|
||||
id: 0,
|
||||
ip: "192.168.1.50",
|
||||
last_seen: 0,
|
||||
learned_name: "",
|
||||
name: "laptop",
|
||||
},
|
||||
],
|
||||
@@ -268,6 +273,7 @@ export const sample_update_client: Client = {
|
||||
id: 0,
|
||||
ip: "192.168.1.50",
|
||||
last_seen: 0,
|
||||
learned_name: "",
|
||||
name: "laptop-renamed",
|
||||
};
|
||||
|
||||
|
||||
@@ -160,7 +160,9 @@ export interface Blocklist {
|
||||
last_updated: number | null;
|
||||
domain_count: number;
|
||||
wildcard_count: number;
|
||||
exception_count: number;
|
||||
skipped_regex_count: number;
|
||||
skipped_unsupported_count: number;
|
||||
checksum: string | null;
|
||||
}
|
||||
|
||||
@@ -189,10 +191,12 @@ export interface SourceStatus {
|
||||
last_error: string;
|
||||
domains: number;
|
||||
wildcards: number;
|
||||
exceptions: number;
|
||||
skipped_regex: number;
|
||||
skipped_unsupported: number;
|
||||
}
|
||||
|
||||
export type RuleKind = "exact" | "wildcard";
|
||||
export type RuleKind = "exact" | "wildcard" | "regex";
|
||||
export type RuleAction = "allow" | "block";
|
||||
|
||||
export interface Rule {
|
||||
@@ -252,6 +256,8 @@ export interface Client {
|
||||
id: number;
|
||||
ip: string;
|
||||
name: string;
|
||||
/** Learned over reverse DNS. `name` wins whenever it is non-empty. */
|
||||
learned_name: string;
|
||||
group_id: number;
|
||||
group: string;
|
||||
hand_edited: boolean;
|
||||
|
||||
Reference in New Issue
Block a user