From 16c9de24149b316f20b92903e704805d5ad98fdb Mon Sep 17 00:00:00 2001 From: m5r Date: Sun, 2 Aug 2026 18:02:58 +0200 Subject: [PATCH] milestone 13: restructure docs to diataxis, tutorial, every command executed --- PLAN.md | 4 +- README.md | 22 +- docs/README.md | 72 +++ docs/architecture.md | 182 -------- docs/docs.zig | 6 +- docs/explanation/architecture.md | 264 +++++++++++ docs/explanation/configuration-model.md | 178 ++++++++ docs/explanation/performance-and-testing.md | 161 +++++++ docs/how-to/back-up-and-restore.md | 192 ++++++++ docs/how-to/enable-doh-and-dot.md | 296 +++++++++++++ docs/how-to/install-with-docker.md | 170 +++++++ docs/how-to/install-with-systemd.md | 262 +++++++++++ docs/how-to/measure-performance.md | 163 +++++++ docs/how-to/set-up-admin-authentication.md | 268 +++++++++++ docs/how-to/troubleshoot.md | 295 ++++++++++++ docs/how-to/upgrade.md | 214 +++++++++ docs/operator.md | 418 ------------------ docs/performance.md | 81 ---- docs/{ => reference}/api.md | 150 ++++--- docs/reference/cli.md | 150 +++++++ .../configuration.md} | 275 ++++++------ docs/reference/files-and-directories.md | 117 +++++ docs/reference/performance.md | 69 +++ docs/tutorial/first-run.md | 300 +++++++++++++ specs/milestone-13.md | 183 ++++++++ src/docs_drift_test.zig | 36 +- 26 files changed, 3627 insertions(+), 901 deletions(-) create mode 100644 docs/README.md delete mode 100644 docs/architecture.md create mode 100644 docs/explanation/architecture.md create mode 100644 docs/explanation/configuration-model.md create mode 100644 docs/explanation/performance-and-testing.md create mode 100644 docs/how-to/back-up-and-restore.md create mode 100644 docs/how-to/enable-doh-and-dot.md create mode 100644 docs/how-to/install-with-docker.md create mode 100644 docs/how-to/install-with-systemd.md create mode 100644 docs/how-to/measure-performance.md create mode 100644 docs/how-to/set-up-admin-authentication.md create mode 100644 docs/how-to/troubleshoot.md create mode 100644 docs/how-to/upgrade.md delete mode 100644 docs/operator.md delete mode 100644 docs/performance.md rename docs/{ => reference}/api.md (62%) create mode 100644 docs/reference/cli.md rename docs/{config-reference.md => reference/configuration.md} (56%) create mode 100644 docs/reference/files-and-directories.md create mode 100644 docs/reference/performance.md create mode 100644 docs/tutorial/first-run.md create mode 100644 specs/milestone-13.md diff --git a/PLAN.md b/PLAN.md index f279cd4..f74249d 100644 --- a/PLAN.md +++ b/PLAN.md @@ -233,7 +233,7 @@ src/ web/ # Vite + React + TS + Tailwind + TanStack vendor/ # sqlite3 amalgamation, mbedtls (pinned) -docs/ # operator/ architecture/ config-reference/ api/ +docs/ # tutorial/ how-to/ reference/ explanation/ (Diátaxis) tests/ # dns/ integration/ fuzz/ ``` @@ -550,7 +550,7 @@ Scalars in `settings(key, value)`; ordered/structured items in dedicated tables. ### 13.2 OpenAPI -Hand-maintained `openapi.yaml`, served + rendered into `docs/api/`. CI contract tests spin up a seeded server, hit every documented endpoint, and validate bodies/status/auth against the schema. Drift fails CI. +Hand-maintained `openapi.yaml`, served at `GET /api/openapi.yaml` and mirrored by hand in `docs/reference/api.md`. No renderer is vendored. Two drift guards fail the build instead: `src/web/openapi.zig` asserts every served route appears in the spec, and `src/docs_drift_test.zig` asserts every served route has its own table row in the reference page. --- diff --git a/README.md b/README.md index 9f4bdc9..42f5f45 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,9 @@ DNS is on port 53, the web UI on . The config file seeds the database on first boot only; from then on the database is the truth and changes go through the UI, the API, or `nxdns export` / `nxdns import`. Full install instructions, including the systemd path and -the Pi 5 recipe, are in [docs/operator.md](docs/operator.md). +the Pi 5 recipe, are in +[docs/how-to/install-with-systemd.md](docs/how-to/install-with-systemd.md) and +[docs/how-to/install-with-docker.md](docs/how-to/install-with-docker.md). ## Building from source @@ -62,11 +64,19 @@ zig build test --summary all # unit tests ## Documentation -- [docs/operator.md](docs/operator.md) — install, configure, back up, - upgrade, troubleshoot -- [docs/architecture.md](docs/architecture.md) — module map and design -- [docs/config-reference.md](docs/config-reference.md) — every +Start at [docs/README.md](docs/README.md), which splits the documentation +into a tutorial, how-to guides, reference and explanation. + +- [docs/tutorial/first-run.md](docs/tutorial/first-run.md) — build it, resolve + a name, block a domain, on a scratch directory +- [docs/how-to/install-with-systemd.md](docs/how-to/install-with-systemd.md) — + a real install, including the Raspberry Pi 5 +- [docs/reference/configuration.md](docs/reference/configuration.md) — every configuration field -- [docs/api.md](docs/api.md) — REST API, auth and SSE +- [docs/reference/api.md](docs/reference/api.md) — REST API, auth and SSE +- [docs/reference/cli.md](docs/reference/cli.md) — subcommands, flags and exit + codes +- [docs/explanation/architecture.md](docs/explanation/architecture.md) — module + map and design - [PLAN.md](PLAN.md) and [specs/](specs/) — scope, design decisions and per-milestone contracts diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..9bf4172 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,72 @@ +# nxdns documentation + +The pages are split by what you are trying to do, following +[Diátaxis](https://diataxis.fr/). Each page serves one of four purposes, and +knowing which one you want is the fastest way to the right page. + +| Mode | For | Read it when | +| --- | --- | --- | +| [Tutorial](#tutorial) | Someone who has never run nxdns | You want to learn what it does by making it work once | +| [How-to guides](#how-to-guides) | An operator with a job to do | You know what you want and need the steps | +| [Reference](#reference) | Anyone who needs an exact answer | You want a field, a flag, a route or an exit code | +| [Explanation](#explanation) | Anyone deciding or debugging | You want to know why it works the way it does | + +## Tutorial + +A lesson, not a procedure: one path with one outcome, on a scratch directory you +can delete afterwards. + +- [tutorial/first-run.md](tutorial/first-run.md) — build nxdns, resolve a name, + block a domain from a real blocklist, open the web interface, stop cleanly. + +## How-to guides + +Steps for a goal you already have. They assume you know what nxdns is. + +- [how-to/install-with-systemd.md](how-to/install-with-systemd.md) — a real + install as a system service, including the Raspberry Pi 5 aarch64 binary. +- [how-to/install-with-docker.md](how-to/install-with-docker.md) — the container + image and compose file. +- [how-to/upgrade.md](how-to/upgrade.md) — move to a new build without losing + state. +- [how-to/troubleshoot.md](how-to/troubleshoot.md) — what to do when it does not + answer, does not block, or will not start. +- [how-to/enable-doh-and-dot.md](how-to/enable-doh-and-dot.md) — serve encrypted + DNS with certificates. +- [how-to/set-up-admin-authentication.md](how-to/set-up-admin-authentication.md) + — put a password on the web interface and the API. +- [how-to/back-up-and-restore.md](how-to/back-up-and-restore.md) — export and + import the configuration, and what to copy. +- [how-to/measure-performance.md](how-to/measure-performance.md) — run the + benchmark harness on your own hardware. + +## Reference + +Descriptions of what is there. No procedures, no advice. + +- [reference/configuration.md](reference/configuration.md) — every + configuration section, field, default and range. +- [reference/api.md](reference/api.md) — every REST route, authentication and + the event stream. +- [reference/cli.md](reference/cli.md) — the six subcommands, every flag, every + exit code. +- [reference/files-and-directories.md](reference/files-and-directories.md) — the + data directory layout and file modes. +- [reference/performance.md](reference/performance.md) — the targets and the + measured numbers. + +## Explanation + +Background. Nothing here is needed to operate nxdns; it is here so the decisions +are inspectable. + +- [explanation/architecture.md](explanation/architecture.md) — the module map + and the design it comes from. +- [explanation/configuration-model.md](explanation/configuration-model.md) — why + the file seeds the database once and the database is the truth afterwards. +- [explanation/performance-and-testing.md](explanation/performance-and-testing.md) + — why the targets exist, why CI does not gate on them, and what the hermetic + tests do and do not prove. + +Scope and per-milestone contracts live outside this directory, in +[../PLAN.md](../PLAN.md) and [../specs/](../specs/). diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index b321ced..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,182 +0,0 @@ -# Architecture - -nxdns is a self-hosted DNS sinkhole for a household LAN: one static Zig binary -that answers DNS on UDP/TCP 53 (optionally DoH and DoT), filters against -blocklists, and serves an embedded admin SPA over HTTP. This document is a map -of the source tree and the few design rules that hold everywhere. - -## Module map - -Top-level files: - -| File | Role | -|---|---| -| `src/main.zig` | Process shell: writers, argv, dispatch, exit code. | -| `src/cli.zig` | Every command body (`run`, `check`, `export`, `import`, `version`, `help`); takes its writers as parameters so tests capture output without a process. | -| `src/app.zig` | The composition root: everything `nxdns run` owns, built in order. Nothing else constructs a collaborator. | -| `src/version.zig` | Build-time version strings. | -| `src/tests.zig` | Test root; imports each file directly. | - -Directories: - -| Directory | Role | -|---|---| -| `src/dns/` | Pure DNS wire format: header, names, questions, records, whole packets, EDNS(0)/ECS (`edns.zig`), enums and limits (`types.zig`). No allocation, no `std.Io` beyond writing to a caller's writer. | -| `src/filter/` | Blocklist pipeline: line parsers (hosts, domains, ABP), the compiler that turns a downloaded list into `.list`/`.wild` bodies, `domain_set.zig` (exact-match set, no Bloom filter), `matcher.zig` (the immutable snapshot every query evaluates against), per-group `rules.zig`, `wildcard.zig`, `safesearch.zig`, blocked-response synthesis (`response.zig`). Two I/O edges live here too: `fetcher.zig` (HTTP download) and `manager.zig` (files + DB + snapshot swap). | -| `src/local/` | Local DNS records and conditional forward zones: immutable lookup tables built once from DB rows (`records.zig`, `forward_zones.zig`), plus the plain UDP/TCP client for LAN resolvers (`forward_client.zig`). | -| `src/cache/` | `dns_cache.zig`: bounded in-memory TTL cache of whole response messages, keyed by the question. The clock arrives as a parameter. | -| `src/upstream/` | Upstream resolution: shared vocabulary and the `Client` interface (`transport.zig`), DoH client (RFC 8484), DoT client (RFC 7858), per-endpoint health/backoff (`health.zig`), and `pool.zig` — priority-ordered sequential failover that is itself a `transport.Client`, so the handler sees one interface. | -| `src/server/` | The serving side: UDP/TCP/DoH/DoT listeners, `handler.zig` (the whole query pipeline), `cert_store.zig` (refcounted TLS cert holder), `rate_limiter.zig`, `pause.zig`, `clients.zig` (client auto-materialisation), `local_tables.zig` (published local-answer tables), `query_sink.zig` (log/SSE fanout), `shutdown.zig` (SIGINT/SIGTERM → one `std.Io.Event`). | -| `src/storage/` | SQLite ownership: `db.zig` is the only file that calls SQLite, `config_schema.zig` + `migrations.zig` for `config.db`, `querylog_schema.zig` (open-or-recreate), async query `logger.zig`, `retention.zig`, `disk_monitor.zig`, and one repository per table under `repositories/`. | -| `src/config/` | The one configuration model (`model.zig`), the pure validator (`validate.zig`), `import.zig`/`export.zig` (ZON ⇄ `config.db`, byte-stable round trip), `bootstrap.zig` (first-start seeding — a policy wrapper over import). | -| `src/web/` | The admin HTTP layer: `server.zig` (listener), `router.zig`/`routes.zig`, one file per resource under `handlers/`, `auth.zig` (sessions), `sse.zig` (live query fanout), `static.zig` (embedded SPA), `metrics.zig` (Prometheus), `openapi.zig` (served contract), `api_limiter.zig`, `http_util.zig`. | -| `src/platform/` | OS and TLS edges: IP address values, the `std.log` sink (`logging.zig`), `statfs.zig` (free-space query via libc), client TLS over `std.crypto.tls` (`tls_client.zig`), server TLS over vendored Mbed TLS (`tls_server.zig`). | - -The SPA source lives in `web/` at the repo root; the build embeds its `dist/` -output as the `web_assets` module (`-Dweb-dist`). - -``` -main.zig ── cli.zig ── app.zig (composition root) - │ injects std.Io + collaborators - ┌──────────────────────┴───────────────────────┐ - │ server/ web/ upstream/ storage/ │ I/O edge - │ platform/ config/{import,export,bootstrap} │ - ├──────────────────────────────────────────────┤ - │ dns/ filter/* local/* cache/ │ pure core: - │ config/{model,validate} │ bytes in, bytes out - └──────────────────────────────────────────────┘ - * except filter/{fetcher,manager}.zig and local/forward_client.zig, - which are those directories' named I/O edges -``` - -## The purity rule - -`dns/`, `filter/`, `local/` and `cache/` take bytes and return bytes: no -`std.Io`, no sockets, no clocks hidden inside (AGENTS.md). Anything that needs -a timestamp takes it as a parameter — the cache, the rate limiter and the -pause flag all work this way, so every decision is testable without a backend. -The exceptions are deliberate and few: `filter/fetcher.zig` downloads lists, -`filter/manager.zig` owns the compiled files, the DB columns and the snapshot -swap, and `local/forward_client.zig` speaks UDP/TCP to a LAN resolver. The -decision path a query takes through these directories allocates nothing and -opens nothing. - -## std.Io injection - -There is one `std.Io` in the process. `main` receives it through -`std.process.Init` — on the standard start path this is the Threaded backend -(`std.Io.Threaded`, constructed in the stdlib's start code) — and hands it to -`cli.Runner`, from which `app.zig` threads it into every collaborator as a -parameter. No module constructs its own event loop or reads an ambient clock; -tests build their own `std.Io.Threaded` instance and pass it the same way. -The one deliberate exception is `storage/db.zig`: SQLite performs its own file -I/O through its VFS, so that file takes no `std.Io` at all. - -## Life of one query - -The pipeline in `src/server/handler.zig` (its order is PLAN §4; the stages -below are the code's actual call chain — `Handler.handle` then `Context.run`): - -``` -UDP/53 TCP/53 DoH DoT (src/server/{udp,tcp,doh,dot}_server.zig) - └──────┴──────┴────┘ - │ raw query bytes, listener-owned buffers - ▼ - handler.handle - ├─ header parse (too short / QR set → counted drop) - ├─ rate limit (over budget → REFUSED) - ├─ packet + EDNS validation (FORMERR / NOTIMP) - ├─ client tracking, group lookup (snapshot.groupForClient) - │ - ├─ local records ────────────────► authoritative answer - ├─ forward zones ─► cache ─► LAN resolver ─► answer - │ - └─ upstream path - ├─ filter snapshot evaluate ─► blocked? synthesized block reply - ├─ safe-search rewrite (per group) - ├─ cache get ─► hit? answer - ├─ upstream pool: priority failover across DoH/DoT endpoints - ├─ CNAME uncloak: walk the answer's chain, re-evaluate each target - └─ cache put - ▼ - reply bytes ─► listener sends - │ - └─► QuerySink ─► SSE hub (GET /api/queries/live) - └─► async logger ─► querylog.db -``` - -Local records win over forward zones, and both win over filtering: a name -nxdns answers itself never reaches a blocklist. Pause suspends filtering only; -local records, forward zones, cache, upstream and the query log keep running. -`handle` returns no error union — every failure is either a DNS response the -client can act on or a counted drop. The query path never waits on the -database: `QuerySink` copies the entry, the SSE hub gets it first, and one -writer task owns the `querylog.db` handle behind an `std.Io.Queue`. - -## Storage - -Two databases with opposite contracts: - -- **`config.db` is the truth.** Schema DDL is carried verbatim by - `migrations.zig` as step 1; a schema change is a new migration step, applied - inside one transaction. `nxdns import` replaces its whole content atomically - (`BEGIN IMMEDIATE`; a failed import changes nothing), `nxdns export` renders - it back as canonical ZON, byte-identical across round trips. A config file - seeds the DB exactly once at first start (`config/bootstrap.zig`); the DB is - truth thereafter. -- **`querylog.db` is expendable.** It is never migrated: its schema carries a - fingerprint derived from the DDL text, and a mismatch at open replaces the - file (`storage/querylog_schema.zig`). Retention deletes old rows daily and - periodically rewrites the file; `config.db` is walled off from that churn. - -## Web stack - -`web/server.zig` runs one `std.http.Server` per connection over its own accept -loop, with fixed pre-allocated connection slots, optionally behind TLS. The -SPA is embedded at build time: `static.zig` serves the `web_assets` module — -bytes, content type, strong ETag, and a pre-compressed `.gz` sibling where it -paid off — via a linear scan, no filesystem at runtime. `GET /api/queries/live` -is server-sent events over chunked transfer, fed by the same `QuerySink` the -logger reads. Routing is a flat table (`routes.zig`) matched linearly; the -OpenAPI YAML is hand-written, embedded and served at `GET /api/openapi.yaml`, -kept honest by tests that assert every served route appears in it. - -Auth (`web/auth.zig`): the operator's password is verified against an argon2id -PHC string (`web.password_hash` — the plaintext is hashed on import and never -stored). A successful login mints a 256-bit token carried in a cookie; the -in-memory session table holds only SHA-256 digests of tokens, compared in -constant time, capped at 32 sessions with LRU eviction. Nothing is persisted — -a restart logs everyone out. Monitoring endpoints (health, version, metrics), -the served OpenAPI contract and login itself are unauthenticated; everything -else requires the cookie, and -the API has its own token-bucket rate limiter. - -## DoH, DoT and certificate hot-reload - -`server/doh_server.zig` (RFC 8484 over HTTP/1.1 + TLS) and -`server/dot_server.zig` (RFC 7858) mirror the plain listeners' shape. Server -TLS terminates in Mbed TLS (`platform/tls_server.zig`), exposing plaintext as -`std.Io.Reader`/`std.Io.Writer`. - -Certificates hot-reload through `server/cert_store.zig`: one refcounted -`CertStore` per endpoint owns the published TLS context generation; listeners -`acquire` it per connection and `release` it when the connection ends, so a -reload never frees a context mid-handshake. Reload publishes nothing on -failure — both PEM files are read and a whole new context built before -anything swaps, and any failure leaves the old generation serving. A watcher -polls mtime+size of both files every 30 s; `POST /api/certs/reload` triggers -the same path on demand and reports the per-endpoint outcome as its payload. - -## Failure visibility - -Every failure mode must be visible, and the surface is counters, not log -lines (AGENTS.md). The handler counts every outcome — drops, FORMERR, -SERVFAIL, blocked, truncated, cache hits, paused and unfiltered queries — in -atomics; listeners count dropped datagrams instead of queueing them -unboundedly. `GET /metrics` renders all of it as Prometheus text 0.0.4, and -`GET /api/health` rolls it up for a monitor (always 200: "degraded" is a fact -about the box, not a failed request). The disk monitor classifies free space -against thresholds and gates non-essential writes; the query logger holds its -batches while writes are disallowed. `std.log` is reserved for failures -nobody else records, with upstream-error deduplication so a flapping resolver -cannot fill a disk. diff --git a/docs/docs.zig b/docs/docs.zig index f340b58..5e8180f 100644 --- a/docs/docs.zig +++ b/docs/docs.zig @@ -2,6 +2,6 @@ //! `docs_files` anonymous import (test builds only) — @embedFile paths resolve //! relative to this file. -pub const api_md = @embedFile("api.md"); -pub const config_reference_md = @embedFile("config-reference.md"); -pub const operator_md = @embedFile("operator.md"); +pub const reference_api_md = @embedFile("reference/api.md"); +pub const reference_configuration_md = @embedFile("reference/configuration.md"); +pub const reference_cli_md = @embedFile("reference/cli.md"); diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md new file mode 100644 index 0000000..909dfd9 --- /dev/null +++ b/docs/explanation/architecture.md @@ -0,0 +1,264 @@ +# Architecture + +nxdns is a self-hosted DNS sinkhole for a household LAN: one static Zig binary +that answers DNS on UDP/TCP 53 (optionally DoH and DoT), filters against +blocklists, and serves an embedded admin SPA over HTTP. This page maps the +source tree and explains the few design rules that hold everywhere, and why +they are the rules. + +For what the configuration fields, API routes and CLI flags actually are, see +[reference/configuration.md](../reference/configuration.md), +[reference/api.md](../reference/api.md) and +[reference/cli.md](../reference/cli.md). This page does not repeat them. + +## Module map + +Top-level files: + +| File | Role | +|---|---| +| `src/main.zig` | Process shell: writers, argv, dispatch, exit code. | +| `src/cli.zig` | Every command body (`run`, `check`, `export`, `import`, `version`, `help`); takes its writers as parameters so tests capture output without a process. | +| `src/app.zig` | The composition root: everything `nxdns run` owns, built in order. Nothing else constructs a collaborator. | +| `src/version.zig` | Build-time version strings. | +| `src/tests.zig` | Test root; imports each file directly. | +| `src/docs_drift_test.zig` | Guards that keep the reference pages in step with the code. | + +Directories: + +| Directory | Role | +|---|---| +| `src/dns/` | Pure DNS wire format: header, names, questions, records, whole packets, EDNS(0)/ECS (`edns.zig`), enums and limits (`types.zig`). No allocation, no `std.Io` beyond writing to a caller's writer. | +| `src/filter/` | Blocklist pipeline: line parsers (hosts, domains, ABP), the compiler that turns a downloaded list into `.list`/`.wild` bodies, `domain_set.zig` (exact-match set, no Bloom filter), `matcher.zig` (the immutable snapshot every query evaluates against), per-group `rules.zig`, `wildcard.zig`, `safesearch.zig`, blocked-response synthesis (`response.zig`). Two I/O edges live here too: `fetcher.zig` (HTTP download) and `manager.zig` (files + DB + snapshot swap). | +| `src/local/` | Local DNS records and conditional forward zones: immutable lookup tables built once from DB rows (`records.zig`, `forward_zones.zig`), plus the plain UDP/TCP client for LAN resolvers (`forward_client.zig`). | +| `src/cache/` | `dns_cache.zig`: bounded in-memory TTL cache of whole response messages, keyed by the question. The clock arrives as a parameter. | +| `src/upstream/` | Upstream resolution: shared vocabulary and the `Client` interface (`transport.zig`), DoH client (RFC 8484), DoT client (RFC 7858), per-endpoint health and backoff (`health.zig`), and `pool.zig` — priority-ordered failover that is itself a `transport.Client`, so the handler sees one interface. | +| `src/server/` | The serving side: UDP/TCP/DoH/DoT listeners, `handler.zig` (the whole query pipeline), `cert_store.zig` (refcounted TLS cert holder), `rate_limiter.zig`, `pause.zig`, `clients.zig` (client auto-materialisation), `local_tables.zig` (published local-answer tables), `query_sink.zig` (log and SSE fanout), `shutdown.zig` (SIGINT/SIGTERM into one `std.Io.Event`). | +| `src/storage/` | SQLite ownership: `db.zig` is the only file that calls SQLite, `config_schema.zig` + `migrations.zig` for `config.db`, `querylog_schema.zig` (open-or-recreate), async query `logger.zig`, `retention.zig`, `disk_monitor.zig`, and one repository per table under `repositories/`. | +| `src/config/` | The one configuration model (`model.zig`), the pure validator (`validate.zig`), `import.zig`/`export.zig` (ZON to and from `config.db`, byte-stable round trip), `bootstrap.zig` (first-start seeding — a policy wrapper over import). | +| `src/web/` | The admin HTTP layer: `server.zig` (listener), `router.zig`/`routes.zig`, one file per resource under `handlers/`, `auth.zig` (sessions), `sse.zig` (live query fanout), `static.zig` (embedded SPA), `metrics.zig` (Prometheus), `openapi.zig` (served contract), `api_limiter.zig`, `http_util.zig`. | +| `src/platform/` | OS and TLS edges: IP address values, the `std.log` sink (`logging.zig`), `statfs.zig` (free-space query via libc), client TLS over `std.crypto.tls` (`tls_client.zig`), server TLS over vendored Mbed TLS (`tls_server.zig`). | + +The SPA source lives in `web/` at the repo root; the build embeds its `dist/` +output as the `web_assets` module (`-Dweb-dist`). + +``` +main.zig ── cli.zig ── app.zig (composition root) + │ injects std.Io + collaborators + ┌──────────────────────┴───────────────────────┐ + │ server/ web/ upstream/ storage/ │ I/O edge + │ platform/ config/{import,export,bootstrap} │ + ├──────────────────────────────────────────────┤ + │ dns/ filter/* local/* cache/ │ pure core: + │ config/{model,validate} │ bytes in, bytes out + └──────────────────────────────────────────────┘ + * except filter/{fetcher,manager}.zig and local/forward_client.zig, + which are those directories' named I/O edges +``` + +## The purity rule + +`dns/`, `filter/`, `local/` and `cache/` take bytes and return bytes: no +`std.Io`, no sockets, no clocks hidden inside (AGENTS.md). Anything that needs +a timestamp takes it as a parameter — the cache, the rate limiter and the +pause flag all work this way, so every decision is testable without a backend. + +The point is not purity for its own sake. A decision that depends on a hidden +clock or a hidden socket can only be tested by arranging the world around it; +one that takes the clock as an argument is tested by passing a number. The +whole filtering and caching pipeline can therefore be exercised in the plain, +network-free test suite, which is what makes that suite worth gating CI on. + +The exceptions are deliberate, few, and named: `filter/fetcher.zig` downloads +lists, `filter/manager.zig` owns the compiled files, the DB columns and the +snapshot swap, and `local/forward_client.zig` speaks UDP/TCP to a LAN +resolver. Those three files are the only ones under those four directories +that take a `std.Io`. The decision path a query takes through them allocates +nothing and opens nothing. + +The honest cost of the exceptions shows up in +[performance-and-testing.md](performance-and-testing.md): `fetcher.zig` is +where the one production crash came from, precisely because it is the file the +pure suite cannot reach. + +## std.Io injection + +There is one `std.Io` in the process. `main` receives it through +`std.process.Init` — on the standard start path this is the Threaded backend +(`std.Io.Threaded`, constructed in the stdlib's start code) — and hands it to +`cli.Runner`, from which `app.zig` threads it into every collaborator as a +parameter. No module constructs its own event loop or reads an ambient clock; +tests build their own `std.Io.Threaded` instance and pass it the same way. + +The one deliberate exception is `storage/db.zig`: SQLite performs its own file +I/O through its VFS, so that file takes no `std.Io` at all. Wrapping SQLite's +VFS to route through `std.Io` would be a large amount of C-boundary code to +make one dependency match a convention it does not need. + +## Life of one query + +The pipeline lives in `src/server/handler.zig` — `Handler.handle` does +validation and setup, then `Context.run` decides the answer. Its order is +PLAN §4; the stages below are the code's actual call chain: + +``` +UDP/53 TCP/53 DoH DoT (src/server/{udp,tcp,doh,dot}_server.zig) + └──────┴──────┴────┘ + │ raw query bytes, listener-owned buffers + ▼ + handler.handle + ├─ header parse (too short / QR set → counted drop) + ├─ rate limit (over budget → REFUSED) + ├─ packet + EDNS validation (FORMERR / NOTIMP, qdcount != 1 → FORMERR) + ├─ one clock read, client tracking, snapshot + local-table acquire + ├─ group lookup (snapshot.groupForClient) + │ + └─ Context.run + ├─ qclass != IN ──► upstream, unfiltered and uncached + ├─ local records ────────────────► authoritative answer + ├─ forward zones ─► cache ─► LAN resolver ─► cache put ─► answer + │ + └─ upstream path (filtering off while paused) + ├─ filter snapshot evaluate ─► blocked? synthesized block reply + ├─ safe-search rewrite (per group) + ├─ cache get ─► hit? answer + ├─ upstream pool: priority failover across DoH/DoT endpoints + ├─ CNAME uncloak: walk the answer's chain, re-evaluate each target + └─ cache put (skipped for safe-search answers) + ▼ + reply: UDP size check ─► TC=1 if over the limit ─► listener sends + │ + └─► QuerySink ─► SSE hub (GET /api/queries/live) + └─► async logger ─► querylog.db +``` + +Local records win over forward zones, and both win over filtering: a name +nxdns answers itself never reaches a blocklist. Pause suspends filtering only; +local records, forward zones, cache, upstream and the query log keep running, +which is what makes pause safe to hand to a household member. A question whose +class is not IN bypasses local answers, filtering and the cache entirely and +goes straight upstream — nxdns has no opinion about CHAOS or HESIOD names and +declines to cache answers it does not model. + +`handle` returns no error union. Every failure is either a DNS response the +client can act on or a counted drop, because there is no caller above it that +could do anything useful with a Zig error. Two consequences are worth knowing: +answers synthesized from a safe-search rewrite are never cached (the rewrite is +per group, and the cache is not), and a SERVFAIL reply short-circuits before the +query sink, so it appears in the counters but not in the query log. + +The query path never waits on the database. `QuerySink` copies the entry, the +SSE hub gets it first, and one writer task owns the `querylog.db` handle behind +an `std.Io.Queue`. A slow disk delays logging, never resolution. + +Two smaller decisions in the same spirit: the upstream pool makes a second +pass that ignores backoff, so "every endpoint is in backoff" degrades to +trying anyway rather than to a blanket SERVFAIL; and a handler that has no +filter snapshot yet answers unfiltered rather than refusing. Both prefer a +working resolver over a correct-looking failure. + +## Storage + +Two databases with opposite contracts, in one data directory (see +[reference/files-and-directories.md](../reference/files-and-directories.md)). + +**`config.db` is the truth.** Its schema is versioned: `migrations.zig` holds +an ordered list of steps, step 1 being the verbatim DDL from +`config_schema.zig`, each applied inside one transaction. `nxdns import` +replaces the whole content atomically under `BEGIN IMMEDIATE`, so a failed +import changes nothing; `nxdns export` renders it back as canonical ZON, +byte-identical across round trips. A config file seeds this database exactly +once at first start. Why it works that way is +[configuration-model.md](configuration-model.md). + +**`querylog.db` is expendable.** It is never migrated. Its schema carries a +fingerprint derived from the DDL text, and at open, a missing, corrupt, +non-database, `quick_check`-failing or fingerprint-mismatched file is moved +aside and recreated empty — the old file is kept under a new name rather than +deleted, so an operator can still look at it. Retention deletes old rows daily +and periodically rewrites the file to reclaim space. + +The split exists so that the churn of the second database can never endanger +the first. Query logs are high-volume, disposable, and the thing most likely +to be corrupted by a power cut on an SD card; configuration is small, +irreplaceable, and the thing an operator would have to reconstruct by hand. +Giving them one file would force the careful contract onto the noisy data or +the loose contract onto the valuable data. + +## Web stack + +`web/server.zig` runs one `std.http.Server` per connection over its own accept +loop, with a fixed set of pre-allocated connection slots, optionally behind +TLS. Over capacity it answers 503 rather than queueing without bound — the +admin UI is not the product, and it must not be able to starve DNS. + +The SPA is embedded at build time: `static.zig` serves the `web_assets` module +— bytes, content type, strong ETag, and a pre-compressed `.gz` sibling where +it paid off — via a linear scan with no filesystem access at runtime. (The one +exception is `nxdns run --web-dev DIR`, which serves from disk with no cache +headers, for developing the SPA against a running server.) `GET +/api/queries/live` is server-sent events over chunked transfer, fed by the +same `QuerySink` the logger reads. Routing is a flat table (`routes.zig`) +matched linearly; a few dozen routes do not justify a trie. The OpenAPI YAML +is hand-written, embedded and served at `GET /api/openapi.yaml`, kept honest +by tests that assert every served route appears in it. + +Authentication (`web/auth.zig`): the operator's password is verified against +an argon2id PHC string (`web.password_hash`; the plaintext is hashed on import +and never stored). A successful login mints a 256-bit token carried in a +cookie; the in-memory session table holds only SHA-256 digests of tokens, +compared in constant time, capped at 32 sessions with LRU eviction. Nothing is +persisted, so a restart logs everyone out — for a household LAN that is a +feature, not a gap. Unauthenticated by design: the monitoring endpoints +(health, version, metrics), the served OpenAPI contract, login itself, and the +static SPA assets, which the router hands to the SPA fallback before any auth +check. Everything else requires the cookie, and the API has its own +token-bucket rate limiter. See +[how-to/set-up-admin-authentication.md](../how-to/set-up-admin-authentication.md). + +## DoH, DoT and certificate hot-reload + +`server/doh_server.zig` (RFC 8484 over HTTP/1.1 and TLS) and +`server/dot_server.zig` (RFC 7858) mirror the plain listeners' shape. Server +TLS terminates in Mbed TLS (`platform/tls_server.zig`), exposing plaintext as +`std.Io.Reader`/`std.Io.Writer`, so the listeners above it do not know whether +they are encrypted. + +Certificates hot-reload through `server/cert_store.zig`. One refcounted +`CertStore` per endpoint owns the published TLS context generation; listeners +`acquire` it per connection and `release` it when the connection ends, so a +reload never frees a context mid-handshake. Reload publishes nothing on +failure: both PEM files are read and a whole new context is built before +anything swaps, and any failure leaves the old generation serving. A watcher +polls mtime and size of both files every 30 seconds +(`cert_store.poll_interval_s`); `POST /api/certs/reload` triggers the same +path on demand and reports the per-endpoint outcome as its payload. + +The requirement driving all of this is that a certbot renewal must not need a +restart and must not be able to break DNS. A half-swapped context or a +free-while-in-use would do exactly that, so the store is built so neither is +representable. See +[how-to/enable-doh-and-dot.md](../how-to/enable-doh-and-dot.md). + +## Failure visibility + +Every failure mode must be visible, and the surface is counters, not log lines +(AGENTS.md). Log lines are a bad primitive for this: they are unbounded, they +are only read after someone already suspects a problem, and on an SD card they +are a way to fill a disk. + +So the handler counts every outcome in atomics — drops, FORMERR, NOTIMP, +REFUSED, SERVFAIL, blocked, uncloak-blocked, truncated, cache hits, local and +forward-zone answers, safe-search rewrites, paused and unfiltered queries, and +the tracker-full condition. Listeners count dropped datagrams instead of +queueing them unboundedly. `GET /metrics` renders all of it as Prometheus text +0.0.4, and `GET /api/health` rolls it up for a monitor. Health always answers +200: "degraded" is a fact about the box, not a failed request, and a monitor +that cannot distinguish the two is worse than no monitor. + +The disk monitor classifies free space against thresholds and gates +non-essential writes; the query logger holds its batches while writes are +disallowed rather than dropping them silently or writing until the filesystem +fills. `std.log` is reserved for failures nobody else records, with +upstream-error deduplication so a flapping resolver cannot fill a disk with +identical lines. diff --git a/docs/explanation/configuration-model.md b/docs/explanation/configuration-model.md new file mode 100644 index 0000000..f3d68bb --- /dev/null +++ b/docs/explanation/configuration-model.md @@ -0,0 +1,178 @@ +# The configuration model + +nxdns is configured two ways — a ZON file and a web UI — and only one of them +can be the truth. This page explains which, and why that choice is the one +that leaves the fewest ways to lose an operator's work. + +For the fields themselves see +[reference/configuration.md](../reference/configuration.md); for the commands +and their exit codes see [reference/cli.md](../reference/cli.md). + +## The rule + +The database is the truth. The file is a seed. + +`config.db` in the data directory holds the running configuration. The ZON +file (default `/etc/nxdns/config.zon`, overridable with `--config`) is read on +`nxdns run` only while the database is still empty: if the file exists and the +database holds no configuration, it is imported. Once the import has put rows +in, the file is not opened again. + +The condition is the state of the database, not a one-shot flag. A `run` whose +seed fails — unreadable file, parse error, failed validation, a constraint +violation inside the import transaction — leaves the database empty, so the +next `run` reads the file again. That is what makes fixing a typo and starting +again work. + +`src/config/bootstrap.zig` is that policy and nothing else — a wrapper over +the same import path `nxdns import` uses. Three outcomes: + +- no file: the database is used as it is; +- file present, database empty: seed it; +- file present, database already configured: skip, without reading the file. + +A file that exists but is unreadable, unparseable or invalid fails the start, +with every problem printed. nxdns never falls back to silent defaults over a +file an operator wrote — a resolver that boots "successfully" with a +configuration nobody chose is the worst outcome available, because it looks +like it worked. + +## Why the database wins + +The alternative designs all lose data. + +If the file were the truth, the admin UI could not write. Every change would +be an SSH session and a restart, which defeats the reason the UI exists: the +household member who wants to unblock one domain is not going to edit ZON. + +If both were the truth, they would disagree. The UI writes a rule; the file +still says otherwise; the next restart either silently reverts the rule or +silently ignores the file. Both are silent, and both destroy work someone +intended to keep. There is no merge rule that fixes this, because the system +cannot know which of two conflicting statements is the newer intention. + +So the file's authority ends the moment the database has content. Editing +`config.zon` after first boot does nothing — no partial effect, no warning +that some fields took and others did not. That is a blunt rule, and it is the +point: the failure mode is "my edit did nothing", which is visible the first +time you look at the UI, rather than "my edit was applied and then quietly +undone next Tuesday". + +The emptiness check is a real query over the content tables, not a flag: the +database counts as configured when any content table has rows, or when the +default group has been altered. One consequence is worth knowing, because it +is not obvious: clients are auto-materialised when they first send a query, so +a server that has answered even one query is "configured" and will ignore a +seed file placed there afterwards. + +## The round trip + +Losing the file as an editing surface would be a real loss — text is +diffable, reviewable and easy to back up — so the file is kept as a +*rendering* of the database rather than a rival to it. That is what +`export`/`import` are for: + +``` +nxdns export → canonical ZON → edit → nxdns import --force → config.db +``` + +`nxdns export` renders the database as canonical ZON: a fixed two-line +header, every default emitted, deterministic ordering from the model's field +order and the repositories' `ORDER BY` clauses, and no timestamps or hostnames +anywhere. Runtime columns — first-seen, last-seen, per-source counters — are +absent from the configuration model on purpose, so two exports taken from a +live, busy server are identical. The round trip `export → import → export` is +byte-identical, and a test asserts it. + +Byte-stability is not cosmetic. It is what makes an exported file usable in +version control and what makes a diff of two exports mean something: any +difference is a configuration change, never noise from when the export ran. + +`nxdns import` replaces the whole database content in one transaction. Not a +merge, not a patch: delete every content table in foreign-key-safe order, then +insert what the file says. A failed import — bad syntax, failed validation, a +constraint violation halfway through — leaves the database exactly as it was, +because everything happens inside a single `BEGIN IMMEDIATE`. + +Without `--force`, import refuses a database that already has content. That +check runs *inside* the transaction, after the lock is taken, so it cannot be +raced by a concurrent write. The effect is that a plain `import` can never +clobber a configured server by accident, and clobbering it deliberately takes +one visible extra word on the command line. See +[how-to/back-up-and-restore.md](../how-to/back-up-and-restore.md). + +## One model, three surfaces + +`src/config/model.zig` defines exactly one `Config` type. The ZON parser +produces it, the database reader produces it, the validator consumes it, the +composition root consumes it, and the settings API derives its key list and +its patch struct from its type information rather than mirroring the fields by +hand. Adding a field in one place therefore cannot leave the other surfaces +behind; the alternative — a file schema, a DB schema and an API schema kept in +step by discipline — is the standard way configuration systems rot. + +The model has two shapes of field, and the difference is structural rather +than stylistic. Struct-typed fields are scalar sections (`dns`, `cache`, +`web`, …) and live in a single key/value `settings` table as +`section.field` text rows. Slice-typed fields are collections (groups, +upstreams, clients, rules, local records, forward zones, …) and each gets its +own table with foreign keys. `toSettings` and `fromSettings` are the two +halves of the scalar bridge, generated by an `inline for` over the model, so +the key list is a consequence of the type rather than a second list to +maintain. + +## Unknown keys, and why the asymmetry is deliberate + +An unknown key in the **database** is warned about and ignored. An unknown key +in the **file** is a hard error. + +They are different situations. A settings row the running binary does not +recognise is almost always an older binary reading a database written by a +newer one — a downgrade, or a rollback after a bad upgrade. Refusing to start +there would mean a downgrade bricks the config database, and the operator +would have to hand-edit SQLite to recover. Warning and ignoring means the +downgrade works, the unknown setting sits inert, and the upgrade back picks it +up again. + +A key in a file, by contrast, is something a human just typed. The likeliest +cause is a typo, and the second likeliest is a field that no longer exists. +Silently ignoring it would mean the setting the operator believes they applied +was never applied — exactly the silent-divergence failure the whole model is +built to avoid. So the ZON parser rejects unknown fields with a line and +column. + +The tolerance has a boundary worth stating plainly: it covers unknown *keys*, +not unparseable *values*. A known key whose stored text does not decode into +its type is an error, not a warning. + +## The password + +`web.password` is a write-only input. It is never a stored value. + +At import time, a non-empty `web.password` is hashed with argon2id (PHC +encoding, OWASP argon2id parameters) into `web.password_hash`, and the +plaintext field is cleared before anything is written. There is no settings +row that can hold it: the model explicitly skips `web.password` in both +directions of the settings bridge, so the plaintext has nowhere to go even by +accident. `nxdns export` always writes `.password = ""` and carries the hash +instead — which is also what makes the round trip stable, since an export that +tried to reproduce a plaintext it never had could not be byte-identical. + +Setting both `password` and `password_hash` in one file is an error rather +than a precedence rule. The two say different things about what the password +is, and picking a winner would mean the operator's other statement was +silently discarded. The file has to say one thing. + +The practical shape of a password change is therefore export, set `.password` +to the new value, clear `.password_hash`, import with `--force`. The procedure +is in +[how-to/set-up-admin-authentication.md](../how-to/set-up-admin-authentication.md). + +## What is not configuration + +Storage paths are process arguments, not configuration fields: `--data-dir`, +`--config`, `--web-dev`. They cannot live in the file, because the file is +found by way of them — a path that told you where to find the thing that told +you the path would be circular. They are also the settings a supervisor +(systemd, Docker) owns rather than the operator's policy about DNS. See +[reference/files-and-directories.md](../reference/files-and-directories.md). diff --git a/docs/explanation/performance-and-testing.md b/docs/explanation/performance-and-testing.md new file mode 100644 index 0000000..8b5ad4c --- /dev/null +++ b/docs/explanation/performance-and-testing.md @@ -0,0 +1,161 @@ +# Performance targets and what the tests prove + +Two related questions: why the performance numbers are the numbers, and why CI +does not enforce them — and then, less comfortably, what a green test suite +here does and does not tell you. + +For the targets and the measured results as data, see +[reference/performance.md](../reference/performance.md); to run the bench +yourself, [how-to/measure-performance.md](../how-to/measure-performance.md). + +## Where the targets come from + +PLAN §18 sets five: + +- sustained ≥ 100 qps on a Raspberry Pi 5; +- blocklist lookup p95 < 1 ms; +- cached response p95 < 5 ms; +- memory with ~1M blocked domains < 100 MB; +- stripped static binary < 10 MB per arch, < 15 MB with the embedded frontend. + +They are household-scale numbers, and they are deliberately unambitious. 100 +qps is far more than a house generates; the point of the target is not speed +but that a Pi 5 with an SD card never becomes the reason the internet feels +broken. The latency targets exist for the same reason: DNS sits in front of +every connection anyone makes, so the failure people notice is not throughput +but a stall. The memory target is what keeps a 1M-entry blocklist from +competing with everything else on a 4 GB board. The binary-size target is +about what a static single-binary deployment is for — if it does not fit on a +constrained box and copy over a slow link in one step, the packaging decision +has not paid for itself. + +`tools/bench.zig` (`zig build bench`) measures the three that are measurable +in-process: `filter` (normalize plus snapshot evaluate against a ~1M-entry +snapshot), `cache` (key build plus cache get plus id patch), and `compile` +(the blocklist compiler over a 1M-line body, informational — there is no §18 +target for it because no prior datapoint exists). Memory comes from +`/proc/self/status` VmRSS. The qps target is not in the harness at all: it is +end-to-end against the real binary with a DNS load generator, because a +harness number for "queries per second" would measure the harness. + +The bench is `tools/`, not `src/`, on purpose: `src/` is the shipped product, +and `src/tests.zig` aggregates everything shippable. + +## Why CI does not gate on performance + +Required CI stays deterministic (AGENTS.md). Latency assertions on shared +runners measure the runner's noisy neighbours; the same commit passes and +fails depending on what else the host is doing. A gate that flakes does not +protect anything — it trains people to re-run the job, and once re-running is +routine, a real regression gets re-run too. The flaky gate is worse than no +gate, because it also consumes the attention a real gate would need. + +So the bench defaults to informational, and `--assert` — which exits non-zero +on a missed target — exists for hardware you control. Run it on the Pi, where +the numbers describe the machine the software actually has to run on. The +x86_64 development-host numbers in +[reference/performance.md](../reference/performance.md) are a regression +baseline for the machine development happens on, not a claim about the target +platform; a Cortex-A76 is far slower and those numbers do not transfer. + +CI does gate on the one performance property that *is* deterministic: binary +size. The `cross` job strips the release binaries and asserts them under the +§18 budgets. Size is a function of the input, not of the runner's mood, so it +is exactly the kind of thing a shared runner can measure honestly. + +## What the test suite is + +The blocking CI (Gitea Actions, `.gitea/workflows/ci.yml`) runs five jobs, all +required: the Zig suite with `-Dintegration`; the same suite cross-built for +aarch64 and executed under qemu-user; the frontend (format, lint, typecheck, +121 vitest cases, build); the cross-build with the two stripped-size asserts; +and a Docker smoke run that boots the image and polls `/api/health`. + +The Zig suite has three tiers, gated by build flags: + +- **plain `zig build test`** — pure logic. No sockets, no threads, no clock + budgets. This is the tier the purity rule + ([architecture.md](architecture.md)) exists to make possible. +- **`-Dintegration`** — hermetic integration: loopback sockets, `:memory:` + databases, temp directories. Nothing leaves the host. +- **`-Dlive`** — the only tests that reach the public internet (DoH and DoT + handshakes against real resolvers). Four tests, and they run in a + manual-dispatch workflow, never on push or pull request. + +The aarch64 job runs the plain tier only. The integration tests are +multithreaded loopback TLS with wall-clock budgets, and qemu-user's slowdown +turns those budgets into a flake source — the same reasoning that keeps the +bench out of CI. What aarch64 needs to prove is portable correctness of the +DNS, filter and cache logic, and the plain tier is exactly that. + +At the time of writing, plain `zig build test` is 1175 of 1288 passing with +113 skipped and 0 failed, the skips being the integration-gated tests. +Milestone 12 recorded the other two tiers on the same tree: 1280 of 1284 with +`-Dintegration` (the 4 skips are the live-network tests) and 1159 passing +under qemu, 0 failed in each. + +## What it does not prove + +The suite is hermetic by design. That is the right default: it is fast, it is +deterministic, it can gate merges. But hermetic and correct are different +properties, and the gap has already cost this project twice. + +**The blocklist download aborted the process on first real use.** The fetcher +constructed the HTTP response reader over `transfer_buf` and then read *into* +that same buffer. `Reader.readSliceShort` starts by `@memcpy`-ing the reader's +already-buffered bytes into the caller's destination — so source and +destination were the same allocation, and Zig's `@memcpy` requires them not to +overlap. It aborts. + +The reason no test caught it is precise and instructive. The copy length is +zero whenever the reader has nothing buffered, and a zero-length `@memcpy` is +fine. Bytes only accumulate in the reader's own buffer when a read comes back +short of filling the destination and the loop goes round again — that is, when +the body arrives in more than one stream call. The loopback fixture answers +every request with one small in-memory body that lands in a single read, and +the one over-size test never streams a byte, because the fetcher refuses an +oversized `content-length` on the response head. Every test in the suite was +on the zero-length-memcpy side of the branch. The first real download — a +multi-megabyte list over TLS across the WAN, arriving in many TCP segments — +was on the other side, and took the process down. The fix (commit 35f2324) +streams the body straight into the caller's writer, so the reader's buffer is +never a destination slice, and it came with four regression tests that put a +fully-buffered reader into exactly the state the old code could not survive. + +**A stale embedded SPA bundle shipped a settings page that crashed on load, +while 121 web tests passed.** `web/dist/` is gitignored and +`-Dweb-dist=web/dist` embeds whatever bytes are sitting in that directory. The +frontend tests ran against the sources, in jsdom, and were green; the binary +carried an older build. The tests were testing something the artifact did not +contain. + +Note what these two have in common. Neither was a logic bug that a better unit +test would have caught. One lived in the seam between the pure core and its +one I/O edge; the other lived in the seam between two build systems. Hermetic +tests are constructed to exclude exactly those seams — that is what makes them +hermetic. + +## The lesson, and where it now lives + +Green hermetic tests are a floor, not a ceiling. They prove the logic is +consistent with itself. They cannot prove the program works, because the +things they deliberately exclude — real network reads, real TLS, real file +sizes, real build artifacts — are where a program meets reality. + +The response is not to make CI non-deterministic. It is to require that the +real paths get exercised by a human before work is called done. That is now +ruling 3 of `specs/milestone-13.md`: every command block in the tutorial and +the how-to pages is executed verbatim, on the host, by the session that writes +it, and a command that cannot run there is marked in the page as unverified +with the reason. Documentation written from source-reading alone is how both +of these shipped; documentation that has been run is a second, independent +test suite that exercises precisely the paths the hermetic one skips. + +Two honest gaps remain, stated so nobody has to rediscover them: + +- Nothing in the suite drives a multi-read HTTP body through the fetcher end + to end. The regression tests cover `pumpBody` directly over a pre-buffered + reader; the loopback fixture still sends one small body per connection. +- There is no freshness check on `web/dist`. CI cannot embed a stale bundle, + because the jobs that pass `-Dweb-dist` rebuild the frontend immediately + beforehand. A local build can, and will do it without a warning. diff --git a/docs/how-to/back-up-and-restore.md b/docs/how-to/back-up-and-restore.md new file mode 100644 index 0000000..899d83c --- /dev/null +++ b/docs/how-to/back-up-and-restore.md @@ -0,0 +1,192 @@ +# Back up and restore + +The configuration database is the only state worth keeping. `nxdns export` +writes it out as a ZON file and `nxdns import` writes one back. The query log is +deliberately not part of a backup: it is expendable history, and if it is +missing it gets recreated empty. + +The commands below use the scratch lab from +[enable DoH and DoT](enable-doh-and-dot.md), data directory +`/tmp/nxdns-lab/data`. On a real install drop `--data-dir` and the default +`/var/lib/nxdns` applies. + +## Back up + +One command, against a stopped or a running server: + +```sh +nxdns export --data-dir /tmp/nxdns-lab/data --out /tmp/nxdns-lab/backup.zon +``` + +``` +wrote /tmp/nxdns-lab/backup.zon +``` + +The write is a temp file plus a rename, so an interrupted export leaves no +half-written backup, and the result is mode 0600: + +```sh +stat -c '%a %n' /tmp/nxdns-lab/backup.zon +``` + +``` +600 /tmp/nxdns-lab/backup.zon +``` + +That mode is not decoration. The file carries `web.password_hash`: + +```sh +grep password /tmp/nxdns-lab/backup.zon +``` + +``` + .password = "", + .password_hash = "$argon2id$v=19$m=19456,t=2,p=1$Gh+zg9xke6BqSVOiouRbqG50+Bs8ZGcXA6oKgs7lrKg$crTNMu5OI8yKBkp31r4+Y1OUQmLiAlH/qvsIxjBQRq4", +``` + +Treat backups as secrets. `.password` is always exported as `""` — the +plaintext is never stored anywhere — so the file re-imports without anyone +knowing the password. + +Without `--out` the export goes to stdout, where the file mode is your +redirect's problem: + +```sh +nxdns export --data-dir /tmp/nxdns-lab/data | head -6 +``` + +``` +// nxdns configuration +// generated by `nxdns export` — the database is the source of truth +.{ + .upstream = .{ .read_timeout_ms = 3000, .total_timeout_ms = 5000 }, + .dns = .{ + .bind_ipv4 = "127.0.0.1", +``` + +Runtime facts are left out on purpose: client first-seen and last-seen times, +rule creation times, per-source domain counts and checksums. They are things a +running server produces, not configuration, and including them would make two +exports taken minutes apart differ. + +## Restore onto a fresh data directory + +This is the normal restore: new machine, new disk, empty data directory. No +`--force`, because there is nothing to overwrite. + +```sh +nxdns import /tmp/nxdns-lab/backup.zon --data-dir /tmp/nxdns-lab/data-restored +``` + +``` +info(migrations): config.db migrated from schema version 0 to 2 +imported /tmp/nxdns-lab/backup.zon +``` + +The migration line is expected: `import` creates and migrates the database +before writing to it. + +## Restore over an existing database + +`import` refuses a database that already has content, so a plain `import` can +never clobber a configured server by accident: + +```sh +nxdns import /tmp/nxdns-lab/backup.zon --data-dir /tmp/nxdns-lab/data +``` + +``` +import failed: DatabaseNotEmpty +``` + +That exits 2. Say `--force` when replacing is what you mean: + +```sh +nxdns import /tmp/nxdns-lab/backup.zon --force --data-dir /tmp/nxdns-lab/data +``` + +``` +imported /tmp/nxdns-lab/backup.zon +``` + +Stop the server first. `import` replaces the whole configuration underneath a +process that has already read it, and a running server will not notice. + +On a real install that is the systemd unit: + +```sh +systemctl stop nxdns +nxdns import /var/backups/nxdns-config.zon --force +systemctl start nxdns +``` + +**Not verified on this host.** Those three lines are the only commands on this +page that were not run: this machine has no installed nxdns systemd unit +(`systemctl status nxdns` answers `Unit nxdns.service could not be found.`) and +`systemctl stop`/`start` need root. The lab equivalent below was run, and it +exercises the same stop-import-start sequence. In the lab the server is a +foreground `nxdns run`, so stopping it is Ctrl-C in its own terminal: + +```sh +nxdns export --data-dir /tmp/nxdns-lab/data --out /tmp/nxdns-lab/pre-restore.zon +# Ctrl-C the `nxdns run` terminal, or `kill` its pid from another shell +nxdns import /tmp/nxdns-lab/pre-restore.zon --force --data-dir /tmp/nxdns-lab/data +nxdns run --data-dir /tmp/nxdns-lab/data --config /tmp/nxdns-lab/etc/config.zon +``` + +``` +wrote /tmp/nxdns-lab/pre-restore.zon +imported /tmp/nxdns-lab/pre-restore.zon +``` + +## Verify a backup + +The round trip is byte-stable: exporting, importing and exporting again gives +an identical file. That is the cheapest check that a backup is complete and +that it will load. + +```sh +nxdns export --data-dir /tmp/nxdns-lab/data --out /tmp/nxdns-lab/backup2.zon +diff /tmp/nxdns-lab/backup.zon /tmp/nxdns-lab/backup2.zon \ + && echo "round trip is byte-identical" +``` + +``` +wrote /tmp/nxdns-lab/backup2.zon +round trip is byte-identical +``` + +The same check works across data directories — export from the restored copy +and diff against the backup you restored from: + +```sh +nxdns export --data-dir /tmp/nxdns-lab/data-restored --out /tmp/nxdns-lab/backup3.zon +diff /tmp/nxdns-lab/backup.zon /tmp/nxdns-lab/backup3.zon \ + && echo "restored database exports identically" +``` + +``` +wrote /tmp/nxdns-lab/backup3.zon +restored database exports identically +``` + +## What is not in the backup + +Everything under the data directory other than `config.db`: + +- `querylog.db` — expendable history, recreated empty when absent. +- `blocklists/` — compiled snapshots. They are rebuilt from the sources named in + the configuration, so restoring the configuration is enough; the first refresh + after a restore downloads them again. + +The data directory layout is in +[files and directories](../reference/files-and-directories.md). + +## A backup before every upgrade + +There is no downgrade path. Schema migrations run forward automatically at +startup and before `check`, `export` and `import`; nothing walks them back. Take +an export before installing a new binary — see [upgrade](upgrade.md). + +Every command on this page was executed on this host as written, except the +`systemctl` block marked **Not verified on this host** above. diff --git a/docs/how-to/enable-doh-and-dot.md b/docs/how-to/enable-doh-and-dot.md new file mode 100644 index 0000000..bbc589b --- /dev/null +++ b/docs/how-to/enable-doh-and-dot.md @@ -0,0 +1,296 @@ +# Enable DoH and DoT + +nxdns can answer encrypted queries on two extra listeners: DNS over HTTPS +(`doh_server`) and DNS over TLS (`dot_server`). Both are off by default and both +need a certificate and a private key in PEM form. + +This page uses a scratch lab under `/tmp/nxdns-lab` so the commands run without +root and without touching a real install. On a real install the files live under +`/etc/nxdns` and the data directory is `/var/lib/nxdns`; the ports are 443 and +853 rather than the unprivileged ones below. + +Every field mentioned here is documented in +[configuration reference](../reference/configuration.md). + +## 1. Get a certificate and key + +For a LAN service the practical options are a certificate from your ACME client +(certbot, lego, caddy) for a name you control, or a self-signed pair. The lab +below uses a self-signed pair, because it needs no domain: + +```sh +mkdir -p /tmp/nxdns-lab/etc +cd /tmp/nxdns-lab +openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout etc/key.pem -out etc/cert.pem -days 365 \ + -subj "/CN=nxdns.lan" \ + -addext "subjectAltName=DNS:nxdns.lan,DNS:localhost,IP:127.0.0.1" +chmod 0600 etc/key.pem +``` + +A self-signed certificate means every client has to be told to trust it, or told +to skip verification. That is why the client commands further down pass +`--insecure` and `+tls` without a CA. A real deployment uses a real certificate +and drops those flags. + +## 2. Turn the listeners on + +`cert_path` and `key_path` must be absolute, or relative to the process working +directory. Point both endpoints at the same pair unless you have a reason not +to: + +```zon +.{ + .groups = .{.{ .name = "default" }}, + .upstreams = .{.{ .url = "https://cloudflare-dns.com/dns-query" }}, + .dns = .{ .bind_ipv4 = "127.0.0.1", .bind_ipv6 = "::1", .port = 15400 }, + .web = .{ .bind = "127.0.0.1", .port = 8451, .password = "lab-password" }, + .doh_server = .{ + .enabled = true, + .bind = "127.0.0.1", + .port = 8443, + .cert_path = "/tmp/nxdns-lab/etc/cert.pem", + .key_path = "/tmp/nxdns-lab/etc/key.pem", + }, + .dot_server = .{ + .enabled = true, + .bind = "127.0.0.1", + .port = 8853, + .cert_path = "/tmp/nxdns-lab/etc/cert.pem", + .key_path = "/tmp/nxdns-lab/etc/key.pem", + }, +} +``` + +Write that to `/tmp/nxdns-lab/etc/config.zon`. The configuration file seeds an +empty database and is then ignored; to change these settings on a server that +already has a database, edit them through the API or through +`export`/`import` — see [the configuration model](../explanation/configuration-model.md). + +## 3. Check the files before starting + +```sh +nxdns check --data-dir /tmp/nxdns-lab/data --config /tmp/nxdns-lab/etc/config.zon +``` + +``` +checking configuration file /tmp/nxdns-lab/etc/config.zon +OK https://cloudflare-dns.com/dns-query +OK: no problems found +``` + +`check` reads both endpoints' certificate and key. An unreadable file is a +failure and exits 2: + +``` +FAIL doh_server.cert_path: '/tmp/nxdns-lab/etc/cert.pem' is not readable +FAIL dot_server.cert_path: '/tmp/nxdns-lab/etc/cert.pem' is not readable +``` + +A key readable by anyone but its owner is a warning, and does not change the +exit code, because the service still starts: + +``` +WARN doh_server.key_path: '/tmp/nxdns-lab/etc/key.pem' is mode 644; a TLS key must be readable by its owner only +``` + +## 4. Start and confirm the listeners + +```sh +nxdns run --data-dir /tmp/nxdns-lab/data --config /tmp/nxdns-lab/etc/config.zon +``` + +Two lines in the log say the listeners bound: + +``` +info(nxdns): doh listener on 127.0.0.1:8443 +info(nxdns): dot listener on 127.0.0.1:8853 +``` + +If a certificate cannot be loaded while its endpoint is enabled, nxdns refuses +to start and exits 2 rather than serving DNS without the listener you asked +for: + +``` +doh_server: '/tmp/nxdns-lab/etc/cert.pem' + '/tmp/nxdns-lab/etc/key.pem': private key file is not readable +nxdns run failed: BadCertificate +run `nxdns check` to see the configuration in full +``` + +## 5. Query DoT + +Recent `dig` speaks DNS over TLS with `+tls` (this page was checked with BIND +9.20.26): + +```sh +dig @127.0.0.1 -p 8853 +tls example.com A +short +``` + +``` +172.66.147.243 +104.20.23.154 +``` + +## 6. Query DoH + +The only path the DoH listener serves is `/dns-query`; anything else is a 404. +It accepts both the POST form (the query as an `application/dns-message` body) +and the GET form (`?dns=` with base64url of the same bytes). + +The body is a raw DNS query in wire format. Build one for `example.com A` — +header with the recursion-desired bit, one question, then the QNAME as +length-prefixed labels: + +```sh +printf '%s' '000001000001000000000000076578616d706c6503636f6d0000010001' \ + | xxd -r -p > /tmp/nxdns-lab/query.bin +``` + +POST it: + +```sh +curl -sS --insecure --http1.1 \ + -H 'content-type: application/dns-message' \ + --data-binary @/tmp/nxdns-lab/query.bin \ + --output /tmp/nxdns-lab/answer.bin \ + -w 'http %{http_code}, %{size_download} bytes\n' \ + https://127.0.0.1:8443/dns-query +xxd /tmp/nxdns-lab/answer.bin +``` + +``` +http 200, 72 bytes +00000000: 0000 8180 0001 0002 0000 0001 0765 7861 .............exa +00000010: 6d70 6c65 0363 6f6d 0000 0100 01c0 0c00 mple.com........ +00000020: 0100 0100 0000 0400 04ac 4293 f3c0 0c00 ..........B..... +00000030: 0100 0100 0000 0400 0468 1417 9a00 0029 .........h.....) +00000040: 04d0 0000 0000 0000 ........ +``` + +The second flag byte `80` and the third answer-count field `0002` say: response, +no error, two answer records. + +The GET form takes the same bytes, base64url-encoded with the padding removed: + +```sh +Q=$(base64 -w0 /tmp/nxdns-lab/query.bin | tr '+/' '-_' | tr -d '=') +curl -sS --insecure --http1.1 -o /tmp/nxdns-lab/get.bin \ + -w 'GET http %{http_code}, %{size_download} bytes\n' \ + "https://127.0.0.1:8443/dns-query?dns=$Q" +``` + +``` +GET http 200, 61 bytes +``` + +`--http1.1` matters: without it curl offers HTTP/2 over ALPN, and the DoH +listener negotiates only what it advertises. `--insecure` is only needed for the +self-signed lab certificate. + +## 7. Renewals + +A watcher polls both files every 30 seconds and compares their modification time +and size against the pair currently loaded. When either differs it reloads and +swaps the new pair in; connections already open finish on the old certificate. +Nothing has to restart. + +Replace the pair and wait one poll interval: + +```sh +openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout /tmp/nxdns-lab/etc/key.pem -out /tmp/nxdns-lab/etc/cert.pem -days 365 \ + -subj "/CN=nxdns.lan" \ + -addext "subjectAltName=DNS:nxdns.lan,DNS:localhost,IP:127.0.0.1" +chmod 0600 /tmp/nxdns-lab/etc/key.pem +``` + +Within 30 seconds the log says so, once per endpoint that watches the file: + +``` +info(cert_store): certificate reloaded from /tmp/nxdns-lab/etc/cert.pem +info(cert_store): certificate reloaded from /tmp/nxdns-lab/etc/cert.pem +``` + +## 8. Reload immediately + +To skip the wait — from an ACME deploy hook, for example — call +`POST /api/certs/reload`. It needs a session; see +[set up admin authentication](set-up-admin-authentication.md) for the login +call that fills `cookies.txt`. + +```sh +curl -sS -b /tmp/nxdns-lab/cookies.txt -X POST http://127.0.0.1:8451/api/certs/reload +``` + +```json +{"doh":{"enabled":true,"reloaded":true,"error":null},"dot":{"enabled":true,"reloaded":true,"error":null}} +``` + +The route always answers 200: the per-endpoint outcome is the payload, not the +status code. A disabled endpoint reports `"enabled":false`. A reload that fails +names the reason and leaves the old certificate serving: + +```sh +chmod 000 /tmp/nxdns-lab/etc/key.pem +curl -sS -b /tmp/nxdns-lab/cookies.txt -X POST http://127.0.0.1:8451/api/certs/reload +``` + +```json +{"doh":{"enabled":true,"reloaded":false,"error":"private key file is not readable"},"dot":{"enabled":true,"reloaded":false,"error":"private key file is not readable"}} +``` + +The listeners keep working through that failure: + +```sh +dig @127.0.0.1 -p 8853 +tls example.com A +short +``` + +``` +104.20.23.154 +172.66.147.243 +``` + +Undo it with `chmod 0600 /tmp/nxdns-lab/etc/key.pem` and reload again. The +counters are on `/metrics`: + +``` +nxdns_cert_reloads_total{endpoint="doh"} 3 +nxdns_cert_reloads_total{endpoint="dot"} 3 +nxdns_cert_reload_failures_total{endpoint="doh"} 2 +nxdns_cert_reload_failures_total{endpoint="dot"} 2 +``` + +## File ownership on a real install + +The key must be readable by the user nxdns runs as, and by nobody else. + +Under systemd the service runs as the `nxdns` user: + +```sh +chown nxdns:nxdns /etc/nxdns/cert.pem /etc/nxdns/key.pem +chmod 0644 /etc/nxdns/cert.pem +chmod 0600 /etc/nxdns/key.pem +``` + +Under Docker the container runs as uid 65532, fixed in the image, and +`/etc/nxdns` is a read-only bind mount — the container cannot fix permissions +itself, so the host-side files must already be readable by that uid. It has no +name on the host, so chown it numerically: + +```sh +cd deploy/docker +chown 65532:65532 etc-nxdns/cert.pem etc-nxdns/key.pem +chmod 0644 etc-nxdns/cert.pem +chmod 0600 etc-nxdns/key.pem +``` + +**Not verified on this host:** the two `chown` blocks above. Both need root, and +the systemd one needs an `nxdns` user this development machine does not have. +Everything else on this page was executed as written. + +## Ports 443 and 853 + +The defaults are the standard ports, which are privileged. Under the packaged +systemd unit that is already handled: it grants `CAP_NET_BIND_SERVICE` for port +53 and the same capability covers 443 and 853. See +[install with systemd](install-with-systemd.md). diff --git a/docs/how-to/install-with-docker.md b/docs/how-to/install-with-docker.md new file mode 100644 index 0000000..e1552b0 --- /dev/null +++ b/docs/how-to/install-with-docker.md @@ -0,0 +1,170 @@ +# Install nxdns with Docker + +Builds the nxdns image and runs it with Docker Compose. At the end a container +answers DNS on port 53 and keeps its data in a named volume. + +For what each configuration field means, see +[the configuration reference](../reference/configuration.md). + +> Verification: every command on this page was run on the machine that wrote +> it, with three exceptions marked below — the `chown` to uid 65532 needs root, +> the arm64 image was built but not run, and pushing to a registry needs +> credentials. One command was run in altered form: host port 8080 was occupied +> here, so the run and the two verification commands in step 3 were executed +> with the host side of the port mappings moved to 25353 and 28088 rather than +> the 53 and 8080 printed below. The container side was unchanged. See the note +> in step 3. + +## 1. Build the image + +The Dockerfile does not compile anything. It assembles a filesystem around a +binary you build first, so build the admin interface and the binaries from the +repository root: + +```sh +(cd web && npm ci && npm run build) +zig build cross -Dweb-dist=web/dist -Doptimize=ReleaseSafe +docker build -t nxdns -f deploy/docker/Dockerfile . +``` + +Build `web/dist` every time, before the binary. A stale bundle is embedded +silently and ships an admin interface that does not match its API. + +The context has to be the repository root, because the Dockerfile copies +`zig-out/cross`. The result is a `scratch` image holding the binary, a CA +bundle and two empty directories — 28.2 MB here. + +Compose runs the same build with the right context: + +```sh +docker compose -f deploy/docker/compose.yaml build +``` + +Every block on this page runs from the repository root, and none of them change +directory, so they can be pasted in order. `-f` is what makes that work: +Compose resolves the relative paths inside `compose.yaml` — the build context, +the `etc-nxdns` bind mount — against the directory holding the file, not +against your shell, and it takes the project name `docker` from that directory +either way, which is why the container is `docker-nxdns-1`. + +## 2. Write the seed configuration + +Compose bind-mounts `deploy/docker/etc-nxdns` read-only at `/etc/nxdns`. Create +it and put the seed file in it: + +```sh +mkdir -p deploy/docker/etc-nxdns +$EDITOR deploy/docker/etc-nxdns/config.zon +``` + +The smallest file that starts is one group named `default` and one enabled +upstream: + +```zon +.{ + .groups = .{ .{ .name = "default" } }, + .upstreams = .{ .{ .url = "https://cloudflare-dns.com/dns-query" } }, + .web = .{ .password = "choose-a-real-password" }, +} +``` + +Without that file the container exits with code 2 on a fresh volume: an empty +database has nothing to forward to. The log is +`no configuration file at '/etc/nxdns/config.zon'; using the database as it is` +followed by `nxdns run failed: NoUsableUpstreams`. + +A file that is present but rejected is a different failure and a different exit +code. No `default` group, no enabled upstream, a syntax error — `run` prints the +diagnostic and exits **1**, for example `nxdns run failed: MissingDefaultGroup`. +Both were run here against this image: a seed file whose only group was named +`other` exited 1, and an empty `/etc/nxdns` exited 2 with `NoUsableUpstreams`. +Under `restart: unless-stopped` either one is a restart loop, so read the exit +code from `docker inspect` to tell them apart; see +[Troubleshoot nxdns](troubleshoot.md). + +The container runs as uid 65532, and the mount is read-only, so the container +cannot repair permissions itself. Mode 0644 works and was used here. If the +file carries a secret — `web.password`, or a `web.password_hash` from a +restored export — give it to that uid instead: + +```sh +chown 65532:65532 deploy/docker/etc-nxdns/config.zon +chmod 0600 deploy/docker/etc-nxdns/config.zon +``` + +> Not verified on this host: `chown` to a uid you do not own needs root. What +> was verified is the failure it prevents — a seed file at 0600 owned by +> another uid makes the container log `nxdns run failed: AccessDenied` and +> restart in a loop. See [Troubleshoot nxdns](troubleshoot.md). + +## 3. Run it + +```sh +docker compose -f deploy/docker/compose.yaml up -d +docker compose -f deploy/docker/compose.yaml logs -f +``` + +A healthy first start logs the seeding and the bound sockets: + +``` +info(config_bootstrap): seeded the database from '/etc/nxdns/config.zon' +info(nxdns): nxdns 0.1.0-dev serving on udp [::]:53 tcp [::]:53 tcp 0.0.0.0:53; 1 upstream(s); blocklist generation 1 +info(web_server): web interface listening on 0.0.0.0:8080 +``` + +Confirm it answers and that the admin interface is up: + +```sh +dig @127.0.0.1 example.com A +short +curl -s -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8080/ +``` + +> On the machine that wrote this page, host port 8080 was already taken by an +> unrelated process, so the container was verified with the host side of the +> port mappings moved to 25353 and 28088 — everything inside the container was +> unchanged, and the log still reads `serving on udp [::]:53`. Against those +> ports `dig` returned the A records for `example.com` and `curl` returned 200. +> `docker compose up -d` fails with +> `failed to bind host port 0.0.0.0:8080/tcp: address already in use` when a +> host port is occupied; free the port or edit the `ports:` list. + +The compose file publishes 53/udp, 53/tcp and 8080, keeps `/var/lib/nxdns` in +the named volume `nxdns-data`, and sets the per-namespace sysctl +`net.ipv4.ip_unprivileged_port_start=0` so uid 65532 can bind port 53 without +any capability. Uncomment the 443 and 853 mappings when you enable the DoH or +DoT listener; see [Enable DoH and DoT](enable-doh-and-dot.md). + +## 4. Do not point the host at the container + +The container resolves its own upstream DoH and DoT hostnames through the +host's DNS configuration. If you set the host's `/etc/resolv.conf` to the nxdns +container, the container's startup lookups depend on the service that is trying +to start. Point LAN clients at nxdns; leave the container's host on its own +resolver. + +## Build for a Raspberry Pi 5 + +The Dockerfile maps buildx's `TARGETARCH` onto the cross-target directory, so +the aarch64 image comes from the same `zig-out/cross` tree with no second +compile: + +```sh +docker buildx build --platform linux/arm64 -t nxdns:arm64 -f deploy/docker/Dockerfile . +``` + +This was run here and completed; add `--push` or `--load` to keep the result, +since the default buildx driver leaves it in the build cache. + +> Not verified on this host: the arm64 image was not started. Running it needs +> an aarch64 machine or qemu binfmt emulation, neither of which is available +> here. + +## Publish the image to a registry + +There is no registry push in CI on purpose: credentials and the choice of +registry are infrastructure decisions, not this repository's. Publish by hand +with `docker login `, then `docker tag nxdns +//nxdns:`, then `docker push +//nxdns:`. + +> Not verified on this host: pushing needs credentials for a registry. diff --git a/docs/how-to/install-with-systemd.md b/docs/how-to/install-with-systemd.md new file mode 100644 index 0000000..b0da3c1 --- /dev/null +++ b/docs/how-to/install-with-systemd.md @@ -0,0 +1,262 @@ +# Install nxdns with systemd + +Installs nxdns as a system service on a Linux host with systemd, including a +Raspberry Pi 5. At the end the service answers DNS on port 53 and starts on +boot. + +For what each flag does, see [the CLI reference](../reference/cli.md); for what +each configuration field means, see +[the configuration reference](../reference/configuration.md). + +> Verification: the build steps and `systemd-analyze verify` were run on the +> machine that wrote this page. `nxdns check` and `nxdns run` were run there +> too, but against a scratch `--data-dir` and `--config` on an unprivileged +> port, because that machine is not a deploy target and has no `/etc/nxdns`, +> no `/var/lib/nxdns` and no root. The steps that need root on a target host — +> `install`, `systemd-sysusers`, `systemctl` — were not run; they are marked +> where they appear. + +## 1. Build the binary + +Requires Zig 0.16.0 and Node.js. From the repository root: + +```sh +(cd web && npm ci && npm run build) +zig build cross -Dweb-dist=web/dist -Doptimize=ReleaseSafe +``` + +The first command builds the admin interface into `web/dist`; the second +embeds that directory in the binary. Build the interface every time, before the +binary: a stale `web/dist` ships an admin UI that does not match the API it +talks to. + +Two static musl binaries come out, one per deploy target: + +- `zig-out/cross/x86_64-linux-musl/nxdns` +- `zig-out/cross/aarch64-linux-musl/nxdns` + +Both are statically linked and need nothing installed on the target host. + +## 2. Copy the files to the target + +```sh +scp zig-out/cross/x86_64-linux-musl/nxdns target:/tmp/nxdns +scp deploy/systemd/nxdns.service deploy/systemd/sysusers.conf target:/tmp/ +``` + +For a Raspberry Pi 5, copy `zig-out/cross/aarch64-linux-musl/nxdns` instead — +see [Raspberry Pi 5](#raspberry-pi-5) below. + +> Not verified on this host: `target` is a placeholder for your server's +> hostname, and the machine that wrote this page has no second host to copy to. +> What was verified is that both source paths exist after step 1 and that the +> aarch64 file is a statically linked aarch64 ELF executable. + +Before copying, you can confirm the unit file parses: + +```sh +systemd-analyze verify deploy/systemd/nxdns.service +``` + +Off the target host this prints one complaint and exits 1: + +``` +nxdns.service: Command /usr/local/bin/nxdns is not executable: No such file or directory +``` + +That is the ExecStart path check finding no binary yet. Any other message is a +real problem with the unit. On the target, after step 3, the same command +should print nothing. + +## 3. Install the binary, the user and the unit + +Run as root on the target: + +```sh +install -m 0755 /tmp/nxdns /usr/local/bin/nxdns + +install -m 0644 /tmp/sysusers.conf /usr/lib/sysusers.d/nxdns.conf +systemd-sysusers + +install -m 0644 /tmp/nxdns.service /etc/systemd/system/nxdns.service +systemctl daemon-reload + +mkdir -p -m 0755 /etc/nxdns +``` + +> Not verified on this host: these commands need root on a target machine. The +> files they install were read at HEAD and the unit was checked with +> `systemd-analyze verify`. + +The service user is a static one, not `DynamicUser`: a TLS key for the DoH or +DoT listener has to be chown-able to a uid that survives a restart. + +Do not create `/var/lib/nxdns` or `/var/log/nxdns` by hand. The unit's +`StateDirectory` and `LogsDirectory` settings make systemd create them on first +start, `/var/lib/nxdns` at mode 0700 owned by `nxdns`. + +`/etc/nxdns` is the one directory the `mkdir` above is for. The unit's +`ConfigurationDirectory=nxdns` also creates it, but not until the first start +in step 5, and step 4 has to write a file into it before then. systemd does not +mind finding the directory already there; it adjusts the mode and ownership to +what the unit asks for. + +## 4. Write the seed configuration + +nxdns starts from an empty database only if a configuration file tells it what +to forward to. Write `/etc/nxdns/config.zon`. The smallest file that starts is +one group named `default` and one enabled upstream: + +```zon +.{ + .groups = .{ .{ .name = "default" } }, + .upstreams = .{ .{ .url = "https://cloudflare-dns.com/dns-query" } }, + .web = .{ .password = "choose-a-real-password" }, +} +``` + +That file holds a password in plain text, so restrict it as soon as you have +written it: + +```sh +chown root:nxdns /etc/nxdns/config.zon +chmod 0640 /etc/nxdns/config.zon +``` + +Root's umask is 022 on most distributions, so a freshly written +`/etc/nxdns/config.zon` is mode 0644 and every account on the host can read the +password out of it. The unit's `UMask=0077` does not help here: it applies to +files the service creates once it is running, and never re-chmods a file that +was written before the first start. + +0640 with group `nxdns` rather than 0600: `/etc/nxdns` is a +`ConfigurationDirectory`, which systemd leaves owned by root, and the service +runs as `nxdns` and has to read this file on the first start. A root-owned 0600 +file would be unreadable to it. + +Do not expect `nxdns check` to catch a permissive mode here. Its only +permission warning is for a TLS private key +(`WARN doh_server.key_path: ... is mode 644; a TLS key must be readable by its +owner only`, from `checkTlsFiles` in `src/cli.zig`); it never stats the +configuration file. A mode 0644 `config.zon` passes `check` in silence, so the +`chmod` above is yours to remember. + +Check it before you start the service: + +```sh +nxdns check --config /etc/nxdns/config.zon +``` + +A good file prints the source it checked, one `OK` line per upstream, and +`OK: no problems found`: + +``` +checking configuration file /etc/nxdns/config.zon +OK https://cloudflare-dns.com/dns-query +OK: no problems found +``` + +The upstream probe sends a real query, so this needs working DNS on the host at +the time you run it. Exit 2 means `check` found something to fix and printed +every problem it found, not only the first. + +The file seeds the database once. From the second start onwards it is ignored +and the database is the configuration; see +[the configuration model](../explanation/configuration-model.md) and +[Upgrade nxdns](upgrade.md) for how to change settings after that. + +Once the seed has been consumed — after step 6 confirms you can log in — the +plaintext in it is dead weight that only carries risk. The seed's +`web.password` is hashed into `web.password_hash` at import time and the +plaintext is never stored; `nxdns export` writes `.password = ""` back out +alongside the hash. Nothing downstream ever reads the plaintext again, so +delete the file: + +```sh +rm /etc/nxdns/config.zon +``` + +Keep it only if you want the seed as a record of the intended starting +configuration, and if you keep it, leave it at 0640 root:nxdns. Note that a +kept seed is not a backup — `nxdns export` is +(see [Back up and restore](back-up-and-restore.md)), and the export carries the +password hash rather than the password. + +> Verified on this host, with a scratch `--config` and `--data-dir` in place of +> `/etc/nxdns` and `/var/lib/nxdns`: a seed written under umask 022 came out +> 0644, `nxdns check --config` on it printed `OK: no problems found` with no +> mode warning, and after `nxdns import` of that seed an `nxdns export` wrote +> `.password = ""` next to a populated `.password_hash = +> "$argon2id$v=19$..."`. The `chown`, `chmod` and `rm` lines above are the +> ordinary root-owned-file operations and were not run against a real +> `/etc/nxdns`, which this host does not have. + +## 5. Start it + +```sh +systemctl enable --now nxdns +journalctl -u nxdns -f +``` + +> Not verified on this host: needs root and an installed unit. + +A healthy start logs a line naming every socket it bound: + +``` +info(nxdns): nxdns 0.1.0-dev serving on udp [::]:53 tcp [::]:53 tcp 0.0.0.0:53; 1 upstream(s); blocklist generation 1 +``` + +nxdns writes to stderr and systemd captures that into the journal; logging +needs no further configuration. Port 53 is privileged, and the unit grants +`CAP_NET_BIND_SERVICE` through `AmbientCapabilities`. + +If the start fails, read [Troubleshoot nxdns](troubleshoot.md). The two common +first-install failures are a port 53 already held by `systemd-resolved` and a +seed file that does not parse. + +## 6. Confirm it answers + +From another machine on the LAN: + +```sh +dig @ example.com A +short +``` + +The admin interface is on port 8080 by default; log in with the password from +the seed file. `http://:8080/api/health` reports upstream +availability and disk state without a login. + +> Not verified on this host as written: `` is a placeholder, and a +> LAN client to run it from is a second machine this host does not have. What +> was verified is the same two checks against a local nxdns started from a +> scratch data directory on an unprivileged port — `dig @127.0.0.1 -p 15353 +> example.com A +short` returned the A records, and `curl` against the web +> port returned 200. Only the address and the port differ from the lines +> above. + +## Raspberry Pi 5 + +The Pi 5 is aarch64. Nothing about the procedure changes except which binary +you copy — the cross build needs no toolchain on the Pi and no toolchain beyond +Zig on the build machine: + +```sh +(cd web && npm ci && npm run build) +zig build cross -Dweb-dist=web/dist -Doptimize=ReleaseSafe +scp zig-out/cross/aarch64-linux-musl/nxdns pi:/tmp/nxdns +scp deploy/systemd/nxdns.service deploy/systemd/sysusers.conf pi:/tmp/ +``` + +Then follow steps 3 to 6 on the Pi. + +The build was run on the machine that wrote this page and +`zig-out/cross/aarch64-linux-musl/nxdns` is a statically linked aarch64 ELF +executable. + +> Not verified on this host: the two `scp` lines. `pi` is a placeholder for +> your Pi's hostname, and this page was written on an x86_64 machine with no Pi +> attached. The build steps above it were run; the copy was not. + +> Not verified on this host: the aarch64 binary was not executed. This host is +> x86_64 and has no `qemu-aarch64` to run it under. Running it needs a +> Raspberry Pi 5 or another aarch64 machine. diff --git a/docs/how-to/measure-performance.md b/docs/how-to/measure-performance.md new file mode 100644 index 0000000..52b5487 --- /dev/null +++ b/docs/how-to/measure-performance.md @@ -0,0 +1,163 @@ +# Measure performance + +`tools/bench.zig` measures the three things nxdns can measure in-process: +blocklist lookup latency, cache-hit latency, and blocklist compile throughput. +Sustained query rate is not one of them — that one is end-to-end and needs a +load generator pointed at a running server. + +The numbers this project treats as targets, and the numbers measured so far, +are in the [performance reference](../reference/performance.md). Why those +targets exist and why CI does not gate on them is in +[performance and testing](../explanation/performance-and-testing.md). + +## Run the whole bench + +```sh +zig build bench -Doptimize=ReleaseFast +``` + +That runs all three suites with the defaults: 1,000,000 domains, 200,000 +iterations per suite, seed `0x5eed`. It takes minutes, most of it generating and +loading the million-domain list. + +`-Doptimize=ReleaseFast` is not optional if you want the numbers to mean +anything. A Debug build says so before it prints: + +``` +warning: Debug build; run with -Doptimize=ReleaseFast for meaningful numbers +``` + +## Run one suite, smaller + +Everything after `--` goes to the harness. A suite name selects one of +`filter`, `cache`, `compile` (the default is `all`), and `--domains` / +`--iters` shrink the load: + +```sh +zig build bench -Doptimize=ReleaseFast -- filter --domains=100000 --iters=20000 +``` + +``` +nxdns bench suite=filter domains=100000 iters=20000 seed=0x5eed optimize=ReleaseFast + +suite ops p50(us) p95(us) p99(us) max(us) +filter 20000 0.14 0.25 0.27 0.51 + blocked 6670/20000, Snapshot.memoryBytes 3.0 MiB, VmRSS 5.4 MiB + target p95 < 1ms: PASS + target VmRSS < 100 MiB: PASS +``` + +A reduced run is good for checking the harness works and for a rough +regression signal. It is not a result: the memory figure scales with +`--domains`, so 100,000 domains says nothing about the 1,000,000-domain memory +target. + +The other two suites: + +```sh +zig build bench -Doptimize=ReleaseFast -- cache --iters=20000 +``` + +``` +suite ops p50(us) p95(us) p99(us) max(us) +cache 20000 0.12 0.21 0.23 0.54 + hits 10000/20000, DnsCache.memoryBytes 4.3 MiB, VmRSS 6.2 MiB + target p95 < 5ms: PASS +``` + +```sh +zig build bench -Doptimize=ReleaseFast -- compile --domains=100000 +``` + +``` +suite ops p50(us) p95(us) p99(us) max(us) +compile 100000 wall 11.512ms, 8686215 lines/s, 100000 domains kept (informational) +``` + +`--seed=N` changes the generated domains and the query order; the default is +`0x5eed`, so two runs on the same machine are comparable. `--domains` caps at +4,000,000, and the `compile` suite additionally refuses more than 2,000,000 — +the compiler's own limit. + +An argument the harness does not recognise stops it before any measuring: + +``` +error: unknown argument 'nosuch' +usage: zig build bench -Doptimize=ReleaseFast -- [filter|cache|compile|all] [--domains=N] [--iters=N] [--seed=N] [--assert] +``` + +## Read the output + +- `p50`/`p95`/`p99`/`max` are per-operation microseconds, nearest-rank over + every iteration. What one operation means differs per suite: for `filter` it + is normalising a name plus evaluating it against the snapshot; for `cache` it + is building the key, getting the entry and stamping the response id. +- `blocked N/M` and `hits N/M` are sanity counters. The harness aborts if either + is zero — a suite that never hits its own path measures nothing. +- Two memory figures appear on purpose. `Snapshot.memoryBytes` and + `DnsCache.memoryBytes` are the in-repo accounting of those structures; `VmRSS` + is what the kernel holds resident for the whole process, allocator slack and + code included. The truth is between them, and the memory target is judged on + `VmRSS`. +- `target ...: PASS` / `FAIL` lines appear for the targets a suite covers. On a + plain run they are informational and the exit code stays 0. + +## Fail the run when a target is missed + +`--assert` turns those lines into an exit code — 1 when any target was +exceeded, 0 otherwise. This is meant for an acceptance run on hardware you +control, not for CI: + +```sh +zig build bench -Doptimize=ReleaseFast -- --assert +``` + +The full-scale form is the one worth asserting on, because the memory target +only means something at a million domains. On this development host the reduced +form was used to check the flag itself: + +```sh +zig build bench -Doptimize=ReleaseFast -- filter --domains=100000 --iters=20000 --assert +``` + +``` +filter 20000 0.14 0.26 0.27 2.42 + blocked 6670/20000, Snapshot.memoryBytes 3.0 MiB, VmRSS 5.4 MiB + target p95 < 1ms: PASS + target VmRSS < 100 MiB: PASS +``` + +**Not verified on this host at full scale:** the plain +`zig build bench -Doptimize=ReleaseFast -- --assert` above was not run during +the writing of this page — the default run takes minutes. The reduced runs +shown were all executed as written. The full-scale numbers already recorded for +this host are in the [performance reference](../reference/performance.md). + +## Measure sustained query rate + +The bench harness cannot do this. Query rate is a property of the whole server +— sockets, upstreams, the query log writer — so it has to be driven from +outside, against the real binary, on the machine you care about. + +Start nxdns with real blocklists configured, then drive it from another host on +the LAN with a DNS load generator such as `dnsperf`: + +```sh +dnsperf -s 192.168.1.10 -p 53 -d queries.txt -c 20 -Q 200 -l 60 +``` + +Read the client's own rate and the server's `/metrics` together: a load +generator that reports 200 qps while the server counts fewer has lost queries +somewhere, and that is the interesting number. + +**Not verified on this host:** `dnsperf` is not installed here and the target +platform is a Raspberry Pi 5, not this development machine. The command above +is the shape of the measurement, not a transcript. + +## Where to run it + +The target platform is a Raspberry Pi 5. Numbers from a development x86_64 box +do not transfer — the Pi's Cortex-A76 is far slower — so a passing run here is +evidence the harness works and a baseline for spotting regressions on the +machine development happens on, and nothing more. Run `--assert` on the Pi, +where the numbers mean something. diff --git a/docs/how-to/set-up-admin-authentication.md b/docs/how-to/set-up-admin-authentication.md new file mode 100644 index 0000000..604673a --- /dev/null +++ b/docs/how-to/set-up-admin-authentication.md @@ -0,0 +1,268 @@ +# Set up admin authentication + +The admin interface and its API are protected by a single operator password. +With no password set, every route is open to anything that can reach the web +port. Set one. + +The commands below run against the scratch lab from +[enable DoH and DoT](enable-doh-and-dot.md): data directory +`/tmp/nxdns-lab/data`, web listener on `127.0.0.1:8451`. On a real install the +data directory is `/var/lib/nxdns` and the web port is 8080. + +## 1. Set the password + +Put it in the seed configuration file, under `web`: + +```zon +.{ + .groups = .{.{ .name = "default" }}, + .upstreams = .{.{ .url = "https://cloudflare-dns.com/dns-query" }}, + .web = .{ .bind = "127.0.0.1", .port = 8451, .password = "lab-password" }, +} +``` + +At import time the plaintext is hashed with argon2id into `web.password_hash` +and discarded. It becomes no database row and appears in no log line. Setting +both `password` and `password_hash` in one file is refused: + +``` +web.password: password and password_hash are both set; ambiguity in a security setting is refused +import failed: PasswordAndHashBothSet +``` + +The seed file is read only while the database is empty. On a server that +already has a database, use step 4 or step 5 instead. + +## 2. Log in + +Login is `POST /api/auth/login` with a JSON body. Without a session, the API +answers 401: + +```sh +curl -sS -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8451/api/stats +``` + +``` +401 +``` + +Log in and keep the cookie: + +```sh +curl -sS -c /tmp/nxdns-lab/cookies.txt \ + -X POST http://127.0.0.1:8451/api/auth/login \ + -H 'content-type: application/json' \ + -d '{"password":"lab-password"}' +``` + +```json +{"authenticated":true,"auth_required":true} +``` + +The session token comes back in a `Set-Cookie` header, not in the body. In the +jar it looks like this (value redacted here): + +``` +#HttpOnly_127.0.0.1 FALSE / FALSE 1785770178 nxdns_session +``` + +The cookie is named `nxdns_session` and carries `HttpOnly; SameSite=Lax; +Path=/`. Its `Max-Age` comes from `web.session_ttl_hours`. Send it back on every +later call: + +```sh +curl -sS -b /tmp/nxdns-lab/cookies.txt -o /dev/null -w '%{http_code}\n' \ + http://127.0.0.1:8451/api/stats +``` + +``` +200 +``` + +A wrong password and an unknown one are the same answer, so a guess learns +nothing: + +```sh +curl -sS -X POST http://127.0.0.1:8451/api/auth/login \ + -H 'content-type: application/json' -d '{"password":"wrong"}' \ + -w ' (http %{http_code})\n' +``` + +``` +{"error":"invalid password"} (http 401) +``` + +The server logs the address and the outcome, never the password: + +``` +info(web_auth): web login accepted for 127.0.0.1:34040 +warning(web_auth): web login refused for 127.0.0.1:59670 +``` + +Sessions live in memory only. A restart logs everyone out. Thirty-two +concurrent sessions are kept; a thirty-third login evicts the least recently +used one. + +## 3. Log out + +```sh +curl -sS -b /tmp/nxdns-lab/cookies.txt -c /tmp/nxdns-lab/cookies.txt \ + -X POST http://127.0.0.1:8451/api/auth/logout +curl -sS -b /tmp/nxdns-lab/cookies.txt -o /dev/null -w 'stats: %{http_code}\n' \ + http://127.0.0.1:8451/api/stats +``` + +``` +{"authenticated":false} +stats: 401 +``` + +Logging out with a stale cookie, or with none, answers the same way. The point +of logging out is to end up logged out, and that is where such a request +already is. + +## 4. Change the password on a running server + +Send the new one to `PUT /api/settings` as `web.password`. The response is the +full settings document; `password` is write-only and `password_hash` is neither +readable nor directly writable, so neither value comes back. + +```sh +curl -sS -c /tmp/nxdns-lab/c2.txt -X POST http://127.0.0.1:8451/api/auth/login \ + -H 'content-type: application/json' -d '{"password":"lab-password"}' +curl -sS -b /tmp/nxdns-lab/c2.txt -X PUT http://127.0.0.1:8451/api/settings \ + -H 'content-type: application/json' \ + -d '{"web":{"password":"a-new-password"}}' +``` + +Changing the password ends every session, including the one that made the +change: + +```sh +curl -sS -b /tmp/nxdns-lab/c2.txt -o /dev/null -w 'old session: %{http_code}\n' \ + http://127.0.0.1:8451/api/stats +curl -sS -X POST http://127.0.0.1:8451/api/auth/login \ + -H 'content-type: application/json' -d '{"password":"lab-password"}' \ + -w ' (old password)\n' +curl -sS -c /tmp/nxdns-lab/c3.txt -X POST http://127.0.0.1:8451/api/auth/login \ + -H 'content-type: application/json' -d '{"password":"a-new-password"}' \ + -w ' (new password)\n' +``` + +``` +old session: 401 +{"error":"invalid password"} (old password) +{"authenticated":true,"auth_required":true} (new password) +``` + +Log back in with the new password. That is the whole rotation. + +## 5. Change the password without the API + +If you have lost the password, the admin interface cannot help — go through the +database instead. Export, edit, import. `nxdns export` always writes +`.password = ""` and carries the hash, so an exported file re-imports without +anyone knowing the password. To install a new one, put it in `.password` and +clear `.password_hash`: + +```sh +nxdns export --data-dir /tmp/nxdns-lab/data --out /tmp/nxdns-lab/rekeyed.zon +``` + +Edit the `web` section of `/tmp/nxdns-lab/rekeyed.zon` so it reads: + +```zon + .password = "offline-password", + .password_hash = "", +``` + +Stop the server before importing. `import` rewrites the stored hash underneath a +process that read it at startup; a running server keeps verifying against the +old one, so skipping the stop leaves the new password not working until the next +restart. In the lab the server is a foreground `nxdns run`, so Ctrl-C in its +terminal stops it, and it goes back up with the same command: + +```sh +# Ctrl-C the `nxdns run` terminal, or `kill` its pid from another shell +nxdns import /tmp/nxdns-lab/rekeyed.zon --force --data-dir /tmp/nxdns-lab/data +nxdns run --data-dir /tmp/nxdns-lab/data --config /tmp/nxdns-lab/etc/config.zon +``` + +``` +imported /tmp/nxdns-lab/rekeyed.zon +``` + +On a real install the stop and start are `systemctl stop nxdns` and +`systemctl start nxdns` around the same `import` — **not verified on this +host**, which has no installed nxdns systemd unit (`systemctl status nxdns` +answers `Unit nxdns.service could not be found.`) and where `systemctl` needs +root. See [back up and restore](back-up-and-restore.md). + +Once it is back up the old password is refused and the new one works: + +```sh +curl -sS -X POST http://127.0.0.1:8451/api/auth/login \ + -H 'content-type: application/json' -d '{"password":"a-new-password"}' \ + -w ' (old password, http %{http_code})\n' +curl -sS -c /tmp/nxdns-lab/c5.txt -X POST http://127.0.0.1:8451/api/auth/login \ + -H 'content-type: application/json' -d '{"password":"offline-password"}' \ + -w ' (new password, http %{http_code})\n' +curl -sS -b /tmp/nxdns-lab/c5.txt -o /dev/null -w 'stats: %{http_code}\n' \ + http://127.0.0.1:8451/api/stats +``` + +``` +{"error":"invalid password"} (old password, http 401) +{"authenticated":true,"auth_required":true} (new password, http 200) +stats: 200 +``` + +The next export shows the new hash and an empty `password` again: + +```sh +nxdns export --data-dir /tmp/nxdns-lab/data | grep password +``` + +``` + .password = "", + .password_hash = "$argon2id$v=19$m=19456,t=2,p=1$kvlRj1tdGul3MlfbvzLncLKWirNpJRJ3howFA9/ysgg$7elW7PPQ3WXHwI4YOmOpZ/1KNEQo7ZDLRJhnYOPMjqw", +``` + +`--force` is required because the database already has content. See +[back up and restore](back-up-and-restore.md). + +## What happens with no password set + +Authentication is off. Every route is open, and a login attempt succeeds +without minting anything — there is nothing to log in to, and a session that +authorises nothing would be a lie for the browser to store: + +```sh +curl -sS -o /dev/null -w '%{http_code}\n' http://127.0.0.1:8453/api/stats +curl -sS -X POST http://127.0.0.1:8453/api/auth/login \ + -H 'content-type: application/json' -d '{"password":"anything"}' +``` + +``` +200 +{"authenticated":true,"auth_required":false} +``` + +`auth_required: false` is how the admin interface knows to stop showing a login +form. Treat this as a lab-only state: bind the web listener to a trusted +interface at the very least, and preferably set a password. + +## Notes + +- A stored hash this build cannot parse is a 500, not a 401. Answering 401 would + tell an operator with a corrupted `web.password_hash` that their password is + wrong, and they would retype a password that can never verify. +- Requests from the box itself skip the API rate limit by default + (`web.api_localhost_exempt`). +- `web.session_ttl_hours`, `web.api_rate_limit_per_min` and the rest are in the + [configuration reference](../reference/configuration.md); the routes are in + the [API reference](../reference/api.md). + +Every command on this page was executed on this host as written, except the +`systemctl` stop and start named in step 5 and marked **not verified on this +host** there. diff --git a/docs/how-to/troubleshoot.md b/docs/how-to/troubleshoot.md new file mode 100644 index 0000000..593613a --- /dev/null +++ b/docs/how-to/troubleshoot.md @@ -0,0 +1,295 @@ +# Troubleshoot nxdns + +Symptoms an nxdns install actually produces, what to run to identify each one, +and what to change. Every symptom on this page was reproduced on the machine +that wrote it, and every diagnosis command was run there. Two details differ +from a real install and cannot be otherwise on that machine: it has no +installed service, so the log lines were read from a foreground run instead of +`journalctl -u nxdns`, and ports 53 and 8080 were occupied, so DNS and the API +were exercised on unprivileged ports. Fixes that need root are marked. + +The exit codes themselves are listed in +[the CLI reference](../reference/cli.md). + +## The service exits with code 2 + +**Symptom.** The process stops immediately. The last two lines are the error +and a pointer: + +``` +nxdns run failed: NoUsableUpstreams +run `nxdns check` to see the configuration in full +``` + +Exit 2 is reserved for a small set of faults `run` raises itself: +`NoUsableUpstreams`, `BadBindAddress`, `BadRateLimit` and `BadCertificate`. + +**Diagnosis.** + +```sh +nxdns check +``` + +`check` prints every problem it finds, not the first, and names the source it +checked on its first line. + +**Fixes by cause.** + +- `NoUsableUpstreams` — the database has no enabled upstream. On a fresh + install this means the seed file was missing or in the wrong place; the start + log says `no configuration file at '/etc/nxdns/config.zon'; using the + database as it is`. Write the seed file and start again against the still + empty database, or `nxdns import --force`. +- `BadCertificate` — a DoH or DoT listener is enabled and its certificate or + key is unreadable, too large, unparseable, or the key does not belong to the + certificate. `run` names both paths before it exits: + `doh_server: '' + '': certificate file is not readable`. + + **`check` does not catch most of this.** It tests only that each file is + readable, and warns when the key is readable beyond its owner; it never opens + the PEM. Parsing and the key/certificate pairing happen when `run` builds the + TLS context, so `check` can print `OK: no problems found` on a configuration + `run` then refuses. Reproduced here with a self-signed pair and the key from a + second, unrelated pair: + + ``` + $ nxdns check --config config.zon + checking configuration file config.zon + OK https://cloudflare-dns.com/dns-query + OK: no problems found # exit 0 + + $ nxdns run --config config.zon --data-dir ./data + warning(tls_server): mbedtls_pk_check_pair failed: RSA - Key failed to pass the validity check of the library (-16896) + doh_server: 'cert.pem' + 'mismatched-key.pem': private key does not belong to the certificate + nxdns run failed: BadCertificate # exit 2 + ``` + + A cert file containing `not a certificate` behaves the same way — `check` + exits 0, `run` exits 2 with `certificate PEM could not be parsed`. So a + successful `check` means the paths and permissions are right, not that the + certificate is usable; the only test of that is starting the service. Fix the + path, the ownership, or the pair; see + [Enable DoH and DoT](enable-doh-and-dot.md). +- `BadRateLimit` — a rate limit or window is zero. `import` refuses such a + configuration, so this only reaches a database that was edited by hand. +- `BadBindAddress` — `dns.bind_ipv4` or `dns.bind_ipv6` is not an address of + that family. + +## The service exits with code 1 on a seed file you just wrote + +**Symptom.** A first start against an empty database prints the validation +problem and stops, but with exit code 1, not 2: + +``` +groups: no group named 'default'; every unknown client is assigned to it +nxdns run failed: MissingDefaultGroup +``` + +A syntax error behaves the same way: + +``` +config: 2:42: error: expected ',' after initializer +nxdns run failed: ParseZon +``` + +So does a seed file whose upstream list is empty or all disabled: + +``` +upstreams: at least one upstream must be enabled +nxdns run failed: NoUpstreams +``` + +`NoUpstreams` from a seed file is not the same fault as `NoUsableUpstreams` +above: the first is a file `run` refused, the second is a database `run` +accepted and found empty. Only the second is exit 2. + +**Diagnosis.** Run the same file through `check`, which reports it as a +configuration problem and exits 2: + +```sh +nxdns check --config /etc/nxdns/config.zon +``` + +**Fix.** Correct the file the diagnostics name and start again. The database is +still empty after a failed seed, so the next start re-reads the file. Note that +`nxdns check` and `nxdns import` of the same bad file exit 2 while `nxdns run` +exits 1 — the exit code differs by command, the diagnostics do not. All three +commands were run here against a file missing its `default` group, one with a +syntax error and one with no enabled upstream, and every pair came out that +way. + +## Port 53 is already taken + +**Symptom.** The process exits 1, having named the socket it could not have: + +``` +cannot bind udp [::1]:53: AddressInUse +nxdns run failed: AddressInUse +``` + +A bind conflict is a runtime failure, not a configuration fault, so this is +exit 1 and `nxdns check` will not find it. + +**Diagnosis.** + +```sh +ss -lnup 'sport = :53' +ss -lntp 'sport = :53' +systemctl is-active systemd-resolved +``` + +On most systemd distributions the holder is `systemd-resolved`, which runs a +stub listener on `127.0.0.53:53` and on some setups binds `0.0.0.0:53`. + +**Fix.** Turn off the stub listener and keep resolved for the host's own +lookups: + +```sh +mkdir -p /etc/systemd/resolved.conf.d +printf '[Resolve]\nDNSStubListener=no\n' > /etc/systemd/resolved.conf.d/nxdns.conf +systemctl restart systemd-resolved +``` + +If `/etc/resolv.conf` is a symlink to `/run/systemd/resolve/stub-resolv.conf`, +repoint it at `/run/systemd/resolve/resolv.conf` so the host still resolves. + +> Not verified on this host: this needs root, and `systemd-resolved` is +> inactive here with port 53 free, so the conflict could not be reproduced +> against it. The bind failure itself was reproduced by starting a second nxdns +> on a port the first already held, which is the same error path. + +Do not fix this by pointing the host's `/etc/resolv.conf` at nxdns when that +host is where nxdns resolves its own upstream DoH and DoT hostnames. That is a +startup cycle, not a fix. + +## The container restarts in a loop + +**Symptom.** `docker compose ps` shows the container restarting, and the log is +one line repeated: + +``` +nxdns run failed: AccessDenied +``` + +**Diagnosis.** + +```sh +docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}} restarts={{.RestartCount}}' docker-nxdns-1 +stat -c '%a %u:%g %n' deploy/docker/etc-nxdns/config.zon +``` + +Exit 1 with `AccessDenied` means the container could not read the seed file. +The container runs as uid 65532 and `/etc/nxdns` is mounted read-only, so a +file at mode 0600 owned by your own uid is unreadable to it and the container +cannot repair it. + +**Fix.** Either make the file world-readable, when it holds no secret: + +```sh +chmod 0644 deploy/docker/etc-nxdns/config.zon +``` + +or give it to the container's uid: + +```sh +chown 65532:65532 deploy/docker/etc-nxdns/config.zon +chmod 0600 deploy/docker/etc-nxdns/config.zon +``` + +The 0644 path was verified here, including the recovery: after the `chmod` the +container started and answered queries. The `chown` needs root and was not run +here. + +A container that exits 2 instead — `nxdns run failed: NoUsableUpstreams` after +`no configuration file at '/etc/nxdns/config.zon'` — has no seed file at all on +a fresh volume. Create `deploy/docker/etc-nxdns/config.zon` and bring it up +again; see [Install with Docker](install-with-docker.md). + +## The container cannot reach its upstreams + +**Symptom.** The container starts, but every query fails and `nxdns check` +inside it reports each upstream as unreachable. + +**Diagnosis.** Look at what the host resolves with: + +```sh +cat /etc/resolv.conf +``` + +**Fix.** If it points at the nxdns container, repoint it at a real resolver. +The container resolves its upstream DoH and DoT hostnames through the host's +DNS configuration, so pointing that at nxdns makes nxdns depend on itself to +start. LAN clients point at nxdns; the container's own host does not. + +## The disk is filling up + +**Symptom.** Writes stop but DNS keeps answering. The journal shows the +transition: + +``` +warning(disk_monitor): disk state ok -> critical: 33349095424 bytes free on /var/lib/nxdns +``` + +**Diagnosis.** + +```sh +curl -s http://127.0.0.1:8080/api/health +``` + +`/api/health` needs no login and reports the state and what has been gated: + +```json +{"status":"degraded","disk":{"state":"critical","free_bytes":33349079040,"db_bytes":180224,"log_bytes":0,"sample_failures":0},"upstreams":{"available":1,"total":1},"queries_dropped":0,"writer_failed":false,"refreshes_gated":1,"snapshot_generation":2} +``` + +`/metrics` carries the same free, database and log byte gauges as +`nxdns_disk_free_bytes`, `nxdns_disk_db_bytes` and `nxdns_disk_log_bytes`; the +state itself is on `/api/health`, not in the metrics output. + +**What the state means.** The monitor samples free space and database sizes +once a minute. Below `disk.warn_free_mb` it logs the transition. Below +`disk.min_free_mb` it gates every non-essential write: the query logger holds +its batches, the client tracker stops persisting, and blocklist refreshes are +skipped and counted in `refreshes_gated`. Resolution never degrades because the +disk is full — this was verified by setting the thresholds above the free space +on the volume: the state went critical, a refresh was gated, and queries kept +being answered. + +**Fix.** Recover space — lower `logging.retention_days`, or stop the service +and delete `querylog.db` — and writes resume on the next sample. + +## Blocklists are not filtering + +**Symptom.** Domains that should be blocked resolve normally. + +**Diagnosis.** Read the startup line: + +```sh +journalctl -u nxdns | grep 'serving on' +``` + +It ends in either `blocklist generation N` or +`unfiltered (no blocklist snapshot)`. + +**Fix.** `unfiltered` means no snapshot loaded at all; the download or compile +warning that explains it is earlier in the same start. nxdns serves anyway on +purpose — a household loses more from DNS that refuses to start than from a +window of unfiltered answers. + +A generation number with nothing being blocked is a different problem: the +snapshot loaded but has no sources in it. The line +`blocklist snapshot generation 1: 0 of 0 sources loaded` says exactly that. Add +a source in the admin interface, or in the seed file before the first start. + +## A database stamped by a newer binary + +**Symptom.** After putting an older binary back, it will not start: + +``` +warning(migrations): config.db is at schema version 99; this nxdns binary supports 2 +nxdns run failed: SchemaTooNew +``` + +**Fix.** There is no downgrade. Import the export you took before upgrading +into a fresh data directory with the older binary; see +[Upgrade nxdns](upgrade.md). diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md new file mode 100644 index 0000000..5e1d43e --- /dev/null +++ b/docs/how-to/upgrade.md @@ -0,0 +1,214 @@ +# Upgrade nxdns + +Replaces a running nxdns with a newer build without losing its configuration. +The database is migrated in place on the first start of the new binary. + +> Verification: the export, the migration behaviour and the `version`/`check` +> steps below were run on the machine that wrote this page, against a +> populated scratch data directory and with an explicit `--data-dir`, since +> that machine has no `/var/lib/nxdns`. The commands were not run exactly as +> printed — the page uses the defaults and placeholders a real operator would +> have (`/var/lib/nxdns`, `/some/backup`, a `target` host), and every block +> where the substitution matters, or which was not run at all, carries its own +> note. Nothing here was verified except where a note says so. + +## 1. Take an export first + +There is no downgrade path, so the export is what you fall back to: + +```sh +nxdns export --out /some/backup/nxdns-config.zon +``` + +`/some/backup` is a stand-in for a directory you keep backups in, and the +command relies on the default `--data-dir /var/lib/nxdns` that a systemd +install has. + +> Verified on this host with both paths substituted, since it has neither +> `/var/lib/nxdns` nor `/some/backup`. `SCRATCH` below is a scratch directory, +> and its `data/` was populated beforehand with `nxdns import`: +> +> ``` +> $ nxdns export --data-dir $SCRATCH/data --out $SCRATCH/nxdns-config.zon +> wrote /…/scratchpad/nxdns-config.zon +> $ stat -c '%a %n' $SCRATCH/nxdns-config.zon +> 600 /…/scratchpad/nxdns-config.zon +> $ grep password $SCRATCH/nxdns-config.zon +> .password = "", +> .password_hash = "$argon2id$v=19$m=19456,t=2,p=1$mCNEo…$i3DMz…", +> ``` +> +> The shell umask was 022, so the 0600 is `export` setting it, not the umask. +> Only the two paths differ from the command above. + +The file is written atomically at mode 0600 and carries +`web.password_hash`, so treat it as a secret. See +[Back up and restore](back-up-and-restore.md) for the full backup story. The +query log is deliberately not part of it. + +## 2. Build the new binary + +```sh +(cd web && npm ci && npm run build) +zig build cross -Dweb-dist=web/dist -Doptimize=ReleaseSafe +``` + +Rebuild `web/dist` before the binary on every upgrade. The admin interface is +embedded at build time, and an old bundle against a new API is a broken +settings page. + +## 3. Replace the binary + +### systemd + +Step 2 leaves the new binary under `zig-out/cross`, one per target. Copy the +one that matches the host — `aarch64-linux-musl` for a Raspberry Pi 5: + +```sh +scp zig-out/cross/x86_64-linux-musl/nxdns target:/tmp/nxdns +``` + +> Not run on this host: `target` is a placeholder for the machine running +> nxdns, and this host has no such second machine to copy to. What exists here +> is the local half — `zig build cross` produced +> `zig-out/cross/x86_64-linux-musl/nxdns`. + +Then, as root on the target: + +```sh +install -m 0755 /tmp/nxdns /usr/local/bin/nxdns +systemctl restart nxdns +journalctl -u nxdns -f +``` + +> Not run on this host: all three lines need root and an installed service. +> The migration half of what a restart does is checkable without either, and +> was — see the note under +> [What happens to the database](#what-happens-to-the-database). The swap of +> an older binary for a newer one on a live service was not reproduced here. + +### Docker + +```sh +cd deploy/docker +docker compose build +docker compose up -d +``` + +Compose recreates the container against the same `nxdns-data` volume. The seed +file in `etc-nxdns` is not read again; the database in the volume is the +configuration. + +> Verified on this host for the first two lines: `docker compose config -q` +> exited 0, and `docker compose build` finished with `Image nxdns Built`. +> `docker compose up -d` was not run — it publishes host ports 53/udp, 53/tcp +> and 8080, which this workstation is not a deploy target for. + +## 4. Confirm the upgrade + +```sh +nxdns version +nxdns check +dig @127.0.0.1 example.com A +short +``` + +`nxdns check` with no `--config` checks the database, which is what you want +after an upgrade — it names its source on the first line and migrates a +database that is one schema version behind before checking it: + +``` +checking database /var/lib/nxdns/config.db +OK https://cloudflare-dns.com/dns-query +OK: no problems found +``` + +> Verified on this host for the first two commands, with `--data-dir` pointing +> at the scratch data directory instead of `/var/lib/nxdns`: +> +> ``` +> $ nxdns version +> nxdns 0.1.0-dev (unknown) +> zig 0.16.0 +> $ nxdns check --data-dir $SCRATCH/data +> checking database /…/scratchpad/data/config.db +> OK https://cloudflare-dns.com/dns-query +> OK: no problems found +> ``` +> +> The database path on the first line is the only difference from the block +> above. The `dig` line was not run in this round: nothing is listening on +> 127.0.0.1:53 here, and port 53 needs root. + +## What happens to the database + +Migrations run at startup, and also before `check`, `export` and `import`, so +whichever of those you run first performs the upgrade. A fresh database is +created at the current schema version; an older one is stepped up to it. The +log line names both versions: + +``` +info(migrations): config.db migrated from schema version 0 to 2 +``` + +> Verified on this host: that exact line is what `nxdns import` printed when it +> created the scratch database used throughout this page. An empty data +> directory is schema version 0, which is why a first run reports a migration +> rather than nothing. The step from a populated older schema to 2 was not +> reproduced here — it needs a database written by an older binary, which this +> host does not have. + +Rolling back is the case that has no answer. A database stamped by a newer +binary refuses to open, so an older binary against an upgraded data directory +fails to start: + +``` +warning(migrations): config.db is at schema version 99; this nxdns binary supports 2 +nxdns run failed: SchemaTooNew +``` + +> Not reproduced on this host: the same missing ingredient as above, a +> database at a schema version this binary does not support. The two lines +> are the messages `src/storage/migrations.zig` emits, not a run captured +> here. + +That run exits 1. Recovering means importing the export you took in step 1 into +a fresh data directory with the older binary. + +## Changing settings, not the binary + +An upgrade never re-reads `/etc/nxdns/config.zon`. After the first successful +seed the file is ignored, and the start log says so: + +``` +info(config_bootstrap): configuration file ignored; the database is already configured +``` + +Change settings through the admin interface, through the API, or with an +export–edit–import cycle against a stopped server: + +```sh +nxdns export --out config-backup.zon +$EDITOR config-backup.zon +systemctl stop nxdns +nxdns import config-backup.zon --force +systemctl start nxdns +``` + +`--force` is required here. A plain `import` into a database that already has +content fails with `import failed: DatabaseNotEmpty` and exits 2, so it cannot +clobber a configured server by accident. + +> Verified on this host for the two `nxdns` lines, against a populated scratch +> data directory: +> +> ``` +> $ nxdns import $SCRATCH/nxdns-config.zon --data-dir $SCRATCH/data +> import failed: DatabaseNotEmpty +> (exit 2) +> $ nxdns import $SCRATCH/nxdns-config.zon --data-dir $SCRATCH/data --force +> imported /…/scratchpad/nxdns-config.zon +> (exit 0) +> ``` +> +> The `systemctl stop`/`start` lines around them need root and an installed +> service and were not run; `$EDITOR` is yours to run. diff --git a/docs/operator.md b/docs/operator.md deleted file mode 100644 index 20b1397..0000000 --- a/docs/operator.md +++ /dev/null @@ -1,418 +0,0 @@ -# Operating nxdns - -How to install, configure, back up, upgrade and troubleshoot an nxdns server. -For the meaning of every configuration field, see -[config-reference.md](config-reference.md); for the HTTP API, -[api.md](api.md). - -## Install: systemd - -nxdns ships as one static musl binary. Build it (see -[Building](#building-the-binary)) or take it from CI, then: - -```sh -# 1. The binary. -install -m 0755 nxdns /usr/local/bin/nxdns - -# 2. The service user. Static, not DynamicUser: the TLS key for DoH/DoT -# must be chown-able to a stable uid. -install -m 0644 deploy/systemd/sysusers.conf /usr/lib/sysusers.d/nxdns.conf -systemd-sysusers - -# 3. The unit. -install -m 0644 deploy/systemd/nxdns.service /etc/systemd/system/nxdns.service -systemctl daemon-reload -``` - -Do not create directories by hand. The unit's `StateDirectory=nxdns`, -`LogsDirectory=nxdns` and `ConfigurationDirectory=nxdns` make systemd create -`/var/lib/nxdns` (mode 0700, owned by `nxdns`), `/var/log/nxdns` and -`/etc/nxdns` on first start. - -Write a seed configuration to `/etc/nxdns/config.zon`. The minimum that -starts is one group named `default` and one enabled upstream: - -```zon -.{ - .groups = .{ .{ .name = "default" } }, - .upstreams = .{ .{ .url = "https://cloudflare-dns.com/dns-query" } }, - .web = .{ .password = "choose-a-real-password" }, -} -``` - -Then: - -```sh -systemctl enable --now nxdns -journalctl -u nxdns -f -``` - -nxdns logs to stderr by default and systemd captures that into the journal; -nothing else needs configuring for logs. Port 53 needs -`CAP_NET_BIND_SERVICE`, which the unit grants via `AmbientCapabilities`. - -If port 53 is already taken, see -[Port 53 conflicts](#port-53-is-taken-systemd-resolved). - -### Building the binary - -Requires Zig 0.16.0 and Node.js 24 (for the web UI). From the repository -root: - -```sh -(cd web && npm ci && npm run build) -zig build cross -Dweb-dist=web/dist -Doptimize=ReleaseSafe -``` - -This produces static binaries for both deploy targets: - -- `zig-out/cross/x86_64-linux-musl/nxdns` -- `zig-out/cross/aarch64-linux-musl/nxdns` - -### Raspberry Pi 5 recipe - -The Pi 5 is aarch64. Build on any machine (the cross build needs no -toolchain beyond Zig itself), copy the binary over, then follow the systemd -steps above on the Pi: - -```sh -(cd web && npm ci && npm run build) -zig build cross -Dweb-dist=web/dist -Doptimize=ReleaseSafe -scp zig-out/cross/aarch64-linux-musl/nxdns pi:/tmp/nxdns -scp deploy/systemd/nxdns.service deploy/systemd/sysusers.conf pi:/tmp/ - -# on the Pi, as root: -install -m 0755 /tmp/nxdns /usr/local/bin/nxdns -install -m 0644 /tmp/sysusers.conf /usr/lib/sysusers.d/nxdns.conf -systemd-sysusers -install -m 0644 /tmp/nxdns.service /etc/systemd/system/nxdns.service -systemctl daemon-reload -# write /etc/nxdns/config.zon, then: -systemctl enable --now nxdns -``` - -The binary is statically linked against musl; it has no runtime -dependencies on the Pi. - -## Install: Docker - -The image is built from a binary you compile first; the Dockerfile only -assembles the filesystem. From the repository root: - -```sh -(cd web && npm ci && npm run build) -zig build cross -Dweb-dist=web/dist -Doptimize=ReleaseSafe -docker build -t nxdns -f deploy/docker/Dockerfile . -``` - -or, with compose (which runs the same build with the repository root as -context): - -```sh -cd deploy/docker -docker compose build -``` - -The Dockerfile maps buildx's `TARGETARCH` onto the cross-target directory, -so `docker buildx build --platform linux/arm64` produces the aarch64 image -from the same `zig-out/cross` tree. - -Before the first `docker compose up`, create the seed configuration the -compose file bind-mounts read-only at `/etc/nxdns`: - -```sh -cd deploy/docker -mkdir -p etc-nxdns -$EDITOR etc-nxdns/config.zon # the same minimal seed as the systemd path -``` - -The seed must be readable by uid 65532, the fixed container user; the bind -mount is read-only, so the container cannot adjust permissions itself. -World-readable (0644) is fine when the seed carries no secret; if it holds -`web.password` or `web.password_hash` (a restored export), restrict it instead: -`chown 65532:65532 etc-nxdns/config.zon && chmod 0600 etc-nxdns/config.zon`. - -Without a valid seed — at least a `default` group and one enabled -upstream — the container exits with code 2, because a fresh volume holds an -empty database and an empty database has no upstream to forward to. - -The compose file publishes 53/udp, 53/tcp and 8080, keeps the data in a -named volume mounted at `/var/lib/nxdns`, and sets the per-network-namespace -sysctl `net.ipv4.ip_unprivileged_port_start=0` so the nonroot user -(uid 65532) can bind port 53. Uncomment the 443/853 port mappings when you -enable the DoH or DoT listener. - -Do not point the host's `/etc/resolv.conf` at the nxdns container. The -container resolves its upstream DoH/DoT hostnames through the host's DNS -configuration; pointing that at nxdns itself makes the container's own -lookups depend on the service they are trying to start. - -### Publishing the image to a private registry - -There is deliberately no registry push in CI — credentials and registry -choice are an infrastructure decision, not this repository's. To publish -manually, log in to your registry, tag the local image with the registry's -name, and push: `docker login `, then -`docker tag nxdns //nxdns:`, then -`docker push //nxdns:`. The same works for a -self-hosted Gitea registry such as git.mial.net. - -## First boot and configuration semantics - -The ZON file at `/etc/nxdns/config.zon` (or `--config`) seeds the database -exactly once: - -- **No file:** normal steady state; the database is used as it is. -- **File present, database empty:** the file is imported. A file that is - unreadable, unparseable or invalid is an error (exit 2, every problem - printed) — nxdns never falls back to silent defaults over a file you - wrote. -- **File present, database already configured:** the file is ignored. The - database is the truth from the first successful seed onward. - -After the first boot, editing `config.zon` changes nothing. Change the -configuration through the web UI, the REST API, or the export→edit→import -cycle: - -```sh -nxdns export --out config-backup.zon -$EDITOR config-backup.zon -systemctl stop nxdns -nxdns import config-backup.zon --force -systemctl start nxdns -``` - -`import` without `--force` refuses a database that already has content -(exit 2), so a plain `import` can never clobber a configured server by -accident. - -## Authentication setup - -Set `web.password` in the seed file (or in a file you `import`). At import -time it is hashed with argon2id into `web.password_hash` and discarded; the -plaintext is never stored anywhere. `nxdns export` always writes -`.password = ""` and carries the hash instead, so an exported file -re-imports without knowing the password. Setting both `password` and -`password_hash` in one file is an error (exit 2). To change the password, -export, set `.password` to the new value, clear `.password_hash` to `""`, -and import with `--force`. - -## TLS for the DoH/DoT listeners - -Both listeners are disabled by default. To enable one, set -`doh_server.enabled` / `dot_server.enabled` and point `cert_path` and -`key_path` at a PEM certificate chain and key, conventionally under -`/etc/nxdns`. Ownership depends on how you deploy. - -Under systemd, the service runs as the `nxdns` user: - -```sh -chown nxdns:nxdns /etc/nxdns/cert.pem /etc/nxdns/key.pem -chmod 0644 /etc/nxdns/cert.pem -chmod 0600 /etc/nxdns/key.pem -``` - -Under Docker, the container runs as uid 65532 (fixed in the image) and -`/etc/nxdns` is a read-only bind mount, so the container cannot fix -permissions itself — the host-side files must already be readable by that -uid. It has no name on the host or in the scratch image, so chown it -numerically: - -```sh -cd deploy/docker -chown 65532:65532 etc-nxdns/cert.pem etc-nxdns/key.pem -chmod 0644 etc-nxdns/cert.pem -chmod 0600 etc-nxdns/key.pem -``` - -The key must be readable by the user nxdns runs as and only by its owner: -`nxdns check` prints a WARN for a key with any group or other permission -bits, and a FAIL (exit 2) for a cert or key the user cannot read. A -certificate that fails to load at boot while its listener is enabled exits 2. - -Renewals need no restart. A watcher polls both files every 30 seconds and -swaps the new pair in atomically; in-flight connections finish on the old -certificate. To pick up a renewal immediately — for example from a certbot -deploy hook — call `POST /api/certs/reload` (session-authenticated; see -[api.md](api.md)). A reload that fails to parse leaves the old certificate -serving and reports the error. - -## Backup, restore and upgrades - -**Backup** is one command against a stopped or running server: - -```sh -nxdns export --out /some/backup/nxdns-config.zon -``` - -The write is atomic (temp file + rename) and mode 0600, because the file -carries `web.password_hash` — treat backups as secrets. Without `--out` the -export goes to stdout, where file permissions are your redirect's problem. -The query log is deliberately not part of the backup; it is expendable -history. - -**Restore** onto a fresh data directory or over an existing one: - -```sh -nxdns import /some/backup/nxdns-config.zon --force -``` - -**Upgrades:** install the new binary, restart the service. Schema -migrations run automatically at startup (and before `check`, `export` and -`import`), so a database one schema version behind is upgraded in place. -There is no downgrade path; take an export before upgrading. - -## Data directory layout - -Everything lives under the data directory (default `/var/lib/nxdns`, -override with `--data-dir`), mode 0700: - -| Path | What it is | -| --- | --- | -| `config.db` (+ `-wal`, `-shm`) | The configuration database — the single source of truth, including `web.password_hash`. Mode 0600. Back it up via `nxdns export`. | -| `querylog.db` (+ `-wal`, `-shm`) | The query log. Mode 0600 — it records every domain every client asked for. Expendable: if it is missing or unusable it is recreated empty. | -| `blocklists/` | Compiled blocklist snapshots, two files per source: `.list` (exact domains) and `.wild` (wildcards). `.raw.tmp` / `.list.tmp` / `.wild.tmp` files are transient refresh state. | - -## CLI reference - -``` -nxdns [options] -``` - -Flags take both spellings: `--flag value` and `--flag=value`. - -### `run` - -Serves DNS until SIGINT or SIGTERM. - -| Flag | Meaning | -| --- | --- | -| `--data-dir DIR` | Data directory (default `/var/lib/nxdns`). Created at 0700 if missing. | -| `--config FILE` | Seed configuration file (default `/etc/nxdns/config.zon`). Read only when the database is empty. | -| `--web-dev DIR` | Serve the web interface from DIR instead of the embedded assets, with no cache headers. Development only. | - -### `check` - -Validates the configuration and probes the upstreams. Exit 0 when clean, -2 when it found problems, and it always reports every problem, not just the -first. What it checks, in order: - -1. Which source to check (see [source selection](#check-source-selection)). -2. Full validation — the same rules `import` enforces. -3. For each enabled DoH/DoT listener: cert and key are readable (FAIL if - not), key permissions are owner-only (WARN if not). -4. A live probe: one real A query for `example.com` through every enabled - upstream, using the same failover machinery the server uses. A FAIL line - names the upstream and the concrete cause. This probe leaves the machine, - so `check` needs network access to pass. - -| Flag | Meaning | -| --- | --- | -| `--data-dir DIR` | Data directory to look for `config.db` in. | -| `--config FILE` | Check this file instead of the database. | - -#### check source selection - -- `--config FILE` given explicitly: check that file, nothing else. -- Otherwise, if `/config.db` exists: check the database — the - right default, since the database is the truth on a configured server. -- Otherwise, if the default config file exists: check it. -- Otherwise: "nothing to check", exit 2. - -The first line of output always names which source was checked. - -### `export` - -Writes the configuration as ZON to stdout, or atomically at mode 0600 to -`--out FILE`. `--out` paths are relative to the shell's working directory. - -| Flag | Meaning | -| --- | --- | -| `--data-dir DIR` | Data directory holding `config.db`. | -| `--out FILE` | Write to FILE instead of stdout. | - -### `import FILE` - -Validates FILE and replaces the configuration with it. Prints every -validation problem on failure. Refuses a non-empty database without -`--force`. - -| Flag | Meaning | -| --- | --- | -| `--data-dir DIR` | Data directory holding `config.db` (created if missing). | -| `--force` | Replace a database that already has content. | - -### `version` - -Prints the nxdns version, git commit and Zig version. - -### `help` - -Prints usage. `--help` and `-h` do the same. - -## Exit codes - -| Code | Meaning | -| --- | --- | -| 0 | Success. | -| 1 | Runtime failure — I/O, database, out of memory. | -| 2 | A configuration problem the operator can fix, or a `check` that found one. | -| 64 | Usage error — unknown command or flag, missing argument. | - -## Troubleshooting - -### The service exits with code 2 - -Exit 2 always means a configuration you can fix; `nxdns run` prints the -cause and suggests `nxdns check`, which shows the full list. The usual -causes: - -- Empty database and no seed file, or a seed file with no `default` group - or no enabled upstream. On a fresh install this means `config.zon` is - missing, in the wrong place, or invalid. -- A seed or imported file that fails validation — every problem is printed - with its field name. -- `dns.bind_ipv4` / `dns.bind_ipv6` is not an IP address, or a rate limit - is zero (possible only in a hand-edited database; `import` refuses both). -- A DoH/DoT listener is enabled but its certificate or key is unreadable or - unparseable at boot. -- `import` into a non-empty database without `--force`. - -### Port 53 is taken (systemd-resolved) - -On most systemd distributions, `systemd-resolved` owns a stub listener on -`127.0.0.53:53`, and on some setups binds `0.0.0.0:53`. Turn the stub off -and keep resolved for the host's own lookups: - -```sh -mkdir -p /etc/systemd/resolved.conf.d -printf '[Resolve]\nDNSStubListener=no\n' > /etc/systemd/resolved.conf.d/nxdns.conf -systemctl restart systemd-resolved -``` - -If `/etc/resolv.conf` is a symlink to the stub -(`/run/systemd/resolve/stub-resolv.conf`), repoint it at -`/run/systemd/resolve/resolv.conf` so the host still resolves. Do not point -the host running nxdns at nxdns itself if that host is where nxdns resolves -its upstream DoH/DoT hostnames — that is a bootstrap cycle. - -### Disk is filling up - -The disk monitor samples free space and database sizes once a minute and -classifies the state against `disk.warn_free_mb` and `disk.min_free_mb`. -Below the warn threshold it logs the transition; below `min_free_mb` it -gates every non-essential write: the query logger holds its batches, the -client tracker stops persisting, and the blocklist scheduler skips its -refresh passes. DNS keeps answering throughout — resolution never degrades -because the disk is full. The state and the size gauges are visible on -`/metrics` and in the web UI. Recover space (lower -`logging.retention_days`, or delete `querylog.db` with the service -stopped) and writes resume on the next sample. - -### Blocklists are not filtering - -Serving starts even when no blocklist snapshot loads — a household loses -more from DNS that refuses to start than from a window of unfiltered -answers. The startup journal line says either `blocklist generation N` or -`unfiltered (no blocklist snapshot)`. If it says unfiltered, check the -journal for the download or compile warning that preceded it. diff --git a/docs/performance.md b/docs/performance.md deleted file mode 100644 index 8a34df8..0000000 --- a/docs/performance.md +++ /dev/null @@ -1,81 +0,0 @@ -# Performance - -PLAN §18 sets the targets; `tools/bench.zig` measures the three that are -measurable in-process. Run it with: - -``` -zig build bench -Doptimize=ReleaseFast -``` - -Subcommands `filter|cache|compile|all` (default `all`) select a suite; flags -`--domains=N` (default 1,000,000), `--iters=N` (default 200,000) and `--seed=N` -(default 0x5eed) shape the load. The default run is informational; `--assert` -exits non-zero when a target below is exceeded. - -## Targets (PLAN §18) - -| Target | Where it is checked | -| --- | --- | -| Sustained ≥ 100 qps on Raspberry Pi 5 | End-to-end against the real binary on the Pi (see below); not a harness number | -| Blocklist lookup p95 < 1 ms | `bench filter`: `matcher.normalize` + `Snapshot.evaluate` per op | -| Cached response p95 < 5 ms | `bench cache`: `buildKey` + `DnsCache.get` + `packet.setId` per op | -| Memory with ~1M blocked domains < 100 MB | `bench filter`: VmRSS with the 1M-domain snapshot loaded | -| Stripped static binary < 10 MB per arch (< 15 MB with embedded frontend) | CI size assert on the `cross` artifacts | - -## Measured: x86_64 development host (2026-08-02) - -Intel Core i7-14700K, Linux 6.18, Zig 0.16.0, `-Doptimize=ReleaseFast`, -defaults (1,000,000 domains, 200,000 iterations per suite, seed 0x5eed). -**This is not the target platform** — the Pi 5's Cortex-A76 is far slower and -these numbers do not transfer; they establish the harness works and set a -baseline for regressions on the machine development happens on. - -``` -suite ops p50(us) p95(us) p99(us) max(us) -filter 200000 0.11 0.18 0.27 16.41 - blocked 66699/200000, Snapshot.memoryBytes 28.0 MiB, VmRSS 31.8 MiB - target p95 < 1ms: PASS - target VmRSS < 100 MiB: PASS -cache 200000 0.10 0.14 0.17 3.53 - hits 100000/200000, DnsCache.memoryBytes 4.3 MiB, VmRSS 7.6 MiB - target p95 < 5ms: PASS -compile 1000000 wall 96.025ms, 10413949 lines/s, 1000000 domains kept (informational) -``` - -Every in-process §18 target passes on this host: the two latency targets by -three-to-four orders of magnitude, the memory target by about 3x. - -Two memory figures appear on purpose. `Snapshot.memoryBytes` / -`DnsCache.memoryBytes` are the in-repo accounting of the structures themselves -(the regression guard); VmRSS is what the kernel actually holds resident for -the whole process, allocator slack and code included. The truth sits between -them, and the §18 memory target is judged on VmRSS. The filter suite frees the -generated list source before reading VmRSS, so the number reflects the loaded -snapshot rather than the generator. The cache line's VmRSS is lower because the -filter suite's snapshot has been freed by then. - -## Raspberry Pi 5 (target platform) - -To be measured on hardware. One command, run on the Pi: - -``` -zig build bench -Doptimize=ReleaseFast -- --assert -``` - -| Target | Result | -| --- | --- | -| Blocklist lookup p95 < 1 ms | to be measured on hardware | -| Cached response p95 < 5 ms | to be measured on hardware | -| Memory with ~1M blocked domains < 100 MB | to be measured on hardware | -| Sustained ≥ 100 qps | to be measured on hardware, end-to-end | - -The qps target is end-to-end and belongs to the real binary, not the harness: -run `nxdns run` on the Pi and drive it over the LAN with a DNS load generator -(for example `dnsperf`) against real blocklists. - -## Why CI does not gate on performance - -Required CI stays deterministic (AGENTS.md); latency assertions on shared -runners measure the runner's noisy neighbours, not nxdns, and a perf gate that -flakes trains people to re-run it. The bench exists for hardware you control: -run `--assert` on the Pi, where the numbers mean something. diff --git a/docs/api.md b/docs/reference/api.md similarity index 62% rename from docs/api.md rename to docs/reference/api.md index 728286b..1e4b041 100644 --- a/docs/api.md +++ b/docs/reference/api.md @@ -1,13 +1,17 @@ -# nxdns REST API +# REST API reference nxdns serves its admin API itself, on `web.bind:web.port` (default port 8080), -as plain HTTP. TLS termination, where an operator wants it, belongs to a -reverse proxy in front; the session cookie deliberately omits the `Secure` -attribute so the supported plain-HTTP LAN deployment works. +as plain HTTP. TLS termination, where an operator wants it, belongs to a reverse +proxy in front; the session cookie deliberately omits the `Secure` attribute so +the supported plain-HTTP LAN deployment works. -This page is orientation. The machine-readable contract is -`src/web/openapi.yaml`, which the running server hands out unauthenticated at -`GET /api/openapi.yaml`. When this page and the yaml disagree, the yaml wins. +The machine-readable contract is `src/web/openapi.yaml`, which the running +server hands out unauthenticated at `GET /api/openapi.yaml`. Request and +response schemas for every operation live there. When this page and the YAML +disagree, the YAML wins. + +The route table is `src/web/routes.zig`; the [Operations](#operations) table +below carries all 56 of its entries. ## Conventions @@ -16,89 +20,92 @@ This page is orientation. The machine-readable contract is `/api/queries/live` (`text/event-stream`). - Field names are snake_case, matching settings keys and SQL column names. - Every error response carries the envelope `{"error": ""}`. The - message is operator-facing text; internal detail never reaches the wire — - a 500 body is generic and the cause goes to the server log. -- Request bodies are strict: an unknown field is a 400, a body over 1 MiB is - a 413. + message is operator-facing text; internal detail never reaches the wire — a + 500 body is generic and the cause goes to the server log. +- Request bodies are strict: an unknown field is a 400, a body over 1 MiB is a + 413. - A request whose path matches but whose method does not answers 405 with an - `Allow` header. An unknown `/api` path is a JSON 404; unknown non-`/api` - paths fall through to the embedded SPA (`index.html`), so client-side - routing works. + `Allow` header. An unknown `/api` path is a JSON 404; unknown non-`/api` paths + fall through to the embedded SPA (`index.html`), so client-side routing works. - Item routes (`{id}`) match a positive integer id only. -- Mutations to groups, blocklists, rules, local records, forward zones, - clients and client prefixes take effect live. Upstreams and `/api/settings` - are restart-required. +- Mutations to groups, blocklists, rules, local records, forward zones, clients + and client prefixes take effect live. Upstreams and `/api/settings` are + restart-required. ## Authentication -Cookie sessions, in memory, no accounts — one operator password. +Cookie sessions, in memory, no accounts — one operator password. Setting that +password is [set up admin +authentication](../how-to/set-up-admin-authentication.md). -- Authentication is on exactly when `web.password_hash` is set. When no - password is set, every route is open and `POST /api/auth/login` answers +- Authentication is on exactly when `web.password_hash` is set. When no password + is set, every route is open and `POST /api/auth/login` answers `{"authenticated": true, "auth_required": false}` without setting a cookie. -- `POST /api/auth/login` takes `{"password": "..."}`. A correct password - answers 200 with a `Set-Cookie` for `nxdns_session` - (`HttpOnly; SameSite=Lax; Path=/`, `Max-Age` = the session TTL). A wrong - password is a 401; a stored hash the server cannot read is a 500, never a - 401. Login attempts spend rate-limit tokens like any other request, and - argon2id verification is deliberately slow. -- Every route whose auth policy is `session` answers - 401 `{"error": "authentication required"}` without a valid cookie. +- `POST /api/auth/login` takes `{"password": "..."}`. A correct password answers + 200 with a `Set-Cookie` for `nxdns_session` (`HttpOnly; SameSite=Lax; Path=/`, + `Max-Age` = the session TTL). A wrong password is a 401; a stored hash the + server cannot read is a 500, never a 401. Login attempts spend rate-limit + tokens like any other request, and argon2id verification is deliberately slow. +- Every route whose auth policy is `session` answers 401 + `{"error": "authentication required"}` without a valid cookie. - Sessions live `web.session_ttl_hours` (default 24) from login; use does not extend the lifetime. The table holds 32 sessions; a 33rd login evicts the least recently used. Nothing is persisted — a server restart logs every operator out. -- Changing the password through `PUT /api/settings` revokes every live - session immediately; the new password applies without a restart. -- `POST /api/auth/logout` ends the cookie's session and clears the cookie. - It answers 200 whether or not the session was live. +- Changing the password through `PUT /api/settings` revokes every live session + immediately; the new password applies without a restart. +- `POST /api/auth/logout` ends the cookie's session and clears the cookie. The + route is `.session` like any other, so with a password configured the router + answers 401 before the handler runs when the cookie is missing, expired, + revoked or already logged out; only a live session gets the 200. With no + password configured every session route is open and logout answers 200. ## Rate limiting A token bucket per client address: capacity and refill are both -`web.api_rate_limit_per_min` (default 300) per minute, so a page-load burst -up to the capacity is admitted and the long-run rate holds. +`web.api_rate_limit_per_min` (default 300) per minute, so a page-load burst up +to the capacity is admitted and the long-run rate holds. - An over-budget request answers 429 `{"error": "rate limited"}` with a - `Retry-After` header giving the seconds until a token is available - (rounded up, never zero). + `Retry-After` header giving the seconds until a token is available (rounded + up, never zero). - Loopback addresses (127.0.0.0/8 and ::1) are exempt while `web.api_localhost_exempt` is true (the default). -- Exempt routes, which never consult a bucket: `/metrics` and `/api/health` - (a Prometheus scrape must never see 429) and `/api/queries/live` (one - long-lived stream must not drain its address's bucket; it is bounded by - the SSE connection cap instead). -- The limiter tracks at most 4096 addresses. When the table is full and no - slot is reclaimable, requests from unknown addresses are refused with 429. +- Exempt routes, which never consult a bucket: `/metrics` and `/api/health` (a + Prometheus scrape must never see 429) and `/api/queries/live` (one long-lived + stream must not drain its address's bucket; it is bounded by the SSE + connection cap instead). +- The limiter tracks at most 4096 addresses. When the table is full and no slot + is reclaimable, requests from unknown addresses are refused with 429. ## Live query stream (SSE) `GET /api/queries/live` is server-sent events over chunked transfer, `Content-Type: text/event-stream`, `Cache-Control: no-store`. -- The stream opens with `retry: 3000`, so a browser `EventSource` reconnects - on its own after a drop. -- Each query is one frame: `event: query` and a single `data:` line of JSON. - The payload carries the `GET /api/queries` row fields minus `id` (a live - entry precedes persistence): `ts`, `domain`, `client_ip`, `qtype`, - `blocked`, `block_reason`, `response_time_us`, `cache_hit`, `upstream`. +- The stream opens with `retry: 3000`, so a browser `EventSource` reconnects on + its own after a drop. +- Each query is one frame: `event: query` and a single `data:` line of JSON. The + payload carries the `GET /api/queries` row fields minus `id` (a live entry + precedes persistence): `ts`, `domain`, `client_ip`, `qtype`, `blocked`, + `block_reason`, `response_time_us`, `cache_hit`, `upstream`. - A `: ping` comment heartbeat goes out after 15 s of quiet, keeping middleboxes from reaping the idle connection. -- Each subscriber buffers up to 64 entries. A client too slow for the query - rate overflows its buffer and the server ends the stream cleanly after - delivering what the buffer held — queries are never held back for a slow - reader. There is no gap marker: on reconnect, re-sync through - `GET /api/queries`, which has the missed rows. -- Connections per client address are capped at - `web.sse_max_connections_per_ip` (default 3); over the cap is a 429. The - cap binds loopback too. The server holds at most 32 concurrent streams in - total; when all slots are taken, the answer is a 503. +- Each subscriber buffers up to 64 entries. A client too slow for the query rate + overflows its buffer and the server ends the stream cleanly after delivering + what the buffer held — queries are never held back for a slow reader. There is + no gap marker: on reconnect, re-sync through `GET /api/queries`, which has the + missed rows. +- Connections per client address are capped at `web.sse_max_connections_per_ip` + (default 3); over the cap is a 429. The cap binds loopback too. The server + holds at most 32 concurrent streams in total; when all slots are taken, the + answer is a 503. ## Operations -Auth `open` means no session required; `session` means a valid session -cookie is required whenever a password is set. Rate limit `counted` spends a -token; `exempt` never consults the limiter. +Auth `open` means no session is required; `session` means a valid session cookie +is required whenever a password is set. Rate limit `counted` spends a token; +`exempt` never consults the limiter. | Method | Path | Auth | Rate limit | Purpose | |---|---|---|---|---| @@ -159,11 +166,24 @@ token; `exempt` never consults the limiter. | PUT | `/api/settings` | session | counted | Update settings | | POST | `/api/certs/reload` | session | counted | Reload the TLS certificates from disk | -There is no `POST /api/clients`: client rows come from DNS activity or -import, never from the API. +There is no `POST /api/clients`: client rows come from DNS activity or import, +never from the API. + +Static assets are not routes. The router sends unmatched non-`/api` paths to the +embedded SPA before any auth or rate-limit check. + +## Settings keys + +`GET /api/settings` and `PUT /api/settings` speak the `section.field` keys of +[the configuration reference](configuration.md), with the values in their +database spelling — notably `logging.level` is `"error"`, not `"err"`. +Two keys behave differently over the API than in the file: `web.password` is +write-only (accepted on a `PUT`, never returned, hashed before storage), and +`web.password_hash` is neither readable nor directly writable, because a client +that could install a hash could install one whose password it already knows. ## Schemas -Request and response schemas for every operation live in the OpenAPI -document: `src/web/openapi.yaml` in the repository, or -`GET /api/openapi.yaml` from a running server. +Request and response schemas for every operation live in the OpenAPI document: +`src/web/openapi.yaml` in the repository, or `GET /api/openapi.yaml` from a +running server. diff --git a/docs/reference/cli.md b/docs/reference/cli.md new file mode 100644 index 0000000..9287ea0 --- /dev/null +++ b/docs/reference/cli.md @@ -0,0 +1,150 @@ +# CLI reference + +``` +nxdns [options] +``` + +Six subcommands: `run`, `check`, `export`, `import`, `version`, `help`. Source +of truth: `src/cli.zig`. + +Every flag takes both spellings, `--flag value` and `--flag=value`. An attached +value that is empty (`--config=`) is a missing value, not an empty path. +`--force` is boolean and takes no value at all, so `--force=1` is not a spelling +of any flag this program has. A flag is rejected by the subcommand that has no +use for it: `--web-dev` outside `run` is an unknown flag, not a no-op. + +## `run` + +Serves DNS until SIGINT or SIGTERM. + +| Flag | Meaning | +| --- | --- | +| `--data-dir DIR` | Data directory (default `/var/lib/nxdns`). Created at mode 0700 if missing. | +| `--config FILE` | Seed configuration file (default `/etc/nxdns/config.zon`). Read only when the database has never been configured. | +| `--web-dev DIR` | Serve the web interface from DIR instead of the embedded assets, with no cache headers. Development only. | + +Seeding failures are not treated as configuration faults here: a seed file that +is unparseable, oversized or invalid prints its diagnostics and exits 1, not 2. +See [exit codes](#exit-codes). + +## `check` + +Validates the configuration and probes the upstreams. Exit 0 when clean, 2 when +it found problems. It always reports every problem, not just the first. What it +checks, in order: + +1. Which source to check (see [source selection](#source-selection)). +2. Full validation — the same rules `import` enforces. +3. For each enabled DoH/DoT listener: certificate and key are readable (FAIL if + not), and the key's permissions are owner-only (WARN if any group or other + bit is set; the WARN does not change the exit code). +4. A live probe: one real A query for `example.com` through every enabled + upstream, driving the same pool and failover machinery the server uses, with + `upstream.total_timeout_ms` as the per-attempt deadline. A FAIL line names + the upstream and the concrete cause recorded in its health. This probe leaves + the machine, so `check` needs network access to pass. It runs from the + command line but not from unit tests. + +| Flag | Meaning | +| --- | --- | +| `--data-dir DIR` | Data directory to look for `config.db` in. | +| `--config FILE` | Check this file instead of the database. | + +### Source selection + +- `--config FILE` given explicitly: check that file, nothing else. +- Otherwise, if `/config.db` exists: check the database — the right + default, since the database is the truth on a configured server. Pending + schema migrations are applied first, so a `check` immediately after an upgrade + works. +- Otherwise, if the default config file path exists: check it. +- Otherwise: "nothing to check", exit 2. + +The first line of output always names which source was checked. A file larger +than 4 MiB fails with `larger than 4194304 bytes`; a ZON syntax error is +reported with its line and column. + +A missing file is treated differently depending on how the path was reached. An +explicit `--config FILE` that does not exist is an I/O failure: `check failed: +FileNotFound` on stderr, exit 1. A default path that does not exist is one of +the cases above and produces "nothing to check", exit 2. + +## `export` + +Writes the configuration as ZON to stdout, or atomically at mode 0600 to +`--out FILE`. `--out` paths are relative to the shell's working directory, not +to the data directory. The output is canonical: every default emitted, +deterministic ordering, no timestamps, so `export` → `import` → `export` is +byte-identical. + +`export` does not create the data directory; it fails if the directory is not +there. A directory that exists without a `config.db` is not that case: the +database is opened with create semantics, so an empty `config.db` is created and +migrated, and the export is of a default configuration. + +| Flag | Meaning | +| --- | --- | +| `--data-dir DIR` | Data directory holding `config.db`. | +| `--out FILE` | Write to FILE instead of stdout. | + +See [back up and restore](../how-to/back-up-and-restore.md). + +## `import FILE` + +Validates FILE and replaces the whole configuration with it in one transaction. +Prints every validation problem on failure; a failed import leaves the database +untouched. Refuses a database that already has content unless `--force` is +given (`DatabaseNotEmpty`, exit 2). Creates the data directory at mode 0700 if +it is missing. + +`FILE` is positional and may appear before or after the flags. + +| Flag | Meaning | +| --- | --- | +| `--data-dir DIR` | Data directory holding `config.db` (created if missing). | +| `--force` | Replace a database that already has content. | + +## `version` + +Prints two lines: the nxdns version with the git commit, then the Zig version +the binary was built with. Takes no flags and no arguments. + +## `help` + +Prints the usage text to stdout and exits 0. `nxdns --help` and `nxdns -h` do +the same. A usage error prints the same text to stderr and exits 64. + +## Exit codes + +| Code | Meaning | +| --- | --- | +| 0 | Success. | +| 1 | Runtime failure — I/O, database, out of memory. A partial diagnostic report caused by an allocation failure is a runtime failure, not a verdict on the configuration. | +| 2 | A configuration problem the operator can fix, or a `check` that found one. Which faults qualify differs per subcommand; see below. | +| 64 | Usage error — unknown command or flag, a flag without its value, a missing or extra argument. | + +Code 2 is not a single rule shared by every subcommand. `check` and `import` +classify configuration faults; `run` maps only four errors to it and lets +everything else out as a runtime failure. + +`check` exits 2 when validation reported at least one problem, when a +certificate or key named by an enabled listener is unreadable, when a probed +upstream failed, when the file is larger than 4 MiB, when the file has a ZON +syntax error, and when there was nothing to check. Any other error escaping the +run — a missing explicit `--config` file, an unreadable database — prints +`check failed: ` and exits 1. + +`import` exits 2 when validation recorded at least one problem, and for +`DatabaseNotEmpty`, `ConfigTooLarge`, `ParseZon` and `PasswordAndHashBothSet`. +`OutOfMemory` is 1 even when problems were recorded, because the report is then +incomplete. Every other error is 1. + +`run` exits 2 for `NoUsableUpstreams`, `BadBindAddress`, `BadRateLimit` and +`BadCertificate`, and points the operator at `nxdns check` on stderr when it +does. Nothing else is remapped. In particular, seeding the database from +`--config` on first start happens before that classification applies to it: a +seed file that is unparseable (`ParseZon`), larger than 4 MiB +(`ConfigTooLarge`), or rejected by validation prints its diagnostics to stderr +and exits 1. The same file given to `check` or `import` exits 2. + +Where an exit code sends you next: [troubleshoot](../how-to/troubleshoot.md). diff --git a/docs/config-reference.md b/docs/reference/configuration.md similarity index 56% rename from docs/config-reference.md rename to docs/reference/configuration.md index 35b312b..49b1bbf 100644 --- a/docs/config-reference.md +++ b/docs/reference/configuration.md @@ -1,73 +1,51 @@ # Configuration reference Every section, field and collection nxdns accepts, with its type, default, -unit, validation rule and the subsystem that consumes it. Source of truth: -`src/config/model.zig` (the model and defaults), `src/config/validate.zig` -(the rules), `src/config/{bootstrap,import,export}.zig` (the lifecycle). +unit, validation rule and the subsystem that consumes it. -## How configuration works +Source of truth: `src/config/model.zig` (the model and the defaults), +`src/config/validate.zig` (the rules), `src/config/{bootstrap,import,export}.zig` +(the lifecycle). -The database is the truth; the file is a seed. +For how the file, the database and `export`/`import` relate to each other, see +[the configuration model](../explanation/configuration-model.md). For the +commands that read and write configuration, see [the CLI +reference](cli.md). -- On `nxdns run`, the configuration file (default `/etc/nxdns/config.zon`, - overridable with `--config`) is imported into `config.db` **once**: only - when the file exists and the database has never been configured. On every - later start the file is ignored and the database is used as it is - (`src/config/bootstrap.zig`). A file that exists but is unreadable, - unparseable or invalid fails the start — nxdns never falls back to silent - defaults over a file the operator wrote. -- After the seed, changes are made through the web API (or `nxdns import - --force`), never by editing the file. Editing the file after first boot has - no effect. -- `nxdns export` renders the database back as canonical ZON: fixed two-line - header, every default emitted, deterministic ordering, no timestamps. The - round trip `export` → `import` → `export` is byte-identical. With `--out - FILE` the write is atomic and the file is created mode 0600, because the - export carries `web.password_hash`. -- `nxdns import FILE` replaces the whole database content in one transaction. - Without `--force` it refuses a database that already has content - (`error.DatabaseNotEmpty`); a failed import leaves the database untouched. -- `nxdns check` validates without writing. Source selection order: an - explicit `--config FILE` wins; otherwise `config.db` in the data directory - if it exists; otherwise the default config file path if it exists; - otherwise "nothing to check" (exit 2). `check` also verifies TLS - certificate/key readability and, from the command line, probes each enabled - upstream with a real query. +## File format -Absent fields keep their defaults — in the file and in the database. A +The file is ZON: a top-level anonymous struct whose fields are the sections and +collections below. Enum values are ZON enum literals (`.level = .err`, +`.response = .nxdomain`). Strings are double-quoted. The file may be at most +4 MiB (`max_config_bytes` in `src/config/import.zig`); beyond that the error is +`ConfigTooLarge`. A syntax error is reported with its line and column. + +Absent fields keep their defaults, both in the file and in the database. A settings key stored in the database that the running binary does not know is -warned about and ignored, never an error, so a downgrade cannot brick a -config database. +warned about and ignored, never an error. + +### The `logging.level = .err` quirk + +The log level `error` is a Zig keyword, so the ZON and model tag is `.err` while +the database and the settings API store the operator-facing word `"error"`. The +file says `.err`; `GET /api/settings` says `"error"`. `"err"` is not accepted as +database text, and `.error` is not a ZON tag. ## What is not in the file Storage paths are process arguments, not configuration: -- `--data-dir DIR` (default `/var/lib/nxdns`) holds `config.db` and - `querylog.db`. The directory is created mode 0700; both databases and their - WAL sidecars are forced to mode 0600. -- `--config FILE` (default `/etc/nxdns/config.zon`) names the seed file. -- `--web-dev DIR` (`run` only) serves the web interface from a directory - instead of the embedded assets. - -## File format - -The file is ZON: a top-level anonymous struct whose fields are the sections -and collections below. Enum values are written as ZON enum literals -(`.level = .err`, `.response = .nxdomain`). Strings are double-quoted. The -file may be at most 4 MiB (`ConfigTooLarge` beyond that). A syntax error is -reported with its line and column. - -One serialization quirk: the log level `error` is the Zig keyword `error`, -so the ZON/model tag is `.err` while the database stores the operator-facing -word `"error"`. `err` is not accepted as database text, and `error` is not a -ZON tag — the file says `.err`, the settings API says `"error"`. +| Argument | Default | Meaning | +| --- | --- | --- | +| `--data-dir DIR` | `/var/lib/nxdns` | Holds `config.db` and `querylog.db`; see [files and directories](files-and-directories.md). | +| `--config FILE` | `/etc/nxdns/config.zon` | Names the seed file. | +| `--web-dev DIR` | — | `run` only; serves the web interface from a directory instead of the embedded assets. | ## Scalar sections The "Key" column is the settings key as stored in the database (`section.field`); in the file the same field lives inside its section block, -e.g. `.dns = .{ .port = 53 }`. +for example `.dns = .{ .port = 53 }`. ### upstream @@ -75,21 +53,26 @@ Timeouts for talking to upstream resolvers. | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| -| `upstream.read_timeout_ms` | u32 | 3000 | ms | 100–120000 | read deadline on conditional-forward-zone exchanges (`src/server/handler.zig` via `app.zig`) | -| `upstream.total_timeout_ms` | u32 | 5000 | ms | 100–120000, and at least `read_timeout_ms` | per-query budget of the upstream pool (`src/upstream/pool.zig`) — the whole attempt including the connect; also the `nxdns check` probe deadline | +| `upstream.read_timeout_ms` | u32 | 3000 | ms | 100–120000 | read deadline on conditional-forward-zone exchanges (`src/local/forward_client.zig`) | +| `upstream.total_timeout_ms` | u32 | 5000 | ms | 100–120000, and not below `read_timeout_ms` | per-query budget of the upstream pool (`src/upstream/pool.zig`), the whole attempt including the connect; also the `nxdns check` probe deadline | ### dns -The plain DNS listener (UDP + TCP). +The plain DNS listener (UDP and TCP). | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| -| `dns.bind_ipv4` | string | `"0.0.0.0"` | IP address | must parse as an IPv4 address | UDP/TCP listener bind (`src/app.zig`) | -| `dns.bind_ipv6` | string | `"::"` | IP address | must parse as an IPv6 address | UDP/TCP listener bind (`src/app.zig`) | -| `dns.port` | u16 | 53 | port | 1–65535 | UDP/TCP listener port | +| `dns.bind_ipv4` | string | `"0.0.0.0"` | IP address | must parse as an IPv4 literal | UDP/TCP listener bind (`src/app.zig`) | +| `dns.bind_ipv6` | string | `"::"` | IP address | must parse as an IPv6 literal | UDP/TCP listener bind (`src/app.zig`) | +| `dns.port` | u16 | 53 | port | 1–65535 (0 is refused) | UDP/TCP listener port | | `dns.rate_limit` | u32 | 1000 | queries per window | at least 1 | per-client DNS rate limiter (`src/server/rate_limiter.zig`) | | `dns.rate_window_seconds` | u32 | 60 | seconds | 1–3600 | window of the same limiter | +`dns.bind_ipv4` and `dns.bind_ipv6` each name one socket of the dual-stack pair, +so each is required to be a literal of its own family. An IPv4 wildcard in +`dns.bind_ipv6` is refused: it would bind IPv4 as the "v6" socket and make the +real IPv4 bind fail with `AddressInUse`, silently removing the IPv6 service. + ### blocking What a blocked query gets back. @@ -97,13 +80,13 @@ What a blocked query gets back. | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| | `blocking.response` | enum `.zero` \| `.nxdomain` | `.zero` | — | one of the two tags | blocked-response synthesis (`src/filter/response.zig`): `.zero` answers 0.0.0.0 / `::`, `.nxdomain` answers NXDOMAIN | -| `blocking.ttl` | u32 | 5 | seconds | at most 86400 (0 allowed) | TTL on the synthesized block answer | +| `blocking.ttl` | u32 | 5 | seconds | at most 86400; 0 allowed | TTL on the synthesized block answer | ### cache | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| -| `cache.size` | u32 | 10000 | entries | none; `0` disables caching (put short-circuits — `src/cache/dns_cache.zig` test "a cache of zero entries stores nothing") | DNS answer cache capacity (`src/cache/dns_cache.zig`) | +| `cache.size` | u32 | 10000 | entries | none; 0 disables caching (puts short-circuit) | DNS answer cache capacity (`src/cache/dns_cache.zig`) | | `cache.negative_ttl_max` | u32 | 3600 | seconds | at most 86400 | cap on cached negative answers; 0 disables negative caching | ### web @@ -113,25 +96,26 @@ The web interface and REST API. | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| | `web.enabled` | bool | true | — | — | gates the whole web stack: server, sessions, SSE hub, API limiter (`src/app.zig`) | -| `web.bind` | string | `"0.0.0.0"` | IP address | must parse as an IP address | web listener bind (`src/web/server.zig`) | -| `web.port` | u16 | 8080 | port | 1–65535 | web listener port | -| `web.password` | string | `""` | — | must not be set together with `web.password_hash` | operator input only — see "Authentication" below; never stored, never a settings key | +| `web.bind` | string | `"0.0.0.0"` | IP address | must parse as an IP address of either family | web listener bind (`src/web/server.zig`) | +| `web.port` | u16 | 8080 | port | 1–65535 (0 is refused) | web listener port | +| `web.password` | string | `""` | — | must not be set together with `web.password_hash` | operator input only; hashed at import and discarded. Never a settings row — see [Password and hash](#password-and-hash) | | `web.password_hash` | string | `""` | — | — | argon2id PHC string verified at login (`src/web/auth.zig`); `""` disables authentication | -| `web.session_ttl_hours` | u16 | 24 | hours | at least 1 | session expiry and cookie Max-Age (`src/web/auth.zig`) | +| `web.session_ttl_hours` | u16 | 24 | hours | at least 1 | session expiry and cookie `Max-Age` (`src/web/auth.zig`) | | `web.api_rate_limit_per_min` | u32 | 300 | requests per minute | at least 1 | API token-bucket limiter (`src/web/api_limiter.zig`) | | `web.api_localhost_exempt` | bool | true | — | — | loopback requests skip the API limiter | -| `web.sse_max_connections_per_ip` | u16 | 3 | connections | at least 1 | cap on concurrent SSE streams per client IP | +| `web.sse_max_connections_per_ip` | u16 | 3 | connections | at least 1 | cap on concurrent SSE streams per client address | ### doh_server -The DNS-over-HTTPS listener (server side, for clients on the LAN). +The DNS-over-HTTPS listener (server side, for clients on the LAN). See +[enable DoH and DoT](../how-to/enable-doh-and-dot.md). | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| | `doh_server.enabled` | bool | false | — | — | gates the DoH listener (`src/server/doh_server.zig`) | -| `doh_server.bind` | string | `"0.0.0.0"` | IP address | must parse as an IP address | DoH listener bind | -| `doh_server.port` | u16 | 443 | port | 1–65535 | DoH listener port | -| `doh_server.cert_path` | string | `"/etc/nxdns/cert.pem"` | path | non-empty when enabled | certificate loaded into the hot-reloading `CertStore`; readability is checked by `nxdns check`, not the validator | +| `doh_server.bind` | string | `"0.0.0.0"` | IP address | must parse as an IP address of either family | DoH listener bind | +| `doh_server.port` | u16 | 443 | port | 1–65535 (0 is refused, enabled or not) | DoH listener port | +| `doh_server.cert_path` | string | `"/etc/nxdns/cert.pem"` | path | non-empty when enabled | certificate loaded into the hot-reloading `CertStore`; readability is checked by `nxdns check`, not by the validator | | `doh_server.key_path` | string | `"/etc/nxdns/key.pem"` | path | non-empty when enabled | private key for the same; `nxdns check` warns when it is readable beyond its owner | ### dot_server @@ -142,8 +126,8 @@ differs. | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| | `dot_server.enabled` | bool | false | — | — | gates the DoT listener (`src/server/dot_server.zig`) | -| `dot_server.bind` | string | `"0.0.0.0"` | IP address | must parse as an IP address | DoT listener bind | -| `dot_server.port` | u16 | 853 | port | 1–65535 | DoT listener port | +| `dot_server.bind` | string | `"0.0.0.0"` | IP address | must parse as an IP address of either family | DoT listener bind | +| `dot_server.port` | u16 | 853 | port | 1–65535 (0 is refused, enabled or not) | DoT listener port | | `dot_server.cert_path` | string | `"/etc/nxdns/cert.pem"` | path | non-empty when enabled | certificate, shared `CertStore` with hot reload | | `dot_server.key_path` | string | `"/etc/nxdns/key.pem"` | path | non-empty when enabled | private key for the same | @@ -159,21 +143,21 @@ Process log and query log behavior. | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| -| `logging.level` | enum `.err` \| `.warn` \| `.info` \| `.debug` | `.info` | — | one of the four tags; stored as `"error"`/`"warn"`/`"info"`/`"debug"` | log threshold (`src/platform/logging.zig`) | +| `logging.level` | enum `.err` \| `.warn` \| `.info` \| `.debug` | `.info` | — | one of the four tags; stored as `"error"` / `"warn"` / `"info"` / `"debug"` | log threshold (`src/platform/logging.zig`) | | `logging.retention_days` | u16 | 30 | days | at least 1 | query-log pruning cutoff (`src/storage/retention.zig`) and the client tracker's last-seen cutoff (`src/server/clients.zig`) | | `logging.query_log_buffer_max` | u32 | 10000 | entries | at least 1 | in-memory query-log ring size and backpressure cap (`src/storage/logger.zig`) | -| `logging.hide_domains` | bool | false | — | — | query log stores a hidden marker instead of the domain | -| `logging.hide_client_ips` | bool | false | — | — | query log stores a hidden marker instead of the client IP | +| `logging.hide_domains` | bool | false | — | — | the query log stores a hidden marker instead of the domain | +| `logging.hide_client_ips` | bool | false | — | — | the query log stores a hidden marker instead of the client address | | `logging.output` | enum `.stderr` \| `.syslog` \| `.file` | `.stderr` | — | one of the three tags | log sink selection (`src/platform/logging.zig`); `.stderr` and `.syslog` both write to stderr (journald captures it), `.file` rotates | -| `logging.file_path` | string | `"/var/log/nxdns/nxdns.log"` | path | when `output` is `.file`: non-empty absolute path | rotating log file; its directory also feeds the disk monitor. The binary does not create the directory | +| `logging.file_path` | string | `"/var/log/nxdns/nxdns.log"` | path | when `output` is `.file`: non-empty and starting with `/` | rotating log file; its directory also feeds the disk monitor. The binary does not create the directory | | `logging.max_size_mb` | u32 | 50 | MiB | at least 1 | rotation trigger for the log file | -| `logging.max_files` | u8 | 5 | files | at least 1 | rotated generations kept | +| `logging.max_files` | u8 | 5 | files | at least 1 | log files kept in total, the live one included, so the highest rotated generation is `max_files - 1`; the default 5 keeps `nxdns.log` plus `nxdns.log.1` through `nxdns.log.4`, and a value of 1 keeps only the live file, which rotation deletes rather than renames | ### disk -Free-space thresholds for the data directory. When free space falls below -them, the query-log writer, client tracker and blocklist scheduler are -throttled (`src/storage/disk_monitor.zig`). +Free-space thresholds for the data directory. Below them the query-log writer, +the client tracker and the blocklist scheduler are throttled +(`src/storage/disk_monitor.zig`); DNS resolution is never gated. | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| @@ -184,20 +168,20 @@ throttled (`src/storage/disk_monitor.zig`). | Key | Type | Default | Unit | Validation | Consumed by | |---|---|---|---|---|---| -| `blocklist_update.enabled` | bool | true | — | — | blocklist refresh scheduler (`src/filter/manager.zig`); when false only the startup pass runs | +| `blocklist_update.enabled` | bool | true | — | — | blocklist refresh scheduler (`src/filter/manager.zig`); when false the scheduler stops after the startup pass and only a manual refresh runs | | `blocklist_update.interval_hours` | u16 | 24 | hours | at least 1 | sleep between refresh passes and the per-source staleness test | ## Collections Collections are ZON lists of structs. Fields without a default are required. -Runtime columns (first/last seen timestamps, per-source download counters) -are deliberately not part of the model: import sets timestamps to the import -time, and export omits them, which is what keeps the round trip byte-stable. +Runtime columns (first/last seen timestamps, per-source download counters) are +deliberately not part of the model: import sets timestamps to the import time +and export omits them, which is what keeps the round trip byte-stable. ### groups -Client groups. A group named `default` is **required**; every client not -assigned elsewhere lands in it, and import guarantees it keeps database id 1. +Client groups. A group named `default` is required; every client not assigned +elsewhere lands in it, and import guarantees it keeps database id 1. | Field | Type | Default | Validation | |---|---|---|---| @@ -209,20 +193,19 @@ triggers the safe-search rewrite in the query path. ### upstreams -Upstream resolvers. **At least one enabled upstream is required.** +Upstream resolvers. At least one enabled upstream is required. | Field | Type | Default | Validation | |---|---|---|---| -| `url` | string | required | `https://` (DoH) or `tls://` (DoT) endpoint; unique | +| `url` | string | required | an `https://` (DoH) or `tls://` (DoT) endpoint accepted by `transport.Endpoint.parse`; unique | | `priority` | i32 | 100 | — (lower is tried first) | | `enabled` | bool | true | — | -| `tls_name` | string | `""` | DoT only — a `tls_name` on an `https://` upstream is an error; must be a valid domain name | +| `tls_name` | string | `""` | DoT only — a `tls_name` on an `https://` upstream is an error; when set it must be a valid domain name | -Consumed by the upstream pool (`src/upstream/pool.zig`): entries are sorted -by ascending priority and tried in order with failover. `tls_name` sets SNI -and the certificate verification name for a `tls://` upstream written as an -IP literal; empty means "verify by the URL host" -(`src/upstream/dot_client.zig`). +Consumed by the upstream pool (`src/upstream/pool.zig`): entries are sorted by +ascending priority and tried in order with failover. `tls_name` sets SNI and the +certificate verification name for a `tls://` upstream written as an IP literal; +empty means "verify by the URL host" (`src/upstream/dot_client.zig`). ### clients @@ -230,11 +213,11 @@ Known clients with a fixed group assignment. | Field | Type | Default | Validation | |---|---|---|---| -| `ip` | string | required | IP address; unique after canonicalization (`FD00::1` and `fd00:0:0:0:0:0:0:1` collide) | +| `ip` | string | required | an IP address; unique after canonicalization (`FD00::1` and `fd00:0:0:0:0:0:0:1` collide) | | `name` | string | `""` | — (display only, never read by the resolver) | | `group` | string | `"default"` | must name a declared group | -Consumed by the filter engine's exact IP → group lookup +Consumed by the filter engine's exact address-to-group lookup (`src/filter/matcher.zig`). ### client_prefixes @@ -243,7 +226,7 @@ Group assignment by CIDR prefix, for clients without an exact entry. | Field | Type | Default | Validation | |---|---|---|---| -| `prefix` | string | required | CIDR prefix (`192.168.2.0/24`, `fd00:abcd::/48`); unique after canonicalization | +| `prefix` | string | required | a CIDR prefix (`192.168.2.0/24`, `fd00:abcd::/48`); unique after canonicalization | | `group` | string | `"default"` | must name a declared group | | `priority` | i32 | 100 | — (ties on match are broken by lower priority) | @@ -256,14 +239,14 @@ Downloadable blocklists. | Field | Type | Default | Validation | |---|---|---|---| -| `url` | string | required | `http://` or `https://` URL with a host; unique | +| `url` | string | required | an `http://` or `https://` URL with a host; unique | | `name` | string | required | non-empty | | `enabled` | bool | true | — | | `is_suggested` | bool | false | — (web UI hint only, never read by the resolver) | -Consumed by the blocklist manager (`src/filter/manager.zig`): downloaded by -the fetcher, compiled into domain sets; a disabled source is neither -downloaded nor loaded. +Consumed by the blocklist manager (`src/filter/manager.zig`): downloaded by the +fetcher and compiled into domain sets. A disabled source is neither downloaded +nor loaded. ### group_sources @@ -274,13 +257,12 @@ Which groups consult which blocklist sources. | `group` | string | required | must name a declared group | | `source_url` | string | required | must name a declared blocklist source's `url`; the (group, source_url) pair is unique | -Consumed by the filter engine when assembling each group's compiled domain -sets (`src/filter/matcher.zig`). A link to a disabled source is silently -skipped. +Consumed by the filter engine when assembling each group's compiled domain sets +(`src/filter/matcher.zig`). A link to a disabled source is skipped. ### rules -Per-group allow/block overrides, checked before the blocklists. +Per-group allow and block overrides, checked before the blocklists. | Field | Type | Default | Validation | |---|---|---|---| @@ -290,9 +272,9 @@ Per-group allow/block overrides, checked before the blocklists. | `action` | enum `.allow` \| `.block` | required | — | Pattern rules: an `.exact` pattern is a plain domain name and may not contain -`*`; a `.wildcard` pattern must contain at least one label that is exactly -`*` (`*.tracker.example`, or `*` alone), and every other label must be a -legal DNS label. `ads*.example` is not a valid wildcard. +`*`. A `.wildcard` pattern must contain at least one label that is exactly `*` +(`*.tracker.example`, or `*` alone), and every other label must be a legal DNS +label. `ads*.example` is not a valid wildcard. Consumed by the filter engine's rule sets (`src/filter/rules.zig`). @@ -302,48 +284,84 @@ Local DNS answers, served without touching any upstream. | Field | Type | Default | Validation | |---|---|---|---| -| `name` | string | required | valid domain name | -| `rtype` | enum `.a` \| `.aaaa` \| `.cname` | required | stored as `A`/`AAAA`/`CNAME` | -| `value` | string | required | IPv4 address for `.a`, IPv6 for `.aaaa`, domain name for `.cname` | +| `name` | string | required | a valid domain name | +| `rtype` | enum `.a` \| `.aaaa` \| `.cname` | required | stored as `A` / `AAAA` / `CNAME` | +| `value` | string | required | an IPv4 address for `.a`, an IPv6 address for `.aaaa`, a domain name for `.cname` | | `ttl` | u32 | 300 | 1–604800 seconds | -The (name, rtype, value) triple is unique. Consumed by the local records -table in the query path (`src/local/records.zig`). +The (name, rtype, value) triple is unique. Consumed by the local records table +in the query path (`src/local/records.zig`). ### forward_zones -Zones resolved by a specific resolver instead of the configured upstreams — -for LAN or corporate domains. +Zones resolved by a specific resolver instead of the configured upstreams, for +LAN or corporate domains. | Field | Type | Default | Validation | |---|---|---|---| -| `zone` | string | required | valid domain name; unique | +| `zone` | string | required | a valid domain name; unique | | `resolver` | string | required | `udp://IP:port` or `tcp://IP:port`; the host must be an IP literal and the port is mandatory | The resolver host must be an IP literal because resolving the resolver's own name would be a bootstrap problem. Matching is longest suffix -(`src/local/forward_zones.zig`); the exchange is UDP-then-TCP +(`src/local/forward_zones.zig`); the exchange is UDP then TCP (`src/local/forward_client.zig`) with `upstream.read_timeout_ms` as the read deadline. -## Authentication: web.password vs web.password_hash +## Password and hash -Exactly one of the two may be set; setting both is refused -(`PasswordAndHashBothSet` — ambiguity in a security setting). +Exactly one of `web.password` and `web.password_hash` may be set; setting both +is refused (`PasswordAndHashBothSet` — ambiguity in a security setting). - `web.password` is operator input only. At import time it is hashed with argon2id (OWASP parameters: t=2, m=19 MiB, p=1, PHC encoding) into - `web.password_hash` and discarded. It is never stored — there is no - `web.password` settings row, and `nxdns export` always writes - `.password = ""`. -- `web.password_hash` is the stored argon2id PHC string. Supplying it - directly (for example from a previous export) is how a backup restores - authentication without knowing the password. + `web.password_hash` and discarded. There is no `web.password` settings row, + and `nxdns export` always writes `.password = ""`. +- `web.password_hash` is the stored argon2id PHC string. Supplying it directly, + for example from a previous export, is how a backup restores authentication + without knowing the password. - Both empty disables web authentication entirely. -Because the export carries the hash and re-importing an exported file takes -the "password is empty" branch, the export/import round trip preserves the -hash byte-for-byte. +Because the export carries the hash and re-importing an exported file takes the +"password is empty" branch, the export/import round trip preserves the hash +byte for byte. See [set up admin +authentication](../how-to/set-up-admin-authentication.md). + +## Validation errors + +`nxdns check` and `nxdns import` print one `path: message` line per problem and +report every problem, not just the first. The error set is +`validate.ValidateError` in `src/config/validate.zig`: + +| Error | Raised by | +| --- | --- | +| `NoUpstreams` | no upstream has `enabled = true` | +| `BadUpstreamUrl` | an upstream `url` is not an `https://` or `tls://` endpoint | +| `DuplicateUpstreamUrl` | two upstreams share a `url` | +| `BadTlsName` | a `tls_name` is not a valid domain name | +| `TlsNameOnNonTlsUpstream` | a `tls_name` on a non-`tls://` upstream | +| `MissingDefaultGroup` | no group is named `default` | +| `DuplicateGroupName` | two groups share a `name` | +| `EmptyGroupName` | a group `name` is empty | +| `UnknownGroup` | a client, prefix, group source or rule names a group that is not declared | +| `BadClientIp` / `DuplicateClientIp` | a client `ip` is unparseable, or collides after canonicalization | +| `BadClientPrefix` / `DuplicateClientPrefix` | the same for a `client_prefixes.prefix` | +| `BadSourceUrl` / `DuplicateSourceUrl` / `EmptySourceName` | blocklist source fields | +| `UnknownSource` / `DuplicateGroupSource` | `group_sources` links | +| `BadRulePattern` | a rule `pattern` does not match its `kind` | +| `BadLocalRecordName` / `BadLocalRecordValue` / `DuplicateLocalRecord` | local record fields | +| `BadForwardZone` / `DuplicateForwardZone` / `BadResolverUrl` | forward zone fields | +| `BadPort` | a port field is 0 | +| `BadTimeout` | a timeout is outside 100–120000 ms, or `total` is below `read` | +| `BadTtl` | `blocking.ttl`, `cache.negative_ttl_max`, a record `ttl`, `web.session_ttl_hours` or `blocklist_update.interval_hours` outside its range | +| `BadRetention` | `logging.retention_days` or `logging.query_log_buffer_max` below 1 | +| `BadLogRotation` | `logging.max_size_mb` or `logging.max_files` below 1 | +| `BadDiskThresholds` | a threshold below 1, or `min_free_mb` above `warn_free_mb` | +| `BadRateLimit` | `dns.rate_limit`, `dns.rate_window_seconds`, `web.api_rate_limit_per_min` or `web.sse_max_connections_per_ip` out of range | +| `BadBindAddress` | a bind field is not an IP address, or not of the required family | +| `MissingCertPath` / `MissingKeyPath` | an enabled TLS listener with an empty path | +| `MissingLogPath` | `logging.output = .file` with an empty or relative `file_path` | +| `PasswordAndHashBothSet` | both `web.password` and `web.password_hash` are set | ## Minimal working example @@ -448,4 +466,3 @@ upstream. Everything else keeps its default. }, } ``` - diff --git a/docs/reference/files-and-directories.md b/docs/reference/files-and-directories.md new file mode 100644 index 0000000..0d40aa7 --- /dev/null +++ b/docs/reference/files-and-directories.md @@ -0,0 +1,117 @@ +# Files and directories + +Every path nxdns reads or writes, and the mode it is created with. Source of +truth: `src/cli.zig` (`DataDir`), `src/storage/querylog_schema.zig` (the +preserved query-log databases), `src/filter/manager.zig` (the blocklist +snapshots), `src/platform/logging.zig` (the log file). + +## The data directory + +Default `/var/lib/nxdns`, overridable with `--data-dir DIR`. `nxdns run` and +`nxdns import` create it and its parents at mode 0700 when it is missing; +`nxdns check` and `nxdns export` do not create it. `export` fails if it is not +there. `check` opens it only on the branch that resolved to the database, so an +absent data directory is not in itself a failure: an explicit `--config FILE` +never looks at the directory, and without one `check` falls back to the default +configuration file, or prints "nothing to check" and exits 2 when neither source +exists. + +No subcommand opens the directory read-only. `run`, `import`, `export` and a +`check` that resolved to the database all go through the same +`DataDir.openConfigDb`, which opens `config.db` read/write, chmods it to 0600, +enables WAL — creating `config.db-wal` and `config.db-shm` — and then runs any +pending schema migrations. So `nxdns export` and `nxdns check` write to the data +directory, and on a database one schema version behind they migrate it. `export` +additionally creates an empty `config.db` if the directory exists without one; +`check` reaches the database branch only when `config.db` is already there. + +| Path | What it is | Mode | +| --- | --- | --- | +| `config.db` | The configuration database — the single source of truth, including `web.password_hash`. | 0600 | +| `config.db-wal`, `config.db-shm` | SQLite write-ahead log and shared-memory index for `config.db`. Created when WAL is enabled, inheriting the main file's permissions. | 0600 | +| `querylog.db` | The query log: every domain every client asked for. Expendable — if it is missing or unusable it is recreated empty. | 0600 | +| `querylog.db-wal`, `querylog.db-shm` | WAL sidecars for `querylog.db`. | 0600 | +| `querylog.db.corrupt-` | A `querylog.db` that could not be used, moved aside before an empty one was created in its place. Kept, never overwritten. | Whatever the renamed file had — no chmod reaches it | +| `querylog.db.corrupt--` | The same, when the plain name is taken — `` counts from 1 and rises until the name is free. Two recreates within one second is the case it exists for. | The same | +| `blocklists/` | Compiled blocklist snapshots, one subdirectory of the data directory. | 0700 | +| `blocklists/.list` | Exact domains for blocklist source ``, one per line, behind a header. | 0600 | +| `blocklists/.wild` | Wildcard entries for the same source. | 0600 | +| `blocklists/.raw.tmp`, `.list.tmp`, `.wild.tmp` | Transient refresh state: the downloaded body and the two compile outputs before they are published by rename. | 0600 | + +`` is the `blocklist_sources` row id. A `.list` or `.wild` file whose id is +no longer a `blocklist_sources` row is deleted by the orphan sweep; files of a +live source are left alone whatever their state, and the sweep matches only the +two published suffixes, so a `.tmp` file left by an interrupted refresh is not +swept — the next refresh of that source overwrites it. + +A `querylog.db` is moved aside when it is missing nothing but usability: +SQLite reports it corrupt or not a database, `PRAGMA quick_check` does not +answer `ok`, or its `user_version` fingerprint does not match the schema. Only +the main file is renamed — its `-wal` and `-shm` are deleted, because a stale +WAL would be replayed into the fresh database. A missing `querylog.db` is +created without any aside file. The rename happens inside +`querylog_schema.open`, before the 0600 chmod, and that chmod names +`querylog.db` and its two sidecars only — so an aside file keeps the mode the +file had at rename time, which for a `querylog.db` nxdns itself created is 0600 +and for one an operator put there is whatever they left it at. Nothing prunes +the aside files; they accumulate until an operator removes them, and each one +holds the same browsing history the live query log holds. + +The 0600 modes are not cosmetic. `config.db` holds the argon2id password hash +and `querylog.db` holds the browsing history of every client on the LAN, so both +are as sensitive as each other, and a WAL file holds the same rows as the +database it belongs to. SQLite creates the main database at `0644 & ~umask`; +nxdns chmods it to 0600 before enabling WAL, so the sidecars inherit 0600 rather +than being created world-readable. + +## The configuration file + +Default `/etc/nxdns/config.zon`, overridable with `--config FILE`. nxdns reads +it and never writes it: it seeds an unconfigured database once and is ignored +afterwards. nxdns does not create the file or its directory. + +`nxdns export --out FILE` writes a ZON file at mode 0600 through a temporary +file and a rename. That file carries `web.password_hash`, so treat exports as +secrets. Without `--out` the export goes to stdout, where permissions are the +redirect's business. + +## The log file + +Only when `logging.output = .file`. The path is `logging.file_path`, default +`/var/log/nxdns/nxdns.log`, and validation requires it to be absolute. + +**nxdns does not create the log directory.** It must exist and be writable by +the user the service runs as before the process starts; under systemd the unit's +`LogsDirectory=nxdns` does that. + +Rotation triggers at `logging.max_size_mb` MiB and keeps `logging.max_files` +files in total, the live one included, so the highest generation on disk is +`logging.max_files - 1`. With the default 5 that is `nxdns.log` plus `nxdns.log.1` +through `nxdns.log.4`. Rotation deletes the highest generation, renames each +remaining one up by one, then renames the live file to `.1`. A `max_files` of 1 +or 0 deletes the live file instead of renaming it. The directory holding the log +file is also sampled by the disk monitor. + +| Path | What it is | Mode | +| --- | --- | --- | +| `logging.file_path` (default `/var/log/nxdns/nxdns.log`) | The live process log. Opened for append; created when it does not exist. | `0666 & ~umask` — nxdns never chmods it | +| `.1` … `.` | Rotated generations, newest first. | Inherited from the live file they were renamed from | + +The log file is the one path here nxdns does not set a mode on. `config.db` and +`querylog.db` are chmodded to 0600 whatever the umask; the log file is left at +whatever the umask gives it. Under the shipped unit that is 0600, because +`deploy/systemd/nxdns.service` sets `UMask=0077`. Run from a shell with the +usual `umask 022` it is 0644, and `LogsDirectory=nxdns` leaves the directory at +systemd's default 0755, so a log written there is readable by any local user. + +With `logging.output = .stderr` or `.syslog` nxdns writes to stderr and creates +no file; under systemd the journal captures it. + +## Certificate and key files + +`doh_server.cert_path` / `key_path` and `dot_server.cert_path` / `key_path`, +conventionally under `/etc/nxdns`. nxdns reads them, never writes or creates +them. Both must be readable by the user nxdns runs as; the key should be +readable by its owner only, which `nxdns check` warns about when it is not. A +watcher polls both files and swaps a renewed pair in without a restart. See +[enable DoH and DoT](../how-to/enable-doh-and-dot.md). diff --git a/docs/reference/performance.md b/docs/reference/performance.md new file mode 100644 index 0000000..9b79ede --- /dev/null +++ b/docs/reference/performance.md @@ -0,0 +1,69 @@ +# Performance reference + +The PLAN §18 targets and the numbers measured against them. Why the targets are +these targets, and why CI does not gate on them, is [performance and +testing](../explanation/performance-and-testing.md). To reproduce the numbers, +see [measure performance](../how-to/measure-performance.md). + +## Targets (PLAN §18) + +| Target | Where it is checked | +| --- | --- | +| Sustained ≥ 100 qps on Raspberry Pi 5 | End-to-end against the real binary on the Pi; not a harness number | +| Blocklist lookup p95 < 1 ms | `bench filter`: `matcher.normalize` + `Snapshot.evaluate` per op | +| Cached response p95 < 5 ms | `bench cache`: `buildKey` + `DnsCache.get` + `packet.setId` per op | +| Memory with ~1M blocked domains < 100 MB | `bench filter`: VmRSS with the 1M-domain snapshot loaded | +| Stripped static binary < 10 MB per arch (< 15 MB with the embedded frontend) | CI size assert on the `cross` artifacts | + +The harness is `tools/bench.zig`. It measures the three targets that are +measurable in process; the qps target is end to end and the binary-size target +belongs to CI. + +## Measured: x86_64 development host + +Date: 2026-08-02. Hardware and build: Intel Core i7-14700K, Linux 6.18, +Zig 0.16.0, `-Doptimize=ReleaseFast`, harness defaults (1,000,000 domains, +200,000 iterations per suite, seed 0x5eed). + +This is not the target platform. The Pi 5's Cortex-A76 is far slower and these +numbers do not transfer; they establish that the harness works and set a +baseline for regressions on the machine development happens on. + +``` +suite ops p50(us) p95(us) p99(us) max(us) +filter 200000 0.11 0.18 0.27 16.41 + blocked 66699/200000, Snapshot.memoryBytes 28.0 MiB, VmRSS 31.8 MiB + target p95 < 1ms: PASS + target VmRSS < 100 MiB: PASS +cache 200000 0.10 0.14 0.17 3.53 + hits 100000/200000, DnsCache.memoryBytes 4.3 MiB, VmRSS 7.6 MiB + target p95 < 5ms: PASS +compile 1000000 wall 96.025ms, 10413949 lines/s, 1000000 domains kept (informational) +``` + +Every in-process §18 target passes on this host: the two latency targets by +three to four orders of magnitude, the memory target by about 3x. + +### The two memory figures + +`Snapshot.memoryBytes` and `DnsCache.memoryBytes` are the in-repo accounting of +the structures themselves, which is the regression guard. VmRSS is what the +kernel holds resident for the whole process, allocator slack and code included. +The truth sits between them, and the §18 memory target is judged on VmRSS. + +The filter suite frees the generated list source before reading VmRSS, so its +number reflects the loaded snapshot rather than the generator. The cache suite's +VmRSS is lower because the filter suite's snapshot has been freed by then. + +## Raspberry Pi 5 (target platform) + +| Target | Result | +| --- | --- | +| Blocklist lookup p95 < 1 ms | to be measured on hardware | +| Cached response p95 < 5 ms | to be measured on hardware | +| Memory with ~1M blocked domains < 100 MB | to be measured on hardware | +| Sustained ≥ 100 qps | to be measured on hardware, end to end | + +The qps target belongs to the real binary rather than the harness: it means +`nxdns run` on the Pi, driven over the LAN by a DNS load generator against real +blocklists. diff --git a/docs/tutorial/first-run.md b/docs/tutorial/first-run.md new file mode 100644 index 0000000..6389edf --- /dev/null +++ b/docs/tutorial/first-run.md @@ -0,0 +1,300 @@ +# Your first nxdns + +This page takes you from a checkout of the repository to a running nxdns that +answers DNS queries, blocks domains from a real blocklist, and shows you the web +interface. It runs on unprivileged ports in a scratch directory, so nothing on +your machine changes and nothing needs root. At the end you stop the server and +delete the directory. + +Follow the steps in order. Each one says what it did. + +Every command below was executed on x86_64 Linux with Zig 0.16.0, Node.js +24.14.1, dig 9.20.26 and curl 8.21.0. The only thing substituted during that run +was the tutorial directory. + +## What you need + +- Zig 0.16.0 and Node.js 24, both on your `PATH`. +- `dig` (from bind-tools or dnsutils) and `curl`. +- Working internet access: nxdns queries an upstream resolver over HTTPS and + downloads a 3 MB blocklist. +- Two terminals. The server runs in the first one; you type into the second. + +Build commands run from the repository root. The server and the queries use +`~/nxdns-tutorial` for everything they write. + +## 1. Build the web interface + +```sh +cd web && npm ci && npm run build && cd .. +``` + +This produces `web/dist`. Do not skip it. A plain `zig build` embeds +`web/dist-placeholder`, a one-page status stub, and you would reach step 10 and +find no admin interface there. + +## 2. Build nxdns + +```sh +zig build -Dweb-dist=web/dist +``` + +The binary is `zig-out/bin/nxdns`, with `web/dist` embedded in it. SQLite and +mbedTLS are vendored and built by this command, so the first build takes a +while; later builds are cached. + +## 3. Write a configuration file + +```sh +mkdir -p ~/nxdns-tutorial +cat > ~/nxdns-tutorial/config.zon <<'EOF' +.{ + .dns = .{ .bind_ipv4 = "127.0.0.1", .bind_ipv6 = "::1", .port = 15353 }, + .web = .{ .bind = "127.0.0.1", .port = 8080 }, + .groups = .{ + .{ .name = "default" }, + }, + .upstreams = .{ + .{ .url = "https://cloudflare-dns.com/dns-query" }, + }, +} +EOF +``` + +Four things are set, and the rest of nxdns keeps its defaults. + +DNS is on port 15353 instead of 53, and both listeners are bound to localhost. +Port 53 is privileged: binding it needs root or `CAP_NET_BIND_SERVICE`, which is +an install decision, not a first-run decision. A real install is covered in +[how-to/install-with-systemd.md](../how-to/install-with-systemd.md). + +The `default` group and one enabled upstream are the two things nxdns will not +start without. Every client that nxdns has never seen is assigned to `default`, +and with no usable upstream there is nowhere to send a query it cannot answer +itself, so a configuration missing either one is rejected. Seeding happens +inside `run`, so `run` prints the problem and exits 1; `nxdns check` and `nxdns +import` reject the same file with exit code 2. + +## 4. Check the configuration before starting + +```sh +zig-out/bin/nxdns check --data-dir ~/nxdns-tutorial/data --config ~/nxdns-tutorial/config.zon +``` + +``` +checking configuration file /home/you/nxdns-tutorial/config.zon +OK https://cloudflare-dns.com/dns-query +OK: no problems found +``` + +`check` parses the file, validates it, and contacts each upstream to confirm it +answers. It exits 0 when there is nothing to fix and 2 when there is. It does +not start any listener, so you can run it as often as you like. + +## 5. Start the server + +In your first terminal: + +```sh +zig-out/bin/nxdns run --data-dir ~/nxdns-tutorial/data --config ~/nxdns-tutorial/config.zon +``` + +``` +info(migrations): config.db migrated from schema version 0 to 2 +info(config_bootstrap): seeded the database from '/home/you/nxdns-tutorial/config.zon' +info(querylog_schema): created querylog database '/home/you/nxdns-tutorial/data/querylog.db' +info(blocklist_manager): blocklist snapshot generation 1: 0 of 0 sources loaded, 199 bytes +info(nxdns): nxdns 0.1.0-dev serving on udp [::1]:15353 udp 127.0.0.1:15353 tcp [::1]:15353 tcp 127.0.0.1:15353; 1 upstream(s); blocklist generation 1 +info(web_server): web interface listening on 127.0.0.1:8080 +``` + +The data directory did not exist; nxdns created it at mode 0700 along with +`config.db` and `querylog.db`. The line that matters is `seeded the database +from`: the configuration file was read into the database this once. From now on +the database is the truth and the file is ignored on every later start, which +you will see for yourself in step 11. Configuration changes go through the API, +the web interface, or `nxdns import`. + +Leave this terminal running and switch to the second one. + +## 6. Resolve a name + +```sh +dig @127.0.0.1 -p 15353 example.com A +noall +answer +``` + +``` +example.com. 90 IN A 172.66.147.243 +example.com. 90 IN A 104.20.23.154 +``` + +nxdns had no answer cached, so it forwarded the query to +`https://cloudflare-dns.com/dns-query` over HTTPS and returned what came back. +You now have a working resolver. It blocks nothing yet: the log line in step 5 +said `0 of 0 sources loaded`. + +## 7. Add a blocklist source + +```sh +curl -s -X POST http://127.0.0.1:8080/api/blocklists \ + -H 'Content-Type: application/json' \ + -d '{"url":"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts","name":"StevenBlack hosts"}' +``` + +```json +{"id":1,"url":"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts","name":"StevenBlack hosts","enabled":true,"is_suggested":false} +``` + +The API is unauthenticated here because the configuration set no `web.password`. +That is fine for a localhost tutorial and wrong for anything else; see +[how-to/set-up-admin-authentication.md](../how-to/set-up-admin-authentication.md). + +Note the `"id":1` in the response. You need it in the next step. + +## 8. Attach the source to the `default` group + +A blocklist source belongs to the installation. Which groups use it is a +separate decision, which is what lets one group get a strict list and another +get none. A source that is attached to no group is downloaded and then filters +nothing. + +Ask which groups exist: + +```sh +curl -s http://127.0.0.1:8080/api/groups +``` + +```json +{"groups":[{"id":1,"name":"default","safe_search":false}]} +``` + +The `default` group is id 1. Give it the source you just created — the request +body is the complete set of sources for that group, so sending `[1]` replaces +whatever was there: + +```sh +curl -s -X PUT http://127.0.0.1:8080/api/groups/1/sources \ + -H 'Content-Type: application/json' \ + -d '{"source_ids":[1]}' +``` + +```json +{"source_ids":[1]} +``` + +## 9. Download the list + +Adding a source registers it; it does not fetch it. Ask for a refresh: + +```sh +curl -s -X POST http://127.0.0.1:8080/api/blocklists/update +``` + +```json +{"sources":[{"id":1,"state":"ok","loaded":true,"last_attempt":1785683777,"last_success":1785683777,"url":"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts","last_error":"","domains":99277,"wildcards":0,"skipped_regex":0}]} +``` + +The download is about 3 MB and takes a few seconds. Watch the first terminal +until this appears: + +``` +info(blocklist_manager): blocklist snapshot generation 5: 1 of 1 sources loaded, 3088703 bytes +``` + +`1 of 1 sources loaded` is the line to wait for. nxdns builds each blocklist +snapshot in full and swaps it in atomically, so queries keep being answered from +the previous snapshot the whole time the new one is being built. After this, the +domain count in the JSON above — 99277 on the day this was run — is live. + +From here on, the list is on disk under `~/nxdns-tutorial/data/blocklists`. +Restarting nxdns does not re-download it. + +## 10. Watch a domain get blocked + +```sh +dig @127.0.0.1 -p 15353 doubleclick.net A +noall +answer +``` + +``` +doubleclick.net. 5 IN A 0.0.0.0 +``` + +`0.0.0.0` with a TTL of 5 is a block, not an answer. The default block response +is the zero address and the default block TTL is 5 seconds, so a client that +caches the answer forgets it quickly after you unblock something. + +A name that is not on the list still resolves normally: + +```sh +dig @127.0.0.1 -p 15353 wikipedia.org A +noall +answer +``` + +``` +wikipedia.org. 17 IN A 185.15.58.224 +``` + +One thing to know before you try other names: a blocklist entry blocks exactly +the name it names. `doubleclick.net` on the list does not block +`ads.doubleclick.net`; that name is blocked because the list happens to contain +it too. Blocklist entries do not walk up the parent chain — only rules you write +yourself can, with a wildcard pattern such as `*.doubleclick.net`. So when you +pick a domain to test, pick one that is literally in the file. +`www.google-analytics.com` is another that is. + +## 11. Open the web interface + +Visit in a browser. This is the single-page application +you built in step 1, served out of the binary. The dashboard shows query and +block counts, and the Blocklists page shows the source you added with its +domain count. (The endpoints behind those two pages were checked while writing +this; the browser page itself was not opened on the verification host.) + +## 12. Stop it + +In the first terminal, press Ctrl-C. + +``` +info(nxdns): shutting down +``` + +nxdns catches SIGINT and SIGTERM, stops serving and exits 0. + +Start it again with the same command as in step 5 and read the first log line: + +``` +info(config_bootstrap): configuration file ignored; the database is already configured +info(blocklist_manager): blocklist snapshot generation 1: 1 of 1 sources loaded, 3088703 bytes +``` + +The configuration file was not even opened, and the blocklist came off disk +rather than the network. The blocklist source, the group it is attached to, and +the query log all survived the restart because they live in +`~/nxdns-tutorial/data`. + +Press Ctrl-C again to stop this second process. Nothing is listening on 15353 or +8080 now, and nothing of nxdns is running. + +## What you have now + +A resolver that answers real queries, a real blocklist of about 99000 domains +attached to the default group, a query log, and a web interface — all inside one +directory you can delete: + +```sh +rm -rf ~/nxdns-tutorial +``` + +You also saw the two things that surprise people most: the configuration file +seeds the database once and is ignored afterwards, and a blocklist source does +nothing until a group uses it. + +## Where to go next + +- [how-to/install-with-systemd.md](../how-to/install-with-systemd.md) — the same + thing as a real service on port 53, including a Raspberry Pi 5. +- [how-to/set-up-admin-authentication.md](../how-to/set-up-admin-authentication.md) + — put a password on the web interface before it leaves localhost. +- [reference/configuration.md](../reference/configuration.md) — every field you + did not set. +- [explanation/configuration-model.md](../explanation/configuration-model.md) — + why the file seeds and the database rules. diff --git a/specs/milestone-13.md b/specs/milestone-13.md new file mode 100644 index 0000000..24b3448 --- /dev/null +++ b/specs/milestone-13.md @@ -0,0 +1,183 @@ +# Milestone 13: restructure the documentation to Diátaxis + +Goal: replace the four mixed-mode documents with the four Diátaxis modes, add the +missing tutorial, and prove every instruction by running it. + +## Rulings (binding) + +1. **Four directories plus an index.** `docs/tutorial/`, `docs/how-to/`, + `docs/reference/`, `docs/explanation/`, and `docs/README.md` — the index names + the four modes, says who each is for, and links every page. The old + `docs/{operator,architecture,config-reference,api,performance}.md` are deleted + by the orchestrator once the new pages exist; no redirect stubs (greenfield + repo, AGENTS.md forbids compatibility leftovers). + +2. **A document serves one mode.** The defect being fixed is `operator.md`, which + interleaves install procedure, CLI reference, exit-code tables and + troubleshooting in 418 lines. Reference pages state what is; how-to pages state + what to do; explanation states why; the tutorial teaches. When porting text, + move a paragraph to the mode it belongs to rather than keeping it where it was. + +3. **Every command block in `tutorial/` and `how-to/` is executed verbatim by the + session that writes it**, on this host, before the session reports. A command + that cannot run here (needs a Raspberry Pi, root, a domain, hardware TLS) is + marked in the page itself as not verified on this host, with the reason. This + ruling exists because milestone 11's docs were written from source-reading + alone: the first real blocklist download aborted the process (commit 35f2324) + and a stale SPA bundle crashed the settings page — both would have surfaced if + the documented paths had been run. No session may report a page complete on the + strength of having read the code. + +4. **Content is verified against `src/`, not copied from the old pages.** Defaults, + flags, exit codes, paths and route names come from the code at HEAD. Report any + discrepancy found; do not fix source in this milestone (docs-only, ruling 9). + +5. **The tutorial has one guaranteed outcome.** `docs/tutorial/first-run.md` takes + a reader from a clean checkout to a running nxdns that answers a query and + blocks a domain from a real blocklist, on unprivileged ports in a scratch + directory, then stops it cleanly. It teaches by doing and states what the reader + will have at the end. No branching, no options menu, no "you may also". + +6. **File map.** + - `docs/tutorial/first-run.md` — new. + - `docs/how-to/`: `install-with-systemd.md` (incl. the Raspberry Pi 5 aarch64 + binary), `install-with-docker.md`, `upgrade.md`, `troubleshoot.md`, + `enable-doh-and-dot.md`, `set-up-admin-authentication.md`, + `back-up-and-restore.md`, `measure-performance.md`. + - `docs/reference/`: `configuration.md` (every section, field, default, range), + `api.md` (all 56 operations), `cli.md` (six subcommands, every flag, exit + codes), `files-and-directories.md` (data dir layout, file modes), + `performance.md` (targets + measured numbers). + - `docs/explanation/`: `architecture.md`, `configuration-model.md` (the file + seeds the database once, the database is truth, export/import round trip), + `performance-and-testing.md` (why the targets exist, why CI does not gate on + them, what hermetic tests do and do not prove). + +7. **Drift guards repointed and kept honest.** `docs/docs.zig` embeds + `reference/api.md`, `reference/configuration.md`, `reference/cli.md`. + `src/docs_drift_test.zig` keeps its three guards against those paths: every + served route as a full `| METHOD | \`pattern\` |` row; every `toSettings` key + verbatim; every subcommand as its own heading. `reference/cli.md` gives each + subcommand a `## \`name\`` heading so the guard anchors on structure, not prose. + +8. **Style.** Plain sentences. No banned vocabulary (leverage, seamless, robust as + filler, rule-of-three padding, "it's important to note"). No badges, no + marketing. Code blocks are copy-pasteable and use the scratch paths the + tutorial establishes, not `/var/lib/nxdns`, unless the page is about a real + install. + +9. **Docs-only.** The only non-docs edits are the orchestrator's: `docs/docs.zig`, + `src/docs_drift_test.zig`, `README.md` links, and PLAN.md's repo-layout line. + No behavior changes. + +## Sessions + +X1, X2, X3, X4, X5 run in parallel — every session owns distinct files. + +## Session X1: index + tutorial + README links + +Owns `docs/README.md`, `docs/tutorial/first-run.md`, `README.md` (links section +only). Rulings 1, 5, 3 (the tutorial is executed end to end). + +## Session X2: how-to, install and operations + +Owns `docs/how-to/{install-with-systemd,install-with-docker,upgrade,troubleshoot}.md`. +Rulings 2, 3, 4. + +## Session X3: how-to, security and measurement + +Owns `docs/how-to/{enable-doh-and-dot,set-up-admin-authentication,back-up-and-restore,measure-performance}.md`. +Rulings 2, 3, 4. + +## Session X4: reference + +Owns `docs/reference/{configuration,api,cli,files-and-directories,performance}.md`. +Rulings 2, 4, 7 (heading convention). + +## Session X5: explanation + +Owns `docs/explanation/{architecture,configuration-model,performance-and-testing}.md`. +Rulings 2, 4. + +## File ownership + +X1 docs/README.md + docs/tutorial/* + README.md; X2 and X3 disjoint files under +docs/how-to/; X4 docs/reference/*; X5 docs/explanation/*. Orchestrator: +docs/docs.zig, src/docs_drift_test.zig, PLAN.md, deletion of the old pages, spec. + +## Delivered + +Seventeen pages: `docs/README.md`, one tutorial, eight how-to guides, five +reference pages, three explanation pages. The five old documents are deleted. + +The orchestrator repointed `docs/docs.zig` at `reference/{api,configuration,cli}.md` +and `src/docs_drift_test.zig` at the same three, moved the subcommand anchor from +`### \`name` to `## \`name`, fixed the README Quickstart link that pointed at the +deleted `operator.md`, and corrected PLAN.md's repo-layout line and its §13.2 +OpenAPI paragraph (which still promised a renderer and contract tests that were +never built). + +All three guards were proven able to fail: deleting one API table row, one +settings key row and one `## \`export` heading each produced a named build +failure. The settings guard was strengthened during review — it searched for a +bare key, which prose and the annotated example could satisfy after a field's row +was deleted; it now anchors on `| \`key\` |`. + +Ruling 3 held. The tutorial ran end to end twice, the second time after the review +fixes. DoH and DoT answered real queries, `POST /api/certs/reload` returned both +its success and its failure payload, the export/import round trip was +byte-identical, a Docker image was built and its container served DNS, and the +offline password change was proven (old password 401, new 200). What could not run +here — root-only steps, a second LAN host, a Raspberry Pi, a registry push, the +full-scale bench, `dnsperf` — is marked in the page that documents it, with the +reason. + +Five review rounds: 15 findings, then 8, then 1, then 1, then clean. One round-1 +finding was rejected on evidence (`troubleshoot.md` already stated the verified +exit codes; the contradiction was in `reference/cli.md`). + +## Discrepancies found, not fixed + +Docs-only per ruling 9. These are source-side and belong to a later milestone: + +1. `nxdns run` exits 1 for seed-file errors (`ParseZon`, `MissingDefaultGroup`, + `NoUpstreams`) while `check` and `import` exit 2 for the same file, because + `app.isConfigFault` lists only `NoUsableUpstreams`, `BadBindAddress`, + `BadRateLimit`, `BadCertificate`. +2. `nxdns check` prints `OK: no problems found` and exits 0 after emitting WARN + lines. +3. `nxdns check` never parses a certificate or tests that the key matches it — it + checks path readability and key mode only. A green `check` is followed by + `run` exiting 2 with `BadCertificate` on a mismatched pair. +4. `check --config ` exits 1; the implicit path prints `nothing to + check` and exits 2. +5. A blocklist source attached to no group is silently inert, and `validate.zig` + has no diagnostic for it. +6. `nxdns check` says it validates without writing, but the `config.db` branch + chmods, enables WAL and runs migrations. +7. `pruneOrphans` matches only `.list`/`.wild`, so an orphaned `.raw.tmp` is never + swept. +8. `isEmpty` counts auto-materialised client rows, so a server that has answered + one query ignores a seed file placed afterwards. +9. PLAN §18 and the docs say 10 MB / 15 MB; CI asserts 10 MiB / 15 MiB. + +## Acceptance (milestone complete) + +- [x] The five old `docs/*.md` files are gone; the four directories and the index + exist with every page in ruling 6. +- [x] Every command in `tutorial/` and `how-to/` was executed on this host, or is + marked in-page as unverified with a reason. +- [x] `zig build test` passes with the three drift guards pointing at the new + reference pages: 16/16 steps, 1175 pass, 113 skip (integration-gated), 0 failed. +- [x] `zig build test -Dintegration` passes: 1284 pass, 4 skip (the live-network + TLS tests excluded by milestone-1 design), 0 failed. No src/ behavior change. +- [x] No page mixes modes: no procedure in `reference/`, no field table in any + `how-to/` or `tutorial/` page. + +## Anti-requirements + +- No new documentation subjects (no metrics-families reference, no security-model + page) — this milestone moves and splits existing content plus the tutorial. +- No redirect stubs or `docs/legacy/`. +- No source fixes; report discrepancies instead. +- No doc generator, no site builder, no MkDocs. diff --git a/src/docs_drift_test.zig b/src/docs_drift_test.zig index cdd61f0..87d3a57 100644 --- a/src/docs_drift_test.zig +++ b/src/docs_drift_test.zig @@ -1,5 +1,6 @@ //! Textual-containment guards that keep the hand-written docs honest -//! (milestone-11 ruling 3). They assert presence, not correctness — the same +//! (milestone-11 ruling 3, repointed at the Diátaxis reference pages by +//! milestone-13 ruling 7). They assert presence, not correctness — the same //! contract as openapi.zig's route guard. const std = @import("std"); @@ -7,7 +8,7 @@ const docs = @import("docs_files"); const routes = @import("web/routes.zig"); const model = @import("config/model.zig"); -test "every served operation has its own table row in docs/api.md" { +test "every served operation has its own table row in docs/reference/api.md" { const gpa = std.testing.allocator; for (routes.table) |route| { // Matches one full method + path cell pair ("| GET | `/api/groups` |"), @@ -17,14 +18,14 @@ test "every served operation has its own table row in docs/api.md" { @tagName(route.method), route.pattern, }); defer gpa.free(needle); - if (std.mem.indexOf(u8, docs.api_md, needle) == null) { - std.debug.print("operation row missing from docs/api.md: {s}\n", .{needle}); + if (std.mem.indexOf(u8, docs.reference_api_md, needle) == null) { + std.debug.print("operation row missing from docs/reference/api.md: {s}\n", .{needle}); return error.OperationMissingFromApiDoc; } } } -test "every settings key appears in docs/config-reference.md" { +test "every settings key appears in docs/reference/configuration.md" { const gpa = std.testing.allocator; var pairs: std.ArrayList(model.SettingPair) = .empty; defer { @@ -33,25 +34,30 @@ test "every settings key appears in docs/config-reference.md" { } try model.toSettings(.{}, gpa, &pairs); for (pairs.items) |pair| { - if (std.mem.indexOf(u8, docs.config_reference_md, pair.key) == null) { - std.debug.print("settings key missing from docs/config-reference.md: {s}\n", .{pair.key}); + // Matches the key's own leading table cell ("| `dns.port` |"), not a + // bare mention: prose and the annotated example both name most keys, so + // a substring search would stay green after a field's row is deleted. + const needle = try std.fmt.allocPrint(gpa, "| `{s}` |", .{pair.key}); + defer gpa.free(needle); + if (std.mem.indexOf(u8, docs.reference_configuration_md, needle) == null) { + std.debug.print("settings key row missing from docs/reference/configuration.md: {s}\n", .{needle}); return error.SettingsKeyMissingFromConfigDoc; } } } -test "every cli subcommand has its own reference heading in docs/operator.md" { +test "every cli subcommand has its own reference heading in docs/reference/cli.md" { const gpa = std.testing.allocator; const subcommands = [_][]const u8{ "run", "check", "export", "import", "version", "help" }; for (subcommands) |name| { - // Anchors on the reference-section heading ("### `import FILE`" starts - // with "### `import"), so prose mentions elsewhere cannot mask a - // removed command section. - const needle = try std.fmt.allocPrint(gpa, "### `{s}", .{name}); + // Anchors on the reference-section heading ("## `import FILE`" starts + // with "## `import"), so prose mentions elsewhere cannot mask a removed + // command section. + const needle = try std.fmt.allocPrint(gpa, "## `{s}", .{name}); defer gpa.free(needle); - if (std.mem.indexOf(u8, docs.operator_md, needle) == null) { - std.debug.print("subcommand heading missing from docs/operator.md: {s}\n", .{needle}); - return error.SubcommandMissingFromOperatorDoc; + if (std.mem.indexOf(u8, docs.reference_cli_md, needle) == null) { + std.debug.print("subcommand heading missing from docs/reference/cli.md: {s}\n", .{needle}); + return error.SubcommandMissingFromCliDoc; } } }