Compare commits
11
Commits
d596fd788a
...
v0.0.20
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6e9a36903e
|
||
|
|
f6fa43a8b4
|
||
|
|
f4562cac26
|
||
|
|
250bdca7e7
|
||
|
|
63f4f96a48
|
||
|
|
52158198cf
|
||
|
|
bf79a22584
|
||
|
|
bcd9bce16c
|
||
|
|
2b790c4c3e
|
||
|
|
1860ff59f5
|
||
|
|
f168247b33
|
+61
-45
@@ -41,6 +41,12 @@ on:
|
|||||||
|
|
||||||
env:
|
env:
|
||||||
ZIG_VERSION: "0.16.0"
|
ZIG_VERSION: "0.16.0"
|
||||||
|
# The bytes of that zig, not just its version string: a distribution package
|
||||||
|
# of the same version is built against a different LLVM and emits different
|
||||||
|
# machine code, so its tarballs hash to something no CI run reproduces.
|
||||||
|
# setup-zig verifies these same bytes by minisign; `zig build cut` checks this
|
||||||
|
# digest before it builds a release.
|
||||||
|
ZIG_TARBALL_SHA256: "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00"
|
||||||
# Exact patch, not a floating "24" (milestone-14 ruling 12): the bundled npm
|
# Exact patch, not a floating "24" (milestone-14 ruling 12): the bundled npm
|
||||||
# and the emitted bundle change under a floating major.
|
# and the emitted bundle change under a floating major.
|
||||||
NODE_VERSION: "24.19.0"
|
NODE_VERSION: "24.19.0"
|
||||||
@@ -287,52 +293,11 @@ jobs:
|
|||||||
-Dadmin-dist=admin-dist-ci \
|
-Dadmin-dist=admin-dist-ci \
|
||||||
-Doptimize=ReleaseSafe
|
-Doptimize=ReleaseSafe
|
||||||
|
|
||||||
# The cut writes the release hashes into flake.nix before it makes the
|
# This upload runs before the pin check below: a pin mismatch is exactly
|
||||||
# bump commit, so the bump commit is the one commit whose pins nothing has
|
# the failure whose diagnosis needs the built bytes, so they must already
|
||||||
# verified yet — the tag's own run (release.yml) is the next chance, and by
|
# be downloadable when that step fails. The container job's dependency on
|
||||||
# then the tag is public. This step is that first chance.
|
# the artifact is unchanged.
|
||||||
#
|
#
|
||||||
# It is deliberately NOT part of verify-dist. Every other commit on master
|
|
||||||
# builds the same build.zig.zon version from a different tree, so its bytes
|
|
||||||
# legitimately differ from the pins and a check there would fail the whole
|
|
||||||
# branch. The commit is identified by the version it declares, not by its
|
|
||||||
# message: a message is a string anyone can write, and the pins follow
|
|
||||||
# the manifest.
|
|
||||||
#
|
|
||||||
# A root commit has no first parent. That is an error rather than a skip:
|
|
||||||
# this repository has history, so `HEAD^` failing means the checkout is
|
|
||||||
# shallower than the depth 2 declared above and the question went
|
|
||||||
# unanswered, which must never read as "nothing to check".
|
|
||||||
#
|
|
||||||
# The predicate is the declared VERSION, not the file: build.zig.zon also
|
|
||||||
# carries the dependency pins, and updating a sqlite or mbedTLS hash
|
|
||||||
# changes the file without cutting a release. Such a commit builds the
|
|
||||||
# same version from a different tree, so its bytes are not the pinned
|
|
||||||
# ones and this check would fail it.
|
|
||||||
#
|
|
||||||
# HEAD's version is CI_VERSION, parsed out of the working tree by the
|
|
||||||
# container gate tool. The parent's is read with `sed`, because that tool
|
|
||||||
# reads `build.zig.zon` at a fixed path and has no mode for a blob out of
|
|
||||||
# history. An empty parse is a failure, not a bump: it means the manifest
|
|
||||||
# moved and the question went unanswered.
|
|
||||||
- name: Verify the flake pins on a version bump
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
parent="$(git rev-parse --verify HEAD^)"
|
|
||||||
parent_version="$(git show "$parent":build.zig.zon | sed -n 's/^[[:space:]]*\.version = "\([^"]*\)".*/\1/p')"
|
|
||||||
if [ -z "$parent_version" ]; then
|
|
||||||
echo "cannot read .version out of $parent:build.zig.zon" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
if [ "$parent_version" != "$CI_VERSION" ]; then
|
|
||||||
zig build verify-pins \
|
|
||||||
-Dversion-string="$CI_VERSION" \
|
|
||||||
-Dadmin-dist=admin-dist-ci \
|
|
||||||
-Doptimize=ReleaseSafe
|
|
||||||
else
|
|
||||||
echo "skipped: $GITHUB_SHA declares version $CI_VERSION and $parent already declared $parent_version, so it is not a version bump and its bytes are not the ones flake.nix pins"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# deploy/docker/Dockerfile copies both of these trees and nothing else
|
# deploy/docker/Dockerfile copies both of these trees and nothing else
|
||||||
# out of zig-out/dist: the binary comes from dist/bin/<triple>/, and
|
# out of zig-out/dist: the binary comes from dist/bin/<triple>/, and
|
||||||
# /LICENSE and /THIRD-PARTY-NOTICES come from the matching dist/stage/
|
# /LICENSE and /THIRD-PARTY-NOTICES come from the matching dist/stage/
|
||||||
@@ -360,6 +325,57 @@ jobs:
|
|||||||
zig-out/bin/container_check
|
zig-out/bin/container_check
|
||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
|
|
||||||
|
# The cut writes the release hashes into flake.nix before it makes the
|
||||||
|
# bump commit, so the bump commit is the one commit whose pins nothing has
|
||||||
|
# verified yet — the tag's own run (release.yml) is the next chance, and by
|
||||||
|
# then the tag is public. This step is that first chance.
|
||||||
|
#
|
||||||
|
# It is deliberately NOT part of verify-dist. Every other commit on master
|
||||||
|
# builds the same build.zig.zon version from a different tree, so its bytes
|
||||||
|
# legitimately differ from the pins and a check there would fail the whole
|
||||||
|
# branch. The commit is identified by the version it declares, not by its
|
||||||
|
# message: a message is a string anyone can write, and the pins follow
|
||||||
|
# the manifest.
|
||||||
|
#
|
||||||
|
# A root commit has no first parent. That is an error rather than a skip:
|
||||||
|
# this repository has history, so `HEAD^` failing means the checkout is
|
||||||
|
# shallower than the depth 2 declared above and the question went
|
||||||
|
# unanswered, which must never read as "nothing to check".
|
||||||
|
#
|
||||||
|
# The predicate is the declared VERSION, not the file: build.zig.zon also
|
||||||
|
# carries the dependency pins, and updating a sqlite or mbedTLS hash
|
||||||
|
# changes the file without cutting a release. Such a commit builds the
|
||||||
|
# same version from a different tree, so its bytes are not the pinned
|
||||||
|
# ones and this check would fail it.
|
||||||
|
#
|
||||||
|
# A commit that rewrites flake.nix runs the check too: a re-pin during a
|
||||||
|
# cut changes the pins without changing the version, and those pins must
|
||||||
|
# be verified before the tag as well.
|
||||||
|
#
|
||||||
|
# HEAD's version is CI_VERSION, parsed out of the working tree by the
|
||||||
|
# container gate tool. The parent's is read with `sed`, because that tool
|
||||||
|
# reads `build.zig.zon` at a fixed path and has no mode for a blob out of
|
||||||
|
# history. An empty parse is a failure, not a bump: it means the manifest
|
||||||
|
# moved and the question went unanswered.
|
||||||
|
- name: Verify the flake pins on a version bump
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
parent="$(git rev-parse --verify HEAD^)"
|
||||||
|
parent_version="$(git show "$parent":build.zig.zon | sed -n 's/^[[:space:]]*\.version = "\([^"]*\)".*/\1/p')"
|
||||||
|
if [ -z "$parent_version" ]; then
|
||||||
|
echo "cannot read .version out of $parent:build.zig.zon" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ "$parent_version" != "$CI_VERSION" ] || ! git diff --quiet "$parent" HEAD -- flake.nix; then
|
||||||
|
zig build verify-pins \
|
||||||
|
-Dversion-string="$CI_VERSION" \
|
||||||
|
-Dadmin-dist=admin-dist-ci \
|
||||||
|
-Doptimize=ReleaseSafe
|
||||||
|
else
|
||||||
|
echo "skipped: $GITHUB_SHA declares version $CI_VERSION, $parent already declared $parent_version and flake.nix is unchanged, so it is neither a version bump nor a re-pin and its bytes are not the ones flake.nix pins"
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
container:
|
container:
|
||||||
needs: [package]
|
needs: [package]
|
||||||
runs-on: ubuntu-24.04
|
runs-on: ubuntu-24.04
|
||||||
|
|||||||
@@ -60,6 +60,12 @@ env:
|
|||||||
REGISTRY_HOST: "git.mial.net"
|
REGISTRY_HOST: "git.mial.net"
|
||||||
|
|
||||||
ZIG_VERSION: "0.16.0"
|
ZIG_VERSION: "0.16.0"
|
||||||
|
# The bytes of that zig, not just its version string: a distribution package
|
||||||
|
# of the same version is built against a different LLVM and emits different
|
||||||
|
# machine code, so its tarballs hash to something no CI run reproduces.
|
||||||
|
# setup-zig verifies these same bytes by minisign; `zig build cut` checks this
|
||||||
|
# digest before it builds a release.
|
||||||
|
ZIG_TARBALL_SHA256: "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00"
|
||||||
# Exact patch, not a floating "24" (ruling 12).
|
# Exact patch, not a floating "24" (ruling 12).
|
||||||
NODE_VERSION: "24.19.0"
|
NODE_VERSION: "24.19.0"
|
||||||
NPM_VERSION: "11.17.0"
|
NPM_VERSION: "11.17.0"
|
||||||
|
|||||||
@@ -4,6 +4,24 @@ All notable changes to nxdns are recorded here. The format follows [Keep a Chang
|
|||||||
|
|
||||||
Sections are written by hand. Nothing here is generated from commit messages: the point of the file is to say what changed for an operator, which a commit subject rarely does.
|
Sections are written by hand. Nothing here is generated from commit messages: the point of the file is to say what changed for an operator, which a commit subject rarely does.
|
||||||
|
|
||||||
|
## [0.0.20] - 2026-09-12
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **A blocklist source no longer fails on a stale pooled connection.** The fetcher reused a keep-alive connection from an earlier pass that the server had closed, and the standard client never retries one; every later download from that host failed with `HttpConnectionClosing` at 0 ms. Downloads now send `connection: close` and never enter the pool.
|
||||||
|
|
||||||
|
## [0.0.19] - 2026-09-12
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **A failed blocklist download says why.** The warning line and the Diagnostics event detail now carry the phase that failed, the concrete cause behind the classification (a reset connection, a truncated chunk, a TLS handshake fault), the HTTP status if a head arrived, the bytes received, and the elapsed time. The separate `http status` warning is folded into that line. Same one line per failed source per pass as before.
|
||||||
|
|
||||||
|
## [0.0.18] - 2026-09-09
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **The Overview scope pickers no longer sit apart, and a long device name no longer pushes the period picker off a phone screen.** The Device and Period dropdowns fill their wrappers and stand side by side; a long name truncates with an ellipsis in the trigger and stays fully readable in the list.
|
||||||
|
|
||||||
## [0.0.17] - 2026-09-08
|
## [0.0.17] - 2026-09-08
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
@@ -60,10 +60,14 @@ const styles = stylex.create({
|
|||||||
toolbar: {
|
toolbar: {
|
||||||
display: "flex",
|
display: "flex",
|
||||||
gap: "0.5rem",
|
gap: "0.5rem",
|
||||||
|
// Without this the row's minimum is both labels at full length, and a
|
||||||
|
// long device name pushes the period picker past the viewport edge.
|
||||||
|
minWidth: 0,
|
||||||
flexBasis: { default: null, [NARROW]: "100%" },
|
flexBasis: { default: null, [NARROW]: "100%" },
|
||||||
},
|
},
|
||||||
control: {
|
control: {
|
||||||
minWidth: { default: "11rem", [NARROW]: 0 },
|
minWidth: { default: "11rem", [NARROW]: 0 },
|
||||||
|
maxWidth: { default: "20rem", [NARROW]: "none" },
|
||||||
flex: { default: null, [NARROW]: 1 },
|
flex: { default: null, [NARROW]: 1 },
|
||||||
},
|
},
|
||||||
/** Under the Device control: the list behind it did not load, so the control offers the household only. */
|
/** Under the Device control: the list behind it did not load, so the control offers the household only. */
|
||||||
|
|||||||
@@ -80,6 +80,9 @@ const styles = stylex.create({
|
|||||||
},
|
},
|
||||||
/** Rectangular, hairline, full hit height: the decision record's toolbar selector. */
|
/** Rectangular, hairline, full hit height: the decision record's toolbar selector. */
|
||||||
toolbar: {
|
toolbar: {
|
||||||
|
// A button keeps its content width even as a flex container, so without
|
||||||
|
// this the scope picker's 11rem wrapper shows as dead space beside it.
|
||||||
|
width: "100%",
|
||||||
minHeight: metrics.hitTarget,
|
minHeight: metrics.hitTarget,
|
||||||
borderRadius: metrics.radius,
|
borderRadius: metrics.radius,
|
||||||
borderWidth: 1,
|
borderWidth: 1,
|
||||||
|
|||||||
@@ -334,6 +334,18 @@ pub fn build(b: *std.Build) void {
|
|||||||
});
|
});
|
||||||
test_step.dependOn(&b.addRunArtifact(dist_stage_tests).step);
|
test_step.dependOn(&b.addRunArtifact(dist_stage_tests).step);
|
||||||
|
|
||||||
|
// `gen_admin_assets` decides which files reach the binary, and that set must
|
||||||
|
// not depend on the bundle's provenance, so it is tested here too.
|
||||||
|
const gen_admin_assets_tests = b.addTest(.{
|
||||||
|
.name = "gen-admin-assets-tool",
|
||||||
|
.root_module = b.createModule(.{
|
||||||
|
.root_source_file = b.path("tools/gen_admin_assets.zig"),
|
||||||
|
.target = b.graph.host,
|
||||||
|
.optimize = optimize,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
test_step.dependOn(&b.addRunArtifact(gen_admin_assets_tests).step);
|
||||||
|
|
||||||
addDist(b, options, admin_assets, .{
|
addDist(b, options, admin_assets, .{
|
||||||
.version = version_option,
|
.version = version_option,
|
||||||
.version_string = version_string,
|
.version_string = version_string,
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
.{
|
.{
|
||||||
.name = .nxdns,
|
.name = .nxdns,
|
||||||
.version = "0.0.17",
|
.version = "0.0.20",
|
||||||
.minimum_zig_version = "0.16.0",
|
.minimum_zig_version = "0.16.0",
|
||||||
.paths = .{""},
|
.paths = .{""},
|
||||||
.fingerprint = 0x3307b311dded1d91,
|
.fingerprint = 0x3307b311dded1d91,
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Install nxdns with Nix
|
# Install nxdns with Nix
|
||||||
|
|
||||||
Adds nxdns to a NixOS machine as a flake input pinned to a release tag. At the end `pkgs`-style references to `inputs.nxdns.packages.${system}.default` resolve to the published release binary, and Renovate opens a pull request when a new tag appears.
|
Adds nxdns to a NixOS machine as a flake input pinned to a release tag. At the end `pkgs`-style references to `inputs.nxdns.packages.${system}.default` resolve to the published release binary, and a Renovate custom manager opens a pull request when a new tag appears.
|
||||||
|
|
||||||
nxdns publishes its own `flake.nix`. Its packages do not build nxdns from source: each one fetches the release tarball for the target and pins its SHA-256 hash, so a changed byte fails the build. The two supported systems are `aarch64-linux` and `x86_64-linux`, both static musl builds that need nothing on the host.
|
nxdns publishes its own `flake.nix`. Its packages do not build nxdns from source: each one fetches the release tarball for the target and pins its SHA-256 hash, so a changed byte fails the build. The two supported systems are `aarch64-linux` and `x86_64-linux`, both static musl builds that need nothing on the host.
|
||||||
|
|
||||||
@@ -57,19 +57,29 @@ The derivation installs `bin/nxdns`, plus `LICENSE` and `THIRD-PARTY-NOTICES` un
|
|||||||
|
|
||||||
## 4. Let Renovate bump the tag
|
## 4. Let Renovate bump the tag
|
||||||
|
|
||||||
Enable the Nix manager in the consuming repository's `renovate.json`:
|
Renovate's built-in `nix` manager does not do this. It only advances `flake.lock` along the ref an input already tracks, and a `refs/tags/vX.Y.Z` ref never moves, so it reports no releases for a tag-pinned input. Verified on 2026-09-09 with Renovate 42.99.0 against a Gitea host: all flake inputs, GitHub-hosted ones included, came back with an empty release list.
|
||||||
|
|
||||||
|
What works is a regex custom manager that treats the tag in the input URL as a version string, with the `gitea-tags` datasource:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||||
"extends": ["config:recommended"],
|
"extends": ["config:recommended"],
|
||||||
"nix": {
|
"customManagers": [
|
||||||
"enabled": true
|
{
|
||||||
}
|
"customType": "regex",
|
||||||
|
"managerFilePatterns": ["/^flake\\.nix$/"],
|
||||||
|
"matchStrings": ["git\\+https://git\\.mial\\.net/mokhtar/nxdns\\.git\\?ref=refs/tags/(?<currentValue>v\\d+\\.\\d+\\.\\d+)"],
|
||||||
|
"depNameTemplate": "mokhtar/nxdns",
|
||||||
|
"datasourceTemplate": "gitea-tags",
|
||||||
|
"registryUrlTemplate": "https://git.mial.net",
|
||||||
|
"versioningTemplate": "semver"
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
Renovate reads the flake inputs and the lock file, sees the `refs/tags/v0.0.16` ref, and opens a pull request when a newer tag exists. It proposes only tags that already exist, so it never pins a release that has not been cut.
|
Verified on 2026-09-09: with this manager, Renovate opened the pull request for v0.0.18 on the day of the tag. That pull request rewrites the tag in `flake.nix` only. `flake.lock` still records the old revision, so the same pull request must also refresh the lock, either with a `postUpgradeTasks` command (`nix flake update nxdns`, which needs the self-hosted `allowedCommands` setting) or with a CI job on the Renovate branch that commits the lock. The datasource proposes only tags that already exist, so it never pins a release that has not been cut.
|
||||||
|
|
||||||
Between the tag push and the asset upload there is a window in which the tag exists and the release tarballs do not. A pin written by hand during that window evaluates, then fails at build time with a 404 from the release download URL. Wait for the release to be published, or re-run the build once it is. Renovate's own pull requests are not affected by the window in practice, because it runs on a schedule rather than on the tag push.
|
Between the tag push and the asset upload there is a window in which the tag exists and the release tarballs do not. A pin written by hand during that window evaluates, then fails at build time with a 404 from the release download URL. Wait for the release to be published, or re-run the build once it is. Renovate's own pull requests are not affected by the window in practice, because it runs on a schedule rather than on the tag push.
|
||||||
|
|
||||||
|
|||||||
@@ -268,7 +268,7 @@ The signing key is a subkey rather than the primary key, which limits the damage
|
|||||||
|
|
||||||
Build the same version from source and compare the hashes. Match the toolchain first: `ZIG_VERSION`, `NODE_VERSION` and `NPM_VERSION` at the top of `.gitea/workflows/gates.yml` are the exact versions the release used, and a different patch release of any of them changes the bytes.
|
Build the same version from source and compare the hashes. Match the toolchain first: `ZIG_VERSION`, `NODE_VERSION` and `NPM_VERSION` at the top of `.gitea/workflows/gates.yml` are the exact versions the release used, and a different patch release of any of them changes the bytes.
|
||||||
|
|
||||||
The host matters too: the admin bundle is built with host-native Rolldown and Lightning CSS bindings, which the lockfile ships per platform and libc, so the recipe reproduces the release only on x86_64 Linux with glibc, the runner CI uses. The two `npm_config_*` paths must not exist on your machine; the recipe relies on npm finding no config file there. The environment matters as much as the toolchain. The release cut and CI both build the bundle under exactly nine variables and a `022` umask, so the recipe below does the same: a locale, a time zone, a build timestamp or a file mode picked up from your shell each move the bytes. The Zig build gets a private `--cache-dir` so a stale local cache cannot leak into them; the global Zig cache stays shared, because it is content-addressed and a fresh one refetches every dependency (see `specs/release-cut.md`).
|
Zig has to be the official ziglang.org tarball for that version, not your distribution's package of it. A distro package of `0.16.0` is built against the system LLVM and emits different machine code than the official build, and both print `0.16.0`, so the version string tells you nothing. This is not hypothetical: the first cut of `flake.nix` pinned hashes CI could not reproduce for exactly that reason, and the release cut now downloads the official tarball itself rather than trusting `PATH`. Fetch and verify it the same way, against the `ZIG_TARBALL_SHA256` pin that sits next to `ZIG_VERSION` in `gates.yml` (CI installs the same bytes through setup-zig, which checks them by minisign):
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
git clone https://git.mial.net/mokhtar/nxdns
|
git clone https://git.mial.net/mokhtar/nxdns
|
||||||
@@ -276,11 +276,22 @@ cd nxdns
|
|||||||
git checkout "v$VERSION"
|
git checkout "v$VERSION"
|
||||||
git verify-tag "v$VERSION"
|
git verify-tag "v$VERSION"
|
||||||
|
|
||||||
|
ZIG_VERSION=$(sed -n 's/^ ZIG_VERSION: "\(.*\)"$/\1/p' .gitea/workflows/gates.yml)
|
||||||
|
ZIG_TARBALL_SHA256=$(sed -n 's/^ ZIG_TARBALL_SHA256: "\(.*\)"$/\1/p' .gitea/workflows/gates.yml)
|
||||||
|
curl -fsSL -o ../zig.tar.xz "https://ziglang.org/download/$ZIG_VERSION/zig-x86_64-linux-$ZIG_VERSION.tar.xz"
|
||||||
|
(cd .. && echo "$ZIG_TARBALL_SHA256 zig.tar.xz" | sha256sum -c - && tar -xJf zig.tar.xz)
|
||||||
|
ZIG="$(cd .. && pwd)/zig-x86_64-linux-$ZIG_VERSION/zig"
|
||||||
|
"$ZIG" version
|
||||||
|
```
|
||||||
|
|
||||||
|
The host matters too: the admin bundle is built with host-native Rolldown and Lightning CSS bindings, which the lockfile ships per platform and libc, so the recipe reproduces the release only on x86_64 Linux with glibc, the runner CI uses. The two `npm_config_*` paths must not exist on your machine; the recipe relies on npm finding no config file there. The environment matters as much as the toolchain. The release cut and CI both build the bundle under exactly nine variables and a `022` umask, so the recipe below does the same: a locale, a time zone, a build timestamp or a file mode picked up from your shell each move the bytes. The Zig build gets a private `--cache-dir` so a stale local cache cannot leak into them; the global Zig cache stays shared, because it is content-addressed and a fresh one refetches every dependency (see `specs/release-cut.md`).
|
||||||
|
|
||||||
|
```sh
|
||||||
env -i PATH="$PATH" HOME="$HOME" LC_ALL=C LANG=C TZ=UTC SOURCE_DATE_EPOCH=0 CI=true npm_config_userconfig=/nonexistent/npmrc-user npm_config_globalconfig=/nonexistent/npmrc-global \
|
env -i PATH="$PATH" HOME="$HOME" LC_ALL=C LANG=C TZ=UTC SOURCE_DATE_EPOCH=0 CI=true npm_config_userconfig=/nonexistent/npmrc-user npm_config_globalconfig=/nonexistent/npmrc-global \
|
||||||
sh -c 'cd admin && umask 022 && npm ci && npm run build'
|
sh -c 'cd admin && umask 022 && npm ci && npm run build'
|
||||||
|
|
||||||
env -i PATH="$PATH" HOME="$HOME" LC_ALL=C LANG=C TZ=UTC SOURCE_DATE_EPOCH=0 CI=true npm_config_userconfig=/nonexistent/npmrc-user npm_config_globalconfig=/nonexistent/npmrc-global \
|
env -i PATH="$PATH" HOME="$HOME" LC_ALL=C LANG=C TZ=UTC SOURCE_DATE_EPOCH=0 CI=true npm_config_userconfig=/nonexistent/npmrc-user npm_config_globalconfig=/nonexistent/npmrc-global \
|
||||||
sh -c 'umask 022 && exec zig build dist \
|
sh -c 'umask 022 && exec "'"$ZIG"'" build dist \
|
||||||
-Dversion-string="'"$VERSION"'" \
|
-Dversion-string="'"$VERSION"'" \
|
||||||
-Dadmin-dist=admin/dist -Doptimize=ReleaseSafe \
|
-Dadmin-dist=admin/dist -Doptimize=ReleaseSafe \
|
||||||
--cache-dir "$(mktemp -d)"'
|
--cache-dir "$(mktemp -d)"'
|
||||||
@@ -296,7 +307,7 @@ The tarball is written by `zig build dist` itself rather than by the host's `tar
|
|||||||
|
|
||||||
Compare your two tarball hashes against two things: the published `SHA256SUMS.txt`, and the `hashes` block of `flake.nix` at the tag, which carries the same digests in SRI form. All three agree on a matching toolchain, and the release pipeline fails before it uploads anything if they do not.
|
Compare your two tarball hashes against two things: the published `SHA256SUMS.txt`, and the `hashes` block of `flake.nix` at the tag, which carries the same digests in SRI form. All three agree on a matching toolchain, and the release pipeline fails before it uploads anything if they do not.
|
||||||
|
|
||||||
A hash that differs is a signal to check the toolchain and the environment first. An unpinned Zig, Node or npm version is the ordinary explanation, and a build run outside the normalized environment above is the next one. Rule both out before you conclude anything about the release itself.
|
A hash that differs is a signal to check the toolchain and the environment first. A distribution's Zig package instead of the official tarball is the first thing to rule out; an unpinned Node or npm version is the next ordinary explanation, and a build run outside the normalized environment above is the next one. Rule both out before you conclude anything about the release itself.
|
||||||
|
|
||||||
> Verified against `v0.0.1`, before the build was reproducible: the recipe ran from a fresh clone, `git verify-tag v0.0.1` printed `Good signature` under the release subkey, and the rebuilt tarball hashes did not match the published `SHA256SUMS.txt` (Node 24.14.1 against the pinned 24.19.0, a different build path, and an archive written by the host's `tar`). Releases from 0.0.17 on are built and checked by the pinned pipeline this page describes, and the cut records the local hashes in `flake.nix` before CI rebuilds them.
|
> Verified against `v0.0.1`, before the build was reproducible: the recipe ran from a fresh clone, `git verify-tag v0.0.1` printed `Good signature` under the release subkey, and the rebuilt tarball hashes did not match the published `SHA256SUMS.txt` (Node 24.14.1 against the pinned 24.19.0, a different build path, and an archive written by the host's `tar`). Releases from 0.0.17 on are built and checked by the pinned pipeline this page describes, and the cut records the local hashes in `flake.nix` before CI rebuilds them.
|
||||||
|
|
||||||
|
|||||||
@@ -8,10 +8,10 @@
|
|||||||
let
|
let
|
||||||
# `zig build cut` rewrites the lines between the delimiters by text replacement.
|
# `zig build cut` rewrites the lines between the delimiters by text replacement.
|
||||||
# BEGIN GENERATED BY zig build cut
|
# BEGIN GENERATED BY zig build cut
|
||||||
version = "0.0.17";
|
version = "0.0.20";
|
||||||
hashes = {
|
hashes = {
|
||||||
"aarch64-linux" = "sha256-BwWuS4OhtBTY+VbRQztHg8ZmlqRv2QioCt92NprBlrY=";
|
"aarch64-linux" = "sha256-2p86FImDgi4SDDzmV6w0qkrkp9dtDRyi/ZMOZ8WAEm4=";
|
||||||
"x86_64-linux" = "sha256-hSgsXuLRcoOJvlcnuOuQ0ifS12i6gAmIVBFWMCopZQY=";
|
"x86_64-linux" = "sha256-rnTpjo7J9uY+pINxJ4ZgwbqU7E2o7N0gr9z7VRhQiZ0=";
|
||||||
};
|
};
|
||||||
# END GENERATED BY zig build cut
|
# END GENERATED BY zig build cut
|
||||||
|
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ The tool finds the two delimiter lines by their text after leading whitespace, k
|
|||||||
|
|
||||||
Where CI runs the pin check:
|
Where CI runs the pin check:
|
||||||
|
|
||||||
- `.gitea/workflows/gates.yml` package job: after `verify-dist`, run `zig build verify-pins` only when the pushed commit changed `build.zig.zon` (compare `HEAD` with its first parent; the checkout needs depth 2). That is the bump commit, the one commit whose pins are otherwise unverified before the tag. Every other commit skips the step, and the skip is printed, not silent.
|
- `.gitea/workflows/gates.yml` package job: after `verify-dist` and after the release payload is uploaded (so a mismatch still leaves the bytes downloadable), run `zig build verify-pins` when the pushed commit changed the declared `.version` against its first parent or changed `flake.nix` against it (the checkout needs depth 2). That covers the bump commit and a re-pin during a cut, the commits whose pins are otherwise unverified before the tag. Every other commit skips the step, and the skip is printed, not silent.
|
||||||
- `.gitea/workflows/release.yml` publish job: after `verify-dist` and before the image push, the draft, and every upload, run `zig build verify-pins` unconditionally. The tag's tree is the bump commit's tree, and the flake block's version must equal the tag.
|
- `.gitea/workflows/release.yml` publish job: after `verify-dist` and before the image push, the draft, and every upload, run `zig build verify-pins` unconditionally. The tag's tree is the bump commit's tree, and the flake block's version must equal the tag.
|
||||||
|
|
||||||
The remaining stages (push, ci.yml wait, tag, release.yml wait) are unchanged. CI rebuilds the same bytes from the same inputs; `verify-pins` in the package job fails the run before the tag if the bytes differ, and the same step in the publish job fails before any image push, draft, or upload. `specs/release-cut.md` gains the pin stage and the parity requirement.
|
The remaining stages (push, ci.yml wait, tag, release.yml wait) are unchanged. CI rebuilds the same bytes from the same inputs; `verify-pins` in the package job fails the run before the tag if the bytes differ, and the same step in the publish job fails before any image push, draft, or upload. `specs/release-cut.md` gains the pin stage and the parity requirement.
|
||||||
@@ -81,6 +81,8 @@ Deviation (Session D): the pin stage is tested in pieces rather than on a fixtur
|
|||||||
|
|
||||||
Signing the flake outputs, a Hydra or cachix binary cache, Darwin packages, and a NixOS module. Renovate tag discovery on Gitea is a consumer-side acceptance test run once on `rpi.mial.net` after the first tagged release with a flake.
|
Signing the flake outputs, a Hydra or cachix binary cache, Darwin packages, and a NixOS module. Renovate tag discovery on Gitea is a consumer-side acceptance test run once on `rpi.mial.net` after the first tagged release with a flake.
|
||||||
|
|
||||||
|
Result of that test (2026-09-09, v0.0.18, Renovate 42.99.0): no pull request. Renovate's `nix` manager advances `flake.lock` along the ref an input already tracks and never moves a `refs/tags/vX.Y.Z` ref; it returned an empty release list for every flake input, GitHub-hosted ones included. The flake itself is correct and v0.0.17 deployed through it. Tag bumps need a regex custom manager with the `gitea-tags` datasource plus a lock refresh in the same pull request, which the consumer owns; `docs/how-to/install-with-nix.md` section 4 now says so instead of claiming the `nix` manager does it. With that manager in place, Renovate opened rpi.mial.net PR 5 the same day, bumping the input to v0.0.18 and refreshing `flake.lock`: the acceptance test passes.
|
||||||
|
|
||||||
## Acceptance
|
## Acceptance
|
||||||
|
|
||||||
- `zig build test` green, including the two-path reproducibility test.
|
- `zig build test` green, including the two-path reproducibility test.
|
||||||
|
|||||||
@@ -1377,3 +1377,26 @@ Deviations from the text above, recorded after the first six sessions verified.
|
|||||||
**S9 integration + fuzz.** Case 3 splits into the two real failure modes: a checksum-mismatched file marks the source `.load_failed` and the reload succeeds without it (generation advances); only a checksum-clean but malformed body fails `reload` and leaves the previous snapshot serving. Case 7 proves the cap from the response head: an explicit `content-length` of 100 MiB returns `error.BodyTooLarge` before any body streams. Case 8 back-dates `last_updated` through `updateSourceStats` and asserts inode and mtime of the compiled file are unchanged. Case 15 asserts `budget/2 <= elapsed < 2*budget` (the POSIX backend wakes ~0.8 ms early on a 200 ms deadline). The fuzz corpus is inline — `tests/fuzz/corpus.zig` imports the `dns` module, which the blocklist-fuzz module does not have. Fuzz targets assert properties, not only absence of crashes: `Line.text` windows the input (pointer containment), `covers_apex` only on `.wildcard`, `detectFormat`'s answer survives `parseLine` over the same bytes, and `matches` is exercised on rejected patterns.
|
**S9 integration + fuzz.** Case 3 splits into the two real failure modes: a checksum-mismatched file marks the source `.load_failed` and the reload succeeds without it (generation advances); only a checksum-clean but malformed body fails `reload` and leaves the previous snapshot serving. Case 7 proves the cap from the response head: an explicit `content-length` of 100 MiB returns `error.BodyTooLarge` before any body streams. Case 8 back-dates `last_updated` through `updateSourceStats` and asserts inode and mtime of the compiled file are unchanged. Case 15 asserts `budget/2 <= elapsed < 2*budget` (the POSIX backend wakes ~0.8 ms early on a 200 ms deadline). The fuzz corpus is inline — `tests/fuzz/corpus.zig` imports the `dns` module, which the blocklist-fuzz module does not have. Fuzz targets assert properties, not only absence of crashes: `Line.text` windows the input (pointer containment), `covers_apex` only on `.wildcard`, `detectFormat`'s answer survives `parseLine` over the same bytes, and `matches` is exercised on rejected patterns.
|
||||||
|
|
||||||
**Final wiring.** `build.zig` gained the `blocklist-fuzz` artifact (module import `parsers` → `src/filter/parsers.zig`, LLVM backend under `-Dfuzz`), hung off `test_step` beside the dns fuzz artifact. Evaluation: `zig build test`, `zig build test -Dintegration` and `zig build cross` all exit 0; both cross executables are statically linked.
|
**Final wiring.** `build.zig` gained the `blocklist-fuzz` artifact (module import `parsers` → `src/filter/parsers.zig`, LLVM backend under `-Dfuzz`), hung off `test_step` beside the dns fuzz artifact. Evaluation: `zig build test`, `zig build test -Dintegration` and `zig build cross` all exit 0; both cross executables are statically linked.
|
||||||
|
|
||||||
|
## Addendum (2026-09-12): a download failure names its cause
|
||||||
|
|
||||||
|
Observed on the Pi (nxdns 0.0.17): four of eight sources fail on every refresh pass since 2026-09-11, the same four each time, and the log line for each is `download failed: TlsFailed` one pass and `download failed: ReceiveFailed` the next. The same binary loads all eight from another machine. The line cannot tell a TLS alert from a reset connection from a truncated chunk, because `fetcher.mapError` collapses the cause into the six-member taxonomy and `reportDownloadFailure` logs only the taxonomy name. The Diagnostics event carries the same text. A reader of either has nothing to act on. `upstream/doh_client.zig` already unwraps the concrete cause that `std.http.Client` stashes on the connection (`sendCause`, `headCause`, `bodyCause`, `readCause`); the fetcher does not.
|
||||||
|
|
||||||
|
Change:
|
||||||
|
|
||||||
|
- `fetcher.zig`: `Fetcher` gains `last_failure: ?Failure`, cleared at the top of `fetch` beside `last_status`, set on every error return. `Failure` is `{ phase: Phase, cause: anyerror, status: ?std.http.Status, bytes_read: u64 }`. `Phase` gains `receive_body`, distinct from the head. `cause` is the unwrapped concrete error: the three unwrap helpers move from `doh_client.zig` into `upstream/transport.zig` (or a sibling file both import; the coder picks the smaller diff), keep their tests, and both clients call them. The taxonomy `Error` and `mapError` are unchanged; a caller that ignores `last_failure` sees exactly what it saw before. `bytes_read` counts body bytes delivered to the writer before the failure, so a body that dies at byte 0 and one that dies at 5 MB read differently.
|
||||||
|
- `manager.zig`: `fetchWithin` measures elapsed milliseconds on the awake clock. `reportDownloadFailure` takes the failure and elapsed time and logs one line: `blocklist <label>: download failed: <taxonomy> (<phase>, cause <CauseName>, http <status or none>, <bytes> bytes, <ms> ms)`. On the `Timeout` outcome of the expiry race the losing fetch is cancelled before the line is built, and it records itself on the way out, so the line names the phase it was in, `cause Canceled`, the status if a head arrived, and the bytes delivered so far: `Timeout (receive_body, cause Canceled, http 200, 5242880 bytes, 300000 ms)`. `download` clears `last_failure` and `last_status` before anything else runs, so a null record means no fetch reached the network for this source (a local file failure, or no concurrency to spawn the race); that prints as `<taxonomy> (no fetch, http none, - bytes, <ms> ms)`. Elapsed is captured when the race resolves, before the loser is cancelled, so a timeout's milliseconds are the time the transfer was given. `SourceStatus.fail` receives the same compact text, so the Diagnostics event detail shows it; `max_error_len` rises from 128 to 192 so the longest cause name and the numbers fit without truncation (check the longest `std.crypto.tls.Client.ReadError` and `std.http.Client` error names; `events.Store.max_detail_len` is 512 and is not touched).
|
||||||
|
- Bound: this changes the content of an existing warning line, not its frequency. One line per failed source per pass, at most `sources × passes` per day; a pass runs once per `blocklist_update.interval_hours` or on a manual update. No per-chunk or per-retry logging is added. Nothing new is written at info or debug level.
|
||||||
|
- Tests: `mapError` tests unchanged; a test that a head failure with a stashed `ConnectionResetByPeer` yields `last_failure.cause == error.ConnectionResetByPeer` and phase `receive` (the doh tests' stub connection pattern); a test that a body failure records `bytes_read` and `receive_body`; a `reportDownloadFailure` format test with the fixed text `blocklist source 3 'ads' 'https://lists.example': download failed: ReceiveFailed (receive_body, cause HttpChunkTruncated, http 200, 1048576 bytes, 812 ms)`; the existing secrecy assertions on the label stay.
|
||||||
|
|
||||||
|
Out of scope: a TLS handshake failure stays `TlsInitializationFailed`, because `std.http.Client` collapses every handshake fault at `Client.zig:1470` before this code sees it; distinguishing an alert from a certificate failure would mean driving `std.crypto.tls.Client` by hand. Retries and a per-source backoff are not added.
|
||||||
|
|
||||||
|
## Addendum (2026-09-12): a blocklist download never reuses a pooled connection
|
||||||
|
|
||||||
|
The line above found the cause the same day nxdns 0.0.19 reached the Pi. A manual refresh at 14:28 failed the same four sources with one record each: `ReceiveFailed (receive, cause HttpConnectionClosing, http none, 0 bytes, 0 ms)`. Under one millisecond of elapsed time, and no response head byte after the request was sent, means the peer had already closed the socket. `std.http.Client` returns `HttpConnectionClosing` when a reused connection hits EOF before the first head byte (`std/http.zig:400`), and `std/http/test.zig:1331` pins that a client never retries a stale pooled connection. The startup pass at 13:03 had fetched `big.oisd.nl` successfully and the client parked that keep-alive connection in `fetch_http`'s pool; the server closed it long before 14:28, and the next request to the same host picked it up. The same shape explains every earlier observation: the first fetch to a host in a process succeeds, a later fetch to that host fails, and the daily pass reuses connections a day old. The `TlsFailed` variant of the older lines is the same dead socket failing inside the TLS layer instead of at the head.
|
||||||
|
|
||||||
|
Change: `fetcher.fetch` requests with `.keep_alive = false`, and `fetch_http` in app.zig is created with `connection_pool.free_size = 0`, so `release` destroys every connection regardless of the request flag (`Client.zig:133`), including the one a failed send leaves in the `.ready` state that `Request.deinit` would otherwise pool, and no connection is ever taken from that pool. The flag alone is not enough: `Client.request` takes a pooled connection before it stores the flag (`Client.zig:1725`, `:1742`). The request then carries `connection: close`, the client marks the connection closing at the head, and `release` destroys it instead of pooling it (`Client.zig:133`, `:1167`). A blocklist download is one bulk transfer per source per day; there is nothing to gain from a pooled connection and a stale one costs a source for a day. `fetch_http` stays a separate client from `dns_http`, so the DoH pool is untouched.
|
||||||
|
|
||||||
|
Tests: an integration test against `HttpFixture` where the server closes the socket after each response: two consecutive fetches to the same URL both succeed. Without the change the second fails with `HttpConnectionClosing`, which is the exact record the Pi produced. A unit test that the request header the fetcher sends carries `connection: close` if the fixture route is the cheaper path.
|
||||||
|
|
||||||
|
Out of scope: a retry on `HttpConnectionClosing` for the DoH client, which keeps pooling by design and has its own health logic.
|
||||||
|
|||||||
@@ -80,10 +80,10 @@ Fixing a FAIL is a sentence in the changelog, not a flag: there is no override,
|
|||||||
|
|
||||||
This amends step 3 of the sequence above. What was one write and one commit is now three moves, in this order and no other:
|
This amends step 3 of the sequence above. What was one write and one commit is now three moves, in this order and no other:
|
||||||
|
|
||||||
1. **Toolchain parity.** Read `ZIG_VERSION`, `NODE_VERSION` and `NPM_VERSION` out of the top-level `env:` block of `.gitea/workflows/gates.yml` — a line parser, not a YAML library — and refuse unless `zig version`, `node --version` (less its leading `v`) and `npm --version` report those exact strings, and unless the host is x86_64 Linux with glibc (the admin bundle uses host-native Rolldown and Lightning CSS bindings, shipped per platform and libc, and CI is an x86_64 Ubuntu runner). The cut and CI must build with one toolchain, because the cut writes the hashes and CI recomputes them. A mismatch found here costs nothing; found in the release run it costs a public tag. The pins live in `gates.yml` and are read from there rather than copied, so the two can never drift. Both halves run before the manifest is rewritten, so a refusal leaves the working tree clean.
|
1. **Toolchain parity.** Read `ZIG_VERSION`, `ZIG_TARBALL_SHA256`, `NODE_VERSION` and `NPM_VERSION` out of the top-level `env:` block of `.gitea/workflows/gates.yml` — a line parser, not a YAML library — and refuse unless `node --version` (less its leading `v`) and `npm --version` report those exact strings, and unless the host is x86_64 Linux with glibc (the admin bundle uses host-native Rolldown and Lightning CSS bindings, shipped per platform and libc, and CI is an x86_64 Ubuntu runner). Zig is not taken from `PATH` at all: the cut obtains the official ziglang.org tarball `https://ziglang.org/download/<ZIG_VERSION>/zig-x86_64-linux-<ZIG_VERSION>.tar.xz`, caches it under `$HOME/.cache/nxdns-cut/zig-<ZIG_VERSION>-<first 12 hex of ZIG_TARBALL_SHA256>/` (keyed by the digest too, so a changed pin never reuses an old extraction), and builds every release byte with that binary by absolute path. The first real cut pinned hashes CI could not reproduce because the Arch Linux 0.16.0 package is built against the system LLVM and emits different machine code than the tarball CI installs, and both print `0.16.0` — a version string cannot see the difference, so the bytes are pinned instead. When the cached binary is absent the tarball is downloaded with `curl -fsSL`, checked against `ZIG_TARBALL_SHA256` (setup-zig verifies the same bytes by minisign on the CI side), extracted with `tar -xJf`, and the extracted `zig version` is asserted; when it is present that last assertion still runs. There is no fall back to `PATH`. The `zig` on `PATH` still compiles the cut tool itself — that is `zig build cut`, whose bytes nobody hashes. The cut and CI must build with one toolchain, because the cut writes the hashes and CI recomputes them. A mismatch found here costs nothing; found in the release run it costs a public tag. The pins live in `gates.yml` and are read from there rather than copied, so the two can never drift; `release.yml` mirrors them and a test asserts the two files agree. Both halves run before the manifest is rewritten, so a refusal leaves the working tree clean.
|
||||||
2. **Write `build.zig.zon`.** The manifest declares the new version, atomically, and is reparsed off disk. Nothing is committed yet.
|
2. **Write `build.zig.zon`.** The manifest declares the new version, atomically, and is reparsed off disk. Nothing is committed yet.
|
||||||
3. **Build the release.** In `admin/`, `npm ci` then `npm run build`; then `zig build dist -Dversion-string=<next> -Dadmin-dist=admin/dist -Doptimize=ReleaseSafe`, with `--cache-dir` under a scratch directory that is deleted and recreated first (the global cache stays shared: it is content-addressed, and a fresh one refetches every dependency and trips on zig 0.16.0's unzip, which expects `<global>/tmp` to exist). Every one of these runs under a constructed environment holding exactly nine variables — `PATH` and `HOME` from the parent, and `LC_ALL=C`, `LANG=C`, `TZ=UTC`, `SOURCE_DATE_EPOCH=0`, `CI=true`, `npm_config_userconfig=/nonexistent/npmrc-user`, `npm_config_globalconfig=/nonexistent/npmrc-global` (so neither `~/.npmrc` under the passed-through `HOME` nor the node install's `etc/npmrc` is read; the parity check refuses to run if either path exists, and CI asserts the same) — with `umask 022`. The umask arrives through an `sh -c 'umask 022 && exec "$@"'` wrapper because zig 0.16.0 exposes `umask(2)` only as a libc extern that these tools do not link, and `SpawnOptions` has no field for it. The wrapper's `exec` resolves the real command through the child's `PATH`, which is why `PATH` is a passthrough and not a pinned value.
|
3. **Build the release.** In `admin/`, `npm ci` then `npm run build`; then `dist` through the official zig: `<cached zig> build dist -Dversion-string=<next> -Dadmin-dist=admin/dist -Doptimize=ReleaseSafe`, with `--cache-dir` under a scratch directory that is deleted and recreated first (the global cache stays shared: it is content-addressed, and a fresh one refetches every dependency and trips on zig 0.16.0's unzip, which expects `<global>/tmp` to exist). Every one of these runs under a constructed environment holding exactly nine variables — `PATH` and `HOME` from the parent, and `LC_ALL=C`, `LANG=C`, `TZ=UTC`, `SOURCE_DATE_EPOCH=0`, `CI=true`, `npm_config_userconfig=/nonexistent/npmrc-user`, `npm_config_globalconfig=/nonexistent/npmrc-global` (so neither `~/.npmrc` under the passed-through `HOME` nor the node install's `etc/npmrc` is read; the parity check refuses to run if either path exists, and CI asserts the same) — with `umask 022`. The umask arrives through an `sh -c 'umask 022 && exec "$@"'` wrapper because zig 0.16.0 exposes `umask(2)` only as a libc extern that these tools do not link, and `SpawnOptions` has no field for it. The wrapper's `exec` resolves the real command through the child's `PATH`, which is why `PATH` is a passthrough and not a pinned value.
|
||||||
4. **Pin and verify.** `zig build pin-flake` rewrites the generated block of `flake.nix` from `zig-out/dist/SHA256SUMS`; then `zig build verify-dist` and `zig build verify-pins` with the same flags and cache directories, then `nix flake check --no-build` — evaluation only, because the tarballs the block now names do not exist until the release run uploads them.
|
4. **Pin and verify.** Through the same official zig, `build pin-flake` rewrites the generated block of `flake.nix` from `zig-out/dist/SHA256SUMS`; then `build verify-dist` and `build verify-pins` with the same flags and cache directories, then `nix flake check --no-build` — evaluation only, because the tarballs the block now names do not exist until the release run uploads them.
|
||||||
5. **Commit.** One `git commit -S`, taking `build.zig.zon` and `flake.nix`. The working-tree diff is asserted to be a subset of those two paths and to contain the manifest.
|
5. **Commit.** One `git commit -S`, taking `build.zig.zon` and `flake.nix`. The working-tree diff is asserted to be a subset of those two paths and to contain the manifest.
|
||||||
|
|
||||||
The bump must precede the build because `verify-dist` refuses a build whose `-Dversion-string` disagrees with the manifest; the build must precede the commit because the pins belong to the commit. There is no ordering that satisfies both differently.
|
The bump must precede the build because `verify-dist` refuses a build whose `-Dversion-string` disagrees with the manifest; the build must precede the commit because the pins belong to the commit. There is no ordering that satisfies both differently.
|
||||||
|
|||||||
+13
-1
@@ -482,7 +482,19 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
|||||||
// Two HTTP clients on purpose. A blocklist download streams tens of
|
// Two HTTP clients on purpose. A blocklist download streams tens of
|
||||||
// megabytes and holds its connection for the whole of it; DoH queries must
|
// megabytes and holds its connection for the whole of it; DoH queries must
|
||||||
// not queue behind that, and the two have nothing to share but a type.
|
// not queue behind that, and the two have nothing to share but a type.
|
||||||
var fetch_http: std.http.Client = .{ .allocator = gpa, .io = io };
|
// `free_size = 0` makes `release` destroy every connection and the pool
|
||||||
|
// hand none back (`Client.zig:133`). The fetcher's `.keep_alive = false` is
|
||||||
|
// what sends `connection: close`, but it cannot carry this alone: `request`
|
||||||
|
// takes a pooled connection before it stores the flag (`Client.zig:1725`,
|
||||||
|
// `:1742`), and a failed send leaves a `.ready` connection that
|
||||||
|
// `Request.deinit` would pool (`Client.zig:890`). A pass runs once a day,
|
||||||
|
// so any kept connection is closed at the peer by the next one, and the
|
||||||
|
// client never retries a stale one — it fails the source for the day.
|
||||||
|
var fetch_http: std.http.Client = .{
|
||||||
|
.allocator = gpa,
|
||||||
|
.io = io,
|
||||||
|
.connection_pool = .{ .free_size = 0 },
|
||||||
|
};
|
||||||
defer fetch_http.deinit();
|
defer fetch_http.deinit();
|
||||||
var dns_http: std.http.Client = .{ .allocator = gpa, .io = io };
|
var dns_http: std.http.Client = .{ .allocator = gpa, .io = io };
|
||||||
defer dns_http.deinit();
|
defer dns_http.deinit();
|
||||||
|
|||||||
+312
-33
@@ -1,16 +1,21 @@
|
|||||||
//! Blocklist download over HTTP/1.1.
|
//! Blocklist download over HTTP/1.1.
|
||||||
//!
|
//!
|
||||||
//! One `Fetcher` wraps a caller-owned `std.http.Client`, which owns the
|
//! One `Fetcher` wraps a caller-owned `std.http.Client`, which owns the CA
|
||||||
//! connection pool and the CA bundle, exactly as `upstream/doh_client.zig`
|
//! bundle, as `upstream/doh_client.zig` does. Nothing here is pooled: every
|
||||||
//! does. This file knows nothing about parsing, files or the database: it GETs
|
//! request asks for `connection: close`, and the caller is expected to give
|
||||||
//! a URL and streams the bytes into a writer the caller supplies.
|
//! this file a client whose pool holds nothing (`free_size = 0`, set at the
|
||||||
|
//! one construction site in `app.zig`). This file knows nothing about parsing,
|
||||||
|
//! files or the database: it GETs a URL and streams the bytes into a writer
|
||||||
|
//! the caller supplies.
|
||||||
//!
|
//!
|
||||||
//! The body is never held whole. A blocklist can reach `max_body_bytes`, and
|
//! The body is never held whole. A blocklist can reach `max_body_bytes`, and
|
||||||
//! the caller writes into a temporary file anyway, so nothing here allocates.
|
//! the caller writes into a temporary file anyway, so nothing here allocates.
|
||||||
//!
|
//!
|
||||||
//! There is no timeout parameter and no sleep. `std.http.Client` has no
|
//! There is no timeout parameter and no sleep. `std.http.Client` has no
|
||||||
//! per-request deadline, so the caller runs `fetch` under `io.concurrent` and
|
//! per-request deadline, so the caller runs `fetch` under `io.concurrent` and
|
||||||
//! cancels the future; this file only propagates `error.Canceled`.
|
//! cancels the future. A cancel during the body copy surfaces as
|
||||||
|
//! `error.ReceiveFailed` with `Canceled` in `last_failure`; elsewhere it
|
||||||
|
//! propagates as `error.Canceled`.
|
||||||
|
|
||||||
const std = @import("std");
|
const std = @import("std");
|
||||||
const transport = @import("../upstream/transport.zig");
|
const transport = @import("../upstream/transport.zig");
|
||||||
@@ -43,6 +48,30 @@ pub const Result = struct {
|
|||||||
status: std.http.Status,
|
status: std.http.Status,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// The concrete fault behind an `Error`. `cause` is unwrapped from whatever
|
||||||
|
/// `std.http.Client` stashed on the connection or the response, so it names a
|
||||||
|
/// record-layer TLS fault or an HTTP framing fault rather than the collapsed
|
||||||
|
/// `error.ReadFailed`.
|
||||||
|
pub const Failure = struct {
|
||||||
|
phase: Phase,
|
||||||
|
cause: anyerror,
|
||||||
|
/// The response status if a head arrived before the failure.
|
||||||
|
status: ?std.http.Status,
|
||||||
|
/// Body bytes delivered to the caller's writer before the failure, and a
|
||||||
|
/// lower bound rather than an exact count: at least this many reached `w`.
|
||||||
|
/// A single `Reader.stream` call can hand bytes to `w` and then fail, and
|
||||||
|
/// that partial delivery is not counted. Zero on every phase before
|
||||||
|
/// `receive_body`.
|
||||||
|
bytes_read: u64,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Which call failed. The phase is what decides the classification, and only
|
||||||
|
/// the call site knows it — guessing it from an error name would be wrong the
|
||||||
|
/// first time two phases shared an error. `receive_body` is distinct from
|
||||||
|
/// `receive`: the head arrived, so the fault is in the transfer, not the
|
||||||
|
/// response.
|
||||||
|
pub const Phase = enum { connect, send, receive, receive_body };
|
||||||
|
|
||||||
pub const Fetcher = struct {
|
pub const Fetcher = struct {
|
||||||
/// Caller-owned; shared across sources, pools connections.
|
/// Caller-owned; shared across sources, pools connections.
|
||||||
http: *std.http.Client,
|
http: *std.http.Client,
|
||||||
@@ -55,6 +84,12 @@ pub const Fetcher = struct {
|
|||||||
/// one. `error.HttpStatus` carries no `Result`, and the operator's message
|
/// one. `error.HttpStatus` carries no `Result`, and the operator's message
|
||||||
/// needs the number, so it is readable here after a failed `fetch`.
|
/// needs the number, so it is readable here after a failed `fetch`.
|
||||||
last_status: ?std.http.Status = null,
|
last_status: ?std.http.Status = null,
|
||||||
|
/// What the most recent `fetch` failed on, or null if it succeeded or has
|
||||||
|
/// not run. The taxonomy `Error` a caller receives names six outcomes; this
|
||||||
|
/// names the one concrete fault behind the outcome, so an operator reading
|
||||||
|
/// the log can tell a TLS alert from a reset connection from a truncated
|
||||||
|
/// chunk.
|
||||||
|
last_failure: ?Failure = null,
|
||||||
|
|
||||||
/// GETs `url` and streams the body into `w`.
|
/// GETs `url` and streams the body into `w`.
|
||||||
pub fn fetch(
|
pub fn fetch(
|
||||||
@@ -71,37 +106,110 @@ pub const Fetcher = struct {
|
|||||||
std.debug.assert(self.transfer_buf.len >= min_transfer_buf);
|
std.debug.assert(self.transfer_buf.len >= min_transfer_buf);
|
||||||
std.debug.assert(self.redirect_buf.len >= redirect_buffer_len);
|
std.debug.assert(self.redirect_buf.len >= redirect_buffer_len);
|
||||||
|
|
||||||
const uri = try parseUrl(url);
|
|
||||||
|
|
||||||
self.last_status = null;
|
self.last_status = null;
|
||||||
|
self.last_failure = null;
|
||||||
|
|
||||||
|
const uri = parseUrl(url) catch |err| return self.record(err, .connect, err, null, 0);
|
||||||
|
|
||||||
var req = self.http.request(.GET, uri, .{
|
var req = self.http.request(.GET, uri, .{
|
||||||
.keep_alive = true,
|
// Sends `connection: close`, so the peer ends the connection and
|
||||||
|
// `release` destroys it rather than pooling it (`Client.zig:133`,
|
||||||
|
// `:1167`). A pass runs once a day: a kept connection is one the
|
||||||
|
// peer has already closed by the next pass, and `std.http.Client`
|
||||||
|
// never retries a reused connection that reaches EOF before the
|
||||||
|
// first head byte — it returns `error.HttpConnectionClosing` and
|
||||||
|
// the source fails for the day. This flag cannot carry that alone,
|
||||||
|
// because `request` takes a pooled connection before it reads the
|
||||||
|
// flag; `app.zig` empties the pool itself.
|
||||||
|
.keep_alive = false,
|
||||||
.headers = .{
|
.headers = .{
|
||||||
// Identity only: a compressed transfer encoding would need
|
// Identity only: a compressed transfer encoding would need
|
||||||
// `Response.readerDecompressing`, a decompression buffer and a
|
// `Response.readerDecompressing`, a decompression buffer and a
|
||||||
// second failure surface, for a download that runs once a day.
|
// second failure surface, for a download that runs once a day.
|
||||||
.accept_encoding = .{ .override = "identity" },
|
.accept_encoding = .{ .override = "identity" },
|
||||||
},
|
},
|
||||||
}) catch |err| return mapError(err, .connect);
|
}) catch |err| return self.record(mapError(err, .connect), .connect, err, null, 0);
|
||||||
defer req.deinit();
|
defer req.deinit();
|
||||||
|
|
||||||
req.sendBodiless() catch |err| return mapError(err, .send);
|
req.sendBodiless() catch |err|
|
||||||
|
return self.record(mapError(err, .send), .send, transport.sendCause(&req, err), null, 0);
|
||||||
|
|
||||||
var resp = req.receiveHead(self.redirect_buf) catch |err| return mapError(err, .receive);
|
var resp = req.receiveHead(self.redirect_buf) catch |err|
|
||||||
|
return self.record(mapError(err, .receive), .receive, transport.headCause(&req, err), null, 0);
|
||||||
|
|
||||||
self.last_status = resp.head.status;
|
self.last_status = resp.head.status;
|
||||||
if (resp.head.status != .ok) return error.HttpStatus;
|
if (resp.head.status != .ok) {
|
||||||
|
return self.record(error.HttpStatus, .receive, error.HttpStatus, resp.head.status, 0);
|
||||||
|
}
|
||||||
|
|
||||||
// `content-type` is deliberately not checked: blocklists are served as
|
// `content-type` is deliberately not checked: blocklists are served as
|
||||||
// text/plain, application/octet-stream and text/html alike, and the
|
// text/plain, application/octet-stream and text/html alike, and the
|
||||||
// compiler's invalid-line counters are the honest signal about content.
|
// compiler's invalid-line counters are the honest signal about content.
|
||||||
if (resp.head.content_length) |declared| {
|
if (resp.head.content_length) |declared| {
|
||||||
if (declared > max_body_bytes) return error.BodyTooLarge;
|
if (declared > max_body_bytes) {
|
||||||
|
return self.record(
|
||||||
|
error.BodyTooLarge,
|
||||||
|
.receive,
|
||||||
|
error.BodyTooLarge,
|
||||||
|
resp.head.status,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const body = resp.reader(self.transfer_buf);
|
const body = resp.reader(self.transfer_buf);
|
||||||
return .{ .bytes_read = try pumpBody(body, w, max_body_bytes), .status = resp.head.status };
|
var pump: Pump = .{};
|
||||||
|
const bytes_read = pumpBody(body, w, max_body_bytes, &pump) catch |err| return self.record(
|
||||||
|
err,
|
||||||
|
.receive_body,
|
||||||
|
transport.bodyCause(&resp, pump.cause),
|
||||||
|
resp.head.status,
|
||||||
|
pump.delivered,
|
||||||
|
);
|
||||||
|
return .{ .bytes_read = bytes_read, .status = resp.head.status };
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stores the failure and hands back `mapped` unchanged.
|
||||||
|
///
|
||||||
|
/// `mapped` is the taxonomy error the call site already decided, never one
|
||||||
|
/// this function derives. `mapError` answers for a raw std error, and the
|
||||||
|
/// call sites that raise a taxonomy member directly — a non-200 head, a
|
||||||
|
/// declared length over the cap — pass that member: feeding either back
|
||||||
|
/// through `mapError` would find no case for it and fall to the phase
|
||||||
|
/// default, turning `error.HttpStatus` into `error.ReceiveFailed`.
|
||||||
|
///
|
||||||
|
/// `cause` never reaches the returned error. The unwrap is for the
|
||||||
|
/// operator's line, and a caller that ignores `last_failure` sees exactly
|
||||||
|
/// what it saw before the unwrap existed.
|
||||||
|
fn record(
|
||||||
|
self: *Fetcher,
|
||||||
|
mapped: Error,
|
||||||
|
phase: Phase,
|
||||||
|
cause: anyerror,
|
||||||
|
status: ?std.http.Status,
|
||||||
|
bytes_read: u64,
|
||||||
|
) Error {
|
||||||
|
self.last_failure = .{
|
||||||
|
.phase = phase,
|
||||||
|
.cause = cause,
|
||||||
|
.status = status,
|
||||||
|
.bytes_read = bytes_read,
|
||||||
|
};
|
||||||
|
return mapped;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
/// What `pumpBody` reports alongside its error: the reader error before
|
||||||
|
/// `mapError` collapses it, and the bytes already handed to `w`. `pumpBody`
|
||||||
|
/// knows nothing of `std.http`, so the unwrap of `cause` happens in `fetch`,
|
||||||
|
/// which holds the response.
|
||||||
|
const Pump = struct {
|
||||||
|
cause: anyerror = error.Unexpected,
|
||||||
|
delivered: u64 = 0,
|
||||||
|
|
||||||
|
fn fail(self: *Pump, cause: anyerror, mapped: Error) Error {
|
||||||
|
self.cause = cause;
|
||||||
|
return mapped;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -116,17 +224,19 @@ pub const Fetcher = struct {
|
|||||||
/// `self.transfer_buf` — makes `@memcpy` copy the buffer onto itself
|
/// `self.transfer_buf` — makes `@memcpy` copy the buffer onto itself
|
||||||
/// (Reader.zig:677) and panics the process on the first read that finds bytes
|
/// (Reader.zig:677) and panics the process on the first read that finds bytes
|
||||||
/// already buffered.
|
/// already buffered.
|
||||||
fn pumpBody(body: *std.Io.Reader, w: *std.Io.Writer, cap: usize) Error!u64 {
|
fn pumpBody(body: *std.Io.Reader, w: *std.Io.Writer, cap: usize, out: *Pump) Error!u64 {
|
||||||
var total: usize = 0;
|
var total: usize = 0;
|
||||||
while (total < cap) {
|
while (total < cap) {
|
||||||
|
out.delivered = total;
|
||||||
total += body.stream(w, .limited(cap - total)) catch |err| switch (err) {
|
total += body.stream(w, .limited(cap - total)) catch |err| switch (err) {
|
||||||
error.EndOfStream => return total,
|
error.EndOfStream => return total,
|
||||||
// The caller owns `w` and can read the concrete failure from its
|
// The caller owns `w` and can read the concrete failure from its
|
||||||
// own writer; this taxonomy has no member for a failing sink.
|
// own writer; this taxonomy has no member for a failing sink.
|
||||||
error.WriteFailed => return error.Unexpected,
|
error.WriteFailed => return out.fail(err, error.Unexpected),
|
||||||
else => |e| return mapError(e, .receive),
|
else => |e| return out.fail(e, mapError(e, .receive_body)),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
out.delivered = total;
|
||||||
|
|
||||||
// `cap` written exactly. One more byte separates a body that fits from one
|
// `cap` written exactly. One more byte separates a body that fits from one
|
||||||
// that was cut off, and it must not reach `w`.
|
// that was cut off, and it must not reach `w`.
|
||||||
@@ -134,10 +244,10 @@ fn pumpBody(body: *std.Io.Reader, w: *std.Io.Writer, cap: usize) Error!u64 {
|
|||||||
var probe: std.Io.Writer.Discarding = .init(&probe_buf);
|
var probe: std.Io.Writer.Discarding = .init(&probe_buf);
|
||||||
const extra = body.stream(&probe.writer, .limited(1)) catch |err| switch (err) {
|
const extra = body.stream(&probe.writer, .limited(1)) catch |err| switch (err) {
|
||||||
error.EndOfStream => return total,
|
error.EndOfStream => return total,
|
||||||
error.WriteFailed => return error.Unexpected,
|
error.WriteFailed => return out.fail(err, error.Unexpected),
|
||||||
else => |e| return mapError(e, .receive),
|
else => |e| return out.fail(e, mapError(e, .receive_body)),
|
||||||
};
|
};
|
||||||
return if (extra == 0) total else error.BodyTooLarge;
|
return if (extra == 0) total else out.fail(error.BodyTooLarge, error.BodyTooLarge);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A scheme other than `http`/`https`, an unparseable URL and a URL with no
|
/// A scheme other than `http`/`https`, an unparseable URL and a URL with no
|
||||||
@@ -151,11 +261,6 @@ fn parseUrl(url: []const u8) Error!std.Uri {
|
|||||||
return uri;
|
return uri;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Which call failed. The phase is what decides the classification, and only
|
|
||||||
/// the call site knows it — guessing it from an error name would be wrong the
|
|
||||||
/// first time two phases shared an error.
|
|
||||||
const Phase = enum { connect, send, receive };
|
|
||||||
|
|
||||||
// `error.X` in an expression names a member into existence rather than
|
// `error.X` in an expression names a member into existence rather than
|
||||||
// referring to one, so the switch in `mapError` would keep compiling — and
|
// referring to one, so the switch in `mapError` would keep compiling — and
|
||||||
// silently stop matching — if std renamed either of these. This is what fails
|
// silently stop matching — if std renamed either of these. This is what fails
|
||||||
@@ -179,9 +284,10 @@ fn errorSetHas(comptime Set: type, comptime name: []const u8) bool {
|
|||||||
/// `std.http.Client` collapses every handshake fault into
|
/// `std.http.Client` collapses every handshake fault into
|
||||||
/// `error.TlsInitializationFailed` (Client.zig:1470) and every bundle fault into
|
/// `error.TlsInitializationFailed` (Client.zig:1470) and every bundle fault into
|
||||||
/// `error.CertificateBundleLoadFailure`, and this file never unwraps a
|
/// `error.CertificateBundleLoadFailure`, and this file never unwraps a
|
||||||
/// connection's stashed read cause — it maps the collapsed `error.ReadFailed` by
|
/// connection's stashed read cause *for classification* — it maps the collapsed
|
||||||
/// phase — so the record-layer members of `std.crypto.tls.Client.ReadError`
|
/// `error.ReadFailed` by phase — so the record-layer members of
|
||||||
/// cannot arrive here. `doh_client.zig` does unwrap, and names them.
|
/// `std.crypto.tls.Client.ReadError` cannot arrive here. `fetch` does unwrap
|
||||||
|
/// them, into `Failure.cause`, which no classification reads.
|
||||||
fn mapError(err: anyerror, phase: Phase) Error {
|
fn mapError(err: anyerror, phase: Phase) Error {
|
||||||
if (transport.mapLocal(err)) |local| return narrowLocal(local);
|
if (transport.mapLocal(err)) |local| return narrowLocal(local);
|
||||||
switch (err) {
|
switch (err) {
|
||||||
@@ -198,7 +304,7 @@ fn mapError(err: anyerror, phase: Phase) Error {
|
|||||||
return switch (phase) {
|
return switch (phase) {
|
||||||
.connect => error.ConnectFailed,
|
.connect => error.ConnectFailed,
|
||||||
.send => error.SendFailed,
|
.send => error.SendFailed,
|
||||||
.receive => error.ReceiveFailed,
|
.receive, .receive_body => error.ReceiveFailed,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -230,7 +336,8 @@ test "pumpBody streams a fully buffered body without aliasing its source" {
|
|||||||
var out: [payload.len]u8 = undefined;
|
var out: [payload.len]u8 = undefined;
|
||||||
var sink: std.Io.Writer = .fixed(&out);
|
var sink: std.Io.Writer = .fixed(&out);
|
||||||
|
|
||||||
const n = try pumpBody(&body, &sink, max_body_bytes);
|
var pump: Pump = .{};
|
||||||
|
const n = try pumpBody(&body, &sink, max_body_bytes, &pump);
|
||||||
try testing.expectEqual(@as(u64, payload.len), n);
|
try testing.expectEqual(@as(u64, payload.len), n);
|
||||||
try testing.expectEqualStrings(payload, out[0..payload.len]);
|
try testing.expectEqualStrings(payload, out[0..payload.len]);
|
||||||
}
|
}
|
||||||
@@ -241,7 +348,8 @@ test "pumpBody accepts a body of exactly the cap" {
|
|||||||
var out: [payload.len]u8 = undefined;
|
var out: [payload.len]u8 = undefined;
|
||||||
var sink: std.Io.Writer = .fixed(&out);
|
var sink: std.Io.Writer = .fixed(&out);
|
||||||
|
|
||||||
const n = try pumpBody(&body, &sink, payload.len);
|
var pump: Pump = .{};
|
||||||
|
const n = try pumpBody(&body, &sink, payload.len, &pump);
|
||||||
try testing.expectEqual(@as(u64, payload.len), n);
|
try testing.expectEqual(@as(u64, payload.len), n);
|
||||||
try testing.expectEqualStrings(payload, out[0..payload.len]);
|
try testing.expectEqualStrings(payload, out[0..payload.len]);
|
||||||
}
|
}
|
||||||
@@ -252,10 +360,12 @@ test "pumpBody refuses a body one byte over the cap" {
|
|||||||
var out: [payload.len]u8 = undefined;
|
var out: [payload.len]u8 = undefined;
|
||||||
var sink: std.Io.Writer = .fixed(&out);
|
var sink: std.Io.Writer = .fixed(&out);
|
||||||
|
|
||||||
|
var pump: Pump = .{};
|
||||||
try testing.expectError(
|
try testing.expectError(
|
||||||
error.BodyTooLarge,
|
error.BodyTooLarge,
|
||||||
pumpBody(&body, &sink, payload.len - 1),
|
pumpBody(&body, &sink, payload.len - 1, &pump),
|
||||||
);
|
);
|
||||||
|
try testing.expectEqual(error.BodyTooLarge, pump.cause);
|
||||||
}
|
}
|
||||||
|
|
||||||
test "pumpBody reports an empty body as zero bytes" {
|
test "pumpBody reports an empty body as zero bytes" {
|
||||||
@@ -263,7 +373,8 @@ test "pumpBody reports an empty body as zero bytes" {
|
|||||||
var sink_buf: [0]u8 = .{};
|
var sink_buf: [0]u8 = .{};
|
||||||
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||||
|
|
||||||
try testing.expectEqual(@as(u64, 0), try pumpBody(&body, &discarding.writer, max_body_bytes));
|
var pump: Pump = .{};
|
||||||
|
try testing.expectEqual(@as(u64, 0), try pumpBody(&body, &discarding.writer, max_body_bytes, &pump));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn undefinedFetcher(transfer_buf: []u8, redirect_buf: []u8) Fetcher {
|
fn undefinedFetcher(transfer_buf: []u8, redirect_buf: []u8) Fetcher {
|
||||||
@@ -359,3 +470,171 @@ test "caps are the values the memory budget was sized against" {
|
|||||||
try testing.expectEqual(@as(usize, 64 * 1024 * 1024), max_body_bytes);
|
try testing.expectEqual(@as(usize, 64 * 1024 * 1024), max_body_bytes);
|
||||||
try testing.expectEqual(@as(usize, 8192), redirect_buffer_len);
|
try testing.expectEqual(@as(usize, 8192), redirect_buffer_len);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Only the fields the unwrap helpers read are set; the rest of a `Connection`
|
||||||
|
/// is two buffered streams, a host name and a pool node. `.plain` for the
|
||||||
|
/// reason `doh_client.zig` gives: `getReadError` reaches a TLS connection's
|
||||||
|
/// cause through `@fieldParentPtr`, which on a stub would read memory that was
|
||||||
|
/// never a `Tls`.
|
||||||
|
fn stubConnection(read_err: ?std.Io.net.Stream.Reader.Error) std.http.Client.Connection {
|
||||||
|
var connection: std.http.Client.Connection = undefined;
|
||||||
|
connection.protocol = .plain;
|
||||||
|
connection.stream_reader.err = read_err;
|
||||||
|
connection.stream_writer.err = null;
|
||||||
|
return connection;
|
||||||
|
}
|
||||||
|
|
||||||
|
test "a head failure records the stashed cause, not the collapsed read error" {
|
||||||
|
var connection = stubConnection(error.ConnectionResetByPeer);
|
||||||
|
var req: std.http.Client.Request = undefined;
|
||||||
|
req.connection = &connection;
|
||||||
|
|
||||||
|
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||||
|
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||||
|
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||||
|
|
||||||
|
const err = error.ReadFailed;
|
||||||
|
const mapped = f.record(mapError(err, .receive), .receive, transport.headCause(&req, err), null, 0);
|
||||||
|
|
||||||
|
// The taxonomy the caller sees is what it was before the unwrap existed.
|
||||||
|
try testing.expectEqual(error.ReceiveFailed, mapped);
|
||||||
|
const failure = f.last_failure.?;
|
||||||
|
try testing.expectEqual(Phase.receive, failure.phase);
|
||||||
|
try testing.expectEqual(error.ConnectionResetByPeer, failure.cause);
|
||||||
|
try testing.expectEqual(@as(?std.http.Status, null), failure.status);
|
||||||
|
try testing.expectEqual(@as(u64, 0), failure.bytes_read);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "a body failure records the bytes delivered before it and the body phase" {
|
||||||
|
const payload = "0.0.0.0 ads.example.com\n";
|
||||||
|
var buffer: [payload.len]u8 = (payload ++ "").*;
|
||||||
|
// Buffered bytes reach `w` first; the vtable is only asked for more once
|
||||||
|
// the buffer is drained, which is the shape of a transfer cut off mid-body.
|
||||||
|
var body: std.Io.Reader = .{
|
||||||
|
.vtable = std.Io.Reader.failing.vtable,
|
||||||
|
.buffer = &buffer,
|
||||||
|
.seek = 0,
|
||||||
|
.end = buffer.len,
|
||||||
|
};
|
||||||
|
var sink_buf: [0]u8 = .{};
|
||||||
|
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||||
|
|
||||||
|
var pump: Pump = .{};
|
||||||
|
try testing.expectError(
|
||||||
|
error.ReceiveFailed,
|
||||||
|
pumpBody(&body, &discarding.writer, max_body_bytes, &pump),
|
||||||
|
);
|
||||||
|
try testing.expectEqual(@as(u64, payload.len), pump.delivered);
|
||||||
|
|
||||||
|
var connection = stubConnection(null);
|
||||||
|
var req: std.http.Client.Request = undefined;
|
||||||
|
req.connection = &connection;
|
||||||
|
req.reader.body_err = error.HttpChunkTruncated;
|
||||||
|
const resp: std.http.Client.Response = .{ .request = &req, .head = undefined };
|
||||||
|
|
||||||
|
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||||
|
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||||
|
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||||
|
const mapped = f.record(
|
||||||
|
mapError(error.ReadFailed, .receive_body),
|
||||||
|
.receive_body,
|
||||||
|
transport.bodyCause(&resp, pump.cause),
|
||||||
|
.ok,
|
||||||
|
pump.delivered,
|
||||||
|
);
|
||||||
|
|
||||||
|
try testing.expectEqual(error.ReceiveFailed, mapped);
|
||||||
|
const failure = f.last_failure.?;
|
||||||
|
try testing.expectEqual(Phase.receive_body, failure.phase);
|
||||||
|
try testing.expectEqual(error.HttpChunkTruncated, failure.cause);
|
||||||
|
try testing.expectEqual(@as(?std.http.Status, .ok), failure.status);
|
||||||
|
try testing.expectEqual(@as(u64, payload.len), failure.bytes_read);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "a rejected url records the phase without claiming a connect failure" {
|
||||||
|
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||||
|
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||||
|
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||||
|
var sink_buf: [0]u8 = .{};
|
||||||
|
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||||
|
|
||||||
|
try testing.expectError(
|
||||||
|
error.BadUrl,
|
||||||
|
f.fetch(undefined, "ftp://example.com/list.txt", &discarding.writer),
|
||||||
|
);
|
||||||
|
try testing.expectEqual(error.BadUrl, f.last_failure.?.cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "record returns the taxonomy member the call site raised, not a remapped one" {
|
||||||
|
// `error.HttpStatus` and `error.BodyTooLarge` are raised by `fetch` itself
|
||||||
|
// rather than by std, so `mapError` has no case for either and would fall
|
||||||
|
// to the phase default. `record` must not run them through it.
|
||||||
|
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||||
|
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||||
|
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||||
|
|
||||||
|
try testing.expectEqual(
|
||||||
|
error.HttpStatus,
|
||||||
|
f.record(error.HttpStatus, .receive, error.HttpStatus, .not_found, 0),
|
||||||
|
);
|
||||||
|
try testing.expectEqual(@as(?std.http.Status, .not_found), f.last_failure.?.status);
|
||||||
|
try testing.expectEqual(error.HttpStatus, f.last_failure.?.cause);
|
||||||
|
|
||||||
|
try testing.expectEqual(
|
||||||
|
error.BodyTooLarge,
|
||||||
|
f.record(error.BodyTooLarge, .receive, error.BodyTooLarge, .ok, 0),
|
||||||
|
);
|
||||||
|
try testing.expectEqual(error.BodyTooLarge, f.last_failure.?.cause);
|
||||||
|
|
||||||
|
// This is why: `mapError` is unchanged and does collapse both to the phase
|
||||||
|
// default. `fetch` must never route these two through it.
|
||||||
|
try testing.expectEqual(error.ReceiveFailed, mapError(error.HttpStatus, .receive));
|
||||||
|
try testing.expectEqual(error.ReceiveFailed, mapError(error.BodyTooLarge, .receive));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "a cancelled body read records the cancellation and the progress so far" {
|
||||||
|
// This is the shape the expiry race leaves behind. `Select.cancelDiscard`
|
||||||
|
// cancels the fetch, the cancellation surfaces from the socket as a
|
||||||
|
// stashed `error.Canceled` under a collapsed `error.ReadFailed`, and the
|
||||||
|
// fetch records it on its way out. The manager then reports its own
|
||||||
|
// `error.Timeout` beside this record.
|
||||||
|
const payload = "0.0.0.0 ads.example.com\n";
|
||||||
|
var buffer: [payload.len]u8 = (payload ++ "").*;
|
||||||
|
var body: std.Io.Reader = .{
|
||||||
|
.vtable = std.Io.Reader.failing.vtable,
|
||||||
|
.buffer = &buffer,
|
||||||
|
.seek = 0,
|
||||||
|
.end = buffer.len,
|
||||||
|
};
|
||||||
|
var sink_buf: [0]u8 = .{};
|
||||||
|
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||||
|
|
||||||
|
var pump: Pump = .{};
|
||||||
|
try testing.expectError(
|
||||||
|
error.ReceiveFailed,
|
||||||
|
pumpBody(&body, &discarding.writer, max_body_bytes, &pump),
|
||||||
|
);
|
||||||
|
|
||||||
|
var connection = stubConnection(error.Canceled);
|
||||||
|
var req: std.http.Client.Request = undefined;
|
||||||
|
req.connection = &connection;
|
||||||
|
req.reader.body_err = null;
|
||||||
|
const resp: std.http.Client.Response = .{ .request = &req, .head = undefined };
|
||||||
|
|
||||||
|
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||||
|
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||||
|
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||||
|
// `pumpBody` maps the collapsed `error.ReadFailed` it was handed, not the
|
||||||
|
// unwrapped cause, so the taxonomy stays `ReceiveFailed` here. That is what
|
||||||
|
// this path returned before `Failure` existed, and the addendum keeps it:
|
||||||
|
// only the recorded cause is new. The manager reports `error.Timeout` from
|
||||||
|
// the expiry race anyway, so nothing reads this return value on this path.
|
||||||
|
const cause = transport.bodyCause(&resp, error.ReadFailed);
|
||||||
|
const mapped = f.record(error.ReceiveFailed, .receive_body, cause, .ok, pump.delivered);
|
||||||
|
|
||||||
|
try testing.expectEqual(error.ReceiveFailed, mapped);
|
||||||
|
try testing.expectEqual(Phase.receive_body, f.last_failure.?.phase);
|
||||||
|
try testing.expectEqual(error.Canceled, f.last_failure.?.cause);
|
||||||
|
try testing.expectEqual(@as(?std.http.Status, .ok), f.last_failure.?.status);
|
||||||
|
try testing.expectEqual(@as(u64, payload.len), f.last_failure.?.bytes_read);
|
||||||
|
}
|
||||||
|
|||||||
@@ -421,7 +421,7 @@ const Env = struct {
|
|||||||
// fixtures: the loopback http server
|
// fixtures: the loopback http server
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const Route = enum(u8) { body, redirect, not_found, oversize, chunked, stall, changed };
|
const Route = enum(u8) { body, redirect, not_found, oversize, chunked, stall, changed, stale_keep_alive };
|
||||||
|
|
||||||
/// How long the `stall` route holds a reply open when nothing releases it.
|
/// How long the `stall` route holds a reply open when nothing releases it.
|
||||||
///
|
///
|
||||||
@@ -542,9 +542,10 @@ const HttpFixture = struct {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Every reply closes the connection. A keep-alive reply would leave the
|
/// Every reply but `stale_keep_alive` closes the connection. A keep-alive
|
||||||
/// fetcher holding the connection open while this server waits to accept a
|
/// reply would leave the fetcher holding the connection open while this
|
||||||
/// second one that never comes (milestone-5 spec, S9 note from S8).
|
/// server waits to accept a second one that never comes (milestone-5 spec,
|
||||||
|
/// S9 note from S8).
|
||||||
fn respond(self: *HttpFixture, io: std.Io, request: *std.http.Server.Request) !void {
|
fn respond(self: *HttpFixture, io: std.Io, request: *std.http.Server.Request) !void {
|
||||||
switch (@as(Route, @enumFromInt(self.route.load(.acquire)))) {
|
switch (@as(Route, @enumFromInt(self.route.load(.acquire)))) {
|
||||||
.body => try request.respond(self.body, .{ .keep_alive = false }),
|
.body => try request.respond(self.body, .{ .keep_alive = false }),
|
||||||
@@ -566,6 +567,11 @@ const HttpFixture = struct {
|
|||||||
.transfer_encoding = .none,
|
.transfer_encoding = .none,
|
||||||
.extra_headers = &.{.{ .name = "content-length", .value = oversize_length }},
|
.extra_headers = &.{.{ .name = "content-length", .value = oversize_length }},
|
||||||
}),
|
}),
|
||||||
|
// Advertises keep-alive and then closes anyway, because `serve`
|
||||||
|
// takes one request per connection. That is the Pi's failure in
|
||||||
|
// miniature: a client that pools this connection reuses one the
|
||||||
|
// peer has already closed on its next request to the host.
|
||||||
|
.stale_keep_alive => try request.respond(self.body, .{ .keep_alive = true }),
|
||||||
.chunked => try self.respondChunked(request),
|
.chunked => try self.respondChunked(request),
|
||||||
.stall => try self.respondStalled(io, request),
|
.stall => try self.respondStalled(io, request),
|
||||||
}
|
}
|
||||||
@@ -1003,6 +1009,19 @@ test "5: a redirect is followed to the same result" {
|
|||||||
try testing.expect(decision.blocked);
|
try testing.expect(decision.blocked);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The download failure line ends in numbers no test can predict — an elapsed
|
||||||
|
/// time always, and a byte count where the failure is a cancelled transfer. The
|
||||||
|
/// fixed part is compared whole; the tail only has to be digits, spaces and the
|
||||||
|
/// two words those numbers carry.
|
||||||
|
fn expectFailureText(expected_prefix: []const u8, actual: []const u8) !void {
|
||||||
|
try testing.expectEqualStrings(expected_prefix, actual[0..@min(expected_prefix.len, actual.len)]);
|
||||||
|
const tail = actual[expected_prefix.len..];
|
||||||
|
try testing.expect(std.mem.endsWith(u8, tail, " ms)"));
|
||||||
|
for (tail[0 .. tail.len - " ms)".len]) |c| {
|
||||||
|
try testing.expect(std.ascii.isDigit(c) or c == ' ' or c == ',' or std.ascii.isAlphabetic(c));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
test "6: a 404 leaves the compiled files and the snapshot untouched" {
|
test "6: a 404 leaves the compiled files and the snapshot untouched" {
|
||||||
if (!build_options.integration) return error.SkipZigTest;
|
if (!build_options.integration) return error.SkipZigTest;
|
||||||
|
|
||||||
@@ -1034,7 +1053,13 @@ test "6: a 404 leaves the compiled files and the snapshot untouched" {
|
|||||||
|
|
||||||
const failed = try env.status(id);
|
const failed = try env.status(id);
|
||||||
try testing.expectEqual(manager.State.fetch_failed, failed.state);
|
try testing.expectEqual(manager.State.fetch_failed, failed.state);
|
||||||
try testing.expectEqualStrings("HttpStatus", failed.errorText());
|
// The whole line, not just the taxonomy: a 404 must stay `HttpStatus` and
|
||||||
|
// name the status it saw. Routing it back through `fetcher.mapError` would
|
||||||
|
// read `ReceiveFailed (receive, cause ReceiveFailed, ...)` here.
|
||||||
|
try expectFailureText(
|
||||||
|
"HttpStatus (receive, cause HttpStatus, http 404, 0 bytes, ",
|
||||||
|
failed.errorText(),
|
||||||
|
);
|
||||||
|
|
||||||
var after = try Bodies.read(gpa, io, dir, "1");
|
var after = try Bodies.read(gpa, io, dir, "1");
|
||||||
defer after.deinit(gpa);
|
defer after.deinit(gpa);
|
||||||
@@ -1046,6 +1071,51 @@ test "6: a 404 leaves the compiled files and the snapshot untouched" {
|
|||||||
try testing.expect(decision.blocked);
|
try testing.expect(decision.blocked);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "6b: a local failure reports itself, not the previous source's network fault" {
|
||||||
|
if (!build_options.integration) return error.SkipZigTest;
|
||||||
|
// Mode bits do not apply to root, so the denial the test needs cannot happen.
|
||||||
|
if (std.c.geteuid() == 0) return error.SkipZigTest;
|
||||||
|
|
||||||
|
const gpa = testing.allocator;
|
||||||
|
const env = try Env.create(gpa);
|
||||||
|
defer env.destroy();
|
||||||
|
const io = env.io();
|
||||||
|
|
||||||
|
var fixture = try HttpFixture.init(io, http_body);
|
||||||
|
defer fixture.deinit(io);
|
||||||
|
fixture.setRoute(.not_found);
|
||||||
|
var group: std.Io.Group = .init;
|
||||||
|
defer group.cancel(io);
|
||||||
|
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||||
|
|
||||||
|
var url_buf: [64]u8 = undefined;
|
||||||
|
const url = try fixture.url(&url_buf);
|
||||||
|
const id = try seedSource(&env.database, url);
|
||||||
|
|
||||||
|
// First a real network failure, so the fetcher is holding a record.
|
||||||
|
try testing.expect(!try refreshOnce(env, url));
|
||||||
|
try expectFailureText(
|
||||||
|
"HttpStatus (receive, cause HttpStatus, http 404, 0 bytes, ",
|
||||||
|
(try env.status(id)).errorText(),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Then a failure that never reaches the network: the raw file cannot be
|
||||||
|
// created. Without the clear at the top of `download` this would report the
|
||||||
|
// 404 above as this refresh's cause.
|
||||||
|
var dir = try env.blocklistDir();
|
||||||
|
defer dir.close(io);
|
||||||
|
try dir.setPermissions(io, .fromMode(0o500));
|
||||||
|
defer dir.setPermissions(io, .fromMode(0o700)) catch {};
|
||||||
|
|
||||||
|
try testing.expect(!try refreshOnce(env, url));
|
||||||
|
|
||||||
|
const failed = try env.status(id);
|
||||||
|
try testing.expectEqual(manager.State.fetch_failed, failed.state);
|
||||||
|
try expectFailureText("AccessDenied (no fetch, http none, - bytes, ", failed.errorText());
|
||||||
|
try testing.expect(!std.mem.containsAtLeast(u8, failed.errorText(), 1, "HttpStatus"));
|
||||||
|
try testing.expect(!std.mem.containsAtLeast(u8, failed.errorText(), 1, "404"));
|
||||||
|
}
|
||||||
|
|
||||||
test "7: a body over the cap fails the refresh and leaves no temporary file" {
|
test "7: a body over the cap fails the refresh and leaves no temporary file" {
|
||||||
if (!build_options.integration) return error.SkipZigTest;
|
if (!build_options.integration) return error.SkipZigTest;
|
||||||
|
|
||||||
@@ -1069,7 +1139,10 @@ test "7: a body over the cap fails the refresh and leaves no temporary file" {
|
|||||||
|
|
||||||
const failed = try env.status(id);
|
const failed = try env.status(id);
|
||||||
try testing.expectEqual(manager.State.fetch_failed, failed.state);
|
try testing.expectEqual(manager.State.fetch_failed, failed.state);
|
||||||
try testing.expectEqualStrings("BodyTooLarge", failed.errorText());
|
try expectFailureText(
|
||||||
|
"BodyTooLarge (receive, cause BodyTooLarge, http 200, 0 bytes, ",
|
||||||
|
failed.errorText(),
|
||||||
|
);
|
||||||
|
|
||||||
var dir = try env.blocklistDir();
|
var dir = try env.blocklistDir();
|
||||||
defer dir.close(io);
|
defer dir.close(io);
|
||||||
@@ -1187,6 +1260,44 @@ fn publishFixtureFiles(env: *Env, id: i64, list_body: []const u8, wild_body: []c
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "8b: a second download to the same host never reuses a dead pooled connection" {
|
||||||
|
if (!build_options.integration) return error.SkipZigTest;
|
||||||
|
|
||||||
|
const gpa = testing.allocator;
|
||||||
|
const env = try Env.create(gpa);
|
||||||
|
defer env.destroy();
|
||||||
|
const io = env.io();
|
||||||
|
|
||||||
|
var fixture = try HttpFixture.init(io, http_body);
|
||||||
|
defer fixture.deinit(io);
|
||||||
|
// The server says keep-alive and then drops the socket, which is what the
|
||||||
|
// Pi's upstreams do between one daily pass and the next.
|
||||||
|
fixture.setRoute(.stale_keep_alive);
|
||||||
|
var group: std.Io.Group = .init;
|
||||||
|
defer group.cancel(io);
|
||||||
|
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||||
|
|
||||||
|
var url_buf: [64]u8 = undefined;
|
||||||
|
const url = try fixture.url(&url_buf);
|
||||||
|
|
||||||
|
var sink_buf: [0]u8 = .{};
|
||||||
|
var first: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||||
|
const one = try env.f.fetch(io, url, &first.writer);
|
||||||
|
try testing.expectEqual(@as(u64, http_body.len), one.bytes_read);
|
||||||
|
|
||||||
|
// The whole test is this second call. With a pooling client the request
|
||||||
|
// goes out over a connection the peer has already closed, no response head
|
||||||
|
// byte arrives, and `error.HttpConnectionClosing` comes back with no retry
|
||||||
|
// — the exact record the Pi produced at 0 bytes and under a millisecond.
|
||||||
|
var second: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||||
|
const two = try env.f.fetch(io, url, &second.writer);
|
||||||
|
try testing.expectEqual(@as(u64, http_body.len), two.bytes_read);
|
||||||
|
try testing.expectEqual(@as(?fetcher.Failure, null), env.f.last_failure);
|
||||||
|
|
||||||
|
// Two connections, not one reused: the proof the first was not pooled.
|
||||||
|
try testing.expectEqual(@as(u32, 2), fixture.accepted.load(.monotonic));
|
||||||
|
}
|
||||||
|
|
||||||
test "9: a reload swaps under a held handle and the new generation follows the release" {
|
test "9: a reload swaps under a held handle and the new generation follows the release" {
|
||||||
if (!build_options.integration) return error.SkipZigTest;
|
if (!build_options.integration) return error.SkipZigTest;
|
||||||
|
|
||||||
|
|||||||
+196
-15
@@ -69,8 +69,13 @@ const log = std.log.scoped(.blocklist_manager);
|
|||||||
const Sha256 = std.crypto.hash.sha2.Sha256;
|
const Sha256 = std.crypto.hash.sha2.Sha256;
|
||||||
|
|
||||||
/// `SourceStatus.last_error` is fixed-size so the failure path allocates
|
/// `SourceStatus.last_error` is fixed-size so the failure path allocates
|
||||||
/// nothing.
|
/// nothing. Sized so the longest `downloadFailureText` fits whole: the widest
|
||||||
pub const max_error_len: usize = 128;
|
/// cause name the unwraps can produce is `DetectingNetworkConfigurationFailed`
|
||||||
|
/// at 35 bytes, which with a 64 MiB byte count and an hour of milliseconds
|
||||||
|
/// makes a 111-byte line. The test below recomputes that worst case from the
|
||||||
|
/// std error sets, so a wider name std adds fails the build's test run rather
|
||||||
|
/// than silently truncating an operator's only diagnostic.
|
||||||
|
pub const max_error_len: usize = 192;
|
||||||
|
|
||||||
/// `SourceStatus.url` is fixed-size so a copied status borrows nothing. A
|
/// `SourceStatus.url` is fixed-size so a copied status borrows nothing. A
|
||||||
/// blocklist url longer than this is truncated in the status only; the row
|
/// blocklist url longer than this is truncated in the status only; the row
|
||||||
@@ -131,6 +136,55 @@ const SourceLabel = struct {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// The one place the download failure text is formatted. `phase`, `cause` and
|
||||||
|
/// the byte count come from the fetcher's record of the concrete fault.
|
||||||
|
///
|
||||||
|
/// A timeout carries a record like any other failure: `fetchWithin` cancels the
|
||||||
|
/// fetch on its way out of the race, and the cancelled fetch records the phase
|
||||||
|
/// it was in, the status if a head had arrived and the bytes delivered so far,
|
||||||
|
/// all before this function reads it. So an expiry reads
|
||||||
|
/// `Timeout (receive_body, cause Canceled, http 200, 5242880 bytes, 300000 ms)`.
|
||||||
|
/// `no fetch` and `-` are for a failure that never reached the network at all:
|
||||||
|
/// `download` clears the record before anything can fail, so a local fault —
|
||||||
|
/// the temporary file, an allocation, a refused task — reads as its own
|
||||||
|
/// taxonomy name with nothing borrowed from the source fetched before it.
|
||||||
|
fn downloadFailureText(
|
||||||
|
buf: []u8,
|
||||||
|
err: anyerror,
|
||||||
|
failure: ?fetcher.Failure,
|
||||||
|
last_status: ?std.http.Status,
|
||||||
|
elapsed_ms: u64,
|
||||||
|
) []const u8 {
|
||||||
|
var status_buf: [8]u8 = undefined;
|
||||||
|
const status_text = if (if (failure) |f| f.status else last_status) |code|
|
||||||
|
std.fmt.bufPrint(&status_buf, "{d}", .{@intFromEnum(code)}) catch "none"
|
||||||
|
else
|
||||||
|
"none";
|
||||||
|
|
||||||
|
var bytes_buf: [24]u8 = undefined;
|
||||||
|
const bytes_text = if (failure) |f|
|
||||||
|
std.fmt.bufPrint(&bytes_buf, "{d}", .{f.bytes_read}) catch "-"
|
||||||
|
else
|
||||||
|
"-";
|
||||||
|
|
||||||
|
if (failure) |f| {
|
||||||
|
return std.fmt.bufPrint(buf, "{s} ({s}, cause {s}, http {s}, {s} bytes, {d} ms)", .{
|
||||||
|
@errorName(err),
|
||||||
|
@tagName(f.phase),
|
||||||
|
@errorName(f.cause),
|
||||||
|
status_text,
|
||||||
|
bytes_text,
|
||||||
|
elapsed_ms,
|
||||||
|
}) catch @errorName(err);
|
||||||
|
}
|
||||||
|
return std.fmt.bufPrint(buf, "{s} (no fetch, http {s}, {s} bytes, {d} ms)", .{
|
||||||
|
@errorName(err),
|
||||||
|
status_text,
|
||||||
|
bytes_text,
|
||||||
|
elapsed_ms,
|
||||||
|
}) catch @errorName(err);
|
||||||
|
}
|
||||||
|
|
||||||
pub const Paths = struct {
|
pub const Paths = struct {
|
||||||
/// `<data_dir>`, owned by the caller and left open for the manager's life.
|
/// `<data_dir>`, owned by the caller and left open for the manager's life.
|
||||||
dir: std.Io.Dir,
|
dir: std.Io.Dir,
|
||||||
@@ -1169,11 +1223,12 @@ pub const Manager = struct {
|
|||||||
raw_name: []const u8,
|
raw_name: []const u8,
|
||||||
tmp: TempNames,
|
tmp: TempNames,
|
||||||
) Error!Prepared {
|
) Error!Prepared {
|
||||||
self.download(io, dir, raw_name, row) catch |err| switch (err) {
|
var elapsed_ms: u64 = 0;
|
||||||
|
self.download(io, dir, raw_name, row, &elapsed_ms) catch |err| switch (err) {
|
||||||
error.OutOfMemory => return error.OutOfMemory,
|
error.OutOfMemory => return error.OutOfMemory,
|
||||||
error.Canceled => return error.Canceled,
|
error.Canceled => return error.Canceled,
|
||||||
else => {
|
else => {
|
||||||
self.reportFetchFailure(row, status, err);
|
self.reportDownloadFailure(row, status, err, elapsed_ms);
|
||||||
return .failed;
|
return .failed;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -1297,7 +1352,18 @@ pub const Manager = struct {
|
|||||||
dir: std.Io.Dir,
|
dir: std.Io.Dir,
|
||||||
raw_name: []const u8,
|
raw_name: []const u8,
|
||||||
row: sources_repo.SourceRow,
|
row: sources_repo.SourceRow,
|
||||||
|
/// Milliseconds the download itself took, set whether it succeeded or
|
||||||
|
/// failed, so the failure line can say how long the peer had.
|
||||||
|
elapsed_ms: *u64,
|
||||||
) !void {
|
) !void {
|
||||||
|
// Ahead of everything that can fail. The record is the fetcher's, not
|
||||||
|
// this source's, and a local failure here — the create, the allocation,
|
||||||
|
// a refused task — would otherwise leave the PREVIOUS source's network
|
||||||
|
// cause in place for `reportDownloadFailure` to print as this one's.
|
||||||
|
// Cleared here, a null record means no fetch reached the network.
|
||||||
|
self.fetcher.last_failure = null;
|
||||||
|
self.fetcher.last_status = null;
|
||||||
|
|
||||||
const file = try dir.createFile(io, raw_name, .{ .permissions = .fromMode(0o600) });
|
const file = try dir.createFile(io, raw_name, .{ .permissions = .fromMode(0o600) });
|
||||||
defer file.close(io);
|
defer file.close(io);
|
||||||
|
|
||||||
@@ -1305,16 +1371,14 @@ pub const Manager = struct {
|
|||||||
defer self.gpa.free(buffer);
|
defer self.gpa.free(buffer);
|
||||||
|
|
||||||
var fw = file.writer(io, buffer);
|
var fw = file.writer(io, buffer);
|
||||||
const result = self.fetchWithin(io, row.url, &fw.interface) catch |err| {
|
const result = self.fetchWithin(io, row.url, &fw.interface, elapsed_ms) catch |err| {
|
||||||
// `fetcher.Error.Unexpected` is what a failing sink surfaces as;
|
// `fetcher.Error.Unexpected` is what a failing sink surfaces as;
|
||||||
// the concrete cause is on this writer, which the fetcher does not
|
// the concrete cause is on this writer, which the fetcher does not
|
||||||
// own.
|
// own.
|
||||||
if (fw.err) |cause| return cause;
|
if (fw.err) |cause| return cause;
|
||||||
if (err == error.HttpStatus) {
|
// The status used to get a second warning of its own. It is a field
|
||||||
if (self.fetcher.last_status) |status| {
|
// of the one download-failure line now, so a second line would only
|
||||||
log.warn("blocklist {f}: http status {d}", .{ SourceLabel.of(row), @intFromEnum(status) });
|
// repeat it.
|
||||||
}
|
|
||||||
}
|
|
||||||
return err;
|
return err;
|
||||||
};
|
};
|
||||||
try fw.interface.flush();
|
try fw.interface.flush();
|
||||||
@@ -1332,7 +1396,12 @@ pub const Manager = struct {
|
|||||||
io: std.Io,
|
io: std.Io,
|
||||||
url: []const u8,
|
url: []const u8,
|
||||||
w: *std.Io.Writer,
|
w: *std.Io.Writer,
|
||||||
|
elapsed_ms: *u64,
|
||||||
) fetcher.Error!fetcher.Result {
|
) fetcher.Error!fetcher.Result {
|
||||||
|
// `awake` and not `real`: an operator reading "812 ms" wants the time
|
||||||
|
// the transfer was given, which a stepped wall clock would misreport.
|
||||||
|
const started = std.Io.Clock.awake.now(io);
|
||||||
|
|
||||||
var outcomes: [2]Outcome = undefined;
|
var outcomes: [2]Outcome = undefined;
|
||||||
var race: std.Io.Select(Outcome) = .init(io, &outcomes);
|
var race: std.Io.Select(Outcome) = .init(io, &outcomes);
|
||||||
defer race.cancelDiscard();
|
defer race.cancelDiscard();
|
||||||
@@ -1344,7 +1413,12 @@ pub const Manager = struct {
|
|||||||
error.ConcurrencyUnavailable => return error.SystemResources,
|
error.ConcurrencyUnavailable => return error.SystemResources,
|
||||||
};
|
};
|
||||||
|
|
||||||
switch (try race.await()) {
|
// Read before the deferred `cancelDiscard`, which tears the loser down:
|
||||||
|
// the number is the time the peer had, not that plus the teardown.
|
||||||
|
const outcome = race.await();
|
||||||
|
elapsed_ms.* = @intCast(@max(0, started.durationTo(std.Io.Clock.awake.now(io)).toMilliseconds()));
|
||||||
|
|
||||||
|
switch (try outcome) {
|
||||||
.fetch => |result| return result,
|
.fetch => |result| return result,
|
||||||
.expiry => |result| {
|
.expiry => |result| {
|
||||||
// A canceled sleep means this task is being torn down, not that
|
// A canceled sleep means this task is being torn down, not that
|
||||||
@@ -1514,15 +1588,21 @@ pub const Manager = struct {
|
|||||||
return compiledBodiesMatch(list_bytes, wild_bytes, allow_bytes, expected);
|
return compiledBodiesMatch(list_bytes, wild_bytes, allow_bytes, expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reportFetchFailure(
|
/// One line per failed source per pass, and the only place the download
|
||||||
|
/// failure text is built. The Diagnostics event detail gets the same text,
|
||||||
|
/// so an operator reading either can tell a TLS alert from a reset
|
||||||
|
/// connection from a truncated chunk.
|
||||||
|
fn reportDownloadFailure(
|
||||||
self: *Manager,
|
self: *Manager,
|
||||||
row: sources_repo.SourceRow,
|
row: sources_repo.SourceRow,
|
||||||
status: *SourceStatus,
|
status: *SourceStatus,
|
||||||
err: anyerror,
|
err: anyerror,
|
||||||
|
elapsed_ms: u64,
|
||||||
) void {
|
) void {
|
||||||
_ = self;
|
var buf: [max_error_len]u8 = undefined;
|
||||||
log.warn("blocklist {f}: download failed: {s}", .{ SourceLabel.of(row), @errorName(err) });
|
const text = downloadFailureText(&buf, err, self.fetcher.last_failure, self.fetcher.last_status, elapsed_ms);
|
||||||
status.fail(.fetch_failed, @errorName(err));
|
log.warn("blocklist {f}: download failed: {s}", .{ SourceLabel.of(row), text });
|
||||||
|
status.fail(.fetch_failed, text);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reportCompileFailure(
|
fn reportCompileFailure(
|
||||||
@@ -3580,3 +3660,104 @@ test "setSchedule is what the live schedule readers see" {
|
|||||||
try testing.expectEqual(@as(u16, 6), live.interval_hours);
|
try testing.expectEqual(@as(u16, 6), live.interval_hours);
|
||||||
try testing.expect(manager.schedule_event.isSet());
|
try testing.expect(manager.schedule_event.isSet());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "a download failure line names the concrete cause behind the taxonomy" {
|
||||||
|
var buf: [1024]u8 = undefined;
|
||||||
|
const row: sources_repo.SourceRow = .{
|
||||||
|
.id = 3,
|
||||||
|
.url = "https://lists.example/download/token/hunter2/hosts.txt?apikey=s3cr3t",
|
||||||
|
.name = "ads",
|
||||||
|
.enabled = true,
|
||||||
|
.last_updated = null,
|
||||||
|
.domain_count = 0,
|
||||||
|
.wildcard_count = 0,
|
||||||
|
.skipped_regex_count = 0,
|
||||||
|
.skipped_unsupported_count = 0,
|
||||||
|
.checksum = null,
|
||||||
|
};
|
||||||
|
var text_buf: [max_error_len]u8 = undefined;
|
||||||
|
const text = downloadFailureText(&text_buf, error.ReceiveFailed, .{
|
||||||
|
.phase = .receive_body,
|
||||||
|
.cause = error.HttpChunkTruncated,
|
||||||
|
.status = .ok,
|
||||||
|
.bytes_read = 1024 * 1024,
|
||||||
|
}, .ok, 812);
|
||||||
|
|
||||||
|
const printed = try std.fmt.bufPrint(&buf, "blocklist {f}: download failed: {s}", .{
|
||||||
|
SourceLabel.of(row),
|
||||||
|
text,
|
||||||
|
});
|
||||||
|
try testing.expectEqualStrings(
|
||||||
|
"blocklist source 3 'ads' 'https://lists.example': download failed:" ++
|
||||||
|
" ReceiveFailed (receive_body, cause HttpChunkTruncated, http 200, 1048576 bytes, 812 ms)",
|
||||||
|
printed,
|
||||||
|
);
|
||||||
|
try testing.expect(!std.mem.containsAtLeast(u8, printed, 1, "hunter2"));
|
||||||
|
try testing.expect(!std.mem.containsAtLeast(u8, printed, 1, "s3cr3t"));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "an expiry names the phase and the progress the cancelled fetch had made" {
|
||||||
|
// `fetchWithin` cancels the fetch before it reports, so a timeout normally
|
||||||
|
// does have a record: the cancellation the fetch saw, under the manager's
|
||||||
|
// own `error.Timeout`.
|
||||||
|
var text_buf: [max_error_len]u8 = undefined;
|
||||||
|
try testing.expectEqualStrings(
|
||||||
|
"Timeout (receive_body, cause Canceled, http 200, 5242880 bytes, 300000 ms)",
|
||||||
|
downloadFailureText(&text_buf, error.Timeout, .{
|
||||||
|
.phase = .receive_body,
|
||||||
|
.cause = error.Canceled,
|
||||||
|
.status = .ok,
|
||||||
|
.bytes_read = 5 * 1024 * 1024,
|
||||||
|
}, .ok, 300000),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "a failure that never reached the network says so and borrows nothing" {
|
||||||
|
// `download` clears the fetcher's record before anything can fail, so a
|
||||||
|
// local fault reads as itself. Printing a phase and a cause here would be
|
||||||
|
// printing the previous source's network fault against this source.
|
||||||
|
var text_buf: [max_error_len]u8 = undefined;
|
||||||
|
try testing.expectEqualStrings(
|
||||||
|
"SystemResources (no fetch, http none, - bytes, 0 ms)",
|
||||||
|
downloadFailureText(&text_buf, error.SystemResources, null, null, 0),
|
||||||
|
);
|
||||||
|
try testing.expectEqualStrings(
|
||||||
|
"AccessDenied (no fetch, http none, - bytes, 3 ms)",
|
||||||
|
downloadFailureText(&text_buf, error.AccessDenied, null, null, 3),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "max_error_len holds the widest download failure line whole" {
|
||||||
|
// `@errorName` of the unwrapped cause is the only unbounded-looking part.
|
||||||
|
// The two sets below are every set the unwraps in `upstream/transport.zig`
|
||||||
|
// can return a member of.
|
||||||
|
const widest_cause = comptime blk: {
|
||||||
|
var widest: []const u8 = "";
|
||||||
|
for (@typeInfo(std.crypto.tls.Client.ReadError).error_set.?) |member| {
|
||||||
|
if (member.name.len > widest.len) widest = member.name;
|
||||||
|
}
|
||||||
|
for (@typeInfo(std.http.Client.RequestError).error_set.?) |member| {
|
||||||
|
if (member.name.len > widest.len) widest = member.name;
|
||||||
|
}
|
||||||
|
for (@typeInfo(std.http.Reader.BodyError).error_set.?) |member| {
|
||||||
|
if (member.name.len > widest.len) widest = member.name;
|
||||||
|
}
|
||||||
|
break :blk widest;
|
||||||
|
};
|
||||||
|
|
||||||
|
var buf: [max_error_len]u8 = undefined;
|
||||||
|
const text = std.fmt.bufPrint(
|
||||||
|
&buf,
|
||||||
|
"{s} ({s}, cause {s}, http {d}, {d} bytes, {d} ms)",
|
||||||
|
.{
|
||||||
|
"SystemResources",
|
||||||
|
@tagName(fetcher.Phase.receive_body),
|
||||||
|
widest_cause,
|
||||||
|
@as(u16, 599),
|
||||||
|
@as(u64, fetcher.max_body_bytes),
|
||||||
|
@as(u64, std.time.ms_per_hour),
|
||||||
|
},
|
||||||
|
) catch unreachable;
|
||||||
|
try testing.expect(text.len <= max_error_len);
|
||||||
|
try testing.expectEqualStrings("DetectingNetworkConfigurationFailed", widest_cause);
|
||||||
|
}
|
||||||
|
|||||||
@@ -226,43 +226,12 @@ const Connection = std.http.Client.Connection;
|
|||||||
const Request = std.http.Client.Request;
|
const Request = std.http.Client.Request;
|
||||||
const Response = std.http.Client.Response;
|
const Response = std.http.Client.Response;
|
||||||
|
|
||||||
/// `std.Io.Writer` collapses every send failure to `error.WriteFailed` and
|
// The unwraps live in `transport.zig` because the blocklist fetcher needs the
|
||||||
/// stashes the cause on the connection's socket writer. Unwrapping it is what
|
// same concrete causes. They are aliased rather than qualified so the call
|
||||||
/// keeps `error.Canceled` and the local resource errors out of the peer fault
|
// sites and the tests below read as they did when they were local.
|
||||||
/// group, exactly as `concreteWrite` does for DoT.
|
const sendCause = transport.sendCause;
|
||||||
fn sendCause(req: *const Request, err: anyerror) anyerror {
|
const headCause = transport.headCause;
|
||||||
if (err != error.WriteFailed) return err;
|
const bodyCause = transport.bodyCause;
|
||||||
const connection = req.connection orelse return err;
|
|
||||||
return connection.stream_writer.err orelse err;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `receiveHead` collapses a transport failure to `error.ReadFailed` and names
|
|
||||||
/// `Connection.getReadError` as the accessor for the concrete cause.
|
|
||||||
fn headCause(req: *const Request, err: anyerror) anyerror {
|
|
||||||
if (err != error.ReadFailed) return err;
|
|
||||||
const connection = req.connection orelse return err;
|
|
||||||
return readCause(connection, err);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// A body read reports two different kinds of failure through the same
|
|
||||||
/// `error.ReadFailed`. An HTTP framing fault lands on the response, and only a
|
|
||||||
/// read that never reached the framing leaves the connection's cause, so the
|
|
||||||
/// response is consulted first.
|
|
||||||
fn bodyCause(resp: *const Response, err: anyerror) anyerror {
|
|
||||||
if (err != error.ReadFailed) return err;
|
|
||||||
if (resp.bodyErr()) |cause| return cause;
|
|
||||||
const connection = resp.request.connection orelse return err;
|
|
||||||
return readCause(connection, err);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `Connection.getReadError` reads the socket reader's stashed cause with `.?`.
|
|
||||||
/// On a plain connection that is its only source, so calling it with nothing
|
|
||||||
/// stashed would panic rather than return null; the guard keeps this unwrap
|
|
||||||
/// total on the path this client can reach without TLS.
|
|
||||||
fn readCause(connection: *const Connection, err: anyerror) anyerror {
|
|
||||||
if (connection.protocol == .plain and connection.stream_reader.err == null) return err;
|
|
||||||
return connection.getReadError() orelse err;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// RFC 8484 §6: the response media type is `application/dns-message`. The
|
/// RFC 8484 §6: the response media type is `application/dns-message`. The
|
||||||
/// header may carry parameters (`; charset=…`) and the type is case-insensitive
|
/// header may carry parameters (`; charset=…`) and the type is case-insensitive
|
||||||
|
|||||||
+17
-12
@@ -1951,11 +1951,9 @@ test "a waiter that spends its whole budget queueing blames nobody but the budge
|
|||||||
try testing.expectEqual(@as(usize, 2), fake.calls.load(.acquire));
|
try testing.expectEqual(@as(usize, 2), fake.calls.load(.acquire));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The saturated-primary test's numbers. The holders stall longer than the
|
/// The saturated-primary test's budgets. The holders do not stall on a clock at
|
||||||
/// probe's whole budget, so the probe provably cannot be waiting for one of
|
/// all: they are held on a gate the test releases, so the probe's own budget is
|
||||||
/// their slots; the stall still ends inside the test so the holders' futures
|
/// the only duration the test depends on.
|
||||||
/// can be awaited rather than abandoned.
|
|
||||||
const saturating_stall_ms = 800;
|
|
||||||
const probe_attempt_ms = 150;
|
const probe_attempt_ms = 150;
|
||||||
const probe_total_ms = 300;
|
const probe_total_ms = 300;
|
||||||
|
|
||||||
@@ -1968,10 +1966,11 @@ test "a saturated primary defers to a standby that has capacity" {
|
|||||||
// slow exchange, and the standby is idle. Priority orders the entries that
|
// slow exchange, and the standby is idle. Priority orders the entries that
|
||||||
// can be admitted now; it does not entitle a saturated entry to hold a
|
// can be admitted now; it does not entitle a saturated entry to hold a
|
||||||
// query until the whole budget is gone.
|
// query until the whole budget is gone.
|
||||||
var primary: Fake = .{ .behavior = .{ .slow = .{
|
// The holders are released by the test, not by a clock: a fixed stall can
|
||||||
.duration = .{ .raw = .fromMilliseconds(saturating_stall_ms), .clock = .awake },
|
// expire before a loaded scheduler lets the poller observe both calls, and
|
||||||
.reply = response_bytes,
|
// an overlap that already passed cannot be recovered by polling.
|
||||||
} } };
|
var gate: std.Io.Semaphore = .{};
|
||||||
|
var primary: Fake = .{ .behavior = .{ .hold = .{ .gate = &gate, .reply = response_bytes } } };
|
||||||
var standby: Fake = .{ .behavior = .{ .reply = alt_response_bytes } };
|
var standby: Fake = .{ .behavior = .{ .reply = alt_response_bytes } };
|
||||||
var entries = [_]Entry{
|
var entries = [_]Entry{
|
||||||
testEntrySlots("https://primary.example/dns-query", &primary, 10, 2),
|
testEntrySlots("https://primary.example/dns-query", &primary, 10, 2),
|
||||||
@@ -1989,14 +1988,20 @@ test "a saturated primary defers to a standby that has capacity" {
|
|||||||
|
|
||||||
var bufs: [2][512]u8 = undefined;
|
var bufs: [2][512]u8 = undefined;
|
||||||
var holders: [2]std.Io.Future(transport.ExchangeError!Attributed) = undefined;
|
var holders: [2]std.Io.Future(transport.ExchangeError!Attributed) = undefined;
|
||||||
|
var spawned: usize = 0;
|
||||||
|
// Every spawned holder is released and awaited on any exit, including the
|
||||||
|
// skip when the second spawn is refused: a holder left blocked on the gate
|
||||||
|
// would outlive the stack it reads.
|
||||||
|
defer for (holders[0..spawned]) |*holder| {
|
||||||
|
_ = holder.await(io) catch Attributed.discarded;
|
||||||
|
};
|
||||||
|
defer for (0..spawned) |_| gate.post(io);
|
||||||
for (&holders, &bufs) |*holder, *buf| {
|
for (&holders, &bufs) |*holder, *buf| {
|
||||||
holder.* = io.concurrent(exchangeAttributed, .{ &holder_pool, io, buf }) catch |err| switch (err) {
|
holder.* = io.concurrent(exchangeAttributed, .{ &holder_pool, io, buf }) catch |err| switch (err) {
|
||||||
error.ConcurrencyUnavailable => return error.SkipZigTest,
|
error.ConcurrencyUnavailable => return error.SkipZigTest,
|
||||||
};
|
};
|
||||||
|
spawned += 1;
|
||||||
}
|
}
|
||||||
defer for (&holders) |*holder| {
|
|
||||||
_ = holder.await(io) catch Attributed.discarded;
|
|
||||||
};
|
|
||||||
// Both permits provably taken before the probe asks for one.
|
// Both permits provably taken before the probe asks for one.
|
||||||
try awaitInFlight(io, &primary, 2);
|
try awaitInFlight(io, &primary, 2);
|
||||||
|
|
||||||
|
|||||||
@@ -419,6 +419,52 @@ pub const Client = struct {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// The unwrap helpers below turn the single collapsed error `std.http.Client`
|
||||||
|
/// reports into the concrete cause it stashed. Every HTTP caller in this tree
|
||||||
|
/// uses them: the DoH client classifies by the unwrapped cause, the blocklist
|
||||||
|
/// fetcher keeps its own classification and records the cause for the operator.
|
||||||
|
const Connection = std.http.Client.Connection;
|
||||||
|
const Request = std.http.Client.Request;
|
||||||
|
const Response = std.http.Client.Response;
|
||||||
|
|
||||||
|
/// `std.Io.Writer` collapses every send failure to `error.WriteFailed` and
|
||||||
|
/// stashes the cause on the connection's socket writer. Unwrapping it is what
|
||||||
|
/// keeps `error.Canceled` and the local resource errors out of the peer fault
|
||||||
|
/// group.
|
||||||
|
pub fn sendCause(req: *const Request, err: anyerror) anyerror {
|
||||||
|
if (err != error.WriteFailed) return err;
|
||||||
|
const connection = req.connection orelse return err;
|
||||||
|
return connection.stream_writer.err orelse err;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `receiveHead` collapses a transport failure to `error.ReadFailed` and names
|
||||||
|
/// `Connection.getReadError` as the accessor for the concrete cause.
|
||||||
|
pub fn headCause(req: *const Request, err: anyerror) anyerror {
|
||||||
|
if (err != error.ReadFailed) return err;
|
||||||
|
const connection = req.connection orelse return err;
|
||||||
|
return readCause(connection, err);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A body read reports two different kinds of failure through the same
|
||||||
|
/// `error.ReadFailed`. An HTTP framing fault lands on the response, and only a
|
||||||
|
/// read that never reached the framing leaves the connection's cause, so the
|
||||||
|
/// response is consulted first.
|
||||||
|
pub fn bodyCause(resp: *const Response, err: anyerror) anyerror {
|
||||||
|
if (err != error.ReadFailed) return err;
|
||||||
|
if (resp.bodyErr()) |cause| return cause;
|
||||||
|
const connection = resp.request.connection orelse return err;
|
||||||
|
return readCause(connection, err);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `Connection.getReadError` reads the socket reader's stashed cause with `.?`.
|
||||||
|
/// On a plain connection that is its only source, so calling it with nothing
|
||||||
|
/// stashed would panic rather than return null; the guard keeps this unwrap
|
||||||
|
/// total on the path a plain connection can reach without TLS.
|
||||||
|
pub fn readCause(connection: *const Connection, err: anyerror) anyerror {
|
||||||
|
if (connection.protocol == .plain and connection.stream_reader.err == null) return err;
|
||||||
|
return connection.getReadError() orelse err;
|
||||||
|
}
|
||||||
|
|
||||||
pub const ValidateError = error{ BadResponse, ResponseMismatch };
|
pub const ValidateError = error{ BadResponse, ResponseMismatch };
|
||||||
|
|
||||||
/// RFC 9619: exactly one question on both sides. RFC 4343: names compare
|
/// RFC 9619: exactly one question on both sides. RFC 4343: names compare
|
||||||
|
|||||||
+207
-26
@@ -135,7 +135,7 @@ const poll_interval_ns: u64 = 15 * std.time.ns_per_s;
|
|||||||
const progress_every_polls: usize = 8;
|
const progress_every_polls: usize = 8;
|
||||||
|
|
||||||
/// The bound on a captured git command that talks to the network. The
|
/// The bound on a captured git command that talks to the network. The
|
||||||
/// interactive commands are not captured and not bounded — see `gitInherit`.
|
/// interactive commands are not captured and not bounded — see `runInherited`.
|
||||||
const git_network_timeout_s: u64 = 120;
|
const git_network_timeout_s: u64 = 120;
|
||||||
const git_local_timeout_s: u64 = 30;
|
const git_local_timeout_s: u64 = 30;
|
||||||
|
|
||||||
@@ -1424,16 +1424,18 @@ fn attemptBudgetNs(started_ns: i96, now_ns: i96, budget_ns: u64, ceiling_ns: u64
|
|||||||
/// than carrying its own copy: two pins that can drift are not a pin.
|
/// than carrying its own copy: two pins that can drift are not a pin.
|
||||||
const gates_workflow_path = ".gitea/workflows/gates.yml";
|
const gates_workflow_path = ".gitea/workflows/gates.yml";
|
||||||
|
|
||||||
/// The three toolchain versions `gates.yml` declares.
|
/// The toolchain `gates.yml` declares: three versions and the hash of the zig
|
||||||
|
/// tarball those versions are meaningless without.
|
||||||
const ToolchainPins = struct {
|
const ToolchainPins = struct {
|
||||||
zig: []const u8,
|
zig: []const u8,
|
||||||
|
zig_tarball_sha256: []const u8,
|
||||||
node: []const u8,
|
node: []const u8,
|
||||||
npm: []const u8,
|
npm: []const u8,
|
||||||
};
|
};
|
||||||
|
|
||||||
const ToolchainKey = enum { ZIG_VERSION, NODE_VERSION, NPM_VERSION };
|
const ToolchainKey = enum { ZIG_VERSION, ZIG_TARBALL_SHA256, NODE_VERSION, NPM_VERSION };
|
||||||
|
|
||||||
/// The three pins out of the top-level `env:` block of `gates.yml`.
|
/// The four pins out of the top-level `env:` block of `gates.yml`.
|
||||||
///
|
///
|
||||||
/// A line parser and not a YAML library: the one shape this has to read is
|
/// A line parser and not a YAML library: the one shape this has to read is
|
||||||
/// ` KEY: "value"` under a column-zero `env:`, and a dependency that can parse
|
/// ` KEY: "value"` under a column-zero `env:`, and a dependency that can parse
|
||||||
@@ -1442,10 +1444,10 @@ const ToolchainKey = enum { ZIG_VERSION, NODE_VERSION, NPM_VERSION };
|
|||||||
/// non-comment line that is not indented, which is how the file's own `jobs:`
|
/// non-comment line that is not indented, which is how the file's own `jobs:`
|
||||||
/// key terminates it.
|
/// key terminates it.
|
||||||
///
|
///
|
||||||
/// Null when any of the three is missing, because a parity check that silently
|
/// Null when any of the four is missing, because a parity check that silently
|
||||||
/// dropped one of them would report parity it never established.
|
/// dropped one of them would report parity it never established.
|
||||||
fn parseToolchainPins(source: []const u8) ?ToolchainPins {
|
fn parseToolchainPins(source: []const u8) ?ToolchainPins {
|
||||||
var found: [3]?[]const u8 = .{ null, null, null };
|
var found: [@typeInfo(ToolchainKey).@"enum".fields.len]?[]const u8 = @splat(null);
|
||||||
|
|
||||||
var lines = std.mem.splitScalar(u8, source, '\n');
|
var lines = std.mem.splitScalar(u8, source, '\n');
|
||||||
var inside = false;
|
var inside = false;
|
||||||
@@ -1468,13 +1470,28 @@ fn parseToolchainPins(source: []const u8) ?ToolchainPins {
|
|||||||
found[@intFromEnum(which)] = value;
|
found[@intFromEnum(which)] = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const digest = found[@intFromEnum(ToolchainKey.ZIG_TARBALL_SHA256)] orelse return null;
|
||||||
|
if (!isSha256Hex(digest)) return null;
|
||||||
|
|
||||||
return .{
|
return .{
|
||||||
.zig = found[@intFromEnum(ToolchainKey.ZIG_VERSION)] orelse return null,
|
.zig = found[@intFromEnum(ToolchainKey.ZIG_VERSION)] orelse return null,
|
||||||
|
.zig_tarball_sha256 = digest,
|
||||||
.node = found[@intFromEnum(ToolchainKey.NODE_VERSION)] orelse return null,
|
.node = found[@intFromEnum(ToolchainKey.NODE_VERSION)] orelse return null,
|
||||||
.npm = found[@intFromEnum(ToolchainKey.NPM_VERSION)] orelse return null,
|
.npm = found[@intFromEnum(ToolchainKey.NPM_VERSION)] orelse return null,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A digest that is anything but 64 lowercase hex digits cannot name a cache
|
||||||
|
/// directory or be compared to a computed hash, so the parser refuses it.
|
||||||
|
fn isSha256Hex(digest: []const u8) bool {
|
||||||
|
if (digest.len != 64) return false;
|
||||||
|
for (digest) |c| switch (c) {
|
||||||
|
'0'...'9', 'a'...'f' => {},
|
||||||
|
else => return false,
|
||||||
|
};
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/// Every variable a release build is allowed to see, and nothing else.
|
/// Every variable a release build is allowed to see, and nothing else.
|
||||||
///
|
///
|
||||||
/// The bundle and the tarballs are hashed into `flake.nix` before CI rebuilds
|
/// The bundle and the tarballs are hashed into `flake.nix` before CI rebuilds
|
||||||
@@ -1524,7 +1541,7 @@ fn scratchRoot(ctx: *Ctx, version: []const u8) []const u8 {
|
|||||||
///
|
///
|
||||||
/// stdio is inherited: these commands take minutes and an operator watching a
|
/// stdio is inherited: these commands take minutes and an operator watching a
|
||||||
/// cut needs to see npm and zig make progress. Their success is read from the
|
/// cut needs to see npm and zig make progress. Their success is read from the
|
||||||
/// termination state, exactly as `gitInherit` reads it.
|
/// termination state, exactly as `runInherited` reads it.
|
||||||
///
|
///
|
||||||
/// The umask arrives through `sh` rather than through this process: zig 0.16.0's
|
/// The umask arrives through `sh` rather than through this process: zig 0.16.0's
|
||||||
/// standard library exposes `umask(2)` only as a libc extern (`std.c.umask`),
|
/// standard library exposes `umask(2)` only as a libc extern (`std.c.umask`),
|
||||||
@@ -1581,7 +1598,8 @@ fn runPinned(
|
|||||||
/// recompute. A mismatch is a refusal and not a warning: the alternative is a
|
/// recompute. A mismatch is a refusal and not a warning: the alternative is a
|
||||||
/// tag whose flake pins bytes no CI run can reproduce, which is only discovered
|
/// tag whose flake pins bytes no CI run can reproduce, which is only discovered
|
||||||
/// after the tag is public.
|
/// after the tag is public.
|
||||||
fn toolchainParity(ctx: *Ctx) !void {
|
/// Returns the absolute path of the official zig this cut must build with.
|
||||||
|
fn toolchainParity(ctx: *Ctx) ![]const u8 {
|
||||||
// The admin bundle is built by Rolldown and Lightning CSS, whose native
|
// The admin bundle is built by Rolldown and Lightning CSS, whose native
|
||||||
// bindings are chosen per host; CI builds it on an x86_64 Ubuntu runner.
|
// bindings are chosen per host; CI builds it on an x86_64 Ubuntu runner.
|
||||||
// A bundle built on another architecture is a different set of bytes, so
|
// A bundle built on another architecture is a different set of bytes, so
|
||||||
@@ -1598,7 +1616,7 @@ fn toolchainParity(ctx: *Ctx) !void {
|
|||||||
return CheckFailed;
|
return CheckFailed;
|
||||||
};
|
};
|
||||||
const pins = parseToolchainPins(source) orelse {
|
const pins = parseToolchainPins(source) orelse {
|
||||||
ctx.soft("toolchain", "{s} does not declare ZIG_VERSION, NODE_VERSION and NPM_VERSION in its top-level `env:` block", .{gates_workflow_path});
|
ctx.soft("toolchain", "{s} does not declare ZIG_VERSION, ZIG_TARBALL_SHA256, NODE_VERSION and NPM_VERSION in its top-level `env:` block", .{gates_workflow_path});
|
||||||
return CheckFailed;
|
return CheckFailed;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1607,7 +1625,14 @@ fn toolchainParity(ctx: *Ctx) !void {
|
|||||||
// print the bare number already.
|
// print the bare number already.
|
||||||
try assertVersion(ctx, "node", &.{ "node", "--version" }, "v", pins.node);
|
try assertVersion(ctx, "node", &.{ "node", "--version" }, "v", pins.node);
|
||||||
try assertVersion(ctx, "npm", &.{ "npm", "--version" }, "", pins.npm);
|
try assertVersion(ctx, "npm", &.{ "npm", "--version" }, "", pins.npm);
|
||||||
try assertVersion(ctx, "zig", &.{ "zig", "version" }, "", pins.zig);
|
|
||||||
|
// The zig on PATH is not consulted here, and `zig version` would not settle
|
||||||
|
// it anyway: the Arch Linux 0.16.0 package is built against the system LLVM
|
||||||
|
// and emits different machine code from the ziglang.org tarball CI installs,
|
||||||
|
// while both print `0.16.0`. The first real cut pinned hashes CI could not
|
||||||
|
// reproduce for exactly that reason. PATH's zig still compiles this program
|
||||||
|
// — that is `zig build cut`, whose bytes nobody hashes.
|
||||||
|
const zig_exe = try officialZig(ctx, pins);
|
||||||
|
|
||||||
// The npm config paths in the release environment silence `~/.npmrc` and
|
// The npm config paths in the release environment silence `~/.npmrc` and
|
||||||
// the node install's `etc/npmrc` only while nothing exists at them.
|
// the node install's `etc/npmrc` only while nothing exists at them.
|
||||||
@@ -1623,7 +1648,110 @@ fn toolchainParity(ctx: *Ctx) !void {
|
|||||||
ctx.soft("toolchain", "{s} exists, and npm would read it as {s}: the release environment relies on that path being absent", .{ entry.value, entry.key });
|
ctx.soft("toolchain", "{s} exists, and npm would read it as {s}: the release environment relies on that path being absent", .{ entry.value, entry.key });
|
||||||
return CheckFailed;
|
return CheckFailed;
|
||||||
}
|
}
|
||||||
ctx.pass("toolchain", "x86_64 Linux glibc with zig {s}, node {s} and npm {s}, as {s} pins them", .{ pins.zig, pins.node, pins.npm, gates_workflow_path });
|
ctx.pass("toolchain", "x86_64 Linux glibc with node {s}, npm {s}, and the official zig {s} at {s}, as {s} pins them", .{ pins.node, pins.npm, pins.zig, zig_exe, gates_workflow_path });
|
||||||
|
return zig_exe;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The official toolchain: the host is asserted x86_64 Linux above, so the
|
||||||
|
/// triple is a constant and not a lookup.
|
||||||
|
const zig_tarball_triple = "x86_64-linux";
|
||||||
|
|
||||||
|
fn zigTarballUrl(arena: Allocator, version: []const u8) []const u8 {
|
||||||
|
return std.fmt.allocPrint(
|
||||||
|
arena,
|
||||||
|
"https://ziglang.org/download/{s}/zig-" ++ zig_tarball_triple ++ "-{s}.tar.xz",
|
||||||
|
.{ version, version },
|
||||||
|
) catch @panic("OOM");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cache is keyed by version and lives outside the repository: it is a
|
||||||
|
/// toolchain, not a build output, and a download per cut would make the network
|
||||||
|
/// an input of every release.
|
||||||
|
/// Keyed by the pinned digest as well as the version: a changed pin for the
|
||||||
|
/// same version must not find yesterday's extraction and skip the download.
|
||||||
|
fn zigCacheRoot(arena: Allocator, home: []const u8, version: []const u8, digest: []const u8) []const u8 {
|
||||||
|
return std.fmt.allocPrint(arena, "{s}/.cache/nxdns-cut/zig-{s}-{s}", .{ home, version, digest }) catch @panic("OOM");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The tarball unpacks to one directory named after itself, which is why the
|
||||||
|
/// cache root is not that directory.
|
||||||
|
fn zigBinaryPath(arena: Allocator, home: []const u8, version: []const u8, digest: []const u8) []const u8 {
|
||||||
|
return std.fmt.allocPrint(
|
||||||
|
arena,
|
||||||
|
"{s}/zig-" ++ zig_tarball_triple ++ "-{s}/zig",
|
||||||
|
.{ zigCacheRoot(arena, home, version, digest), version },
|
||||||
|
) catch @panic("OOM");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The pinned zig from ziglang.org, downloaded once per version and verified
|
||||||
|
/// every run.
|
||||||
|
///
|
||||||
|
/// There is no fall back to PATH: a cut that silently built with whatever zig
|
||||||
|
/// the operator's distribution ships is the failure this exists to prevent.
|
||||||
|
fn officialZig(ctx: *Ctx, pins: ToolchainPins) ![]const u8 {
|
||||||
|
const home = ctx.get("HOME");
|
||||||
|
if (home.len == 0) {
|
||||||
|
ctx.soft("toolchain", "HOME is not set, and the official zig {s} is cached under $HOME/.cache/nxdns-cut", .{pins.zig});
|
||||||
|
return CheckFailed;
|
||||||
|
}
|
||||||
|
const root = zigCacheRoot(ctx.arena, home, pins.zig, pins.zig_tarball_sha256);
|
||||||
|
const exe = zigBinaryPath(ctx.arena, home, pins.zig, pins.zig_tarball_sha256);
|
||||||
|
|
||||||
|
const cached = if (Io.Dir.accessAbsolute(ctx.io, exe, .{})) |_| true else |err| switch (err) {
|
||||||
|
error.FileNotFound => false,
|
||||||
|
else => {
|
||||||
|
ctx.soft("toolchain", "cannot probe {s}: {t}", .{ exe, err });
|
||||||
|
return CheckFailed;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!cached) {
|
||||||
|
// A partially extracted cache from an interrupted run is not a
|
||||||
|
// toolchain, so the directory is rebuilt rather than added to.
|
||||||
|
Io.Dir.cwd().deleteTree(ctx.io, root) catch |err| {
|
||||||
|
ctx.soft("toolchain", "cannot clear {s}: {t}", .{ root, err });
|
||||||
|
return CheckFailed;
|
||||||
|
};
|
||||||
|
Io.Dir.cwd().createDirPath(ctx.io, root) catch |err| {
|
||||||
|
ctx.soft("toolchain", "cannot create {s}: {t}", .{ root, err });
|
||||||
|
return CheckFailed;
|
||||||
|
};
|
||||||
|
|
||||||
|
const tarball = ctx.fmt("{s}/zig.tar.xz", .{root});
|
||||||
|
const url = zigTarballUrl(ctx.arena, pins.zig);
|
||||||
|
ctx.note("toolchain: downloading {s}", .{url});
|
||||||
|
try runInherited(ctx, "toolchain", &.{ "curl", "-fsSL", "-o", tarball, url });
|
||||||
|
try assertTarballDigest(ctx, tarball, pins.zig_tarball_sha256);
|
||||||
|
try runInherited(ctx, "toolchain", &.{ "tar", "-xJf", tarball, "-C", root });
|
||||||
|
Io.Dir.cwd().deleteFile(ctx.io, tarball) catch |err| {
|
||||||
|
ctx.soft("toolchain", "cannot remove {s}: {t}", .{ tarball, err });
|
||||||
|
return CheckFailed;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
try assertVersion(ctx, "zig", &.{ exe, "version" }, "", pins.zig);
|
||||||
|
return exe;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// CI installs the same tarball through setup-zig, which checks it by minisign
|
||||||
|
/// against ziglang.org's public key; this digest is the cut's half of that same
|
||||||
|
/// assertion, so both machines build with bytes they each verified.
|
||||||
|
fn assertTarballDigest(ctx: *Ctx, path: []const u8, expected: []const u8) !void {
|
||||||
|
const bytes = Io.Dir.cwd().readFileAlloc(ctx.io, path, ctx.gpa, .limited(max_input_bytes)) catch |err| {
|
||||||
|
ctx.soft("toolchain", "cannot read {s}: {t}", .{ path, err });
|
||||||
|
return CheckFailed;
|
||||||
|
};
|
||||||
|
defer ctx.gpa.free(bytes);
|
||||||
|
|
||||||
|
var digest: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined;
|
||||||
|
std.crypto.hash.sha2.Sha256.hash(bytes, &digest, .{});
|
||||||
|
const hex = std.fmt.bytesToHex(digest, .lower);
|
||||||
|
if (!std.mem.eql(u8, &hex, expected)) {
|
||||||
|
ctx.soft("toolchain", "{s} hashes to {s}, and {s} pins ZIG_TARBALL_SHA256 {s}", .{
|
||||||
|
path, hex, gates_workflow_path, expected,
|
||||||
|
});
|
||||||
|
return CheckFailed;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn assertVersion(
|
fn assertVersion(
|
||||||
@@ -1661,7 +1789,10 @@ fn assertVersion(
|
|||||||
/// `-Dversion-string` disagrees with the manifest, so the manifest has to be
|
/// `-Dversion-string` disagrees with the manifest, so the manifest has to be
|
||||||
/// bumped first; and `flake.nix` is part of the bump commit's diff, so the pins
|
/// bumped first; and `flake.nix` is part of the bump commit's diff, so the pins
|
||||||
/// have to exist before the commit is made.
|
/// have to exist before the commit is made.
|
||||||
fn pinStage(ctx: *Ctx, version: []const u8, environ: *const std.process.Environ.Map) !void {
|
///
|
||||||
|
/// `zig_exe` is the official ziglang.org toolchain the parity check resolved.
|
||||||
|
/// Every build here goes through it by absolute path, never through PATH.
|
||||||
|
fn pinStage(ctx: *Ctx, version: []const u8, environ: *const std.process.Environ.Map, zig_exe: []const u8) !void {
|
||||||
// A fresh local cache each run: the one input of these bytes that lives
|
// A fresh local cache each run: the one input of these bytes that lives
|
||||||
// outside the repository and the toolchain. The global cache stays shared:
|
// outside the repository and the toolchain. The global cache stays shared:
|
||||||
// it is content-addressed, and a fresh one would refetch every dependency
|
// it is content-addressed, and a fresh one would refetch every dependency
|
||||||
@@ -1690,12 +1821,12 @@ fn pinStage(ctx: *Ctx, version: []const u8, environ: *const std.process.Environ.
|
|||||||
const dist_flags = [_][]const u8{ version_flag, "-Dadmin-dist=admin/dist", "-Doptimize=ReleaseSafe" };
|
const dist_flags = [_][]const u8{ version_flag, "-Dadmin-dist=admin/dist", "-Doptimize=ReleaseSafe" };
|
||||||
const cache_flags = [_][]const u8{ "--cache-dir", cache_dir };
|
const cache_flags = [_][]const u8{ "--cache-dir", cache_dir };
|
||||||
|
|
||||||
try runPinned(ctx, "dist", environ, repo_root, try zigBuild(ctx, "dist", &dist_flags, &cache_flags));
|
try runPinned(ctx, "dist", environ, repo_root, try zigBuild(ctx, zig_exe, "dist", &dist_flags, &cache_flags));
|
||||||
ctx.pass("dist", "the {s} tarballs and SHA256SUMS are under zig-out/dist", .{version});
|
ctx.pass("dist", "the {s} tarballs and SHA256SUMS are under zig-out/dist", .{version});
|
||||||
|
|
||||||
try runPinned(ctx, "pin", environ, repo_root, try zigBuild(ctx, "pin-flake", &dist_flags, &cache_flags));
|
try runPinned(ctx, "pin", environ, repo_root, try zigBuild(ctx, zig_exe, "pin-flake", &dist_flags, &cache_flags));
|
||||||
try runPinned(ctx, "verify-dist", environ, repo_root, try zigBuild(ctx, "verify-dist", &dist_flags, &cache_flags));
|
try runPinned(ctx, "verify-dist", environ, repo_root, try zigBuild(ctx, zig_exe, "verify-dist", &dist_flags, &cache_flags));
|
||||||
try runPinned(ctx, "verify-pins", environ, repo_root, try zigBuild(ctx, "verify-pins", &dist_flags, &cache_flags));
|
try runPinned(ctx, "verify-pins", environ, repo_root, try zigBuild(ctx, zig_exe, "verify-pins", &dist_flags, &cache_flags));
|
||||||
ctx.pass("verify-pins", "flake.nix pins the hashes of the {s} tarballs this machine just built", .{version});
|
ctx.pass("verify-pins", "flake.nix pins the hashes of the {s} tarballs this machine just built", .{version});
|
||||||
|
|
||||||
// Evaluation only: `--no-build` keeps this from fetching the tarballs the
|
// Evaluation only: `--no-build` keeps this from fetching the tarballs the
|
||||||
@@ -1704,14 +1835,16 @@ fn pinStage(ctx: *Ctx, version: []const u8, environ: *const std.process.Environ.
|
|||||||
ctx.pass("flake-check", "`nix flake check --no-build` accepts the rewritten flake.nix", .{});
|
ctx.pass("flake-check", "`nix flake check --no-build` accepts the rewritten flake.nix", .{});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every release byte comes out of the official zig, addressed by absolute path.
|
||||||
fn zigBuild(
|
fn zigBuild(
|
||||||
ctx: *Ctx,
|
ctx: *Ctx,
|
||||||
|
zig_exe: []const u8,
|
||||||
step: []const u8,
|
step: []const u8,
|
||||||
dist_flags: []const []const u8,
|
dist_flags: []const []const u8,
|
||||||
cache_flags: []const []const u8,
|
cache_flags: []const []const u8,
|
||||||
) ![]const []const u8 {
|
) ![]const []const u8 {
|
||||||
var argv: std.ArrayList([]const u8) = .empty;
|
var argv: std.ArrayList([]const u8) = .empty;
|
||||||
try argv.appendSlice(ctx.arena, &.{ "zig", "build", step });
|
try argv.appendSlice(ctx.arena, &.{ zig_exe, "build", step });
|
||||||
try argv.appendSlice(ctx.arena, dist_flags);
|
try argv.appendSlice(ctx.arena, dist_flags);
|
||||||
try argv.appendSlice(ctx.arena, cache_flags);
|
try argv.appendSlice(ctx.arena, cache_flags);
|
||||||
return argv.items;
|
return argv.items;
|
||||||
@@ -1778,13 +1911,15 @@ fn capture(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8, time
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A git command that must be able to talk to the operator: `commit -S`, `tag
|
/// A command that must be able to talk to the operator: `commit -S`, `tag -s`
|
||||||
/// -s` and both pushes reach gpg and ssh, either of which may need a passphrase
|
/// and both pushes reach gpg and ssh, either of which may need a passphrase
|
||||||
/// from a terminal. Piping their stdio would turn a pinentry prompt into a hang.
|
/// from a terminal. Piping their stdio would turn a pinentry prompt into a hang.
|
||||||
|
/// The toolchain download uses it too, so curl and tar report their own
|
||||||
|
/// failures where the operator can read them.
|
||||||
///
|
///
|
||||||
/// The termination state is read rather than assumed: a gpg-agent that dies
|
/// The termination state is read rather than assumed: a gpg-agent that dies
|
||||||
/// takes git with it by signal, and `.signal` is not `.exited 0`.
|
/// takes git with it by signal, and `.signal` is not `.exited 0`.
|
||||||
fn gitInherit(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8) !void {
|
fn runInherited(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8) !void {
|
||||||
var child = std.process.spawn(ctx.io, .{
|
var child = std.process.spawn(ctx.io, .{
|
||||||
.argv = argv,
|
.argv = argv,
|
||||||
.stdin = .inherit,
|
.stdin = .inherit,
|
||||||
@@ -1820,7 +1955,7 @@ fn gitInherit(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8) !
|
|||||||
/// stdout is a pipe because the porcelain report is the only account of whether
|
/// stdout is a pipe because the porcelain report is the only account of whether
|
||||||
/// the ref actually moved. stdin and stderr stay on the terminal: ssh writes its
|
/// the ref actually moved. stdin and stderr stay on the terminal: ssh writes its
|
||||||
/// prompts and progress there, and piping them would turn a key passphrase into
|
/// prompts and progress there, and piping them would turn a key passphrase into
|
||||||
/// a hang — the same reason `gitInherit` exists.
|
/// a hang — the same reason `runInherited` exists.
|
||||||
fn gitPushPorcelain(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8) ![]const u8 {
|
fn gitPushPorcelain(ctx: *Ctx, comptime check: []const u8, argv: []const []const u8) ![]const u8 {
|
||||||
var child = std.process.spawn(ctx.io, .{
|
var child = std.process.spawn(ctx.io, .{
|
||||||
.argv = argv,
|
.argv = argv,
|
||||||
@@ -2116,7 +2251,7 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void {
|
|||||||
// Both of these run before the manifest is rewritten: they can refuse, and
|
// Both of these run before the manifest is rewritten: they can refuse, and
|
||||||
// a refusal after the rewrite leaves a dirty build.zig.zon that the next
|
// a refusal after the rewrite leaves a dirty build.zig.zon that the next
|
||||||
// run's clean-tree gate rejects.
|
// run's clean-tree gate rejects.
|
||||||
try toolchainParity(ctx);
|
const zig_exe = try toolchainParity(ctx);
|
||||||
var environ = normalizedEnvironment(ctx.gpa, ctx.env) catch |err| {
|
var environ = normalizedEnvironment(ctx.gpa, ctx.env) catch |err| {
|
||||||
ctx.soft("pin", "cannot build the release environment: {t}", .{err});
|
ctx.soft("pin", "cannot build the release environment: {t}", .{err});
|
||||||
return CheckFailed;
|
return CheckFailed;
|
||||||
@@ -2124,7 +2259,7 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void {
|
|||||||
defer environ.deinit();
|
defer environ.deinit();
|
||||||
|
|
||||||
if (bump_needed) try writeZonVersion(ctx, version, zon_source);
|
if (bump_needed) try writeZonVersion(ctx, version, zon_source);
|
||||||
try pinStage(ctx, version, &environ);
|
try pinStage(ctx, version, &environ, zig_exe);
|
||||||
try commitBump(ctx, version, bump_needed);
|
try commitBump(ctx, version, bump_needed);
|
||||||
|
|
||||||
const sha = try headSha(ctx);
|
const sha = try headSha(ctx);
|
||||||
@@ -2182,7 +2317,7 @@ fn cut(ctx: *Ctx, kind_text: []const u8) !void {
|
|||||||
}
|
}
|
||||||
ctx.pass("tag", "the verified tag {s} ({s}) is unchanged", .{ tag, object });
|
ctx.pass("tag", "the verified tag {s} ({s}) is unchanged", .{ tag, object });
|
||||||
} else {
|
} else {
|
||||||
try gitInherit(ctx, "tag", &.{ "git", "tag", "-s", tag, "-m", tag, sha });
|
try runInherited(ctx, "tag", &.{ "git", "tag", "-s", tag, "-m", tag, sha });
|
||||||
try verifyTag(ctx, tag, sha);
|
try verifyTag(ctx, tag, sha);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2927,7 +3062,7 @@ fn commitBump(ctx: *Ctx, version: []const u8, bump_needed: bool) !void {
|
|||||||
return CheckFailed;
|
return CheckFailed;
|
||||||
}
|
}
|
||||||
|
|
||||||
try gitInherit(ctx, "bump", &.{
|
try runInherited(ctx, "bump", &.{
|
||||||
"git", "commit",
|
"git", "commit",
|
||||||
"-S", "-m",
|
"-S", "-m",
|
||||||
ctx.fmt("build: bump version to {s}", .{version}), "--",
|
ctx.fmt("build: bump version to {s}", .{version}), "--",
|
||||||
@@ -4656,6 +4791,7 @@ test "the toolchain pins come out of the top-level env block of gates.yml" {
|
|||||||
\\
|
\\
|
||||||
\\env:
|
\\env:
|
||||||
\\ ZIG_VERSION: "0.16.0"
|
\\ ZIG_VERSION: "0.16.0"
|
||||||
|
\\ ZIG_TARBALL_SHA256: "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00"
|
||||||
\\ # A comment between two entries, which the file has.
|
\\ # A comment between two entries, which the file has.
|
||||||
\\ NODE_VERSION: "24.19.0"
|
\\ NODE_VERSION: "24.19.0"
|
||||||
\\ NPM_VERSION: "11.17.0"
|
\\ NPM_VERSION: "11.17.0"
|
||||||
@@ -4668,12 +4804,13 @@ test "the toolchain pins come out of the top-level env block of gates.yml" {
|
|||||||
;
|
;
|
||||||
const pins = parseToolchainPins(source).?;
|
const pins = parseToolchainPins(source).?;
|
||||||
try testing.expectEqualStrings("0.16.0", pins.zig);
|
try testing.expectEqualStrings("0.16.0", pins.zig);
|
||||||
|
try testing.expectEqualStrings("70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", pins.zig_tarball_sha256);
|
||||||
try testing.expectEqualStrings("24.19.0", pins.node);
|
try testing.expectEqualStrings("24.19.0", pins.node);
|
||||||
try testing.expectEqualStrings("11.17.0", pins.npm);
|
try testing.expectEqualStrings("11.17.0", pins.npm);
|
||||||
}
|
}
|
||||||
|
|
||||||
test "a gates.yml missing any one pin yields no pins at all" {
|
test "a gates.yml missing any one pin yields no pins at all" {
|
||||||
// Parity established for two of three tools is not parity, so there is no
|
// Parity established for some of the pins is not parity, so there is no
|
||||||
// partial answer to return.
|
// partial answer to return.
|
||||||
try testing.expect(parseToolchainPins(
|
try testing.expect(parseToolchainPins(
|
||||||
\\env:
|
\\env:
|
||||||
@@ -4684,6 +4821,25 @@ test "a gates.yml missing any one pin yields no pins at all" {
|
|||||||
try testing.expect(parseToolchainPins("jobs:\n package:\n") == null);
|
try testing.expect(parseToolchainPins("jobs:\n package:\n") == null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "a zig tarball digest that is not 64 lowercase hex digits yields no pins" {
|
||||||
|
// The digest names the cache directory and is compared to a computed hash;
|
||||||
|
// a truncated or uppercase value would do neither correctly.
|
||||||
|
const shapes = [_][]const u8{
|
||||||
|
"70e49664a743",
|
||||||
|
"70E49664A74374B48B51E6F3FDFBF437F6395D42509050588BD49ABE52BA3D00",
|
||||||
|
"70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d0g",
|
||||||
|
};
|
||||||
|
for (shapes) |digest| {
|
||||||
|
const source = std.fmt.allocPrint(
|
||||||
|
testing.allocator,
|
||||||
|
"env:\n ZIG_VERSION: \"0.16.0\"\n ZIG_TARBALL_SHA256: \"{s}\"\n NODE_VERSION: \"24.19.0\"\n NPM_VERSION: \"11.17.0\"\n",
|
||||||
|
.{digest},
|
||||||
|
) catch unreachable;
|
||||||
|
defer testing.allocator.free(source);
|
||||||
|
try testing.expect(parseToolchainPins(source) == null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
test "the release environment is exactly nine variables" {
|
test "the release environment is exactly nine variables" {
|
||||||
var parent: std.process.Environ.Map = .init(testing.allocator);
|
var parent: std.process.Environ.Map = .init(testing.allocator);
|
||||||
defer parent.deinit();
|
defer parent.deinit();
|
||||||
@@ -4746,6 +4902,7 @@ test "this repository's gates.yml pins the toolchain the cut asserts" {
|
|||||||
try testing.expect(pins.zig.len != 0);
|
try testing.expect(pins.zig.len != 0);
|
||||||
try testing.expect(pins.node.len != 0);
|
try testing.expect(pins.node.len != 0);
|
||||||
try testing.expect(pins.npm.len != 0);
|
try testing.expect(pins.npm.len != 0);
|
||||||
|
try testing.expectEqual(@as(usize, 64), pins.zig_tarball_sha256.len);
|
||||||
}
|
}
|
||||||
|
|
||||||
test "release.yml pins the same toolchain as gates.yml" {
|
test "release.yml pins the same toolchain as gates.yml" {
|
||||||
@@ -4764,10 +4921,34 @@ test "release.yml pins the same toolchain as gates.yml" {
|
|||||||
const expected = parseToolchainPins(gates) orelse return error.NoToolchainPins;
|
const expected = parseToolchainPins(gates) orelse return error.NoToolchainPins;
|
||||||
const found = parseToolchainPins(release) orelse return error.NoToolchainPins;
|
const found = parseToolchainPins(release) orelse return error.NoToolchainPins;
|
||||||
try testing.expectEqualStrings(expected.zig, found.zig);
|
try testing.expectEqualStrings(expected.zig, found.zig);
|
||||||
|
try testing.expectEqualStrings(expected.zig_tarball_sha256, found.zig_tarball_sha256);
|
||||||
try testing.expectEqualStrings(expected.node, found.node);
|
try testing.expectEqualStrings(expected.node, found.node);
|
||||||
try testing.expectEqualStrings(expected.npm, found.npm);
|
try testing.expectEqualStrings(expected.npm, found.npm);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "the official zig is addressed by version, under HOME and never on PATH" {
|
||||||
|
// The download and the cache path are the whole of what makes the release
|
||||||
|
// bytes reproducible off this machine, so their text is asserted rather
|
||||||
|
// than trusted to a format string. Nothing here touches the network.
|
||||||
|
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
|
||||||
|
defer arena_state.deinit();
|
||||||
|
const arena = arena_state.allocator();
|
||||||
|
|
||||||
|
try testing.expectEqualStrings(
|
||||||
|
"https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz",
|
||||||
|
zigTarballUrl(arena, "0.16.0"),
|
||||||
|
);
|
||||||
|
const digest = "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00";
|
||||||
|
try testing.expectEqualStrings(
|
||||||
|
"/home/someone/.cache/nxdns-cut/zig-0.16.0-70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00",
|
||||||
|
zigCacheRoot(arena, "/home/someone", "0.16.0", digest),
|
||||||
|
);
|
||||||
|
try testing.expectEqualStrings(
|
||||||
|
"/home/someone/.cache/nxdns-cut/zig-0.16.0-70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00/zig-x86_64-linux-0.16.0/zig",
|
||||||
|
zigBinaryPath(arena, "/home/someone", "0.16.0", digest),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
test "the sh wrapper sets the umask, keeps the real command and reports its exit code" {
|
test "the sh wrapper sets the umask, keeps the real command and reports its exit code" {
|
||||||
// The umask is the one release input this program cannot set on itself:
|
// The umask is the one release input this program cannot set on itself:
|
||||||
// zig 0.16.0 exposes `umask(2)` only through libc, which these tools do not
|
// zig 0.16.0 exposes `umask(2)` only through libc, which these tools do not
|
||||||
|
|||||||
@@ -102,6 +102,11 @@ fn collectSorted(arena: Allocator, io: std.Io, dist: std.Io.Dir) ![]const []cons
|
|||||||
defer walker.deinit();
|
defer walker.deinit();
|
||||||
while (try walker.next(io)) |entry| {
|
while (try walker.next(io)) |entry| {
|
||||||
if (entry.kind != .file) continue;
|
if (entry.kind != .file) continue;
|
||||||
|
// A hidden file is build metadata, not a web asset: the freshness stamp
|
||||||
|
// `admin/dist/.src-hash` exists in a local tree and not in a bundle that
|
||||||
|
// went through a CI artifact round trip, and embedding it would make the
|
||||||
|
// two binaries differ.
|
||||||
|
if (entry.basename.len != 0 and entry.basename[0] == '.') continue;
|
||||||
for (entry.path) |c| {
|
for (entry.path) |c| {
|
||||||
if (!std.ascii.isAlphanumeric(c) and std.mem.findScalar(u8, "._-/", c) == null) {
|
if (!std.ascii.isAlphanumeric(c) and std.mem.findScalar(u8, "._-/", c) == null) {
|
||||||
std.process.fatal("asset name '{s}' has a character the index cannot carry", .{entry.path});
|
std.process.fatal("asset name '{s}' has a character the index cannot carry", .{entry.path});
|
||||||
@@ -231,3 +236,23 @@ fn renderIndex(arena: Allocator, assets: []const Asset) ![]const u8 {
|
|||||||
try w.writeAll("};\n");
|
try w.writeAll("};\n");
|
||||||
return sink.written();
|
return sink.written();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const testing = std.testing;
|
||||||
|
|
||||||
|
test "a hidden file is not indexed" {
|
||||||
|
const io = testing.io;
|
||||||
|
|
||||||
|
var arena_state: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||||
|
defer arena_state.deinit();
|
||||||
|
const arena = arena_state.allocator();
|
||||||
|
|
||||||
|
var tmp = testing.tmpDir(.{ .iterate = true });
|
||||||
|
defer tmp.cleanup();
|
||||||
|
|
||||||
|
try tmp.dir.writeFile(io, .{ .sub_path = "index.html", .data = "<!doctype html>" });
|
||||||
|
try tmp.dir.writeFile(io, .{ .sub_path = ".src-hash", .data = "deadbeef" });
|
||||||
|
|
||||||
|
const names = try collectSorted(arena, io, tmp.dir);
|
||||||
|
try testing.expectEqual(@as(usize, 1), names.len);
|
||||||
|
try testing.expectEqualStrings("index.html", names[0]);
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user