milestone 27: diagnostics — operational failures land in one curated log, resolved history purgeable
Gates / frontend (push) Successful in 1m33s
Gates / test (push) Successful in 1m48s
Gates / test-aarch64 (push) Successful in 7m10s
Gates / package (push) Successful in 5m31s
Gates / container (push) Successful in 15s
CI / gates (push) Successful in 14m51s
Gates / frontend (push) Successful in 1m33s
Gates / test (push) Successful in 1m48s
Gates / test-aarch64 (push) Successful in 7m10s
Gates / package (push) Successful in 5m31s
Gates / container (push) Successful in 15s
CI / gates (push) Successful in 14m51s
This commit is contained in:
@@ -6,6 +6,10 @@ Sections are written by hand. Nothing here is generated from commit messages: th
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- **A diagnostics page.** Operational failures now land in one curated log instead of only journald: blocklist download failures, certificate reload failures, disk pressure, query-log writer and maintenance failures, upstream exchange and history failures, client tracking failures, listener and configuration problems at boot, and the query-log recreation an upgrade causes. One entry per failing subject — an entry opens on the first failure, counts repeats, and closes itself when the subject recovers; nothing needs dismissing. Each entry says what it means for the service and what to do about it. `GET /api/diagnostics` serves the log, `GET /api/health` reports the active counts and degrades while the diagnostics store itself cannot write, and `/metrics` gains `nxdns_diagnostics_active_warnings`, `nxdns_diagnostics_active_errors` and `nxdns_diagnostics_write_failures_total`. Resolved entries can be purged when you decide the history has served its purpose — one entry from its row or its detail page, or the whole resolved history at once with "Purge all resolved" (`DELETE /api/diagnostics/{id}` and `DELETE /api/diagnostics`). An entry that is still failing is the current state of the box, not history, so it has no purge action and the API answers 409.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **An upstream success rate no longer rounds up to 100.0% while failures stand.** One decimal place cannot hold 12,696 successes out of 12,698 attempts: it rounded to `100.0%`, so the row claimed perfect reliability next to a failure count of 2. Neither end of the scale is reachable by rounding any more — `100.0%` needs an actual absence of failures and `0.0%` an actual absence of successes, and a rate a hair off either end shows `99.9%` or `0.1%` instead.
|
||||
|
||||
@@ -435,8 +435,28 @@ CREATE TABLE forward_zones (
|
||||
);
|
||||
|
||||
CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
|
||||
CREATE TABLE operational_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
code TEXT NOT NULL,
|
||||
subject_key TEXT NOT NULL,
|
||||
subject_label TEXT NOT NULL,
|
||||
severity TEXT NOT NULL CHECK (severity IN ('warning', 'error')),
|
||||
first_seen INTEGER NOT NULL,
|
||||
last_seen INTEGER NOT NULL,
|
||||
occurrences INTEGER NOT NULL CHECK (occurrences > 0),
|
||||
resolved_at INTEGER,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
CHECK (resolved_at IS NULL OR resolved_at >= first_seen)
|
||||
);
|
||||
CREATE UNIQUE INDEX idx_operational_events_active
|
||||
ON operational_events(code, subject_key) WHERE resolved_at IS NULL;
|
||||
CREATE INDEX idx_operational_events_last_seen
|
||||
ON operational_events(last_seen DESC);
|
||||
```
|
||||
|
||||
`operational_events` is the one table here that is **not** configuration. It is the diagnostics log of `src/storage/events.zig`: one row per failure episode, opened on the first failure and resolved when the same subject succeeds again. It is deliberately absent from `config_schema.table_names` and `config_schema.delete_order`, so `nxdns export` never emits it and `nxdns import` never wipes it.
|
||||
|
||||
### 11.3 querylog.db Schema
|
||||
|
||||
```sql
|
||||
|
||||
@@ -61,6 +61,7 @@ const RESPONSES: Record<string, unknown> = {
|
||||
writer_failed: false,
|
||||
refreshes_gated: 0,
|
||||
snapshot_generation: 3,
|
||||
diagnostics: { state: "recording", active_warnings: 1, active_errors: 0 },
|
||||
},
|
||||
"/api/upstream/health?period=24h": {
|
||||
period: "24h",
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
|
||||
import { QueryClientProvider } from "@tanstack/react-query";
|
||||
import { RouterProvider, createMemoryHistory } from "@tanstack/react-router";
|
||||
import { AuthProvider } from "@/auth/store";
|
||||
import { createQueryClient } from "@/lib/queryClient";
|
||||
import { createAppRouter } from "@/routes";
|
||||
import { DIAGNOSTIC_CODES, type DiagnosticEvent } from "@/lib/types";
|
||||
import { EVENT_COPY } from "./eventCopy";
|
||||
|
||||
const NOW_S = Math.floor(Date.now() / 1000);
|
||||
|
||||
function event(overrides: Partial<DiagnosticEvent> = {}): DiagnosticEvent {
|
||||
return {
|
||||
id: 42,
|
||||
code: "blocklist.refresh",
|
||||
component: "blocklist",
|
||||
subject: "StevenBlack",
|
||||
severity: "warning",
|
||||
first_seen: NOW_S - 7200,
|
||||
last_seen: NOW_S - 600,
|
||||
occurrences: 4,
|
||||
resolved_at: null,
|
||||
detail: "download failed: ConnectionTimedOut",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
/** A 204: what `DELETE /api/diagnostics/{id}` answers on a purge. */
|
||||
const NO_CONTENT = Symbol("204");
|
||||
|
||||
let responses: Record<string, unknown>;
|
||||
let requested: string[];
|
||||
|
||||
beforeEach(() => {
|
||||
requested = [];
|
||||
responses = {
|
||||
"/api/version": { version: "0.0.0-test", git_commit: "0000000", zig_version: "0.16.0", uptime_seconds: 1 },
|
||||
};
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (input: RequestInfo | URL, init?: { method?: string }) => {
|
||||
const url = String(input);
|
||||
const method = init?.method ?? "GET";
|
||||
const key = method === "GET" ? url : `${method} ${url}`;
|
||||
requested.push(key);
|
||||
const payload = responses[key];
|
||||
if (payload === undefined)
|
||||
return new Response(JSON.stringify({ error: "no such event" }), {
|
||||
status: 404,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
if (payload === NO_CONTENT) return new Response(null, { status: 204 });
|
||||
return new Response(JSON.stringify(payload), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
/**
|
||||
* `retry` is off in the failure test: the shared client backs 5xx off for
|
||||
* seconds, which the render assertions would sit through for nothing.
|
||||
*/
|
||||
function renderDetail(id: number, { retry = true } = {}) {
|
||||
const queryClient = createQueryClient();
|
||||
if (!retry) {
|
||||
const defaults = queryClient.getDefaultOptions();
|
||||
queryClient.setDefaultOptions({ ...defaults, queries: { ...defaults.queries, retry: false } });
|
||||
}
|
||||
const router = createAppRouter(createMemoryHistory({ initialEntries: [`/diagnostics/${id}`] }), queryClient);
|
||||
render(
|
||||
<AuthProvider>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<RouterProvider router={router} />
|
||||
</QueryClientProvider>
|
||||
</AuthProvider>,
|
||||
);
|
||||
return router;
|
||||
}
|
||||
|
||||
test("an open episode shows its facts, its copy and the error the server sent", async () => {
|
||||
responses["/api/diagnostics/42"] = event();
|
||||
renderDetail(42);
|
||||
|
||||
await screen.findByRole("heading", { name: "Blocklist source failed to update" });
|
||||
expect(screen.getByText("Warning")).toBeTruthy();
|
||||
expect(screen.getByText("StevenBlack")).toBeTruthy();
|
||||
expect(screen.getByText("Active for 2h")).toBeTruthy();
|
||||
expect(screen.getByText("Not yet — still failing")).toBeTruthy();
|
||||
expect(screen.getByText("4")).toBeTruthy();
|
||||
expect(screen.getByText("blocklist.refresh")).toBeTruthy();
|
||||
expect(screen.getByText(EVENT_COPY["blocklist.refresh"].impact)).toBeTruthy();
|
||||
expect(screen.getByText(EVENT_COPY["blocklist.refresh"].remediation)).toBeTruthy();
|
||||
expect(screen.getByText("download failed: ConnectionTimedOut")).toBeTruthy();
|
||||
expect(screen.getByRole("link", { name: "Go to Blocklists" }).getAttribute("href")).toBe("/blocklists");
|
||||
});
|
||||
|
||||
test("a resolved episode states how long it lasted, not how long it has run", async () => {
|
||||
responses["/api/diagnostics/7"] = event({ id: 7, resolved_at: NOW_S - 3600 });
|
||||
renderDetail(7);
|
||||
|
||||
await screen.findByRole("heading", { name: "Blocklist source failed to update" });
|
||||
expect(screen.getByText("Resolved after 1h")).toBeTruthy();
|
||||
expect(screen.queryByText("Not yet — still failing")).toBeNull();
|
||||
});
|
||||
|
||||
test("an open episode offers no purge", async () => {
|
||||
responses["/api/diagnostics/42"] = event();
|
||||
renderDetail(42);
|
||||
|
||||
await screen.findByRole("heading", { name: "Blocklist source failed to update" });
|
||||
expect(screen.queryByRole("button", { name: "Purge" })).toBeNull();
|
||||
});
|
||||
|
||||
test("purging a resolved episode asks first, then returns to the list", async () => {
|
||||
responses["/api/diagnostics/7"] = event({ id: 7, resolved_at: NOW_S - 3600 });
|
||||
responses["DELETE /api/diagnostics/7"] = NO_CONTENT;
|
||||
responses["/api/diagnostics?state=active"] = { events: [], next_before: null, active: { warnings: 0, errors: 0 } };
|
||||
responses["/api/diagnostics?state=resolved"] = {
|
||||
events: [],
|
||||
next_before: null,
|
||||
active: { warnings: 0, errors: 0 },
|
||||
};
|
||||
const router = renderDetail(7);
|
||||
|
||||
await screen.findByRole("heading", { name: "Blocklist source failed to update" });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Purge" }));
|
||||
|
||||
const dialog = await screen.findByRole("alertdialog");
|
||||
expect(dialog.textContent).toContain("Purge this resolved event? Its history is gone for good.");
|
||||
fireEvent.click(within(dialog).getByRole("button", { name: "Cancel" }));
|
||||
expect(requested).not.toContain("DELETE /api/diagnostics/7");
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Purge" }));
|
||||
fireEvent.click(within(await screen.findByRole("alertdialog")).getByRole("button", { name: "Purge" }));
|
||||
|
||||
await waitFor(() => expect(requested).toContain("DELETE /api/diagnostics/7"));
|
||||
// The row it was showing no longer exists, so the page it navigates to is
|
||||
// the list rather than a 404 of its own.
|
||||
await waitFor(() => expect(router.state.location.pathname).toBe("/diagnostics"));
|
||||
});
|
||||
|
||||
test("a refused purge stays on the event and shows why", async () => {
|
||||
responses["/api/diagnostics/7"] = event({ id: 7, resolved_at: NOW_S - 3600 });
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (input: RequestInfo | URL, init?: { method?: string }) => {
|
||||
if (init?.method === "DELETE")
|
||||
return new Response(JSON.stringify({ error: "the event is still active" }), {
|
||||
status: 409,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
return new Response(JSON.stringify(responses[String(input)] ?? {}), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}),
|
||||
);
|
||||
const router = renderDetail(7, { retry: false });
|
||||
|
||||
await screen.findByRole("heading", { name: "Blocklist source failed to update" });
|
||||
fireEvent.click(screen.getByRole("button", { name: "Purge" }));
|
||||
fireEvent.click(within(await screen.findByRole("alertdialog")).getByRole("button", { name: "Purge" }));
|
||||
|
||||
const alert = await screen.findByRole("alert");
|
||||
expect(alert.textContent).toContain("the event is still active");
|
||||
expect(router.state.location.pathname).toBe("/diagnostics/7");
|
||||
});
|
||||
|
||||
test("every code renders its own title, impact and remediation", async () => {
|
||||
for (const [index, code] of DIAGNOSTIC_CODES.entries()) {
|
||||
const id = 100 + index;
|
||||
responses[`/api/diagnostics/${id}`] = event({ id, code, component: code.slice(0, code.indexOf(".")) });
|
||||
renderDetail(id);
|
||||
|
||||
const copy = EVENT_COPY[code];
|
||||
await screen.findByRole("heading", { name: copy.title });
|
||||
expect(screen.getByText(copy.impact), code).toBeTruthy();
|
||||
expect(screen.getByText(copy.remediation), code).toBeTruthy();
|
||||
screen.getByText(code);
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("an event retention has removed shows the server's message, not an empty page", async () => {
|
||||
renderDetail(999);
|
||||
await screen.findByText("no such event");
|
||||
expect(screen.getByRole("link", { name: "← All diagnostics" })).toBeTruthy();
|
||||
});
|
||||
|
||||
test("an unavailable store reports the failure instead of loading forever", async () => {
|
||||
responses["/api/diagnostics/42"] = event();
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (input: RequestInfo | URL) =>
|
||||
String(input).startsWith("/api/diagnostics/")
|
||||
? new Response(JSON.stringify({ error: "store unavailable" }), {
|
||||
status: 503,
|
||||
headers: { "content-type": "application/json" },
|
||||
})
|
||||
: new Response(JSON.stringify(responses[String(input)] ?? {}), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
),
|
||||
);
|
||||
renderDetail(42, { retry: false });
|
||||
|
||||
const alert = await screen.findByRole("alert");
|
||||
expect(alert.textContent).toContain("The server is starting or degraded.");
|
||||
expect(screen.queryByText("Loading event…")).toBeNull();
|
||||
expect(screen.getByRole("link", { name: "← All diagnostics" })).toBeTruthy();
|
||||
});
|
||||
@@ -0,0 +1,226 @@
|
||||
import { useState } from "react";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { Link, useNavigate, useParams } from "@tanstack/react-router";
|
||||
import * as stylex from "@stylexjs/stylex";
|
||||
import InlineError from "@/lib/InlineError";
|
||||
import { formatDuration, formatTime } from "@/lib/format";
|
||||
import { diagnosticPurgeMutation, diagnosticQuery } from "@/lib/queries";
|
||||
import ConfirmDialog from "@/ui/ConfirmDialog";
|
||||
import { styles as shared } from "@/ui/styles";
|
||||
import { colors } from "@/ui/tokens.stylex";
|
||||
import SeverityBadge from "./SeverityBadge";
|
||||
import { componentLabel, copyFor } from "./eventCopy";
|
||||
|
||||
const styles = stylex.create({
|
||||
back: {
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
color: colors.primaryOnSurface,
|
||||
textDecorationLine: "none",
|
||||
},
|
||||
headingRow: {
|
||||
marginTop: "0.5rem",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
flexWrap: "wrap",
|
||||
gap: "0.5rem",
|
||||
},
|
||||
heading: {
|
||||
fontSize: "1.5rem",
|
||||
lineHeight: "2rem",
|
||||
fontWeight: 600,
|
||||
},
|
||||
purgeAction: {
|
||||
marginInlineStart: "auto",
|
||||
},
|
||||
subject: {
|
||||
marginTop: "0.25rem",
|
||||
color: colors.textSecondary,
|
||||
wordBreak: "break-all",
|
||||
},
|
||||
panel: {
|
||||
marginTop: "1rem",
|
||||
maxWidth: "48rem",
|
||||
borderRadius: "0.25rem",
|
||||
borderWidth: 1,
|
||||
borderStyle: "solid",
|
||||
borderColor: colors.border,
|
||||
backgroundColor: colors.surfaceRaised,
|
||||
padding: "1rem",
|
||||
},
|
||||
facts: {
|
||||
display: "grid",
|
||||
gap: "0.5rem 1rem",
|
||||
gridTemplateColumns: {
|
||||
default: "auto",
|
||||
"@media (min-width: 640px)": "max-content 1fr",
|
||||
},
|
||||
margin: 0,
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
},
|
||||
term: {
|
||||
color: colors.textMuted,
|
||||
},
|
||||
value: {
|
||||
margin: 0,
|
||||
},
|
||||
sectionHeading: {
|
||||
marginTop: "1.5rem",
|
||||
fontSize: "1.125rem",
|
||||
lineHeight: "1.75rem",
|
||||
fontWeight: 600,
|
||||
},
|
||||
prose: {
|
||||
marginTop: "0.5rem",
|
||||
maxWidth: "48rem",
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.5rem",
|
||||
},
|
||||
detail: {
|
||||
marginTop: "0.5rem",
|
||||
maxWidth: "48rem",
|
||||
overflowX: "auto",
|
||||
borderRadius: "0.25rem",
|
||||
borderWidth: 1,
|
||||
borderStyle: "solid",
|
||||
borderColor: colors.border,
|
||||
padding: "0.75rem",
|
||||
fontSize: "0.8125rem",
|
||||
lineHeight: "1.25rem",
|
||||
whiteSpace: "pre-wrap",
|
||||
wordBreak: "break-all",
|
||||
},
|
||||
links: {
|
||||
marginTop: "1rem",
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
},
|
||||
link: {
|
||||
color: colors.primaryOnSurface,
|
||||
},
|
||||
loading: {
|
||||
marginTop: "1rem",
|
||||
color: colors.textMuted,
|
||||
},
|
||||
});
|
||||
|
||||
export default function DiagnosticDetailPage() {
|
||||
const { id } = useParams({ from: "/shell/diagnostics/$id" });
|
||||
const eventId = Number(id);
|
||||
const { data, error, isPending, refetch } = useQuery(diagnosticQuery(eventId));
|
||||
|
||||
const navigate = useNavigate();
|
||||
const queryClient = useQueryClient();
|
||||
const purge = useMutation(diagnosticPurgeMutation(queryClient));
|
||||
const [confirming, setConfirming] = useState(false);
|
||||
|
||||
function confirmPurge() {
|
||||
setConfirming(false);
|
||||
// The row this page is about is gone, so staying here would show the
|
||||
// 404 the purge itself caused.
|
||||
purge.mutate(eventId, { onSuccess: () => void navigate({ to: "/diagnostics" }) });
|
||||
}
|
||||
|
||||
if (isPending) {
|
||||
return (
|
||||
<p {...stylex.props(styles.loading, shared.pulse)} role="status">
|
||||
Loading event…
|
||||
</p>
|
||||
);
|
||||
}
|
||||
if (data === undefined) {
|
||||
return (
|
||||
<section>
|
||||
<Link to="/diagnostics" {...stylex.props(styles.back, shared.focusRing)}>
|
||||
← All diagnostics
|
||||
</Link>
|
||||
<InlineError error={error} onRetry={() => void refetch()} />
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
const copy = copyFor(data.code);
|
||||
const resolvedAt = data.resolved_at;
|
||||
const span = (resolvedAt ?? Math.floor(Date.now() / 1000)) - data.first_seen;
|
||||
|
||||
return (
|
||||
<section>
|
||||
<Link to="/diagnostics" {...stylex.props(styles.back, shared.focusRing)}>
|
||||
← All diagnostics
|
||||
</Link>
|
||||
<div {...stylex.props(styles.headingRow)}>
|
||||
<h1 {...stylex.props(styles.heading)}>{copy.title}</h1>
|
||||
<SeverityBadge severity={data.severity} />
|
||||
{/* Only history can be purged: an open episode is the current state of the box. */}
|
||||
{resolvedAt !== null && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setConfirming(true)}
|
||||
disabled={purge.isPending}
|
||||
{...stylex.props(styles.purgeAction, shared.dangerLinkButton, shared.focusRing)}
|
||||
>
|
||||
Purge
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<p {...stylex.props(styles.subject)}>{data.subject}</p>
|
||||
<InlineError error={purge.error} />
|
||||
|
||||
<div {...stylex.props(styles.panel)}>
|
||||
<dl {...stylex.props(styles.facts)}>
|
||||
<dt {...stylex.props(styles.term)}>State</dt>
|
||||
<dd {...stylex.props(styles.value)}>
|
||||
{resolvedAt === null
|
||||
? `Active for ${formatDuration(span)}`
|
||||
: `Resolved after ${formatDuration(span)}`}
|
||||
</dd>
|
||||
<dt {...stylex.props(styles.term)}>First seen</dt>
|
||||
<dd {...stylex.props(styles.value)}>{formatTime(data.first_seen)}</dd>
|
||||
<dt {...stylex.props(styles.term)}>Last seen</dt>
|
||||
<dd {...stylex.props(styles.value)}>{formatTime(data.last_seen)}</dd>
|
||||
<dt {...stylex.props(styles.term)}>Occurrences</dt>
|
||||
<dd {...stylex.props(styles.value, shared.tabularNums)}>{data.occurrences}</dd>
|
||||
<dt {...stylex.props(styles.term)}>Resolved</dt>
|
||||
<dd {...stylex.props(styles.value)}>
|
||||
{data.resolved_at === null ? "Not yet — still failing" : formatTime(data.resolved_at)}
|
||||
</dd>
|
||||
<dt {...stylex.props(styles.term)}>Component</dt>
|
||||
<dd {...stylex.props(styles.value)}>{componentLabel(data.component)}</dd>
|
||||
<dt {...stylex.props(styles.term)}>Code</dt>
|
||||
<dd {...stylex.props(styles.value, shared.mono)}>{data.code}</dd>
|
||||
</dl>
|
||||
</div>
|
||||
|
||||
<h2 {...stylex.props(styles.sectionHeading)}>Impact</h2>
|
||||
<p {...stylex.props(styles.prose)}>{copy.impact}</p>
|
||||
|
||||
<h2 {...stylex.props(styles.sectionHeading)}>What to do</h2>
|
||||
<p {...stylex.props(styles.prose)}>{copy.remediation}</p>
|
||||
|
||||
<h2 {...stylex.props(styles.sectionHeading)}>Last error</h2>
|
||||
{data.detail === "" ? (
|
||||
<p {...stylex.props(styles.prose)}>The server recorded no error text for this event.</p>
|
||||
) : (
|
||||
<pre {...stylex.props(styles.detail, shared.mono)}>{data.detail}</pre>
|
||||
)}
|
||||
|
||||
{copy.link !== undefined && (
|
||||
<p {...stylex.props(styles.links)}>
|
||||
<Link to={copy.link.to} {...stylex.props(styles.link, shared.focusRing)}>
|
||||
Go to {copy.link.label}
|
||||
</Link>
|
||||
</p>
|
||||
)}
|
||||
|
||||
<ConfirmDialog
|
||||
isOpen={confirming}
|
||||
title="Purge event"
|
||||
message="Purge this resolved event? Its history is gone for good."
|
||||
confirmLabel="Purge"
|
||||
onConfirm={confirmPurge}
|
||||
onCancel={() => setConfirming(false)}
|
||||
/>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
import { fireEvent, render, screen, waitFor, within } from "@testing-library/react";
|
||||
import { QueryClientProvider } from "@tanstack/react-query";
|
||||
import { RouterProvider, createMemoryHistory } from "@tanstack/react-router";
|
||||
import { AuthProvider } from "@/auth/store";
|
||||
import { createQueryClient } from "@/lib/queryClient";
|
||||
import { createAppRouter } from "@/routes";
|
||||
import type { DiagnosticEvent, DiagnosticsPage } from "@/lib/types";
|
||||
|
||||
// Ages are rendered against the wall clock, so the fixtures are anchored to it
|
||||
// rather than to a frozen instant: faking time here would fight the query
|
||||
// client's own timers for no gain.
|
||||
const NOW_S = Math.floor(Date.now() / 1000);
|
||||
|
||||
function event(id: number, overrides: Partial<DiagnosticEvent> = {}): DiagnosticEvent {
|
||||
return {
|
||||
id,
|
||||
code: "blocklist.refresh",
|
||||
component: "blocklist",
|
||||
subject: "StevenBlack",
|
||||
severity: "warning",
|
||||
first_seen: NOW_S - 3600,
|
||||
last_seen: NOW_S - 300,
|
||||
occurrences: 3,
|
||||
resolved_at: null,
|
||||
detail: "download failed: ConnectionTimedOut",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function page(events: DiagnosticEvent[], nextBefore: number | null = null): DiagnosticsPage {
|
||||
return { events, next_before: nextBefore, active: { warnings: 1, errors: 1 } };
|
||||
}
|
||||
|
||||
const ACTIVE = page([
|
||||
event(42),
|
||||
event(41, {
|
||||
code: "upstream.exchange",
|
||||
component: "upstream",
|
||||
subject: "tls://dns.example:853",
|
||||
severity: "error",
|
||||
occurrences: 1,
|
||||
}),
|
||||
]);
|
||||
|
||||
const RESOLVED = page([
|
||||
event(30, { code: "disk.space", component: "disk", subject: "data", resolved_at: NOW_S - 7200 }),
|
||||
]);
|
||||
|
||||
/** A stubbed response that carries a non-200 status instead of a payload. */
|
||||
class Failure {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
readonly body: unknown,
|
||||
) {}
|
||||
}
|
||||
|
||||
function fail(status: number, message: string): Failure {
|
||||
return new Failure(status, { error: message });
|
||||
}
|
||||
|
||||
/** A 204: what `DELETE /api/diagnostics/{id}` answers on a purge. */
|
||||
const NO_CONTENT = Symbol("204");
|
||||
|
||||
let responses: Record<string, unknown>;
|
||||
let requested: string[];
|
||||
|
||||
beforeEach(() => {
|
||||
requested = [];
|
||||
responses = {
|
||||
"/api/version": { version: "0.0.0-test", git_commit: "0000000", zig_version: "0.16.0", uptime_seconds: 1 },
|
||||
"/api/diagnostics?state=active": ACTIVE,
|
||||
"/api/diagnostics?state=resolved": RESOLVED,
|
||||
};
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (input: RequestInfo | URL, init?: { method?: string }) => {
|
||||
const url = String(input);
|
||||
const method = init?.method ?? "GET";
|
||||
// Reads stay keyed by url alone, so the assertions below read as the
|
||||
// request line they are; writes carry their method.
|
||||
const key = method === "GET" ? url : `${method} ${url}`;
|
||||
requested.push(key);
|
||||
const payload = responses[key];
|
||||
if (payload === undefined) return new Response(JSON.stringify({ error: "not stubbed" }), { status: 404 });
|
||||
if (payload === NO_CONTENT) return new Response(null, { status: 204 });
|
||||
if (payload instanceof Failure) {
|
||||
return new Response(JSON.stringify(payload.body), {
|
||||
status: payload.status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
return new Response(JSON.stringify(payload), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
/**
|
||||
* `retry` is off in the failure tests: the shared client backs 5xx off for
|
||||
* seconds, which the render assertions would sit through for nothing.
|
||||
*/
|
||||
function renderRoute(path = "/diagnostics", { retry = true } = {}) {
|
||||
const queryClient = createQueryClient();
|
||||
if (!retry) {
|
||||
const defaults = queryClient.getDefaultOptions();
|
||||
queryClient.setDefaultOptions({ ...defaults, queries: { ...defaults.queries, retry: false } });
|
||||
}
|
||||
const router = createAppRouter(createMemoryHistory({ initialEntries: [path] }), queryClient);
|
||||
render(
|
||||
<AuthProvider>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<RouterProvider router={router} />
|
||||
</QueryClientProvider>
|
||||
</AuthProvider>,
|
||||
);
|
||||
return router;
|
||||
}
|
||||
|
||||
/** A RAC Select names its trigger with the current value and then the label. */
|
||||
function trigger(label: string): HTMLElement {
|
||||
return screen.getByRole("button", { name: new RegExp(`${label}$`) });
|
||||
}
|
||||
|
||||
async function pick(label: string, option: string) {
|
||||
fireEvent.click(trigger(label));
|
||||
fireEvent.click(await screen.findByRole("option", { name: option }));
|
||||
await waitFor(() => expect(screen.queryByRole("listbox")).toBeNull());
|
||||
}
|
||||
|
||||
test("active episodes come first, each with its title, subject, age and count", async () => {
|
||||
renderRoute();
|
||||
await screen.findByRole("heading", { name: "Diagnostics" });
|
||||
|
||||
const active = screen.getByText("Blocklist source failed to update").closest("li")!;
|
||||
expect(within(active).getByText("Warning")).toBeTruthy();
|
||||
expect(within(active).getByText("StevenBlack")).toBeTruthy();
|
||||
expect(within(active).getByText(/Active for 1h · 3 occurrences/)).toBeTruthy();
|
||||
|
||||
const failing = screen.getByText("Upstream failing").closest("li")!;
|
||||
expect(within(failing).getByText("Error")).toBeTruthy();
|
||||
expect(within(failing).getByText(/1 occurrence(?!s)/)).toBeTruthy();
|
||||
|
||||
// The resolved history is a separate section, below the active list.
|
||||
const table = within(screen.getByRole("table"));
|
||||
expect(table.getByText("Disk space low")).toBeTruthy();
|
||||
expect(screen.getByText(/Showing 1 resolved entry — end of history/)).toBeTruthy();
|
||||
});
|
||||
|
||||
test("nothing open reads as good news, not as a broken page", async () => {
|
||||
responses["/api/diagnostics?state=active"] = page([]);
|
||||
renderRoute();
|
||||
await screen.findByRole("heading", { name: "Diagnostics" });
|
||||
|
||||
const healthy = await screen.findByText("No active operational issues.");
|
||||
expect(healthy.getAttribute("role")).toBe("status");
|
||||
// Quiet: no alert anywhere on the page, and no empty table standing in.
|
||||
expect(screen.queryByRole("alert")).toBeNull();
|
||||
});
|
||||
|
||||
test("a filter lands in the url and refetches both sections through it", async () => {
|
||||
responses["/api/diagnostics?severity=error&state=active"] = page([
|
||||
event(41, { code: "upstream.exchange", component: "upstream", severity: "error" }),
|
||||
]);
|
||||
responses["/api/diagnostics?severity=error&state=resolved"] = page([]);
|
||||
const router = renderRoute();
|
||||
await screen.findByRole("heading", { name: "Diagnostics" });
|
||||
|
||||
await pick("Severity", "Errors");
|
||||
|
||||
await waitFor(() => expect(router.state.location.search).toEqual({ severity: "error" }));
|
||||
await waitFor(() => expect(screen.queryByText("Blocklist source failed to update")).toBeNull());
|
||||
expect(requested).toContain("/api/diagnostics?severity=error&state=active");
|
||||
expect(requested).toContain("/api/diagnostics?severity=error&state=resolved");
|
||||
});
|
||||
|
||||
test("the state filter hides the section it excludes", async () => {
|
||||
const router = renderRoute();
|
||||
await screen.findByRole("heading", { name: "Diagnostics" });
|
||||
|
||||
await pick("Show", "Active only");
|
||||
|
||||
await waitFor(() => expect(router.state.location.search).toEqual({ state: "active" }));
|
||||
expect(screen.queryByRole("heading", { name: "Resolved" })).toBeNull();
|
||||
expect(screen.getByRole("heading", { name: "Active" })).toBeTruthy();
|
||||
});
|
||||
|
||||
test("a url written by hand starts on the filters it names", async () => {
|
||||
responses["/api/diagnostics?component=disk&state=resolved"] = RESOLVED;
|
||||
renderRoute("/diagnostics?state=resolved&component=disk");
|
||||
await screen.findByRole("heading", { name: "Diagnostics" });
|
||||
|
||||
await screen.findByText("Disk space low");
|
||||
expect(screen.queryByRole("heading", { name: "Active" })).toBeNull();
|
||||
expect(requested).toContain("/api/diagnostics?component=disk&state=resolved");
|
||||
});
|
||||
|
||||
test("load more appends the next page of resolved history", async () => {
|
||||
responses["/api/diagnostics?state=resolved"] = page(
|
||||
[event(30, { code: "disk.space", component: "disk", subject: "data", resolved_at: NOW_S - 7200 })],
|
||||
30,
|
||||
);
|
||||
responses["/api/diagnostics?state=resolved&before=30"] = page([
|
||||
event(12, {
|
||||
code: "certificate.reload",
|
||||
component: "certificate",
|
||||
subject: "doh",
|
||||
resolved_at: NOW_S - 90_000,
|
||||
}),
|
||||
]);
|
||||
renderRoute();
|
||||
await screen.findByText("Disk space low");
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Load more" }));
|
||||
|
||||
await screen.findByText("TLS certificate reload failed");
|
||||
expect(screen.getByText(/Showing 2 resolved entries — end of history/)).toBeTruthy();
|
||||
});
|
||||
|
||||
test("an unavailable store reports the failure instead of loading forever", async () => {
|
||||
responses["/api/diagnostics?state=active"] = fail(503, "store unavailable");
|
||||
renderRoute("/diagnostics", { retry: false });
|
||||
await screen.findByRole("heading", { name: "Diagnostics" });
|
||||
|
||||
const alert = await screen.findByRole("alert");
|
||||
expect(alert.textContent).toContain("The server is starting or degraded.");
|
||||
expect(screen.queryByText("Loading diagnostics…")).toBeNull();
|
||||
// The resolved section answered, so it still renders its own history.
|
||||
expect(screen.getByText("Disk space low")).toBeTruthy();
|
||||
});
|
||||
|
||||
test("a failed history query reports the failure and retries on demand", async () => {
|
||||
responses["/api/diagnostics?state=resolved"] = fail(500, "diagnostics store read failed");
|
||||
renderRoute("/diagnostics", { retry: false });
|
||||
await screen.findByRole("heading", { name: "Diagnostics" });
|
||||
|
||||
const alert = await screen.findByRole("alert");
|
||||
expect(alert.textContent).toContain("diagnostics store read failed");
|
||||
expect(screen.queryByText("Loading history…")).toBeNull();
|
||||
|
||||
responses["/api/diagnostics?state=resolved"] = RESOLVED;
|
||||
fireEvent.click(screen.getByRole("button", { name: "Retry" }));
|
||||
|
||||
await screen.findByText("Disk space low");
|
||||
expect(screen.queryByRole("alert")).toBeNull();
|
||||
});
|
||||
|
||||
test("only the resolved history offers a purge", async () => {
|
||||
renderRoute();
|
||||
await screen.findByRole("heading", { name: "Diagnostics" });
|
||||
|
||||
// An episode still failing is the state of the box, not history: no purge
|
||||
// affordance anywhere on its card.
|
||||
const active = screen.getByText("Blocklist source failed to update").closest("li")!;
|
||||
expect(within(active).queryByRole("button", { name: "Purge" })).toBeNull();
|
||||
|
||||
const row = screen.getByText("Disk space low").closest("tr")!;
|
||||
expect(within(row).getByRole("button", { name: "Purge" })).toBeTruthy();
|
||||
expect(screen.getByRole("button", { name: "Purge all resolved" })).toBeTruthy();
|
||||
});
|
||||
|
||||
test("with no resolved history there is nothing to purge in bulk", async () => {
|
||||
responses["/api/diagnostics?state=resolved"] = page([]);
|
||||
renderRoute();
|
||||
await screen.findByRole("heading", { name: "Diagnostics" });
|
||||
|
||||
await screen.findByText("Nothing has failed and recovered in the retained window.");
|
||||
expect(screen.queryByRole("button", { name: "Purge all resolved" })).toBeNull();
|
||||
});
|
||||
|
||||
test("purging one row asks first, then sends the DELETE and refetches the lists", async () => {
|
||||
responses["DELETE /api/diagnostics/30"] = NO_CONTENT;
|
||||
renderRoute();
|
||||
await screen.findByText("Disk space low");
|
||||
|
||||
fireEvent.click(within(screen.getByText("Disk space low").closest("tr")!).getByRole("button", { name: "Purge" }));
|
||||
|
||||
const dialog = await screen.findByRole("alertdialog");
|
||||
expect(dialog.textContent).toContain("Purge this resolved event? Its history is gone for good.");
|
||||
fireEvent.click(within(dialog).getByRole("button", { name: "Cancel" }));
|
||||
expect(requested).not.toContain("DELETE /api/diagnostics/30");
|
||||
|
||||
fireEvent.click(within(screen.getByText("Disk space low").closest("tr")!).getByRole("button", { name: "Purge" }));
|
||||
fireEvent.click(within(await screen.findByRole("alertdialog")).getByRole("button", { name: "Purge" }));
|
||||
|
||||
await waitFor(() => expect(requested).toContain("DELETE /api/diagnostics/30"));
|
||||
// The invalidation covers both sections: the page the row left and the
|
||||
// active list, whose `active` counts come from the same table.
|
||||
await waitFor(() =>
|
||||
expect(requested.filter((url) => url === "/api/diagnostics?state=resolved").length).toBeGreaterThan(1),
|
||||
);
|
||||
await waitFor(() =>
|
||||
expect(requested.filter((url) => url === "/api/diagnostics?state=active").length).toBeGreaterThan(1),
|
||||
);
|
||||
});
|
||||
|
||||
test("purging the whole history asks first and sends one DELETE", async () => {
|
||||
responses["DELETE /api/diagnostics"] = { purged: 1 };
|
||||
renderRoute();
|
||||
await screen.findByText("Disk space low");
|
||||
|
||||
fireEvent.click(screen.getByRole("button", { name: "Purge all resolved" }));
|
||||
const dialog = await screen.findByRole("alertdialog");
|
||||
expect(dialog.textContent).toContain("Purge all resolved events? Active events are kept.");
|
||||
|
||||
// What the server will answer once the purge has landed; the refetch the
|
||||
// mutation triggers is what has to pick it up.
|
||||
responses["/api/diagnostics?state=resolved"] = page([]);
|
||||
fireEvent.click(within(dialog).getByRole("button", { name: "Purge all" }));
|
||||
|
||||
await waitFor(() => expect(requested).toContain("DELETE /api/diagnostics"));
|
||||
await waitFor(() => expect(screen.queryByText("Disk space low")).toBeNull());
|
||||
expect(screen.getByText("Blocklist source failed to update")).toBeTruthy();
|
||||
});
|
||||
|
||||
test("a refused purge reports the server's reason and keeps the row", async () => {
|
||||
responses["DELETE /api/diagnostics/30"] = fail(409, "the event is still active; it can be purged once it resolves");
|
||||
renderRoute("/diagnostics", { retry: false });
|
||||
await screen.findByText("Disk space low");
|
||||
|
||||
fireEvent.click(within(screen.getByText("Disk space low").closest("tr")!).getByRole("button", { name: "Purge" }));
|
||||
fireEvent.click(within(await screen.findByRole("alertdialog")).getByRole("button", { name: "Purge" }));
|
||||
|
||||
const alert = await screen.findByRole("alert");
|
||||
expect(alert.textContent).toContain("the event is still active");
|
||||
expect(screen.getByText("Disk space low")).toBeTruthy();
|
||||
});
|
||||
|
||||
test("an episode links to its own detail page", async () => {
|
||||
responses["/api/diagnostics/42"] = event(42);
|
||||
renderRoute();
|
||||
const link = await screen.findByRole("link", { name: "Blocklist source failed to update" });
|
||||
expect(link.getAttribute("href")).toBe("/diagnostics/42");
|
||||
});
|
||||
@@ -0,0 +1,498 @@
|
||||
import { useState } from "react";
|
||||
import {
|
||||
useInfiniteQuery,
|
||||
useMutation,
|
||||
useQueryClient,
|
||||
type InfiniteData,
|
||||
type UseInfiniteQueryResult,
|
||||
} from "@tanstack/react-query";
|
||||
import { Link, useNavigate, useSearch } from "@tanstack/react-router";
|
||||
import * as stylex from "@stylexjs/stylex";
|
||||
import * as api from "@/lib/api";
|
||||
import InlineError from "@/lib/InlineError";
|
||||
import { formatDuration, formatTime } from "@/lib/format";
|
||||
import { diagnosticPurgeMutation, diagnosticsInfiniteQuery, diagnosticsPurgeResolvedMutation } from "@/lib/queries";
|
||||
import type {
|
||||
DiagnosticEvent,
|
||||
DiagnosticSeverity,
|
||||
DiagnosticState,
|
||||
DiagnosticsFilter,
|
||||
DiagnosticsPage as Page,
|
||||
} from "@/lib/types";
|
||||
import ConfirmDialog from "@/ui/ConfirmDialog";
|
||||
import Select from "@/ui/Select";
|
||||
import { styles as shared } from "@/ui/styles";
|
||||
import { colors } from "@/ui/tokens.stylex";
|
||||
import SeverityBadge from "./SeverityBadge";
|
||||
import { DIAGNOSTIC_COMPONENTS, componentLabel, copyFor } from "./eventCopy";
|
||||
|
||||
const DARK = "@media (prefers-color-scheme: dark)";
|
||||
|
||||
const STATE_OPTIONS = [
|
||||
{ value: "all", label: "Active and resolved" },
|
||||
{ value: "active", label: "Active only" },
|
||||
{ value: "resolved", label: "Resolved only" },
|
||||
];
|
||||
|
||||
const SEVERITY_OPTIONS = [
|
||||
{ value: "any", label: "Any severity" },
|
||||
{ value: "warning", label: "Warnings" },
|
||||
{ value: "error", label: "Errors" },
|
||||
];
|
||||
|
||||
const COMPONENT_OPTIONS = [
|
||||
{ value: "any", label: "All components" },
|
||||
...DIAGNOSTIC_COMPONENTS.map((component) => ({ value: component, label: componentLabel(component) })),
|
||||
];
|
||||
|
||||
const styles = stylex.create({
|
||||
heading: {
|
||||
fontSize: "1.5rem",
|
||||
lineHeight: "2rem",
|
||||
fontWeight: 600,
|
||||
},
|
||||
intro: {
|
||||
marginTop: "0.25rem",
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
color: colors.textMuted,
|
||||
maxWidth: "48rem",
|
||||
},
|
||||
filterGrid: {
|
||||
marginTop: "1rem",
|
||||
display: "grid",
|
||||
gap: "0.75rem",
|
||||
gridTemplateColumns: {
|
||||
default: "repeat(1, minmax(0, 1fr))",
|
||||
"@media (min-width: 640px)": "repeat(3, minmax(0, 1fr))",
|
||||
},
|
||||
maxWidth: "48rem",
|
||||
},
|
||||
sectionHeading: {
|
||||
marginTop: "1.5rem",
|
||||
fontSize: "1.125rem",
|
||||
lineHeight: "1.75rem",
|
||||
fontWeight: 600,
|
||||
},
|
||||
sectionHeadingRow: {
|
||||
display: "flex",
|
||||
alignItems: "baseline",
|
||||
flexWrap: "wrap",
|
||||
justifyContent: "space-between",
|
||||
gap: "0.75rem",
|
||||
},
|
||||
/**
|
||||
* Nothing open is the normal state of a working install, so it gets one
|
||||
* quiet muted line — no border, no icon, no alert role. A panel here would
|
||||
* read as a broken page rather than as good news.
|
||||
*/
|
||||
healthy: {
|
||||
marginTop: "0.5rem",
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
color: colors.textSecondary,
|
||||
},
|
||||
empty: {
|
||||
marginTop: "0.5rem",
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
color: colors.textMuted,
|
||||
},
|
||||
cardList: {
|
||||
marginTop: "0.75rem",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: "0.5rem",
|
||||
listStyleType: "none",
|
||||
padding: 0,
|
||||
},
|
||||
card: {
|
||||
borderRadius: "0.25rem",
|
||||
borderWidth: 1,
|
||||
borderStyle: "solid",
|
||||
borderColor: colors.border,
|
||||
backgroundColor: colors.surfaceRaised,
|
||||
paddingInline: "0.75rem",
|
||||
paddingBlock: "0.625rem",
|
||||
},
|
||||
cardTop: {
|
||||
display: "flex",
|
||||
alignItems: "baseline",
|
||||
flexWrap: "wrap",
|
||||
gap: "0.5rem",
|
||||
},
|
||||
cardTitle: {
|
||||
fontWeight: 500,
|
||||
color: colors.primaryOnSurface,
|
||||
textDecorationLine: "none",
|
||||
},
|
||||
subject: {
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
color: colors.textSecondary,
|
||||
wordBreak: "break-all",
|
||||
},
|
||||
meta: {
|
||||
marginTop: "0.25rem",
|
||||
fontSize: "0.75rem",
|
||||
lineHeight: "1rem",
|
||||
color: colors.textMuted,
|
||||
},
|
||||
tableWrap: {
|
||||
marginTop: "0.75rem",
|
||||
overflowX: "auto",
|
||||
borderRadius: "0.25rem",
|
||||
borderWidth: 1,
|
||||
borderStyle: "solid",
|
||||
borderColor: colors.border,
|
||||
},
|
||||
table: {
|
||||
width: "100%",
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
},
|
||||
head: {
|
||||
backgroundColor: { default: "oklch(98.5% 0 none)", [DARK]: "oklch(21% 0.006 285.885)" },
|
||||
textAlign: "left",
|
||||
},
|
||||
th: {
|
||||
paddingInline: "0.75rem",
|
||||
paddingBlock: "0.5rem",
|
||||
fontWeight: 500,
|
||||
color: colors.textSecondary,
|
||||
whiteSpace: "nowrap",
|
||||
},
|
||||
row: {
|
||||
borderTopWidth: { default: 1, ":first-child": 0 },
|
||||
borderTopStyle: "solid",
|
||||
borderTopColor: colors.border,
|
||||
},
|
||||
cell: {
|
||||
paddingInline: "0.75rem",
|
||||
paddingBlock: "0.5rem",
|
||||
},
|
||||
nowrap: {
|
||||
whiteSpace: "nowrap",
|
||||
},
|
||||
rowLink: {
|
||||
color: colors.primaryOnSurface,
|
||||
textDecorationLine: "none",
|
||||
},
|
||||
footer: {
|
||||
marginTop: "0.75rem",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: "0.75rem",
|
||||
},
|
||||
note: {
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
color: colors.textMuted,
|
||||
},
|
||||
moreError: {
|
||||
marginTop: "0.5rem",
|
||||
fontSize: "0.875rem",
|
||||
lineHeight: "1.25rem",
|
||||
color: colors.dangerText,
|
||||
},
|
||||
});
|
||||
|
||||
type Section = UseInfiniteQueryResult<InfiniteData<Page, unknown>, Error>;
|
||||
|
||||
/** The two enum filters, narrowed from the picker's string rather than cast. */
|
||||
function asState(value: string): DiagnosticState | undefined {
|
||||
return value === "active" || value === "resolved" ? value : undefined;
|
||||
}
|
||||
|
||||
function asSeverity(value: string): DiagnosticSeverity | undefined {
|
||||
return value === "warning" || value === "error" ? value : undefined;
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function occurrenceText(count: number): string {
|
||||
return `${count} ${count === 1 ? "occurrence" : "occurrences"}`;
|
||||
}
|
||||
|
||||
function rowsOf(section: Section): DiagnosticEvent[] {
|
||||
return (section.data?.pages ?? []).flatMap((page) => page.events);
|
||||
}
|
||||
|
||||
/**
|
||||
* The cursor comes from the newest page on screen, not from `hasNextPage`:
|
||||
* while placeholder data stands in for a filter change the query state is
|
||||
* empty, and the button would flash away and back.
|
||||
*/
|
||||
function hasMore(section: Section): boolean {
|
||||
const pages = section.data?.pages ?? [];
|
||||
const last = pages[pages.length - 1];
|
||||
return last !== undefined && last.next_before !== null;
|
||||
}
|
||||
|
||||
function MoreButton({ section }: { section: Section }) {
|
||||
const more = hasMore(section);
|
||||
// A 401 is already redirecting via the cache-level handleUnauthorized.
|
||||
const isUnauthorized = section.error instanceof api.ApiError && section.error.status === 401;
|
||||
const failed = section.isFetchNextPageError && !isUnauthorized ? errorMessage(section.error) : null;
|
||||
if (!more && failed === null) return null;
|
||||
return (
|
||||
<>
|
||||
<div {...stylex.props(styles.footer)}>
|
||||
{more && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
if (section.isFetchingNextPage || section.isPlaceholderData) return;
|
||||
void section.fetchNextPage();
|
||||
}}
|
||||
disabled={section.isFetchingNextPage || section.isPlaceholderData}
|
||||
{...stylex.props(shared.button, shared.focusRing)}
|
||||
>
|
||||
{section.isFetchingNextPage ? "Loading…" : "Load more"}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
{failed !== null && (
|
||||
<p role="alert" {...stylex.props(styles.moreError)}>
|
||||
Failed to load more: {failed}
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function ActiveCard({ event, now }: { event: DiagnosticEvent; now: number }) {
|
||||
const copy = copyFor(event.code);
|
||||
return (
|
||||
<li {...stylex.props(styles.card)}>
|
||||
<div {...stylex.props(styles.cardTop)}>
|
||||
<SeverityBadge severity={event.severity} />
|
||||
<Link
|
||||
to="/diagnostics/$id"
|
||||
params={{ id: String(event.id) }}
|
||||
{...stylex.props(styles.cardTitle, shared.focusRing)}
|
||||
>
|
||||
{copy.title}
|
||||
</Link>
|
||||
<span {...stylex.props(styles.subject)}>{event.subject}</span>
|
||||
</div>
|
||||
<p {...stylex.props(styles.meta)}>
|
||||
Active for {formatDuration(now - event.first_seen)} · {occurrenceText(event.occurrences)} · last failure{" "}
|
||||
{formatTime(event.last_seen)}
|
||||
</p>
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
function HistoryRow({ event, onPurge, busy }: { event: DiagnosticEvent; onPurge: () => void; busy: boolean }) {
|
||||
const copy = copyFor(event.code);
|
||||
return (
|
||||
<tr {...stylex.props(styles.row)}>
|
||||
<td {...stylex.props(styles.cell)}>
|
||||
<SeverityBadge severity={event.severity} />
|
||||
</td>
|
||||
<td {...stylex.props(styles.cell)}>
|
||||
<Link
|
||||
to="/diagnostics/$id"
|
||||
params={{ id: String(event.id) }}
|
||||
{...stylex.props(styles.rowLink, shared.focusRing)}
|
||||
>
|
||||
{copy.title}
|
||||
</Link>
|
||||
</td>
|
||||
<td {...stylex.props(styles.cell)}>{event.subject}</td>
|
||||
<td {...stylex.props(styles.cell, styles.nowrap)}>{formatTime(event.first_seen)}</td>
|
||||
<td {...stylex.props(styles.cell, styles.nowrap)}>
|
||||
{event.resolved_at === null ? "—" : formatTime(event.resolved_at)}
|
||||
</td>
|
||||
<td {...stylex.props(styles.cell, styles.nowrap, shared.tabularNums)}>{event.occurrences}</td>
|
||||
<td {...stylex.props(styles.cell, styles.nowrap)}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onPurge}
|
||||
disabled={busy}
|
||||
{...stylex.props(shared.dangerLinkButton, shared.focusRing)}
|
||||
>
|
||||
Purge
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
}
|
||||
|
||||
export default function DiagnosticsPage() {
|
||||
const search = useSearch({ from: "/shell/diagnostics" });
|
||||
const navigate = useNavigate({ from: "/diagnostics" });
|
||||
const state = search.state ?? "all";
|
||||
|
||||
const base: DiagnosticsFilter = {};
|
||||
if (search.severity !== undefined) base.severity = search.severity;
|
||||
if (search.component !== undefined) base.component = search.component;
|
||||
|
||||
const active = useInfiniteQuery(diagnosticsInfiniteQuery({ ...base, state: "active" }, state !== "resolved"));
|
||||
const history = useInfiniteQuery(diagnosticsInfiniteQuery({ ...base, state: "resolved" }, state !== "active"));
|
||||
|
||||
const queryClient = useQueryClient();
|
||||
const purgeOne = useMutation(diagnosticPurgeMutation(queryClient));
|
||||
const purgeAll = useMutation(diagnosticsPurgeResolvedMutation(queryClient));
|
||||
// `null` is "no dialog"; the id is which row it is about, and `"all"` the
|
||||
// whole history. One piece of state, so the two dialogs cannot both be open.
|
||||
const [pendingPurge, setPendingPurge] = useState<number | "all" | null>(null);
|
||||
|
||||
const activeRows = rowsOf(active);
|
||||
const historyRows = rowsOf(history);
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const purging = purgeOne.isPending || purgeAll.isPending;
|
||||
|
||||
function setSearch(patch: Partial<typeof search>) {
|
||||
void navigate({ search: (prev) => ({ ...prev, ...patch }) });
|
||||
}
|
||||
|
||||
function confirmPurge() {
|
||||
if (pendingPurge === null) return;
|
||||
if (pendingPurge === "all") {
|
||||
purgeAll.mutate();
|
||||
} else {
|
||||
purgeOne.mutate(pendingPurge);
|
||||
}
|
||||
setPendingPurge(null);
|
||||
}
|
||||
|
||||
return (
|
||||
<section>
|
||||
<h1 {...stylex.props(styles.heading)}>Diagnostics</h1>
|
||||
<p {...stylex.props(styles.intro)}>
|
||||
Operational failures, one entry per subject that failed. An entry opens on the first failure, counts
|
||||
repeats, and closes when the subject recovers.
|
||||
</p>
|
||||
|
||||
<div {...stylex.props(styles.filterGrid)}>
|
||||
<Select
|
||||
variant="compactField"
|
||||
label="Show"
|
||||
value={state}
|
||||
onChange={(value) => setSearch({ state: asState(value) })}
|
||||
options={STATE_OPTIONS}
|
||||
/>
|
||||
<Select
|
||||
variant="compactField"
|
||||
label="Severity"
|
||||
value={search.severity ?? "any"}
|
||||
onChange={(value) => setSearch({ severity: asSeverity(value) })}
|
||||
options={SEVERITY_OPTIONS}
|
||||
/>
|
||||
<Select
|
||||
variant="compactField"
|
||||
label="Component"
|
||||
value={search.component ?? "any"}
|
||||
onChange={(value) => setSearch({ component: value === "any" ? undefined : value })}
|
||||
options={COMPONENT_OPTIONS}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{state !== "resolved" && (
|
||||
<>
|
||||
<h2 {...stylex.props(styles.sectionHeading)}>Active</h2>
|
||||
{active.status === "error" ? (
|
||||
<InlineError error={active.error} onRetry={() => void active.refetch()} />
|
||||
) : active.data === undefined ? (
|
||||
<p {...stylex.props(styles.empty, shared.pulse)} role="status">
|
||||
Loading diagnostics…
|
||||
</p>
|
||||
) : activeRows.length === 0 ? (
|
||||
<p {...stylex.props(styles.healthy)} role="status">
|
||||
No active operational issues.
|
||||
</p>
|
||||
) : (
|
||||
<>
|
||||
<ul {...stylex.props(styles.cardList)}>
|
||||
{activeRows.map((event) => (
|
||||
<ActiveCard key={event.id} event={event} now={now} />
|
||||
))}
|
||||
</ul>
|
||||
<MoreButton section={active} />
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{state !== "active" && (
|
||||
<>
|
||||
<div {...stylex.props(styles.sectionHeading, styles.sectionHeadingRow)}>
|
||||
<h2>Resolved</h2>
|
||||
{historyRows.length > 0 && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setPendingPurge("all")}
|
||||
disabled={purging}
|
||||
{...stylex.props(shared.dangerLinkButton, shared.focusRing)}
|
||||
>
|
||||
Purge all resolved
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
{history.status === "error" ? (
|
||||
<InlineError error={history.error} onRetry={() => void history.refetch()} />
|
||||
) : history.data === undefined ? (
|
||||
<p {...stylex.props(styles.empty, shared.pulse)} role="status">
|
||||
Loading history…
|
||||
</p>
|
||||
) : historyRows.length === 0 ? (
|
||||
<p {...stylex.props(styles.empty)}>Nothing has failed and recovered in the retained window.</p>
|
||||
) : (
|
||||
<>
|
||||
<div {...stylex.props(styles.tableWrap)}>
|
||||
<table {...stylex.props(styles.table)}>
|
||||
<thead {...stylex.props(styles.head)}>
|
||||
<tr>
|
||||
<th {...stylex.props(styles.th)}>Severity</th>
|
||||
<th {...stylex.props(styles.th)}>Event</th>
|
||||
<th {...stylex.props(styles.th)}>Subject</th>
|
||||
<th {...stylex.props(styles.th)}>Started</th>
|
||||
<th {...stylex.props(styles.th)}>Resolved</th>
|
||||
<th {...stylex.props(styles.th)}>Occurrences</th>
|
||||
<th {...stylex.props(styles.th)}>
|
||||
<span {...stylex.props(shared.srOnly)}>Actions</span>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{historyRows.map((event) => (
|
||||
<HistoryRow
|
||||
key={event.id}
|
||||
event={event}
|
||||
busy={purging}
|
||||
onPurge={() => setPendingPurge(event.id)}
|
||||
/>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p {...stylex.props(styles.footer, styles.note)}>
|
||||
Showing {historyRows.length} resolved {historyRows.length === 1 ? "entry" : "entries"}
|
||||
{hasMore(history) ? "" : " — end of history"}
|
||||
</p>
|
||||
<MoreButton section={history} />
|
||||
</>
|
||||
)}
|
||||
<InlineError error={purgeOne.error ?? purgeAll.error} />
|
||||
</>
|
||||
)}
|
||||
|
||||
<ConfirmDialog
|
||||
isOpen={pendingPurge !== null}
|
||||
title={pendingPurge === "all" ? "Purge resolved history" : "Purge event"}
|
||||
message={
|
||||
pendingPurge === "all"
|
||||
? "Purge all resolved events? Active events are kept."
|
||||
: "Purge this resolved event? Its history is gone for good."
|
||||
}
|
||||
confirmLabel={pendingPurge === "all" ? "Purge all" : "Purge"}
|
||||
onConfirm={confirmPurge}
|
||||
onCancel={() => setPendingPurge(null)}
|
||||
/>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/**
|
||||
* The severity chip both diagnostics views carry. The word is the affordance —
|
||||
* colour alone would leave the severity unreadable to a screen reader and to
|
||||
* anyone who does not separate the amber from the red.
|
||||
*/
|
||||
|
||||
import * as stylex from "@stylexjs/stylex";
|
||||
import type { DiagnosticSeverity } from "@/lib/types";
|
||||
import { colors } from "@/ui/tokens.stylex";
|
||||
|
||||
const styles = stylex.create({
|
||||
badge: {
|
||||
display: "inline-block",
|
||||
borderRadius: "0.25rem",
|
||||
borderWidth: 1,
|
||||
borderStyle: "solid",
|
||||
paddingInline: "0.375rem",
|
||||
paddingBlock: "0.125rem",
|
||||
fontSize: "0.75rem",
|
||||
lineHeight: "1rem",
|
||||
fontWeight: 500,
|
||||
whiteSpace: "nowrap",
|
||||
},
|
||||
warning: {
|
||||
borderColor: colors.warnBorder,
|
||||
backgroundColor: colors.warnSurface,
|
||||
color: colors.warnText,
|
||||
},
|
||||
error: {
|
||||
borderColor: colors.dangerBorder,
|
||||
backgroundColor: colors.dangerSurface,
|
||||
color: colors.dangerText,
|
||||
},
|
||||
});
|
||||
|
||||
export default function SeverityBadge({ severity }: { severity: DiagnosticSeverity }) {
|
||||
return (
|
||||
<span {...stylex.props(styles.badge, severity === "error" ? styles.error : styles.warning)}>
|
||||
{severity === "error" ? "Error" : "Warning"}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { DIAGNOSTIC_CODES } from "@/lib/types";
|
||||
import { DIAGNOSTIC_COMPONENTS, EVENT_COPY, componentLabel, copyFor } from "./eventCopy";
|
||||
|
||||
test("the enum holds the fifteen codes the store defines", () => {
|
||||
expect(DIAGNOSTIC_CODES).toHaveLength(15);
|
||||
expect(new Set(DIAGNOSTIC_CODES).size).toBe(15);
|
||||
});
|
||||
|
||||
test("every code has copy, and no copy belongs to a code that does not exist", () => {
|
||||
for (const code of DIAGNOSTIC_CODES) {
|
||||
const copy = EVENT_COPY[code];
|
||||
expect(copy, code).toBeDefined();
|
||||
expect(copy.title.length, code).toBeGreaterThan(0);
|
||||
expect(copy.impact.length, code).toBeGreaterThan(0);
|
||||
expect(copy.remediation.length, code).toBeGreaterThan(0);
|
||||
}
|
||||
expect(Object.keys(EVENT_COPY).sort()).toEqual([...DIAGNOSTIC_CODES].sort());
|
||||
});
|
||||
|
||||
test("titles are distinct, so two open episodes never read as the same event", () => {
|
||||
const titles = DIAGNOSTIC_CODES.map((code) => EVENT_COPY[code].title);
|
||||
expect(new Set(titles).size).toBe(titles.length);
|
||||
});
|
||||
|
||||
test("a code this build has never heard of falls back to the code itself", () => {
|
||||
// The server is the authority on the enum; a newer one can send a sixteenth.
|
||||
const copy = copyFor("nonsense.code" as (typeof DIAGNOSTIC_CODES)[number]);
|
||||
expect(copy.title).toBe("nonsense.code");
|
||||
expect(copy.remediation.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test("the component options are the code prefixes, deduplicated and in enum order", () => {
|
||||
expect(DIAGNOSTIC_COMPONENTS).toEqual([
|
||||
"disk",
|
||||
"blocklist",
|
||||
"certificate",
|
||||
"query_log",
|
||||
"upstream_history",
|
||||
"upstream",
|
||||
"client_names",
|
||||
"clients",
|
||||
"listener",
|
||||
"configuration",
|
||||
]);
|
||||
});
|
||||
|
||||
test("component labels read as prose without inventing a name", () => {
|
||||
expect(componentLabel("query_log")).toBe("Query log");
|
||||
expect(componentLabel("disk")).toBe("Disk");
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
/**
|
||||
* What each event code means to the operator, in three fixed fields: what the
|
||||
* episode is (`title`), what it costs while it stays open (`impact`), and what
|
||||
* to do about it (`remediation`). The server sends a code and an error string;
|
||||
* every word of explanation the page shows comes from here.
|
||||
*
|
||||
* The record is exhaustive over `DiagnosticCode` by type, and a test walks
|
||||
* `DIAGNOSTIC_CODES` to prove it at runtime too. A sixteenth code added to the
|
||||
* enum fails `tsc` here before it can reach the page as a bare dotted string.
|
||||
*
|
||||
* `link` points at the configuration surface that governs the failure. Those
|
||||
* are today's routes; the navigation restructure re-points them.
|
||||
*/
|
||||
|
||||
import { DIAGNOSTIC_CODES, type DiagnosticCode } from "@/lib/types";
|
||||
|
||||
/** The literal paths keep `link.to` assignable to a typed router `Link`. */
|
||||
export type CopyLinkPath = "/settings" | "/blocklists" | "/upstreams" | "/clients";
|
||||
|
||||
export interface EventCopy {
|
||||
title: string;
|
||||
impact: string;
|
||||
remediation: string;
|
||||
link?: { to: CopyLinkPath; label: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* The copy for a code, with a floor under it. `tsc` proves the record covers
|
||||
* the union, but a server one release ahead can send a code this build has
|
||||
* never heard of; showing the raw code beats rendering "undefined".
|
||||
*/
|
||||
export function copyFor(code: DiagnosticCode): EventCopy {
|
||||
return (
|
||||
EVENT_COPY[code] ?? {
|
||||
title: code,
|
||||
impact: "This build has no description for this event code.",
|
||||
remediation: "The error detail below is the whole of what the server reported.",
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const SETTINGS = { to: "/settings", label: "Settings" } as const;
|
||||
const BLOCKLISTS = { to: "/blocklists", label: "Blocklists" } as const;
|
||||
const UPSTREAMS = { to: "/upstreams", label: "Upstreams" } as const;
|
||||
const CLIENTS = { to: "/clients", label: "Clients" } as const;
|
||||
|
||||
/**
|
||||
* The component filter's options, derived from the codes rather than listed
|
||||
* again: the server matches `component` against the part of `code` before the
|
||||
* dot, so any list written by hand here could drift from the enum.
|
||||
*/
|
||||
export const DIAGNOSTIC_COMPONENTS: readonly string[] = [
|
||||
...new Set(DIAGNOSTIC_CODES.map((code) => code.slice(0, code.indexOf(".")))),
|
||||
];
|
||||
|
||||
/** `query_log` → "Query log". Display only; the filter sends the raw component. */
|
||||
export function componentLabel(component: string): string {
|
||||
const spaced = component.replaceAll("_", " ");
|
||||
return spaced.charAt(0).toUpperCase() + spaced.slice(1);
|
||||
}
|
||||
|
||||
export const EVENT_COPY: Record<DiagnosticCode, EventCopy> = {
|
||||
"disk.space": {
|
||||
title: "Disk space low",
|
||||
impact: "Below the critical threshold nxdns stops blocklist updates and query log flushes to protect the disk.",
|
||||
remediation: "Free space on the data volume, or lower the retention window so the query log holds fewer days.",
|
||||
link: SETTINGS,
|
||||
},
|
||||
"disk.probe": {
|
||||
title: "Disk usage probe failed",
|
||||
impact: "Free space is unknown, so the low-disk guard cannot act until a probe succeeds.",
|
||||
remediation: "Check that the data and log directories exist and that the service user can read them.",
|
||||
link: SETTINGS,
|
||||
},
|
||||
"blocklist.refresh": {
|
||||
title: "Blocklist source failed to update",
|
||||
impact: "The source keeps serving its last good snapshot, so blocking continues but the list ages.",
|
||||
remediation: "Check the source url and the machine's internet access, then update the lists again.",
|
||||
link: BLOCKLISTS,
|
||||
},
|
||||
"blocklist.snapshot": {
|
||||
title: "Filter snapshot failed to publish",
|
||||
impact: "The resolver keeps the snapshot it already holds; blocklist edits do not take effect until one publishes.",
|
||||
remediation: "Check free disk space and the data directory's permissions, then update the lists again.",
|
||||
link: BLOCKLISTS,
|
||||
},
|
||||
"blocklist.storage": {
|
||||
title: "Blocklist storage operation failed",
|
||||
impact: "Cached list files or their database rows are out of step; a later pass can redownload what is missing.",
|
||||
remediation: "Check free disk space and the data directory's permissions.",
|
||||
link: BLOCKLISTS,
|
||||
},
|
||||
"certificate.reload": {
|
||||
title: "TLS certificate reload failed",
|
||||
impact: "The endpoint keeps serving the certificate it already loaded, which expires on its own schedule.",
|
||||
remediation:
|
||||
"Check the certificate and key paths, and that renewal writes both files the service user can read.",
|
||||
link: SETTINGS,
|
||||
},
|
||||
"query_log.write": {
|
||||
title: "Query log write failed",
|
||||
impact: "Queries are resolved and answered as usual, but they are not being recorded.",
|
||||
remediation: "Check free disk space and the log database's permissions, then restart nxdns.",
|
||||
link: SETTINGS,
|
||||
},
|
||||
"query_log.maintenance": {
|
||||
title: "Query log maintenance failed",
|
||||
impact: "Old rows are not being trimmed, so the log database grows past its retention window.",
|
||||
remediation: "Check free disk space; the next maintenance pass retries on its own.",
|
||||
link: SETTINGS,
|
||||
},
|
||||
"query_log.recreated": {
|
||||
title: "Query log recreated",
|
||||
impact: "The old log database was unreadable and was moved aside; the history it held is not in the new one.",
|
||||
remediation: "Keep or delete the aside file named below. Nothing else is required — logging is running.",
|
||||
link: SETTINGS,
|
||||
},
|
||||
"upstream_history.write": {
|
||||
title: "Upstream history write failed",
|
||||
impact: "Resolution is unaffected; the per-upstream success and failure aggregates lose the affected window.",
|
||||
remediation: "Check free disk space and the configuration database's permissions.",
|
||||
link: UPSTREAMS,
|
||||
},
|
||||
"upstream.exchange": {
|
||||
title: "Upstream failing",
|
||||
impact: "Queries fall through to the remaining upstreams; answers are slower while this one backs off.",
|
||||
remediation: "Check the upstream's reachability and its TLS name. Remove it if it stays down.",
|
||||
link: UPSTREAMS,
|
||||
},
|
||||
"client_names.storage": {
|
||||
title: "Client name storage failed",
|
||||
impact: "Learned reverse-DNS names are not persisted, so clients can show as bare addresses after a restart.",
|
||||
remediation: "Check free disk space and the configuration database's permissions.",
|
||||
link: CLIENTS,
|
||||
},
|
||||
"clients.storage": {
|
||||
title: "Client record storage failed",
|
||||
impact: "New clients may not appear in the list and stale ones may not be pruned.",
|
||||
remediation: "Check free disk space and the configuration database's permissions.",
|
||||
link: CLIENTS,
|
||||
},
|
||||
"listener.start": {
|
||||
title: "Encrypted DNS listener failed to start",
|
||||
impact: "That endpoint is not accepting queries. Plain DNS on port 53 is unaffected.",
|
||||
remediation:
|
||||
"Check the bind address, the port, and the certificate paths, then restart nxdns. The episode closes on a clean start.",
|
||||
link: SETTINGS,
|
||||
},
|
||||
"configuration.load": {
|
||||
title: "Configuration problem at startup",
|
||||
impact: "The setting named below was rejected or replaced by its default for this run.",
|
||||
remediation: "Correct the setting and restart nxdns. The episode closes on a clean start.",
|
||||
link: SETTINGS,
|
||||
},
|
||||
};
|
||||
@@ -6,6 +6,10 @@ import type {
|
||||
ClientEdit,
|
||||
ClientPrefix,
|
||||
ClientPrefixInput,
|
||||
DiagnosticEvent,
|
||||
DiagnosticsFilter,
|
||||
DiagnosticsPage,
|
||||
DiagnosticsPurge,
|
||||
ForwardZone,
|
||||
ForwardZoneInput,
|
||||
Group,
|
||||
@@ -121,6 +125,20 @@ export const getLookup = (domain: string, groupId?: number): Promise<LookupResul
|
||||
export const getUpstreamHealth = (period?: Period): Promise<UpstreamHealth> =>
|
||||
request(`/api/upstream/health${qs({ period })}`);
|
||||
|
||||
// Diagnostics
|
||||
|
||||
export const getDiagnostics = (filter: DiagnosticsFilter = {}): Promise<DiagnosticsPage> =>
|
||||
request(`/api/diagnostics${qs({ ...filter })}`);
|
||||
|
||||
export const getDiagnostic = (id: number): Promise<DiagnosticEvent> => request(`/api/diagnostics/${id}`);
|
||||
|
||||
/** Purges one resolved event. An event still active answers 409, an unknown id 404. */
|
||||
export const purgeDiagnostic = (id: number): Promise<void> => request(`/api/diagnostics/${id}`, { method: "DELETE" });
|
||||
|
||||
/** Purges the whole resolved history; active events are never touched. */
|
||||
export const purgeResolvedDiagnostics = (): Promise<DiagnosticsPurge> =>
|
||||
request("/api/diagnostics", { method: "DELETE" });
|
||||
|
||||
// Groups
|
||||
|
||||
export const listGroups = async (): Promise<Group[]> => (await request<{ groups: Group[] }>("/api/groups")).groups;
|
||||
|
||||
@@ -16,6 +16,9 @@ import type {
|
||||
BlocklistEcho,
|
||||
Client,
|
||||
ClientPrefix,
|
||||
DiagnosticEvent,
|
||||
DiagnosticsPage,
|
||||
DiagnosticsPurge,
|
||||
ErrorEnvelope,
|
||||
ForwardZone,
|
||||
Group,
|
||||
@@ -39,6 +42,11 @@ import type {
|
||||
} from "@/lib/types";
|
||||
|
||||
export const sample_get_health: Health = {
|
||||
diagnostics: {
|
||||
active_errors: 0,
|
||||
active_warnings: 0,
|
||||
state: "recording",
|
||||
},
|
||||
disk: {
|
||||
db_bytes: 0,
|
||||
free_bytes: 0,
|
||||
@@ -73,6 +81,57 @@ export const sample_logout: LogoutResponse = {
|
||||
authenticated: false,
|
||||
};
|
||||
|
||||
export const sample_get_diagnostics: DiagnosticsPage = {
|
||||
active: {
|
||||
errors: 0,
|
||||
warnings: 0,
|
||||
},
|
||||
events: [
|
||||
{
|
||||
code: "upstream_history.write",
|
||||
component: "upstream_history",
|
||||
detail: "Busy",
|
||||
first_seen: 0,
|
||||
id: 0,
|
||||
last_seen: 0,
|
||||
occurrences: 0,
|
||||
resolved_at: 0,
|
||||
severity: "warning",
|
||||
subject: "history",
|
||||
},
|
||||
{
|
||||
code: "blocklist.refresh",
|
||||
component: "blocklist",
|
||||
detail: "download failed: ConnectionTimedOut",
|
||||
first_seen: 0,
|
||||
id: 0,
|
||||
last_seen: 0,
|
||||
occurrences: 0,
|
||||
resolved_at: null,
|
||||
severity: "warning",
|
||||
subject: "StevenBlack",
|
||||
},
|
||||
],
|
||||
next_before: null,
|
||||
};
|
||||
|
||||
export const sample_get_diagnostic: DiagnosticEvent = {
|
||||
code: "blocklist.refresh",
|
||||
component: "blocklist",
|
||||
detail: "download failed: ConnectionTimedOut",
|
||||
first_seen: 0,
|
||||
id: 0,
|
||||
last_seen: 0,
|
||||
occurrences: 0,
|
||||
resolved_at: null,
|
||||
severity: "warning",
|
||||
subject: "StevenBlack",
|
||||
};
|
||||
|
||||
export const sample_purge_diagnostics: DiagnosticsPurge = {
|
||||
purged: 0,
|
||||
};
|
||||
|
||||
export const sample_create_blocklist: BlocklistEcho = {
|
||||
enabled: false,
|
||||
id: 0,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { formatAge, formatBytes, formatMicros, formatTime } from "@/lib/format";
|
||||
import { formatAge, formatBytes, formatDuration, formatMicros, formatTime } from "@/lib/format";
|
||||
|
||||
test("formatTime renders unix seconds in the given locale and zone", () => {
|
||||
// 2024-01-01T00:00:00Z; ICU emits U+202F before AM/PM in recent Node.
|
||||
@@ -27,6 +27,15 @@ test("formatAge steps up a unit at each boundary and truncates", () => {
|
||||
expect(formatAge(400000)).toBe("4d ago");
|
||||
});
|
||||
|
||||
test("formatDuration is the same span without the 'ago', and never negative", () => {
|
||||
expect(formatDuration(0)).toBe("0s");
|
||||
expect(formatDuration(59)).toBe("59s");
|
||||
expect(formatDuration(3600)).toBe("1h");
|
||||
expect(formatDuration(86400)).toBe("1d");
|
||||
// Clock skew between the server's timestamps and the browser's clock.
|
||||
expect(formatDuration(-5)).toBe("0s");
|
||||
});
|
||||
|
||||
test("formatMicros renders milliseconds with one decimal", () => {
|
||||
expect(formatMicros(0)).toBe("0.0 ms");
|
||||
expect(formatMicros(1234)).toBe("1.2 ms");
|
||||
|
||||
@@ -39,6 +39,18 @@ export function formatAge(seconds: number): string {
|
||||
return `${Math.floor(seconds)}s ago`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Seconds of elapsed time → a coarse "3h", the same single truncated unit as
|
||||
* `formatAge` without the "ago". For a span the caller labels itself, as in
|
||||
* "active for 3h". A negative span reads "0s": clock skew is not a duration.
|
||||
*/
|
||||
export function formatDuration(seconds: number): string {
|
||||
for (const unit of AGE_UNITS) {
|
||||
if (seconds >= unit.seconds) return `${Math.floor(seconds / unit.seconds)}${unit.suffix}`;
|
||||
}
|
||||
return `${Math.max(0, Math.floor(seconds))}s`;
|
||||
}
|
||||
|
||||
/** Microseconds → milliseconds with one decimal, e.g. 1234 → "1.2 ms". */
|
||||
export function formatMicros(micros: number): string {
|
||||
return `${(micros / 1000).toFixed(1)} ms`;
|
||||
|
||||
@@ -5,6 +5,8 @@ import type {
|
||||
BlocklistInput,
|
||||
ClientEdit,
|
||||
ClientPrefixInput,
|
||||
DiagnosticsFilter,
|
||||
DiagnosticsPage,
|
||||
ForwardZoneInput,
|
||||
GroupInput,
|
||||
LocalRecordInput,
|
||||
@@ -23,6 +25,10 @@ export const queryKeys = {
|
||||
stats: (period: Period) => ["stats", period] as const,
|
||||
timeseries: (period: Period) => ["stats", "timeseries", period] as const,
|
||||
queriesInfinite: (filter: QueriesFilter) => ["queries", "infinite", filter] as const,
|
||||
diagnosticsInfinite: (filter: DiagnosticsFilter) => ["diagnostics", "infinite", filter] as const,
|
||||
diagnostic: (id: number) => ["diagnostics", "event", id] as const,
|
||||
/** Prefix of every diagnostics entry, page and detail alike; the purge target. */
|
||||
diagnosticsAll: ["diagnostics"] as const,
|
||||
upstreamHealth: (period: Period) => ["upstream-health", period] as const,
|
||||
lookup: (domain: string, groupId?: number) => ["lookup", domain, groupId ?? null] as const,
|
||||
/** Prefix of every `lookup` entry; the invalidation target after any verdict input changes. */
|
||||
@@ -69,6 +75,28 @@ export const queriesInfiniteQuery = (filter: QueriesFilter = {}) =>
|
||||
placeholderData: keepPreviousData,
|
||||
});
|
||||
|
||||
// Keyset pagination on `next_before`, exactly as the query log pages
|
||||
// (handlers/diagnostics.zig copies the /api/queries contract). The active view
|
||||
// polls on healthQuery's cadence because an episode opening is the same news a
|
||||
// health banner carries; a resolved-history page is settled and does not poll.
|
||||
// `enabled` belongs to the factory rather than to a spread at the call site:
|
||||
// spreading the options object loses the page-param type, and the Diagnostics
|
||||
// page turns one of its two sections off whenever a filter excludes it.
|
||||
export const diagnosticsInfiniteQuery = (filter: DiagnosticsFilter = {}, enabled = true) =>
|
||||
infiniteQueryOptions({
|
||||
enabled,
|
||||
queryKey: queryKeys.diagnosticsInfinite(filter),
|
||||
queryFn: ({ pageParam }: { pageParam: number | undefined }) =>
|
||||
api.getDiagnostics(pageParam === undefined ? filter : { ...filter, before: pageParam }),
|
||||
initialPageParam: undefined as number | undefined,
|
||||
getNextPageParam: (last: DiagnosticsPage) => last.next_before ?? undefined,
|
||||
placeholderData: keepPreviousData,
|
||||
refetchInterval: filter.state === "active" ? 10_000 : undefined,
|
||||
});
|
||||
|
||||
export const diagnosticQuery = (id: number) =>
|
||||
queryOptions({ queryKey: queryKeys.diagnostic(id), queryFn: () => api.getDiagnostic(id) });
|
||||
|
||||
// The period is part of the key: the upstream aggregates are ranged like the
|
||||
// stats ones, so the picker has to refetch them rather than reuse a cached
|
||||
// window under a new label.
|
||||
@@ -112,6 +140,19 @@ export const settingsQuery = () => queryOptions({ queryKey: queryKeys.settings,
|
||||
// Group membership and names feed lookup verdicts and the group columns on
|
||||
// clients, prefixes and rules, hence the wide invalidation on group mutations.
|
||||
|
||||
// Both purges invalidate the whole `diagnostics` prefix rather than one page
|
||||
// key: the resolved list, the active list (whose `active` counts ride along) and
|
||||
// the detail query of the row just deleted all describe the table that changed.
|
||||
export const diagnosticPurgeMutation = (qc: QueryClient) => ({
|
||||
mutationFn: (id: number) => api.purgeDiagnostic(id),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: queryKeys.diagnosticsAll }),
|
||||
});
|
||||
|
||||
export const diagnosticsPurgeResolvedMutation = (qc: QueryClient) => ({
|
||||
mutationFn: () => api.purgeResolvedDiagnostics(),
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: queryKeys.diagnosticsAll }),
|
||||
});
|
||||
|
||||
function invalidateGroupWorld(qc: QueryClient): Promise<unknown> {
|
||||
return Promise.all([
|
||||
qc.invalidateQueries({ queryKey: queryKeys.groups }),
|
||||
|
||||
@@ -28,6 +28,16 @@ export interface Health {
|
||||
writer_failed: boolean;
|
||||
refreshes_gated: number;
|
||||
snapshot_generation: number | null;
|
||||
/**
|
||||
* The diagnostics store's own state, not a summary of what it holds:
|
||||
* `unavailable` means the store is missing or its last write failed, so the
|
||||
* counts below are the last ones it managed to observe.
|
||||
*/
|
||||
diagnostics: {
|
||||
state: "recording" | "unavailable";
|
||||
active_warnings: number;
|
||||
active_errors: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface Version {
|
||||
@@ -81,6 +91,77 @@ export interface QueriesFilter {
|
||||
until?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The fifteen operational event codes, in the order `src/storage/events.zig`
|
||||
* declares them. A value, not only a type, because the copy map has to be
|
||||
* proven exhaustive at runtime as well as by `tsc`.
|
||||
*/
|
||||
export const DIAGNOSTIC_CODES = [
|
||||
"disk.space",
|
||||
"disk.probe",
|
||||
"blocklist.refresh",
|
||||
"blocklist.snapshot",
|
||||
"blocklist.storage",
|
||||
"certificate.reload",
|
||||
"query_log.write",
|
||||
"query_log.maintenance",
|
||||
"query_log.recreated",
|
||||
"upstream_history.write",
|
||||
"upstream.exchange",
|
||||
"client_names.storage",
|
||||
"clients.storage",
|
||||
"listener.start",
|
||||
"configuration.load",
|
||||
] as const;
|
||||
|
||||
export type DiagnosticCode = (typeof DIAGNOSTIC_CODES)[number];
|
||||
|
||||
export type DiagnosticSeverity = "warning" | "error";
|
||||
|
||||
/** Which episodes a query selects. `all` is the server's default. */
|
||||
export type DiagnosticState = "active" | "resolved" | "all";
|
||||
|
||||
export interface DiagnosticEvent {
|
||||
id: number;
|
||||
code: DiagnosticCode;
|
||||
/** The part of `code` before the dot, repeated by the server for filtering. */
|
||||
component: string;
|
||||
/** The display identity of what failed; redacted where it derives from a url. */
|
||||
subject: string;
|
||||
severity: DiagnosticSeverity;
|
||||
first_seen: number;
|
||||
last_seen: number;
|
||||
occurrences: number;
|
||||
/** Null while the episode is still open. */
|
||||
resolved_at: number | null;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface DiagnosticsPage {
|
||||
events: DiagnosticEvent[];
|
||||
next_before: number | null;
|
||||
/** Episodes open right now, whatever this page filtered to. */
|
||||
active: {
|
||||
warnings: number;
|
||||
errors: number;
|
||||
};
|
||||
}
|
||||
|
||||
/** `DELETE /api/diagnostics` — how many resolved events the purge removed. */
|
||||
export interface DiagnosticsPurge {
|
||||
purged: number;
|
||||
}
|
||||
|
||||
export interface DiagnosticsFilter {
|
||||
state?: DiagnosticState;
|
||||
severity?: DiagnosticSeverity;
|
||||
component?: string;
|
||||
since?: number;
|
||||
until?: number;
|
||||
limit?: number;
|
||||
before?: number;
|
||||
}
|
||||
|
||||
export interface StatsTotals {
|
||||
period: Period;
|
||||
since: number;
|
||||
|
||||
@@ -12,10 +12,13 @@ import {
|
||||
import AppShell from "@/shell/AppShell";
|
||||
import { ApiError } from "@/lib/api";
|
||||
import { createQueryClient } from "@/lib/queryClient";
|
||||
import type { DiagnosticSeverity, DiagnosticState, DiagnosticsFilter } from "@/lib/types";
|
||||
import {
|
||||
blocklistsQuery,
|
||||
clientPrefixesQuery,
|
||||
clientsQuery,
|
||||
diagnosticQuery,
|
||||
diagnosticsInfiniteQuery,
|
||||
forwardZonesQuery,
|
||||
groupsQuery,
|
||||
healthQuery,
|
||||
@@ -214,6 +217,52 @@ const lookupRoute = createRoute({
|
||||
component: lazyRouteComponent(() => import("@/features/lookup/LookupPage")),
|
||||
});
|
||||
|
||||
/**
|
||||
* The three filters live in the url so an episode can be linked to as it was
|
||||
* read. Anything else in the search object is dropped: an unknown value would
|
||||
* reach the api as a query parameter the handler rejects with a 400.
|
||||
*/
|
||||
const diagnosticsRoute = createRoute({
|
||||
getParentRoute: () => shellRoute,
|
||||
path: "/diagnostics",
|
||||
validateSearch: (
|
||||
search: Record<string, unknown>,
|
||||
): { state?: DiagnosticState; severity?: DiagnosticSeverity; component?: string } => {
|
||||
const state = search["state"];
|
||||
const severity = search["severity"];
|
||||
const component = search["component"];
|
||||
return {
|
||||
state: state === "active" || state === "resolved" ? state : undefined,
|
||||
severity: severity === "warning" || severity === "error" ? severity : undefined,
|
||||
component: typeof component === "string" && component !== "" ? component : undefined,
|
||||
};
|
||||
},
|
||||
loaderDeps: ({ search }) => search,
|
||||
// allSettled: the two sections render their own state, and the resolved
|
||||
// history failing must not replace the active list with the error page.
|
||||
loader: ({ context, deps }) => {
|
||||
const base: DiagnosticsFilter = {};
|
||||
if (deps.severity !== undefined) base.severity = deps.severity;
|
||||
if (deps.component !== undefined) base.component = deps.component;
|
||||
return Promise.allSettled([
|
||||
context.queryClient.ensureInfiniteQueryData(diagnosticsInfiniteQuery({ ...base, state: "active" })),
|
||||
context.queryClient.ensureInfiniteQueryData(diagnosticsInfiniteQuery({ ...base, state: "resolved" })),
|
||||
]);
|
||||
},
|
||||
component: lazyRouteComponent(() => import("@/features/diagnostics/DiagnosticsPage")),
|
||||
});
|
||||
|
||||
const diagnosticDetailRoute = createRoute({
|
||||
getParentRoute: () => shellRoute,
|
||||
path: "/diagnostics/$id",
|
||||
// Swallowed on purpose: an event retention has removed is a 404 the page
|
||||
// itself explains, with the way back to the list. The whole-page error
|
||||
// component would state it as a request failure instead.
|
||||
loader: ({ context, params }) =>
|
||||
context.queryClient.ensureQueryData(diagnosticQuery(Number(params.id))).catch(() => undefined),
|
||||
component: lazyRouteComponent(() => import("@/features/diagnostics/DiagnosticDetailPage")),
|
||||
});
|
||||
|
||||
const settingsRoute = createRoute({
|
||||
getParentRoute: () => shellRoute,
|
||||
path: "/settings",
|
||||
@@ -234,6 +283,8 @@ const routeTree = rootRoute.addChildren([
|
||||
localDnsRoute,
|
||||
upstreamsRoute,
|
||||
lookupRoute,
|
||||
diagnosticsRoute,
|
||||
diagnosticDetailRoute,
|
||||
settingsRoute,
|
||||
]),
|
||||
]);
|
||||
|
||||
@@ -16,6 +16,7 @@ const NAV_LABELS = [
|
||||
"Local DNS",
|
||||
"Upstreams",
|
||||
"Lookup",
|
||||
"Diagnostics",
|
||||
"Settings",
|
||||
];
|
||||
|
||||
@@ -39,6 +40,7 @@ const RESPONSES: Record<string, unknown> = {
|
||||
writer_failed: false,
|
||||
refreshes_gated: 0,
|
||||
snapshot_generation: null,
|
||||
diagnostics: { state: "recording", active_warnings: 0, active_errors: 0 },
|
||||
},
|
||||
"/api/upstream/health": { upstreams: [], available: 1, total: 1 },
|
||||
"/api/version": { version: "0.0.0-test", git_commit: "0000000", zig_version: "0.16.0", uptime_seconds: 1 },
|
||||
|
||||
@@ -26,6 +26,7 @@ const NAV_ITEMS = [
|
||||
{ to: "/local-dns", label: "Local DNS" },
|
||||
{ to: "/upstreams", label: "Upstreams" },
|
||||
{ to: "/lookup", label: "Lookup" },
|
||||
{ to: "/diagnostics", label: "Diagnostics" },
|
||||
{ to: "/settings", label: "Settings" },
|
||||
] as const;
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ nxdns serves its admin API itself, on `web.bind:web.port` (default port 8080), a
|
||||
|
||||
The machine-readable contract is `src/web/openapi.yaml`, which the running server hands out unauthenticated at `GET /api/openapi.yaml`. Request and response schemas for every operation live there. When this page and the YAML disagree, the YAML wins.
|
||||
|
||||
The route table is `src/web/routes.zig`; the [Operations](#operations) table below carries all 56 of its entries.
|
||||
The route table is `src/web/routes.zig`; the [Operations](#operations) table below carries all 60 of its entries.
|
||||
|
||||
## Conventions
|
||||
|
||||
@@ -109,6 +109,10 @@ Auth `open` means no session is required; `session` means a valid session cookie
|
||||
| GET | `/api/stats/timeseries` | session | counted | read | Bucketed counts for a period |
|
||||
| GET | `/api/lookup` | session | counted | read | Explain a domain |
|
||||
| GET | `/api/upstream/health` | session | counted | read | Upstream pool health |
|
||||
| GET | `/api/diagnostics` | session | counted | read | Operational event log |
|
||||
| DELETE | `/api/diagnostics` | session | counted | runtime action | Purge every resolved event |
|
||||
| GET | `/api/diagnostics/{id}` | session | counted | read | One operational event |
|
||||
| DELETE | `/api/diagnostics/{id}` | session | counted | runtime action | Purge one resolved event |
|
||||
| GET | `/api/groups` | session | counted | read | List groups |
|
||||
| POST | `/api/groups` | session | counted | config write | Create a group |
|
||||
| GET | `/api/groups/{id}` | session | counted | read | Read a group |
|
||||
@@ -156,6 +160,8 @@ Auth `open` means no session is required; `session` means a valid session cookie
|
||||
|
||||
There is no `POST /api/clients`: client rows come from DNS activity or import, never from the API.
|
||||
|
||||
The two diagnostics deletes purge history only. `DELETE /api/diagnostics/{id}` answers 204 for a resolved event, 409 for one that is still active — an open episode is the current state of the box, not history — and 404 for an id no row holds. `DELETE /api/diagnostics` removes every resolved event and answers `{"purged": N}`, leaving the active ones. Events still resolve on their own; these only decide when the resolved rows go.
|
||||
|
||||
Static assets are not routes. The router sends unmatched non-`/api` paths to the embedded SPA before any auth or rate-limit check.
|
||||
|
||||
## Settings keys
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
# Milestone 27: diagnostics vertical slice
|
||||
|
||||
Step 1 of the UI redesign (`specs/ui-redesign.md`). A curated log of operational failure episodes: one `operational_events` table in `config.db`, a serialized event store, fifteen typed event codes emitted at existing failure sites, `GET /api/diagnostics`, and a Diagnostics page in the admin SPA. The rest of the UI stays intact; the full navigation restructure is later steps.
|
||||
|
||||
Design authority: `specs/ui-redesign.md` §Diagnostics, as amended by the Fable review rulings recorded in that file. No `api.storage` code. No periodic probe. No new index beyond the two specced.
|
||||
|
||||
## Sessions
|
||||
|
||||
S1 (store + API) → S2 (emitters) and S3 (SPA) in parallel → orchestrator integration (contract samples regen, final wiring check).
|
||||
|
||||
---
|
||||
|
||||
## Session S1: event store, schema, API, health, metrics
|
||||
|
||||
### S1.1 Schema — `src/storage/config_schema.zig`
|
||||
|
||||
Append to `ddl_v1` (pre-v0.1: edit the baseline, no migration step; update PLAN §11.2 to match):
|
||||
|
||||
```sql
|
||||
CREATE TABLE operational_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
code TEXT NOT NULL,
|
||||
subject_key TEXT NOT NULL,
|
||||
subject_label TEXT NOT NULL,
|
||||
severity TEXT NOT NULL CHECK (severity IN ('warning', 'error')),
|
||||
first_seen INTEGER NOT NULL,
|
||||
last_seen INTEGER NOT NULL,
|
||||
occurrences INTEGER NOT NULL CHECK (occurrences > 0),
|
||||
resolved_at INTEGER,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
CHECK (resolved_at IS NULL OR resolved_at >= first_seen)
|
||||
);
|
||||
CREATE UNIQUE INDEX idx_operational_events_active
|
||||
ON operational_events(code, subject_key) WHERE resolved_at IS NULL;
|
||||
CREATE INDEX idx_operational_events_last_seen
|
||||
ON operational_events(last_seen DESC);
|
||||
```
|
||||
|
||||
`operational_events` is runtime state, NOT configuration. It stays out of `table_names` and `delete_order` (`config_schema.zig:112-129`). Tests that must be updated deliberately, not silently: `migrations.zig:365` asserts `count(tables) == delete_order.len + 1` — becomes `+ 2`, and its comment names `operational_events` as the second exclusion alongside `schema_version`. `src/config/reconcile.zig:268` (`comptime assert(delete_order.len == 10)`) is untouched — the list does not change. Add a test asserting export/import round-trips leave `operational_events` rows intact.
|
||||
|
||||
Editing `ddl_v1` changes nothing for existing installs pre-0.1 EXCEPT the silent-divergence hazard already on record: the Pi's `config.db` is stamped version 1 and will not get the new table. This milestone accepts that with a bridge placed at the END of `migrations.migrate`, after the version stamp: execute the three `operational_events` statements with `IF NOT EXISTS` added. It must NOT live in `cli.openConfigDb` — `openConfigDb` runs before migration everywhere (`app.zig:333`, `cli.zig:484`, `cli.zig:516`), and creating the table there would make a fresh database's non-`IF NOT EXISTS` `ddl_v1` `CREATE TABLE` fail. The bridge is explicit, commented as removable when the 0.1 adoption gate lands, and touches no other table. Test: a version-1 database created WITHOUT the table gains exactly it after `migrate`, and a fresh database migrates cleanly (no double-create).
|
||||
|
||||
### S1.2 Event codes — `src/storage/events.zig` (new)
|
||||
|
||||
```zig
|
||||
pub const Code = enum {
|
||||
disk_space, disk_probe,
|
||||
blocklist_refresh, blocklist_snapshot, blocklist_storage,
|
||||
certificate_reload,
|
||||
query_log_write, query_log_maintenance, query_log_recreated,
|
||||
upstream_history_write, upstream_exchange,
|
||||
client_names_storage, clients_storage,
|
||||
listener_start, configuration_load,
|
||||
};
|
||||
```
|
||||
|
||||
Fifteen enum members — that count is the single cardinality every exhaustive test, route fixture and frontend copy union uses. The enum is the truth; the wire form is the dotted string (`disk_space` → `"disk.space"`, `blocklist_refresh` → `"blocklist.refresh"`, etc.) via `pub fn wire(code: Code) []const u8` — an exhaustive switch, tested against every member. `component(code)` returns the prefix before the dot, also exhaustive.
|
||||
|
||||
Severity is fixed per emit call, not per code (a disk transition to `warn` is warning, to `critical` is error).
|
||||
|
||||
### S1.3 Event store — `src/storage/events.zig`
|
||||
|
||||
```zig
|
||||
pub const Store = struct {
|
||||
pub const max_detail_len = 512;
|
||||
pub const max_subject_key_len = 256; // longer keys become sha256: digests — see the identity rule below
|
||||
pub const max_subject_label_len = 128;
|
||||
pub const resolved_retention_s: i64 = 90 * 86_400;
|
||||
pub const max_resolved_rows: i64 = 5_000;
|
||||
|
||||
mutex: std.Io.Mutex,
|
||||
database: *db.Db, // dedicated connection; ALL access goes through the Store's mutex
|
||||
write_failed: std.atomic.Value(bool),
|
||||
write_failures: std.atomic.Value(u64), // feeds the metrics counter
|
||||
active: ActiveSet, // in-memory mirror of active (code, subject_key) rows
|
||||
untracked_active_count: u32, // active rows NOT in the mirror (overflow); loaded at init
|
||||
|
||||
pub fn init(io, database, now_s) db.Error!Store // loads ActiveSet + untracked count, prunes; failure = no store
|
||||
pub const max_kept_keys = 64; // resolveExcept bound; over it the call is refused, counted and latched — truncating the kept list would close episodes that are still true
|
||||
pub fn report(self, io, now_s, code, subject_key, subject_label, severity, detail) void
|
||||
pub fn resolve(self, io, now_s, code, subject_key) void
|
||||
pub fn reportResolved(self, io, now_s, code, subject_key, subject_label, severity, detail) void
|
||||
pub fn resolveExcept(self, io, now_s, code, kept_keys: []const []const u8) void
|
||||
pub fn prune(self, io, now_s) void
|
||||
pub fn writeFailed(self) bool
|
||||
pub fn activeCounts(self, io) struct { warnings: u32, errors: u32 }
|
||||
pub fn selectEvents(self, io, arena, filter) db.Error!EventsPage
|
||||
pub fn selectOne(self, io, arena, id: i64) db.Error!?Event
|
||||
};
|
||||
```
|
||||
|
||||
Contract:
|
||||
|
||||
- **Time is a parameter, not a stored seam.** Every mutating method takes `now_s: i64` from the caller, matching how `history.zig:132` receives `wall_s` and `logger.zig:72` receives entry timestamps. Production callers compute it from `Clock.real`; tests pass literals. No function pointer, no clock inside the store.
|
||||
- **`resolve` is hot-path safe.** It checks `active` under the mutex and, when the key is absent AND `untracked_active_count == 0`, returns without any SQLite statement. `pool.recordSuccess` calls it on every successful exchange; steady state must cost a mutex acquire and a lookup. The no-SQL guarantee is tested through a debug-only statement counter on the Store (incremented before every repo call) — NOT via `sqlite3_total_changes`, which a `SELECT` probe would not move.
|
||||
- **Overflow is exact, not heuristic.** `ActiveSet` capacity is 256. The no-SQL fast path belongs to `resolve` ONLY: mirror miss + `untracked_active_count == 0` → return, no SQL (that is the steady-state success). `report` on a mirror miss always writes — when the count is zero it inserts directly (a new episode in normal state, no probe needed); when the count is nonzero it probes and upserts (touch the untracked active row if one exists — a second insert would collide with the partial unique index — else insert), incrementing `untracked_active_count` only when a new row was inserted AND the mirror is full. A successful slow-path `resolve` decrements the count. `init` loads the count as `active rows - mirrored rows`. `events_repo.resolveExcept` performs the bulk resolve and a count of TOTAL remaining active rows in ONE transaction, returning that total only after commit — the repo knows nothing of the mirror. The Store then removes resolved keys from its mirror and sets `untracked_active_count = total_active - active.len`; on any failure it changes neither.
|
||||
- **Key canonicalization happens at every entry point.** `report`, `resolve`, `reportResolved` and each `kept_keys` element of `resolveExcept` all pass the caller's key through the same digest-if-over-length rule before any lookup or SQL. Tests cover a long-key report resolved with the same long key, and `resolveExcept` keeping a long kept key.
|
||||
- **All reads go through the Store.** The handler calls `store.selectEvents` / `store.selectOne`, which lock the same mutex around `events_repo` — nothing touches `store.database` from outside. One connection, one owner.
|
||||
- `report` upserts on the active row: present → `last_seen = now_s`, `occurrences += 1`, `detail` replaced, severity raised to the worse of the two, never lowered. Absent → insert new active row and add to `active`.
|
||||
- `resolve` on an active row sets `resolved_at = now_s` and removes it from the mirror. A later failure inserts a NEW row (new episode) — the partial unique index enforces one active row per key.
|
||||
- `reportResolved` inserts a row with `resolved_at = first_seen = last_seen = now_s`, `occurrences = 1`, and never touches `active` (one-shot events: `query_log_recreated`).
|
||||
- `resolveExcept(code, kept_keys)`: resolves every active row of `code` whose `subject_key` is not in `kept_keys`, in one serialized operation. Exists for the boot-finalized codes (S2); nothing else may use it.
|
||||
- Write failures: on `db.Error`, set `write_failed = true`, increment `write_failures`, drop the event. **Log only on the `false → true` transition** — a broken diagnostics database plus a busy pool must not produce warnings at query rate; the counter and the health surface carry the ongoing state. The next successful write clears the latch (and that recovery may log once).
|
||||
- **No error propagates to a producer** — mutating methods return `void` by design; a diagnostics failure must never break the subsystem reporting it. `init` is the exception: it returns `db.Error`, and `app.zig` responds by running with no store (`null` everywhere) and logging once — a store built on an unverified mirror would produce false no-op resolves, which is worse than no store.
|
||||
- `subject_key` identity is exact at any length: a key at or under `max_subject_key_len` is stored verbatim; a longer one (operator URLs are unbounded — `safe_url.zig:12` imposes no input limit, and `manager.zig:179`'s 255 cap bounds only a display copy) is replaced by `"sha256:" ++ hex(SHA-256(key))` — 71 bytes, deterministic, collision-free in practice, so distinct long URLs never merge and the same URL always maps to the same episode. Never truncate and never reject a key. `subject_label` truncates to `max_subject_label_len`, `detail` to `max_detail_len` — those are display fields. `subject_key` never leaves the process; `subject_label` is the redacted display identity (`safe_url.redactQuoted` where the subject is a URL).
|
||||
- `prune`: delete resolved rows older than `resolved_retention_s`, then oldest resolved rows beyond `max_resolved_rows`. Active rows are never pruned. Called at store init and once per retention pass (S2).
|
||||
|
||||
### S1.4 Repository — `src/storage/repositories/events_repo.zig` (new)
|
||||
|
||||
Free functions on `*db.Db`, matching `upstream_history_repo.zig` conventions (file-scope SQL constants, by-value row structs with fixed buffers, prose test names, `:memory:` fixtures). Functions: `insertActive`, `touchActive`, `resolveActive`, `resolveActiveByKey`, `selectActiveId`, `countActive`, `resolveExcept`, `insertResolved`, `loadActive`, `selectEvents(filter)`, `selectOne(id)`, `pruneResolved`, `activeCounts` (the three beyond the original list serve the exact-overflow contract). Only the `Store` calls these in production. `selectEvents` filter: `state` (active/resolved/all), `severity`, `component` (matched on `code` prefix), `since`/`until` with the repo's `[since, until)` convention: an empty range (`since >= until`) returns nothing, and overlap is `first_seen < until AND (resolved_at IS NULL OR resolved_at > since)` — strict `>`, an episode resolved exactly at `since` does not overlap (matches `queries_repo.zig:211`). `limit`, `before` (keyset on id descending).
|
||||
|
||||
### S1.5 API — `src/web/handlers/diagnostics.zig` (new)
|
||||
|
||||
`GET /api/diagnostics` — params `state` (`active`|`resolved`|`all`, default `all`), `severity` (`warning`|`error`), `component`, `since`, `until`, `limit` (1..1000, default 100), `before` (positive id). 400 with a naming message on any bad param (copy `queries.zig` conventions: `parseFilter` + `message(err)`). 503 when the store is absent. Response:
|
||||
|
||||
```json
|
||||
{ "events": [ { "id": 42, "code": "blocklist.refresh", "component": "blocklist",
|
||||
"subject": "StevenBlack", "severity": "warning",
|
||||
"first_seen": 1787118000, "last_seen": 1787118300, "occurrences": 3,
|
||||
"resolved_at": null, "detail": "download failed: ConnectionTimedOut" } ],
|
||||
"next_before": null,
|
||||
"active": { "warnings": 1, "errors": 0 } }
|
||||
```
|
||||
|
||||
`subject` serializes `subject_label`. `subject_key` has no wire form — assert that in a test. Pagination contract identical to `/api/queries` (full page carries cursor, short page nulls it).
|
||||
|
||||
`GET /api/diagnostics/{id}` — the same event object, 404 after retention or for an unknown id.
|
||||
|
||||
Routes: two entries in `routes.zig`, `.auth = .session`, `.policy = .read`. Update the pinned route-count test (`routes.zig:145`, 56 → 58) and every routing invariant test that enumerates. OpenAPI: paths + `DiagnosticsPage` / `DiagnosticEvent` schemas, following the `/api/queries` exemplar; the openapi drift tests must stay green.
|
||||
|
||||
### S1.6 Health — `src/web/handlers/health.zig`
|
||||
|
||||
`Input` gains `diagnostics_present: bool = true` (benign default, matching the all-defaulted convention), `diagnostics_write_failed: bool = false`, `diagnostics_active_warnings: u32 = 0`, `diagnostics_active_errors: u32 = 0`. Body gains `"diagnostics": { "state": "recording"|"unavailable", "active_warnings": N, "active_errors": N }` — `unavailable` when `!present` or `write_failed`. `degraded` adds `diagnostics_write_failed or !diagnostics_present`: in a serving process the store is absent only when `Store.init` failed, which is a real degradation, and the health endpoint never runs in subcommands. `collect` must assign `diagnostics_present = state.events != null` explicitly — the natural `if (state.events) |store|` shape would leave an absent store reported as recording under the benign default. **This milestone does NOT yet remove `history_flush_failing` or restructure health to the full redesign shape** — that lands with step 4 (Overview replacement); here health only gains the diagnostics block. Update the degraded-matrix test to cover both new inputs.
|
||||
|
||||
### S1.7 Metrics — `src/web/metrics.zig`
|
||||
|
||||
Gauges `nxdns_diagnostics_active_warnings`, `nxdns_diagnostics_active_errors`; counter `nxdns_diagnostics_write_failures_total` (increment in the store on each failed write). Follow the upstream-history block pattern.
|
||||
|
||||
### S1.8 WebState — `src/web/server.zig`
|
||||
|
||||
`events: ?*events.Store = null`.
|
||||
|
||||
### S1.9 Acceptance criteria (S1)
|
||||
|
||||
- [ ] `zig build test` green; new store tests cover: episode open/dedupe/severity-raise, resolve-then-new-episode, one-shot insert, no-op resolve executes no SQL, write-failure latch set and cleared (log only on transition), label/detail truncation, over-length subject_key digested to the same identity twice, prune retention + cap, overflow slow paths for report (upsert, no unique-index collision) and resolve (count decrements), resolveExcept keeping mirror and count exact.
|
||||
- [ ] `events_repo` tests cover every function against `:memory:` including overlap-window selection and keyset pagination.
|
||||
- [ ] Handler tests: each param rejected with 400 + naming message, 503 without store, pagination cursor behavior, active counts, 404 on `{id}`.
|
||||
- [ ] Route-pin, openapi-drift, health-matrix, schema-count tests all updated intentionally and green.
|
||||
- [ ] Export/import round-trip leaves `operational_events` intact (test).
|
||||
|
||||
---
|
||||
|
||||
## Session S2: emitters (after S1)
|
||||
|
||||
Thread the store as `?*events.Store` using the established `gate: ?*disk_monitor.Monitor` idiom (`app.zig:776`): per-call parameter for run-loops, optional post-init field for `Manager`/`Pool`/`CertStore`. Every subsystem must build and test with `null` (no store). app.zig opens the dedicated connection right after migration (`app.zig:335`), unconditionally — not gated on `cfg.web.enabled`; diagnostics record whether or not the UI is on:
|
||||
|
||||
```zig
|
||||
var events_db = try data.openConfigDb(io);
|
||||
defer events_db.close();
|
||||
const boot_now = std.Io.Clock.real.now(io).toSeconds();
|
||||
var event_store_storage: ?events.Store = events.Store.init(io, &events_db, boot_now) catch |err| blk: {
|
||||
log.warn("diagnostics store unavailable: {s}", .{@errorName(err)});
|
||||
break :blk null;
|
||||
};
|
||||
const event_store: ?*events.Store = if (event_store_storage) |*s| s else null;
|
||||
```
|
||||
|
||||
`event_store` is what gets threaded; a failed init leaves it null and health reports `unavailable` + degraded.
|
||||
|
||||
Emit sites, from the verified survey. Each row: failure emit, recovery resolve, subject_key / subject_label, severity.
|
||||
|
||||
| Code | Failure (file:line today) | Recovery | subject_key → label | Severity |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `disk.space` | `disk_monitor.zig:137-147` transition to warn/critical | same `publish`, transition to ok | `"data"` singleton | warn→warning, critical→error |
|
||||
| `disk.probe` | `disk_monitor.zig:94-97` (change `catch {}` → `catch \|err\|`), `:101-105`, `:109-113` | next successful branch of the same probe | operation name (`statvfs`/`data_dir`/`log_dir`) | warning |
|
||||
| `blocklist.refresh` | `manager.zig:1102-1138` (fetch/compile/empty), `:1520-1522` (load) | `SourceStatus.succeed` paths `:840`, `:870`, `:1552` | source URL → source name | warning |
|
||||
| `blocklist.snapshot` | `app.zig:572-577`, `manager.zig:1164-1167`, `:1184-1187` | `manager.zig:539-544` (post-swap) | singleton | error |
|
||||
| `blocklist.storage` | `manager.zig:1200-1203`, `:1336`, `:1451`, `:1458`, `:1470` | **the success branch of each matching operation, NOT pass-end** — `deleteQuietly` absorbs failures at `manager.zig:1466` and `pruneOrphans` still returns success at `:1350`, so a pass-end resolve would close the very event its own pass emitted | operation name | warning |
|
||||
| `certificate.reload` | `cert_store.zig:256-262` (capture the discarded err), `:273-279` | `:271-273` success reload | endpoint kind — **`CertStore` gains a `kind: enum { doh, dot }` field set at `openCertStore` (`app.zig:611/:617`)** | warning — a stat failure can be a transient rename window and a failed reload keeps the loaded certificate serving (`cert_store.zig:250`); nothing is down |
|
||||
| `query_log.write` | `logger.zig:252-260` (init, from the failing `BatchWriter.init` and its catch), `:400-404` (batch) | `:405` successful batch. Init failure has no recovery (writer exits) — the event stays active, which is the truth | `writer`/`batch` | error |
|
||||
| `query_log.maintenance` | `retention.zig:113`, `:127`, `:133`, `:143`, `:151` | matching success branch same pass | operation name | warning |
|
||||
| `query_log.recreated` | `querylog_schema.open` result. **Two changes: `OpenResult` gains the aside name (fixed `[path_buf_len]u8` + len — today it exists only in a stack buffer inside `open`, `querylog_schema.zig:137`), and `cli.openQuerylogDb` (`cli.zig:335`) stops discarding `recreated` and returns it to `app.zig`**, which calls `reportResolved` once the store exists. **Never emitted for `.missing`** — first creation has no aside file and is logged as "created" (`querylog_schema.zig:131`); a fresh install must not record a warning with an impossible aside path | one-shot | reason tag; detail carries the aside filename | warning |
|
||||
| `upstream_history.write` | `history.zig:262-266` | `:257-261` | singleton | warning |
|
||||
| `upstream.exchange` | `pool.zig:332-346` `recordFailure` — emit OUTSIDE the pool mutex, same placement discipline as `recordHistory` | `pool.zig:316-330` `recordSuccess` → `resolve` (no-op hot path) | upstream URL → `safe_url.redactQuoted` | warning |
|
||||
| `client_names.storage` | `client_names.zig:125-131`, `:141-147` | clean-pass determination `client_names.zig:161-164` | `read`/`write` | warning |
|
||||
| `clients.storage` | `clients.zig:196-202`, `:218-224` | success branches `:195-197`, `:214-217` | `materialise`/`prune` | warning |
|
||||
| `listener.start` | `app.zig:896/:900` (DoH), `:915/:919` (DoT) | boot-finalized: after the bind phase, one `resolveExcept(.listener_start, failed_keys)` closes prior episodes for endpoints that started clean this boot — including endpoints now disabled | `doh`/`dot` | error |
|
||||
| `configuration.load` | `app.zig:686`, `:1039-1045`, `:1056-1061` and managed-file `validate.Diagnostics` warnings rendered at `app.zig:204` | boot-finalized: after all boot findings are emitted, one `resolveExcept(.configuration_load, emitted_keys)` | setting/upstream identity (redacted) | warning |
|
||||
|
||||
Rules that bind every site:
|
||||
|
||||
- The existing `log.warn`/`log.err` lines STAY. Events are additive; journald keeps the raw stream.
|
||||
- Failure text passed as `detail` is `@errorName(err)` plus the site's existing message fragment — no new prose invented, no allocation: format into a stack buffer of `Store.max_detail_len`.
|
||||
- **Lock discipline is collect-then-flush, not "emit after the mutex closes".** Several manager sites cannot simply move: `publishRefresh` runs under `writer_lock` by contract (`manager.zig:792`) and `pruneOrphans` holds both `refresh_lock` and `writer_lock` through its filesystem work (`manager.zig:1299`). At such sites, record outcomes while the locks are held and flush to the store after the outer locked operation returns. The collection must not lose events: per-source outcomes ride the manager's existing per-source status allocations (one outcome per source, bounded by the source count), and same-operation failures within one pass aggregate into a single `report` per pass (the row's `occurrences` then counts failing passes, and `detail` carries the last error plus how many failures that pass held) — never an unbounded list, never a silent drop. Pool follows the existing `recordHistory` placement.
|
||||
- Boot-finalized codes (`listener.start`, `configuration.load`) use `resolveExcept` exactly once each, after their boot phase completes; restart is the recovery, matching the design table.
|
||||
- `retention.zig` `runOnce` gains `events: ?*events.Store` and calls `store.prune` once per pass.
|
||||
|
||||
### S2 acceptance criteria
|
||||
|
||||
Emitters fall into three classes, each with its own test obligation (the author watches each fail first, repo ruling F-f):
|
||||
|
||||
- **Episodic** (disk, blocklist ×3, certificate, query_log.write batch, query_log.maintenance, upstream_history.write, upstream.exchange, client_names, clients): force the failure (existing seams: injected db errors, missing files, dead fetch server), assert the `(code, subject_key, severity)` row; then force recovery and assert resolution.
|
||||
- **Boot-finalized** (`listener.start`, `configuration.load`): assert the emit on a failing boot, and assert `resolveExcept` closes a pre-seeded stale episode on a clean boot.
|
||||
- **One-shot / permanent** (`query_log.recreated`; `query_log.write` init): assert the one-shot row inserts already resolved; assert the init-failure episode exists and that no recovery path claims it.
|
||||
- [ ] `pool` exchange tests: success with no active episode performs no store SQL; failure→success round-trip produces exactly one resolved episode with correct occurrences.
|
||||
- [ ] All subsystems still pass with `events = null` (the existing suites running unchanged).
|
||||
|
||||
---
|
||||
|
||||
## Session S3: admin SPA (after S1, parallel with S2)
|
||||
|
||||
- `lib/types.ts`: `DiagnosticEvent`, `DiagnosticsPage`, plus the health body's new `diagnostics` block.
|
||||
- `lib/api.ts`: `getDiagnostics(filter)`, `getDiagnostic(id)`.
|
||||
- `lib/queries.ts`: `diagnosticsQuery` (infinite, keyset via `next_before`, `keepPreviousData`), `diagnosticQuery(id)`; refetch interval matching healthQuery's cadence for the active view.
|
||||
- `routes.tsx`: `/diagnostics` route (loader + lazy component) and `/diagnostics/$id`. Nav: add `Diagnostics` to `NAV_ITEMS` between Lookup and Settings (full nav restructure is later milestones); update `AppShell.test.tsx`.
|
||||
- `features/diagnostics/DiagnosticsPage.tsx`: active episodes first (severity, title from an exhaustive code→copy map, subject, "active for …" age, occurrences), then resolved history with the standard filters (state, severity, component) reflected in the URL search params. **Empty state is the healthy state**: one line, "No active operational issues.", quiet styling — it must read as good news, not as a broken page (Fable's caution).
|
||||
- `features/diagnostics/DiagnosticDetailPage.tsx`: the ordered detail from the design — state, first/last seen, occurrences, resolution, impact and remediation text from the same exhaustive code map, last error detail, and links (the config surfaces linked are today's routes; they get re-pointed when later milestones move them).
|
||||
- The code→copy map lives in `features/diagnostics/eventCopy.ts`: an exhaustive `Record<Code, {title, impact, remediation, link?}>` over a string-literal union of the fifteen wire codes; a test iterates the union and asserts every member has copy.
|
||||
- Tests: page renders fixtures (active + resolved + empty), filters drive the URL, detail renders every code's copy, pagination fetches more.
|
||||
|
||||
### S3 acceptance criteria
|
||||
|
||||
- [ ] `npm run typecheck`, `npx vitest run`, `npx prettier --check`, lint, build, assert-bundled all green; byte budget respected.
|
||||
|
||||
---
|
||||
|
||||
## Orchestrator integration (after S2+S3)
|
||||
|
||||
- Regenerate `contractSamples.gen.ts` (`zig build test -Dintegration -Dcontract-samples-out=…`) — requires the integration capture in `web_integration_test.zig` to seed at least one active and one resolved event.
|
||||
- Update `docs/reference/api.md` and any drift-guarded reference pages; changelog entry under Unreleased.
|
||||
- Live smoke on the real binary before anything ships: force a blocklist failure (dead URL), watch the episode appear, fix the URL, watch it resolve; screenshot the page per standing rule.
|
||||
|
||||
## Module layout (new files)
|
||||
|
||||
| File | Purpose |
|
||||
| --- | --- |
|
||||
| `src/storage/events.zig` | `Code`, wire mapping, `Store` |
|
||||
| `src/storage/repositories/events_repo.zig` | SQL |
|
||||
| `src/web/handlers/diagnostics.zig` | list + detail handlers |
|
||||
| `admin/src/features/diagnostics/…` | page, detail, copy map, tests |
|
||||
|
||||
## File ownership
|
||||
|
||||
S1: `config_schema.zig`, `events.zig`, `events_repo.zig`, `diagnostics.zig` (handler), `routes.zig`, `openapi.yaml`, `health.zig`, `metrics.zig`, `server.zig`, `migrations.zig` (bridge + test), PLAN §11.2. S2: every emitter file + `app.zig` + `retention.zig` + `cert_store.zig` + `cli.zig` (openQuerylogDb) + `querylog_schema.zig` (OpenResult aside name) and its tests — S2 starts after S1 lands, so nothing is shared in parallel. S3: `admin/` only. No parallel writers on any file (S2 and S3 run concurrently and are disjoint).
|
||||
|
||||
## Anti-requirements
|
||||
|
||||
- No `info` severity, no acknowledgement/dismissal state, no manual-resolve endpoint, no raw-log endpoint, no generic remediation action schema.
|
||||
- No `api.storage` code. No periodic probe for `write_failed`. No configurable retention.
|
||||
- No changes to the existing dashboard, query log, live, or lookup pages.
|
||||
- No new dependency, front or back.
|
||||
|
||||
## Acceptance criteria (milestone complete)
|
||||
|
||||
- [ ] All session criteria; full `zig build test` + `-Dintegration`, admin suite, byte budgets.
|
||||
- [ ] Live smoke: forced failure → visible episode → recovery → resolved row; `/api/health` shows the diagnostics block; `/metrics` shows the three series.
|
||||
- [ ] Screenshots of the page (active, resolved, empty states) shown before push.
|
||||
|
||||
## Recorded (as built)
|
||||
|
||||
Deviations from the sections above, found during build, review and live smoke. The code is the authority; this section says where it moved.
|
||||
|
||||
- `src/storage/events_fixture.zig` exists (test-only): a shared migrated-config.db + Store fixture the storage and filter tests use.
|
||||
- The threading fields through `app.zig`/`server.zig` are named `.diagnostics`; `WebState` carries `.events`. The live smoke caught that `.events` was never assigned in `app.zig` while every suite stayed green — the integration tests build their own `WebState`. One line fixed it; the lesson is the standing "verify against the real network" rule.
|
||||
- `runScheduler`'s loop body is the pub function `scheduledPass`, so tests drive one pass deterministically. `runScheduler` flushes only in its catch branch (for the snapshot note its failure path adds).
|
||||
- Occurrences count failing passes, not flushes: `SourceStatus` carries `pass_outcome`/`pass_failures`, set in `fail()`/`succeed()`, drained by `flushSourceDiagnostics`. Detail format: `"{state}: {error} ({N} this pass)"`.
|
||||
- Pass accounting lives in exactly one table copy at a time: `mergeStatuses` zeroes the pass fields on carried entries, `installStatuses` folds the live table's unflushed accounting in by id under the exclusive lock, and the flush drains by claiming one outcome-bearing entry per lock round (immune to concurrent table replacement).
|
||||
- A pass flushes before it releases the lock that serializes it: the flush defer registers after the unlock defer in `refreshSource`, `refreshAll`, `startupPass`; standalone `reload` takes `writer_lock` itself and flushes inside it. This amends the collect-then-flush rule — the store runs on its own connection and mutex, so no shared resource is held across the store call.
|
||||
- Deleted sources: `flushSourceDiagnostics` ends in `resolveDeletedSources` — `resolveExcept(.blocklist_refresh, all current keys)`, gated on `generation != 0` (an empty pre-reload table means "not read yet", not "all deleted") and skipped above 64 keyed entries (`max_kept_keys`; the episode then lingers until the count drops). `Store.resolveExcept` is no longer boot-finalized-only; the rule is the kept list must be the whole current subject set.
|
||||
- `events_repo.resolveExcept` is mark-then-unmark (sentinel `maxInt(i64)`, one transaction): the prior resolve-all-then-revive matched the revive on `resolved_at = now_s` and revived an episode resolved by the drain in the same second.
|
||||
|
||||
### Addendum: manual purge of resolved events
|
||||
|
||||
Resolution stays automatic; the operator decides when resolved history disappears. Two endpoints, both mutations under the usual auth, both allowed in file mode (diagnostics are runtime state, not configuration):
|
||||
|
||||
- `DELETE /api/diagnostics/{id}` — purges one resolved event. 409 `{ "error": ... }` when the event is active; 404 when no row has that id.
|
||||
- `DELETE /api/diagnostics` — purges every resolved event, returns `{ "purged": N }`.
|
||||
|
||||
Store: purge functions under the store mutex; they touch only rows with `resolved_at NOT NULL`, so the ActiveSet mirror and untracked count never change. UI: a purge action on each resolved row and on the resolved detail page, plus a "Purge all resolved" control on the list when at least one resolved event shows; active events show no purge affordance. OpenAPI, routes count, contract goldens, api.md updated.
|
||||
|
||||
Accepted limitations (reviewed with Codex, ruled in proportion to household scale; each is bounded and self-correcting, counts stay visible in the detail text):
|
||||
|
||||
- A standalone web reload interleaving with a refresh pass can merge two passes' accounting into one occurrence.
|
||||
- `pruneOrphans` flushes its storage aggregates with the writer locks released; two concurrent prunes can merge into one report.
|
||||
- The deletion sweep's kept-key snapshot can go stale against a concurrently added source: its fresh episode can be resolved once and reopens on the next failing pass with `first_seen`/`occurrences` reset.
|
||||
@@ -0,0 +1,304 @@
|
||||
# UI redesign proposal
|
||||
|
||||
Author: Codex (gpt-5.6-sol, extra-high effort), 2026-08-19. **Not accepted yet.** Untracked on purpose until Mokhtar rules on the open questions at the end.
|
||||
|
||||
Answers that shaped it: the server and API may change; the surface-ownership split is right; file-mode configuration pages are read-only; diagnostics are curated structured events in the vein of Pi-hole's; time scoping is per workflow; a past query must be explainable exactly; Query Log and Live merge.
|
||||
|
||||
## Navigation
|
||||
|
||||
Five primary items. Configuration expands to three task-shaped subpages and holds no landing route of its own.
|
||||
|
||||
| Navigation | Route | Operator question | Replaces |
|
||||
| --- | --- | --- | --- |
|
||||
| Overview | `/overview` | Is DNS healthy and protecting the household now, and what happened in this period? | Dashboard |
|
||||
| Activity | `/activity` | What requests are happening or happened, and why did nxdns handle them that way? | Query Log, Live, Lookup |
|
||||
| Clients | `/clients` | Who is this address, which policy applies, and what has it been querying? | Clients |
|
||||
| Diagnostics | `/diagnostics` | What is failing or has failed, what is affected, what should I do? | new |
|
||||
| Protection | `/configuration/protection` | What policy governs each group, and which rules and lists produce it? | Groups, Blocklists, Rules |
|
||||
| Resolution | `/configuration/resolution` | Where does nxdns answer or forward permitted names? | Local DNS, Upstreams |
|
||||
| System | `/configuration/system` | What service, storage, logging, TLS and web settings is this process running with? | Settings |
|
||||
|
||||
Secondary routes, reached from those surfaces rather than the nav: `/activity/queries/:id`, `/activity/test`, `/clients/:id`, `/diagnostics/:id`.
|
||||
|
||||
No current page survives unchanged. Login, logout and pause survive functionally, restyled into the new shell.
|
||||
|
||||
### Activity
|
||||
|
||||
Two modes over the same columns and filters. History is persisted queries with keyset pagination and an absolute range. Live is follow-by-default with Freeze/Resume over the existing bounded 500-row buffer.
|
||||
|
||||
Columns: Time, Domain, Client, Type, Result, Route, Duration. Rule matches, source URLs and upstream errors live in the detail view, never on every row.
|
||||
|
||||
Domain testing stays as an Activity action labelled "Current policy simulation". It must never read as an explanation of a historical query.
|
||||
|
||||
### Protection
|
||||
|
||||
Group-centred: group list, selected group detail, effective safe-search setting, assigned blocklist sources, rules scoped to the group, client count linking to matching clients. A Sources tab holds the shared blocklist catalogue and the "Update now" runtime action.
|
||||
|
||||
### Resolution
|
||||
|
||||
Three tabs: upstream pool, local records, forward zones.
|
||||
|
||||
### Clients
|
||||
|
||||
Keeps primary navigation because identifying and naming unknown devices is an operational job, not configuration. The *learned* marker appears only here, beside the name. Prefix assignments live here as "Network assignments".
|
||||
|
||||
## Overview
|
||||
|
||||
Three sections, nothing else.
|
||||
|
||||
**1. Current status.** Five current facts, each conveyed by text and icon as well as colour. Healthy rows stay quiet; degraded rows link to the diagnostic or configuration surface that explains them.
|
||||
|
||||
| Status | Shows | Why it belongs |
|
||||
| --- | --- | --- |
|
||||
| Protection | Active, paused until a timestamp, or unavailable, with Pause/Resume | Confirms filtering is in force, and carries the valid runtime action |
|
||||
| Upstreams | available / configured, now | Confirms DNS can leave the network |
|
||||
| Query history | Recording, losing rows, or writer failed | Says whether Activity can be trusted |
|
||||
| Diagnostics | Recording or unavailable | A failure reporter that cannot record failures must itself be visible |
|
||||
| Storage | ok / low / critical, free bytes | Says whether writes are safe, and explains write gating |
|
||||
|
||||
The shell carries a small global "Protection active/paused" indicator linking back to Overview. The controls themselves stay on Overview and beside blocked-query details.
|
||||
|
||||
**2. Active issues.** Severity, short title, affected object, how long it has been active, link to the detail. When none exist, one restrained line: "No active operational issues." Resolved failures never appear here, and healthy subsystems never get permanent green cards.
|
||||
|
||||
**3. Activity over a period.** The existing 1h / 24h / 7d / 30d control. Every value uses exactly the returned `[since, until)` window: queries, blocked count and rate, distinct clients, average response time, one query-volume timeline split blocked/cached/other, and "Open activity for this period" carrying the exact bounds. The timeline stays the existing lightweight SVG; no charting dependency.
|
||||
|
||||
If the selected period predates available data, the section says "Query history is available from …" rather than charting the missing span as zero.
|
||||
|
||||
Removed from Overview: the historical upstream table, the "last failure" text, per-upstream period rates, the database and log byte breakdown, the standalone cache card.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Not a journald viewer, and it does not subscribe to `std.log`. Producers emit a finite set of typed events at the failure boundary.
|
||||
|
||||
### Event model
|
||||
|
||||
One row is one failure episode.
|
||||
|
||||
| Field | Meaning |
|
||||
| --- | --- |
|
||||
| `id` | Durable identifier |
|
||||
| `code` | Fixed machine-readable kind |
|
||||
| `subject_key` | Internal stable identity, may hold a full URL, never serialized |
|
||||
| `subject_label` | Bounded, redacted, operator-facing identity |
|
||||
| `severity` | `warning` or `error` |
|
||||
| `first_seen` / `last_seen` | Episode bounds |
|
||||
| `occurrences` | Deduplicated report count |
|
||||
| `resolved_at` | Null while active |
|
||||
| `detail` | Bounded last error or current condition |
|
||||
|
||||
No `info` severity. Normal starts, refreshes and reloads do not become entries; a success resolves its prior failure.
|
||||
|
||||
Active dedup key is `(code, subject_key)`. A repeat updates `last_seen`, `occurrences`, severity and detail. A success resolves the row. A later failure opens a new episode rather than reopening the old one. Severity records the worst state reached. There is no acknowledgement or manual dismissal: active means the component has not demonstrated recovery. One-shot material events, such as a query-log recreation, are inserted already resolved.
|
||||
|
||||
### Storage
|
||||
|
||||
Events live in `config.db` as runtime state, excluded from export, import and file reconciliation. Active rows are never pruned. Resolved rows keep 90 days, with a hard cap of the newest 5,000. `detail` caps at 512 bytes. Pruning runs at startup and from existing maintenance; no new scheduler.
|
||||
|
||||
One table, one repository, one fixed event-code enum. A small serialized event store owns a dedicated `config.db` connection; background producers report synchronously through its mutex. This is the existing pattern, not a new one: `app.zig:333` opens a connection for migration and reconciliation and `app.zig:544` opens a separate `web_config_db`, and the background producers (fetcher, disk monitor, logger writer) have no other safe path into `config.db`.
|
||||
|
||||
If the store itself cannot write, an atomic `event_store_failed` state appears in `/api/health` and journald. It clears on the next successful write — no periodic probe. The store receives a write whenever anything fails or recovers, and a flag left set while nothing needs writing costs nothing.
|
||||
|
||||
### Event sources
|
||||
|
||||
| Source | Event identity | Recovery |
|
||||
| --- | --- | --- |
|
||||
| Disk warn/critical transitions | `disk.space`, singleton | next `ok` sample |
|
||||
| Failed `statvfs` or directory sizing | `disk.probe`, keyed by operation/path | next successful probe |
|
||||
| Per-source download, HTTP, parse, compile or file-read failure | `blocklist.refresh`, keyed by source URL | that source refreshes |
|
||||
| Initial snapshot or whole-pass failure | `blocklist.snapshot`, singleton | a snapshot publishes |
|
||||
| Blocklist file cleanup failure | `blocklist.storage`, keyed by operation | that operation succeeds |
|
||||
| Certificate stat or reload failure | `certificate.reload`, keyed by `doh`/`dot` | files readable and reload succeeds |
|
||||
| Query writer init or batch failure | `query_log.write`, keyed by `writer`/`batch`/`queue` | writer starts, or a batch succeeds without drops |
|
||||
| Query retention prune/checkpoint/vacuum failure | `query_log.maintenance`, keyed by operation | that operation succeeds |
|
||||
| Upstream-history flush failure | `upstream_history.write`, singleton | next flush succeeds |
|
||||
| Client-name selection or persistence failure | `client_names.storage`, keyed by operation | next pass succeeds |
|
||||
| Client materialisation or pruning failure | `clients.storage`, keyed by operation | next pass succeeds |
|
||||
| Upstream exchange failure | `upstream.exchange`, keyed by upstream URL | next successful exchange |
|
||||
| Enabled DoH/DoT listener that cannot start | `listener.start`, keyed by endpoint | successful start after restart |
|
||||
| Query-log recreation | `query_log.recreated`, one-shot | inserted resolved |
|
||||
| Configuration warning leaving a capability skipped | `configuration.load`, keyed by setting | clean load after restart |
|
||||
|
||||
Sixteen codes. Codex proposed a seventeenth, `api.storage`, for a storage failure that produced an HTTP 500 — cut on review, because it breaks this section's own exclusion rule: a 500 already answered its caller. Do not merge the remaining codes to shrink the count either. A merged code forces `subject_key` to carry what the code no longer says.
|
||||
|
||||
An upstream event describes a consecutive failure episode, not one row per retry. One timeout followed by success is one resolved episode.
|
||||
|
||||
Outside Diagnostics: invalid requests, conflicts, rate limits and failed logins already answered to their caller; expected reverse-DNS outcomes; individual TLS handshake failures already counted; development asset-server warnings; fatal startup failures that stop the UI existing. Disk-gated skips do not duplicate — the active disk event explains the cause and counters keep the totals.
|
||||
|
||||
### From event to remediation
|
||||
|
||||
The detail shows current or resolved state; first seen, last seen, occurrences, resolution time; impact in operator language; the last bounded underlying error; the exact next action; how nxdns will verify recovery; links to the relevant configuration and activity window.
|
||||
|
||||
The frontend uses one exhaustive `switch` over the fixed codes for titles, impact, remediation and routes. No generic action schema, no plugin mechanism.
|
||||
|
||||
## Query provenance
|
||||
|
||||
Stays in the expendable `querylog.db`. Does not go into the operational-events table.
|
||||
|
||||
`block_reason` is replaced by the fuller `policy_reason`. New columns:
|
||||
|
||||
| Column | Type | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `qclass` | INTEGER NOT NULL | explains filtering bypass for non-IN questions |
|
||||
| `rcode` | INTEGER NOT NULL | the client-visible result, including SERVFAIL |
|
||||
| `group_id` | INTEGER | group at query time, not a foreign key |
|
||||
| `group_name` | TEXT | historical label, survives a rename |
|
||||
| `policy_action` | TEXT NOT NULL | `not_evaluated`, `allow`, `block` |
|
||||
| `policy_reason` | TEXT NOT NULL | pipeline or matcher reason |
|
||||
| `matched` | TEXT | exact rule pattern or list entry |
|
||||
| `source_id` | INTEGER | blocklist source at query time |
|
||||
| `source_name` | TEXT | historical source label |
|
||||
| `cname_target` | TEXT | target that caused an uncloaked block |
|
||||
| `safe_search_target` | TEXT | name used for the rewrite |
|
||||
| `route_kind` | TEXT NOT NULL | `blocked`, `local`, `forward_zone`, `upstream` |
|
||||
| `forward_zone` | TEXT | exact matched zone |
|
||||
|
||||
`upstream` changes from the unhelpful `"pool"` marker to the actual configured upstream or forward resolver **on the exchange that actually happened**. On a cache hit `route_kind` is `cache` and `upstream` is NULL. Credentials are redacted at the serialization boundary.
|
||||
|
||||
Codex wanted the upstream recorded on cache hits too. Cut on review: it would widen every `src/cache/dns_cache.zig` entry to carry an upstream label, and it states a half-truth, because on a cache hit no upstream answered. The interface change it does need is real and worth doing — `handler.zig:79-83` records ruling 20, which deliberately keeps the pool's answering endpoint out of `transport.Client`'s reach. Exposing it touches `src/upstream/`, not the pure core. Do not fall back to `"pool"`: the exact upstream on a SERVFAIL row is the single most useful correlation this redesign adds.
|
||||
|
||||
`policy_reason` is a closed enum: `local_record`, `forward_zone`, `non_in_class`, `paused`, `snapshot_unavailable`, `no_match`, plus the existing rule allow/block and blocklist exception/domain/wildcard reasons.
|
||||
|
||||
For a CNAME-uncloaked block, the policy fields describe the target's decision and `cname_target` preserves the target responsible.
|
||||
|
||||
Every syntactically parsed request that receives a response is logged, including synthesized SERVFAIL. Requests too malformed to identify a question stay counters, not fabricated rows.
|
||||
|
||||
No response payloads, answer RR sets, EDNS data or packet bytes are stored. The record explains nxdns's own decision, not the external resolver's answer.
|
||||
|
||||
Privacy transforms apply to every new domain-bearing field, not only `domain`: with `hide_domains` on, matched names, CNAME targets and safe-search targets hide consistently.
|
||||
|
||||
A one-row `querylog_meta (created_at INTEGER NOT NULL)` table lets the stats and query APIs return a conservative `available_since`, which distinguishes "zero queries" from "history does not exist".
|
||||
|
||||
### Historical query detail
|
||||
|
||||
Ordered explanation: request (time, domain, client, type, class); group (historical id and name); policy (evaluated or not, allow/block, exact matched rule or list candidate, historical source, whether filtering was paused or unavailable); rewrites (safe-search target, CNAME target); route (local, forward zone, cache, or selected upstream); response (rcode, duration); related actions (test the domain against current policy, view the client, view activity for the same domain or client, view diagnostics in a five-minute window around the query).
|
||||
|
||||
Historical and current facts are visually separated. A rule, source or group that no longer exists stays visible as a historical value and is not linked to a different current object.
|
||||
|
||||
Live SSE events carry the same provenance shape without a persisted `id`. A frozen live row shows its in-memory detail; no correlation id is invented to link it to a row SQLite has not written.
|
||||
|
||||
## Time scoping
|
||||
|
||||
One contract: unix seconds UTC, `since` inclusive, `until` exclusive, point data qualifies on `since <= ts < until`, diagnostic episodes qualify when their active interval overlaps the range, current state is labelled "Now" and no historical selector touches it.
|
||||
|
||||
URLs: `/overview?period=24h` with the server returning the exact aligned bounds; `/activity?mode=history&since=…&until=…` with `domain`, `client`, `blocked` and the other filters in the URL; `/diagnostics?since=…&until=…&severity=…&component=…`; `/activity?mode=live` with Follow/Freeze as ephemeral UI state. Investigation links always carry absolute bounds, so a viewed incident does not drift as time passes. Timestamps display in the browser's timezone; URLs and APIs stay timezone-independent.
|
||||
|
||||
## File mode
|
||||
|
||||
One persistent authority line in the shell:
|
||||
|
||||
> File-managed · `/etc/nxdns/config.zon` · loaded 19 Aug 2026, 08:42
|
||||
|
||||
It sits in the Configuration sub-navigation and appears elsewhere as a compact lock indicator. No full-width banner on every route. The wording is "running configuration loaded from", not "file contents" — the server cannot prove a since-edited file still matches the running process.
|
||||
|
||||
A file-managed page uses definition lists for scalars and tables or cards for collections, with human labels and the exact ZON key shown secondarily (`logging.retention_days`). No text inputs, no checkboxes, no Add/Edit/Delete, no disabled form shells, no simulated Save. A short page note says where edits happen and that a restart may be needed.
|
||||
|
||||
Runtime actions stay ordinary enabled buttons: pause/resume, update blocklists now, reload certificates, delete an observed undeclared client, login/logout.
|
||||
|
||||
Database mode uses the same information architecture with real edit actions, plus a server-owned `restart_pending` boolean. The current client-only restart banner and its local store are removed, so a browser refresh cannot erase the warning.
|
||||
|
||||
## API changes
|
||||
|
||||
`GET /api/config/status` → `{authority, path, reconciled_at, restart_pending}`. `restart_pending` is process state: database-mode mutations that need a restart set it, a successful restart clears it.
|
||||
|
||||
`GET /api/health` becomes explicit about every condition that contributes to degradation — `protection`, `upstreams`, `query_history`, `upstream_history`, `diagnostics`, `disk`, each an object with its own state. The current hidden `history_flush_failing` contribution is eliminated: nothing may degrade the rollup without appearing in the response.
|
||||
|
||||
`GET /api/diagnostics?state=&severity=&component=&since=&until=&limit=&before=` returns `{events[], next_before, active:{warnings, errors}}`. `GET /api/diagnostics/{id}` returns one event or 404 after retention. No acknowledgement, dismissal, generic-action or raw-log endpoints.
|
||||
|
||||
`GET /api/queries` keeps keyset pagination and its filters; rows gain `rcode`, `route_kind`, `policy_action` and the short policy reason the table needs, and the body gains `coverage: {complete, available_since}`. `GET /api/queries/{id}` returns nested `request` / `policy` / `route` / `response` provenance. `GET /api/queries/live` sends the same object without `id`.
|
||||
|
||||
`GET /api/stats` and `/api/stats/timeseries` add `complete` and `available_since`.
|
||||
|
||||
Existing mutation endpoints stay specific. Diagnostics introduces no generic "perform remediation" endpoint; it invokes the existing blocklist-refresh and certificate-reload operations.
|
||||
|
||||
## Schema changes
|
||||
|
||||
`config.db`:
|
||||
|
||||
```sql
|
||||
CREATE TABLE operational_events (
|
||||
id INTEGER PRIMARY KEY,
|
||||
code TEXT NOT NULL,
|
||||
subject_key TEXT NOT NULL,
|
||||
subject_label TEXT NOT NULL,
|
||||
severity TEXT NOT NULL CHECK (severity IN ('warning', 'error')),
|
||||
first_seen INTEGER NOT NULL,
|
||||
last_seen INTEGER NOT NULL,
|
||||
occurrences INTEGER NOT NULL CHECK (occurrences > 0),
|
||||
resolved_at INTEGER,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
CHECK (resolved_at IS NULL OR resolved_at >= first_seen)
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX idx_operational_events_active
|
||||
ON operational_events(code, subject_key)
|
||||
WHERE resolved_at IS NULL;
|
||||
|
||||
CREATE INDEX idx_operational_events_last_seen
|
||||
ON operational_events(last_seen DESC);
|
||||
```
|
||||
|
||||
Absent from the configuration table lists and the reconciliation delete order.
|
||||
|
||||
`querylog.db`: `querylog_meta` and the provenance columns above. No new index — Codex proposed `idx_query_log_rcode`, cut on review, because every rcode question the UI asks is time-scoped and `idx_query_log_ts` already bounds the scan. 700k rows on a Pi 5 do not need a second index for a rare filter, and every index taxes the hot insert path in `logger.zig`.
|
||||
|
||||
No new provenance table, no key/value store — household retention makes nullable columns cheaper than a normalized graph of decision objects.
|
||||
|
||||
### Entry buffer widths
|
||||
|
||||
`logger.zig`'s `Entry` uses fixed buffers sized by `max_reason_len = 32` and `max_upstream_len = 64`, and travels through the `Io.Queue` by value. The new fields roughly triple it: `matched` holds a full pattern, and `cname_target`, `safe_search_target` and `forward_zone` each hold up to 253 bytes. That is fine at `flush_batch = 100`, but the widths are part of this contract and must be set explicitly, not left to whatever the first implementation picks.
|
||||
|
||||
`transformed()` must hide `matched`, `cname_target` and `safe_search_target` under `hide_domains`, not only `domain`.
|
||||
|
||||
On recreation: query rows, provenance and upstream-minute history reset together as today; the old file stays aside; `config.db` diagnostics survive; a resolved `query_log.recreated` event records the reason, the aside path and the new coverage start; Overview and Activity report the incomplete range instead of charting zero.
|
||||
|
||||
## Deletions and their cost
|
||||
|
||||
| Deleted | Lost |
|
||||
| --- | --- |
|
||||
| Separate Query Log and Live pages | separate bookmarks; both modes remain in Activity |
|
||||
| Standalone Lookup page | a top-level bookmark; testing remains under Activity |
|
||||
| Top-level Groups, Blocklists, Rules, Local DNS, Upstreams, Settings | direct resource navigation; all capabilities remain under task-shaped configuration |
|
||||
| Historical upstream table on Overview | at-a-glance period rates; availability stays, failures move to Diagnostics |
|
||||
| "last failure · 9h ago" text | nothing actionable; the episode becomes a diagnostic |
|
||||
| Detailed DB/log byte gauges | exact component sizes stay in Prometheus; free space stays on Overview |
|
||||
| Standalone cache card | one prominent number; cache stays in the timeline and metrics |
|
||||
| Ephemeral blocklist `SourceStatusSection` | transient success detail after navigation; durable counters and diagnostics remain |
|
||||
| Disabled configuration forms in file mode | the illusion that fields can be edited |
|
||||
| Global read-only banner | repeated warning text; authority stays visible once in the shell |
|
||||
| Client-only restart banner state | nothing reliable; server-owned `restart_pending` replaces it |
|
||||
| Old route aliases and redirects | existing bookmarks break; no permanent duplicate routing layer |
|
||||
|
||||
## Build sequence
|
||||
|
||||
Each step leaves the app working and shippable, and updates its OpenAPI contract, generated sample, TypeScript types and deterministic tests before landing.
|
||||
|
||||
1. **Diagnostics vertical slice.** `operational_events` schema, repository, serialized store, retention, health state, every typed emitter, API, UI, navigation. Recovery paths and deterministic failure-injection tests per event code. The rest of the UI stays intact.
|
||||
2. **Query provenance vertical slice.** Query-log fingerprint and schema, metadata table, provenance capture in the handler and logger, parsed-SERVFAIL logging, detail and coverage APIs, the historical detail route, and the `query_log.recreated` emission. Existing list summary fields stay so the current pages keep working.
|
||||
|
||||
**This step destroys the existing query history.** The provenance DDL edit changes the fingerprint, so `querylog_schema.open` recreates the file and keeps the old one aside as `querylog.db.schema-changed-<unix seconds>`. Acceptable pre-v0.1, and `available_since` carries the story in the UI, but it is a consequence of this step and must be stated in its spec and its changelog entry. Its acceptance tests cover the recreate, the aside name and the coverage sequence — which is also the natural first `query_log.recreated` emission.
|
||||
|
||||
Parsed-SERVFAIL logging reverses ruling 20: `handler.zig:507` counts today rather than logging. The `Context` exists at every `servFail` site that follows question parsing. Pre-parse failures correctly stay counters.
|
||||
3. **Activity consolidation.** The unified History/Live surface, URL filters, freeze/follow, live detail, current-policy test, historical detail links. Query Log, Live and Lookup routes and code are removed in the same change. Route, SSE, accessibility, reconnect and bounded-buffer tests.
|
||||
4. **Overview replacement.** Current status, active diagnostics, one coherent activity section. New health contract and completeness states. The upstream-history table, stale-failure text, detailed DiskCard and cache card go.
|
||||
5. **Task-shaped configuration and file mode.** `/api/config/status` and server-owned `restart_pending`. Protection, Resolution and System in both read-only and editable forms. Clients and its detail route redesigned. Old configuration routes replaced atomically; global banner and disabled forms removed.
|
||||
6. **Contract closure.** Remove obsolete queries, types, stores, CSS, tests and route fixtures. Regenerate contract samples, update OpenAPI and reference docs, add cross-surface acceptance tests for investigation links, file authority, query-log recreation, active-event recovery and time bounds. Zig, frontend, integration, accessibility and byte-budget checks; no new dependency.
|
||||
|
||||
## Codex's least-certain calls
|
||||
|
||||
- **Clients in primary navigation.** It earns the slot if identifying unknown devices and checking their group is routine. If Mokhtar almost always reaches a client from a query, Clients moves under Protection and leaves the nav.
|
||||
- **Recording the exact selected upstream per query.** It materially improves correlating SERVFAIL queries with upstream events, but needs the pool exchange result to expose the selected target. Drop back to `"pool"` only if that interface change proves invasive and exact resolver identity never changes an action.
|
||||
- **90-day / 5,000-event retention.** Conservative fixed bounds, not settings. Change only after measuring real row size and event rate on the Pi; do not add configurable retention pre-emptively.
|
||||
|
||||
## Rulings (Fable review, 2026-08-20)
|
||||
|
||||
Verdict: build it, with the four cuts folded in above and the rulings below.
|
||||
|
||||
**Diagnostics live in `config.db`. Accepted.** The `querylog.db` alternative is self-refuting: that file is recreated on any schema edit or corruption, so the recreation event dies with the thing it describes. A third database needs either its own migration discipline or a recreate policy that loses the events — the same problem with more files. The write-traffic objection is overstated twice: event volume is failure-rate volume, deduplicated, with no `info` severity, so it is near zero in steady state; and `config.db` already takes operational writes, because client rows are materialized from traffic (PLAN §3.5, `clients.last_seen`, `learned_name`). Pre-v0.1 the table is one edit to `ddl_v1` with no migration step. Keeping it out of `delete_order` and `table_names` is enforceable — `config_schema.zig` has tests pinning those lists.
|
||||
|
||||
**`group_name`, `source_name` and `matched` stay TEXT. Accepted, and my inconsistency objection was weaker than I put it.** The `domains` table exists because a domain appears on every row, runs to 253 bytes, and feeds `GROUP BY` stats. None of that holds here. The closer precedent is `client_ip`, which is TEXT with the comment "not a FK: log rows are immutable facts" (`querylog_schema.zig:37`). `group_name` is short and mostly `default`; `source_name` and `matched` are non-NULL only on blocked rows; `matched` has cardinality high enough that normalizing buys nothing. Keep both halves of each id/name pair: the id links to the same object across a rename, the name survives a delete, and the detail view needs both.
|
||||
|
||||
**Provenance capture does not violate the pure core.** Verified against the code. The policy decision is made in `src/server/handler.zig`, which is already impure and already holds `Io`, the clock and the log call. `matcher.Decision` (`matcher.zig:39-51`) already returns `reason`, `matched` and `source`; the handler throws `matched` and `source` away at `handler.zig:501`. Capture is mostly widening `logger.zig`'s `Entry` and `LogFields`, not threading state through `dns/`, `filter/`, `local/` or `cache/`.
|
||||
|
||||
**Six milestones under the harness, one per step.** Do not fold step 6 into step 5: the closure sweep regenerates the contract samples and adds the cross-surface acceptance tests, and it deserves its own verify gate. If anything needs splitting it is step 1, which touches about ten subsystems — store, API and UI first, then the emitters. An event store with three emitters is already shippable and honest.
|
||||
|
||||
**One caution, not a blocker.** Diagnostics will be an empty page most of the year. The "No active operational issues" line has to make empty read as healthy, not broken.
|
||||
+336
-14
@@ -44,6 +44,7 @@ const doh_client = @import("upstream/doh_client.zig");
|
||||
const doh_server = @import("server/doh_server.zig");
|
||||
const dot_client = @import("upstream/dot_client.zig");
|
||||
const dot_server = @import("server/dot_server.zig");
|
||||
const events = @import("storage/events.zig");
|
||||
const faults = @import("config/faults.zig");
|
||||
const fetcher = @import("filter/fetcher.zig");
|
||||
const forward_zones = @import("local/forward_zones.zig");
|
||||
@@ -154,6 +155,7 @@ fn reconcileFromFile(
|
||||
config_db: *db.Db,
|
||||
dir: std.Io.Dir,
|
||||
config_path: []const u8,
|
||||
config_load: ?*ConfigLoad,
|
||||
) !i64 {
|
||||
return reconcileFromFileAt(
|
||||
r,
|
||||
@@ -161,6 +163,7 @@ fn reconcileFromFile(
|
||||
dir,
|
||||
config_path,
|
||||
std.Io.Clock.real.now(r.io).toSeconds(),
|
||||
config_load,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -184,6 +187,7 @@ fn reconcileFromFileAt(
|
||||
dir: std.Io.Dir,
|
||||
config_path: []const u8,
|
||||
pass_now: i64,
|
||||
config_load: ?*ConfigLoad,
|
||||
) !i64 {
|
||||
var arena_state: std.heap.ArenaAllocator = .init(r.gpa);
|
||||
defer arena_state.deinit();
|
||||
@@ -203,9 +207,32 @@ fn reconcileFromFileAt(
|
||||
diags.writeAll(r.err) catch {};
|
||||
r.err.flush() catch {};
|
||||
|
||||
// Warnings only. A `.fail` rejects the file and the process exits, so there
|
||||
// is nobody left to read a diagnostics row about it; a warning is the case
|
||||
// where the box serves on with a setting the operator did not mean.
|
||||
if (config_load) |collector| {
|
||||
for (diags.problems.items) |problem| {
|
||||
if (problem.severity != .warn) continue;
|
||||
collector.note(problem.path, problem.path, problem.message);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// An upstream's identity is its url: the whole url is the key, and the
|
||||
/// redaction is the label, because a url can carry an account token.
|
||||
fn noteUpstream(config_load: *ConfigLoad, url: []const u8, message: []const u8) void {
|
||||
var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
|
||||
const label = std.fmt.bufPrint(&label_buf, "{f}", .{safe_url.redact(url)}) catch &label_buf;
|
||||
var detail_buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&detail_buf, "upstream {f} {s}", .{
|
||||
safe_url.redactQuoted(url),
|
||||
message,
|
||||
}) catch &detail_buf;
|
||||
config_load.note(url, label, detail);
|
||||
}
|
||||
|
||||
/// Returns the moment the transaction committed, which is what the settings
|
||||
/// envelope reports as `reconciled_at`.
|
||||
fn applyManagedFile(
|
||||
@@ -318,6 +345,55 @@ fn printSummary(
|
||||
try r.out.flush();
|
||||
}
|
||||
|
||||
/// The `configuration.load` findings of one boot.
|
||||
///
|
||||
/// This code is boot-finalized: nothing during the run can fix a setting, so a
|
||||
/// restart is its recovery. Every finding is reported as it is made and its key
|
||||
/// kept, and one `resolveExcept` after the last of them closes the episodes of
|
||||
/// settings that were wrong last boot and are not wrong now.
|
||||
const ConfigLoad = struct {
|
||||
store: ?*events.Store,
|
||||
io: std.Io,
|
||||
now_s: i64,
|
||||
keys: [events.Store.max_kept_keys][events.Store.max_subject_key_len]u8 = undefined,
|
||||
lens: [events.Store.max_kept_keys]u16 = @splat(0),
|
||||
len: usize = 0,
|
||||
/// Set when a boot produced more distinct findings than `resolveExcept`
|
||||
/// carries. The bulk resolve is then refused rather than truncated: a stale
|
||||
/// episode left open is honest, and one that is still true closed is not.
|
||||
/// The refusal goes through the store, so it is counted and latched.
|
||||
overflowed: bool = false,
|
||||
|
||||
fn note(self: *ConfigLoad, key: []const u8, label: []const u8, detail: []const u8) void {
|
||||
const store = self.store orelse return;
|
||||
store.report(self.io, self.now_s, .configuration_load, key, label, .warning, detail);
|
||||
self.keep(key);
|
||||
}
|
||||
|
||||
fn keep(self: *ConfigLoad, key: []const u8) void {
|
||||
var canon_buf: [events.Store.max_subject_key_len]u8 = undefined;
|
||||
const canon = events.canonicalKey(key, &canon_buf);
|
||||
for (0..self.len) |i| {
|
||||
if (std.mem.eql(u8, self.keys[i][0..self.lens[i]], canon)) return;
|
||||
}
|
||||
if (self.len == self.keys.len) {
|
||||
self.overflowed = true;
|
||||
return;
|
||||
}
|
||||
@memcpy(self.keys[self.len][0..canon.len], canon);
|
||||
self.lens[self.len] = @intCast(canon.len);
|
||||
self.len += 1;
|
||||
}
|
||||
|
||||
fn finalize(self: *ConfigLoad) void {
|
||||
const store = self.store orelse return;
|
||||
if (self.overflowed) return store.refuseResolveExcept(self.io, self.now_s);
|
||||
var kept: [events.Store.max_kept_keys][]const u8 = undefined;
|
||||
for (0..self.len) |i| kept[i] = self.keys[i][0..self.lens[i]];
|
||||
store.resolveExcept(self.io, self.now_s, .configuration_load, kept[0..self.len]);
|
||||
}
|
||||
};
|
||||
|
||||
fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
const io = r.io;
|
||||
const gpa = r.gpa;
|
||||
@@ -334,13 +410,29 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
defer config_db.close();
|
||||
_ = try migrations.migrate(&config_db);
|
||||
|
||||
// Opened unconditionally, not gated on `cfg.web.enabled`: diagnostics record
|
||||
// what went wrong whether or not anyone is running the UI, and the store
|
||||
// owns this connection outright (`storage/events.zig`).
|
||||
var events_db = try data.openConfigDb(io);
|
||||
defer events_db.close();
|
||||
const boot_now_s = std.Io.Clock.real.now(io).toSeconds();
|
||||
var event_store_storage: ?events.Store = events.Store.init(io, &events_db, boot_now_s) catch |err| blk: {
|
||||
// No store rather than a store on a mirror it could not verify: the
|
||||
// latter answers `resolve` with confident no-ops. `/api/health` reports
|
||||
// the absence as `unavailable` and degrades on it.
|
||||
log.warn("diagnostics store unavailable: {s}", .{@errorName(err)});
|
||||
break :blk null;
|
||||
};
|
||||
const event_store: ?*events.Store = if (event_store_storage) |*s| s else null;
|
||||
var config_load: ConfigLoad = .{ .store = event_store, .io = io, .now_s = boot_now_s };
|
||||
|
||||
// Ruling 1: the presence of `--config` is the whole authority decision. With
|
||||
// it, the file is the sole declarative source and the database is converged
|
||||
// onto it here, before anything reads the database. Without it the database
|
||||
// is authority and this step does not exist — a file on disk that no flag
|
||||
// names changes nothing.
|
||||
const reconciled_at: ?i64 = if (args.config) |config_path|
|
||||
try reconcileFromFile(r, &config_db, std.Io.Dir.cwd(), config_path)
|
||||
try reconcileFromFile(r, &config_db, std.Io.Dir.cwd(), config_path, &config_load)
|
||||
else
|
||||
null;
|
||||
|
||||
@@ -437,7 +529,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
defer bundle.deinit(gpa);
|
||||
var bundle_lock: std.Io.RwLock = .init;
|
||||
|
||||
var upstreams = try Upstreams.build(gpa, cfg.upstreams, &dns_http, &bundle, &bundle_lock);
|
||||
var upstreams = try Upstreams.build(gpa, cfg.upstreams, &dns_http, &bundle, &bundle_lock, &config_load);
|
||||
defer upstreams.deinit(gpa);
|
||||
|
||||
var pool: pool_mod.Pool = .init(
|
||||
@@ -455,7 +547,9 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
const history = try gpa.create(history_mod.Accumulator);
|
||||
defer gpa.destroy(history);
|
||||
history.* = .init;
|
||||
history.diagnostics = event_store;
|
||||
pool.history = history;
|
||||
pool.diagnostics = event_store;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// per-query state
|
||||
@@ -472,15 +566,18 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
|
||||
var paused: pause.Pause = .{};
|
||||
var tracker: clients.Tracker = .init(cfg.logging.retention_days);
|
||||
tracker.diagnostics = event_store;
|
||||
// Naming rides the tracker's pass, on the tracker's task and connection
|
||||
// (milestone-25 ruling 1), and reads the live forward zones.
|
||||
var client_names_resolver: client_names.Resolver = .init(&tables);
|
||||
client_names_resolver.diagnostics = event_store;
|
||||
|
||||
// The queue holds waiting tasks in intrusive lists, so neither the buffer
|
||||
// nor the `Logger` may move once a task has touched either.
|
||||
const queue_buf = try gpa.alloc(logger_mod.Entry, cfg.logging.query_log_buffer_max);
|
||||
defer gpa.free(queue_buf);
|
||||
var query_logger: logger_mod.Logger = .init(cfg.logging, queue_buf);
|
||||
query_logger.diagnostics = event_store;
|
||||
|
||||
// Milestone 8 fans every logged query out to the SSE hub as well. The hub
|
||||
// exists only when the web interface does (ruling 6) — without it the sink
|
||||
@@ -510,6 +607,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
// Ruling 17. The scheduler consults it before every scheduled pass; the
|
||||
// startup `reload` below is an operator action and stays ungated.
|
||||
manager.monitor = &monitor;
|
||||
manager.diagnostics = event_store;
|
||||
|
||||
// One synchronous sample before anything can consult the gate. `Monitor`
|
||||
// initializes to `.ok`, and `Monitor.run` takes its first sample inside the
|
||||
@@ -525,12 +623,37 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
// fails therefore leaves the state at `.ok` — an unreadable filesystem is
|
||||
// not evidence that the disk is full, which is the monitor's documented
|
||||
// policy and the right one here too.
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, event_store, boot_now_s);
|
||||
|
||||
var retention: retention_mod.Retention = .init(cfg.logging);
|
||||
|
||||
var querylog_writer_db = try data.openQuerylogDb(io);
|
||||
var querylog_opened = try data.openQuerylogDb(io);
|
||||
var querylog_writer_db = querylog_opened.database;
|
||||
defer querylog_writer_db.close();
|
||||
|
||||
// One-shot and already over: the file was recreated during this boot, and
|
||||
// there is nothing to recover from. Never emitted for `.missing` — a first
|
||||
// creation renames nothing aside, so the event would carry an aside path
|
||||
// that does not exist and would greet every fresh install with a warning.
|
||||
if (querylog_opened.recreated) |cause| {
|
||||
if (cause != .missing) {
|
||||
if (event_store) |store| {
|
||||
var detail_buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&detail_buf, "previous file kept as '{s}'", .{
|
||||
querylog_opened.aside(),
|
||||
}) catch detail_buf[0..];
|
||||
store.reportResolved(
|
||||
io,
|
||||
boot_now_s,
|
||||
.query_log_recreated,
|
||||
@tagName(cause),
|
||||
@tagName(cause),
|
||||
.warning,
|
||||
detail,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
var querylog_retention_db = try data.reopenQuerylogDb(io);
|
||||
defer querylog_retention_db.close();
|
||||
var querylog_history_db = try data.reopenQuerylogDb(io);
|
||||
@@ -574,6 +697,17 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
"loading the blocklist snapshot failed ({s}); serving unfiltered until the next refresh",
|
||||
.{@errorName(err)},
|
||||
);
|
||||
// No manager lock is held here, so this reports directly rather than
|
||||
// through the manager's collector.
|
||||
if (event_store) |store| {
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(
|
||||
&buf,
|
||||
"loading the blocklist snapshot failed: {s}",
|
||||
.{@errorName(err)},
|
||||
) catch buf[0..];
|
||||
store.report(io, boot_now_s, .blocklist_snapshot, "snapshot", "blocklist snapshot", .@"error", detail);
|
||||
}
|
||||
};
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -609,12 +743,16 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
defer if (doh_certs) |*store| store.deinit(io);
|
||||
if (cfg.doh_server.enabled) {
|
||||
doh_certs = try openCertStore(r, gpa, io, cfg.doh_server, "doh_server", doh_server.alpn_protocols);
|
||||
doh_certs.?.kind = .doh;
|
||||
doh_certs.?.diagnostics = event_store;
|
||||
}
|
||||
|
||||
var dot_certs: ?cert_store.CertStore = null;
|
||||
defer if (dot_certs) |*store| store.deinit(io);
|
||||
if (cfg.dot_server.enabled) {
|
||||
dot_certs = try openCertStore(r, gpa, io, cfg.dot_server, "dot_server", dot_alpn);
|
||||
dot_certs.?.kind = .dot;
|
||||
dot_certs.?.diagnostics = event_store;
|
||||
}
|
||||
|
||||
// Bound here, in this frame, rather than through doh_server's module-level
|
||||
@@ -622,13 +760,35 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
// only a listener that lives in this frame has an address to wire there.
|
||||
// A failed bind warns and stays off (ruling 1, the web precedent): TLS DNS
|
||||
// failing to come up must not stop the plain-DNS side this box exists for.
|
||||
var failed_listeners: [2][]const u8 = undefined;
|
||||
var failed_listener_count: usize = 0;
|
||||
|
||||
var doh: ?doh_server.DohServer = null;
|
||||
defer if (doh) |*server| server.deinit(io);
|
||||
if (doh_certs) |*store| doh = bindDoh(gpa, io, cfg.doh_server, &h, store);
|
||||
if (doh_certs) |*store| {
|
||||
doh = bindDoh(gpa, io, cfg.doh_server, &h, store, event_store, boot_now_s);
|
||||
if (doh == null) {
|
||||
failed_listeners[failed_listener_count] = "doh";
|
||||
failed_listener_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
var dot: ?dot_server.DotServer = null;
|
||||
defer if (dot) |*server| server.deinit(io);
|
||||
if (dot_certs) |*store| dot = bindDot(gpa, io, cfg.dot_server, &h, store);
|
||||
if (dot_certs) |*store| {
|
||||
dot = bindDot(gpa, io, cfg.dot_server, &h, store, event_store, boot_now_s);
|
||||
if (dot == null) {
|
||||
failed_listeners[failed_listener_count] = "dot";
|
||||
failed_listener_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Boot-finalized: one call closes whatever the last boot left open for an
|
||||
// endpoint that started clean this time — including an endpoint now
|
||||
// disabled, which contributes no key and so is not kept.
|
||||
if (event_store) |store| {
|
||||
store.resolveExcept(io, boot_now_s, .listener_start, failed_listeners[0..failed_listener_count]);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// web interface (ruling 26)
|
||||
@@ -668,6 +828,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
.dot_listener = if (dot) |*server| server else null,
|
||||
.config_db = if (web_config_db) |*database| database else null,
|
||||
.querylog_db = if (web_querylog_db) |*database| database else null,
|
||||
.events = event_store,
|
||||
.version = version.string,
|
||||
.started_unix = std.Io.Clock.real.now(io).toSeconds(),
|
||||
// Ruling 24: `--admin-dev` serves from disk with no cache headers;
|
||||
@@ -684,6 +845,7 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
var udp6: ?udp_server.UdpServer = udp_server.UdpServer.bind(gpa, io, v6_bind, &h, .{}) catch |err| bound: {
|
||||
if (!ipv6Unavailable(err)) return reportBind(r, "udp", v6_bind, err);
|
||||
log.warn("this system has no IPv6; serving IPv4 only", .{});
|
||||
config_load.note("dns.bind_ipv6", "dns.bind_ipv6", "this system has no IPv6; serving IPv4 only");
|
||||
break :bound null;
|
||||
};
|
||||
defer if (udp6) |*s| s.deinit(gpa, io);
|
||||
@@ -712,6 +874,12 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
};
|
||||
defer if (tcp4) |*s| s.deinit(io);
|
||||
|
||||
// Every `configuration.load` finding of this boot is in by here: the managed
|
||||
// file, the upstream table and the IPv6 bind above. Finalizing any earlier
|
||||
// would resolve an episode this boot is about to reopen, so consecutive
|
||||
// IPv6-less boots would read as a new episode each time.
|
||||
config_load.finalize();
|
||||
|
||||
// Ruling 13: `/metrics` sums each transport's listeners into one family, so
|
||||
// the web state carries pointers to whichever of the four came up. The
|
||||
// arrays are declared here rather than beside `web_state` because a
|
||||
@@ -793,11 +961,11 @@ fn serve(r: cli.Runner, args: cli.RunArgs) !u8 {
|
||||
if (doh_certs) |*store| try group.concurrent(io, cert_store.CertStore.watch, .{ store, io });
|
||||
if (dot_certs) |*store| try group.concurrent(io, cert_store.CertStore.watch, .{ store, io });
|
||||
|
||||
try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db, gate });
|
||||
try group.concurrent(io, retention_mod.Retention.run, .{ &retention, io, &querylog_retention_db, gate, event_store });
|
||||
// Ungated: a flush writes at most one row per upstream per minute, the same
|
||||
// category as the query logger's own writes, which are ungated too.
|
||||
try group.concurrent(io, history_mod.Accumulator.run, .{ history, io, &querylog_history_db });
|
||||
try group.concurrent(io, disk_monitor.Monitor.run, .{ &monitor, io });
|
||||
try group.concurrent(io, disk_monitor.Monitor.run, .{ &monitor, io, event_store });
|
||||
try group.concurrent(io, manager_mod.Manager.runScheduler, .{ &manager, io });
|
||||
try group.concurrent(io, clients.Tracker.run, .{ &tracker, io, &tracker_db, gate, &client_names_resolver });
|
||||
try group.concurrent(io, runMaintenance, .{ &h, if (web_limiter) |*l| l else null, io });
|
||||
@@ -882,6 +1050,23 @@ fn openCertStore(
|
||||
};
|
||||
}
|
||||
|
||||
/// An error, not a warning: an endpoint the operator enabled is not serving.
|
||||
fn reportListener(
|
||||
store: ?*events.Store,
|
||||
io: std.Io,
|
||||
now_s: i64,
|
||||
kind: []const u8,
|
||||
comptime fmt: []const u8,
|
||||
args: anytype,
|
||||
) void {
|
||||
const s = store orelse return;
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
var w: std.Io.Writer = .fixed(&buf);
|
||||
w.print("{s} listener ", .{kind}) catch {};
|
||||
w.print(fmt, args) catch {};
|
||||
s.report(io, now_s, .listener_start, kind, kind, .@"error", w.buffered());
|
||||
}
|
||||
|
||||
/// Ruling 1: a listener that cannot bind warns and stays off. The bind text
|
||||
/// itself gets the same treatment — `validate` refuses it, but a hand-edited
|
||||
/// database can still carry one, and it is not worth taking DNS down over.
|
||||
@@ -891,13 +1076,17 @@ fn bindDoh(
|
||||
endpoint: model.TlsEndpoint,
|
||||
h: *handler.Handler,
|
||||
store: *cert_store.CertStore,
|
||||
diagnostics: ?*events.Store,
|
||||
now_s: i64,
|
||||
) ?doh_server.DohServer {
|
||||
const bind_address = net.IpAddress.parse(endpoint.bind, endpoint.port) catch {
|
||||
log.warn("doh_server.bind '{s}' is not an IP address; DoH is disabled", .{endpoint.bind});
|
||||
reportListener(diagnostics, io, now_s, "doh", "bind '{s}' is not an IP address", .{endpoint.bind});
|
||||
return null;
|
||||
};
|
||||
const server = doh_server.DohServer.listen(gpa, io, bind_address, h, store, .{}) catch |err| {
|
||||
log.warn("doh listener cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
|
||||
reportListener(diagnostics, io, now_s, "doh", "cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
|
||||
return null;
|
||||
};
|
||||
log.info("doh listener on {f}", .{server.boundAddress()});
|
||||
@@ -910,13 +1099,17 @@ fn bindDot(
|
||||
endpoint: model.TlsEndpoint,
|
||||
h: *handler.Handler,
|
||||
store: *cert_store.CertStore,
|
||||
diagnostics: ?*events.Store,
|
||||
now_s: i64,
|
||||
) ?dot_server.DotServer {
|
||||
const bind_address = net.IpAddress.parse(endpoint.bind, endpoint.port) catch {
|
||||
log.warn("dot_server.bind '{s}' is not an IP address; DoT is disabled", .{endpoint.bind});
|
||||
reportListener(diagnostics, io, now_s, "dot", "bind '{s}' is not an IP address", .{endpoint.bind});
|
||||
return null;
|
||||
};
|
||||
const server = dot_server.DotServer.listen(gpa, io, bind_address, h, store, .{}) catch |err| {
|
||||
log.warn("dot listener cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
|
||||
reportListener(diagnostics, io, now_s, "dot", "cannot listen on {s}:{d}: {t}", .{ endpoint.bind, endpoint.port, err });
|
||||
return null;
|
||||
};
|
||||
log.info("dot listener on {f}", .{server.boundAddress()});
|
||||
@@ -1006,6 +1199,7 @@ const Upstreams = struct {
|
||||
http: *std.http.Client,
|
||||
bundle: *Certificate.Bundle,
|
||||
bundle_lock: *std.Io.RwLock,
|
||||
config_load: *ConfigLoad,
|
||||
) (Allocator.Error || error{NoUsableUpstreams})!Upstreams {
|
||||
var enabled: usize = 0;
|
||||
for (servers) |server| {
|
||||
@@ -1041,6 +1235,7 @@ const Upstreams = struct {
|
||||
"upstream {f} is not an https:// or tls:// endpoint; skipped",
|
||||
.{safe_url.redactQuoted(server.url)},
|
||||
);
|
||||
noteUpstream(config_load, server.url, "not an https:// or tls:// endpoint; skipped");
|
||||
continue;
|
||||
};
|
||||
|
||||
@@ -1058,6 +1253,7 @@ const Upstreams = struct {
|
||||
"upstream {f} is not a usable DoH url; skipped",
|
||||
.{safe_url.redactQuoted(server.url)},
|
||||
);
|
||||
noteUpstream(config_load, server.url, "not a usable DoH url; skipped");
|
||||
continue;
|
||||
};
|
||||
doh_count += 1;
|
||||
@@ -1154,6 +1350,9 @@ fn parseBind(
|
||||
return addr;
|
||||
}
|
||||
|
||||
const events_fixture = @import("storage/events_fixture.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
test "parseBind refuses a bind address of the wrong family" {
|
||||
var out_buf: [8]u8 = undefined;
|
||||
var err_buf: [256]u8 = undefined;
|
||||
@@ -1325,7 +1524,7 @@ test "a start in file mode prints the warnings the file earned" {
|
||||
|
||||
// The file is valid, so the start succeeds and the database converges onto
|
||||
// it. The returned stamp is what the settings envelope reports.
|
||||
const reconciled_at = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
|
||||
const reconciled_at = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
|
||||
try std.testing.expect(reconciled_at > 0);
|
||||
try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
|
||||
|
||||
@@ -1431,7 +1630,7 @@ test "reconciled_at is stamped after the commit, not from the clock the pass wro
|
||||
|
||||
const pass_now: i64 = 42;
|
||||
const before = std.Io.Clock.real.now(io).toSeconds();
|
||||
const reconciled_at = try reconcileFromFileAt(r, &database, tmp.dir, "config.zon", pass_now);
|
||||
const reconciled_at = try reconcileFromFileAt(r, &database, tmp.dir, "config.zon", pass_now, null);
|
||||
|
||||
// The pinned clock reached the engine, so the two values really are separate
|
||||
// inputs rather than the same read twice.
|
||||
@@ -1474,7 +1673,7 @@ test "the startup summary reports what the reconcile changed, then that nothing
|
||||
var err: Writer = .fixed(&err_buf);
|
||||
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out_writer.interface, .err = &err };
|
||||
|
||||
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
|
||||
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
|
||||
{
|
||||
// Read back through the file: `serve` does not return for as long as the
|
||||
// service runs, so a summary still in the buffer is a summary nobody
|
||||
@@ -1491,7 +1690,7 @@ test "the startup summary reports what the reconcile changed, then that nothing
|
||||
}
|
||||
|
||||
try tmp.dir.writeFile(io, .{ .sub_path = "stdout.txt", .data = "" });
|
||||
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
|
||||
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
|
||||
{
|
||||
const printed = try tmp.dir.readFileAlloc(io, "stdout.txt", gpa, .limited(8192));
|
||||
defer gpa.free(printed);
|
||||
@@ -1548,7 +1747,7 @@ test "a broken error writer does not replace the reason a managed file was rejec
|
||||
|
||||
try std.testing.expectError(
|
||||
error.MissingDefaultGroup,
|
||||
reconcileFromFile(r, &database, tmp.dir, "config.zon"),
|
||||
reconcileFromFile(r, &database, tmp.dir, "config.zon", null),
|
||||
);
|
||||
|
||||
// Empty, so the writer did fail — without this the assertion above would
|
||||
@@ -1594,7 +1793,7 @@ test "a broken error writer does not stop a start whose file applied" {
|
||||
var err_writer = brokenErrWriter(io, err_file, &err_buf);
|
||||
const r: cli.Runner = .{ .io = io, .gpa = gpa, .out = &out, .err = &err_writer.interface };
|
||||
|
||||
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon");
|
||||
_ = try reconcileFromFile(r, &database, tmp.dir, "config.zon", null);
|
||||
try std.testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM upstreams"));
|
||||
|
||||
// Nothing reached the file, so the flush really did fail.
|
||||
@@ -1779,3 +1978,126 @@ test "one maintenance pass drops the api limiter's stale buckets" {
|
||||
// A limiter the app did not build is not a reason for the pass to fail.
|
||||
try maintenanceOnce(&h, null, io);
|
||||
}
|
||||
|
||||
test "a failing listener bind opens an error episode a clean boot closes" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
reportListener(&fx.store, io, 1000, "dot", "cannot listen on {s}:{d}: {t}", .{
|
||||
"0.0.0.0",
|
||||
@as(u16, 853),
|
||||
error.AddressInUse,
|
||||
});
|
||||
|
||||
try testing.expectEqualStrings("listener.start", try fx.text("SELECT code FROM operational_events"));
|
||||
try testing.expectEqualStrings("dot", try fx.text("SELECT subject_key FROM operational_events"));
|
||||
try testing.expectEqualStrings("error", try fx.text("SELECT severity FROM operational_events"));
|
||||
|
||||
// The next boot: DoH failed, DoT started clean. One `resolveExcept` over
|
||||
// the failed keys closes the stale DoT episode and leaves the DoH one open.
|
||||
reportListener(&fx.store, io, 1100, "doh", "cannot listen on {s}:{d}: {t}", .{
|
||||
"0.0.0.0",
|
||||
@as(u16, 443),
|
||||
error.AddressInUse,
|
||||
});
|
||||
fx.store.resolveExcept(io, 1100, .listener_start, &.{"doh"});
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
try testing.expectEqualStrings("doh", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
}
|
||||
|
||||
test "a boot with no listener finding closes every listener episode" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
fx.store.report(io, 900, .listener_start, "doh", "doh", .@"error", "stale");
|
||||
fx.store.report(io, 900, .listener_start, "dot", "dot", .@"error", "stale");
|
||||
|
||||
fx.store.resolveExcept(io, 1000, .listener_start, &.{});
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a configuration finding is reported once and finalize closes the rest" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
// Two stale episodes from a previous boot.
|
||||
fx.store.report(io, 900, .configuration_load, "dns.bind_ipv6", "dns.bind_ipv6", .warning, "stale");
|
||||
fx.store.report(io, 900, .configuration_load, "upstreams[0].url", "upstreams[0].url", .warning, "stale");
|
||||
|
||||
var collector: ConfigLoad = .{ .store = &fx.store, .io = io, .now_s = 1000 };
|
||||
collector.note("dns.bind_ipv6", "dns.bind_ipv6", "this system has no IPv6; serving IPv4 only");
|
||||
// The same finding twice is one episode and one kept key.
|
||||
collector.note("dns.bind_ipv6", "dns.bind_ipv6", "this system has no IPv6; serving IPv4 only");
|
||||
try testing.expectEqual(@as(usize, 1), collector.len);
|
||||
|
||||
collector.finalize();
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
try testing.expectEqualStrings("dns.bind_ipv6", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
// The stale episode is still the same one: this boot bumped it twice.
|
||||
try testing.expectEqual(@as(i64, 3), try fx.count(
|
||||
"SELECT occurrences FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
}
|
||||
|
||||
test "an over-long boot finding list refuses to finalize rather than truncate" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
fx.store.report(io, 900, .configuration_load, "stale.setting", "stale.setting", .warning, "stale");
|
||||
|
||||
var collector: ConfigLoad = .{ .store = &fx.store, .io = io, .now_s = 1000 };
|
||||
var key_buf: [32]u8 = undefined;
|
||||
for (0..events.Store.max_kept_keys + 1) |i| {
|
||||
const key = try std.fmt.bufPrint(&key_buf, "upstreams[{d}].url", .{i});
|
||||
collector.note(key, key, "not a usable url");
|
||||
}
|
||||
try testing.expect(collector.overflowed);
|
||||
|
||||
collector.finalize();
|
||||
|
||||
// Closing more than the boot meant would resolve episodes that are still
|
||||
// true, so the stale one stays open instead.
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count(
|
||||
"SELECT count(*) FROM operational_events WHERE subject_key = 'stale.setting' AND resolved_at IS NULL",
|
||||
));
|
||||
// Refused, and refused out loud: the store counted and latched it, so
|
||||
// `/api/health` reports the diagnostics log as not recording.
|
||||
try testing.expect(fx.store.writeFailed());
|
||||
try testing.expectEqual(@as(u64, 1), fx.store.writeFailures());
|
||||
}
|
||||
|
||||
+7
-5
@@ -332,12 +332,14 @@ pub const DataDir = struct {
|
||||
/// `querylog_schema.open` resolves the path through SQLite's VFS as well as
|
||||
/// through the directory handle, so it is given `cwd` and the joined path
|
||||
/// rather than `self.dir` and a name (see its doc comment).
|
||||
pub fn openQuerylogDb(self: *const DataDir, io: std.Io) !db.Db {
|
||||
const opened = try querylog_schema.open(io, std.Io.Dir.cwd(), self.querylog_db_path);
|
||||
var database = opened.database;
|
||||
errdefer database.close();
|
||||
/// Returns the whole `OpenResult`, recreate reason and aside name included:
|
||||
/// the composition root records a recreate as a one-shot diagnostics event,
|
||||
/// and the aside name is what tells an operator where the old file went.
|
||||
pub fn openQuerylogDb(self: *const DataDir, io: std.Io) !querylog_schema.OpenResult {
|
||||
var opened = try querylog_schema.open(io, std.Io.Dir.cwd(), self.querylog_db_path);
|
||||
errdefer opened.database.close();
|
||||
try self.restrictQuerylogPermissions(io);
|
||||
return database;
|
||||
return opened;
|
||||
}
|
||||
|
||||
/// An additional connection to a `querylog.db` that `openQuerylogDb` has
|
||||
|
||||
@@ -151,6 +151,7 @@ fn reportDeletes(diags: *validate.Diagnostics, summary: reconcile.Summary) error
|
||||
|
||||
const testing = std.testing;
|
||||
const config_schema = @import("../storage/config_schema.zig");
|
||||
const export_mod = @import("export.zig");
|
||||
const migrations = @import("../storage/migrations.zig");
|
||||
|
||||
fn openMigrated() !db.Db {
|
||||
@@ -282,6 +283,55 @@ test "importSource converges a migrated database and group 'default' keeps id 1"
|
||||
);
|
||||
}
|
||||
|
||||
test "an export and re-import leaves the operational_events log untouched" {
|
||||
// `operational_events` is runtime state, not configuration: it is out of
|
||||
// `table_names` and out of `delete_order`, so an export must not emit it and
|
||||
// an import's wipe must not reach it. A diagnostics log destroyed by a
|
||||
// routine `nxdns import` would take the record of what the box has been
|
||||
// doing with it.
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
const gpa = testing.allocator;
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
try importText(io, &database, full_source, .{});
|
||||
try database.exec(
|
||||
\\INSERT INTO operational_events
|
||||
\\ (code, subject_key, subject_label, severity, first_seen, last_seen, occurrences, resolved_at)
|
||||
\\VALUES ('disk.space', 'data', 'data', 'warning', 100, 200, 3, NULL),
|
||||
\\ ('blocklist.refresh', 'https://a.example', 'A', 'warning', 100, 150, 1, 300);
|
||||
);
|
||||
|
||||
var rendered: std.Io.Writer.Allocating = .init(gpa);
|
||||
defer rendered.deinit();
|
||||
try export_mod.writeToWriter(gpa, &database, &rendered.writer);
|
||||
const source = try gpa.dupeZ(u8, rendered.written());
|
||||
defer gpa.free(source);
|
||||
|
||||
// The export is the whole declared configuration and says nothing about
|
||||
// the log.
|
||||
try testing.expect(!std.mem.containsAtLeast(u8, source, 1, "operational_events"));
|
||||
try testing.expect(!std.mem.containsAtLeast(u8, source, 1, "disk.space"));
|
||||
|
||||
// A round trip is by definition delete-free for the config tables, and the
|
||||
// events survive it untouched, resolved and active alike.
|
||||
try importText(io, &database, source, .{ .allow_delete = true });
|
||||
var stmt = try database.prepare(
|
||||
"SELECT code, occurrences, resolved_at FROM operational_events ORDER BY id",
|
||||
);
|
||||
defer stmt.deinit();
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqualStrings("disk.space", stmt.columnText(0));
|
||||
try testing.expectEqual(@as(i64, 3), stmt.columnInt(1));
|
||||
try testing.expect(stmt.isNull(2));
|
||||
try testing.expect(try stmt.step());
|
||||
try testing.expectEqualStrings("blocklist.refresh", stmt.columnText(0));
|
||||
try testing.expectEqual(@as(i64, 300), stmt.columnInt(2));
|
||||
try testing.expect(!try stmt.step());
|
||||
}
|
||||
|
||||
test "an import whose diff deletes rows is refused, names the tables, and changes nothing" {
|
||||
// Ruling 6: the emptiness guard is gone, so this is what stops
|
||||
// `nxdns import ./wrong.zon` from emptying a configured database.
|
||||
|
||||
@@ -24,6 +24,7 @@ const net = std.Io.net;
|
||||
const model = @import("../config/model.zig");
|
||||
const reconcile = @import("../config/reconcile.zig");
|
||||
const db = @import("../storage/db.zig");
|
||||
const events_fixture = @import("../storage/events_fixture.zig");
|
||||
const migrations = @import("../storage/migrations.zig");
|
||||
const context = @import("../storage/repositories/context.zig");
|
||||
const groups_repo = @import("../storage/repositories/groups_repo.zig");
|
||||
@@ -2328,7 +2329,6 @@ test "23: a name moving from the list body to the wild body forces a republish"
|
||||
try testing.expectEqual(@as(i64, 0), row.wildcard_count);
|
||||
@memcpy(&first_checksum, row.checksum orelse return error.TestNoChecksum);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
test "22: a list that changed only its skipped lines still updates both skip counters" {
|
||||
@@ -2405,3 +2405,162 @@ test "22: a list that changed only its skipped lines still updates both skip cou
|
||||
try testing.expectEqual(@as(u32, 1), restored.counts.skipped_regex);
|
||||
try testing.expectEqual(@as(u32, 2), restored.counts.skipped_unsupported);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// diagnostics: the events the manager records (milestone 27)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test "27: a failing refresh opens a blocklist.refresh episode a good one closes" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
env.mgr.diagnostics = &fx.store;
|
||||
|
||||
var fixture = try HttpFixture.init(io, http_body);
|
||||
defer fixture.deinit(io);
|
||||
fixture.setRoute(.oversize);
|
||||
var group: std.Io.Group = .init;
|
||||
defer group.cancel(io);
|
||||
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||
|
||||
var url_buf: [64]u8 = undefined;
|
||||
const url = try fixture.url(&url_buf);
|
||||
_ = try seedSource(&env.database, url);
|
||||
|
||||
try testing.expect(!try refreshOnce(env, url));
|
||||
|
||||
try testing.expectEqualStrings("blocklist.refresh", try fx.text(
|
||||
"SELECT code FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
// The whole url, not the display copy: `subject_key` is the identity.
|
||||
try testing.expectEqualStrings(url, try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
try testing.expectEqualStrings("warning", try fx.text(
|
||||
"SELECT severity FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
|
||||
fixture.setRoute(.body);
|
||||
try testing.expect(try refreshOnce(env, url));
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "27: one refreshAll pass records one occurrence of a failing source" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
env.mgr.diagnostics = &fx.store;
|
||||
|
||||
var fixture = try HttpFixture.init(io, http_body);
|
||||
defer fixture.deinit(io);
|
||||
fixture.setRoute(.oversize);
|
||||
var group: std.Io.Group = .init;
|
||||
defer group.cancel(io);
|
||||
try group.concurrent(io, HttpFixture.serve, .{ &fixture, io });
|
||||
|
||||
var url_buf: [64]u8 = undefined;
|
||||
const url = try fixture.url(&url_buf);
|
||||
_ = try seedSource(&env.database, url);
|
||||
|
||||
// `scheduledPass` is what an elapsed interval runs, and it ends in
|
||||
// `refreshAll`, which ends in a reload. One pass is one flush: a reload
|
||||
// that flushed on its own, or a scheduler that flushed after a pass that
|
||||
// already had, would report this failure twice.
|
||||
try env.mgr.scheduledPass(io);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count(
|
||||
"SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
|
||||
));
|
||||
|
||||
// A second pass on a still-failing source is one more occurrence: the
|
||||
// failures one pass held ride the detail, and the flushes a pass makes are
|
||||
// not what `occurrences` counts.
|
||||
try env.mgr.scheduledPass(io);
|
||||
try testing.expectEqual(@as(i64, 2), try fx.count(
|
||||
"SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
|
||||
));
|
||||
const detail = try fx.text("SELECT detail FROM operational_events WHERE code = 'blocklist.refresh'");
|
||||
try testing.expect(std.mem.indexOf(u8, detail, "this pass") != null);
|
||||
}
|
||||
|
||||
test "27: a clean reload resolves blocklist.snapshot and records no storage failure" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
// A stale episode from a previous boot, which a published snapshot closes.
|
||||
fx.store.report(io, 900, .blocklist_snapshot, "snapshot", "blocklist snapshot", .@"error", "stale");
|
||||
env.mgr.diagnostics = &fx.store;
|
||||
|
||||
try env.mgr.reload(io);
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
try testing.expectEqual(@as(i64, 900), try fx.count("SELECT first_seen FROM operational_events"));
|
||||
}
|
||||
|
||||
test "27: an unreadable blocklist directory opens a blocklist.storage episode" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
// Mode bits do not apply to root, so the denial the test needs cannot happen.
|
||||
if (std.c.geteuid() == 0) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
const env = try Env.create(gpa);
|
||||
defer env.destroy();
|
||||
const io = env.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
env.mgr.diagnostics = &fx.store;
|
||||
|
||||
var dir = try env.blocklistDir();
|
||||
dir.close(io);
|
||||
try env.tmp.dir.setPermissions(io, .fromMode(0o600));
|
||||
const failed = env.mgr.pruneOrphans(io);
|
||||
try env.tmp.dir.setPermissions(io, .fromMode(0o700));
|
||||
try testing.expectError(error.FileSystem, failed);
|
||||
|
||||
try testing.expectEqualStrings("blocklist.storage", try fx.text(
|
||||
"SELECT code FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
// `createDirPathStatus` is the first call to touch the unreadable parent,
|
||||
// so it is the operation that fails.
|
||||
try testing.expectEqualStrings("create_dir", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
|
||||
// A pass that can read the directory again closes it.
|
||||
try env.mgr.pruneOrphans(io);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
+624
-22
@@ -59,6 +59,7 @@ const groups_repo = @import("../storage/repositories/groups_repo.zig");
|
||||
const rules_repo = @import("../storage/repositories/rules_repo.zig");
|
||||
const sources_repo = @import("../storage/repositories/sources_repo.zig");
|
||||
const disk_monitor = @import("../storage/disk_monitor.zig");
|
||||
const events = @import("../storage/events.zig");
|
||||
const compiler = @import("compiler.zig");
|
||||
const fetcher = @import("fetcher.zig");
|
||||
const matcher = @import("matcher.zig");
|
||||
@@ -162,6 +163,51 @@ pub const State = enum {
|
||||
}
|
||||
};
|
||||
|
||||
/// The `blocklist.storage` operations, each its own episode subject.
|
||||
///
|
||||
/// A fixed set on purpose: several of these fail once per file in a pass, and
|
||||
/// one slot per operation is what turns that into one report per pass instead
|
||||
/// of an unbounded list of them.
|
||||
pub const StorageOp = enum { sweep, directory_read, create_dir, open_dir, delete };
|
||||
|
||||
/// One operation's outcome across one pass. `detail` keeps the last failure,
|
||||
/// and `failures` says how many that pass held — the row's `occurrences` counts
|
||||
/// failing passes, so the count belongs in the text.
|
||||
const Aggregate = struct {
|
||||
failures: u32 = 0,
|
||||
succeeded: bool = false,
|
||||
detail: [events.Store.max_detail_len]u8 = @splat(0),
|
||||
detail_len: u16 = 0,
|
||||
|
||||
fn detailText(self: *const Aggregate) []const u8 {
|
||||
return self.detail[0..self.detail_len];
|
||||
}
|
||||
};
|
||||
|
||||
/// What a locked body observed, held until `Manager.flushDiagnostics` can
|
||||
/// report it with no manager lock held.
|
||||
///
|
||||
/// Two of this file's operations cannot report from where they stand:
|
||||
/// `publishRefresh` runs under `writer_lock` by contract and `pruneOrphans`
|
||||
/// holds both writer mutexes through its filesystem work. Collect-then-flush is
|
||||
/// what keeps their outcomes without holding a lock across a store call, and
|
||||
/// nothing here can grow: the storage slots are an enum array and a refresh
|
||||
/// outcome rides the status entry the source already has.
|
||||
///
|
||||
/// The flush still happens inside the lock that serializes passes — a pass
|
||||
/// drains its own outcomes before it releases `refresh_lock` (or, for a
|
||||
/// standalone `reload`, `writer_lock`). What collect-then-flush avoids is
|
||||
/// holding a *manager* lock across a store call, not deferring the report until
|
||||
/// the next pass could merge into it.
|
||||
const Pending = struct {
|
||||
mutex: std.Io.Mutex = .init,
|
||||
storage: std.EnumArray(StorageOp, Aggregate) = .initFill(.{}),
|
||||
/// Null until a pass observes a snapshot outcome at all.
|
||||
snapshot_failed: ?bool = null,
|
||||
snapshot_detail: [events.Store.max_detail_len]u8 = @splat(0),
|
||||
snapshot_detail_len: u16 = 0,
|
||||
};
|
||||
|
||||
/// A status is a value with no borrowed memory, so a copy handed to the API
|
||||
/// outlives every reload. The url is held inline for that reason.
|
||||
pub const SourceStatus = struct {
|
||||
@@ -182,6 +228,32 @@ pub const SourceStatus = struct {
|
||||
url_len: u8 = 0,
|
||||
last_error: [max_error_len]u8 = @splat(0),
|
||||
last_error_len: u8 = 0,
|
||||
/// The diagnostics identity of this source, canonicalized from the WHOLE
|
||||
/// url by `setUrl`. `url` above is a display copy truncated at
|
||||
/// `max_url_len`, and two urls sharing a 255-byte prefix would share one
|
||||
/// episode if that copy were the key.
|
||||
event_key: [events.Store.max_subject_key_len]u8 = @splat(0),
|
||||
event_key_len: u16 = 0,
|
||||
/// Diagnostics accounting for the pass in progress, cleared by every
|
||||
/// `flushDiagnostics`. `pass_outcome` says this source recorded one at all;
|
||||
/// `pass_failures` counts the failing ones, which a pass can hold more than
|
||||
/// one of (a refresh that failed, then the reload that could not load the
|
||||
/// files it did not write). One flush reports one `blocklist.refresh`
|
||||
/// occurrence per source, so `occurrences` counts failing passes rather
|
||||
/// than flushes, and the detail carries how many failures the pass held.
|
||||
///
|
||||
/// These two fields live in exactly one copy of the status table at a time,
|
||||
/// which is what makes that count right while reloads replace the table
|
||||
/// underneath: a candidate built by `mergeStatuses` carries none of them,
|
||||
/// `installStatuses` folds the live table's in as it swaps, and the flush
|
||||
/// claims an entry by copying it and zeroing both fields in one locked
|
||||
/// step. Copy them anywhere else and the outcome gets reported twice.
|
||||
pass_outcome: bool = false,
|
||||
pass_failures: u16 = 0,
|
||||
|
||||
pub fn eventKey(self: *const SourceStatus) []const u8 {
|
||||
return self.event_key[0..self.event_key_len];
|
||||
}
|
||||
|
||||
pub fn errorText(self: *const SourceStatus) []const u8 {
|
||||
return self.last_error[0..self.last_error_len];
|
||||
@@ -196,9 +268,12 @@ pub const SourceStatus = struct {
|
||||
@memcpy(self.url[0..kept], url[0..kept]);
|
||||
@memset(self.url[kept..], 0);
|
||||
self.url_len = @intCast(kept);
|
||||
self.event_key_len = @intCast(events.canonicalKey(url, &self.event_key).len);
|
||||
}
|
||||
|
||||
fn fail(self: *SourceStatus, state: State, text: []const u8) void {
|
||||
self.pass_failures +|= 1;
|
||||
self.pass_outcome = true;
|
||||
self.state = state;
|
||||
const kept = @min(text.len, max_error_len);
|
||||
@memcpy(self.last_error[0..kept], text[0..kept]);
|
||||
@@ -207,6 +282,7 @@ pub const SourceStatus = struct {
|
||||
}
|
||||
|
||||
fn succeed(self: *SourceStatus, at: i64, counts: compiler.Counts) void {
|
||||
self.pass_outcome = true;
|
||||
self.state = .ok;
|
||||
self.counts = counts;
|
||||
self.last_success = at;
|
||||
@@ -294,6 +370,14 @@ pub const Manager = struct {
|
||||
/// what every test and `nxdns check` want. Only the scheduler consults it —
|
||||
/// see `refreshGated`.
|
||||
monitor: ?*disk_monitor.Monitor = null,
|
||||
/// The diagnostics store, wired the same way as `monitor` and null
|
||||
/// everywhere else. Never touched while a manager lock is held: see
|
||||
/// `flushDiagnostics`.
|
||||
diagnostics: ?*events.Store = null,
|
||||
/// What the locked bodies observed and could not report from where they
|
||||
/// stood. Bounded by construction — one slot per storage operation, one
|
||||
/// snapshot outcome — and drained by `flushDiagnostics`.
|
||||
pending: Pending = .{},
|
||||
/// Scheduled refresh passes skipped by the disk gate. The `/api/health`
|
||||
/// rollup reads it through `refreshesGated`.
|
||||
refreshes_gated: std.atomic.Value(u64) = .init(0),
|
||||
@@ -388,6 +472,228 @@ pub const Manager = struct {
|
||||
return kept;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// diagnostics
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// Records one storage operation's failure. Callable from anywhere,
|
||||
/// including under both writer mutexes: it touches `pending` only.
|
||||
fn noteStorageFailure(
|
||||
self: *Manager,
|
||||
io: std.Io,
|
||||
op: StorageOp,
|
||||
comptime fmt: []const u8,
|
||||
args: anytype,
|
||||
) void {
|
||||
if (self.diagnostics == null) return;
|
||||
self.pending.mutex.lockUncancelable(io);
|
||||
defer self.pending.mutex.unlock(io);
|
||||
const slot = self.pending.storage.getPtr(op);
|
||||
slot.failures +|= 1;
|
||||
var w: std.Io.Writer = .fixed(&slot.detail);
|
||||
w.print(fmt, args) catch {};
|
||||
slot.detail_len = @intCast(w.end);
|
||||
}
|
||||
|
||||
fn noteStorageSuccess(self: *Manager, io: std.Io, op: StorageOp) void {
|
||||
if (self.diagnostics == null) return;
|
||||
self.pending.mutex.lockUncancelable(io);
|
||||
defer self.pending.mutex.unlock(io);
|
||||
self.pending.storage.getPtr(op).succeeded = true;
|
||||
}
|
||||
|
||||
/// Records whether a snapshot was published. `reason` null is the post-swap
|
||||
/// success; anything else is the pass that could not publish one.
|
||||
fn noteSnapshot(self: *Manager, io: std.Io, reason: ?[]const u8) void {
|
||||
if (self.diagnostics == null) return;
|
||||
self.pending.mutex.lockUncancelable(io);
|
||||
defer self.pending.mutex.unlock(io);
|
||||
self.pending.snapshot_failed = reason != null;
|
||||
const text = reason orelse "";
|
||||
const kept = @min(text.len, self.pending.snapshot_detail.len);
|
||||
@memcpy(self.pending.snapshot_detail[0..kept], text[0..kept]);
|
||||
self.pending.snapshot_detail_len = @intCast(kept);
|
||||
}
|
||||
|
||||
/// Drains `pending` and the status table into the store, holding no manager
|
||||
/// lock across a store call.
|
||||
///
|
||||
/// Called by every pass that can fill either one, and *before that pass
|
||||
/// releases the lock serializing it* — `refresh_lock` for a refresh pass,
|
||||
/// `writer_lock` for a standalone `reload`. Draining after the release
|
||||
/// would let the next pass record its own outcomes on the same entries
|
||||
/// first, and two failing passes would reach the store as one occurrence.
|
||||
/// The `defer` that calls this is registered after the unlock `defer` for
|
||||
/// that reason; defers run last-registered-first.
|
||||
///
|
||||
/// It is idempotent: a drained collector reports nothing.
|
||||
pub fn flushDiagnostics(self: *Manager, io: std.Io) void {
|
||||
const store = self.diagnostics orelse return;
|
||||
const now_s = std.Io.Clock.real.now(io).toSeconds();
|
||||
|
||||
var storage: std.EnumArray(StorageOp, Aggregate) = undefined;
|
||||
var snapshot_failed: ?bool = null;
|
||||
var snapshot_detail: [events.Store.max_detail_len]u8 = undefined;
|
||||
var snapshot_detail_len: u16 = 0;
|
||||
{
|
||||
self.pending.mutex.lockUncancelable(io);
|
||||
defer self.pending.mutex.unlock(io);
|
||||
storage = self.pending.storage;
|
||||
snapshot_failed = self.pending.snapshot_failed;
|
||||
snapshot_detail = self.pending.snapshot_detail;
|
||||
snapshot_detail_len = self.pending.snapshot_detail_len;
|
||||
self.pending.storage = .initFill(.{});
|
||||
self.pending.snapshot_failed = null;
|
||||
self.pending.snapshot_detail_len = 0;
|
||||
}
|
||||
|
||||
var it = storage.iterator();
|
||||
while (it.next()) |kv| {
|
||||
const op = @tagName(kv.key);
|
||||
if (kv.value.failures != 0) {
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "{s} ({d} this pass)", .{
|
||||
kv.value.detailText(),
|
||||
kv.value.failures,
|
||||
}) catch buf[0..];
|
||||
store.report(io, now_s, .blocklist_storage, op, op, .warning, detail);
|
||||
} else if (kv.value.succeeded) {
|
||||
store.resolve(io, now_s, .blocklist_storage, op);
|
||||
}
|
||||
}
|
||||
|
||||
if (snapshot_failed) |failed| {
|
||||
if (failed) {
|
||||
store.report(
|
||||
io,
|
||||
now_s,
|
||||
.blocklist_snapshot,
|
||||
snapshot_key,
|
||||
"blocklist snapshot",
|
||||
.@"error",
|
||||
snapshot_detail[0..snapshot_detail_len],
|
||||
);
|
||||
} else {
|
||||
store.resolve(io, now_s, .blocklist_snapshot, snapshot_key);
|
||||
}
|
||||
}
|
||||
|
||||
self.flushSourceDiagnostics(io, store, now_s);
|
||||
}
|
||||
|
||||
/// One `blocklist.refresh` episode per source, from the status table.
|
||||
///
|
||||
/// The table IS the per-source collection the collect-then-flush rule asks
|
||||
/// for: `prepareRefresh`, `publishRefresh` and the reload's load outcomes
|
||||
/// all write their result into the entry, under locks this cannot take. So
|
||||
/// one entry is copied out at a time under the exclusive lock and the store
|
||||
/// is called with nothing held.
|
||||
///
|
||||
/// The walk is a drain, not an index scan: a reload can replace the whole
|
||||
/// table between two iterations, and an index into the table it replaced
|
||||
/// would skip or repeat entries. Each round takes the lock, claims the
|
||||
/// first entry that still carries pass accounting by copying it out and
|
||||
/// zeroing the two fields, and reports it with nothing held. Claiming and
|
||||
/// clearing are one locked step, so an outcome is reported once: a table
|
||||
/// swapped in mid-drain carries the entries this flush has not claimed yet,
|
||||
/// and `installStatuses` folded them in for exactly that reason. The drain
|
||||
/// ends when a scan finds nothing left to claim.
|
||||
///
|
||||
/// The drain reaches only the sources the table still holds, so the sweep
|
||||
/// below is what closes the episode of one that is gone.
|
||||
fn flushSourceDiagnostics(self: *Manager, io: std.Io, store: *events.Store, now_s: i64) void {
|
||||
drain: while (true) {
|
||||
var status: SourceStatus = undefined;
|
||||
{
|
||||
self.lock.lockUncancelable(io);
|
||||
defer self.lock.unlock(io);
|
||||
const claimed = for (self.statuses) |*entry| {
|
||||
if (!entry.pass_outcome) continue;
|
||||
status = entry.*;
|
||||
entry.pass_outcome = false;
|
||||
entry.pass_failures = 0;
|
||||
// A source with no diagnostics identity has nothing to
|
||||
// report under, but its accounting is cleared all the same:
|
||||
// left set, it would make every later scan claim it and the
|
||||
// drain would never end.
|
||||
if (entry.event_key_len == 0) continue;
|
||||
break true;
|
||||
} else false;
|
||||
if (!claimed) break :drain;
|
||||
}
|
||||
if (!status.state.isRefreshFailure() and status.state != .load_failed) {
|
||||
store.resolve(io, now_s, .blocklist_refresh, status.eventKey());
|
||||
continue;
|
||||
}
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "{t}: {s} ({d} this pass)", .{
|
||||
status.state,
|
||||
status.errorText(),
|
||||
status.pass_failures,
|
||||
}) catch buf[0..];
|
||||
var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
|
||||
const label = std.fmt.bufPrint(&label_buf, "{f}", .{
|
||||
safe_url.redact(status.urlText()),
|
||||
}) catch &label_buf;
|
||||
store.report(io, now_s, .blocklist_refresh, status.eventKey(), label, .warning, detail);
|
||||
}
|
||||
|
||||
self.resolveDeletedSources(io, store, now_s);
|
||||
}
|
||||
|
||||
/// Closes the `blocklist.refresh` episode of a source that no longer exists.
|
||||
///
|
||||
/// Nothing else can. An episode of this code is closed by its source
|
||||
/// succeeding, and a source deleted through the API or dropped by a config
|
||||
/// import never succeeds again: the drain above walks the status table, the
|
||||
/// deleted source has no entry in it, and the resolved-row pruning never
|
||||
/// touches an active row. Without this the operator keeps a warning about a
|
||||
/// list they removed on purpose, and no restart clears it.
|
||||
///
|
||||
/// The status table holds every source at every flush site — `refreshAll`
|
||||
/// syncs it before it refreshes anything and a reload rebuilds it from the
|
||||
/// rows — so its keys are exactly the episodes that may stay open.
|
||||
fn resolveDeletedSources(self: *Manager, io: std.Io, store: *events.Store, now_s: i64) void {
|
||||
var storage: [events.Store.max_kept_keys][events.Store.max_subject_key_len]u8 = undefined;
|
||||
var lens: [events.Store.max_kept_keys]u16 = undefined;
|
||||
var len: usize = 0;
|
||||
{
|
||||
// Shared: this reads the table and changes nothing in it. The keys
|
||||
// are copied out because the arena they live in is freed by the
|
||||
// next `installStatuses`, and the store is called below with
|
||||
// nothing held.
|
||||
self.lock.lockSharedUncancelable(io);
|
||||
defer self.lock.unlockShared(io);
|
||||
|
||||
// An empty table before the first published snapshot means "no
|
||||
// source set has been read yet", not "every source was deleted".
|
||||
// Sweeping on it would close every episode the last run left open,
|
||||
// and the pass that follows would reopen each one as a new episode
|
||||
// with its history reset.
|
||||
if (self.generation == 0) return;
|
||||
|
||||
for (self.statuses) |*entry| {
|
||||
if (entry.event_key_len == 0) continue;
|
||||
// `resolveExcept` refuses a kept list longer than
|
||||
// `max_kept_keys`, because it canonicalizes onto the stack.
|
||||
// Over that many keyed sources the sweep is skipped whole: the
|
||||
// alternative is a truncated kept list, which would close
|
||||
// episodes that are still true. A source deleted while the
|
||||
// household is over the cap keeps its episode until the count
|
||||
// falls back under it.
|
||||
if (len == storage.len) return;
|
||||
const key = entry.eventKey();
|
||||
@memcpy(storage[len][0..key.len], key);
|
||||
lens[len] = entry.event_key_len;
|
||||
len += 1;
|
||||
}
|
||||
}
|
||||
|
||||
var kept: [events.Store.max_kept_keys][]const u8 = undefined;
|
||||
for (0..len) |i| kept[i] = storage[i][0..lens[i]];
|
||||
store.resolveExcept(io, now_s, .blocklist_refresh, kept[0..len]);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// reload
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -407,12 +713,35 @@ pub const Manager = struct {
|
||||
/// table keeps describing that snapshot too — the table is rebuilt off to
|
||||
/// the side and the load findings are written into it there, so a reload
|
||||
/// that never publishes changes neither.
|
||||
///
|
||||
/// A standalone reload is its own pass, and `writer_lock` is what serializes
|
||||
/// it against every other writer of the status table. So it flushes inside
|
||||
/// that lock: the load outcomes it wrote at the swap are drained before any
|
||||
/// other pass can add its own to the same entries, which is what keeps two
|
||||
/// failing passes two occurrences instead of one.
|
||||
pub fn reload(self: *Manager, io: std.Io) Error!void {
|
||||
// Must not be entered with `writer_lock` held.
|
||||
self.writer_lock.lockUncancelable(io);
|
||||
defer self.writer_lock.unlock(io);
|
||||
// Registered after the unlock so it runs before it, and `defer` and not
|
||||
// straight-line code after the call: a reload that fails has already
|
||||
// collected the outcomes that explain why, and leaving them pending
|
||||
// would hold them until some later pass flushed them under the wrong
|
||||
// timestamp.
|
||||
defer self.flushDiagnostics(io);
|
||||
return self.reloadLocked(io);
|
||||
}
|
||||
|
||||
/// `reload` without the flush, for a caller that is inside a pass with a
|
||||
/// flush of its own. One pass flushes once: flushing here as well would
|
||||
/// split the pass's outcomes across two reports.
|
||||
fn reloadCollecting(self: *Manager, io: std.Io) Error!void {
|
||||
// Must not be entered with `writer_lock` held.
|
||||
self.writer_lock.lockUncancelable(io);
|
||||
defer self.writer_lock.unlock(io);
|
||||
try self.reloadLocked(io);
|
||||
}
|
||||
|
||||
fn reloadLocked(self: *Manager, io: std.Io) Error!void {
|
||||
var rows = try sources_repo.listSourceRows(self.database, self.gpa);
|
||||
defer rows.deinit(self.gpa);
|
||||
@@ -543,6 +872,7 @@ pub const Manager = struct {
|
||||
rows.items.len,
|
||||
memory_bytes,
|
||||
});
|
||||
self.noteSnapshot(io, null);
|
||||
}
|
||||
|
||||
/// What one enabled source contributes to the snapshot being built. Nothing
|
||||
@@ -642,6 +972,11 @@ pub const Manager = struct {
|
||||
pub fn refreshSource(self: *Manager, io: std.Io, row: sources_repo.SourceRow) Error!bool {
|
||||
self.refresh_lock.lockUncancelable(io);
|
||||
defer self.refresh_lock.unlock(io);
|
||||
// Registered after the unlock, so it runs before it: a pass drains its
|
||||
// own outcomes while it still holds `refresh_lock`. `defer` at all, so
|
||||
// a refresh that fails outright still reports what it collected instead
|
||||
// of leaving it for an unrelated later flush.
|
||||
defer self.flushDiagnostics(io);
|
||||
return self.refreshSourceLocked(io, row);
|
||||
}
|
||||
|
||||
@@ -706,6 +1041,9 @@ pub const Manager = struct {
|
||||
pub fn refreshAll(self: *Manager, io: std.Io) Error!void {
|
||||
self.refresh_lock.lockUncancelable(io);
|
||||
defer self.refresh_lock.unlock(io);
|
||||
// Inside `refresh_lock`, by being registered after the unlock: see
|
||||
// `refreshSource`.
|
||||
defer self.flushDiagnostics(io);
|
||||
|
||||
var rows = try sources_repo.listSourceRows(self.database, self.gpa);
|
||||
defer rows.deinit(self.gpa);
|
||||
@@ -718,8 +1056,9 @@ pub const Manager = struct {
|
||||
_ = try self.refreshSourceLocked(io, row);
|
||||
}
|
||||
// `reload` takes `writer_lock`, which the pass has been careful not to
|
||||
// hold: the order is `refresh_lock` first, always.
|
||||
return self.reload(io);
|
||||
// hold: the order is `refresh_lock` first, always. The collecting
|
||||
// variant, because the `defer` above is this pass's one flush.
|
||||
return self.reloadCollecting(io);
|
||||
}
|
||||
|
||||
/// The three temporary files one refresh compiles into, before the header
|
||||
@@ -1161,9 +1500,16 @@ pub const Manager = struct {
|
||||
// ask the same filesystem for.
|
||||
try self.sweepOrphans(io);
|
||||
|
||||
// `startupPass` flushes its own outcomes before it releases
|
||||
// `refresh_lock`, so the only flush left here is the one the failure
|
||||
// note below needs.
|
||||
self.startupPass(io) catch |err| switch (err) {
|
||||
error.Canceled => return error.Canceled,
|
||||
else => log.warn("blocklist startup pass failed: {s}", .{@errorName(err)}),
|
||||
else => {
|
||||
log.warn("blocklist startup pass failed: {s}", .{@errorName(err)});
|
||||
self.noteSnapshot(io, @errorName(err));
|
||||
self.flushDiagnostics(io);
|
||||
},
|
||||
};
|
||||
if (!self.update.enabled) return;
|
||||
|
||||
@@ -1175,18 +1521,32 @@ pub const Manager = struct {
|
||||
};
|
||||
while (true) {
|
||||
try interval.sleep(io);
|
||||
// Ahead of the gate as well as ahead of the pass: the sweep only
|
||||
// unlinks, so it is the one thing here that can give a critically
|
||||
// full disk room back, and gating it would keep the residue that
|
||||
// helped fill the disk in the first place.
|
||||
try self.scheduledPass(io);
|
||||
}
|
||||
}
|
||||
|
||||
/// What one elapsed interval does. Split from the loop above so a test can
|
||||
/// run the pass without waiting the interval out; nothing in production
|
||||
/// calls it but `runScheduler`.
|
||||
pub fn scheduledPass(self: *Manager, io: std.Io) std.Io.Cancelable!void {
|
||||
// Ahead of the gate as well as ahead of the refresh: the sweep only
|
||||
// unlinks, so it is the one thing here that can give a critically full
|
||||
// disk room back, and gating it would keep the residue that helped fill
|
||||
// the disk in the first place.
|
||||
try self.sweepOrphans(io);
|
||||
if (self.refreshGated()) continue;
|
||||
if (self.refreshGated()) return;
|
||||
// `refreshAll` flushes the pass itself, so the only flush left here is
|
||||
// the one the failure note below needs: flushing unconditionally would
|
||||
// report every outcome of the pass a second time.
|
||||
self.refreshAll(io) catch |err| switch (err) {
|
||||
error.Canceled => return error.Canceled,
|
||||
else => log.warn("blocklist refresh pass failed: {s}", .{@errorName(err)}),
|
||||
else => {
|
||||
log.warn("blocklist refresh pass failed: {s}", .{@errorName(err)});
|
||||
self.noteSnapshot(io, @errorName(err));
|
||||
self.flushDiagnostics(io);
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// `pruneOrphans` with its failure absorbed. Leftover bytes under
|
||||
/// `<data_dir>/blocklists/` are not an outage, and a sweep that could not
|
||||
@@ -1197,10 +1557,16 @@ pub const Manager = struct {
|
||||
/// Taken from outside every `*Locked` body: `pruneOrphans` takes both
|
||||
/// writer mutexes itself and neither is reentrant.
|
||||
fn sweepOrphans(self: *Manager, io: std.Io) std.Io.Cancelable!void {
|
||||
self.pruneOrphans(io) catch |err| switch (err) {
|
||||
if (self.pruneOrphans(io)) {
|
||||
self.noteStorageSuccess(io, .sweep);
|
||||
} else |err| switch (err) {
|
||||
error.Canceled => return error.Canceled,
|
||||
else => log.warn("pruning orphaned blocklist files failed: {s}", .{@errorName(err)}),
|
||||
};
|
||||
else => {
|
||||
log.warn("pruning orphaned blocklist files failed: {s}", .{@errorName(err)});
|
||||
self.noteStorageFailure(io, .sweep, "pruning orphaned blocklist files failed: {s}", .{@errorName(err)});
|
||||
},
|
||||
}
|
||||
self.flushDiagnostics(io);
|
||||
}
|
||||
|
||||
/// The §11.6 gate, consulted by scheduled passes only (ruling 17). A
|
||||
@@ -1232,11 +1598,14 @@ pub const Manager = struct {
|
||||
fn startupPass(self: *Manager, io: std.Io) Error!void {
|
||||
self.refresh_lock.lockUncancelable(io);
|
||||
defer self.refresh_lock.unlock(io);
|
||||
// Inside `refresh_lock`, by being registered after the unlock: see
|
||||
// `refreshSource`.
|
||||
defer self.flushDiagnostics(io);
|
||||
|
||||
// Ahead of the gate on purpose: loading the compiled files that already
|
||||
// exist is a read. A full disk must not cost the household its
|
||||
// filtering as well as its downloads.
|
||||
try self.reload(io);
|
||||
try self.reloadCollecting(io);
|
||||
|
||||
if (self.refreshGated()) return;
|
||||
|
||||
@@ -1251,7 +1620,7 @@ pub const Manager = struct {
|
||||
if (!self.needsRefresh(io, row, now)) continue;
|
||||
if (try self.refreshSourceLocked(io, row)) refreshed = true;
|
||||
}
|
||||
if (refreshed) try self.reload(io);
|
||||
if (refreshed) try self.reloadCollecting(io);
|
||||
}
|
||||
|
||||
fn needsRefresh(self: *Manager, io: std.Io, row: sources_repo.SourceRow, now: i64) bool {
|
||||
@@ -1297,6 +1666,13 @@ pub const Manager = struct {
|
||||
/// `<data_dir>/blocklists/` if nothing has yet, and an empty directory
|
||||
/// sweeps to nothing.
|
||||
pub fn pruneOrphans(self: *Manager, io: std.Io) Error!void {
|
||||
defer self.flushDiagnostics(io);
|
||||
return self.pruneOrphansLocked(io);
|
||||
}
|
||||
|
||||
/// Assumes nothing and takes both writer mutexes itself. Split from
|
||||
/// `pruneOrphans` so the diagnostics flush above happens with neither held.
|
||||
fn pruneOrphansLocked(self: *Manager, io: std.Io) Error!void {
|
||||
// `refresh_lock` first, and for the reason it exists: the download and
|
||||
// the compile are the only writers of `.raw.tmp`, `.list.tmp`,
|
||||
// `.wild.tmp` and `.allow.tmp`, and they hold it for as long as they
|
||||
@@ -1334,6 +1710,7 @@ pub const Manager = struct {
|
||||
error.Canceled => return error.Canceled,
|
||||
else => {
|
||||
log.warn("pruning blocklists: reading the directory failed: {s}", .{@errorName(err)});
|
||||
self.noteStorageFailure(io, .directory_read, "reading the blocklist directory failed: {s}", .{@errorName(err)});
|
||||
return error.FileSystem;
|
||||
},
|
||||
} orelse break;
|
||||
@@ -1343,6 +1720,8 @@ pub const Manager = struct {
|
||||
try doomed.append(self.gpa, try self.gpa.dupe(u8, entry.name));
|
||||
}
|
||||
|
||||
self.noteStorageSuccess(io, .directory_read);
|
||||
|
||||
for (doomed.items) |name| {
|
||||
self.deleteQuietly(io, dir, name);
|
||||
log.info("pruned orphaned blocklist file {s}", .{name});
|
||||
@@ -1381,8 +1760,21 @@ pub const Manager = struct {
|
||||
}
|
||||
|
||||
/// Publishes a built table and frees the one it replaces. The caller holds
|
||||
/// the exclusive lock, so no reader is inside the old table.
|
||||
/// the exclusive lock, so no reader is inside the old table and no flush is
|
||||
/// half way through draining it.
|
||||
///
|
||||
/// The pass accounting the live table still holds is folded into the
|
||||
/// incoming entry of the same id first. `mergeStatuses` left the candidate
|
||||
/// carrying none, so an outcome recorded after the candidate was built —
|
||||
/// and any a flush has not drained yet — survives the swap exactly once. An
|
||||
/// outcome a flush already reported is zero in the live table, so nothing
|
||||
/// here resurrects it.
|
||||
fn installStatuses(self: *Manager, table: StatusTable) void {
|
||||
for (table.items) |*incoming| {
|
||||
const live = entryFor(self.statuses, incoming.id) orelse continue;
|
||||
incoming.pass_failures +|= live.pass_failures;
|
||||
incoming.pass_outcome = incoming.pass_outcome or live.pass_outcome;
|
||||
}
|
||||
self.status_arena.deinit();
|
||||
self.status_arena = table.arena;
|
||||
self.statuses = table.items;
|
||||
@@ -1449,29 +1841,46 @@ pub const Manager = struct {
|
||||
error.Canceled => return error.Canceled,
|
||||
else => {
|
||||
log.warn("creating {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
|
||||
self.noteStorageFailure(io, .create_dir, "creating {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
|
||||
return error.FileSystem;
|
||||
},
|
||||
};
|
||||
return self.paths.dir.openDir(io, self.paths.subdir, options) catch |err| switch (err) {
|
||||
self.noteStorageSuccess(io, .create_dir);
|
||||
|
||||
const dir = self.paths.dir.openDir(io, self.paths.subdir, options) catch |err| switch (err) {
|
||||
error.Canceled => return error.Canceled,
|
||||
else => {
|
||||
log.warn("opening {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
|
||||
self.noteStorageFailure(io, .open_dir, "opening {s} failed: {s}", .{ self.paths.subdir, @errorName(err) });
|
||||
return error.FileSystem;
|
||||
},
|
||||
};
|
||||
self.noteStorageSuccess(io, .open_dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
/// A temporary that cannot be removed is not a failure of the operation
|
||||
/// that made it, but it is not nothing either: it is left visible.
|
||||
fn deleteQuietly(self: *Manager, io: std.Io, dir: std.Io.Dir, name: []const u8) void {
|
||||
_ = self;
|
||||
dir.deleteFile(io, name) catch |err| switch (err) {
|
||||
error.FileNotFound => {},
|
||||
else => log.warn("deleting {s} failed: {s}", .{ name, @errorName(err) }),
|
||||
};
|
||||
if (dir.deleteFile(io, name)) {
|
||||
self.noteStorageSuccess(io, .delete);
|
||||
} else |err| switch (err) {
|
||||
// A name that was never created is the ordinary case: the temporary
|
||||
// deletes are installed before the files exist.
|
||||
error.FileNotFound => self.noteStorageSuccess(io, .delete),
|
||||
else => {
|
||||
log.warn("deleting {s} failed: {s}", .{ name, @errorName(err) });
|
||||
self.noteStorageFailure(io, .delete, "deleting {s} failed: {s}", .{ name, @errorName(err) });
|
||||
},
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// The one subject `blocklist.snapshot` ever has: a box publishes exactly one
|
||||
/// snapshot, and every source that failed to load is its own
|
||||
/// `blocklist.refresh` episode.
|
||||
const snapshot_key = "snapshot";
|
||||
|
||||
/// A status table and the arena holding it. Until `installStatuses` takes it,
|
||||
/// it is a candidate nobody can see, and `deinit` frees it whole.
|
||||
const StatusTable = struct {
|
||||
@@ -1488,6 +1897,14 @@ const StatusTable = struct {
|
||||
/// over from `previous`. A source deleted since `previous` was built is gone; a
|
||||
/// source added since starts blank. `previous` is only read, so the caller's
|
||||
/// published table is untouched by this.
|
||||
///
|
||||
/// The pass accounting is *not* carried: it lives in exactly one table copy at
|
||||
/// a time. `previous` is a snapshot of the published table taken outside the
|
||||
/// swap, so copying its counters here would leave the same outcomes in two
|
||||
/// tables — the live one for a flush to drain, and this candidate for the
|
||||
/// reload's own flush to report a second time. A candidate holds only what
|
||||
/// `applyLoadOutcomes` writes into it; what the live table holds is folded in
|
||||
/// by `installStatuses` under the exclusive lock.
|
||||
fn mergeStatuses(
|
||||
table: []SourceStatus,
|
||||
rows: []const sources_repo.SourceRow,
|
||||
@@ -1500,6 +1917,8 @@ fn mergeStatuses(
|
||||
status.* = prior;
|
||||
break;
|
||||
}
|
||||
status.pass_outcome = false;
|
||||
status.pass_failures = 0;
|
||||
// After the carry-over: a url edited on the row wins over the one the
|
||||
// prior entry recorded.
|
||||
status.setUrl(row.url);
|
||||
@@ -1716,6 +2135,7 @@ fn containsId(rows: []const sources_repo.SourceRow, id: i64) bool {
|
||||
// real swaps under load are the integration suite's (S9).
|
||||
|
||||
const testing = std.testing;
|
||||
const events_fixture = @import("../storage/events_fixture.zig");
|
||||
const migrations = @import("../storage/migrations.zig");
|
||||
|
||||
fn openMigrated() !db.Db {
|
||||
@@ -2275,6 +2695,13 @@ test "a candidate table carries prior entries over and leaves the published one
|
||||
try testing.expectEqual(State.never_fetched, candidate[1].state);
|
||||
try testing.expect(!candidate[1].loaded);
|
||||
|
||||
// The one thing a candidate does not carry. `published[0]` is holding a
|
||||
// failure no flush has drained yet; copying its accounting here would leave
|
||||
// the same outcome in two tables, and the flush of each would report it.
|
||||
try testing.expect(published[0].pass_outcome);
|
||||
try testing.expect(!candidate[0].pass_outcome);
|
||||
try testing.expectEqual(@as(u16, 0), candidate[0].pass_failures);
|
||||
|
||||
// The published table is untouched, so a reload that fails before the swap
|
||||
// leaves it describing the snapshot that is still serving — including the
|
||||
// entry of the deleted source, which that snapshot still enforces.
|
||||
@@ -2284,6 +2711,181 @@ test "a candidate table carries prior entries over and leaves the published one
|
||||
try testing.expectEqualStrings("https://lists.example/one.txt", published[0].urlText());
|
||||
}
|
||||
|
||||
test "installing a table folds the live pass accounting in by id" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
var f: fetcher.Fetcher = undefined;
|
||||
var mgr = try testManager(&database, &f);
|
||||
defer mgr.deinit(io);
|
||||
|
||||
// Source 1 recorded a failure the flush has not drained; source 2 was
|
||||
// drained already; source 3 recorded one this reload knows nothing about.
|
||||
var live: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
const live_items = try live.allocator().alloc(SourceStatus, 3);
|
||||
live_items[0] = .{ .id = 1, .pass_outcome = true, .pass_failures = 2 };
|
||||
live_items[1] = .{ .id = 2 };
|
||||
live_items[2] = .{ .id = 3, .pass_outcome = true, .pass_failures = 1 };
|
||||
{
|
||||
mgr.lock.lockUncancelable(io);
|
||||
defer mgr.lock.unlock(io);
|
||||
mgr.installStatuses(.{ .arena = live, .items = live_items });
|
||||
}
|
||||
|
||||
// What a reload built beside it, carrying only its own load outcomes.
|
||||
var incoming: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
const incoming_items = try incoming.allocator().alloc(SourceStatus, 3);
|
||||
incoming_items[0] = .{ .id = 1, .pass_outcome = true, .pass_failures = 1 };
|
||||
incoming_items[1] = .{ .id = 2, .pass_outcome = true, .pass_failures = 4 };
|
||||
incoming_items[2] = .{ .id = 3 };
|
||||
{
|
||||
mgr.lock.lockUncancelable(io);
|
||||
defer mgr.lock.unlock(io);
|
||||
mgr.installStatuses(.{ .arena = incoming, .items = incoming_items });
|
||||
}
|
||||
|
||||
try testing.expectEqual(@as(u16, 3), mgr.statuses[0].pass_failures);
|
||||
try testing.expect(mgr.statuses[0].pass_outcome);
|
||||
|
||||
// A drained entry adds nothing: what the swap publishes is the reload's own
|
||||
// accounting and no resurrection of what was already reported.
|
||||
try testing.expectEqual(@as(u16, 4), mgr.statuses[1].pass_failures);
|
||||
try testing.expect(mgr.statuses[1].pass_outcome);
|
||||
|
||||
// The half the swap used to lose: an outcome the live table held and the
|
||||
// candidate never saw.
|
||||
try testing.expectEqual(@as(u16, 1), mgr.statuses[2].pass_failures);
|
||||
try testing.expect(mgr.statuses[2].pass_outcome);
|
||||
}
|
||||
|
||||
test "the flush claims every entry that carries pass accounting, once" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1_700_000_000);
|
||||
defer fx.deinit();
|
||||
|
||||
var f: fetcher.Fetcher = undefined;
|
||||
var mgr = try testManager(&database, &f);
|
||||
defer mgr.deinit(io);
|
||||
mgr.diagnostics = &fx.store;
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
const items = try arena.allocator().alloc(SourceStatus, 3);
|
||||
items[0] = .{ .id = 1 };
|
||||
items[0].setUrl("https://lists.example/one.txt");
|
||||
items[0].fail(.fetch_failed, "HttpStatus");
|
||||
// No url, so no episode to report under. The drain has to claim it anyway:
|
||||
// an entry left with `pass_outcome` set is the one every later scan finds
|
||||
// first, and the entry behind it would never be reached.
|
||||
items[1] = .{ .id = 2, .pass_outcome = true, .pass_failures = 1 };
|
||||
items[2] = .{ .id = 3 };
|
||||
items[2].setUrl("https://lists.example/three.txt");
|
||||
items[2].succeed(1_700_000_000, .{ .domains = 3 });
|
||||
{
|
||||
mgr.lock.lockUncancelable(io);
|
||||
defer mgr.lock.unlock(io);
|
||||
mgr.installStatuses(.{ .arena = arena, .items = items });
|
||||
}
|
||||
|
||||
mgr.flushDiagnostics(io);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count(
|
||||
"SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh'",
|
||||
));
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count(
|
||||
"SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
|
||||
));
|
||||
for (mgr.statuses) |entry| {
|
||||
try testing.expect(!entry.pass_outcome);
|
||||
try testing.expectEqual(@as(u16, 0), entry.pass_failures);
|
||||
}
|
||||
|
||||
// Drained: flushing the same table again reports nothing a second time.
|
||||
mgr.flushDiagnostics(io);
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count(
|
||||
"SELECT occurrences FROM operational_events WHERE code = 'blocklist.refresh'",
|
||||
));
|
||||
}
|
||||
|
||||
test "the flush closes the episode of a source that is no longer in the table" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1_700_000_000);
|
||||
defer fx.deinit();
|
||||
|
||||
var f: fetcher.Fetcher = undefined;
|
||||
var mgr = try testManager(&database, &f);
|
||||
defer mgr.deinit(io);
|
||||
mgr.diagnostics = &fx.store;
|
||||
|
||||
// What the operator deleted while it was failing. Nothing will ever record
|
||||
// a success for it, so nothing but the sweep can close this.
|
||||
fx.store.report(
|
||||
io,
|
||||
1_700_000_000,
|
||||
.blocklist_refresh,
|
||||
"https://lists.example/deleted.txt",
|
||||
"lists.example/deleted.txt",
|
||||
.warning,
|
||||
"fetch_failed: HttpStatus (1 this pass)",
|
||||
);
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
const items = try arena.allocator().alloc(SourceStatus, 1);
|
||||
items[0] = .{ .id = 1 };
|
||||
items[0].setUrl("https://lists.example/one.txt");
|
||||
items[0].fail(.fetch_failed, "HttpStatus");
|
||||
{
|
||||
mgr.lock.lockUncancelable(io);
|
||||
defer mgr.lock.unlock(io);
|
||||
mgr.installStatuses(.{ .arena = arena, .items = items });
|
||||
}
|
||||
|
||||
// No snapshot published yet, so the table is not known to describe the
|
||||
// source set and the sweep must not run: the drain reports the failing
|
||||
// source and the deleted one's episode is left alone.
|
||||
mgr.flushDiagnostics(io);
|
||||
try testing.expectEqual(@as(i64, 2), try fx.count(
|
||||
"SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL",
|
||||
));
|
||||
|
||||
mgr.generation = 1;
|
||||
mgr.flushDiagnostics(io);
|
||||
|
||||
// One left active, and it is the source that still exists.
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count(
|
||||
"SELECT count(*) FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL",
|
||||
));
|
||||
try testing.expectEqualStrings(
|
||||
"https://lists.example/one.txt",
|
||||
try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NULL",
|
||||
),
|
||||
);
|
||||
try testing.expectEqualStrings(
|
||||
"https://lists.example/deleted.txt",
|
||||
try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE code = 'blocklist.refresh' AND resolved_at IS NOT NULL",
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
test "a disabled source stops being loaded" {
|
||||
var statuses = [_]SourceStatus{.{ .id = 1 }};
|
||||
statuses[0].succeed(1_700_000_000, .{ .domains = 9 });
|
||||
|
||||
+130
-10
@@ -13,6 +13,8 @@
|
||||
//! through the size.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const events = @import("../storage/events.zig");
|
||||
const tls_server = @import("../platform/tls_server.zig");
|
||||
|
||||
const log = std.log.scoped(.cert_store);
|
||||
@@ -101,6 +103,11 @@ pub const ReloadHook = struct {
|
||||
};
|
||||
|
||||
pub const CertStore = struct {
|
||||
/// Which endpoint's certificate this store holds. It is the subject of
|
||||
/// every `certificate.reload` event, and a box serving both DoH and DoT
|
||||
/// runs two stores over two file pairs.
|
||||
pub const Kind = enum { doh, dot };
|
||||
|
||||
gpa: std.mem.Allocator,
|
||||
/// Borrowed from the config; must outlive the store.
|
||||
cert_path: []const u8,
|
||||
@@ -132,6 +139,12 @@ pub const CertStore = struct {
|
||||
/// `reload` synchronously (that would self-deadlock on `reload_mutex`).
|
||||
after_load_hook: ?ReloadHook,
|
||||
|
||||
/// Set by the composition root right after `init`, with `diagnostics`.
|
||||
kind: Kind = .doh,
|
||||
/// Wired the same way and for the same reason as every other subsystem's:
|
||||
/// the store is fully usable without it, and `nxdns check` has none.
|
||||
diagnostics: ?*events.Store = null,
|
||||
|
||||
reloads: std.atomic.Value(u64),
|
||||
reload_failures: std.atomic.Value(u64),
|
||||
/// Wall-clock second of the last successful load, including the one in
|
||||
@@ -243,7 +256,7 @@ pub const CertStore = struct {
|
||||
};
|
||||
while (true) {
|
||||
try interval.sleep(io);
|
||||
self.pollOnce(io);
|
||||
self.pollOnce(io, std.Io.Clock.real.now(io).toSeconds());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,13 +265,15 @@ pub const CertStore = struct {
|
||||
/// changed, and the old one keeps serving either way. A failed reload
|
||||
/// warns and counts (`reload_failures`); the signature stays at the loaded
|
||||
/// pair, so every subsequent poll retries until the files parse.
|
||||
pub fn pollOnce(self: *CertStore, io: std.Io) void {
|
||||
const cert_sig = statSig(io, self.cert_path) catch {
|
||||
pub fn pollOnce(self: *CertStore, io: std.Io, now_s: i64) void {
|
||||
const cert_sig = statSig(io, self.cert_path) catch |err| {
|
||||
log.warn("stat {s} failed; keeping the loaded certificate", .{self.cert_path});
|
||||
self.reportReload(io, now_s, "stat of the certificate failed", @errorName(err));
|
||||
return;
|
||||
};
|
||||
const key_sig = statSig(io, self.key_path) catch {
|
||||
const key_sig = statSig(io, self.key_path) catch |err| {
|
||||
log.warn("stat {s} failed; keeping the loaded certificate", .{self.key_path});
|
||||
self.reportReload(io, now_s, "stat of the private key failed", @errorName(err));
|
||||
return;
|
||||
};
|
||||
const observed: Signature = .{ .cert = cert_sig, .key = key_sig };
|
||||
@@ -266,18 +281,36 @@ pub const CertStore = struct {
|
||||
self.mutex.lockUncancelable(io);
|
||||
const loaded = self.loaded;
|
||||
self.mutex.unlock(io);
|
||||
if (!changed(loaded, observed)) return;
|
||||
if (!changed(loaded, observed)) {
|
||||
// A poll that stat'ed both files and found nothing to do is a
|
||||
// fully healthy pass, so it closes any episode a transient stat
|
||||
// failure opened. Without this, a file that never changes again
|
||||
// would leave that episode open forever.
|
||||
if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
|
||||
return;
|
||||
}
|
||||
|
||||
if (self.reload(io)) {
|
||||
log.info("certificate reloaded from {s}", .{self.cert_path});
|
||||
if (self.diagnostics) |store| store.resolve(io, now_s, .certificate_reload, @tagName(self.kind));
|
||||
} else |err| {
|
||||
log.warn("certificate reload from {s} failed ({s}); the old certificate keeps serving", .{
|
||||
self.cert_path,
|
||||
humanMessage(err),
|
||||
});
|
||||
self.reportReload(io, now_s, "certificate reload failed", humanMessage(err));
|
||||
}
|
||||
}
|
||||
|
||||
/// A warning, never an error: a stat failure can be a rename window, and a
|
||||
/// failed reload leaves the loaded certificate serving. Nothing is down.
|
||||
fn reportReload(self: *CertStore, io: std.Io, now_s: i64, message: []const u8, reason: []const u8) void {
|
||||
const store = self.diagnostics orelse return;
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "{s}: {s}", .{ message, reason }) catch buf[0..];
|
||||
store.report(io, now_s, .certificate_reload, @tagName(self.kind), @tagName(self.kind), .warning, detail);
|
||||
}
|
||||
|
||||
pub fn snapshotStats(self: *const CertStore) Stats {
|
||||
return .{
|
||||
.reloads = self.reloads.load(.monotonic),
|
||||
@@ -406,6 +439,7 @@ fn statSig(io: std.Io, path: []const u8) !FileSig {
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const events_fixture = @import("../storage/events_fixture.zig");
|
||||
const fixtures = @import("test_fixtures");
|
||||
const testing = std.testing;
|
||||
|
||||
@@ -699,7 +733,7 @@ test "pollOnce reloads on a changed stat pair and stays put on an unchanged one"
|
||||
var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
|
||||
defer store.deinit(io);
|
||||
|
||||
store.pollOnce(io);
|
||||
store.pollOnce(io, 1000);
|
||||
try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
|
||||
|
||||
// Same certificate plus a trailing newline: the PEM still parses and the
|
||||
@@ -710,14 +744,14 @@ test "pollOnce reloads on a changed stat pair and stays put on an unchanged one"
|
||||
|
||||
const old = store.acquire(io);
|
||||
store.release(io, old);
|
||||
store.pollOnce(io);
|
||||
store.pollOnce(io, 1000);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
|
||||
const fresh = store.acquire(io);
|
||||
try testing.expect(fresh != old);
|
||||
store.release(io, fresh);
|
||||
|
||||
store.pollOnce(io);
|
||||
store.pollOnce(io, 1000);
|
||||
try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
|
||||
}
|
||||
|
||||
@@ -734,7 +768,7 @@ test "pollOnce warns and keeps serving when a reload fails" {
|
||||
store.release(io, before);
|
||||
|
||||
try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
|
||||
store.pollOnce(io);
|
||||
store.pollOnce(io, 1000);
|
||||
|
||||
const stats = store.snapshotStats();
|
||||
try testing.expectEqual(@as(u64, 0), stats.reloads);
|
||||
@@ -755,7 +789,7 @@ test "pollOnce does nothing when a file cannot be stat'ed" {
|
||||
defer store.deinit(io);
|
||||
|
||||
try env.tmp.dir.deleteFile(io, "cert.pem");
|
||||
store.pollOnce(io);
|
||||
store.pollOnce(io, 1000);
|
||||
|
||||
const stats = store.snapshotStats();
|
||||
try testing.expectEqual(@as(u64, 0), stats.reloads);
|
||||
@@ -873,3 +907,89 @@ test "a reload overlapping another reload's window publishes last" {
|
||||
store.mutex.unlock(io);
|
||||
try testing.expectEqual(@as(u64, grown.len), final.cert.size);
|
||||
}
|
||||
|
||||
test "a failed reload opens an episode keyed by endpoint kind and a good one closes it" {
|
||||
var env: TestEnv = undefined;
|
||||
try env.init();
|
||||
defer env.deinit();
|
||||
const io = env.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
|
||||
defer store.deinit(io);
|
||||
store.kind = .dot;
|
||||
store.diagnostics = &fx.store;
|
||||
|
||||
try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = "still not a certificate" });
|
||||
store.pollOnce(io, 1000);
|
||||
|
||||
try testing.expectEqualStrings("certificate.reload", try fx.text("SELECT code FROM operational_events"));
|
||||
try testing.expectEqualStrings("dot", try fx.text("SELECT subject_key FROM operational_events"));
|
||||
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
|
||||
|
||||
// The same certificate plus a newline: it parses, and the size differs even
|
||||
// within one timestamp granule.
|
||||
const grown = try std.mem.concat(testing.allocator, u8, &.{ fixtures.cert_pem, "\n" });
|
||||
defer testing.allocator.free(grown);
|
||||
try env.tmp.dir.writeFile(io, .{ .sub_path = "cert.pem", .data = grown });
|
||||
store.pollOnce(io, 1100);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), store.snapshotStats().reloads);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a failed stat opens the same episode a failed reload would" {
|
||||
var env: TestEnv = undefined;
|
||||
try env.init();
|
||||
defer env.deinit();
|
||||
const io = env.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
|
||||
defer store.deinit(io);
|
||||
store.diagnostics = &fx.store;
|
||||
|
||||
try env.tmp.dir.deleteFile(io, "cert.pem");
|
||||
store.pollOnce(io, 1000);
|
||||
|
||||
try testing.expectEqualStrings("doh", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
try testing.expectEqual(@as(u64, 0), store.snapshotStats().reload_failures);
|
||||
}
|
||||
|
||||
test "an unchanged poll closes the episode a transient stat failure opened" {
|
||||
var env: TestEnv = undefined;
|
||||
try env.init();
|
||||
defer env.deinit();
|
||||
const io = env.io();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
var store = try CertStore.init(testing.allocator, io, env.cert_path, env.key_path, null);
|
||||
defer store.deinit(io);
|
||||
store.diagnostics = &fx.store;
|
||||
|
||||
// What a stat failure in a rename window left open. The file it named is
|
||||
// back and unchanged, so no reload will ever close this episode.
|
||||
fx.store.report(io, 1000, .certificate_reload, "doh", "doh", .warning, "stat of the certificate failed: FileNotFound");
|
||||
|
||||
store.pollOnce(io, 1100);
|
||||
|
||||
try testing.expectEqual(@as(u64, 0), store.snapshotStats().reloads);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
+105
-1
@@ -30,6 +30,7 @@ const clients_repo = @import("../storage/repositories/clients_repo.zig");
|
||||
const db = @import("../storage/db.zig");
|
||||
const dns_header = @import("../dns/header.zig");
|
||||
const edns = @import("../dns/edns.zig");
|
||||
const events = @import("../storage/events.zig");
|
||||
const forward_client = @import("../local/forward_client.zig");
|
||||
const local_tables = @import("local_tables.zig");
|
||||
const name_mod = @import("../dns/name.zig");
|
||||
@@ -103,6 +104,9 @@ pub const Resolver = struct {
|
||||
tables: *local_tables.LocalTables,
|
||||
stats: Stats = .{},
|
||||
exchange_fn: ExchangeFn = defaultExchange,
|
||||
/// Wired by the composition root after `init`, following the
|
||||
/// `gate: ?*disk_monitor.Monitor` idiom. Null in every unit test here.
|
||||
diagnostics: ?*events.Store = null,
|
||||
|
||||
pub fn init(tables: *local_tables.LocalTables) Resolver {
|
||||
return .{ .tables = tables };
|
||||
@@ -127,10 +131,13 @@ pub const Resolver = struct {
|
||||
self.mutex.lockUncancelable(io);
|
||||
self.stats.read_failures += 1;
|
||||
self.mutex.unlock(io);
|
||||
self.reportStorage(io, now_s, "read", "selecting clients to name failed", @errorName(err), 1);
|
||||
return;
|
||||
};
|
||||
if (self.diagnostics) |store| store.resolve(io, now_s, .client_names_storage, "read");
|
||||
|
||||
var write_failures: u64 = 0;
|
||||
var last_write_error: []const u8 = "";
|
||||
for (candidates[0..count]) |*candidate| {
|
||||
var learned_buf: [types.max_name_len]u8 = undefined;
|
||||
const result = self.attempt(io, candidate.ip(), &learned_buf);
|
||||
@@ -144,6 +151,7 @@ pub const Resolver = struct {
|
||||
log.warn("recording the name of {s} failed: {s}", .{ candidate.ip(), @errorName(err) });
|
||||
}
|
||||
write_failures += 1;
|
||||
last_write_error = @errorName(err);
|
||||
}
|
||||
|
||||
self.mutex.lockUncancelable(io);
|
||||
@@ -158,10 +166,38 @@ pub const Resolver = struct {
|
||||
self.mutex.unlock(io);
|
||||
}
|
||||
|
||||
if (write_failures == 0) return;
|
||||
// The clean-pass determination is what closes a `write` episode: one
|
||||
// aggregated outcome per pass, so a row's `occurrences` counts failing
|
||||
// passes rather than failing rows.
|
||||
if (write_failures == 0) {
|
||||
if (self.diagnostics) |store| store.resolve(io, now_s, .client_names_storage, "write");
|
||||
return;
|
||||
}
|
||||
self.mutex.lockUncancelable(io);
|
||||
self.stats.write_failures += write_failures;
|
||||
self.mutex.unlock(io);
|
||||
self.reportStorage(io, now_s, "write", "recording a client name failed", last_write_error, write_failures);
|
||||
}
|
||||
|
||||
/// One aggregated report per pass. `count` is how many failures that pass
|
||||
/// held; the detail carries it because the row itself counts passes.
|
||||
fn reportStorage(
|
||||
self: *Resolver,
|
||||
io: std.Io,
|
||||
now_s: i64,
|
||||
operation: []const u8,
|
||||
message: []const u8,
|
||||
error_name: []const u8,
|
||||
count: u64,
|
||||
) void {
|
||||
const store = self.diagnostics orelse return;
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "{s}: {s} ({d} this pass)", .{
|
||||
message,
|
||||
error_name,
|
||||
count,
|
||||
}) catch buf[0..];
|
||||
store.report(io, now_s, .client_names_storage, operation, operation, .warning, detail);
|
||||
}
|
||||
|
||||
const Attempt = struct {
|
||||
@@ -355,6 +391,7 @@ fn exchangeOnce(
|
||||
|
||||
const forward_zones = @import("../local/forward_zones.zig");
|
||||
const migrations = @import("../storage/migrations.zig");
|
||||
const events_fixture = @import("../storage/events_fixture.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
/// A stub exchange whose reply and error are set by the test. `calls` is the
|
||||
@@ -1157,3 +1194,70 @@ test "an extended rcode of zero still reads as the header's rcode" {
|
||||
try testing.expectEqualStrings("nas.lan", (try f.learned("192.168.1.10", &buf)).?);
|
||||
try testing.expectEqual(@as(u64, 1), f.resolver.snapshotStats(f.io()).answered);
|
||||
}
|
||||
|
||||
test "a failing write opens one episode per pass and a clean pass closes it" {
|
||||
var f: Fixture = undefined;
|
||||
try fixture(&f, "168.192.in-addr.arpa");
|
||||
defer f.deinit();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(f.io(), 1000);
|
||||
defer fx.deinit();
|
||||
f.resolver.diagnostics = &fx.store;
|
||||
|
||||
// Two addresses outside the declared zone: both reach the write step
|
||||
// without any exchange, which is not what this test is about.
|
||||
try clients_repo.upsertSeen(&f.database, "10.0.0.1", 1700000000);
|
||||
try clients_repo.upsertSeen(&f.database, "10.0.0.2", 1700000000);
|
||||
try f.database.exec(
|
||||
\\CREATE TRIGGER refuse_update BEFORE UPDATE ON clients
|
||||
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
||||
);
|
||||
|
||||
f.resolver.runPass(f.io(), &f.database, 1700000000);
|
||||
|
||||
// Two failing rows, one aggregated report: the row counts failing passes.
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
try testing.expectEqualStrings("client_names.storage", try fx.text("SELECT code FROM operational_events"));
|
||||
try testing.expectEqualStrings("write", try fx.text("SELECT subject_key FROM operational_events"));
|
||||
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT occurrences FROM operational_events"));
|
||||
|
||||
f.resolver.runPass(f.io(), &f.database, 1700000060);
|
||||
try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
|
||||
|
||||
try f.database.exec("DROP TRIGGER refuse_update;");
|
||||
f.resolver.runPass(f.io(), &f.database, 1700000120);
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a failing candidate select opens a read episode the next clean pass closes" {
|
||||
var f: Fixture = undefined;
|
||||
try fixture(&f, "168.192.in-addr.arpa");
|
||||
defer f.deinit();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(f.io(), 1000);
|
||||
defer fx.deinit();
|
||||
f.resolver.diagnostics = &fx.store;
|
||||
|
||||
try clients_repo.upsertSeen(&f.database, "10.0.0.1", 1700000000);
|
||||
try f.database.exec("ALTER TABLE clients RENAME TO clients_aside;");
|
||||
f.resolver.runPass(f.io(), &f.database, 1700000000);
|
||||
|
||||
try testing.expectEqualStrings("read", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
|
||||
try f.database.exec("ALTER TABLE clients_aside RENAME TO clients;");
|
||||
f.resolver.runPass(f.io(), &f.database, 1700000060);
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ const client_names = @import("client_names.zig");
|
||||
const clients_repo = @import("../storage/repositories/clients_repo.zig");
|
||||
const db = @import("../storage/db.zig");
|
||||
const disk_monitor = @import("../storage/disk_monitor.zig");
|
||||
const events = @import("../storage/events.zig");
|
||||
const logger = @import("../storage/logger.zig");
|
||||
|
||||
const log = std.log.scoped(.clients);
|
||||
@@ -63,6 +64,9 @@ pub const Tracker = struct {
|
||||
count: u32,
|
||||
passes: u64,
|
||||
stats: Stats,
|
||||
/// Wired by the composition root after `init`, following the
|
||||
/// `gate: ?*disk_monitor.Monitor` idiom. Null in every unit test here.
|
||||
diagnostics: ?*events.Store = null,
|
||||
|
||||
/// `retention_days` is `logging.retention_days`, the same knob the query log
|
||||
/// prunes by (milestone-7 ruling 16). A client silent for that long is as
|
||||
@@ -185,6 +189,7 @@ pub const Tracker = struct {
|
||||
|
||||
var flushed: u64 = 0;
|
||||
var failures: u64 = 0;
|
||||
var last_failure: []const u8 = "";
|
||||
for (batch) |entry| {
|
||||
// `logger.max_client_len` is the RFC 5952 bound every address text
|
||||
// in this program is sized by, so `format` cannot fail here.
|
||||
@@ -199,9 +204,19 @@ pub const Tracker = struct {
|
||||
log.warn("materialising client {s} failed: {s}", .{ w.buffered(), @errorName(err) });
|
||||
}
|
||||
failures += 1;
|
||||
last_failure = @errorName(err);
|
||||
}
|
||||
}
|
||||
|
||||
// One aggregated outcome per pass, so the row's `occurrences` counts
|
||||
// failing passes rather than failing addresses. A pass that wrote
|
||||
// nothing resolves nothing: an empty batch is not evidence of success.
|
||||
if (failures != 0) {
|
||||
self.reportStorage(io, now_s, "materialise", "materialising a client failed", last_failure, failures);
|
||||
} else if (flushed != 0) {
|
||||
if (self.diagnostics) |store| store.resolve(io, now_s, .clients_storage, "materialise");
|
||||
}
|
||||
|
||||
self.mutex.lockUncancelable(io);
|
||||
self.passes += 1;
|
||||
self.stats.flushed += flushed;
|
||||
@@ -215,17 +230,39 @@ pub const Tracker = struct {
|
||||
self.mutex.lockUncancelable(io);
|
||||
self.stats.pruned += deleted;
|
||||
self.mutex.unlock(io);
|
||||
if (self.diagnostics) |store| store.resolve(io, now_s, .clients_storage, "prune");
|
||||
} else |err| {
|
||||
log.warn("pruning clients before {d} failed: {s}", .{ cutoff, @errorName(err) });
|
||||
self.mutex.lockUncancelable(io);
|
||||
self.stats.flush_failures += 1;
|
||||
self.mutex.unlock(io);
|
||||
self.reportStorage(io, now_s, "prune", "pruning stale clients failed", @errorName(err), 1);
|
||||
}
|
||||
}
|
||||
|
||||
if (names) |resolver| resolver.runPass(io, database, now_s);
|
||||
}
|
||||
|
||||
/// One aggregated report per pass; `count` is how many failures it held.
|
||||
fn reportStorage(
|
||||
self: *Tracker,
|
||||
io: std.Io,
|
||||
now_s: i64,
|
||||
operation: []const u8,
|
||||
message: []const u8,
|
||||
error_name: []const u8,
|
||||
count: u64,
|
||||
) void {
|
||||
const store = self.diagnostics orelse return;
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "{s}: {s} ({d} this pass)", .{
|
||||
message,
|
||||
error_name,
|
||||
count,
|
||||
}) catch buf[0..];
|
||||
store.report(io, now_s, .clients_storage, operation, operation, .warning, detail);
|
||||
}
|
||||
|
||||
pub fn snapshotStats(self: *Tracker, io: std.Io) Stats {
|
||||
self.mutex.lockUncancelable(io);
|
||||
defer self.mutex.unlock(io);
|
||||
@@ -258,6 +295,7 @@ pub const Tracker = struct {
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const events_fixture = @import("../storage/events_fixture.zig");
|
||||
const migrations = @import("../storage/migrations.zig");
|
||||
|
||||
const testing = std.testing;
|
||||
@@ -649,3 +687,83 @@ test "a gated pass attempts no naming either" {
|
||||
try testing.expectEqual(@as(usize, 0), CountingExchange.calls);
|
||||
try testing.expectEqual(@as(u64, 0), names.snapshotStats(io).attempted);
|
||||
}
|
||||
|
||||
test "a failing materialise opens one episode per pass and a clean pass closes it" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
var tracker: Tracker = .init(30);
|
||||
tracker.diagnostics = &fx.store;
|
||||
|
||||
try database.exec(
|
||||
\\CREATE TRIGGER refuse_insert BEFORE INSERT ON clients
|
||||
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
||||
);
|
||||
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000000);
|
||||
_ = tracker.trackAt(io, parsed("192.168.1.11"), 1700000001);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
try testing.expectEqualStrings("clients.storage", try fx.text("SELECT code FROM operational_events"));
|
||||
try testing.expectEqualStrings("materialise", try fx.text("SELECT subject_key FROM operational_events"));
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT occurrences FROM operational_events"));
|
||||
|
||||
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000060);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
|
||||
|
||||
try database.exec("DROP TRIGGER refuse_insert;");
|
||||
_ = tracker.trackAt(io, parsed("192.168.1.10"), 1700000120);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a failing prune opens its own episode the next due pass closes" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
var tracker: Tracker = .init(30);
|
||||
tracker.diagnostics = &fx.store;
|
||||
// One pass short of due, so the pass below is the pruning one.
|
||||
tracker.passes = Tracker.prune_every_passes - 1;
|
||||
|
||||
try database.exec(
|
||||
\\CREATE TRIGGER refuse_delete BEFORE DELETE ON clients
|
||||
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
||||
);
|
||||
try clients_repo.upsertSeen(&database, "192.168.1.10", 1);
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqualStrings("prune", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
|
||||
try database.exec("DROP TRIGGER refuse_delete;");
|
||||
tracker.passes = Tracker.prune_every_passes - 1;
|
||||
tracker.flushOnce(io, &database, true, null);
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -98,6 +98,57 @@ pub const ddl_v1: [:0]const u8 =
|
||||
\\);
|
||||
\\
|
||||
\\CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
\\
|
||||
\\CREATE TABLE operational_events (
|
||||
\\ id INTEGER PRIMARY KEY,
|
||||
\\ code TEXT NOT NULL,
|
||||
\\ subject_key TEXT NOT NULL,
|
||||
\\ subject_label TEXT NOT NULL,
|
||||
\\ severity TEXT NOT NULL CHECK (severity IN ('warning', 'error')),
|
||||
\\ first_seen INTEGER NOT NULL,
|
||||
\\ last_seen INTEGER NOT NULL,
|
||||
\\ occurrences INTEGER NOT NULL CHECK (occurrences > 0),
|
||||
\\ resolved_at INTEGER,
|
||||
\\ detail TEXT NOT NULL DEFAULT '',
|
||||
\\ CHECK (resolved_at IS NULL OR resolved_at >= first_seen)
|
||||
\\);
|
||||
\\CREATE UNIQUE INDEX idx_operational_events_active
|
||||
\\ ON operational_events(code, subject_key) WHERE resolved_at IS NULL;
|
||||
\\CREATE INDEX idx_operational_events_last_seen
|
||||
\\ ON operational_events(last_seen DESC);
|
||||
;
|
||||
|
||||
/// The same three statements, each made conditional, for the bridge at the end
|
||||
/// of `migrations.migrate`.
|
||||
///
|
||||
/// A database stamped version 1 before `operational_events` joined `ddl_v1`
|
||||
/// never runs step 1 again, so nothing would ever create the table there. The
|
||||
/// bridge closes that divergence for the pre-0.1 installs that exist; it is
|
||||
/// **removable the moment the v0.1 adoption gate lands**, because from then on
|
||||
/// a schema change is an append-only migration step and this hazard cannot
|
||||
/// recur.
|
||||
///
|
||||
/// It must not be folded into `ddl_v1`: a fresh database would then create the
|
||||
/// table twice, and the unconditional `CREATE TABLE` above is what proves the
|
||||
/// baseline and this text stay in step.
|
||||
pub const operational_events_bridge: [:0]const u8 =
|
||||
\\CREATE TABLE IF NOT EXISTS operational_events (
|
||||
\\ id INTEGER PRIMARY KEY,
|
||||
\\ code TEXT NOT NULL,
|
||||
\\ subject_key TEXT NOT NULL,
|
||||
\\ subject_label TEXT NOT NULL,
|
||||
\\ severity TEXT NOT NULL CHECK (severity IN ('warning', 'error')),
|
||||
\\ first_seen INTEGER NOT NULL,
|
||||
\\ last_seen INTEGER NOT NULL,
|
||||
\\ occurrences INTEGER NOT NULL CHECK (occurrences > 0),
|
||||
\\ resolved_at INTEGER,
|
||||
\\ detail TEXT NOT NULL DEFAULT '',
|
||||
\\ CHECK (resolved_at IS NULL OR resolved_at >= first_seen)
|
||||
\\);
|
||||
\\CREATE UNIQUE INDEX IF NOT EXISTS idx_operational_events_active
|
||||
\\ ON operational_events(code, subject_key) WHERE resolved_at IS NULL;
|
||||
\\CREATE INDEX IF NOT EXISTS idx_operational_events_last_seen
|
||||
\\ ON operational_events(last_seen DESC);
|
||||
;
|
||||
|
||||
/// Child-before-parent, and correct under `foreign_keys = ON`. The reconcile
|
||||
@@ -108,7 +159,10 @@ pub const ddl_v1: [:0]const u8 =
|
||||
/// `upstreams`, `local_records`, `forward_zones` and `settings` have no foreign
|
||||
/// keys, so their position is free; `groups` and `blocklist_sources` must come
|
||||
/// last, after every referrer. `schema_version` is deliberately absent — an
|
||||
/// import must never erase the stamped migration version.
|
||||
/// import must never erase the stamped migration version — and so is
|
||||
/// `operational_events`, which is runtime state rather than configuration: an
|
||||
/// import that wiped the diagnostics log would destroy the record of what the
|
||||
/// box has been doing.
|
||||
pub const delete_order = [_][]const u8{
|
||||
"group_sources", "rules", "client_prefixes", "clients",
|
||||
"upstreams", "local_records", "forward_zones", "settings",
|
||||
@@ -120,12 +174,16 @@ pub const delete_order = [_][]const u8{
|
||||
/// that renumbered a group.
|
||||
///
|
||||
/// `schema_version` is absent: it is the migration's, not the operator's.
|
||||
/// `operational_events` is absent for the same class of reason: it is the
|
||||
/// program's own record of its failures, and an export of it would be a log
|
||||
/// dump, not a configuration.
|
||||
pub const table_names = [_][]const u8{
|
||||
"groups", "clients", "client_prefixes", "upstreams",
|
||||
"blocklist_sources", "group_sources", "rules", "local_records",
|
||||
"forward_zones", "settings",
|
||||
};
|
||||
|
||||
const db = @import("db.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
test "table_names names exactly the tables delete_order does" {
|
||||
@@ -135,6 +193,48 @@ test "table_names names exactly the tables delete_order does" {
|
||||
}
|
||||
try testing.expect(indexOf(&table_names, "groups") != null);
|
||||
try testing.expect(indexOf(&table_names, "schema_version") == null);
|
||||
// Runtime state, not configuration: neither list may reach it, or an
|
||||
// import would wipe the diagnostics log and an export would emit it.
|
||||
try testing.expect(indexOf(&table_names, "operational_events") == null);
|
||||
try testing.expect(indexOf(&delete_order, "operational_events") == null);
|
||||
}
|
||||
|
||||
test "the bridge creates exactly what the baseline does" {
|
||||
// The two texts are separate on purpose (a fresh database must not create
|
||||
// the table twice), which is exactly how they could drift. Applying each to
|
||||
// its own database and comparing `sqlite_schema` is what keeps them equal.
|
||||
const baseline = try schemaOf(ddl_v1);
|
||||
defer testing.allocator.free(baseline);
|
||||
const bridged = try schemaOf(operational_events_bridge);
|
||||
defer testing.allocator.free(bridged);
|
||||
|
||||
// SQLite stores the `CREATE` text verbatim, so the conditional is the one
|
||||
// difference the two are allowed to have.
|
||||
const size = std.mem.replacementSize(u8, bridged, " IF NOT EXISTS", "");
|
||||
const plain = try testing.allocator.alloc(u8, size);
|
||||
defer testing.allocator.free(plain);
|
||||
_ = std.mem.replace(u8, bridged, " IF NOT EXISTS", "", plain);
|
||||
|
||||
try testing.expectEqualStrings(baseline, plain);
|
||||
}
|
||||
|
||||
/// Every `sqlite_schema` row of `operational_events`, after applying `sql`.
|
||||
fn schemaOf(sql: [:0]const u8) ![]u8 {
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
defer database.close();
|
||||
try database.exec(sql);
|
||||
|
||||
var out: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
errdefer out.deinit();
|
||||
var stmt = try database.prepare(
|
||||
\\SELECT type, name, sql FROM sqlite_schema
|
||||
\\ WHERE tbl_name = 'operational_events' ORDER BY name
|
||||
);
|
||||
defer stmt.deinit();
|
||||
while (try stmt.step()) {
|
||||
try out.writer.print("{s} {s}\n{s}\n", .{ stmt.columnText(0), stmt.columnText(1), stmt.columnText(2) });
|
||||
}
|
||||
return out.toOwnedSlice();
|
||||
}
|
||||
|
||||
test "delete_order lists every referrer before the table it references" {
|
||||
|
||||
+168
-13
@@ -9,6 +9,7 @@
|
||||
//! milestone-5 file; the gate is pulled, not pushed.
|
||||
|
||||
const std = @import("std");
|
||||
const events = @import("events.zig");
|
||||
const model = @import("../config/model.zig");
|
||||
const statfs = @import("../platform/statfs.zig");
|
||||
|
||||
@@ -89,43 +90,49 @@ pub const Monitor = struct {
|
||||
/// evidence that the disk filled — and a failed size scan leaves that one
|
||||
/// gauge at its previous reading. Every failure increments
|
||||
/// `sample_failures` and logs one line at `warn`.
|
||||
pub fn sample(self: *Monitor, io: std.Io) void {
|
||||
const free = statfs.freeBytes(self.data_path) catch {
|
||||
pub fn sample(self: *Monitor, io: std.Io, store: ?*events.Store, now_s: i64) void {
|
||||
const free = statfs.freeBytes(self.data_path) catch |err| {
|
||||
self.countFailure();
|
||||
log.warn("statvfs on {s} failed", .{self.data_path});
|
||||
probeFailed(store, io, now_s, "statvfs", "statvfs on the data path failed", err);
|
||||
return;
|
||||
};
|
||||
self.free_bytes.store(free, .monotonic);
|
||||
if (store) |s| s.resolve(io, now_s, .disk_probe, "statvfs");
|
||||
|
||||
if (sumDir(io, self.data_dir, isDatabaseFile)) |bytes| {
|
||||
self.db_bytes.store(bytes, .monotonic);
|
||||
if (store) |s| s.resolve(io, now_s, .disk_probe, "data_dir");
|
||||
} else |err| {
|
||||
self.countFailure();
|
||||
log.warn("sizing the data directory failed: {s}", .{@errorName(err)});
|
||||
probeFailed(store, io, now_s, "data_dir", "sizing the data directory failed", err);
|
||||
}
|
||||
|
||||
if (self.log_dir_path) |path| {
|
||||
if (self.sumLogDir(io, path)) |bytes| {
|
||||
self.log_bytes.store(bytes, .monotonic);
|
||||
if (store) |s| s.resolve(io, now_s, .disk_probe, "log_dir");
|
||||
} else |err| {
|
||||
self.countFailure();
|
||||
log.warn("sizing {s} failed: {s}", .{ path, @errorName(err) });
|
||||
probeFailed(store, io, now_s, "log_dir", "sizing the log directory failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
self.publish(classify(free, self.cfg), free);
|
||||
self.publish(io, store, now_s, classify(free, self.cfg), free);
|
||||
}
|
||||
|
||||
/// Sample first, then sleep: a process that starts on a full disk must not
|
||||
/// serve a whole interval believing the state is `.ok`. `.boot` so a
|
||||
/// suspended box still sees the interval elapse.
|
||||
pub fn run(self: *Monitor, io: std.Io) std.Io.Cancelable!void {
|
||||
pub fn run(self: *Monitor, io: std.Io, store: ?*events.Store) std.Io.Cancelable!void {
|
||||
const interval: std.Io.Clock.Duration = .{
|
||||
.raw = .fromSeconds(sample_interval_s),
|
||||
.clock = .boot,
|
||||
};
|
||||
while (true) {
|
||||
self.sample(io);
|
||||
self.sample(io, store, std.Io.Clock.real.now(io).toSeconds());
|
||||
try interval.sleep(io);
|
||||
}
|
||||
}
|
||||
@@ -136,7 +143,14 @@ pub const Monitor = struct {
|
||||
|
||||
/// Logs on transitions only. A disk that sits at `.warn` for a week
|
||||
/// produces one line, not ten thousand.
|
||||
fn publish(self: *Monitor, next: State, free: u64) void {
|
||||
fn publish(
|
||||
self: *Monitor,
|
||||
io: std.Io,
|
||||
store: ?*events.Store,
|
||||
now_s: i64,
|
||||
next: State,
|
||||
free: u64,
|
||||
) void {
|
||||
const previous: State = @enumFromInt(self.state_raw.swap(@intFromEnum(next), .monotonic));
|
||||
if (previous == next) return;
|
||||
log.warn("disk state {t} -> {t}: {d} bytes free on {s}", .{
|
||||
@@ -145,6 +159,24 @@ pub const Monitor = struct {
|
||||
free,
|
||||
self.data_path,
|
||||
});
|
||||
|
||||
const s = store orelse return;
|
||||
if (next == .ok) {
|
||||
s.resolve(io, now_s, .disk_space, disk_space_key);
|
||||
return;
|
||||
}
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "disk state {t} -> {t}: {d} bytes free on {s}", .{
|
||||
previous,
|
||||
next,
|
||||
free,
|
||||
self.data_path,
|
||||
}) catch buf[0..];
|
||||
s.report(io, now_s, .disk_space, disk_space_key, "data directory", switch (next) {
|
||||
.warn => .warning,
|
||||
.critical => .@"error",
|
||||
.ok => unreachable,
|
||||
}, detail);
|
||||
}
|
||||
|
||||
fn sumLogDir(self: *Monitor, io: std.Io, path: [:0]const u8) !u64 {
|
||||
@@ -155,6 +187,26 @@ pub const Monitor = struct {
|
||||
}
|
||||
};
|
||||
|
||||
/// The one subject `disk.space` ever has: this box has exactly one data
|
||||
/// directory, and its filesystem is what the thresholds classify.
|
||||
const disk_space_key = "data";
|
||||
|
||||
/// Every probe failure is a warning, not an error: an unreadable filesystem is
|
||||
/// a gap in what the monitor can see, and the state it published last stands.
|
||||
fn probeFailed(
|
||||
store: ?*events.Store,
|
||||
io: std.Io,
|
||||
now_s: i64,
|
||||
operation: []const u8,
|
||||
message: []const u8,
|
||||
err: anyerror,
|
||||
) void {
|
||||
const s = store orelse return;
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "{s}: {s}", .{ message, @errorName(err) }) catch buf[0..];
|
||||
s.report(io, now_s, .disk_probe, operation, operation, .warning, detail);
|
||||
}
|
||||
|
||||
fn everyFile(_: []const u8) bool {
|
||||
return true;
|
||||
}
|
||||
@@ -190,6 +242,7 @@ fn sumDir(io: std.Io, dir: std.Io.Dir, accept: *const fn ([]const u8) bool) !u64
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const events_fixture = @import("events_fixture.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
const mb = 1024 * 1024;
|
||||
@@ -280,7 +333,7 @@ test "a sample sizes the databases and ignores every other file" {
|
||||
try tmp.dir.writeFile(io, .{ .sub_path = "notes.txt", .data = &[_]u8{'d'} ** 4096 });
|
||||
|
||||
var monitor: Monitor = .init(.{ .min_free_mb = 0, .warn_free_mb = 0 }, tmp.dir, ".", null);
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, null, 0);
|
||||
|
||||
const g = monitor.gauges();
|
||||
try testing.expectEqual(@as(u64, 160), g.db_bytes);
|
||||
@@ -308,7 +361,7 @@ test "a sample sizes every file in the log directory" {
|
||||
const log_path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/logs", .{tmp.sub_path});
|
||||
|
||||
var monitor: Monitor = .init(.{ .min_free_mb = 0, .warn_free_mb = 0 }, tmp.dir, ".", log_path);
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, null, 0);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1000), monitor.gauges().log_bytes);
|
||||
try testing.expectEqual(@as(u64, 0), monitor.sample_failures.load(.monotonic));
|
||||
@@ -321,7 +374,7 @@ test "a failed statvfs counts and keeps the previous state" {
|
||||
|
||||
var monitor: Monitor = .init(.{}, std.Io.Dir.cwd(), "./nxdns-no-such-path-7c21", null);
|
||||
monitor.state_raw.store(@intFromEnum(State.warn), .monotonic);
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, null, 0);
|
||||
|
||||
try testing.expectEqual(State.warn, monitor.state());
|
||||
try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
|
||||
@@ -342,7 +395,7 @@ test "an unreadable log directory counts a failure but still publishes a state"
|
||||
".",
|
||||
"./nxdns-no-such-dir-4f8a",
|
||||
);
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, null, 0);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
|
||||
try testing.expectEqual(State.ok, monitor.state());
|
||||
@@ -386,7 +439,7 @@ test "an unreadable data directory fails the scan and keeps the previous gauge"
|
||||
// The handle keeps its read permission from open time, so `iterate` still
|
||||
// lists the file, but path resolution under the directory now fails.
|
||||
try tmp.dir.setPermissions(io, .fromMode(0o600));
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, null, 0);
|
||||
try tmp.dir.setPermissions(io, .fromMode(0o700));
|
||||
|
||||
try testing.expectEqual(@as(u64, 4096), monitor.gauges().db_bytes);
|
||||
@@ -409,13 +462,115 @@ test "a threshold above the real free space drives the state to critical" {
|
||||
".",
|
||||
null,
|
||||
);
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, null, 0);
|
||||
try testing.expectEqual(State.critical, monitor.state());
|
||||
try testing.expect(!monitor.writesAllowed());
|
||||
|
||||
monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, null, 0);
|
||||
try testing.expectEqual(State.ok, monitor.state());
|
||||
try testing.expect(monitor.writesAllowed());
|
||||
try testing.expectEqual(@as(u64, 0), monitor.sample_failures.load(.monotonic));
|
||||
}
|
||||
|
||||
test "a disk transition records an episode per severity and closes it on recovery" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var tmp = testing.tmpDir(.{ .iterate = true });
|
||||
defer tmp.cleanup();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
const unreachable_mb = std.math.maxInt(u32);
|
||||
var monitor: Monitor = .init(
|
||||
.{ .min_free_mb = unreachable_mb, .warn_free_mb = unreachable_mb },
|
||||
tmp.dir,
|
||||
".",
|
||||
null,
|
||||
);
|
||||
monitor.sample(io, &fx.store, 1000);
|
||||
try testing.expectEqual(State.critical, monitor.state());
|
||||
try testing.expectEqualStrings("disk.space", try fx.text(
|
||||
"SELECT code FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
try testing.expectEqualStrings("error", try fx.text(
|
||||
"SELECT severity FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
try testing.expectEqualStrings("data", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
|
||||
// A second critical sample is the same episode, not a second row: `publish`
|
||||
// only reports on a transition.
|
||||
monitor.sample(io, &fx.store, 1060);
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
|
||||
monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
|
||||
monitor.sample(io, &fx.store, 1120);
|
||||
try testing.expectEqual(State.ok, monitor.state());
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
try testing.expectEqual(@as(i64, 1120), try fx.count("SELECT resolved_at FROM operational_events"));
|
||||
}
|
||||
|
||||
test "a failed probe opens an episode the next clean pass closes" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var tmp = testing.tmpDir(.{ .iterate = true });
|
||||
defer tmp.cleanup();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
var monitor: Monitor = .init(
|
||||
.{ .min_free_mb = 0, .warn_free_mb = 0 },
|
||||
tmp.dir,
|
||||
".",
|
||||
"./nxdns-no-such-dir-4f8a",
|
||||
);
|
||||
monitor.sample(io, &fx.store, 1000);
|
||||
|
||||
try testing.expectEqualStrings("disk.probe", try fx.text(
|
||||
"SELECT code FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
try testing.expectEqualStrings("log_dir", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
try testing.expectEqualStrings("warning", try fx.text(
|
||||
"SELECT severity FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
|
||||
try tmp.dir.createDirPath(io, "logs");
|
||||
var path_buf: [256]u8 = undefined;
|
||||
monitor.log_dir_path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/logs", .{tmp.sub_path});
|
||||
monitor.sample(io, &fx.store, 1100);
|
||||
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "every emit site is inert when the store is absent" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var monitor: Monitor = .init(
|
||||
.{ .min_free_mb = std.math.maxInt(u32), .warn_free_mb = std.math.maxInt(u32) },
|
||||
std.Io.Dir.cwd(),
|
||||
"./nxdns-no-such-path-7c21",
|
||||
"./nxdns-no-such-dir-4f8a",
|
||||
);
|
||||
monitor.sample(io, null, 0);
|
||||
try testing.expectEqual(@as(u64, 1), monitor.sample_failures.load(.monotonic));
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,47 @@
|
||||
//! Test support: a migrated in-memory `config.db` with an `events.Store` over
|
||||
//! it, for the emitter tests that live beside their own subsystem.
|
||||
//!
|
||||
//! Every emitter takes `?*events.Store` and must work with `null`; the tests
|
||||
//! that prove an emitter *does* record something need a real store, and nine
|
||||
//! subsystems needing the same six lines is what this file removes.
|
||||
//!
|
||||
//! Built in place rather than returned by value: a `Store` holds a `*db.Db`, so
|
||||
//! a fixture that moved after `Store.init` would leave that pointer behind.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const db = @import("db.zig");
|
||||
const events = @import("events.zig");
|
||||
const migrations = @import("migrations.zig");
|
||||
|
||||
pub const Fixture = struct {
|
||||
database: db.Db = undefined,
|
||||
store: events.Store = undefined,
|
||||
text_buf: [1024]u8 = undefined,
|
||||
|
||||
pub fn init(self: *Fixture, io: std.Io, now_s: i64) !void {
|
||||
self.database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
errdefer self.database.close();
|
||||
try db.applyPragmas(&self.database, .{});
|
||||
_ = try migrations.migrate(&self.database);
|
||||
self.store = try events.Store.init(io, &self.database, now_s);
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Fixture) void {
|
||||
self.database.close();
|
||||
}
|
||||
|
||||
pub fn count(self: *Fixture, sql: []const u8) !i64 {
|
||||
return self.database.queryInt(sql);
|
||||
}
|
||||
|
||||
/// The one column a test names most often, for the newest row of `code`.
|
||||
pub fn text(self: *Fixture, sql: []const u8) ![]const u8 {
|
||||
var stmt = try self.database.prepare(sql);
|
||||
defer stmt.deinit();
|
||||
if (!try stmt.step()) return error.NoRow;
|
||||
const value = stmt.columnText(0);
|
||||
@memcpy(self.text_buf[0..value.len], value);
|
||||
return self.text_buf[0..value.len];
|
||||
}
|
||||
};
|
||||
@@ -23,6 +23,7 @@ const std = @import("std");
|
||||
|
||||
const db = @import("db.zig");
|
||||
const disk_monitor = @import("disk_monitor.zig");
|
||||
const events = @import("events.zig");
|
||||
const model = @import("../config/model.zig");
|
||||
const queries_repo = @import("repositories/queries_repo.zig");
|
||||
|
||||
@@ -177,6 +178,9 @@ pub const Logger = struct {
|
||||
/// closed and every entry counts as dropped from that point, so a caller
|
||||
/// that sees this must not expect rows.
|
||||
writer_failed: std.atomic.Value(bool),
|
||||
/// Wired by the composition root after `init`, following the
|
||||
/// `gate: ?*disk_monitor.Monitor` idiom. Null in every unit test here.
|
||||
diagnostics: ?*events.Store = null,
|
||||
|
||||
/// `queue_buf.len` is the backpressure cap — the composition root
|
||||
/// (`app.zig:311`) allocates `cfg.logging.query_log_buffer_max` entries,
|
||||
@@ -254,6 +258,9 @@ pub const Logger = struct {
|
||||
// Without a writer there is no consumer, so leaving the queue open
|
||||
// would silently swallow every later entry.
|
||||
self.writer_failed.store(true, .release);
|
||||
// No recovery path claims this episode: the writer is gone for the
|
||||
// life of the process, so the row stays active, which is the truth.
|
||||
self.reportWrite(io, "writer", "preparing the batch statements failed", @errorName(err), 0);
|
||||
self.queue.close(io);
|
||||
self.dropRemaining(io);
|
||||
return;
|
||||
@@ -400,9 +407,41 @@ pub const Logger = struct {
|
||||
writer.writeBatch(rows[0..entries.len]) catch |err| {
|
||||
scope.warn("query log batch of {d} rows dropped: {s}", .{ entries.len, @errorName(err) });
|
||||
self.countDropped(entries.len);
|
||||
self.reportWrite(io, "batch", "a query log batch was dropped", @errorName(err), entries.len);
|
||||
return;
|
||||
};
|
||||
_ = self.rows_written.fetchAdd(entries.len, .monotonic);
|
||||
if (self.diagnostics) |store| {
|
||||
store.resolve(io, std.Io.Clock.real.now(io).toSeconds(), .query_log_write, "batch");
|
||||
}
|
||||
}
|
||||
|
||||
/// An error, not a warning: dropped query rows are gone, and a writer that
|
||||
/// never started means every later row is gone too.
|
||||
fn reportWrite(
|
||||
self: *Logger,
|
||||
io: std.Io,
|
||||
operation: []const u8,
|
||||
message: []const u8,
|
||||
error_name: []const u8,
|
||||
rows: usize,
|
||||
) void {
|
||||
const store = self.diagnostics orelse return;
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "{s}: {s} ({d} rows)", .{
|
||||
message,
|
||||
error_name,
|
||||
rows,
|
||||
}) catch buf[0..];
|
||||
store.report(
|
||||
io,
|
||||
std.Io.Clock.real.now(io).toSeconds(),
|
||||
.query_log_write,
|
||||
operation,
|
||||
operation,
|
||||
.@"error",
|
||||
detail,
|
||||
);
|
||||
}
|
||||
|
||||
fn countDropped(self: *Logger, n: usize) void {
|
||||
@@ -441,6 +480,7 @@ fn outcomeEntry(outcome: Outcome) ?Entry {
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const events_fixture = @import("events_fixture.zig");
|
||||
const querylog_schema = @import("querylog_schema.zig");
|
||||
|
||||
const testing = std.testing;
|
||||
@@ -881,3 +921,81 @@ test "an empty batch touches neither the database nor the counters" {
|
||||
try testing.expectEqual(@as(u64, 0), logger.rows_written.load(.monotonic));
|
||||
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
|
||||
}
|
||||
|
||||
test "a dropped batch opens an error episode the next good batch closes" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
try database.exec(
|
||||
\\CREATE TRIGGER refuse_boom BEFORE INSERT ON query_log
|
||||
\\WHEN new.client_ip = 'boom'
|
||||
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
||||
);
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
var writer = try queries_repo.BatchWriter.init(&database);
|
||||
defer writer.deinit();
|
||||
|
||||
var buf: [4]Entry = undefined;
|
||||
var logger: Logger = .init(.{}, &buf);
|
||||
logger.diagnostics = &fx.store;
|
||||
|
||||
var doomed = sampleEntry(10, "poison.example");
|
||||
doomed.setClientIp("boom");
|
||||
const bad = [_]Entry{doomed};
|
||||
try logger.flush(io, &writer, &bad, null);
|
||||
|
||||
try testing.expectEqualStrings("query_log.write", try fx.text("SELECT code FROM operational_events"));
|
||||
try testing.expectEqualStrings("batch", try fx.text("SELECT subject_key FROM operational_events"));
|
||||
try testing.expectEqualStrings("error", try fx.text("SELECT severity FROM operational_events"));
|
||||
|
||||
const good = [_]Entry{sampleEntry(11, "next.example")};
|
||||
try logger.flush(io, &writer, &good, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a writer that cannot prepare leaves an episode no recovery path claims" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
// No schema: `BatchWriter.init` cannot prepare against a missing table.
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
defer database.close();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
var buf: [8]Entry = undefined;
|
||||
var logger: Logger = .init(.{}, &buf);
|
||||
logger.diagnostics = &fx.store;
|
||||
|
||||
try logger.runWriter(io, &database, null);
|
||||
|
||||
try testing.expectEqualStrings("writer", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
try testing.expectEqualStrings("error", try fx.text(
|
||||
"SELECT severity FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
|
||||
// The writer returned, so nothing can ever close this. A second run finds
|
||||
// the queue closed and adds no second episode.
|
||||
try logger.runWriter(io, &database, null);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
+113
-4
@@ -55,7 +55,24 @@ fn assertOrdered(list: []const Step) void {
|
||||
/// foreign_keys` is a no-op inside a transaction, so applying it afterwards
|
||||
/// would silently leave referential integrity off.
|
||||
pub fn migrate(database: *db.Db) Error!u32 {
|
||||
return migrateSteps(database, &steps);
|
||||
const version = try migrateSteps(database, &steps);
|
||||
try bridgeOperationalEvents(database);
|
||||
return version;
|
||||
}
|
||||
|
||||
/// **Removable when the v0.1 adoption gate lands.**
|
||||
///
|
||||
/// `operational_events` joined `config_schema.ddl_v1` after databases stamped
|
||||
/// version 1 already existed, and a stamped database never runs step 1 again —
|
||||
/// so on those installs nothing would ever create the table, silently, and the
|
||||
/// diagnostics store would fail to open forever. This runs after the stamp,
|
||||
/// with every statement conditional, and touches no other table.
|
||||
///
|
||||
/// It belongs here and not in `cli.openConfigDb`: that runs *before* migration
|
||||
/// everywhere (`app.zig`, `cli.zig`), so creating the table there would make a
|
||||
/// fresh database's unconditional `CREATE TABLE` in `ddl_v1` fail.
|
||||
fn bridgeOperationalEvents(database: *db.Db) db.Error!void {
|
||||
return database.exec(config_schema.operational_events_bridge);
|
||||
}
|
||||
|
||||
/// Same logic against an injected step list. The seam exists for the rollback
|
||||
@@ -159,7 +176,7 @@ test "migrate on a fresh database creates every table and seeds the default grou
|
||||
const expected = [_][]const u8{
|
||||
"schema_version", "groups", "clients", "client_prefixes",
|
||||
"upstreams", "rules", "local_records", "forward_zones",
|
||||
"blocklist_sources", "group_sources", "settings",
|
||||
"blocklist_sources", "group_sources", "settings", "operational_events",
|
||||
};
|
||||
for (expected) |name| {
|
||||
try testing.expect(try tableExists(&database, name));
|
||||
@@ -373,9 +390,101 @@ test "delete_order and table_names name exactly the tables the schema creates" {
|
||||
for (config_schema.table_names) |name| {
|
||||
try testing.expect(try tableExists(&database, name));
|
||||
}
|
||||
// delete_order covers every table except `schema_version`.
|
||||
// delete_order covers every table except two: `schema_version`, which is
|
||||
// the migration's own, and `operational_events`, which is runtime state an
|
||||
// import must never wipe.
|
||||
try testing.expect(!try tableExists(&database, "no_such_table"));
|
||||
try testing.expect(try tableExists(&database, "operational_events"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, config_schema.delete_order.len + 1),
|
||||
@as(i64, config_schema.delete_order.len + 2),
|
||||
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a version-1 database created without operational_events gains exactly it" {
|
||||
// The silent divergence the bridge exists for: this is what the Pi's
|
||||
// `config.db` looks like — stamped 1, so step 1 never runs again.
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
try database.exec(config_schema.ddl_v1);
|
||||
try database.exec("DROP TABLE operational_events;");
|
||||
try database.exec("INSERT INTO schema_version (version) VALUES (1);");
|
||||
try testing.expect(!try tableExists(&database, "operational_events"));
|
||||
|
||||
const tables_before = try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'");
|
||||
const groups_before = try database.queryInt("SELECT count(*) FROM groups");
|
||||
|
||||
try testing.expectEqual(@as(u32, 1), try migrate(&database));
|
||||
|
||||
try testing.expect(try tableExists(&database, "operational_events"));
|
||||
try testing.expectEqual(
|
||||
tables_before + 1,
|
||||
try database.queryInt("SELECT count(*) FROM sqlite_schema WHERE type='table'"),
|
||||
);
|
||||
// Both indexes came with it, and no other table moved.
|
||||
try testing.expectEqual(
|
||||
@as(i64, 2),
|
||||
try database.queryInt(
|
||||
"SELECT count(*) FROM sqlite_schema WHERE type='index' AND tbl_name='operational_events'",
|
||||
),
|
||||
);
|
||||
try testing.expectEqual(groups_before, try database.queryInt("SELECT count(*) FROM groups"));
|
||||
}
|
||||
|
||||
test "the bridge does not double-create on a fresh database or on a second run" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
|
||||
// `ddl_v1` creates the table unconditionally, so a bridge that ran as part
|
||||
// of the step list would fail here rather than be a no-op.
|
||||
try testing.expectEqual(@as(u32, 1), try migrate(&database));
|
||||
const schema_rows = try database.queryInt("SELECT count(*) FROM sqlite_schema");
|
||||
|
||||
try database.exec(
|
||||
\\INSERT INTO operational_events
|
||||
\\ (code, subject_key, subject_label, severity, first_seen, last_seen, occurrences)
|
||||
\\VALUES ('disk.space', 'data', 'data', 'warning', 100, 100, 1);
|
||||
);
|
||||
try testing.expectEqual(@as(u32, 1), try migrate(&database));
|
||||
try testing.expectEqual(schema_rows, try database.queryInt("SELECT count(*) FROM sqlite_schema"));
|
||||
// A `CREATE TABLE IF NOT EXISTS` that had somehow replaced the table would
|
||||
// show up as a lost row, not as a schema difference.
|
||||
try testing.expectEqual(@as(i64, 1), try database.queryInt("SELECT count(*) FROM operational_events"));
|
||||
}
|
||||
|
||||
test "the partial unique index allows one active row per key and any number of resolved ones" {
|
||||
var database = try openMigrated();
|
||||
defer database.close();
|
||||
_ = try migrate(&database);
|
||||
|
||||
const insert =
|
||||
\\INSERT INTO operational_events
|
||||
\\ (code, subject_key, subject_label, severity, first_seen, last_seen, occurrences, resolved_at)
|
||||
\\VALUES ('blocklist.refresh', 'https://a.example', 'a', 'warning', 100, 100, 1, ?1)
|
||||
;
|
||||
{
|
||||
var stmt = try database.prepare(insert);
|
||||
defer stmt.deinit();
|
||||
try stmt.bindNull(1);
|
||||
try stmt.exec();
|
||||
}
|
||||
{
|
||||
// A second active row for the same (code, subject_key) is what
|
||||
// `report`'s overflow probe exists to avoid, and the index proves it.
|
||||
// Its own statement: `Stmt.reset` re-reports the code of a failed step,
|
||||
// so a reused one would answer `error.Constraint` a second time.
|
||||
var stmt = try database.prepare(insert);
|
||||
defer stmt.deinit();
|
||||
try stmt.bindNull(1);
|
||||
try testing.expectError(error.Constraint, stmt.step());
|
||||
}
|
||||
|
||||
var resolved = try database.prepare(insert);
|
||||
defer resolved.deinit();
|
||||
for ([_]i64{ 200, 300 }) |resolved_at| {
|
||||
try resolved.reset();
|
||||
try resolved.bindInt(1, resolved_at);
|
||||
try resolved.exec();
|
||||
}
|
||||
try testing.expectEqual(@as(i64, 3), try database.queryInt("SELECT count(*) FROM operational_events"));
|
||||
}
|
||||
|
||||
@@ -360,7 +360,7 @@ test "S8 case 5: a retention pass prunes the old rows and truncates the write-ah
|
||||
try testing.expect(try f.sizeOf("querylog.db-wal") > 0);
|
||||
|
||||
var pass: retention.Retention = .init(.{ .retention_days = 30 });
|
||||
pass.runOnce(io, log_db.database(), null);
|
||||
pass.runOnce(io, log_db.database(), null, null);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), pass.snapshotStats().passes);
|
||||
try testing.expectEqual(@as(u64, 2), pass.snapshotStats().rows_pruned);
|
||||
@@ -396,7 +396,7 @@ test "S8 case 6: a critical disk gates the flushes and recovery releases them" {
|
||||
data_path,
|
||||
null,
|
||||
);
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, null, 0);
|
||||
try testing.expectEqual(disk_monitor.State.critical, monitor.state());
|
||||
try testing.expect(!monitor.writesAllowed());
|
||||
try testing.expect(monitor.gauges().free_bytes > 0);
|
||||
@@ -418,7 +418,7 @@ test "S8 case 6: a critical disk gates the flushes and recovery releases them" {
|
||||
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(log_db.database()));
|
||||
|
||||
monitor.cfg = .{ .min_free_mb = 0, .warn_free_mb = 0 };
|
||||
monitor.sample(io);
|
||||
monitor.sample(io, null, 0);
|
||||
try testing.expectEqual(disk_monitor.State.ok, monitor.state());
|
||||
try testing.expect(monitor.writesAllowed());
|
||||
|
||||
|
||||
@@ -88,6 +88,18 @@ pub const OpenResult = struct {
|
||||
database: db.Db,
|
||||
/// Non-null feeds a counter and the `/api/health` rollup.
|
||||
recreated: ?RecreateReason,
|
||||
/// The path the previous file was kept as, by value. It existed only in a
|
||||
/// stack buffer inside `open` before the diagnostics event needed it, and a
|
||||
/// slice of that buffer would dangle the moment `open` returned.
|
||||
///
|
||||
/// Empty when nothing was renamed aside, which `.missing` and a clean open
|
||||
/// both are.
|
||||
aside_buf: [path_buf_len]u8 = undefined,
|
||||
aside_len: u16 = 0,
|
||||
|
||||
pub fn aside(self: *const OpenResult) []const u8 {
|
||||
return self.aside_buf[0..self.aside_len];
|
||||
}
|
||||
};
|
||||
|
||||
pub const Error = db.Error || error{AsideNameCollision} ||
|
||||
@@ -156,7 +168,12 @@ pub fn open(io: std.Io, dir: std.Io.Dir, path: [:0]const u8) Error!OpenResult {
|
||||
aside.?,
|
||||
});
|
||||
}
|
||||
return .{ .database = fresh, .recreated = cause };
|
||||
var result: OpenResult = .{ .database = fresh, .recreated = cause };
|
||||
if (aside) |name| {
|
||||
result.aside_len = @intCast(name.len);
|
||||
@memcpy(result.aside_buf[0..name.len], name);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// The whitelist. `null` means "propagate, do not touch the file".
|
||||
@@ -321,3 +338,53 @@ test "recreatable selects exactly two of db.Error's members" {
|
||||
}
|
||||
try testing.expectEqual(@as(usize, 2), whitelisted);
|
||||
}
|
||||
|
||||
test "a recreate returns the aside name by value and a fresh create returns none" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var tmp = testing.tmpDir(.{ .iterate = true });
|
||||
defer tmp.cleanup();
|
||||
|
||||
var path_buf: [path_buf_len]u8 = undefined;
|
||||
const path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/querylog.db", .{tmp.sub_path});
|
||||
|
||||
// First open: the file is missing, so nothing is renamed aside. The
|
||||
// one-shot event is deliberately not emitted for this case.
|
||||
var created = try open(io, std.Io.Dir.cwd(), path);
|
||||
created.database.close();
|
||||
try testing.expectEqual(RecreateReason.missing, created.recreated.?);
|
||||
try testing.expectEqualStrings("", created.aside());
|
||||
|
||||
try tmp.dir.writeFile(io, .{ .sub_path = "querylog.db", .data = "not a database at all" });
|
||||
|
||||
var recreated = try open(io, std.Io.Dir.cwd(), path);
|
||||
recreated.database.close();
|
||||
try testing.expectEqual(RecreateReason.not_a_database, recreated.recreated.?);
|
||||
try testing.expect(recreated.aside().len != 0);
|
||||
|
||||
// The name is a real file, which is the whole reason it travels out.
|
||||
const kept = std.fs.path.basename(recreated.aside());
|
||||
try tmp.dir.access(io, kept, .{});
|
||||
}
|
||||
|
||||
test "a clean reopen reports no recreate and no aside" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var tmp = testing.tmpDir(.{ .iterate = true });
|
||||
defer tmp.cleanup();
|
||||
|
||||
var path_buf: [path_buf_len]u8 = undefined;
|
||||
const path = try std.fmt.bufPrintZ(&path_buf, ".zig-cache/tmp/{s}/querylog.db", .{tmp.sub_path});
|
||||
|
||||
var first = try open(io, std.Io.Dir.cwd(), path);
|
||||
first.database.close();
|
||||
|
||||
var second = try open(io, std.Io.Dir.cwd(), path);
|
||||
second.database.close();
|
||||
try testing.expectEqual(@as(?RecreateReason, null), second.recreated);
|
||||
try testing.expectEqualStrings("", second.aside());
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+149
-19
@@ -13,6 +13,7 @@ const std = @import("std");
|
||||
|
||||
const db = @import("db.zig");
|
||||
const disk_monitor = @import("disk_monitor.zig");
|
||||
const events = @import("events.zig");
|
||||
const model = @import("../config/model.zig");
|
||||
const queries_repo = @import("repositories/queries_repo.zig");
|
||||
const upstream_history_repo = @import("repositories/upstream_history_repo.zig");
|
||||
@@ -102,15 +103,23 @@ pub const Retention = struct {
|
||||
io: std.Io,
|
||||
database: *db.Db,
|
||||
monitor: ?*disk_monitor.Monitor,
|
||||
store: ?*events.Store,
|
||||
) void {
|
||||
add(&self.counters.passes, 1);
|
||||
const now = std.Io.Clock.real.now(io).toSeconds();
|
||||
const cutoff = now - model.retentionSeconds(self.cfg);
|
||||
|
||||
// Diagnostics retention rides this pass rather than a schedule of its
|
||||
// own: one daily housekeeping task, and a box restarted every night
|
||||
// still prunes through `Store.init`.
|
||||
if (store) |s| s.prune(io, now);
|
||||
|
||||
if (queries_repo.pruneOlderThan(database, cutoff)) |deleted| {
|
||||
add(&self.counters.rows_pruned, @intCast(deleted));
|
||||
maintenance(store, io, now, "prune", null);
|
||||
} else |err| {
|
||||
log.warn("retention prune before {d} failed: {s}", .{ cutoff, @errorName(err) });
|
||||
maintenance(store, io, now, "prune", @errorName(err));
|
||||
}
|
||||
|
||||
// Before the vacuum-cadence logic below, which returns early on six
|
||||
@@ -123,14 +132,18 @@ pub const Retention = struct {
|
||||
const history_cutoff = now - upstream_history_repo.retention_window_s;
|
||||
if (upstream_history_repo.pruneOlderThan(database, history_cutoff)) |deleted| {
|
||||
add(&self.counters.upstream_rows_pruned, @intCast(deleted));
|
||||
maintenance(store, io, now, "history_prune", null);
|
||||
} else |err| {
|
||||
log.warn("upstream history prune before {d} failed: {s}", .{ history_cutoff, @errorName(err) });
|
||||
maintenance(store, io, now, "history_prune", @errorName(err));
|
||||
}
|
||||
|
||||
if (queries_repo.checkpointTruncate(database)) {
|
||||
add(&self.counters.checkpoints, 1);
|
||||
maintenance(store, io, now, "checkpoint", null);
|
||||
} else |err| {
|
||||
log.warn("retention checkpoint failed: {s}", .{@errorName(err)});
|
||||
maintenance(store, io, now, "checkpoint", @errorName(err));
|
||||
}
|
||||
|
||||
self.passes_since_vacuum += 1;
|
||||
@@ -141,17 +154,37 @@ pub const Retention = struct {
|
||||
if (monitor) |m| if (!m.writesAllowed()) {
|
||||
add(&self.counters.vacuums_gated, 1);
|
||||
log.warn("retention vacuum skipped: the disk monitor refuses writes", .{});
|
||||
maintenance(store, io, now, "vacuum", "the disk monitor refuses writes");
|
||||
return;
|
||||
};
|
||||
|
||||
if (queries_repo.vacuum(database)) {
|
||||
add(&self.counters.vacuums, 1);
|
||||
self.passes_since_vacuum = 0;
|
||||
maintenance(store, io, now, "vacuum", null);
|
||||
} else |err| {
|
||||
log.warn("retention vacuum failed: {s}", .{@errorName(err)});
|
||||
maintenance(store, io, now, "vacuum", @errorName(err));
|
||||
}
|
||||
}
|
||||
|
||||
/// One step's outcome. `reason` null is the success branch of that same
|
||||
/// step in that same pass, which is what closes its episode; a gated vacuum
|
||||
/// is a failure of the step, because the work it owes is still owed.
|
||||
fn maintenance(
|
||||
store: ?*events.Store,
|
||||
io: std.Io,
|
||||
now_s: i64,
|
||||
operation: []const u8,
|
||||
reason: ?[]const u8,
|
||||
) void {
|
||||
const s = store orelse return;
|
||||
const text = reason orelse return s.resolve(io, now_s, .query_log_maintenance, operation);
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "retention {s} failed: {s}", .{ operation, text }) catch buf[0..];
|
||||
s.report(io, now_s, .query_log_maintenance, operation, operation, .warning, detail);
|
||||
}
|
||||
|
||||
fn add(counter: *std.atomic.Value(u64), delta: u64) void {
|
||||
_ = counter.fetchAdd(delta, .monotonic);
|
||||
}
|
||||
@@ -182,13 +215,14 @@ pub const Retention = struct {
|
||||
io: std.Io,
|
||||
database: *db.Db,
|
||||
monitor: ?*disk_monitor.Monitor,
|
||||
store: ?*events.Store,
|
||||
) std.Io.Cancelable!void {
|
||||
const interval: std.Io.Clock.Duration = .{
|
||||
.raw = .fromSeconds(pass_interval_s),
|
||||
.clock = .boot,
|
||||
};
|
||||
while (true) {
|
||||
self.runOnce(io, database, monitor);
|
||||
self.runOnce(io, database, monitor, store);
|
||||
try interval.sleep(io);
|
||||
}
|
||||
}
|
||||
@@ -198,6 +232,7 @@ pub const Retention = struct {
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const events_fixture = @import("events_fixture.zig");
|
||||
const querylog_schema = @import("querylog_schema.zig");
|
||||
|
||||
const testing = std.testing;
|
||||
@@ -243,7 +278,7 @@ test "a pass prunes the rows past the retention window and keeps the rest" {
|
||||
try writeRows(&database, &.{ now - 40 * day, now - 31 * day, now - 29 * day, now - 60 });
|
||||
|
||||
var retention: Retention = .init(.{ .retention_days = 30 });
|
||||
retention.runOnce(io, &database, null);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
|
||||
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
|
||||
@@ -267,12 +302,12 @@ test "the cutoff follows retention_days" {
|
||||
try writeRows(&database, &.{now - 3 * day});
|
||||
|
||||
var keeps: Retention = .init(.{ .retention_days = 7 });
|
||||
keeps.runOnce(io, &database, null);
|
||||
keeps.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
|
||||
try testing.expectEqual(@as(u64, 0), keeps.snapshotStats().rows_pruned);
|
||||
|
||||
var prunes: Retention = .init(.{ .retention_days = 1 });
|
||||
prunes.runOnce(io, &database, null);
|
||||
prunes.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
|
||||
try testing.expectEqual(@as(u64, 1), prunes.snapshotStats().rows_pruned);
|
||||
}
|
||||
@@ -287,16 +322,16 @@ test "the seventh pass vacuums and the six before it do not" {
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
for (0..6) |_| {
|
||||
retention.runOnce(io, &database, null);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums);
|
||||
}
|
||||
retention.runOnce(io, &database, null);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
|
||||
try testing.expectEqual(@as(u64, 7), retention.snapshotStats().passes);
|
||||
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
|
||||
try testing.expectEqual(@as(u64, 7), retention.snapshotStats().checkpoints);
|
||||
|
||||
for (0..7) |_| retention.runOnce(io, &database, null);
|
||||
for (0..7) |_| retention.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(u64, 14), retention.snapshotStats().passes);
|
||||
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().vacuums);
|
||||
}
|
||||
@@ -315,7 +350,7 @@ test "a gated pass skips the vacuum, counts it, and vacuums on the next pass" {
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
|
||||
|
||||
var gated: Retention = .init(.{});
|
||||
for (0..vacuum_every_passes) |_| gated.runOnce(io, &database, &monitor);
|
||||
for (0..vacuum_every_passes) |_| gated.runOnce(io, &database, &monitor, null);
|
||||
|
||||
// Prune and checkpoint ran on every pass; only the vacuum was refused.
|
||||
try testing.expectEqual(@as(u64, vacuum_every_passes), gated.snapshotStats().passes);
|
||||
@@ -325,12 +360,12 @@ test "a gated pass skips the vacuum, counts it, and vacuums on the next pass" {
|
||||
|
||||
// The vacuum is due again immediately, not seven passes later.
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.ok), .monotonic);
|
||||
gated.runOnce(io, &database, &monitor);
|
||||
gated.runOnce(io, &database, &monitor, null);
|
||||
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums);
|
||||
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums_gated);
|
||||
|
||||
// And the counter reset, so the next six passes vacuum nothing.
|
||||
for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor);
|
||||
for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor, null);
|
||||
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums);
|
||||
}
|
||||
|
||||
@@ -346,7 +381,7 @@ test "a warn state still allows the vacuum" {
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.warn), .monotonic);
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor);
|
||||
for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor, null);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
|
||||
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().vacuums_gated);
|
||||
@@ -361,7 +396,7 @@ test "a pass over an empty database still counts" {
|
||||
defer database.close();
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
retention.runOnce(io, &database, null);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
|
||||
try testing.expectEqual(@as(u64, 0), retention.snapshotStats().rows_pruned);
|
||||
@@ -385,7 +420,7 @@ test "a failing prune counts the pass and leaves the rows alone" {
|
||||
);
|
||||
|
||||
var retention: Retention = .init(.{ .retention_days = 30 });
|
||||
retention.runOnce(io, &database, null);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try queries_repo.countRows(&database));
|
||||
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().passes);
|
||||
@@ -412,7 +447,7 @@ test "the upstream-history window is its own, and a one-day query log does not s
|
||||
|
||||
// A query log kept for one day, and 30 days of upstream minutes beside it.
|
||||
var retention: Retention = .init(.{ .retention_days = 1 });
|
||||
retention.runOnce(io, &database, null);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
|
||||
const stats = retention.snapshotStats();
|
||||
// One query-log row is older than one day; one minute row is older than the
|
||||
@@ -450,7 +485,7 @@ test "the history prune runs on the passes where the vacuum logic returns early"
|
||||
.{ .url = "https://a.example", .minute_ts = old_minute, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
|
||||
});
|
||||
var early: Retention = .init(.{});
|
||||
early.runOnce(io, &database, null);
|
||||
early.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(u64, 1), early.snapshotStats().upstream_rows_pruned);
|
||||
try testing.expectEqual(@as(i64, 0), try upstream_history_repo.countMinutes(&database));
|
||||
|
||||
@@ -460,11 +495,11 @@ test "the history prune runs on the passes where the vacuum logic returns early"
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
|
||||
|
||||
var gated: Retention = .init(.{});
|
||||
for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor);
|
||||
for (0..vacuum_every_passes - 1) |_| gated.runOnce(io, &database, &monitor, null);
|
||||
try upstream_history_repo.flush(&database, &.{
|
||||
.{ .url = "https://b.example", .minute_ts = old_minute, .successes = 1, .failures = 0, .last_failure_ts = null, .last_error = "" },
|
||||
});
|
||||
gated.runOnce(io, &database, &monitor);
|
||||
gated.runOnce(io, &database, &monitor, null);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().vacuums_gated);
|
||||
try testing.expectEqual(@as(u64, 1), gated.snapshotStats().upstream_rows_pruned);
|
||||
@@ -487,13 +522,108 @@ test "the next pass retries what the failed one could not do" {
|
||||
);
|
||||
|
||||
var retention: Retention = .init(.{ .retention_days = 30 });
|
||||
retention.runOnce(io, &database, null);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
try testing.expectEqual(@as(i64, 2), try queries_repo.countRows(&database));
|
||||
|
||||
try database.exec("DROP TRIGGER refuse_delete;");
|
||||
retention.runOnce(io, &database, null);
|
||||
retention.runOnce(io, &database, null, null);
|
||||
|
||||
try testing.expectEqual(@as(i64, 0), try queries_repo.countRows(&database));
|
||||
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().passes);
|
||||
try testing.expectEqual(@as(u64, 2), retention.snapshotStats().rows_pruned);
|
||||
}
|
||||
|
||||
test "a failing prune opens a maintenance episode the next clean pass closes" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
const now = std.Io.Clock.real.now(io).toSeconds();
|
||||
try writeRows(&database, &.{now - 40 * 86_400});
|
||||
try database.exec(
|
||||
\\CREATE TRIGGER refuse_delete BEFORE DELETE ON query_log
|
||||
\\BEGIN SELECT RAISE(ABORT, 'refused'); END;
|
||||
);
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
retention.runOnce(io, &database, null, &fx.store);
|
||||
|
||||
// Only the prune failed; checkpoint and history prune succeeded, and a
|
||||
// success writes no row of its own.
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
try testing.expectEqualStrings("query_log.maintenance", try fx.text("SELECT code FROM operational_events"));
|
||||
try testing.expectEqualStrings("prune", try fx.text("SELECT subject_key FROM operational_events"));
|
||||
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
|
||||
|
||||
try database.exec("DROP TRIGGER refuse_delete;");
|
||||
retention.runOnce(io, &database, null, &fx.store);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a gated vacuum is a maintenance failure the next ungated pass closes" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
var monitor: disk_monitor.Monitor = .init(.{}, std.Io.Dir.cwd(), ".", null);
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.critical), .monotonic);
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
for (0..vacuum_every_passes) |_| retention.runOnce(io, &database, &monitor, &fx.store);
|
||||
|
||||
try testing.expectEqualStrings("vacuum", try fx.text(
|
||||
"SELECT subject_key FROM operational_events WHERE resolved_at IS NULL",
|
||||
));
|
||||
|
||||
monitor.state_raw.store(@intFromEnum(disk_monitor.State.ok), .monotonic);
|
||||
retention.runOnce(io, &database, &monitor, &fx.store);
|
||||
|
||||
try testing.expectEqual(@as(u64, 1), retention.snapshotStats().vacuums);
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a pass prunes the diagnostics store once" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
// Resolved further back than the retention window, so the pass must drop it.
|
||||
const now = std.Io.Clock.real.now(io).toSeconds();
|
||||
const stale = now - events.Store.resolved_retention_s - 86_400;
|
||||
fx.store.reportResolved(io, stale, .query_log_recreated, "one-shot", "one-shot", .warning, "aside kept");
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
|
||||
var retention: Retention = .init(.{});
|
||||
retention.runOnce(io, &database, null, &fx.store);
|
||||
|
||||
try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
}
|
||||
|
||||
@@ -76,6 +76,9 @@ comptime {
|
||||
_ = @import("server/rate_limiter.zig");
|
||||
_ = @import("storage/repositories/queries_repo.zig");
|
||||
_ = @import("storage/repositories/upstream_history_repo.zig");
|
||||
_ = @import("storage/repositories/events_repo.zig");
|
||||
_ = @import("storage/events.zig");
|
||||
_ = @import("storage/events_fixture.zig");
|
||||
_ = @import("upstream/history.zig");
|
||||
_ = @import("storage/logger.zig");
|
||||
_ = @import("platform/statfs.zig");
|
||||
@@ -100,6 +103,7 @@ comptime {
|
||||
_ = @import("web/metrics.zig");
|
||||
_ = @import("web/handlers/stats.zig");
|
||||
_ = @import("web/handlers/queries.zig");
|
||||
_ = @import("web/handlers/diagnostics.zig");
|
||||
_ = @import("web/handlers/lookup.zig");
|
||||
_ = @import("web/handlers/upstream_health.zig");
|
||||
_ = @import("web/handlers/health.zig");
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
const std = @import("std");
|
||||
|
||||
const db = @import("../storage/db.zig");
|
||||
const events = @import("../storage/events.zig");
|
||||
const health = @import("health.zig");
|
||||
const upstream_history_repo = @import("../storage/repositories/upstream_history_repo.zig");
|
||||
|
||||
@@ -114,6 +115,10 @@ pub const Accumulator = struct {
|
||||
flush_cells: [max_pending]Cell,
|
||||
flush_rows: [max_pending]upstream_history_repo.FlushRow,
|
||||
flush_count: u32,
|
||||
/// Set once by the composition root after `init`, following the
|
||||
/// `gate: ?*disk_monitor.Monitor` idiom. Null everywhere else, and every
|
||||
/// emit site below is inert when it is.
|
||||
diagnostics: ?*events.Store = null,
|
||||
|
||||
/// `cells` and `flush_cells` are `undefined`: a cell is always written
|
||||
/// before it is read, and `count` is what says which ones exist.
|
||||
@@ -255,6 +260,9 @@ pub const Accumulator = struct {
|
||||
|
||||
if (write(database, rows)) {
|
||||
_ = self.counters.flushes.fetchAdd(1, .monotonic);
|
||||
if (self.diagnostics) |store| {
|
||||
store.resolve(io, std.Io.Clock.real.now(io).toSeconds(), .upstream_history_write, flush_key);
|
||||
}
|
||||
self.mutex.lockUncancelable(io);
|
||||
defer self.mutex.unlock(io);
|
||||
self.last_flush_failed = false;
|
||||
@@ -262,6 +270,22 @@ pub const Accumulator = struct {
|
||||
} else |err| {
|
||||
_ = self.counters.flush_failures.fetchAdd(1, .monotonic);
|
||||
log.warn("flushing {d} upstream history rows failed: {s}", .{ rows.len, @errorName(err) });
|
||||
if (self.diagnostics) |store| {
|
||||
var buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&buf, "flushing {d} upstream history rows failed: {s}", .{
|
||||
rows.len,
|
||||
@errorName(err),
|
||||
}) catch buf[0..];
|
||||
store.report(
|
||||
io,
|
||||
std.Io.Clock.real.now(io).toSeconds(),
|
||||
.upstream_history_write,
|
||||
flush_key,
|
||||
"upstream history flush",
|
||||
.warning,
|
||||
detail,
|
||||
);
|
||||
}
|
||||
self.mergeBack(io);
|
||||
}
|
||||
}
|
||||
@@ -296,6 +320,10 @@ pub const Accumulator = struct {
|
||||
}
|
||||
};
|
||||
|
||||
/// One accumulator, one flush task, one table: the subject of every
|
||||
/// `upstream_history.write` episode is that single writer.
|
||||
const flush_key = "flush";
|
||||
|
||||
/// Max-wins, matching the SQL upsert exactly: the newest failure in the minute
|
||||
/// is the one whose name the cell keeps.
|
||||
fn noteFailure(cell: *Accumulator.Cell, at: i64, error_name: []const u8) void {
|
||||
@@ -312,6 +340,7 @@ fn noteFailure(cell: *Accumulator.Cell, at: i64, error_name: []const u8) void {
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const events_fixture = @import("../storage/events_fixture.zig");
|
||||
const querylog_schema = @import("../storage/querylog_schema.zig");
|
||||
|
||||
const testing = std.testing;
|
||||
@@ -632,3 +661,56 @@ test "a flush against the real repository writes the minute rows" {
|
||||
try testing.expectEqual(@as(i64, 2), try upstream_history_repo.countMinutes(&database));
|
||||
try testing.expectEqual(@as(u64, 2), acc.snapshotStats(io).flushes);
|
||||
}
|
||||
|
||||
test "a failed flush opens one episode and the next successful flush closes it" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
|
||||
const acc = try newAccumulator();
|
||||
defer testing.allocator.destroy(acc);
|
||||
acc.diagnostics = &fx.store;
|
||||
|
||||
acc.recordFailure(io, "https://a.example", 1_700_000_000, "Timeout");
|
||||
acc.flushOnce(io, &database, failingWrite);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
try testing.expectEqualStrings("upstream_history.write", try fx.text("SELECT code FROM operational_events"));
|
||||
try testing.expectEqualStrings("flush", try fx.text("SELECT subject_key FROM operational_events"));
|
||||
try testing.expectEqualStrings("warning", try fx.text("SELECT severity FROM operational_events"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, 1),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
|
||||
acc.flushOnce(io, &database, upstream_history_repo.flush);
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
}
|
||||
|
||||
test "a flush with no store attached records nothing and still flushes" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try openLog();
|
||||
defer database.close();
|
||||
|
||||
const acc = try newAccumulator();
|
||||
defer testing.allocator.destroy(acc);
|
||||
|
||||
acc.recordFailure(io, "https://a.example", 1_700_000_000, "Timeout");
|
||||
acc.flushOnce(io, &database, failingWrite);
|
||||
acc.flushOnce(io, &database, upstream_history_repo.flush);
|
||||
try testing.expectEqual(@as(u64, 1), acc.counters.flushes.load(.monotonic));
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const events = @import("../storage/events.zig");
|
||||
const health = @import("health.zig");
|
||||
const history_mod = @import("history.zig");
|
||||
const safe_url = @import("../safe_url.zig");
|
||||
@@ -136,6 +137,9 @@ pub const Pool = struct {
|
||||
/// pool is fully usable without it — `nxdns check` and every unit test here
|
||||
/// run with no history at all.
|
||||
history: ?*history_mod.Accumulator = null,
|
||||
/// The diagnostics store, wired the same way and for the same reason as
|
||||
/// `history`. Every emit here sits outside `mutex`; see `recordHistory`.
|
||||
diagnostics: ?*events.Store = null,
|
||||
|
||||
pub fn init(
|
||||
entries: []Entry,
|
||||
@@ -327,6 +331,7 @@ pub const Pool = struct {
|
||||
// constraint: the accumulator takes a mutex of its own, and no task may
|
||||
// hold one of the two while it takes the other.
|
||||
self.recordHistory(io, entry, .success);
|
||||
self.recordDiagnostics(io, entry, .success);
|
||||
}
|
||||
|
||||
fn recordFailure(
|
||||
@@ -344,6 +349,7 @@ pub const Pool = struct {
|
||||
// After the pool mutex is released, for the reason `recordSuccess`
|
||||
// states.
|
||||
self.recordHistory(io, entry, .{ .failure = @errorName(err) });
|
||||
self.recordDiagnostics(io, entry, .{ .failure = @errorName(err) });
|
||||
}
|
||||
|
||||
const Outcome = union(enum) { success, failure: []const u8 };
|
||||
@@ -359,9 +365,34 @@ pub const Pool = struct {
|
||||
.failure => |name| history.recordFailure(io, entry.endpoint.url, wall_s, name),
|
||||
}
|
||||
}
|
||||
|
||||
/// The same placement discipline as `recordHistory`: the store takes a
|
||||
/// mutex of its own, so this runs after the pool's is released.
|
||||
///
|
||||
/// A success is the steady state of the whole program, so `resolve` is
|
||||
/// built to issue no SQL when nothing is open (`storage/events.zig`).
|
||||
fn recordDiagnostics(self: *Pool, io: std.Io, entry: *Entry, outcome: Outcome) void {
|
||||
const store = self.diagnostics orelse return;
|
||||
const url = entry.endpoint.url;
|
||||
const now_s = std.Io.Clock.real.now(io).toSeconds();
|
||||
switch (outcome) {
|
||||
.success => store.resolve(io, now_s, .upstream_exchange, url),
|
||||
.failure => |name| {
|
||||
var label_buf: [events.Store.max_subject_label_len]u8 = undefined;
|
||||
const label = std.fmt.bufPrint(&label_buf, "{f}", .{safe_url.redact(url)}) catch &label_buf;
|
||||
var detail_buf: [events.Store.max_detail_len]u8 = undefined;
|
||||
const detail = std.fmt.bufPrint(&detail_buf, "upstream {f} failed: {s}", .{
|
||||
safe_url.redactQuoted(url),
|
||||
name,
|
||||
}) catch &detail_buf;
|
||||
store.report(io, now_s, .upstream_exchange, url, label, .warning, detail);
|
||||
},
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const db = @import("../storage/db.zig");
|
||||
const events_fixture = @import("../storage/events_fixture.zig");
|
||||
const querylog_schema = @import("../storage/querylog_schema.zig");
|
||||
const upstream_history_repo = @import("../storage/repositories/upstream_history_repo.zig");
|
||||
|
||||
@@ -948,3 +979,69 @@ test "an entry that enters backoff while a task waits on it is not attempted" {
|
||||
try testing.expectEqual(@as(u64, 1), entries[0].health.total_failures);
|
||||
try testing.expect(entries[0].health.backoff_until != null);
|
||||
}
|
||||
|
||||
test "a successful exchange with nothing open costs the store no statement" {
|
||||
if (@FieldType(events.Store, "statements") != u64) return error.SkipZigTest;
|
||||
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var good: Fake = .{ .behavior = .{ .reply = response_bytes } };
|
||||
var entries = [_]Entry{testEntry("https://good.example/dns-query", &good, 10)};
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
pool.diagnostics = &fx.store;
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
const before = fx.store.statements;
|
||||
for (0..20) |_| _ = try pool.exchange(io, query_bytes, &buf);
|
||||
try testing.expectEqual(before, fx.store.statements);
|
||||
try testing.expectEqual(@as(i64, 0), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
}
|
||||
|
||||
test "a failing then recovering upstream leaves exactly one resolved episode" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var flaky: Fake = .{ .behavior = .{ .fail = error.Timeout } };
|
||||
var standby: Fake = .{ .behavior = .{ .reply = response_bytes } };
|
||||
var entries = [_]Entry{
|
||||
testEntry("https://flaky.example/dns-query", &flaky, 10),
|
||||
testEntry("https://standby.example/dns-query", &standby, 20),
|
||||
};
|
||||
var pool: Pool = .init(&entries, test_cfg, test_timeouts, 1);
|
||||
|
||||
var fx: events_fixture.Fixture = .{};
|
||||
try fx.init(io, 1000);
|
||||
defer fx.deinit();
|
||||
pool.diagnostics = &fx.store;
|
||||
|
||||
var buf: [512]u8 = undefined;
|
||||
_ = try pool.exchange(io, query_bytes, &buf);
|
||||
// Backoff would park the failing entry, so the second failure is driven
|
||||
// through `recordFailure` itself rather than through another exchange.
|
||||
pool.recordFailure(io, &entries[0], std.Io.Clock.awake.now(io), error.ConnectFailed);
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
|
||||
try testing.expectEqualStrings("upstream.exchange", try fx.text("SELECT code FROM operational_events"));
|
||||
try testing.expectEqualStrings(
|
||||
"https://flaky.example/dns-query",
|
||||
try fx.text("SELECT subject_key FROM operational_events"),
|
||||
);
|
||||
|
||||
flaky.behavior = .{ .reply = response_bytes };
|
||||
pool.recordSuccess(io, &entries[0], std.Io.Clock.awake.now(io));
|
||||
|
||||
try testing.expectEqual(@as(i64, 1), try fx.count("SELECT count(*) FROM operational_events"));
|
||||
try testing.expectEqual(
|
||||
@as(i64, 0),
|
||||
try fx.count("SELECT count(*) FROM operational_events WHERE resolved_at IS NULL"),
|
||||
);
|
||||
try testing.expectEqual(@as(i64, 2), try fx.count("SELECT occurrences FROM operational_events"));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,448 @@
|
||||
//! `GET /api/diagnostics` and `GET /api/diagnostics/{id}` — the operational
|
||||
//! event log, newest first.
|
||||
//!
|
||||
//! Keyset pagination and the same page envelope as `/api/queries`, for the same
|
||||
//! reason: the table is append-only at the head, so `id < before` is one index
|
||||
//! seek however deep a client has scrolled, and rows arriving between two pages
|
||||
//! cannot shift the window and duplicate one.
|
||||
//!
|
||||
//! Filter parsing is separated from fetching, because parsing is where PLAN
|
||||
//! §19's input validation lives and it is worth testing on its own. Every value
|
||||
//! is length-capped here and bound as a SQL parameter by the repository;
|
||||
//! nothing this file reads is ever concatenated into a statement.
|
||||
//!
|
||||
//! No SQL and no connection of its own: `events.Store` owns the one diagnostics
|
||||
//! connection and locks its mutex around every read, so this file cannot race
|
||||
//! the emitters writing through it.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const db = @import("../../storage/db.zig");
|
||||
const events = @import("../../storage/events.zig");
|
||||
const http_util = @import("../http_util.zig");
|
||||
const server = @import("../server.zig");
|
||||
|
||||
const log = std.log.scoped(.web_diagnostics);
|
||||
|
||||
pub const default_limit: u32 = 100;
|
||||
pub const max_limit: u32 = events.max_limit;
|
||||
|
||||
/// Wide enough for every component `events.component` can produce, and for a
|
||||
/// mistyped one to still be reported as a bad filter rather than a truncated
|
||||
/// match.
|
||||
pub const max_component_len = 64;
|
||||
|
||||
/// Where the string filters are copied to. The parsed filter borrows them, so
|
||||
/// it must not outlive the buffers — in the handler both live in the same stack
|
||||
/// frame.
|
||||
pub const Buffers = struct {
|
||||
state: [16]u8 = undefined,
|
||||
severity: [16]u8 = undefined,
|
||||
component: [max_component_len]u8 = undefined,
|
||||
};
|
||||
|
||||
pub const FilterError = error{
|
||||
BadState,
|
||||
BadSeverity,
|
||||
BadComponent,
|
||||
BadSince,
|
||||
BadUntil,
|
||||
BadLimit,
|
||||
BadBefore,
|
||||
};
|
||||
|
||||
/// An absent parameter drops the filter; a malformed one is a 400 rather than a
|
||||
/// filter silently left off, which would answer a question the client did not
|
||||
/// ask.
|
||||
pub fn parseFilter(query: []const u8, buffers: *Buffers) FilterError!events.Filter {
|
||||
var filter: events.Filter = .{ .limit = default_limit };
|
||||
|
||||
if (http_util.queryValue(query, "state", &buffers.state) catch return error.BadState) |text| {
|
||||
filter.state = std.meta.stringToEnum(events.State, text) orelse return error.BadState;
|
||||
}
|
||||
|
||||
if (http_util.queryValue(query, "severity", &buffers.severity) catch return error.BadSeverity) |text| {
|
||||
// Bound as text by the repository, so it is normalised to one of the
|
||||
// two stored spellings here rather than passed through.
|
||||
const severity = std.meta.stringToEnum(events.Severity, text) orelse return error.BadSeverity;
|
||||
filter.severity = severity.text();
|
||||
}
|
||||
|
||||
if (http_util.queryValue(query, "component", &buffers.component) catch return error.BadComponent) |text| {
|
||||
if (text.len != 0) filter.component = text;
|
||||
}
|
||||
|
||||
filter.since = http_util.queryInt(i64, query, "since") catch return error.BadSince;
|
||||
filter.until = http_util.queryInt(i64, query, "until") catch return error.BadUntil;
|
||||
|
||||
if (http_util.queryInt(u32, query, "limit") catch return error.BadLimit) |limit| {
|
||||
if (limit == 0 or limit > max_limit) return error.BadLimit;
|
||||
filter.limit = limit;
|
||||
}
|
||||
|
||||
if (http_util.queryInt(i64, query, "before") catch return error.BadBefore) |before| {
|
||||
// Row ids are positive, so a non-positive cursor is a client bug, not
|
||||
// an empty page.
|
||||
if (before <= 0) return error.BadBefore;
|
||||
filter.before = before;
|
||||
}
|
||||
|
||||
return filter;
|
||||
}
|
||||
|
||||
pub fn message(err: FilterError) []const u8 {
|
||||
return switch (err) {
|
||||
error.BadState => "state must be active, resolved or all",
|
||||
error.BadSeverity => "severity must be warning or error",
|
||||
error.BadComponent => "component is not a valid filter",
|
||||
error.BadSince => "since must be a unix timestamp in seconds",
|
||||
error.BadUntil => "until must be a unix timestamp in seconds",
|
||||
error.BadLimit => "limit must be between 1 and 1000",
|
||||
error.BadBefore => "before must be a positive row id",
|
||||
};
|
||||
}
|
||||
|
||||
const unavailable_message = "diagnostics unavailable";
|
||||
|
||||
pub fn list(
|
||||
state: *server.WebState,
|
||||
io: std.Io,
|
||||
request: *http_util.Request,
|
||||
) http_util.HandlerError!void {
|
||||
var buffers: Buffers = .{};
|
||||
const filter = parseFilter(request.query, &buffers) catch |err| {
|
||||
return http_util.respondError(request, .bad_request, message(err));
|
||||
};
|
||||
|
||||
const store = state.events orelse
|
||||
return http_util.respondError(request, .service_unavailable, unavailable_message);
|
||||
|
||||
const page = store.selectEvents(io, request.arena, filter) catch |err| {
|
||||
// The one thing this handler logs: a database fault is a property of
|
||||
// the box, not of the request, and the client is told nothing about it.
|
||||
log.warn("diagnostics read failed: {s}", .{@errorName(err)});
|
||||
return http_util.respondError(request, .internal_server_error, "internal error");
|
||||
};
|
||||
|
||||
return http_util.respondJson(request, .ok, page, &.{});
|
||||
}
|
||||
|
||||
pub fn get(
|
||||
state: *server.WebState,
|
||||
io: std.Io,
|
||||
request: *http_util.Request,
|
||||
) http_util.HandlerError!void {
|
||||
const store = state.events orelse
|
||||
return http_util.respondError(request, .service_unavailable, unavailable_message);
|
||||
|
||||
const row = store.selectOne(io, request.arena, request.id.?) catch |err| {
|
||||
log.warn("diagnostics read failed: {s}", .{@errorName(err)});
|
||||
return http_util.respondError(request, .internal_server_error, "internal error");
|
||||
};
|
||||
|
||||
// An id retention has removed and one that never existed are the same
|
||||
// answer, and the API does not pretend to tell them apart.
|
||||
const event = row orelse return http_util.respondError(request, .not_found, "not found");
|
||||
return http_util.respondJson(request, .ok, event, &.{});
|
||||
}
|
||||
|
||||
/// An open episode is the current state of the box, so it is not history to
|
||||
/// throw away — and the message says what would make it purgeable.
|
||||
const still_active_message = "the event is still active; it can be purged once it resolves";
|
||||
|
||||
/// `DELETE /api/diagnostics` — how many resolved events went.
|
||||
pub const PurgeResult = struct {
|
||||
purged: i64,
|
||||
};
|
||||
|
||||
/// `DELETE /api/diagnostics/{id}`. Resolution stays automatic; this is only
|
||||
/// about when the history disappears, which is the operator's call.
|
||||
///
|
||||
/// Classified `runtime_action` in the route table, not `config_write`: the
|
||||
/// event log is runtime state that no configuration file declares, so file
|
||||
/// authority has nothing to say about it and the router lets this through in
|
||||
/// both modes.
|
||||
pub fn purge(
|
||||
state: *server.WebState,
|
||||
io: std.Io,
|
||||
request: *http_util.Request,
|
||||
) http_util.HandlerError!void {
|
||||
const store = state.events orelse
|
||||
return http_util.respondError(request, .service_unavailable, unavailable_message);
|
||||
|
||||
// No log here: the store already latches and logs the false→true
|
||||
// transition, and a warning per retried request would spam.
|
||||
const outcome = store.purge(io, request.id.?) catch
|
||||
return http_util.respondError(request, .internal_server_error, "internal error");
|
||||
|
||||
return switch (outcome) {
|
||||
.deleted => http_util.respondEmpty(request, .no_content),
|
||||
.active => http_util.respondError(request, .conflict, still_active_message),
|
||||
.absent => http_util.respondError(request, .not_found, "not found"),
|
||||
};
|
||||
}
|
||||
|
||||
/// `DELETE /api/diagnostics` — the whole resolved history at once. Active
|
||||
/// episodes are never touched, so an operator clearing the page cannot lose the
|
||||
/// events that are still true.
|
||||
pub fn purgeAll(
|
||||
state: *server.WebState,
|
||||
io: std.Io,
|
||||
request: *http_util.Request,
|
||||
) http_util.HandlerError!void {
|
||||
const store = state.events orelse
|
||||
return http_util.respondError(request, .service_unavailable, unavailable_message);
|
||||
|
||||
const purged = store.purgeAll(io) catch
|
||||
return http_util.respondError(request, .internal_server_error, "internal error");
|
||||
|
||||
return http_util.respondJson(request, .ok, PurgeResult{ .purged = purged }, &.{});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const migrations = @import("../../storage/migrations.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
test "an empty query string is the default page over everything" {
|
||||
var buffers: Buffers = .{};
|
||||
const filter = try parseFilter("", &buffers);
|
||||
try testing.expectEqual(default_limit, filter.limit);
|
||||
try testing.expectEqual(events.State.all, filter.state);
|
||||
try testing.expectEqual(@as(?[]const u8, null), filter.severity);
|
||||
try testing.expectEqual(@as(?[]const u8, null), filter.component);
|
||||
try testing.expectEqual(@as(?i64, null), filter.before);
|
||||
try testing.expectEqual(@as(?i64, null), filter.since);
|
||||
}
|
||||
|
||||
test "every filter reaches the store untouched" {
|
||||
var buffers: Buffers = .{};
|
||||
const filter = try parseFilter(
|
||||
"state=resolved&severity=error&component=query_log&since=100&until=200&limit=250&before=900",
|
||||
&buffers,
|
||||
);
|
||||
try testing.expectEqual(events.State.resolved, filter.state);
|
||||
try testing.expectEqualStrings("error", filter.severity.?);
|
||||
try testing.expectEqualStrings("query_log", filter.component.?);
|
||||
try testing.expectEqual(@as(?i64, 100), filter.since);
|
||||
try testing.expectEqual(@as(?i64, 200), filter.until);
|
||||
try testing.expectEqual(@as(u32, 250), filter.limit);
|
||||
try testing.expectEqual(@as(?i64, 900), filter.before);
|
||||
}
|
||||
|
||||
test "an empty component is no filter at all" {
|
||||
var buffers: Buffers = .{};
|
||||
try testing.expectEqual(@as(?[]const u8, null), (try parseFilter("component=", &buffers)).component);
|
||||
}
|
||||
|
||||
test "each malformed parameter names itself in a 400" {
|
||||
var buffers: Buffers = .{};
|
||||
try testing.expectError(error.BadState, parseFilter("state=open", &buffers));
|
||||
try testing.expectError(error.BadState, parseFilter("state=", &buffers));
|
||||
try testing.expectError(error.BadSeverity, parseFilter("severity=info", &buffers));
|
||||
try testing.expectError(error.BadSeverity, parseFilter("severity=WARNING", &buffers));
|
||||
try testing.expectError(error.BadSince, parseFilter("since=yesterday", &buffers));
|
||||
try testing.expectError(error.BadUntil, parseFilter("until=", &buffers));
|
||||
try testing.expectError(error.BadLimit, parseFilter("limit=0", &buffers));
|
||||
try testing.expectError(error.BadLimit, parseFilter("limit=1001", &buffers));
|
||||
try testing.expectError(error.BadLimit, parseFilter("limit=ten", &buffers));
|
||||
try testing.expectError(error.BadBefore, parseFilter("before=0", &buffers));
|
||||
try testing.expectError(error.BadBefore, parseFilter("before=-4", &buffers));
|
||||
try testing.expectError(error.BadComponent, parseFilter("component=%zz", &buffers));
|
||||
|
||||
var long: [max_component_len + 8]u8 = @splat('a');
|
||||
var text: std.ArrayList(u8) = .empty;
|
||||
defer text.deinit(testing.allocator);
|
||||
try text.appendSlice(testing.allocator, "component=");
|
||||
try text.appendSlice(testing.allocator, &long);
|
||||
try testing.expectError(error.BadComponent, parseFilter(text.items, &buffers));
|
||||
|
||||
// Every member of the error set has its own wording, and none of them is
|
||||
// the empty string.
|
||||
inline for (comptime std.meta.fieldNames(FilterError)) |name| {
|
||||
try testing.expect(message(@field(FilterError, name)).len != 0);
|
||||
}
|
||||
}
|
||||
|
||||
test "the limit cap is the store's" {
|
||||
var buffers: Buffers = .{};
|
||||
try testing.expectEqual(max_limit, (try parseFilter("limit=1000", &buffers)).limit);
|
||||
try testing.expectEqual(@as(u32, 1000), events.max_limit);
|
||||
}
|
||||
|
||||
/// A store over a migrated in-memory `config.db`, built in place: a `Store`
|
||||
/// holds a `*db.Db`, so a fixture that moved after `init` would leave that
|
||||
/// pointer behind.
|
||||
const Fixture = struct {
|
||||
threaded: std.Io.Threaded = undefined,
|
||||
io: std.Io = undefined,
|
||||
database: db.Db = undefined,
|
||||
store: events.Store = undefined,
|
||||
state: server.WebState = undefined,
|
||||
|
||||
fn init(self: *Fixture) !void {
|
||||
self.threaded = .init(testing.allocator, .{});
|
||||
errdefer self.threaded.deinit();
|
||||
self.io = self.threaded.io();
|
||||
|
||||
self.database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
errdefer self.database.close();
|
||||
try db.applyPragmas(&self.database, .{});
|
||||
_ = try migrations.migrate(&self.database);
|
||||
|
||||
self.store = try events.Store.init(self.io, &self.database, 1000);
|
||||
self.state = .{ .gpa = testing.allocator, .events = &self.store };
|
||||
}
|
||||
|
||||
fn deinit(self: *Fixture) void {
|
||||
self.database.close();
|
||||
self.threaded.deinit();
|
||||
}
|
||||
|
||||
fn seed(self: *Fixture) void {
|
||||
const store = &self.store;
|
||||
store.report(self.io, 1000, .blocklist_refresh, "https://a.example", "StevenBlack", .warning, "ConnectionTimedOut");
|
||||
store.report(self.io, 1100, .blocklist_refresh, "https://a.example", "StevenBlack", .warning, "ConnectionTimedOut");
|
||||
store.report(self.io, 1200, .listener_start, "doh", "doh", .@"error", "AddressInUse");
|
||||
store.report(self.io, 1300, .query_log_write, "batch", "batch", .@"error", "Busy");
|
||||
store.resolve(self.io, 1400, .query_log_write, "batch");
|
||||
}
|
||||
};
|
||||
|
||||
test "a page carries the events, the cursor and the active counts" {
|
||||
var fx: Fixture = .{};
|
||||
try fx.init();
|
||||
defer fx.deinit();
|
||||
fx.seed();
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const page = try fx.store.selectEvents(fx.io, arena.allocator(), .{ .limit = 100 });
|
||||
try testing.expectEqual(@as(usize, 3), page.events.len);
|
||||
try testing.expectEqual(@as(?i64, null), page.next_before);
|
||||
try testing.expectEqual(events.Counts{ .warnings = 1, .errors = 1 }, page.active);
|
||||
|
||||
// Newest first, and the episode that repeated counts rather than repeats.
|
||||
try testing.expectEqualStrings("query_log.write", page.events[0].code);
|
||||
try testing.expectEqualStrings("blocklist.refresh", page.events[2].code);
|
||||
try testing.expectEqual(@as(i64, 2), page.events[2].occurrences);
|
||||
try testing.expectEqualStrings("StevenBlack", page.events[2].subject);
|
||||
}
|
||||
|
||||
test "a full page carries a cursor and the last page does not" {
|
||||
var fx: Fixture = .{};
|
||||
try fx.init();
|
||||
defer fx.deinit();
|
||||
fx.seed();
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
const gpa = arena.allocator();
|
||||
|
||||
const first = try fx.store.selectEvents(fx.io, gpa, .{ .limit = 2 });
|
||||
try testing.expectEqual(@as(usize, 2), first.events.len);
|
||||
try testing.expectEqual(first.events[1].id, first.next_before.?);
|
||||
|
||||
const second = try fx.store.selectEvents(fx.io, gpa, .{ .limit = 2, .before = first.next_before });
|
||||
try testing.expectEqual(@as(usize, 1), second.events.len);
|
||||
try testing.expectEqual(@as(?i64, null), second.next_before);
|
||||
}
|
||||
|
||||
test "the parsed filters narrow the page the store answers with" {
|
||||
var fx: Fixture = .{};
|
||||
try fx.init();
|
||||
defer fx.deinit();
|
||||
fx.seed();
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
const gpa = arena.allocator();
|
||||
var buffers: Buffers = .{};
|
||||
|
||||
const active = try fx.store.selectEvents(fx.io, gpa, try parseFilter("state=active", &buffers));
|
||||
try testing.expectEqual(@as(usize, 2), active.events.len);
|
||||
|
||||
const resolved = try fx.store.selectEvents(fx.io, gpa, try parseFilter("state=resolved", &buffers));
|
||||
try testing.expectEqual(@as(usize, 1), resolved.events.len);
|
||||
try testing.expectEqual(@as(?i64, 1400), resolved.events[0].resolved_at);
|
||||
|
||||
const errors = try fx.store.selectEvents(fx.io, gpa, try parseFilter("severity=error", &buffers));
|
||||
try testing.expectEqual(@as(usize, 2), errors.events.len);
|
||||
|
||||
const blocklist = try fx.store.selectEvents(fx.io, gpa, try parseFilter("component=blocklist", &buffers));
|
||||
try testing.expectEqual(@as(usize, 1), blocklist.events.len);
|
||||
try testing.expectEqualStrings("blocklist", blocklist.events[0].component);
|
||||
}
|
||||
|
||||
test "the wire object carries the label and never the subject key" {
|
||||
var fx: Fixture = .{};
|
||||
try fx.init();
|
||||
defer fx.deinit();
|
||||
|
||||
// A key that would be unmistakable on the wire if it ever leaked: the
|
||||
// upstream urls this store keys on can carry a token.
|
||||
fx.store.report(fx.io, 1000, .upstream_exchange, "https://dns.example/secret-token", "dns.example", .warning, "Timeout");
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
const page = try fx.store.selectEvents(fx.io, arena.allocator(), .{});
|
||||
|
||||
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer allocating.deinit();
|
||||
try std.json.Stringify.value(page, .{}, &allocating.writer);
|
||||
const text = allocating.written();
|
||||
|
||||
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "secret-token"));
|
||||
try testing.expect(!std.mem.containsAtLeast(u8, text, 1, "subject_key"));
|
||||
try testing.expect(std.mem.startsWith(u8, text, "{\"events\":["));
|
||||
for ([_][]const u8{
|
||||
"\"id\":", "\"code\":\"upstream.exchange\"",
|
||||
"\"component\":\"upstream\"", "\"subject\":\"dns.example\"",
|
||||
"\"severity\":\"warning\"", "\"first_seen\":",
|
||||
"\"last_seen\":", "\"occurrences\":",
|
||||
"\"resolved_at\":null", "\"detail\":\"Timeout\"",
|
||||
"\"next_before\":null", "\"active\":{\"warnings\":1,\"errors\":0}",
|
||||
}) |field| {
|
||||
errdefer std.debug.print("missing {s} in {s}\n", .{ field, text });
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, field));
|
||||
}
|
||||
}
|
||||
|
||||
test "a detail page answers by id and reports an unknown one as absent" {
|
||||
var fx: Fixture = .{};
|
||||
try fx.init();
|
||||
defer fx.deinit();
|
||||
fx.seed();
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const found = (try fx.store.selectOne(fx.io, arena.allocator(), 1)).?;
|
||||
try testing.expectEqualStrings("blocklist.refresh", found.code);
|
||||
// The 404 the handler answers with is this null.
|
||||
try testing.expect((try fx.store.selectOne(fx.io, arena.allocator(), 9999)) == null);
|
||||
}
|
||||
|
||||
test "the purge-all body is the count and nothing else" {
|
||||
var allocating: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer allocating.deinit();
|
||||
try std.json.Stringify.value(PurgeResult{ .purged = 3 }, .{}, &allocating.writer);
|
||||
try testing.expectEqualStrings("{\"purged\":3}", allocating.written());
|
||||
|
||||
// The 409 says what would make the event purgeable, so it is not the
|
||||
// generic conflict wording.
|
||||
try testing.expect(still_active_message.len != 0);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, still_active_message, 1, "resolves"));
|
||||
}
|
||||
|
||||
test "a state with no store answers 503 rather than an empty page" {
|
||||
// The handler's only branch that does not need a request: a `WebState`
|
||||
// whose store failed to open reports the endpoint unavailable, and never
|
||||
// an empty list that would read as "nothing is wrong".
|
||||
const state: server.WebState = .{ .gpa = testing.allocator };
|
||||
try testing.expectEqual(@as(?*events.Store, null), state.events);
|
||||
try testing.expect(unavailable_message.len != 0);
|
||||
}
|
||||
+114
-1
@@ -25,6 +25,15 @@ pub const Disk = struct {
|
||||
sample_failures: u64,
|
||||
};
|
||||
|
||||
/// The diagnostics store's own state, not a summary of what it holds: `state`
|
||||
/// answers "is the operational log recording", and the two counts answer "what
|
||||
/// is open right now".
|
||||
pub const Diagnostics = struct {
|
||||
state: []const u8,
|
||||
active_warnings: u32,
|
||||
active_errors: u32,
|
||||
};
|
||||
|
||||
pub const Upstreams = struct {
|
||||
available: u32,
|
||||
total: u32,
|
||||
@@ -34,6 +43,7 @@ pub const Body = struct {
|
||||
status: []const u8,
|
||||
disk: Disk,
|
||||
upstreams: Upstreams,
|
||||
diagnostics: Diagnostics,
|
||||
queries_dropped: u64,
|
||||
writer_failed: bool,
|
||||
refreshes_gated: u64,
|
||||
@@ -61,6 +71,16 @@ pub const Input = struct {
|
||||
/// one overflow. Drops surface through the metric and through the API's
|
||||
/// per-window `complete` instead.
|
||||
history_flush_failing: bool = false,
|
||||
/// The diagnostics store exists. The benign default matches every other
|
||||
/// field here — a half-wired `Input` reports a box with nothing wrong — but
|
||||
/// `collect` must assign it explicitly, because in a serving process an
|
||||
/// absent store means `Store.init` failed.
|
||||
diagnostics_present: bool = true,
|
||||
/// The last diagnostics write failed. Current state, cleared by the next
|
||||
/// write that succeeds, like `history_flush_failing`.
|
||||
diagnostics_write_failed: bool = false,
|
||||
diagnostics_active_warnings: u32 = 0,
|
||||
diagnostics_active_errors: u32 = 0,
|
||||
refreshes_gated: u64 = 0,
|
||||
snapshot_generation: ?u64 = null,
|
||||
};
|
||||
@@ -68,6 +88,16 @@ pub const Input = struct {
|
||||
pub const status_ok = "ok";
|
||||
pub const status_degraded = "degraded";
|
||||
|
||||
pub const diagnostics_recording = "recording";
|
||||
pub const diagnostics_unavailable = "unavailable";
|
||||
|
||||
/// The operational log is not recording — either the store never opened or its
|
||||
/// writes are failing. Both mean the same thing to an operator: the record of
|
||||
/// what went wrong is not being kept.
|
||||
pub fn diagnosticsUnavailable(input: Input) bool {
|
||||
return !input.diagnostics_present or input.diagnostics_write_failed;
|
||||
}
|
||||
|
||||
/// Conditions an operator must act on, and every one of them is a fact about
|
||||
/// now rather than a count of the past: a disk that is filling stops the query
|
||||
/// log, a pool with nothing available stops resolution, a failed writer means
|
||||
@@ -76,7 +106,7 @@ pub const status_degraded = "degraded";
|
||||
/// the underlying condition does.
|
||||
pub fn degraded(input: Input) bool {
|
||||
return input.disk_state != .ok or input.upstreams_available == 0 or
|
||||
input.writer_failed or input.history_flush_failing;
|
||||
input.writer_failed or input.history_flush_failing or diagnosticsUnavailable(input);
|
||||
}
|
||||
|
||||
pub fn rollup(input: Input) Body {
|
||||
@@ -90,6 +120,11 @@ pub fn rollup(input: Input) Body {
|
||||
.sample_failures = input.disk_sample_failures,
|
||||
},
|
||||
.upstreams = .{ .available = input.upstreams_available, .total = input.upstreams_total },
|
||||
.diagnostics = .{
|
||||
.state = if (diagnosticsUnavailable(input)) diagnostics_unavailable else diagnostics_recording,
|
||||
.active_warnings = input.diagnostics_active_warnings,
|
||||
.active_errors = input.diagnostics_active_errors,
|
||||
},
|
||||
.queries_dropped = input.queries_dropped,
|
||||
.writer_failed = input.writer_failed,
|
||||
.refreshes_gated = input.refreshes_gated,
|
||||
@@ -128,6 +163,17 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
|
||||
input.writer_failed = logger.writer_failed.load(.monotonic);
|
||||
}
|
||||
|
||||
// Assigned before the `if`, not inside it: the field's benign default is
|
||||
// `true`, so the natural `if (state.events) |store|` shape would report an
|
||||
// absent store as recording — the one case that must degrade.
|
||||
input.diagnostics_present = state.events != null;
|
||||
if (state.events) |store| {
|
||||
input.diagnostics_write_failed = store.writeFailed();
|
||||
const counts = store.activeCounts(io);
|
||||
input.diagnostics_active_warnings = counts.warnings;
|
||||
input.diagnostics_active_errors = counts.errors;
|
||||
}
|
||||
|
||||
if (state.history) |history| {
|
||||
input.history_flush_failing = history.snapshotStats(io).last_flush_failed;
|
||||
}
|
||||
@@ -148,6 +194,8 @@ pub fn collect(state: *server.WebState, io: std.Io) Input {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const db = @import("../../storage/db.zig");
|
||||
const events_mod = @import("../../storage/events.zig");
|
||||
const migrations = @import("../../storage/migrations.zig");
|
||||
const history_mod = @import("../../upstream/history.zig");
|
||||
const logger_mod = @import("../../storage/logger.zig");
|
||||
const upstream_history_repo = @import("../../storage/repositories/upstream_history_repo.zig");
|
||||
@@ -172,6 +220,11 @@ test "the degraded matrix covers disk state, availability and the writer" {
|
||||
// right now, and it recovers on its own the moment a flush succeeds.
|
||||
.{ .input = withHistoryFailing(healthy, true), .degraded = true },
|
||||
.{ .input = withHistoryFailing(healthy, false), .degraded = false },
|
||||
// The operational log not recording is itself a fault an operator must
|
||||
// act on: whatever fails next will leave no record of having failed.
|
||||
.{ .input = withDiagnostics(healthy, false, false), .degraded = true },
|
||||
.{ .input = withDiagnostics(healthy, true, true), .degraded = true },
|
||||
.{ .input = withDiagnostics(healthy, true, false), .degraded = false },
|
||||
// Two faults at once still report one status.
|
||||
.{ .input = withWriterFailed(withDisk(healthy, .critical)), .degraded = true },
|
||||
// Some upstreams down is not degraded while one still answers.
|
||||
@@ -212,6 +265,66 @@ fn withHistoryFailing(input: Input, failing: bool) Input {
|
||||
return out;
|
||||
}
|
||||
|
||||
fn withDiagnostics(input: Input, present: bool, write_failed: bool) Input {
|
||||
var out = input;
|
||||
out.diagnostics_present = present;
|
||||
out.diagnostics_write_failed = write_failed;
|
||||
return out;
|
||||
}
|
||||
|
||||
test "the diagnostics block reports the state and the open counts" {
|
||||
const recording = rollup(.{
|
||||
.upstreams_available = 1,
|
||||
.diagnostics_active_warnings = 3,
|
||||
.diagnostics_active_errors = 1,
|
||||
});
|
||||
try testing.expectEqualStrings(diagnostics_recording, recording.diagnostics.state);
|
||||
try testing.expectEqual(@as(u32, 3), recording.diagnostics.active_warnings);
|
||||
try testing.expectEqual(@as(u32, 1), recording.diagnostics.active_errors);
|
||||
// Open episodes are what the box is doing, not a fault of the log: they do
|
||||
// not degrade on their own.
|
||||
try testing.expectEqualStrings(status_ok, recording.status);
|
||||
|
||||
// Failing writes: the counts are whatever was last read, and the state is
|
||||
// the honest one.
|
||||
const failing = rollup(.{ .upstreams_available = 1, .diagnostics_write_failed = true });
|
||||
try testing.expectEqualStrings(diagnostics_unavailable, failing.diagnostics.state);
|
||||
try testing.expectEqualStrings(status_degraded, failing.status);
|
||||
|
||||
const absent = rollup(.{ .upstreams_available = 1, .diagnostics_present = false });
|
||||
try testing.expectEqualStrings(diagnostics_unavailable, absent.diagnostics.state);
|
||||
try testing.expectEqualStrings(status_degraded, absent.status);
|
||||
}
|
||||
|
||||
test "collect reports an absent store as unavailable rather than as recording" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
// `diagnostics_present` defaults to true like every other benign default,
|
||||
// so an assignment `collect` forgot would read as a healthy log here.
|
||||
var state: server.WebState = .{ .gpa = testing.allocator };
|
||||
const absent = collect(&state, io);
|
||||
try testing.expect(!absent.diagnostics_present);
|
||||
try testing.expectEqualStrings(diagnostics_unavailable, rollup(absent).diagnostics.state);
|
||||
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
defer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
_ = try migrations.migrate(&database);
|
||||
var store = try events_mod.Store.init(io, &database, 1000);
|
||||
store.report(io, 1000, .disk_space, "data", "data", .warning, "low");
|
||||
store.report(io, 1000, .listener_start, "doh", "doh", .@"error", "AddressInUse");
|
||||
|
||||
state.events = &store;
|
||||
const present = collect(&state, io);
|
||||
try testing.expect(present.diagnostics_present);
|
||||
try testing.expect(!present.diagnostics_write_failed);
|
||||
try testing.expectEqual(@as(u32, 1), present.diagnostics_active_warnings);
|
||||
try testing.expectEqual(@as(u32, 1), present.diagnostics_active_errors);
|
||||
try testing.expectEqualStrings(diagnostics_recording, rollup(present).diagnostics.state);
|
||||
}
|
||||
|
||||
test "a history overflow that already happened does not degrade the rollup" {
|
||||
// `rows_dropped` is cumulative and the rollup is stateless, so the only
|
||||
// thing it could do with a drop count is latch on it. The accumulator's
|
||||
|
||||
@@ -30,6 +30,7 @@ const dns_cache = @import("../cache/dns_cache.zig");
|
||||
const dns_handler = @import("../server/handler.zig");
|
||||
const disk_monitor = @import("../storage/disk_monitor.zig");
|
||||
const dot_server = @import("../server/dot_server.zig");
|
||||
const events_mod = @import("../storage/events.zig");
|
||||
const history_mod = @import("../upstream/history.zig");
|
||||
const http_util = @import("http_util.zig");
|
||||
const logging = @import("../platform/logging.zig");
|
||||
@@ -115,6 +116,15 @@ pub const UpstreamSample = struct {
|
||||
success_rate: f32,
|
||||
};
|
||||
|
||||
/// The diagnostics store, as one scrape sees it. Two gauges and a counter,
|
||||
/// written out rather than reflected over a stats struct because they are not
|
||||
/// all the same kind of number.
|
||||
pub const DiagnosticsSample = struct {
|
||||
active_warnings: u32,
|
||||
active_errors: u32,
|
||||
write_failures: u64,
|
||||
};
|
||||
|
||||
/// Everything one scrape reports. A null section is a collaborator the state
|
||||
/// does not have.
|
||||
pub const Sample = struct {
|
||||
@@ -129,6 +139,11 @@ pub const Sample = struct {
|
||||
/// The upstream-history flush loop's counters (m26 ruling 7). Absent while
|
||||
/// no accumulator is wired, like every other collaborator.
|
||||
history: ?history_mod.Accumulator.Stats = null,
|
||||
/// The diagnostics store's open episodes and its failed writes. Absent
|
||||
/// while no store is wired, like every other collaborator — an operator
|
||||
/// distinguishes "no series" from "zero episodes" through `/api/health`,
|
||||
/// which says which of the two it is.
|
||||
diagnostics: ?DiagnosticsSample = null,
|
||||
blocklist: ?BlocklistSample = null,
|
||||
disk: ?DiskSample = null,
|
||||
/// One entry per enabled TLS endpoint (milestone-10 ruling 10). Rendered
|
||||
@@ -205,6 +220,15 @@ pub fn collect(state: *server.WebState, io: std.Io, arena: Allocator) Allocator.
|
||||
|
||||
if (state.history) |history| sample.history = history.snapshotStats(io);
|
||||
|
||||
if (state.events) |store| {
|
||||
const counts = store.activeCounts(io);
|
||||
sample.diagnostics = .{
|
||||
.active_warnings = counts.warnings,
|
||||
.active_errors = counts.errors,
|
||||
.write_failures = store.writeFailures(),
|
||||
};
|
||||
}
|
||||
|
||||
if (state.manager) |manager| {
|
||||
const generation: ?u64 = if (manager.acquire(io)) |acquired| gen: {
|
||||
defer acquired.release(io);
|
||||
@@ -388,6 +412,27 @@ pub fn render(w: *std.Io.Writer, sample: Sample) std.Io.Writer.Error!void {
|
||||
);
|
||||
}
|
||||
|
||||
if (sample.diagnostics) |diagnostics| {
|
||||
try gauge(
|
||||
w,
|
||||
"nxdns_diagnostics_active_warnings",
|
||||
"Operational event episodes open right now at warning severity.",
|
||||
diagnostics.active_warnings,
|
||||
);
|
||||
try gauge(
|
||||
w,
|
||||
"nxdns_diagnostics_active_errors",
|
||||
"Operational event episodes open right now at error severity.",
|
||||
diagnostics.active_errors,
|
||||
);
|
||||
try counter(
|
||||
w,
|
||||
"nxdns_diagnostics_write_failures_total",
|
||||
"Operational events dropped because the diagnostics database refused the write.",
|
||||
diagnostics.write_failures,
|
||||
);
|
||||
}
|
||||
|
||||
if (sample.blocklist) |blocklist| {
|
||||
try counter(
|
||||
w,
|
||||
@@ -638,8 +683,10 @@ fn writeLabelValue(w: *std.Io.Writer, value: []const u8) std.Io.Writer.Error!voi
|
||||
// tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const db = @import("../storage/db.zig");
|
||||
const local_tables = @import("../server/local_tables.zig");
|
||||
const logger_mod = @import("../storage/logger.zig");
|
||||
const migrations = @import("../storage/migrations.zig");
|
||||
const testing = std.testing;
|
||||
|
||||
/// A handler with no upstream reachable: every test here reads counters and
|
||||
@@ -781,6 +828,55 @@ test "the upstream-history family renders three counters and one gauge" {
|
||||
try testing.expect(!std.mem.containsAtLeast(u8, bare, 1, "nxdns_upstream_history_"));
|
||||
}
|
||||
|
||||
test "the diagnostics family renders two gauges and one counter" {
|
||||
const text = try renderToString(testing.allocator, .{
|
||||
.diagnostics = .{ .active_warnings = 3, .active_errors = 1, .write_failures = 7 },
|
||||
});
|
||||
defer testing.allocator.free(text);
|
||||
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_diagnostics_active_warnings gauge\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_diagnostics_active_warnings 3\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_diagnostics_active_errors gauge\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_diagnostics_active_errors 1\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "# TYPE nxdns_diagnostics_write_failures_total counter\n"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, text, 1, "nxdns_diagnostics_write_failures_total 7\n"));
|
||||
|
||||
// No store is an absent family, not a family of zeros: "no episodes open"
|
||||
// and "nothing is recording them" must not render the same.
|
||||
const bare = try renderToString(testing.allocator, .{});
|
||||
defer testing.allocator.free(bare);
|
||||
try testing.expect(!std.mem.containsAtLeast(u8, bare, 1, "nxdns_diagnostics_"));
|
||||
}
|
||||
|
||||
test "collect reads the diagnostics store's open episodes and failed writes" {
|
||||
var threaded: std.Io.Threaded = .init(testing.allocator, .{});
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
|
||||
var database = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
defer database.close();
|
||||
try db.applyPragmas(&database, .{});
|
||||
_ = try migrations.migrate(&database);
|
||||
var store = try events_mod.Store.init(io, &database, 1000);
|
||||
store.report(io, 1000, .disk_space, "data", "data", .warning, "low");
|
||||
|
||||
var state: server.WebState = .{ .gpa = testing.allocator, .events = &store };
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
|
||||
const sample = try collect(&state, io, arena.allocator());
|
||||
try testing.expectEqual(@as(u32, 1), sample.diagnostics.?.active_warnings);
|
||||
try testing.expectEqual(@as(u32, 0), sample.diagnostics.?.active_errors);
|
||||
try testing.expectEqual(@as(u64, 0), sample.diagnostics.?.write_failures);
|
||||
|
||||
// A write that cannot land is counted here, through the production path
|
||||
// rather than by poking the field.
|
||||
try database.exec("DROP TABLE operational_events;");
|
||||
store.report(io, 1100, .disk_space, "data", "data", .warning, "low");
|
||||
const after = try collect(&state, io, arena.allocator());
|
||||
try testing.expect(after.diagnostics.?.write_failures >= 1);
|
||||
}
|
||||
|
||||
test "every HELP line has a TYPE line and a sample, and every sample a name" {
|
||||
const text = try renderToString(testing.allocator, .{});
|
||||
defer testing.allocator.free(text);
|
||||
|
||||
+222
-1
@@ -256,6 +256,131 @@ paths:
|
||||
"503":
|
||||
$ref: "#/components/responses/Unavailable"
|
||||
|
||||
/api/diagnostics:
|
||||
get:
|
||||
summary: Operational event log
|
||||
description: |
|
||||
Failure episodes, newest first. One event is one subject failing
|
||||
continuously: it opens on the first failure, counts repeats in
|
||||
`occurrences`, and gets a `resolved_at` when the subject recovers. A
|
||||
subject that fails again opens a new event rather than reopening the
|
||||
old one. Keyset pagination — follow `next_before` until it is null.
|
||||
parameters:
|
||||
- name: state
|
||||
in: query
|
||||
schema: { type: string, enum: [active, resolved, all], default: all }
|
||||
- name: severity
|
||||
in: query
|
||||
schema: { type: string, enum: [warning, error] }
|
||||
- name: component
|
||||
in: query
|
||||
description: Matches the part of `code` before the dot, exactly.
|
||||
schema: { type: string, maxLength: 64 }
|
||||
- name: since
|
||||
in: query
|
||||
description: |
|
||||
Unix seconds. With `until`, selects episodes overlapping the
|
||||
window; an episode resolved exactly at `since` does not overlap.
|
||||
schema: { type: integer }
|
||||
- name: until
|
||||
in: query
|
||||
description: Unix seconds, exclusive.
|
||||
schema: { type: integer }
|
||||
- name: limit
|
||||
in: query
|
||||
schema: { type: integer, minimum: 1, maximum: 1000, default: 100 }
|
||||
- name: before
|
||||
in: query
|
||||
description: Return events with id strictly below this cursor.
|
||||
schema: { type: integer, minimum: 1 }
|
||||
responses:
|
||||
"200":
|
||||
description: One page.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/DiagnosticsPage"
|
||||
"400":
|
||||
$ref: "#/components/responses/BadRequest"
|
||||
"401":
|
||||
$ref: "#/components/responses/Unauthorized"
|
||||
"429":
|
||||
$ref: "#/components/responses/RateLimited"
|
||||
"500":
|
||||
$ref: "#/components/responses/Internal"
|
||||
"503":
|
||||
$ref: "#/components/responses/Unavailable"
|
||||
delete:
|
||||
summary: Purge every resolved event
|
||||
description: |
|
||||
Deletes the resolved history and answers with how many rows went.
|
||||
Active events are never touched, so clearing the page cannot lose an
|
||||
episode that is still failing. Resolution stays automatic; this only
|
||||
decides when the history disappears. A runtime action, served in file
|
||||
mode too — the event log is not configuration.
|
||||
responses:
|
||||
"200":
|
||||
description: How many resolved events were removed.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/DiagnosticsPurge"
|
||||
"401":
|
||||
$ref: "#/components/responses/Unauthorized"
|
||||
"429":
|
||||
$ref: "#/components/responses/RateLimited"
|
||||
"500":
|
||||
$ref: "#/components/responses/Internal"
|
||||
"503":
|
||||
$ref: "#/components/responses/Unavailable"
|
||||
|
||||
/api/diagnostics/{id}:
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/RowId"
|
||||
get:
|
||||
summary: One operational event
|
||||
description: |
|
||||
404 for an id that never existed and for one retention has removed —
|
||||
the API does not distinguish them.
|
||||
responses:
|
||||
"200":
|
||||
description: The event.
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/DiagnosticEvent"
|
||||
"401":
|
||||
$ref: "#/components/responses/Unauthorized"
|
||||
"404":
|
||||
$ref: "#/components/responses/NotFound"
|
||||
"429":
|
||||
$ref: "#/components/responses/RateLimited"
|
||||
"500":
|
||||
$ref: "#/components/responses/Internal"
|
||||
"503":
|
||||
$ref: "#/components/responses/Unavailable"
|
||||
delete:
|
||||
summary: Purge one resolved event
|
||||
description: |
|
||||
Deletes a resolved event. An event that is still active answers 409 —
|
||||
an open episode is the current state of the box, not history — and an
|
||||
id no row holds answers 404. A runtime action, served in file mode too.
|
||||
responses:
|
||||
"204":
|
||||
description: Purged.
|
||||
"401":
|
||||
$ref: "#/components/responses/Unauthorized"
|
||||
"404":
|
||||
$ref: "#/components/responses/NotFound"
|
||||
"409":
|
||||
$ref: "#/components/responses/Conflict"
|
||||
"429":
|
||||
$ref: "#/components/responses/RateLimited"
|
||||
"500":
|
||||
$ref: "#/components/responses/Internal"
|
||||
"503":
|
||||
$ref: "#/components/responses/Unavailable"
|
||||
|
||||
/api/stats:
|
||||
get:
|
||||
summary: Totals for a period
|
||||
@@ -1664,11 +1789,21 @@ components:
|
||||
|
||||
Health:
|
||||
type: object
|
||||
required: [status, disk, upstreams, queries_dropped, writer_failed, refreshes_gated, snapshot_generation]
|
||||
required: [status, disk, upstreams, diagnostics, queries_dropped, writer_failed, refreshes_gated, snapshot_generation]
|
||||
properties:
|
||||
status:
|
||||
type: string
|
||||
enum: [ok, degraded]
|
||||
diagnostics:
|
||||
type: object
|
||||
required: [state, active_warnings, active_errors]
|
||||
properties:
|
||||
state:
|
||||
type: string
|
||||
enum: [recording, unavailable]
|
||||
description: unavailable when the event store failed to open or its writes are failing; either state degrades health.
|
||||
active_warnings: { type: integer }
|
||||
active_errors: { type: integer }
|
||||
disk:
|
||||
type: object
|
||||
required: [state, free_bytes, db_bytes, log_bytes, sample_failures]
|
||||
@@ -1760,6 +1895,92 @@ components:
|
||||
nullable: true
|
||||
description: Cursor for the next page; null on the last page.
|
||||
|
||||
DiagnosticEvent:
|
||||
type: object
|
||||
required: [id, code, component, subject, severity, first_seen, last_seen, occurrences, resolved_at, detail]
|
||||
properties:
|
||||
id: { type: integer }
|
||||
code:
|
||||
type: string
|
||||
description: |
|
||||
The failure kind, as `component.name`. One of a fixed set of
|
||||
fifteen; new codes are added with new releases.
|
||||
enum:
|
||||
- disk.space
|
||||
- disk.probe
|
||||
- blocklist.refresh
|
||||
- blocklist.snapshot
|
||||
- blocklist.storage
|
||||
- certificate.reload
|
||||
- query_log.write
|
||||
- query_log.maintenance
|
||||
- query_log.recreated
|
||||
- upstream_history.write
|
||||
- upstream.exchange
|
||||
- client_names.storage
|
||||
- clients.storage
|
||||
- listener.start
|
||||
- configuration.load
|
||||
component:
|
||||
type: string
|
||||
description: The part of `code` before the dot, repeated for filtering.
|
||||
subject:
|
||||
type: string
|
||||
description: |
|
||||
What failed, as a display name: a blocklist source name, an
|
||||
endpoint, an operation. Redacted where it derives from a url; the
|
||||
store's internal identity for the subject is never exposed.
|
||||
severity:
|
||||
type: string
|
||||
enum: [warning, error]
|
||||
first_seen:
|
||||
type: integer
|
||||
description: When this episode opened, unix seconds.
|
||||
last_seen:
|
||||
type: integer
|
||||
description: The most recent failure of this episode, unix seconds.
|
||||
occurrences:
|
||||
type: integer
|
||||
description: How many failures this episode has held; at least 1.
|
||||
resolved_at:
|
||||
type: integer
|
||||
nullable: true
|
||||
description: |
|
||||
When the subject recovered, unix seconds. Null while the episode is
|
||||
still open. A subject that fails again opens a new event rather than
|
||||
reopening this one.
|
||||
detail:
|
||||
type: string
|
||||
description: The last error of this episode, truncated to 512 bytes.
|
||||
|
||||
DiagnosticsPage:
|
||||
type: object
|
||||
required: [events, next_before, active]
|
||||
properties:
|
||||
events:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/DiagnosticEvent"
|
||||
next_before:
|
||||
type: integer
|
||||
nullable: true
|
||||
description: Cursor for the next page; null on the last page.
|
||||
active:
|
||||
type: object
|
||||
required: [warnings, errors]
|
||||
description: Episodes open right now, whatever this page filtered to.
|
||||
properties:
|
||||
warnings: { type: integer }
|
||||
errors: { type: integer }
|
||||
|
||||
DiagnosticsPurge:
|
||||
type: object
|
||||
required: [purged]
|
||||
properties:
|
||||
purged:
|
||||
type: integer
|
||||
description: How many resolved events the purge removed; zero when there were none.
|
||||
|
||||
StatsTotals:
|
||||
type: object
|
||||
required: [period, since, until, queries, blocked, cached, clients, avg_response_time_us]
|
||||
|
||||
+12
-1
@@ -36,6 +36,7 @@ const auth = @import("handlers/auth.zig");
|
||||
const blocklists = @import("handlers/blocklists.zig");
|
||||
const certs = @import("handlers/certs.zig");
|
||||
const clients = @import("handlers/clients.zig");
|
||||
const diagnostics = @import("handlers/diagnostics.zig");
|
||||
const groups = @import("handlers/groups.zig");
|
||||
const health = @import("handlers/health.zig");
|
||||
const live = @import("handlers/live.zig");
|
||||
@@ -71,6 +72,14 @@ pub const table: []const router.RouteInfo = &.{
|
||||
.{ .method = .GET, .pattern = "/api/lookup", .auth = .session, .policy = .read, .handler = lookup.handle },
|
||||
.{ .method = .GET, .pattern = "/api/upstream/health", .auth = .session, .policy = .read, .handler = upstream_health.handle },
|
||||
|
||||
// Diagnostics: the operational event log (milestone 27). The two purges are
|
||||
// `runtime_action` — the event log is runtime state no configuration file
|
||||
// declares, so file authority has nothing to say about deleting from it.
|
||||
.{ .method = .GET, .pattern = "/api/diagnostics", .auth = .session, .policy = .read, .handler = diagnostics.list },
|
||||
.{ .method = .DELETE, .pattern = "/api/diagnostics", .auth = .session, .policy = .runtime_action, .handler = diagnostics.purgeAll },
|
||||
.{ .method = .GET, .pattern = "/api/diagnostics/{id}", .auth = .session, .policy = .read, .handler = diagnostics.get },
|
||||
.{ .method = .DELETE, .pattern = "/api/diagnostics/{id}", .auth = .session, .policy = .runtime_action, .handler = diagnostics.purge },
|
||||
|
||||
// Groups.
|
||||
.{ .method = .GET, .pattern = "/api/groups", .auth = .session, .policy = .read, .handler = groups.list },
|
||||
.{ .method = .POST, .pattern = "/api/groups", .auth = .session, .policy = .config_write, .handler = groups.create },
|
||||
@@ -143,7 +152,7 @@ const std = @import("std");
|
||||
const testing = std.testing;
|
||||
|
||||
test "the table carries every endpoint of the milestone" {
|
||||
try testing.expectEqual(@as(usize, 56), table.len);
|
||||
try testing.expectEqual(@as(usize, 60), table.len);
|
||||
}
|
||||
|
||||
test "no two entries claim the same method and pattern" {
|
||||
@@ -231,6 +240,8 @@ test "the runtime actions are exactly ruling 7's list" {
|
||||
"POST /api/auth/login",
|
||||
"POST /api/auth/logout",
|
||||
"POST /api/blocklists/update",
|
||||
"DELETE /api/diagnostics",
|
||||
"DELETE /api/diagnostics/{id}",
|
||||
"DELETE /api/clients/{id}",
|
||||
"POST /api/pause",
|
||||
"POST /api/certs/reload",
|
||||
|
||||
@@ -32,6 +32,7 @@ const disk_monitor = @import("../storage/disk_monitor.zig");
|
||||
const dns_handler = @import("../server/handler.zig");
|
||||
const doh_server = @import("../server/doh_server.zig");
|
||||
const dot_server = @import("../server/dot_server.zig");
|
||||
const events_mod = @import("../storage/events.zig");
|
||||
const http_util = @import("http_util.zig");
|
||||
const listener_core = @import("../server/listener.zig");
|
||||
const local_tables_mod = @import("../server/local_tables.zig");
|
||||
@@ -183,6 +184,11 @@ pub const WebState = struct {
|
||||
/// concurrent writes would misread each other's row counts.
|
||||
config_lock: std.Io.Mutex = .init,
|
||||
querylog_db: ?*db.Db = null,
|
||||
/// The diagnostics event store, which owns a third connection of its own
|
||||
/// and serializes every access — read and write — through its mutex. Null
|
||||
/// when `Store.init` failed, which `/api/health` reports as `unavailable`
|
||||
/// and treats as degraded.
|
||||
events: ?*events_mod.Store = null,
|
||||
|
||||
version: []const u8 = "",
|
||||
/// The `--admin-dev` asset directory, read by the dev-mode fallback. Empty
|
||||
|
||||
@@ -31,6 +31,7 @@ const auth = @import("auth.zig");
|
||||
const clients_repo = @import("../storage/repositories/clients_repo.zig");
|
||||
const db = @import("../storage/db.zig");
|
||||
const dns_handler = @import("../server/handler.zig");
|
||||
const events_mod = @import("../storage/events.zig");
|
||||
const fetcher = @import("../filter/fetcher.zig");
|
||||
const groups_repo = @import("../storage/repositories/groups_repo.zig");
|
||||
const header = @import("../dns/header.zig");
|
||||
@@ -58,6 +59,7 @@ const upstreams_repo = @import("../storage/repositories/upstreams_repo.zig");
|
||||
|
||||
const handlers_blocklists = @import("handlers/blocklists.zig");
|
||||
const handlers_certs = @import("handlers/certs.zig");
|
||||
const handlers_diagnostics = @import("handlers/diagnostics.zig");
|
||||
const handlers_health = @import("handlers/health.zig");
|
||||
const handlers_live = @import("handlers/live.zig");
|
||||
const handlers_lookup = @import("handlers/lookup.zig");
|
||||
@@ -277,6 +279,10 @@ const Env = struct {
|
||||
tmp: testing.TmpDir,
|
||||
config_db: db.Db,
|
||||
querylog_db: db.Db,
|
||||
/// The diagnostics store's own connection, as in production: the store
|
||||
/// serializes every access through its mutex and shares it with nobody.
|
||||
events_db: db.Db,
|
||||
events_store: events_mod.Store,
|
||||
http_client: std.http.Client,
|
||||
transfer_buf: [fetcher.min_transfer_buf]u8,
|
||||
redirect_buf: [fetcher.redirect_buffer_len]u8,
|
||||
@@ -317,6 +323,13 @@ const Env = struct {
|
||||
try self.querylog_db.exec(querylog_schema.ddl);
|
||||
try seedQueryLog(&self.querylog_db);
|
||||
|
||||
self.events_db = try db.Db.open(":memory:", .{ .mode = .memory });
|
||||
errdefer self.events_db.close();
|
||||
try db.applyPragmas(&self.events_db, .{});
|
||||
_ = try migrations.migrate(&self.events_db);
|
||||
self.events_store = try events_mod.Store.init(ioh, &self.events_db, seeded_now);
|
||||
seedEvents(ioh, &self.events_store);
|
||||
|
||||
// Real fetcher wiring; nothing in this suite downloads (the one
|
||||
// refreshAll in the contract walk runs with zero source rows).
|
||||
self.http_client = .{ .allocator = gpa, .io = ioh };
|
||||
@@ -388,6 +401,7 @@ const Env = struct {
|
||||
.hub = self.hub,
|
||||
.config_db = &self.config_db,
|
||||
.querylog_db = &self.querylog_db,
|
||||
.events = &self.events_store,
|
||||
.version = "w10-test",
|
||||
.started_unix = std.Io.Clock.real.now(ioh).toSeconds(),
|
||||
.fallback = options.fallback,
|
||||
@@ -419,6 +433,7 @@ const Env = struct {
|
||||
self.limiter.deinit();
|
||||
self.mgr.deinit(ioh);
|
||||
self.http_client.deinit();
|
||||
self.events_db.close();
|
||||
self.querylog_db.close();
|
||||
self.config_db.close();
|
||||
self.tmp.cleanup();
|
||||
@@ -480,6 +495,21 @@ fn seedQueryLog(database: *db.Db) !void {
|
||||
}
|
||||
}
|
||||
|
||||
/// A fixed instant, like every other seeded timestamp here: the contract
|
||||
/// samples are byte-compared, so nothing the walk writes may come from a clock.
|
||||
const seeded_now: i64 = 1_787_118_000;
|
||||
|
||||
/// One active episode and one resolved one, so `/api/diagnostics` answers with
|
||||
/// both states and the committed contract sample describes a real page rather
|
||||
/// than an empty one.
|
||||
fn seedEvents(io: std.Io, store: *events_mod.Store) void {
|
||||
store.report(io, seeded_now, .blocklist_refresh, "https://lists.example/ads.txt", "StevenBlack", .warning, "download failed: ConnectionTimedOut");
|
||||
store.report(io, seeded_now + 300, .blocklist_refresh, "https://lists.example/ads.txt", "StevenBlack", .warning, "download failed: ConnectionTimedOut");
|
||||
|
||||
store.report(io, seeded_now + 60, .upstream_history_write, "history", "history", .warning, "Busy");
|
||||
store.resolve(io, seeded_now + 120, .upstream_history_write, "history");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// the contract table (ruling 23)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -627,6 +657,15 @@ const contract = [_]Contract{
|
||||
.{ .method = .GET, .pattern = "/api/stats/timeseries", .auth = .session, .policy = .read, .target = "/api/stats/timeseries?period=1h", .status = 200, .check = jsonShape(handlers_stats.TimeseriesBody) },
|
||||
.{ .method = .GET, .pattern = "/api/upstream/health", .auth = .session, .policy = .read, .target = "/api/upstream/health", .status = 200, .check = jsonShape(handlers_upstream_health.Body) },
|
||||
|
||||
// Diagnostics. The seeded store holds one active episode (id 1) and one
|
||||
// resolved one, so both the page and the detail answer with real rows.
|
||||
.{ .method = .GET, .pattern = "/api/diagnostics", .auth = .session, .policy = .read, .target = "/api/diagnostics?limit=10", .status = 200, .check = jsonShape(events_mod.EventsPage) },
|
||||
.{ .method = .GET, .pattern = "/api/diagnostics/{id}", .auth = .session, .policy = .read, .target = "/api/diagnostics/1", .status = 200, .check = jsonShape(events_mod.Event) },
|
||||
// The purges follow the reads: id 2 is the seeded resolved episode, and the
|
||||
// sweep after it takes whatever resolved history is left (none).
|
||||
.{ .method = .DELETE, .pattern = "/api/diagnostics/{id}", .auth = .session, .policy = .runtime_action, .target = "/api/diagnostics/2", .status = 204, .kind = .none },
|
||||
.{ .method = .DELETE, .pattern = "/api/diagnostics", .auth = .session, .policy = .runtime_action, .target = "/api/diagnostics", .status = 200, .check = jsonShape(handlers_diagnostics.PurgeResult) },
|
||||
|
||||
// Groups. The migrated schema seeds `default` as id 1; the POST creates
|
||||
// id 2, which the delete at the end of the walk removes.
|
||||
.{ .method = .GET, .pattern = "/api/groups", .auth = .session, .policy = .read, .target = "/api/groups", .status = 200, .check = jsonShape(GroupsList) },
|
||||
@@ -1000,6 +1039,157 @@ fn fileModeClasses(io: std.Io, env: *Env) anyerror!void {
|
||||
try conn.request("POST", "/api/certs/reload", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), response.status);
|
||||
|
||||
// Diagnostics are runtime state, not configuration: purging resolved
|
||||
// history is served under file authority like any other runtime action.
|
||||
try conn.request("DELETE", "/api/diagnostics", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), response.status);
|
||||
try testing.expectEqualStrings("{\"purged\":1}", response.body);
|
||||
|
||||
try conn.request("DELETE", "/api/diagnostics/1", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 409), response.status);
|
||||
}
|
||||
|
||||
fn diagnosticsRejections(io: std.Io, env: *Env) anyerror!void {
|
||||
var body_buf: [8192]u8 = undefined;
|
||||
var conn: Conn = undefined;
|
||||
try conn.connect(io, env.addr);
|
||||
defer conn.close(io);
|
||||
|
||||
// Every bad parameter is a 400 whose message names the parameter, rather
|
||||
// than a filter silently dropped — which would answer a question the client
|
||||
// did not ask.
|
||||
const bad = [_]struct { target: []const u8, needle: []const u8 }{
|
||||
.{ .target = "/api/diagnostics?state=open", .needle = "state" },
|
||||
.{ .target = "/api/diagnostics?severity=info", .needle = "severity" },
|
||||
.{ .target = "/api/diagnostics?since=yesterday", .needle = "since" },
|
||||
.{ .target = "/api/diagnostics?until=", .needle = "until" },
|
||||
.{ .target = "/api/diagnostics?limit=0", .needle = "limit" },
|
||||
.{ .target = "/api/diagnostics?limit=1001", .needle = "limit" },
|
||||
.{ .target = "/api/diagnostics?before=0", .needle = "before" },
|
||||
};
|
||||
for (bad) |case| {
|
||||
try conn.request("GET", case.target, null, null);
|
||||
const response = try conn.receive(&body_buf);
|
||||
errdefer std.debug.print("{s}: {d} {s}\n", .{ case.target, response.status, response.body });
|
||||
try testing.expectEqual(@as(u16, 400), response.status);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, case.needle));
|
||||
}
|
||||
|
||||
// The resolved episode the seed left behind is reachable by id, and an id
|
||||
// nothing holds is a 404 rather than an empty object.
|
||||
try conn.request("GET", "/api/diagnostics?state=resolved", null, null);
|
||||
var response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), response.status);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "upstream_history.write"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "\"active\":{\"warnings\":1,\"errors\":0}"));
|
||||
|
||||
try conn.request("GET", "/api/diagnostics/999999", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 404), response.status);
|
||||
}
|
||||
|
||||
fn diagnosticsPurge(io: std.Io, env: *Env) anyerror!void {
|
||||
var body_buf: [8192]u8 = undefined;
|
||||
var conn: Conn = undefined;
|
||||
try conn.connect(io, env.addr);
|
||||
defer conn.close(io);
|
||||
|
||||
// Row 1 is the seeded active episode: still the state of the box, so the
|
||||
// purge is refused with a message that says what would change that.
|
||||
try conn.request("DELETE", "/api/diagnostics/1", null, null);
|
||||
var response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 409), response.status);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "still active"));
|
||||
|
||||
try conn.request("DELETE", "/api/diagnostics/999999", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 404), response.status);
|
||||
|
||||
// Row 2 is the seeded resolved episode.
|
||||
try conn.request("DELETE", "/api/diagnostics/2", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 204), response.status);
|
||||
try testing.expectEqualStrings("", response.body);
|
||||
|
||||
// Gone is a different answer from still open, even for a row that existed a
|
||||
// moment ago.
|
||||
try conn.request("DELETE", "/api/diagnostics/2", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 404), response.status);
|
||||
|
||||
// Nothing resolved is left, and the sweep says so rather than failing.
|
||||
try conn.request("DELETE", "/api/diagnostics", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), response.status);
|
||||
try testing.expectEqualStrings("{\"purged\":0}", response.body);
|
||||
|
||||
// The active episode survived every one of those, counts included.
|
||||
try conn.request("GET", "/api/diagnostics", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), response.status);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "blocklist.refresh"));
|
||||
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "\"active\":{\"warnings\":1,\"errors\":0}"));
|
||||
try testing.expect(!std.mem.containsAtLeast(u8, response.body, 1, "upstream_history.write"));
|
||||
}
|
||||
|
||||
test "W10 milestone 27: a purge takes resolved events only, and says which of the three answers it gave" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
var env = try Env.create(gpa, .{});
|
||||
defer env.destroy();
|
||||
|
||||
try bounded(env.io(), default_budget, diagnosticsPurge, .{ env.io(), env });
|
||||
}
|
||||
|
||||
fn diagnosticsPurgeAll(io: std.Io, env: *Env) anyerror!void {
|
||||
var body_buf: [8192]u8 = undefined;
|
||||
var conn: Conn = undefined;
|
||||
try conn.connect(io, env.addr);
|
||||
defer conn.close(io);
|
||||
|
||||
// A second resolved episode, so the count the sweep reports is a number it
|
||||
// had to compute rather than the one row the seed leaves.
|
||||
env.events_store.reportResolved(io, seeded_now, .query_log_recreated, "one-shot", "corrupt", .warning, "aside");
|
||||
|
||||
try conn.request("DELETE", "/api/diagnostics", null, null);
|
||||
var response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), response.status);
|
||||
try testing.expectEqualStrings("{\"purged\":2}", response.body);
|
||||
|
||||
try conn.request("GET", "/api/diagnostics?state=resolved", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expectEqual(@as(u16, 200), response.status);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "\"events\":[]"));
|
||||
|
||||
// And the episode that is still failing is untouched: the operator clearing
|
||||
// the page cannot lose what is still true.
|
||||
try conn.request("GET", "/api/diagnostics?state=active", null, null);
|
||||
response = try conn.receive(&body_buf);
|
||||
try testing.expect(std.mem.containsAtLeast(u8, response.body, 1, "blocklist.refresh"));
|
||||
}
|
||||
|
||||
test "W10 milestone 27: purging all resolved events counts them and leaves the active ones" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
var env = try Env.create(gpa, .{});
|
||||
defer env.destroy();
|
||||
|
||||
try bounded(env.io(), default_budget, diagnosticsPurgeAll, .{ env.io(), env });
|
||||
}
|
||||
|
||||
test "W10 milestone 27: every diagnostics filter names itself in a 400, and an unknown id is a 404" {
|
||||
if (!build_options.integration) return error.SkipZigTest;
|
||||
|
||||
const gpa = testing.allocator;
|
||||
var env = try Env.create(gpa, .{});
|
||||
defer env.destroy();
|
||||
|
||||
try bounded(env.io(), default_budget, diagnosticsRejections, .{ env.io(), env });
|
||||
}
|
||||
|
||||
test "W10 milestone 20: file authority rejects configuration writes and spares the rest" {
|
||||
@@ -2197,6 +2387,15 @@ const contract_sample_walk = [_]ContractSample{
|
||||
.{ .name = "login", .ts_type = "LoginResponse", .method = "POST", .target = "/api/auth/login", .body = "{\"password\":\"\"}", .status = 200 },
|
||||
.{ .name = "logout", .ts_type = "LogoutResponse", .method = "POST", .target = "/api/auth/logout", .body = "{}", .status = 200 },
|
||||
|
||||
// Diagnostics, ahead of every write below: the seeded store holds one
|
||||
// active episode (id 1) and one resolved one, and a later pass that
|
||||
// reported an event of its own would move the page under the golden.
|
||||
.{ .name = "get_diagnostics", .ts_type = "DiagnosticsPage", .method = "GET", .target = "/api/diagnostics?limit=10", .status = 200 },
|
||||
.{ .name = "get_diagnostic", .ts_type = "DiagnosticEvent", .method = "GET", .target = "/api/diagnostics/1", .status = 200 },
|
||||
// The sweep runs after both reads and takes the seeded resolved episode;
|
||||
// the per-id purge answers 204, which has no body to sample.
|
||||
.{ .name = "purge_diagnostics", .ts_type = "DiagnosticsPurge", .method = "DELETE", .target = "/api/diagnostics", .status = 200 },
|
||||
|
||||
// Blocklists. The row is created disabled so the refresh below has a status
|
||||
// to report and still downloads nothing.
|
||||
.{ .name = "create_blocklist", .ts_type = "BlocklistEcho", .method = "POST", .target = "/api/blocklists", .body = "{\"url\":\"https://lists.example/ads.txt\",\"name\":\"ads\",\"enabled\":false}", .status = 201 },
|
||||
|
||||
Reference in New Issue
Block a user