website: demo household and the admin screenshots it produced
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
"""Print the id of the newest blocked app-measurement.com lookup from the iPhone: the query the
|
||||
detail screenshot shows. usage: blocked_query_id.py DATA_DIR"""
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
row = sqlite3.connect(f"file:{sys.argv[1]}/querylog.db?mode=ro", uri=True).execute(
|
||||
"SELECT q.id FROM query_log q JOIN domains d ON d.id = q.domain_id"
|
||||
" WHERE q.client_ip = '192.168.50.52' AND q.policy_reason = 'blocklist_domain'"
|
||||
" AND d.domain = 'app-measurement.com' ORDER BY q.id DESC LIMIT 1"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
sys.exit("no blocked app-measurement.com lookup from 192.168.50.52 in the log yet")
|
||||
print(row[0])
|
||||
@@ -0,0 +1,62 @@
|
||||
"""The repository's projection-coherence oracle (`recompute_checks` and
|
||||
`expectProjectionsMatchRecompute` in src/storage/repositories/queries_repo.zig),
|
||||
plus file-level invariants. Exits 1 on any violation. usage: check_coherence.py DATA_DIR"""
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
DATA = sys.argv[1]
|
||||
BUCKET = "(CASE WHEN timestamp >= 0 THEN timestamp / 1800 * 1800 ELSE (timestamp - 1799) / 1800 * 1800 END)"
|
||||
CHECKS = [
|
||||
("SELECT bucket, queries, blocked, cached, rt_sum, rt_count FROM bucket_totals",
|
||||
f"SELECT {BUCKET}, count(*), coalesce(sum(blocked <> 0), 0), coalesce(sum(cache_hit = 1), 0),"
|
||||
" coalesce(sum(response_time_us), 0), count(response_time_us) FROM query_log GROUP BY 1"),
|
||||
("SELECT bucket, client_ip, queries FROM bucket_clients",
|
||||
f"SELECT {BUCKET}, client_ip, count(*) FROM query_log GROUP BY 1, 2"),
|
||||
("SELECT bucket, qtype, count FROM bucket_types",
|
||||
f"SELECT {BUCKET}, coalesce(qtype, -1), count(*) FROM query_log GROUP BY 1, 2"),
|
||||
("SELECT bucket, route_kind, source_present, source_text, count FROM bucket_routes",
|
||||
f"SELECT {BUCKET}, route_kind, source IS NOT NULL, coalesce(source, ''), count(*)"
|
||||
" FROM (SELECT timestamp, route_kind, CASE route_kind WHEN 'upstream' THEN upstream"
|
||||
" WHEN 'forward_zone' THEN forward_zone END AS source FROM query_log) GROUP BY 1, 2, 3, 4"),
|
||||
]
|
||||
|
||||
ql = sqlite3.connect(f"file:{DATA}/querylog.db?mode=ro", uri=True)
|
||||
failures = 0
|
||||
|
||||
|
||||
def expect(label, ok, detail=""):
|
||||
global failures
|
||||
print(("ok " if ok else "FAIL ") + label + (f" ({detail})" if detail else ""))
|
||||
failures += not ok
|
||||
|
||||
|
||||
for projection, recompute in CHECKS:
|
||||
diff = ql.execute(
|
||||
f"SELECT (SELECT count(*) FROM ({projection} EXCEPT {recompute}))"
|
||||
f" + (SELECT count(*) FROM ({recompute} EXCEPT {projection}))"
|
||||
).fetchone()[0]
|
||||
expect(f"projection == recompute: {projection.split('FROM ')[1]}", diff == 0, f"{diff} differing rows")
|
||||
|
||||
integrity = ql.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
expect("querylog integrity_check", integrity == "ok", integrity)
|
||||
expect("querylog user_version == 1", ql.execute("PRAGMA user_version").fetchone()[0] == 1)
|
||||
orphans = ql.execute("SELECT count(*) FROM query_log WHERE domain_id NOT IN (SELECT id FROM domains)").fetchone()[0]
|
||||
expect("every query_log.domain_id names a domain", orphans == 0, f"{orphans} orphans")
|
||||
created, since, first = ql.execute(
|
||||
"SELECT created_at, available_since, (SELECT min(timestamp) FROM query_log) FROM querylog_meta").fetchone()
|
||||
expect("available_since = created_at + 1 <= first row", since == created + 1 and (first is None or since <= first),
|
||||
f"created_at={created} available_since={since} first_row={first}")
|
||||
disorder = ql.execute(
|
||||
"SELECT count(*) FROM query_log a JOIN query_log b ON b.id = a.id + 1 WHERE b.timestamp < a.timestamp").fetchone()[0]
|
||||
expect("query_log ids ascend with timestamps", disorder == 0, f"{disorder} inversions")
|
||||
|
||||
cfg = sqlite3.connect(f"file:{DATA}/config.db?mode=ro", uri=True)
|
||||
cfg.execute(f"ATTACH 'file:{DATA}/querylog.db?mode=ro' AS ql")
|
||||
bad = cfg.execute("""SELECT count(*) FROM clients c WHERE EXISTS (SELECT 1 FROM ql.query_log q WHERE q.client_ip = c.ip)
|
||||
AND (c.first_seen > (SELECT min(timestamp) FROM ql.query_log WHERE client_ip = c.ip)
|
||||
OR c.last_seen < (SELECT max(timestamp) FROM ql.query_log WHERE client_ip = c.ip))""").fetchone()[0]
|
||||
expect("clients first_seen/last_seen bracket their logged rows", bad == 0, f"{bad} clients out of range")
|
||||
unknown = cfg.execute("SELECT count(DISTINCT client_ip) FROM ql.query_log WHERE client_ip NOT IN (SELECT ip FROM clients)").fetchone()[0]
|
||||
expect("every logged client is a named client row", unknown == 0, f"{unknown} unknown")
|
||||
expect("config integrity_check", cfg.execute("PRAGMA integrity_check").fetchone()[0] == "ok")
|
||||
sys.exit(1 if failures else 0)
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Print each profile's expected blocked share (by query weight) via /api/lookup,
|
||||
and every domain that does not resolve on the host."""
|
||||
import json
|
||||
import socket
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
from profiles import PROFILES # noqa: E402
|
||||
|
||||
COOKIE = open(sys.argv[2]).read().split("nxdns_session")[1].split()[0]
|
||||
BASE = "http://127.0.0.1:18080"
|
||||
GROUP = {"kids": 2}
|
||||
|
||||
|
||||
def lookup(domain, group_id):
|
||||
req = urllib.request.Request(f"{BASE}/api/lookup?domain={domain}&group_id={group_id}")
|
||||
req.add_header("Cookie", f"nxdns_session={COOKIE}")
|
||||
return json.load(urllib.request.urlopen(req))
|
||||
|
||||
|
||||
cache = {}
|
||||
for name, prof in PROFILES.items():
|
||||
gid = GROUP.get(name, 1)
|
||||
total = blocked = 0
|
||||
for domain, weight in prof["domains"]:
|
||||
res = lookup(domain, gid)
|
||||
total += weight
|
||||
if sys.argv[3:] == ["-v"]:
|
||||
print(name, domain, json.dumps(res)[:200])
|
||||
is_blocked = res.get("blocked") or (res.get("decision") or {}).get("action") == "block"
|
||||
if is_blocked:
|
||||
blocked += weight
|
||||
elif domain not in cache and not domain.endswith(".lan"):
|
||||
try:
|
||||
socket.getaddrinfo(domain, None)
|
||||
cache[domain] = True
|
||||
except OSError:
|
||||
cache[domain] = False
|
||||
print("UNRESOLVED", name, domain)
|
||||
print(f"{name}: blocked weight share {blocked / total:.1%}")
|
||||
@@ -0,0 +1,47 @@
|
||||
"""usage: contact_sheet.py SHOTS_DIR OUT_HTML — light and dark side by side, images inlined."""
|
||||
import base64
|
||||
import html
|
||||
import sys
|
||||
|
||||
SHOTS, OUT = sys.argv[1], sys.argv[2]
|
||||
PAGES = [
|
||||
("overview", "Overview — 24 hours of a six-device household: total and blocked queries, and which device made them."),
|
||||
("activity", "Activity — recent queries, each with its client, result and route."),
|
||||
("query-detail", "Query detail — a blocked app-analytics lookup and the provenance that blocked it."),
|
||||
("clients", "Clients — each device by name and group; the kids' phone runs with safe search and extra lists."),
|
||||
("diagnostics", "Diagnostics — a resolved upstream episode: the primary DoH resolver was unreachable for about two minutes."),
|
||||
]
|
||||
|
||||
|
||||
def img(name, scheme):
|
||||
with open(f"{SHOTS}/{name}-{scheme}.png", "rb") as f:
|
||||
data = base64.b64encode(f.read()).decode()
|
||||
return f'<figure><img alt="{name} {scheme}" src="data:image/png;base64,{data}"><figcaption>{scheme}</figcaption></figure>'
|
||||
|
||||
|
||||
rows = "\n".join(
|
||||
f'<section><h2>{html.escape(caption)}</h2><div class="pair">{img(name, "light")}{img(name, "dark")}</div></section>'
|
||||
for name, caption in PAGES
|
||||
)
|
||||
page = f"""<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>nxdns screenshots</title>
|
||||
<style>
|
||||
:root {{ --bg: #f6f6f7; --fg: #1d1d22; --muted: #66666e; --line: #d9d9de; }}
|
||||
@media (prefers-color-scheme: dark) {{ :root {{ --bg: #141417; --fg: #ececf0; --muted: #9a9aa3; --line: #2c2c33; }} }}
|
||||
body {{ margin: 0; padding: 24px 16px; background: var(--bg); color: var(--fg); font: 15px/1.5 system-ui, sans-serif; }}
|
||||
h1 {{ font-size: 20px; margin: 0 0 24px; }}
|
||||
h2 {{ font-size: 15px; font-weight: 500; margin: 32px 0 12px; }}
|
||||
.pair {{ display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }}
|
||||
@media (max-width: 800px) {{ .pair {{ grid-template-columns: 1fr; }} }}
|
||||
figure {{ margin: 0; }}
|
||||
img {{ width: 100%; height: auto; border: 1px solid var(--line); border-radius: 6px; display: block; }}
|
||||
figcaption {{ color: var(--muted); font-size: 13px; margin-top: 4px; }}
|
||||
</style></head>
|
||||
<body><h1>nxdns admin — marketing screenshots (1440×900 @2x)</h1>
|
||||
{rows}
|
||||
</body></html>
|
||||
"""
|
||||
with open(OUT, "w") as f:
|
||||
f.write(page)
|
||||
print(OUT)
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Per-device domain profiles: (domain, weight). Weights are relative lookup
|
||||
frequencies within a device; each lookup emits the device's query-type set."""
|
||||
|
||||
# Query-type sets per lookup, with their probabilities.
|
||||
A, AAAA, HTTPS = 1, 28, 65
|
||||
APPLE_TYPES = [((A, AAAA, HTTPS), 0.75), ((A, AAAA), 0.15), ((A,), 0.10)]
|
||||
ANDROID_TYPES = [((A, AAAA), 0.55), ((A, AAAA, HTTPS), 0.30), ((A,), 0.15)]
|
||||
WINDOWS_TYPES = [((A, AAAA), 0.60), ((A, AAAA, HTTPS), 0.20), ((A,), 0.20)]
|
||||
TV_TYPES = [((A,), 0.70), ((A, AAAA), 0.30)]
|
||||
|
||||
COMMON_WEB = [
|
||||
("www.google.com", 30), ("www.gstatic.com", 18), ("fonts.googleapis.com", 8),
|
||||
("fonts.gstatic.com", 8), ("www.wikipedia.org", 3), ("en.wikipedia.org", 4),
|
||||
("upload.wikimedia.org", 3), ("www.bbc.co.uk", 3), ("www.theguardian.com", 3),
|
||||
("www.reddit.com", 6), ("www.redditstatic.com", 3), ("i.redd.it", 3),
|
||||
("www.amazon.com", 4), ("m.media-amazon.com", 4), ("www.ebay.com", 2),
|
||||
("github.com", 2), ("cdn.jsdelivr.net", 3), ("cdnjs.cloudflare.com", 3),
|
||||
("www.youtube.com", 10), ("i.ytimg.com", 8), ("yt3.ggpht.com", 4),
|
||||
("www.nytimes.com", 2), ("static01.nyt.com", 2), ("www.weather.com", 2),
|
||||
("duckduckgo.com", 2), ("www.bing.com", 2), ("stackoverflow.com", 2),
|
||||
("www.imdb.com", 1), ("www.booking.com", 1), ("www.paypal.com", 1),
|
||||
("ajax.googleapis.com", 3), ("maps.googleapis.com", 3), ("www.google-analytics.com", 6),
|
||||
("www.googletagmanager.com", 6), ("googleads.g.doubleclick.net", 4),
|
||||
("securepubads.g.doubleclick.net", 3), ("pagead2.googlesyndication.com", 3),
|
||||
("connect.facebook.net", 2), ("bat.bing.com", 2), ("static.ads-twitter.com", 1),
|
||||
("c.amazon-adsystem.com", 2), ("cdn.taboola.com", 1), ("widgets.outbrain.com", 1),
|
||||
("sb.scorecardresearch.com", 2), ("js-agent.newrelic.com", 1), ("browser.sentry-cdn.com", 1),
|
||||
]
|
||||
|
||||
PROFILES = {
|
||||
"tv": {
|
||||
"types": TV_TYPES,
|
||||
"domains": [
|
||||
("api-global.netflix.com", 40), ("nrdp.prod.ftl.netflix.com", 18), ("assets.nflxext.com", 14),
|
||||
("ichnaea.netflix.com", 10), ("www.youtube.com", 22), ("youtubei.googleapis.com", 28),
|
||||
("i.ytimg.com", 22), ("redirector.googlevideo.com", 14), ("www.googleapis.com", 8),
|
||||
("disney.api.edge.bamgrid.com", 12), ("bam-sdk-configs.bamgrid.com", 4),
|
||||
("appconfigs.disney-plus.net", 5), ("apresolve.spotify.com", 4), ("spclient.wg.spotify.com", 5),
|
||||
("time.samsungcloudsolution.com", 6), ("otn.samsungcloudsolution.com", 4),
|
||||
("www.samsungotn.net", 2), ("osb-apps.samsungqbe.com", 4), ("gpm.samsungqbe.com", 3),
|
||||
("cdn.samsungcloudsolution.com", 3),
|
||||
("connectivitycheck.gstatic.com", 4),
|
||||
# Trackers and ads a smart TV really emits.
|
||||
("log-config.samsungacr.com", 9), ("log-ingestion-eu.samsungacr.com", 8),
|
||||
("config.samsungads.com", 7), ("samsung-com.112.2o7.net", 5), ("ad.samsungadhub.com", 4),
|
||||
("tracking.samsungads.com", 4), ("googleads.g.doubleclick.net", 6),
|
||||
("pubads.g.doubleclick.net", 5), ("s0.2mdn.net", 3), ("imasdk.googleapis.com", 4),
|
||||
("logs.netflix.com", 4), ("events.samsungads.com", 3),
|
||||
],
|
||||
"lookups": 3900,
|
||||
},
|
||||
"iphone": {
|
||||
"types": APPLE_TYPES,
|
||||
"domains": COMMON_WEB + [
|
||||
("gateway.icloud.com", 14), ("p53-caldav.icloud.com", 4), ("api.apple-cloudkit.com", 8),
|
||||
("gspe1-ssl.ls.apple.com", 6), ("mesu.apple.com", 2), ("init.itunes.apple.com", 4),
|
||||
("bag.itunes.apple.com", 3), ("play.itunes.apple.com", 2), ("captive.apple.com", 6),
|
||||
("weather-data.apple.com", 4), ("configuration.ls.apple.com", 3),
|
||||
("graph.instagram.com", 8), ("i.instagram.com", 12), ("scontent.cdninstagram.com", 10),
|
||||
("static.cdninstagram.com", 4), ("graph.facebook.com", 5), ("g.whatsapp.net", 12),
|
||||
("mmg.whatsapp.net", 6), ("api.spotify.com", 5),
|
||||
("spclient.wg.spotify.com", 6), ("i.scdn.co", 4), ("www.google.com", 6),
|
||||
("maps.apple.com", 2), ("mask.icloud.com", 3),
|
||||
# App analytics and ad SDKs.
|
||||
("app-measurement.com", 10), ("firebaselogging-pa.googleapis.com", 6),
|
||||
("app-analytics-services.com", 4), ("api.mixpanel.com", 3), ("api2.amplitude.com", 3),
|
||||
("graph.facebook.com", 2), ("ads.tiktok.com", 2), ("cdn.branch.io", 2), ("api2.branch.io", 2),
|
||||
("app.adjust.com", 3), ("in.appcenter.ms", 2), ("events.appsflyer.com", 3),
|
||||
("sessions.bugsnag.com", 2), ("iadsdk.apple.com", 3), ("metrics.icloud.com", 3),
|
||||
("notify.bugsnag.com", 1), ("ads.mopub.com", 1),
|
||||
],
|
||||
"lookups": 1300,
|
||||
},
|
||||
"pixel": {
|
||||
"types": ANDROID_TYPES,
|
||||
"domains": COMMON_WEB + [
|
||||
("android.googleapis.com", 10), ("play.googleapis.com", 8), ("www.googleapis.com", 10),
|
||||
("android.clients.google.com", 6), ("mtalk.google.com", 10), ("connectivitycheck.gstatic.com", 8),
|
||||
("clients4.google.com", 4), ("play-lh.googleusercontent.com", 6), ("lh3.googleusercontent.com", 5),
|
||||
("photos.googleapis.com", 3), ("gmail.googleapis.com", 4), ("inbox.google.com", 2),
|
||||
("youtubei.googleapis.com", 10), ("i.ytimg.com", 6), ("rr1---sn-4g5e6nzz.googlevideo.com", 3),
|
||||
("g.whatsapp.net", 10), ("mmg.whatsapp.net", 5), ("web.whatsapp.com", 1),
|
||||
("i.instagram.com", 8), ("scontent.cdninstagram.com", 7), ("api.spotify.com", 4),
|
||||
("spclient.wg.spotify.com", 5), ("discord.com", 3), ("gateway.discord.gg", 3),
|
||||
("cdn.discordapp.com", 3), ("time.android.com", 3),
|
||||
# App analytics and ad SDKs.
|
||||
("app-measurement.com", 10), ("firebaselogging-pa.googleapis.com", 8),
|
||||
("crashlyticsreports-pa.googleapis.com", 4), ("settings.crashlytics.com", 3),
|
||||
("googleads.g.doubleclick.net", 5), ("adservice.google.com", 3),
|
||||
("graph.facebook.com", 3), ("an.facebook.com", 2), ("api.mixpanel.com", 2),
|
||||
("app.adjust.com", 2), ("t.appsflyer.com", 2), ("ads.mopub.com", 1),
|
||||
],
|
||||
"lookups": 1450,
|
||||
},
|
||||
"kids": {
|
||||
"types": ANDROID_TYPES,
|
||||
"domains": [
|
||||
("www.google.com", 12), ("www.youtube.com", 20), ("youtubei.googleapis.com", 18),
|
||||
("i.ytimg.com", 14), ("yt3.ggpht.com", 6), ("android.googleapis.com", 6),
|
||||
("mtalk.google.com", 6), ("connectivitycheck.gstatic.com", 5), ("www.gstatic.com", 5),
|
||||
("play.googleapis.com", 5), ("www.roblox.com", 10), ("apis.roblox.com", 10),
|
||||
("tr.rbxcdn.com", 6), ("www.minecraft.net", 3),
|
||||
("api.minecraftservices.com", 4), ("pocket.realms.minecraft.net", 2),
|
||||
("api.spotify.com", 3), ("www.duolingo.com", 3), ("android-api.duolingo.com", 4),
|
||||
("en.wikipedia.org", 2), ("www.bing.com", 2), ("g.whatsapp.net", 4),
|
||||
("www.tiktok.com", 4), ("api16-normal-c-useast1a.tiktokv.com", 4),
|
||||
("v16-webapp.tiktok.com", 3), ("discord.com", 2),
|
||||
# Game ad networks and analytics.
|
||||
("unityads.unity3d.com", 7), ("config.unityads.unity3d.com", 4),
|
||||
("auction.unityads.unity3d.com", 4), ("ms.applovin.com", 5), ("a.applovin.com", 4),
|
||||
("rt.applovin.com", 3), ("googleads.g.doubleclick.net", 6), ("app-measurement.com", 7),
|
||||
("firebaselogging-pa.googleapis.com", 4), ("supersonicads.com", 2),
|
||||
("outcome-ssp.supersonicads.com", 2), ("init.supersonicads.com", 2),
|
||||
("events.appsflyer.com", 2), ("ads.vungle.com", 2), ("api.vungle.com", 2),
|
||||
# Casino ads in free games: only the kids group blocks these.
|
||||
("www.pokerstars.com", 1), ("www.bet365.com", 1), ("www.888casino.com", 1)],
|
||||
"lookups": 1150,
|
||||
},
|
||||
"desktop": {
|
||||
"types": WINDOWS_TYPES,
|
||||
"domains": COMMON_WEB + [
|
||||
("www.msftconnecttest.com", 6), ("login.live.com", 6), ("login.microsoftonline.com", 4),
|
||||
("settings-win.data.microsoft.com", 6), ("v10.events.data.microsoft.com", 10),
|
||||
("watson.telemetry.microsoft.com", 4), ("self.events.data.microsoft.com", 5),
|
||||
("browser.events.data.msn.com", 4), ("arc.msn.com", 3), ("ris.api.iris.microsoft.com", 3),
|
||||
("www.bing.com", 6), ("assets.msn.com", 4), ("ntp.msn.com", 4), ("img-s-msn-com.akamaized.net", 3),
|
||||
("fe3cr.delivery.mp.microsoft.com", 3), ("tsfe.trafficshaping.dsp.mp.microsoft.com", 2),
|
||||
("dl.delivery.mp.microsoft.com", 2), ("ctldl.windowsupdate.com", 3), ("time.windows.com", 2),
|
||||
("outlook.office365.com", 6), ("teams.microsoft.com", 4), ("graph.microsoft.com", 4),
|
||||
("onedrive.live.com", 2), ("store.steampowered.com", 3), ("api.steampowered.com", 4),
|
||||
("steamcdn-a.akamaihd.net", 2), ("discord.com", 3), ("gateway.discord.gg", 3),
|
||||
("clients2.google.com", 3), ("update.googleapis.com", 2), ("nas.lan", 3), ("printer.lan", 1),
|
||||
("vortex.data.microsoft.com", 4), ("mobile.events.data.microsoft.com", 3),
|
||||
("telemetry.nvidia.com", 2), ("events.gfe.nvidia.com", 2),
|
||||
],
|
||||
"lookups": 900,
|
||||
},
|
||||
"mac": {
|
||||
"types": APPLE_TYPES,
|
||||
"domains": COMMON_WEB + [
|
||||
("gateway.icloud.com", 10), ("api.apple-cloudkit.com", 6), ("gs-loc.apple.com", 3),
|
||||
("swscan.apple.com", 2), ("swdist.apple.com", 1), ("xp.apple.com", 4),
|
||||
("ocsp2.apple.com", 6), ("valid.apple.com", 2), ("captive.apple.com", 4),
|
||||
("configuration.ls.apple.com", 2), ("time.apple.com", 2), ("api.github.com", 4),
|
||||
("avatars.githubusercontent.com", 3), ("objects.githubusercontent.com", 2),
|
||||
("registry.npmjs.org", 3), ("slack.com", 6), ("wss-primary.slack.com", 4),
|
||||
("edgeapi.slack.com", 3), ("a.slack-edge.com", 3), ("zoom.us", 3), ("us05web.zoom.us", 2),
|
||||
("www.notion.so", 4), ("api.spotify.com", 3), ("spclient.wg.spotify.com", 4),
|
||||
("docs.google.com", 5), ("drive.google.com", 3), ("mail.google.com", 6),
|
||||
("nas.lan", 4), ("files.lan", 2), ("printer.lan", 2),
|
||||
("metrics.icloud.com", 3), ("api-adservices.apple.com", 2), ("notes-analytics-events.apple.com", 2),
|
||||
("stats.g.doubleclick.net", 3), ("o0.ingest.sentry.io", 2), ("api.segment.io", 2),
|
||||
("cdn.segment.com", 2), ("api.mixpanel.com", 1), ("in.hotjar.com", 1), ("static.hotjar.com", 1),
|
||||
],
|
||||
"lookups": 800,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Spread the bulk-generated query log over the previous day with a household
|
||||
rhythm, then rebuild every projection from the rewritten rows.
|
||||
|
||||
Run only while nxdns is stopped. usage: rewrite_history.py DATA_DIR
|
||||
"""
|
||||
import datetime
|
||||
import random
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
|
||||
DATA = sys.argv[1]
|
||||
NOW = int(time.time())
|
||||
START = NOW - 24 * 3600 + 120
|
||||
END = NOW - 300
|
||||
CREATED_AT = START - 1200
|
||||
|
||||
# Relative activity per local hour of day, per device.
|
||||
CURVES = {
|
||||
"192.168.50.41": [2, 1, 1, 1, 1, 1, 3, 20, 25, 8, 6, 6, 10, 8, 6, 8, 15, 45, 80, 100, 100, 90, 60, 20], # TV
|
||||
"192.168.50.52": [6, 3, 3, 3, 3, 5, 25, 70, 60, 35, 35, 40, 60, 45, 35, 40, 45, 60, 70, 80, 85, 80, 60, 25], # iPhone
|
||||
"192.168.50.53": [5, 3, 3, 3, 3, 4, 15, 60, 70, 40, 40, 45, 65, 50, 40, 45, 50, 65, 75, 80, 75, 70, 50, 20], # Pixel
|
||||
"192.168.50.54": [1, 1, 1, 1, 1, 1, 4, 35, 20, 2, 2, 2, 4, 2, 2, 40, 90, 100, 85, 80, 60, 25, 4, 2], # Kids' phone
|
||||
"192.168.50.20": [2, 1, 1, 1, 1, 1, 1, 4, 10, 15, 15, 15, 20, 15, 15, 20, 30, 50, 80, 100, 100, 85, 50, 15], # Desktop PC
|
||||
"192.168.50.21": [2, 1, 1, 1, 1, 1, 2, 15, 60, 100, 100, 95, 60, 90, 100, 95, 80, 40, 25, 25, 20, 12, 5, 3], # MacBook Air
|
||||
}
|
||||
|
||||
|
||||
def weight(ip, ts):
|
||||
return CURVES[ip][datetime.datetime.fromtimestamp(ts).hour]
|
||||
|
||||
|
||||
def sample_times(ip, n):
|
||||
minutes = list(range(START, END, 60))
|
||||
weights = [weight(ip, m) for m in minutes]
|
||||
return sorted(m + random.randrange(60) for m in random.choices(minutes, weights, k=n))
|
||||
|
||||
|
||||
def lookups(rows):
|
||||
"""Group a client's rows into lookups: the A/AAAA/HTTPS burst of one name."""
|
||||
groups, last = [], None
|
||||
for row_id, ts, domain_id in rows:
|
||||
if last and last[0] == domain_id and ts - last[1] <= 1:
|
||||
groups[-1].append(row_id)
|
||||
else:
|
||||
groups.append([row_id])
|
||||
last = (domain_id, ts)
|
||||
return groups
|
||||
|
||||
|
||||
# The repository's own bucket recompute (src/storage/repositories/queries_repo.zig,
|
||||
# `recompute_bucket_sql`), verbatim; ?1 is the bucket start, ?2 its exclusive end.
|
||||
RECOMPUTE_BUCKET_SQL = [
|
||||
"""INSERT INTO bucket_totals (bucket, queries, blocked, cached, rt_sum, rt_count)
|
||||
SELECT ?1, count(*), coalesce(sum(blocked <> 0), 0), coalesce(sum(cache_hit = 1), 0),
|
||||
coalesce(sum(response_time_us), 0), count(response_time_us)
|
||||
FROM query_log
|
||||
WHERE timestamp >= ?1 AND timestamp < ?2
|
||||
HAVING count(*) > 0""",
|
||||
"""INSERT INTO bucket_clients (bucket, client_ip, queries)
|
||||
SELECT ?1, client_ip, count(*)
|
||||
FROM query_log
|
||||
WHERE timestamp >= ?1 AND timestamp < ?2
|
||||
GROUP BY client_ip""",
|
||||
"""INSERT INTO bucket_types (bucket, qtype, count)
|
||||
SELECT ?1, coalesce(qtype, -1), count(*)
|
||||
FROM query_log
|
||||
WHERE timestamp >= ?1 AND timestamp < ?2
|
||||
GROUP BY coalesce(qtype, -1)""",
|
||||
"""INSERT INTO bucket_routes (bucket, route_kind, source_present, source_text, count)
|
||||
SELECT ?1, route_kind, source IS NOT NULL, coalesce(source, ''), count(*)
|
||||
FROM (SELECT route_kind,
|
||||
CASE route_kind
|
||||
WHEN 'upstream' THEN upstream
|
||||
WHEN 'forward_zone' THEN forward_zone
|
||||
END AS source
|
||||
FROM query_log
|
||||
WHERE timestamp >= ?1 AND timestamp < ?2)
|
||||
GROUP BY route_kind, source""",
|
||||
]
|
||||
GRAIN = 1800
|
||||
|
||||
COLUMNS = (
|
||||
"timestamp, domain_id, client_ip, qtype, blocked, response_time_us, cache_hit, upstream, qclass, rcode,"
|
||||
" group_id, group_name, policy_action, policy_reason, matched, source_id, source_name, cname_target,"
|
||||
" safe_search_target, route_kind, forward_zone"
|
||||
)
|
||||
|
||||
ql = sqlite3.connect(f"{DATA}/querylog.db", isolation_level=None)
|
||||
ql.execute("BEGIN IMMEDIATE")
|
||||
new_ts = {}
|
||||
for (ip,) in ql.execute("SELECT DISTINCT client_ip FROM query_log").fetchall():
|
||||
rows = ql.execute(
|
||||
"SELECT id, timestamp, domain_id FROM query_log WHERE client_ip = ? ORDER BY timestamp, id", (ip,)
|
||||
).fetchall()
|
||||
groups = lookups(rows)
|
||||
for group, ts in zip(groups, sample_times(ip, len(groups))):
|
||||
for row_id in group:
|
||||
new_ts[row_id] = ts
|
||||
ql.execute("CREATE TEMP TABLE remap (old_id INTEGER PRIMARY KEY, ts INTEGER NOT NULL)")
|
||||
ql.executemany("INSERT INTO remap VALUES (?, ?)", new_ts.items())
|
||||
rest = COLUMNS.split(", ", 1)[1]
|
||||
ql.execute(f"CREATE TEMP TABLE rewritten AS SELECT r.ts AS timestamp, {rest}, q.id AS old_id"
|
||||
" FROM query_log q JOIN remap r ON r.old_id = q.id")
|
||||
before = ql.execute("SELECT count(*) FROM query_log").fetchone()[0]
|
||||
ql.execute("DELETE FROM query_log")
|
||||
ql.execute(f"INSERT INTO query_log ({COLUMNS}) SELECT {COLUMNS} FROM rewritten ORDER BY timestamp, old_id")
|
||||
after = ql.execute("SELECT count(*) FROM query_log").fetchone()[0]
|
||||
assert before == after == len(new_ts), (before, after, len(new_ts))
|
||||
|
||||
for table in ("bucket_totals", "bucket_clients", "bucket_types", "bucket_routes"):
|
||||
ql.execute(f"DELETE FROM {table}")
|
||||
buckets = [b for (b,) in ql.execute(f"SELECT DISTINCT timestamp / {GRAIN} * {GRAIN} FROM query_log")]
|
||||
for bucket in buckets:
|
||||
for sql in RECOMPUTE_BUCKET_SQL:
|
||||
ql.execute(sql, (bucket, bucket + GRAIN))
|
||||
ql.execute("UPDATE querylog_meta SET created_at = ?, available_since = ? WHERE id = 1", (CREATED_AT, CREATED_AT + 1))
|
||||
ql.execute("COMMIT")
|
||||
ql.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
span = ql.execute("SELECT min(timestamp), max(timestamp), count(*) FROM query_log").fetchone()
|
||||
print(f"querylog: {span[2]} rows from {span[0]} to {span[1]}, {len(buckets)} buckets rebuilt")
|
||||
|
||||
cfg = sqlite3.connect(f"{DATA}/config.db", isolation_level=None)
|
||||
cfg.execute("ATTACH ? AS ql", (f"{DATA}/querylog.db",))
|
||||
cfg.execute("BEGIN IMMEDIATE")
|
||||
cfg.execute("""UPDATE clients SET
|
||||
first_seen = coalesce((SELECT min(timestamp) FROM ql.query_log WHERE client_ip = clients.ip), first_seen),
|
||||
last_seen = coalesce((SELECT max(timestamp) FROM ql.query_log WHERE client_ip = clients.ip), last_seen)""")
|
||||
cfg.execute("UPDATE rules SET created_at = ?", (CREATED_AT,))
|
||||
cfg.execute("COMMIT")
|
||||
cfg.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
for row in cfg.execute("SELECT ip, name, first_seen, last_seen FROM clients ORDER BY ip"):
|
||||
print("client", row)
|
||||
@@ -0,0 +1,50 @@
|
||||
// usage: node shoot.mjs OUT_DIR QUERY_ID
|
||||
import { chromium } from "playwright";
|
||||
|
||||
const [outDir, queryId] = process.argv.slice(2);
|
||||
const BASE = "http://127.0.0.1:18080";
|
||||
const PAGES = [
|
||||
["overview", "/overview?period=24h"],
|
||||
["activity", "/activity"],
|
||||
["query-detail", `/activity/queries/${queryId}`],
|
||||
["clients", "/clients"],
|
||||
["diagnostics", "/diagnostics"],
|
||||
];
|
||||
|
||||
async function settle(page) {
|
||||
await page.waitForLoadState("networkidle").catch(() => {});
|
||||
await page.waitForFunction(
|
||||
() => !document.querySelector('[aria-busy="true"], [role="progressbar"]'),
|
||||
null,
|
||||
{ timeout: 15000 },
|
||||
).catch(() => console.log(" still busy after 15s"));
|
||||
await page.waitForTimeout(2500);
|
||||
}
|
||||
|
||||
const browser = await chromium.launch();
|
||||
for (const scheme of ["light", "dark"]) {
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 1440, height: 900 },
|
||||
deviceScaleFactor: 2,
|
||||
colorScheme: scheme,
|
||||
timezoneId: "Europe/Paris",
|
||||
locale: "en-GB",
|
||||
});
|
||||
const page = await context.newPage();
|
||||
page.on("console", (m) => m.type() === "error" && console.log(` console error: ${m.text()}`));
|
||||
await page.goto(`${BASE}/login`);
|
||||
await page.locator("#password").fill("nxdns-demo");
|
||||
await page.locator('button[type="submit"]').click();
|
||||
await page.waitForURL((u) => !u.pathname.startsWith("/login"));
|
||||
for (const [name, path] of PAGES) {
|
||||
await page.goto(BASE + path);
|
||||
await page.mouse.move(1430, 890);
|
||||
await settle(page);
|
||||
if (name === "activity") await page.waitForTimeout(8000);
|
||||
const file = `${outDir}/${name}-${scheme}.png`;
|
||||
await page.screenshot({ path: file });
|
||||
console.log(file);
|
||||
}
|
||||
await context.close();
|
||||
}
|
||||
await browser.close();
|
||||
@@ -0,0 +1,67 @@
|
||||
"""One household device's DNS traffic, stdlib only.
|
||||
|
||||
usage: traffic.py PROFILE bulk send the profile's lookup budget fast, then exit
|
||||
traffic.py PROFILE live MEAN_S one lookup every ~MEAN_S seconds, forever
|
||||
"""
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from profiles import PROFILES # noqa: E402
|
||||
|
||||
SERVER = (os.environ.get("NXDNS", "192.168.50.2"), 53)
|
||||
|
||||
|
||||
def build_query(qid, name, qtype):
|
||||
header = struct.pack("!HHHHHH", qid, 0x0100, 1, 0, 0, 1)
|
||||
qname = b"".join(bytes([len(label)]) + label.encode() for label in name.split(".")) + b"\0"
|
||||
opt = b"\0" + struct.pack("!HHIH", 41, 1232, 0, 0)
|
||||
return header + qname + struct.pack("!HH", qtype, 1) + opt
|
||||
|
||||
|
||||
def pick_types(type_sets):
|
||||
sets, weights = zip(*type_sets)
|
||||
return random.choices(sets, weights)[0]
|
||||
|
||||
|
||||
def main():
|
||||
name, mode = sys.argv[1], sys.argv[2]
|
||||
profile = PROFILES[name]
|
||||
domains, weights = zip(*profile["domains"])
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
sock.settimeout(6)
|
||||
sent = answered = failed = 0
|
||||
lookups = profile["lookups"] if mode == "bulk" else None
|
||||
mean = float(sys.argv[3]) if mode == "live" else 0.0
|
||||
done = 0
|
||||
while lookups is None or done < lookups:
|
||||
domain = random.choices(domains, weights)[0]
|
||||
for qtype in pick_types(profile["types"]):
|
||||
qid = random.randrange(65536)
|
||||
sock.sendto(build_query(qid, domain, qtype), SERVER)
|
||||
sent += 1
|
||||
try:
|
||||
while True:
|
||||
data, _ = sock.recvfrom(4096)
|
||||
if struct.unpack("!H", data[:2])[0] == qid:
|
||||
answered += 1
|
||||
break
|
||||
except socket.timeout:
|
||||
failed += 1
|
||||
print(f"{name}: no answer for {domain} type {qtype}", flush=True)
|
||||
done += 1
|
||||
if mode == "bulk":
|
||||
time.sleep(random.uniform(0.05, 0.25))
|
||||
else:
|
||||
time.sleep(random.expovariate(1.0 / mean))
|
||||
if done % 500 == 0:
|
||||
print(f"{name}: {done} lookups, {sent} sent, {answered} answered, {failed} unanswered", flush=True)
|
||||
print(f"{name}: finished {done} lookups, {sent} sent, {answered} answered, {failed} unanswered", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user