milestone 5: blocklist filtering, local records and conditional forwarding
This commit is contained in:
@@ -0,0 +1,265 @@
|
||||
//! Blocklist download over HTTP/1.1.
|
||||
//!
|
||||
//! One `Fetcher` wraps a caller-owned `std.http.Client`, which owns the
|
||||
//! connection pool and the CA bundle, exactly as `upstream/doh_client.zig`
|
||||
//! does. This file knows nothing about parsing, files or the database: it GETs
|
||||
//! a URL and streams the bytes into a writer the caller supplies.
|
||||
//!
|
||||
//! The body is never held whole. A blocklist can reach `max_body_bytes`, and
|
||||
//! the caller writes into a temporary file anyway, so nothing here allocates.
|
||||
//!
|
||||
//! There is no timeout parameter and no sleep. `std.http.Client` has no
|
||||
//! per-request deadline, so the caller runs `fetch` under `io.concurrent` and
|
||||
//! cancels the future; this file only propagates `error.Canceled`.
|
||||
|
||||
const std = @import("std");
|
||||
const transport = @import("../upstream/transport.zig");
|
||||
|
||||
pub const max_body_bytes: usize = 64 * 1024 * 1024;
|
||||
|
||||
/// RFC 9110 recommends at least 8000 bytes for the redirect buffer
|
||||
/// (`std.http.Client` doc comment, Client.zig:1128).
|
||||
pub const redirect_buffer_len: usize = 8192;
|
||||
|
||||
pub const min_transfer_buf: usize = 16 * 1024;
|
||||
|
||||
pub const Error = error{
|
||||
BadUrl,
|
||||
ConnectFailed,
|
||||
TlsFailed,
|
||||
SendFailed,
|
||||
ReceiveFailed,
|
||||
HttpStatus,
|
||||
BodyTooLarge,
|
||||
Timeout,
|
||||
Canceled,
|
||||
OutOfMemory,
|
||||
SystemResources,
|
||||
Unexpected,
|
||||
};
|
||||
|
||||
pub const Result = struct {
|
||||
bytes_read: u64,
|
||||
status: std.http.Status,
|
||||
};
|
||||
|
||||
pub const Fetcher = struct {
|
||||
/// Caller-owned; shared across sources, pools connections.
|
||||
http: *std.http.Client,
|
||||
/// Caller-owned HTTP body transfer buffer, at least `min_transfer_buf`.
|
||||
transfer_buf: []u8,
|
||||
/// Caller-owned. `receiveHead` follows redirects itself and needs this to
|
||||
/// outlive `Request.uri`. At least `redirect_buffer_len`.
|
||||
redirect_buf: []u8,
|
||||
/// The status of the most recent response head, or null before the first
|
||||
/// one. `error.HttpStatus` carries no `Result`, and the operator's message
|
||||
/// needs the number, so it is readable here after a failed `fetch`.
|
||||
last_status: ?std.http.Status = null,
|
||||
|
||||
/// GETs `url` and streams the body into `w`.
|
||||
pub fn fetch(
|
||||
self: *Fetcher,
|
||||
io: std.Io,
|
||||
url: []const u8,
|
||||
w: *std.Io.Writer,
|
||||
) Error!Result {
|
||||
// `std.http.Client` carries the `std.Io` it was constructed with and
|
||||
// takes none per request. The parameter stays in the signature because
|
||||
// the manager drives every fetch through one `std.Io`.
|
||||
_ = io;
|
||||
|
||||
std.debug.assert(self.transfer_buf.len >= min_transfer_buf);
|
||||
std.debug.assert(self.redirect_buf.len >= redirect_buffer_len);
|
||||
|
||||
const uri = try parseUrl(url);
|
||||
|
||||
self.last_status = null;
|
||||
|
||||
var req = self.http.request(.GET, uri, .{
|
||||
.keep_alive = true,
|
||||
.headers = .{
|
||||
// Identity only: a compressed transfer encoding would need
|
||||
// `Response.readerDecompressing`, a decompression buffer and a
|
||||
// second failure surface, for a download that runs once a day.
|
||||
.accept_encoding = .{ .override = "identity" },
|
||||
},
|
||||
}) catch |err| return mapError(err, .connect);
|
||||
defer req.deinit();
|
||||
|
||||
req.sendBodiless() catch |err| return mapError(err, .send);
|
||||
|
||||
var resp = req.receiveHead(self.redirect_buf) catch |err| return mapError(err, .receive);
|
||||
|
||||
self.last_status = resp.head.status;
|
||||
if (resp.head.status != .ok) return error.HttpStatus;
|
||||
|
||||
// `content-type` is deliberately not checked: blocklists are served as
|
||||
// text/plain, application/octet-stream and text/html alike, and the
|
||||
// compiler's invalid-line counters are the honest signal about content.
|
||||
if (resp.head.content_length) |declared| {
|
||||
if (declared > max_body_bytes) return error.BodyTooLarge;
|
||||
}
|
||||
|
||||
const body = resp.reader(self.transfer_buf);
|
||||
var total: u64 = 0;
|
||||
while (true) {
|
||||
const n = body.readSliceShort(self.transfer_buf) catch |err|
|
||||
return mapError(err, .receive);
|
||||
if (n == 0) break;
|
||||
total += n;
|
||||
if (total > max_body_bytes) return error.BodyTooLarge;
|
||||
// The caller owns `w` and can read the concrete failure from its
|
||||
// own writer; this taxonomy has no member for a failing sink.
|
||||
w.writeAll(self.transfer_buf[0..n]) catch return error.Unexpected;
|
||||
}
|
||||
|
||||
return .{ .bytes_read = total, .status = resp.head.status };
|
||||
}
|
||||
};
|
||||
|
||||
/// A scheme other than `http`/`https`, an unparseable URL and a URL with no
|
||||
/// host are one fault to the operator: the source row is unusable.
|
||||
fn parseUrl(url: []const u8) Error!std.Uri {
|
||||
const uri = std.Uri.parse(url) catch return error.BadUrl;
|
||||
if (!std.mem.eql(u8, uri.scheme, "http") and
|
||||
!std.mem.eql(u8, uri.scheme, "https")) return error.BadUrl;
|
||||
const host = uri.host orelse return error.BadUrl;
|
||||
if (host.isEmpty()) return error.BadUrl;
|
||||
return uri;
|
||||
}
|
||||
|
||||
/// Which call failed. The phase is what decides the classification, and only
|
||||
/// the call site knows it — guessing it from an error name would be wrong the
|
||||
/// first time two phases shared an error.
|
||||
const Phase = enum { connect, send, receive };
|
||||
|
||||
fn mapError(err: anyerror, phase: Phase) Error {
|
||||
if (transport.mapLocal(err)) |local| return narrowLocal(local);
|
||||
switch (err) {
|
||||
error.Timeout => return error.Timeout,
|
||||
// Both are ruled out by `parseUrl` before the client is touched.
|
||||
error.UnsupportedUriScheme, error.UriMissingHost => return error.BadUrl,
|
||||
error.TooManyHttpRedirects => return error.HttpStatus,
|
||||
else => {},
|
||||
}
|
||||
const err_name = @errorName(err);
|
||||
if (std.mem.startsWith(u8, err_name, "Tls") or
|
||||
std.mem.startsWith(u8, err_name, "Certificate")) return error.TlsFailed;
|
||||
return switch (phase) {
|
||||
.connect => error.ConnectFailed,
|
||||
.send => error.SendFailed,
|
||||
.receive => error.ReceiveFailed,
|
||||
};
|
||||
}
|
||||
|
||||
/// `transport.mapLocal` answers in `transport.ExchangeError`, which is wider
|
||||
/// than this file's taxonomy. Both file-descriptor quotas are the same
|
||||
/// exhaustion to a downloader, and `error.BufferTooSmall` cannot occur because
|
||||
/// this file hands the client no undersized buffer.
|
||||
fn narrowLocal(local: transport.ExchangeError) Error {
|
||||
return switch (local) {
|
||||
error.OutOfMemory => error.OutOfMemory,
|
||||
error.SystemResources,
|
||||
error.ProcessFdQuotaExceeded,
|
||||
error.SystemFdQuotaExceeded,
|
||||
=> error.SystemResources,
|
||||
error.Canceled => error.Canceled,
|
||||
else => error.Unexpected,
|
||||
};
|
||||
}
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
fn undefinedFetcher(transfer_buf: []u8, redirect_buf: []u8) Fetcher {
|
||||
// `http` is never driven: every test below asserts a rejection that
|
||||
// happens before the first client call.
|
||||
const http: *std.http.Client = undefined;
|
||||
return .{ .http = http, .transfer_buf = transfer_buf, .redirect_buf = redirect_buf };
|
||||
}
|
||||
|
||||
test "fetch rejects a non-http scheme before any client use" {
|
||||
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||
var sink_buf: [0]u8 = .{};
|
||||
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||
try testing.expectError(
|
||||
error.BadUrl,
|
||||
f.fetch(undefined, "ftp://example.com/list.txt", &discarding.writer),
|
||||
);
|
||||
}
|
||||
|
||||
test "fetch rejects a url with no scheme before any client use" {
|
||||
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||
var sink_buf: [0]u8 = .{};
|
||||
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||
try testing.expectError(error.BadUrl, f.fetch(undefined, "x", &discarding.writer));
|
||||
}
|
||||
|
||||
test "fetch rejects a url with no host before any client use" {
|
||||
var transfer_buf: [min_transfer_buf]u8 = undefined;
|
||||
var redirect_buf: [redirect_buffer_len]u8 = undefined;
|
||||
var f = undefinedFetcher(&transfer_buf, &redirect_buf);
|
||||
var sink_buf: [0]u8 = .{};
|
||||
var discarding: std.Io.Writer.Discarding = .init(&sink_buf);
|
||||
try testing.expectError(error.BadUrl, f.fetch(undefined, "https://", &discarding.writer));
|
||||
}
|
||||
|
||||
test "parseUrl accepts http and https urls" {
|
||||
const plain = try parseUrl("http://example.com/hosts.txt");
|
||||
try testing.expectEqualStrings("http", plain.scheme);
|
||||
const secure = try parseUrl("https://example.com:8443/hosts.txt");
|
||||
try testing.expectEqualStrings("https", secure.scheme);
|
||||
try testing.expectEqual(@as(?u16, 8443), secure.port);
|
||||
}
|
||||
|
||||
test "parseUrl rejects the url forms the source table can hold" {
|
||||
try testing.expectError(error.BadUrl, parseUrl("ftp://example.com/list"));
|
||||
try testing.expectError(error.BadUrl, parseUrl("file:///etc/hosts"));
|
||||
try testing.expectError(error.BadUrl, parseUrl("x"));
|
||||
try testing.expectError(error.BadUrl, parseUrl(""));
|
||||
try testing.expectError(error.BadUrl, parseUrl("https://"));
|
||||
}
|
||||
|
||||
test "mapError maps local errors before phase errors" {
|
||||
try testing.expectEqual(error.OutOfMemory, mapError(error.OutOfMemory, .connect));
|
||||
try testing.expectEqual(error.Canceled, mapError(error.Canceled, .receive));
|
||||
try testing.expectEqual(error.Unexpected, mapError(error.Unexpected, .send));
|
||||
try testing.expectEqual(error.SystemResources, mapError(error.SystemResources, .connect));
|
||||
try testing.expectEqual(
|
||||
error.SystemResources,
|
||||
mapError(error.ProcessFdQuotaExceeded, .connect),
|
||||
);
|
||||
try testing.expectEqual(
|
||||
error.SystemResources,
|
||||
mapError(error.SystemFdQuotaExceeded, .connect),
|
||||
);
|
||||
}
|
||||
|
||||
test "mapError maps tls errors regardless of phase" {
|
||||
try testing.expectEqual(error.TlsFailed, mapError(error.TlsInitializationFailed, .connect));
|
||||
try testing.expectEqual(error.TlsFailed, mapError(error.TlsAlert, .receive));
|
||||
try testing.expectEqual(error.TlsFailed, mapError(error.CertificateExpired, .connect));
|
||||
}
|
||||
|
||||
test "mapError maps a redirect overrun to HttpStatus" {
|
||||
try testing.expectEqual(error.HttpStatus, mapError(error.TooManyHttpRedirects, .receive));
|
||||
}
|
||||
|
||||
test "mapError maps a connect timeout to Timeout" {
|
||||
try testing.expectEqual(error.Timeout, mapError(error.Timeout, .connect));
|
||||
}
|
||||
|
||||
test "mapError maps remaining errors by phase" {
|
||||
try testing.expectEqual(error.ConnectFailed, mapError(error.ConnectionRefused, .connect));
|
||||
try testing.expectEqual(error.SendFailed, mapError(error.WriteFailed, .send));
|
||||
try testing.expectEqual(error.ReceiveFailed, mapError(error.ReadFailed, .receive));
|
||||
try testing.expectEqual(error.ReceiveFailed, mapError(error.HttpHeadersInvalid, .receive));
|
||||
}
|
||||
|
||||
test "caps are the values the memory budget was sized against" {
|
||||
try testing.expectEqual(@as(usize, 64 * 1024 * 1024), max_body_bytes);
|
||||
try testing.expectEqual(@as(usize, 8192), redirect_buffer_len);
|
||||
}
|
||||
Reference in New Issue
Block a user