//! `/api/blocklists` — the blocklist sources table, and the manual refresh. //! //! The resource is `blocklist_sources`: its four configuration columns are what //! an operator edits, and the counters the refresh writes ride along in the //! read shape so the UI can show a list's size next to its url (ruling 9). //! //! `POST /api/blocklists/update` runs `Manager.refreshAll` and then the reload //! seam, and answers 202 with the status of every source (ruling 12). The //! refresh downloads and compiles before the response is written: 202 is //! "accepted and done as far as this connection is concerned", and the status //! table in the body is what tells the operator which sources actually landed. //! //! `DELETE /api/blocklists/{id}` removes the row, reloads, and then sweeps the //! compiled files that row named, so `/blocklists/` follows the table //! the operator is looking at rather than the scheduler's next pass. const std = @import("std"); const Allocator = std.mem.Allocator; const http_util = @import("../http_util.zig"); const manager_mod = @import("../../filter/manager.zig"); const model = @import("../../config/model.zig"); const mutations = @import("mutations.zig"); const server = @import("../server.zig"); const sources_repo = @import("../../storage/repositories/sources_repo.zig"); const Failure = mutations.Failure; const Request = http_util.Request; const HandlerError = http_util.HandlerError; const log = std.log.scoped(.web_api); const url_conflict = "a blocklist with that url already exists"; /// How many source statuses one refresh response carries. A household runs a /// handful of lists; a table longer than this is truncated in the response /// only, never in the refresh. pub const max_statuses = 64; const Body = struct { url: []const u8, name: []const u8, enabled: bool = true, is_suggested: bool = false, }; const Created = union(enum) { id: i64, fail: Failure }; /// One source's status, in the shape the API speaks: the fixed-size text fields /// of `manager.SourceStatus` become plain strings, and the compile counts are /// flattened next to them. pub const StatusView = struct { id: i64, state: []const u8, loaded: bool, last_attempt: i64, last_success: i64, url: []const u8, last_error: []const u8, domains: u32, wildcards: u32, skipped_regex: u32, pub fn from(status: *const manager_mod.SourceStatus) StatusView { return .{ .id = status.id, .state = @tagName(status.state), .loaded = status.loaded, .last_attempt = status.last_attempt, .last_success = status.last_success, .url = status.urlText(), .last_error = status.errorText(), .domains = status.counts.domains, .wildcards = status.counts.wildcards, .skipped_regex = status.counts.skipped_regex, }; } }; // --------------------------------------------------------------------------- // decisions // --------------------------------------------------------------------------- pub fn applyCreate( state: *server.WebState, io: std.Io, arena: Allocator, item: model.BlocklistSource, ) error{OutOfMemory}!Created { const database = switch (mutations.configDb(state)) { .database => |value| value, .fail => |failure| return .{ .fail = failure }, }; if (try mutations.checkSource(arena, item)) |problem| return .{ .fail = .{ .invalid = problem } }; state.config_lock.lockUncancelable(io); const inserted = sources_repo.insertSourceRow(database, item); state.config_lock.unlock(io); const id = inserted catch |err| return .{ .fail = mutations.dbFailure(err, url_conflict) }; if (mutations.reload(state, io)) |failure| return .{ .fail = failure }; return .{ .id = id }; } pub fn applyUpdate( state: *server.WebState, io: std.Io, arena: Allocator, id: i64, item: model.BlocklistSource, ) error{OutOfMemory}!?Failure { const database = switch (mutations.configDb(state)) { .database => |value| value, .fail => |failure| return failure, }; if (try mutations.checkSource(arena, item)) |problem| return .{ .invalid = problem }; state.config_lock.lockUncancelable(io); const written = sources_repo.updateSource(database, id, item); state.config_lock.unlock(io); written catch |err| return mutations.dbFailure(err, url_conflict); return mutations.reload(state, io); } pub fn applyDelete(state: *server.WebState, io: std.Io, id: i64) ?Failure { const database = switch (mutations.configDb(state)) { .database => |value| value, .fail => |failure| return failure, }; state.config_lock.lockUncancelable(io); const written = sources_repo.deleteSource(database, id); state.config_lock.unlock(io); written catch |err| return mutations.dbFailure(err, url_conflict); const failure = mutations.reload(state, io); pruneFiles(state, io); return failure; } /// Removes the compiled files the deleted row leaves behind. /// /// This is the moment an orphan is made during normal operation, and the only /// other sweep is the scheduler's — up to `blocklist_update.interval_hours` /// away. Without this call a deleted list keeps its megabytes on disk for a day. /// /// After the reload and never part of the response: the row is gone and the /// snapshot has stopped enforcing the list, so bytes still on disk are not a /// failed delete. `Manager.pruneOrphans` takes the manager's refresh lock and /// then its writer lock; the reload above has already taken and released the /// writer lock — nothing here holds either, and `state.config_lock` was /// released before both. fn pruneFiles(state: *server.WebState, io: std.Io) void { const manager = state.manager orelse return; manager.pruneOrphans(io) catch |err| { log.warn("pruning the deleted blocklist's files failed: {s}", .{@errorName(err)}); }; } /// Refreshes every enabled source, then applies the result (ruling 12). /// /// `refreshAll` already ends in the manager's own reload; the seam is called /// too, because it is how the composition root learns that a change landed and /// the only reload a test can observe. pub fn applyRefresh(state: *server.WebState, io: std.Io, out: []manager_mod.SourceStatus) union(enum) { statuses: usize, fail: Failure, } { const manager = state.manager orelse return .{ .fail = .{ .unavailable = "no blocklist manager" } }; manager.refreshAll(io) catch |err| switch (err) { error.Canceled => return .{ .fail = .{ .unavailable = "shutting down" } }, error.OutOfMemory => return .{ .fail = .{ .internal = error.OutOfMemory } }, // A source that fails to fetch or compile records that in the status // table and returns cleanly, so reaching here means the pass itself // broke. `Manager.Error` is wider than `db.Error`, so the cause is // logged here and the client is told only that it was internal. else => { log.warn("refreshing the blocklists failed: {s}", .{@errorName(err)}); return .{ .fail = .{ .internal = error.Unexpected } }; }, }; if (mutations.reload(state, io)) |failure| return .{ .fail = failure }; return .{ .statuses = manager.statusSnapshot(io, out) }; } // --------------------------------------------------------------------------- // routes // --------------------------------------------------------------------------- pub fn list(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void { _ = io; const database = switch (mutations.configDb(state)) { .database => |value| value, .fail => |failure| return mutations.respondFailure(request, failure, "listing blocklists"), }; const rows = sources_repo.listSourceRows(database, request.arena) catch |err| return mutations.respondFailure(request, .{ .internal = err }, "listing blocklists"); return http_util.respondJson(request, .ok, .{ .blocklists = rows.items }, &.{}); } pub fn get(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void { _ = io; const database = switch (mutations.configDb(state)) { .database => |value| value, .fail => |failure| return mutations.respondFailure(request, failure, "reading a blocklist"), }; const row = sources_repo.getSource(database, request.arena, request.id.?) catch |err| return mutations.respondFailure(request, .{ .internal = err }, "reading a blocklist"); const found = row orelse return mutations.respondFailure(request, .not_found, ""); return http_util.respondJson(request, .ok, found, &.{}); } pub fn create(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void { const parsed = http_util.parseBody(Body, request) catch |err| return mutations.respondBadBody(request, err); const item = toModel(parsed.value); return switch (try applyCreate(state, io, request.arena, item)) { .fail => |failure| mutations.respondFailure(request, failure, "creating a blocklist"), .id => |id| http_util.respondJson(request, .created, .{ .id = id, .url = item.url, .name = item.name, .enabled = item.enabled, .is_suggested = item.is_suggested, }, &.{}), }; } pub fn update(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void { const parsed = http_util.parseBody(Body, request) catch |err| return mutations.respondBadBody(request, err); const item = toModel(parsed.value); const id = request.id.?; if (try applyUpdate(state, io, request.arena, id, item)) |failure| { return mutations.respondFailure(request, failure, "updating a blocklist"); } return http_util.respondJson(request, .ok, .{ .id = id, .url = item.url, .name = item.name, .enabled = item.enabled, .is_suggested = item.is_suggested, }, &.{}); } pub fn remove(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void { if (applyDelete(state, io, request.id.?)) |failure| { return mutations.respondFailure(request, failure, "deleting a blocklist"); } return http_util.respondEmpty(request, .no_content); } /// `POST /api/blocklists/update`. pub fn refresh(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void { const statuses = try request.arena.alloc(manager_mod.SourceStatus, max_statuses); return switch (applyRefresh(state, io, statuses)) { .fail => |failure| mutations.respondFailure(request, failure, "refreshing the blocklists"), .statuses => |count| respondStatuses(request, statuses[0..count]), }; } fn respondStatuses(request: *Request, statuses: []const manager_mod.SourceStatus) HandlerError!void { const views = try request.arena.alloc(StatusView, statuses.len); for (views, statuses) |*view, *status| view.* = .from(status); return http_util.respondJson(request, .accepted, .{ .sources = views }, &.{}); } fn toModel(body: Body) model.BlocklistSource { return .{ .url = body.url, .name = body.name, .enabled = body.enabled, .is_suggested = body.is_suggested, }; } // --------------------------------------------------------------------------- // tests // --------------------------------------------------------------------------- const testing = std.testing; const valid: model.BlocklistSource = .{ .url = "https://a.test/list.txt", .name = "a" }; test "a created blocklist is stored with its runtime columns at their defaults" { var bench: mutations.Bench = undefined; try bench.init(testing.allocator); defer bench.deinit(testing.allocator); const created = try applyCreate(&bench.state, bench.io(), bench.arena(), valid); try testing.expectEqual(@as(usize, 1), bench.reloads); const row = (try sources_repo.getSource(&bench.database, bench.arena(), created.id)).?; try testing.expectEqualStrings("https://a.test/list.txt", row.url); try testing.expect(row.enabled); try testing.expectEqual(@as(?i64, null), row.last_updated); try testing.expectEqual(@as(i64, 0), row.domain_count); } test "a url the validator refuses never reaches the database" { var bench: mutations.Bench = undefined; try bench.init(testing.allocator); defer bench.deinit(testing.allocator); const created = try applyCreate(&bench.state, bench.io(), bench.arena(), .{ .url = "ftp://a.test/list.txt", .name = "a", }); try testing.expect(created.fail == .invalid); try testing.expectEqual(@as(i64, 0), try bench.queryInt("SELECT count(*) FROM blocklist_sources")); const unnamed = try applyCreate(&bench.state, bench.io(), bench.arena(), .{ .url = "https://a.test/list.txt", .name = "", }); try testing.expect(unnamed.fail == .invalid); try testing.expectEqual(@as(usize, 0), bench.reloads); } test "a duplicate url is a conflict" { var bench: mutations.Bench = undefined; try bench.init(testing.allocator); defer bench.deinit(testing.allocator); _ = try applyCreate(&bench.state, bench.io(), bench.arena(), valid); const again = try applyCreate(&bench.state, bench.io(), bench.arena(), valid); try testing.expectEqualStrings(url_conflict, again.fail.conflict); } test "editing a blocklist keeps the counters the refresh wrote" { var bench: mutations.Bench = undefined; try bench.init(testing.allocator); defer bench.deinit(testing.allocator); const created = try applyCreate(&bench.state, bench.io(), bench.arena(), valid); try sources_repo.updateSourceStats(&bench.database, created.id, .{ .last_updated = 1700, .domain_count = 42, .wildcard_count = 3, .skipped_regex_count = 1, .checksum = "abc", }); const failure = try applyUpdate(&bench.state, bench.io(), bench.arena(), created.id, .{ .url = "https://a.test/list.txt", .name = "renamed", .enabled = false, }); try testing.expectEqual(@as(?Failure, null), failure); const row = (try sources_repo.getSource(&bench.database, bench.arena(), created.id)).?; try testing.expectEqualStrings("renamed", row.name); try testing.expect(!row.enabled); try testing.expectEqual(@as(i64, 42), row.domain_count); try testing.expectEqual(@as(usize, 2), bench.reloads); } test "updating and deleting an id no row holds is a 404" { var bench: mutations.Bench = undefined; try bench.init(testing.allocator); defer bench.deinit(testing.allocator); try testing.expectEqual( Failure.not_found, (try applyUpdate(&bench.state, bench.io(), bench.arena(), 999, valid)).?, ); try testing.expectEqual(Failure.not_found, applyDelete(&bench.state, bench.io(), 999).?); try testing.expectEqual(@as(usize, 0), bench.reloads); } test "deleting a blocklist takes its group assignments with it" { var bench: mutations.Bench = undefined; try bench.init(testing.allocator); defer bench.deinit(testing.allocator); const created = try applyCreate(&bench.state, bench.io(), bench.arena(), valid); try bench.exec("INSERT INTO group_sources (group_id, source_id) VALUES (1, 1);"); try testing.expectEqual(@as(?Failure, null), applyDelete(&bench.state, bench.io(), created.id)); try testing.expectEqual(@as(i64, 0), try bench.queryInt("SELECT count(*) FROM group_sources")); try testing.expectEqual(@as(usize, 2), bench.reloads); } test "a refresh with no manager is unavailable rather than a silent success" { var bench: mutations.Bench = undefined; try bench.init(testing.allocator); defer bench.deinit(testing.allocator); var statuses: [4]manager_mod.SourceStatus = undefined; const outcome = applyRefresh(&bench.state, bench.io(), &statuses); try testing.expect(outcome.fail == .unavailable); try testing.expectEqual(@as(usize, 0), bench.reloads); } test "a status becomes the flat shape the API answers with" { var status: manager_mod.SourceStatus = .{ .id = 7, .state = .fetch_failed, .loaded = true }; const url = "https://a.test/list.txt"; @memcpy(status.url[0..url.len], url); status.url_len = url.len; const message = "connection refused"; @memcpy(status.last_error[0..message.len], message); status.last_error_len = message.len; status.counts = .{ .domains = 10, .wildcards = 2, .skipped_regex = 1 }; const view: StatusView = .from(&status); try testing.expectEqual(@as(i64, 7), view.id); try testing.expectEqualStrings("fetch_failed", view.state); try testing.expect(view.loaded); try testing.expectEqualStrings(url, view.url); try testing.expectEqualStrings(message, view.last_error); try testing.expectEqual(@as(u32, 10), view.domains); }