422 lines
17 KiB
Zig
422 lines
17 KiB
Zig
//! `/api/blocklists` — the blocklist sources table, and the manual refresh.
|
|
//!
|
|
//! The resource is `blocklist_sources`: its four configuration columns are what
|
|
//! an operator edits, and the counters the refresh writes ride along in the
|
|
//! read shape so the UI can show a list's size next to its url (ruling 9).
|
|
//!
|
|
//! `POST /api/blocklists/update` runs `Manager.refreshAll` and then the reload
|
|
//! seam, and answers 202 with the status of every source (ruling 12). The
|
|
//! refresh downloads and compiles before the response is written: 202 is
|
|
//! "accepted and done as far as this connection is concerned", and the status
|
|
//! table in the body is what tells the operator which sources actually landed.
|
|
//!
|
|
//! `DELETE /api/blocklists/{id}` removes the row, reloads, and then sweeps the
|
|
//! compiled files that row named, so `<data_dir>/blocklists/` follows the table
|
|
//! the operator is looking at rather than the scheduler's next pass.
|
|
|
|
const std = @import("std");
|
|
const Allocator = std.mem.Allocator;
|
|
|
|
const http_util = @import("../http_util.zig");
|
|
const manager_mod = @import("../../filter/manager.zig");
|
|
const model = @import("../../config/model.zig");
|
|
const mutations = @import("mutations.zig");
|
|
const server = @import("../server.zig");
|
|
const sources_repo = @import("../../storage/repositories/sources_repo.zig");
|
|
|
|
const Failure = mutations.Failure;
|
|
const Request = http_util.Request;
|
|
const HandlerError = http_util.HandlerError;
|
|
|
|
const log = std.log.scoped(.web_api);
|
|
|
|
const url_conflict = "a blocklist with that url already exists";
|
|
|
|
/// How many source statuses one refresh response carries. A household runs a
|
|
/// handful of lists; a table longer than this is truncated in the response
|
|
/// only, never in the refresh.
|
|
pub const max_statuses = 64;
|
|
|
|
const Body = struct {
|
|
url: []const u8,
|
|
name: []const u8,
|
|
enabled: bool = true,
|
|
is_suggested: bool = false,
|
|
};
|
|
|
|
const Created = union(enum) { id: i64, fail: Failure };
|
|
|
|
/// One source's status, in the shape the API speaks: the fixed-size text fields
|
|
/// of `manager.SourceStatus` become plain strings, and the compile counts are
|
|
/// flattened next to them.
|
|
pub const StatusView = struct {
|
|
id: i64,
|
|
state: []const u8,
|
|
loaded: bool,
|
|
last_attempt: i64,
|
|
last_success: i64,
|
|
url: []const u8,
|
|
last_error: []const u8,
|
|
domains: u32,
|
|
wildcards: u32,
|
|
skipped_regex: u32,
|
|
|
|
pub fn from(status: *const manager_mod.SourceStatus) StatusView {
|
|
return .{
|
|
.id = status.id,
|
|
.state = @tagName(status.state),
|
|
.loaded = status.loaded,
|
|
.last_attempt = status.last_attempt,
|
|
.last_success = status.last_success,
|
|
.url = status.urlText(),
|
|
.last_error = status.errorText(),
|
|
.domains = status.counts.domains,
|
|
.wildcards = status.counts.wildcards,
|
|
.skipped_regex = status.counts.skipped_regex,
|
|
};
|
|
}
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// decisions
|
|
// ---------------------------------------------------------------------------
|
|
|
|
pub fn applyCreate(
|
|
state: *server.WebState,
|
|
io: std.Io,
|
|
arena: Allocator,
|
|
item: model.BlocklistSource,
|
|
) error{OutOfMemory}!Created {
|
|
const database = switch (mutations.configDb(state)) {
|
|
.database => |value| value,
|
|
.fail => |failure| return .{ .fail = failure },
|
|
};
|
|
if (try mutations.checkSource(arena, item)) |problem| return .{ .fail = .{ .invalid = problem } };
|
|
|
|
state.config_lock.lockUncancelable(io);
|
|
const inserted = sources_repo.insertSourceRow(database, item);
|
|
state.config_lock.unlock(io);
|
|
|
|
const id = inserted catch |err| return .{ .fail = mutations.dbFailure(err, url_conflict) };
|
|
if (mutations.reload(state, io)) |failure| return .{ .fail = failure };
|
|
return .{ .id = id };
|
|
}
|
|
|
|
pub fn applyUpdate(
|
|
state: *server.WebState,
|
|
io: std.Io,
|
|
arena: Allocator,
|
|
id: i64,
|
|
item: model.BlocklistSource,
|
|
) error{OutOfMemory}!?Failure {
|
|
const database = switch (mutations.configDb(state)) {
|
|
.database => |value| value,
|
|
.fail => |failure| return failure,
|
|
};
|
|
if (try mutations.checkSource(arena, item)) |problem| return .{ .invalid = problem };
|
|
|
|
state.config_lock.lockUncancelable(io);
|
|
const written = sources_repo.updateSource(database, id, item);
|
|
state.config_lock.unlock(io);
|
|
|
|
written catch |err| return mutations.dbFailure(err, url_conflict);
|
|
return mutations.reload(state, io);
|
|
}
|
|
|
|
pub fn applyDelete(state: *server.WebState, io: std.Io, id: i64) ?Failure {
|
|
const database = switch (mutations.configDb(state)) {
|
|
.database => |value| value,
|
|
.fail => |failure| return failure,
|
|
};
|
|
|
|
state.config_lock.lockUncancelable(io);
|
|
const written = sources_repo.deleteSource(database, id);
|
|
state.config_lock.unlock(io);
|
|
|
|
written catch |err| return mutations.dbFailure(err, url_conflict);
|
|
const failure = mutations.reload(state, io);
|
|
pruneFiles(state, io);
|
|
return failure;
|
|
}
|
|
|
|
/// Removes the compiled files the deleted row leaves behind.
|
|
///
|
|
/// This is the moment an orphan is made during normal operation, and the only
|
|
/// other sweep is the scheduler's — up to `blocklist_update.interval_hours`
|
|
/// away. Without this call a deleted list keeps its megabytes on disk for a day.
|
|
///
|
|
/// After the reload and never part of the response: the row is gone and the
|
|
/// snapshot has stopped enforcing the list, so bytes still on disk are not a
|
|
/// failed delete. `Manager.pruneOrphans` takes the manager's refresh lock and
|
|
/// then its writer lock; the reload above has already taken and released the
|
|
/// writer lock — nothing here holds either, and `state.config_lock` was
|
|
/// released before both.
|
|
fn pruneFiles(state: *server.WebState, io: std.Io) void {
|
|
const manager = state.manager orelse return;
|
|
manager.pruneOrphans(io) catch |err| {
|
|
log.warn("pruning the deleted blocklist's files failed: {s}", .{@errorName(err)});
|
|
};
|
|
}
|
|
|
|
/// Refreshes every enabled source, then applies the result (ruling 12).
|
|
///
|
|
/// `refreshAll` already ends in the manager's own reload; the seam is called
|
|
/// too, because it is how the composition root learns that a change landed and
|
|
/// the only reload a test can observe.
|
|
pub fn applyRefresh(state: *server.WebState, io: std.Io, out: []manager_mod.SourceStatus) union(enum) {
|
|
statuses: usize,
|
|
fail: Failure,
|
|
} {
|
|
const manager = state.manager orelse return .{ .fail = .{ .unavailable = "no blocklist manager" } };
|
|
|
|
manager.refreshAll(io) catch |err| switch (err) {
|
|
error.Canceled => return .{ .fail = .{ .unavailable = "shutting down" } },
|
|
error.OutOfMemory => return .{ .fail = .{ .internal = error.OutOfMemory } },
|
|
// A source that fails to fetch or compile records that in the status
|
|
// table and returns cleanly, so reaching here means the pass itself
|
|
// broke. `Manager.Error` is wider than `db.Error`, so the cause is
|
|
// logged here and the client is told only that it was internal.
|
|
else => {
|
|
log.warn("refreshing the blocklists failed: {s}", .{@errorName(err)});
|
|
return .{ .fail = .{ .internal = error.Unexpected } };
|
|
},
|
|
};
|
|
if (mutations.reload(state, io)) |failure| return .{ .fail = failure };
|
|
return .{ .statuses = manager.statusSnapshot(io, out) };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// routes
|
|
// ---------------------------------------------------------------------------
|
|
|
|
pub fn list(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void {
|
|
_ = io;
|
|
const database = switch (mutations.configDb(state)) {
|
|
.database => |value| value,
|
|
.fail => |failure| return mutations.respondFailure(request, failure, "listing blocklists"),
|
|
};
|
|
|
|
const rows = sources_repo.listSourceRows(database, request.arena) catch |err|
|
|
return mutations.respondFailure(request, .{ .internal = err }, "listing blocklists");
|
|
|
|
return http_util.respondJson(request, .ok, .{ .blocklists = rows.items }, &.{});
|
|
}
|
|
|
|
pub fn get(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void {
|
|
_ = io;
|
|
const database = switch (mutations.configDb(state)) {
|
|
.database => |value| value,
|
|
.fail => |failure| return mutations.respondFailure(request, failure, "reading a blocklist"),
|
|
};
|
|
|
|
const row = sources_repo.getSource(database, request.arena, request.id.?) catch |err|
|
|
return mutations.respondFailure(request, .{ .internal = err }, "reading a blocklist");
|
|
const found = row orelse return mutations.respondFailure(request, .not_found, "");
|
|
|
|
return http_util.respondJson(request, .ok, found, &.{});
|
|
}
|
|
|
|
pub fn create(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void {
|
|
const parsed = http_util.parseBody(Body, request) catch |err|
|
|
return mutations.respondBadBody(request, err);
|
|
const item = toModel(parsed.value);
|
|
|
|
return switch (try applyCreate(state, io, request.arena, item)) {
|
|
.fail => |failure| mutations.respondFailure(request, failure, "creating a blocklist"),
|
|
.id => |id| http_util.respondJson(request, .created, .{
|
|
.id = id,
|
|
.url = item.url,
|
|
.name = item.name,
|
|
.enabled = item.enabled,
|
|
.is_suggested = item.is_suggested,
|
|
}, &.{}),
|
|
};
|
|
}
|
|
|
|
pub fn update(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void {
|
|
const parsed = http_util.parseBody(Body, request) catch |err|
|
|
return mutations.respondBadBody(request, err);
|
|
const item = toModel(parsed.value);
|
|
const id = request.id.?;
|
|
|
|
if (try applyUpdate(state, io, request.arena, id, item)) |failure| {
|
|
return mutations.respondFailure(request, failure, "updating a blocklist");
|
|
}
|
|
return http_util.respondJson(request, .ok, .{
|
|
.id = id,
|
|
.url = item.url,
|
|
.name = item.name,
|
|
.enabled = item.enabled,
|
|
.is_suggested = item.is_suggested,
|
|
}, &.{});
|
|
}
|
|
|
|
pub fn remove(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void {
|
|
if (applyDelete(state, io, request.id.?)) |failure| {
|
|
return mutations.respondFailure(request, failure, "deleting a blocklist");
|
|
}
|
|
return http_util.respondEmpty(request, .no_content);
|
|
}
|
|
|
|
/// `POST /api/blocklists/update`.
|
|
pub fn refresh(state: *server.WebState, io: std.Io, request: *Request) HandlerError!void {
|
|
const statuses = try request.arena.alloc(manager_mod.SourceStatus, max_statuses);
|
|
|
|
return switch (applyRefresh(state, io, statuses)) {
|
|
.fail => |failure| mutations.respondFailure(request, failure, "refreshing the blocklists"),
|
|
.statuses => |count| respondStatuses(request, statuses[0..count]),
|
|
};
|
|
}
|
|
|
|
fn respondStatuses(request: *Request, statuses: []const manager_mod.SourceStatus) HandlerError!void {
|
|
const views = try request.arena.alloc(StatusView, statuses.len);
|
|
for (views, statuses) |*view, *status| view.* = .from(status);
|
|
return http_util.respondJson(request, .accepted, .{ .sources = views }, &.{});
|
|
}
|
|
|
|
fn toModel(body: Body) model.BlocklistSource {
|
|
return .{
|
|
.url = body.url,
|
|
.name = body.name,
|
|
.enabled = body.enabled,
|
|
.is_suggested = body.is_suggested,
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// tests
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const testing = std.testing;
|
|
|
|
const valid: model.BlocklistSource = .{ .url = "https://a.test/list.txt", .name = "a" };
|
|
|
|
test "a created blocklist is stored with its runtime columns at their defaults" {
|
|
var bench: mutations.Bench = undefined;
|
|
try bench.init(testing.allocator);
|
|
defer bench.deinit(testing.allocator);
|
|
|
|
const created = try applyCreate(&bench.state, bench.io(), bench.arena(), valid);
|
|
try testing.expectEqual(@as(usize, 1), bench.reloads);
|
|
|
|
const row = (try sources_repo.getSource(&bench.database, bench.arena(), created.id)).?;
|
|
try testing.expectEqualStrings("https://a.test/list.txt", row.url);
|
|
try testing.expect(row.enabled);
|
|
try testing.expectEqual(@as(?i64, null), row.last_updated);
|
|
try testing.expectEqual(@as(i64, 0), row.domain_count);
|
|
}
|
|
|
|
test "a url the validator refuses never reaches the database" {
|
|
var bench: mutations.Bench = undefined;
|
|
try bench.init(testing.allocator);
|
|
defer bench.deinit(testing.allocator);
|
|
|
|
const created = try applyCreate(&bench.state, bench.io(), bench.arena(), .{
|
|
.url = "ftp://a.test/list.txt",
|
|
.name = "a",
|
|
});
|
|
try testing.expect(created.fail == .invalid);
|
|
try testing.expectEqual(@as(i64, 0), try bench.queryInt("SELECT count(*) FROM blocklist_sources"));
|
|
|
|
const unnamed = try applyCreate(&bench.state, bench.io(), bench.arena(), .{
|
|
.url = "https://a.test/list.txt",
|
|
.name = "",
|
|
});
|
|
try testing.expect(unnamed.fail == .invalid);
|
|
try testing.expectEqual(@as(usize, 0), bench.reloads);
|
|
}
|
|
|
|
test "a duplicate url is a conflict" {
|
|
var bench: mutations.Bench = undefined;
|
|
try bench.init(testing.allocator);
|
|
defer bench.deinit(testing.allocator);
|
|
|
|
_ = try applyCreate(&bench.state, bench.io(), bench.arena(), valid);
|
|
const again = try applyCreate(&bench.state, bench.io(), bench.arena(), valid);
|
|
try testing.expectEqualStrings(url_conflict, again.fail.conflict);
|
|
}
|
|
|
|
test "editing a blocklist keeps the counters the refresh wrote" {
|
|
var bench: mutations.Bench = undefined;
|
|
try bench.init(testing.allocator);
|
|
defer bench.deinit(testing.allocator);
|
|
|
|
const created = try applyCreate(&bench.state, bench.io(), bench.arena(), valid);
|
|
try sources_repo.updateSourceStats(&bench.database, created.id, .{
|
|
.last_updated = 1700,
|
|
.domain_count = 42,
|
|
.wildcard_count = 3,
|
|
.skipped_regex_count = 1,
|
|
.checksum = "abc",
|
|
});
|
|
|
|
const failure = try applyUpdate(&bench.state, bench.io(), bench.arena(), created.id, .{
|
|
.url = "https://a.test/list.txt",
|
|
.name = "renamed",
|
|
.enabled = false,
|
|
});
|
|
try testing.expectEqual(@as(?Failure, null), failure);
|
|
|
|
const row = (try sources_repo.getSource(&bench.database, bench.arena(), created.id)).?;
|
|
try testing.expectEqualStrings("renamed", row.name);
|
|
try testing.expect(!row.enabled);
|
|
try testing.expectEqual(@as(i64, 42), row.domain_count);
|
|
try testing.expectEqual(@as(usize, 2), bench.reloads);
|
|
}
|
|
|
|
test "updating and deleting an id no row holds is a 404" {
|
|
var bench: mutations.Bench = undefined;
|
|
try bench.init(testing.allocator);
|
|
defer bench.deinit(testing.allocator);
|
|
|
|
try testing.expectEqual(
|
|
Failure.not_found,
|
|
(try applyUpdate(&bench.state, bench.io(), bench.arena(), 999, valid)).?,
|
|
);
|
|
try testing.expectEqual(Failure.not_found, applyDelete(&bench.state, bench.io(), 999).?);
|
|
try testing.expectEqual(@as(usize, 0), bench.reloads);
|
|
}
|
|
|
|
test "deleting a blocklist takes its group assignments with it" {
|
|
var bench: mutations.Bench = undefined;
|
|
try bench.init(testing.allocator);
|
|
defer bench.deinit(testing.allocator);
|
|
|
|
const created = try applyCreate(&bench.state, bench.io(), bench.arena(), valid);
|
|
try bench.exec("INSERT INTO group_sources (group_id, source_id) VALUES (1, 1);");
|
|
|
|
try testing.expectEqual(@as(?Failure, null), applyDelete(&bench.state, bench.io(), created.id));
|
|
try testing.expectEqual(@as(i64, 0), try bench.queryInt("SELECT count(*) FROM group_sources"));
|
|
try testing.expectEqual(@as(usize, 2), bench.reloads);
|
|
}
|
|
|
|
test "a refresh with no manager is unavailable rather than a silent success" {
|
|
var bench: mutations.Bench = undefined;
|
|
try bench.init(testing.allocator);
|
|
defer bench.deinit(testing.allocator);
|
|
|
|
var statuses: [4]manager_mod.SourceStatus = undefined;
|
|
const outcome = applyRefresh(&bench.state, bench.io(), &statuses);
|
|
try testing.expect(outcome.fail == .unavailable);
|
|
try testing.expectEqual(@as(usize, 0), bench.reloads);
|
|
}
|
|
|
|
test "a status becomes the flat shape the API answers with" {
|
|
var status: manager_mod.SourceStatus = .{ .id = 7, .state = .fetch_failed, .loaded = true };
|
|
const url = "https://a.test/list.txt";
|
|
@memcpy(status.url[0..url.len], url);
|
|
status.url_len = url.len;
|
|
const message = "connection refused";
|
|
@memcpy(status.last_error[0..message.len], message);
|
|
status.last_error_len = message.len;
|
|
status.counts = .{ .domains = 10, .wildcards = 2, .skipped_regex = 1 };
|
|
|
|
const view: StatusView = .from(&status);
|
|
try testing.expectEqual(@as(i64, 7), view.id);
|
|
try testing.expectEqualStrings("fetch_failed", view.state);
|
|
try testing.expect(view.loaded);
|
|
try testing.expectEqualStrings(url, view.url);
|
|
try testing.expectEqualStrings(message, view.last_error);
|
|
try testing.expectEqual(@as(u32, 10), view.domains);
|
|
}
|