const std = @import("std"); const http = std.http; const Allocator = std.mem.Allocator; const Writer = std.Io.Writer; const Database = @import("../storage/db.zig").Database; const schema = @import("../storage/schema.zig"); const events = @import("../events.zig"); const DenylistStatus = schema.DenylistStatus; /// Maximum output size for denylist fetch (100MB) const MAX_DENYLIST_SIZE = 100 * 1024 * 1024; /// Denylist format types pub const DenylistFormat = enum { hosts, // 0.0.0.0 domain.com or 127.0.0.1 domain.com domains, // one domain per line adblock, // ||domain.com^ /// Detect format from content sample pub fn detect(content: []const u8) DenylistFormat { var lines = std.mem.splitScalar(u8, content, '\n'); var hosts_count: usize = 0; var adblock_count: usize = 0; var checked: usize = 0; while (lines.next()) |line| { const trimmed = std.mem.trim(u8, line, " \t\r"); if (trimmed.len == 0 or trimmed[0] == '#') continue; if (std.mem.startsWith(u8, trimmed, "||") and std.mem.endsWith(u8, trimmed, "^")) { adblock_count += 1; } else if (std.mem.startsWith(u8, trimmed, "0.0.0.0 ") or std.mem.startsWith(u8, trimmed, "127.0.0.1 ")) { hosts_count += 1; } checked += 1; if (checked >= 50) break; } if (adblock_count > hosts_count) return .adblock; if (hosts_count > 0) return .hosts; return .domains; } }; /// Result of parsing a denylist pub const ParseResult = struct { domains: std.ArrayListUnmanaged([]const u8), invalid_count: usize, }; /// Parse denylist content and return domains plus count of invalid lines pub fn parseDenylist(content: []const u8, format: DenylistFormat, allocator: Allocator) !ParseResult { var domains = std.ArrayListUnmanaged([]const u8){}; errdefer { for (domains.items) |d| allocator.free(d); domains.deinit(allocator); } var invalid_count: usize = 0; var lines = std.mem.splitScalar(u8, content, '\n'); while (lines.next()) |line| { const trimmed = std.mem.trim(u8, line, " \t\r"); if (trimmed.len == 0 or trimmed[0] == '#' or trimmed[0] == '!') continue; const domain = switch (format) { .hosts => parseHostsLine(trimmed), .domains => parseDomainLine(trimmed), .adblock => parseAdblockLine(trimmed), }; if (domain) |d| { if (isValidDomain(d)) { const copy = try allocator.dupe(u8, d); try domains.append(allocator, copy); } else { invalid_count += 1; } } else { // Line matched format but couldn't be parsed (e.g., wrong IP in hosts) invalid_count += 1; } } return .{ .domains = domains, .invalid_count = invalid_count }; } fn parseHostsLine(line: []const u8) ?[]const u8 { // Format: 0.0.0.0 domain.com or 127.0.0.1 domain.com var parts = std.mem.splitAny(u8, line, " \t"); const ip = parts.first(); if (!std.mem.eql(u8, ip, "0.0.0.0") and !std.mem.eql(u8, ip, "127.0.0.1")) { return null; } const domain = parts.next() orelse return null; // Skip localhost entries if (std.mem.eql(u8, domain, "localhost") or std.mem.eql(u8, domain, "localhost.localdomain") or std.mem.eql(u8, domain, "local") or std.mem.eql(u8, domain, "broadcasthost")) { return null; } return domain; } fn parseDomainLine(line: []const u8) ?[]const u8 { // Just the domain, skip comments if (line.len == 0) return null; // Check for inline comments var domain = line; if (std.mem.indexOf(u8, line, "#")) |idx| { domain = std.mem.trim(u8, line[0..idx], " \t"); } if (domain.len == 0) return null; // Handle wildcard format: *.domain.com -> domain.com if (std.mem.startsWith(u8, domain, "*.")) { domain = domain[2..]; } return if (domain.len > 0) domain else null; } fn parseAdblockLine(line: []const u8) ?[]const u8 { // Format: ||domain.com^ or ||domain.com^$... if (!std.mem.startsWith(u8, line, "||")) return null; var end = line.len; // Find the end marker (^ or $) if (std.mem.indexOf(u8, line[2..], "^")) |idx| { end = idx + 2; } else if (std.mem.indexOf(u8, line[2..], "$")) |idx| { end = idx + 2; } const domain = line[2..end]; return if (domain.len > 0) domain else null; } fn isValidDomain(domain: []const u8) bool { if (domain.len == 0 or domain.len > 253) return false; // Must contain at least one dot (except for special cases) if (std.mem.indexOf(u8, domain, ".") == null) return false; // Check for valid characters for (domain) |c| { if (!std.ascii.isAlphanumeric(c) and c != '.' and c != '-' and c != '_') { return false; } } // Don't start or end with hyphen or dot if (domain[0] == '-' or domain[0] == '.') return false; if (domain[domain.len - 1] == '-' or domain[domain.len - 1] == '.') return false; return true; } /// Denylist fetcher for downloading and updating denylists pub const DenylistFetcher = struct { db: *Database, allocator: Allocator, pub fn init(db: *Database, allocator: Allocator) DenylistFetcher { return DenylistFetcher{ .db = db, .allocator = allocator, }; } /// Fetch and update a denylist from URL pub fn fetchAndUpdate(self: *DenylistFetcher, source_id: i64, url: []const u8) !usize { // Fetch content via HTTP const content = self.httpGet(url) catch |err| { // Fetch failed - check if we have cached data const has_cached = self.hasCachedDomains(source_id); const status: DenylistStatus = if (has_cached) .cached else .failed; self.updateStatus(source_id, status); return err; }; defer self.allocator.free(content); // Compute content hash var hasher = std.hash.XxHash3.init(0); hasher.update(content); const hash = hasher.final(); var hash_str: [16]u8 = undefined; _ = std.fmt.bufPrint(&hash_str, "{x:0>16}", .{hash}) catch unreachable; // Check if content changed const old_hash = self.getContentHash(source_id); if (old_hash) |h| { if (std.mem.eql(u8, &hash_str, h)) { // Content unchanged self.updateStatusAndHash(source_id, .unchanged, &hash_str); return self.getDomainCount(source_id); } } // Detect format const format = DenylistFormat.detect(content); // Parse domains var result = try parseDenylist(content, format, self.allocator); defer { for (result.domains.items) |d| self.allocator.free(d); result.domains.deinit(self.allocator); } // Update database with new domains and invalid count const count = try self.updateDatabase(source_id, result.domains.items, result.invalid_count, &hash_str); return count; } fn hasCachedDomains(self: *DenylistFetcher, source_id: i64) bool { var stmt = self.db.prepare("SELECT domain_count FROM denylist_sources WHERE id = ?") catch return false; defer stmt.finalize(); stmt.bindInt(1, source_id) catch return false; if (stmt.step() catch false) { return stmt.getInt(0) > 0; } return false; } fn getContentHash(self: *DenylistFetcher, source_id: i64) ?[]const u8 { var stmt = self.db.prepare("SELECT content_hash FROM denylist_sources WHERE id = ?") catch return null; defer stmt.finalize(); stmt.bindInt(1, source_id) catch return null; if (stmt.step() catch false) { return stmt.getText(0); } return null; } fn getDomainCount(self: *DenylistFetcher, source_id: i64) usize { var stmt = self.db.prepare("SELECT domain_count FROM denylist_sources WHERE id = ?") catch return 0; defer stmt.finalize(); stmt.bindInt(1, source_id) catch return 0; if (stmt.step() catch false) { const count = stmt.getInt(0); return if (count >= 0) @intCast(count) else 0; } return 0; } fn updateStatus(self: *DenylistFetcher, source_id: i64, status: DenylistStatus) void { var stmt = self.db.prepare("UPDATE denylist_sources SET status = ? WHERE id = ?") catch return; defer stmt.finalize(); stmt.bindInt(1, @intFromEnum(status)) catch return; stmt.bindInt(2, source_id) catch return; _ = stmt.step() catch {}; } fn updateStatusAndHash(self: *DenylistFetcher, source_id: i64, status: DenylistStatus, hash: []const u8) void { var stmt = self.db.prepare( "UPDATE denylist_sources SET status = ?, content_hash = ?, last_updated = strftime('%s', 'now') WHERE id = ?", ) catch return; defer stmt.finalize(); stmt.bindInt(1, @intFromEnum(status)) catch return; stmt.bindText(2, hash) catch return; stmt.bindInt(3, source_id) catch return; _ = stmt.step() catch {}; } fn updateDatabase(self: *DenylistFetcher, source_id: i64, domains: []const []const u8, invalid_count: usize, hash: []const u8) !usize { // Get source type to determine which table to use (Gap 5) const source_type = self.getSourceType(source_id); const is_allowlist = source_type == .allowlist; // Start transaction with RAII - auto-rollback if not committed var tx = try self.db.begin(); defer tx.deinit(); // Delete existing domains for this source from appropriate table if (is_allowlist) { var delete_stmt = try self.db.prepare("DELETE FROM allowlist_domains WHERE source_id = ?"); defer delete_stmt.finalize(); try delete_stmt.bindInt(1, source_id); _ = try delete_stmt.step(); } else { var delete_stmt = try self.db.prepare("DELETE FROM denylist_domains WHERE source_id = ?"); defer delete_stmt.finalize(); try delete_stmt.bindInt(1, source_id); _ = try delete_stmt.step(); } // Insert new domains into appropriate table const insert_sql = if (is_allowlist) "INSERT OR IGNORE INTO allowlist_domains (domain, source_id) VALUES (?, ?)" else "INSERT OR IGNORE INTO denylist_domains (domain, source_id) VALUES (?, ?)"; var insert_stmt = try self.db.prepare(insert_sql); defer insert_stmt.finalize(); for (domains) |domain| { insert_stmt.reset(); try insert_stmt.bindText(1, domain); try insert_stmt.bindInt(2, source_id); _ = try insert_stmt.step(); } // Update source metadata with status = updated, including invalid_domains count (Gap 6) var update_stmt = try self.db.prepare( \\UPDATE denylist_sources SET \\ domain_count = ?, \\ invalid_domains = ?, \\ status = ?, \\ content_hash = ?, \\ last_updated = strftime('%s', 'now'), \\ date_modified = strftime('%s', 'now') \\WHERE id = ? ); defer update_stmt.finalize(); try update_stmt.bindInt(1, @intCast(domains.len)); try update_stmt.bindInt(2, @intCast(invalid_count)); try update_stmt.bindInt(3, @intFromEnum(DenylistStatus.updated)); try update_stmt.bindText(4, hash); try update_stmt.bindInt(5, source_id); _ = try update_stmt.step(); try tx.commit(); return domains.len; } fn getSourceType(self: *DenylistFetcher, source_id: i64) schema.SourceType { var stmt = self.db.prepare("SELECT type FROM denylist_sources WHERE id = ?") catch return .denylist; defer stmt.finalize(); stmt.bindInt(1, source_id) catch return .denylist; if (stmt.step() catch false) { return schema.SourceType.fromInt(stmt.getInt(0)); } return .denylist; } fn httpGet(self: *DenylistFetcher, url: []const u8) ![]const u8 { var client = http.Client{ .allocator = self.allocator }; defer client.deinit(); var response_writer = Writer.Allocating.init(self.allocator); errdefer response_writer.deinit(); const result = client.fetch(.{ .location = .{ .url = url }, .response_writer = &response_writer.writer, }) catch |err| { std.log.warn("HTTP fetch failed for {s}: {}", .{ url, err }); return error.FetchFailed; }; if (result.status != .ok) { std.log.warn("HTTP {d} response for {s}", .{ @intFromEnum(result.status), url }); return error.FetchFailed; // errdefer handles cleanup } return response_writer.toOwnedSlice() catch error.FetchFailed; } /// Update all enabled denylists pub fn updateAll(self: *DenylistFetcher) !void { var stmt = try self.db.prepare("SELECT id, url, comment FROM denylist_sources WHERE enabled = 1"); defer stmt.finalize(); while (try stmt.step()) { const id = stmt.getInt(0); const url = stmt.getText(1) orelse continue; const label = stmt.getText(2) orelse url; std.log.info("Updating denylist: {s}", .{label}); const url_copy = try self.allocator.dupe(u8, url); defer self.allocator.free(url_copy); const count = self.fetchAndUpdate(id, url_copy) catch |err| { std.log.warn("Failed to update denylist {s}: {}", .{ label, err }); continue; }; std.log.info("Denylist {s}: {d} domains", .{ label, count }); } // Signal that denylist needs to be reloaded into memory events.signalDenylistReload(); } /// Fetch denylists that have 0 domains (incomplete/interrupted fetches) /// Called at startup to resume any denylists that were mid-fetch when server stopped pub fn fetchIncomplete(self: *DenylistFetcher) !usize { var stmt = try self.db.prepare( "SELECT id, url, comment FROM denylist_sources WHERE enabled = 1 AND domain_count = 0", ); defer stmt.finalize(); var fetched: usize = 0; while (try stmt.step()) { const id = stmt.getInt(0); const url = stmt.getText(1) orelse continue; const label = stmt.getText(2) orelse url; std.log.info("Resuming incomplete denylist: {s}", .{label}); const url_copy = try self.allocator.dupe(u8, url); defer self.allocator.free(url_copy); const count = self.fetchAndUpdate(id, url_copy) catch |err| { std.log.warn("Failed to fetch denylist {s}: {}", .{ label, err }); continue; }; std.log.info("Denylist {s}: {d} domains", .{ label, count }); fetched += 1; } if (fetched > 0) { events.signalDenylistReload(); } return fetched; } }; test "detect hosts format" { const testing = std.testing; const hosts_content = \\# Comment \\0.0.0.0 ads.example.com \\127.0.0.1 tracker.example.com \\0.0.0.0 malware.example.com ; const format = DenylistFormat.detect(hosts_content); try testing.expectEqual(DenylistFormat.hosts, format); } test "detect adblock format" { const testing = std.testing; const adblock_content = \\! AdBlock list \\||ads.example.com^ \\||tracker.example.com^$third-party \\||malware.example.com^ ; const format = DenylistFormat.detect(adblock_content); try testing.expectEqual(DenylistFormat.adblock, format); } test "parse hosts format" { const testing = std.testing; const allocator = testing.allocator; const content = \\# Comment \\0.0.0.0 ads.example.com \\127.0.0.1 tracker.example.com \\0.0.0.0 localhost ; var result = try parseDenylist(content, .hosts, allocator); defer { for (result.domains.items) |d| allocator.free(d); result.domains.deinit(allocator); } try testing.expectEqual(@as(usize, 2), result.domains.items.len); try testing.expectEqualStrings("ads.example.com", result.domains.items[0]); try testing.expectEqualStrings("tracker.example.com", result.domains.items[1]); } test "parse adblock format" { const testing = std.testing; const allocator = testing.allocator; const content = \\! Comment \\||ads.example.com^ \\||tracker.example.com^$third-party ; var result = try parseDenylist(content, .adblock, allocator); defer { for (result.domains.items) |d| allocator.free(d); result.domains.deinit(allocator); } try testing.expectEqual(@as(usize, 2), result.domains.items.len); try testing.expectEqualStrings("ads.example.com", result.domains.items[0]); try testing.expectEqualStrings("tracker.example.com", result.domains.items[1]); } test "parse domain list format" { const testing = std.testing; const allocator = testing.allocator; const content = \\# Comment \\ads.example.com \\tracker.example.com # inline comment \\malware.example.com ; var result = try parseDenylist(content, .domains, allocator); defer { for (result.domains.items) |d| allocator.free(d); result.domains.deinit(allocator); } try testing.expectEqual(@as(usize, 3), result.domains.items.len); } test "parse tracks invalid domains" { const testing = std.testing; const allocator = testing.allocator; const content = \\# Comment \\valid.example.com \\-invalid-start.com \\also.valid.org \\toolong12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345.com ; var result = try parseDenylist(content, .domains, allocator); defer { for (result.domains.items) |d| allocator.free(d); result.domains.deinit(allocator); } try testing.expectEqual(@as(usize, 2), result.domains.items.len); try testing.expectEqual(@as(usize, 2), result.invalid_count); }