mokhtar/nxdns
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

initial commit

Browse Source
This commit is contained in:
mokhtar
2025-12-26 18:42:04 +01:00
commit d8d9ddfc53
52 changed files with 16863 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/filter/fetcher.zig
+549
View File
@@ -0,0 +1,549 @@
const std = @import("std");
const http = std.http;
const Allocator = std.mem.Allocator;
const Writer = std.Io.Writer;
const Database = @import("../storage/db.zig").Database;
const schema = @import("../storage/schema.zig");
const events = @import("../events.zig");
const DenylistStatus = schema.DenylistStatus;
/// Maximum output size for denylist fetch (100MB)
const MAX_DENYLIST_SIZE = 100 * 1024 * 1024;
/// Denylist format types
pub const DenylistFormat = enum {
hosts, // 0.0.0.0 domain.com or 127.0.0.1 domain.com
domains, // one domain per line
adblock, // ||domain.com^
/// Detect format from content sample
pub fn detect(content: []const u8) DenylistFormat {
var lines = std.mem.splitScalar(u8, content, '\n');
var hosts_count: usize = 0;
var adblock_count: usize = 0;
var checked: usize = 0;
while (lines.next()) |line| {
const trimmed = std.mem.trim(u8, line, " \t\r");
if (trimmed.len == 0 or trimmed[0] == '#') continue;
if (std.mem.startsWith(u8, trimmed, "||") and std.mem.endsWith(u8, trimmed, "^")) {
adblock_count += 1;
} else if (std.mem.startsWith(u8, trimmed, "0.0.0.0 ") or
std.mem.startsWith(u8, trimmed, "127.0.0.1 "))
{
hosts_count += 1;
}
checked += 1;
if (checked >= 50) break;
}
if (adblock_count > hosts_count) return .adblock;
if (hosts_count > 0) return .hosts;
return .domains;
}
};
/// Result of parsing a denylist
pub const ParseResult = struct {
domains: std.ArrayListUnmanaged([]const u8),
invalid_count: usize,
};
/// Parse denylist content and return domains plus count of invalid lines
pub fn parseDenylist(content: []const u8, format: DenylistFormat, allocator: Allocator) !ParseResult {
var domains = std.ArrayListUnmanaged([]const u8){};
errdefer {
for (domains.items) |d| allocator.free(d);
domains.deinit(allocator);
}
var invalid_count: usize = 0;
var lines = std.mem.splitScalar(u8, content, '\n');
while (lines.next()) |line| {
const trimmed = std.mem.trim(u8, line, " \t\r");
if (trimmed.len == 0 or trimmed[0] == '#' or trimmed[0] == '!') continue;
const domain = switch (format) {
.hosts => parseHostsLine(trimmed),
.domains => parseDomainLine(trimmed),
.adblock => parseAdblockLine(trimmed),
};
if (domain) |d| {
if (isValidDomain(d)) {
const copy = try allocator.dupe(u8, d);
try domains.append(allocator, copy);
} else {
invalid_count += 1;
}
} else {
// Line matched format but couldn't be parsed (e.g., wrong IP in hosts)
invalid_count += 1;
}
}
return .{ .domains = domains, .invalid_count = invalid_count };
}
fn parseHostsLine(line: []const u8) ?[]const u8 {
// Format: 0.0.0.0 domain.com or 127.0.0.1 domain.com
var parts = std.mem.splitAny(u8, line, " \t");
const ip = parts.first();
if (!std.mem.eql(u8, ip, "0.0.0.0") and !std.mem.eql(u8, ip, "127.0.0.1")) {
return null;
}
const domain = parts.next() orelse return null;
// Skip localhost entries
if (std.mem.eql(u8, domain, "localhost") or
std.mem.eql(u8, domain, "localhost.localdomain") or
std.mem.eql(u8, domain, "local") or
std.mem.eql(u8, domain, "broadcasthost"))
{
return null;
}
return domain;
}
fn parseDomainLine(line: []const u8) ?[]const u8 {
// Just the domain, skip comments
if (line.len == 0) return null;
// Check for inline comments
var domain = line;
if (std.mem.indexOf(u8, line, "#")) |idx| {
domain = std.mem.trim(u8, line[0..idx], " \t");
}
if (domain.len == 0) return null;
// Handle wildcard format: *.domain.com -> domain.com
if (std.mem.startsWith(u8, domain, "*.")) {
domain = domain[2..];
}
return if (domain.len > 0) domain else null;
}
fn parseAdblockLine(line: []const u8) ?[]const u8 {
// Format: ||domain.com^ or ||domain.com^$...
if (!std.mem.startsWith(u8, line, "||")) return null;
var end = line.len;
// Find the end marker (^ or $)
if (std.mem.indexOf(u8, line[2..], "^")) |idx| {
end = idx + 2;
} else if (std.mem.indexOf(u8, line[2..], "$")) |idx| {
end = idx + 2;
}
const domain = line[2..end];
return if (domain.len > 0) domain else null;
}
fn isValidDomain(domain: []const u8) bool {
if (domain.len == 0 or domain.len > 253) return false;
// Must contain at least one dot (except for special cases)
if (std.mem.indexOf(u8, domain, ".") == null) return false;
// Check for valid characters
for (domain) |c| {
if (!std.ascii.isAlphanumeric(c) and c != '.' and c != '-' and c != '_') {
return false;
}
}
// Don't start or end with hyphen or dot
if (domain[0] == '-' or domain[0] == '.') return false;
if (domain[domain.len - 1] == '-' or domain[domain.len - 1] == '.') return false;
return true;
}
/// Denylist fetcher for downloading and updating denylists
pub const DenylistFetcher = struct {
db: *Database,
allocator: Allocator,
pub fn init(db: *Database, allocator: Allocator) DenylistFetcher {
return DenylistFetcher{
.db = db,
.allocator = allocator,
};
}
/// Fetch and update a denylist from URL
pub fn fetchAndUpdate(self: *DenylistFetcher, source_id: i64, url: []const u8) !usize {
// Fetch content via HTTP
const content = self.httpGet(url) catch |err| {
// Fetch failed - check if we have cached data
const has_cached = self.hasCachedDomains(source_id);
const status: DenylistStatus = if (has_cached) .cached else .failed;
self.updateStatus(source_id, status);
return err;
};
defer self.allocator.free(content);
// Compute content hash
var hasher = std.hash.XxHash3.init(0);
hasher.update(content);
const hash = hasher.final();
var hash_str: [16]u8 = undefined;
_ = std.fmt.bufPrint(&hash_str, "{x:0>16}", .{hash}) catch unreachable;
// Check if content changed
const old_hash = self.getContentHash(source_id);
if (old_hash) |h| {
if (std.mem.eql(u8, &hash_str, h)) {
// Content unchanged
self.updateStatusAndHash(source_id, .unchanged, &hash_str);
return self.getDomainCount(source_id);
}
}
// Detect format
const format = DenylistFormat.detect(content);
// Parse domains
var result = try parseDenylist(content, format, self.allocator);
defer {
for (result.domains.items) |d| self.allocator.free(d);
result.domains.deinit(self.allocator);
}
// Update database with new domains and invalid count
const count = try self.updateDatabase(source_id, result.domains.items, result.invalid_count, &hash_str);
return count;
}
fn hasCachedDomains(self: *DenylistFetcher, source_id: i64) bool {
var stmt = self.db.prepare("SELECT domain_count FROM denylist_sources WHERE id = ?") catch return false;
defer stmt.finalize();
stmt.bindInt(1, source_id) catch return false;
if (stmt.step() catch false) {
return stmt.getInt(0) > 0;
}
return false;
}
fn getContentHash(self: *DenylistFetcher, source_id: i64) ?[]const u8 {
var stmt = self.db.prepare("SELECT content_hash FROM denylist_sources WHERE id = ?") catch return null;
defer stmt.finalize();
stmt.bindInt(1, source_id) catch return null;
if (stmt.step() catch false) {
return stmt.getText(0);
}
return null;
}
fn getDomainCount(self: *DenylistFetcher, source_id: i64) usize {
var stmt = self.db.prepare("SELECT domain_count FROM denylist_sources WHERE id = ?") catch return 0;
defer stmt.finalize();
stmt.bindInt(1, source_id) catch return 0;
if (stmt.step() catch false) {
const count = stmt.getInt(0);
return if (count >= 0) @intCast(count) else 0;
}
return 0;
}
fn updateStatus(self: *DenylistFetcher, source_id: i64, status: DenylistStatus) void {
var stmt = self.db.prepare("UPDATE denylist_sources SET status = ? WHERE id = ?") catch return;
defer stmt.finalize();
stmt.bindInt(1, @intFromEnum(status)) catch return;
stmt.bindInt(2, source_id) catch return;
_ = stmt.step() catch {};
}
fn updateStatusAndHash(self: *DenylistFetcher, source_id: i64, status: DenylistStatus, hash: []const u8) void {
var stmt = self.db.prepare(
"UPDATE denylist_sources SET status = ?, content_hash = ?, last_updated = strftime('%s', 'now') WHERE id = ?",
) catch return;
defer stmt.finalize();
stmt.bindInt(1, @intFromEnum(status)) catch return;
stmt.bindText(2, hash) catch return;
stmt.bindInt(3, source_id) catch return;
_ = stmt.step() catch {};
}
fn updateDatabase(self: *DenylistFetcher, source_id: i64, domains: []const []const u8, invalid_count: usize, hash: []const u8) !usize {
// Get source type to determine which table to use (Gap 5)
const source_type = self.getSourceType(source_id);
const is_allowlist = source_type == .allowlist;
// Start transaction with RAII - auto-rollback if not committed
var tx = try self.db.begin();
defer tx.deinit();
// Delete existing domains for this source from appropriate table
if (is_allowlist) {
var delete_stmt = try self.db.prepare("DELETE FROM allowlist_domains WHERE source_id = ?");
defer delete_stmt.finalize();
try delete_stmt.bindInt(1, source_id);
_ = try delete_stmt.step();
} else {
var delete_stmt = try self.db.prepare("DELETE FROM denylist_domains WHERE source_id = ?");
defer delete_stmt.finalize();
try delete_stmt.bindInt(1, source_id);
_ = try delete_stmt.step();
}
// Insert new domains into appropriate table
const insert_sql = if (is_allowlist)
"INSERT OR IGNORE INTO allowlist_domains (domain, source_id) VALUES (?, ?)"
else
"INSERT OR IGNORE INTO denylist_domains (domain, source_id) VALUES (?, ?)";
var insert_stmt = try self.db.prepare(insert_sql);
defer insert_stmt.finalize();
for (domains) |domain| {
insert_stmt.reset();
try insert_stmt.bindText(1, domain);
try insert_stmt.bindInt(2, source_id);
_ = try insert_stmt.step();
}
// Update source metadata with status = updated, including invalid_domains count (Gap 6)
var update_stmt = try self.db.prepare(
\\UPDATE denylist_sources SET
\\ domain_count = ?,
\\ invalid_domains = ?,
\\ status = ?,
\\ content_hash = ?,
\\ last_updated = strftime('%s', 'now'),
\\ date_modified = strftime('%s', 'now')
\\WHERE id = ?
);
defer update_stmt.finalize();
try update_stmt.bindInt(1, @intCast(domains.len));
try update_stmt.bindInt(2, @intCast(invalid_count));
try update_stmt.bindInt(3, @intFromEnum(DenylistStatus.updated));
try update_stmt.bindText(4, hash);
try update_stmt.bindInt(5, source_id);
_ = try update_stmt.step();
try tx.commit();
return domains.len;
}
fn getSourceType(self: *DenylistFetcher, source_id: i64) schema.SourceType {
var stmt = self.db.prepare("SELECT type FROM denylist_sources WHERE id = ?") catch return .denylist;
defer stmt.finalize();
stmt.bindInt(1, source_id) catch return .denylist;
if (stmt.step() catch false) {
return schema.SourceType.fromInt(stmt.getInt(0));
}
return .denylist;
}
fn httpGet(self: *DenylistFetcher, url: []const u8) ![]const u8 {
var client = http.Client{ .allocator = self.allocator };
defer client.deinit();
var response_writer = Writer.Allocating.init(self.allocator);
errdefer response_writer.deinit();
const result = client.fetch(.{
.location = .{ .url = url },
.response_writer = &response_writer.writer,
}) catch |err| {
std.log.warn("HTTP fetch failed for {s}: {}", .{ url, err });
return error.FetchFailed;
};
if (result.status != .ok) {
std.log.warn("HTTP {d} response for {s}", .{ @intFromEnum(result.status), url });
return error.FetchFailed; // errdefer handles cleanup
}
return response_writer.toOwnedSlice() catch error.FetchFailed;
}
/// Update all enabled denylists
pub fn updateAll(self: *DenylistFetcher) !void {
var stmt = try self.db.prepare("SELECT id, url, comment FROM denylist_sources WHERE enabled = 1");
defer stmt.finalize();
while (try stmt.step()) {
const id = stmt.getInt(0);
const url = stmt.getText(1) orelse continue;
const label = stmt.getText(2) orelse url;
std.log.info("Updating denylist: {s}", .{label});
const url_copy = try self.allocator.dupe(u8, url);
defer self.allocator.free(url_copy);
const count = self.fetchAndUpdate(id, url_copy) catch |err| {
std.log.warn("Failed to update denylist {s}: {}", .{ label, err });
continue;
};
std.log.info("Denylist {s}: {d} domains", .{ label, count });
}
// Signal that denylist needs to be reloaded into memory
events.signalDenylistReload();
}
/// Fetch denylists that have 0 domains (incomplete/interrupted fetches)
/// Called at startup to resume any denylists that were mid-fetch when server stopped
pub fn fetchIncomplete(self: *DenylistFetcher) !usize {
var stmt = try self.db.prepare(
"SELECT id, url, comment FROM denylist_sources WHERE enabled = 1 AND domain_count = 0",
);
defer stmt.finalize();
var fetched: usize = 0;
while (try stmt.step()) {
const id = stmt.getInt(0);
const url = stmt.getText(1) orelse continue;
const label = stmt.getText(2) orelse url;
std.log.info("Resuming incomplete denylist: {s}", .{label});
const url_copy = try self.allocator.dupe(u8, url);
defer self.allocator.free(url_copy);
const count = self.fetchAndUpdate(id, url_copy) catch |err| {
std.log.warn("Failed to fetch denylist {s}: {}", .{ label, err });
continue;
};
std.log.info("Denylist {s}: {d} domains", .{ label, count });
fetched += 1;
}
if (fetched > 0) {
events.signalDenylistReload();
}
return fetched;
}
};
test "detect hosts format" {
const testing = std.testing;
const hosts_content =
\\# Comment
\\0.0.0.0 ads.example.com
\\127.0.0.1 tracker.example.com
\\0.0.0.0 malware.example.com
;
const format = DenylistFormat.detect(hosts_content);
try testing.expectEqual(DenylistFormat.hosts, format);
}
test "detect adblock format" {
const testing = std.testing;
const adblock_content =
\\! AdBlock list
\\||ads.example.com^
\\||tracker.example.com^$third-party
\\||malware.example.com^
;
const format = DenylistFormat.detect(adblock_content);
try testing.expectEqual(DenylistFormat.adblock, format);
}
test "parse hosts format" {
const testing = std.testing;
const allocator = testing.allocator;
const content =
\\# Comment
\\0.0.0.0 ads.example.com
\\127.0.0.1 tracker.example.com
\\0.0.0.0 localhost
;
var result = try parseDenylist(content, .hosts, allocator);
defer {
for (result.domains.items) |d| allocator.free(d);
result.domains.deinit(allocator);
}
try testing.expectEqual(@as(usize, 2), result.domains.items.len);
try testing.expectEqualStrings("ads.example.com", result.domains.items[0]);
try testing.expectEqualStrings("tracker.example.com", result.domains.items[1]);
}
test "parse adblock format" {
const testing = std.testing;
const allocator = testing.allocator;
const content =
\\! Comment
\\||ads.example.com^
\\||tracker.example.com^$third-party
;
var result = try parseDenylist(content, .adblock, allocator);
defer {
for (result.domains.items) |d| allocator.free(d);
result.domains.deinit(allocator);
}
try testing.expectEqual(@as(usize, 2), result.domains.items.len);
try testing.expectEqualStrings("ads.example.com", result.domains.items[0]);
try testing.expectEqualStrings("tracker.example.com", result.domains.items[1]);
}
test "parse domain list format" {
const testing = std.testing;
const allocator = testing.allocator;
const content =
\\# Comment
\\ads.example.com
\\tracker.example.com # inline comment
\\malware.example.com
;
var result = try parseDenylist(content, .domains, allocator);
defer {
for (result.domains.items) |d| allocator.free(d);
result.domains.deinit(allocator);
}
try testing.expectEqual(@as(usize, 3), result.domains.items.len);
}
test "parse tracks invalid domains" {
const testing = std.testing;
const allocator = testing.allocator;
const content =
\\# Comment
\\valid.example.com
\\-invalid-start.com
\\also.valid.org
\\toolong12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345.com
;
var result = try parseDenylist(content, .domains, allocator);
defer {
for (result.domains.items) |d| allocator.free(d);
result.domains.deinit(allocator);
}
try testing.expectEqual(@as(usize, 2), result.domains.items.len);
try testing.expectEqual(@as(usize, 2), result.invalid_count);
}