milestone 24: persist and surface the unsupported-line count

src/storage/repositories/sources_repo.zig

@@ -1,9 +1,10 @@
 //! `blocklist_sources`.
 //!
 //! Only the four configuration columns are read and written. `last_updated`,
-//! `domain_count`, `wildcard_count`, `exception_count`, `skipped_regex_count`
-//! and `checksum` are facts a running server produces; an insert leaves them at
-//! their column defaults so two exports taken minutes apart stay identical.
+//! `domain_count`, `wildcard_count`, `exception_count`, `skipped_regex_count`,
+//! `skipped_unsupported_count` and `checksum` are facts a running server
+//! produces; an insert leaves them at their column defaults so two exports taken
+//! minutes apart stay identical.
 //!
 //! The import path is list / insert / deleteAll / count; the runtime columns and
 //! the REST surface follow it, both keyed by row id.
@@ -93,6 +94,12 @@ pub const SourceRow = struct {
     /// builds `SourceRow` values from the refresh columns alone.
     exception_count: i64 = 0,
     skipped_regex_count: i64,
+    /// Lines the compiler read and could not translate into a DNS decision:
+    /// cosmetic element hiding, `$`-modifier rules (save the tolerated
+    /// `$important` exception suffix, which lands in `exception_count`), scheme
+    /// anchors. Counted and not written, like `skipped_regex_count` and unlike
+    /// the three counts above.
+    skipped_unsupported_count: i64,
     checksum: ?[]const u8,
 };
 
@@ -102,6 +109,7 @@ pub const SourceStats = struct {
     wildcard_count: i64,
     exception_count: i64,
     skipped_regex_count: i64,
+    skipped_unsupported_count: i64,
     /// Lowercase hex sha256 over the `.list` body, then the `.wild` body, then
     /// the `.allow` body. The allow body is hashed last so an empty one leaves
     /// the digest of a two-body compile unchanged, which is what keeps a
@@ -112,7 +120,7 @@ pub const SourceStats = struct {
 const row_columns_sql =
     \\SELECT id, url, name, enabled, last_updated,
     \\       domain_count, wildcard_count, skipped_regex_count, checksum,
-    \\       is_suggested, exception_count
+    \\       is_suggested, exception_count, skipped_unsupported_count
     \\  FROM blocklist_sources
 ;
 
@@ -143,6 +151,7 @@ fn readSourceRow(stmt: *db.Stmt, gpa: Allocator) db.Error!SourceRow {
         .wildcard_count = stmt.columnInt(6),
         .exception_count = stmt.columnInt(10),
         .skipped_regex_count = stmt.columnInt(7),
+        .skipped_unsupported_count = stmt.columnInt(11),
         .checksum = checksum,
     };
 }
@@ -159,7 +168,8 @@ pub fn freeSourceRows(gpa: Allocator, items: []const SourceRow) void {
 const update_stats_sql =
     \\UPDATE blocklist_sources
     \\   SET last_updated = ?2, domain_count = ?3, wildcard_count = ?4,
-    \\       skipped_regex_count = ?5, checksum = ?6, exception_count = ?7
+    \\       skipped_regex_count = ?5, checksum = ?6, exception_count = ?7,
+    \\       skipped_unsupported_count = ?8
     \\ WHERE id = ?1
 ;
 
@@ -175,6 +185,7 @@ pub fn updateSourceStats(database: *db.Db, id: i64, stats: SourceStats) db.Error
     try stmt.bindInt(5, stats.skipped_regex_count);
     try stmt.bindText(6, stats.checksum);
     try stmt.bindInt(7, stats.exception_count);
+    try stmt.bindInt(8, stats.skipped_unsupported_count);
     try stmt.exec();
 }
 
@@ -315,8 +326,11 @@ test "insertBlocklistSource leaves the runtime columns at their defaults" {
     try testing.expectEqual(
         @as(i64, 0),
         try database.queryInt(
-            "SELECT sum(domain_count + wildcard_count + exception_count + skipped_regex_count)" ++
-                " FROM blocklist_sources",
+            // Every runtime counter, summed only because this asserts they are
+            // all 0. No production query may add the two skip counters to the
+            // three written ones: a skipped line was never written.
+            "SELECT sum(domain_count + wildcard_count + exception_count + skipped_regex_count" ++
+                " + skipped_unsupported_count) FROM blocklist_sources",
         ),
     );
 }
@@ -370,6 +384,7 @@ test "listSourceRows returns row ids and the runtime columns in url order" {
         try testing.expectEqual(@as(i64, 0), row.wildcard_count);
         try testing.expectEqual(@as(i64, 0), row.exception_count);
         try testing.expectEqual(@as(i64, 0), row.skipped_regex_count);
+        try testing.expectEqual(@as(i64, 0), row.skipped_unsupported_count);
     }
 }
 
@@ -389,6 +404,7 @@ test "updateSourceStats writes the runtime columns of one source only" {
         .wildcard_count = 21,
         .exception_count = 9,
         .skipped_regex_count = 7,
+        .skipped_unsupported_count = 15,
         .checksum = "a" ** 64,
     });
 
@@ -406,6 +422,7 @@ test "updateSourceStats writes the runtime columns of one source only" {
     try testing.expectEqual(@as(i64, 21), updated.wildcard_count);
     try testing.expectEqual(@as(i64, 9), updated.exception_count);
     try testing.expectEqual(@as(i64, 7), updated.skipped_regex_count);
+    try testing.expectEqual(@as(i64, 15), updated.skipped_unsupported_count);
     try testing.expectEqualStrings("a" ** 64, updated.checksum.?);
 
     // The two untouched rows kept their defaults.
@@ -423,6 +440,7 @@ fn listSourceRowsUnderFailure(gpa: Allocator) !void {
         .wildcard_count = 3,
         .exception_count = 5,
         .skipped_regex_count = 4,
+        .skipped_unsupported_count = 6,
         .checksum = "b" ** 64,
     });
 
@@ -489,6 +507,7 @@ test "updateSource leaves the runtime columns where the refresh path left them"
         .wildcard_count = 3,
         .exception_count = 2,
         .skipped_regex_count = 1,
+        .skipped_unsupported_count = 4,
         .checksum = "c" ** 64,
     });