From dc9d1c1399abd82d7eed62d424d19ac826f4ab21 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Tue, 12 May 2026 02:56:12 +0400 Subject: [PATCH 01/28] Refactor is bad filter (0.13.x) (#641) The PR: * removes IS_BADFILTER from NetworkFilterMask * adds NetworkFilterFeatureMask to store parse-only features * rids off get_id_without_badfilter in favor of get_id() * removes impl NetworkFilterMaskHelper for CheckedResult --- src/blocker.rs | 14 +++++------ src/content_blocking.rs | 7 ++++-- src/filters/fb_network_builder.rs | 2 +- src/filters/network.rs | 41 ++++++++++++++++--------------- src/filters/network_matchers.rs | 4 --- src/lists.rs | 3 +-- src/network_filter_list.rs | 11 +-------- 7 files changed, 36 insertions(+), 46 deletions(-) diff --git a/src/blocker.rs b/src/blocker.rs index a7ac53b3..bef17c90 100644 --- a/src/blocker.rs +++ b/src/blocker.rs @@ -211,7 +211,7 @@ impl Blocker { } None => None, // If matched an important filter, exceptions don't atter - Some(f) if f.is_important() => None, + Some(f) if f.filter_mask.is_important() => None, Some(_) => self .exceptions() .check(request, &self.tags_enabled, &mut regex_manager), @@ -227,7 +227,7 @@ impl Blocker { let redirect_resource = { let mut exceptions = vec![]; for redirect_filter in redirect_filters.iter() { - if redirect_filter.is_exception() { + if redirect_filter.filter_mask.is_exception() { if let Some(redirect) = redirect_filter.modifier_option.as_ref() { exceptions.push(redirect); } @@ -235,7 +235,7 @@ impl Blocker { } let mut resource_and_priority = None; for redirect_filter in redirect_filters.iter() { - if !redirect_filter.is_exception() { + if !redirect_filter.filter_mask.is_exception() { if let Some(redirect) = redirect_filter.modifier_option.as_ref() { if !exceptions.contains(&redirect) { // parse redirect + priority @@ -278,7 +278,7 @@ impl Blocker { let important = filter.is_some() && filter .as_ref() - .map(|f| f.is_important()) + .map(|f| f.filter_mask.is_important()) .unwrap_or_else(|| false); let rewritten_url = if important { @@ -410,8 +410,8 @@ impl Blocker { let mut enabled_directives: HashSet<&str> = HashSet::new(); for filter in filters.iter() { - if filter.is_exception() { - if filter.is_csp() { + if filter.filter_mask.is_exception() { + if filter.filter_mask.is_csp() { if let Some(csp_directive) = &filter.modifier_option { disabled_directives.insert(csp_directive); } else { @@ -420,7 +420,7 @@ impl Blocker { return None; } } - } else if filter.is_csp() { + } else if filter.filter_mask.is_csp() { if let Some(csp_directive) = &filter.modifier_option { enabled_directives.insert(csp_directive); } diff --git a/src/content_blocking.rs b/src/content_blocking.rs index 128d202f..5dbff53d 100644 --- a/src/content_blocking.rs +++ b/src/content_blocking.rs @@ -1,7 +1,9 @@ //! Transforms filter rules into content blocking syntax used on iOS and MacOS. use crate::filters::cosmetic::CosmeticFilter; -use crate::filters::network::{NetworkFilter, NetworkFilterMask, NetworkFilterMaskHelper}; +use crate::filters::network::{ + NetworkFilter, NetworkFilterFeaturesMask, NetworkFilterMask, NetworkFilterMaskHelper, +}; use crate::lists::ParsedFilter; use memchr::{memchr as find_char, memmem}; @@ -319,7 +321,8 @@ impl TryFrom for CbRuleEquivalent { return Err(CbRuleCreationFailure::NetworkGenerichideUnsupported); } debug_assert!( - !v.mask.contains(NetworkFilterMask::BAD_FILTER), + !v.features_mask + .contains(NetworkFilterFeaturesMask::BAD_FILTER), "BAD_FILTER should be filtered out" ); if v.is_csp() { diff --git a/src/filters/fb_network_builder.rs b/src/filters/fb_network_builder.rs index 115bcbcc..6f54ae8b 100644 --- a/src/filters/fb_network_builder.rs +++ b/src/filters/fb_network_builder.rs @@ -228,7 +228,7 @@ impl NetworkRulesBuilder { // Collect badfilter ids in advance. for filter in network_filters.iter() { if filter.is_badfilter() { - badfilter_ids.insert(filter.get_id_without_badfilter()); + badfilter_ids.insert(filter.get_id()); } } diff --git a/src/filters/network.rs b/src/filters/network.rs index 9bd66729..16eee562 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -21,6 +21,16 @@ use crate::utils::{self, Hash, TokensBuffer}; /// For now, only support `$removeparam` with simple alphanumeric/dash/underscore patterns. static VALID_PARAM: LazyLock = LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9_\-]+$").unwrap()); +bitflags::bitflags! { + /// Features that are properties used to classify the filter, but not stored + /// in the flatbuffer serialized format. For that reason, not available + /// for FlatNetworkFilter (use NetworkFilterMask instead). + #[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Default)] + pub struct NetworkFilterFeaturesMask: u32 { + const BAD_FILTER = 1 << 0; + } +} + #[non_exhaustive] #[derive(Debug, Error, PartialEq, Clone)] pub enum NetworkFilterError { @@ -101,7 +111,6 @@ bitflags::bitflags! { const THIRD_PARTY = 1 << 16; const FIRST_PARTY = 1 << 17; const IS_REDIRECT = 1 << 26; - const BAD_FILTER = 1 << 27; const GENERIC_HIDE = 1 << 30; // Full document rules are not implied by negated types. @@ -202,11 +211,6 @@ pub trait NetworkFilterMaskHelper { self.has_flag(NetworkFilterMask::ALSO_BLOCK_REDIRECT) } - #[inline] - fn is_badfilter(&self) -> bool { - self.has_flag(NetworkFilterMask::BAD_FILTER) - } - #[inline] fn is_generic_hide(&self) -> bool { self.has_flag(NetworkFilterMask::GENERIC_HIDE) @@ -380,6 +384,7 @@ impl FilterPart { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NetworkFilter { pub mask: NetworkFilterMask, + pub features_mask: NetworkFilterFeaturesMask, pub filter: FilterPart, pub opt_domains: Option>, pub opt_not_domains: Option>, @@ -446,6 +451,7 @@ impl NetworkFilter { | NetworkFilterMask::FIRST_PARTY | NetworkFilterMask::FROM_HTTPS | NetworkFilterMask::FROM_HTTP; + let mut features_mask: NetworkFilterFeaturesMask = Default::default(); // Temporary masks for positive (e.g.: $script) and negative (e.g.: $~script) // content type options. @@ -505,7 +511,9 @@ impl NetworkFilter { opt_not_domains = Some(opt_not_domains_array); } } - NetworkFilterOption::Badfilter => mask.set(NetworkFilterMask::BAD_FILTER, true), + NetworkFilterOption::Badfilter => { + features_mask.set(NetworkFilterFeaturesMask::BAD_FILTER, true) + } NetworkFilterOption::Important => { mask.set(NetworkFilterMask::IS_IMPORTANT, true) } @@ -802,6 +810,7 @@ impl NetworkFilter { }, hostname: hostname_decoded?, mask, + features_mask, opt_domains, opt_not_domains, tag, @@ -838,19 +847,6 @@ impl NetworkFilter { NetworkFilter::parse(&rule, debug, Default::default()) } - pub fn get_id_without_badfilter(&self) -> Hash { - let mut mask = self.mask; - mask.set(NetworkFilterMask::BAD_FILTER, false); - compute_filter_id( - self.modifier_option.as_deref(), - mask, - self.filter.string_view().as_deref(), - self.hostname.as_deref(), - self.opt_domains.as_ref(), - self.opt_not_domains.as_ref(), - ) - } - pub fn get_id(&self) -> Hash { compute_filter_id( self.modifier_option.as_deref(), @@ -937,6 +933,11 @@ impl NetworkFilter { } } + pub fn is_badfilter(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::BAD_FILTER) + } + #[cfg(test)] pub(crate) fn matches_test(&self, request: &request::Request) -> bool { let filter_set = crate::FilterSet::new_with_rules(vec![self.clone()], vec![], true); diff --git a/src/filters/network_matchers.rs b/src/filters/network_matchers.rs index 6d640a17..aac604ac 100644 --- a/src/filters/network_matchers.rs +++ b/src/filters/network_matchers.rs @@ -404,10 +404,6 @@ where #[inline] pub fn check_options(mask: NetworkFilterMask, request: &request::Request) -> bool { - // Bad filter never matches - if mask.is_badfilter() { - return false; - } // We first discard requests based on type, protocol and party. This is really // cheap and should be done first. if !mask.check_cpt_allowed(&request.request_type) diff --git a/src/lists.rs b/src/lists.rs index 857b41e8..c52ce660 100644 --- a/src/lists.rs +++ b/src/lists.rs @@ -285,7 +285,6 @@ impl FilterSet { self, ) -> Result<(Vec, Vec), ()> { use crate::content_blocking; - use crate::filters::network::NetworkFilterMaskHelper; use std::collections::HashSet; if !self.debug { @@ -296,7 +295,7 @@ impl FilterSet { let mut bad_filter_ids = HashSet::new(); for filter in self.network_filters.iter() { if filter.is_badfilter() { - bad_filter_ids.insert(filter.get_id_without_badfilter()); + bad_filter_ids.insert(filter.get_id()); } } diff --git a/src/network_filter_list.rs b/src/network_filter_list.rs index 4e642061..01e67f94 100644 --- a/src/network_filter_list.rs +++ b/src/network_filter_list.rs @@ -7,9 +7,7 @@ use flatbuffers::ForwardsUOffset; use crate::filters::fb_network::FlatNetworkFilter; use crate::filters::filter_data_context::FilterDataContext; use crate::filters::flatbuffer_generated::fb; -use crate::filters::network::{ - NetworkFilter, NetworkFilterMask, NetworkFilterMaskHelper, NetworkMatchable, -}; +use crate::filters::network::{NetworkFilter, NetworkFilterMask, NetworkMatchable}; use crate::flatbuffers::containers::flat_multimap::FlatMultiMapView; use crate::flatbuffers::unsafe_tools::fb_vector_to_slice; use crate::regex_manager::RegexManager; @@ -44,13 +42,6 @@ impl fmt::Display for CheckResult { } } -impl NetworkFilterMaskHelper for CheckResult { - #[inline] - fn has_flag(&self, v: NetworkFilterMask) -> bool { - self.filter_mask.contains(v) - } -} - /// Internal structure to keep track of a collection of network filters. pub(crate) struct NetworkFilterList<'a> { pub(crate) list: fb::NetworkFilterList<'a>, From dc8018797fec7eaef5d475a321419c7b4e2e570c Mon Sep 17 00:00:00 2001 From: Mikhail Date: Wed, 13 May 2026 01:13:00 +0400 Subject: [PATCH 02/28] Refactor more mask items (0.13.x) (#642) The PR refactors `NetworkFilterMask` by moving some parse-only bits to `NetworkFilterFeaturesMask`. * Refactor IS_REMOVEPARAM * Refactor IS_GENERIC_HIDE * Refactor IS_CSP * Refactor IS_REDIRECT + ALSO_BLOCK_REDIRECT * Fix compilation and tests * remove unused import * Add feature_mask to get_id() --- benches/bench_redirect_performance.rs | 2 +- src/blocker.rs | 16 ++-- src/content_blocking.rs | 6 +- src/data_format/mod.rs | 2 +- src/filters/network.rs | 103 ++++++++++++++------------ tests/unit/engine.rs | 8 +- 6 files changed, 71 insertions(+), 66 deletions(-) diff --git a/benches/bench_redirect_performance.rs b/benches/bench_redirect_performance.rs index 1ef2dbc2..56da371f 100644 --- a/benches/bench_redirect_performance.rs +++ b/benches/bench_redirect_performance.rs @@ -2,7 +2,7 @@ use adblock::{Engine, FilterSet}; use criterion::*; use tokio::runtime::Runtime; -use adblock::filters::network::{NetworkFilter, NetworkFilterMask, NetworkFilterMaskHelper}; +use adblock::filters::network::{NetworkFilter, NetworkFilterMask}; use adblock::request::Request; use adblock::resources::Resource; diff --git a/src/blocker.rs b/src/blocker.rs index bef17c90..19a41c33 100644 --- a/src/blocker.rs +++ b/src/blocker.rs @@ -411,16 +411,14 @@ impl Blocker { for filter in filters.iter() { if filter.filter_mask.is_exception() { - if filter.filter_mask.is_csp() { - if let Some(csp_directive) = &filter.modifier_option { - disabled_directives.insert(csp_directive); - } else { - // Exception filters with empty `csp` options will disable all CSP - // injections for matching pages. - return None; - } + if let Some(csp_directive) = &filter.modifier_option { + disabled_directives.insert(csp_directive); + } else { + // Exception filters with empty `csp` options will disable all CSP + // injections for matching pages. + return None; } - } else if filter.filter_mask.is_csp() { + } else { if let Some(csp_directive) = &filter.modifier_option { enabled_directives.insert(csp_directive); } diff --git a/src/content_blocking.rs b/src/content_blocking.rs index 5dbff53d..fd4fe905 100644 --- a/src/content_blocking.rs +++ b/src/content_blocking.rs @@ -1,9 +1,7 @@ //! Transforms filter rules into content blocking syntax used on iOS and MacOS. use crate::filters::cosmetic::CosmeticFilter; -use crate::filters::network::{ - NetworkFilter, NetworkFilterFeaturesMask, NetworkFilterMask, NetworkFilterMaskHelper, -}; +use crate::filters::network::{NetworkFilter, NetworkFilterFeaturesMask, NetworkFilterMask}; use crate::lists::ParsedFilter; use memchr::{memchr as find_char, memmem}; @@ -317,7 +315,7 @@ impl TryFrom for CbRuleEquivalent { if v.is_redirect() { return Err(CbRuleCreationFailure::NetworkRedirectUnsupported); } - if v.mask.contains(NetworkFilterMask::GENERIC_HIDE) { + if v.is_generic_hide() { return Err(CbRuleCreationFailure::NetworkGenerichideUnsupported); } debug_assert!( diff --git a/src/data_format/mod.rs b/src/data_format/mod.rs index 35d0ff94..fb3ffebd 100644 --- a/src/data_format/mod.rs +++ b/src/data_format/mod.rs @@ -17,7 +17,7 @@ const ADBLOCK_RUST_DAT_MAGIC: [u8; 4] = [0xd1, 0xd9, 0x3a, 0xaf]; /// The version of the data format. /// If the data format version is incremented, the data is considered as incompatible. -const ADBLOCK_RUST_DAT_VERSION: u8 = 3; +const ADBLOCK_RUST_DAT_VERSION: u8 = 4; /// The total length of the header prefix (magic + version + seahash) const HEADER_PREFIX_LENGTH: usize = 4 + 1 + 8; diff --git a/src/filters/network.rs b/src/filters/network.rs index 16eee562..306950cd 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -28,6 +28,14 @@ bitflags::bitflags! { #[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Default)] pub struct NetworkFilterFeaturesMask: u32 { const BAD_FILTER = 1 << 0; + const IS_REMOVEPARAM = 1 << 1; + const GENERIC_HIDE = 1 << 2; + const IS_CSP = 1 << 3; + const IS_REDIRECT = 1 << 4; + + /// Specifies that a redirect rule should also create a corresponding block rule. + /// This is used to avoid returning two separate rules from `NetworkFilter::parse`. + const ALSO_BLOCK_REDIRECT = 1 << 5; } } @@ -107,12 +115,8 @@ bitflags::bitflags! { const FROM_HTTPS = 1 << 12; const IS_IMPORTANT = 1 << 13; const MATCH_CASE = 1 << 14; - const IS_REMOVEPARAM = 1 << 15; const THIRD_PARTY = 1 << 16; const FIRST_PARTY = 1 << 17; - const IS_REDIRECT = 1 << 26; - const GENERIC_HIDE = 1 << 30; - // Full document rules are not implied by negated types. const FROM_DOCUMENT = 1 << 29; @@ -122,14 +126,9 @@ bitflags::bitflags! { const IS_RIGHT_ANCHOR = 1 << 20; const IS_HOSTNAME_ANCHOR = 1 << 21; const IS_EXCEPTION = 1 << 22; - const IS_CSP = 1 << 23; const IS_COMPLETE_REGEX = 1 << 24; const IS_HOSTNAME_REGEX = 1 << 28; - // Specifies that a redirect rule should also create a corresponding block rule. - // This is used to avoid returning two separate rules from `NetworkFilter::parse`. - const ALSO_BLOCK_REDIRECT = 1 << 31; - // "Other" network request types const UNMATCHED = 1 << 25; @@ -196,26 +195,6 @@ pub trait NetworkFilterMaskHelper { self.has_flag(NetworkFilterMask::IS_IMPORTANT) } - #[inline] - fn is_redirect(&self) -> bool { - self.has_flag(NetworkFilterMask::IS_REDIRECT) - } - - #[inline] - fn is_removeparam(&self) -> bool { - self.has_flag(NetworkFilterMask::IS_REMOVEPARAM) - } - - #[inline] - fn also_block_redirect(&self) -> bool { - self.has_flag(NetworkFilterMask::ALSO_BLOCK_REDIRECT) - } - - #[inline] - fn is_generic_hide(&self) -> bool { - self.has_flag(NetworkFilterMask::GENERIC_HIDE) - } - #[inline] fn is_regex(&self) -> bool { self.has_flag(NetworkFilterMask::IS_REGEX) @@ -231,11 +210,6 @@ pub trait NetworkFilterMaskHelper { !self.is_regex() } - #[inline] - fn is_csp(&self) -> bool { - self.has_flag(NetworkFilterMask::IS_CSP) - } - #[inline] fn third_party(&self) -> bool { self.has_flag(NetworkFilterMask::THIRD_PARTY) @@ -528,28 +502,28 @@ impl NetworkFilter { } NetworkFilterOption::Tag(value) => tag = Some(value), NetworkFilterOption::Redirect(value) => { - mask.set(NetworkFilterMask::IS_REDIRECT, true); - mask.set(NetworkFilterMask::ALSO_BLOCK_REDIRECT, true); + features_mask.set(NetworkFilterFeaturesMask::IS_REDIRECT, true); + features_mask.set(NetworkFilterFeaturesMask::ALSO_BLOCK_REDIRECT, true); modifier_option = Some(value); } NetworkFilterOption::RedirectRule(value) => { - mask.set(NetworkFilterMask::IS_REDIRECT, true); + features_mask.set(NetworkFilterFeaturesMask::IS_REDIRECT, true); modifier_option = Some(value); } NetworkFilterOption::Removeparam(value) => { - mask.set(NetworkFilterMask::IS_REMOVEPARAM, true); + features_mask.set(NetworkFilterFeaturesMask::IS_REMOVEPARAM, true); modifier_option = Some(value); } NetworkFilterOption::Csp(value) => { - mask.set(NetworkFilterMask::IS_CSP, true); + features_mask.set(NetworkFilterFeaturesMask::IS_CSP, true); // CSP rules can never have content types, and should always match against // subdocument and document rules. Rules do not match against document - // requests by default, so this must be explictly added. + // requests by default, so this must be explicitly added. mask.set(NetworkFilterMask::FROM_DOCUMENT, true); modifier_option = value; } NetworkFilterOption::Generichide => { - mask.set(NetworkFilterMask::GENERIC_HIDE, true) + features_mask.set(NetworkFilterFeaturesMask::GENERIC_HIDE, true) } NetworkFilterOption::Document => { cpt_mask_positive.set(NetworkFilterMask::FROM_DOCUMENT, true) @@ -588,7 +562,7 @@ impl NetworkFilter { // The negated types will be applied later. // // This doesn't apply to removeparam filters. - if !mask.contains(NetworkFilterMask::IS_REMOVEPARAM) + if !features_mask.contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) && (cpt_mask_negative & NetworkFilterMask::FROM_NETWORK_TYPES) != NetworkFilterMask::NONE { @@ -597,7 +571,7 @@ impl NetworkFilter { // If no positive types were set, then the filter should apply to all network types. if (cpt_mask_positive & NetworkFilterMask::FROM_ALL_TYPES).is_empty() { // Removeparam is again a special case. - if mask.contains(NetworkFilterMask::IS_REMOVEPARAM) { + if features_mask.contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) { mask |= NetworkFilterMask::FROM_DOCUMENT | NetworkFilterMask::FROM_SUBDOCUMENT | NetworkFilterMask::FROM_XMLHTTPREQUEST; @@ -772,11 +746,11 @@ impl NetworkFilter { }) .transpose(); - if mask.contains(NetworkFilterMask::GENERIC_HIDE) && !parsed.exception { + if features_mask.contains(NetworkFilterFeaturesMask::GENERIC_HIDE) && !parsed.exception { return Err(NetworkFilterError::GenericHideWithoutException); } - if mask.contains(NetworkFilterMask::IS_REMOVEPARAM) && parsed.exception { + if features_mask.contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) && parsed.exception { return Err(NetworkFilterError::RemoveparamWithException); } @@ -795,7 +769,7 @@ impl NetworkFilter { && mask.contains(NetworkFilterMask::IS_HOSTNAME_ANCHOR) && mask.contains(NetworkFilterMask::IS_RIGHT_ANCHOR) && !end_url_anchor - && !mask.contains(NetworkFilterMask::IS_REMOVEPARAM) + && !features_mask.contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) { mask |= NetworkFilterMask::FROM_ALL_TYPES; } @@ -851,6 +825,7 @@ impl NetworkFilter { compute_filter_id( self.modifier_option.as_deref(), self.mask, + self.features_mask, self.filter.string_view().as_deref(), self.hostname.as_deref(), self.opt_domains.as_ref(), @@ -903,7 +878,11 @@ impl NetworkFilter { } } - if tokens_buffer.is_empty() && self.mask.contains(NetworkFilterMask::IS_REMOVEPARAM) { + if tokens_buffer.is_empty() + && self + .features_mask + .contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) + { if let Some(removeparam) = &self.modifier_option { if VALID_PARAM.is_match(removeparam) { utils::tokenize_to(&removeparam.to_ascii_lowercase(), tokens_buffer); @@ -938,6 +917,31 @@ impl NetworkFilter { .contains(NetworkFilterFeaturesMask::BAD_FILTER) } + pub fn is_removeparam(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) + } + + pub fn is_generic_hide(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::GENERIC_HIDE) + } + + pub fn is_csp(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::IS_CSP) + } + + pub fn is_redirect(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::IS_REDIRECT) + } + + pub fn also_block_redirect(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::ALSO_BLOCK_REDIRECT) + } + #[cfg(test)] pub(crate) fn matches_test(&self, request: &request::Request) -> bool { let filter_set = crate::FilterSet::new_with_rules(vec![self.clone()], vec![], true); @@ -977,6 +981,7 @@ pub(crate) trait NetworkMatchable { fn compute_filter_id( modifier_option: Option<&str>, mask: NetworkFilterMask, + features_mask: NetworkFilterFeaturesMask, filter: Option<&str>, hostname: Option<&str>, opt_domains: Option<&Vec>, @@ -984,6 +989,10 @@ fn compute_filter_id( ) -> Hash { let mut hash: Hash = (5408 * 33) ^ Hash::from(mask.bits()); + // Exclude BAD_FILTER from the hash + let features_mask_bits = features_mask.bits() & !NetworkFilterFeaturesMask::BAD_FILTER.bits(); + hash = hash.wrapping_mul(33) ^ Hash::from(features_mask_bits); + if let Some(s) = modifier_option { let chars = s.chars(); for c in chars { diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index ce31d490..8239345c 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -183,7 +183,7 @@ mod tests { fn deserialization_generate_simple() { let mut engine = Engine::from_rules(["ad-banner"], Default::default()); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 10945714988765761881; + const EXPECTED_HASH: u64 = 4588487935723956783; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } @@ -193,7 +193,7 @@ mod tests { let mut engine = Engine::from_rules(["ad-banner$tag=abc"], Default::default()); engine.use_tags(&["abc"]); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 4608037684406751718; + const EXPECTED_HASH: u64 = 7781508779107365248; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } @@ -237,9 +237,9 @@ mod tests { ); } let expected_hash: u64 = if cfg!(feature = "css-validation") { - 18006544876284176656 + 11452175769852090292 } else { - 7678096075090154250 + 9645570723907119088 }; assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}"); From 2e8345f3a0786a84159638d75b9da09c7232cb95 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Wed, 13 May 2026 01:51:08 +0400 Subject: [PATCH 03/28] Set mask default to save memory (#639) The PR adjusts mask default value to consume less memory. Flatbuffer builder doesn't serialize the mask if it's equal to the default. The current default covers about 50% of the network rules and saves 200KB+ of memory. --- src/flatbuffers/fb_network_filter.fbs | 5 ++++- src/flatbuffers/fb_network_filter_generated.rs | 9 +++++---- tests/unit/engine.rs | 8 ++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/flatbuffers/fb_network_filter.fbs b/src/flatbuffers/fb_network_filter.fbs index c4b7ed01..f36bde07 100644 --- a/src/flatbuffers/fb_network_filter.fbs +++ b/src/flatbuffers/fb_network_filter.fbs @@ -7,7 +7,10 @@ namespace fb; table NetworkFilter { - mask: uint32; // NetworkFilterMask (network.rs) + // NetworkFilterMask (network.rs). + // Default = DEFAULT_OPTIONS | IS_HOSTNAME_ANCHOR | IS_RIGHT_ANCHOR | FROM_DOCUMENT + // (the most common mask, covering ~50% of filters). + mask: uint32 = 540221439; // 0x20331FFF /// These arrays contain sorted (ascending) indices in the |unique_domains_hashes| /// instead of the hashes themselves. This approach saves memory, as there diff --git a/src/flatbuffers/fb_network_filter_generated.rs b/src/flatbuffers/fb_network_filter_generated.rs index 1d906be4..a90e295e 100644 --- a/src/flatbuffers/fb_network_filter_generated.rs +++ b/src/flatbuffers/fb_network_filter_generated.rs @@ -114,7 +114,7 @@ pub mod fb { // which contains a valid value in this slot unsafe { self._tab - .get::(NetworkFilter::VT_MASK, Some(0)) + .get::(NetworkFilter::VT_MASK, Some(540221439)) .unwrap() } } @@ -262,7 +262,7 @@ pub mod fb { #[inline] fn default() -> Self { NetworkFilterArgs { - mask: 0, + mask: 540221439, opt_domains: None, opt_not_domains: None, patterns: None, @@ -281,7 +281,8 @@ pub mod fb { impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> NetworkFilterBuilder<'a, 'b, A> { #[inline] pub fn add_mask(&mut self, mask: u32) { - self.fbb_.push_slot::(NetworkFilter::VT_MASK, mask, 0); + self.fbb_ + .push_slot::(NetworkFilter::VT_MASK, mask, 540221439); } #[inline] pub fn add_opt_domains( @@ -387,7 +388,7 @@ pub mod fb { impl Default for NetworkFilterT { fn default() -> Self { Self { - mask: 0, + mask: 540221439, opt_domains: None, opt_not_domains: None, patterns: None, diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index 8239345c..a68c9a4b 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -221,8 +221,8 @@ mod tests { #[cfg(feature = "debug-info")] { let debug_info = engine.get_debug_info(); - let low_bound = 9_000_000; - let high_bound = 9_500_000; + let low_bound = 8_700_000; + let high_bound = 9_200_000; assert!( debug_info.flatbuffer_size >= low_bound, "Expected size >= {} bytes, got {}", @@ -237,9 +237,9 @@ mod tests { ); } let expected_hash: u64 = if cfg!(feature = "css-validation") { - 11452175769852090292 + 1278886013448413771 } else { - 9645570723907119088 + 6460458707531433656 }; assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}"); From aaa99d1391c5dbeaa00674aa56f55d78c04b4d17 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Wed, 13 May 2026 01:59:56 +0400 Subject: [PATCH 04/28] [perf, memory] Add single/multi_pattern to optimize memory usage (0.13.x) (#643) The PR optimizes memory usage and building time by adding a simplified pattern storage. Most filters (95%+) have patterns.len() <= 1. Despite that fact we always store patterns as a vector. This implies storing an extra pointer (to the vector itself) + the vector length. The PR adds an optional ability to store pattern as a single string eliminating the overhead above. --- src/filters/fb_network.rs | 61 +++++++++---- src/filters/fb_network_builder.rs | 26 ++++-- src/filters/network.rs | 4 +- src/flatbuffers/fb_network_filter.fbs | 6 +- .../fb_network_filter_generated.rs | 85 +++++++++++++------ tests/unit/engine.rs | 8 +- 6 files changed, 135 insertions(+), 55 deletions(-) diff --git a/src/filters/fb_network.rs b/src/filters/fb_network.rs index 505caa17..9c8be08d 100644 --- a/src/filters/fb_network.rs +++ b/src/filters/fb_network.rs @@ -8,24 +8,37 @@ use crate::regex_manager::RegexManager; use crate::request::Request; use crate::filters::flatbuffer_generated::fb; + /// A list of string parts that can be matched against a URL. -pub(crate) struct FlatPatterns<'a> { - patterns: Option>>, +pub(crate) enum FlatPatterns<'a> { + /// No patterns to match + Empty, + /// Memory-usage optimization - ~95% of filters have <= 1 pattern. Special-casing avoids the + /// need to hold an extra pointer and vector length. + Single(&'a str), + /// More than 1 pattern to match + Multi(flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<&'a str>>), } impl<'a> FlatPatterns<'a> { #[inline(always)] pub fn new( - patterns: Option>>, + single_pattern: Option<&'a str>, + multi_patterns: Option>>, ) -> Self { - Self { patterns } + if let Some(single_pattern) = single_pattern { + FlatPatterns::Single(single_pattern) + } else if let Some(patterns) = multi_patterns { + FlatPatterns::Multi(patterns) + } else { + FlatPatterns::Empty + } } #[inline(always)] pub fn iter(&self) -> FlatPatternsIterator<'_> { FlatPatternsIterator { patterns: self, - len: self.patterns.map_or(0, |d| d.len()), index: 0, } } @@ -34,7 +47,6 @@ impl<'a> FlatPatterns<'a> { /// Iterator over [FlatPatterns]. pub(crate) struct FlatPatternsIterator<'a> { patterns: &'a FlatPatterns<'a>, - len: usize, index: usize, } @@ -43,21 +55,37 @@ impl<'a> Iterator for FlatPatternsIterator<'a> { #[inline(always)] fn next(&mut self) -> Option { - self.patterns.patterns.and_then(|fi| { - if self.index < self.len { - self.index += 1; - Some(fi.get(self.index - 1)) - } else { - None + match &self.patterns { + FlatPatterns::Empty => None, + FlatPatterns::Single(s) => { + if self.index == 0 { + self.index += 1; + Some(*s) + } else { + None + } + } + FlatPatterns::Multi(v) => { + if self.index < v.len() { + let result = v.get(self.index); + self.index += 1; + Some(result) + } else { + None + } } - }) + } } } impl ExactSizeIterator for FlatPatternsIterator<'_> { #[inline(always)] fn len(&self) -> usize { - self.len + match &self.patterns { + FlatPatterns::Empty => 0, + FlatPatterns::Single(_) => 1_usize.saturating_sub(self.index), + FlatPatterns::Multi(v) => v.len().saturating_sub(self.index), + } } } @@ -123,7 +151,10 @@ impl<'a> FlatNetworkFilter<'a> { #[inline(always)] pub fn patterns(&self) -> FlatPatterns<'_> { - FlatPatterns::new(self.fb_filter.patterns()) + FlatPatterns::new( + self.fb_filter.single_pattern(), + self.fb_filter.multi_patterns(), + ) } #[inline(always)] diff --git a/src/filters/fb_network_builder.rs b/src/filters/fb_network_builder.rs index 6f54ae8b..d07cdbdf 100644 --- a/src/filters/fb_network_builder.rs +++ b/src/filters/fb_network_builder.rs @@ -81,15 +81,22 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for &NetworkFilter { .as_ref() .map(|s| builder.create_string(s)); - let patterns = if network_filter.filter.iter().len() > 0 { - let offsets: Vec> = network_filter - .filter - .iter() - .map(|s| builder.create_string(s)) - .collect(); - Some(FlatSerialize::serialize(offsets, builder)) + let mut filter_iter = network_filter.filter.iter(); + let filter_count = filter_iter.len(); + + // Use single_pattern for the common case of 0 or 1 patterns to avoid + // the overhead of a FlatBuffers vector (extra table + offset indirection). + let (single_pattern, multi_patterns) = if filter_count <= 1 { + let single = filter_iter.next().map(|s| builder.create_string(s)); + (single, None) } else { - None + ( + None, + Some(FlatSerialize::serialize( + filter_iter.collect::>(), + builder, + )), + ) }; let raw_line = network_filter @@ -101,7 +108,8 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for &NetworkFilter { builder.raw_builder(), &fb::NetworkFilterArgs { mask: network_filter.mask.bits(), - patterns, + single_pattern, + multi_patterns, modifier_option, opt_domains, opt_not_domains, diff --git a/src/filters/network.rs b/src/filters/network.rs index 306950cd..c78e3ea8 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -332,8 +332,8 @@ impl ExactSizeIterator for FilterPartIterator<'_> { fn len(&self) -> usize { match self.filter_part { FilterPart::Empty => 0, - FilterPart::Simple(_) => 1, - FilterPart::AnyOf(vec) => vec.len(), + FilterPart::Simple(_) => 1usize.saturating_sub(self.index), + FilterPart::AnyOf(vec) => vec.len().saturating_sub(self.index), } } } diff --git a/src/flatbuffers/fb_network_filter.fbs b/src/flatbuffers/fb_network_filter.fbs index f36bde07..599847d0 100644 --- a/src/flatbuffers/fb_network_filter.fbs +++ b/src/flatbuffers/fb_network_filter.fbs @@ -18,7 +18,11 @@ table NetworkFilter { opt_domains: [uint32]; opt_not_domains: [uint32]; - patterns: [string]; + // A union-like storage for pattern. Prefer |single_pattern| where + // possible for performance concerns. + single_pattern: string; + multi_patterns: [string]; + modifier_option: string; hostname: string; diff --git a/src/flatbuffers/fb_network_filter_generated.rs b/src/flatbuffers/fb_network_filter_generated.rs index a90e295e..df1cc3fb 100644 --- a/src/flatbuffers/fb_network_filter_generated.rs +++ b/src/flatbuffers/fb_network_filter_generated.rs @@ -38,11 +38,12 @@ pub mod fb { pub const VT_MASK: flatbuffers::VOffsetT = 4; pub const VT_OPT_DOMAINS: flatbuffers::VOffsetT = 6; pub const VT_OPT_NOT_DOMAINS: flatbuffers::VOffsetT = 8; - pub const VT_PATTERNS: flatbuffers::VOffsetT = 10; - pub const VT_MODIFIER_OPTION: flatbuffers::VOffsetT = 12; - pub const VT_HOSTNAME: flatbuffers::VOffsetT = 14; - pub const VT_TAG: flatbuffers::VOffsetT = 16; - pub const VT_RAW_LINE: flatbuffers::VOffsetT = 18; + pub const VT_SINGLE_PATTERN: flatbuffers::VOffsetT = 10; + pub const VT_MULTI_PATTERNS: flatbuffers::VOffsetT = 12; + pub const VT_MODIFIER_OPTION: flatbuffers::VOffsetT = 14; + pub const VT_HOSTNAME: flatbuffers::VOffsetT = 16; + pub const VT_TAG: flatbuffers::VOffsetT = 18; + pub const VT_RAW_LINE: flatbuffers::VOffsetT = 20; #[inline] pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { @@ -71,8 +72,11 @@ pub mod fb { if let Some(x) = args.modifier_option { builder.add_modifier_option(x); } - if let Some(x) = args.patterns { - builder.add_patterns(x); + if let Some(x) = args.multi_patterns { + builder.add_multi_patterns(x); + } + if let Some(x) = args.single_pattern { + builder.add_single_pattern(x); } if let Some(x) = args.opt_not_domains { builder.add_opt_not_domains(x); @@ -88,8 +92,9 @@ pub mod fb { let mask = self.mask(); let opt_domains = self.opt_domains().map(|x| x.into_iter().collect()); let opt_not_domains = self.opt_not_domains().map(|x| x.into_iter().collect()); - let patterns = self - .patterns() + let single_pattern = self.single_pattern().map(|x| x.to_string()); + let multi_patterns = self + .multi_patterns() .map(|x| x.iter().map(|s| s.to_string()).collect()); let modifier_option = self.modifier_option().map(|x| x.to_string()); let hostname = self.hostname().map(|x| x.to_string()); @@ -99,7 +104,8 @@ pub mod fb { mask, opt_domains, opt_not_domains, - patterns, + single_pattern, + multi_patterns, modifier_option, hostname, tag, @@ -148,7 +154,19 @@ pub mod fb { } } #[inline] - pub fn patterns( + pub fn single_pattern(&self) -> Option<&'a str> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab.get::>( + NetworkFilter::VT_SINGLE_PATTERN, + None, + ) + } + } + #[inline] + pub fn multi_patterns( &self, ) -> Option>> { // Safety: @@ -157,7 +175,7 @@ pub mod fb { unsafe { self._tab.get::>, - >>(NetworkFilter::VT_PATTERNS, None) + >>(NetworkFilter::VT_MULTI_PATTERNS, None) } } #[inline] @@ -223,9 +241,14 @@ pub mod fb { Self::VT_OPT_NOT_DOMAINS, false, )? + .visit_field::>( + "single_pattern", + Self::VT_SINGLE_PATTERN, + false, + )? .visit_field::>, - >>("patterns", Self::VT_PATTERNS, false)? + >>("multi_patterns", Self::VT_MULTI_PATTERNS, false)? .visit_field::>( "modifier_option", Self::VT_MODIFIER_OPTION, @@ -250,7 +273,8 @@ pub mod fb { pub mask: u32, pub opt_domains: Option>>, pub opt_not_domains: Option>>, - pub patterns: Option< + pub single_pattern: Option>, + pub multi_patterns: Option< flatbuffers::WIPOffset>>, >, pub modifier_option: Option>, @@ -265,7 +289,8 @@ pub mod fb { mask: 540221439, opt_domains: None, opt_not_domains: None, - patterns: None, + single_pattern: None, + multi_patterns: None, modifier_option: None, hostname: None, tag: None, @@ -305,15 +330,22 @@ pub mod fb { ); } #[inline] - pub fn add_patterns( + pub fn add_single_pattern(&mut self, single_pattern: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_SINGLE_PATTERN, + single_pattern, + ); + } + #[inline] + pub fn add_multi_patterns( &mut self, - patterns: flatbuffers::WIPOffset< + multi_patterns: flatbuffers::WIPOffset< flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset<&'b str>>, >, ) { self.fbb_.push_slot_always::>( - NetworkFilter::VT_PATTERNS, - patterns, + NetworkFilter::VT_MULTI_PATTERNS, + multi_patterns, ); } #[inline] @@ -365,7 +397,8 @@ pub mod fb { ds.field("mask", &self.mask()); ds.field("opt_domains", &self.opt_domains()); ds.field("opt_not_domains", &self.opt_not_domains()); - ds.field("patterns", &self.patterns()); + ds.field("single_pattern", &self.single_pattern()); + ds.field("multi_patterns", &self.multi_patterns()); ds.field("modifier_option", &self.modifier_option()); ds.field("hostname", &self.hostname()); ds.field("tag", &self.tag()); @@ -379,7 +412,8 @@ pub mod fb { pub mask: u32, pub opt_domains: Option>, pub opt_not_domains: Option>, - pub patterns: Option>, + pub single_pattern: Option, + pub multi_patterns: Option>, pub modifier_option: Option, pub hostname: Option, pub tag: Option, @@ -391,7 +425,8 @@ pub mod fb { mask: 540221439, opt_domains: None, opt_not_domains: None, - patterns: None, + single_pattern: None, + multi_patterns: None, modifier_option: None, hostname: None, tag: None, @@ -407,7 +442,8 @@ pub mod fb { let mask = self.mask; let opt_domains = self.opt_domains.as_ref().map(|x| _fbb.create_vector(x)); let opt_not_domains = self.opt_not_domains.as_ref().map(|x| _fbb.create_vector(x)); - let patterns = self.patterns.as_ref().map(|x| { + let single_pattern = self.single_pattern.as_ref().map(|x| _fbb.create_string(x)); + let multi_patterns = self.multi_patterns.as_ref().map(|x| { let w: Vec<_> = x.iter().map(|s| _fbb.create_string(s)).collect(); _fbb.create_vector(&w) }); @@ -421,7 +457,8 @@ pub mod fb { mask, opt_domains, opt_not_domains, - patterns, + single_pattern, + multi_patterns, modifier_option, hostname, tag, diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index a68c9a4b..c579aac1 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -183,7 +183,7 @@ mod tests { fn deserialization_generate_simple() { let mut engine = Engine::from_rules(["ad-banner"], Default::default()); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 4588487935723956783; + const EXPECTED_HASH: u64 = 16556115079021991714; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } @@ -193,7 +193,7 @@ mod tests { let mut engine = Engine::from_rules(["ad-banner$tag=abc"], Default::default()); engine.use_tags(&["abc"]); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 7781508779107365248; + const EXPECTED_HASH: u64 = 4864047469838009851; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } @@ -237,9 +237,9 @@ mod tests { ); } let expected_hash: u64 = if cfg!(feature = "css-validation") { - 1278886013448413771 + 10068943306137168952 } else { - 6460458707531433656 + 6262681569722546672 }; assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}"); From a7028ad44ee36f965443df7b6b793be7bb0c40ad Mon Sep 17 00:00:00 2001 From: Anton Lazarev Date: Tue, 12 May 2026 15:24:13 -0700 Subject: [PATCH 05/28] add test for default flatbuffer NetworkFilterMask --- src/flatbuffers/mod.rs | 4 ++++ tests/unit/flatbuffers/mod.rs | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/unit/flatbuffers/mod.rs diff --git a/src/flatbuffers/mod.rs b/src/flatbuffers/mod.rs index 61dc0bd6..7b93f063 100644 --- a/src/flatbuffers/mod.rs +++ b/src/flatbuffers/mod.rs @@ -1,2 +1,6 @@ pub(crate) mod containers; pub(crate) mod unsafe_tools; + +#[cfg(test)] +#[path = "../../tests/unit/flatbuffers/mod.rs"] +mod unit_tests; diff --git a/tests/unit/flatbuffers/mod.rs b/tests/unit/flatbuffers/mod.rs new file mode 100644 index 00000000..57f15853 --- /dev/null +++ b/tests/unit/flatbuffers/mod.rs @@ -0,0 +1,16 @@ +use crate::filters::flatbuffer_generated::fb::NetworkFilterArgs; +use crate::filters::network::NetworkFilterMask; + +/// If this ever changes, confirm that the default value hardcoded in `fb_network_filter.fbs` +/// continues to track the most common mask. +#[test] +fn default_network_filter_mask() { + assert_eq!( + NetworkFilterArgs::default().mask, + (NetworkFilterMask::DEFAULT_OPTIONS + | NetworkFilterMask::IS_HOSTNAME_ANCHOR + | NetworkFilterMask::IS_RIGHT_ANCHOR + | NetworkFilterMask::FROM_DOCUMENT) + .bits() + ); +} From a7b4277f1a70532a795672b39ab875cdf2633a2c Mon Sep 17 00:00:00 2001 From: Mikhail Date: Fri, 29 May 2026 02:27:21 +0400 Subject: [PATCH 06/28] New API for FilterSet & Engine (#652) The PR introduces a new API to move from Vec in FilterSet. This is a necessary step to make streaming parsing work. --- benches/bench_cosmetic_matching.rs | 4 +- benches/bench_matching.rs | 66 ++++---- benches/bench_memory.rs | 10 +- benches/bench_redirect_performance.rs | 5 +- benches/bench_rules.rs | 56 +++---- benches/bench_serialization.rs | 24 +-- benches/bench_url.rs | 5 +- examples/example.rs | 17 +- examples/generate-dat.rs | 7 +- js/example.mjs | 4 +- js/index.js | 2 +- js/src/lib.rs | 65 ++++---- js/test/bindings.test.mjs | 141 ++++++++--------- src/blocker.rs | 17 +- src/content_blocking.rs | 10 +- src/cosmetic_filter_cache.rs | 10 +- src/engine.rs | 72 +++++---- src/filters/fb_builder.rs | 7 +- src/filters/network.rs | 4 +- src/lists.rs | 191 ++++++++++------------- src/resources/mod.rs | 10 +- tests/legacy_harness.rs | 44 ++++-- tests/live.rs | 8 +- tests/matching.rs | 13 +- tests/simple_use.rs | 2 +- tests/test_utils.rs | 19 +-- tests/ublock-coverage.rs | 19 +-- tests/unit/blocker.rs | 85 +++------- tests/unit/cosmetic_filter_cache.rs | 56 ++----- tests/unit/engine.rs | 91 ++++++----- tests/unit/filters/network_matchers.rs | 2 +- tests/unit/lists.rs | 16 +- tests/unit/regex_manager.rs | 2 +- tests/unit/resources/resource_storage.rs | 12 +- 34 files changed, 495 insertions(+), 601 deletions(-) diff --git a/benches/bench_cosmetic_matching.rs b/benches/bench_cosmetic_matching.rs index 7e226a50..0a735716 100644 --- a/benches/bench_cosmetic_matching.rs +++ b/benches/bench_cosmetic_matching.rs @@ -7,10 +7,10 @@ mod test_utils; pub fn make_engine() -> Engine { use adblock::resources::Resource; - let rules = test_utils::rules_from_lists(&["data/brave/brave-main-list.txt"]); + let rules = test_utils::rules_from_lists(["data/brave/brave-main-list.txt"]); let resource_json = std::fs::read_to_string("data/brave/brave-resources.json").unwrap(); let resource_list: Vec = serde_json::from_str(&resource_json).unwrap(); - let mut engine = Engine::from_rules_parametrised(rules, Default::default(), true, true); + let mut engine = Engine::new_with_list_text_parametrised(rules, Default::default(), true, true); engine.use_resources(resource_list); engine } diff --git a/benches/bench_matching.rs b/benches/bench_matching.rs index 4a1f3287..016c7956 100644 --- a/benches/bench_matching.rs +++ b/benches/bench_matching.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use adblock::request::Request; use adblock::url_parser::parse_url; -use adblock::{Engine, FilterSet}; +use adblock::Engine; #[path = "../tests/test_utils.rs"] mod test_utils; @@ -25,25 +25,15 @@ impl From<&TestRequest> for Request { } fn load_requests() -> Vec { - let requests_str = rules_from_lists(&["data/requests.json"]); + let requests_str = rules_from_lists(["data/requests.json"]); let reqs: Vec = requests_str - .into_iter() - .map(|r| serde_json::from_str(&r)) + .lines() + .map(serde_json::from_str) .filter_map(Result::ok) .collect(); reqs } -fn get_engine(rules: impl IntoIterator>) -> Engine { - let (network_filters, cosmetic_filters) = - adblock::lists::parse_filters(rules, false, Default::default()); - - Engine::from_filter_set( - FilterSet::new_with_rules(network_filters, cosmetic_filters, false), - true, - ) -} - fn bench_rule_matching(engine: &Engine, requests: &[TestRequest]) -> (u32, u32) { let mut matches = 0; let mut passes = 0; @@ -112,21 +102,21 @@ fn rule_match(c: &mut Criterion) { group.sample_size(10); group.bench_function("el+ep", move |b| { - let rules = rules_from_lists(&[ + let rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = Engine::from_rules(rules, Default::default()); + let engine = Engine::new_with_list_text(rules, Default::default()); b.iter(|| bench_rule_matching(&engine, &elep_req)) }); group.bench_function("easylist", move |b| { - let rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); - let engine = Engine::from_rules(rules, Default::default()); + let rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); + let engine = Engine::new_with_list_text(rules, Default::default()); b.iter(|| bench_rule_matching(&engine, &el_req)) }); group.bench_function("slimlist", move |b| { - let rules = rules_from_lists(&["data/slim-list.txt"]); - let engine = Engine::from_rules(rules, Default::default()); + let rules = rules_from_lists(["data/slim-list.txt"]); + let engine = Engine::new_with_list_text(rules, Default::default()); b.iter(|| bench_rule_matching(&engine, &slim_req)) }); @@ -136,7 +126,7 @@ fn rule_match(c: &mut Criterion) { fn rule_match_parsed_el(c: &mut Criterion) { let mut group = c.benchmark_group("rule-match-parsed"); - let rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); + let rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); let requests = load_requests(); let requests_parsed: Vec<_> = requests .into_iter() @@ -144,7 +134,7 @@ fn rule_match_parsed_el(c: &mut Criterion) { .filter_map(Result::ok) .collect(); let requests_len = requests_parsed.len() as u64; - let engine = get_engine(rules); + let engine = Engine::new_with_list_text(rules, Default::default()); group.throughput(Throughput::Elements(requests_len)); group.sample_size(10); @@ -159,11 +149,11 @@ fn rule_match_parsed_el(c: &mut Criterion) { fn rule_match_parsed_elep_slimlist(c: &mut Criterion) { let mut group = c.benchmark_group("rule-match-parsed"); - let full_rules = rules_from_lists(&[ + let full_rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = get_engine(full_rules); + let engine = Engine::new_with_list_text(full_rules, Default::default()); let requests = load_requests(); let requests_parsed: Vec<_> = requests @@ -173,8 +163,8 @@ fn rule_match_parsed_elep_slimlist(c: &mut Criterion) { .collect(); let requests_len = requests_parsed.len() as u64; - let slim_rules = rules_from_lists(&["data/slim-list.txt"]); - let slim_engine = get_engine(slim_rules); + let slim_rules = rules_from_lists(["data/slim-list.txt"]); + let slim_engine = Engine::new_with_list_text(slim_rules, Default::default()); let requests_copy = load_requests(); let requests_parsed_copy: Vec<_> = requests_copy @@ -245,26 +235,30 @@ fn rule_match_browserlike_comparable(c: &mut Criterion) { let requests = requests_parsed(&requests); group.bench_function("el+ep", |b| { - let rules = rules_from_lists(&[ + let rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); b.iter(|| bench_rule_matching_browserlike(&engine, &requests)) }); group.bench_function("el", |b| { - let rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); - let engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); + let engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); b.iter(|| bench_rule_matching_browserlike(&engine, &requests)) }); group.bench_function("slimlist", |b| { - let rules = rules_from_lists(&["data/slim-list.txt"]); - let engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let rules = rules_from_lists(["data/slim-list.txt"]); + let engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); b.iter(|| bench_rule_matching_browserlike(&engine, &requests)) }); group.bench_function("brave-list", |b| { - let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); - let engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let rules = rules_from_lists(["data/brave/brave-main-list.txt"]); + let engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); b.iter(|| bench_rule_matching_browserlike(&engine, &requests)) }); @@ -288,9 +282,9 @@ fn rule_match_first_request(c: &mut Criterion) { b.iter_custom(|iters| { let mut total_time = std::time::Duration::ZERO; for _ in 0..iters { - let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); + let rules = rules_from_lists(["data/brave/brave-main-list.txt"]); let engine = - Engine::from_rules_parametrised(rules, Default::default(), false, true); + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); // Measure only the matching time, skip setup and destruction let start_time = std::time::Instant::now(); diff --git a/benches/bench_memory.rs b/benches/bench_memory.rs index 1e764414..7eca9310 100644 --- a/benches/bench_memory.rs +++ b/benches/bench_memory.rs @@ -133,10 +133,10 @@ impl From<&TestRequest> for Request { } fn load_requests() -> Vec { - let requests_str = rules_from_lists(&["data/requests.json"]); + let requests_str = rules_from_lists(["data/requests.json"]); let reqs: Vec = requests_str - .into_iter() - .map(|r| serde_json::from_str(&r)) + .lines() + .map(serde_json::from_str) .filter_map(Result::ok) .collect(); reqs @@ -243,8 +243,8 @@ fn bench_cb( ) { let single_run = || { ALLOCATOR.reset(); - let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); - let mut engine = Engine::from_rules(rules, Default::default()); + let rules = rules_from_lists(["data/brave/brave-main-list.txt"]); + let mut engine = Engine::new_with_list_text(rules, Default::default()); let resource_json = std::fs::read_to_string("data/brave/brave-resources.json").unwrap(); let resource_list: Vec = serde_json::from_str(&resource_json).unwrap(); std::mem::drop(resource_json); diff --git a/benches/bench_redirect_performance.rs b/benches/bench_redirect_performance.rs index 56da371f..82dce5c6 100644 --- a/benches/bench_redirect_performance.rs +++ b/benches/bench_redirect_performance.rs @@ -1,4 +1,4 @@ -use adblock::{Engine, FilterSet}; +use adblock::Engine; use criterion::*; use tokio::runtime::Runtime; @@ -86,8 +86,7 @@ fn get_redirect_rules() -> Vec { /// Loads the supplied rules, and the test set of resources, into a Engine fn get_preloaded_engine(rules: Vec) -> Engine { - let filter_set = FilterSet::new_with_rules(rules, vec![], false); - Engine::from_filter_set(filter_set, true /* optimize */) + Engine::new_with_parsed_rules(rules, vec![], true) } fn get_resources_for_filters(#[allow(unused)] filters: &[NetworkFilter]) -> Vec { diff --git a/benches/bench_rules.rs b/benches/bench_rules.rs index e4c35f72..c753f631 100644 --- a/benches/bench_rules.rs +++ b/benches/bench_rules.rs @@ -1,26 +1,26 @@ use criterion::*; use std::sync::LazyLock; -use adblock::{Engine, FilterSet}; +use adblock::Engine; #[path = "../tests/test_utils.rs"] mod test_utils; use test_utils::rules_from_lists; -static DEFAULT_LISTS: LazyLock> = - LazyLock::new(|| rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]).collect()); +static DEFAULT_LISTS: LazyLock = + LazyLock::new(|| rules_from_lists(["data/easylist.to/easylist/easylist.txt"])); -fn bench_string_hashing(filters: &Vec) -> adblock::utils::Hash { +fn bench_string_hashing(filters: &str) -> adblock::utils::Hash { let mut dummy: adblock::utils::Hash = 0; - for filter in filters { + for filter in filters.lines() { dummy = (dummy + adblock::utils::fast_hash(filter)) % 1000000000; } dummy } -fn bench_string_tokenize(filters: &Vec) -> usize { +fn bench_string_tokenize(filters: &str) -> usize { let mut dummy: usize = 0; - for filter in filters { + for filter in filters.lines() { dummy = (dummy + adblock::utils::tokenize(filter).len()) % 1000000000; } dummy @@ -50,15 +50,10 @@ fn string_tokenize(c: &mut Criterion) { group.finish(); } -fn bench_parsing_impl(lists: &Vec<&Vec>) -> usize { - let mut dummy = 0; - - for list in lists { - let (network_filters, _) = adblock::lists::parse_filters(*list, false, Default::default()); - dummy += network_filters.len() % 1000000; - } - - dummy +fn bench_parsing_impl(lists: &str) -> usize { + let (network_filters, _) = + adblock::lists::parse_filters(lists.lines(), false, Default::default()); + network_filters.len() % 1000000 } fn list_parse(c: &mut Criterion) { @@ -68,44 +63,35 @@ fn list_parse(c: &mut Criterion) { group.sample_size(10); group.bench_function("network filters", |b| { - b.iter(|| bench_parsing_impl(&vec![DEFAULT_LISTS.as_ref()])) + b.iter(|| bench_parsing_impl(&DEFAULT_LISTS)) }); group.bench_function("all filters", |b| { - b.iter(|| bench_parsing_impl(&vec![DEFAULT_LISTS.as_ref()])) + b.iter(|| bench_parsing_impl(&DEFAULT_LISTS)) }); group.finish(); } -fn get_engine(rules: impl IntoIterator>) -> Engine { - let (network_filters, cosmetic_filters) = - adblock::lists::parse_filters(rules, false, Default::default()); - - Engine::from_filter_set( - FilterSet::new_with_rules(network_filters, cosmetic_filters, false), - true, - ) -} - fn blocker_new(c: &mut Criterion) { let mut group = c.benchmark_group("blocker_new"); group.throughput(Throughput::Elements(1)); group.sample_size(10); - let easylist_rules: Vec<_> = rules_from_lists(&[ + let easylist_rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", - ]) - .collect(); - let brave_list_rules: Vec<_> = rules_from_lists(&["data/brave/brave-main-list.txt"]).collect(); - let engine = Engine::from_rules(&brave_list_rules, Default::default()); + ]); + let brave_list_rules = rules_from_lists(["data/brave/brave-main-list.txt"]); + let engine = Engine::new_with_list_text(brave_list_rules.clone(), Default::default()); let engine_serialized = engine.serialize().to_vec(); - group.bench_function("el+ep", move |b| b.iter(|| get_engine(&easylist_rules))); + group.bench_function("el+ep", move |b| { + b.iter(|| Engine::new_with_list_text(easylist_rules.clone(), Default::default())) + }); group.bench_function("brave-list", move |b| { - b.iter(|| get_engine(&brave_list_rules)) + b.iter(|| Engine::new_with_list_text(brave_list_rules.clone(), Default::default())) }); group.bench_function("brave-list-deserialize", move |b| { b.iter(|| { diff --git a/benches/bench_serialization.rs b/benches/bench_serialization.rs index ff84d17a..87820c05 100644 --- a/benches/bench_serialization.rs +++ b/benches/bench_serialization.rs @@ -12,24 +12,24 @@ fn serialization(c: &mut Criterion) { group.sample_size(20); group.bench_function("el+ep", move |b| { - let full_rules = rules_from_lists(&[ + let full_rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); b.iter(|| assert!(!engine.serialize().to_vec().is_empty())) }); group.bench_function("el", move |b| { - let full_rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); + let full_rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); b.iter(|| assert!(!engine.serialize().to_vec().is_empty())) }); group.bench_function("slimlist", move |b| { - let full_rules = rules_from_lists(&["data/slim-list.txt"]); + let full_rules = rules_from_lists(["data/slim-list.txt"]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); b.iter(|| assert!(!engine.serialize().to_vec().is_empty())) }); @@ -42,12 +42,12 @@ fn deserialization(c: &mut Criterion) { group.sample_size(20); group.bench_function("el+ep", move |b| { - let full_rules = rules_from_lists(&[ + let full_rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); let serialized = engine.serialize().to_vec(); b.iter(|| { @@ -56,9 +56,9 @@ fn deserialization(c: &mut Criterion) { }) }); group.bench_function("el", move |b| { - let full_rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); + let full_rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); let serialized = engine.serialize().to_vec(); b.iter(|| { @@ -67,9 +67,9 @@ fn deserialization(c: &mut Criterion) { }) }); group.bench_function("slimlist", move |b| { - let full_rules = rules_from_lists(&["data/slim-list.txt"]); + let full_rules = rules_from_lists(["data/slim-list.txt"]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); let serialized = engine.serialize().to_vec(); b.iter(|| { diff --git a/benches/bench_url.rs b/benches/bench_url.rs index c9d90ebe..cee526fd 100644 --- a/benches/bench_url.rs +++ b/benches/bench_url.rs @@ -18,8 +18,9 @@ struct TestRequest { } fn load_requests() -> Vec { - rules_from_lists(&["data/requests.json"]) - .map(|r| serde_json::from_str(&r)) + rules_from_lists(["data/requests.json"]) + .lines() + .map(serde_json::from_str) .filter_map(Result::ok) .collect::>() } diff --git a/examples/example.rs b/examples/example.rs index 9f2f74ac..7d0e29bf 100644 --- a/examples/example.rs +++ b/examples/example.rs @@ -5,18 +5,19 @@ use adblock::{ }; fn main() { - let rules = vec![ - String::from("-advertisement-icon."), - String::from("-advertisement-management/"), - String::from("-advertisement."), - String::from("-advertisement/script."), - ]; + let rules = [ + "-advertisement-icon.", + "-advertisement-management/", + "-advertisement.", + "-advertisement/script.", + ] + .join("\n"); let debug_info = true; let mut filter_set = FilterSet::new(debug_info); - filter_set.add_filters(&rules, ParseOptions::default()); + filter_set.add_filter_list(rules, ParseOptions::default()); - let engine = Engine::from_filter_set(filter_set, true); + let engine = Engine::new_with_filter_set(filter_set, true); let request = Request::new( "http://example.com/-advertisement-icon.", diff --git a/examples/generate-dat.rs b/examples/generate-dat.rs index 513db3f7..33c2d513 100644 --- a/examples/generate-dat.rs +++ b/examples/generate-dat.rs @@ -5,13 +5,14 @@ use std::io::prelude::*; fn main() { // Rules we want to serialize - let rules = vec![ + let rules = [ String::from("||platform.twitter.com/$tag=twitter-embeds"), String::from("@@||platform.twitter.com/$tag=twitter-embeds"), - ]; + ] + .join("\n"); // Serialize - let mut engine = Engine::from_rules_debug(&rules, Default::default()); + let mut engine = Engine::new_with_list_text(rules, Default::default()); engine.use_tags(&["twitter-embeds"]); let request = Request::new( diff --git a/js/example.mjs b/js/example.mjs index f1a22df2..882c371d 100644 --- a/js/example.mjs +++ b/js/example.mjs @@ -8,13 +8,13 @@ const filterSet = new adblockRust.FilterSet(debugInfo); const easylistFilters = fs.readFileSync( dataPath + 'easylist.to/easylist/easylist.txt', { encoding: 'utf-8' }, -).split('\n'); +); filterSet.addFilters(easylistFilters); const uboUnbreakFilters = fs.readFileSync( dataPath + 'uBlockOrigin/unbreak.txt', { encoding: 'utf-8' }, -).split('\n'); +); filterSet.addFilters(uboUnbreakFilters); const resources = adblockRust.uBlockResources( diff --git a/js/index.js b/js/index.js index e7f389d1..a8207639 100644 --- a/js/index.js +++ b/js/index.js @@ -15,7 +15,7 @@ class FilterSet { this.boxed = blocker.FilterSet_constructor(...args); } } -forwardClassMethods(FilterSet, ['addFilters', 'addFilter', 'intoContentBlocking']); +forwardClassMethods(FilterSet, ['addFilters', 'intoContentBlocking']); class Engine { constructor(filter_set, ...args) { diff --git a/js/src/lib.rs b/js/src/lib.rs index d6577086..0572d10b 100644 --- a/js/src/lib.rs +++ b/js/src/lib.rs @@ -1,6 +1,4 @@ -use adblock::lists::{ - FilterFormat, FilterListMetadata, FilterSet as FilterSetInternal, ParseOptions, RuleTypes, -}; +use adblock::lists::{FilterFormat, FilterSet as FilterSetInternal, ParseOptions, RuleTypes}; use adblock::resources::resource_assembler::assemble_web_accessible_resources; use adblock::resources::Resource; use adblock::Engine as EngineInternal; @@ -53,6 +51,31 @@ mod json_ffi { } } +fn console_warn<'a, C: Context<'a>>(cx: &mut C, message: &str) -> NeonResult<()> { + let console: Handle = cx.global().get(cx, "console")?; + let warn: Handle = console.get(cx, "warn")?; + let msg = JsString::new(cx, message).upcast::(); + warn.call(cx, console, [msg])?; + Ok(()) +} + +/// Accepts a single filter list string, or a deprecated array of list strings (joined with `\n`). +fn filter_rules_from_js<'a, C: Context<'a>>( + cx: &mut C, + input: Handle, +) -> NeonResult { + if input.downcast::(cx).is_ok() { + console_warn( + cx, + "FilterSet.addFilters: passing an array of strings is deprecated; \ + pass a single newline-separated filter list string instead.", + )?; + let parts: Vec = json_ffi::from_js(cx, input)?; + return Ok(parts.join("\n")); + } + json_ffi::from_js(cx, input) +} + #[derive(Serialize, Deserialize)] struct EngineOptions { pub optimize: Option, @@ -64,16 +87,10 @@ impl FilterSet { fn new(debug: bool) -> Self { Self(RefCell::new(FilterSetInternal::new(debug))) } - fn add_filters(&self, rules: &[String], opts: ParseOptions) -> FilterListMetadata { - self.0.borrow_mut().add_filters(rules, opts) - } - fn add_filter( - &self, - filter: &str, - opts: ParseOptions, - ) -> Result<(), adblock::lists::FilterParseError> { - self.0.borrow_mut().add_filter(filter, opts) + fn add_filters(&self, rules: String, opts: ParseOptions) -> adblock::lists::FilterListMetadata { + self.0.borrow_mut().add_filter_list(rules, opts) } + fn into_content_blocking( &self, ) -> Result<(Vec, Vec), ()> { @@ -99,7 +116,6 @@ fn create_filter_set(mut cx: FunctionContext) -> JsResult> { fn filter_set_add_filters(mut cx: FunctionContext) -> JsResult { let this = cx.argument::>(0)?; - // Take the first argument, which must be an array let rules_handle: Handle = cx.argument(1)?; // Second argument is optional parse options. All fields are optional. ParseOptions::default() // if unspecified. @@ -108,27 +124,13 @@ fn filter_set_add_filters(mut cx: FunctionContext) -> JsResult { None => ParseOptions::default(), }; - let rules: Vec = json_ffi::from_js(&mut cx, rules_handle)?; + let rules: String = filter_rules_from_js(&mut cx, rules_handle)?; - let metadata = this.add_filters(&rules, parse_opts); + let metadata = this.add_filters(rules, parse_opts); json_ffi::to_js(&mut cx, &metadata) } -fn filter_set_add_filter(mut cx: FunctionContext) -> JsResult { - let this = cx.argument::>(0)?; - - let filter: String = cx.argument::(1)?.value(&mut cx); - let parse_opts = match cx.argument_opt(2) { - Some(parse_opts_arg) => json_ffi::from_js(&mut cx, parse_opts_arg)?, - None => ParseOptions::default(), - }; - - let ok = this.add_filter(&filter, parse_opts).is_ok(); - // Return true/false depending on whether or not the filter could be added - Ok(JsBoolean::new(&mut cx, ok)) -} - #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct ContentBlockingConversionResult { @@ -171,9 +173,9 @@ fn engine_constructor(mut cx: FunctionContext) -> JsResult> { config.optimize.unwrap_or(true) } }; - EngineInternal::from_filter_set(rules, optimize) + EngineInternal::new_with_filter_set(rules, optimize) } - None => EngineInternal::from_filter_set(rules, true), + None => EngineInternal::new_with_filter_set(rules, true), }; Ok(cx.boxed(Engine(Mutex::new(engine_internal)))) } @@ -391,7 +393,6 @@ fn build_rule_types_enum<'a, C: Context<'a>>(cx: &mut C) -> JsResult<'a, JsObjec register_module!(mut m, { m.export_function("FilterSet_constructor", create_filter_set)?; m.export_function("FilterSet_addFilters", filter_set_add_filters)?; - m.export_function("FilterSet_addFilter", filter_set_add_filter)?; m.export_function( "FilterSet_intoContentBlocking", filter_set_into_content_blocking, diff --git a/js/test/bindings.test.mjs b/js/test/bindings.test.mjs index 2cca2147..7f288778 100644 --- a/js/test/bindings.test.mjs +++ b/js/test/bindings.test.mjs @@ -23,7 +23,7 @@ const { FilterSet, Engine, FilterFormat, RuleTypes, uBlockResources } = // --------------------------------------------------------------------------- describe('FilterSet.addFilters', () => { - it('parses metadata comments (title, homepage, expires, redirect)', () => { + it('parses metadata comments (title, homepage, expires, redirect)', () => { const fs = new FilterSet(); const meta = fs.addFilters([ '! Title: Test List', @@ -31,16 +31,15 @@ describe('FilterSet.addFilters', () => { '! Expires: 2 days', '! Redirect: https://example.com/new-list.txt', '||ads.com^', - ]); + ].join('\n')); assert.equal(meta.title, 'Test List'); assert.equal(meta.homepage, 'https://example.com'); assert.ok(meta.expires != null); assert.equal(meta.redirect, 'https://example.com/new-list.txt'); }); - it('hosts format parses IP-hostname entries', () => { const fs = new FilterSet(); - fs.addFilters(['127.0.0.1 ads.example.com'], { format: FilterFormat.HOSTS }); + fs.addFilters('127.0.0.1 ads.example.com', { format: FilterFormat.HOSTS }); const engine = new Engine(fs, true); assert.equal( engine.check('https://ads.example.com/', 'https://pub.com', 'script'), @@ -51,7 +50,7 @@ describe('FilterSet.addFilters', () => { it('NETWORK_ONLY skips cosmetic rules', () => { const fs = new FilterSet(); fs.addFilters( - ['||example.com^', 'example.com##.ad'], + ['||example.com^', 'example.com##.ad'].join('\n'), { rule_types: RuleTypes.NETWORK_ONLY }, ); const engine = new Engine(fs, true); @@ -63,10 +62,26 @@ describe('FilterSet.addFilters', () => { assert.equal(cosmetic.hide_selectors.length, 0); }); + it("Legacy addFilters syntax works", () => { + const fs = new FilterSet(); + fs.addFilters(["||example.com^", "example.com##.ad"], { + rule_types: RuleTypes.NETWORK_ONLY, + }); + const engine = new Engine(fs, true); + assert.equal( + engine.check( + "https://example.com/test.js", + "https://pub.com", + "script", + ), + true, + ); + }); + it('COSMETIC_ONLY skips network rules', () => { const fs = new FilterSet(); fs.addFilters( - ['||example.com^', 'example.com##.ad'], + ['||example.com^', 'example.com##.ad'].join('\n'), { rule_types: RuleTypes.COSMETIC_ONLY }, ); const engine = new Engine(fs, true); @@ -79,40 +94,6 @@ describe('FilterSet.addFilters', () => { }); }); -// --------------------------------------------------------------------------- -// FilterSet.addFilter -// --------------------------------------------------------------------------- - -describe('FilterSet.addFilter', () => { - it('returns true for a valid network filter', () => { - const fs = new FilterSet(); - assert.equal(fs.addFilter('||example.com^'), true); - }); - - it('returns true for a valid cosmetic filter', () => { - const fs = new FilterSet(); - assert.equal(fs.addFilter('example.com##.banner'), true); - }); - - it('returns false for an empty string', () => { - const fs = new FilterSet(); - assert.equal(fs.addFilter(''), false); - }); - - it('returns false for a comment line', () => { - const fs = new FilterSet(); - assert.equal(fs.addFilter('! this is a comment'), false); - }); - - it('accepts hosts format via ParseOptions', () => { - const fs = new FilterSet(); - assert.equal( - fs.addFilter('127.0.0.1 ads.example.com', { format: FilterFormat.HOSTS }), - true, - ); - }); -}); - // --------------------------------------------------------------------------- // FilterSet.intoContentBlocking // --------------------------------------------------------------------------- @@ -120,7 +101,7 @@ describe('FilterSet.addFilter', () => { describe('FilterSet.intoContentBlocking', () => { it('converts network filters to content blocking rules with trigger/action', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); const result = fs.intoContentBlocking(); assert.notEqual(result, undefined); assert.ok(result.contentBlockingRules.length > 0); @@ -131,7 +112,7 @@ describe('FilterSet.intoContentBlocking', () => { it('returns undefined when debug=false', () => { const fs = new FilterSet(false); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); assert.equal(fs.intoContentBlocking(), undefined); }); }); @@ -143,9 +124,9 @@ describe('FilterSet.intoContentBlocking', () => { describe('FilterSet survives Engine construction (clone semantics)', () => { it('FilterSet is still usable after being passed to Engine constructor', () => { const fs = new FilterSet(); - fs.addFilters(['||example.com^']); + fs.addFilters('||example.com^'); const _engine = new Engine(fs, true); - assert.doesNotThrow(() => fs.addFilter('||another.com^')); + assert.doesNotThrow(() => fs.addFilters('||another.com^')); }); }); @@ -156,7 +137,7 @@ describe('FilterSet survives Engine construction (clone semantics)', () => { describe('Engine.check — basic blocking', () => { it('blocks matching requests, allows non-matching', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), true); assert.equal(engine.check('https://safe.com/t.js', 'https://pub.com', 'script'), false); @@ -164,7 +145,7 @@ describe('Engine.check — basic blocking', () => { it('debug=true returns BlockerResult with filter text', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); const engine = new Engine(fs, true); const result = engine.check( 'https://ads.example.com/t.js', 'https://pub.com', 'script', true, @@ -180,7 +161,7 @@ describe('Engine.check — basic blocking', () => { it('EngineOptions object works as alternative to boolean', () => { const fs = new FilterSet(); - fs.addFilters(['||blocked.com^']); + fs.addFilters('||blocked.com^'); const engine = new Engine(fs, { optimize: false }); assert.equal( engine.check('https://blocked.com/img.png', 'https://pub.com', 'image'), @@ -196,7 +177,7 @@ describe('Engine.check — basic blocking', () => { describe('Engine.check — exception rules', () => { it('exception rule prevents blocking, populates exception field', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^', '@@||ads.example.com^$domain=publisher.com']); + fs.addFilters(['||ads.example.com^', '@@||ads.example.com^$domain=publisher.com'].join('\n')); const engine = new Engine(fs, true); const result = engine.check( 'https://ads.example.com/tracker.js', 'https://publisher.com', 'script', true, @@ -208,7 +189,7 @@ describe('Engine.check — exception rules', () => { it('$important overrides exception rules', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^$important', '@@||ads.example.com^']); + fs.addFilters(['||ads.example.com^$important', '@@||ads.example.com^'].join('\n')); const engine = new Engine(fs, true); const result = engine.check( 'https://ads.example.com/t.js', 'https://pub.com', 'script', true, @@ -226,7 +207,7 @@ describe('Engine.check — exception rules', () => { describe('Engine.check — $third-party and $1p modifiers', () => { it('$third-party rule blocks 3p, allows 1p', () => { const fs = new FilterSet(); - fs.addFilters(['||tracker.com^$third-party']); + fs.addFilters('||tracker.com^$third-party'); const engine = new Engine(fs, true); assert.equal(engine.check('https://tracker.com/t.js', 'https://other.com', 'script'), true); assert.equal(engine.check('https://tracker.com/t.js', 'https://tracker.com', 'script'), false); @@ -234,7 +215,7 @@ describe('Engine.check — $third-party and $1p modifiers', () => { it('$1p rule blocks 1p, allows 3p', () => { const fs = new FilterSet(); - fs.addFilters(['/bad-path$1p']); + fs.addFilters('/bad-path$1p'); const engine = new Engine(fs, true); assert.equal(engine.check('https://example.com/bad-path', 'https://example.com', 'script'), true); assert.equal(engine.check('https://example.com/bad-path', 'https://other.com', 'script'), false); @@ -248,7 +229,7 @@ describe('Engine.check — $third-party and $1p modifiers', () => { describe('Engine.check — type-specific rules', () => { it('$script blocks only script requests', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^$script']); + fs.addFilters('||ads.example.com^$script'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), true); assert.equal(engine.check('https://ads.example.com/t.png', 'https://pub.com', 'image'), false); @@ -256,7 +237,7 @@ describe('Engine.check — type-specific rules', () => { it('$image blocks only image requests', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^$image']); + fs.addFilters('||ads.example.com^$image'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.png', 'https://pub.com', 'image'), true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), false); @@ -264,7 +245,7 @@ describe('Engine.check — type-specific rules', () => { it('$stylesheet blocks only stylesheet requests', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^$stylesheet']); + fs.addFilters('||ads.example.com^$stylesheet'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/s.css', 'https://pub.com', 'stylesheet'), true); assert.equal(engine.check('https://ads.example.com/s.js', 'https://pub.com', 'script'), false); @@ -272,7 +253,7 @@ describe('Engine.check — type-specific rules', () => { it('$xmlhttprequest blocks only XHR requests', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^$xmlhttprequest']); + fs.addFilters('||ads.example.com^$xmlhttprequest'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/api', 'https://pub.com', 'xmlhttprequest'), true); assert.equal(engine.check('https://ads.example.com/api', 'https://pub.com', 'image'), false); @@ -286,7 +267,7 @@ describe('Engine.check — type-specific rules', () => { describe('Engine.check — $domain modifier', () => { it('blocks only when source matches the domain option', () => { const fs = new FilterSet(); - fs.addFilters(['/ads.js$domain=publisher.com']); + fs.addFilters('/ads.js$domain=publisher.com'); const engine = new Engine(fs, true); assert.equal(engine.check('https://cdn.example.com/ads.js', 'https://publisher.com', 'script'), true); assert.equal(engine.check('https://cdn.example.com/ads.js', 'https://other.com', 'script'), false); @@ -294,7 +275,7 @@ describe('Engine.check — $domain modifier', () => { it('~domain excludes the specified domain', () => { const fs = new FilterSet(); - fs.addFilters(['/ads.js$domain=~safe.com']); + fs.addFilters('/ads.js$domain=~safe.com'); const engine = new Engine(fs, true); assert.equal(engine.check('https://cdn.example.com/ads.js', 'https://safe.com', 'script'), false); assert.equal(engine.check('https://cdn.example.com/ads.js', 'https://other.com', 'script'), true); @@ -308,14 +289,14 @@ describe('Engine.check — $domain modifier', () => { describe('Engine.check — $badfilter modifier', () => { it('cancels a matching blocking rule', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^', '||ads.example.com^$badfilter']); + fs.addFilters(['||ads.example.com^', '||ads.example.com^$badfilter'].join('\n')); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), false); }); it('does not cancel a dissimilar rule', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^', '||other.com^$badfilter']); + fs.addFilters(['||ads.example.com^', '||other.com^$badfilter'].join('\n')); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), true); }); @@ -328,7 +309,7 @@ describe('Engine.check — $badfilter modifier', () => { describe('Engine.check — redirect rules', () => { it('redirect field is set when $redirect rule matches and resource is loaded', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^$script,redirect=noopjs']); + fs.addFilters('||ads.example.com^$script,redirect=noopjs'); const engine = new Engine(fs, true); engine.useResources([{ name: 'noopjs', @@ -346,7 +327,7 @@ describe('Engine.check — redirect rules', () => { it('redirect is null when no redirect rule applies', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); const engine = new Engine(fs, true); const result = engine.check( 'https://ads.example.com/t.js', 'https://pub.com', 'script', true, @@ -364,7 +345,7 @@ describe('Engine.check — redirect rules', () => { describe('Engine.check — $removeparam modifier', () => { it('strips the specified parameter, preserves others', () => { const fs = new FilterSet(true); - fs.addFilters(['||example.com^$removeparam=tracking_id']); + fs.addFilters('||example.com^$removeparam=tracking_id'); const engine = new Engine(fs, true); const result = engine.check( 'https://example.com/page?tracking_id=abc&keep=1', @@ -377,7 +358,7 @@ describe('Engine.check — $removeparam modifier', () => { it('rewritten_url is null when the parameter is absent', () => { const fs = new FilterSet(true); - fs.addFilters(['||example.com^$removeparam=tracking_id']); + fs.addFilters('||example.com^$removeparam=tracking_id'); const engine = new Engine(fs, true); const result = engine.check( 'https://example.com/page?unrelated=1', @@ -394,7 +375,7 @@ describe('Engine.check — $removeparam modifier', () => { describe('Engine.check — exception rules with tags', () => { it('tagged exception activates only after enableTag', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^', '@@||ads.example.com^$tag=unbreak']); + fs.addFilters(['||ads.example.com^', '@@||ads.example.com^$tag=unbreak'].join('\n')); const engine = new Engine(fs, true); const before = engine.check( @@ -419,7 +400,7 @@ describe('Engine.check — exception rules with tags', () => { describe('Engine.urlCosmeticResources', () => { it('returns matching hide_selectors for the URL', () => { const fs = new FilterSet(); - fs.addFilters(['example.com##.ad-banner', 'example.com##.sponsored-post']); + fs.addFilters(['example.com##.ad-banner', 'example.com##.sponsored-post'].join('\n')); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://example.com/page'); assert.ok(result.hide_selectors.includes('.ad-banner')); @@ -428,7 +409,7 @@ describe('Engine.urlCosmeticResources', () => { it('returns empty hide_selectors for an unmatched URL', () => { const fs = new FilterSet(); - fs.addFilters(['example.com##.ad-banner']); + fs.addFilters('example.com##.ad-banner'); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://other-site.com/page'); assert.equal(result.hide_selectors.length, 0); @@ -436,7 +417,7 @@ describe('Engine.urlCosmeticResources', () => { it('generichide exception sets generichide=true', () => { const fs = new FilterSet(); - fs.addFilters(['##.generic-ad', '@@||example.com^$generichide']); + fs.addFilters(['##.generic-ad', '@@||example.com^$generichide'].join('\n')); const engine = new Engine(fs, true); assert.equal(engine.urlCosmeticResources('https://example.com/page').generichide, true); assert.equal(engine.urlCosmeticResources('https://other.com/page').generichide, false); @@ -444,7 +425,7 @@ describe('Engine.urlCosmeticResources', () => { it('site-specific unhide (#@#) prevents selector from appearing', () => { const fs = new FilterSet(); - fs.addFilters(['example.com##.ad-banner', 'example.com#@#.ad-banner']); + fs.addFilters(['example.com##.ad-banner', 'example.com#@#.ad-banner'].join('\n')); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://example.com/page'); assert.ok(!result.hide_selectors.includes('.ad-banner')); @@ -452,7 +433,7 @@ describe('Engine.urlCosmeticResources', () => { it('generic unhide (#@#) adds to exceptions list', () => { const fs = new FilterSet(); - fs.addFilters(['##.generic-ad', 'example.com#@#.generic-ad']); + fs.addFilters(['##.generic-ad', 'example.com#@#.generic-ad'].join('\n')); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://example.com/page'); assert.ok(result.exceptions.includes('.generic-ad')); @@ -460,7 +441,7 @@ describe('Engine.urlCosmeticResources', () => { it('negated domain (~sub.example.com) excludes subdomain', () => { const fs = new FilterSet(); - fs.addFilters(['example.com,~sub.example.com##.ad']); + fs.addFilters('example.com,~sub.example.com##.ad'); const engine = new Engine(fs, true); assert.ok(engine.urlCosmeticResources('https://example.com/page').hide_selectors.includes('.ad')); assert.ok(!engine.urlCosmeticResources('https://sub.example.com/page').hide_selectors.includes('.ad')); @@ -471,7 +452,7 @@ describe('Engine.urlCosmeticResources', () => { fs.addFilters([ 'example.com##.items:has-text(Sponsored)', 'example.com##.ad-banner:remove()', - ]); + ].join('\n')); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://example.com/page'); assert.ok(result.procedural_actions.length >= 2); @@ -479,7 +460,7 @@ describe('Engine.urlCosmeticResources', () => { it('scriptlet injection populates injected_script', () => { const fs = new FilterSet(); - fs.addFilters(['example.com##+js(noopjs)']); + fs.addFilters('example.com##+js(noopjs)'); const engine = new Engine(fs, true); // ##+js(noopjs) looks up "noopjs.js"; scriptlets use kind: "template" engine.useResources([{ @@ -500,7 +481,7 @@ describe('Engine.urlCosmeticResources', () => { describe('Engine.hiddenClassIdSelectors', () => { it('returns selectors matching class and id names', () => { const fs = new FilterSet(); - fs.addFilters(['##.a-class', '###simple-id']); + fs.addFilters(['##.a-class', '###simple-id'].join('\n')); const engine = new Engine(fs, true); assert.ok(engine.hiddenClassIdSelectors(['a-class'], [], []).includes('.a-class')); assert.ok(engine.hiddenClassIdSelectors([], ['simple-id'], []).includes('#simple-id')); @@ -508,14 +489,14 @@ describe('Engine.hiddenClassIdSelectors', () => { it('returns empty for unknown class/id names', () => { const fs = new FilterSet(); - fs.addFilters(['##.a-class']); + fs.addFilters('##.a-class'); const engine = new Engine(fs, true); assert.deepEqual(engine.hiddenClassIdSelectors(['unknown'], ['unknown'], []), []); }); it('exceptions array filters out results', () => { const fs = new FilterSet(); - fs.addFilters(['##.a-class']); + fs.addFilters('##.a-class'); const engine = new Engine(fs, true); assert.ok(engine.hiddenClassIdSelectors(['a-class'], [], []).includes('.a-class')); assert.ok(!engine.hiddenClassIdSelectors(['a-class'], [], ['.a-class']).includes('.a-class')); @@ -529,7 +510,7 @@ describe('Engine.hiddenClassIdSelectors', () => { describe('Engine serialization', () => { it('roundtrip preserves blocking and exception rules', () => { const fs = new FilterSet(); - fs.addFilters(['||blocked.com^', '@@||exception.blocked.com^']); + fs.addFilters(['||blocked.com^', '@@||exception.blocked.com^'].join('\n')); const src = new Engine(fs, true); const buf = src.serialize(); @@ -542,7 +523,7 @@ describe('Engine serialization', () => { it('tag enablement is NOT serialized — must re-enable after deserialize', () => { const fs = new FilterSet(); - fs.addFilters(['adv$tag=stuff', '||blocked.com^']); + fs.addFilters(['adv$tag=stuff', '||blocked.com^'].join('\n')); const src = new Engine(fs, true); src.enableTag('stuff'); const buf = src.serialize(); @@ -560,7 +541,7 @@ describe('Engine serialization', () => { it('resources are NOT serialized — must reload after deserialize', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^$script,redirect=noopjs']); + fs.addFilters(['||ads.example.com^$script,redirect=noopjs'].join('\n')); const resource = { name: 'noopjs', aliases: [], @@ -597,7 +578,7 @@ describe('Engine serialization', () => { describe('Engine tags', () => { it('tagged filter is inactive before enableTag, active after', () => { const fs = new FilterSet(); - fs.addFilters(['adv$tag=stuff']); + fs.addFilters(['adv$tag=stuff'].join('\n')); const engine = new Engine(fs, true); assert.equal(engine.check('https://example.com/adv', 'https://example.com', 'other'), false); engine.enableTag('stuff'); @@ -607,7 +588,7 @@ describe('Engine tags', () => { it('clearTags deactivates all enabled tags', () => { const fs = new FilterSet(); - fs.addFilters(['adv$tag=stuff', '||brianbondy.com/$tag=brian']); + fs.addFilters(['adv$tag=stuff', '||brianbondy.com/$tag=brian'].join('\n')); const engine = new Engine(fs, true); engine.enableTag('stuff'); engine.enableTag('brian'); diff --git a/src/blocker.rs b/src/blocker.rs index 19a41c33..b654521a 100644 --- a/src/blocker.rs +++ b/src/blocker.rs @@ -451,15 +451,26 @@ impl Blocker { #[cfg(test)] pub fn new( - network_filters: Vec, + network_filters: impl IntoIterator>, options: &BlockerOptions, ) -> Self { use crate::engine::Engine; use crate::FilterSet; let mut filter_set = FilterSet::new(true); - filter_set.network_filters = network_filters; - let engine = Engine::from_filter_set(filter_set, options.enable_optimizations); + filter_set.add_filters(network_filters, Default::default()); + let engine = Engine::new_with_filter_set(filter_set, options.enable_optimizations); + Self::from_context(engine.filter_data_context()) + } + + #[cfg(test)] + pub fn new_with_list_text(text: String, options: &BlockerOptions) -> Self { + use crate::engine::Engine; + use crate::FilterSet; + + let mut filter_set = FilterSet::new(true); + filter_set.add_filter_list(text, Default::default()); + let engine = Engine::new_with_filter_set(filter_set, options.enable_optimizations); Self::from_context(engine.filter_data_context()) } diff --git a/src/content_blocking.rs b/src/content_blocking.rs index fd4fe905..6ba393af 100644 --- a/src/content_blocking.rs +++ b/src/content_blocking.rs @@ -2,7 +2,7 @@ use crate::filters::cosmetic::CosmeticFilter; use crate::filters::network::{NetworkFilter, NetworkFilterFeaturesMask, NetworkFilterMask}; -use crate::lists::ParsedFilter; +use crate::lists::ParsedLine; use memchr::{memchr as find_char, memmem}; use regex::Regex; @@ -229,13 +229,13 @@ pub enum CbRuleCreationFailure { ProceduralCosmeticFiltersUnsupported, } -impl TryFrom for CbRuleEquivalent { +impl TryFrom for CbRuleEquivalent { type Error = CbRuleCreationFailure; - fn try_from(v: ParsedFilter) -> Result { + fn try_from(v: ParsedLine) -> Result { match v { - ParsedFilter::Network(f) => f.try_into(), - ParsedFilter::Cosmetic(f) => Ok(Self::SingleRule(f.try_into()?)), + ParsedLine::Network(f) => f.try_into(), + ParsedLine::Cosmetic(f) => Ok(Self::SingleRule(f.try_into()?)), } } } diff --git a/src/cosmetic_filter_cache.rs b/src/cosmetic_filter_cache.rs index d60b37a5..5f17c6b5 100644 --- a/src/cosmetic_filter_cache.rs +++ b/src/cosmetic_filter_cache.rs @@ -10,8 +10,6 @@ //! To build `CosmeticFilterCache`, use `CosmeticFilterCacheBuilder`. use crate::cosmetic_filter_utils::decode_script_with_permission; -#[cfg(test)] -use crate::filters::cosmetic::CosmeticFilter; use crate::filters::cosmetic::{CosmeticFilterAction, CosmeticFilterOperator}; use crate::filters::filter_data_context::FilterDataContextRef; @@ -134,13 +132,13 @@ impl CosmeticFilterCache { } #[cfg(test)] - pub fn from_rules(rules: Vec) -> Self { + pub fn from_rules(rules: impl IntoIterator>) -> Self { use crate::engine::Engine; use crate::FilterSet; - let mut filter_set = FilterSet::new(true); - filter_set.cosmetic_filters = rules; - let engine = Engine::from_filter_set(filter_set, true); + let mut filter_set = FilterSet::new(false); + filter_set.add_filters(rules, Default::default()); + let engine = Engine::new_with_filter_set(filter_set, true); engine.cosmetic_cache() } diff --git a/src/engine.rs b/src/engine.rs index 7867e0cd..4ca8aa8a 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -4,18 +4,18 @@ use crate::blocker::{Blocker, BlockerResult}; use crate::cosmetic_filter_cache::{CosmeticFilterCache, UrlSpecificResources}; use crate::cosmetic_filter_cache_builder::CosmeticFilterCacheBuilder; use crate::data_format::{deserialize_dat_file, serialize_dat_file, DeserializationError}; -use crate::filters::cosmetic::CosmeticFilter; use crate::filters::fb_builder::EngineFlatBuilder; use crate::filters::fb_network_builder::NetworkRulesBuilder; use crate::filters::filter_data_context::{FilterDataContext, FilterDataContextRef}; -use crate::filters::network::NetworkFilter; use crate::flatbuffers::containers::flat_serialize::FlatSerialize; use crate::flatbuffers::unsafe_tools::VerifiedFlatbufferMemory; -use crate::lists::{FilterSet, ParseOptions}; +use crate::lists::{parse_filter, FilterSet, ParseOptions, ParsedLine}; use crate::regex_manager::RegexManagerDiscardPolicy; use crate::request::Request; use crate::resources::{Resource, ResourceStorage, ResourceStorageBackend}; +use crate::filters::{cosmetic::CosmeticFilter, network::NetworkFilter}; + use std::collections::HashSet; /// Drives high-level blocking logic and is responsible for loading filter lists into an optimized @@ -66,38 +66,28 @@ pub struct EngineDebugInfo { impl Default for Engine { fn default() -> Self { - Self::from_filter_set(FilterSet::new(false), false) + Self::new_with_filter_set(FilterSet::new(false), false) } } impl Engine { - /// Loads rules in a single format, enabling optimizations and discarding debug information. - pub fn from_rules( - rules: impl IntoIterator>, - opts: ParseOptions, - ) -> Self { - let mut filter_set = FilterSet::new(false); - filter_set.add_filters(rules, opts); - Self::from_filter_set(filter_set, true) - } - - /// Loads rules, enabling optimizations and including debug information. - pub fn from_rules_debug( - rules: impl IntoIterator>, - opts: ParseOptions, - ) -> Self { - Self::from_rules_parametrised(rules, opts, true, true) + /// A helper for tests and benchmarks. Use [`Engine::new_with_filter_set`] instead. + #[doc(hidden)] + pub fn new_with_list_text(list_text: impl Into, opts: ParseOptions) -> Self { + Self::new_with_list_text_parametrised(list_text.into(), opts, false, true) } - pub fn from_rules_parametrised( - filter_rules: impl IntoIterator>, + /// A helper for tests and benchmarks. Use [`Engine::new_with_filter_set`] instead. + #[doc(hidden)] + pub fn new_with_list_text_parametrised( + list_text: impl Into, opts: ParseOptions, debug: bool, optimize: bool, ) -> Self { let mut filter_set = FilterSet::new(debug); - filter_set.add_filters(filter_rules, opts); - Self::from_filter_set(filter_set, optimize) + filter_set.add_filter_list(list_text.into(), opts); + Self::new_with_filter_set(filter_set, optimize) } #[cfg(test)] @@ -110,15 +100,13 @@ impl Engine { self.filter_data_context } - /// Loads rules from the given `FilterSet`. It is recommended to use a `FilterSet` when adding - /// rules from multiple sources. - pub fn from_filter_set(set: FilterSet, optimize: bool) -> Self { - let FilterSet { - network_filters, - cosmetic_filters, - .. - } = set; - + /// A helper for tests and benchmarks. Use [`Engine::new_with_filter_set`] instead. + #[doc(hidden)] + pub fn new_with_parsed_rules( + network_filters: Vec, + cosmetic_filters: Vec, + optimize: bool, + ) -> Self { let memory = make_flatbuffer(network_filters, cosmetic_filters, optimize); let filter_data_context = FilterDataContext::new(memory); @@ -133,6 +121,24 @@ impl Engine { } } + /// Loads rules from the given `FilterSet`. + pub fn new_with_filter_set(set: FilterSet, optimize: bool) -> Self { + let mut network_filters = vec![]; + let mut cosmetic_filters = vec![]; + + for list_source in set.list_sources { + for line in list_source.list_text.lines() { + let parsed_line = parse_filter(line, set.debug, list_source.parse_options); + match parsed_line { + Ok(ParsedLine::Network(filter)) => network_filters.push(filter), + Ok(ParsedLine::Cosmetic(filter)) => cosmetic_filters.push(filter), + Err(_) => {} + } + } + } + Self::new_with_parsed_rules(network_filters, cosmetic_filters, optimize) + } + /// Check if a request for a network resource from `url`, of type `request_type`, initiated by /// `source_url`, should be blocked. pub fn check_network_request(&self, request: &Request) -> BlockerResult { diff --git a/src/filters/fb_builder.rs b/src/filters/fb_builder.rs index 6ea470dd..b6a757e4 100644 --- a/src/filters/fb_builder.rs +++ b/src/filters/fb_builder.rs @@ -4,8 +4,7 @@ use std::collections::HashMap; use flatbuffers::WIPOffset; -use crate::filters::fb_network_builder::NetworkFilterListBuilder; -use crate::flatbuffers::containers::flat_serialize::{FlatBuilder, WIPFlatVec}; +use crate::flatbuffers::containers::flat_serialize::FlatBuilder; use crate::flatbuffers::unsafe_tools::VerifiedFlatbufferMemory; use crate::utils::Hash; @@ -31,7 +30,9 @@ impl<'a> EngineFlatBuilder<'a> { pub fn finish( &mut self, - network_rules: WIPFlatVec<'a, NetworkFilterListBuilder, EngineFlatBuilder<'a>>, + network_rules: flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, cosmetic_rules: WIPOffset>, ) -> VerifiedFlatbufferMemory { let unique_domains_hashes = diff --git a/src/filters/network.rs b/src/filters/network.rs index c78e3ea8..fef5d53d 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -944,9 +944,7 @@ impl NetworkFilter { #[cfg(test)] pub(crate) fn matches_test(&self, request: &request::Request) -> bool { - let filter_set = crate::FilterSet::new_with_rules(vec![self.clone()], vec![], true); - let engine = crate::Engine::from_filter_set(filter_set, true); - + let engine = crate::Engine::new_with_parsed_rules(vec![self.clone()], vec![], true); if self.is_exception() { engine.check_network_request_exceptions(request) } else { diff --git a/src/lists.rs b/src/lists.rs index c52ce660..cac924ae 100644 --- a/src/lists.rs +++ b/src/lists.rs @@ -69,6 +69,12 @@ impl Default for ParseOptions { } } +#[derive(Clone)] +pub(crate) struct ListSource { + pub(crate) list_text: String, + pub(crate) parse_options: ParseOptions, +} + /// Manages a set of rules to be added to an [`crate::Engine`]. /// /// To be able to efficiently handle special options like `$badfilter`, and to allow optimizations, @@ -76,33 +82,28 @@ impl Default for ParseOptions { /// compound list from multiple different sources before compiling the rules into an `Engine`. #[derive(Clone)] pub struct FilterSet { - debug: bool, - pub(crate) network_filters: Vec, - pub(crate) cosmetic_filters: Vec, + pub(crate) debug: bool, + pub(crate) list_sources: Vec, } /// Collects metadata for the list by reading just until the first non-comment line. pub fn read_list_metadata(list: &str) -> FilterListMetadata { let mut metadata = FilterListMetadata::default(); - - // uBO only searches within the first 1024 characters; the same optimization can be useful here - let mut cutoff = list.len().min(1024); - - while !list.is_char_boundary(cutoff) { - cutoff -= 1; - } - - // String slice is safe here because `cutoff` is guaranteed to be a character boundary - for line in list[0..cutoff].lines() { - if line.starts_with('!') { - metadata.try_add(line); - } else if line.starts_with('[') { + let mut chars_read = 0; + for line in list.lines() { + if chars_read > 1024 { + break; // stop parsing metadata + } + chars_read += line.len(); + if line.starts_with("[") { continue; + } + if let Some(comment_content) = line.strip_prefix("!") { + metadata.try_add_metadata(comment_content); } else { - break; + break; // stop parsing metadata } } - metadata } @@ -188,27 +189,16 @@ pub struct FilterListMetadata { } impl FilterListMetadata { - /// Attempts to add a line of a filter list to this collection of metadata. Only comment lines - /// with valid metadata content will be added. Previously added information will not be - /// rewritten. - fn try_add(&mut self, line: &str) { - if let Some(kv) = line.strip_prefix("! ") { - if let Some((key, value)) = kv.split_once(": ") { - match key { - "Homepage" if self.homepage.is_none() => { - self.homepage = Some(value.to_string()) - } - "Title" if self.title.is_none() => self.title = Some(value.to_string()), - "Expires" if self.expires.is_none() => { - if let Ok(expires) = ExpiresInterval::try_from(value) { - self.expires = Some(expires); - } - } - "Redirect" if self.redirect.is_none() => { - self.redirect = Some(value.to_string()) - } - _ => (), + pub(crate) fn try_add_metadata(&mut self, metadata: &str) { + if let Some((key, value)) = metadata.trim().split_once(": ") { + match key { + "Homepage" if self.homepage.is_none() => self.homepage = Some(value.to_string()), + "Title" if self.title.is_none() => self.title = Some(value.to_string()), + "Expires" if self.expires.is_none() => { + self.expires = Some(ExpiresInterval::try_from(value).unwrap()) } + "Redirect" if self.redirect.is_none() => self.redirect = Some(value.to_string()), + _ => (), } } } @@ -221,54 +211,48 @@ impl FilterSet { pub fn new(debug: bool) -> Self { Self { debug, - network_filters: Vec::new(), - cosmetic_filters: Vec::new(), - } - } - - // Used in benchmarks to avoid parsing the rules twice. - #[doc(hidden)] - pub fn new_with_rules( - network_filters: Vec, - cosmetic_filters: Vec, - debug: bool, - ) -> Self { - Self { - debug, - network_filters, - cosmetic_filters, + list_sources: Vec::new(), } } /// Adds the contents of an entire filter list to this `FilterSet`. Filters that cannot be /// parsed successfully are ignored. Returns any discovered metadata about the list of rules /// added. - pub fn add_filter_list(&mut self, filter_list: &str, opts: ParseOptions) -> FilterListMetadata { - self.add_filters(filter_list.lines(), opts) + pub fn add_filter_list(&mut self, list_text: String, opts: ParseOptions) -> FilterListMetadata { + let metadata = match opts.format { + FilterFormat::Standard => read_list_metadata(&list_text), + FilterFormat::Hosts => FilterListMetadata::default(), + }; + self.list_sources.push(ListSource { + list_text, + parse_options: opts, + }); + metadata } /// Adds a collection of filter rules to this `FilterSet`. Filters that cannot be parsed /// successfully are ignored. Returns any discovered metadata about the list of rules added. + #[cfg(test)] pub fn add_filters( &mut self, filters: impl IntoIterator>, opts: ParseOptions, - ) -> FilterListMetadata { - let (metadata, parsed_network_filters, parsed_cosmetic_filters) = - parse_filters_with_metadata(filters, self.debug, opts); - self.network_filters.extend(parsed_network_filters); - self.cosmetic_filters.extend(parsed_cosmetic_filters); - metadata + ) { + let list_text = filters.into_iter().fold(String::new(), |mut acc, rule| { + acc.push_str(rule.as_ref()); + acc.push('\n'); + acc + }); + self.list_sources.push(ListSource { + list_text, + parse_options: opts, + }); } /// Adds the string representation of a single filter rule to this `FilterSet`. - pub fn add_filter(&mut self, filter: &str, opts: ParseOptions) -> Result<(), FilterParseError> { - let filter_parsed = parse_filter(filter, self.debug, opts); - match filter_parsed? { - ParsedFilter::Network(filter) => self.network_filters.push(filter), - ParsedFilter::Cosmetic(filter) => self.cosmetic_filters.push(filter), - } - Ok(()) + #[cfg(test)] + pub fn add_filter(&mut self, filter: &str, opts: ParseOptions) { + self.add_filters([filter], opts) } /// Consumes this `FilterSet`, returning an equivalent list of content blocking rules and a @@ -291,9 +275,20 @@ impl FilterSet { return Err(()); } + let mut network_filters = vec![]; + let mut cosmetic_filters = vec![]; + for list_source in self.list_sources.iter() { + let list_text = list_source.list_text.lines(); + let parse_options = list_source.parse_options; + let (list_network_filters, list_cosmetic_filters) = + parse_filters(list_text, self.debug, parse_options); + network_filters.extend(list_network_filters); + cosmetic_filters.extend(list_cosmetic_filters); + } + // Store bad filter id to skip them later. let mut bad_filter_ids = HashSet::new(); - for filter in self.network_filters.iter() { + for filter in network_filters.iter() { if filter.is_badfilter() { bad_filter_ids.insert(filter.get_id()); } @@ -304,7 +299,7 @@ impl FilterSet { let mut filters_used = vec![]; - self.network_filters.into_iter().for_each(|filter| { + network_filters.into_iter().for_each(|filter| { // Don't process bad filter rules or matching bad filter rules. if bad_filter_ids.contains(&filter.get_id()) || filter.is_badfilter() { return; @@ -329,7 +324,7 @@ impl FilterSet { let add_fp_document_exception = !filters_used.is_empty(); - self.cosmetic_filters.into_iter().for_each(|filter| { + cosmetic_filters.into_iter().for_each(|filter| { let original_rule = *filter .raw_line .clone() @@ -391,21 +386,21 @@ pub enum FilterType { NotSupported, } -/// Successful result of parsing a single filter rule -pub enum ParsedFilter { +/// Successful result of parsing a single line from a filter list +pub enum ParsedLine { Network(NetworkFilter), Cosmetic(CosmeticFilter), } -impl From for ParsedFilter { +impl From for ParsedLine { fn from(v: NetworkFilter) -> Self { - ParsedFilter::Network(v) + ParsedLine::Network(v) } } -impl From for ParsedFilter { +impl From for ParsedLine { fn from(v: CosmeticFilter) -> Self { - ParsedFilter::Cosmetic(v) + ParsedLine::Cosmetic(v) } } @@ -420,6 +415,8 @@ pub enum FilterParseError { Unsupported, #[error("empty")] Empty, + #[error("invalid expires interval")] + InvalidExpiresInterval, } impl From for FilterParseError { @@ -434,12 +431,12 @@ impl From for FilterParseError { } } -/// Parse a single filter rule +/// Parse a single line from a filter list pub fn parse_filter( line: &str, debug: bool, opts: ParseOptions, -) -> Result { +) -> Result { let filter = line.trim(); if filter.is_empty() { @@ -516,34 +513,16 @@ pub fn parse_filters( debug: bool, opts: ParseOptions, ) -> (Vec, Vec) { - let (_metadata, network_filters, cosmetic_filters) = - parse_filters_with_metadata(list, debug, opts); - - (network_filters, cosmetic_filters) -} - -/// Parse an entire list of filters, ignoring any errors -pub fn parse_filters_with_metadata( - list: impl IntoIterator>, - debug: bool, - opts: ParseOptions, -) -> (FilterListMetadata, Vec, Vec) { - let mut metadata = FilterListMetadata::default(); - - let list_iter = list.into_iter(); - - let (network_filters, cosmetic_filters): (Vec<_>, Vec<_>) = list_iter - .map(|line| { - metadata.try_add(line.as_ref()); - parse_filter(line.as_ref(), debug, opts) + let (network_filters, cosmetic_filters): (Vec<_>, Vec<_>) = list + .into_iter() + .filter_map(|line| match parse_filter(line.as_ref(), debug, opts) { + Ok(ParsedLine::Network(f)) => Some(Either::Left(f)), + Ok(ParsedLine::Cosmetic(f)) => Some(Either::Right(f)), + _ => None, }) - .filter_map(Result::ok) - .partition_map(|filter| match filter { - ParsedFilter::Network(f) => Either::Left(f), - ParsedFilter::Cosmetic(f) => Either::Right(f), - }); + .partition_map(|x| x); - (metadata, network_filters, cosmetic_filters) + (network_filters, cosmetic_filters) } /// Given a single line, checks if this would likely be a cosmetic filter, a diff --git a/src/resources/mod.rs b/src/resources/mod.rs index d3dc7c52..5d29a825 100644 --- a/src/resources/mod.rs +++ b/src/resources/mod.rs @@ -39,19 +39,19 @@ use serde::{Deserialize, Serialize}; /// # use adblock::lists::ParseOptions; /// # use adblock::resources::{MimeType, PermissionMask, Resource, ResourceStorage, ResourceType}; /// # let mut filter_set = adblock::lists::FilterSet::default(); -/// # let untrusted_filters = vec![""]; -/// # let trusted_filters = vec![""]; +/// # let untrusted_filters = "".to_string(); +/// # let trusted_filters = "".to_string(); /// const COOKIE_ACCESS: PermissionMask = PermissionMask::from_bits(0b00000001); /// const LOCALSTORAGE_ACCESS: PermissionMask = PermissionMask::from_bits(0b00000010); /// /// // `untrusted_filters` will not be able to use privileged scriptlet injections. -/// filter_set.add_filters( +/// filter_set.add_filter_list( /// untrusted_filters, /// Default::default(), /// ); /// // `trusted_filters` will be able to inject scriptlets requiring `COOKIE_ACCESS` /// // permissions or `LOCALSTORAGE_ACCESS` permissions. -/// filter_set.add_filters( +/// filter_set.add_filter_list( /// trusted_filters, /// ParseOptions { /// permissions: COOKIE_ACCESS | LOCALSTORAGE_ACCESS, @@ -59,7 +59,7 @@ use serde::{Deserialize, Serialize}; /// }, /// ); /// -/// let mut engine = Engine::from_filter_set(filter_set, true); +/// let mut engine = Engine::new_with_filter_set(filter_set, true); /// // The `trusted-set-cookie` scriptlet cannot be injected without `COOKIE_ACCESS` /// // permission. /// engine.use_resources([Resource { diff --git a/tests/legacy_harness.rs b/tests/legacy_harness.rs index 1cb133d6..3d7dba43 100644 --- a/tests/legacy_harness.rs +++ b/tests/legacy_harness.rs @@ -34,7 +34,7 @@ mod legacy_test_filters { filter.filter ); - let engine = Engine::from_rules_debug([raw_filter], Default::default()); + let engine = Engine::new_with_list_text(raw_filter, Default::default()); for to_block in blocked { assert!( @@ -306,8 +306,8 @@ mod legacy_test_filters { // explicit, separate testcase construction of the "script" option as it is not the deafult let request = Request::new("http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html#xpc=sf-gdn-exp-2&p=http%3A//slashdot.org;", "https://this-is-always-third-party.com", "script").unwrap(); - let engine = Engine::from_rules_debug( - ["||googlesyndication.com/safeframe/$third-party,script"], + let engine = Engine::new_with_list_text( + "||googlesyndication.com/safeframe/$third-party,script", Default::default(), ); assert!(engine.check_network_request(&request).matched); @@ -324,7 +324,7 @@ mod legacy_check_match { not_blocked: &[&'a str], tags: &[&'a str], ) { - let mut engine = Engine::from_rules(rules, Default::default()); // first one with the provided rules + let mut engine = Engine::new_with_list_text(rules.join("\n"), Default::default()); // first one with the provided rules engine.use_tags(tags); let mut engine_deserialized = Engine::default(); // second empty @@ -395,11 +395,12 @@ mod legacy_check_match { { // Explicitly write out the full case instead of using check_match helper // or tweaking it to allow passing in the source domain for this one case - let engine = Engine::from_rules( + let engine = Engine::new_with_list_text( [ "/ads/freewheel/*", "@@||turner.com^*/ads/freewheel/*/AdManager.js$domain=cnn.com", - ], + ] + .join("\n"), Default::default(), ); let mut engine_deserialized = Engine::default(); // second empty @@ -505,7 +506,7 @@ mod legacy_check_options { use adblock::Engine; fn check_option_rule<'a>(rules: &[&'a str], tests: &[(&'a str, &'a str, &'a str, bool)]) { - let engine = Engine::from_rules(rules, Default::default()); // first one with the provided rules + let engine = Engine::new_with_list_text(rules.join("\n"), Default::default()); // first one with the provided rules for (url, source_url, request_type, expectation) in tests { let request = Request::new(url, source_url, request_type).unwrap(); @@ -851,8 +852,8 @@ mod legacy_misc_tests { #[test] fn demo_app() { // Demo app test - let engine = Engine::from_rules( - ["||googlesyndication.com/safeframe/$third-party"], + let engine = Engine::new_with_list_text( + "||googlesyndication.com/safeframe/$third-party", Default::default(), ); @@ -882,12 +883,13 @@ mod legacy_misc_tests { #[test] fn serialization_tests() { - let engine = Engine::from_rules_parametrised( + let engine = Engine::new_with_list_text_parametrised( [ "||googlesyndication.com$third-party", "@@||googlesyndication.ca", "a$explicitcancel", - ], + ] + .join("\n"), Default::default(), true, false, @@ -974,12 +976,15 @@ mod legacy_misc_tests { #[test] fn find_matching_filters() { - let engine = Engine::from_rules_debug( + let engine = Engine::new_with_list_text_parametrised( [ "||googlesyndication.com/safeframe/$third-party", "||brianbondy.com/ads", - ], + ] + .join("\n"), Default::default(), + true, + true, ); let current_page_frame = "http://slashdot.org"; @@ -1018,13 +1023,16 @@ mod legacy_misc_tests { #[test] fn find_matching_filters_exceptions() { - let engine = Engine::from_rules_debug( + let engine = Engine::new_with_list_text_parametrised( [ "||googlesyndication.com/safeframe/$third-party", "||brianbondy.com/ads", "@@safeframe", - ], + ] + .join("\n"), Default::default(), + true, + true, ); let current_page_frame = "http://slashdot.org"; @@ -1056,9 +1064,11 @@ mod legacy_misc_tests { #[test] fn matches_with_filter_info_preserves_important() { // exceptions have not effect if important filter matches - let engine = Engine::from_rules_debug( - ["||brianbondy.com^$important", "@@||brianbondy.com^"], + let engine = Engine::new_with_list_text_parametrised( + ["||brianbondy.com^$important", "@@||brianbondy.com^"].join("\n"), Default::default(), + true, + true, ); let request = diff --git a/tests/live.rs b/tests/live.rs index 444a31dc..3f48acd5 100644 --- a/tests/live.rs +++ b/tests/live.rs @@ -124,8 +124,8 @@ fn get_all_filters() -> &'static adblock::lists::FilterSet { .await .iter() .for_each(|(format, list)| { - filter_set.add_filters( - list.lines().map(|s| s.to_owned()).collect::>(), + filter_set.add_filter_list( + list.clone(), adblock::lists::ParseOptions { format: *format, ..Default::default() @@ -150,7 +150,7 @@ fn troubleshoot() { } fn get_blocker_engine() -> Engine { - let mut engine = Engine::from_filter_set(get_all_filters().clone(), true); + let mut engine = Engine::new_with_filter_set(get_all_filters().clone(), true); engine.use_tags(&["fb-embeds", "twitter-embeds"]); @@ -278,7 +278,7 @@ fn check_live_redirects() { /// Ensure that one engine's serialization result can be exactly reproduced by another engine after /// deserializing from it. fn stable_serialization_through_load() { - let engine1 = Engine::from_filter_set(get_all_filters().clone(), true); + let engine1 = Engine::new_with_filter_set(get_all_filters().clone(), true); let ser1 = engine1.serialize().to_vec(); let mut engine2 = Engine::default(); diff --git a/tests/matching.rs b/tests/matching.rs index d72c417f..14759d8c 100644 --- a/tests/matching.rs +++ b/tests/matching.rs @@ -77,8 +77,7 @@ fn check_filter_matching() { .set(NetworkFilterMask::IS_EXCEPTION, false); filters.push(original_filter); } - let filter_set = adblock::FilterSet::new_with_rules(filters, vec![], true); - let engine = adblock::Engine::from_filter_set(filter_set, true); + let engine = adblock::Engine::new_with_parsed_rules(filters, vec![], true); let request_res = Request::new(&req.url, &req.sourceUrl, &req.r#type); // The dataset has cases where URL is set to just "http://" or "https://", which we do not support @@ -121,7 +120,7 @@ fn check_engine_matching() { } for filter in req.filters { let opts = ParseOptions::default(); - let mut engine = Engine::from_rules_debug(std::slice::from_ref(&filter), opts); + let mut engine = Engine::new_with_list_text(filter.clone(), opts); let resources = build_resources_from_filters(std::slice::from_ref(&filter)); engine.use_resources(resources); @@ -191,10 +190,10 @@ fn check_rule_matching_browserlike() { } fn load_requests() -> Vec { - let requests_str = rules_from_lists(&["data/requests.json"]); + let requests_str = rules_from_lists(["data/requests.json"]); requests_str - .into_iter() - .filter_map(|r| serde_json::from_str(&r).ok()) + .lines() + .filter_map(|r| serde_json::from_str(r).ok()) .collect() } @@ -214,7 +213,7 @@ fn check_rule_matching_browserlike() { let requests = load_requests(); let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); - let engine = Engine::from_rules(rules, Default::default()); + let engine = Engine::new_with_list_text(rules, Default::default()); let (blocked, passes) = bench_rule_matching_browserlike(&engine, &requests); let msg = "The number of blocked/passed requests has changed. ".to_string() + "If this is expected, update the expected values in the test."; diff --git a/tests/simple_use.rs b/tests/simple_use.rs index 4b4f9259..1b05780a 100644 --- a/tests/simple_use.rs +++ b/tests/simple_use.rs @@ -10,7 +10,7 @@ fn check_simple_use() { "-advertisement/script.", ]; - let engine = Engine::from_rules(rules, Default::default()); + let engine = Engine::new_with_list_text(rules.join("\n"), Default::default()); let request = Request::new( "http://example.com/-advertisement-icon.", diff --git a/tests/test_utils.rs b/tests/test_utils.rs index a4e3a3db..71d5d3ff 100644 --- a/tests/test_utils.rs +++ b/tests/test_utils.rs @@ -2,18 +2,11 @@ //! needed outside of this directory. #[cfg(not(target_arch = "wasm32"))] -pub fn rules_from_lists( - lists: impl IntoIterator>, -) -> impl Iterator { - fn read_file_lines(filename: &str) -> impl Iterator { - use std::fs::File; - use std::io::{BufRead, BufReader}; - - let reader = BufReader::new(File::open(filename).unwrap()); - reader.lines().map(|r| r.unwrap()) +pub fn rules_from_lists(list_files: impl IntoIterator>) -> String { + let mut contents = String::new(); + for file in list_files { + contents.push_str(&std::fs::read_to_string(file.as_ref()).unwrap()); + contents.push('\n'); } - - lists - .into_iter() - .flat_map(|filename| read_file_lines(filename.as_ref())) + contents } diff --git a/tests/ublock-coverage.rs b/tests/ublock-coverage.rs index 94aa7aca..02de72d7 100644 --- a/tests/ublock-coverage.rs +++ b/tests/ublock-coverage.rs @@ -48,11 +48,11 @@ fn get_blocker_engine() -> Engine { "data/regression-testing/easyprivacy.txt", ]); - Engine::from_rules_parametrised(rules, Default::default(), true, false) + Engine::new_with_list_text_parametrised(rules, Default::default(), true, false) } fn get_blocker_engine_default(extra_rules: impl IntoIterator>) -> Engine { - let rules = rules_from_lists([ + let mut rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", "data/uBlockOrigin/unbreak.txt", @@ -62,16 +62,19 @@ fn get_blocker_engine_default(extra_rules: impl IntoIterator>, requests: &[(Request, bool)], ) { - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); requests.iter().for_each(|(req, expected_result)| { let matched_rule = blocker.check(req, &Default::default()); @@ -78,13 +73,11 @@ mod blocker_tests { ) .unwrap(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::in_memory_from_resources([Resource::simple( "noop-0.1s.mp3", crate::resources::MimeType::AudioMp3, @@ -118,13 +111,11 @@ mod blocker_tests { ) .unwrap(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::in_memory_from_resources([Resource::simple( "noop-0.1s.mp3", crate::resources::MimeType::AudioMp3, @@ -153,13 +144,11 @@ mod blocker_tests { let request = Request::new("https://www3.doubleclick.net", "https://lineups.fun", "xhr").unwrap(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::in_memory_from_resources([Resource::simple( "noop.txt", crate::resources::MimeType::TextPlain, @@ -225,13 +214,11 @@ mod blocker_tests { fn trailing_dot_domain() { let filters = ["||dot.example.com.^", "||test.example.com^"]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let request = Request::new( "https://dot.example.com", @@ -311,13 +298,11 @@ mod blocker_tests { ), ]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::default(); url_results.into_iter().for_each(|(req, expected_result)| { @@ -346,13 +331,11 @@ mod blocker_tests { "^first-party-only^$csp=script-src 'none',1p", ]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: false, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); { // No directives should be returned for requests that are not `document` or `subdocument` content types. @@ -540,13 +523,11 @@ mod blocker_tests { "$removeparam=testCase,~xhr", ]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::in_memory_from_resources([Resource::simple( "noopjs", crate::resources::MimeType::ApplicationJavascript, @@ -938,13 +919,11 @@ mod blocker_tests { .map(|s| format!("*$removeparam={s}")) .collect::>(); - let (network_filters, _) = parse_filters(&filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::default(); for (original, expected) in testcases.into_iter() { @@ -968,13 +947,11 @@ mod blocker_tests { fn test_removeparam_same_tokens() { let filters = ["$removeparam=example1_", "$removeparam=example1-"]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let result = blocker.check( &Request::new( @@ -1000,13 +977,11 @@ mod blocker_tests { "@@^exceptc20^$redirect-rule=c:20", ]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); fn simple_resource(identifier: &str) -> Resource { Resource::simple( identifier, @@ -1162,13 +1137,11 @@ mod blocker_tests { }) .collect(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let mut blocker = Blocker::new(network_filters, &blocker_options); + let mut blocker = Blocker::new(filters, &blocker_options); let resources = Default::default(); blocker.enable_tags(&["stuff"]); assert_eq!( @@ -1215,13 +1188,11 @@ mod blocker_tests { }) .collect(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let mut blocker = Blocker::new(network_filters, &blocker_options); + let mut blocker = Blocker::new(filters, &blocker_options); let resources = Default::default(); blocker.enable_tags(&["stuff"]); blocker.enable_tags(&["brian"]); @@ -1269,13 +1240,11 @@ mod blocker_tests { }) .collect(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let mut blocker = Blocker::new(network_filters, &blocker_options); + let mut blocker = Blocker::new(filters, &blocker_options); let resources = Default::default(); blocker.enable_tags(&["brian", "stuff"]); assert_eq!( @@ -1310,12 +1279,7 @@ mod blocker_tests { enable_optimizations: true, }; - let mut filter_set = crate::lists::FilterSet::new(true); - filter_set - .add_filter("@@*ad_banner.png", Default::default()) - .unwrap(); - - let blocker = Blocker::new(filter_set.network_filters, &blocker_options); + let blocker = Blocker::new(["@@*ad_banner.png"], &blocker_options); let resources = Default::default(); @@ -1337,12 +1301,7 @@ mod blocker_tests { enable_optimizations: true, }; - let mut filter_set = crate::lists::FilterSet::new(true); - filter_set - .add_filter("@@||example.com$generichide", Default::default()) - .unwrap(); - - let blocker = Blocker::new(filter_set.network_filters, &blocker_options); + let blocker = Blocker::new(["@@||example.com$generichide\n"], &blocker_options); assert!(blocker.check_generic_hide( &Request::new("https://example.com", "https://example.com", "other").unwrap() @@ -1356,10 +1315,8 @@ mod placeholder_string_tests { #[test] fn test_constant_placeholder_string() { let mut filter_set = crate::lists::FilterSet::new(false); - filter_set - .add_filter("||example.com^", Default::default()) - .unwrap(); - let engine = crate::Engine::from_filter_set(filter_set, true); + filter_set.add_filter_list("||example.com^\n".to_string(), Default::default()); + let engine = crate::Engine::new_with_filter_set(filter_set, true); let block = engine.check_network_request( &crate::request::Request::new("https://example.com", "https://example.com", "document") .unwrap(), @@ -1472,7 +1429,7 @@ mod legacy_rule_parsing_tests { let rules = rules_from_lists(rule_lists); let (network_filters, cosmetic_filters) = parse_filters( - rules, + rules.lines(), true, ParseOptions { format, @@ -1498,7 +1455,7 @@ mod legacy_rule_parsing_tests { enable_optimizations: false, // optimizations will reduce number of rules }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new_with_list_text(rules, &blocker_options); // Some filters in the filter_map are pointed at by multiple tokens, increasing the total number of items assert!( diff --git a/tests/unit/cosmetic_filter_cache.rs b/tests/unit/cosmetic_filter_cache.rs index 7b937f46..cb544336 100644 --- a/tests/unit/cosmetic_filter_cache.rs +++ b/tests/unit/cosmetic_filter_cache.rs @@ -82,18 +82,12 @@ mod cosmetic_cache_tests { use crate::resources::Resource; use base64::{engine::Engine as _, prelude::BASE64_STANDARD}; - fn cache_from_rules(rules: Vec<&str>) -> CosmeticFilterCache { - let parsed_rules = rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(); - - CosmeticFilterCache::from_rules(parsed_rules) - } - #[test] fn exceptions() { - let cfcache = cache_from_rules(vec!["~example.com##.item", "sub.example.com#@#.item2"]); + let cfcache = CosmeticFilterCache::from_rules(vec![ + "~example.com##.item", + "sub.example.com#@#.item2", + ]); let resources = ResourceStorage::default(); let out = cfcache.hostname_cosmetic_resources(&resources, "test.com", false); @@ -111,7 +105,7 @@ mod cosmetic_cache_tests { #[test] fn exceptions2() { - let cfcache = cache_from_rules(vec!["example.com,~sub.example.com##.item"]); + let cfcache = CosmeticFilterCache::from_rules(vec!["example.com,~sub.example.com##.item"]); let resources = ResourceStorage::default(); let out = cfcache.hostname_cosmetic_resources(&resources, "test.com", false); @@ -130,7 +124,7 @@ mod cosmetic_cache_tests { #[test] fn style_exceptions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##.element:style(background: #fff)", "sub.test.example.com#@#.element:style(background: #fff)", "a1.sub.example.com##.element", @@ -195,7 +189,7 @@ mod cosmetic_cache_tests { fn script_exceptions() { use crate::resources::{MimeType, ResourceType}; - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##+js(set-constant.js, atob, trueFunc)", "sub.test.example.com#@#+js(set-constant.js, atob, trueFunc)", "cosmetic.net##+js(nowebrtc.js)", @@ -259,7 +253,7 @@ mod cosmetic_cache_tests { #[test] fn remove_exceptions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##.element:remove()", "sub.test.example.com#@#.element:remove()", "a1.sub.example.com##.element", @@ -308,7 +302,7 @@ mod cosmetic_cache_tests { #[test] fn remove_attr_exceptions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##.element:remove-attr(style)", "sub.test.example.com#@#.element:remove-attr(style)", "a1.sub.example.com##.element", @@ -371,7 +365,7 @@ mod cosmetic_cache_tests { #[test] fn remove_class_exceptions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##.element:remove-class(overlay)", "sub.test.example.com#@#.element:remove-class(overlay)", "a1.sub.example.com##.element", @@ -435,7 +429,7 @@ mod cosmetic_cache_tests { #[test] #[cfg(feature = "css-validation")] fn procedural_actions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com##div:has(video):remove()", "example.com##div:has-text(Ad):remove()", "example.com##div:has-text(Sponsored) > p", @@ -501,12 +495,7 @@ mod cosmetic_cache_tests { "##.children .including #simple-id", "##a.a-class", ]; - let cfcache = CosmeticFilterCache::from_rules( - rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(), - ); + let cfcache = CosmeticFilterCache::from_rules(rules); let out = cfcache.hidden_class_id_selectors(["with"], EMPTY, &HashSet::default()); assert_eq!(out, Vec::::new()); @@ -564,12 +553,7 @@ mod cosmetic_cache_tests { "example.*#@#.a-class", "~test.com###test-element", ]; - let cfcache = CosmeticFilterCache::from_rules( - rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(), - ); + let cfcache = CosmeticFilterCache::from_rules(rules); let resources = ResourceStorage::default(); let exceptions = cfcache .hostname_cosmetic_resources(&resources, "example.co.uk", false) @@ -624,12 +608,7 @@ mod cosmetic_cache_tests { "example.com#@#div > p", "~example.com##a[href=\"notbad.com\"]", ]; - let cfcache = CosmeticFilterCache::from_rules( - rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(), - ); + let cfcache = CosmeticFilterCache::from_rules(rules); let resources = ResourceStorage::default(); let hide_selectors = cfcache @@ -659,12 +638,7 @@ mod cosmetic_cache_tests { "toolforge.org##+js(abort-on-property-read, noAdBlockers)", "github.io##div.adToBlock", ]; - let cfcache = CosmeticFilterCache::from_rules( - rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(), - ); + let cfcache = CosmeticFilterCache::from_rules(rules); let resources = ResourceStorage::in_memory_from_resources([Resource { name: "abort-on-property-read.js".into(), aliases: vec!["aopr".to_string()], diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index c579aac1..cffe102b 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -21,7 +21,7 @@ mod tests { ("https://brave.com/about", true), ]; - let mut engine = Engine::from_rules(filters, Default::default()); + let mut engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); engine.enable_tags(&["stuff"]); engine.enable_tags(&["brian"]); @@ -55,7 +55,7 @@ mod tests { ("https://brave.com/about", true), ]; - let mut engine = Engine::from_rules(filters, Default::default()); + let mut engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); engine.enable_tags(&["brian", "stuff"]); engine.disable_tags(&["stuff"]); @@ -87,7 +87,7 @@ mod tests { ("https://brianbondy.com/advert", true), ]; - let engine = Engine::from_rules(filters, Default::default()); + let engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); url_results.into_iter().for_each(|(url, expected_result)| { let request = Request::new(url, "", "").unwrap(); @@ -117,7 +117,7 @@ mod tests { ("https://brianbondy.com/advert", false), ]; - let mut engine = Engine::from_rules(filters, Default::default()); + let mut engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); engine.enable_tags(&["brian", "stuff"]); url_results.into_iter().for_each(|(url, expected_result)| { @@ -150,7 +150,7 @@ mod tests { ("https://brave.com/about", false), ]; - let mut engine = Engine::from_rules(filters, Default::default()); + let mut engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); engine.enable_tags(&["stuff"]); engine.enable_tags(&["brian"]); let serialized = engine.serialize(); @@ -181,7 +181,7 @@ mod tests { #[test] fn deserialization_generate_simple() { - let mut engine = Engine::from_rules(["ad-banner"], Default::default()); + let mut engine = Engine::new_with_list_text("ad-banner", Default::default()); let data = engine.serialize().to_vec(); const EXPECTED_HASH: u64 = 16556115079021991714; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); @@ -190,7 +190,7 @@ mod tests { #[test] fn deserialization_generate_tags() { - let mut engine = Engine::from_rules(["ad-banner$tag=abc"], Default::default()); + let mut engine = Engine::new_with_list_text("ad-banner$tag=abc", Default::default()); engine.use_tags(&["abc"]); let data = engine.serialize().to_vec(); const EXPECTED_HASH: u64 = 4864047469838009851; @@ -200,7 +200,8 @@ mod tests { #[test] fn deserialization_generate_resources() { - let mut engine = Engine::from_rules(["ad-banner$redirect=nooptext"], Default::default()); + let mut engine = + Engine::new_with_list_text("ad-banner$redirect=nooptext", Default::default()); engine.use_resources([ Resource::simple("nooptext", MimeType::TextPlain, ""), @@ -214,8 +215,9 @@ mod tests { #[test] fn deserialization_brave_list() { - let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); - let mut engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let rules = rules_from_lists(["data/brave/brave-main-list.txt"]); + let mut engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); let data = engine.serialize().to_vec(); #[cfg(feature = "debug-info")] @@ -249,8 +251,8 @@ mod tests { #[test] fn redirect_resource_insertion_works() { - let mut engine = Engine::from_rules( - ["ad-banner$redirect=nooptext", "script.js$redirect=noop.js"], + let mut engine = Engine::new_with_list_text( + ["ad-banner$redirect=nooptext", "script.js$redirect=noop.js"].join("\n"), Default::default(), ); @@ -295,7 +297,7 @@ mod tests { fn document() { let filters = ["||example.com$document", "@@||sub.example.com$document"]; - let engine = Engine::from_rules_debug(filters, Default::default()); + let engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); assert!( engine @@ -328,7 +330,7 @@ mod tests { #[test] fn implicit_all() { { - let engine = Engine::from_rules_debug(["||example.com^"], Default::default()); + let engine = Engine::new_with_list_text("||example.com^", Default::default()); assert!( engine .check_network_request( @@ -340,7 +342,7 @@ mod tests { } { let engine = - Engine::from_rules_debug(["||example.com^$first-party"], Default::default()); + Engine::new_with_list_text("||example.com^$first-party", Default::default()); assert!( engine .check_network_request( @@ -351,7 +353,7 @@ mod tests { ); } { - let engine = Engine::from_rules_debug(["||example.com^$script"], Default::default()); + let engine = Engine::new_with_list_text("||example.com^$script", Default::default()); assert!( !engine .check_network_request( @@ -362,7 +364,7 @@ mod tests { ); } { - let engine = Engine::from_rules_debug(["||example.com^$~script"], Default::default()); + let engine = Engine::new_with_list_text("||example.com^$~script", Default::default()); assert!( !engine .check_network_request( @@ -373,8 +375,8 @@ mod tests { ); } { - let engine = Engine::from_rules_debug( - ["||example.com^$document", "@@||example.com^$generichide"], + let engine = Engine::new_with_list_text( + ["||example.com^$document", "@@||example.com^$generichide"].join("\n"), Default::default(), ); assert!( @@ -387,8 +389,8 @@ mod tests { ); } { - let engine = Engine::from_rules_debug( - ["example.com"], + let engine = Engine::new_with_list_text( + "example.com", ParseOptions { format: FilterFormat::Hosts, ..Default::default() @@ -404,7 +406,7 @@ mod tests { ); } { - let engine = Engine::from_rules_debug(["||example.com/path"], Default::default()); + let engine = Engine::new_with_list_text("||example.com/path", Default::default()); assert!( !engine .check_network_request( @@ -419,7 +421,7 @@ mod tests { ); } { - let engine = Engine::from_rules_debug(["||example.com/path^"], Default::default()); + let engine = Engine::new_with_list_text("||example.com/path^", Default::default()); assert!( !engine .check_network_request( @@ -437,8 +439,8 @@ mod tests { #[test] fn explicit_all() { - let engine = Engine::from_rules_debug( - ["*$all,domain=rarvinzp.click|ytrqcxat.click"], + let engine = Engine::new_with_list_text( + "*$all,domain=rarvinzp.click|ytrqcxat.click", Default::default(), ); for content_type in [ @@ -472,7 +474,8 @@ mod tests { "example.com##.block", "@@||example2.com/test.html$generichide", "example2.com##.block", - ]; + ] + .join("\n"); let url_results = [ ("https://example.com", vec![".block"], true), ("https://example.com/test.html", vec![".block"], true), @@ -484,7 +487,7 @@ mod tests { ("https://example2.com/test.html", vec![".block"], true), ]; - let engine = Engine::from_rules(filters, Default::default()); + let engine = Engine::new_with_list_text(filters, Default::default()); url_results .into_iter() @@ -508,7 +511,7 @@ mod tests { "||addthis.com^$important,3p,domain=~missingkids.com|~missingkids.org|~sainsburys.jobs|~sitecore.com|~amd.com", "||addthis.com/*/addthis_widget.js$script,redirect=addthis.com/addthis_widget.js", ], Default::default()); - let mut engine = Engine::from_filter_set(filter_set, false); + let mut engine = Engine::new_with_filter_set(filter_set, false); engine.use_resources([Resource::simple( "addthis.com/addthis_widget.js", @@ -526,17 +529,15 @@ mod tests { fn check_match_case_regex_filtering() { { // match case without regex is discarded - let engine = Engine::from_rules_debug(["ad.png$match-case"], Default::default()); + let engine = Engine::new_with_list_text("ad.png$match-case", Default::default()); let request = Request::new("https://example.com/ad.png", "https://example.com", "image").unwrap(); assert!(!engine.check_network_request(&request).matched); } { // /^https:\/\/[0-9a-z]{3,}\.[-a-z]{10,}\.(?:li[fv]e|top|xyz)\/[a-z]{8}\/\?utm_campaign=\w{40,}/$doc,match-case,domain=life|live|top|xyz - let engine = Engine::from_rules_debug( - [ - r#"/^https:\/\/[0-9a-z]{3,}\.[-a-z]{10,}\.(?:li[fv]e|top|xyz)\/[a-z]{8}\/\?utm_campaign=\w{40,}/$doc,match-case,domain=life|live|top|xyz"#, - ], + let engine = Engine::new_with_list_text( + r#"/^https:\/\/[0-9a-z]{3,}\.[-a-z]{10,}\.(?:li[fv]e|top|xyz)\/[a-z]{8}\/\?utm_campaign=\w{40,}/$doc,match-case,domain=life|live|top|xyz"#, Default::default(), ); let request = Request::new("https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "document").unwrap(); @@ -571,10 +572,8 @@ mod tests { }*/ { // /^http:\/\/[a-z]{5}\.[a-z]{5}\.com\/[a-z]{10}\.apk$/$doc,match-case,domain=com - let engine = Engine::from_rules_debug( - [ - r#"/^http:\/\/[a-z]{5}\.[a-z]{5}\.com\/[a-z]{10}\.apk$/$doc,match-case,domain=com"#, - ], + let engine = Engine::new_with_list_text( + r#"/^http:\/\/[a-z]{5}\.[a-z]{5}\.com\/[a-z]{10}\.apk$/$doc,match-case,domain=com"#, Default::default(), ); let request = Request::new( @@ -623,8 +622,8 @@ mod tests { }*/ { // /^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.json$/$xhr,3p,match-case - let engine = Engine::from_rules_debug( - [r#"/^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.json$/$xhr,3p,match-case"#], + let engine = Engine::new_with_list_text( + r#"/^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.json$/$xhr,3p,match-case"#, Default::default(), ); let request = Request::new( @@ -664,8 +663,8 @@ mod tests { }*/ { // /^https?:\/\/cdn\.[a-z]{4,6}\.xyz\/app\.js$/$script,3p,match-case - let engine = Engine::from_rules_debug( - [r#"/^https?:\/\/cdn\.[a-z]{4,6}\.xyz\/app\.js$/$script,3p,match-case"#], + let engine = Engine::new_with_list_text( + r#"/^https?:\/\/cdn\.[a-z]{4,6}\.xyz\/app\.js$/$script,3p,match-case"#, Default::default(), ); let request = Request::new( @@ -687,10 +686,8 @@ mod tests { }*/ { // /^https:\/\/cdn\.jsdelivr\.net\/npm\/[-a-z_]{4,22}@latest\/dist\/script\.min\.js$/$script,3p,match-case - let engine = Engine::from_rules_debug( - [ - r#"/^https:\/\/cdn\.jsdelivr\.net\/npm\/[-a-z_]{4,22}@latest\/dist\/script\.min\.js$/$script,3p,match-case"#, - ], + let engine = Engine::new_with_list_text( + r#"/^https:\/\/cdn\.jsdelivr\.net\/npm\/[-a-z_]{4,22}@latest\/dist\/script\.min\.js$/$script,3p,match-case"#, Default::default(), ); let request = Request::new( @@ -790,7 +787,7 @@ mod tests { }, ); - let mut engine = Engine::from_filter_set(filter_set, true); + let mut engine = Engine::new_with_filter_set(filter_set, true); engine.use_resources(resources); fn wrap_try(scriptlet_content: &str) -> String { @@ -879,7 +876,7 @@ mod tests { r#"example.com##+js(trusted-set-local-storage-item, "test"test, 3)"#, ], Default::default()); - let mut engine = Engine::from_filter_set(filter_set, true); + let mut engine = Engine::new_with_filter_set(filter_set, true); engine.use_resources(resources); assert_eq!(engine.url_cosmetic_resources("https://dailymail.co.uk").injected_script, r#"function trustedSetLocalStorageItem(key = '', value = '') { setLocalStorageItemFn('local', true, key, value); } diff --git a/tests/unit/filters/network_matchers.rs b/tests/unit/filters/network_matchers.rs index 2a000f46..6b80d407 100644 --- a/tests/unit/filters/network_matchers.rs +++ b/tests/unit/filters/network_matchers.rs @@ -693,7 +693,7 @@ mod match_tests { { use crate::Engine; let filter = r#"/^https?:\/\/([0-9a-z\-]+\.)?(9anime|animeland|animenova|animeplus|animetoon|animewow|gamestorrent|goodanime|gogoanime|igg-games|kimcartoon|memecenter|readcomiconline|toonget|toonova|watchcartoononline)\.[a-z]{2,4}\/(?!([Ee]xternal|[Ii]mages|[Ss]cripts|[Uu]ploads|ac|ajax|assets|combined|content|cov|cover|(img\/bg)|(img\/icon)|inc|jwplayer|player|playlist-cat-rss|static|thumbs|wp-content|wp-includes)\/)(.*)/$image,other,script,~third-party,xmlhttprequest,domain=~animeland.hu"#; - let engine = Engine::from_rules(vec![filter], Default::default()); + let engine = Engine::new_with_list_text(filter, Default::default()); let url = "https://9anime.to/watch/episode-1"; let source = "https://9anime.to"; let request = request::Request::new(url, source, "script").unwrap(); diff --git a/tests/unit/lists.rs b/tests/unit/lists.rs index b67a26d2..2d24e476 100644 --- a/tests/unit/lists.rs +++ b/tests/unit/lists.rs @@ -290,10 +290,11 @@ mod tests { "! Version: 20220411", "", "! => https://austinhuang.me/0131-block-list/list.txt", - ]; + ] + .join("\n"); let mut filter_set = FilterSet::new(false); - let metadata = filter_set.add_filters(list, ParseOptions::default()); + let metadata = filter_set.add_filter_list(list, ParseOptions::default()); assert_eq!(metadata.title, Some("0131 Block List".to_string())); assert_eq!( @@ -319,10 +320,11 @@ mod tests { "! Last modified: 09/03/2021", "! Expires: 7 days (update frequency)", "! Homepage: https://www.haopro.net/", - ]; + ] + .join("\n"); let mut filter_set = FilterSet::new(false); - let metadata = filter_set.add_filters(list, ParseOptions::default()); + let metadata = filter_set.add_filter_list(list, ParseOptions::default()); assert_eq!(metadata.title, Some("ABPVN Advanced".to_string())); assert_eq!( @@ -392,18 +394,18 @@ mod tests { { let input = "example.com##.selector"; let result = parse_filter(input, true, Default::default()); - assert!(matches!(result, Ok(ParsedFilter::Cosmetic(..)))); + assert!(matches!(result, Ok(ParsedLine::Cosmetic(..)))); } { let input = "9gag.com#?#article:-abp-has(.promoted)"; let result = parse_filter(input, true, Default::default()); - assert!(matches!(result, Ok(ParsedFilter::Cosmetic(..)))); + assert!(matches!(result, Ok(ParsedLine::Cosmetic(..)))); } #[cfg(feature = "css-validation")] { let input = "sportowefakty.wp.pl#@?#body > [class]:not([id]):matches-css(position: fixed):matches-css(top: 0px)"; let result = parse_filter(input, true, Default::default()); - assert!(matches!(result, Ok(ParsedFilter::Cosmetic(..)))); + assert!(matches!(result, Ok(ParsedLine::Cosmetic(..)))); } { let input = diff --git a/tests/unit/regex_manager.rs b/tests/unit/regex_manager.rs index 739781dd..2f7dbd00 100644 --- a/tests/unit/regex_manager.rs +++ b/tests/unit/regex_manager.rs @@ -7,7 +7,7 @@ mod tests { use mock_instant::thread_local::MockClock; fn make_engine(line: &str) -> Engine { - Engine::from_rules(vec![line], Default::default()) + Engine::new_with_list_text(line, Default::default()) } fn make_request(url: &str) -> request::Request { diff --git a/tests/unit/resources/resource_storage.rs b/tests/unit/resources/resource_storage.rs index 2052705e..c73842e4 100644 --- a/tests/unit/resources/resource_storage.rs +++ b/tests/unit/resources/resource_storage.rs @@ -834,14 +834,18 @@ mod shared_storage_tests { let shared_storage = Rc::new(in_memory_storage); - let mut engine1 = - crate::Engine::from_rules(["example1.com##+js(test-scriptlet)"], Default::default()); + let mut engine1 = crate::Engine::new_with_list_text( + "example1.com##+js(test-scriptlet)", + Default::default(), + ); engine1.use_resource_storage(BraveCoreResourceStorage { shared_storage: Rc::clone(&shared_storage), }); - let mut engine2 = - crate::Engine::from_rules(["example2.com##+js(test-scriptlet)"], Default::default()); + let mut engine2 = crate::Engine::new_with_list_text( + "example2.com##+js(test-scriptlet)", + Default::default(), + ); engine2.use_resource_storage(BraveCoreResourceStorage { shared_storage: Rc::clone(&shared_storage), }); From be9365f9fde17d428d73aa7994a1d388a74b5606 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Fri, 5 Jun 2026 12:13:12 +0400 Subject: [PATCH 07/28] Streaming filter parsing (#653) The PR refactor the filter parsing code to allow processing filters one-by-one. * new_with_filter_set parses filter one by one and immediately drops them after processing. * The filters, matched to bad filters, are always serialized. The offsets are dropped from the hash table on the last stage. The negative impact is minimal, because $badfilter is rare thing. * get_tokens uses an extra output param tokens_buffer to avoid unwanted allocations. The PR dramatically reduces allocation count (-25%) and peak memory usage (about -50%). --- src/cosmetic_filter_cache_builder.rs | 10 - src/engine.rs | 76 +++-- src/filters/fb_network_builder.rs | 317 ++++++++++---------- src/filters/network.rs | 24 +- src/flatbuffers/containers/flat_multimap.rs | 4 + tests/test_utils.rs | 1 + tests/unit/engine.rs | 4 +- tests/unit/filters/network.rs | 5 +- 8 files changed, 238 insertions(+), 203 deletions(-) diff --git a/src/cosmetic_filter_cache_builder.rs b/src/cosmetic_filter_cache_builder.rs index 2f674c5c..2d5cd2e1 100644 --- a/src/cosmetic_filter_cache_builder.rs +++ b/src/cosmetic_filter_cache_builder.rs @@ -77,16 +77,6 @@ pub(crate) struct CosmeticFilterCacheBuilder<'a> { } impl<'a> CosmeticFilterCacheBuilder<'a> { - pub fn from_rules(rules: Vec, builder: &mut EngineFlatBuilder<'a>) -> Self { - let mut self_ = Self::default(); - - for rule in rules { - self_.add_filter(rule, builder); - } - - self_ - } - pub fn add_filter(&mut self, rule: CosmeticFilter, builder: &mut EngineFlatBuilder<'a>) { if rule.has_hostname_constraint() { if let Some(generic_rule) = rule.hidden_generic_rule() { diff --git a/src/engine.rs b/src/engine.rs index 4ca8aa8a..4aa7ac4f 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -107,36 +107,69 @@ impl Engine { cosmetic_filters: Vec, optimize: bool, ) -> Self { - let memory = make_flatbuffer(network_filters, cosmetic_filters, optimize); + let mut builder = EngineFlatBuilder::default(); + let mut network_rules_builder = NetworkRulesBuilder::new(optimize); + let mut cosmetic_filter_cache_builder = CosmeticFilterCacheBuilder::default(); - let filter_data_context = FilterDataContext::new(memory); - - Self { - blocker: Blocker::from_context(FilterDataContextRef::clone(&filter_data_context)), - cosmetic_cache: CosmeticFilterCache::from_context(FilterDataContextRef::clone( - &filter_data_context, - )), - resources: ResourceStorage::default(), - filter_data_context, + for filter in network_filters { + network_rules_builder.add_filter(filter, &mut builder); } + for filter in cosmetic_filters { + cosmetic_filter_cache_builder.add_filter(filter, &mut builder); + } + + Self::new_with_builders( + network_rules_builder, + cosmetic_filter_cache_builder, + builder, + ) } /// Loads rules from the given `FilterSet`. pub fn new_with_filter_set(set: FilterSet, optimize: bool) -> Self { - let mut network_filters = vec![]; - let mut cosmetic_filters = vec![]; + let mut builder = EngineFlatBuilder::default(); + let mut network_rules_builder = NetworkRulesBuilder::new(optimize); + let mut cosmetic_filter_cache_builder = CosmeticFilterCacheBuilder::default(); for list_source in set.list_sources { for line in list_source.list_text.lines() { let parsed_line = parse_filter(line, set.debug, list_source.parse_options); match parsed_line { - Ok(ParsedLine::Network(filter)) => network_filters.push(filter), - Ok(ParsedLine::Cosmetic(filter)) => cosmetic_filters.push(filter), + Ok(ParsedLine::Network(filter)) => { + network_rules_builder.add_filter(filter, &mut builder) + } + Ok(ParsedLine::Cosmetic(filter)) => { + cosmetic_filter_cache_builder.add_filter(filter, &mut builder) + } Err(_) => {} } } } - Self::new_with_parsed_rules(network_filters, cosmetic_filters, optimize) + + Self::new_with_builders( + network_rules_builder, + cosmetic_filter_cache_builder, + builder, + ) + } + + fn new_with_builders<'a>( + network_rules_builder: NetworkRulesBuilder<'a>, + cosmetic_filter_cache_builder: CosmeticFilterCacheBuilder<'a>, + mut builder: EngineFlatBuilder<'a>, + ) -> Self { + let network_rules = FlatSerialize::serialize(network_rules_builder, &mut builder); + let cosmetic_rules = FlatSerialize::serialize(cosmetic_filter_cache_builder, &mut builder); + let memory = builder.finish(network_rules, cosmetic_rules); + let filter_data_context = FilterDataContext::new(memory); + Self { + blocker: Blocker::from_context(FilterDataContextRef::clone(&filter_data_context)), + cosmetic_cache: CosmeticFilterCache::from_context(FilterDataContextRef::clone( + &filter_data_context, + )), + resources: ResourceStorage::default(), + filter_data_context, + } } /// Check if a request for a network resource from `url`, of type `request_type`, initiated by @@ -336,19 +369,6 @@ fn _assertions() { _assert_sync::(); } -fn make_flatbuffer( - network_filters: Vec, - cosmetic_filters: Vec, - optimize: bool, -) -> VerifiedFlatbufferMemory { - let mut builder = EngineFlatBuilder::default(); - let network_rules_builder = NetworkRulesBuilder::from_rules(network_filters, optimize); - let network_rules = FlatSerialize::serialize(network_rules_builder, &mut builder); - let cosmetic_rules = CosmeticFilterCacheBuilder::from_rules(cosmetic_filters, &mut builder); - let cosmetic_rules = FlatSerialize::serialize(cosmetic_rules, &mut builder); - builder.finish(network_rules, cosmetic_rules) -} - #[cfg(test)] #[path = "../tests/unit/engine.rs"] mod unit_tests; diff --git a/src/filters/fb_network_builder.rs b/src/filters/fb_network_builder.rs index d07cdbdf..5929c395 100644 --- a/src/filters/fb_network_builder.rs +++ b/src/filters/fb_network_builder.rs @@ -11,7 +11,7 @@ use crate::utils::TokensBuffer; use crate::filters::network::NetworkFilterMaskHelper; use crate::flatbuffers::containers::flat_multimap::FlatMultiMapBuilder; -use crate::flatbuffers::containers::flat_serialize::{FlatBuilder, FlatSerialize, WIPFlatVec}; +use crate::flatbuffers::containers::flat_serialize::{FlatBuilder, FlatSerialize}; use crate::optimizer; use crate::utils::{to_short_hash, Hash, ShortHash}; @@ -29,21 +29,29 @@ pub(crate) enum NetworkFilterListId { Size = 8, } -#[derive(Default, Clone)] -pub(crate) struct NetworkFilterListBuilder { - filters: Vec, +struct NetworkFilterFlatEntry<'a> { + filter: WIPOffset>, + id: Hash, +} + +struct NetworkFilterListBuilder<'a> { + flat_map_builder: FlatMultiMapBuilder>, + token_frequencies: TokenSelector, + filters_to_optimize: HashMap>, + tokens_buffer: TokensBuffer, optimize: bool, } -pub(crate) struct NetworkRulesBuilder { - lists: Vec, +pub(crate) struct NetworkRulesBuilder<'a> { + lists: Vec>, + bad_filter_ids: HashSet, } -impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for &NetworkFilter { +impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilter { type Output = WIPOffset>; fn serialize( - network_filter: &NetworkFilter, + network_filter: NetworkFilter, builder: &mut EngineFlatBuilder<'a>, ) -> WIPOffset> { let opt_domains = network_filter.opt_domains.as_ref().map(|v| { @@ -104,7 +112,7 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for &NetworkFilter { .as_ref() .map(|v| builder.create_string(v.as_str())); - let network_filter = fb::NetworkFilter::create( + fb::NetworkFilter::create( builder.raw_builder(), &fb::NetworkFilterArgs { mask: network_filter.mask.bits(), @@ -117,177 +125,182 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for &NetworkFilter { tag, raw_line, }, - ); - - network_filter + ) } } -impl NetworkFilterListBuilder { +impl<'a> NetworkFilterListBuilder<'a> { fn new(optimize: bool) -> Self { Self { - filters: vec![], + flat_map_builder: FlatMultiMapBuilder::with_capacity(1024), + token_frequencies: TokenSelector::new(1024), + filters_to_optimize: HashMap::new(), + tokens_buffer: TokensBuffer::default(), optimize, } } -} -impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterListBuilder { - type Output = WIPOffset>; - fn serialize( - rule_list: Self, - builder: &mut EngineFlatBuilder<'a>, - ) -> WIPOffset> { - let mut filter_map_builder = FlatMultiMapBuilder::with_capacity(rule_list.filters.len()); - - let mut optimizable = HashMap::>::new(); - - let mut token_frequencies = TokenSelector::new(rule_list.filters.len()); - let mut tokens_buffer = TokensBuffer::default(); - - { - for network_filter in rule_list.filters { - let flat_filter = if !rule_list.optimize - || !optimizer::is_filter_optimizable_by_patterns(&network_filter) - { - Some(FlatSerialize::serialize(&network_filter, builder)) - } else { - None - }; - - let mut store_filter = |token: Hash| { - let short_token = to_short_hash(token); - if let Some(flat_filter) = flat_filter { - filter_map_builder.insert(short_token, flat_filter); - } else { - optimizable - .entry(short_token) - .or_default() - .push(network_filter.clone()); - } - }; - - let multi_tokens = network_filter.get_tokens(&mut tokens_buffer); - match multi_tokens { - FilterTokens::Empty => { - // No tokens, add to fallback bucket (token 0) - store_filter(0); - } - FilterTokens::OptDomains(opt_domains) => { - // For OptDomains, each domain is treated as a separate token group - for &token in opt_domains.iter() { - store_filter(token); - token_frequencies.record_usage(token); - } - } - FilterTokens::Other(tokens) => { - let best_token = token_frequencies.select_least_used_token(tokens); - token_frequencies.record_usage(best_token); - store_filter(best_token); - } - } + fn add_filter(&mut self, network_filter: NetworkFilter, builder: &mut EngineFlatBuilder<'a>) { + let multi_tokens = network_filter.get_tokens(&mut self.tokens_buffer); + let id = network_filter.get_id(); + + // Resolve token(s) and record frequencies up-front so the + // serialized/optimizable branches share no token logic. + let single_token: Hash; + let tokens: &[Hash] = match multi_tokens { + FilterTokens::Empty => { + // No tokens, add to fallback bucket (token 0) + &[0] } - } - - if rule_list.optimize { - // Sort the entries to ensure deterministic iteration order - let mut optimizable_entries: Vec<_> = optimizable.drain().collect(); - optimizable_entries.sort_unstable_by_key(|(token, _)| *token); - - for (token, v) in optimizable_entries { - let optimized = optimizer::optimize(v); + FilterTokens::OptDomains => { + // tokens_buffer has been populated by get_tokens + self.tokens_buffer.as_slice() + } + FilterTokens::Other => { + single_token = self + .token_frequencies + .select_least_used_token(self.tokens_buffer.as_slice()); + std::slice::from_ref(&single_token) + } + }; - for filter in optimized { - let flat_filter = FlatSerialize::serialize(&filter, builder); - filter_map_builder.insert(token, flat_filter); - } + if !self.optimize || !optimizer::is_filter_optimizable_by_patterns(&network_filter) { + // Serialize now (even if it matches to a bad filter later); + // Although store the id for later bad-filter pruning. + let filter = FlatSerialize::serialize(network_filter, builder); + for &token in tokens { + self.token_frequencies.record_usage(token); + self.flat_map_builder + .insert(to_short_hash(token), NetworkFilterFlatEntry { filter, id }); } } else { - debug_assert!( - optimizable.is_empty(), - "Should be empty if optimization is off" - ); + // Defer serialization to the optimizer + for &token in tokens { + self.token_frequencies.record_usage(token); + self.filters_to_optimize + .entry(to_short_hash(token)) + .or_default() + .push(network_filter.clone()); + } } - - let flat_filter_map = FlatMultiMapBuilder::finish(filter_map_builder, builder); - - fb::NetworkFilterList::create( - builder.raw_builder(), - &fb::NetworkFilterListArgs { - filter_map_index: Some(flat_filter_map.keys), - filter_map_values: Some(flat_filter_map.values), - }, - ) } } -impl NetworkRulesBuilder { - pub fn from_rules(network_filters: Vec, optimize: bool) -> Self { - let mut lists = vec![]; - for list_id in 0..NetworkFilterListId::Size as usize { - // Don't optimize removeparam, since it can fuse filters without respecting distinct - let optimize = optimize && list_id != NetworkFilterListId::RemoveParam as usize; - lists.push(NetworkFilterListBuilder::new(optimize)); +impl<'a> NetworkRulesBuilder<'a> { + pub fn new(optimize: bool) -> Self { + let lists = (0..NetworkFilterListId::Size as usize) + .map(|list_id| { + // Don't optimize removeparam, since it can fuse filters without respecting distinct + let optimize = optimize && list_id != NetworkFilterListId::RemoveParam as usize; + NetworkFilterListBuilder::new(optimize) + }) + .collect::>(); + Self { + lists, + bad_filter_ids: HashSet::with_capacity(1024), } - let mut self_ = Self { lists }; - - let mut badfilter_ids: HashSet = HashSet::new(); + } - // Collect badfilter ids in advance. - for filter in network_filters.iter() { - if filter.is_badfilter() { - badfilter_ids.insert(filter.get_id()); - } + pub fn add_filter(&mut self, filter: NetworkFilter, builder: &mut EngineFlatBuilder<'a>) { + if filter.is_badfilter() { + // Note: `get_id()` doesn't include BAD_FILTER bit. + self.bad_filter_ids.insert(filter.get_id()); + return; } - for filter in network_filters.into_iter() { - // skip any bad filters - let filter_id = filter.get_id(); - if badfilter_ids.contains(&filter_id) || filter.is_badfilter() { - continue; - } - - // Redirects are independent of blocking behavior. - if filter.is_redirect() { - self_.add_filter(filter.clone(), NetworkFilterListId::Redirects); - } - type FilterId = NetworkFilterListId; - - let list_id: FilterId = if filter.is_csp() { - FilterId::Csp - } else if filter.is_removeparam() { - FilterId::RemoveParam - } else if filter.is_generic_hide() { - FilterId::GenericHide - } else if filter.is_exception() { - FilterId::Exceptions - } else if filter.is_important() { - FilterId::Importants - } else if filter.tag.is_some() && !filter.is_redirect() { - // `tag` + `redirect` is unsupported for now. - FilterId::TaggedFiltersAll - } else if (filter.is_redirect() && filter.also_block_redirect()) - || !filter.is_redirect() - { - FilterId::Filters - } else { - continue; - }; - - self_.add_filter(filter, list_id); + // Redirects are independent of blocking behavior. + if filter.is_redirect() { + self.add_filter_internal(filter.clone(), NetworkFilterListId::Redirects, builder); } + type FilterId = NetworkFilterListId; + + let list_id: FilterId = if filter.is_csp() { + FilterId::Csp + } else if filter.is_removeparam() { + FilterId::RemoveParam + } else if filter.is_generic_hide() { + FilterId::GenericHide + } else if filter.is_exception() { + FilterId::Exceptions + } else if filter.is_important() { + FilterId::Importants + } else if filter.tag.is_some() && !filter.is_redirect() { + // `tag` + `redirect` is unsupported for now. + FilterId::TaggedFiltersAll + } else if (filter.is_redirect() && filter.also_block_redirect()) || !filter.is_redirect() { + FilterId::Filters + } else { + return; + }; + + self.add_filter_internal(filter, list_id, builder); + } - self_ + fn add_filter_internal( + &mut self, + network_filter: NetworkFilter, + list_id: NetworkFilterListId, + builder: &mut EngineFlatBuilder<'a>, + ) { + self.lists[list_id as usize].add_filter(network_filter, builder); } +} - fn add_filter(&mut self, network_filter: NetworkFilter, list_id: NetworkFilterListId) { - self.lists[list_id as usize].filters.push(network_filter); +impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterFlatEntry<'a> { + type Output = WIPOffset>; + + fn serialize( + value: Self, + _builder: &mut EngineFlatBuilder<'a>, + ) -> WIPOffset> { + value.filter } } -impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder { - type Output = WIPFlatVec<'a, NetworkFilterListBuilder, EngineFlatBuilder<'a>>; +impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder<'a> { + type Output = + WIPOffset>>>; + fn serialize(value: Self, builder: &mut EngineFlatBuilder<'a>) -> Self::Output { - FlatSerialize::serialize(value.lists, builder) + let mut serialized_lists = vec![]; + + for mut rule_list in value.lists { + if !rule_list.filters_to_optimize.is_empty() { + // Sort entries for deterministic iteration order. + let mut optimizable_entries: Vec<_> = + rule_list.filters_to_optimize.drain().collect(); + optimizable_entries.sort_unstable_by_key(|(token, _)| *token); + + for (token, mut v) in optimizable_entries { + v.retain(|f| !value.bad_filter_ids.contains(&f.get_id())); + let optimized = optimizer::optimize(v); + + for filter in optimized { + let id = filter.get_id(); + let filter = FlatSerialize::serialize(filter, builder); + rule_list + .flat_map_builder + .insert(token, NetworkFilterFlatEntry { filter, id }); + } + } + } + + // Prune already-serialized entries that were cancelled by a $badfilter. + rule_list + .flat_map_builder + .retain_by_value(|entry| !value.bad_filter_ids.contains(&entry.id)); + + let flat_filter_map = FlatMultiMapBuilder::finish(rule_list.flat_map_builder, builder); + + serialized_lists.push(fb::NetworkFilterList::create( + builder.raw_builder(), + &fb::NetworkFilterListArgs { + filter_map_index: Some(flat_filter_map.keys), + filter_map_values: Some(flat_filter_map.values), + }, + )); + } + + FlatSerialize::serialize(serialized_lists, builder) } } diff --git a/src/filters/network.rs b/src/filters/network.rs index fef5d53d..7f89f2b4 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -289,10 +289,10 @@ pub enum FilterPart { } #[derive(Debug, PartialEq)] -pub(crate) enum FilterTokens<'a> { +pub(crate) enum FilterTokens { Empty, - OptDomains(&'a [Hash]), - Other(&'a [Hash]), + OptDomains, + Other, } pub struct FilterPartIterator<'a> { @@ -833,10 +833,10 @@ impl NetworkFilter { ) } - pub(crate) fn get_tokens<'a>( - &'a self, - tokens_buffer: &'a mut TokensBuffer, - ) -> FilterTokens<'a> { + /// Returns the tokens that the filter matches to. + /// + /// For `!FilterTokens::Empty`, the result is stored in the given `tokens_buffer`. + pub(crate) fn get_tokens(&self, tokens_buffer: &mut TokensBuffer) -> FilterTokens { tokens_buffer.clear(); // If there is only one domain and no domain negation, we also use this @@ -896,7 +896,13 @@ impl NetworkFilter { { if let Some(opt_domains) = self.opt_domains.as_ref() { if !opt_domains.is_empty() { - return FilterTokens::OptDomains(opt_domains); + let cap = tokens_buffer.remaining_capacity(); + if opt_domains.len() <= cap { + tokens_buffer.extend(opt_domains.iter().copied()); + return FilterTokens::OptDomains; + } + // Too many domains to bucket individually; fall back to the catch-all + // bucket (token 0). } } FilterTokens::Empty @@ -908,7 +914,7 @@ impl NetworkFilter { tokens_buffer.push(utils::fast_hash("https")); } - FilterTokens::Other(tokens_buffer.as_slice()) + FilterTokens::Other } } diff --git a/src/flatbuffers/containers/flat_multimap.rs b/src/flatbuffers/containers/flat_multimap.rs index d19e82c0..b21c6993 100644 --- a/src/flatbuffers/containers/flat_multimap.rs +++ b/src/flatbuffers/containers/flat_multimap.rs @@ -107,6 +107,10 @@ impl FlatMultiMapBuilder { self.entries.push((key, value)); } + pub fn retain_by_value(&mut self, mut retain_if: impl FnMut(&V) -> bool) { + self.entries.retain(|(_, value)| retain_if(value)); + } + pub fn finish<'a, B: FlatBuilder<'a>>( value: Self, builder: &mut B, diff --git a/tests/test_utils.rs b/tests/test_utils.rs index 71d5d3ff..5a1575c0 100644 --- a/tests/test_utils.rs +++ b/tests/test_utils.rs @@ -8,5 +8,6 @@ pub fn rules_from_lists(list_files: impl IntoIterator>) - contents.push_str(&std::fs::read_to_string(file.as_ref()).unwrap()); contents.push('\n'); } + contents.shrink_to_fit(); contents } diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index cffe102b..ea751986 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -239,9 +239,9 @@ mod tests { ); } let expected_hash: u64 = if cfg!(feature = "css-validation") { - 10068943306137168952 + 16703814344905873613 } else { - 6262681569722546672 + 13012744135002827554 }; assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}"); diff --git a/tests/unit/filters/network.rs b/tests/unit/filters/network.rs index af9a5e35..7af7ec26 100644 --- a/tests/unit/filters/network.rs +++ b/tests/unit/filters/network.rs @@ -1252,9 +1252,10 @@ mod parse_tests { let rule = "||some.primewire.c*/sw$script,1p"; let filter = NetworkFilter::parse(rule, true, ParseOptions::default()).unwrap(); let mut tokens_buffer = utils::TokensBuffer::default(); + assert_eq!(filter.get_tokens(&mut tokens_buffer), FilterTokens::Other); assert_eq!( - filter.get_tokens(&mut tokens_buffer), - FilterTokens::Other(&[utils::fast_hash("some"), utils::fast_hash("primewire")]) + tokens_buffer.as_slice(), + &[utils::fast_hash("some"), utils::fast_hash("primewire")] ); } } From ab6df434d1b475d9f3213fecc7093989e0d167d3 Mon Sep 17 00:00:00 2001 From: Mikhail Date: Tue, 9 Jun 2026 00:06:32 +0400 Subject: [PATCH 08/28] [perf] Use Cow/&str in network filter parsing (#664) The PR makes NetworkFilter::parse reuse the original string via &str or Cow<> --- benches/bench_redirect_performance.rs | 37 +++--- src/content_blocking.rs | 8 +- src/engine.rs | 37 +++--- src/filters/abstract_network.rs | 60 +++++----- src/filters/fb_network_builder.rs | 38 +++---- src/filters/network.rs | 155 ++++++++++++++++---------- src/lists.rs | 48 ++++---- src/network_filter_list.rs | 6 +- src/optimizer.rs | 51 +++++---- tests/legacy_harness.rs | 6 +- tests/matching.rs | 4 +- tests/unit/filters/network.rs | 74 ++++++------ tests/unit/lists.rs | 7 +- 13 files changed, 296 insertions(+), 235 deletions(-) diff --git a/benches/bench_redirect_performance.rs b/benches/bench_redirect_performance.rs index 82dce5c6..f9cb0245 100644 --- a/benches/bench_redirect_performance.rs +++ b/benches/bench_redirect_performance.rs @@ -1,3 +1,6 @@ +use std::borrow::Cow; +use std::sync::OnceLock; + use adblock::Engine; use criterion::*; use tokio::runtime::Runtime; @@ -55,23 +58,31 @@ async fn get_all_filters() -> Vec { .collect() } -/// Gets all rules with redirects, and modifies them to apply to resources at `a{0-n}.com/bad.js` -fn get_redirect_rules() -> Vec { - let async_runtime = Runtime::new().expect("Could not start Tokio runtime"); +static ALL_FILTERS: OnceLock> = OnceLock::new(); - let filters = async_runtime.block_on(get_all_filters()); - let (network_filters, _) = adblock::lists::parse_filters(&filters, true, Default::default()); +fn all_filters() -> &'static [String] { + ALL_FILTERS.get_or_init(|| { + let async_runtime = Runtime::new().expect("Could not start Tokio runtime"); + async_runtime.block_on(get_all_filters()).into_boxed_slice() + }) +} + +/// Gets all rules with redirects, and modifies them to apply to resources at `a{0-n}.com/bad.js` +fn get_redirect_rules() -> Vec> { + let filters = all_filters(); + let (network_filters, _) = + adblock::lists::parse_filters(filters.iter().map(|s| s.as_str()), true, Default::default()); network_filters .into_iter() .filter(NetworkFilter::is_redirect) .filter(NetworkFilter::also_block_redirect) - .filter(|rule| rule.modifier_option.as_ref().unwrap() != "none") + .filter(|rule| rule.modifier_option.unwrap() != "none") .enumerate() .map(|(index, mut rule)| { rule.mask.insert(NetworkFilterMask::IS_LEFT_ANCHOR); rule.mask.insert(NetworkFilterMask::IS_RIGHT_ANCHOR); - rule.hostname = Some(format!("a{index}.com/bad.js")); + rule.hostname = Some(Cow::Owned(format!("a{index}.com/bad.js"))); rule.filter = adblock::filters::network::FilterPart::Empty; rule.mask.remove(NetworkFilterMask::IS_HOSTNAME_ANCHOR); @@ -117,17 +128,17 @@ fn get_resources_for_filters(#[allow(unused)] filters: &[NetworkFilter]) -> Vec< .iter() .filter(|f| f.is_redirect()) .map(|f| { - let mut redirect = f.modifier_option.as_ref().unwrap().as_str(); + let mut redirect = f.modifier_option.as_ref().unwrap().to_string(); // strip priority, if present if let Some(i) = redirect.rfind(':') { - redirect = &redirect[0..i]; + redirect.truncate(i); } Resource { - name: redirect.to_owned(), + name: redirect.clone(), aliases: vec![], - kind: ResourceType::Mime(MimeType::from_extension(redirect)), - content: BASE64_STANDARD.encode(redirect), + kind: ResourceType::Mime(MimeType::from_extension(&redirect)), + content: BASE64_STANDARD.encode(&redirect), dependencies: vec![], permission: Default::default(), } @@ -174,7 +185,7 @@ pub fn build_custom_requests(rules: Vec) -> Vec { let domain = &rule_hostname[..rule_hostname.find('/').unwrap()]; let hostname = domain; - let raw_line = rule.raw_line.clone().unwrap(); + let raw_line = rule.raw_line.as_deref().unwrap().to_string(); let source_hostname = if rule.opt_domains.is_some() { let domain_start = raw_line.rfind("domain=").unwrap() + "domain=".len(); let from_start = &raw_line[domain_start..]; diff --git a/src/content_blocking.rs b/src/content_blocking.rs index 6ba393af..5ad01239 100644 --- a/src/content_blocking.rs +++ b/src/content_blocking.rs @@ -229,7 +229,7 @@ pub enum CbRuleCreationFailure { ProceduralCosmeticFiltersUnsupported, } -impl TryFrom for CbRuleEquivalent { +impl TryFrom> for CbRuleEquivalent { type Error = CbRuleCreationFailure; fn try_from(v: ParsedLine) -> Result { @@ -301,7 +301,7 @@ impl Iterator for CbRuleEquivalentIterator { } } -impl TryFrom for CbRuleEquivalent { +impl TryFrom> for CbRuleEquivalent { type Error = CbRuleCreationFailure; fn try_from(v: NetworkFilter) -> Result { @@ -311,7 +311,7 @@ impl TryFrom for CbRuleEquivalent { LazyLock::new(|| Regex::new(r##"\*"##).unwrap()); static TRAILING_SEPARATOR: LazyLock = LazyLock::new(|| Regex::new(r##"\^$"##).unwrap()); - if let Some(raw_line) = &v.raw_line { + if let Some(raw_line) = v.raw_line.as_deref() { if v.is_redirect() { return Err(CbRuleCreationFailure::NetworkRedirectUnsupported); } @@ -603,7 +603,7 @@ impl TryFrom for CbRule { return Err(CbRuleCreationFailure::ScriptletInjectionsNotSupported); } - if let Some(raw_line) = &v.raw_line { + if let Some(raw_line) = v.raw_line.as_deref() { let mut hostnames_vec = vec![]; let mut not_hostnames_vec = vec![]; diff --git a/src/engine.rs b/src/engine.rs index 4aa7ac4f..1e64a6e3 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -7,6 +7,7 @@ use crate::data_format::{deserialize_dat_file, serialize_dat_file, Deserializati use crate::filters::fb_builder::EngineFlatBuilder; use crate::filters::fb_network_builder::NetworkRulesBuilder; use crate::filters::filter_data_context::{FilterDataContext, FilterDataContextRef}; +use crate::filters::flatbuffer_generated::fb; use crate::flatbuffers::containers::flat_serialize::FlatSerialize; use crate::flatbuffers::unsafe_tools::VerifiedFlatbufferMemory; use crate::lists::{parse_filter, FilterSet, ParseOptions, ParsedLine}; @@ -118,22 +119,26 @@ impl Engine { cosmetic_filter_cache_builder.add_filter(filter, &mut builder); } - Self::new_with_builders( - network_rules_builder, - cosmetic_filter_cache_builder, + Self::new_with_flatbuffer_offsets( + FlatSerialize::serialize(network_rules_builder, &mut builder), + FlatSerialize::serialize(cosmetic_filter_cache_builder, &mut builder), builder, ) } /// Loads rules from the given `FilterSet`. pub fn new_with_filter_set(set: FilterSet, optimize: bool) -> Self { + let FilterSet { + debug, + list_sources, + } = set; let mut builder = EngineFlatBuilder::default(); let mut network_rules_builder = NetworkRulesBuilder::new(optimize); let mut cosmetic_filter_cache_builder = CosmeticFilterCacheBuilder::default(); - for list_source in set.list_sources { + for list_source in &list_sources { for line in list_source.list_text.lines() { - let parsed_line = parse_filter(line, set.debug, list_source.parse_options); + let parsed_line = parse_filter(line, debug, list_source.parse_options); match parsed_line { Ok(ParsedLine::Network(filter)) => { network_rules_builder.add_filter(filter, &mut builder) @@ -145,21 +150,23 @@ impl Engine { } } } + let network_rules_offset = FlatSerialize::serialize(network_rules_builder, &mut builder); + // Drop the list sources to reduce peak memory usage. + drop(list_sources); - Self::new_with_builders( - network_rules_builder, - cosmetic_filter_cache_builder, - builder, - ) + let cosmetic_rules_offset = + FlatSerialize::serialize(cosmetic_filter_cache_builder, &mut builder); + + Self::new_with_flatbuffer_offsets(network_rules_offset, cosmetic_rules_offset, builder) } - fn new_with_builders<'a>( - network_rules_builder: NetworkRulesBuilder<'a>, - cosmetic_filter_cache_builder: CosmeticFilterCacheBuilder<'a>, + fn new_with_flatbuffer_offsets<'a>( + network_rules: flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + cosmetic_rules: flatbuffers::WIPOffset>, mut builder: EngineFlatBuilder<'a>, ) -> Self { - let network_rules = FlatSerialize::serialize(network_rules_builder, &mut builder); - let cosmetic_rules = FlatSerialize::serialize(cosmetic_filter_cache_builder, &mut builder); let memory = builder.finish(network_rules, cosmetic_rules); let filter_data_context = FilterDataContext::new(memory); Self { diff --git a/src/filters/abstract_network.rs b/src/filters/abstract_network.rs index b97c9b33..7d3658df 100644 --- a/src/filters/abstract_network.rs +++ b/src/filters/abstract_network.rs @@ -26,7 +26,8 @@ pub(crate) enum NetworkFilterRightAnchor { #[derive(Clone)] pub(crate) struct NetworkFilterPattern { pub(crate) left_anchor: Option, - pub(crate) pattern: String, + pub(crate) start: usize, + pub(crate) end: usize, pub(crate) right_anchor: Option, } @@ -34,18 +35,18 @@ pub(crate) struct NetworkFilterPattern { /// All `bool` arguments below are `true` if the option stands alone, or `false` if the option is /// negated using a prepended `~`. #[derive(Clone)] -pub(crate) enum NetworkFilterOption { - Domain(Vec<(bool, String)>), +pub(crate) enum NetworkFilterOption<'a> { + Domain(Vec<(bool, &'a str)>), Badfilter, Important, MatchCase, ThirdParty(bool), FirstParty(bool), - Tag(String), - Redirect(String), - RedirectRule(String), - Csp(Option), - Removeparam(String), + Tag(&'a str), + Redirect(&'a str), + RedirectRule(&'a str), + Csp(Option<&'a str>), + Removeparam(&'a str), Generichide, Document, Image(bool), @@ -62,7 +63,7 @@ pub(crate) enum NetworkFilterOption { All, } -impl NetworkFilterOption { +impl NetworkFilterOption<'_> { pub fn is_content_type(&self) -> bool { matches!( self, @@ -90,14 +91,16 @@ impl NetworkFilterOption { /// Abstract syntax representation of a network filter. This representation can fully specify the /// string representation of a filter as written, with the exception of aliased options like `1p` /// or `ghide`. This allows separation of concerns between parsing and interpretation. -pub(crate) struct AbstractNetworkFilter { +pub(crate) struct AbstractNetworkFilter<'a> { pub(crate) exception: bool, pub(crate) pattern: NetworkFilterPattern, - pub(crate) options: Option>, + pub(crate) options: Option>>, } -impl AbstractNetworkFilter { - pub(crate) fn parse(line: &str) -> Result { +impl AbstractNetworkFilter<'_> { + pub(crate) fn parse<'a>( + line: &'a str, + ) -> Result, NetworkFilterError> { let mut filter_index_start: usize = 0; let mut filter_index_end: usize = line.len(); @@ -139,13 +142,12 @@ impl AbstractNetworkFilter { None }; - let pattern = &line[filter_index_start..filter_index_end]; - Ok(AbstractNetworkFilter { exception, pattern: NetworkFilterPattern { left_anchor, - pattern: pattern.to_string(), + start: filter_index_start, + end: filter_index_end, right_anchor, }, options, @@ -153,7 +155,9 @@ impl AbstractNetworkFilter { } } -fn parse_filter_options(raw_options: &str) -> Result, NetworkFilterError> { +fn parse_filter_options<'a>( + raw_options: &'a str, +) -> Result>, NetworkFilterError> { let mut result = vec![]; for raw_option in raw_options.split(',') { @@ -170,13 +174,13 @@ fn parse_filter_options(raw_options: &str) -> Result, N result.push(match (option, negation) { ("domain", _) | ("from", _) => { - let domains: Vec<(bool, String)> = value + let domains: Vec<(bool, &'a str)> = value .split('|') .map(|domain| { if let Some(negated_domain) = domain.strip_prefix('~') { - (false, negated_domain.to_string()) + (false, negated_domain) } else { - (true, domain.to_string()) + (true, domain) } }) .filter(|(_, d)| !(d.starts_with('/') && d.ends_with('/'))) @@ -195,7 +199,7 @@ fn parse_filter_options(raw_options: &str) -> Result, N ("third-party", negated) | ("3p", negated) => NetworkFilterOption::ThirdParty(!negated), ("first-party", negated) | ("1p", negated) => NetworkFilterOption::FirstParty(!negated), ("tag", true) => return Err(NetworkFilterError::NegatedTag), - ("tag", false) => NetworkFilterOption::Tag(String::from(value)), + ("tag", false) => NetworkFilterOption::Tag(value), ("redirect", true) => return Err(NetworkFilterError::NegatedRedirection), ("redirect", false) => { // Ignore this filter if no redirection resource is specified @@ -203,7 +207,7 @@ fn parse_filter_options(raw_options: &str) -> Result, N return Err(NetworkFilterError::EmptyRedirection); } - NetworkFilterOption::Redirect(String::from(value)) + NetworkFilterOption::Redirect(value) } ("redirect-rule", true) => return Err(NetworkFilterError::NegatedRedirection), ("redirect-rule", false) => { @@ -211,13 +215,11 @@ fn parse_filter_options(raw_options: &str) -> Result, N return Err(NetworkFilterError::EmptyRedirection); } - NetworkFilterOption::RedirectRule(String::from(value)) + NetworkFilterOption::RedirectRule(value) + } + ("csp", _) => { + NetworkFilterOption::Csp(if !value.is_empty() { Some(value) } else { None }) } - ("csp", _) => NetworkFilterOption::Csp(if !value.is_empty() { - Some(String::from(value)) - } else { - None - }), ("removeparam", true) => return Err(NetworkFilterError::NegatedRemoveparam), ("removeparam", false) => { if value.is_empty() { @@ -226,7 +228,7 @@ fn parse_filter_options(raw_options: &str) -> Result, N if !VALID_PARAM.is_match(value) { return Err(NetworkFilterError::RemoveparamRegexUnsupported); } - NetworkFilterOption::Removeparam(String::from(value)) + NetworkFilterOption::Removeparam(value) } ("generichide", true) | ("ghide", true) => { return Err(NetworkFilterError::NegatedGenericHide) diff --git a/src/filters/fb_network_builder.rs b/src/filters/fb_network_builder.rs index 5929c395..f7f80020 100644 --- a/src/filters/fb_network_builder.rs +++ b/src/filters/fb_network_builder.rs @@ -34,24 +34,24 @@ struct NetworkFilterFlatEntry<'a> { id: Hash, } -struct NetworkFilterListBuilder<'a> { +struct NetworkFilterListBuilder<'a, 'f> { flat_map_builder: FlatMultiMapBuilder>, token_frequencies: TokenSelector, - filters_to_optimize: HashMap>, + filters_to_optimize: HashMap>>, tokens_buffer: TokensBuffer, optimize: bool, } -pub(crate) struct NetworkRulesBuilder<'a> { - lists: Vec>, +pub(crate) struct NetworkRulesBuilder<'a, 'f> { + lists: Vec>, bad_filter_ids: HashSet, } -impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilter { +impl<'f, 'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilter<'f> { type Output = WIPOffset>; fn serialize( - network_filter: NetworkFilter, + network_filter: NetworkFilter<'f>, builder: &mut EngineFlatBuilder<'a>, ) -> WIPOffset> { let opt_domains = network_filter.opt_domains.as_ref().map(|v| { @@ -76,18 +76,14 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilter { let modifier_option = network_filter .modifier_option - .as_ref() .map(|s| builder.create_string(s)); let hostname = network_filter .hostname .as_ref() - .map(|s| builder.create_string(s)); + .map(|s| builder.create_string(s.as_ref())); - let tag = network_filter - .tag - .as_ref() - .map(|s| builder.create_string(s)); + let tag = network_filter.tag.map(|s| builder.create_string(s)); let mut filter_iter = network_filter.filter.iter(); let filter_count = filter_iter.len(); @@ -110,7 +106,7 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilter { let raw_line = network_filter .raw_line .as_ref() - .map(|v| builder.create_string(v.as_str())); + .map(|v| builder.create_string(v.as_ref())); fb::NetworkFilter::create( builder.raw_builder(), @@ -129,7 +125,7 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilter { } } -impl<'a> NetworkFilterListBuilder<'a> { +impl<'a, 'f> NetworkFilterListBuilder<'a, 'f> { fn new(optimize: bool) -> Self { Self { flat_map_builder: FlatMultiMapBuilder::with_capacity(1024), @@ -140,7 +136,11 @@ impl<'a> NetworkFilterListBuilder<'a> { } } - fn add_filter(&mut self, network_filter: NetworkFilter, builder: &mut EngineFlatBuilder<'a>) { + fn add_filter( + &mut self, + network_filter: NetworkFilter<'f>, + builder: &mut EngineFlatBuilder<'a>, + ) { let multi_tokens = network_filter.get_tokens(&mut self.tokens_buffer); let id = network_filter.get_id(); @@ -186,7 +186,7 @@ impl<'a> NetworkFilterListBuilder<'a> { } } -impl<'a> NetworkRulesBuilder<'a> { +impl<'a, 'f> NetworkRulesBuilder<'a, 'f> { pub fn new(optimize: bool) -> Self { let lists = (0..NetworkFilterListId::Size as usize) .map(|list_id| { @@ -201,7 +201,7 @@ impl<'a> NetworkRulesBuilder<'a> { } } - pub fn add_filter(&mut self, filter: NetworkFilter, builder: &mut EngineFlatBuilder<'a>) { + pub fn add_filter(&mut self, filter: NetworkFilter<'f>, builder: &mut EngineFlatBuilder<'a>) { if filter.is_badfilter() { // Note: `get_id()` doesn't include BAD_FILTER bit. self.bad_filter_ids.insert(filter.get_id()); @@ -238,7 +238,7 @@ impl<'a> NetworkRulesBuilder<'a> { fn add_filter_internal( &mut self, - network_filter: NetworkFilter, + network_filter: NetworkFilter<'f>, list_id: NetworkFilterListId, builder: &mut EngineFlatBuilder<'a>, ) { @@ -257,7 +257,7 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterFlatEntry<'a> } } -impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder<'a> { +impl<'a, 'f> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder<'a, 'f> { type Output = WIPOffset>>>; diff --git a/src/filters/network.rs b/src/filters/network.rs index 7f89f2b4..930bb485 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -282,9 +282,9 @@ impl From<&request::RequestType> for NetworkFilterMask { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum FilterPart { +pub enum FilterPart<'a> { Empty, - Simple(String), + Simple(Cow<'a, str>), AnyOf(Vec), } @@ -296,7 +296,7 @@ pub(crate) enum FilterTokens { } pub struct FilterPartIterator<'a> { - filter_part: &'a FilterPart, + filter_part: &'a FilterPart<'a>, index: usize, } @@ -309,14 +309,14 @@ impl<'a> Iterator for FilterPartIterator<'a> { FilterPart::Simple(s) => { if self.index == 0 { self.index += 1; - Some(s.as_str()) + Some(s.as_ref()) } else { None } } FilterPart::AnyOf(vec) => { if self.index < vec.len() { - let result = Some(vec[self.index].as_str()); + let result = Some(vec[self.index].as_ref()); self.index += 1; result } else { @@ -338,12 +338,17 @@ impl ExactSizeIterator for FilterPartIterator<'_> { } } -impl FilterPart { +impl FilterPart<'_> { pub fn string_view(&self) -> Option { match &self { FilterPart::Empty => None, - FilterPart::Simple(s) => Some(s.clone()), - FilterPart::AnyOf(s) => Some(s.join("|")), + FilterPart::Simple(s) => Some(s.to_string()), + FilterPart::AnyOf(s) => Some( + s.iter() + .map(|part| part.as_ref()) + .collect::>() + .join("|"), + ), } } @@ -356,19 +361,19 @@ impl FilterPart { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NetworkFilter { +pub struct NetworkFilter<'a> { pub mask: NetworkFilterMask, pub features_mask: NetworkFilterFeaturesMask, - pub filter: FilterPart, + pub filter: FilterPart<'a>, pub opt_domains: Option>, pub opt_not_domains: Option>, /// Used for `$redirect`, `$redirect-rule`, `$csp`, and `$removeparam` - only one of which is /// supported per-rule. - pub modifier_option: Option, - pub hostname: Option, - pub(crate) tag: Option, + pub modifier_option: Option<&'a str>, + pub hostname: Option>, + pub(crate) tag: Option<&'a str>, - pub raw_line: Option>, + pub raw_line: Option>, pub id: Hash, } @@ -377,21 +382,21 @@ pub struct NetworkFilter { // prevent field access, and don't load the ID from the serialized format. /// The ID of a filter is assumed to be correctly calculated for the purposes of this /// implementation. -impl PartialEq for NetworkFilter { +impl PartialEq for NetworkFilter<'_> { fn eq(&self, other: &Self) -> bool { self.id == other.id } } /// Filters are sorted by ID to preserve a stable ordering of data in the serialized format. -impl PartialOrd for NetworkFilter { +impl PartialOrd for NetworkFilter<'_> { fn partial_cmp(&self, other: &Self) -> Option { self.id.partial_cmp(&other.id) } } /// Ensure that no invalid option combinations were provided for a filter. -fn validate_options(options: &[NetworkFilterOption]) -> Result<(), NetworkFilterError> { +fn validate_options(options: &[NetworkFilterOption<'_>]) -> Result<(), NetworkFilterError> { let mut has_csp = false; let mut has_content_type = false; let mut modifier_options = 0; @@ -416,8 +421,25 @@ fn validate_options(options: &[NetworkFilterOption]) -> Result<(), NetworkFilter Ok(()) } -impl NetworkFilter { - pub fn parse(line: &str, debug: bool, _opts: ParseOptions) -> Result { +fn cow_ascii_lowercase<'a>(s: &'a str) -> Cow<'a, str> { + if s.as_bytes().iter().any(|byte| byte.is_ascii_uppercase()) { + Cow::Owned(s.to_ascii_lowercase()) + } else { + Cow::Borrowed(s) + } +} + +fn decode_hostname<'a>(host: &'a [u8]) -> Result, NetworkFilterError> { + idna::domain_to_ascii_cow(host, idna::AsciiDenyList::EMPTY) + .map_err(|_| NetworkFilterError::PunycodeError) +} + +impl<'a> NetworkFilter<'a> { + pub fn parse( + line: &'a str, + debug: bool, + _opts: ParseOptions, + ) -> Result { let parsed = AbstractNetworkFilter::parse(line)?; // Represent options as a bitmask @@ -432,13 +454,13 @@ impl NetworkFilter { let mut cpt_mask_positive: NetworkFilterMask = NetworkFilterMask::NONE; let mut cpt_mask_negative: NetworkFilterMask = NetworkFilterMask::NONE; - let mut hostname: Option = None; + let mut hostname: Option<&'a [u8]> = None; let mut opt_domains: Option> = None; let mut opt_not_domains: Option> = None; - let mut modifier_option: Option = None; - let mut tag: Option = None; + let mut modifier_option: Option<&'a str> = None; + let mut tag: Option<&'a str> = None; if parsed.exception { mask.set(NetworkFilterMask::IS_EXCEPTION, true); @@ -464,7 +486,7 @@ impl NetworkFilter { let mut opt_not_domains_array: Vec = vec![]; for (enabled, domain) in domains { - let domain_hash = utils::fast_hash(&domain); + let domain_hash = utils::fast_hash(domain); if !enabled { opt_not_domains_array.push(domain_hash); } else { @@ -597,7 +619,7 @@ impl NetworkFilter { end_url_anchor = true; } - let pattern = &parsed.pattern.pattern; + let pattern = &line[parsed.pattern.start..parsed.pattern.end]; let is_regex = check_is_regex(pattern); mask.set(NetworkFilterMask::IS_REGEX, is_regex); @@ -638,7 +660,7 @@ impl NetworkFilter { mask.set(NetworkFilterMask::IS_HOSTNAME_REGEX, true); } - hostname = Some(String::from(&pattern[..first_separator_start])); + hostname = Some(&pattern.as_bytes()[..first_separator_start]); filter_index_start = first_separator_start; // If the only symbol remaining for the selector is '^' then ignore it @@ -663,12 +685,12 @@ impl NetworkFilter { let slash_index = find_char(b'/', pattern.as_bytes()); slash_index .map(|i| { - hostname = Some(String::from(&pattern[..i])); + hostname = Some(&pattern.as_bytes()[..i]); filter_index_start += i; mask.set(NetworkFilterMask::IS_LEFT_ANCHOR, true); }) .or_else(|| { - hostname = Some(String::from(pattern)); + hostname = Some(pattern.as_bytes()); filter_index_start = filter_index_end; None }); @@ -720,13 +742,13 @@ impl NetworkFilter { } } - let filter: Option = if filter_index_end > filter_index_start { + let filter = if filter_index_end > filter_index_start { let filter_str = &pattern[filter_index_start..filter_index_end]; mask.set(NetworkFilterMask::IS_REGEX, check_is_regex(filter_str)); if mask.contains(NetworkFilterMask::MATCH_CASE) { - Some(String::from(filter_str)) + Some(Cow::Borrowed(filter_str)) } else { - Some(filter_str.to_ascii_lowercase()) + Some(cow_ascii_lowercase(filter_str)) } } else { None @@ -734,17 +756,7 @@ impl NetworkFilter { // TODO: ignore hostname anchor is not hostname provided - let hostname_decoded = hostname - .map(|host| { - let hostname = - idna::domain_to_ascii_cow(host.as_bytes(), idna::AsciiDenyList::EMPTY) - .map_err(|_| NetworkFilterError::PunycodeError)?; - match hostname { - Cow::Borrowed(_) => Ok(host), - Cow::Owned(h) => Ok(h.to_string()), - } - }) - .transpose(); + let hostname_decoded = hostname.map(decode_hostname).transpose()?; if features_mask.contains(NetworkFilterFeaturesMask::GENERIC_HIDE) && !parsed.exception { return Err(NetworkFilterError::GenericHideWithoutException); @@ -782,14 +794,14 @@ impl NetworkFilter { } else { FilterPart::Empty }, - hostname: hostname_decoded?, + hostname: hostname_decoded, mask, features_mask, opt_domains, opt_not_domains, tag, raw_line: if debug { - Some(Box::new(String::from(line))) + Some(Cow::Borrowed(line)) } else { None }, @@ -800,7 +812,7 @@ impl NetworkFilter { /// Given a hostname, produces an equivalent filter parsed from the form `"||hostname^"`, to /// emulate the behavior of hosts-style blocking. - pub fn parse_hosts_style(hostname: &str, debug: bool) -> Result { + pub fn parse_hosts_style(hostname: &'a str, debug: bool) -> Result { // Make sure the hostname doesn't contain any invalid characters static INVALID_CHARS: LazyLock = LazyLock::new(|| Regex::new("[/^*!?$&(){}\\[\\]+=~`\\s|@,'\"><:;]").unwrap()); @@ -816,14 +828,40 @@ impl NetworkFilter { return Err(NetworkFilterError::FilterParseError); } - // Parse it as a `||hostname^` rule. - let rule = format!("||{hostname}^"); - NetworkFilter::parse(&rule, debug, Default::default()) + let decoded_hostname = decode_hostname(hostname.as_bytes())?; + + let mask = NetworkFilterMask::THIRD_PARTY + | NetworkFilterMask::FIRST_PARTY + | NetworkFilterMask::FROM_HTTPS + | NetworkFilterMask::FROM_HTTP + | NetworkFilterMask::FROM_NETWORK_TYPES + | NetworkFilterMask::FROM_ALL_TYPES + | NetworkFilterMask::IS_HOSTNAME_ANCHOR + | NetworkFilterMask::IS_RIGHT_ANCHOR; + + let mut rule = String::with_capacity(hostname.len() + 3); + rule.push_str("||"); + rule.push_str(hostname); + rule.push('^'); + let id = utils::fast_hash(&rule); + + Ok(NetworkFilter { + filter: FilterPart::Empty, + hostname: Some(decoded_hostname), + mask, + features_mask: Default::default(), + opt_domains: None, + opt_not_domains: None, + tag: None, + raw_line: if debug { Some(Cow::Owned(rule)) } else { None }, + modifier_option: None, + id, + }) } pub fn get_id(&self) -> Hash { compute_filter_id( - self.modifier_option.as_deref(), + self.modifier_option, self.mask, self.features_mask, self.filter.string_view().as_deref(), @@ -859,7 +897,12 @@ impl NetworkFilter { (self.is_plain() || self.is_regex()) && !self.is_right_anchor(); let skip_first_token = self.is_right_anchor(); - utils::tokenize_filter_to(f, skip_first_token, skip_last_token, tokens_buffer); + utils::tokenize_filter_to( + f.as_ref(), + skip_first_token, + skip_last_token, + tokens_buffer, + ); } FilterPart::AnyOf(_) => (), // across AnyOf set of filters no single token is guaranteed to match to a request _ => (), @@ -868,13 +911,13 @@ impl NetworkFilter { // Append tokens from hostname, if any if !self.mask.contains(NetworkFilterMask::IS_HOSTNAME_REGEX) { if let Some(hostname) = self.hostname.as_ref() { - utils::tokenize_to(hostname, tokens_buffer); + utils::tokenize_to(hostname.as_ref(), tokens_buffer); } } else if let Some(hostname) = self.hostname.as_ref() { // Find last dot to tokenize the prefix let last_dot_pos = hostname.rfind('.'); if let Some(last_dot_pos) = last_dot_pos { - utils::tokenize_to(&hostname[..last_dot_pos], tokens_buffer); + utils::tokenize_to(&hostname.as_ref()[..last_dot_pos], tokens_buffer); } } @@ -884,7 +927,7 @@ impl NetworkFilter { .contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) { if let Some(removeparam) = &self.modifier_option { - if VALID_PARAM.is_match(removeparam) { + if VALID_PARAM.is_match(removeparam.as_ref()) { utils::tokenize_to(&removeparam.to_ascii_lowercase(), tokens_buffer); } } @@ -959,16 +1002,16 @@ impl NetworkFilter { } } -impl NetworkFilterMaskHelper for NetworkFilter { +impl NetworkFilterMaskHelper for NetworkFilter<'_> { fn has_flag(&self, v: NetworkFilterMask) -> bool { self.mask.contains(v) } } -impl fmt::Display for NetworkFilter { +impl fmt::Display for NetworkFilter<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { - match self.raw_line.as_ref() { - Some(r) => write!(f, "{}", r.clone()), + match self.raw_line.as_deref() { + Some(r) => write!(f, "{r}"), None => write!(f, "NetworkFilter"), } } diff --git a/src/lists.rs b/src/lists.rs index cac924ae..24c4cc0b 100644 --- a/src/lists.rs +++ b/src/lists.rs @@ -271,17 +271,22 @@ impl FilterSet { use crate::content_blocking; use std::collections::HashSet; - if !self.debug { + let FilterSet { + debug, + list_sources, + } = self; + + if !debug { return Err(()); } let mut network_filters = vec![]; let mut cosmetic_filters = vec![]; - for list_source in self.list_sources.iter() { + for list_source in list_sources.iter() { let list_text = list_source.list_text.lines(); let parse_options = list_source.parse_options; let (list_network_filters, list_cosmetic_filters) = - parse_filters(list_text, self.debug, parse_options); + parse_filters(list_text, debug, parse_options); network_filters.extend(list_network_filters); cosmetic_filters.extend(list_cosmetic_filters); } @@ -304,10 +309,11 @@ impl FilterSet { if bad_filter_ids.contains(&filter.get_id()) || filter.is_badfilter() { return; } - let original_rule = *filter + let original_rule = filter .raw_line - .clone() - .expect("All rules should be in debug mode"); + .as_ref() + .expect("All rules should be in debug mode") + .to_string(); if let Ok(equivalent) = TryInto::::try_into(filter) { filters_used.push(original_rule); @@ -387,18 +393,18 @@ pub enum FilterType { } /// Successful result of parsing a single line from a filter list -pub enum ParsedLine { - Network(NetworkFilter), +pub enum ParsedLine<'a> { + Network(NetworkFilter<'a>), Cosmetic(CosmeticFilter), } -impl From for ParsedLine { - fn from(v: NetworkFilter) -> Self { +impl From> for ParsedLine<'static> { + fn from(v: NetworkFilter<'static>) -> Self { ParsedLine::Network(v) } } -impl From for ParsedLine { +impl From for ParsedLine<'static> { fn from(v: CosmeticFilter) -> Self { ParsedLine::Cosmetic(v) } @@ -432,11 +438,11 @@ impl From for FilterParseError { } /// Parse a single line from a filter list -pub fn parse_filter( - line: &str, +pub fn parse_filter<'a>( + line: &'a str, debug: bool, opts: ParseOptions, -) -> Result { +) -> Result, FilterParseError> { let filter = line.trim(); if filter.is_empty() { @@ -447,12 +453,12 @@ pub fn parse_filter( FilterFormat::Standard => match (detect_filter_type(filter), opts.rule_types) { (FilterType::Network, RuleTypes::All | RuleTypes::NetworkOnly) => { NetworkFilter::parse(filter, debug, opts) - .map(|f| f.into()) + .map(ParsedLine::Network) .map_err(|e| e.into()) } (FilterType::Cosmetic, RuleTypes::All | RuleTypes::CosmeticOnly) => { CosmeticFilter::parse(filter, debug, opts.permissions) - .map(|f| f.into()) + .map(ParsedLine::Cosmetic) .map_err(|e| e.into()) } _ => Err(FilterParseError::Unsupported), @@ -501,21 +507,21 @@ pub fn parse_filter( } NetworkFilter::parse_hosts_style(hostname, debug) - .map(|f| f.into()) + .map(ParsedLine::Network) .map_err(|e| e.into()) } } } /// Parse an entire list of filters, ignoring any errors -pub fn parse_filters( - list: impl IntoIterator>, +pub fn parse_filters<'a>( + list: impl IntoIterator, debug: bool, opts: ParseOptions, -) -> (Vec, Vec) { +) -> (Vec>, Vec) { let (network_filters, cosmetic_filters): (Vec<_>, Vec<_>) = list .into_iter() - .filter_map(|line| match parse_filter(line.as_ref(), debug, opts) { + .filter_map(|line| match parse_filter(line, debug, opts) { Ok(ParsedLine::Network(f)) => Some(Either::Left(f)), Ok(ParsedLine::Cosmetic(f)) => Some(Either::Right(f)), _ => None, diff --git a/src/network_filter_list.rs b/src/network_filter_list.rs index 01e67f94..8a7935ef 100644 --- a/src/network_filter_list.rs +++ b/src/network_filter_list.rs @@ -22,12 +22,12 @@ pub struct CheckResult { pub raw_line: Option, } -impl From<&NetworkFilter> for CheckResult { +impl From<&NetworkFilter<'_>> for CheckResult { fn from(filter: &NetworkFilter) -> Self { Self { filter_mask: filter.mask, - modifier_option: filter.modifier_option.clone(), - raw_line: filter.raw_line.clone().map(|v| *v), + modifier_option: filter.modifier_option.map(|s| s.to_string()), + raw_line: filter.raw_line.as_ref().map(|line| line.to_string()), } } } diff --git a/src/optimizer.rs b/src/optimizer.rs index 73dc274f..e17a15a5 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -2,15 +2,16 @@ use crate::filters::network::{ FilterPart, NetworkFilter, NetworkFilterMask, NetworkFilterMaskHelper, }; use itertools::*; +use std::borrow::Cow; use std::collections::HashMap; trait Optimization { - fn fusion(&self, filters: &[NetworkFilter]) -> NetworkFilter; - fn group_by_criteria(&self, filter: &NetworkFilter) -> String; - fn select(&self, filter: &NetworkFilter) -> bool; + fn fusion<'a>(&self, filters: &[NetworkFilter<'a>]) -> NetworkFilter<'a>; + fn group_by_criteria(&self, filter: &NetworkFilter<'_>) -> String; + fn select(&self, filter: &NetworkFilter<'_>) -> bool; } -pub fn is_filter_optimizable_by_patterns(filter: &NetworkFilter) -> bool { +pub fn is_filter_optimizable_by_patterns(filter: &NetworkFilter<'_>) -> bool { filter.opt_domains.is_none() && filter.opt_not_domains.is_none() && !filter.is_hostname_anchor() @@ -19,8 +20,8 @@ pub fn is_filter_optimizable_by_patterns(filter: &NetworkFilter) -> bool { } /// Fuse `NetworkFilter`s together by applying optimizations sequentially. -pub fn optimize(filters: Vec) -> Vec { - let mut optimized: Vec = Vec::new(); +pub fn optimize<'a>(filters: Vec>) -> Vec> { + let mut optimized: Vec> = Vec::new(); /* let union_domain_group = UnionDomainGroup {}; @@ -40,11 +41,11 @@ pub fn optimize(filters: Vec) -> Vec { optimized } -fn apply_optimisation( +fn apply_optimisation<'a, T: Optimization>( optimization: &T, - filters: Vec, -) -> (Vec, Vec) { - let (positive, mut negative): (Vec, Vec) = + filters: Vec>, +) -> (Vec>, Vec>) { + let (positive, mut negative): (Vec>, Vec>) = filters.into_iter().partition_map(|f| { if optimization.select(&f) { Either::Left(f) @@ -53,7 +54,8 @@ fn apply_optimisation( } }); - let mut to_fuse: HashMap> = HashMap::with_capacity(positive.len()); + let mut to_fuse: HashMap>> = + HashMap::with_capacity(positive.len()); positive .into_iter() .for_each(|f| insert_dup(&mut to_fuse, optimization.group_by_criteria(&f), f)); @@ -85,7 +87,7 @@ struct SimplePatternGroup {} impl Optimization for SimplePatternGroup { // Group simple patterns, into a single filter - fn fusion(&self, filters: &[NetworkFilter]) -> NetworkFilter { + fn fusion<'a>(&self, filters: &[NetworkFilter<'a>]) -> NetworkFilter<'a> { let base_filter = &filters[0]; // FIXME: can technically panic, if filters list is empty let mut filter = base_filter.clone(); @@ -100,15 +102,16 @@ impl Optimization for SimplePatternGroup { for f in filters { match &f.filter { FilterPart::Empty => (), - FilterPart::Simple(s) => flat_patterns.push(s.clone()), - FilterPart::AnyOf(s) => flat_patterns.extend_from_slice(s), + FilterPart::Simple(s) => flat_patterns.push(s.to_string()), + FilterPart::AnyOf(s) => flat_patterns.extend(s.iter().cloned()), } } if flat_patterns.is_empty() { filter.filter = FilterPart::Empty; } else if flat_patterns.len() == 1 { - filter.filter = FilterPart::Simple(flat_patterns[0].clone()) + filter.filter = + FilterPart::Simple(Cow::Owned(flat_patterns.into_iter().next().unwrap())) } else { filter.filter = FilterPart::AnyOf(flat_patterns) } @@ -122,10 +125,10 @@ impl Optimization for SimplePatternGroup { .set(NetworkFilterMask::IS_COMPLETE_REGEX, is_complete_regex); if base_filter.raw_line.is_some() { - filter.raw_line = Some(Box::new( + filter.raw_line = Some(Cow::Owned( filters .iter() - .flat_map(|f| f.raw_line.clone()) + .flat_map(|f| f.raw_line.as_deref()) .join(" <+> "), )) } @@ -133,10 +136,10 @@ impl Optimization for SimplePatternGroup { filter } - fn group_by_criteria(&self, filter: &NetworkFilter) -> String { + fn group_by_criteria(&self, filter: &NetworkFilter<'_>) -> String { format!("{:b}:{:?}", filter.mask, filter.is_complete_regex()) } - fn select(&self, filter: &NetworkFilter) -> bool { + fn select(&self, filter: &NetworkFilter<'_>) -> bool { is_filter_optimizable_by_patterns(filter) } } @@ -145,7 +148,7 @@ impl Optimization for SimplePatternGroup { struct UnionDomainGroup {} impl Optimization for UnionDomainGroup { - fn fusion(&self, filters: &[NetworkFilter]) -> NetworkFilter { + fn fusion<'a>(&self, filters: &[NetworkFilter<'a>]) -> NetworkFilter<'a> { let base_filter = &filters[0]; // FIXME: can technically panic, if filters list is empty let mut filter = base_filter.clone(); let mut domains = HashSet::new(); @@ -178,10 +181,10 @@ impl Optimization for UnionDomainGroup { } if base_filter.raw_line.is_some() { - filter.raw_line = Some(Box::new( + filter.raw_line = Some(Cow::Owned( filters .iter() - .flat_map(|f| f.raw_line.clone()) + .flat_map(|f| f.raw_line.as_deref()) .join(" <+> "), )) } @@ -189,7 +192,7 @@ impl Optimization for UnionDomainGroup { filter } - fn group_by_criteria(&self, filter: &NetworkFilter) -> String { + fn group_by_criteria(&self, filter: &NetworkFilter<'_>) -> String { format!( "{:?}:{}:{:b}:{:?}", filter.hostname.as_ref(), @@ -199,7 +202,7 @@ impl Optimization for UnionDomainGroup { ) } - fn select(&self, filter: &NetworkFilter) -> bool { + fn select(&self, filter: &NetworkFilter<'_>) -> bool { !filter.is_csp() && (filter.opt_domains.is_some() || filter.opt_not_domains.is_some()) } } diff --git a/tests/legacy_harness.rs b/tests/legacy_harness.rs index 3d7dba43..864fefe9 100644 --- a/tests/legacy_harness.rs +++ b/tests/legacy_harness.rs @@ -871,14 +871,14 @@ mod legacy_misc_tests { // Host anchor is calculated correctly let filter = NetworkFilter::parse("||test.com$third-party", false, Default::default()).unwrap(); - assert_eq!(filter.hostname, Some(String::from("test.com"))); + assert_eq!(filter.hostname.as_deref(), Some("test.com")); let filter = NetworkFilter::parse("||test.com/ok$third-party", false, Default::default()).unwrap(); - assert_eq!(filter.hostname, Some(String::from("test.com"))); + assert_eq!(filter.hostname.as_deref(), Some("test.com")); let filter = NetworkFilter::parse("||test.com/ok", false, Default::default()).unwrap(); - assert_eq!(filter.hostname, Some(String::from("test.com"))); + assert_eq!(filter.hostname.as_deref(), Some("test.com")); } #[test] diff --git a/tests/matching.rs b/tests/matching.rs index 14759d8c..11687269 100644 --- a/tests/matching.rs +++ b/tests/matching.rs @@ -40,9 +40,9 @@ fn build_resources_from_filters(filters: &[String]) -> Vec { let redirect = f.modifier_option.unwrap(); Resource { - name: redirect.to_owned(), + name: redirect.to_string(), aliases: vec![], - kind: ResourceType::Mime(MimeType::from_extension(&redirect)), + kind: ResourceType::Mime(MimeType::from_extension(redirect)), content: BASE64_STANDARD.encode(redirect), dependencies: vec![], permission: Default::default(), diff --git a/tests/unit/filters/network.rs b/tests/unit/filters/network.rs index 7af7ec26..d0df847c 100644 --- a/tests/unit/filters/network.rs +++ b/tests/unit/filters/network.rs @@ -39,14 +39,17 @@ mod parse_tests { third_party: bool, } - impl From<&NetworkFilter> for NetworkFilterBreakdown { + impl From<&NetworkFilter<'_>> for NetworkFilterBreakdown { fn from(filter: &NetworkFilter) -> NetworkFilterBreakdown { NetworkFilterBreakdown { filter: filter.filter.string_view(), - hostname: filter.hostname.as_ref().cloned(), + hostname: filter.hostname.as_ref().map(|host| host.to_string()), opt_domains: filter.opt_domains.as_ref().cloned(), opt_not_domains: filter.opt_not_domains.as_ref().cloned(), - modifier_option: filter.modifier_option.as_ref().cloned(), + modifier_option: filter + .modifier_option + .as_ref() + .map(|value| value.to_string()), // filter type is_exception: filter.is_exception(), @@ -587,7 +590,7 @@ mod parse_tests { NetworkFilter::parse(r#"||foo.com$csp=self bar """#, true, Default::default()) .unwrap(); assert!(filter.is_csp()); - assert_eq!(filter.modifier_option, Some(String::from(r#"self bar """#))); + assert_eq!(filter.modifier_option, Some(r#"self bar """#)); } { // parses empty CSP @@ -714,12 +717,12 @@ mod parse_tests { let filter = NetworkFilter::parse("||foo.com$redirect=bar.js", true, Default::default()) .unwrap(); - assert_eq!(filter.modifier_option, Some(String::from("bar.js"))); + assert_eq!(filter.modifier_option, Some("bar.js")); } { let filter = NetworkFilter::parse("$redirect=bar.js", true, Default::default()).unwrap(); - assert_eq!(filter.modifier_option, Some(String::from("bar.js"))); + assert_eq!(filter.modifier_option, Some("bar.js")); } // parses ~redirect { @@ -794,7 +797,7 @@ mod parse_tests { NetworkFilter::parse("||foo.com^$removeparam=test", true, Default::default()) .unwrap(); assert!(filter.is_removeparam()); - assert_eq!(filter.modifier_option, Some("test".into())); + assert_eq!(filter.modifier_option, Some("test")); } } @@ -974,7 +977,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("example.com", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||example.com^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||example.com^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("example.com".to_string()); defaults.is_plain = true; @@ -986,7 +989,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("www.example.com", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||www.example.com^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||www.example.com^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("www.example.com".to_string()); defaults.is_plain = true; @@ -998,7 +1001,10 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("malware.example.com", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||malware.example.com^"); + assert_eq!( + filter.raw_line.as_deref().unwrap(), + "||malware.example.com^" + ); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("malware.example.com".to_string()); defaults.is_plain = true; @@ -1011,7 +1017,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("Example.COM", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||Example.COM^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||Example.COM^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("example.com".to_string()); defaults.is_plain = true; @@ -1024,7 +1030,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("WWW.Example.COM", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||WWW.Example.COM^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||WWW.Example.COM^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("www.example.com".to_string()); defaults.is_plain = true; @@ -1037,7 +1043,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("münchen.de", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||münchen.de^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||münchen.de^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("xn--mnchen-3ya.de".to_string()); defaults.is_plain = true; @@ -1050,7 +1056,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("www.münchen.de", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||www.münchen.de^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||www.münchen.de^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("www.xn--mnchen-3ya.de".to_string()); defaults.is_plain = true; @@ -1074,9 +1080,9 @@ mod parse_tests { let options = vec!["genericblock", "inline-script", "popunder", "popup", "woot"]; for option in options { - let filter = - NetworkFilter::parse(&format!("||foo.com${option}"), true, Default::default()); - assert!(filter.err().is_some()); + let rule = format!("||foo.com${option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()); + assert!(filter.is_err()); } } @@ -1134,9 +1140,8 @@ mod parse_tests { for option in options { // positive { - let filter = - NetworkFilter::parse(&format!("||foo.com${option}"), true, Default::default()) - .unwrap(); + let rule = format!("||foo.com${option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; @@ -1148,12 +1153,8 @@ mod parse_tests { } { - let filter = NetworkFilter::parse( - &format!("||foo.com$object,{option}"), - true, - Default::default(), - ) - .unwrap(); + let rule = format!("||foo.com$object,{option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; @@ -1166,12 +1167,8 @@ mod parse_tests { } { - let filter = NetworkFilter::parse( - &format!("||foo.com$domain=bar.com,{option}"), - true, - Default::default(), - ) - .unwrap(); + let rule = format!("||foo.com$domain=bar.com,{option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; @@ -1185,9 +1182,8 @@ mod parse_tests { // negative { - let filter = - NetworkFilter::parse(&format!("||foo.com$~{option}"), true, Default::default()) - .unwrap(); + let rule = format!("||foo.com$~{option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; @@ -1199,12 +1195,8 @@ mod parse_tests { } { - let filter = NetworkFilter::parse( - &format!("||foo.com${option},~{option}"), - true, - Default::default(), - ) - .unwrap(); + let rule = format!("||foo.com${option},~{option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; diff --git a/tests/unit/lists.rs b/tests/unit/lists.rs index 2d24e476..4b183922 100644 --- a/tests/unit/lists.rs +++ b/tests/unit/lists.rs @@ -197,11 +197,8 @@ mod tests { #[test] fn parse_filter_failed_fuzz_4() { // \\##+js(,\xdd\x8d - let parsed = parse_filter( - &String::from_utf8(vec![92, 35, 35, 43, 106, 115, 40, 44, 221, 141]).unwrap(), - true, - Default::default(), - ); + let rule = String::from_utf8(vec![92, 35, 35, 43, 106, 115, 40, 44, 221, 141]).unwrap(); + let parsed = parse_filter(&rule, true, Default::default()); #[cfg(feature = "css-validation")] assert!(parsed.is_err()); #[cfg(not(feature = "css-validation"))] From f18336c908cdc9358c3da7615e188796a7cec73d Mon Sep 17 00:00:00 2001 From: Mikhail Date: Thu, 11 Jun 2026 12:55:42 +0400 Subject: [PATCH 09/28] Speedup get_id() (#670) * removes an extra String alloc in get_id(); * uses rustc-hash as the impl which is more reliable and a little bit faster; --- src/filters/network.rs | 50 ++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/filters/network.rs b/src/filters/network.rs index 930bb485..4ca61245 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -3,8 +3,10 @@ use memchr::memchr as find_char; use regex::Regex; +use rustc_hash::FxHasher; use serde::{Deserialize, Serialize}; use std::borrow::Cow; +use std::hash::Hasher; use std::sync::LazyLock; use thiserror::Error; @@ -864,7 +866,7 @@ impl<'a> NetworkFilter<'a> { self.modifier_option, self.mask, self.features_mask, - self.filter.string_view().as_deref(), + &self.filter, self.hostname.as_deref(), self.opt_domains.as_ref(), self.opt_not_domains.as_ref(), @@ -1025,55 +1027,61 @@ pub(crate) trait NetworkMatchable { // Filter parsing // --------------------------------------------------------------------------- +#[inline] +fn write_str_to_hasher(hasher: &mut impl Hasher, s: &str) { + hasher.write_usize(s.len()); + hasher.write(s.as_bytes()); +} + fn compute_filter_id( modifier_option: Option<&str>, mask: NetworkFilterMask, features_mask: NetworkFilterFeaturesMask, - filter: Option<&str>, + filter: &FilterPart<'_>, hostname: Option<&str>, opt_domains: Option<&Vec>, opt_not_domains: Option<&Vec>, ) -> Hash { - let mut hash: Hash = (5408 * 33) ^ Hash::from(mask.bits()); + let mut hasher = FxHasher::default(); + + hasher.write_u64(u64::from(mask.bits())); // Exclude BAD_FILTER from the hash let features_mask_bits = features_mask.bits() & !NetworkFilterFeaturesMask::BAD_FILTER.bits(); - hash = hash.wrapping_mul(33) ^ Hash::from(features_mask_bits); + hasher.write_u64(u64::from(features_mask_bits)); if let Some(s) = modifier_option { - let chars = s.chars(); - for c in chars { - hash = hash.wrapping_mul(33) ^ (c as Hash); - } - }; + write_str_to_hasher(&mut hasher, s); + } if let Some(domains) = opt_domains { for d in domains { - hash = hash.wrapping_mul(33) ^ d; + hasher.write_u64(*d); } - }; + } if let Some(domains) = opt_not_domains { for d in domains { - hash = hash.wrapping_mul(33) ^ d; + hasher.write_u64(*d); } } - if let Some(s) = filter { - let chars = s.chars(); - for c in chars { - hash = hash.wrapping_mul(33) ^ (c as Hash); + match filter { + FilterPart::Empty => {} + FilterPart::Simple(s) => write_str_to_hasher(&mut hasher, s.as_ref()), + FilterPart::AnyOf(parts) => { + hasher.write_u64(parts.len() as u64); + for part in parts { + write_str_to_hasher(&mut hasher, part); + } } } if let Some(s) = hostname { - let chars = s.chars(); - for c in chars { - hash = hash.wrapping_mul(33) ^ (c as Hash); - } + write_str_to_hasher(&mut hasher, s); } - hash + hasher.finish() } /// Check if the sub-string contained between the indices start and end is a From 508257a934ff4de97730110f58e1d46c46271587 Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 5 Jun 2026 11:47:01 +0700 Subject: [PATCH 10/28] Add HTTP method (HEAD, POST, GET) support to NetworkFilter parsing, --- src/filters/abstract_network.rs | 37 +++++++++ src/filters/network.rs | 42 +++++++++- tests/unit/filters/network.rs | 134 ++++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 1 deletion(-) diff --git a/src/filters/abstract_network.rs b/src/filters/abstract_network.rs index 7d3658df..69ac61f6 100644 --- a/src/filters/abstract_network.rs +++ b/src/filters/abstract_network.rs @@ -31,6 +31,13 @@ pub(crate) struct NetworkFilterPattern { pub(crate) right_anchor: Option, } +#[derive(Clone, Copy)] +pub(crate) enum HttpMethod { + Get, + Head, + Post, +} + /// Any option that appears on the right side of a network filter as initiated by a `$` character. /// All `bool` arguments below are `true` if the option stands alone, or `false` if the option is /// negated using a prepended `~`. @@ -61,6 +68,7 @@ pub(crate) enum NetworkFilterOption<'a> { Websocket(bool), Font(bool), All, + Method(Vec<(bool, HttpMethod)>), } impl NetworkFilterOption<'_> { @@ -255,6 +263,35 @@ fn parse_filter_options<'a>( ("font", negated) => NetworkFilterOption::Font(!negated), ("all", true) => return Err(NetworkFilterError::NegatedAll), ("all", false) => NetworkFilterOption::All, + ("method", true) => return Err(NetworkFilterError::UnrecognisedOption), + ("method", false) => { + if value.is_empty() { + return Err(NetworkFilterError::UnrecognisedOption); + } + let methods: Vec<(bool, HttpMethod)> = value + .split('|') + .filter_map(|method| { + let (enabled, name) = if let Some(negated) = method.strip_prefix('~') { + (false, negated) + } else { + (true, method) + }; + if name.eq_ignore_ascii_case("get") { + Some((enabled, HttpMethod::Get)) + } else if name.eq_ignore_ascii_case("head") { + Some((enabled, HttpMethod::Head)) + } else if name.eq_ignore_ascii_case("post") { + Some((enabled, HttpMethod::Post)) + } else { + None + } + }) + .collect(); + if methods.is_empty() { + return Err(NetworkFilterError::UnrecognisedOption); + } + NetworkFilterOption::Method(methods) + } (_, _) => return Err(NetworkFilterError::UnrecognisedOption), }); } diff --git a/src/filters/network.rs b/src/filters/network.rs index 4ca61245..491a34fd 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -13,7 +13,8 @@ use thiserror::Error; use std::fmt; use crate::filters::abstract_network::{ - AbstractNetworkFilter, NetworkFilterLeftAnchor, NetworkFilterOption, NetworkFilterRightAnchor, + AbstractNetworkFilter, HttpMethod, NetworkFilterLeftAnchor, NetworkFilterOption, + NetworkFilterRightAnchor, }; use crate::lists::ParseOptions; use crate::regex_manager::RegexManager; @@ -134,6 +135,14 @@ bitflags::bitflags! { // "Other" network request types const UNMATCHED = 1 << 25; + const FROM_GET = 1 << 26; + const FROM_HEAD = 1 << 27; + const FROM_POST = 1 << 30; + + const FROM_METHODS = Self::FROM_GET.bits() | + Self::FROM_HEAD.bits() | + Self::FROM_POST.bits(); + // Includes all request types that are implied by any negated types. const FROM_NETWORK_TYPES = Self::FROM_FONT.bits() | Self::FROM_IMAGE.bits() | @@ -455,6 +464,8 @@ impl<'a> NetworkFilter<'a> { // content type options. let mut cpt_mask_positive: NetworkFilterMask = NetworkFilterMask::NONE; let mut cpt_mask_negative: NetworkFilterMask = NetworkFilterMask::NONE; + let mut method_mask_positive: NetworkFilterMask = NetworkFilterMask::NONE; + let mut method_mask_negative: NetworkFilterMask = NetworkFilterMask::NONE; let mut hostname: Option<&'a [u8]> = None; @@ -481,6 +492,16 @@ impl<'a> NetworkFilter<'a> { }; } + macro_rules! apply_method { + ($method:ident, $enabled:ident) => { + if $enabled { + method_mask_positive.set(NetworkFilterMask::$method, true); + } else { + method_mask_negative.set(NetworkFilterMask::$method, true); + } + }; + } + options.into_iter().for_each(|option| { match option { NetworkFilterOption::Domain(domains) => { @@ -576,6 +597,15 @@ impl<'a> NetworkFilter<'a> { } NetworkFilterOption::Font(enabled) => apply_content_type!(FROM_FONT, enabled), NetworkFilterOption::All => apply_content_type!(FROM_ALL_TYPES, true), + NetworkFilterOption::Method(methods) => { + for (enabled, method) in methods { + match method { + HttpMethod::Get => apply_method!(FROM_GET, enabled), + HttpMethod::Head => apply_method!(FROM_HEAD, enabled), + HttpMethod::Post => apply_method!(FROM_POST, enabled), + } + } + } } }); } @@ -790,6 +820,16 @@ impl<'a> NetworkFilter<'a> { // Finally, apply any explicitly negated request types mask &= !cpt_mask_negative; + if !method_mask_positive.is_empty() || !method_mask_negative.is_empty() { + mask |= method_mask_positive; + if (method_mask_negative & NetworkFilterMask::FROM_METHODS) != NetworkFilterMask::NONE + && (method_mask_positive & NetworkFilterMask::FROM_METHODS).is_empty() + { + mask |= NetworkFilterMask::FROM_METHODS; + } + mask &= !method_mask_negative; + } + Ok(NetworkFilter { filter: if let Some(simple_filter) = filter { FilterPart::Simple(simple_filter) diff --git a/tests/unit/filters/network.rs b/tests/unit/filters/network.rs index d0df847c..ab189731 100644 --- a/tests/unit/filters/network.rs +++ b/tests/unit/filters/network.rs @@ -35,6 +35,9 @@ mod parse_tests { from_websocket: bool, from_xml_http_request: bool, from_document: bool, + from_get: bool, + from_head: bool, + from_post: bool, match_case: bool, third_party: bool, } @@ -76,6 +79,9 @@ mod parse_tests { from_websocket: filter.mask.contains(NetworkFilterMask::FROM_WEBSOCKET), from_xml_http_request: filter.mask.contains(NetworkFilterMask::FROM_XMLHTTPREQUEST), from_document: filter.mask.contains(NetworkFilterMask::FROM_DOCUMENT), + from_get: filter.mask.contains(NetworkFilterMask::FROM_GET), + from_head: filter.mask.contains(NetworkFilterMask::FROM_HEAD), + from_post: filter.mask.contains(NetworkFilterMask::FROM_POST), match_case: filter.match_case(), third_party: filter.third_party(), } @@ -115,6 +121,9 @@ mod parse_tests { from_websocket: true, from_xml_http_request: true, from_document: false, + from_get: false, + from_head: false, + from_post: false, match_case: false, third_party: true, } @@ -1250,4 +1259,129 @@ mod parse_tests { &[utils::fast_hash("some"), utils::fast_hash("primewire")] ); } + + #[test] + fn handles_method_options() { + { + let filter = + NetworkFilter::parse("||foo$method=post", true, Default::default()).unwrap(); + let mut expected = default_network_filter_breakdown(); + expected.hostname = Some(String::from("foo")); + expected.is_hostname_anchor = true; + expected.is_plain = true; + expected.from_post = true; + assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); + } + + { + let filter = + NetworkFilter::parse("||foo$method=post|get", true, Default::default()).unwrap(); + let mut expected = default_network_filter_breakdown(); + expected.hostname = Some(String::from("foo")); + expected.is_hostname_anchor = true; + expected.is_plain = true; + expected.from_post = true; + expected.from_get = true; + assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); + } + + { + let filter = + NetworkFilter::parse("||foo$method=head|get", true, Default::default()).unwrap(); + let mut expected = default_network_filter_breakdown(); + expected.hostname = Some(String::from("foo")); + expected.is_hostname_anchor = true; + expected.is_plain = true; + expected.from_head = true; + expected.from_get = true; + assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); + } + + { + let filter = + NetworkFilter::parse("||foo$method=~get", true, Default::default()).unwrap(); + let mut expected = default_network_filter_breakdown(); + expected.hostname = Some(String::from("foo")); + expected.is_hostname_anchor = true; + expected.is_plain = true; + expected.from_head = true; + expected.from_post = true; + assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); + } + + { + let filter = + NetworkFilter::parse("||foo$method=POST", true, Default::default()).unwrap(); + let mut expected = default_network_filter_breakdown(); + expected.hostname = Some(String::from("foo")); + expected.is_hostname_anchor = true; + expected.is_plain = true; + expected.from_post = true; + assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); + } + + { + let filter = + NetworkFilter::parse("||foo$method=post|put", true, Default::default()).unwrap(); + let mut expected = default_network_filter_breakdown(); + expected.hostname = Some(String::from("foo")); + expected.is_hostname_anchor = true; + expected.is_plain = true; + expected.from_post = true; + assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); + } + + for filter_text in [ + "||lemonde.fr/*?s=$xhr,method=post", + "||perplexity.ai/rest/metrics/collect^$xhr,1p,method=post", + "||apmplus.volces.com/monitor_web/collect$xhr,method=post", + "@@||www.realclear*/esm/assets/js/analytics/chartbeat.js?v=$xhr,1p,method=get,domain=com|org", + "||pagead2.googlesyndication.com^$3p,xhr,method=head,redirect-rule=noop.js,domain=photopea.com", + "||void.nicopr.fr/rec$method=POST", + ] { + assert!( + NetworkFilter::parse(filter_text, true, Default::default()).is_ok(), + "failed to parse: {filter_text}" + ); + } + + { + let filter = NetworkFilter::parse( + "||perplexity.ai/rest/metrics/collect^$xhr,1p,method=post", + true, + Default::default(), + ) + .unwrap(); + assert!(filter.mask.contains(NetworkFilterMask::FROM_POST)); + assert!(filter.mask.contains(NetworkFilterMask::FROM_XMLHTTPREQUEST)); + assert!(!filter.mask.contains(NetworkFilterMask::FROM_NETWORK_TYPES)); + assert!(filter.first_party()); + assert!(!filter.third_party()); + } + + { + let filter = NetworkFilter::parse( + "@@*$xhr,method=head|get,domain=app.axenthost.com,3p", + true, + Default::default(), + ) + .unwrap(); + assert!(filter.is_exception()); + assert!(filter.mask.contains(NetworkFilterMask::FROM_HEAD)); + assert!(filter.mask.contains(NetworkFilterMask::FROM_GET)); + assert!(!filter.mask.contains(NetworkFilterMask::FROM_POST)); + } + + for filter_text in [ + "||foo$method=put", + "||foo$method=connect|delete", + "||foo$method=", + "||foo$~method=post", + ] { + assert!( + NetworkFilter::parse(filter_text, true, Default::default()).is_err(), + "expected parse error: {filter_text}" + ); + } + } } From 36199a0d074f88fea54f69691074cd05aee1049a Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 5 Jun 2026 12:04:54 +0700 Subject: [PATCH 11/28] Added support for HTTP methods (GET, HEAD, POST) in Request and NetworkFilter. --- src/filters/network.rs | 23 +++++++ src/filters/network_matchers.rs | 5 +- src/request.rs | 35 ++++++++++ tests/unit/blocker.rs | 2 +- tests/unit/filters/network_matchers.rs | 89 ++++++++++++++++++++++++++ tests/unit/request.rs | 1 + 6 files changed, 152 insertions(+), 3 deletions(-) diff --git a/src/filters/network.rs b/src/filters/network.rs index 491a34fd..00e4db16 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -173,6 +173,19 @@ bitflags::bitflags! { } } +impl NetworkFilterMask { + #[inline] + pub(crate) fn check_method_allowed(self, method: Option<&request::RequestMethod>) -> bool { + if !self.intersects(Self::FROM_METHODS) { + return true; + } + let Some(method) = method else { + return false; + }; + self.contains(Self::from(method)) + } +} + pub trait NetworkFilterMaskHelper { fn has_flag(&self, v: NetworkFilterMask) -> bool; @@ -268,6 +281,16 @@ impl fmt::Display for NetworkFilterMask { } } +impl From<&request::RequestMethod> for NetworkFilterMask { + fn from(method: &request::RequestMethod) -> NetworkFilterMask { + match method { + request::RequestMethod::Get => NetworkFilterMask::FROM_GET, + request::RequestMethod::Head => NetworkFilterMask::FROM_HEAD, + request::RequestMethod::Post => NetworkFilterMask::FROM_POST, + } + } +} + impl From<&request::RequestType> for NetworkFilterMask { fn from(request_type: &request::RequestType) -> NetworkFilterMask { match request_type { diff --git a/src/filters/network_matchers.rs b/src/filters/network_matchers.rs index aac604ac..830eb4f0 100644 --- a/src/filters/network_matchers.rs +++ b/src/filters/network_matchers.rs @@ -407,10 +407,11 @@ pub fn check_options(mask: NetworkFilterMask, request: &request::Request) -> boo // We first discard requests based on type, protocol and party. This is really // cheap and should be done first. if !mask.check_cpt_allowed(&request.request_type) - || (request.is_https && !mask.for_https()) - || (request.is_http && !mask.for_http()) || (!mask.first_party() && !request.is_third_party) || (!mask.third_party() && request.is_third_party) + || (request.is_https && !mask.for_https()) + || (request.is_http && !mask.for_http()) + || !NetworkFilterMask::check_method_allowed(mask, request.method.as_ref()) { return false; } diff --git a/src/request.rs b/src/request.rs index fd5409f1..4876f362 100644 --- a/src/request.rs +++ b/src/request.rs @@ -5,6 +5,13 @@ use thiserror::Error; use crate::url_parser; use crate::utils; +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum RequestMethod { + Get, + Head, + Post, +} + /// The type of resource requested from the URL endpoint. #[derive(Clone, PartialEq, Debug)] pub enum RequestType { @@ -79,6 +86,7 @@ fn cpt_match_type(cpt: &str) -> RequestType { #[derive(Clone, Debug)] pub struct Request { pub request_type: RequestType, + pub method: Option, pub is_http: bool, pub is_https: bool, @@ -116,6 +124,18 @@ impl Request { &self.request_tokens } + fn parse_method(raw_method: &str) -> Option { + if raw_method.eq_ignore_ascii_case("get") { + Some(RequestMethod::Get) + } else if raw_method.eq_ignore_ascii_case("head") { + Some(RequestMethod::Head) + } else if raw_method.eq_ignore_ascii_case("post") { + Some(RequestMethod::Post) + } else { + None + } + } + #[allow(clippy::too_many_arguments)] fn from_detailed_parameters( raw_type: &str, @@ -125,6 +145,7 @@ impl Request { source_hostname: &str, third_party: bool, original_url: String, + method: Option, ) -> Request { let is_http: bool; let is_https: bool; @@ -167,6 +188,7 @@ impl Request { Request { request_type, + method, url: url.to_owned(), url_lower_cased: url_lower_cased.to_owned(), hostname: hostname.to_owned(), @@ -182,7 +204,17 @@ impl Request { /// Construct a new [`Request`]. pub fn new(url: &str, source_url: &str, request_type: &str) -> Result { + Self::new_with_method(url, source_url, request_type, None) + } + + pub fn new_with_method( + url: &str, + source_url: &str, + request_type: &str, + method: Option<&str>, + ) -> Result { if let Some(parsed_url) = url_parser::parse_url(url) { + let parsed_method = method.and_then(Self::parse_method); if let Some(parsed_source) = url_parser::parse_url(source_url) { let source_domain = parsed_source.domain(); @@ -196,6 +228,7 @@ impl Request { parsed_source.hostname(), third_party, url.to_string(), + parsed_method, )) } else { Ok(Request::from_detailed_parameters( @@ -206,6 +239,7 @@ impl Request { "", true, url.to_string(), + parsed_method, )) } } else { @@ -234,6 +268,7 @@ impl Request { source_hostname, third_party, url.to_string(), + None, ) } } diff --git a/tests/unit/blocker.rs b/tests/unit/blocker.rs index 76dbb432..4b655df9 100644 --- a/tests/unit/blocker.rs +++ b/tests/unit/blocker.rs @@ -1373,7 +1373,7 @@ mod legacy_rule_parsing_tests { }; // differences in counts explained by hashset size underreporting as detailed in the next two cases const EASY_PRIVACY: ListCounts = ListCounts { - filters: 55395 - 818, // total - exceptions + filters: 55396 - 818, // total - exceptions cosmetic_filters: 32, exceptions: 818, duplicates: 2, diff --git a/tests/unit/filters/network_matchers.rs b/tests/unit/filters/network_matchers.rs index 6b80d407..9f898bb2 100644 --- a/tests/unit/filters/network_matchers.rs +++ b/tests/unit/filters/network_matchers.rs @@ -472,6 +472,95 @@ mod match_tests { } } + #[test] + fn check_method_options_works() { + let url = "https://foo.com/bar"; + + { + let filter = + NetworkFilter::parse("||foo$method=post", true, Default::default()).unwrap(); + let request = + request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + assert!(check_options(&filter, &request)); + let request = + request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + assert!(!check_options(&filter, &request)); + } + + { + let filter = NetworkFilter::parse("||foo$method=post|get", true, Default::default()) + .unwrap(); + let request = + request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + assert!(check_options(&filter, &request)); + } + + { + let filter = NetworkFilter::parse("||foo$method=head|get", true, Default::default()) + .unwrap(); + let request = + request::Request::new_with_method(url, "", "xhr", Some("head")).unwrap(); + assert!(check_options(&filter, &request)); + let request = + request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + assert!(!check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$method=~get", true, Default::default()).unwrap(); + let request = + request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + assert!(!check_options(&filter, &request)); + let request = + request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + assert!(check_options(&filter, &request)); + let request = + request::Request::new_with_method(url, "", "xhr", Some("head")).unwrap(); + assert!(check_options(&filter, &request)); + let request = request::Request::new(url, "", "xhr").unwrap(); + assert!(!check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo", true, Default::default()).unwrap(); + let request = + request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + assert!(check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$xhr,method=post", true, Default::default()).unwrap(); + let request = + request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + assert!(check_options(&filter, &request)); + let request = + request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + assert!(!check_options(&filter, &request)); + let request = + request::Request::new_with_method(url, "", "script", Some("post")).unwrap(); + assert!(!check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$xhr,method=get", true, Default::default()).unwrap(); + let request = + request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + assert!(check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$xhr,method=head", true, Default::default()).unwrap(); + let request = + request::Request::new_with_method(url, "", "xhr", Some("head")).unwrap(); + assert!(check_options(&filter, &request)); + } + } + #[test] fn check_domain_option_subsetting_works() { { diff --git a/tests/unit/request.rs b/tests/unit/request.rs index d1486e08..8a0fb551 100644 --- a/tests/unit/request.rs +++ b/tests/unit/request.rs @@ -21,6 +21,7 @@ mod tests { source_hostname, third_party, url.to_string(), + None, ) } From 53f5024ec976269395bfc37c31539432c908ce71 Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 5 Jun 2026 14:09:02 +0700 Subject: [PATCH 12/28] Extended request API to accept optional HTTP method parameter. --- benches/bench_matching.rs | 1 + js/src/lib.rs | 65 ++++++++++++++++++++++++++++++++++----- js/test/bindings.test.mjs | 21 +++++++++++++ src/request.rs | 3 +- tests/ublock-coverage.rs | 1 + tests/unit/engine.rs | 60 ++++++++++++++++++++++++++++++++++++ 6 files changed, 142 insertions(+), 9 deletions(-) diff --git a/benches/bench_matching.rs b/benches/bench_matching.rs index 016c7956..03b0faf9 100644 --- a/benches/bench_matching.rs +++ b/benches/bench_matching.rs @@ -77,6 +77,7 @@ fn bench_rule_matching_browserlike(blocker: &Engine, requests: &[ParsedRequest]) source_hostname, request_type, *third_party, + None, )); if check.matched { matches += 1; diff --git a/js/src/lib.rs b/js/src/lib.rs index 0572d10b..939c79d5 100644 --- a/js/src/lib.rs +++ b/js/src/lib.rs @@ -187,17 +187,46 @@ fn engine_check(mut cx: FunctionContext) -> JsResult { let source_url: String = cx.argument::(2)?.value(&mut cx); let request_type: String = cx.argument::(3)?.value(&mut cx); - let debug = match cx.argument_opt(4) { + // Optional trailing args: + // - legacy: `(debug: boolean)` + // - new: `(method: string | null | undefined, debug?: boolean)` + let (method, debug) = match cx.argument_opt(4) { + None => (None, false), Some(arg) => { - // Throw if the argument exists and it cannot be downcasted to a boolean - arg.downcast::(&mut cx) - .or_throw(&mut cx)? - .value(&mut cx) + if arg.is_a::(&mut cx) || arg.is_a::(&mut cx) { + let debug = match cx.argument_opt(5) { + Some(debug_arg) => debug_arg + .downcast::(&mut cx) + .or_throw(&mut cx)? + .value(&mut cx), + None => false, + }; + (None, debug) + } else if let Ok(debug) = arg.downcast::(&mut cx) { + (None, debug.value(&mut cx)) + } else if let Ok(method) = arg.downcast::(&mut cx) { + let debug = match cx.argument_opt(5) { + Some(debug_arg) => debug_arg + .downcast::(&mut cx) + .or_throw(&mut cx)? + .value(&mut cx), + None => false, + }; + (Some(method.value(&mut cx)), debug) + } else { + cx.throw_error( + "expected boolean (debug), string (method), null, or undefined as 4th argument", + )? + } } - None => false, }; - let request = match adblock::request::Request::new(&url, &source_url, &request_type) { + let request = match adblock::request::Request::new_with_method( + &url, + &source_url, + &request_type, + method.as_deref(), + ) { Ok(r) => r, Err(e) => cx.throw_error(e.to_string())?, }; @@ -329,7 +358,27 @@ fn validate_request(mut cx: FunctionContext) -> JsResult { let url: String = cx.argument::(0)?.value(&mut cx); let source_url: String = cx.argument::(1)?.value(&mut cx); let request_type: String = cx.argument::(2)?.value(&mut cx); - let request_ok = adblock::request::Request::new(&url, &source_url, &request_type).is_ok(); + let method = match cx.argument_opt(3) { + None => None, + Some(arg) => { + if arg.is_a::(&mut cx) || arg.is_a::(&mut cx) { + None + } else { + Some( + arg.downcast::(&mut cx) + .or_throw(&mut cx)? + .value(&mut cx), + ) + } + } + }; + let request_ok = adblock::request::Request::new_with_method( + &url, + &source_url, + &request_type, + method.as_deref(), + ) + .is_ok(); Ok(cx.boolean(request_ok)) } diff --git a/js/test/bindings.test.mjs b/js/test/bindings.test.mjs index 7f288778..c76f659e 100644 --- a/js/test/bindings.test.mjs +++ b/js/test/bindings.test.mjs @@ -258,6 +258,27 @@ describe('Engine.check — type-specific rules', () => { assert.equal(engine.check('https://ads.example.com/api', 'https://pub.com', 'xmlhttprequest'), true); assert.equal(engine.check('https://ads.example.com/api', 'https://pub.com', 'image'), false); }); + + it('$method=post blocks only POST requests', () => { + const fs = new FilterSet(); + fs.addFilters('||api.example.com^$xhr,method=post'); + const engine = new Engine(fs, true); + const url = 'https://api.example.com/collect'; + const source = 'https://pub.com'; + assert.equal(engine.check(url, source, 'xhr', 'post'), true); + assert.equal(engine.check(url, source, 'xhr', 'get'), false); + assert.equal(engine.check(url, source, 'xhr'), false); + }); + + it('debug=true remains backward compatible as 4th boolean arg', () => { + const fs = new FilterSet(true); + fs.addFilters('||api.example.com^$xhr,method=post'); + const engine = new Engine(fs, true); + const result = engine.check( + 'https://api.example.com/collect', 'https://pub.com', 'xhr', true, + ); + assert.equal(result.matched, false); + }); }); // --------------------------------------------------------------------------- diff --git a/src/request.rs b/src/request.rs index 4876f362..39479e2a 100644 --- a/src/request.rs +++ b/src/request.rs @@ -256,6 +256,7 @@ impl Request { source_hostname: &str, request_type: &str, third_party: bool, + method: Option<&str>, ) -> Request { let splitter = memchr::memchr(b':', url.as_bytes()).unwrap_or(0); let schema: &str = &url[..splitter]; @@ -268,7 +269,7 @@ impl Request { source_hostname, third_party, url.to_string(), - None, + method.and_then(Self::parse_method), ) } } diff --git a/tests/ublock-coverage.rs b/tests/ublock-coverage.rs index 02de72d7..b521a930 100644 --- a/tests/ublock-coverage.rs +++ b/tests/ublock-coverage.rs @@ -289,6 +289,7 @@ fn check_matching_hostnames() { source_host.hostname(), &req.r#type, third_party, + None, ); let checked = engine.check_network_request(&request); diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index ea751986..9f13a743 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -892,4 +892,64 @@ trustedSetLocalStorageItem("mol.ads.cmp.tcf.cache", "{\"getTCData\":{\"cmpId\":2 "" ); } + + #[test] + fn method_option_blocks_post_xhr_only() { + let engine = Engine::new_with_list_text( + "||perplexity.ai/rest/metrics/collect^$xhr,1p,method=post", + Default::default(), + ); + let url = "https://perplexity.ai/rest/metrics/collect?foo=bar"; + let source = "https://perplexity.ai/page"; + + let post = Request::new_with_method(url, source, "xhr", Some("post")).unwrap(); + assert!( + engine.check_network_request(&post).matched, + "POST xhr 1p should match" + ); + + let get = Request::new_with_method(url, source, "xhr", Some("get")).unwrap(); + assert!( + !engine.check_network_request(&get).matched, + "GET xhr should not match method=post rule" + ); + + let post_3p = + Request::new_with_method(url, "https://other.com/page", "xhr", Some("post")).unwrap(); + assert!( + !engine.check_network_request(&post_3p).matched, + "POST xhr 3p should not match 1p rule" + ); + + let post_no_method = Request::new(url, source, "xhr").unwrap(); + assert!( + !engine.check_network_request(&post_no_method).matched, + "missing method should not match method=post rule" + ); + } + + #[test] + fn method_option_exception_head_get() { + let engine = Engine::new_with_list_text( + "||tracker.example.com^$xhr\n@@*$xhr,method=head|get,domain=app.axenthost.com,3p", + Default::default(), + ); + let url = "https://tracker.example.com/pixel"; + let source = "https://app.axenthost.com/page"; + + for method in ["get", "head"] { + let request = + Request::new_with_method(url, source, "xhr", Some(method)).unwrap(); + assert!( + !engine.check_network_request(&request).matched, + "{method} xhr should be excepted" + ); + } + + let post = Request::new_with_method(url, source, "xhr", Some("post")).unwrap(); + assert!( + engine.check_network_request(&post).matched, + "POST xhr should still be blocked" + ); + } } From e21d6b9289410dde063428ced6a1a920dbfbc2e0 Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 5 Jun 2026 14:44:51 +0700 Subject: [PATCH 13/28] Removed new_with_method fn. --- benches/bench_matching.rs | 8 +- benches/bench_memory.rs | 2 +- benches/bench_redirect_performance.rs | 2 +- benches/bench_url.rs | 4 +- examples/deserialization.rs | 2 +- examples/example.rs | 1 + examples/generate-dat.rs | 1 + examples/use-dat.rs | 1 + fuzz/fuzz_targets/parse_request.rs | 15 +- js/src/lib.rs | 38 ++--- src/engine.rs | 2 +- src/request.rs | 17 +- tests/legacy_harness.rs | 39 +++-- tests/live.rs | 15 +- tests/matching.rs | 6 +- tests/simple_use.rs | 1 + tests/ublock-coverage.rs | 43 ++---- tests/unit/blocker.rs | 165 ++++++++++++++++---- tests/unit/engine.rs | 130 +++++++--------- tests/unit/filters/network_matchers.rs | 206 ++++++++++++++----------- tests/unit/network_filter_list.rs | 14 +- tests/unit/optimizer.rs | 78 +++------- tests/unit/regex_manager.rs | 2 +- tests/unit/request.rs | 5 +- 24 files changed, 429 insertions(+), 368 deletions(-) diff --git a/benches/bench_matching.rs b/benches/bench_matching.rs index 03b0faf9..226c267e 100644 --- a/benches/bench_matching.rs +++ b/benches/bench_matching.rs @@ -20,7 +20,7 @@ struct TestRequest { impl From<&TestRequest> for Request { fn from(v: &TestRequest) -> Self { - Request::new(&v.url, &v.frameUrl, &v.cpt).unwrap() + Request::new(&v.url, &v.frameUrl, &v.cpt, "").unwrap() } } @@ -131,7 +131,7 @@ fn rule_match_parsed_el(c: &mut Criterion) { let requests = load_requests(); let requests_parsed: Vec<_> = requests .into_iter() - .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt)) + .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt, "")) .filter_map(Result::ok) .collect(); let requests_len = requests_parsed.len() as u64; @@ -159,7 +159,7 @@ fn rule_match_parsed_elep_slimlist(c: &mut Criterion) { let requests = load_requests(); let requests_parsed: Vec<_> = requests .into_iter() - .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt)) + .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt, "")) .filter_map(Result::ok) .collect(); let requests_len = requests_parsed.len() as u64; @@ -170,7 +170,7 @@ fn rule_match_parsed_elep_slimlist(c: &mut Criterion) { let requests_copy = load_requests(); let requests_parsed_copy: Vec<_> = requests_copy .into_iter() - .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt)) + .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt, "")) .filter_map(Result::ok) .collect(); diff --git a/benches/bench_memory.rs b/benches/bench_memory.rs index 7eca9310..e4dc1409 100644 --- a/benches/bench_memory.rs +++ b/benches/bench_memory.rs @@ -128,7 +128,7 @@ struct TestRequest { impl From<&TestRequest> for Request { fn from(v: &TestRequest) -> Self { - Request::new(&v.url, &v.frameUrl, &v.cpt).unwrap() + Request::new(&v.url, &v.frameUrl, &v.cpt, "").unwrap() } } diff --git a/benches/bench_redirect_performance.rs b/benches/bench_redirect_performance.rs index f9cb0245..0108467b 100644 --- a/benches/bench_redirect_performance.rs +++ b/benches/bench_redirect_performance.rs @@ -204,7 +204,7 @@ pub fn build_custom_requests(rules: Vec) -> Vec { let source_url = format!("https://{source_hostname}"); - Request::new(&url, &source_url, raw_type).unwrap() + Request::new(&url, &source_url, raw_type, "").unwrap() }) .collect::>() } diff --git a/benches/bench_url.rs b/benches/bench_url.rs index cee526fd..f45915cc 100644 --- a/benches/bench_url.rs +++ b/benches/bench_url.rs @@ -38,7 +38,7 @@ fn request_parsing_throughput(c: &mut Criterion) { b.iter(|| { let mut successful = 0; requests.iter().for_each(|r| { - let req: Result = Request::new(&r.url, &r.frameUrl, &r.cpt); + let req: Result = Request::new(&r.url, &r.frameUrl, &r.cpt, ""); if req.is_ok() { successful += 1; } @@ -88,7 +88,7 @@ fn request_new_throughput(c: &mut Criterion) { b.iter(|| { let mut successful = 0; requests.iter().for_each(|r| { - Request::new(&r.url, &r.frameUrl, &r.cpt).ok(); + Request::new(&r.url, &r.frameUrl, &r.cpt, "").ok(); successful += 1; }); }) diff --git a/examples/deserialization.rs b/examples/deserialization.rs index 3ef07a72..dd80e663 100644 --- a/examples/deserialization.rs +++ b/examples/deserialization.rs @@ -72,7 +72,7 @@ fn main() { if reqs_processed % 10000 == 0 { println!("{reqs_processed} requests processed"); } - let request = Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap(); + let request = Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(); let checked = engine.check_network_request(&request); if req.blocked == 1 && !checked.matched { mismatch_expected_match += 1; diff --git a/examples/example.rs b/examples/example.rs index 7d0e29bf..fe0bd81e 100644 --- a/examples/example.rs +++ b/examples/example.rs @@ -23,6 +23,7 @@ fn main() { "http://example.com/-advertisement-icon.", "http://example.com/helloworld", "image", + "", ) .unwrap(); let blocker_result = engine.check_network_request(&request); diff --git a/examples/generate-dat.rs b/examples/generate-dat.rs index 33c2d513..d3311cd0 100644 --- a/examples/generate-dat.rs +++ b/examples/generate-dat.rs @@ -19,6 +19,7 @@ fn main() { "https://platform.twitter.com/widgets.js", "https://fmarier.github.io/brave-testing/social-widgets.html", "script", + "", ) .unwrap(); assert!(engine.check_network_request(&request).exception.is_some()); diff --git a/examples/use-dat.rs b/examples/use-dat.rs index bc662d71..6708c3b7 100644 --- a/examples/use-dat.rs +++ b/examples/use-dat.rs @@ -20,6 +20,7 @@ fn main() { "https://platform.twitter.com/widgets.js", "https://fmarier.github.io/brave-testing/social-widgets.html", "script", + "", ) .unwrap(); let checked = engine.check_network_request(&request); diff --git a/fuzz/fuzz_targets/parse_request.rs b/fuzz/fuzz_targets/parse_request.rs index 62ca8944..a84a1ccf 100644 --- a/fuzz/fuzz_targets/parse_request.rs +++ b/fuzz/fuzz_targets/parse_request.rs @@ -6,10 +6,15 @@ use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { if let Ok(url) = std::str::from_utf8(data) { - Request::new(&format!("https://{}", url), "https://example.com", "other"); - Request::new(url, "https://example.com", "script"); - Request::new(url, "", ""); - Request::new(url, url, ""); - Request::new(url, url, url); + Request::new( + &format!("https://{}", url), + "https://example.com", + "other", + "", + ); + Request::new(url, "https://example.com", "script", ""); + Request::new(url, "", "", ""); + Request::new(url, url, "", ""); + Request::new(url, url, url, ""); } }); diff --git a/js/src/lib.rs b/js/src/lib.rs index 939c79d5..e09cf136 100644 --- a/js/src/lib.rs +++ b/js/src/lib.rs @@ -189,9 +189,9 @@ fn engine_check(mut cx: FunctionContext) -> JsResult { // Optional trailing args: // - legacy: `(debug: boolean)` - // - new: `(method: string | null | undefined, debug?: boolean)` + // - new: `(method: string, debug?: boolean)` — empty method string means unspecified let (method, debug) = match cx.argument_opt(4) { - None => (None, false), + None => (String::new(), false), Some(arg) => { if arg.is_a::(&mut cx) || arg.is_a::(&mut cx) { let debug = match cx.argument_opt(5) { @@ -201,9 +201,9 @@ fn engine_check(mut cx: FunctionContext) -> JsResult { .value(&mut cx), None => false, }; - (None, debug) + (String::new(), debug) } else if let Ok(debug) = arg.downcast::(&mut cx) { - (None, debug.value(&mut cx)) + (String::new(), debug.value(&mut cx)) } else if let Ok(method) = arg.downcast::(&mut cx) { let debug = match cx.argument_opt(5) { Some(debug_arg) => debug_arg @@ -212,7 +212,7 @@ fn engine_check(mut cx: FunctionContext) -> JsResult { .value(&mut cx), None => false, }; - (Some(method.value(&mut cx)), debug) + (method.value(&mut cx), debug) } else { cx.throw_error( "expected boolean (debug), string (method), null, or undefined as 4th argument", @@ -221,12 +221,7 @@ fn engine_check(mut cx: FunctionContext) -> JsResult { } }; - let request = match adblock::request::Request::new_with_method( - &url, - &source_url, - &request_type, - method.as_deref(), - ) { + let request = match adblock::request::Request::new(&url, &source_url, &request_type, &method) { Ok(r) => r, Err(e) => cx.throw_error(e.to_string())?, }; @@ -359,26 +354,19 @@ fn validate_request(mut cx: FunctionContext) -> JsResult { let source_url: String = cx.argument::(1)?.value(&mut cx); let request_type: String = cx.argument::(2)?.value(&mut cx); let method = match cx.argument_opt(3) { - None => None, + None => String::new(), Some(arg) => { if arg.is_a::(&mut cx) || arg.is_a::(&mut cx) { - None + String::new() } else { - Some( - arg.downcast::(&mut cx) - .or_throw(&mut cx)? - .value(&mut cx), - ) + arg.downcast::(&mut cx) + .or_throw(&mut cx)? + .value(&mut cx) } } }; - let request_ok = adblock::request::Request::new_with_method( - &url, - &source_url, - &request_type, - method.as_deref(), - ) - .is_ok(); + let request_ok = + adblock::request::Request::new(&url, &source_url, &request_type, &method).is_ok(); Ok(cx.boolean(request_ok)) } diff --git a/src/engine.rs b/src/engine.rs index 1e64a6e3..1305d50c 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -299,7 +299,7 @@ impl Engine { /// `hidden_class_id_selectors` to obtain any stylesheets consisting of generic rules (if the /// returned `generichide` value is false). pub fn url_cosmetic_resources(&self, url: &str) -> UrlSpecificResources { - let request = if let Ok(request) = Request::new(url, url, "document") { + let request = if let Ok(request) = Request::new(url, url, "document", "") { request } else { return UrlSpecificResources::empty(); diff --git a/src/request.rs b/src/request.rs index 39479e2a..7cc32165 100644 --- a/src/request.rs +++ b/src/request.rs @@ -125,6 +125,9 @@ impl Request { } fn parse_method(raw_method: &str) -> Option { + if raw_method.is_empty() { + return None; + } if raw_method.eq_ignore_ascii_case("get") { Some(RequestMethod::Get) } else if raw_method.eq_ignore_ascii_case("head") { @@ -203,18 +206,14 @@ impl Request { } /// Construct a new [`Request`]. - pub fn new(url: &str, source_url: &str, request_type: &str) -> Result { - Self::new_with_method(url, source_url, request_type, None) - } - - pub fn new_with_method( + pub fn new( url: &str, source_url: &str, request_type: &str, - method: Option<&str>, + method: &str, ) -> Result { if let Some(parsed_url) = url_parser::parse_url(url) { - let parsed_method = method.and_then(Self::parse_method); + let parsed_method = Self::parse_method(method); if let Some(parsed_source) = url_parser::parse_url(source_url) { let source_domain = parsed_source.domain(); @@ -256,7 +255,7 @@ impl Request { source_hostname: &str, request_type: &str, third_party: bool, - method: Option<&str>, + method: &str, ) -> Request { let splitter = memchr::memchr(b':', url.as_bytes()).unwrap_or(0); let schema: &str = &url[..splitter]; @@ -269,7 +268,7 @@ impl Request { source_hostname, third_party, url.to_string(), - method.and_then(Self::parse_method), + Self::parse_method(method), ) } } diff --git a/tests/legacy_harness.rs b/tests/legacy_harness.rs index 864fefe9..1939afec 100644 --- a/tests/legacy_harness.rs +++ b/tests/legacy_harness.rs @@ -40,7 +40,7 @@ mod legacy_test_filters { assert!( engine .check_network_request( - &Request::new(to_block, "https://example.com", "other").unwrap(), + &Request::new(to_block, "https://example.com", "other", "").unwrap(), ) .matched, "Expected filter {} to match {}", @@ -53,7 +53,7 @@ mod legacy_test_filters { assert!( !engine .check_network_request( - &Request::new(to_pass, "https://example.com", "other").unwrap(), + &Request::new(to_pass, "https://example.com", "other", "").unwrap(), ) .matched, "Expected filter {} to pass {}", @@ -305,7 +305,7 @@ mod legacy_test_filters { ); // explicit, separate testcase construction of the "script" option as it is not the deafult - let request = Request::new("http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html#xpc=sf-gdn-exp-2&p=http%3A//slashdot.org;", "https://this-is-always-third-party.com", "script").unwrap(); + let request = Request::new("http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html#xpc=sf-gdn-exp-2&p=http%3A//slashdot.org;", "https://this-is-always-third-party.com", "script", "").unwrap(); let engine = Engine::new_with_list_text( "||googlesyndication.com/safeframe/$third-party,script", Default::default(), @@ -335,7 +335,7 @@ mod legacy_check_match { } for to_block in blocked { - let request = Request::new(to_block, "alwaysthirdparty.com", "script").unwrap(); + let request = Request::new(to_block, "alwaysthirdparty.com", "script", "").unwrap(); assert!( engine.check_network_request(&request).matched, @@ -353,7 +353,7 @@ mod legacy_check_match { } for to_pass in not_blocked { - let request = Request::new(to_pass, "alwaysthirdparty.com", "script").unwrap(); + let request = Request::new(to_pass, "alwaysthirdparty.com", "script", "").unwrap(); assert!( !engine.check_network_request(&request).matched, @@ -413,6 +413,7 @@ mod legacy_check_match { "http://z.cdn.turner.com/xslo/cvp/ads/freewheel/js/0/AdManager.js", "http://cnn.com", "", + "", ) .unwrap(); assert!(!engine.check_network_request(&request).matched); @@ -509,7 +510,7 @@ mod legacy_check_options { let engine = Engine::new_with_list_text(rules.join("\n"), Default::default()); // first one with the provided rules for (url, source_url, request_type, expectation) in tests { - let request = Request::new(url, source_url, request_type).unwrap(); + let request = Request::new(url, source_url, request_type, "").unwrap(); assert!( engine.check_network_request(&request).matched == *expectation, "Expected match = {expectation} for {url} from {source_url} typed {request_type} against {rules:?}" @@ -861,6 +862,7 @@ mod legacy_misc_tests { "http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html", "http://slashdot.org", "script", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched) @@ -905,7 +907,8 @@ mod legacy_misc_tests { &Request::new( "https://googlesyndication.com/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -917,7 +920,8 @@ mod legacy_misc_tests { &Request::new( "https://googlesyndication.com/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -929,7 +933,8 @@ mod legacy_misc_tests { &Request::new( "https://googleayndication.com/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -941,7 +946,8 @@ mod legacy_misc_tests { &Request::new( "https://googleayndication.com/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -954,7 +960,8 @@ mod legacy_misc_tests { &Request::new( "https://googlesyndication.ca/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -966,7 +973,8 @@ mod legacy_misc_tests { &Request::new( "https://googlesyndication.ca/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -995,6 +1003,7 @@ mod legacy_misc_tests { "http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html", current_page_frame, request_type, + "", ) .unwrap(); let checked = engine.check_network_request(&request); @@ -1010,7 +1019,8 @@ mod legacy_misc_tests { ); // Test when no filter is found, returns None - let request = Request::new("http://ssafsdf.com", current_page_frame, request_type).unwrap(); + let request = + Request::new("http://ssafsdf.com", current_page_frame, request_type, "").unwrap(); let checked = engine.check_network_request(&request); assert!(!checked.matched, "Expected url to pass"); assert!(checked.filter.is_none(), "Expected no fitler to match"); @@ -1043,6 +1053,7 @@ mod legacy_misc_tests { "http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html", current_page_frame, request_type, + "", ) .unwrap(); let checked = engine.check_network_request(&request); @@ -1072,7 +1083,7 @@ mod legacy_misc_tests { ); let request = - Request::new("https://brianbondy.com/t", "https://test.com", "script").unwrap(); + Request::new("https://brianbondy.com/t", "https://test.com", "script", "").unwrap(); let checked = engine.check_network_request(&request); assert!(checked.matched); diff --git a/tests/live.rs b/tests/live.rs index 3f48acd5..6b24036e 100644 --- a/tests/live.rs +++ b/tests/live.rs @@ -166,6 +166,7 @@ fn check_live_specific_urls() { "https://static.scroll.com/js/scroll.js", "https://www.theverge.com/", "script", + "", ) .unwrap(), ); @@ -182,6 +183,7 @@ fn check_live_specific_urls() { "https://platform.twitter.com/widgets.js", "https://fmarier.github.io/brave-testing/social-widgets.html", "script", + "", ) .unwrap(), ); @@ -194,11 +196,7 @@ fn check_live_specific_urls() { } { engine.disable_tags(&["twitter-embeds"]); - let checked = engine.check_network_request(&Request::new( - "https://imagesrv.adition.com/banners/1337/files/00/0e/6f/09/000000945929.jpg?PQgSgs13hf1fw.jpg", - "https://spiegel.de", - "image", - ).unwrap()); + let checked = engine.check_network_request(&Request::new("https://imagesrv.adition.com/banners/1337/files/00/0e/6f/09/000000945929.jpg?PQgSgs13hf1fw.jpg", "https://spiegel.de", "image", "").unwrap()); assert!( checked.matched, "Expected match, got filter {:?}, exception {:?}", @@ -214,8 +212,9 @@ fn check_live_from_filterlists() { let requests = load_requests(); for req in requests { - let checked = engine - .check_network_request(&Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap()); + let checked = engine.check_network_request( + &Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(), + ); assert_eq!( checked.matched, req.blocked, "Expected match {} for {} at {}, got filter {:?}, exception {:?}", @@ -243,6 +242,7 @@ fn check_live_redirects() { "https://c.amazon-adsystem.com/aax2/amzn_ads.js", "https://aussieexotics.com/", "script", + "", ) .unwrap(), ); @@ -261,6 +261,7 @@ fn check_live_redirects() { "https://www.googletagservices.com/tag/js/gpt.js", "https://tvguide.com/", "script", + "", ) .unwrap(), ); diff --git a/tests/matching.rs b/tests/matching.rs index 11687269..f9b80453 100644 --- a/tests/matching.rs +++ b/tests/matching.rs @@ -79,7 +79,7 @@ fn check_filter_matching() { } let engine = adblock::Engine::new_with_parsed_rules(filters, vec![], true); - let request_res = Request::new(&req.url, &req.sourceUrl, &req.r#type); + let request_res = Request::new(&req.url, &req.sourceUrl, &req.r#type, ""); // The dataset has cases where URL is set to just "http://" or "https://", which we do not support if let Ok(request) = request_res { let result = engine.check_network_request(&request); @@ -131,7 +131,7 @@ fn check_engine_matching() { ); let network_filter = network_filter_res.unwrap(); - let request = Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap(); + let request = Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(); let result = engine.check_network_request(&request); if network_filter.is_exception() { @@ -185,7 +185,7 @@ fn check_rule_matching_browserlike() { impl From<&TestRequest> for Request { fn from(v: &TestRequest) -> Self { - Request::new(&v.url, &v.frameUrl, &v.cpt).unwrap() + Request::new(&v.url, &v.frameUrl, &v.cpt, "").unwrap() } } diff --git a/tests/simple_use.rs b/tests/simple_use.rs index 1b05780a..c07e6bb1 100644 --- a/tests/simple_use.rs +++ b/tests/simple_use.rs @@ -16,6 +16,7 @@ fn check_simple_use() { "http://example.com/-advertisement-icon.", "http://example.com/helloworld", "image", + "", ) .unwrap(); let blocker_result = engine.check_network_request(&request); diff --git a/tests/ublock-coverage.rs b/tests/ublock-coverage.rs index b521a930..ab775bd9 100644 --- a/tests/ublock-coverage.rs +++ b/tests/ublock-coverage.rs @@ -76,8 +76,13 @@ fn check_specific_rules() { // exceptions have not effect if important filter matches let engine = Engine::new_with_list_text("||www.facebook.com/*/plugin", Default::default()); - let request = - Request::new("https://www.facebook.com/v3.2/plugins/comments.ph", "", "").unwrap(); + let request = Request::new( + "https://www.facebook.com/v3.2/plugins/comments.ph", + "", + "", + "", + ) + .unwrap(); let checked = engine.check_network_request(&request); assert!(checked.matched); @@ -102,6 +107,7 @@ fn check_specific_rules() { "http://cdn.taboola.com/libtrc/test/loader.js", "http://cnet.com", "script", + "", ) .unwrap(); let checked = engine.check_network_request(&request); @@ -117,43 +123,27 @@ fn check_specifics_default() { "@@||www.googleadservices.*/aclk?$first-party", ]); { - let request = Request::new("https://www.youtube.com/youtubei/v1/log_event?alt=json&key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", "", "").unwrap(); + let request = Request::new("https://www.youtube.com/youtubei/v1/log_event?alt=json&key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", "", "", "").unwrap(); let checked = engine.check_network_request(&request); assert!(checked.matched); } { - let request = Request::new( - "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl=", - "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl=", - "main_frame", - ).unwrap(); + let request = Request::new("https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl=", "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl=", "main_frame", "").unwrap(); let checked = engine.check_network_request(&request); assert!(!checked.matched, "Matched on {:?}", checked.filter); } { - let request = Request::new( - "https://www.googleadservices.com/pagead/aclk?sa=L&ai=DChcSEwin96uLgYzjAhWH43cKHf0JA7YYABABGgJlZg&ohost=www.google.com&cid=CAASEuRoSkQKbbu2CAjK-zZJnF-wcw&sig=AOD64_1j63JqPtw22vaMasSE4aN1FRKtEw&ctype=5&q=&ved=0ahUKEwivnaWLgYzjAhUERxUIHWzYDTQQ9A4IzgI&adurl=", - "https://www.googleadservices.com/pagead/aclk?sa=L&ai=DChcSEwin96uLgYzjAhWH43cKHf0JA7YYABABGgJlZg&ohost=www.google.com&cid=CAASEuRoSkQKbbu2CAjK-zZJnF-wcw&sig=AOD64_1j63JqPtw22vaMasSE4aN1FRKtEw&ctype=5&q=&ved=0ahUKEwivnaWLgYzjAhUERxUIHWzYDTQQ9A4IzgI&adurl=", - "main_frame", - ).unwrap(); + let request = Request::new("https://www.googleadservices.com/pagead/aclk?sa=L&ai=DChcSEwin96uLgYzjAhWH43cKHf0JA7YYABABGgJlZg&ohost=www.google.com&cid=CAASEuRoSkQKbbu2CAjK-zZJnF-wcw&sig=AOD64_1j63JqPtw22vaMasSE4aN1FRKtEw&ctype=5&q=&ved=0ahUKEwivnaWLgYzjAhUERxUIHWzYDTQQ9A4IzgI&adurl=", "https://www.googleadservices.com/pagead/aclk?sa=L&ai=DChcSEwin96uLgYzjAhWH43cKHf0JA7YYABABGgJlZg&ohost=www.google.com&cid=CAASEuRoSkQKbbu2CAjK-zZJnF-wcw&sig=AOD64_1j63JqPtw22vaMasSE4aN1FRKtEw&ctype=5&q=&ved=0ahUKEwivnaWLgYzjAhUERxUIHWzYDTQQ9A4IzgI&adurl=", "main_frame", "").unwrap(); let checked = engine.check_network_request(&request); assert!(!checked.matched, "Matched on {:?}", checked.filter); } { - let request = Request::new( - "https://www.researchgate.net/profile/Ruofei_Zhang/publication/221653522_Bid_landscape_forecasting_in_online_Ad_exchange_marketplace/links/53f10c1f0cf2711e0c432641.pdf", - "https://www.researchgate.net/profile/Ruofei_Zhang/publication/221653522_Bid_landscape_forecasting_in_online_Ad_exchange_marketplace/links/53f10c1f0cf2711e0c432641.pdf", - "main_frame", - ).unwrap(); + let request = Request::new("https://www.researchgate.net/profile/Ruofei_Zhang/publication/221653522_Bid_landscape_forecasting_in_online_Ad_exchange_marketplace/links/53f10c1f0cf2711e0c432641.pdf", "https://www.researchgate.net/profile/Ruofei_Zhang/publication/221653522_Bid_landscape_forecasting_in_online_Ad_exchange_marketplace/links/53f10c1f0cf2711e0c432641.pdf", "main_frame", "").unwrap(); let checked = engine.check_network_request(&request); assert!(!checked.matched, "Matched on {:?}", checked.filter); } { - let request = Request::new( - "https://www.google.com/search?q=Bid+Landscape+Forecasting+in+Online+Exchange+Marketplace&oq=Landscape+Forecasting+in+Online+Ad+Exchange+Marketplace", - "https://www.google.com/search?q=Bid+Landscape+Forecasting+in+Online+Exchange+Marketplace&oq=Landscape+Forecasting+in+Online+Ad+Exchange+Marketplace", - "main_frame", - ).unwrap(); + let request = Request::new("https://www.google.com/search?q=Bid+Landscape+Forecasting+in+Online+Exchange+Marketplace&oq=Landscape+Forecasting+in+Online+Ad+Exchange+Marketplace", "https://www.google.com/search?q=Bid+Landscape+Forecasting+in+Online+Exchange+Marketplace&oq=Landscape+Forecasting+in+Online+Ad+Exchange+Marketplace", "main_frame", "").unwrap(); let checked = engine.check_network_request(&request); assert!(!checked.matched, "Matched on {:?}", checked.filter); } @@ -163,6 +153,7 @@ fn check_specifics_default() { "https://platform.twitter.com/widgets.js", "https://fmarier.github.io/brave-testing/social-widgets.html", "script", + "", ) .unwrap(); let checked = engine.check_network_request(&request); @@ -180,7 +171,7 @@ fn check_basic_works_after_deserialization() { deserialized_engine.deserialize(&serialized).unwrap(); { - let request = Request::new("https://www.youtube.com/youtubei/v1/log_event?alt=json&key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", "", "").unwrap(); + let request = Request::new("https://www.youtube.com/youtubei/v1/log_event?alt=json&key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", "", "", "").unwrap(); let checked = engine.check_network_request(&request); assert!(checked.matched); } @@ -204,7 +195,7 @@ fn check_matching_equivalent() { let mut false_positive_rules: HashMap = HashMap::new(); let mut false_negative_exceptions: HashMap = HashMap::new(); for req in requests { - let request = Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap(); + let request = Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(); let checked = engine.check_network_request(&request); if req.blocked == 1 && !checked.matched { mismatch_expected_match += 1; @@ -282,7 +273,7 @@ fn check_matching_hostnames() { let source_domain = source_host.domain(); let third_party = source_domain != domain; - let request = Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap(); + let request = Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(); let preparsed_request = Request::preparsed( &req.url, url_host.hostname(), diff --git a/tests/unit/blocker.rs b/tests/unit/blocker.rs index 4b655df9..1f1d4738 100644 --- a/tests/unit/blocker.rs +++ b/tests/unit/blocker.rs @@ -22,6 +22,7 @@ mod blocker_tests { "https://example.com/test/", "https://example.com", "xmlhttprequest", + "", ) .unwrap(); assert!(blocker.check(&request, &Default::default()).matched); @@ -30,6 +31,7 @@ mod blocker_tests { "https://example.com/test", "https://example.com", "xmlhttprequest", + "", ) .unwrap(); assert!(!blocker.check(&request, &Default::default()).matched); @@ -70,6 +72,7 @@ mod blocker_tests { "https://imdb-video.media-imdb.com/kBOeI88k1o23eNAi", "https://www.imdb.com/video/13", "media", + "", ) .unwrap(); @@ -108,6 +111,7 @@ mod blocker_tests { "https://imdb-video.media-imdb.com/kBOeI88k1o23eNAi", "https://www.imdb.com/video/13", "media", + "", ) .unwrap(); @@ -141,8 +145,13 @@ mod blocker_tests { "||www3.doubleclick.net^$xmlhttprequest,redirect-rule=noop.txt,domain=lineups.fun", ]; - let request = - Request::new("https://www3.doubleclick.net", "https://lineups.fun", "xhr").unwrap(); + let request = Request::new( + "https://www3.doubleclick.net", + "https://lineups.fun", + "xhr", + "", + ) + .unwrap(); let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, @@ -169,7 +178,7 @@ mod blocker_tests { fn badfilter_does_not_match() { let filters = ["||foo.com$badfilter"]; let url_results = [( - Request::new("https://foo.com", "https://bar.com", "image").unwrap(), + Request::new("https://foo.com", "https://bar.com", "image", "").unwrap(), false, )]; @@ -185,7 +194,7 @@ mod blocker_tests { "||foo.com$domain=foo.com|bar.com", ]; let url_results = [( - Request::new("https://foo.com", "https://bar.com", "image").unwrap(), + Request::new("https://foo.com", "https://bar.com", "image", "").unwrap(), false, )]; @@ -201,7 +210,7 @@ mod blocker_tests { "||foo.com$domain=foo.com|bar.com,image", ]; let url_results = [( - Request::new("https://foo.com", "https://bar.com", "image").unwrap(), + Request::new("https://foo.com", "https://bar.com", "image", "").unwrap(), true, )]; @@ -224,6 +233,7 @@ mod blocker_tests { "https://dot.example.com", "https://dot.example.com", "document", + "", ) .unwrap(); assert!(!blocker.check(&request, &Default::default()).matched); @@ -232,6 +242,7 @@ mod blocker_tests { "https://dot.example.com.", "https://dot.example.com.", "document", + "", ) .unwrap(); assert!(blocker.check(&request, &Default::default()).matched); @@ -240,6 +251,7 @@ mod blocker_tests { "https://test.example.com", "https://test.example.com", "document", + "", ) .unwrap(); assert!(blocker.check(&request, &Default::default()).matched); @@ -248,6 +260,7 @@ mod blocker_tests { "https://test.example.com.", "https://test.example.com.", "document", + "", ) .unwrap(); assert!(blocker.check(&request, &Default::default()).matched); @@ -265,6 +278,7 @@ mod blocker_tests { "https://r.alimc1.top/test.js", "https://minisite.letv.com/", "script", + "", ) .unwrap(), true, @@ -274,6 +288,7 @@ mod blocker_tests { "https://www.baidu.com/test.js", "https://minisite.letv.com/", "script", + "", ) .unwrap(), false, @@ -283,6 +298,7 @@ mod blocker_tests { "https://r.aabb.top/test.js", "https://example.com/", "script", + "", ) .unwrap(), false, @@ -292,6 +308,7 @@ mod blocker_tests { "https://r.aabb.top/test.js", "https://minisite.letv.com/", "script", + "", ) .unwrap(), true, @@ -344,7 +361,8 @@ mod blocker_tests { &Request::new( "https://pirateproxy.live/static/custom_ads.js", "https://pirateproxy.live", - "script" + "script", + "" ) .unwrap() ), @@ -355,7 +373,8 @@ mod blocker_tests { &Request::new( "https://pirateproxy.live/static/custom_ads.js", "https://pirateproxy.live", - "image" + "image", + "" ) .unwrap() ), @@ -366,7 +385,8 @@ mod blocker_tests { &Request::new( "https://pirateproxy.live/static/custom_ads.js", "https://pirateproxy.live", - "object" + "object", + "" ) .unwrap() ), @@ -377,14 +397,20 @@ mod blocker_tests { // A single directive should be returned if only one match is present in the engine, for both document and subdocument types assert_eq!( blocker.get_csp_directives( - &Request::new("https://example.com", "https://vidoza.co", "document").unwrap() + &Request::new("https://example.com", "https://vidoza.co", "document", "") + .unwrap() ), Some(String::from("worker-src 'none'")) ); assert_eq!( blocker.get_csp_directives( - &Request::new("https://example.com", "https://vidoza.net", "subdocument") - .unwrap() + &Request::new( + "https://example.com", + "https://vidoza.net", + "subdocument", + "" + ) + .unwrap() ), Some(String::from("worker-src 'none'")) ); @@ -404,7 +430,8 @@ mod blocker_tests { &Request::new( "https://example.com", "https://pirateproxy.live", - "document" + "document", + "" ) .unwrap() ) @@ -414,7 +441,8 @@ mod blocker_tests { &Request::new( "https://example.com", "https://pirateproxy.live", - "subdocument" + "subdocument", + "" ) .unwrap() ) @@ -424,14 +452,19 @@ mod blocker_tests { // A directive with an exception should not be returned assert_eq!( blocker.get_csp_directives( - &Request::new("https://1337x.to", "https://1337x.to", "document").unwrap() + &Request::new("https://1337x.to", "https://1337x.to", "document", "").unwrap() ), Some(String::from("script-src 'self' 'unsafe-inline'")) ); assert_eq!( blocker.get_csp_directives( - &Request::new("https://1337x.to/no-csp", "https://1337x.to", "subdocument") - .unwrap() + &Request::new( + "https://1337x.to/no-csp", + "https://1337x.to", + "subdocument", + "" + ) + .unwrap() ), None ); @@ -443,7 +476,8 @@ mod blocker_tests { &Request::new( "https://example.com/duplicated-directive", "https://flashx.cc", - "document" + "document", + "" ) .unwrap() ), @@ -454,7 +488,8 @@ mod blocker_tests { &Request::new( "https://example.com/duplicated-directive", "https://flashx.cc", - "subdocument" + "subdocument", + "" ) .unwrap() ), @@ -468,7 +503,8 @@ mod blocker_tests { &Request::new( "https://1337x.to/duplicated-directive/disable-all", "https://thepiratebay10.org", - "document" + "document", + "" ) .unwrap() ), @@ -479,7 +515,8 @@ mod blocker_tests { &Request::new( "https://1337x.to/duplicated-directive/disable-all", "https://thepiratebay10.org", - "document" + "document", + "" ) .unwrap() ), @@ -493,7 +530,8 @@ mod blocker_tests { &Request::new( "htps://github.com/first-party-only", "https://example.com", - "subdocument" + "subdocument", + "" ) .unwrap() ), @@ -504,7 +542,8 @@ mod blocker_tests { &Request::new( "https://example.com/first-party-only", "https://example.com", - "document" + "document", + "" ) .unwrap() ), @@ -539,6 +578,7 @@ mod blocker_tests { "https://example.com?q=1&test=2#blue", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -554,6 +594,7 @@ mod blocker_tests { "https://example.com?test=2&q=1#blue", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -569,6 +610,7 @@ mod blocker_tests { "https://example.com?test=2#blue", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -580,7 +622,13 @@ mod blocker_tests { assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?q=1#blue", "https://antonok.com", "xhr").unwrap(), + &Request::new( + "https://example.com?q=1#blue", + "https://antonok.com", + "xhr", + "", + ) + .unwrap(), &resources, ); assert_eq!(result.rewritten_url, None); @@ -591,6 +639,7 @@ mod blocker_tests { "https://example.com?q=1&test=2", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -603,6 +652,7 @@ mod blocker_tests { "https://example.com?test=2&q=1", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -611,28 +661,46 @@ mod blocker_tests { assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?test=2", "https://antonok.com", "xhr").unwrap(), + &Request::new( + "https://example.com?test=2", + "https://antonok.com", + "xhr", + "", + ) + .unwrap(), &resources, ); assert_eq!(result.rewritten_url, Some("https://example.com".into())); assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?test=2", "https://antonok.com", "image").unwrap(), + &Request::new( + "https://example.com?test=2", + "https://antonok.com", + "image", + "", + ) + .unwrap(), &resources, ); assert_eq!(result.rewritten_url, None); assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?q=1", "https://antonok.com", "xhr").unwrap(), + &Request::new("https://example.com?q=1", "https://antonok.com", "xhr", "").unwrap(), &resources, ); assert_eq!(result.rewritten_url, None); assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?q=fbclid", "https://antonok.com", "xhr").unwrap(), + &Request::new( + "https://example.com?q=fbclid", + "https://antonok.com", + "xhr", + "", + ) + .unwrap(), &resources, ); assert_eq!(result.rewritten_url, None); @@ -643,6 +711,7 @@ mod blocker_tests { "https://example.com?fbclid=10938&q=1&test=2", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -655,6 +724,7 @@ mod blocker_tests { "https://test.com?fbclid=10938&q=1&test=2", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -670,6 +740,7 @@ mod blocker_tests { "https://example.com?q1=1&q2=2&q3=3&test=2&q4=4&q5=5&fbclid=39", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -685,6 +756,7 @@ mod blocker_tests { "https://example.com?q1=1&q1=2&test=2&test=3", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -700,6 +772,7 @@ mod blocker_tests { "https://example.com/script.js?test=2#blue", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -719,6 +792,7 @@ mod blocker_tests { "https://example.com/block/script.js?test=2", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -735,6 +809,7 @@ mod blocker_tests { "https://example.com/Path/?Test=ABC&testcase=AbC&testCase=aBc", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -747,6 +822,7 @@ mod blocker_tests { "https://example.com/Path/?Test=ABC&testcase=AbC&testCase=aBc", "https://antonok.com", "image", + "", ) .unwrap(), &resources, @@ -759,6 +835,7 @@ mod blocker_tests { "https://example.com/Path/?Test=ABC&testcase=AbC&testCase=aBc", "https://antonok.com", "subdocument", + "", ) .unwrap(), &resources, @@ -774,6 +851,7 @@ mod blocker_tests { "https://example.com/Path/?Test=ABC&testcase=AbC&testCase=aBc", "https://antonok.com", "document", + "", ) .unwrap(), &resources, @@ -789,6 +867,7 @@ mod blocker_tests { "https://example.com?Test=ABC?123&test=3#&test=4#b", "https://antonok.com", "document", + "", ) .unwrap(), &resources, @@ -804,6 +883,7 @@ mod blocker_tests { "https://example.com?Test=ABC&testCase=5", "https://antonok.com", "document", + "", ) .unwrap(), &resources, @@ -819,6 +899,7 @@ mod blocker_tests { "https://example.com?Test=ABC&testCase=5", "https://antonok.com", "image", + "", ) .unwrap(), &resources, @@ -928,7 +1009,7 @@ mod blocker_tests { for (original, expected) in testcases.into_iter() { let result = blocker.check( - &Request::new(original, "https://example.net", "xhr").unwrap(), + &Request::new(original, "https://example.net", "xhr", "").unwrap(), &resources, ); let expected = if original == expected { @@ -958,6 +1039,7 @@ mod blocker_tests { "https://example.com?example1_=1&example1-=2", "https://example.com", "xhr", + "", ) .unwrap(), &Default::default(), @@ -1010,6 +1092,7 @@ mod blocker_tests { "https://example.net/test", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1022,6 +1105,7 @@ mod blocker_tests { "https://example.net/test.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1034,6 +1118,7 @@ mod blocker_tests { "https://example.com/test.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1046,6 +1131,7 @@ mod blocker_tests { "https://example.com/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1058,6 +1144,7 @@ mod blocker_tests { "https://example.com/exceptc20/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1070,6 +1157,7 @@ mod blocker_tests { "https://example.com/exceptb10/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1082,6 +1170,7 @@ mod blocker_tests { "https://example.com/exceptc20/exceptb10/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1094,6 +1183,7 @@ mod blocker_tests { "https://example.com/exceptc20/exceptb10/excepta/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1106,6 +1196,7 @@ mod blocker_tests { "https://example.com/exceptc20/exceptb10/text", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1132,7 +1223,7 @@ mod blocker_tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); @@ -1183,7 +1274,7 @@ mod blocker_tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); @@ -1235,7 +1326,7 @@ mod blocker_tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); @@ -1287,6 +1378,7 @@ mod blocker_tests { "http://example.com/ad_banner.png", "https://example.com", "other", + "", ) .unwrap(); @@ -1304,7 +1396,7 @@ mod blocker_tests { let blocker = Blocker::new(["@@||example.com$generichide\n"], &blocker_options); assert!(blocker.check_generic_hide( - &Request::new("https://example.com", "https://example.com", "other").unwrap() + &Request::new("https://example.com", "https://example.com", "other", "").unwrap() )); } } @@ -1318,8 +1410,13 @@ mod placeholder_string_tests { filter_set.add_filter_list("||example.com^\n".to_string(), Default::default()); let engine = crate::Engine::new_with_filter_set(filter_set, true); let block = engine.check_network_request( - &crate::request::Request::new("https://example.com", "https://example.com", "document") - .unwrap(), + &crate::request::Request::new( + "https://example.com", + "https://example.com", + "document", + "", + ) + .unwrap(), ); assert_eq!( block.filter, diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index 9f13a743..2af05d9e 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -26,7 +26,7 @@ mod tests { engine.enable_tags(&["brian"]); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -60,7 +60,7 @@ mod tests { engine.disable_tags(&["stuff"]); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -90,7 +90,7 @@ mod tests { let engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -121,7 +121,7 @@ mod tests { engine.enable_tags(&["brian", "stuff"]); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -159,7 +159,7 @@ mod tests { deserialized_engine.deserialize(&serialized).unwrap(); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = deserialized_engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -270,7 +270,7 @@ mod tests { engine.use_resources(resources); let url = "http://example.com/ad-banner.gif"; - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); assert!(matched_rule.matched, "Expected match for {url}"); assert_eq!( @@ -280,7 +280,7 @@ mod tests { ); let url = "http://example.com/script.js"; - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); assert!(matched_rule.matched, "Expected match for {url}"); assert_eq!( @@ -302,7 +302,7 @@ mod tests { assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched @@ -310,7 +310,8 @@ mod tests { assert!( !engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "script").unwrap() + &Request::new("https://example.com", "https://example.com", "script", "") + .unwrap() ) .matched ); @@ -319,7 +320,8 @@ mod tests { &Request::new( "https://sub.example.com", "https://sub.example.com", - "document" + "document", + "" ) .unwrap() ) @@ -334,7 +336,7 @@ mod tests { assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched @@ -346,7 +348,7 @@ mod tests { assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched @@ -357,7 +359,7 @@ mod tests { assert!( !engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched @@ -368,7 +370,7 @@ mod tests { assert!( !engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched @@ -382,7 +384,7 @@ mod tests { assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched @@ -399,7 +401,7 @@ mod tests { assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched @@ -413,7 +415,8 @@ mod tests { &Request::new( "https://example.com/path", "https://example.com/path", - "document" + "document", + "" ) .unwrap() ) @@ -428,7 +431,8 @@ mod tests { &Request::new( "https://example.com/path", "https://example.com/path", - "document" + "document", + "" ) .unwrap() ) @@ -456,7 +460,8 @@ mod tests { &Request::new( "https://example.com", "https://rarvinzp.click", - content_type + content_type, + "" ) .unwrap() ) @@ -519,7 +524,7 @@ mod tests { "window.addthis = undefined", )]); - let request = Request::new("https://s7.addthis.com/js/250/addthis_widget.js?pub=resto", "https://www.rhmodern.com/catalog/product/product.jsp?productId=prod14970086&categoryId=cat7150028", "script").unwrap(); + let request = Request::new("https://s7.addthis.com/js/250/addthis_widget.js?pub=resto", "https://www.rhmodern.com/catalog/product/product.jsp?productId=prod14970086&categoryId=cat7150028", "script", "").unwrap(); let result = engine.check_network_request(&request); assert!(result.redirect.is_some()); @@ -530,8 +535,13 @@ mod tests { { // match case without regex is discarded let engine = Engine::new_with_list_text("ad.png$match-case", Default::default()); - let request = - Request::new("https://example.com/ad.png", "https://example.com", "image").unwrap(); + let request = Request::new( + "https://example.com/ad.png", + "https://example.com", + "image", + "", + ) + .unwrap(); assert!(!engine.check_network_request(&request).matched); } { @@ -540,34 +550,28 @@ mod tests { r#"/^https:\/\/[0-9a-z]{3,}\.[-a-z]{10,}\.(?:li[fv]e|top|xyz)\/[a-z]{8}\/\?utm_campaign=\w{40,}/$doc,match-case,domain=life|live|top|xyz"#, Default::default(), ); - let request = Request::new("https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "document").unwrap(); + let request = Request::new("https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "document", "").unwrap(); assert!(engine.check_network_request(&request).matched); } // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/((?!www)[a-z]{3,}|\d{2})?\.?[-0-9a-z]{6,}\.[a-z]{2,6}\/(?:[a-z]{6,8}\/)?\/?\?u=[0-9a-z]{7}&o=[0-9a-z]{7}/$doc,frame,match-case,domain=buzz|com|de|fun|guru|info|life|live|mobi|online|pw|site|space|top|us|xyz let engine = Engine::from_rules_debug([r#"/^https?:\/\/((?!www)[a-z]{3,}|\d{2})?\.?[-0-9a-z]{6,}\.[a-z]{2,6}\/(?:[a-z]{6,8}\/)?\/?\?u=[0-9a-z]{7}&o=[0-9a-z]{7}/$doc,frame,match-case,domain=buzz|com|de|fun|guru|info|life|live|mobi|online|pw|site|space|top|us|xyz"#], Default::default()); - let request = Request::new("https://example.com/aaaaaa/?u=aaaaaaa&o=bbbbbbb", - "https://example.com/aaaaaa/?u=aaaaaaa&o=bbbbbbb", - "document").unwrap(); + let request = Request::new("https://example.com/aaaaaa/?u=aaaaaaa&o=bbbbbbb", "https://example.com/aaaaaa/?u=aaaaaaa&o=bbbbbbb", "document", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https:\/\/(?:www\d\.)?[-a-z]{6,}\.(?:com|info|net|org)\/(?=[-_a-zA-Z]{0,42}\d)(?=[-_0-9a-z]{0,42}[A-Z])[-_0-9a-zA-Z]{43}\/\?cid=[-_0-9a-zA-Z]{16,36}(?:&qs\d=\S+)?&sid=[_0-9a-f]{1,32}$/$doc,match-case,domain=com|info|net|org let engine = Engine::from_rules_debug([r#"/^https:\/\/(?:www\d\.)?[-a-z]{6,}\.(?:com|info|net|org)\/(?=[-_a-zA-Z]{0,42}\d)(?=[-_0-9a-z]{0,42}[A-Z])[-_0-9a-zA-Z]{43}\/\?cid=[-_0-9a-zA-Z]{16,36}(?:&qs\d=\S+)?&sid=[_0-9a-f]{1,32}$/$doc,match-case,domain=com|info|net|org"#], Default::default()); - let request = Request::new("https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?cid=aaaaaaaaaabbbbbb&qs5=\n&sid=a", - "https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?cid=aaaaaaaaaabbbbbb&qs5=\n&sid=a", - "document").unwrap(); + let request = Request::new("https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?cid=aaaaaaaaaabbbbbb&qs5=\n&sid=a", "https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?cid=aaaaaaaaaabbbbbb&qs5=\n&sid=a", "document", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https:\/\/(?:www\d\.)?[-a-z]{6,}\.(?:com|info|net|org)\/(?=[-_a-zA-Z]{0,42}\d)(?=[-_0-9a-z]{0,42}[A-Z])[-_0-9a-zA-Z]{43}\/\?sid=[_0-9a-f]{1,32}(?:&qs\d=\S+)?&cid=[-_0-9a-zA-Z]{16,36}$/$doc,match-case,domain=com|info|net|org let engine = Engine::from_rules_debug([r#"/^https:\/\/(?:www\d\.)?[-a-z]{6,}\.(?:com|info|net|org)\/(?=[-_a-zA-Z]{0,42}\d)(?=[-_0-9a-z]{0,42}[A-Z])[-_0-9a-zA-Z]{43}\/\?cid=[-_0-9a-zA-Z]{16,36}(?:&qs\d=\S+)?&sid=[_0-9a-f]{1,32}$/$doc,match-case,domain=com|info|net|org"#], Default::default()); - let request = Request::new("https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?sid=1&qs1=\n&cid=aaaaaaaaaabbbbbb", - "https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?sid=1&qs1=\n&cid=aaaaaaaaaabbbbbb", - "document").unwrap(); + let request = Request::new("https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?sid=1&qs1=\n&cid=aaaaaaaaaabbbbbb", "https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?sid=1&qs1=\n&cid=aaaaaaaaaabbbbbb", "document", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ { @@ -580,6 +584,7 @@ mod tests { "http://abcde.abcde.com/aaaaabbbbb.apk", "http://abcde.abcde.com/aaaaabbbbb.apk", "document", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched); @@ -588,36 +593,28 @@ mod tests { /*{ // /\/[A-Z]\/[-0-9a-z]{5,}\.com\/(?:[0-9a-f]{2}\/){3}[0-9a-f]{32}\.js$/$script,1p,match-case let engine = Engine::from_rules_debug([r#"/\/[A-Z]\/[-0-9a-z]{5,}\.com\/(?:[0-9a-f]{2}\/){3}[0-9a-f]{32}\.js$/$script,1p,match-case"#], Default::default()); - let request = Request::new("/A/aaaaa.com/aa/bb/cc/aaaaaaaabbbbbbbbccccccccdddddddd.js", - "/A/aaaaa.com/aa/bb/cc/aaaaaaaabbbbbbbbccccccccdddddddd.js", - "script").unwrap(); + let request = Request::new("/A/aaaaa.com/aa/bb/cc/aaaaaaaabbbbbbbbccccccccdddddddd.js", "/A/aaaaa.com/aa/bb/cc/aaaaaaaabbbbbbbbccccccccdddddddd.js", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/(?:[a-z]{2}\.)?[0-9a-z]{7,16}\.com\/[a-z](?=[a-z]{0,25}[0-9A-Z])[0-9a-zA-Z]{3,26}\/(?:[1-5]\d{4}|[3-9]\d{3})\??(?:_=\d+|v=\d)?$/$frame,script,xhr,popup,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/(?:[a-z]{2}\.)?[0-9a-z]{7,16}\.com\/[a-z](?=[a-z]{0,25}[0-9A-Z])[0-9a-zA-Z]{3,26}\/(?:[1-5]\d{4}|[3-9]\d{3})\??(?:_=\d+|v=\d)?$/$frame,script,xhr,popup,3p,match-case"#], Default::default()); - let request = Request::new("https://aa.example.com/aAaaa/12222", - "https://aa.example.net/aAaaa/12222", - "frame").unwrap(); + let request = Request::new("https://aa.example.com/aAaaa/12222", "https://aa.example.net/aAaaa/12222", "frame", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/(?:[a-z]{2}\.)?[0-9a-z]{7,16}\.website\/[a-z](?=[a-z]{0,25}[0-9A-Z])[0-9a-zA-Z]{3,26}\/(?:[1-5]\d{4}|[3-9]\d{3})\??(?:_=\d+|v=\d)?$/$frame,script,xhr,popup,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/(?:[a-z]{2}\.)?[0-9a-z]{7,16}\.website\/[a-z](?=[a-z]{0,25}[0-9A-Z])[0-9a-zA-Z]{3,26}\/(?:[1-5]\d{4}|[3-9]\d{3})\??(?:_=\d+|v=\d)?$/$frame,script,xhr,popup,3p,match-case"#], Default::default()); - let request = Request::new("https://aa.example.website/aAaaa/12222", - "https://aa.example.website/aAaaa/12222", - "frame").unwrap(); + let request = Request::new("https://aa.example.website/aAaaa/12222", "https://aa.example.website/aAaaa/12222", "frame", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/[a-z]{8,15}\.top(\/(?:\d{1,5}|0NaN|articles?|browse|index|movie|news|pages?|static|view|web|wiki)){1,4}(?:\.html|\/)$/$frame,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[a-z]{8,15}\.top(\/(?:\d{1,5}|0NaN|articles?|browse|index|movie|news|pages?|static|view|web|wiki)){1,4}(?:\.html|\/)$/$frame,3p,match-case"#], Default::default()); - let request = Request::new("https://examples.top/articles.html", - "https://examples.top/articles.html", - "frame").unwrap(); + let request = Request::new("https://examples.top/articles.html", "https://examples.top/articles.html", "frame", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ { @@ -630,6 +627,7 @@ mod tests { "https://examples.top/abcd.json", "https://examples.com/abcd.json", "xhr", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched); @@ -638,27 +636,21 @@ mod tests { /*{ // /^https?:\/\/[a-z]{8,15}\.top\/[-a-z]{4,}\.css\?aHR0c[\/0-9a-zA-Z]{33,}=?=?$/$css,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[a-z]{8,15}\.top\/[-a-z]{4,}\.css\?aHR0c[\/0-9a-zA-Z]{33,}=?=?$/$css,3p,match-case"#], Default::default()); - let request = Request::new("https://examples.top/abcd.css?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", - "https://examples.com/abcd.css?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", - "stylesheet").unwrap(); + let request = Request::new("https://examples.top/abcd.css?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", "https://examples.com/abcd.css?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", "stylesheet", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - inferring unescaped `$` inside regex pattern /*{ // /^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.png\?aHR0c[\/0-9a-zA-Z]{33,}=?=?$/$image,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.png\?aHR0c[\/0-9a-zA-Z]{33,}=?=?$/$image,3p,match-case"#], Default::default()); - let request = Request::new("https://examples.top/abcd.png?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", - "https://examples.com/abcd.png?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", - "image").unwrap(); + let request = Request::new("https://examples.top/abcd.png?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", "https://examples.com/abcd.png?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", "image", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/[a-z]{8,15}\.xyz(\/(?:\d{1,5}|0NaN|articles?|browse|index|movie|news|pages?|static|view|web|wiki)){1,4}(?:\.html|\/)$/$frame,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[a-z]{8,15}\.xyz(\/(?:\d{1,5}|0NaN|articles?|browse|index|movie|news|pages?|static|view|web|wiki)){1,4}(?:\.html|\/)$/$frame,3p,match-case"#], Default::default()); - let request = Request::new("https://examples.xyz/articles.html", - "https://examples.xyz/articles.html", - "frame").unwrap(); + let request = Request::new("https://examples.xyz/articles.html", "https://examples.xyz/articles.html", "frame", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ { @@ -671,6 +663,7 @@ mod tests { "https://cdn.abcde.xyz/app.js", "https://cdn.abcde.com/app.js", "script", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched); @@ -679,9 +672,7 @@ mod tests { /*{ // /^https:\/\/a\.[-0-9a-z]{4,16}\.(?:club|com?|cyou|info|net|ru|site|top?|xxx|xyz)\/(?=[a-z]{0,6}[0-9A-Z])[0-9a-zA-Z]{7}\.js$/$script,match-case let engine = Engine::from_rules_debug([r#"/^https:\/\/a\.[-0-9a-z]{4,16}\.(?:club|com?|cyou|info|net|ru|site|top?|xxx|xyz)\/(?=[a-z]{0,6}[0-9A-Z])[0-9a-zA-Z]{7}\.js$/$script,match-case"#], Default::default()); - let request = Request::new("https://a.abcd.club/aaaaaaA.js", - "https://a.abcd.club/aaaaaaA.js", - "script").unwrap(); + let request = Request::new("https://a.abcd.club/aaaaaaA.js", "https://a.abcd.club/aaaaaaA.js", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ { @@ -694,6 +685,7 @@ mod tests { "https://cdn.jsdelivr.net/npm/abcd@latest/dist/script.min.js", "https://cdn.jsdelivr.com/npm/abcd@latest/dist/script.min.js", "script", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched); @@ -702,25 +694,19 @@ mod tests { /*{ // /^https?:\/\/[-.0-9a-z]+\/script\.js$/$script,1p,strict3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[-.0-9a-z]+\/script\.js$/$script,1p,strict3p,match-case"#], Default::default()); - let request = Request::new("https://www.example.com/script.js", - "https://www.abc.com/script.js", - "script").unwrap(); + let request = Request::new("https://www.example.com/script.js", "https://www.abc.com/script.js", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - unicode not supported in network filter /*{ let engine = Engine::from_rules_debug([r#"/tesT߶/$domain=example.com"#], Default::default()); - let request = Request::new("https://example.com/tesT߶", - "https://example.com", - "script").unwrap(); + let request = Request::new("https://example.com/tesT߶", "https://example.com", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - unicode not supported in network filter /*{ let engine = Engine::from_rules_debug([r#"/tesT߶/$domain=example.com"#], Default::default()); - let request = Request::new("https://example-tesT߶.com/tesT", - "https://example.com", - "script").unwrap(); + let request = Request::new("https://example-tesT߶.com/tesT", "https://example.com", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ } @@ -902,26 +888,25 @@ trustedSetLocalStorageItem("mol.ads.cmp.tcf.cache", "{\"getTCData\":{\"cmpId\":2 let url = "https://perplexity.ai/rest/metrics/collect?foo=bar"; let source = "https://perplexity.ai/page"; - let post = Request::new_with_method(url, source, "xhr", Some("post")).unwrap(); + let post = Request::new(url, source, "xhr", "post").unwrap(); assert!( engine.check_network_request(&post).matched, "POST xhr 1p should match" ); - let get = Request::new_with_method(url, source, "xhr", Some("get")).unwrap(); + let get = Request::new(url, source, "xhr", "get").unwrap(); assert!( !engine.check_network_request(&get).matched, "GET xhr should not match method=post rule" ); - let post_3p = - Request::new_with_method(url, "https://other.com/page", "xhr", Some("post")).unwrap(); + let post_3p = Request::new(url, "https://other.com/page", "xhr", "post").unwrap(); assert!( !engine.check_network_request(&post_3p).matched, "POST xhr 3p should not match 1p rule" ); - let post_no_method = Request::new(url, source, "xhr").unwrap(); + let post_no_method = Request::new(url, source, "xhr", "").unwrap(); assert!( !engine.check_network_request(&post_no_method).matched, "missing method should not match method=post rule" @@ -938,15 +923,14 @@ trustedSetLocalStorageItem("mol.ads.cmp.tcf.cache", "{\"getTCData\":{\"cmpId\":2 let source = "https://app.axenthost.com/page"; for method in ["get", "head"] { - let request = - Request::new_with_method(url, source, "xhr", Some(method)).unwrap(); + let request = Request::new(url, source, "xhr", method).unwrap(); assert!( !engine.check_network_request(&request).matched, "{method} xhr should be excepted" ); } - let post = Request::new_with_method(url, source, "xhr", Some("post")).unwrap(); + let post = Request::new(url, source, "xhr", "post").unwrap(); assert!( engine.check_network_request(&post).matched, "POST xhr should still be blocked" diff --git a/tests/unit/filters/network_matchers.rs b/tests/unit/filters/network_matchers.rs index 9f898bb2..7e9612e9 100644 --- a/tests/unit/filters/network_matchers.rs +++ b/tests/unit/filters/network_matchers.rs @@ -96,7 +96,7 @@ mod match_tests { fn filter_match_url(filter: &str, url: &str, matching: bool) { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); - let request = request::Request::new(url, "https://example.com", "other").unwrap(); + let request = request::Request::new(url, "https://example.com", "other", "").unwrap(); assert!( network_filter.matches_test(&request) == matching, @@ -106,7 +106,7 @@ mod match_tests { fn hosts_filter_match_url(filter: &str, url: &str, matching: bool) { let network_filter = NetworkFilter::parse_hosts_style(filter, true).unwrap(); - let request = request::Request::new(url, "https://example.com", "other").unwrap(); + let request = request::Request::new(url, "https://example.com", "other", "").unwrap(); assert!( network_filter.matches_test(&request) == matching, @@ -275,7 +275,7 @@ mod match_tests { let url = "https://0914.global.ssl.fastly.net/ad2/script/x.js?cb=1549980040838"; let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let request = - request::Request::new(url, "https://www.gamespot.com/metro-exodus/", "script") + request::Request::new(url, "https://www.gamespot.com/metro-exodus/", "script", "") .unwrap(); assert!( network_filter.matches_test(&request), @@ -290,6 +290,7 @@ mod match_tests { url, "https://www1.swatchseries.to/serie/roswell_new_mexico", "script", + "", ) .unwrap(); assert!( @@ -319,33 +320,43 @@ mod match_tests { NetworkFilter::parse("|ws://$domain=4shared.com", true, Default::default()).unwrap(); assert!(network_filter.matches_test( - &request::Request::new("ws://example.com", "https://4shared.com", "websocket").unwrap() + &request::Request::new("ws://example.com", "https://4shared.com", "websocket", "") + .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("wss://example.com", "https://4shared.com", "websocket") + &request::Request::new("wss://example.com", "https://4shared.com", "websocket", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.com", "https://4shared.com", "script").unwrap() + &request::Request::new("http://example.com", "https://4shared.com", "script", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("https://example.com", "https://4shared.com", "script").unwrap() + &request::Request::new("https://example.com", "https://4shared.com", "script", "") + .unwrap() )); // The `ws://` and `wss://` protocols should be used, rather than the resource type. assert!(network_filter.matches_test( - &request::Request::new("ws://example.com", "https://4shared.com", "script").unwrap() + &request::Request::new("ws://example.com", "https://4shared.com", "script", "") + .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("wss://example.com", "https://4shared.com", "script").unwrap() + &request::Request::new("wss://example.com", "https://4shared.com", "script", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.com", "https://4shared.com", "websocket") + &request::Request::new("http://example.com", "https://4shared.com", "websocket", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("https://example.com", "https://4shared.com", "websocket") - .unwrap() + &request::Request::new( + "https://example.com", + "https://4shared.com", + "websocket", + "" + ) + .unwrap() )); } @@ -391,19 +402,19 @@ mod match_tests { { let network_filter = NetworkFilter::parse("||foo$image", true, Default::default()).unwrap(); - let request = request::Request::new("https://foo.com/bar", "", "image").unwrap(); + let request = request::Request::new("https://foo.com/bar", "", "image", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$image", true, Default::default()).unwrap(); - let request = request::Request::new("https://foo.com/bar", "", "script").unwrap(); + let request = request::Request::new("https://foo.com/bar", "", "script", "").unwrap(); assert!(!check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$~image", true, Default::default()).unwrap(); - let request = request::Request::new("https://foo.com/bar", "", "script").unwrap(); + let request = request::Request::new("https://foo.com/bar", "", "script", "").unwrap(); assert!(check_options(&network_filter, &request)); } @@ -412,14 +423,14 @@ mod match_tests { let network_filter = NetworkFilter::parse("||foo$~third-party", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://baz.foo.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://baz.foo.com", "", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$~third-party", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://baz.bar.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://baz.bar.com", "", "").unwrap(); assert!(!check_options(&network_filter, &request)); } @@ -428,14 +439,14 @@ mod match_tests { let network_filter = NetworkFilter::parse("||foo$~first-party", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://baz.bar.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://baz.bar.com", "", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$~first-party", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://baz.foo.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://baz.foo.com", "", "").unwrap(); assert!(!check_options(&network_filter, &request)); } @@ -444,14 +455,14 @@ mod match_tests { let network_filter = NetworkFilter::parse("||foo$domain=foo.com", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://foo.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://foo.com", "", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$domain=foo.com", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://bar.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://bar.com", "", "").unwrap(); assert!(!check_options(&network_filter, &request)); } @@ -460,14 +471,14 @@ mod match_tests { let network_filter = NetworkFilter::parse("||foo$domain=~bar.com", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://foo.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://foo.com", "", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$domain=~bar.com", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://bar.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://bar.com", "", "").unwrap(); assert!(!check_options(&network_filter, &request)); } } @@ -479,84 +490,69 @@ mod match_tests { { let filter = NetworkFilter::parse("||foo$method=post", true, Default::default()).unwrap(); - let request = - request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); assert!(check_options(&filter, &request)); - let request = - request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); assert!(!check_options(&filter, &request)); } { - let filter = NetworkFilter::parse("||foo$method=post|get", true, Default::default()) - .unwrap(); - let request = - request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + let filter = + NetworkFilter::parse("||foo$method=post|get", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); assert!(check_options(&filter, &request)); } { - let filter = NetworkFilter::parse("||foo$method=head|get", true, Default::default()) - .unwrap(); - let request = - request::Request::new_with_method(url, "", "xhr", Some("head")).unwrap(); + let filter = + NetworkFilter::parse("||foo$method=head|get", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "head").unwrap(); assert!(check_options(&filter, &request)); - let request = - request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); assert!(!check_options(&filter, &request)); } { let filter = NetworkFilter::parse("||foo$method=~get", true, Default::default()).unwrap(); - let request = - request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); assert!(!check_options(&filter, &request)); - let request = - request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); assert!(check_options(&filter, &request)); - let request = - request::Request::new_with_method(url, "", "xhr", Some("head")).unwrap(); + let request = request::Request::new(url, "", "xhr", "head").unwrap(); assert!(check_options(&filter, &request)); - let request = request::Request::new(url, "", "xhr").unwrap(); + let request = request::Request::new(url, "", "xhr", "").unwrap(); assert!(!check_options(&filter, &request)); } { - let filter = - NetworkFilter::parse("||foo", true, Default::default()).unwrap(); - let request = - request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + let filter = NetworkFilter::parse("||foo", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); assert!(check_options(&filter, &request)); } { let filter = NetworkFilter::parse("||foo$xhr,method=post", true, Default::default()).unwrap(); - let request = - request::Request::new_with_method(url, "", "xhr", Some("post")).unwrap(); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); assert!(check_options(&filter, &request)); - let request = - request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); assert!(!check_options(&filter, &request)); - let request = - request::Request::new_with_method(url, "", "script", Some("post")).unwrap(); + let request = request::Request::new(url, "", "script", "post").unwrap(); assert!(!check_options(&filter, &request)); } { let filter = NetworkFilter::parse("||foo$xhr,method=get", true, Default::default()).unwrap(); - let request = - request::Request::new_with_method(url, "", "xhr", Some("get")).unwrap(); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); assert!(check_options(&filter, &request)); } { let filter = NetworkFilter::parse("||foo$xhr,method=head", true, Default::default()).unwrap(); - let request = - request::Request::new_with_method(url, "", "xhr", Some("head")).unwrap(); + let request = request::Request::new(url, "", "xhr", "head").unwrap(); assert!(check_options(&filter, &request)); } } @@ -571,27 +567,34 @@ mod match_tests { ) .unwrap(); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://example.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://example.com", "", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.example.com", "") + &request::Request::new("http://example.net/adv", "http://foo.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( &request::Request::new( "http://example.net/adv", "http://subfoo.foo.example.com", + "", "" ) .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.example.com", "") + &request::Request::new("http://example.net/adv", "http://bar.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://anotherexample.com", "") - .unwrap() + &request::Request::new( + "http://example.net/adv", + "http://anotherexample.com", + "", + "" + ) + .unwrap() )); } { @@ -602,27 +605,34 @@ mod match_tests { ) .unwrap(); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://example.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://example.com", "", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.example.com", "") + &request::Request::new("http://example.net/adv", "http://foo.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( &request::Request::new( "http://example.net/adv", "http://subfoo.foo.example.com", + "", "" ) .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.example.com", "") + &request::Request::new("http://example.net/adv", "http://bar.example.com", "", "") .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://anotherexample.com", "") - .unwrap() + &request::Request::new( + "http://example.net/adv", + "http://anotherexample.com", + "", + "" + ) + .unwrap() )); } { @@ -633,27 +643,34 @@ mod match_tests { ) .unwrap(); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://example.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://example.com", "", "") + .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.example.com", "") + &request::Request::new("http://example.net/adv", "http://foo.example.com", "", "") .unwrap() )); assert!(network_filter.matches_test( &request::Request::new( "http://example.net/adv", "http://subfoo.foo.example.com", + "", "" ) .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.example.com", "") + &request::Request::new("http://example.net/adv", "http://bar.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://anotherexample.com", "") - .unwrap() + &request::Request::new( + "http://example.net/adv", + "http://anotherexample.com", + "", + "" + ) + .unwrap() )); } { @@ -664,47 +681,54 @@ mod match_tests { ) .unwrap(); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://example.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://example.com", "", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.example.com", "") + &request::Request::new("http://example.net/adv", "http://foo.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( &request::Request::new( "http://example.net/adv", "http://subfoo.foo.example.com", + "", "" ) .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.example.com", "") + &request::Request::new("http://example.net/adv", "http://bar.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://anotherexample.com", "") - .unwrap() + &request::Request::new( + "http://example.net/adv", + "http://anotherexample.com", + "", + "" + ) + .unwrap() )); } { let network_filter = NetworkFilter::parse("adv$domain=com|~foo.com", true, Default::default()).unwrap(); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://com", "", "").unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://foo.com", "", "").unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://subfoo.foo.com", "") + &request::Request::new("http://example.net/adv", "http://subfoo.foo.com", "", "") .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://bar.com", "", "").unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://co.uk", "").unwrap() + &request::Request::new("http://example.net/adv", "http://co.uk", "", "").unwrap() )); } } @@ -744,7 +768,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://bit.ly/bar/"; let source = "http://123movies.com"; - let request = request::Request::new(url, source, "").unwrap(); + let request = request::Request::new(url, source, "", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -756,7 +780,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://data.foo.com/9VjjrjU9Or2aqkb8PDiqTBnULPgeI48WmYEHkYer"; let source = "http://123movies.com"; - let request = request::Request::new(url, source, "xmlhttprequest").unwrap(); + let request = request::Request::new(url, source, "xmlhttprequest", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -768,7 +792,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://hello.club/123.css"; let source = "http://123movies.com"; - let request = request::Request::new(url, source, "stylesheet").unwrap(); + let request = request::Request::new(url, source, "stylesheet", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -785,7 +809,7 @@ mod match_tests { let engine = Engine::new_with_list_text(filter, Default::default()); let url = "https://9anime.to/watch/episode-1"; let source = "https://9anime.to"; - let request = request::Request::new(url, source, "script").unwrap(); + let request = request::Request::new(url, source, "script", "").unwrap(); assert_eq!( engine .get_debug_info() @@ -815,7 +839,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://example.com/ad.js"; let source = "http://auth.wi-fi.ru"; - let request = request::Request::new(url, source, "script").unwrap(); + let request = request::Request::new(url, source, "script", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -826,7 +850,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://example.com/ad.js"; let source = "http://auth.wi-fi.ru"; - let request = request::Request::new(url, source, "script").unwrap(); + let request = request::Request::new(url, source, "script", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -841,7 +865,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl="; let source = "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl="; - let request = request::Request::new(url, source, "document").unwrap(); + let request = request::Request::new(url, source, "document", "").unwrap(); assert!(!request.is_third_party); assert!( network_filter.matches_test(&request), @@ -853,7 +877,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl="; let source = "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl="; - let request = request::Request::new(url, source, "main_frame").unwrap(); + let request = request::Request::new(url, source, "main_frame", "").unwrap(); assert!(!request.is_third_party); assert!( network_filter.matches_test(&request), diff --git a/tests/unit/network_filter_list.rs b/tests/unit/network_filter_list.rs index 10a8c5e7..7670ee64 100644 --- a/tests/unit/network_filter_list.rs +++ b/tests/unit/network_filter_list.rs @@ -191,7 +191,7 @@ mod tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); @@ -226,7 +226,7 @@ mod tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); @@ -254,7 +254,7 @@ mod tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); @@ -271,15 +271,11 @@ mod tests { let url_results = [ ( - Request::new("https://bit.ly/bar/", "http://123movies.com", "").unwrap(), + Request::new("https://bit.ly/bar/", "http://123movies.com", "", "").unwrap(), true, ), ( - Request::new( - "https://data.foo.com/9VjjrjU9Or2aqkb8PDiqTBnULPgeI48WmYEHkYer", - "http://123movies.com", - "xmlhttprequest", - ) + Request::new("https://data.foo.com/9VjjrjU9Or2aqkb8PDiqTBnULPgeI48WmYEHkYer", "http://123movies.com", "xmlhttprequest", "") .unwrap(), true, ), diff --git a/tests/unit/optimizer.rs b/tests/unit/optimizer.rs index d4779d20..7a62b3ce 100644 --- a/tests/unit/optimizer.rs +++ b/tests/unit/optimizer.rs @@ -22,6 +22,7 @@ mod optimization_tests_pattern_group { ("https://example.com/".to_string() + url_path).as_str(), "https://google.com", "", + "", ) .unwrap(), ); @@ -132,6 +133,7 @@ mod optimization_tests_pattern_group { &Request::new( "https://example.com/v1/pixel?", "https://my.leadpages.net", + "", "" ) .unwrap() @@ -148,6 +150,7 @@ mod optimization_tests_pattern_group { &Request::new( "https://example.com/analytics/v1/foobar", "https://foo.leadpages.net", + "", "" ) .unwrap() @@ -194,21 +197,13 @@ mod optimization_tests_pattern_group { assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://google.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://google.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://foo.leadpages.net", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://foo.leadpages.net", "", "") .unwrap() ) == false ); @@ -259,41 +254,25 @@ mod optimization_tests_pattern_group { assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://google.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://google.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://example.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://example.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://exampletwo.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://exampletwo.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://foo.leadpages.net", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://foo.leadpages.net", "", "") .unwrap() ) == false ); @@ -320,6 +299,7 @@ mod optimization_tests_pattern_group { ("https://example.com/".to_string() + url_path).as_str(), "https://google.com", "", + "", ) .unwrap(), ); @@ -430,6 +410,7 @@ mod optimization_tests_pattern_group { &Request::new( "https://example.com/v1/pixel?", "https://my.leadpages.net", + "", "" ) .unwrap() @@ -446,6 +427,7 @@ mod optimization_tests_pattern_group { &Request::new( "https://example.com/analytics/v1/foobar", "https://foo.leadpages.net", + "", "" ) .unwrap() @@ -492,21 +474,13 @@ mod optimization_tests_union_domain { assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://google.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://google.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://foo.leadpages.net", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://foo.leadpages.net", "", "") .unwrap() ) == false ); @@ -557,41 +531,25 @@ mod optimization_tests_union_domain { assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://google.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://google.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://example.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://example.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://exampletwo.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://exampletwo.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://foo.leadpages.net", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://foo.leadpages.net", "", "") .unwrap() ) == false ); diff --git a/tests/unit/regex_manager.rs b/tests/unit/regex_manager.rs index 2f7dbd00..bb842fc9 100644 --- a/tests/unit/regex_manager.rs +++ b/tests/unit/regex_manager.rs @@ -11,7 +11,7 @@ mod tests { } fn make_request(url: &str) -> request::Request { - request::Request::new(url, "https://example.com", "other").unwrap() + request::Request::new(url, "https://example.com", "other", "").unwrap() } fn get_active_regex_count(regex_manager: &RegexManager) -> usize { diff --git a/tests/unit/request.rs b/tests/unit/request.rs index 8a0fb551..7035c178 100644 --- a/tests/unit/request.rs +++ b/tests/unit/request.rs @@ -154,6 +154,7 @@ mod tests { "https://subdomain.example.com/ad", "https://example.com/", "document", + "", ) .unwrap(); assert!(parsed.is_https); @@ -178,6 +179,7 @@ mod tests { "subdomain.example.com/ad", "https://example.com/", "document", + "", ); assert_eq!(bad_url.err(), Some(RequestError::HostnameParseError)); } @@ -185,7 +187,7 @@ mod tests { #[test] fn fuzzing_errors() { { - let parsed = Request::new("https://߶", "https://example.com", "other"); + let parsed = Request::new("https://߶", "https://example.com", "other", ""); assert!(parsed.is_ok()); } { @@ -193,6 +195,7 @@ mod tests { &format!("https://{}", std::str::from_utf8(&[9, 9, 64]).unwrap()), "https://example.com", "other", + "", ); assert!(parsed.is_err()); } From cf09132383e9d7c37601de4a561f8cc641689462 Mon Sep 17 00:00:00 2001 From: boocmp Date: Mon, 8 Jun 2026 17:18:43 +0700 Subject: [PATCH 14/28] Rebased. --- tests/unit/engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index 2af05d9e..5e038289 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -241,7 +241,7 @@ mod tests { let expected_hash: u64 = if cfg!(feature = "css-validation") { 16703814344905873613 } else { - 13012744135002827554 + 14988656075164302061 }; assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}"); From 8ec1c7875fce9b7a0a984e097d9913a091d68d27 Mon Sep 17 00:00:00 2001 From: boocmp Date: Mon, 8 Jun 2026 17:23:17 +0700 Subject: [PATCH 15/28] Compilation. --- benches/bench_matching.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benches/bench_matching.rs b/benches/bench_matching.rs index 226c267e..6cc0d2e0 100644 --- a/benches/bench_matching.rs +++ b/benches/bench_matching.rs @@ -77,7 +77,7 @@ fn bench_rule_matching_browserlike(blocker: &Engine, requests: &[ParsedRequest]) source_hostname, request_type, *third_party, - None, + "", )); if check.matched { matches += 1; From aa1da0b49df9780fa70f5a2a51998073fe30ef39 Mon Sep 17 00:00:00 2001 From: boocmp Date: Mon, 8 Jun 2026 17:29:58 +0700 Subject: [PATCH 16/28] Updated hash. --- tests/unit/engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index 5e038289..7d36c762 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -239,7 +239,7 @@ mod tests { ); } let expected_hash: u64 = if cfg!(feature = "css-validation") { - 16703814344905873613 + 2977715961033378463 } else { 14988656075164302061 }; From 698c95249743155421a22e1ba47d9da455baee36 Mon Sep 17 00:00:00 2001 From: boocmp Date: Mon, 8 Jun 2026 17:34:50 +0700 Subject: [PATCH 17/28] Compilation. --- tests/ublock-coverage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ublock-coverage.rs b/tests/ublock-coverage.rs index ab775bd9..a3642f9a 100644 --- a/tests/ublock-coverage.rs +++ b/tests/ublock-coverage.rs @@ -280,7 +280,7 @@ fn check_matching_hostnames() { source_host.hostname(), &req.r#type, third_party, - None, + "", ); let checked = engine.check_network_request(&request); From 23021cfe640ec1747494a0c57ddbfbbd69053c76 Mon Sep 17 00:00:00 2001 From: boocmp Date: Wed, 10 Jun 2026 12:54:46 +0700 Subject: [PATCH 18/28] Improved NetworkFilter tests readability. --- tests/unit/filters/network.rs | 104 +++++++++++----------------------- 1 file changed, 34 insertions(+), 70 deletions(-) diff --git a/tests/unit/filters/network.rs b/tests/unit/filters/network.rs index ab189731..ee39f25c 100644 --- a/tests/unit/filters/network.rs +++ b/tests/unit/filters/network.rs @@ -129,6 +129,17 @@ mod parse_tests { } } + fn foo_method_breakdown(get: bool, head: bool, post: bool) -> NetworkFilterBreakdown { + let mut expected = default_network_filter_breakdown(); + expected.hostname = Some(String::from("foo")); + expected.is_hostname_anchor = true; + expected.is_plain = true; + expected.from_get = get; + expected.from_head = head; + expected.from_post = post; + expected + } + #[test] // pattern fn parses_plain_pattern() { @@ -1262,83 +1273,34 @@ mod parse_tests { #[test] fn handles_method_options() { - { - let filter = - NetworkFilter::parse("||foo$method=post", true, Default::default()).unwrap(); - let mut expected = default_network_filter_breakdown(); - expected.hostname = Some(String::from("foo")); - expected.is_hostname_anchor = true; - expected.is_plain = true; - expected.from_post = true; - assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); - } - - { - let filter = - NetworkFilter::parse("||foo$method=post|get", true, Default::default()).unwrap(); - let mut expected = default_network_filter_breakdown(); - expected.hostname = Some(String::from("foo")); - expected.is_hostname_anchor = true; - expected.is_plain = true; - expected.from_post = true; - expected.from_get = true; - assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); - } - - { - let filter = - NetworkFilter::parse("||foo$method=head|get", true, Default::default()).unwrap(); - let mut expected = default_network_filter_breakdown(); - expected.hostname = Some(String::from("foo")); - expected.is_hostname_anchor = true; - expected.is_plain = true; - expected.from_head = true; - expected.from_get = true; - assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); - } - - { - let filter = - NetworkFilter::parse("||foo$method=~get", true, Default::default()).unwrap(); - let mut expected = default_network_filter_breakdown(); - expected.hostname = Some(String::from("foo")); - expected.is_hostname_anchor = true; - expected.is_plain = true; - expected.from_head = true; - expected.from_post = true; - assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); - } - - { - let filter = - NetworkFilter::parse("||foo$method=POST", true, Default::default()).unwrap(); - let mut expected = default_network_filter_breakdown(); - expected.hostname = Some(String::from("foo")); - expected.is_hostname_anchor = true; - expected.is_plain = true; - expected.from_post = true; - assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); - } + const CASES: &[(&str, bool, bool, bool)] = &[ + ("||foo$method=post", false, false, true), + ("||foo$method=post|get", true, false, true), + ("||foo$method=head|get", true, true, false), + ("||foo$method=~get", false, true, true), + ("||foo$method=POST", false, false, true), + ("||foo$method=post|put", false, false, true), + ]; - { - let filter = - NetworkFilter::parse("||foo$method=post|put", true, Default::default()).unwrap(); - let mut expected = default_network_filter_breakdown(); - expected.hostname = Some(String::from("foo")); - expected.is_hostname_anchor = true; - expected.is_plain = true; - expected.from_post = true; - assert_eq!(expected, NetworkFilterBreakdown::from(&filter)); + for (rule, get, head, post) in CASES { + let filter = NetworkFilter::parse(rule, true, Default::default()).unwrap(); + assert_eq!( + foo_method_breakdown(*get, *head, *post), + NetworkFilterBreakdown::from(&filter), + "rule: {rule}", + ); } - for filter_text in [ + const RULES: &[&str] = &[ "||lemonde.fr/*?s=$xhr,method=post", "||perplexity.ai/rest/metrics/collect^$xhr,1p,method=post", "||apmplus.volces.com/monitor_web/collect$xhr,method=post", "@@||www.realclear*/esm/assets/js/analytics/chartbeat.js?v=$xhr,1p,method=get,domain=com|org", "||pagead2.googlesyndication.com^$3p,xhr,method=head,redirect-rule=noop.js,domain=photopea.com", "||void.nicopr.fr/rec$method=POST", - ] { + ]; + + for filter_text in RULES { assert!( NetworkFilter::parse(filter_text, true, Default::default()).is_ok(), "failed to parse: {filter_text}" @@ -1372,12 +1334,14 @@ mod parse_tests { assert!(!filter.mask.contains(NetworkFilterMask::FROM_POST)); } - for filter_text in [ + const INVALID_RULES: &[&str] = &[ "||foo$method=put", "||foo$method=connect|delete", "||foo$method=", "||foo$~method=post", - ] { + ]; + + for filter_text in INVALID_RULES { assert!( NetworkFilter::parse(filter_text, true, Default::default()).is_err(), "expected parse error: {filter_text}" From 7d6fa42816afbbeff8d9338d91da96076add0635 Mon Sep 17 00:00:00 2001 From: boocmp Date: Thu, 11 Jun 2026 14:19:12 +0700 Subject: [PATCH 19/28] Added deprecation warning for passing debug as the 4th argument. --- js/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/js/src/lib.rs b/js/src/lib.rs index e09cf136..c4500567 100644 --- a/js/src/lib.rs +++ b/js/src/lib.rs @@ -203,6 +203,12 @@ fn engine_check(mut cx: FunctionContext) -> JsResult { }; (String::new(), debug) } else if let Ok(debug) = arg.downcast::(&mut cx) { + console_warn( + &mut cx, + "Engine.check: passing debug as the 4th boolean argument is deprecated; \ + pass an HTTP method string as the 4th argument (use \"\" if unspecified) \ + and debug as an optional 5th boolean instead.", + )?; (String::new(), debug.value(&mut cx)) } else if let Ok(method) = arg.downcast::(&mut cx) { let debug = match cx.argument_opt(5) { From f6305a33d8e97a59c11b9519ccdeacc36a20bbab Mon Sep 17 00:00:00 2001 From: boocmp Date: Thu, 11 Jun 2026 14:20:58 +0700 Subject: [PATCH 20/28] Renamed FROM_METHODS to FROM_ANY_METHODS. --- src/filters/network.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/filters/network.rs b/src/filters/network.rs index 00e4db16..bf3fb71e 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -139,7 +139,7 @@ bitflags::bitflags! { const FROM_HEAD = 1 << 27; const FROM_POST = 1 << 30; - const FROM_METHODS = Self::FROM_GET.bits() | + const FROM_ANY_METHODS = Self::FROM_GET.bits() | Self::FROM_HEAD.bits() | Self::FROM_POST.bits(); @@ -176,7 +176,7 @@ bitflags::bitflags! { impl NetworkFilterMask { #[inline] pub(crate) fn check_method_allowed(self, method: Option<&request::RequestMethod>) -> bool { - if !self.intersects(Self::FROM_METHODS) { + if !self.intersects(Self::FROM_ANY_METHODS) { return true; } let Some(method) = method else { @@ -845,10 +845,10 @@ impl<'a> NetworkFilter<'a> { if !method_mask_positive.is_empty() || !method_mask_negative.is_empty() { mask |= method_mask_positive; - if (method_mask_negative & NetworkFilterMask::FROM_METHODS) != NetworkFilterMask::NONE - && (method_mask_positive & NetworkFilterMask::FROM_METHODS).is_empty() + if (method_mask_negative & NetworkFilterMask::FROM_ANY_METHODS) != NetworkFilterMask::NONE + && (method_mask_positive & NetworkFilterMask::FROM_ANY_METHODS).is_empty() { - mask |= NetworkFilterMask::FROM_METHODS; + mask |= NetworkFilterMask::FROM_ANY_METHODS; } mask &= !method_mask_negative; } From 85a8e358a702434eb59524c11898b268385b7147 Mon Sep 17 00:00:00 2001 From: boocmp Date: Thu, 11 Jun 2026 14:26:51 +0700 Subject: [PATCH 21/28] Extended RequestMethod enum to include all HTTP methods. --- src/filters/network.rs | 7 ++++++- src/request.rs | 28 ++++++++++++++++++++++------ tests/unit/request.rs | 28 ++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/filters/network.rs b/src/filters/network.rs index bf3fb71e..02a06e43 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -182,7 +182,11 @@ impl NetworkFilterMask { let Some(method) = method else { return false; }; - self.contains(Self::from(method)) + let method_mask = Self::from(method); + if method_mask.is_empty() { + return false; + } + self.contains(method_mask) } } @@ -287,6 +291,7 @@ impl From<&request::RequestMethod> for NetworkFilterMask { request::RequestMethod::Get => NetworkFilterMask::FROM_GET, request::RequestMethod::Head => NetworkFilterMask::FROM_HEAD, request::RequestMethod::Post => NetworkFilterMask::FROM_POST, + _ => NetworkFilterMask::NONE, } } } diff --git a/src/request.rs b/src/request.rs index 7cc32165..ca791670 100644 --- a/src/request.rs +++ b/src/request.rs @@ -7,9 +7,15 @@ use crate::utils; #[derive(Clone, Copy, PartialEq, Debug)] pub enum RequestMethod { + Connect, + Delete, Get, Head, + Options, + Patch, Post, + Put, + Other, } /// The type of resource requested from the URL endpoint. @@ -128,15 +134,25 @@ impl Request { if raw_method.is_empty() { return None; } - if raw_method.eq_ignore_ascii_case("get") { - Some(RequestMethod::Get) + Some(if raw_method.eq_ignore_ascii_case("connect") { + RequestMethod::Connect + } else if raw_method.eq_ignore_ascii_case("delete") { + RequestMethod::Delete + } else if raw_method.eq_ignore_ascii_case("get") { + RequestMethod::Get } else if raw_method.eq_ignore_ascii_case("head") { - Some(RequestMethod::Head) + RequestMethod::Head + } else if raw_method.eq_ignore_ascii_case("options") { + RequestMethod::Options + } else if raw_method.eq_ignore_ascii_case("patch") { + RequestMethod::Patch } else if raw_method.eq_ignore_ascii_case("post") { - Some(RequestMethod::Post) + RequestMethod::Post + } else if raw_method.eq_ignore_ascii_case("put") { + RequestMethod::Put } else { - None - } + RequestMethod::Other + }) } #[allow(clippy::too_many_arguments)] diff --git a/tests/unit/request.rs b/tests/unit/request.rs index 7035c178..c85a4e6f 100644 --- a/tests/unit/request.rs +++ b/tests/unit/request.rs @@ -184,6 +184,34 @@ mod tests { assert_eq!(bad_url.err(), Some(RequestError::HostnameParseError)); } + #[test] + fn parse_method_works() { + assert_eq!( + Request::new("https://example.com", "https://example.com", "xhr", "get") + .unwrap() + .method, + Some(RequestMethod::Get), + ); + assert_eq!( + Request::new("https://example.com", "https://example.com", "xhr", "PUT") + .unwrap() + .method, + Some(RequestMethod::Put), + ); + assert_eq!( + Request::new("https://example.com", "https://example.com", "xhr", "TRACE") + .unwrap() + .method, + Some(RequestMethod::Other), + ); + assert_eq!( + Request::new("https://example.com", "https://example.com", "xhr", "") + .unwrap() + .method, + None, + ); + } + #[test] fn fuzzing_errors() { { From 2d4a6da0c2f03f759c6be792d06e81297cb59c3c Mon Sep 17 00:00:00 2001 From: boocmp Date: Thu, 11 Jun 2026 14:34:07 +0700 Subject: [PATCH 22/28] Using 'get' method and added error handling for generichide method conflicts in NetworkFilter. --- src/engine.rs | 2 +- src/filters/network.rs | 11 ++++++++++- tests/unit/filters/network.rs | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/engine.rs b/src/engine.rs index 1305d50c..63c449f7 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -299,7 +299,7 @@ impl Engine { /// `hidden_class_id_selectors` to obtain any stylesheets consisting of generic rules (if the /// returned `generichide` value is false). pub fn url_cosmetic_resources(&self, url: &str) -> UrlSpecificResources { - let request = if let Ok(request) = Request::new(url, url, "document", "") { + let request = if let Ok(request) = Request::new(url, url, "document", "get") { request } else { return UrlSpecificResources::empty(); diff --git a/src/filters/network.rs b/src/filters/network.rs index 02a06e43..129c4115 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -67,6 +67,8 @@ pub enum NetworkFilterError { NegatedAll, #[error("generichide without exception")] GenericHideWithoutException, + #[error("method with generichide")] + MethodWithGenerichide, #[error("empty redirection")] EmptyRedirection, #[error("empty removeparam")] @@ -850,7 +852,8 @@ impl<'a> NetworkFilter<'a> { if !method_mask_positive.is_empty() || !method_mask_negative.is_empty() { mask |= method_mask_positive; - if (method_mask_negative & NetworkFilterMask::FROM_ANY_METHODS) != NetworkFilterMask::NONE + if (method_mask_negative & NetworkFilterMask::FROM_ANY_METHODS) + != NetworkFilterMask::NONE && (method_mask_positive & NetworkFilterMask::FROM_ANY_METHODS).is_empty() { mask |= NetworkFilterMask::FROM_ANY_METHODS; @@ -858,6 +861,12 @@ impl<'a> NetworkFilter<'a> { mask &= !method_mask_negative; } + if features_mask.contains(NetworkFilterFeaturesMask::GENERIC_HIDE) + && mask.intersects(NetworkFilterMask::FROM_ANY_METHODS) + { + return Err(NetworkFilterError::MethodWithGenerichide); + } + Ok(NetworkFilter { filter: if let Some(simple_filter) = filter { FilterPart::Simple(simple_filter) diff --git a/tests/unit/filters/network.rs b/tests/unit/filters/network.rs index ee39f25c..e2080f2c 100644 --- a/tests/unit/filters/network.rs +++ b/tests/unit/filters/network.rs @@ -1347,5 +1347,19 @@ mod parse_tests { "expected parse error: {filter_text}" ); } + + assert!(NetworkFilter::parse("@@||foo^$generichide", true, Default::default()).is_ok()); + + for filter_text in [ + "@@||foo^$generichide,method=post", + "@@||foo^$ghide,method=get", + "@@||foo^$generichide,method=head|get", + ] { + assert_eq!( + NetworkFilter::parse(filter_text, true, Default::default()).err(), + Some(NetworkFilterError::MethodWithGenerichide), + "expected MethodWithGenerichide: {filter_text}", + ); + } } } From 7f413a5e2b623bd9bf91b13ad16da53ca9e500d4 Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 12 Jun 2026 12:20:33 +0700 Subject: [PATCH 23/28] Added parser for $to option in NetworkFilter. --- src/filters/abstract_network.rs | 77 +++++++++++++++++++++----- src/filters/network.rs | 3 + tests/unit/filters/abstract_network.rs | 42 ++++++++++++++ tests/unit/filters/network.rs | 33 +++++++++++ 4 files changed, 142 insertions(+), 13 deletions(-) create mode 100644 tests/unit/filters/abstract_network.rs diff --git a/src/filters/abstract_network.rs b/src/filters/abstract_network.rs index 69ac61f6..dfc62758 100644 --- a/src/filters/abstract_network.rs +++ b/src/filters/abstract_network.rs @@ -38,12 +38,68 @@ pub(crate) enum HttpMethod { Post, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum DomainValueKind { + Plain, + Entity, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ParsedDomainValue<'a> { + pub included: bool, + pub kind: DomainValueKind, + pub value: &'a str, + pub raw: &'a str, +} + +fn parse_domain_option_values<'a>( + value: &'a str, +) -> Result>, NetworkFilterError> { + let domains: Vec = value + .split('|') + .filter_map(|domain| { + let (included, rest) = if let Some(negated_domain) = domain.strip_prefix('~') { + (false, negated_domain) + } else { + (true, domain) + }; + + if (rest.starts_with('/') && rest.ends_with('/')) || rest.is_empty() { + return None; + } + + let (kind, normalized) = if let Some(entity_prefix) = rest.strip_suffix(".*") { + if entity_prefix.is_empty() { + return None; + } + (DomainValueKind::Entity, entity_prefix) + } else { + (DomainValueKind::Plain, rest) + }; + + Some(ParsedDomainValue { + included, + kind, + value: normalized, + raw: rest, + }) + }) + .collect(); + + if domains.is_empty() { + return Err(NetworkFilterError::NoSupportedDomains); + } + + Ok(domains) +} + /// Any option that appears on the right side of a network filter as initiated by a `$` character. /// All `bool` arguments below are `true` if the option stands alone, or `false` if the option is /// negated using a prepended `~`. #[derive(Clone)] pub(crate) enum NetworkFilterOption<'a> { Domain(Vec<(bool, &'a str)>), + To(Vec>), Badfilter, Important, MatchCase, @@ -182,22 +238,13 @@ fn parse_filter_options<'a>( result.push(match (option, negation) { ("domain", _) | ("from", _) => { - let domains: Vec<(bool, &'a str)> = value - .split('|') - .map(|domain| { - if let Some(negated_domain) = domain.strip_prefix('~') { - (false, negated_domain) - } else { - (true, domain) - } - }) - .filter(|(_, d)| !(d.starts_with('/') && d.ends_with('/'))) + let domains = parse_domain_option_values(value)? + .into_iter() + .map(|entry| (entry.included, entry.raw)) .collect(); - if domains.is_empty() { - return Err(NetworkFilterError::NoSupportedDomains); - } NetworkFilterOption::Domain(domains) } + ("to", _) => NetworkFilterOption::To(parse_domain_option_values(value)?), ("badfilter", true) => return Err(NetworkFilterError::NegatedBadFilter), ("badfilter", false) => NetworkFilterOption::Badfilter, ("important", true) => return Err(NetworkFilterError::NegatedImportant), @@ -297,3 +344,7 @@ fn parse_filter_options<'a>( } Ok(result) } + +#[cfg(test)] +#[path = "../../tests/unit/filters/abstract_network.rs"] +mod unit_tests; diff --git a/src/filters/network.rs b/src/filters/network.rs index 129c4115..e607ee68 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -560,6 +560,9 @@ impl<'a> NetworkFilter<'a> { opt_not_domains = Some(opt_not_domains_array); } } + NetworkFilterOption::To(values) => { + let _ = values; + } NetworkFilterOption::Badfilter => { features_mask.set(NetworkFilterFeaturesMask::BAD_FILTER, true) } diff --git a/tests/unit/filters/abstract_network.rs b/tests/unit/filters/abstract_network.rs new file mode 100644 index 00000000..6d84d822 --- /dev/null +++ b/tests/unit/filters/abstract_network.rs @@ -0,0 +1,42 @@ +use super::*; + +#[test] +fn parse_domain_option_values_plain_and_entity() { + let values = parse_domain_option_values("google.*|gstatic.com|~example.it").unwrap(); + assert_eq!(values.len(), 3); + assert_eq!( + values[0], + ParsedDomainValue { + included: true, + kind: DomainValueKind::Entity, + value: "google", + raw: "google.*", + } + ); + assert_eq!( + values[1], + ParsedDomainValue { + included: true, + kind: DomainValueKind::Plain, + value: "gstatic.com", + raw: "gstatic.com", + } + ); + assert_eq!( + values[2], + ParsedDomainValue { + included: false, + kind: DomainValueKind::Plain, + value: "example.it", + raw: "example.it", + } + ); +} + +#[test] +fn parse_domain_option_values_strips_regex() { + assert!(parse_domain_option_values("/^foo/").is_err()); + let values = parse_domain_option_values("/^foo/|bar.com").unwrap(); + assert_eq!(values.len(), 1); + assert_eq!(values[0].value, "bar.com"); +} diff --git a/tests/unit/filters/network.rs b/tests/unit/filters/network.rs index e2080f2c..72eb0653 100644 --- a/tests/unit/filters/network.rs +++ b/tests/unit/filters/network.rs @@ -730,6 +730,39 @@ mod parse_tests { } } + #[test] + fn parses_to() { + { + let filter = + NetworkFilter::parse("||foo.com$to=bar.com", true, Default::default()).unwrap(); + assert_eq!(filter.opt_domains, None); + } + { + let filter = NetworkFilter::parse( + "||foo.com$to=google.*|gstatic.com", + true, + Default::default(), + ) + .unwrap(); + assert_eq!(filter.opt_domains, None); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=~example.it", true, Default::default()).unwrap(); + assert_eq!(filter.opt_domains, None); + } + { + let filter = NetworkFilter::parse("||foo.com$to=/^foo/", true, Default::default()); + assert_eq!(filter.err(), Some(NetworkFilterError::NoSupportedDomains)); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=/^foo/|bar.com", true, Default::default()) + .unwrap(); + assert_eq!(filter.opt_domains, None); + } + } + #[test] fn parses_redirects() { // parses redirect From 9a83ae598cf735b946ab916d7e1904d6f1e94895 Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 12 Jun 2026 12:32:43 +0700 Subject: [PATCH 24/28] Extended NetworkFilter to support additional options: opt_to_domains, opt_not_to_domains, opt_to_entities, and opt_not_to_entities. --- src/filters/network.rs | 142 ++++++++++++++++++++++++++++++---- src/optimizer.rs | 4 + tests/unit/filters/network.rs | 29 ++++++- 3 files changed, 159 insertions(+), 16 deletions(-) diff --git a/src/filters/network.rs b/src/filters/network.rs index e607ee68..f03bb521 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -13,8 +13,8 @@ use thiserror::Error; use std::fmt; use crate::filters::abstract_network::{ - AbstractNetworkFilter, HttpMethod, NetworkFilterLeftAnchor, NetworkFilterOption, - NetworkFilterRightAnchor, + AbstractNetworkFilter, DomainValueKind, HttpMethod, NetworkFilterLeftAnchor, + NetworkFilterOption, NetworkFilterRightAnchor, }; use crate::lists::ParseOptions; use crate::regex_manager::RegexManager; @@ -408,6 +408,10 @@ pub struct NetworkFilter<'a> { pub filter: FilterPart<'a>, pub opt_domains: Option>, pub opt_not_domains: Option>, + pub opt_to_domains: Option>, + pub opt_not_to_domains: Option>, + pub opt_to_entities: Option>, + pub opt_not_to_entities: Option>, /// Used for `$redirect`, `$redirect-rule`, `$csp`, and `$removeparam` - only one of which is /// supported per-rule. pub modifier_option: Option<&'a str>, @@ -501,6 +505,10 @@ impl<'a> NetworkFilter<'a> { let mut opt_domains: Option> = None; let mut opt_not_domains: Option> = None; + let mut opt_to_domains: Option> = None; + let mut opt_not_to_domains: Option> = None; + let mut opt_to_entities: Option> = None; + let mut opt_not_to_entities: Option> = None; let mut modifier_option: Option<&'a str> = None; let mut tag: Option<&'a str> = None; @@ -561,7 +569,49 @@ impl<'a> NetworkFilter<'a> { } } NetworkFilterOption::To(values) => { - let _ = values; + let mut opt_to_domains_array: Vec = vec![]; + let mut opt_not_to_domains_array: Vec = vec![]; + let mut opt_to_entities_array: Vec = vec![]; + let mut opt_not_to_entities_array: Vec = vec![]; + + for entry in values { + let hash = utils::fast_hash(entry.value); + match (entry.kind, entry.included) { + (DomainValueKind::Plain, true) => { + opt_to_domains_array.push(hash); + } + (DomainValueKind::Plain, false) => { + opt_not_to_domains_array.push(hash); + } + (DomainValueKind::Entity, true) => { + opt_to_entities_array.push(hash); + } + (DomainValueKind::Entity, false) => { + opt_not_to_entities_array.push(hash); + } + } + } + + if !opt_to_domains_array.is_empty() { + opt_to_domains_array.sort_unstable(); + opt_to_domains_array.dedup(); + opt_to_domains = Some(opt_to_domains_array); + } + if !opt_not_to_domains_array.is_empty() { + opt_not_to_domains_array.sort_unstable(); + opt_not_to_domains_array.dedup(); + opt_not_to_domains = Some(opt_not_to_domains_array); + } + if !opt_to_entities_array.is_empty() { + opt_to_entities_array.sort_unstable(); + opt_to_entities_array.dedup(); + opt_to_entities = Some(opt_to_entities_array); + } + if !opt_not_to_entities_array.is_empty() { + opt_not_to_entities_array.sort_unstable(); + opt_not_to_entities_array.dedup(); + opt_not_to_entities = Some(opt_not_to_entities_array); + } } NetworkFilterOption::Badfilter => { features_mask.set(NetworkFilterFeaturesMask::BAD_FILTER, true) @@ -881,6 +931,10 @@ impl<'a> NetworkFilter<'a> { features_mask, opt_domains, opt_not_domains, + opt_to_domains, + opt_not_to_domains, + opt_to_entities, + opt_not_to_entities, tag, raw_line: if debug { Some(Cow::Borrowed(line)) @@ -934,6 +988,10 @@ impl<'a> NetworkFilter<'a> { features_mask: Default::default(), opt_domains: None, opt_not_domains: None, + opt_to_domains: None, + opt_not_to_domains: None, + opt_to_entities: None, + opt_not_to_entities: None, tag: None, raw_line: if debug { Some(Cow::Owned(rule)) } else { None }, modifier_option: None, @@ -950,6 +1008,10 @@ impl<'a> NetworkFilter<'a> { self.hostname.as_deref(), self.opt_domains.as_ref(), self.opt_not_domains.as_ref(), + self.opt_to_domains.as_ref(), + self.opt_not_to_domains.as_ref(), + self.opt_to_entities.as_ref(), + self.opt_not_to_entities.as_ref(), ) } @@ -970,6 +1032,17 @@ impl<'a> NetworkFilter<'a> { tokens_buffer.push(*domain); } } + } else if self.opt_to_domains.is_some() + && self.opt_not_to_domains.is_none() + && self.opt_to_entities.is_none() + && self.opt_not_to_entities.is_none() + && self.opt_to_domains.as_ref().map(|d| d.len()) == Some(1) + { + if let Some(domains) = self.opt_to_domains.as_ref() { + if let Some(domain) = domains.first() { + tokens_buffer.push(*domain); + } + } } // Get tokens from filter @@ -1017,17 +1090,32 @@ impl<'a> NetworkFilter<'a> { // If we got no tokens for the filter/hostname part, then we will dispatch // this filter in multiple buckets based on the domains option. - if tokens_buffer.is_empty() && self.opt_domains.is_some() && self.opt_not_domains.is_none() - { - if let Some(opt_domains) = self.opt_domains.as_ref() { - if !opt_domains.is_empty() { - let cap = tokens_buffer.remaining_capacity(); - if opt_domains.len() <= cap { - tokens_buffer.extend(opt_domains.iter().copied()); - return FilterTokens::OptDomains; + if tokens_buffer.is_empty() { + if self.opt_domains.is_some() && self.opt_not_domains.is_none() { + if let Some(opt_domains) = self.opt_domains.as_ref() { + if !opt_domains.is_empty() { + let cap = tokens_buffer.remaining_capacity(); + if opt_domains.len() <= cap { + tokens_buffer.extend(opt_domains.iter().copied()); + return FilterTokens::OptDomains; + } + // Too many domains to bucket individually; fall back to the catch-all + // bucket (token 0). + } + } + } else if self.opt_to_domains.is_some() + && self.opt_not_to_domains.is_none() + && self.opt_to_entities.is_none() + && self.opt_not_to_entities.is_none() + { + if let Some(opt_to_domains) = self.opt_to_domains.as_ref() { + if !opt_to_domains.is_empty() { + let cap = tokens_buffer.remaining_capacity(); + if opt_to_domains.len() <= cap { + tokens_buffer.extend(opt_to_domains.iter().copied()); + return FilterTokens::OptDomains; + } } - // Too many domains to bucket individually; fall back to the catch-all - // bucket (token 0). } } FilterTokens::Empty @@ -1121,6 +1209,10 @@ fn compute_filter_id( hostname: Option<&str>, opt_domains: Option<&Vec>, opt_not_domains: Option<&Vec>, + opt_to_domains: Option<&Vec>, + opt_not_to_domains: Option<&Vec>, + opt_to_entities: Option<&Vec>, + opt_not_to_entities: Option<&Vec>, ) -> Hash { let mut hasher = FxHasher::default(); @@ -1146,6 +1238,30 @@ fn compute_filter_id( } } + if let Some(domains) = opt_to_domains { + for d in domains { + hasher.write_u64(*d); + } + } + + if let Some(domains) = opt_not_to_domains { + for d in domains { + hasher.write_u64(*d); + } + } + + if let Some(domains) = opt_to_entities { + for d in domains { + hasher.write_u64(*d); + } + } + + if let Some(domains) = opt_not_to_entities { + for d in domains { + hasher.write_u64(*d); + } + } + match filter { FilterPart::Empty => {} FilterPart::Simple(s) => write_str_to_hasher(&mut hasher, s.as_ref()), diff --git a/src/optimizer.rs b/src/optimizer.rs index e17a15a5..b0362a68 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -14,6 +14,10 @@ trait Optimization { pub fn is_filter_optimizable_by_patterns(filter: &NetworkFilter<'_>) -> bool { filter.opt_domains.is_none() && filter.opt_not_domains.is_none() + && filter.opt_to_domains.is_none() + && filter.opt_not_to_domains.is_none() + && filter.opt_to_entities.is_none() + && filter.opt_not_to_entities.is_none() && !filter.is_hostname_anchor() && !filter.is_redirect() && !filter.is_csp() diff --git a/tests/unit/filters/network.rs b/tests/unit/filters/network.rs index 72eb0653..e2de6b52 100644 --- a/tests/unit/filters/network.rs +++ b/tests/unit/filters/network.rs @@ -736,6 +736,13 @@ mod parse_tests { let filter = NetworkFilter::parse("||foo.com$to=bar.com", true, Default::default()).unwrap(); assert_eq!(filter.opt_domains, None); + assert_eq!( + filter.opt_to_domains, + Some(vec![utils::fast_hash("bar.com")]) + ); + assert_eq!(filter.opt_not_to_domains, None); + assert_eq!(filter.opt_to_entities, None); + assert_eq!(filter.opt_not_to_entities, None); } { let filter = NetworkFilter::parse( @@ -744,12 +751,25 @@ mod parse_tests { Default::default(), ) .unwrap(); - assert_eq!(filter.opt_domains, None); + assert_eq!( + filter.opt_to_domains, + Some(vec![utils::fast_hash("gstatic.com")]) + ); + assert_eq!( + filter.opt_to_entities, + Some(vec![utils::fast_hash("google")]) + ); + assert_eq!(filter.opt_not_to_domains, None); + assert_eq!(filter.opt_not_to_entities, None); } { let filter = NetworkFilter::parse("||foo.com$to=~example.it", true, Default::default()).unwrap(); - assert_eq!(filter.opt_domains, None); + assert_eq!(filter.opt_to_domains, None); + assert_eq!( + filter.opt_not_to_domains, + Some(vec![utils::fast_hash("example.it")]) + ); } { let filter = NetworkFilter::parse("||foo.com$to=/^foo/", true, Default::default()); @@ -759,7 +779,10 @@ mod parse_tests { let filter = NetworkFilter::parse("||foo.com$to=/^foo/|bar.com", true, Default::default()) .unwrap(); - assert_eq!(filter.opt_domains, None); + assert_eq!( + filter.opt_to_domains, + Some(vec![utils::fast_hash("bar.com")]) + ); } } From 806f1c3de52f1965637982345d9dbb5589cdf139 Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 12 Jun 2026 12:38:36 +0700 Subject: [PATCH 25/28] Added hostname and entity hash calculations to Request. --- src/request.rs | 45 +++++++++++++++++++++++++++++++++---------- tests/unit/request.rs | 33 +++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/request.rs b/src/request.rs index ca791670..450fcb3c 100644 --- a/src/request.rs +++ b/src/request.rs @@ -2,6 +2,7 @@ use thiserror::Error; +use crate::filters::cosmetic::get_entity_hashes_from_labels; use crate::url_parser; use crate::utils; @@ -101,6 +102,8 @@ pub struct Request { pub url: String, pub hostname: String, pub source_hostname_hashes: Option>, + pub(crate) hostname_hashes: Option>, + pub(crate) entity_hashes: Option>, pub(crate) url_lower_cased: String, pub(crate) request_tokens: Vec, @@ -155,12 +158,27 @@ impl Request { }) } + fn suffix_hostname_hashes(hostname: &str) -> Option> { + if hostname.is_empty() { + return None; + } + let mut hashes = Vec::with_capacity(4); + hashes.push(utils::fast_hash(hostname)); + for (i, c) in hostname.char_indices() { + if c == '.' && i + 1 < hostname.len() { + hashes.push(utils::fast_hash(&hostname[i + 1..])); + } + } + Some(hashes) + } + #[allow(clippy::too_many_arguments)] fn from_detailed_parameters( raw_type: &str, url: &str, schema: &str, hostname: &str, + domain: &str, source_hostname: &str, third_party: bool, original_url: String, @@ -190,17 +208,17 @@ impl Request { } } - let source_hostname_hashes = if !source_hostname.is_empty() { - let mut hashes = Vec::with_capacity(4); - hashes.push(utils::fast_hash(source_hostname)); - for (i, c) in source_hostname.char_indices() { - if c == '.' && i + 1 < source_hostname.len() { - hashes.push(utils::fast_hash(&source_hostname[i + 1..])); - } - } - Some(hashes) - } else { + let source_hostname_hashes = Self::suffix_hostname_hashes(source_hostname); + let hostname_hashes = Self::suffix_hostname_hashes(hostname); + let entity_hashes = if hostname.is_empty() { None + } else { + let hashes = get_entity_hashes_from_labels(hostname, domain); + if hashes.is_empty() { + None + } else { + Some(hashes) + } }; let url_lower_cased = url.to_ascii_lowercase(); @@ -213,6 +231,8 @@ impl Request { hostname: hostname.to_owned(), request_tokens: calculate_tokens(&url_lower_cased), source_hostname_hashes, + hostname_hashes, + entity_hashes, is_third_party: third_party, is_http, is_https, @@ -240,6 +260,7 @@ impl Request { &parsed_url.url, parsed_url.schema(), parsed_url.hostname(), + parsed_url.domain(), parsed_source.hostname(), third_party, url.to_string(), @@ -251,6 +272,7 @@ impl Request { &parsed_url.url, parsed_url.schema(), parsed_url.hostname(), + parsed_url.domain(), "", true, url.to_string(), @@ -275,12 +297,15 @@ impl Request { ) -> Request { let splitter = memchr::memchr(b':', url.as_bytes()).unwrap_or(0); let schema: &str = &url[..splitter]; + let (domain_start, domain_end) = url_parser::get_host_domain(hostname); + let domain = &hostname[domain_start..domain_end]; Request::from_detailed_parameters( request_type, url, schema, hostname, + domain, source_hostname, third_party, url.to_string(), diff --git a/tests/unit/request.rs b/tests/unit/request.rs index c85a4e6f..635134af 100644 --- a/tests/unit/request.rs +++ b/tests/unit/request.rs @@ -18,6 +18,7 @@ mod tests { url, schema, hostname, + domain, source_hostname, third_party, url.to_string(), @@ -148,6 +149,38 @@ mod tests { ) } + #[test] + fn destination_hostname_hashes_works() { + let request = build_request( + "script", + "https://cdn.gstatic.com/foo.js", + "https", + "cdn.gstatic.com", + "gstatic.com", + "example.com", + "example.com", + ); + assert_eq!( + request.hostname_hashes.as_ref().unwrap().as_slice(), + tokenize(&["cdn.gstatic.com", "gstatic.com", "com"], &[]).as_slice(), + ); + } + + #[test] + fn destination_entity_hashes_works() { + let request = build_request( + "script", + "https://www.google.co.uk/foo.js", + "https", + "www.google.co.uk", + "google.co.uk", + "example.com", + "example.com", + ); + let entity_hashes = request.entity_hashes.as_ref().unwrap(); + assert!(entity_hashes.contains(&utils::fast_hash("google"))); + } + #[test] fn parses_urls() { let parsed = Request::new( From 34c168a6f6642f183f8ce6c31c8f9f0f463c20a2 Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 12 Jun 2026 12:49:17 +0700 Subject: [PATCH 26/28] Extended fb to support of opt_to_domains, opt_not_to_domains, opt_to_entities, and opt_not_to_entities. --- src/data_format/mod.rs | 2 +- src/filters/fb_network.rs | 47 ++++- src/filters/fb_network_builder.rs | 44 ++++ src/filters/network_matchers.rs | 100 +++++++-- src/flatbuffers/fb_network_filter.fbs | 4 + .../fb_network_filter_generated.rs | 198 ++++++++++++++++-- tests/unit/engine.rs | 6 +- tests/unit/filters/network_matchers.rs | 155 ++++++++++++++ 8 files changed, 524 insertions(+), 32 deletions(-) diff --git a/src/data_format/mod.rs b/src/data_format/mod.rs index fb3ffebd..37202b05 100644 --- a/src/data_format/mod.rs +++ b/src/data_format/mod.rs @@ -17,7 +17,7 @@ const ADBLOCK_RUST_DAT_MAGIC: [u8; 4] = [0xd1, 0xd9, 0x3a, 0xaf]; /// The version of the data format. /// If the data format version is incremented, the data is considered as incompatible. -const ADBLOCK_RUST_DAT_VERSION: u8 = 4; +const ADBLOCK_RUST_DAT_VERSION: u8 = 5; /// The total length of the header prefix (magic + version + seahash) const HEADER_PREFIX_LENGTH: usize = 4 + 1 + 8; diff --git a/src/filters/fb_network.rs b/src/filters/fb_network.rs index 9c8be08d..0a283350 100644 --- a/src/filters/fb_network.rs +++ b/src/filters/fb_network.rs @@ -140,6 +140,34 @@ impl<'a> FlatNetworkFilter<'a> { .map(|data| fb_vector_to_slice(data)) } + #[inline(always)] + pub fn include_to_domains(&self) -> Option<&[u32]> { + self.fb_filter + .opt_to_domains() + .map(|data| fb_vector_to_slice(data)) + } + + #[inline(always)] + pub fn exclude_to_domains(&self) -> Option<&[u32]> { + self.fb_filter + .opt_not_to_domains() + .map(|data| fb_vector_to_slice(data)) + } + + #[inline(always)] + pub fn include_to_entities(&self) -> Option<&[u32]> { + self.fb_filter + .opt_to_entities() + .map(|data| fb_vector_to_slice(data)) + } + + #[inline(always)] + pub fn exclude_to_entities(&self) -> Option<&[u32]> { + self.fb_filter + .opt_not_to_entities() + .map(|data| fb_vector_to_slice(data)) + } + #[inline(always)] pub fn hostname(&self) -> Option<&'a str> { if self.mask.is_hostname_anchor() { @@ -173,7 +201,8 @@ impl NetworkFilterMaskHelper for FlatNetworkFilter<'_> { impl NetworkMatchable for FlatNetworkFilter<'_> { fn matches(&self, request: &Request, regex_manager: &mut RegexManager) -> bool { use crate::filters::network_matchers::{ - check_excluded_domains_mapped, check_included_domains_mapped, check_options, + check_excluded_domains_mapped, check_excluded_to_options_mapped, + check_included_domains_mapped, check_included_to_options_mapped, check_options, check_pattern, }; if !check_options(self.mask, request) { @@ -193,6 +222,22 @@ impl NetworkMatchable for FlatNetworkFilter<'_> { ) { return false; } + if !check_included_to_options_mapped( + self.include_to_domains(), + self.include_to_entities(), + request, + &self.filter_data_context.unique_domains_hashes_map, + ) { + return false; + } + if !check_excluded_to_options_mapped( + self.exclude_to_domains(), + self.exclude_to_entities(), + request, + &self.filter_data_context.unique_domains_hashes_map, + ) { + return false; + } check_pattern( self.mask, self.patterns().iter(), diff --git a/src/filters/fb_network_builder.rs b/src/filters/fb_network_builder.rs index f7f80020..92ec33ff 100644 --- a/src/filters/fb_network_builder.rs +++ b/src/filters/fb_network_builder.rs @@ -74,6 +74,46 @@ impl<'f, 'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilter<'f> { FlatSerialize::serialize(o, builder) }); + let opt_to_domains = network_filter.opt_to_domains.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + + let opt_not_to_domains = network_filter.opt_not_to_domains.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + + let opt_to_entities = network_filter.opt_to_entities.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + + let opt_not_to_entities = network_filter.opt_not_to_entities.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + let modifier_option = network_filter .modifier_option .map(|s| builder.create_string(s)); @@ -117,6 +157,10 @@ impl<'f, 'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilter<'f> { modifier_option, opt_domains, opt_not_domains, + opt_to_domains, + opt_not_to_domains, + opt_to_entities, + opt_not_to_entities, hostname, tag, raw_line, diff --git a/src/filters/network_matchers.rs b/src/filters/network_matchers.rs index 830eb4f0..ffca5e88 100644 --- a/src/filters/network_matchers.rs +++ b/src/filters/network_matchers.rs @@ -419,15 +419,15 @@ pub fn check_options(mask: NetworkFilterMask, request: &request::Request) -> boo } #[inline] -pub fn check_included_domains_mapped( +fn check_included_domains_mapped_with_hashes( opt_domains: Option<&[u32]>, - request: &request::Request, + hostname_hashes: Option<&[Hash]>, mapping: &HashMap, ) -> bool { // Source URL must be among these domains to match if let Some(included_domains) = opt_domains.as_ref() { - if let Some(source_hashes) = request.source_hostname_hashes.as_ref() { - if source_hashes.iter().all(|h| { + if let Some(hashes) = hostname_hashes { + if hashes.iter().all(|h| { mapping .get(h) .is_none_or(|index| !utils::bin_lookup(included_domains, *index)) @@ -443,30 +443,104 @@ pub fn check_included_domains_mapped( } #[inline] -pub fn check_excluded_domains_mapped( +fn check_excluded_domains_mapped_with_hashes( opt_not_domains: Option<&[u32]>, - request: &request::Request, + hostname_hashes: Option<&[Hash]>, mapping: &HashMap, ) -> bool { if let Some(excluded_domains) = opt_not_domains.as_ref() { - if let Some(source_hashes) = request.source_hostname_hashes.as_ref() { - if source_hashes.iter().any(|h| { + if let Some(hashes) = hostname_hashes { + if hashes.iter().any(|h| { mapping .get(h) .is_some_and(|index| utils::bin_lookup(excluded_domains, *index)) }) { return false; } - } else { - // If there are domain restrictions but no source hostname - // (i.e. about:blank), apply the rule anyway. - return true; } } - true } +#[inline] +pub fn check_included_domains_mapped( + opt_domains: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, +) -> bool { + check_included_domains_mapped_with_hashes( + opt_domains, + request.source_hostname_hashes.as_deref(), + mapping, + ) +} + +#[inline] +pub fn check_excluded_domains_mapped( + opt_not_domains: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, +) -> bool { + check_excluded_domains_mapped_with_hashes( + opt_not_domains, + request.source_hostname_hashes.as_deref(), + mapping, + ) +} + +#[inline] +pub fn check_included_to_options_mapped( + opt_to_domains: Option<&[u32]>, + opt_to_entities: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, +) -> bool { + let plain = opt_to_domains.filter(|domains| !domains.is_empty()); + let entity = opt_to_entities.filter(|domains| !domains.is_empty()); + match (plain, entity) { + (None, None) => true, + (Some(included), None) => check_included_domains_mapped_with_hashes( + Some(included), + request.hostname_hashes.as_deref(), + mapping, + ), + (None, Some(included)) => check_included_domains_mapped_with_hashes( + Some(included), + request.entity_hashes.as_deref(), + mapping, + ), + (Some(included_plain), Some(included_entity)) => { + check_included_domains_mapped_with_hashes( + Some(included_plain), + request.hostname_hashes.as_deref(), + mapping, + ) || check_included_domains_mapped_with_hashes( + Some(included_entity), + request.entity_hashes.as_deref(), + mapping, + ) + } + } +} + +#[inline] +pub fn check_excluded_to_options_mapped( + opt_not_to_domains: Option<&[u32]>, + opt_not_to_entities: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, +) -> bool { + check_excluded_domains_mapped_with_hashes( + opt_not_to_domains, + request.hostname_hashes.as_deref(), + mapping, + ) && check_excluded_domains_mapped_with_hashes( + opt_not_to_entities, + request.entity_hashes.as_deref(), + mapping, + ) +} + #[cfg(test)] #[path = "../../tests/unit/filters/network_matchers.rs"] mod unit_tests; diff --git a/src/flatbuffers/fb_network_filter.fbs b/src/flatbuffers/fb_network_filter.fbs index 599847d0..d2a8e404 100644 --- a/src/flatbuffers/fb_network_filter.fbs +++ b/src/flatbuffers/fb_network_filter.fbs @@ -17,6 +17,10 @@ table NetworkFilter { /// typically aren’t many unique hashes opt_domains: [uint32]; opt_not_domains: [uint32]; + opt_to_domains: [uint32]; + opt_not_to_domains: [uint32]; + opt_to_entities: [uint32]; + opt_not_to_entities: [uint32]; // A union-like storage for pattern. Prefer |single_pattern| where // possible for performance concerns. diff --git a/src/flatbuffers/fb_network_filter_generated.rs b/src/flatbuffers/fb_network_filter_generated.rs index df1cc3fb..b93e5490 100644 --- a/src/flatbuffers/fb_network_filter_generated.rs +++ b/src/flatbuffers/fb_network_filter_generated.rs @@ -29,7 +29,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -38,12 +38,16 @@ pub mod fb { pub const VT_MASK: flatbuffers::VOffsetT = 4; pub const VT_OPT_DOMAINS: flatbuffers::VOffsetT = 6; pub const VT_OPT_NOT_DOMAINS: flatbuffers::VOffsetT = 8; - pub const VT_SINGLE_PATTERN: flatbuffers::VOffsetT = 10; - pub const VT_MULTI_PATTERNS: flatbuffers::VOffsetT = 12; - pub const VT_MODIFIER_OPTION: flatbuffers::VOffsetT = 14; - pub const VT_HOSTNAME: flatbuffers::VOffsetT = 16; - pub const VT_TAG: flatbuffers::VOffsetT = 18; - pub const VT_RAW_LINE: flatbuffers::VOffsetT = 20; + pub const VT_OPT_TO_DOMAINS: flatbuffers::VOffsetT = 10; + pub const VT_OPT_NOT_TO_DOMAINS: flatbuffers::VOffsetT = 12; + pub const VT_OPT_TO_ENTITIES: flatbuffers::VOffsetT = 14; + pub const VT_OPT_NOT_TO_ENTITIES: flatbuffers::VOffsetT = 16; + pub const VT_SINGLE_PATTERN: flatbuffers::VOffsetT = 18; + pub const VT_MULTI_PATTERNS: flatbuffers::VOffsetT = 20; + pub const VT_MODIFIER_OPTION: flatbuffers::VOffsetT = 22; + pub const VT_HOSTNAME: flatbuffers::VOffsetT = 24; + pub const VT_TAG: flatbuffers::VOffsetT = 26; + pub const VT_RAW_LINE: flatbuffers::VOffsetT = 28; #[inline] pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { @@ -78,6 +82,18 @@ pub mod fb { if let Some(x) = args.single_pattern { builder.add_single_pattern(x); } + if let Some(x) = args.opt_not_to_entities { + builder.add_opt_not_to_entities(x); + } + if let Some(x) = args.opt_to_entities { + builder.add_opt_to_entities(x); + } + if let Some(x) = args.opt_not_to_domains { + builder.add_opt_not_to_domains(x); + } + if let Some(x) = args.opt_to_domains { + builder.add_opt_to_domains(x); + } if let Some(x) = args.opt_not_domains { builder.add_opt_not_domains(x); } @@ -92,6 +108,10 @@ pub mod fb { let mask = self.mask(); let opt_domains = self.opt_domains().map(|x| x.into_iter().collect()); let opt_not_domains = self.opt_not_domains().map(|x| x.into_iter().collect()); + let opt_to_domains = self.opt_to_domains().map(|x| x.into_iter().collect()); + let opt_not_to_domains = self.opt_not_to_domains().map(|x| x.into_iter().collect()); + let opt_to_entities = self.opt_to_entities().map(|x| x.into_iter().collect()); + let opt_not_to_entities = self.opt_not_to_entities().map(|x| x.into_iter().collect()); let single_pattern = self.single_pattern().map(|x| x.to_string()); let multi_patterns = self .multi_patterns() @@ -104,6 +124,10 @@ pub mod fb { mask, opt_domains, opt_not_domains, + opt_to_domains, + opt_not_to_domains, + opt_to_entities, + opt_not_to_entities, single_pattern, multi_patterns, modifier_option, @@ -154,6 +178,58 @@ pub mod fb { } } #[inline] + pub fn opt_to_domains(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + NetworkFilter::VT_OPT_TO_DOMAINS, + None, + ) + } + } + #[inline] + pub fn opt_not_to_domains(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + NetworkFilter::VT_OPT_NOT_TO_DOMAINS, + None, + ) + } + } + #[inline] + pub fn opt_to_entities(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + NetworkFilter::VT_OPT_TO_ENTITIES, + None, + ) + } + } + #[inline] + pub fn opt_not_to_entities(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + NetworkFilter::VT_OPT_NOT_TO_ENTITIES, + None, + ) + } + } + #[inline] pub fn single_pattern(&self) -> Option<&'a str> { // Safety: // Created from valid Table for this object @@ -241,6 +317,26 @@ pub mod fb { Self::VT_OPT_NOT_DOMAINS, false, )? + .visit_field::>>( + "opt_to_domains", + Self::VT_OPT_TO_DOMAINS, + false, + )? + .visit_field::>>( + "opt_not_to_domains", + Self::VT_OPT_NOT_TO_DOMAINS, + false, + )? + .visit_field::>>( + "opt_to_entities", + Self::VT_OPT_TO_ENTITIES, + false, + )? + .visit_field::>>( + "opt_not_to_entities", + Self::VT_OPT_NOT_TO_ENTITIES, + false, + )? .visit_field::>( "single_pattern", Self::VT_SINGLE_PATTERN, @@ -273,6 +369,10 @@ pub mod fb { pub mask: u32, pub opt_domains: Option>>, pub opt_not_domains: Option>>, + pub opt_to_domains: Option>>, + pub opt_not_to_domains: Option>>, + pub opt_to_entities: Option>>, + pub opt_not_to_entities: Option>>, pub single_pattern: Option>, pub multi_patterns: Option< flatbuffers::WIPOffset>>, @@ -289,6 +389,10 @@ pub mod fb { mask: 540221439, opt_domains: None, opt_not_domains: None, + opt_to_domains: None, + opt_not_to_domains: None, + opt_to_entities: None, + opt_not_to_entities: None, single_pattern: None, multi_patterns: None, modifier_option: None, @@ -330,6 +434,46 @@ pub mod fb { ); } #[inline] + pub fn add_opt_to_domains( + &mut self, + opt_to_domains: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_OPT_TO_DOMAINS, + opt_to_domains, + ); + } + #[inline] + pub fn add_opt_not_to_domains( + &mut self, + opt_not_to_domains: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_OPT_NOT_TO_DOMAINS, + opt_not_to_domains, + ); + } + #[inline] + pub fn add_opt_to_entities( + &mut self, + opt_to_entities: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_OPT_TO_ENTITIES, + opt_to_entities, + ); + } + #[inline] + pub fn add_opt_not_to_entities( + &mut self, + opt_not_to_entities: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_OPT_NOT_TO_ENTITIES, + opt_not_to_entities, + ); + } + #[inline] pub fn add_single_pattern(&mut self, single_pattern: flatbuffers::WIPOffset<&'b str>) { self.fbb_.push_slot_always::>( NetworkFilter::VT_SINGLE_PATTERN, @@ -397,6 +541,10 @@ pub mod fb { ds.field("mask", &self.mask()); ds.field("opt_domains", &self.opt_domains()); ds.field("opt_not_domains", &self.opt_not_domains()); + ds.field("opt_to_domains", &self.opt_to_domains()); + ds.field("opt_not_to_domains", &self.opt_not_to_domains()); + ds.field("opt_to_entities", &self.opt_to_entities()); + ds.field("opt_not_to_entities", &self.opt_not_to_entities()); ds.field("single_pattern", &self.single_pattern()); ds.field("multi_patterns", &self.multi_patterns()); ds.field("modifier_option", &self.modifier_option()); @@ -412,6 +560,10 @@ pub mod fb { pub mask: u32, pub opt_domains: Option>, pub opt_not_domains: Option>, + pub opt_to_domains: Option>, + pub opt_not_to_domains: Option>, + pub opt_to_entities: Option>, + pub opt_not_to_entities: Option>, pub single_pattern: Option, pub multi_patterns: Option>, pub modifier_option: Option, @@ -425,6 +577,10 @@ pub mod fb { mask: 540221439, opt_domains: None, opt_not_domains: None, + opt_to_domains: None, + opt_not_to_domains: None, + opt_to_entities: None, + opt_not_to_entities: None, single_pattern: None, multi_patterns: None, modifier_option: None, @@ -442,6 +598,16 @@ pub mod fb { let mask = self.mask; let opt_domains = self.opt_domains.as_ref().map(|x| _fbb.create_vector(x)); let opt_not_domains = self.opt_not_domains.as_ref().map(|x| _fbb.create_vector(x)); + let opt_to_domains = self.opt_to_domains.as_ref().map(|x| _fbb.create_vector(x)); + let opt_not_to_domains = self + .opt_not_to_domains + .as_ref() + .map(|x| _fbb.create_vector(x)); + let opt_to_entities = self.opt_to_entities.as_ref().map(|x| _fbb.create_vector(x)); + let opt_not_to_entities = self + .opt_not_to_entities + .as_ref() + .map(|x| _fbb.create_vector(x)); let single_pattern = self.single_pattern.as_ref().map(|x| _fbb.create_string(x)); let multi_patterns = self.multi_patterns.as_ref().map(|x| { let w: Vec<_> = x.iter().map(|s| _fbb.create_string(s)).collect(); @@ -457,6 +623,10 @@ pub mod fb { mask, opt_domains, opt_not_domains, + opt_to_domains, + opt_not_to_domains, + opt_to_entities, + opt_not_to_entities, single_pattern, multi_patterns, modifier_option, @@ -479,7 +649,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -710,7 +880,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -1037,7 +1207,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -1196,7 +1366,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -2103,7 +2273,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -2421,14 +2591,14 @@ pub mod fb { /// # Safety /// Callers must trust the given bytes do indeed contain a valid `Engine`. pub unsafe fn root_as_engine_unchecked(buf: &[u8]) -> Engine { - unsafe { flatbuffers::root_unchecked::(buf) } + flatbuffers::root_unchecked::(buf) } #[inline] /// Assumes, without verification, that a buffer of bytes contains a size prefixed Engine and returns it. /// # Safety /// Callers must trust the given bytes do indeed contain a valid size prefixed `Engine`. pub unsafe fn size_prefixed_root_as_engine_unchecked(buf: &[u8]) -> Engine { - unsafe { flatbuffers::size_prefixed_root_unchecked::(buf) } + flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] pub fn finish_engine_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index 7d36c762..6f322a9c 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -183,7 +183,7 @@ mod tests { fn deserialization_generate_simple() { let mut engine = Engine::new_with_list_text("ad-banner", Default::default()); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 16556115079021991714; + const EXPECTED_HASH: u64 = 9901869967784992350; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } @@ -193,7 +193,7 @@ mod tests { let mut engine = Engine::new_with_list_text("ad-banner$tag=abc", Default::default()); engine.use_tags(&["abc"]); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 4864047469838009851; + const EXPECTED_HASH: u64 = 2168748959804448596; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } @@ -241,7 +241,7 @@ mod tests { let expected_hash: u64 = if cfg!(feature = "css-validation") { 2977715961033378463 } else { - 14988656075164302061 + 16823792549182954096 }; assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}"); diff --git a/tests/unit/filters/network_matchers.rs b/tests/unit/filters/network_matchers.rs index 7e9612e9..d483f62e 100644 --- a/tests/unit/filters/network_matchers.rs +++ b/tests/unit/filters/network_matchers.rs @@ -382,6 +382,22 @@ mod match_tests { .collect::>() }); + let map_opt_hashes = |domains: Option>, mapping: &mut HashMap| { + domains.map(|domains| { + domains + .iter() + .map(|domain| { + mapping.insert(*domain, *domain as u32); + *domain as u32 + }) + .collect::>() + }) + }; + let opt_to_domains = map_opt_hashes(filter.opt_to_domains.clone(), &mut mapping); + let opt_not_to_domains = map_opt_hashes(filter.opt_not_to_domains.clone(), &mut mapping); + let opt_to_entities = map_opt_hashes(filter.opt_to_entities.clone(), &mut mapping); + let opt_not_to_entities = map_opt_hashes(filter.opt_not_to_entities.clone(), &mut mapping); + super::super::check_options(filter.mask, request) && super::super::check_included_domains_mapped( opt_domains.as_deref(), @@ -393,6 +409,18 @@ mod match_tests { request, &mapping, ) + && super::super::check_included_to_options_mapped( + opt_to_domains.as_deref(), + opt_to_entities.as_deref(), + request, + &mapping, + ) + && super::super::check_excluded_to_options_mapped( + opt_not_to_domains.as_deref(), + opt_not_to_entities.as_deref(), + request, + &mapping, + ) } #[test] @@ -481,6 +509,133 @@ mod match_tests { request::Request::new("https://foo.com/bar", "http://bar.com", "", "").unwrap(); assert!(!check_options(&network_filter, &request)); } + + // opt-to-domain (destination hostname) + { + let network_filter = + NetworkFilter::parse("||foo$to=gstatic.com", true, Default::default()).unwrap(); + let request = + request::Request::new("https://cdn.gstatic.com/foo", "https://example.com", "", "") + .unwrap(); + assert!(check_options(&network_filter, &request)); + } + { + let network_filter = + NetworkFilter::parse("||foo$to=gstatic.com", true, Default::default()).unwrap(); + let request = + request::Request::new("https://foo.com/bar", "https://example.com", "", "") + .unwrap(); + assert!(!check_options(&network_filter, &request)); + } + + // opt-not-to-domain + { + let network_filter = + NetworkFilter::parse("||foo$to=~example.it", true, Default::default()).unwrap(); + let request = + request::Request::new("https://foo.com/bar", "https://example.com", "", "") + .unwrap(); + assert!(check_options(&network_filter, &request)); + } + { + let network_filter = + NetworkFilter::parse("||foo$to=~example.it", true, Default::default()).unwrap(); + let request = + request::Request::new("https://example.it/bar", "https://example.com", "", "") + .unwrap(); + assert!(!check_options(&network_filter, &request)); + } + + // opt-to-entity + { + let network_filter = + NetworkFilter::parse("||foo$to=google.*", true, Default::default()).unwrap(); + let request = request::Request::new( + "https://www.google.co.uk/foo", + "https://example.com", + "", + "", + ) + .unwrap(); + assert!(check_options(&network_filter, &request)); + } + { + let network_filter = + NetworkFilter::parse("||foo$to=google.*", true, Default::default()).unwrap(); + let request = + request::Request::new("https://foo.com/bar", "https://example.com", "", "") + .unwrap(); + assert!(!check_options(&network_filter, &request)); + } + } + + #[test] + fn check_to_option_matches_works() { + fn to_filter_match( + filter: &str, + url: &str, + source: &str, + request_type: &str, + matching: bool, + ) { + let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); + let request = request::Request::new(url, source, request_type, "").unwrap(); + assert!( + network_filter.matches_test(&request) == matching, + "Expected match={matching} for {filter} on {url} (source={source})" + ); + } + + to_filter_match( + "*$script,from=beforeitsnews.com,to=google.*|gstatic.com", + "https://cdn.gstatic.com/foo.js", + "https://beforeitsnews.com/article", + "script", + true, + ); + to_filter_match( + "*$script,from=beforeitsnews.com,to=google.*|gstatic.com", + "https://www.google.co.uk/foo.js", + "https://beforeitsnews.com/article", + "script", + true, + ); + to_filter_match( + "*$script,from=beforeitsnews.com,to=google.*|gstatic.com", + "https://example.com/foo.js", + "https://beforeitsnews.com/article", + "script", + false, + ); + to_filter_match( + "*$script,from=beforeitsnews.com,to=google.*|gstatic.com", + "https://cdn.gstatic.com/foo.js", + "https://other.com/article", + "script", + false, + ); + + to_filter_match( + "||it^$3p,to=~example.it", + "https://tracker.foo.it/path", + "https://example.com/page", + "script", + true, + ); + to_filter_match( + "||it^$3p,to=~example.it", + "https://example.it/path", + "https://example.com/page", + "script", + false, + ); + to_filter_match( + "||it^$3p,to=~example.it", + "https://tracker.foo.it/path", + "https://tracker.foo.it/page", + "script", + false, + ); } #[test] From 955bade946759564e4649b4433c40101fea84214 Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 12 Jun 2026 13:17:22 +0700 Subject: [PATCH 27/28] clippy. --- src/filters/network.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/filters/network.rs b/src/filters/network.rs index f03bb521..bddeb626 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -1201,6 +1201,7 @@ fn write_str_to_hasher(hasher: &mut impl Hasher, s: &str) { hasher.write(s.as_bytes()); } +#[allow(clippy::too_many_arguments)] fn compute_filter_id( modifier_option: Option<&str>, mask: NetworkFilterMask, From b2ccbfb095b46519ef689b7634eb576c5e473da9 Mon Sep 17 00:00:00 2001 From: boocmp Date: Fri, 12 Jun 2026 13:39:36 +0700 Subject: [PATCH 28/28] Perf --- src/filters/fb_network.rs | 153 ++++++++++++++++++++++--- src/filters/filter_data_context.rs | 62 ++++++++++ src/filters/network_matchers.rs | 63 +++++++--- src/request.rs | 153 +++++++++++++++++++------ src/utils.rs | 3 + tests/unit/engine.rs | 2 +- tests/unit/filters/network_matchers.rs | 10 ++ tests/unit/request.rs | 23 +++- 8 files changed, 406 insertions(+), 63 deletions(-) diff --git a/src/filters/fb_network.rs b/src/filters/fb_network.rs index 0a283350..7ae32e25 100644 --- a/src/filters/fb_network.rs +++ b/src/filters/fb_network.rs @@ -89,6 +89,103 @@ impl ExactSizeIterator for FlatPatternsIterator<'_> { } } +#[derive(Clone, Copy, Default)] +pub(crate) struct ToOptionsFlags(u8); + +impl ToOptionsFlags { + const PLAIN_INC: u8 = 1 << 0; + const PLAIN_EXC: u8 = 1 << 1; + const ENTITY_INC: u8 = 1 << 2; + const ENTITY_EXC: u8 = 1 << 3; + + #[inline(always)] + pub fn any(self) -> bool { + self.0 != 0 + } + + #[inline(always)] + fn has_plain_inc(self) -> bool { + self.0 & Self::PLAIN_INC != 0 + } + + #[inline(always)] + fn has_plain_exc(self) -> bool { + self.0 & Self::PLAIN_EXC != 0 + } + + #[inline(always)] + fn has_entity_inc(self) -> bool { + self.0 & Self::ENTITY_INC != 0 + } + + #[inline(always)] + fn has_entity_exc(self) -> bool { + self.0 & Self::ENTITY_EXC != 0 + } + + #[inline(always)] + pub(crate) fn needs_plain(self) -> bool { + self.0 & (Self::PLAIN_INC | Self::PLAIN_EXC) != 0 + } + + #[inline(always)] + pub(crate) fn needs_entity(self) -> bool { + self.0 & (Self::ENTITY_INC | Self::ENTITY_EXC) != 0 + } + + #[cfg(test)] + pub(crate) fn from_bucket_parts( + include_plain: bool, + exclude_plain: bool, + include_entity: bool, + exclude_entity: bool, + ) -> Self { + let mut flags = 0u8; + if include_plain { + flags |= Self::PLAIN_INC; + } + if exclude_plain { + flags |= Self::PLAIN_EXC; + } + if include_entity { + flags |= Self::ENTITY_INC; + } + if exclude_entity { + flags |= Self::ENTITY_EXC; + } + Self(flags) + } + + fn from_filter(filter: &fb::NetworkFilter<'_>) -> Self { + let mut flags = 0u8; + if filter + .opt_to_domains() + .is_some_and(|data| !fb_vector_to_slice(data).is_empty()) + { + flags |= Self::PLAIN_INC; + } + if filter + .opt_not_to_domains() + .is_some_and(|data| !fb_vector_to_slice(data).is_empty()) + { + flags |= Self::PLAIN_EXC; + } + if filter + .opt_to_entities() + .is_some_and(|data| !fb_vector_to_slice(data).is_empty()) + { + flags |= Self::ENTITY_INC; + } + if filter + .opt_not_to_entities() + .is_some_and(|data| !fb_vector_to_slice(data).is_empty()) + { + flags |= Self::ENTITY_EXC; + } + Self(flags) + } +} + /// Internal implementation of [NetworkFilter] that is compatible with flatbuffers. pub(crate) struct FlatNetworkFilter<'a> { key: u64, @@ -96,6 +193,7 @@ pub(crate) struct FlatNetworkFilter<'a> { fb_filter: &'a fb::NetworkFilter<'a>, pub(crate) mask: NetworkFilterMask, + to_options: ToOptionsFlags, } impl<'a> FlatNetworkFilter<'a> { @@ -112,6 +210,7 @@ impl<'a> FlatNetworkFilter<'a> { key, fb_filter: filter, mask: NetworkFilterMask::from_bits_retain(filter.mask()), + to_options: ToOptionsFlags::from_filter(filter), filter_data_context, } } @@ -222,21 +321,45 @@ impl NetworkMatchable for FlatNetworkFilter<'_> { ) { return false; } - if !check_included_to_options_mapped( - self.include_to_domains(), - self.include_to_entities(), - request, - &self.filter_data_context.unique_domains_hashes_map, - ) { - return false; - } - if !check_excluded_to_options_mapped( - self.exclude_to_domains(), - self.exclude_to_entities(), - request, - &self.filter_data_context.unique_domains_hashes_map, - ) { - return false; + if self.to_options.any() { + let mapping = &self.filter_data_context.unique_domains_hashes_map; + let to_capability = self.filter_data_context.to_rule_capability(); + if !check_included_to_options_mapped( + self.to_options, + if self.to_options.has_plain_inc() { + self.include_to_domains() + } else { + None + }, + if self.to_options.has_entity_inc() { + self.include_to_entities() + } else { + None + }, + request, + mapping, + to_capability, + ) { + return false; + } + if !check_excluded_to_options_mapped( + self.to_options, + if self.to_options.has_plain_exc() { + self.exclude_to_domains() + } else { + None + }, + if self.to_options.has_entity_exc() { + self.exclude_to_entities() + } else { + None + }, + request, + mapping, + to_capability, + ) { + return false; + } } check_pattern( self.mask, diff --git a/src/filters/filter_data_context.rs b/src/filters/filter_data_context.rs index 985ef1eb..e4ca1feb 100644 --- a/src/filters/filter_data_context.rs +++ b/src/filters/filter_data_context.rs @@ -1,3 +1,4 @@ +use crate::filters::flatbuffer_generated::fb; use crate::flatbuffers::unsafe_tools::VerifiedFlatbufferMemory; use crate::utils::Hash; use std::collections::HashMap; @@ -10,9 +11,67 @@ pub(crate) type FilterDataContextRef = std::sync::Arc; // The struct is used to store the flatbuffer and supporting data // for both network filter and cosmetic filters. // Supposed to be stored via FilterDataContextRef to avoid copying the data. +#[derive(Clone, Copy)] +pub(crate) struct ToRuleCapability { + pub has_plain: bool, + pub has_entity: bool, +} + +impl ToRuleCapability { + #[inline(always)] + #[cfg(test)] + pub const fn all() -> Self { + Self { + has_plain: true, + has_entity: true, + } + } +} + pub(crate) struct FilterDataContext { pub(crate) memory: VerifiedFlatbufferMemory, pub(crate) unique_domains_hashes_map: HashMap, + pub(crate) has_to_plain_rules: bool, + pub(crate) has_to_entity_rules: bool, +} + +impl FilterDataContext { + #[inline(always)] + pub(crate) fn to_rule_capability(&self) -> ToRuleCapability { + ToRuleCapability { + has_plain: self.has_to_plain_rules, + has_entity: self.has_to_entity_rules, + } + } +} + +fn non_empty_fb_slice(data: flatbuffers::Vector<'_, u32>) -> bool { + !data.is_empty() +} + +fn scan_to_option_flags(root: fb::Engine<'_>) -> (bool, bool) { + let mut has_to_plain_rules = false; + let mut has_to_entity_rules = false; + + for list in root.network_rules().iter() { + for filter in list.filter_map_values().iter() { + if filter.opt_to_domains().is_some_and(non_empty_fb_slice) + || filter.opt_not_to_domains().is_some_and(non_empty_fb_slice) + { + has_to_plain_rules = true; + } + if filter.opt_to_entities().is_some_and(non_empty_fb_slice) + || filter.opt_not_to_entities().is_some_and(non_empty_fb_slice) + { + has_to_entity_rules = true; + } + if has_to_plain_rules && has_to_entity_rules { + return (true, true); + } + } + } + + (has_to_plain_rules, has_to_entity_rules) } impl FilterDataContext { @@ -23,9 +82,12 @@ impl FilterDataContext { for (index, hash) in root.unique_domains_hashes().iter().enumerate() { unique_domains_hashes_map.insert(hash, index as u32); } + let (has_to_plain_rules, has_to_entity_rules) = scan_to_option_flags(root); FilterDataContextRef::new(Self { memory, unique_domains_hashes_map, + has_to_plain_rules, + has_to_entity_rules, }) } } diff --git a/src/filters/network_matchers.rs b/src/filters/network_matchers.rs index ffca5e88..abfe2416 100644 --- a/src/filters/network_matchers.rs +++ b/src/filters/network_matchers.rs @@ -7,6 +7,8 @@ use memchr::memmem; +use crate::filters::fb_network::ToOptionsFlags; +use crate::filters::filter_data_context::ToRuleCapability; use crate::filters::network::{NetworkFilterMask, NetworkFilterMaskHelper}; use crate::regex_manager::RegexManager; use crate::request; @@ -490,33 +492,53 @@ pub fn check_excluded_domains_mapped( #[inline] pub fn check_included_to_options_mapped( + to_options: ToOptionsFlags, opt_to_domains: Option<&[u32]>, opt_to_entities: Option<&[u32]>, request: &request::Request, mapping: &HashMap, + to_capability: ToRuleCapability, ) -> bool { + if !to_options.any() { + return true; + } + let plain = opt_to_domains.filter(|domains| !domains.is_empty()); let entity = opt_to_entities.filter(|domains| !domains.is_empty()); + + let destination_suffix_hashes = + if to_options.needs_plain() && to_capability.has_plain && plain.is_some() { + request.destination_suffix_hashes() + } else { + None + }; + let destination_entity_hashes = + if to_options.needs_entity() && to_capability.has_entity && entity.is_some() { + request.destination_entity_hashes() + } else { + None + }; + match (plain, entity) { (None, None) => true, (Some(included), None) => check_included_domains_mapped_with_hashes( Some(included), - request.hostname_hashes.as_deref(), + destination_suffix_hashes, mapping, ), (None, Some(included)) => check_included_domains_mapped_with_hashes( Some(included), - request.entity_hashes.as_deref(), + destination_entity_hashes, mapping, ), (Some(included_plain), Some(included_entity)) => { check_included_domains_mapped_with_hashes( Some(included_plain), - request.hostname_hashes.as_deref(), + destination_suffix_hashes, mapping, ) || check_included_domains_mapped_with_hashes( Some(included_entity), - request.entity_hashes.as_deref(), + destination_entity_hashes, mapping, ) } @@ -525,20 +547,35 @@ pub fn check_included_to_options_mapped( #[inline] pub fn check_excluded_to_options_mapped( + to_options: ToOptionsFlags, opt_not_to_domains: Option<&[u32]>, opt_not_to_entities: Option<&[u32]>, request: &request::Request, mapping: &HashMap, + to_capability: ToRuleCapability, ) -> bool { - check_excluded_domains_mapped_with_hashes( - opt_not_to_domains, - request.hostname_hashes.as_deref(), - mapping, - ) && check_excluded_domains_mapped_with_hashes( - opt_not_to_entities, - request.entity_hashes.as_deref(), - mapping, - ) + if !to_options.any() { + return true; + } + + let plain = opt_not_to_domains.filter(|domains| !domains.is_empty()); + let entity = opt_not_to_entities.filter(|domains| !domains.is_empty()); + + let destination_suffix_hashes = + if to_options.needs_plain() && to_capability.has_plain && plain.is_some() { + request.destination_suffix_hashes() + } else { + None + }; + let destination_entity_hashes = + if to_options.needs_entity() && to_capability.has_entity && entity.is_some() { + request.destination_entity_hashes() + } else { + None + }; + + check_excluded_domains_mapped_with_hashes(plain, destination_suffix_hashes, mapping) + && check_excluded_domains_mapped_with_hashes(entity, destination_entity_hashes, mapping) } #[cfg(test)] diff --git a/src/request.rs b/src/request.rs index 450fcb3c..22602d29 100644 --- a/src/request.rs +++ b/src/request.rs @@ -1,10 +1,12 @@ //! Contains structures needed to describe network requests. +use std::sync::OnceLock; + use thiserror::Error; use crate::filters::cosmetic::get_entity_hashes_from_labels; use crate::url_parser; -use crate::utils; +use crate::utils::{self, HostnameHashBuffer}; #[derive(Clone, Copy, PartialEq, Debug)] pub enum RequestMethod { @@ -90,7 +92,7 @@ fn cpt_match_type(cpt: &str) -> RequestType { } /// A network [`Request`], used as an interface for network blocking in the [`crate::Engine`]. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct Request { pub request_type: RequestType, pub method: Option, @@ -101,15 +103,49 @@ pub struct Request { pub is_third_party: bool, pub url: String, pub hostname: String, + pub(crate) domain: String, + pub(crate) source_hostname: String, pub source_hostname_hashes: Option>, - pub(crate) hostname_hashes: Option>, - pub(crate) entity_hashes: Option>, + + destination_suffix_hashes: OnceLock, + destination_entity_hashes: OnceLock, pub(crate) url_lower_cased: String, pub(crate) request_tokens: Vec, pub(crate) original_url: String, } +impl Clone for Request { + fn clone(&self) -> Self { + Self { + request_type: self.request_type.clone(), + method: self.method, + is_http: self.is_http, + is_https: self.is_https, + is_supported: self.is_supported, + is_third_party: self.is_third_party, + url: self.url.clone(), + hostname: self.hostname.clone(), + domain: self.domain.clone(), + source_hostname: self.source_hostname.clone(), + source_hostname_hashes: self.source_hostname_hashes.clone(), + destination_suffix_hashes: clone_once_lock(&self.destination_suffix_hashes), + destination_entity_hashes: clone_once_lock(&self.destination_entity_hashes), + url_lower_cased: self.url_lower_cased.clone(), + request_tokens: self.request_tokens.clone(), + original_url: self.original_url.clone(), + } + } +} + +fn clone_once_lock(src: &OnceLock) -> OnceLock { + let lock = OnceLock::new(); + if let Some(buffer) = src.get() { + let _ = lock.set(buffer.clone()); + } + lock +} + impl Request { pub(crate) fn get_url(&self, case_sensitive: bool) -> &str { if case_sensitive { @@ -133,6 +169,76 @@ impl Request { &self.request_tokens } + /// Lazily computed destination suffix hashes for `$to=` plain matching. + pub(crate) fn destination_suffix_hashes(&self) -> Option<&[utils::Hash]> { + if self.hostname.is_empty() { + return None; + } + Some( + self.destination_suffix_hashes + .get_or_init(|| self.compute_destination_suffix_hashes()) + .as_slice(), + ) + } + + /// Lazily computed destination entity hashes for `$to=google.*` style matching. + pub(crate) fn destination_entity_hashes(&self) -> Option<&[utils::Hash]> { + if self.hostname.is_empty() { + return None; + } + let buffer = self.destination_entity_hashes.get_or_init(|| { + let mut buffer = HostnameHashBuffer::new(); + for hash in get_entity_hashes_from_labels(&self.hostname, &self.domain) { + if buffer.try_push(hash).is_err() { + break; + } + } + buffer + }); + if buffer.is_empty() { + None + } else { + Some(buffer.as_slice()) + } + } + + fn compute_destination_suffix_hashes(&self) -> HostnameHashBuffer { + let mut buffer = HostnameHashBuffer::new(); + if self.hostname.is_empty() { + return buffer; + } + if self.hostname == self.source_hostname { + if let Some(source_hashes) = self.source_hostname_hashes.as_ref() { + for &hash in source_hashes { + if buffer.try_push(hash).is_err() { + break; + } + } + return buffer; + } + } + Self::push_suffix_hostname_hashes(&self.hostname, &mut buffer); + buffer + } + + fn push_suffix_hostname_hashes(hostname: &str, buffer: &mut HostnameHashBuffer) { + let _ = buffer.try_push(utils::fast_hash(hostname)); + for (i, c) in hostname.char_indices() { + if c == '.' && i + 1 < hostname.len() { + let _ = buffer.try_push(utils::fast_hash(&hostname[i + 1..])); + } + } + } + + fn suffix_hostname_hashes(hostname: &str) -> Option> { + if hostname.is_empty() { + return None; + } + let mut buffer = HostnameHashBuffer::new(); + Self::push_suffix_hostname_hashes(hostname, &mut buffer); + Some(buffer.into_iter().collect()) + } + fn parse_method(raw_method: &str) -> Option { if raw_method.is_empty() { return None; @@ -158,22 +264,8 @@ impl Request { }) } - fn suffix_hostname_hashes(hostname: &str) -> Option> { - if hostname.is_empty() { - return None; - } - let mut hashes = Vec::with_capacity(4); - hashes.push(utils::fast_hash(hostname)); - for (i, c) in hostname.char_indices() { - if c == '.' && i + 1 < hostname.len() { - hashes.push(utils::fast_hash(&hostname[i + 1..])); - } - } - Some(hashes) - } - #[allow(clippy::too_many_arguments)] - fn from_detailed_parameters( + pub(crate) fn from_detailed_parameters( raw_type: &str, url: &str, schema: &str, @@ -209,17 +301,6 @@ impl Request { } let source_hostname_hashes = Self::suffix_hostname_hashes(source_hostname); - let hostname_hashes = Self::suffix_hostname_hashes(hostname); - let entity_hashes = if hostname.is_empty() { - None - } else { - let hashes = get_entity_hashes_from_labels(hostname, domain); - if hashes.is_empty() { - None - } else { - Some(hashes) - } - }; let url_lower_cased = url.to_ascii_lowercase(); @@ -229,10 +310,12 @@ impl Request { url: url.to_owned(), url_lower_cased: url_lower_cased.to_owned(), hostname: hostname.to_owned(), + domain: domain.to_owned(), + source_hostname: source_hostname.to_owned(), request_tokens: calculate_tokens(&url_lower_cased), source_hostname_hashes, - hostname_hashes, - entity_hashes, + destination_suffix_hashes: OnceLock::new(), + destination_entity_hashes: OnceLock::new(), is_third_party: third_party, is_http, is_https, @@ -312,6 +395,12 @@ impl Request { Self::parse_method(method), ) } + + #[cfg(test)] + pub(crate) fn destination_hashes_initialized(&self) -> bool { + self.destination_suffix_hashes.get().is_some() + || self.destination_entity_hashes.get().is_some() + } } fn calculate_tokens(url_lower_cased: &str) -> Vec { diff --git a/src/utils.rs b/src/utils.rs index 6930ac11..b80cc79e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -33,6 +33,9 @@ fn is_allowed_filter(ch: char) -> bool { /// Used instread of Vec to avoid heap allocations. pub(crate) type TokensBuffer = ArrayVec; +/// Suffix-chain and entity hashes for a hostname (typically fewer than a dozen labels). +pub(crate) type HostnameHashBuffer = ArrayVec; + fn fast_tokenizer_no_regex( pattern: &str, is_allowed_code: &dyn Fn(char) -> bool, diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index 6f322a9c..d1444f5b 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -239,7 +239,7 @@ mod tests { ); } let expected_hash: u64 = if cfg!(feature = "css-validation") { - 2977715961033378463 + 14819081262113618355 } else { 16823792549182954096 }; diff --git a/tests/unit/filters/network_matchers.rs b/tests/unit/filters/network_matchers.rs index d483f62e..178dc02f 100644 --- a/tests/unit/filters/network_matchers.rs +++ b/tests/unit/filters/network_matchers.rs @@ -397,6 +397,12 @@ mod match_tests { let opt_not_to_domains = map_opt_hashes(filter.opt_not_to_domains.clone(), &mut mapping); let opt_to_entities = map_opt_hashes(filter.opt_to_entities.clone(), &mut mapping); let opt_not_to_entities = map_opt_hashes(filter.opt_not_to_entities.clone(), &mut mapping); + let to_options = crate::filters::fb_network::ToOptionsFlags::from_bucket_parts( + opt_to_domains.is_some(), + opt_not_to_domains.is_some(), + opt_to_entities.is_some(), + opt_not_to_entities.is_some(), + ); super::super::check_options(filter.mask, request) && super::super::check_included_domains_mapped( @@ -410,16 +416,20 @@ mod match_tests { &mapping, ) && super::super::check_included_to_options_mapped( + to_options, opt_to_domains.as_deref(), opt_to_entities.as_deref(), request, &mapping, + crate::filters::filter_data_context::ToRuleCapability::all(), ) && super::super::check_excluded_to_options_mapped( + to_options, opt_not_to_domains.as_deref(), opt_not_to_entities.as_deref(), request, &mapping, + crate::filters::filter_data_context::ToRuleCapability::all(), ) } diff --git a/tests/unit/request.rs b/tests/unit/request.rs index 635134af..6fff6771 100644 --- a/tests/unit/request.rs +++ b/tests/unit/request.rs @@ -160,8 +160,9 @@ mod tests { "example.com", "example.com", ); + assert!(!request.destination_hashes_initialized()); assert_eq!( - request.hostname_hashes.as_ref().unwrap().as_slice(), + request.destination_suffix_hashes().unwrap(), tokenize(&["cdn.gstatic.com", "gstatic.com", "com"], &[]).as_slice(), ); } @@ -177,10 +178,28 @@ mod tests { "example.com", "example.com", ); - let entity_hashes = request.entity_hashes.as_ref().unwrap(); + assert!(!request.destination_hashes_initialized()); + let entity_hashes = request.destination_entity_hashes().unwrap(); assert!(entity_hashes.contains(&utils::fast_hash("google"))); } + #[test] + fn destination_suffix_hashes_shared_with_source() { + let request = build_request( + "script", + "https://example.com/foo.js", + "https", + "example.com", + "example.com", + "example.com", + "example.com", + ); + assert_eq!( + request.destination_suffix_hashes().unwrap(), + request.source_hostname_hashes.as_ref().unwrap().as_slice(), + ); + } + #[test] fn parses_urls() { let parsed = Request::new(