diff --git a/benches/bench_cosmetic_matching.rs b/benches/bench_cosmetic_matching.rs index 7e226a50..0a735716 100644 --- a/benches/bench_cosmetic_matching.rs +++ b/benches/bench_cosmetic_matching.rs @@ -7,10 +7,10 @@ mod test_utils; pub fn make_engine() -> Engine { use adblock::resources::Resource; - let rules = test_utils::rules_from_lists(&["data/brave/brave-main-list.txt"]); + let rules = test_utils::rules_from_lists(["data/brave/brave-main-list.txt"]); let resource_json = std::fs::read_to_string("data/brave/brave-resources.json").unwrap(); let resource_list: Vec = serde_json::from_str(&resource_json).unwrap(); - let mut engine = Engine::from_rules_parametrised(rules, Default::default(), true, true); + let mut engine = Engine::new_with_list_text_parametrised(rules, Default::default(), true, true); engine.use_resources(resource_list); engine } diff --git a/benches/bench_matching.rs b/benches/bench_matching.rs index 4a1f3287..6cc0d2e0 100644 --- a/benches/bench_matching.rs +++ b/benches/bench_matching.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use adblock::request::Request; use adblock::url_parser::parse_url; -use adblock::{Engine, FilterSet}; +use adblock::Engine; #[path = "../tests/test_utils.rs"] mod test_utils; @@ -20,30 +20,20 @@ struct TestRequest { impl From<&TestRequest> for Request { fn from(v: &TestRequest) -> Self { - Request::new(&v.url, &v.frameUrl, &v.cpt).unwrap() + Request::new(&v.url, &v.frameUrl, &v.cpt, "").unwrap() } } fn load_requests() -> Vec { - let requests_str = rules_from_lists(&["data/requests.json"]); + let requests_str = rules_from_lists(["data/requests.json"]); let reqs: Vec = requests_str - .into_iter() - .map(|r| serde_json::from_str(&r)) + .lines() + .map(serde_json::from_str) .filter_map(Result::ok) .collect(); reqs } -fn get_engine(rules: impl IntoIterator>) -> Engine { - let (network_filters, cosmetic_filters) = - adblock::lists::parse_filters(rules, false, Default::default()); - - Engine::from_filter_set( - FilterSet::new_with_rules(network_filters, cosmetic_filters, false), - true, - ) -} - fn bench_rule_matching(engine: &Engine, requests: &[TestRequest]) -> (u32, u32) { let mut matches = 0; let mut passes = 0; @@ -87,6 +77,7 @@ fn bench_rule_matching_browserlike(blocker: &Engine, requests: &[ParsedRequest]) source_hostname, request_type, *third_party, + "", )); if check.matched { matches += 1; @@ -112,21 +103,21 @@ fn rule_match(c: &mut Criterion) { group.sample_size(10); group.bench_function("el+ep", move |b| { - let rules = rules_from_lists(&[ + let rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = Engine::from_rules(rules, Default::default()); + let engine = Engine::new_with_list_text(rules, Default::default()); b.iter(|| bench_rule_matching(&engine, &elep_req)) }); group.bench_function("easylist", move |b| { - let rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); - let engine = Engine::from_rules(rules, Default::default()); + let rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); + let engine = Engine::new_with_list_text(rules, Default::default()); b.iter(|| bench_rule_matching(&engine, &el_req)) }); group.bench_function("slimlist", move |b| { - let rules = rules_from_lists(&["data/slim-list.txt"]); - let engine = Engine::from_rules(rules, Default::default()); + let rules = rules_from_lists(["data/slim-list.txt"]); + let engine = Engine::new_with_list_text(rules, Default::default()); b.iter(|| bench_rule_matching(&engine, &slim_req)) }); @@ -136,15 +127,15 @@ fn rule_match(c: &mut Criterion) { fn rule_match_parsed_el(c: &mut Criterion) { let mut group = c.benchmark_group("rule-match-parsed"); - let rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); + let rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); let requests = load_requests(); let requests_parsed: Vec<_> = requests .into_iter() - .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt)) + .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt, "")) .filter_map(Result::ok) .collect(); let requests_len = requests_parsed.len() as u64; - let engine = get_engine(rules); + let engine = Engine::new_with_list_text(rules, Default::default()); group.throughput(Throughput::Elements(requests_len)); group.sample_size(10); @@ -159,27 +150,27 @@ fn rule_match_parsed_el(c: &mut Criterion) { fn rule_match_parsed_elep_slimlist(c: &mut Criterion) { let mut group = c.benchmark_group("rule-match-parsed"); - let full_rules = rules_from_lists(&[ + let full_rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = get_engine(full_rules); + let engine = Engine::new_with_list_text(full_rules, Default::default()); let requests = load_requests(); let requests_parsed: Vec<_> = requests .into_iter() - .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt)) + .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt, "")) .filter_map(Result::ok) .collect(); let requests_len = requests_parsed.len() as u64; - let slim_rules = rules_from_lists(&["data/slim-list.txt"]); - let slim_engine = get_engine(slim_rules); + let slim_rules = rules_from_lists(["data/slim-list.txt"]); + let slim_engine = Engine::new_with_list_text(slim_rules, Default::default()); let requests_copy = load_requests(); let requests_parsed_copy: Vec<_> = requests_copy .into_iter() - .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt)) + .map(|r| Request::new(&r.url, &r.frameUrl, &r.cpt, "")) .filter_map(Result::ok) .collect(); @@ -245,26 +236,30 @@ fn rule_match_browserlike_comparable(c: &mut Criterion) { let requests = requests_parsed(&requests); group.bench_function("el+ep", |b| { - let rules = rules_from_lists(&[ + let rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); b.iter(|| bench_rule_matching_browserlike(&engine, &requests)) }); group.bench_function("el", |b| { - let rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); - let engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); + let engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); b.iter(|| bench_rule_matching_browserlike(&engine, &requests)) }); group.bench_function("slimlist", |b| { - let rules = rules_from_lists(&["data/slim-list.txt"]); - let engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let rules = rules_from_lists(["data/slim-list.txt"]); + let engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); b.iter(|| bench_rule_matching_browserlike(&engine, &requests)) }); group.bench_function("brave-list", |b| { - let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); - let engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let rules = rules_from_lists(["data/brave/brave-main-list.txt"]); + let engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); b.iter(|| bench_rule_matching_browserlike(&engine, &requests)) }); @@ -288,9 +283,9 @@ fn rule_match_first_request(c: &mut Criterion) { b.iter_custom(|iters| { let mut total_time = std::time::Duration::ZERO; for _ in 0..iters { - let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); + let rules = rules_from_lists(["data/brave/brave-main-list.txt"]); let engine = - Engine::from_rules_parametrised(rules, Default::default(), false, true); + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); // Measure only the matching time, skip setup and destruction let start_time = std::time::Instant::now(); diff --git a/benches/bench_memory.rs b/benches/bench_memory.rs index 1e764414..e4dc1409 100644 --- a/benches/bench_memory.rs +++ b/benches/bench_memory.rs @@ -128,15 +128,15 @@ struct TestRequest { impl From<&TestRequest> for Request { fn from(v: &TestRequest) -> Self { - Request::new(&v.url, &v.frameUrl, &v.cpt).unwrap() + Request::new(&v.url, &v.frameUrl, &v.cpt, "").unwrap() } } fn load_requests() -> Vec { - let requests_str = rules_from_lists(&["data/requests.json"]); + let requests_str = rules_from_lists(["data/requests.json"]); let reqs: Vec = requests_str - .into_iter() - .map(|r| serde_json::from_str(&r)) + .lines() + .map(serde_json::from_str) .filter_map(Result::ok) .collect(); reqs @@ -243,8 +243,8 @@ fn bench_cb( ) { let single_run = || { ALLOCATOR.reset(); - let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); - let mut engine = Engine::from_rules(rules, Default::default()); + let rules = rules_from_lists(["data/brave/brave-main-list.txt"]); + let mut engine = Engine::new_with_list_text(rules, Default::default()); let resource_json = std::fs::read_to_string("data/brave/brave-resources.json").unwrap(); let resource_list: Vec = serde_json::from_str(&resource_json).unwrap(); std::mem::drop(resource_json); diff --git a/benches/bench_redirect_performance.rs b/benches/bench_redirect_performance.rs index 1ef2dbc2..0108467b 100644 --- a/benches/bench_redirect_performance.rs +++ b/benches/bench_redirect_performance.rs @@ -1,8 +1,11 @@ -use adblock::{Engine, FilterSet}; +use std::borrow::Cow; +use std::sync::OnceLock; + +use adblock::Engine; use criterion::*; use tokio::runtime::Runtime; -use adblock::filters::network::{NetworkFilter, NetworkFilterMask, NetworkFilterMaskHelper}; +use adblock::filters::network::{NetworkFilter, NetworkFilterMask}; use adblock::request::Request; use adblock::resources::Resource; @@ -55,23 +58,31 @@ async fn get_all_filters() -> Vec { .collect() } -/// Gets all rules with redirects, and modifies them to apply to resources at `a{0-n}.com/bad.js` -fn get_redirect_rules() -> Vec { - let async_runtime = Runtime::new().expect("Could not start Tokio runtime"); +static ALL_FILTERS: OnceLock> = OnceLock::new(); - let filters = async_runtime.block_on(get_all_filters()); - let (network_filters, _) = adblock::lists::parse_filters(&filters, true, Default::default()); +fn all_filters() -> &'static [String] { + ALL_FILTERS.get_or_init(|| { + let async_runtime = Runtime::new().expect("Could not start Tokio runtime"); + async_runtime.block_on(get_all_filters()).into_boxed_slice() + }) +} + +/// Gets all rules with redirects, and modifies them to apply to resources at `a{0-n}.com/bad.js` +fn get_redirect_rules() -> Vec> { + let filters = all_filters(); + let (network_filters, _) = + adblock::lists::parse_filters(filters.iter().map(|s| s.as_str()), true, Default::default()); network_filters .into_iter() .filter(NetworkFilter::is_redirect) .filter(NetworkFilter::also_block_redirect) - .filter(|rule| rule.modifier_option.as_ref().unwrap() != "none") + .filter(|rule| rule.modifier_option.unwrap() != "none") .enumerate() .map(|(index, mut rule)| { rule.mask.insert(NetworkFilterMask::IS_LEFT_ANCHOR); rule.mask.insert(NetworkFilterMask::IS_RIGHT_ANCHOR); - rule.hostname = Some(format!("a{index}.com/bad.js")); + rule.hostname = Some(Cow::Owned(format!("a{index}.com/bad.js"))); rule.filter = adblock::filters::network::FilterPart::Empty; rule.mask.remove(NetworkFilterMask::IS_HOSTNAME_ANCHOR); @@ -86,8 +97,7 @@ fn get_redirect_rules() -> Vec { /// Loads the supplied rules, and the test set of resources, into a Engine fn get_preloaded_engine(rules: Vec) -> Engine { - let filter_set = FilterSet::new_with_rules(rules, vec![], false); - Engine::from_filter_set(filter_set, true /* optimize */) + Engine::new_with_parsed_rules(rules, vec![], true) } fn get_resources_for_filters(#[allow(unused)] filters: &[NetworkFilter]) -> Vec { @@ -118,17 +128,17 @@ fn get_resources_for_filters(#[allow(unused)] filters: &[NetworkFilter]) -> Vec< .iter() .filter(|f| f.is_redirect()) .map(|f| { - let mut redirect = f.modifier_option.as_ref().unwrap().as_str(); + let mut redirect = f.modifier_option.as_ref().unwrap().to_string(); // strip priority, if present if let Some(i) = redirect.rfind(':') { - redirect = &redirect[0..i]; + redirect.truncate(i); } Resource { - name: redirect.to_owned(), + name: redirect.clone(), aliases: vec![], - kind: ResourceType::Mime(MimeType::from_extension(redirect)), - content: BASE64_STANDARD.encode(redirect), + kind: ResourceType::Mime(MimeType::from_extension(&redirect)), + content: BASE64_STANDARD.encode(&redirect), dependencies: vec![], permission: Default::default(), } @@ -175,7 +185,7 @@ pub fn build_custom_requests(rules: Vec) -> Vec { let domain = &rule_hostname[..rule_hostname.find('/').unwrap()]; let hostname = domain; - let raw_line = rule.raw_line.clone().unwrap(); + let raw_line = rule.raw_line.as_deref().unwrap().to_string(); let source_hostname = if rule.opt_domains.is_some() { let domain_start = raw_line.rfind("domain=").unwrap() + "domain=".len(); let from_start = &raw_line[domain_start..]; @@ -194,7 +204,7 @@ pub fn build_custom_requests(rules: Vec) -> Vec { let source_url = format!("https://{source_hostname}"); - Request::new(&url, &source_url, raw_type).unwrap() + Request::new(&url, &source_url, raw_type, "").unwrap() }) .collect::>() } diff --git a/benches/bench_rules.rs b/benches/bench_rules.rs index e4c35f72..c753f631 100644 --- a/benches/bench_rules.rs +++ b/benches/bench_rules.rs @@ -1,26 +1,26 @@ use criterion::*; use std::sync::LazyLock; -use adblock::{Engine, FilterSet}; +use adblock::Engine; #[path = "../tests/test_utils.rs"] mod test_utils; use test_utils::rules_from_lists; -static DEFAULT_LISTS: LazyLock> = - LazyLock::new(|| rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]).collect()); +static DEFAULT_LISTS: LazyLock = + LazyLock::new(|| rules_from_lists(["data/easylist.to/easylist/easylist.txt"])); -fn bench_string_hashing(filters: &Vec) -> adblock::utils::Hash { +fn bench_string_hashing(filters: &str) -> adblock::utils::Hash { let mut dummy: adblock::utils::Hash = 0; - for filter in filters { + for filter in filters.lines() { dummy = (dummy + adblock::utils::fast_hash(filter)) % 1000000000; } dummy } -fn bench_string_tokenize(filters: &Vec) -> usize { +fn bench_string_tokenize(filters: &str) -> usize { let mut dummy: usize = 0; - for filter in filters { + for filter in filters.lines() { dummy = (dummy + adblock::utils::tokenize(filter).len()) % 1000000000; } dummy @@ -50,15 +50,10 @@ fn string_tokenize(c: &mut Criterion) { group.finish(); } -fn bench_parsing_impl(lists: &Vec<&Vec>) -> usize { - let mut dummy = 0; - - for list in lists { - let (network_filters, _) = adblock::lists::parse_filters(*list, false, Default::default()); - dummy += network_filters.len() % 1000000; - } - - dummy +fn bench_parsing_impl(lists: &str) -> usize { + let (network_filters, _) = + adblock::lists::parse_filters(lists.lines(), false, Default::default()); + network_filters.len() % 1000000 } fn list_parse(c: &mut Criterion) { @@ -68,44 +63,35 @@ fn list_parse(c: &mut Criterion) { group.sample_size(10); group.bench_function("network filters", |b| { - b.iter(|| bench_parsing_impl(&vec![DEFAULT_LISTS.as_ref()])) + b.iter(|| bench_parsing_impl(&DEFAULT_LISTS)) }); group.bench_function("all filters", |b| { - b.iter(|| bench_parsing_impl(&vec![DEFAULT_LISTS.as_ref()])) + b.iter(|| bench_parsing_impl(&DEFAULT_LISTS)) }); group.finish(); } -fn get_engine(rules: impl IntoIterator>) -> Engine { - let (network_filters, cosmetic_filters) = - adblock::lists::parse_filters(rules, false, Default::default()); - - Engine::from_filter_set( - FilterSet::new_with_rules(network_filters, cosmetic_filters, false), - true, - ) -} - fn blocker_new(c: &mut Criterion) { let mut group = c.benchmark_group("blocker_new"); group.throughput(Throughput::Elements(1)); group.sample_size(10); - let easylist_rules: Vec<_> = rules_from_lists(&[ + let easylist_rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", - ]) - .collect(); - let brave_list_rules: Vec<_> = rules_from_lists(&["data/brave/brave-main-list.txt"]).collect(); - let engine = Engine::from_rules(&brave_list_rules, Default::default()); + ]); + let brave_list_rules = rules_from_lists(["data/brave/brave-main-list.txt"]); + let engine = Engine::new_with_list_text(brave_list_rules.clone(), Default::default()); let engine_serialized = engine.serialize().to_vec(); - group.bench_function("el+ep", move |b| b.iter(|| get_engine(&easylist_rules))); + group.bench_function("el+ep", move |b| { + b.iter(|| Engine::new_with_list_text(easylist_rules.clone(), Default::default())) + }); group.bench_function("brave-list", move |b| { - b.iter(|| get_engine(&brave_list_rules)) + b.iter(|| Engine::new_with_list_text(brave_list_rules.clone(), Default::default())) }); group.bench_function("brave-list-deserialize", move |b| { b.iter(|| { diff --git a/benches/bench_serialization.rs b/benches/bench_serialization.rs index ff84d17a..87820c05 100644 --- a/benches/bench_serialization.rs +++ b/benches/bench_serialization.rs @@ -12,24 +12,24 @@ fn serialization(c: &mut Criterion) { group.sample_size(20); group.bench_function("el+ep", move |b| { - let full_rules = rules_from_lists(&[ + let full_rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); b.iter(|| assert!(!engine.serialize().to_vec().is_empty())) }); group.bench_function("el", move |b| { - let full_rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); + let full_rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); b.iter(|| assert!(!engine.serialize().to_vec().is_empty())) }); group.bench_function("slimlist", move |b| { - let full_rules = rules_from_lists(&["data/slim-list.txt"]); + let full_rules = rules_from_lists(["data/slim-list.txt"]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); b.iter(|| assert!(!engine.serialize().to_vec().is_empty())) }); @@ -42,12 +42,12 @@ fn deserialization(c: &mut Criterion) { group.sample_size(20); group.bench_function("el+ep", move |b| { - let full_rules = rules_from_lists(&[ + let full_rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", ]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); let serialized = engine.serialize().to_vec(); b.iter(|| { @@ -56,9 +56,9 @@ fn deserialization(c: &mut Criterion) { }) }); group.bench_function("el", move |b| { - let full_rules = rules_from_lists(&["data/easylist.to/easylist/easylist.txt"]); + let full_rules = rules_from_lists(["data/easylist.to/easylist/easylist.txt"]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); let serialized = engine.serialize().to_vec(); b.iter(|| { @@ -67,9 +67,9 @@ fn deserialization(c: &mut Criterion) { }) }); group.bench_function("slimlist", move |b| { - let full_rules = rules_from_lists(&["data/slim-list.txt"]); + let full_rules = rules_from_lists(["data/slim-list.txt"]); - let engine = Engine::from_rules(full_rules, Default::default()); + let engine = Engine::new_with_list_text(full_rules, Default::default()); let serialized = engine.serialize().to_vec(); b.iter(|| { diff --git a/benches/bench_url.rs b/benches/bench_url.rs index c9d90ebe..f45915cc 100644 --- a/benches/bench_url.rs +++ b/benches/bench_url.rs @@ -18,8 +18,9 @@ struct TestRequest { } fn load_requests() -> Vec { - rules_from_lists(&["data/requests.json"]) - .map(|r| serde_json::from_str(&r)) + rules_from_lists(["data/requests.json"]) + .lines() + .map(serde_json::from_str) .filter_map(Result::ok) .collect::>() } @@ -37,7 +38,7 @@ fn request_parsing_throughput(c: &mut Criterion) { b.iter(|| { let mut successful = 0; requests.iter().for_each(|r| { - let req: Result = Request::new(&r.url, &r.frameUrl, &r.cpt); + let req: Result = Request::new(&r.url, &r.frameUrl, &r.cpt, ""); if req.is_ok() { successful += 1; } @@ -87,7 +88,7 @@ fn request_new_throughput(c: &mut Criterion) { b.iter(|| { let mut successful = 0; requests.iter().for_each(|r| { - Request::new(&r.url, &r.frameUrl, &r.cpt).ok(); + Request::new(&r.url, &r.frameUrl, &r.cpt, "").ok(); successful += 1; }); }) diff --git a/examples/deserialization.rs b/examples/deserialization.rs index 3ef07a72..dd80e663 100644 --- a/examples/deserialization.rs +++ b/examples/deserialization.rs @@ -72,7 +72,7 @@ fn main() { if reqs_processed % 10000 == 0 { println!("{reqs_processed} requests processed"); } - let request = Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap(); + let request = Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(); let checked = engine.check_network_request(&request); if req.blocked == 1 && !checked.matched { mismatch_expected_match += 1; diff --git a/examples/example.rs b/examples/example.rs index 9f2f74ac..fe0bd81e 100644 --- a/examples/example.rs +++ b/examples/example.rs @@ -5,23 +5,25 @@ use adblock::{ }; fn main() { - let rules = vec![ - String::from("-advertisement-icon."), - String::from("-advertisement-management/"), - String::from("-advertisement."), - String::from("-advertisement/script."), - ]; + let rules = [ + "-advertisement-icon.", + "-advertisement-management/", + "-advertisement.", + "-advertisement/script.", + ] + .join("\n"); let debug_info = true; let mut filter_set = FilterSet::new(debug_info); - filter_set.add_filters(&rules, ParseOptions::default()); + filter_set.add_filter_list(rules, ParseOptions::default()); - let engine = Engine::from_filter_set(filter_set, true); + let engine = Engine::new_with_filter_set(filter_set, true); let request = Request::new( "http://example.com/-advertisement-icon.", "http://example.com/helloworld", "image", + "", ) .unwrap(); let blocker_result = engine.check_network_request(&request); diff --git a/examples/generate-dat.rs b/examples/generate-dat.rs index 513db3f7..d3311cd0 100644 --- a/examples/generate-dat.rs +++ b/examples/generate-dat.rs @@ -5,19 +5,21 @@ use std::io::prelude::*; fn main() { // Rules we want to serialize - let rules = vec![ + let rules = [ String::from("||platform.twitter.com/$tag=twitter-embeds"), String::from("@@||platform.twitter.com/$tag=twitter-embeds"), - ]; + ] + .join("\n"); // Serialize - let mut engine = Engine::from_rules_debug(&rules, Default::default()); + let mut engine = Engine::new_with_list_text(rules, Default::default()); engine.use_tags(&["twitter-embeds"]); let request = Request::new( "https://platform.twitter.com/widgets.js", "https://fmarier.github.io/brave-testing/social-widgets.html", "script", + "", ) .unwrap(); assert!(engine.check_network_request(&request).exception.is_some()); diff --git a/examples/use-dat.rs b/examples/use-dat.rs index bc662d71..6708c3b7 100644 --- a/examples/use-dat.rs +++ b/examples/use-dat.rs @@ -20,6 +20,7 @@ fn main() { "https://platform.twitter.com/widgets.js", "https://fmarier.github.io/brave-testing/social-widgets.html", "script", + "", ) .unwrap(); let checked = engine.check_network_request(&request); diff --git a/fuzz/fuzz_targets/parse_request.rs b/fuzz/fuzz_targets/parse_request.rs index 62ca8944..a84a1ccf 100644 --- a/fuzz/fuzz_targets/parse_request.rs +++ b/fuzz/fuzz_targets/parse_request.rs @@ -6,10 +6,15 @@ use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { if let Ok(url) = std::str::from_utf8(data) { - Request::new(&format!("https://{}", url), "https://example.com", "other"); - Request::new(url, "https://example.com", "script"); - Request::new(url, "", ""); - Request::new(url, url, ""); - Request::new(url, url, url); + Request::new( + &format!("https://{}", url), + "https://example.com", + "other", + "", + ); + Request::new(url, "https://example.com", "script", ""); + Request::new(url, "", "", ""); + Request::new(url, url, "", ""); + Request::new(url, url, url, ""); } }); diff --git a/js/example.mjs b/js/example.mjs index f1a22df2..882c371d 100644 --- a/js/example.mjs +++ b/js/example.mjs @@ -8,13 +8,13 @@ const filterSet = new adblockRust.FilterSet(debugInfo); const easylistFilters = fs.readFileSync( dataPath + 'easylist.to/easylist/easylist.txt', { encoding: 'utf-8' }, -).split('\n'); +); filterSet.addFilters(easylistFilters); const uboUnbreakFilters = fs.readFileSync( dataPath + 'uBlockOrigin/unbreak.txt', { encoding: 'utf-8' }, -).split('\n'); +); filterSet.addFilters(uboUnbreakFilters); const resources = adblockRust.uBlockResources( diff --git a/js/index.js b/js/index.js index e7f389d1..a8207639 100644 --- a/js/index.js +++ b/js/index.js @@ -15,7 +15,7 @@ class FilterSet { this.boxed = blocker.FilterSet_constructor(...args); } } -forwardClassMethods(FilterSet, ['addFilters', 'addFilter', 'intoContentBlocking']); +forwardClassMethods(FilterSet, ['addFilters', 'intoContentBlocking']); class Engine { constructor(filter_set, ...args) { diff --git a/js/src/lib.rs b/js/src/lib.rs index d6577086..c4500567 100644 --- a/js/src/lib.rs +++ b/js/src/lib.rs @@ -1,6 +1,4 @@ -use adblock::lists::{ - FilterFormat, FilterListMetadata, FilterSet as FilterSetInternal, ParseOptions, RuleTypes, -}; +use adblock::lists::{FilterFormat, FilterSet as FilterSetInternal, ParseOptions, RuleTypes}; use adblock::resources::resource_assembler::assemble_web_accessible_resources; use adblock::resources::Resource; use adblock::Engine as EngineInternal; @@ -53,6 +51,31 @@ mod json_ffi { } } +fn console_warn<'a, C: Context<'a>>(cx: &mut C, message: &str) -> NeonResult<()> { + let console: Handle = cx.global().get(cx, "console")?; + let warn: Handle = console.get(cx, "warn")?; + let msg = JsString::new(cx, message).upcast::(); + warn.call(cx, console, [msg])?; + Ok(()) +} + +/// Accepts a single filter list string, or a deprecated array of list strings (joined with `\n`). +fn filter_rules_from_js<'a, C: Context<'a>>( + cx: &mut C, + input: Handle, +) -> NeonResult { + if input.downcast::(cx).is_ok() { + console_warn( + cx, + "FilterSet.addFilters: passing an array of strings is deprecated; \ + pass a single newline-separated filter list string instead.", + )?; + let parts: Vec = json_ffi::from_js(cx, input)?; + return Ok(parts.join("\n")); + } + json_ffi::from_js(cx, input) +} + #[derive(Serialize, Deserialize)] struct EngineOptions { pub optimize: Option, @@ -64,16 +87,10 @@ impl FilterSet { fn new(debug: bool) -> Self { Self(RefCell::new(FilterSetInternal::new(debug))) } - fn add_filters(&self, rules: &[String], opts: ParseOptions) -> FilterListMetadata { - self.0.borrow_mut().add_filters(rules, opts) - } - fn add_filter( - &self, - filter: &str, - opts: ParseOptions, - ) -> Result<(), adblock::lists::FilterParseError> { - self.0.borrow_mut().add_filter(filter, opts) + fn add_filters(&self, rules: String, opts: ParseOptions) -> adblock::lists::FilterListMetadata { + self.0.borrow_mut().add_filter_list(rules, opts) } + fn into_content_blocking( &self, ) -> Result<(Vec, Vec), ()> { @@ -99,7 +116,6 @@ fn create_filter_set(mut cx: FunctionContext) -> JsResult> { fn filter_set_add_filters(mut cx: FunctionContext) -> JsResult { let this = cx.argument::>(0)?; - // Take the first argument, which must be an array let rules_handle: Handle = cx.argument(1)?; // Second argument is optional parse options. All fields are optional. ParseOptions::default() // if unspecified. @@ -108,27 +124,13 @@ fn filter_set_add_filters(mut cx: FunctionContext) -> JsResult { None => ParseOptions::default(), }; - let rules: Vec = json_ffi::from_js(&mut cx, rules_handle)?; + let rules: String = filter_rules_from_js(&mut cx, rules_handle)?; - let metadata = this.add_filters(&rules, parse_opts); + let metadata = this.add_filters(rules, parse_opts); json_ffi::to_js(&mut cx, &metadata) } -fn filter_set_add_filter(mut cx: FunctionContext) -> JsResult { - let this = cx.argument::>(0)?; - - let filter: String = cx.argument::(1)?.value(&mut cx); - let parse_opts = match cx.argument_opt(2) { - Some(parse_opts_arg) => json_ffi::from_js(&mut cx, parse_opts_arg)?, - None => ParseOptions::default(), - }; - - let ok = this.add_filter(&filter, parse_opts).is_ok(); - // Return true/false depending on whether or not the filter could be added - Ok(JsBoolean::new(&mut cx, ok)) -} - #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct ContentBlockingConversionResult { @@ -171,9 +173,9 @@ fn engine_constructor(mut cx: FunctionContext) -> JsResult> { config.optimize.unwrap_or(true) } }; - EngineInternal::from_filter_set(rules, optimize) + EngineInternal::new_with_filter_set(rules, optimize) } - None => EngineInternal::from_filter_set(rules, true), + None => EngineInternal::new_with_filter_set(rules, true), }; Ok(cx.boxed(Engine(Mutex::new(engine_internal)))) } @@ -185,17 +187,47 @@ fn engine_check(mut cx: FunctionContext) -> JsResult { let source_url: String = cx.argument::(2)?.value(&mut cx); let request_type: String = cx.argument::(3)?.value(&mut cx); - let debug = match cx.argument_opt(4) { + // Optional trailing args: + // - legacy: `(debug: boolean)` + // - new: `(method: string, debug?: boolean)` — empty method string means unspecified + let (method, debug) = match cx.argument_opt(4) { + None => (String::new(), false), Some(arg) => { - // Throw if the argument exists and it cannot be downcasted to a boolean - arg.downcast::(&mut cx) - .or_throw(&mut cx)? - .value(&mut cx) + if arg.is_a::(&mut cx) || arg.is_a::(&mut cx) { + let debug = match cx.argument_opt(5) { + Some(debug_arg) => debug_arg + .downcast::(&mut cx) + .or_throw(&mut cx)? + .value(&mut cx), + None => false, + }; + (String::new(), debug) + } else if let Ok(debug) = arg.downcast::(&mut cx) { + console_warn( + &mut cx, + "Engine.check: passing debug as the 4th boolean argument is deprecated; \ + pass an HTTP method string as the 4th argument (use \"\" if unspecified) \ + and debug as an optional 5th boolean instead.", + )?; + (String::new(), debug.value(&mut cx)) + } else if let Ok(method) = arg.downcast::(&mut cx) { + let debug = match cx.argument_opt(5) { + Some(debug_arg) => debug_arg + .downcast::(&mut cx) + .or_throw(&mut cx)? + .value(&mut cx), + None => false, + }; + (method.value(&mut cx), debug) + } else { + cx.throw_error( + "expected boolean (debug), string (method), null, or undefined as 4th argument", + )? + } } - None => false, }; - let request = match adblock::request::Request::new(&url, &source_url, &request_type) { + let request = match adblock::request::Request::new(&url, &source_url, &request_type, &method) { Ok(r) => r, Err(e) => cx.throw_error(e.to_string())?, }; @@ -327,7 +359,20 @@ fn validate_request(mut cx: FunctionContext) -> JsResult { let url: String = cx.argument::(0)?.value(&mut cx); let source_url: String = cx.argument::(1)?.value(&mut cx); let request_type: String = cx.argument::(2)?.value(&mut cx); - let request_ok = adblock::request::Request::new(&url, &source_url, &request_type).is_ok(); + let method = match cx.argument_opt(3) { + None => String::new(), + Some(arg) => { + if arg.is_a::(&mut cx) || arg.is_a::(&mut cx) { + String::new() + } else { + arg.downcast::(&mut cx) + .or_throw(&mut cx)? + .value(&mut cx) + } + } + }; + let request_ok = + adblock::request::Request::new(&url, &source_url, &request_type, &method).is_ok(); Ok(cx.boolean(request_ok)) } @@ -391,7 +436,6 @@ fn build_rule_types_enum<'a, C: Context<'a>>(cx: &mut C) -> JsResult<'a, JsObjec register_module!(mut m, { m.export_function("FilterSet_constructor", create_filter_set)?; m.export_function("FilterSet_addFilters", filter_set_add_filters)?; - m.export_function("FilterSet_addFilter", filter_set_add_filter)?; m.export_function( "FilterSet_intoContentBlocking", filter_set_into_content_blocking, diff --git a/js/test/bindings.test.mjs b/js/test/bindings.test.mjs index 2cca2147..c76f659e 100644 --- a/js/test/bindings.test.mjs +++ b/js/test/bindings.test.mjs @@ -23,7 +23,7 @@ const { FilterSet, Engine, FilterFormat, RuleTypes, uBlockResources } = // --------------------------------------------------------------------------- describe('FilterSet.addFilters', () => { - it('parses metadata comments (title, homepage, expires, redirect)', () => { + it('parses metadata comments (title, homepage, expires, redirect)', () => { const fs = new FilterSet(); const meta = fs.addFilters([ '! Title: Test List', @@ -31,16 +31,15 @@ describe('FilterSet.addFilters', () => { '! Expires: 2 days', '! Redirect: https://example.com/new-list.txt', '||ads.com^', - ]); + ].join('\n')); assert.equal(meta.title, 'Test List'); assert.equal(meta.homepage, 'https://example.com'); assert.ok(meta.expires != null); assert.equal(meta.redirect, 'https://example.com/new-list.txt'); }); - it('hosts format parses IP-hostname entries', () => { const fs = new FilterSet(); - fs.addFilters(['127.0.0.1 ads.example.com'], { format: FilterFormat.HOSTS }); + fs.addFilters('127.0.0.1 ads.example.com', { format: FilterFormat.HOSTS }); const engine = new Engine(fs, true); assert.equal( engine.check('https://ads.example.com/', 'https://pub.com', 'script'), @@ -51,7 +50,7 @@ describe('FilterSet.addFilters', () => { it('NETWORK_ONLY skips cosmetic rules', () => { const fs = new FilterSet(); fs.addFilters( - ['||example.com^', 'example.com##.ad'], + ['||example.com^', 'example.com##.ad'].join('\n'), { rule_types: RuleTypes.NETWORK_ONLY }, ); const engine = new Engine(fs, true); @@ -63,10 +62,26 @@ describe('FilterSet.addFilters', () => { assert.equal(cosmetic.hide_selectors.length, 0); }); + it("Legacy addFilters syntax works", () => { + const fs = new FilterSet(); + fs.addFilters(["||example.com^", "example.com##.ad"], { + rule_types: RuleTypes.NETWORK_ONLY, + }); + const engine = new Engine(fs, true); + assert.equal( + engine.check( + "https://example.com/test.js", + "https://pub.com", + "script", + ), + true, + ); + }); + it('COSMETIC_ONLY skips network rules', () => { const fs = new FilterSet(); fs.addFilters( - ['||example.com^', 'example.com##.ad'], + ['||example.com^', 'example.com##.ad'].join('\n'), { rule_types: RuleTypes.COSMETIC_ONLY }, ); const engine = new Engine(fs, true); @@ -79,40 +94,6 @@ describe('FilterSet.addFilters', () => { }); }); -// --------------------------------------------------------------------------- -// FilterSet.addFilter -// --------------------------------------------------------------------------- - -describe('FilterSet.addFilter', () => { - it('returns true for a valid network filter', () => { - const fs = new FilterSet(); - assert.equal(fs.addFilter('||example.com^'), true); - }); - - it('returns true for a valid cosmetic filter', () => { - const fs = new FilterSet(); - assert.equal(fs.addFilter('example.com##.banner'), true); - }); - - it('returns false for an empty string', () => { - const fs = new FilterSet(); - assert.equal(fs.addFilter(''), false); - }); - - it('returns false for a comment line', () => { - const fs = new FilterSet(); - assert.equal(fs.addFilter('! this is a comment'), false); - }); - - it('accepts hosts format via ParseOptions', () => { - const fs = new FilterSet(); - assert.equal( - fs.addFilter('127.0.0.1 ads.example.com', { format: FilterFormat.HOSTS }), - true, - ); - }); -}); - // --------------------------------------------------------------------------- // FilterSet.intoContentBlocking // --------------------------------------------------------------------------- @@ -120,7 +101,7 @@ describe('FilterSet.addFilter', () => { describe('FilterSet.intoContentBlocking', () => { it('converts network filters to content blocking rules with trigger/action', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); const result = fs.intoContentBlocking(); assert.notEqual(result, undefined); assert.ok(result.contentBlockingRules.length > 0); @@ -131,7 +112,7 @@ describe('FilterSet.intoContentBlocking', () => { it('returns undefined when debug=false', () => { const fs = new FilterSet(false); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); assert.equal(fs.intoContentBlocking(), undefined); }); }); @@ -143,9 +124,9 @@ describe('FilterSet.intoContentBlocking', () => { describe('FilterSet survives Engine construction (clone semantics)', () => { it('FilterSet is still usable after being passed to Engine constructor', () => { const fs = new FilterSet(); - fs.addFilters(['||example.com^']); + fs.addFilters('||example.com^'); const _engine = new Engine(fs, true); - assert.doesNotThrow(() => fs.addFilter('||another.com^')); + assert.doesNotThrow(() => fs.addFilters('||another.com^')); }); }); @@ -156,7 +137,7 @@ describe('FilterSet survives Engine construction (clone semantics)', () => { describe('Engine.check — basic blocking', () => { it('blocks matching requests, allows non-matching', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), true); assert.equal(engine.check('https://safe.com/t.js', 'https://pub.com', 'script'), false); @@ -164,7 +145,7 @@ describe('Engine.check — basic blocking', () => { it('debug=true returns BlockerResult with filter text', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); const engine = new Engine(fs, true); const result = engine.check( 'https://ads.example.com/t.js', 'https://pub.com', 'script', true, @@ -180,7 +161,7 @@ describe('Engine.check — basic blocking', () => { it('EngineOptions object works as alternative to boolean', () => { const fs = new FilterSet(); - fs.addFilters(['||blocked.com^']); + fs.addFilters('||blocked.com^'); const engine = new Engine(fs, { optimize: false }); assert.equal( engine.check('https://blocked.com/img.png', 'https://pub.com', 'image'), @@ -196,7 +177,7 @@ describe('Engine.check — basic blocking', () => { describe('Engine.check — exception rules', () => { it('exception rule prevents blocking, populates exception field', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^', '@@||ads.example.com^$domain=publisher.com']); + fs.addFilters(['||ads.example.com^', '@@||ads.example.com^$domain=publisher.com'].join('\n')); const engine = new Engine(fs, true); const result = engine.check( 'https://ads.example.com/tracker.js', 'https://publisher.com', 'script', true, @@ -208,7 +189,7 @@ describe('Engine.check — exception rules', () => { it('$important overrides exception rules', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^$important', '@@||ads.example.com^']); + fs.addFilters(['||ads.example.com^$important', '@@||ads.example.com^'].join('\n')); const engine = new Engine(fs, true); const result = engine.check( 'https://ads.example.com/t.js', 'https://pub.com', 'script', true, @@ -226,7 +207,7 @@ describe('Engine.check — exception rules', () => { describe('Engine.check — $third-party and $1p modifiers', () => { it('$third-party rule blocks 3p, allows 1p', () => { const fs = new FilterSet(); - fs.addFilters(['||tracker.com^$third-party']); + fs.addFilters('||tracker.com^$third-party'); const engine = new Engine(fs, true); assert.equal(engine.check('https://tracker.com/t.js', 'https://other.com', 'script'), true); assert.equal(engine.check('https://tracker.com/t.js', 'https://tracker.com', 'script'), false); @@ -234,7 +215,7 @@ describe('Engine.check — $third-party and $1p modifiers', () => { it('$1p rule blocks 1p, allows 3p', () => { const fs = new FilterSet(); - fs.addFilters(['/bad-path$1p']); + fs.addFilters('/bad-path$1p'); const engine = new Engine(fs, true); assert.equal(engine.check('https://example.com/bad-path', 'https://example.com', 'script'), true); assert.equal(engine.check('https://example.com/bad-path', 'https://other.com', 'script'), false); @@ -248,7 +229,7 @@ describe('Engine.check — $third-party and $1p modifiers', () => { describe('Engine.check — type-specific rules', () => { it('$script blocks only script requests', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^$script']); + fs.addFilters('||ads.example.com^$script'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), true); assert.equal(engine.check('https://ads.example.com/t.png', 'https://pub.com', 'image'), false); @@ -256,7 +237,7 @@ describe('Engine.check — type-specific rules', () => { it('$image blocks only image requests', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^$image']); + fs.addFilters('||ads.example.com^$image'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.png', 'https://pub.com', 'image'), true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), false); @@ -264,7 +245,7 @@ describe('Engine.check — type-specific rules', () => { it('$stylesheet blocks only stylesheet requests', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^$stylesheet']); + fs.addFilters('||ads.example.com^$stylesheet'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/s.css', 'https://pub.com', 'stylesheet'), true); assert.equal(engine.check('https://ads.example.com/s.js', 'https://pub.com', 'script'), false); @@ -272,11 +253,32 @@ describe('Engine.check — type-specific rules', () => { it('$xmlhttprequest blocks only XHR requests', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^$xmlhttprequest']); + fs.addFilters('||ads.example.com^$xmlhttprequest'); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/api', 'https://pub.com', 'xmlhttprequest'), true); assert.equal(engine.check('https://ads.example.com/api', 'https://pub.com', 'image'), false); }); + + it('$method=post blocks only POST requests', () => { + const fs = new FilterSet(); + fs.addFilters('||api.example.com^$xhr,method=post'); + const engine = new Engine(fs, true); + const url = 'https://api.example.com/collect'; + const source = 'https://pub.com'; + assert.equal(engine.check(url, source, 'xhr', 'post'), true); + assert.equal(engine.check(url, source, 'xhr', 'get'), false); + assert.equal(engine.check(url, source, 'xhr'), false); + }); + + it('debug=true remains backward compatible as 4th boolean arg', () => { + const fs = new FilterSet(true); + fs.addFilters('||api.example.com^$xhr,method=post'); + const engine = new Engine(fs, true); + const result = engine.check( + 'https://api.example.com/collect', 'https://pub.com', 'xhr', true, + ); + assert.equal(result.matched, false); + }); }); // --------------------------------------------------------------------------- @@ -286,7 +288,7 @@ describe('Engine.check — type-specific rules', () => { describe('Engine.check — $domain modifier', () => { it('blocks only when source matches the domain option', () => { const fs = new FilterSet(); - fs.addFilters(['/ads.js$domain=publisher.com']); + fs.addFilters('/ads.js$domain=publisher.com'); const engine = new Engine(fs, true); assert.equal(engine.check('https://cdn.example.com/ads.js', 'https://publisher.com', 'script'), true); assert.equal(engine.check('https://cdn.example.com/ads.js', 'https://other.com', 'script'), false); @@ -294,7 +296,7 @@ describe('Engine.check — $domain modifier', () => { it('~domain excludes the specified domain', () => { const fs = new FilterSet(); - fs.addFilters(['/ads.js$domain=~safe.com']); + fs.addFilters('/ads.js$domain=~safe.com'); const engine = new Engine(fs, true); assert.equal(engine.check('https://cdn.example.com/ads.js', 'https://safe.com', 'script'), false); assert.equal(engine.check('https://cdn.example.com/ads.js', 'https://other.com', 'script'), true); @@ -308,14 +310,14 @@ describe('Engine.check — $domain modifier', () => { describe('Engine.check — $badfilter modifier', () => { it('cancels a matching blocking rule', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^', '||ads.example.com^$badfilter']); + fs.addFilters(['||ads.example.com^', '||ads.example.com^$badfilter'].join('\n')); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), false); }); it('does not cancel a dissimilar rule', () => { const fs = new FilterSet(); - fs.addFilters(['||ads.example.com^', '||other.com^$badfilter']); + fs.addFilters(['||ads.example.com^', '||other.com^$badfilter'].join('\n')); const engine = new Engine(fs, true); assert.equal(engine.check('https://ads.example.com/t.js', 'https://pub.com', 'script'), true); }); @@ -328,7 +330,7 @@ describe('Engine.check — $badfilter modifier', () => { describe('Engine.check — redirect rules', () => { it('redirect field is set when $redirect rule matches and resource is loaded', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^$script,redirect=noopjs']); + fs.addFilters('||ads.example.com^$script,redirect=noopjs'); const engine = new Engine(fs, true); engine.useResources([{ name: 'noopjs', @@ -346,7 +348,7 @@ describe('Engine.check — redirect rules', () => { it('redirect is null when no redirect rule applies', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^']); + fs.addFilters('||ads.example.com^'); const engine = new Engine(fs, true); const result = engine.check( 'https://ads.example.com/t.js', 'https://pub.com', 'script', true, @@ -364,7 +366,7 @@ describe('Engine.check — redirect rules', () => { describe('Engine.check — $removeparam modifier', () => { it('strips the specified parameter, preserves others', () => { const fs = new FilterSet(true); - fs.addFilters(['||example.com^$removeparam=tracking_id']); + fs.addFilters('||example.com^$removeparam=tracking_id'); const engine = new Engine(fs, true); const result = engine.check( 'https://example.com/page?tracking_id=abc&keep=1', @@ -377,7 +379,7 @@ describe('Engine.check — $removeparam modifier', () => { it('rewritten_url is null when the parameter is absent', () => { const fs = new FilterSet(true); - fs.addFilters(['||example.com^$removeparam=tracking_id']); + fs.addFilters('||example.com^$removeparam=tracking_id'); const engine = new Engine(fs, true); const result = engine.check( 'https://example.com/page?unrelated=1', @@ -394,7 +396,7 @@ describe('Engine.check — $removeparam modifier', () => { describe('Engine.check — exception rules with tags', () => { it('tagged exception activates only after enableTag', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^', '@@||ads.example.com^$tag=unbreak']); + fs.addFilters(['||ads.example.com^', '@@||ads.example.com^$tag=unbreak'].join('\n')); const engine = new Engine(fs, true); const before = engine.check( @@ -419,7 +421,7 @@ describe('Engine.check — exception rules with tags', () => { describe('Engine.urlCosmeticResources', () => { it('returns matching hide_selectors for the URL', () => { const fs = new FilterSet(); - fs.addFilters(['example.com##.ad-banner', 'example.com##.sponsored-post']); + fs.addFilters(['example.com##.ad-banner', 'example.com##.sponsored-post'].join('\n')); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://example.com/page'); assert.ok(result.hide_selectors.includes('.ad-banner')); @@ -428,7 +430,7 @@ describe('Engine.urlCosmeticResources', () => { it('returns empty hide_selectors for an unmatched URL', () => { const fs = new FilterSet(); - fs.addFilters(['example.com##.ad-banner']); + fs.addFilters('example.com##.ad-banner'); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://other-site.com/page'); assert.equal(result.hide_selectors.length, 0); @@ -436,7 +438,7 @@ describe('Engine.urlCosmeticResources', () => { it('generichide exception sets generichide=true', () => { const fs = new FilterSet(); - fs.addFilters(['##.generic-ad', '@@||example.com^$generichide']); + fs.addFilters(['##.generic-ad', '@@||example.com^$generichide'].join('\n')); const engine = new Engine(fs, true); assert.equal(engine.urlCosmeticResources('https://example.com/page').generichide, true); assert.equal(engine.urlCosmeticResources('https://other.com/page').generichide, false); @@ -444,7 +446,7 @@ describe('Engine.urlCosmeticResources', () => { it('site-specific unhide (#@#) prevents selector from appearing', () => { const fs = new FilterSet(); - fs.addFilters(['example.com##.ad-banner', 'example.com#@#.ad-banner']); + fs.addFilters(['example.com##.ad-banner', 'example.com#@#.ad-banner'].join('\n')); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://example.com/page'); assert.ok(!result.hide_selectors.includes('.ad-banner')); @@ -452,7 +454,7 @@ describe('Engine.urlCosmeticResources', () => { it('generic unhide (#@#) adds to exceptions list', () => { const fs = new FilterSet(); - fs.addFilters(['##.generic-ad', 'example.com#@#.generic-ad']); + fs.addFilters(['##.generic-ad', 'example.com#@#.generic-ad'].join('\n')); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://example.com/page'); assert.ok(result.exceptions.includes('.generic-ad')); @@ -460,7 +462,7 @@ describe('Engine.urlCosmeticResources', () => { it('negated domain (~sub.example.com) excludes subdomain', () => { const fs = new FilterSet(); - fs.addFilters(['example.com,~sub.example.com##.ad']); + fs.addFilters('example.com,~sub.example.com##.ad'); const engine = new Engine(fs, true); assert.ok(engine.urlCosmeticResources('https://example.com/page').hide_selectors.includes('.ad')); assert.ok(!engine.urlCosmeticResources('https://sub.example.com/page').hide_selectors.includes('.ad')); @@ -471,7 +473,7 @@ describe('Engine.urlCosmeticResources', () => { fs.addFilters([ 'example.com##.items:has-text(Sponsored)', 'example.com##.ad-banner:remove()', - ]); + ].join('\n')); const engine = new Engine(fs, true); const result = engine.urlCosmeticResources('https://example.com/page'); assert.ok(result.procedural_actions.length >= 2); @@ -479,7 +481,7 @@ describe('Engine.urlCosmeticResources', () => { it('scriptlet injection populates injected_script', () => { const fs = new FilterSet(); - fs.addFilters(['example.com##+js(noopjs)']); + fs.addFilters('example.com##+js(noopjs)'); const engine = new Engine(fs, true); // ##+js(noopjs) looks up "noopjs.js"; scriptlets use kind: "template" engine.useResources([{ @@ -500,7 +502,7 @@ describe('Engine.urlCosmeticResources', () => { describe('Engine.hiddenClassIdSelectors', () => { it('returns selectors matching class and id names', () => { const fs = new FilterSet(); - fs.addFilters(['##.a-class', '###simple-id']); + fs.addFilters(['##.a-class', '###simple-id'].join('\n')); const engine = new Engine(fs, true); assert.ok(engine.hiddenClassIdSelectors(['a-class'], [], []).includes('.a-class')); assert.ok(engine.hiddenClassIdSelectors([], ['simple-id'], []).includes('#simple-id')); @@ -508,14 +510,14 @@ describe('Engine.hiddenClassIdSelectors', () => { it('returns empty for unknown class/id names', () => { const fs = new FilterSet(); - fs.addFilters(['##.a-class']); + fs.addFilters('##.a-class'); const engine = new Engine(fs, true); assert.deepEqual(engine.hiddenClassIdSelectors(['unknown'], ['unknown'], []), []); }); it('exceptions array filters out results', () => { const fs = new FilterSet(); - fs.addFilters(['##.a-class']); + fs.addFilters('##.a-class'); const engine = new Engine(fs, true); assert.ok(engine.hiddenClassIdSelectors(['a-class'], [], []).includes('.a-class')); assert.ok(!engine.hiddenClassIdSelectors(['a-class'], [], ['.a-class']).includes('.a-class')); @@ -529,7 +531,7 @@ describe('Engine.hiddenClassIdSelectors', () => { describe('Engine serialization', () => { it('roundtrip preserves blocking and exception rules', () => { const fs = new FilterSet(); - fs.addFilters(['||blocked.com^', '@@||exception.blocked.com^']); + fs.addFilters(['||blocked.com^', '@@||exception.blocked.com^'].join('\n')); const src = new Engine(fs, true); const buf = src.serialize(); @@ -542,7 +544,7 @@ describe('Engine serialization', () => { it('tag enablement is NOT serialized — must re-enable after deserialize', () => { const fs = new FilterSet(); - fs.addFilters(['adv$tag=stuff', '||blocked.com^']); + fs.addFilters(['adv$tag=stuff', '||blocked.com^'].join('\n')); const src = new Engine(fs, true); src.enableTag('stuff'); const buf = src.serialize(); @@ -560,7 +562,7 @@ describe('Engine serialization', () => { it('resources are NOT serialized — must reload after deserialize', () => { const fs = new FilterSet(true); - fs.addFilters(['||ads.example.com^$script,redirect=noopjs']); + fs.addFilters(['||ads.example.com^$script,redirect=noopjs'].join('\n')); const resource = { name: 'noopjs', aliases: [], @@ -597,7 +599,7 @@ describe('Engine serialization', () => { describe('Engine tags', () => { it('tagged filter is inactive before enableTag, active after', () => { const fs = new FilterSet(); - fs.addFilters(['adv$tag=stuff']); + fs.addFilters(['adv$tag=stuff'].join('\n')); const engine = new Engine(fs, true); assert.equal(engine.check('https://example.com/adv', 'https://example.com', 'other'), false); engine.enableTag('stuff'); @@ -607,7 +609,7 @@ describe('Engine tags', () => { it('clearTags deactivates all enabled tags', () => { const fs = new FilterSet(); - fs.addFilters(['adv$tag=stuff', '||brianbondy.com/$tag=brian']); + fs.addFilters(['adv$tag=stuff', '||brianbondy.com/$tag=brian'].join('\n')); const engine = new Engine(fs, true); engine.enableTag('stuff'); engine.enableTag('brian'); diff --git a/src/blocker.rs b/src/blocker.rs index a7ac53b3..b654521a 100644 --- a/src/blocker.rs +++ b/src/blocker.rs @@ -211,7 +211,7 @@ impl Blocker { } None => None, // If matched an important filter, exceptions don't atter - Some(f) if f.is_important() => None, + Some(f) if f.filter_mask.is_important() => None, Some(_) => self .exceptions() .check(request, &self.tags_enabled, &mut regex_manager), @@ -227,7 +227,7 @@ impl Blocker { let redirect_resource = { let mut exceptions = vec![]; for redirect_filter in redirect_filters.iter() { - if redirect_filter.is_exception() { + if redirect_filter.filter_mask.is_exception() { if let Some(redirect) = redirect_filter.modifier_option.as_ref() { exceptions.push(redirect); } @@ -235,7 +235,7 @@ impl Blocker { } let mut resource_and_priority = None; for redirect_filter in redirect_filters.iter() { - if !redirect_filter.is_exception() { + if !redirect_filter.filter_mask.is_exception() { if let Some(redirect) = redirect_filter.modifier_option.as_ref() { if !exceptions.contains(&redirect) { // parse redirect + priority @@ -278,7 +278,7 @@ impl Blocker { let important = filter.is_some() && filter .as_ref() - .map(|f| f.is_important()) + .map(|f| f.filter_mask.is_important()) .unwrap_or_else(|| false); let rewritten_url = if important { @@ -410,17 +410,15 @@ impl Blocker { let mut enabled_directives: HashSet<&str> = HashSet::new(); for filter in filters.iter() { - if filter.is_exception() { - if filter.is_csp() { - if let Some(csp_directive) = &filter.modifier_option { - disabled_directives.insert(csp_directive); - } else { - // Exception filters with empty `csp` options will disable all CSP - // injections for matching pages. - return None; - } + if filter.filter_mask.is_exception() { + if let Some(csp_directive) = &filter.modifier_option { + disabled_directives.insert(csp_directive); + } else { + // Exception filters with empty `csp` options will disable all CSP + // injections for matching pages. + return None; } - } else if filter.is_csp() { + } else { if let Some(csp_directive) = &filter.modifier_option { enabled_directives.insert(csp_directive); } @@ -453,15 +451,26 @@ impl Blocker { #[cfg(test)] pub fn new( - network_filters: Vec, + network_filters: impl IntoIterator>, options: &BlockerOptions, ) -> Self { use crate::engine::Engine; use crate::FilterSet; let mut filter_set = FilterSet::new(true); - filter_set.network_filters = network_filters; - let engine = Engine::from_filter_set(filter_set, options.enable_optimizations); + filter_set.add_filters(network_filters, Default::default()); + let engine = Engine::new_with_filter_set(filter_set, options.enable_optimizations); + Self::from_context(engine.filter_data_context()) + } + + #[cfg(test)] + pub fn new_with_list_text(text: String, options: &BlockerOptions) -> Self { + use crate::engine::Engine; + use crate::FilterSet; + + let mut filter_set = FilterSet::new(true); + filter_set.add_filter_list(text, Default::default()); + let engine = Engine::new_with_filter_set(filter_set, options.enable_optimizations); Self::from_context(engine.filter_data_context()) } diff --git a/src/content_blocking.rs b/src/content_blocking.rs index 128d202f..5ad01239 100644 --- a/src/content_blocking.rs +++ b/src/content_blocking.rs @@ -1,8 +1,8 @@ //! Transforms filter rules into content blocking syntax used on iOS and MacOS. use crate::filters::cosmetic::CosmeticFilter; -use crate::filters::network::{NetworkFilter, NetworkFilterMask, NetworkFilterMaskHelper}; -use crate::lists::ParsedFilter; +use crate::filters::network::{NetworkFilter, NetworkFilterFeaturesMask, NetworkFilterMask}; +use crate::lists::ParsedLine; use memchr::{memchr as find_char, memmem}; use regex::Regex; @@ -229,13 +229,13 @@ pub enum CbRuleCreationFailure { ProceduralCosmeticFiltersUnsupported, } -impl TryFrom for CbRuleEquivalent { +impl TryFrom> for CbRuleEquivalent { type Error = CbRuleCreationFailure; - fn try_from(v: ParsedFilter) -> Result { + fn try_from(v: ParsedLine) -> Result { match v { - ParsedFilter::Network(f) => f.try_into(), - ParsedFilter::Cosmetic(f) => Ok(Self::SingleRule(f.try_into()?)), + ParsedLine::Network(f) => f.try_into(), + ParsedLine::Cosmetic(f) => Ok(Self::SingleRule(f.try_into()?)), } } } @@ -301,7 +301,7 @@ impl Iterator for CbRuleEquivalentIterator { } } -impl TryFrom for CbRuleEquivalent { +impl TryFrom> for CbRuleEquivalent { type Error = CbRuleCreationFailure; fn try_from(v: NetworkFilter) -> Result { @@ -311,15 +311,16 @@ impl TryFrom for CbRuleEquivalent { LazyLock::new(|| Regex::new(r##"\*"##).unwrap()); static TRAILING_SEPARATOR: LazyLock = LazyLock::new(|| Regex::new(r##"\^$"##).unwrap()); - if let Some(raw_line) = &v.raw_line { + if let Some(raw_line) = v.raw_line.as_deref() { if v.is_redirect() { return Err(CbRuleCreationFailure::NetworkRedirectUnsupported); } - if v.mask.contains(NetworkFilterMask::GENERIC_HIDE) { + if v.is_generic_hide() { return Err(CbRuleCreationFailure::NetworkGenerichideUnsupported); } debug_assert!( - !v.mask.contains(NetworkFilterMask::BAD_FILTER), + !v.features_mask + .contains(NetworkFilterFeaturesMask::BAD_FILTER), "BAD_FILTER should be filtered out" ); if v.is_csp() { @@ -602,7 +603,7 @@ impl TryFrom for CbRule { return Err(CbRuleCreationFailure::ScriptletInjectionsNotSupported); } - if let Some(raw_line) = &v.raw_line { + if let Some(raw_line) = v.raw_line.as_deref() { let mut hostnames_vec = vec![]; let mut not_hostnames_vec = vec![]; diff --git a/src/cosmetic_filter_cache.rs b/src/cosmetic_filter_cache.rs index d60b37a5..5f17c6b5 100644 --- a/src/cosmetic_filter_cache.rs +++ b/src/cosmetic_filter_cache.rs @@ -10,8 +10,6 @@ //! To build `CosmeticFilterCache`, use `CosmeticFilterCacheBuilder`. use crate::cosmetic_filter_utils::decode_script_with_permission; -#[cfg(test)] -use crate::filters::cosmetic::CosmeticFilter; use crate::filters::cosmetic::{CosmeticFilterAction, CosmeticFilterOperator}; use crate::filters::filter_data_context::FilterDataContextRef; @@ -134,13 +132,13 @@ impl CosmeticFilterCache { } #[cfg(test)] - pub fn from_rules(rules: Vec) -> Self { + pub fn from_rules(rules: impl IntoIterator>) -> Self { use crate::engine::Engine; use crate::FilterSet; - let mut filter_set = FilterSet::new(true); - filter_set.cosmetic_filters = rules; - let engine = Engine::from_filter_set(filter_set, true); + let mut filter_set = FilterSet::new(false); + filter_set.add_filters(rules, Default::default()); + let engine = Engine::new_with_filter_set(filter_set, true); engine.cosmetic_cache() } diff --git a/src/cosmetic_filter_cache_builder.rs b/src/cosmetic_filter_cache_builder.rs index 2f674c5c..2d5cd2e1 100644 --- a/src/cosmetic_filter_cache_builder.rs +++ b/src/cosmetic_filter_cache_builder.rs @@ -77,16 +77,6 @@ pub(crate) struct CosmeticFilterCacheBuilder<'a> { } impl<'a> CosmeticFilterCacheBuilder<'a> { - pub fn from_rules(rules: Vec, builder: &mut EngineFlatBuilder<'a>) -> Self { - let mut self_ = Self::default(); - - for rule in rules { - self_.add_filter(rule, builder); - } - - self_ - } - pub fn add_filter(&mut self, rule: CosmeticFilter, builder: &mut EngineFlatBuilder<'a>) { if rule.has_hostname_constraint() { if let Some(generic_rule) = rule.hidden_generic_rule() { diff --git a/src/data_format/mod.rs b/src/data_format/mod.rs index 35d0ff94..37202b05 100644 --- a/src/data_format/mod.rs +++ b/src/data_format/mod.rs @@ -17,7 +17,7 @@ const ADBLOCK_RUST_DAT_MAGIC: [u8; 4] = [0xd1, 0xd9, 0x3a, 0xaf]; /// The version of the data format. /// If the data format version is incremented, the data is considered as incompatible. -const ADBLOCK_RUST_DAT_VERSION: u8 = 3; +const ADBLOCK_RUST_DAT_VERSION: u8 = 5; /// The total length of the header prefix (magic + version + seahash) const HEADER_PREFIX_LENGTH: usize = 4 + 1 + 8; diff --git a/src/engine.rs b/src/engine.rs index 7867e0cd..63c449f7 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -4,18 +4,19 @@ use crate::blocker::{Blocker, BlockerResult}; use crate::cosmetic_filter_cache::{CosmeticFilterCache, UrlSpecificResources}; use crate::cosmetic_filter_cache_builder::CosmeticFilterCacheBuilder; use crate::data_format::{deserialize_dat_file, serialize_dat_file, DeserializationError}; -use crate::filters::cosmetic::CosmeticFilter; use crate::filters::fb_builder::EngineFlatBuilder; use crate::filters::fb_network_builder::NetworkRulesBuilder; use crate::filters::filter_data_context::{FilterDataContext, FilterDataContextRef}; -use crate::filters::network::NetworkFilter; +use crate::filters::flatbuffer_generated::fb; use crate::flatbuffers::containers::flat_serialize::FlatSerialize; use crate::flatbuffers::unsafe_tools::VerifiedFlatbufferMemory; -use crate::lists::{FilterSet, ParseOptions}; +use crate::lists::{parse_filter, FilterSet, ParseOptions, ParsedLine}; use crate::regex_manager::RegexManagerDiscardPolicy; use crate::request::Request; use crate::resources::{Resource, ResourceStorage, ResourceStorageBackend}; +use crate::filters::{cosmetic::CosmeticFilter, network::NetworkFilter}; + use std::collections::HashSet; /// Drives high-level blocking logic and is responsible for loading filter lists into an optimized @@ -66,38 +67,28 @@ pub struct EngineDebugInfo { impl Default for Engine { fn default() -> Self { - Self::from_filter_set(FilterSet::new(false), false) + Self::new_with_filter_set(FilterSet::new(false), false) } } impl Engine { - /// Loads rules in a single format, enabling optimizations and discarding debug information. - pub fn from_rules( - rules: impl IntoIterator>, - opts: ParseOptions, - ) -> Self { - let mut filter_set = FilterSet::new(false); - filter_set.add_filters(rules, opts); - Self::from_filter_set(filter_set, true) - } - - /// Loads rules, enabling optimizations and including debug information. - pub fn from_rules_debug( - rules: impl IntoIterator>, - opts: ParseOptions, - ) -> Self { - Self::from_rules_parametrised(rules, opts, true, true) + /// A helper for tests and benchmarks. Use [`Engine::new_with_filter_set`] instead. + #[doc(hidden)] + pub fn new_with_list_text(list_text: impl Into, opts: ParseOptions) -> Self { + Self::new_with_list_text_parametrised(list_text.into(), opts, false, true) } - pub fn from_rules_parametrised( - filter_rules: impl IntoIterator>, + /// A helper for tests and benchmarks. Use [`Engine::new_with_filter_set`] instead. + #[doc(hidden)] + pub fn new_with_list_text_parametrised( + list_text: impl Into, opts: ParseOptions, debug: bool, optimize: bool, ) -> Self { let mut filter_set = FilterSet::new(debug); - filter_set.add_filters(filter_rules, opts); - Self::from_filter_set(filter_set, optimize) + filter_set.add_filter_list(list_text.into(), opts); + Self::new_with_filter_set(filter_set, optimize) } #[cfg(test)] @@ -110,19 +101,74 @@ impl Engine { self.filter_data_context } - /// Loads rules from the given `FilterSet`. It is recommended to use a `FilterSet` when adding - /// rules from multiple sources. - pub fn from_filter_set(set: FilterSet, optimize: bool) -> Self { + /// A helper for tests and benchmarks. Use [`Engine::new_with_filter_set`] instead. + #[doc(hidden)] + pub fn new_with_parsed_rules( + network_filters: Vec, + cosmetic_filters: Vec, + optimize: bool, + ) -> Self { + let mut builder = EngineFlatBuilder::default(); + let mut network_rules_builder = NetworkRulesBuilder::new(optimize); + let mut cosmetic_filter_cache_builder = CosmeticFilterCacheBuilder::default(); + + for filter in network_filters { + network_rules_builder.add_filter(filter, &mut builder); + } + for filter in cosmetic_filters { + cosmetic_filter_cache_builder.add_filter(filter, &mut builder); + } + + Self::new_with_flatbuffer_offsets( + FlatSerialize::serialize(network_rules_builder, &mut builder), + FlatSerialize::serialize(cosmetic_filter_cache_builder, &mut builder), + builder, + ) + } + + /// Loads rules from the given `FilterSet`. + pub fn new_with_filter_set(set: FilterSet, optimize: bool) -> Self { let FilterSet { - network_filters, - cosmetic_filters, - .. + debug, + list_sources, } = set; + let mut builder = EngineFlatBuilder::default(); + let mut network_rules_builder = NetworkRulesBuilder::new(optimize); + let mut cosmetic_filter_cache_builder = CosmeticFilterCacheBuilder::default(); + + for list_source in &list_sources { + for line in list_source.list_text.lines() { + let parsed_line = parse_filter(line, debug, list_source.parse_options); + match parsed_line { + Ok(ParsedLine::Network(filter)) => { + network_rules_builder.add_filter(filter, &mut builder) + } + Ok(ParsedLine::Cosmetic(filter)) => { + cosmetic_filter_cache_builder.add_filter(filter, &mut builder) + } + Err(_) => {} + } + } + } + let network_rules_offset = FlatSerialize::serialize(network_rules_builder, &mut builder); + // Drop the list sources to reduce peak memory usage. + drop(list_sources); - let memory = make_flatbuffer(network_filters, cosmetic_filters, optimize); + let cosmetic_rules_offset = + FlatSerialize::serialize(cosmetic_filter_cache_builder, &mut builder); - let filter_data_context = FilterDataContext::new(memory); + Self::new_with_flatbuffer_offsets(network_rules_offset, cosmetic_rules_offset, builder) + } + fn new_with_flatbuffer_offsets<'a>( + network_rules: flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + cosmetic_rules: flatbuffers::WIPOffset>, + mut builder: EngineFlatBuilder<'a>, + ) -> Self { + let memory = builder.finish(network_rules, cosmetic_rules); + let filter_data_context = FilterDataContext::new(memory); Self { blocker: Blocker::from_context(FilterDataContextRef::clone(&filter_data_context)), cosmetic_cache: CosmeticFilterCache::from_context(FilterDataContextRef::clone( @@ -253,7 +299,7 @@ impl Engine { /// `hidden_class_id_selectors` to obtain any stylesheets consisting of generic rules (if the /// returned `generichide` value is false). pub fn url_cosmetic_resources(&self, url: &str) -> UrlSpecificResources { - let request = if let Ok(request) = Request::new(url, url, "document") { + let request = if let Ok(request) = Request::new(url, url, "document", "get") { request } else { return UrlSpecificResources::empty(); @@ -330,19 +376,6 @@ fn _assertions() { _assert_sync::(); } -fn make_flatbuffer( - network_filters: Vec, - cosmetic_filters: Vec, - optimize: bool, -) -> VerifiedFlatbufferMemory { - let mut builder = EngineFlatBuilder::default(); - let network_rules_builder = NetworkRulesBuilder::from_rules(network_filters, optimize); - let network_rules = FlatSerialize::serialize(network_rules_builder, &mut builder); - let cosmetic_rules = CosmeticFilterCacheBuilder::from_rules(cosmetic_filters, &mut builder); - let cosmetic_rules = FlatSerialize::serialize(cosmetic_rules, &mut builder); - builder.finish(network_rules, cosmetic_rules) -} - #[cfg(test)] #[path = "../tests/unit/engine.rs"] mod unit_tests; diff --git a/src/filters/abstract_network.rs b/src/filters/abstract_network.rs index b97c9b33..dfc62758 100644 --- a/src/filters/abstract_network.rs +++ b/src/filters/abstract_network.rs @@ -26,26 +26,90 @@ pub(crate) enum NetworkFilterRightAnchor { #[derive(Clone)] pub(crate) struct NetworkFilterPattern { pub(crate) left_anchor: Option, - pub(crate) pattern: String, + pub(crate) start: usize, + pub(crate) end: usize, pub(crate) right_anchor: Option, } +#[derive(Clone, Copy)] +pub(crate) enum HttpMethod { + Get, + Head, + Post, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum DomainValueKind { + Plain, + Entity, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ParsedDomainValue<'a> { + pub included: bool, + pub kind: DomainValueKind, + pub value: &'a str, + pub raw: &'a str, +} + +fn parse_domain_option_values<'a>( + value: &'a str, +) -> Result>, NetworkFilterError> { + let domains: Vec = value + .split('|') + .filter_map(|domain| { + let (included, rest) = if let Some(negated_domain) = domain.strip_prefix('~') { + (false, negated_domain) + } else { + (true, domain) + }; + + if (rest.starts_with('/') && rest.ends_with('/')) || rest.is_empty() { + return None; + } + + let (kind, normalized) = if let Some(entity_prefix) = rest.strip_suffix(".*") { + if entity_prefix.is_empty() { + return None; + } + (DomainValueKind::Entity, entity_prefix) + } else { + (DomainValueKind::Plain, rest) + }; + + Some(ParsedDomainValue { + included, + kind, + value: normalized, + raw: rest, + }) + }) + .collect(); + + if domains.is_empty() { + return Err(NetworkFilterError::NoSupportedDomains); + } + + Ok(domains) +} + /// Any option that appears on the right side of a network filter as initiated by a `$` character. /// All `bool` arguments below are `true` if the option stands alone, or `false` if the option is /// negated using a prepended `~`. #[derive(Clone)] -pub(crate) enum NetworkFilterOption { - Domain(Vec<(bool, String)>), +pub(crate) enum NetworkFilterOption<'a> { + Domain(Vec<(bool, &'a str)>), + To(Vec>), Badfilter, Important, MatchCase, ThirdParty(bool), FirstParty(bool), - Tag(String), - Redirect(String), - RedirectRule(String), - Csp(Option), - Removeparam(String), + Tag(&'a str), + Redirect(&'a str), + RedirectRule(&'a str), + Csp(Option<&'a str>), + Removeparam(&'a str), Generichide, Document, Image(bool), @@ -60,9 +124,10 @@ pub(crate) enum NetworkFilterOption { Websocket(bool), Font(bool), All, + Method(Vec<(bool, HttpMethod)>), } -impl NetworkFilterOption { +impl NetworkFilterOption<'_> { pub fn is_content_type(&self) -> bool { matches!( self, @@ -90,14 +155,16 @@ impl NetworkFilterOption { /// Abstract syntax representation of a network filter. This representation can fully specify the /// string representation of a filter as written, with the exception of aliased options like `1p` /// or `ghide`. This allows separation of concerns between parsing and interpretation. -pub(crate) struct AbstractNetworkFilter { +pub(crate) struct AbstractNetworkFilter<'a> { pub(crate) exception: bool, pub(crate) pattern: NetworkFilterPattern, - pub(crate) options: Option>, + pub(crate) options: Option>>, } -impl AbstractNetworkFilter { - pub(crate) fn parse(line: &str) -> Result { +impl AbstractNetworkFilter<'_> { + pub(crate) fn parse<'a>( + line: &'a str, + ) -> Result, NetworkFilterError> { let mut filter_index_start: usize = 0; let mut filter_index_end: usize = line.len(); @@ -139,13 +206,12 @@ impl AbstractNetworkFilter { None }; - let pattern = &line[filter_index_start..filter_index_end]; - Ok(AbstractNetworkFilter { exception, pattern: NetworkFilterPattern { left_anchor, - pattern: pattern.to_string(), + start: filter_index_start, + end: filter_index_end, right_anchor, }, options, @@ -153,7 +219,9 @@ impl AbstractNetworkFilter { } } -fn parse_filter_options(raw_options: &str) -> Result, NetworkFilterError> { +fn parse_filter_options<'a>( + raw_options: &'a str, +) -> Result>, NetworkFilterError> { let mut result = vec![]; for raw_option in raw_options.split(',') { @@ -170,22 +238,13 @@ fn parse_filter_options(raw_options: &str) -> Result, N result.push(match (option, negation) { ("domain", _) | ("from", _) => { - let domains: Vec<(bool, String)> = value - .split('|') - .map(|domain| { - if let Some(negated_domain) = domain.strip_prefix('~') { - (false, negated_domain.to_string()) - } else { - (true, domain.to_string()) - } - }) - .filter(|(_, d)| !(d.starts_with('/') && d.ends_with('/'))) + let domains = parse_domain_option_values(value)? + .into_iter() + .map(|entry| (entry.included, entry.raw)) .collect(); - if domains.is_empty() { - return Err(NetworkFilterError::NoSupportedDomains); - } NetworkFilterOption::Domain(domains) } + ("to", _) => NetworkFilterOption::To(parse_domain_option_values(value)?), ("badfilter", true) => return Err(NetworkFilterError::NegatedBadFilter), ("badfilter", false) => NetworkFilterOption::Badfilter, ("important", true) => return Err(NetworkFilterError::NegatedImportant), @@ -195,7 +254,7 @@ fn parse_filter_options(raw_options: &str) -> Result, N ("third-party", negated) | ("3p", negated) => NetworkFilterOption::ThirdParty(!negated), ("first-party", negated) | ("1p", negated) => NetworkFilterOption::FirstParty(!negated), ("tag", true) => return Err(NetworkFilterError::NegatedTag), - ("tag", false) => NetworkFilterOption::Tag(String::from(value)), + ("tag", false) => NetworkFilterOption::Tag(value), ("redirect", true) => return Err(NetworkFilterError::NegatedRedirection), ("redirect", false) => { // Ignore this filter if no redirection resource is specified @@ -203,7 +262,7 @@ fn parse_filter_options(raw_options: &str) -> Result, N return Err(NetworkFilterError::EmptyRedirection); } - NetworkFilterOption::Redirect(String::from(value)) + NetworkFilterOption::Redirect(value) } ("redirect-rule", true) => return Err(NetworkFilterError::NegatedRedirection), ("redirect-rule", false) => { @@ -211,13 +270,11 @@ fn parse_filter_options(raw_options: &str) -> Result, N return Err(NetworkFilterError::EmptyRedirection); } - NetworkFilterOption::RedirectRule(String::from(value)) + NetworkFilterOption::RedirectRule(value) + } + ("csp", _) => { + NetworkFilterOption::Csp(if !value.is_empty() { Some(value) } else { None }) } - ("csp", _) => NetworkFilterOption::Csp(if !value.is_empty() { - Some(String::from(value)) - } else { - None - }), ("removeparam", true) => return Err(NetworkFilterError::NegatedRemoveparam), ("removeparam", false) => { if value.is_empty() { @@ -226,7 +283,7 @@ fn parse_filter_options(raw_options: &str) -> Result, N if !VALID_PARAM.is_match(value) { return Err(NetworkFilterError::RemoveparamRegexUnsupported); } - NetworkFilterOption::Removeparam(String::from(value)) + NetworkFilterOption::Removeparam(value) } ("generichide", true) | ("ghide", true) => { return Err(NetworkFilterError::NegatedGenericHide) @@ -253,8 +310,41 @@ fn parse_filter_options(raw_options: &str) -> Result, N ("font", negated) => NetworkFilterOption::Font(!negated), ("all", true) => return Err(NetworkFilterError::NegatedAll), ("all", false) => NetworkFilterOption::All, + ("method", true) => return Err(NetworkFilterError::UnrecognisedOption), + ("method", false) => { + if value.is_empty() { + return Err(NetworkFilterError::UnrecognisedOption); + } + let methods: Vec<(bool, HttpMethod)> = value + .split('|') + .filter_map(|method| { + let (enabled, name) = if let Some(negated) = method.strip_prefix('~') { + (false, negated) + } else { + (true, method) + }; + if name.eq_ignore_ascii_case("get") { + Some((enabled, HttpMethod::Get)) + } else if name.eq_ignore_ascii_case("head") { + Some((enabled, HttpMethod::Head)) + } else if name.eq_ignore_ascii_case("post") { + Some((enabled, HttpMethod::Post)) + } else { + None + } + }) + .collect(); + if methods.is_empty() { + return Err(NetworkFilterError::UnrecognisedOption); + } + NetworkFilterOption::Method(methods) + } (_, _) => return Err(NetworkFilterError::UnrecognisedOption), }); } Ok(result) } + +#[cfg(test)] +#[path = "../../tests/unit/filters/abstract_network.rs"] +mod unit_tests; diff --git a/src/filters/fb_builder.rs b/src/filters/fb_builder.rs index 6ea470dd..b6a757e4 100644 --- a/src/filters/fb_builder.rs +++ b/src/filters/fb_builder.rs @@ -4,8 +4,7 @@ use std::collections::HashMap; use flatbuffers::WIPOffset; -use crate::filters::fb_network_builder::NetworkFilterListBuilder; -use crate::flatbuffers::containers::flat_serialize::{FlatBuilder, WIPFlatVec}; +use crate::flatbuffers::containers::flat_serialize::FlatBuilder; use crate::flatbuffers::unsafe_tools::VerifiedFlatbufferMemory; use crate::utils::Hash; @@ -31,7 +30,9 @@ impl<'a> EngineFlatBuilder<'a> { pub fn finish( &mut self, - network_rules: WIPFlatVec<'a, NetworkFilterListBuilder, EngineFlatBuilder<'a>>, + network_rules: flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, cosmetic_rules: WIPOffset>, ) -> VerifiedFlatbufferMemory { let unique_domains_hashes = diff --git a/src/filters/fb_network.rs b/src/filters/fb_network.rs index 505caa17..7ae32e25 100644 --- a/src/filters/fb_network.rs +++ b/src/filters/fb_network.rs @@ -8,24 +8,37 @@ use crate::regex_manager::RegexManager; use crate::request::Request; use crate::filters::flatbuffer_generated::fb; + /// A list of string parts that can be matched against a URL. -pub(crate) struct FlatPatterns<'a> { - patterns: Option>>, +pub(crate) enum FlatPatterns<'a> { + /// No patterns to match + Empty, + /// Memory-usage optimization - ~95% of filters have <= 1 pattern. Special-casing avoids the + /// need to hold an extra pointer and vector length. + Single(&'a str), + /// More than 1 pattern to match + Multi(flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<&'a str>>), } impl<'a> FlatPatterns<'a> { #[inline(always)] pub fn new( - patterns: Option>>, + single_pattern: Option<&'a str>, + multi_patterns: Option>>, ) -> Self { - Self { patterns } + if let Some(single_pattern) = single_pattern { + FlatPatterns::Single(single_pattern) + } else if let Some(patterns) = multi_patterns { + FlatPatterns::Multi(patterns) + } else { + FlatPatterns::Empty + } } #[inline(always)] pub fn iter(&self) -> FlatPatternsIterator<'_> { FlatPatternsIterator { patterns: self, - len: self.patterns.map_or(0, |d| d.len()), index: 0, } } @@ -34,7 +47,6 @@ impl<'a> FlatPatterns<'a> { /// Iterator over [FlatPatterns]. pub(crate) struct FlatPatternsIterator<'a> { patterns: &'a FlatPatterns<'a>, - len: usize, index: usize, } @@ -43,21 +55,134 @@ impl<'a> Iterator for FlatPatternsIterator<'a> { #[inline(always)] fn next(&mut self) -> Option { - self.patterns.patterns.and_then(|fi| { - if self.index < self.len { - self.index += 1; - Some(fi.get(self.index - 1)) - } else { - None + match &self.patterns { + FlatPatterns::Empty => None, + FlatPatterns::Single(s) => { + if self.index == 0 { + self.index += 1; + Some(*s) + } else { + None + } } - }) + FlatPatterns::Multi(v) => { + if self.index < v.len() { + let result = v.get(self.index); + self.index += 1; + Some(result) + } else { + None + } + } + } } } impl ExactSizeIterator for FlatPatternsIterator<'_> { #[inline(always)] fn len(&self) -> usize { - self.len + match &self.patterns { + FlatPatterns::Empty => 0, + FlatPatterns::Single(_) => 1_usize.saturating_sub(self.index), + FlatPatterns::Multi(v) => v.len().saturating_sub(self.index), + } + } +} + +#[derive(Clone, Copy, Default)] +pub(crate) struct ToOptionsFlags(u8); + +impl ToOptionsFlags { + const PLAIN_INC: u8 = 1 << 0; + const PLAIN_EXC: u8 = 1 << 1; + const ENTITY_INC: u8 = 1 << 2; + const ENTITY_EXC: u8 = 1 << 3; + + #[inline(always)] + pub fn any(self) -> bool { + self.0 != 0 + } + + #[inline(always)] + fn has_plain_inc(self) -> bool { + self.0 & Self::PLAIN_INC != 0 + } + + #[inline(always)] + fn has_plain_exc(self) -> bool { + self.0 & Self::PLAIN_EXC != 0 + } + + #[inline(always)] + fn has_entity_inc(self) -> bool { + self.0 & Self::ENTITY_INC != 0 + } + + #[inline(always)] + fn has_entity_exc(self) -> bool { + self.0 & Self::ENTITY_EXC != 0 + } + + #[inline(always)] + pub(crate) fn needs_plain(self) -> bool { + self.0 & (Self::PLAIN_INC | Self::PLAIN_EXC) != 0 + } + + #[inline(always)] + pub(crate) fn needs_entity(self) -> bool { + self.0 & (Self::ENTITY_INC | Self::ENTITY_EXC) != 0 + } + + #[cfg(test)] + pub(crate) fn from_bucket_parts( + include_plain: bool, + exclude_plain: bool, + include_entity: bool, + exclude_entity: bool, + ) -> Self { + let mut flags = 0u8; + if include_plain { + flags |= Self::PLAIN_INC; + } + if exclude_plain { + flags |= Self::PLAIN_EXC; + } + if include_entity { + flags |= Self::ENTITY_INC; + } + if exclude_entity { + flags |= Self::ENTITY_EXC; + } + Self(flags) + } + + fn from_filter(filter: &fb::NetworkFilter<'_>) -> Self { + let mut flags = 0u8; + if filter + .opt_to_domains() + .is_some_and(|data| !fb_vector_to_slice(data).is_empty()) + { + flags |= Self::PLAIN_INC; + } + if filter + .opt_not_to_domains() + .is_some_and(|data| !fb_vector_to_slice(data).is_empty()) + { + flags |= Self::PLAIN_EXC; + } + if filter + .opt_to_entities() + .is_some_and(|data| !fb_vector_to_slice(data).is_empty()) + { + flags |= Self::ENTITY_INC; + } + if filter + .opt_not_to_entities() + .is_some_and(|data| !fb_vector_to_slice(data).is_empty()) + { + flags |= Self::ENTITY_EXC; + } + Self(flags) } } @@ -68,6 +193,7 @@ pub(crate) struct FlatNetworkFilter<'a> { fb_filter: &'a fb::NetworkFilter<'a>, pub(crate) mask: NetworkFilterMask, + to_options: ToOptionsFlags, } impl<'a> FlatNetworkFilter<'a> { @@ -84,6 +210,7 @@ impl<'a> FlatNetworkFilter<'a> { key, fb_filter: filter, mask: NetworkFilterMask::from_bits_retain(filter.mask()), + to_options: ToOptionsFlags::from_filter(filter), filter_data_context, } } @@ -112,6 +239,34 @@ impl<'a> FlatNetworkFilter<'a> { .map(|data| fb_vector_to_slice(data)) } + #[inline(always)] + pub fn include_to_domains(&self) -> Option<&[u32]> { + self.fb_filter + .opt_to_domains() + .map(|data| fb_vector_to_slice(data)) + } + + #[inline(always)] + pub fn exclude_to_domains(&self) -> Option<&[u32]> { + self.fb_filter + .opt_not_to_domains() + .map(|data| fb_vector_to_slice(data)) + } + + #[inline(always)] + pub fn include_to_entities(&self) -> Option<&[u32]> { + self.fb_filter + .opt_to_entities() + .map(|data| fb_vector_to_slice(data)) + } + + #[inline(always)] + pub fn exclude_to_entities(&self) -> Option<&[u32]> { + self.fb_filter + .opt_not_to_entities() + .map(|data| fb_vector_to_slice(data)) + } + #[inline(always)] pub fn hostname(&self) -> Option<&'a str> { if self.mask.is_hostname_anchor() { @@ -123,7 +278,10 @@ impl<'a> FlatNetworkFilter<'a> { #[inline(always)] pub fn patterns(&self) -> FlatPatterns<'_> { - FlatPatterns::new(self.fb_filter.patterns()) + FlatPatterns::new( + self.fb_filter.single_pattern(), + self.fb_filter.multi_patterns(), + ) } #[inline(always)] @@ -142,7 +300,8 @@ impl NetworkFilterMaskHelper for FlatNetworkFilter<'_> { impl NetworkMatchable for FlatNetworkFilter<'_> { fn matches(&self, request: &Request, regex_manager: &mut RegexManager) -> bool { use crate::filters::network_matchers::{ - check_excluded_domains_mapped, check_included_domains_mapped, check_options, + check_excluded_domains_mapped, check_excluded_to_options_mapped, + check_included_domains_mapped, check_included_to_options_mapped, check_options, check_pattern, }; if !check_options(self.mask, request) { @@ -162,6 +321,46 @@ impl NetworkMatchable for FlatNetworkFilter<'_> { ) { return false; } + if self.to_options.any() { + let mapping = &self.filter_data_context.unique_domains_hashes_map; + let to_capability = self.filter_data_context.to_rule_capability(); + if !check_included_to_options_mapped( + self.to_options, + if self.to_options.has_plain_inc() { + self.include_to_domains() + } else { + None + }, + if self.to_options.has_entity_inc() { + self.include_to_entities() + } else { + None + }, + request, + mapping, + to_capability, + ) { + return false; + } + if !check_excluded_to_options_mapped( + self.to_options, + if self.to_options.has_plain_exc() { + self.exclude_to_domains() + } else { + None + }, + if self.to_options.has_entity_exc() { + self.exclude_to_entities() + } else { + None + }, + request, + mapping, + to_capability, + ) { + return false; + } + } check_pattern( self.mask, self.patterns().iter(), diff --git a/src/filters/fb_network_builder.rs b/src/filters/fb_network_builder.rs index 115bcbcc..92ec33ff 100644 --- a/src/filters/fb_network_builder.rs +++ b/src/filters/fb_network_builder.rs @@ -11,7 +11,7 @@ use crate::utils::TokensBuffer; use crate::filters::network::NetworkFilterMaskHelper; use crate::flatbuffers::containers::flat_multimap::FlatMultiMapBuilder; -use crate::flatbuffers::containers::flat_serialize::{FlatBuilder, FlatSerialize, WIPFlatVec}; +use crate::flatbuffers::containers::flat_serialize::{FlatBuilder, FlatSerialize}; use crate::optimizer; use crate::utils::{to_short_hash, Hash, ShortHash}; @@ -29,21 +29,29 @@ pub(crate) enum NetworkFilterListId { Size = 8, } -#[derive(Default, Clone)] -pub(crate) struct NetworkFilterListBuilder { - filters: Vec, +struct NetworkFilterFlatEntry<'a> { + filter: WIPOffset>, + id: Hash, +} + +struct NetworkFilterListBuilder<'a, 'f> { + flat_map_builder: FlatMultiMapBuilder>, + token_frequencies: TokenSelector, + filters_to_optimize: HashMap>>, + tokens_buffer: TokensBuffer, optimize: bool, } -pub(crate) struct NetworkRulesBuilder { - lists: Vec, +pub(crate) struct NetworkRulesBuilder<'a, 'f> { + lists: Vec>, + bad_filter_ids: HashSet, } -impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for &NetworkFilter { +impl<'f, 'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilter<'f> { type Output = WIPOffset>; fn serialize( - network_filter: &NetworkFilter, + network_filter: NetworkFilter<'f>, builder: &mut EngineFlatBuilder<'a>, ) -> WIPOffset> { let opt_domains = network_filter.opt_domains.as_ref().map(|v| { @@ -66,220 +74,277 @@ impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for &NetworkFilter { FlatSerialize::serialize(o, builder) }); + let opt_to_domains = network_filter.opt_to_domains.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + + let opt_not_to_domains = network_filter.opt_not_to_domains.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + + let opt_to_entities = network_filter.opt_to_entities.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + + let opt_not_to_entities = network_filter.opt_not_to_entities.as_ref().map(|v| { + let mut o: Vec = v + .iter() + .map(|x| builder.get_or_insert_unique_domain_hash(x)) + .collect(); + o.sort_unstable(); + o.dedup(); + FlatSerialize::serialize(o, builder) + }); + let modifier_option = network_filter .modifier_option - .as_ref() .map(|s| builder.create_string(s)); let hostname = network_filter .hostname .as_ref() - .map(|s| builder.create_string(s)); + .map(|s| builder.create_string(s.as_ref())); - let tag = network_filter - .tag - .as_ref() - .map(|s| builder.create_string(s)); + let tag = network_filter.tag.map(|s| builder.create_string(s)); - let patterns = if network_filter.filter.iter().len() > 0 { - let offsets: Vec> = network_filter - .filter - .iter() - .map(|s| builder.create_string(s)) - .collect(); - Some(FlatSerialize::serialize(offsets, builder)) + let mut filter_iter = network_filter.filter.iter(); + let filter_count = filter_iter.len(); + + // Use single_pattern for the common case of 0 or 1 patterns to avoid + // the overhead of a FlatBuffers vector (extra table + offset indirection). + let (single_pattern, multi_patterns) = if filter_count <= 1 { + let single = filter_iter.next().map(|s| builder.create_string(s)); + (single, None) } else { - None + ( + None, + Some(FlatSerialize::serialize( + filter_iter.collect::>(), + builder, + )), + ) }; let raw_line = network_filter .raw_line .as_ref() - .map(|v| builder.create_string(v.as_str())); + .map(|v| builder.create_string(v.as_ref())); - let network_filter = fb::NetworkFilter::create( + fb::NetworkFilter::create( builder.raw_builder(), &fb::NetworkFilterArgs { mask: network_filter.mask.bits(), - patterns, + single_pattern, + multi_patterns, modifier_option, opt_domains, opt_not_domains, + opt_to_domains, + opt_not_to_domains, + opt_to_entities, + opt_not_to_entities, hostname, tag, raw_line, }, - ); - - network_filter + ) } } -impl NetworkFilterListBuilder { +impl<'a, 'f> NetworkFilterListBuilder<'a, 'f> { fn new(optimize: bool) -> Self { Self { - filters: vec![], + flat_map_builder: FlatMultiMapBuilder::with_capacity(1024), + token_frequencies: TokenSelector::new(1024), + filters_to_optimize: HashMap::new(), + tokens_buffer: TokensBuffer::default(), optimize, } } -} -impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterListBuilder { - type Output = WIPOffset>; - fn serialize( - rule_list: Self, + fn add_filter( + &mut self, + network_filter: NetworkFilter<'f>, builder: &mut EngineFlatBuilder<'a>, - ) -> WIPOffset> { - let mut filter_map_builder = FlatMultiMapBuilder::with_capacity(rule_list.filters.len()); - - let mut optimizable = HashMap::>::new(); - - let mut token_frequencies = TokenSelector::new(rule_list.filters.len()); - let mut tokens_buffer = TokensBuffer::default(); - - { - for network_filter in rule_list.filters { - let flat_filter = if !rule_list.optimize - || !optimizer::is_filter_optimizable_by_patterns(&network_filter) - { - Some(FlatSerialize::serialize(&network_filter, builder)) - } else { - None - }; - - let mut store_filter = |token: Hash| { - let short_token = to_short_hash(token); - if let Some(flat_filter) = flat_filter { - filter_map_builder.insert(short_token, flat_filter); - } else { - optimizable - .entry(short_token) - .or_default() - .push(network_filter.clone()); - } - }; - - let multi_tokens = network_filter.get_tokens(&mut tokens_buffer); - match multi_tokens { - FilterTokens::Empty => { - // No tokens, add to fallback bucket (token 0) - store_filter(0); - } - FilterTokens::OptDomains(opt_domains) => { - // For OptDomains, each domain is treated as a separate token group - for &token in opt_domains.iter() { - store_filter(token); - token_frequencies.record_usage(token); - } - } - FilterTokens::Other(tokens) => { - let best_token = token_frequencies.select_least_used_token(tokens); - token_frequencies.record_usage(best_token); - store_filter(best_token); - } - } + ) { + let multi_tokens = network_filter.get_tokens(&mut self.tokens_buffer); + let id = network_filter.get_id(); + + // Resolve token(s) and record frequencies up-front so the + // serialized/optimizable branches share no token logic. + let single_token: Hash; + let tokens: &[Hash] = match multi_tokens { + FilterTokens::Empty => { + // No tokens, add to fallback bucket (token 0) + &[0] } - } - - if rule_list.optimize { - // Sort the entries to ensure deterministic iteration order - let mut optimizable_entries: Vec<_> = optimizable.drain().collect(); - optimizable_entries.sort_unstable_by_key(|(token, _)| *token); - - for (token, v) in optimizable_entries { - let optimized = optimizer::optimize(v); + FilterTokens::OptDomains => { + // tokens_buffer has been populated by get_tokens + self.tokens_buffer.as_slice() + } + FilterTokens::Other => { + single_token = self + .token_frequencies + .select_least_used_token(self.tokens_buffer.as_slice()); + std::slice::from_ref(&single_token) + } + }; - for filter in optimized { - let flat_filter = FlatSerialize::serialize(&filter, builder); - filter_map_builder.insert(token, flat_filter); - } + if !self.optimize || !optimizer::is_filter_optimizable_by_patterns(&network_filter) { + // Serialize now (even if it matches to a bad filter later); + // Although store the id for later bad-filter pruning. + let filter = FlatSerialize::serialize(network_filter, builder); + for &token in tokens { + self.token_frequencies.record_usage(token); + self.flat_map_builder + .insert(to_short_hash(token), NetworkFilterFlatEntry { filter, id }); } } else { - debug_assert!( - optimizable.is_empty(), - "Should be empty if optimization is off" - ); + // Defer serialization to the optimizer + for &token in tokens { + self.token_frequencies.record_usage(token); + self.filters_to_optimize + .entry(to_short_hash(token)) + .or_default() + .push(network_filter.clone()); + } } - - let flat_filter_map = FlatMultiMapBuilder::finish(filter_map_builder, builder); - - fb::NetworkFilterList::create( - builder.raw_builder(), - &fb::NetworkFilterListArgs { - filter_map_index: Some(flat_filter_map.keys), - filter_map_values: Some(flat_filter_map.values), - }, - ) } } -impl NetworkRulesBuilder { - pub fn from_rules(network_filters: Vec, optimize: bool) -> Self { - let mut lists = vec![]; - for list_id in 0..NetworkFilterListId::Size as usize { - // Don't optimize removeparam, since it can fuse filters without respecting distinct - let optimize = optimize && list_id != NetworkFilterListId::RemoveParam as usize; - lists.push(NetworkFilterListBuilder::new(optimize)); +impl<'a, 'f> NetworkRulesBuilder<'a, 'f> { + pub fn new(optimize: bool) -> Self { + let lists = (0..NetworkFilterListId::Size as usize) + .map(|list_id| { + // Don't optimize removeparam, since it can fuse filters without respecting distinct + let optimize = optimize && list_id != NetworkFilterListId::RemoveParam as usize; + NetworkFilterListBuilder::new(optimize) + }) + .collect::>(); + Self { + lists, + bad_filter_ids: HashSet::with_capacity(1024), } - let mut self_ = Self { lists }; - - let mut badfilter_ids: HashSet = HashSet::new(); + } - // Collect badfilter ids in advance. - for filter in network_filters.iter() { - if filter.is_badfilter() { - badfilter_ids.insert(filter.get_id_without_badfilter()); - } + pub fn add_filter(&mut self, filter: NetworkFilter<'f>, builder: &mut EngineFlatBuilder<'a>) { + if filter.is_badfilter() { + // Note: `get_id()` doesn't include BAD_FILTER bit. + self.bad_filter_ids.insert(filter.get_id()); + return; } - for filter in network_filters.into_iter() { - // skip any bad filters - let filter_id = filter.get_id(); - if badfilter_ids.contains(&filter_id) || filter.is_badfilter() { - continue; - } - - // Redirects are independent of blocking behavior. - if filter.is_redirect() { - self_.add_filter(filter.clone(), NetworkFilterListId::Redirects); - } - type FilterId = NetworkFilterListId; - - let list_id: FilterId = if filter.is_csp() { - FilterId::Csp - } else if filter.is_removeparam() { - FilterId::RemoveParam - } else if filter.is_generic_hide() { - FilterId::GenericHide - } else if filter.is_exception() { - FilterId::Exceptions - } else if filter.is_important() { - FilterId::Importants - } else if filter.tag.is_some() && !filter.is_redirect() { - // `tag` + `redirect` is unsupported for now. - FilterId::TaggedFiltersAll - } else if (filter.is_redirect() && filter.also_block_redirect()) - || !filter.is_redirect() - { - FilterId::Filters - } else { - continue; - }; - - self_.add_filter(filter, list_id); + // Redirects are independent of blocking behavior. + if filter.is_redirect() { + self.add_filter_internal(filter.clone(), NetworkFilterListId::Redirects, builder); } + type FilterId = NetworkFilterListId; + + let list_id: FilterId = if filter.is_csp() { + FilterId::Csp + } else if filter.is_removeparam() { + FilterId::RemoveParam + } else if filter.is_generic_hide() { + FilterId::GenericHide + } else if filter.is_exception() { + FilterId::Exceptions + } else if filter.is_important() { + FilterId::Importants + } else if filter.tag.is_some() && !filter.is_redirect() { + // `tag` + `redirect` is unsupported for now. + FilterId::TaggedFiltersAll + } else if (filter.is_redirect() && filter.also_block_redirect()) || !filter.is_redirect() { + FilterId::Filters + } else { + return; + }; - self_ + self.add_filter_internal(filter, list_id, builder); } - fn add_filter(&mut self, network_filter: NetworkFilter, list_id: NetworkFilterListId) { - self.lists[list_id as usize].filters.push(network_filter); + fn add_filter_internal( + &mut self, + network_filter: NetworkFilter<'f>, + list_id: NetworkFilterListId, + builder: &mut EngineFlatBuilder<'a>, + ) { + self.lists[list_id as usize].add_filter(network_filter, builder); } } -impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder { - type Output = WIPFlatVec<'a, NetworkFilterListBuilder, EngineFlatBuilder<'a>>; +impl<'a> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkFilterFlatEntry<'a> { + type Output = WIPOffset>; + + fn serialize( + value: Self, + _builder: &mut EngineFlatBuilder<'a>, + ) -> WIPOffset> { + value.filter + } +} + +impl<'a, 'f> FlatSerialize<'a, EngineFlatBuilder<'a>> for NetworkRulesBuilder<'a, 'f> { + type Output = + WIPOffset>>>; + fn serialize(value: Self, builder: &mut EngineFlatBuilder<'a>) -> Self::Output { - FlatSerialize::serialize(value.lists, builder) + let mut serialized_lists = vec![]; + + for mut rule_list in value.lists { + if !rule_list.filters_to_optimize.is_empty() { + // Sort entries for deterministic iteration order. + let mut optimizable_entries: Vec<_> = + rule_list.filters_to_optimize.drain().collect(); + optimizable_entries.sort_unstable_by_key(|(token, _)| *token); + + for (token, mut v) in optimizable_entries { + v.retain(|f| !value.bad_filter_ids.contains(&f.get_id())); + let optimized = optimizer::optimize(v); + + for filter in optimized { + let id = filter.get_id(); + let filter = FlatSerialize::serialize(filter, builder); + rule_list + .flat_map_builder + .insert(token, NetworkFilterFlatEntry { filter, id }); + } + } + } + + // Prune already-serialized entries that were cancelled by a $badfilter. + rule_list + .flat_map_builder + .retain_by_value(|entry| !value.bad_filter_ids.contains(&entry.id)); + + let flat_filter_map = FlatMultiMapBuilder::finish(rule_list.flat_map_builder, builder); + + serialized_lists.push(fb::NetworkFilterList::create( + builder.raw_builder(), + &fb::NetworkFilterListArgs { + filter_map_index: Some(flat_filter_map.keys), + filter_map_values: Some(flat_filter_map.values), + }, + )); + } + + FlatSerialize::serialize(serialized_lists, builder) } } diff --git a/src/filters/filter_data_context.rs b/src/filters/filter_data_context.rs index 985ef1eb..e4ca1feb 100644 --- a/src/filters/filter_data_context.rs +++ b/src/filters/filter_data_context.rs @@ -1,3 +1,4 @@ +use crate::filters::flatbuffer_generated::fb; use crate::flatbuffers::unsafe_tools::VerifiedFlatbufferMemory; use crate::utils::Hash; use std::collections::HashMap; @@ -10,9 +11,67 @@ pub(crate) type FilterDataContextRef = std::sync::Arc; // The struct is used to store the flatbuffer and supporting data // for both network filter and cosmetic filters. // Supposed to be stored via FilterDataContextRef to avoid copying the data. +#[derive(Clone, Copy)] +pub(crate) struct ToRuleCapability { + pub has_plain: bool, + pub has_entity: bool, +} + +impl ToRuleCapability { + #[inline(always)] + #[cfg(test)] + pub const fn all() -> Self { + Self { + has_plain: true, + has_entity: true, + } + } +} + pub(crate) struct FilterDataContext { pub(crate) memory: VerifiedFlatbufferMemory, pub(crate) unique_domains_hashes_map: HashMap, + pub(crate) has_to_plain_rules: bool, + pub(crate) has_to_entity_rules: bool, +} + +impl FilterDataContext { + #[inline(always)] + pub(crate) fn to_rule_capability(&self) -> ToRuleCapability { + ToRuleCapability { + has_plain: self.has_to_plain_rules, + has_entity: self.has_to_entity_rules, + } + } +} + +fn non_empty_fb_slice(data: flatbuffers::Vector<'_, u32>) -> bool { + !data.is_empty() +} + +fn scan_to_option_flags(root: fb::Engine<'_>) -> (bool, bool) { + let mut has_to_plain_rules = false; + let mut has_to_entity_rules = false; + + for list in root.network_rules().iter() { + for filter in list.filter_map_values().iter() { + if filter.opt_to_domains().is_some_and(non_empty_fb_slice) + || filter.opt_not_to_domains().is_some_and(non_empty_fb_slice) + { + has_to_plain_rules = true; + } + if filter.opt_to_entities().is_some_and(non_empty_fb_slice) + || filter.opt_not_to_entities().is_some_and(non_empty_fb_slice) + { + has_to_entity_rules = true; + } + if has_to_plain_rules && has_to_entity_rules { + return (true, true); + } + } + } + + (has_to_plain_rules, has_to_entity_rules) } impl FilterDataContext { @@ -23,9 +82,12 @@ impl FilterDataContext { for (index, hash) in root.unique_domains_hashes().iter().enumerate() { unique_domains_hashes_map.insert(hash, index as u32); } + let (has_to_plain_rules, has_to_entity_rules) = scan_to_option_flags(root); FilterDataContextRef::new(Self { memory, unique_domains_hashes_map, + has_to_plain_rules, + has_to_entity_rules, }) } } diff --git a/src/filters/network.rs b/src/filters/network.rs index 9bd66729..bddeb626 100644 --- a/src/filters/network.rs +++ b/src/filters/network.rs @@ -3,15 +3,18 @@ use memchr::memchr as find_char; use regex::Regex; +use rustc_hash::FxHasher; use serde::{Deserialize, Serialize}; use std::borrow::Cow; +use std::hash::Hasher; use std::sync::LazyLock; use thiserror::Error; use std::fmt; use crate::filters::abstract_network::{ - AbstractNetworkFilter, NetworkFilterLeftAnchor, NetworkFilterOption, NetworkFilterRightAnchor, + AbstractNetworkFilter, DomainValueKind, HttpMethod, NetworkFilterLeftAnchor, + NetworkFilterOption, NetworkFilterRightAnchor, }; use crate::lists::ParseOptions; use crate::regex_manager::RegexManager; @@ -21,6 +24,24 @@ use crate::utils::{self, Hash, TokensBuffer}; /// For now, only support `$removeparam` with simple alphanumeric/dash/underscore patterns. static VALID_PARAM: LazyLock = LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9_\-]+$").unwrap()); +bitflags::bitflags! { + /// Features that are properties used to classify the filter, but not stored + /// in the flatbuffer serialized format. For that reason, not available + /// for FlatNetworkFilter (use NetworkFilterMask instead). + #[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Default)] + pub struct NetworkFilterFeaturesMask: u32 { + const BAD_FILTER = 1 << 0; + const IS_REMOVEPARAM = 1 << 1; + const GENERIC_HIDE = 1 << 2; + const IS_CSP = 1 << 3; + const IS_REDIRECT = 1 << 4; + + /// Specifies that a redirect rule should also create a corresponding block rule. + /// This is used to avoid returning two separate rules from `NetworkFilter::parse`. + const ALSO_BLOCK_REDIRECT = 1 << 5; + } +} + #[non_exhaustive] #[derive(Debug, Error, PartialEq, Clone)] pub enum NetworkFilterError { @@ -46,6 +67,8 @@ pub enum NetworkFilterError { NegatedAll, #[error("generichide without exception")] GenericHideWithoutException, + #[error("method with generichide")] + MethodWithGenerichide, #[error("empty redirection")] EmptyRedirection, #[error("empty removeparam")] @@ -97,13 +120,8 @@ bitflags::bitflags! { const FROM_HTTPS = 1 << 12; const IS_IMPORTANT = 1 << 13; const MATCH_CASE = 1 << 14; - const IS_REMOVEPARAM = 1 << 15; const THIRD_PARTY = 1 << 16; const FIRST_PARTY = 1 << 17; - const IS_REDIRECT = 1 << 26; - const BAD_FILTER = 1 << 27; - const GENERIC_HIDE = 1 << 30; - // Full document rules are not implied by negated types. const FROM_DOCUMENT = 1 << 29; @@ -113,17 +131,20 @@ bitflags::bitflags! { const IS_RIGHT_ANCHOR = 1 << 20; const IS_HOSTNAME_ANCHOR = 1 << 21; const IS_EXCEPTION = 1 << 22; - const IS_CSP = 1 << 23; const IS_COMPLETE_REGEX = 1 << 24; const IS_HOSTNAME_REGEX = 1 << 28; - // Specifies that a redirect rule should also create a corresponding block rule. - // This is used to avoid returning two separate rules from `NetworkFilter::parse`. - const ALSO_BLOCK_REDIRECT = 1 << 31; - // "Other" network request types const UNMATCHED = 1 << 25; + const FROM_GET = 1 << 26; + const FROM_HEAD = 1 << 27; + const FROM_POST = 1 << 30; + + const FROM_ANY_METHODS = Self::FROM_GET.bits() | + Self::FROM_HEAD.bits() | + Self::FROM_POST.bits(); + // Includes all request types that are implied by any negated types. const FROM_NETWORK_TYPES = Self::FROM_FONT.bits() | Self::FROM_IMAGE.bits() | @@ -154,6 +175,23 @@ bitflags::bitflags! { } } +impl NetworkFilterMask { + #[inline] + pub(crate) fn check_method_allowed(self, method: Option<&request::RequestMethod>) -> bool { + if !self.intersects(Self::FROM_ANY_METHODS) { + return true; + } + let Some(method) = method else { + return false; + }; + let method_mask = Self::from(method); + if method_mask.is_empty() { + return false; + } + self.contains(method_mask) + } +} + pub trait NetworkFilterMaskHelper { fn has_flag(&self, v: NetworkFilterMask) -> bool; @@ -187,31 +225,6 @@ pub trait NetworkFilterMaskHelper { self.has_flag(NetworkFilterMask::IS_IMPORTANT) } - #[inline] - fn is_redirect(&self) -> bool { - self.has_flag(NetworkFilterMask::IS_REDIRECT) - } - - #[inline] - fn is_removeparam(&self) -> bool { - self.has_flag(NetworkFilterMask::IS_REMOVEPARAM) - } - - #[inline] - fn also_block_redirect(&self) -> bool { - self.has_flag(NetworkFilterMask::ALSO_BLOCK_REDIRECT) - } - - #[inline] - fn is_badfilter(&self) -> bool { - self.has_flag(NetworkFilterMask::BAD_FILTER) - } - - #[inline] - fn is_generic_hide(&self) -> bool { - self.has_flag(NetworkFilterMask::GENERIC_HIDE) - } - #[inline] fn is_regex(&self) -> bool { self.has_flag(NetworkFilterMask::IS_REGEX) @@ -227,11 +240,6 @@ pub trait NetworkFilterMaskHelper { !self.is_regex() } - #[inline] - fn is_csp(&self) -> bool { - self.has_flag(NetworkFilterMask::IS_CSP) - } - #[inline] fn third_party(&self) -> bool { self.has_flag(NetworkFilterMask::THIRD_PARTY) @@ -279,6 +287,17 @@ impl fmt::Display for NetworkFilterMask { } } +impl From<&request::RequestMethod> for NetworkFilterMask { + fn from(method: &request::RequestMethod) -> NetworkFilterMask { + match method { + request::RequestMethod::Get => NetworkFilterMask::FROM_GET, + request::RequestMethod::Head => NetworkFilterMask::FROM_HEAD, + request::RequestMethod::Post => NetworkFilterMask::FROM_POST, + _ => NetworkFilterMask::NONE, + } + } +} + impl From<&request::RequestType> for NetworkFilterMask { fn from(request_type: &request::RequestType) -> NetworkFilterMask { match request_type { @@ -304,21 +323,21 @@ impl From<&request::RequestType> for NetworkFilterMask { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum FilterPart { +pub enum FilterPart<'a> { Empty, - Simple(String), + Simple(Cow<'a, str>), AnyOf(Vec), } #[derive(Debug, PartialEq)] -pub(crate) enum FilterTokens<'a> { +pub(crate) enum FilterTokens { Empty, - OptDomains(&'a [Hash]), - Other(&'a [Hash]), + OptDomains, + Other, } pub struct FilterPartIterator<'a> { - filter_part: &'a FilterPart, + filter_part: &'a FilterPart<'a>, index: usize, } @@ -331,14 +350,14 @@ impl<'a> Iterator for FilterPartIterator<'a> { FilterPart::Simple(s) => { if self.index == 0 { self.index += 1; - Some(s.as_str()) + Some(s.as_ref()) } else { None } } FilterPart::AnyOf(vec) => { if self.index < vec.len() { - let result = Some(vec[self.index].as_str()); + let result = Some(vec[self.index].as_ref()); self.index += 1; result } else { @@ -354,18 +373,23 @@ impl ExactSizeIterator for FilterPartIterator<'_> { fn len(&self) -> usize { match self.filter_part { FilterPart::Empty => 0, - FilterPart::Simple(_) => 1, - FilterPart::AnyOf(vec) => vec.len(), + FilterPart::Simple(_) => 1usize.saturating_sub(self.index), + FilterPart::AnyOf(vec) => vec.len().saturating_sub(self.index), } } } -impl FilterPart { +impl FilterPart<'_> { pub fn string_view(&self) -> Option { match &self { FilterPart::Empty => None, - FilterPart::Simple(s) => Some(s.clone()), - FilterPart::AnyOf(s) => Some(s.join("|")), + FilterPart::Simple(s) => Some(s.to_string()), + FilterPart::AnyOf(s) => Some( + s.iter() + .map(|part| part.as_ref()) + .collect::>() + .join("|"), + ), } } @@ -378,18 +402,23 @@ impl FilterPart { } #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NetworkFilter { +pub struct NetworkFilter<'a> { pub mask: NetworkFilterMask, - pub filter: FilterPart, + pub features_mask: NetworkFilterFeaturesMask, + pub filter: FilterPart<'a>, pub opt_domains: Option>, pub opt_not_domains: Option>, + pub opt_to_domains: Option>, + pub opt_not_to_domains: Option>, + pub opt_to_entities: Option>, + pub opt_not_to_entities: Option>, /// Used for `$redirect`, `$redirect-rule`, `$csp`, and `$removeparam` - only one of which is /// supported per-rule. - pub modifier_option: Option, - pub hostname: Option, - pub(crate) tag: Option, + pub modifier_option: Option<&'a str>, + pub hostname: Option>, + pub(crate) tag: Option<&'a str>, - pub raw_line: Option>, + pub raw_line: Option>, pub id: Hash, } @@ -398,21 +427,21 @@ pub struct NetworkFilter { // prevent field access, and don't load the ID from the serialized format. /// The ID of a filter is assumed to be correctly calculated for the purposes of this /// implementation. -impl PartialEq for NetworkFilter { +impl PartialEq for NetworkFilter<'_> { fn eq(&self, other: &Self) -> bool { self.id == other.id } } /// Filters are sorted by ID to preserve a stable ordering of data in the serialized format. -impl PartialOrd for NetworkFilter { +impl PartialOrd for NetworkFilter<'_> { fn partial_cmp(&self, other: &Self) -> Option { self.id.partial_cmp(&other.id) } } /// Ensure that no invalid option combinations were provided for a filter. -fn validate_options(options: &[NetworkFilterOption]) -> Result<(), NetworkFilterError> { +fn validate_options(options: &[NetworkFilterOption<'_>]) -> Result<(), NetworkFilterError> { let mut has_csp = false; let mut has_content_type = false; let mut modifier_options = 0; @@ -437,8 +466,25 @@ fn validate_options(options: &[NetworkFilterOption]) -> Result<(), NetworkFilter Ok(()) } -impl NetworkFilter { - pub fn parse(line: &str, debug: bool, _opts: ParseOptions) -> Result { +fn cow_ascii_lowercase<'a>(s: &'a str) -> Cow<'a, str> { + if s.as_bytes().iter().any(|byte| byte.is_ascii_uppercase()) { + Cow::Owned(s.to_ascii_lowercase()) + } else { + Cow::Borrowed(s) + } +} + +fn decode_hostname<'a>(host: &'a [u8]) -> Result, NetworkFilterError> { + idna::domain_to_ascii_cow(host, idna::AsciiDenyList::EMPTY) + .map_err(|_| NetworkFilterError::PunycodeError) +} + +impl<'a> NetworkFilter<'a> { + pub fn parse( + line: &'a str, + debug: bool, + _opts: ParseOptions, + ) -> Result { let parsed = AbstractNetworkFilter::parse(line)?; // Represent options as a bitmask @@ -446,19 +492,26 @@ impl NetworkFilter { | NetworkFilterMask::FIRST_PARTY | NetworkFilterMask::FROM_HTTPS | NetworkFilterMask::FROM_HTTP; + let mut features_mask: NetworkFilterFeaturesMask = Default::default(); // Temporary masks for positive (e.g.: $script) and negative (e.g.: $~script) // content type options. let mut cpt_mask_positive: NetworkFilterMask = NetworkFilterMask::NONE; let mut cpt_mask_negative: NetworkFilterMask = NetworkFilterMask::NONE; + let mut method_mask_positive: NetworkFilterMask = NetworkFilterMask::NONE; + let mut method_mask_negative: NetworkFilterMask = NetworkFilterMask::NONE; - let mut hostname: Option = None; + let mut hostname: Option<&'a [u8]> = None; let mut opt_domains: Option> = None; let mut opt_not_domains: Option> = None; + let mut opt_to_domains: Option> = None; + let mut opt_not_to_domains: Option> = None; + let mut opt_to_entities: Option> = None; + let mut opt_not_to_entities: Option> = None; - let mut modifier_option: Option = None; - let mut tag: Option = None; + let mut modifier_option: Option<&'a str> = None; + let mut tag: Option<&'a str> = None; if parsed.exception { mask.set(NetworkFilterMask::IS_EXCEPTION, true); @@ -477,6 +530,16 @@ impl NetworkFilter { }; } + macro_rules! apply_method { + ($method:ident, $enabled:ident) => { + if $enabled { + method_mask_positive.set(NetworkFilterMask::$method, true); + } else { + method_mask_negative.set(NetworkFilterMask::$method, true); + } + }; + } + options.into_iter().for_each(|option| { match option { NetworkFilterOption::Domain(domains) => { @@ -484,7 +547,7 @@ impl NetworkFilter { let mut opt_not_domains_array: Vec = vec![]; for (enabled, domain) in domains { - let domain_hash = utils::fast_hash(&domain); + let domain_hash = utils::fast_hash(domain); if !enabled { opt_not_domains_array.push(domain_hash); } else { @@ -505,7 +568,54 @@ impl NetworkFilter { opt_not_domains = Some(opt_not_domains_array); } } - NetworkFilterOption::Badfilter => mask.set(NetworkFilterMask::BAD_FILTER, true), + NetworkFilterOption::To(values) => { + let mut opt_to_domains_array: Vec = vec![]; + let mut opt_not_to_domains_array: Vec = vec![]; + let mut opt_to_entities_array: Vec = vec![]; + let mut opt_not_to_entities_array: Vec = vec![]; + + for entry in values { + let hash = utils::fast_hash(entry.value); + match (entry.kind, entry.included) { + (DomainValueKind::Plain, true) => { + opt_to_domains_array.push(hash); + } + (DomainValueKind::Plain, false) => { + opt_not_to_domains_array.push(hash); + } + (DomainValueKind::Entity, true) => { + opt_to_entities_array.push(hash); + } + (DomainValueKind::Entity, false) => { + opt_not_to_entities_array.push(hash); + } + } + } + + if !opt_to_domains_array.is_empty() { + opt_to_domains_array.sort_unstable(); + opt_to_domains_array.dedup(); + opt_to_domains = Some(opt_to_domains_array); + } + if !opt_not_to_domains_array.is_empty() { + opt_not_to_domains_array.sort_unstable(); + opt_not_to_domains_array.dedup(); + opt_not_to_domains = Some(opt_not_to_domains_array); + } + if !opt_to_entities_array.is_empty() { + opt_to_entities_array.sort_unstable(); + opt_to_entities_array.dedup(); + opt_to_entities = Some(opt_to_entities_array); + } + if !opt_not_to_entities_array.is_empty() { + opt_not_to_entities_array.sort_unstable(); + opt_not_to_entities_array.dedup(); + opt_not_to_entities = Some(opt_not_to_entities_array); + } + } + NetworkFilterOption::Badfilter => { + features_mask.set(NetworkFilterFeaturesMask::BAD_FILTER, true) + } NetworkFilterOption::Important => { mask.set(NetworkFilterMask::IS_IMPORTANT, true) } @@ -520,28 +630,28 @@ impl NetworkFilter { } NetworkFilterOption::Tag(value) => tag = Some(value), NetworkFilterOption::Redirect(value) => { - mask.set(NetworkFilterMask::IS_REDIRECT, true); - mask.set(NetworkFilterMask::ALSO_BLOCK_REDIRECT, true); + features_mask.set(NetworkFilterFeaturesMask::IS_REDIRECT, true); + features_mask.set(NetworkFilterFeaturesMask::ALSO_BLOCK_REDIRECT, true); modifier_option = Some(value); } NetworkFilterOption::RedirectRule(value) => { - mask.set(NetworkFilterMask::IS_REDIRECT, true); + features_mask.set(NetworkFilterFeaturesMask::IS_REDIRECT, true); modifier_option = Some(value); } NetworkFilterOption::Removeparam(value) => { - mask.set(NetworkFilterMask::IS_REMOVEPARAM, true); + features_mask.set(NetworkFilterFeaturesMask::IS_REMOVEPARAM, true); modifier_option = Some(value); } NetworkFilterOption::Csp(value) => { - mask.set(NetworkFilterMask::IS_CSP, true); + features_mask.set(NetworkFilterFeaturesMask::IS_CSP, true); // CSP rules can never have content types, and should always match against // subdocument and document rules. Rules do not match against document - // requests by default, so this must be explictly added. + // requests by default, so this must be explicitly added. mask.set(NetworkFilterMask::FROM_DOCUMENT, true); modifier_option = value; } NetworkFilterOption::Generichide => { - mask.set(NetworkFilterMask::GENERIC_HIDE, true) + features_mask.set(NetworkFilterFeaturesMask::GENERIC_HIDE, true) } NetworkFilterOption::Document => { cpt_mask_positive.set(NetworkFilterMask::FROM_DOCUMENT, true) @@ -570,6 +680,15 @@ impl NetworkFilter { } NetworkFilterOption::Font(enabled) => apply_content_type!(FROM_FONT, enabled), NetworkFilterOption::All => apply_content_type!(FROM_ALL_TYPES, true), + NetworkFilterOption::Method(methods) => { + for (enabled, method) in methods { + match method { + HttpMethod::Get => apply_method!(FROM_GET, enabled), + HttpMethod::Head => apply_method!(FROM_HEAD, enabled), + HttpMethod::Post => apply_method!(FROM_POST, enabled), + } + } + } } }); } @@ -580,7 +699,7 @@ impl NetworkFilter { // The negated types will be applied later. // // This doesn't apply to removeparam filters. - if !mask.contains(NetworkFilterMask::IS_REMOVEPARAM) + if !features_mask.contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) && (cpt_mask_negative & NetworkFilterMask::FROM_NETWORK_TYPES) != NetworkFilterMask::NONE { @@ -589,7 +708,7 @@ impl NetworkFilter { // If no positive types were set, then the filter should apply to all network types. if (cpt_mask_positive & NetworkFilterMask::FROM_ALL_TYPES).is_empty() { // Removeparam is again a special case. - if mask.contains(NetworkFilterMask::IS_REMOVEPARAM) { + if features_mask.contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) { mask |= NetworkFilterMask::FROM_DOCUMENT | NetworkFilterMask::FROM_SUBDOCUMENT | NetworkFilterMask::FROM_XMLHTTPREQUEST; @@ -615,7 +734,7 @@ impl NetworkFilter { end_url_anchor = true; } - let pattern = &parsed.pattern.pattern; + let pattern = &line[parsed.pattern.start..parsed.pattern.end]; let is_regex = check_is_regex(pattern); mask.set(NetworkFilterMask::IS_REGEX, is_regex); @@ -656,7 +775,7 @@ impl NetworkFilter { mask.set(NetworkFilterMask::IS_HOSTNAME_REGEX, true); } - hostname = Some(String::from(&pattern[..first_separator_start])); + hostname = Some(&pattern.as_bytes()[..first_separator_start]); filter_index_start = first_separator_start; // If the only symbol remaining for the selector is '^' then ignore it @@ -681,12 +800,12 @@ impl NetworkFilter { let slash_index = find_char(b'/', pattern.as_bytes()); slash_index .map(|i| { - hostname = Some(String::from(&pattern[..i])); + hostname = Some(&pattern.as_bytes()[..i]); filter_index_start += i; mask.set(NetworkFilterMask::IS_LEFT_ANCHOR, true); }) .or_else(|| { - hostname = Some(String::from(pattern)); + hostname = Some(pattern.as_bytes()); filter_index_start = filter_index_end; None }); @@ -738,13 +857,13 @@ impl NetworkFilter { } } - let filter: Option = if filter_index_end > filter_index_start { + let filter = if filter_index_end > filter_index_start { let filter_str = &pattern[filter_index_start..filter_index_end]; mask.set(NetworkFilterMask::IS_REGEX, check_is_regex(filter_str)); if mask.contains(NetworkFilterMask::MATCH_CASE) { - Some(String::from(filter_str)) + Some(Cow::Borrowed(filter_str)) } else { - Some(filter_str.to_ascii_lowercase()) + Some(cow_ascii_lowercase(filter_str)) } } else { None @@ -752,23 +871,13 @@ impl NetworkFilter { // TODO: ignore hostname anchor is not hostname provided - let hostname_decoded = hostname - .map(|host| { - let hostname = - idna::domain_to_ascii_cow(host.as_bytes(), idna::AsciiDenyList::EMPTY) - .map_err(|_| NetworkFilterError::PunycodeError)?; - match hostname { - Cow::Borrowed(_) => Ok(host), - Cow::Owned(h) => Ok(h.to_string()), - } - }) - .transpose(); + let hostname_decoded = hostname.map(decode_hostname).transpose()?; - if mask.contains(NetworkFilterMask::GENERIC_HIDE) && !parsed.exception { + if features_mask.contains(NetworkFilterFeaturesMask::GENERIC_HIDE) && !parsed.exception { return Err(NetworkFilterError::GenericHideWithoutException); } - if mask.contains(NetworkFilterMask::IS_REMOVEPARAM) && parsed.exception { + if features_mask.contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) && parsed.exception { return Err(NetworkFilterError::RemoveparamWithException); } @@ -787,26 +896,48 @@ impl NetworkFilter { && mask.contains(NetworkFilterMask::IS_HOSTNAME_ANCHOR) && mask.contains(NetworkFilterMask::IS_RIGHT_ANCHOR) && !end_url_anchor - && !mask.contains(NetworkFilterMask::IS_REMOVEPARAM) + && !features_mask.contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) { mask |= NetworkFilterMask::FROM_ALL_TYPES; } // Finally, apply any explicitly negated request types mask &= !cpt_mask_negative; + if !method_mask_positive.is_empty() || !method_mask_negative.is_empty() { + mask |= method_mask_positive; + if (method_mask_negative & NetworkFilterMask::FROM_ANY_METHODS) + != NetworkFilterMask::NONE + && (method_mask_positive & NetworkFilterMask::FROM_ANY_METHODS).is_empty() + { + mask |= NetworkFilterMask::FROM_ANY_METHODS; + } + mask &= !method_mask_negative; + } + + if features_mask.contains(NetworkFilterFeaturesMask::GENERIC_HIDE) + && mask.intersects(NetworkFilterMask::FROM_ANY_METHODS) + { + return Err(NetworkFilterError::MethodWithGenerichide); + } + Ok(NetworkFilter { filter: if let Some(simple_filter) = filter { FilterPart::Simple(simple_filter) } else { FilterPart::Empty }, - hostname: hostname_decoded?, + hostname: hostname_decoded, mask, + features_mask, opt_domains, opt_not_domains, + opt_to_domains, + opt_not_to_domains, + opt_to_entities, + opt_not_to_entities, tag, raw_line: if debug { - Some(Box::new(String::from(line))) + Some(Cow::Borrowed(line)) } else { None }, @@ -817,7 +948,7 @@ impl NetworkFilter { /// Given a hostname, produces an equivalent filter parsed from the form `"||hostname^"`, to /// emulate the behavior of hosts-style blocking. - pub fn parse_hosts_style(hostname: &str, debug: bool) -> Result { + pub fn parse_hosts_style(hostname: &'a str, debug: bool) -> Result { // Make sure the hostname doesn't contain any invalid characters static INVALID_CHARS: LazyLock = LazyLock::new(|| Regex::new("[/^*!?$&(){}\\[\\]+=~`\\s|@,'\"><:;]").unwrap()); @@ -833,39 +964,61 @@ impl NetworkFilter { return Err(NetworkFilterError::FilterParseError); } - // Parse it as a `||hostname^` rule. - let rule = format!("||{hostname}^"); - NetworkFilter::parse(&rule, debug, Default::default()) - } + let decoded_hostname = decode_hostname(hostname.as_bytes())?; - pub fn get_id_without_badfilter(&self) -> Hash { - let mut mask = self.mask; - mask.set(NetworkFilterMask::BAD_FILTER, false); - compute_filter_id( - self.modifier_option.as_deref(), + let mask = NetworkFilterMask::THIRD_PARTY + | NetworkFilterMask::FIRST_PARTY + | NetworkFilterMask::FROM_HTTPS + | NetworkFilterMask::FROM_HTTP + | NetworkFilterMask::FROM_NETWORK_TYPES + | NetworkFilterMask::FROM_ALL_TYPES + | NetworkFilterMask::IS_HOSTNAME_ANCHOR + | NetworkFilterMask::IS_RIGHT_ANCHOR; + + let mut rule = String::with_capacity(hostname.len() + 3); + rule.push_str("||"); + rule.push_str(hostname); + rule.push('^'); + let id = utils::fast_hash(&rule); + + Ok(NetworkFilter { + filter: FilterPart::Empty, + hostname: Some(decoded_hostname), mask, - self.filter.string_view().as_deref(), - self.hostname.as_deref(), - self.opt_domains.as_ref(), - self.opt_not_domains.as_ref(), - ) + features_mask: Default::default(), + opt_domains: None, + opt_not_domains: None, + opt_to_domains: None, + opt_not_to_domains: None, + opt_to_entities: None, + opt_not_to_entities: None, + tag: None, + raw_line: if debug { Some(Cow::Owned(rule)) } else { None }, + modifier_option: None, + id, + }) } pub fn get_id(&self) -> Hash { compute_filter_id( - self.modifier_option.as_deref(), + self.modifier_option, self.mask, - self.filter.string_view().as_deref(), + self.features_mask, + &self.filter, self.hostname.as_deref(), self.opt_domains.as_ref(), self.opt_not_domains.as_ref(), + self.opt_to_domains.as_ref(), + self.opt_not_to_domains.as_ref(), + self.opt_to_entities.as_ref(), + self.opt_not_to_entities.as_ref(), ) } - pub(crate) fn get_tokens<'a>( - &'a self, - tokens_buffer: &'a mut TokensBuffer, - ) -> FilterTokens<'a> { + /// Returns the tokens that the filter matches to. + /// + /// For `!FilterTokens::Empty`, the result is stored in the given `tokens_buffer`. + pub(crate) fn get_tokens(&self, tokens_buffer: &mut TokensBuffer) -> FilterTokens { tokens_buffer.clear(); // If there is only one domain and no domain negation, we also use this @@ -879,6 +1032,17 @@ impl NetworkFilter { tokens_buffer.push(*domain); } } + } else if self.opt_to_domains.is_some() + && self.opt_not_to_domains.is_none() + && self.opt_to_entities.is_none() + && self.opt_not_to_entities.is_none() + && self.opt_to_domains.as_ref().map(|d| d.len()) == Some(1) + { + if let Some(domains) = self.opt_to_domains.as_ref() { + if let Some(domain) = domains.first() { + tokens_buffer.push(*domain); + } + } } // Get tokens from filter @@ -888,7 +1052,12 @@ impl NetworkFilter { (self.is_plain() || self.is_regex()) && !self.is_right_anchor(); let skip_first_token = self.is_right_anchor(); - utils::tokenize_filter_to(f, skip_first_token, skip_last_token, tokens_buffer); + utils::tokenize_filter_to( + f.as_ref(), + skip_first_token, + skip_last_token, + tokens_buffer, + ); } FilterPart::AnyOf(_) => (), // across AnyOf set of filters no single token is guaranteed to match to a request _ => (), @@ -897,19 +1066,23 @@ impl NetworkFilter { // Append tokens from hostname, if any if !self.mask.contains(NetworkFilterMask::IS_HOSTNAME_REGEX) { if let Some(hostname) = self.hostname.as_ref() { - utils::tokenize_to(hostname, tokens_buffer); + utils::tokenize_to(hostname.as_ref(), tokens_buffer); } } else if let Some(hostname) = self.hostname.as_ref() { // Find last dot to tokenize the prefix let last_dot_pos = hostname.rfind('.'); if let Some(last_dot_pos) = last_dot_pos { - utils::tokenize_to(&hostname[..last_dot_pos], tokens_buffer); + utils::tokenize_to(&hostname.as_ref()[..last_dot_pos], tokens_buffer); } } - if tokens_buffer.is_empty() && self.mask.contains(NetworkFilterMask::IS_REMOVEPARAM) { + if tokens_buffer.is_empty() + && self + .features_mask + .contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) + { if let Some(removeparam) = &self.modifier_option { - if VALID_PARAM.is_match(removeparam) { + if VALID_PARAM.is_match(removeparam.as_ref()) { utils::tokenize_to(&removeparam.to_ascii_lowercase(), tokens_buffer); } } @@ -917,11 +1090,32 @@ impl NetworkFilter { // If we got no tokens for the filter/hostname part, then we will dispatch // this filter in multiple buckets based on the domains option. - if tokens_buffer.is_empty() && self.opt_domains.is_some() && self.opt_not_domains.is_none() - { - if let Some(opt_domains) = self.opt_domains.as_ref() { - if !opt_domains.is_empty() { - return FilterTokens::OptDomains(opt_domains); + if tokens_buffer.is_empty() { + if self.opt_domains.is_some() && self.opt_not_domains.is_none() { + if let Some(opt_domains) = self.opt_domains.as_ref() { + if !opt_domains.is_empty() { + let cap = tokens_buffer.remaining_capacity(); + if opt_domains.len() <= cap { + tokens_buffer.extend(opt_domains.iter().copied()); + return FilterTokens::OptDomains; + } + // Too many domains to bucket individually; fall back to the catch-all + // bucket (token 0). + } + } + } else if self.opt_to_domains.is_some() + && self.opt_not_to_domains.is_none() + && self.opt_to_entities.is_none() + && self.opt_not_to_entities.is_none() + { + if let Some(opt_to_domains) = self.opt_to_domains.as_ref() { + if !opt_to_domains.is_empty() { + let cap = tokens_buffer.remaining_capacity(); + if opt_to_domains.len() <= cap { + tokens_buffer.extend(opt_to_domains.iter().copied()); + return FilterTokens::OptDomains; + } + } } } FilterTokens::Empty @@ -933,15 +1127,43 @@ impl NetworkFilter { tokens_buffer.push(utils::fast_hash("https")); } - FilterTokens::Other(tokens_buffer.as_slice()) + FilterTokens::Other } } + pub fn is_badfilter(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::BAD_FILTER) + } + + pub fn is_removeparam(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::IS_REMOVEPARAM) + } + + pub fn is_generic_hide(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::GENERIC_HIDE) + } + + pub fn is_csp(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::IS_CSP) + } + + pub fn is_redirect(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::IS_REDIRECT) + } + + pub fn also_block_redirect(&self) -> bool { + self.features_mask + .contains(NetworkFilterFeaturesMask::ALSO_BLOCK_REDIRECT) + } + #[cfg(test)] pub(crate) fn matches_test(&self, request: &request::Request) -> bool { - let filter_set = crate::FilterSet::new_with_rules(vec![self.clone()], vec![], true); - let engine = crate::Engine::from_filter_set(filter_set, true); - + let engine = crate::Engine::new_with_parsed_rules(vec![self.clone()], vec![], true); if self.is_exception() { engine.check_network_request_exceptions(request) } else { @@ -950,16 +1172,16 @@ impl NetworkFilter { } } -impl NetworkFilterMaskHelper for NetworkFilter { +impl NetworkFilterMaskHelper for NetworkFilter<'_> { fn has_flag(&self, v: NetworkFilterMask) -> bool { self.mask.contains(v) } } -impl fmt::Display for NetworkFilter { +impl fmt::Display for NetworkFilter<'_> { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { - match self.raw_line.as_ref() { - Some(r) => write!(f, "{}", r.clone()), + match self.raw_line.as_deref() { + Some(r) => write!(f, "{r}"), None => write!(f, "NetworkFilter"), } } @@ -973,50 +1195,90 @@ pub(crate) trait NetworkMatchable { // Filter parsing // --------------------------------------------------------------------------- +#[inline] +fn write_str_to_hasher(hasher: &mut impl Hasher, s: &str) { + hasher.write_usize(s.len()); + hasher.write(s.as_bytes()); +} + +#[allow(clippy::too_many_arguments)] fn compute_filter_id( modifier_option: Option<&str>, mask: NetworkFilterMask, - filter: Option<&str>, + features_mask: NetworkFilterFeaturesMask, + filter: &FilterPart<'_>, hostname: Option<&str>, opt_domains: Option<&Vec>, opt_not_domains: Option<&Vec>, + opt_to_domains: Option<&Vec>, + opt_not_to_domains: Option<&Vec>, + opt_to_entities: Option<&Vec>, + opt_not_to_entities: Option<&Vec>, ) -> Hash { - let mut hash: Hash = (5408 * 33) ^ Hash::from(mask.bits()); + let mut hasher = FxHasher::default(); + + hasher.write_u64(u64::from(mask.bits())); + + // Exclude BAD_FILTER from the hash + let features_mask_bits = features_mask.bits() & !NetworkFilterFeaturesMask::BAD_FILTER.bits(); + hasher.write_u64(u64::from(features_mask_bits)); if let Some(s) = modifier_option { - let chars = s.chars(); - for c in chars { - hash = hash.wrapping_mul(33) ^ (c as Hash); - } - }; + write_str_to_hasher(&mut hasher, s); + } if let Some(domains) = opt_domains { for d in domains { - hash = hash.wrapping_mul(33) ^ d; + hasher.write_u64(*d); } - }; + } if let Some(domains) = opt_not_domains { for d in domains { - hash = hash.wrapping_mul(33) ^ d; + hasher.write_u64(*d); } } - if let Some(s) = filter { - let chars = s.chars(); - for c in chars { - hash = hash.wrapping_mul(33) ^ (c as Hash); + if let Some(domains) = opt_to_domains { + for d in domains { + hasher.write_u64(*d); } } - if let Some(s) = hostname { - let chars = s.chars(); - for c in chars { - hash = hash.wrapping_mul(33) ^ (c as Hash); + if let Some(domains) = opt_not_to_domains { + for d in domains { + hasher.write_u64(*d); + } + } + + if let Some(domains) = opt_to_entities { + for d in domains { + hasher.write_u64(*d); + } + } + + if let Some(domains) = opt_not_to_entities { + for d in domains { + hasher.write_u64(*d); + } + } + + match filter { + FilterPart::Empty => {} + FilterPart::Simple(s) => write_str_to_hasher(&mut hasher, s.as_ref()), + FilterPart::AnyOf(parts) => { + hasher.write_u64(parts.len() as u64); + for part in parts { + write_str_to_hasher(&mut hasher, part); + } } } - hash + if let Some(s) = hostname { + write_str_to_hasher(&mut hasher, s); + } + + hasher.finish() } /// Check if the sub-string contained between the indices start and end is a diff --git a/src/filters/network_matchers.rs b/src/filters/network_matchers.rs index 6d640a17..abfe2416 100644 --- a/src/filters/network_matchers.rs +++ b/src/filters/network_matchers.rs @@ -7,6 +7,8 @@ use memchr::memmem; +use crate::filters::fb_network::ToOptionsFlags; +use crate::filters::filter_data_context::ToRuleCapability; use crate::filters::network::{NetworkFilterMask, NetworkFilterMaskHelper}; use crate::regex_manager::RegexManager; use crate::request; @@ -404,17 +406,14 @@ where #[inline] pub fn check_options(mask: NetworkFilterMask, request: &request::Request) -> bool { - // Bad filter never matches - if mask.is_badfilter() { - return false; - } // We first discard requests based on type, protocol and party. This is really // cheap and should be done first. if !mask.check_cpt_allowed(&request.request_type) - || (request.is_https && !mask.for_https()) - || (request.is_http && !mask.for_http()) || (!mask.first_party() && !request.is_third_party) || (!mask.third_party() && request.is_third_party) + || (request.is_https && !mask.for_https()) + || (request.is_http && !mask.for_http()) + || !NetworkFilterMask::check_method_allowed(mask, request.method.as_ref()) { return false; } @@ -422,15 +421,15 @@ pub fn check_options(mask: NetworkFilterMask, request: &request::Request) -> boo } #[inline] -pub fn check_included_domains_mapped( +fn check_included_domains_mapped_with_hashes( opt_domains: Option<&[u32]>, - request: &request::Request, + hostname_hashes: Option<&[Hash]>, mapping: &HashMap, ) -> bool { // Source URL must be among these domains to match if let Some(included_domains) = opt_domains.as_ref() { - if let Some(source_hashes) = request.source_hostname_hashes.as_ref() { - if source_hashes.iter().all(|h| { + if let Some(hashes) = hostname_hashes { + if hashes.iter().all(|h| { mapping .get(h) .is_none_or(|index| !utils::bin_lookup(included_domains, *index)) @@ -446,28 +445,137 @@ pub fn check_included_domains_mapped( } #[inline] -pub fn check_excluded_domains_mapped( +fn check_excluded_domains_mapped_with_hashes( opt_not_domains: Option<&[u32]>, - request: &request::Request, + hostname_hashes: Option<&[Hash]>, mapping: &HashMap, ) -> bool { if let Some(excluded_domains) = opt_not_domains.as_ref() { - if let Some(source_hashes) = request.source_hostname_hashes.as_ref() { - if source_hashes.iter().any(|h| { + if let Some(hashes) = hostname_hashes { + if hashes.iter().any(|h| { mapping .get(h) .is_some_and(|index| utils::bin_lookup(excluded_domains, *index)) }) { return false; } + } + } + true +} + +#[inline] +pub fn check_included_domains_mapped( + opt_domains: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, +) -> bool { + check_included_domains_mapped_with_hashes( + opt_domains, + request.source_hostname_hashes.as_deref(), + mapping, + ) +} + +#[inline] +pub fn check_excluded_domains_mapped( + opt_not_domains: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, +) -> bool { + check_excluded_domains_mapped_with_hashes( + opt_not_domains, + request.source_hostname_hashes.as_deref(), + mapping, + ) +} + +#[inline] +pub fn check_included_to_options_mapped( + to_options: ToOptionsFlags, + opt_to_domains: Option<&[u32]>, + opt_to_entities: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, + to_capability: ToRuleCapability, +) -> bool { + if !to_options.any() { + return true; + } + + let plain = opt_to_domains.filter(|domains| !domains.is_empty()); + let entity = opt_to_entities.filter(|domains| !domains.is_empty()); + + let destination_suffix_hashes = + if to_options.needs_plain() && to_capability.has_plain && plain.is_some() { + request.destination_suffix_hashes() } else { - // If there are domain restrictions but no source hostname - // (i.e. about:blank), apply the rule anyway. - return true; + None + }; + let destination_entity_hashes = + if to_options.needs_entity() && to_capability.has_entity && entity.is_some() { + request.destination_entity_hashes() + } else { + None + }; + + match (plain, entity) { + (None, None) => true, + (Some(included), None) => check_included_domains_mapped_with_hashes( + Some(included), + destination_suffix_hashes, + mapping, + ), + (None, Some(included)) => check_included_domains_mapped_with_hashes( + Some(included), + destination_entity_hashes, + mapping, + ), + (Some(included_plain), Some(included_entity)) => { + check_included_domains_mapped_with_hashes( + Some(included_plain), + destination_suffix_hashes, + mapping, + ) || check_included_domains_mapped_with_hashes( + Some(included_entity), + destination_entity_hashes, + mapping, + ) } } +} - true +#[inline] +pub fn check_excluded_to_options_mapped( + to_options: ToOptionsFlags, + opt_not_to_domains: Option<&[u32]>, + opt_not_to_entities: Option<&[u32]>, + request: &request::Request, + mapping: &HashMap, + to_capability: ToRuleCapability, +) -> bool { + if !to_options.any() { + return true; + } + + let plain = opt_not_to_domains.filter(|domains| !domains.is_empty()); + let entity = opt_not_to_entities.filter(|domains| !domains.is_empty()); + + let destination_suffix_hashes = + if to_options.needs_plain() && to_capability.has_plain && plain.is_some() { + request.destination_suffix_hashes() + } else { + None + }; + let destination_entity_hashes = + if to_options.needs_entity() && to_capability.has_entity && entity.is_some() { + request.destination_entity_hashes() + } else { + None + }; + + check_excluded_domains_mapped_with_hashes(plain, destination_suffix_hashes, mapping) + && check_excluded_domains_mapped_with_hashes(entity, destination_entity_hashes, mapping) } #[cfg(test)] diff --git a/src/flatbuffers/containers/flat_multimap.rs b/src/flatbuffers/containers/flat_multimap.rs index d19e82c0..b21c6993 100644 --- a/src/flatbuffers/containers/flat_multimap.rs +++ b/src/flatbuffers/containers/flat_multimap.rs @@ -107,6 +107,10 @@ impl FlatMultiMapBuilder { self.entries.push((key, value)); } + pub fn retain_by_value(&mut self, mut retain_if: impl FnMut(&V) -> bool) { + self.entries.retain(|(_, value)| retain_if(value)); + } + pub fn finish<'a, B: FlatBuilder<'a>>( value: Self, builder: &mut B, diff --git a/src/flatbuffers/fb_network_filter.fbs b/src/flatbuffers/fb_network_filter.fbs index c4b7ed01..d2a8e404 100644 --- a/src/flatbuffers/fb_network_filter.fbs +++ b/src/flatbuffers/fb_network_filter.fbs @@ -7,15 +7,26 @@ namespace fb; table NetworkFilter { - mask: uint32; // NetworkFilterMask (network.rs) + // NetworkFilterMask (network.rs). + // Default = DEFAULT_OPTIONS | IS_HOSTNAME_ANCHOR | IS_RIGHT_ANCHOR | FROM_DOCUMENT + // (the most common mask, covering ~50% of filters). + mask: uint32 = 540221439; // 0x20331FFF /// These arrays contain sorted (ascending) indices in the |unique_domains_hashes| /// instead of the hashes themselves. This approach saves memory, as there /// typically aren’t many unique hashes opt_domains: [uint32]; opt_not_domains: [uint32]; + opt_to_domains: [uint32]; + opt_not_to_domains: [uint32]; + opt_to_entities: [uint32]; + opt_not_to_entities: [uint32]; + + // A union-like storage for pattern. Prefer |single_pattern| where + // possible for performance concerns. + single_pattern: string; + multi_patterns: [string]; - patterns: [string]; modifier_option: string; hostname: string; diff --git a/src/flatbuffers/fb_network_filter_generated.rs b/src/flatbuffers/fb_network_filter_generated.rs index 1d906be4..b93e5490 100644 --- a/src/flatbuffers/fb_network_filter_generated.rs +++ b/src/flatbuffers/fb_network_filter_generated.rs @@ -29,7 +29,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -38,11 +38,16 @@ pub mod fb { pub const VT_MASK: flatbuffers::VOffsetT = 4; pub const VT_OPT_DOMAINS: flatbuffers::VOffsetT = 6; pub const VT_OPT_NOT_DOMAINS: flatbuffers::VOffsetT = 8; - pub const VT_PATTERNS: flatbuffers::VOffsetT = 10; - pub const VT_MODIFIER_OPTION: flatbuffers::VOffsetT = 12; - pub const VT_HOSTNAME: flatbuffers::VOffsetT = 14; - pub const VT_TAG: flatbuffers::VOffsetT = 16; - pub const VT_RAW_LINE: flatbuffers::VOffsetT = 18; + pub const VT_OPT_TO_DOMAINS: flatbuffers::VOffsetT = 10; + pub const VT_OPT_NOT_TO_DOMAINS: flatbuffers::VOffsetT = 12; + pub const VT_OPT_TO_ENTITIES: flatbuffers::VOffsetT = 14; + pub const VT_OPT_NOT_TO_ENTITIES: flatbuffers::VOffsetT = 16; + pub const VT_SINGLE_PATTERN: flatbuffers::VOffsetT = 18; + pub const VT_MULTI_PATTERNS: flatbuffers::VOffsetT = 20; + pub const VT_MODIFIER_OPTION: flatbuffers::VOffsetT = 22; + pub const VT_HOSTNAME: flatbuffers::VOffsetT = 24; + pub const VT_TAG: flatbuffers::VOffsetT = 26; + pub const VT_RAW_LINE: flatbuffers::VOffsetT = 28; #[inline] pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { @@ -71,8 +76,23 @@ pub mod fb { if let Some(x) = args.modifier_option { builder.add_modifier_option(x); } - if let Some(x) = args.patterns { - builder.add_patterns(x); + if let Some(x) = args.multi_patterns { + builder.add_multi_patterns(x); + } + if let Some(x) = args.single_pattern { + builder.add_single_pattern(x); + } + if let Some(x) = args.opt_not_to_entities { + builder.add_opt_not_to_entities(x); + } + if let Some(x) = args.opt_to_entities { + builder.add_opt_to_entities(x); + } + if let Some(x) = args.opt_not_to_domains { + builder.add_opt_not_to_domains(x); + } + if let Some(x) = args.opt_to_domains { + builder.add_opt_to_domains(x); } if let Some(x) = args.opt_not_domains { builder.add_opt_not_domains(x); @@ -88,8 +108,13 @@ pub mod fb { let mask = self.mask(); let opt_domains = self.opt_domains().map(|x| x.into_iter().collect()); let opt_not_domains = self.opt_not_domains().map(|x| x.into_iter().collect()); - let patterns = self - .patterns() + let opt_to_domains = self.opt_to_domains().map(|x| x.into_iter().collect()); + let opt_not_to_domains = self.opt_not_to_domains().map(|x| x.into_iter().collect()); + let opt_to_entities = self.opt_to_entities().map(|x| x.into_iter().collect()); + let opt_not_to_entities = self.opt_not_to_entities().map(|x| x.into_iter().collect()); + let single_pattern = self.single_pattern().map(|x| x.to_string()); + let multi_patterns = self + .multi_patterns() .map(|x| x.iter().map(|s| s.to_string()).collect()); let modifier_option = self.modifier_option().map(|x| x.to_string()); let hostname = self.hostname().map(|x| x.to_string()); @@ -99,7 +124,12 @@ pub mod fb { mask, opt_domains, opt_not_domains, - patterns, + opt_to_domains, + opt_not_to_domains, + opt_to_entities, + opt_not_to_entities, + single_pattern, + multi_patterns, modifier_option, hostname, tag, @@ -114,7 +144,7 @@ pub mod fb { // which contains a valid value in this slot unsafe { self._tab - .get::(NetworkFilter::VT_MASK, Some(0)) + .get::(NetworkFilter::VT_MASK, Some(540221439)) .unwrap() } } @@ -148,7 +178,71 @@ pub mod fb { } } #[inline] - pub fn patterns( + pub fn opt_to_domains(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + NetworkFilter::VT_OPT_TO_DOMAINS, + None, + ) + } + } + #[inline] + pub fn opt_not_to_domains(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + NetworkFilter::VT_OPT_NOT_TO_DOMAINS, + None, + ) + } + } + #[inline] + pub fn opt_to_entities(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + NetworkFilter::VT_OPT_TO_ENTITIES, + None, + ) + } + } + #[inline] + pub fn opt_not_to_entities(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>( + NetworkFilter::VT_OPT_NOT_TO_ENTITIES, + None, + ) + } + } + #[inline] + pub fn single_pattern(&self) -> Option<&'a str> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab.get::>( + NetworkFilter::VT_SINGLE_PATTERN, + None, + ) + } + } + #[inline] + pub fn multi_patterns( &self, ) -> Option>> { // Safety: @@ -157,7 +251,7 @@ pub mod fb { unsafe { self._tab.get::>, - >>(NetworkFilter::VT_PATTERNS, None) + >>(NetworkFilter::VT_MULTI_PATTERNS, None) } } #[inline] @@ -223,9 +317,34 @@ pub mod fb { Self::VT_OPT_NOT_DOMAINS, false, )? + .visit_field::>>( + "opt_to_domains", + Self::VT_OPT_TO_DOMAINS, + false, + )? + .visit_field::>>( + "opt_not_to_domains", + Self::VT_OPT_NOT_TO_DOMAINS, + false, + )? + .visit_field::>>( + "opt_to_entities", + Self::VT_OPT_TO_ENTITIES, + false, + )? + .visit_field::>>( + "opt_not_to_entities", + Self::VT_OPT_NOT_TO_ENTITIES, + false, + )? + .visit_field::>( + "single_pattern", + Self::VT_SINGLE_PATTERN, + false, + )? .visit_field::>, - >>("patterns", Self::VT_PATTERNS, false)? + >>("multi_patterns", Self::VT_MULTI_PATTERNS, false)? .visit_field::>( "modifier_option", Self::VT_MODIFIER_OPTION, @@ -250,7 +369,12 @@ pub mod fb { pub mask: u32, pub opt_domains: Option>>, pub opt_not_domains: Option>>, - pub patterns: Option< + pub opt_to_domains: Option>>, + pub opt_not_to_domains: Option>>, + pub opt_to_entities: Option>>, + pub opt_not_to_entities: Option>>, + pub single_pattern: Option>, + pub multi_patterns: Option< flatbuffers::WIPOffset>>, >, pub modifier_option: Option>, @@ -262,10 +386,15 @@ pub mod fb { #[inline] fn default() -> Self { NetworkFilterArgs { - mask: 0, + mask: 540221439, opt_domains: None, opt_not_domains: None, - patterns: None, + opt_to_domains: None, + opt_not_to_domains: None, + opt_to_entities: None, + opt_not_to_entities: None, + single_pattern: None, + multi_patterns: None, modifier_option: None, hostname: None, tag: None, @@ -281,7 +410,8 @@ pub mod fb { impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> NetworkFilterBuilder<'a, 'b, A> { #[inline] pub fn add_mask(&mut self, mask: u32) { - self.fbb_.push_slot::(NetworkFilter::VT_MASK, mask, 0); + self.fbb_ + .push_slot::(NetworkFilter::VT_MASK, mask, 540221439); } #[inline] pub fn add_opt_domains( @@ -304,15 +434,62 @@ pub mod fb { ); } #[inline] - pub fn add_patterns( + pub fn add_opt_to_domains( + &mut self, + opt_to_domains: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_OPT_TO_DOMAINS, + opt_to_domains, + ); + } + #[inline] + pub fn add_opt_not_to_domains( + &mut self, + opt_not_to_domains: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_OPT_NOT_TO_DOMAINS, + opt_not_to_domains, + ); + } + #[inline] + pub fn add_opt_to_entities( + &mut self, + opt_to_entities: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_OPT_TO_ENTITIES, + opt_to_entities, + ); + } + #[inline] + pub fn add_opt_not_to_entities( + &mut self, + opt_not_to_entities: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_OPT_NOT_TO_ENTITIES, + opt_not_to_entities, + ); + } + #[inline] + pub fn add_single_pattern(&mut self, single_pattern: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>( + NetworkFilter::VT_SINGLE_PATTERN, + single_pattern, + ); + } + #[inline] + pub fn add_multi_patterns( &mut self, - patterns: flatbuffers::WIPOffset< + multi_patterns: flatbuffers::WIPOffset< flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset<&'b str>>, >, ) { self.fbb_.push_slot_always::>( - NetworkFilter::VT_PATTERNS, - patterns, + NetworkFilter::VT_MULTI_PATTERNS, + multi_patterns, ); } #[inline] @@ -364,7 +541,12 @@ pub mod fb { ds.field("mask", &self.mask()); ds.field("opt_domains", &self.opt_domains()); ds.field("opt_not_domains", &self.opt_not_domains()); - ds.field("patterns", &self.patterns()); + ds.field("opt_to_domains", &self.opt_to_domains()); + ds.field("opt_not_to_domains", &self.opt_not_to_domains()); + ds.field("opt_to_entities", &self.opt_to_entities()); + ds.field("opt_not_to_entities", &self.opt_not_to_entities()); + ds.field("single_pattern", &self.single_pattern()); + ds.field("multi_patterns", &self.multi_patterns()); ds.field("modifier_option", &self.modifier_option()); ds.field("hostname", &self.hostname()); ds.field("tag", &self.tag()); @@ -378,7 +560,12 @@ pub mod fb { pub mask: u32, pub opt_domains: Option>, pub opt_not_domains: Option>, - pub patterns: Option>, + pub opt_to_domains: Option>, + pub opt_not_to_domains: Option>, + pub opt_to_entities: Option>, + pub opt_not_to_entities: Option>, + pub single_pattern: Option, + pub multi_patterns: Option>, pub modifier_option: Option, pub hostname: Option, pub tag: Option, @@ -387,10 +574,15 @@ pub mod fb { impl Default for NetworkFilterT { fn default() -> Self { Self { - mask: 0, + mask: 540221439, opt_domains: None, opt_not_domains: None, - patterns: None, + opt_to_domains: None, + opt_not_to_domains: None, + opt_to_entities: None, + opt_not_to_entities: None, + single_pattern: None, + multi_patterns: None, modifier_option: None, hostname: None, tag: None, @@ -406,7 +598,18 @@ pub mod fb { let mask = self.mask; let opt_domains = self.opt_domains.as_ref().map(|x| _fbb.create_vector(x)); let opt_not_domains = self.opt_not_domains.as_ref().map(|x| _fbb.create_vector(x)); - let patterns = self.patterns.as_ref().map(|x| { + let opt_to_domains = self.opt_to_domains.as_ref().map(|x| _fbb.create_vector(x)); + let opt_not_to_domains = self + .opt_not_to_domains + .as_ref() + .map(|x| _fbb.create_vector(x)); + let opt_to_entities = self.opt_to_entities.as_ref().map(|x| _fbb.create_vector(x)); + let opt_not_to_entities = self + .opt_not_to_entities + .as_ref() + .map(|x| _fbb.create_vector(x)); + let single_pattern = self.single_pattern.as_ref().map(|x| _fbb.create_string(x)); + let multi_patterns = self.multi_patterns.as_ref().map(|x| { let w: Vec<_> = x.iter().map(|s| _fbb.create_string(s)).collect(); _fbb.create_vector(&w) }); @@ -420,7 +623,12 @@ pub mod fb { mask, opt_domains, opt_not_domains, - patterns, + opt_to_domains, + opt_not_to_domains, + opt_to_entities, + opt_not_to_entities, + single_pattern, + multi_patterns, modifier_option, hostname, tag, @@ -441,7 +649,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -672,7 +880,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -999,7 +1207,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -1158,7 +1366,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -2065,7 +2273,7 @@ pub mod fb { #[inline] unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { Self { - _tab: unsafe { flatbuffers::Table::new(buf, loc) }, + _tab: flatbuffers::Table::new(buf, loc), } } } @@ -2383,14 +2591,14 @@ pub mod fb { /// # Safety /// Callers must trust the given bytes do indeed contain a valid `Engine`. pub unsafe fn root_as_engine_unchecked(buf: &[u8]) -> Engine { - unsafe { flatbuffers::root_unchecked::(buf) } + flatbuffers::root_unchecked::(buf) } #[inline] /// Assumes, without verification, that a buffer of bytes contains a size prefixed Engine and returns it. /// # Safety /// Callers must trust the given bytes do indeed contain a valid size prefixed `Engine`. pub unsafe fn size_prefixed_root_as_engine_unchecked(buf: &[u8]) -> Engine { - unsafe { flatbuffers::size_prefixed_root_unchecked::(buf) } + flatbuffers::size_prefixed_root_unchecked::(buf) } #[inline] pub fn finish_engine_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>( diff --git a/src/flatbuffers/mod.rs b/src/flatbuffers/mod.rs index 61dc0bd6..7b93f063 100644 --- a/src/flatbuffers/mod.rs +++ b/src/flatbuffers/mod.rs @@ -1,2 +1,6 @@ pub(crate) mod containers; pub(crate) mod unsafe_tools; + +#[cfg(test)] +#[path = "../../tests/unit/flatbuffers/mod.rs"] +mod unit_tests; diff --git a/src/lists.rs b/src/lists.rs index 857b41e8..24c4cc0b 100644 --- a/src/lists.rs +++ b/src/lists.rs @@ -69,6 +69,12 @@ impl Default for ParseOptions { } } +#[derive(Clone)] +pub(crate) struct ListSource { + pub(crate) list_text: String, + pub(crate) parse_options: ParseOptions, +} + /// Manages a set of rules to be added to an [`crate::Engine`]. /// /// To be able to efficiently handle special options like `$badfilter`, and to allow optimizations, @@ -76,33 +82,28 @@ impl Default for ParseOptions { /// compound list from multiple different sources before compiling the rules into an `Engine`. #[derive(Clone)] pub struct FilterSet { - debug: bool, - pub(crate) network_filters: Vec, - pub(crate) cosmetic_filters: Vec, + pub(crate) debug: bool, + pub(crate) list_sources: Vec, } /// Collects metadata for the list by reading just until the first non-comment line. pub fn read_list_metadata(list: &str) -> FilterListMetadata { let mut metadata = FilterListMetadata::default(); - - // uBO only searches within the first 1024 characters; the same optimization can be useful here - let mut cutoff = list.len().min(1024); - - while !list.is_char_boundary(cutoff) { - cutoff -= 1; - } - - // String slice is safe here because `cutoff` is guaranteed to be a character boundary - for line in list[0..cutoff].lines() { - if line.starts_with('!') { - metadata.try_add(line); - } else if line.starts_with('[') { + let mut chars_read = 0; + for line in list.lines() { + if chars_read > 1024 { + break; // stop parsing metadata + } + chars_read += line.len(); + if line.starts_with("[") { continue; + } + if let Some(comment_content) = line.strip_prefix("!") { + metadata.try_add_metadata(comment_content); } else { - break; + break; // stop parsing metadata } } - metadata } @@ -188,27 +189,16 @@ pub struct FilterListMetadata { } impl FilterListMetadata { - /// Attempts to add a line of a filter list to this collection of metadata. Only comment lines - /// with valid metadata content will be added. Previously added information will not be - /// rewritten. - fn try_add(&mut self, line: &str) { - if let Some(kv) = line.strip_prefix("! ") { - if let Some((key, value)) = kv.split_once(": ") { - match key { - "Homepage" if self.homepage.is_none() => { - self.homepage = Some(value.to_string()) - } - "Title" if self.title.is_none() => self.title = Some(value.to_string()), - "Expires" if self.expires.is_none() => { - if let Ok(expires) = ExpiresInterval::try_from(value) { - self.expires = Some(expires); - } - } - "Redirect" if self.redirect.is_none() => { - self.redirect = Some(value.to_string()) - } - _ => (), + pub(crate) fn try_add_metadata(&mut self, metadata: &str) { + if let Some((key, value)) = metadata.trim().split_once(": ") { + match key { + "Homepage" if self.homepage.is_none() => self.homepage = Some(value.to_string()), + "Title" if self.title.is_none() => self.title = Some(value.to_string()), + "Expires" if self.expires.is_none() => { + self.expires = Some(ExpiresInterval::try_from(value).unwrap()) } + "Redirect" if self.redirect.is_none() => self.redirect = Some(value.to_string()), + _ => (), } } } @@ -221,54 +211,48 @@ impl FilterSet { pub fn new(debug: bool) -> Self { Self { debug, - network_filters: Vec::new(), - cosmetic_filters: Vec::new(), - } - } - - // Used in benchmarks to avoid parsing the rules twice. - #[doc(hidden)] - pub fn new_with_rules( - network_filters: Vec, - cosmetic_filters: Vec, - debug: bool, - ) -> Self { - Self { - debug, - network_filters, - cosmetic_filters, + list_sources: Vec::new(), } } /// Adds the contents of an entire filter list to this `FilterSet`. Filters that cannot be /// parsed successfully are ignored. Returns any discovered metadata about the list of rules /// added. - pub fn add_filter_list(&mut self, filter_list: &str, opts: ParseOptions) -> FilterListMetadata { - self.add_filters(filter_list.lines(), opts) + pub fn add_filter_list(&mut self, list_text: String, opts: ParseOptions) -> FilterListMetadata { + let metadata = match opts.format { + FilterFormat::Standard => read_list_metadata(&list_text), + FilterFormat::Hosts => FilterListMetadata::default(), + }; + self.list_sources.push(ListSource { + list_text, + parse_options: opts, + }); + metadata } /// Adds a collection of filter rules to this `FilterSet`. Filters that cannot be parsed /// successfully are ignored. Returns any discovered metadata about the list of rules added. + #[cfg(test)] pub fn add_filters( &mut self, filters: impl IntoIterator>, opts: ParseOptions, - ) -> FilterListMetadata { - let (metadata, parsed_network_filters, parsed_cosmetic_filters) = - parse_filters_with_metadata(filters, self.debug, opts); - self.network_filters.extend(parsed_network_filters); - self.cosmetic_filters.extend(parsed_cosmetic_filters); - metadata + ) { + let list_text = filters.into_iter().fold(String::new(), |mut acc, rule| { + acc.push_str(rule.as_ref()); + acc.push('\n'); + acc + }); + self.list_sources.push(ListSource { + list_text, + parse_options: opts, + }); } /// Adds the string representation of a single filter rule to this `FilterSet`. - pub fn add_filter(&mut self, filter: &str, opts: ParseOptions) -> Result<(), FilterParseError> { - let filter_parsed = parse_filter(filter, self.debug, opts); - match filter_parsed? { - ParsedFilter::Network(filter) => self.network_filters.push(filter), - ParsedFilter::Cosmetic(filter) => self.cosmetic_filters.push(filter), - } - Ok(()) + #[cfg(test)] + pub fn add_filter(&mut self, filter: &str, opts: ParseOptions) { + self.add_filters([filter], opts) } /// Consumes this `FilterSet`, returning an equivalent list of content blocking rules and a @@ -285,18 +269,33 @@ impl FilterSet { self, ) -> Result<(Vec, Vec), ()> { use crate::content_blocking; - use crate::filters::network::NetworkFilterMaskHelper; use std::collections::HashSet; - if !self.debug { + let FilterSet { + debug, + list_sources, + } = self; + + if !debug { return Err(()); } + let mut network_filters = vec![]; + let mut cosmetic_filters = vec![]; + for list_source in list_sources.iter() { + let list_text = list_source.list_text.lines(); + let parse_options = list_source.parse_options; + let (list_network_filters, list_cosmetic_filters) = + parse_filters(list_text, debug, parse_options); + network_filters.extend(list_network_filters); + cosmetic_filters.extend(list_cosmetic_filters); + } + // Store bad filter id to skip them later. let mut bad_filter_ids = HashSet::new(); - for filter in self.network_filters.iter() { + for filter in network_filters.iter() { if filter.is_badfilter() { - bad_filter_ids.insert(filter.get_id_without_badfilter()); + bad_filter_ids.insert(filter.get_id()); } } @@ -305,15 +304,16 @@ impl FilterSet { let mut filters_used = vec![]; - self.network_filters.into_iter().for_each(|filter| { + network_filters.into_iter().for_each(|filter| { // Don't process bad filter rules or matching bad filter rules. if bad_filter_ids.contains(&filter.get_id()) || filter.is_badfilter() { return; } - let original_rule = *filter + let original_rule = filter .raw_line - .clone() - .expect("All rules should be in debug mode"); + .as_ref() + .expect("All rules should be in debug mode") + .to_string(); if let Ok(equivalent) = TryInto::::try_into(filter) { filters_used.push(original_rule); @@ -330,7 +330,7 @@ impl FilterSet { let add_fp_document_exception = !filters_used.is_empty(); - self.cosmetic_filters.into_iter().for_each(|filter| { + cosmetic_filters.into_iter().for_each(|filter| { let original_rule = *filter .raw_line .clone() @@ -392,21 +392,21 @@ pub enum FilterType { NotSupported, } -/// Successful result of parsing a single filter rule -pub enum ParsedFilter { - Network(NetworkFilter), +/// Successful result of parsing a single line from a filter list +pub enum ParsedLine<'a> { + Network(NetworkFilter<'a>), Cosmetic(CosmeticFilter), } -impl From for ParsedFilter { - fn from(v: NetworkFilter) -> Self { - ParsedFilter::Network(v) +impl From> for ParsedLine<'static> { + fn from(v: NetworkFilter<'static>) -> Self { + ParsedLine::Network(v) } } -impl From for ParsedFilter { +impl From for ParsedLine<'static> { fn from(v: CosmeticFilter) -> Self { - ParsedFilter::Cosmetic(v) + ParsedLine::Cosmetic(v) } } @@ -421,6 +421,8 @@ pub enum FilterParseError { Unsupported, #[error("empty")] Empty, + #[error("invalid expires interval")] + InvalidExpiresInterval, } impl From for FilterParseError { @@ -435,12 +437,12 @@ impl From for FilterParseError { } } -/// Parse a single filter rule -pub fn parse_filter( - line: &str, +/// Parse a single line from a filter list +pub fn parse_filter<'a>( + line: &'a str, debug: bool, opts: ParseOptions, -) -> Result { +) -> Result, FilterParseError> { let filter = line.trim(); if filter.is_empty() { @@ -451,12 +453,12 @@ pub fn parse_filter( FilterFormat::Standard => match (detect_filter_type(filter), opts.rule_types) { (FilterType::Network, RuleTypes::All | RuleTypes::NetworkOnly) => { NetworkFilter::parse(filter, debug, opts) - .map(|f| f.into()) + .map(ParsedLine::Network) .map_err(|e| e.into()) } (FilterType::Cosmetic, RuleTypes::All | RuleTypes::CosmeticOnly) => { CosmeticFilter::parse(filter, debug, opts.permissions) - .map(|f| f.into()) + .map(ParsedLine::Cosmetic) .map_err(|e| e.into()) } _ => Err(FilterParseError::Unsupported), @@ -505,46 +507,28 @@ pub fn parse_filter( } NetworkFilter::parse_hosts_style(hostname, debug) - .map(|f| f.into()) + .map(ParsedLine::Network) .map_err(|e| e.into()) } } } /// Parse an entire list of filters, ignoring any errors -pub fn parse_filters( - list: impl IntoIterator>, - debug: bool, - opts: ParseOptions, -) -> (Vec, Vec) { - let (_metadata, network_filters, cosmetic_filters) = - parse_filters_with_metadata(list, debug, opts); - - (network_filters, cosmetic_filters) -} - -/// Parse an entire list of filters, ignoring any errors -pub fn parse_filters_with_metadata( - list: impl IntoIterator>, +pub fn parse_filters<'a>( + list: impl IntoIterator, debug: bool, opts: ParseOptions, -) -> (FilterListMetadata, Vec, Vec) { - let mut metadata = FilterListMetadata::default(); - - let list_iter = list.into_iter(); - - let (network_filters, cosmetic_filters): (Vec<_>, Vec<_>) = list_iter - .map(|line| { - metadata.try_add(line.as_ref()); - parse_filter(line.as_ref(), debug, opts) +) -> (Vec>, Vec) { + let (network_filters, cosmetic_filters): (Vec<_>, Vec<_>) = list + .into_iter() + .filter_map(|line| match parse_filter(line, debug, opts) { + Ok(ParsedLine::Network(f)) => Some(Either::Left(f)), + Ok(ParsedLine::Cosmetic(f)) => Some(Either::Right(f)), + _ => None, }) - .filter_map(Result::ok) - .partition_map(|filter| match filter { - ParsedFilter::Network(f) => Either::Left(f), - ParsedFilter::Cosmetic(f) => Either::Right(f), - }); + .partition_map(|x| x); - (metadata, network_filters, cosmetic_filters) + (network_filters, cosmetic_filters) } /// Given a single line, checks if this would likely be a cosmetic filter, a diff --git a/src/network_filter_list.rs b/src/network_filter_list.rs index 4e642061..8a7935ef 100644 --- a/src/network_filter_list.rs +++ b/src/network_filter_list.rs @@ -7,9 +7,7 @@ use flatbuffers::ForwardsUOffset; use crate::filters::fb_network::FlatNetworkFilter; use crate::filters::filter_data_context::FilterDataContext; use crate::filters::flatbuffer_generated::fb; -use crate::filters::network::{ - NetworkFilter, NetworkFilterMask, NetworkFilterMaskHelper, NetworkMatchable, -}; +use crate::filters::network::{NetworkFilter, NetworkFilterMask, NetworkMatchable}; use crate::flatbuffers::containers::flat_multimap::FlatMultiMapView; use crate::flatbuffers::unsafe_tools::fb_vector_to_slice; use crate::regex_manager::RegexManager; @@ -24,12 +22,12 @@ pub struct CheckResult { pub raw_line: Option, } -impl From<&NetworkFilter> for CheckResult { +impl From<&NetworkFilter<'_>> for CheckResult { fn from(filter: &NetworkFilter) -> Self { Self { filter_mask: filter.mask, - modifier_option: filter.modifier_option.clone(), - raw_line: filter.raw_line.clone().map(|v| *v), + modifier_option: filter.modifier_option.map(|s| s.to_string()), + raw_line: filter.raw_line.as_ref().map(|line| line.to_string()), } } } @@ -44,13 +42,6 @@ impl fmt::Display for CheckResult { } } -impl NetworkFilterMaskHelper for CheckResult { - #[inline] - fn has_flag(&self, v: NetworkFilterMask) -> bool { - self.filter_mask.contains(v) - } -} - /// Internal structure to keep track of a collection of network filters. pub(crate) struct NetworkFilterList<'a> { pub(crate) list: fb::NetworkFilterList<'a>, diff --git a/src/optimizer.rs b/src/optimizer.rs index 73dc274f..b0362a68 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -2,25 +2,30 @@ use crate::filters::network::{ FilterPart, NetworkFilter, NetworkFilterMask, NetworkFilterMaskHelper, }; use itertools::*; +use std::borrow::Cow; use std::collections::HashMap; trait Optimization { - fn fusion(&self, filters: &[NetworkFilter]) -> NetworkFilter; - fn group_by_criteria(&self, filter: &NetworkFilter) -> String; - fn select(&self, filter: &NetworkFilter) -> bool; + fn fusion<'a>(&self, filters: &[NetworkFilter<'a>]) -> NetworkFilter<'a>; + fn group_by_criteria(&self, filter: &NetworkFilter<'_>) -> String; + fn select(&self, filter: &NetworkFilter<'_>) -> bool; } -pub fn is_filter_optimizable_by_patterns(filter: &NetworkFilter) -> bool { +pub fn is_filter_optimizable_by_patterns(filter: &NetworkFilter<'_>) -> bool { filter.opt_domains.is_none() && filter.opt_not_domains.is_none() + && filter.opt_to_domains.is_none() + && filter.opt_not_to_domains.is_none() + && filter.opt_to_entities.is_none() + && filter.opt_not_to_entities.is_none() && !filter.is_hostname_anchor() && !filter.is_redirect() && !filter.is_csp() } /// Fuse `NetworkFilter`s together by applying optimizations sequentially. -pub fn optimize(filters: Vec) -> Vec { - let mut optimized: Vec = Vec::new(); +pub fn optimize<'a>(filters: Vec>) -> Vec> { + let mut optimized: Vec> = Vec::new(); /* let union_domain_group = UnionDomainGroup {}; @@ -40,11 +45,11 @@ pub fn optimize(filters: Vec) -> Vec { optimized } -fn apply_optimisation( +fn apply_optimisation<'a, T: Optimization>( optimization: &T, - filters: Vec, -) -> (Vec, Vec) { - let (positive, mut negative): (Vec, Vec) = + filters: Vec>, +) -> (Vec>, Vec>) { + let (positive, mut negative): (Vec>, Vec>) = filters.into_iter().partition_map(|f| { if optimization.select(&f) { Either::Left(f) @@ -53,7 +58,8 @@ fn apply_optimisation( } }); - let mut to_fuse: HashMap> = HashMap::with_capacity(positive.len()); + let mut to_fuse: HashMap>> = + HashMap::with_capacity(positive.len()); positive .into_iter() .for_each(|f| insert_dup(&mut to_fuse, optimization.group_by_criteria(&f), f)); @@ -85,7 +91,7 @@ struct SimplePatternGroup {} impl Optimization for SimplePatternGroup { // Group simple patterns, into a single filter - fn fusion(&self, filters: &[NetworkFilter]) -> NetworkFilter { + fn fusion<'a>(&self, filters: &[NetworkFilter<'a>]) -> NetworkFilter<'a> { let base_filter = &filters[0]; // FIXME: can technically panic, if filters list is empty let mut filter = base_filter.clone(); @@ -100,15 +106,16 @@ impl Optimization for SimplePatternGroup { for f in filters { match &f.filter { FilterPart::Empty => (), - FilterPart::Simple(s) => flat_patterns.push(s.clone()), - FilterPart::AnyOf(s) => flat_patterns.extend_from_slice(s), + FilterPart::Simple(s) => flat_patterns.push(s.to_string()), + FilterPart::AnyOf(s) => flat_patterns.extend(s.iter().cloned()), } } if flat_patterns.is_empty() { filter.filter = FilterPart::Empty; } else if flat_patterns.len() == 1 { - filter.filter = FilterPart::Simple(flat_patterns[0].clone()) + filter.filter = + FilterPart::Simple(Cow::Owned(flat_patterns.into_iter().next().unwrap())) } else { filter.filter = FilterPart::AnyOf(flat_patterns) } @@ -122,10 +129,10 @@ impl Optimization for SimplePatternGroup { .set(NetworkFilterMask::IS_COMPLETE_REGEX, is_complete_regex); if base_filter.raw_line.is_some() { - filter.raw_line = Some(Box::new( + filter.raw_line = Some(Cow::Owned( filters .iter() - .flat_map(|f| f.raw_line.clone()) + .flat_map(|f| f.raw_line.as_deref()) .join(" <+> "), )) } @@ -133,10 +140,10 @@ impl Optimization for SimplePatternGroup { filter } - fn group_by_criteria(&self, filter: &NetworkFilter) -> String { + fn group_by_criteria(&self, filter: &NetworkFilter<'_>) -> String { format!("{:b}:{:?}", filter.mask, filter.is_complete_regex()) } - fn select(&self, filter: &NetworkFilter) -> bool { + fn select(&self, filter: &NetworkFilter<'_>) -> bool { is_filter_optimizable_by_patterns(filter) } } @@ -145,7 +152,7 @@ impl Optimization for SimplePatternGroup { struct UnionDomainGroup {} impl Optimization for UnionDomainGroup { - fn fusion(&self, filters: &[NetworkFilter]) -> NetworkFilter { + fn fusion<'a>(&self, filters: &[NetworkFilter<'a>]) -> NetworkFilter<'a> { let base_filter = &filters[0]; // FIXME: can technically panic, if filters list is empty let mut filter = base_filter.clone(); let mut domains = HashSet::new(); @@ -178,10 +185,10 @@ impl Optimization for UnionDomainGroup { } if base_filter.raw_line.is_some() { - filter.raw_line = Some(Box::new( + filter.raw_line = Some(Cow::Owned( filters .iter() - .flat_map(|f| f.raw_line.clone()) + .flat_map(|f| f.raw_line.as_deref()) .join(" <+> "), )) } @@ -189,7 +196,7 @@ impl Optimization for UnionDomainGroup { filter } - fn group_by_criteria(&self, filter: &NetworkFilter) -> String { + fn group_by_criteria(&self, filter: &NetworkFilter<'_>) -> String { format!( "{:?}:{}:{:b}:{:?}", filter.hostname.as_ref(), @@ -199,7 +206,7 @@ impl Optimization for UnionDomainGroup { ) } - fn select(&self, filter: &NetworkFilter) -> bool { + fn select(&self, filter: &NetworkFilter<'_>) -> bool { !filter.is_csp() && (filter.opt_domains.is_some() || filter.opt_not_domains.is_some()) } } diff --git a/src/request.rs b/src/request.rs index fd5409f1..22602d29 100644 --- a/src/request.rs +++ b/src/request.rs @@ -1,9 +1,25 @@ //! Contains structures needed to describe network requests. +use std::sync::OnceLock; + use thiserror::Error; +use crate::filters::cosmetic::get_entity_hashes_from_labels; use crate::url_parser; -use crate::utils; +use crate::utils::{self, HostnameHashBuffer}; + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum RequestMethod { + Connect, + Delete, + Get, + Head, + Options, + Patch, + Post, + Put, + Other, +} /// The type of resource requested from the URL endpoint. #[derive(Clone, PartialEq, Debug)] @@ -76,9 +92,10 @@ fn cpt_match_type(cpt: &str) -> RequestType { } /// A network [`Request`], used as an interface for network blocking in the [`crate::Engine`]. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct Request { pub request_type: RequestType, + pub method: Option, pub is_http: bool, pub is_https: bool, @@ -86,13 +103,49 @@ pub struct Request { pub is_third_party: bool, pub url: String, pub hostname: String, + pub(crate) domain: String, + pub(crate) source_hostname: String, pub source_hostname_hashes: Option>, + destination_suffix_hashes: OnceLock, + destination_entity_hashes: OnceLock, + pub(crate) url_lower_cased: String, pub(crate) request_tokens: Vec, pub(crate) original_url: String, } +impl Clone for Request { + fn clone(&self) -> Self { + Self { + request_type: self.request_type.clone(), + method: self.method, + is_http: self.is_http, + is_https: self.is_https, + is_supported: self.is_supported, + is_third_party: self.is_third_party, + url: self.url.clone(), + hostname: self.hostname.clone(), + domain: self.domain.clone(), + source_hostname: self.source_hostname.clone(), + source_hostname_hashes: self.source_hostname_hashes.clone(), + destination_suffix_hashes: clone_once_lock(&self.destination_suffix_hashes), + destination_entity_hashes: clone_once_lock(&self.destination_entity_hashes), + url_lower_cased: self.url_lower_cased.clone(), + request_tokens: self.request_tokens.clone(), + original_url: self.original_url.clone(), + } + } +} + +fn clone_once_lock(src: &OnceLock) -> OnceLock { + let lock = OnceLock::new(); + if let Some(buffer) = src.get() { + let _ = lock.set(buffer.clone()); + } + lock +} + impl Request { pub(crate) fn get_url(&self, case_sensitive: bool) -> &str { if case_sensitive { @@ -116,15 +169,112 @@ impl Request { &self.request_tokens } + /// Lazily computed destination suffix hashes for `$to=` plain matching. + pub(crate) fn destination_suffix_hashes(&self) -> Option<&[utils::Hash]> { + if self.hostname.is_empty() { + return None; + } + Some( + self.destination_suffix_hashes + .get_or_init(|| self.compute_destination_suffix_hashes()) + .as_slice(), + ) + } + + /// Lazily computed destination entity hashes for `$to=google.*` style matching. + pub(crate) fn destination_entity_hashes(&self) -> Option<&[utils::Hash]> { + if self.hostname.is_empty() { + return None; + } + let buffer = self.destination_entity_hashes.get_or_init(|| { + let mut buffer = HostnameHashBuffer::new(); + for hash in get_entity_hashes_from_labels(&self.hostname, &self.domain) { + if buffer.try_push(hash).is_err() { + break; + } + } + buffer + }); + if buffer.is_empty() { + None + } else { + Some(buffer.as_slice()) + } + } + + fn compute_destination_suffix_hashes(&self) -> HostnameHashBuffer { + let mut buffer = HostnameHashBuffer::new(); + if self.hostname.is_empty() { + return buffer; + } + if self.hostname == self.source_hostname { + if let Some(source_hashes) = self.source_hostname_hashes.as_ref() { + for &hash in source_hashes { + if buffer.try_push(hash).is_err() { + break; + } + } + return buffer; + } + } + Self::push_suffix_hostname_hashes(&self.hostname, &mut buffer); + buffer + } + + fn push_suffix_hostname_hashes(hostname: &str, buffer: &mut HostnameHashBuffer) { + let _ = buffer.try_push(utils::fast_hash(hostname)); + for (i, c) in hostname.char_indices() { + if c == '.' && i + 1 < hostname.len() { + let _ = buffer.try_push(utils::fast_hash(&hostname[i + 1..])); + } + } + } + + fn suffix_hostname_hashes(hostname: &str) -> Option> { + if hostname.is_empty() { + return None; + } + let mut buffer = HostnameHashBuffer::new(); + Self::push_suffix_hostname_hashes(hostname, &mut buffer); + Some(buffer.into_iter().collect()) + } + + fn parse_method(raw_method: &str) -> Option { + if raw_method.is_empty() { + return None; + } + Some(if raw_method.eq_ignore_ascii_case("connect") { + RequestMethod::Connect + } else if raw_method.eq_ignore_ascii_case("delete") { + RequestMethod::Delete + } else if raw_method.eq_ignore_ascii_case("get") { + RequestMethod::Get + } else if raw_method.eq_ignore_ascii_case("head") { + RequestMethod::Head + } else if raw_method.eq_ignore_ascii_case("options") { + RequestMethod::Options + } else if raw_method.eq_ignore_ascii_case("patch") { + RequestMethod::Patch + } else if raw_method.eq_ignore_ascii_case("post") { + RequestMethod::Post + } else if raw_method.eq_ignore_ascii_case("put") { + RequestMethod::Put + } else { + RequestMethod::Other + }) + } + #[allow(clippy::too_many_arguments)] - fn from_detailed_parameters( + pub(crate) fn from_detailed_parameters( raw_type: &str, url: &str, schema: &str, hostname: &str, + domain: &str, source_hostname: &str, third_party: bool, original_url: String, + method: Option, ) -> Request { let is_http: bool; let is_https: bool; @@ -150,28 +300,22 @@ impl Request { } } - let source_hostname_hashes = if !source_hostname.is_empty() { - let mut hashes = Vec::with_capacity(4); - hashes.push(utils::fast_hash(source_hostname)); - for (i, c) in source_hostname.char_indices() { - if c == '.' && i + 1 < source_hostname.len() { - hashes.push(utils::fast_hash(&source_hostname[i + 1..])); - } - } - Some(hashes) - } else { - None - }; + let source_hostname_hashes = Self::suffix_hostname_hashes(source_hostname); let url_lower_cased = url.to_ascii_lowercase(); Request { request_type, + method, url: url.to_owned(), url_lower_cased: url_lower_cased.to_owned(), hostname: hostname.to_owned(), + domain: domain.to_owned(), + source_hostname: source_hostname.to_owned(), request_tokens: calculate_tokens(&url_lower_cased), source_hostname_hashes, + destination_suffix_hashes: OnceLock::new(), + destination_entity_hashes: OnceLock::new(), is_third_party: third_party, is_http, is_https, @@ -181,8 +325,14 @@ impl Request { } /// Construct a new [`Request`]. - pub fn new(url: &str, source_url: &str, request_type: &str) -> Result { + pub fn new( + url: &str, + source_url: &str, + request_type: &str, + method: &str, + ) -> Result { if let Some(parsed_url) = url_parser::parse_url(url) { + let parsed_method = Self::parse_method(method); if let Some(parsed_source) = url_parser::parse_url(source_url) { let source_domain = parsed_source.domain(); @@ -193,9 +343,11 @@ impl Request { &parsed_url.url, parsed_url.schema(), parsed_url.hostname(), + parsed_url.domain(), parsed_source.hostname(), third_party, url.to_string(), + parsed_method, )) } else { Ok(Request::from_detailed_parameters( @@ -203,9 +355,11 @@ impl Request { &parsed_url.url, parsed_url.schema(), parsed_url.hostname(), + parsed_url.domain(), "", true, url.to_string(), + parsed_method, )) } } else { @@ -222,20 +376,31 @@ impl Request { source_hostname: &str, request_type: &str, third_party: bool, + method: &str, ) -> Request { let splitter = memchr::memchr(b':', url.as_bytes()).unwrap_or(0); let schema: &str = &url[..splitter]; + let (domain_start, domain_end) = url_parser::get_host_domain(hostname); + let domain = &hostname[domain_start..domain_end]; Request::from_detailed_parameters( request_type, url, schema, hostname, + domain, source_hostname, third_party, url.to_string(), + Self::parse_method(method), ) } + + #[cfg(test)] + pub(crate) fn destination_hashes_initialized(&self) -> bool { + self.destination_suffix_hashes.get().is_some() + || self.destination_entity_hashes.get().is_some() + } } fn calculate_tokens(url_lower_cased: &str) -> Vec { diff --git a/src/resources/mod.rs b/src/resources/mod.rs index d3dc7c52..5d29a825 100644 --- a/src/resources/mod.rs +++ b/src/resources/mod.rs @@ -39,19 +39,19 @@ use serde::{Deserialize, Serialize}; /// # use adblock::lists::ParseOptions; /// # use adblock::resources::{MimeType, PermissionMask, Resource, ResourceStorage, ResourceType}; /// # let mut filter_set = adblock::lists::FilterSet::default(); -/// # let untrusted_filters = vec![""]; -/// # let trusted_filters = vec![""]; +/// # let untrusted_filters = "".to_string(); +/// # let trusted_filters = "".to_string(); /// const COOKIE_ACCESS: PermissionMask = PermissionMask::from_bits(0b00000001); /// const LOCALSTORAGE_ACCESS: PermissionMask = PermissionMask::from_bits(0b00000010); /// /// // `untrusted_filters` will not be able to use privileged scriptlet injections. -/// filter_set.add_filters( +/// filter_set.add_filter_list( /// untrusted_filters, /// Default::default(), /// ); /// // `trusted_filters` will be able to inject scriptlets requiring `COOKIE_ACCESS` /// // permissions or `LOCALSTORAGE_ACCESS` permissions. -/// filter_set.add_filters( +/// filter_set.add_filter_list( /// trusted_filters, /// ParseOptions { /// permissions: COOKIE_ACCESS | LOCALSTORAGE_ACCESS, @@ -59,7 +59,7 @@ use serde::{Deserialize, Serialize}; /// }, /// ); /// -/// let mut engine = Engine::from_filter_set(filter_set, true); +/// let mut engine = Engine::new_with_filter_set(filter_set, true); /// // The `trusted-set-cookie` scriptlet cannot be injected without `COOKIE_ACCESS` /// // permission. /// engine.use_resources([Resource { diff --git a/src/utils.rs b/src/utils.rs index 6930ac11..b80cc79e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -33,6 +33,9 @@ fn is_allowed_filter(ch: char) -> bool { /// Used instread of Vec to avoid heap allocations. pub(crate) type TokensBuffer = ArrayVec; +/// Suffix-chain and entity hashes for a hostname (typically fewer than a dozen labels). +pub(crate) type HostnameHashBuffer = ArrayVec; + fn fast_tokenizer_no_regex( pattern: &str, is_allowed_code: &dyn Fn(char) -> bool, diff --git a/tests/legacy_harness.rs b/tests/legacy_harness.rs index 1cb133d6..1939afec 100644 --- a/tests/legacy_harness.rs +++ b/tests/legacy_harness.rs @@ -34,13 +34,13 @@ mod legacy_test_filters { filter.filter ); - let engine = Engine::from_rules_debug([raw_filter], Default::default()); + let engine = Engine::new_with_list_text(raw_filter, Default::default()); for to_block in blocked { assert!( engine .check_network_request( - &Request::new(to_block, "https://example.com", "other").unwrap(), + &Request::new(to_block, "https://example.com", "other", "").unwrap(), ) .matched, "Expected filter {} to match {}", @@ -53,7 +53,7 @@ mod legacy_test_filters { assert!( !engine .check_network_request( - &Request::new(to_pass, "https://example.com", "other").unwrap(), + &Request::new(to_pass, "https://example.com", "other", "").unwrap(), ) .matched, "Expected filter {} to pass {}", @@ -305,9 +305,9 @@ mod legacy_test_filters { ); // explicit, separate testcase construction of the "script" option as it is not the deafult - let request = Request::new("http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html#xpc=sf-gdn-exp-2&p=http%3A//slashdot.org;", "https://this-is-always-third-party.com", "script").unwrap(); - let engine = Engine::from_rules_debug( - ["||googlesyndication.com/safeframe/$third-party,script"], + let request = Request::new("http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html#xpc=sf-gdn-exp-2&p=http%3A//slashdot.org;", "https://this-is-always-third-party.com", "script", "").unwrap(); + let engine = Engine::new_with_list_text( + "||googlesyndication.com/safeframe/$third-party,script", Default::default(), ); assert!(engine.check_network_request(&request).matched); @@ -324,7 +324,7 @@ mod legacy_check_match { not_blocked: &[&'a str], tags: &[&'a str], ) { - let mut engine = Engine::from_rules(rules, Default::default()); // first one with the provided rules + let mut engine = Engine::new_with_list_text(rules.join("\n"), Default::default()); // first one with the provided rules engine.use_tags(tags); let mut engine_deserialized = Engine::default(); // second empty @@ -335,7 +335,7 @@ mod legacy_check_match { } for to_block in blocked { - let request = Request::new(to_block, "alwaysthirdparty.com", "script").unwrap(); + let request = Request::new(to_block, "alwaysthirdparty.com", "script", "").unwrap(); assert!( engine.check_network_request(&request).matched, @@ -353,7 +353,7 @@ mod legacy_check_match { } for to_pass in not_blocked { - let request = Request::new(to_pass, "alwaysthirdparty.com", "script").unwrap(); + let request = Request::new(to_pass, "alwaysthirdparty.com", "script", "").unwrap(); assert!( !engine.check_network_request(&request).matched, @@ -395,11 +395,12 @@ mod legacy_check_match { { // Explicitly write out the full case instead of using check_match helper // or tweaking it to allow passing in the source domain for this one case - let engine = Engine::from_rules( + let engine = Engine::new_with_list_text( [ "/ads/freewheel/*", "@@||turner.com^*/ads/freewheel/*/AdManager.js$domain=cnn.com", - ], + ] + .join("\n"), Default::default(), ); let mut engine_deserialized = Engine::default(); // second empty @@ -412,6 +413,7 @@ mod legacy_check_match { "http://z.cdn.turner.com/xslo/cvp/ads/freewheel/js/0/AdManager.js", "http://cnn.com", "", + "", ) .unwrap(); assert!(!engine.check_network_request(&request).matched); @@ -505,10 +507,10 @@ mod legacy_check_options { use adblock::Engine; fn check_option_rule<'a>(rules: &[&'a str], tests: &[(&'a str, &'a str, &'a str, bool)]) { - let engine = Engine::from_rules(rules, Default::default()); // first one with the provided rules + let engine = Engine::new_with_list_text(rules.join("\n"), Default::default()); // first one with the provided rules for (url, source_url, request_type, expectation) in tests { - let request = Request::new(url, source_url, request_type).unwrap(); + let request = Request::new(url, source_url, request_type, "").unwrap(); assert!( engine.check_network_request(&request).matched == *expectation, "Expected match = {expectation} for {url} from {source_url} typed {request_type} against {rules:?}" @@ -851,8 +853,8 @@ mod legacy_misc_tests { #[test] fn demo_app() { // Demo app test - let engine = Engine::from_rules( - ["||googlesyndication.com/safeframe/$third-party"], + let engine = Engine::new_with_list_text( + "||googlesyndication.com/safeframe/$third-party", Default::default(), ); @@ -860,6 +862,7 @@ mod legacy_misc_tests { "http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html", "http://slashdot.org", "script", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched) @@ -870,24 +873,25 @@ mod legacy_misc_tests { // Host anchor is calculated correctly let filter = NetworkFilter::parse("||test.com$third-party", false, Default::default()).unwrap(); - assert_eq!(filter.hostname, Some(String::from("test.com"))); + assert_eq!(filter.hostname.as_deref(), Some("test.com")); let filter = NetworkFilter::parse("||test.com/ok$third-party", false, Default::default()).unwrap(); - assert_eq!(filter.hostname, Some(String::from("test.com"))); + assert_eq!(filter.hostname.as_deref(), Some("test.com")); let filter = NetworkFilter::parse("||test.com/ok", false, Default::default()).unwrap(); - assert_eq!(filter.hostname, Some(String::from("test.com"))); + assert_eq!(filter.hostname.as_deref(), Some("test.com")); } #[test] fn serialization_tests() { - let engine = Engine::from_rules_parametrised( + let engine = Engine::new_with_list_text_parametrised( [ "||googlesyndication.com$third-party", "@@||googlesyndication.ca", "a$explicitcancel", - ], + ] + .join("\n"), Default::default(), true, false, @@ -903,7 +907,8 @@ mod legacy_misc_tests { &Request::new( "https://googlesyndication.com/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -915,7 +920,8 @@ mod legacy_misc_tests { &Request::new( "https://googlesyndication.com/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -927,7 +933,8 @@ mod legacy_misc_tests { &Request::new( "https://googleayndication.com/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -939,7 +946,8 @@ mod legacy_misc_tests { &Request::new( "https://googleayndication.com/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -952,7 +960,8 @@ mod legacy_misc_tests { &Request::new( "https://googlesyndication.ca/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -964,7 +973,8 @@ mod legacy_misc_tests { &Request::new( "https://googlesyndication.ca/script.js", "https://example.com", - "script" + "script", + "" ) .unwrap() ) @@ -974,12 +984,15 @@ mod legacy_misc_tests { #[test] fn find_matching_filters() { - let engine = Engine::from_rules_debug( + let engine = Engine::new_with_list_text_parametrised( [ "||googlesyndication.com/safeframe/$third-party", "||brianbondy.com/ads", - ], + ] + .join("\n"), Default::default(), + true, + true, ); let current_page_frame = "http://slashdot.org"; @@ -990,6 +1003,7 @@ mod legacy_misc_tests { "http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html", current_page_frame, request_type, + "", ) .unwrap(); let checked = engine.check_network_request(&request); @@ -1005,7 +1019,8 @@ mod legacy_misc_tests { ); // Test when no filter is found, returns None - let request = Request::new("http://ssafsdf.com", current_page_frame, request_type).unwrap(); + let request = + Request::new("http://ssafsdf.com", current_page_frame, request_type, "").unwrap(); let checked = engine.check_network_request(&request); assert!(!checked.matched, "Expected url to pass"); assert!(checked.filter.is_none(), "Expected no fitler to match"); @@ -1018,13 +1033,16 @@ mod legacy_misc_tests { #[test] fn find_matching_filters_exceptions() { - let engine = Engine::from_rules_debug( + let engine = Engine::new_with_list_text_parametrised( [ "||googlesyndication.com/safeframe/$third-party", "||brianbondy.com/ads", "@@safeframe", - ], + ] + .join("\n"), Default::default(), + true, + true, ); let current_page_frame = "http://slashdot.org"; @@ -1035,6 +1053,7 @@ mod legacy_misc_tests { "http://tpc.googlesyndication.com/safeframe/1-0-2/html/container.html", current_page_frame, request_type, + "", ) .unwrap(); let checked = engine.check_network_request(&request); @@ -1056,13 +1075,15 @@ mod legacy_misc_tests { #[test] fn matches_with_filter_info_preserves_important() { // exceptions have not effect if important filter matches - let engine = Engine::from_rules_debug( - ["||brianbondy.com^$important", "@@||brianbondy.com^"], + let engine = Engine::new_with_list_text_parametrised( + ["||brianbondy.com^$important", "@@||brianbondy.com^"].join("\n"), Default::default(), + true, + true, ); let request = - Request::new("https://brianbondy.com/t", "https://test.com", "script").unwrap(); + Request::new("https://brianbondy.com/t", "https://test.com", "script", "").unwrap(); let checked = engine.check_network_request(&request); assert!(checked.matched); diff --git a/tests/live.rs b/tests/live.rs index 444a31dc..6b24036e 100644 --- a/tests/live.rs +++ b/tests/live.rs @@ -124,8 +124,8 @@ fn get_all_filters() -> &'static adblock::lists::FilterSet { .await .iter() .for_each(|(format, list)| { - filter_set.add_filters( - list.lines().map(|s| s.to_owned()).collect::>(), + filter_set.add_filter_list( + list.clone(), adblock::lists::ParseOptions { format: *format, ..Default::default() @@ -150,7 +150,7 @@ fn troubleshoot() { } fn get_blocker_engine() -> Engine { - let mut engine = Engine::from_filter_set(get_all_filters().clone(), true); + let mut engine = Engine::new_with_filter_set(get_all_filters().clone(), true); engine.use_tags(&["fb-embeds", "twitter-embeds"]); @@ -166,6 +166,7 @@ fn check_live_specific_urls() { "https://static.scroll.com/js/scroll.js", "https://www.theverge.com/", "script", + "", ) .unwrap(), ); @@ -182,6 +183,7 @@ fn check_live_specific_urls() { "https://platform.twitter.com/widgets.js", "https://fmarier.github.io/brave-testing/social-widgets.html", "script", + "", ) .unwrap(), ); @@ -194,11 +196,7 @@ fn check_live_specific_urls() { } { engine.disable_tags(&["twitter-embeds"]); - let checked = engine.check_network_request(&Request::new( - "https://imagesrv.adition.com/banners/1337/files/00/0e/6f/09/000000945929.jpg?PQgSgs13hf1fw.jpg", - "https://spiegel.de", - "image", - ).unwrap()); + let checked = engine.check_network_request(&Request::new("https://imagesrv.adition.com/banners/1337/files/00/0e/6f/09/000000945929.jpg?PQgSgs13hf1fw.jpg", "https://spiegel.de", "image", "").unwrap()); assert!( checked.matched, "Expected match, got filter {:?}, exception {:?}", @@ -214,8 +212,9 @@ fn check_live_from_filterlists() { let requests = load_requests(); for req in requests { - let checked = engine - .check_network_request(&Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap()); + let checked = engine.check_network_request( + &Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(), + ); assert_eq!( checked.matched, req.blocked, "Expected match {} for {} at {}, got filter {:?}, exception {:?}", @@ -243,6 +242,7 @@ fn check_live_redirects() { "https://c.amazon-adsystem.com/aax2/amzn_ads.js", "https://aussieexotics.com/", "script", + "", ) .unwrap(), ); @@ -261,6 +261,7 @@ fn check_live_redirects() { "https://www.googletagservices.com/tag/js/gpt.js", "https://tvguide.com/", "script", + "", ) .unwrap(), ); @@ -278,7 +279,7 @@ fn check_live_redirects() { /// Ensure that one engine's serialization result can be exactly reproduced by another engine after /// deserializing from it. fn stable_serialization_through_load() { - let engine1 = Engine::from_filter_set(get_all_filters().clone(), true); + let engine1 = Engine::new_with_filter_set(get_all_filters().clone(), true); let ser1 = engine1.serialize().to_vec(); let mut engine2 = Engine::default(); diff --git a/tests/matching.rs b/tests/matching.rs index d72c417f..f9b80453 100644 --- a/tests/matching.rs +++ b/tests/matching.rs @@ -40,9 +40,9 @@ fn build_resources_from_filters(filters: &[String]) -> Vec { let redirect = f.modifier_option.unwrap(); Resource { - name: redirect.to_owned(), + name: redirect.to_string(), aliases: vec![], - kind: ResourceType::Mime(MimeType::from_extension(&redirect)), + kind: ResourceType::Mime(MimeType::from_extension(redirect)), content: BASE64_STANDARD.encode(redirect), dependencies: vec![], permission: Default::default(), @@ -77,10 +77,9 @@ fn check_filter_matching() { .set(NetworkFilterMask::IS_EXCEPTION, false); filters.push(original_filter); } - let filter_set = adblock::FilterSet::new_with_rules(filters, vec![], true); - let engine = adblock::Engine::from_filter_set(filter_set, true); + let engine = adblock::Engine::new_with_parsed_rules(filters, vec![], true); - let request_res = Request::new(&req.url, &req.sourceUrl, &req.r#type); + let request_res = Request::new(&req.url, &req.sourceUrl, &req.r#type, ""); // The dataset has cases where URL is set to just "http://" or "https://", which we do not support if let Ok(request) = request_res { let result = engine.check_network_request(&request); @@ -121,7 +120,7 @@ fn check_engine_matching() { } for filter in req.filters { let opts = ParseOptions::default(); - let mut engine = Engine::from_rules_debug(std::slice::from_ref(&filter), opts); + let mut engine = Engine::new_with_list_text(filter.clone(), opts); let resources = build_resources_from_filters(std::slice::from_ref(&filter)); engine.use_resources(resources); @@ -132,7 +131,7 @@ fn check_engine_matching() { ); let network_filter = network_filter_res.unwrap(); - let request = Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap(); + let request = Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(); let result = engine.check_network_request(&request); if network_filter.is_exception() { @@ -186,15 +185,15 @@ fn check_rule_matching_browserlike() { impl From<&TestRequest> for Request { fn from(v: &TestRequest) -> Self { - Request::new(&v.url, &v.frameUrl, &v.cpt).unwrap() + Request::new(&v.url, &v.frameUrl, &v.cpt, "").unwrap() } } fn load_requests() -> Vec { - let requests_str = rules_from_lists(&["data/requests.json"]); + let requests_str = rules_from_lists(["data/requests.json"]); requests_str - .into_iter() - .filter_map(|r| serde_json::from_str(&r).ok()) + .lines() + .filter_map(|r| serde_json::from_str(r).ok()) .collect() } @@ -214,7 +213,7 @@ fn check_rule_matching_browserlike() { let requests = load_requests(); let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); - let engine = Engine::from_rules(rules, Default::default()); + let engine = Engine::new_with_list_text(rules, Default::default()); let (blocked, passes) = bench_rule_matching_browserlike(&engine, &requests); let msg = "The number of blocked/passed requests has changed. ".to_string() + "If this is expected, update the expected values in the test."; diff --git a/tests/simple_use.rs b/tests/simple_use.rs index 4b4f9259..c07e6bb1 100644 --- a/tests/simple_use.rs +++ b/tests/simple_use.rs @@ -10,12 +10,13 @@ fn check_simple_use() { "-advertisement/script.", ]; - let engine = Engine::from_rules(rules, Default::default()); + let engine = Engine::new_with_list_text(rules.join("\n"), Default::default()); let request = Request::new( "http://example.com/-advertisement-icon.", "http://example.com/helloworld", "image", + "", ) .unwrap(); let blocker_result = engine.check_network_request(&request); diff --git a/tests/test_utils.rs b/tests/test_utils.rs index a4e3a3db..5a1575c0 100644 --- a/tests/test_utils.rs +++ b/tests/test_utils.rs @@ -2,18 +2,12 @@ //! needed outside of this directory. #[cfg(not(target_arch = "wasm32"))] -pub fn rules_from_lists( - lists: impl IntoIterator>, -) -> impl Iterator { - fn read_file_lines(filename: &str) -> impl Iterator { - use std::fs::File; - use std::io::{BufRead, BufReader}; - - let reader = BufReader::new(File::open(filename).unwrap()); - reader.lines().map(|r| r.unwrap()) +pub fn rules_from_lists(list_files: impl IntoIterator>) -> String { + let mut contents = String::new(); + for file in list_files { + contents.push_str(&std::fs::read_to_string(file.as_ref()).unwrap()); + contents.push('\n'); } - - lists - .into_iter() - .flat_map(|filename| read_file_lines(filename.as_ref())) + contents.shrink_to_fit(); + contents } diff --git a/tests/ublock-coverage.rs b/tests/ublock-coverage.rs index 94aa7aca..a3642f9a 100644 --- a/tests/ublock-coverage.rs +++ b/tests/ublock-coverage.rs @@ -48,11 +48,11 @@ fn get_blocker_engine() -> Engine { "data/regression-testing/easyprivacy.txt", ]); - Engine::from_rules_parametrised(rules, Default::default(), true, false) + Engine::new_with_list_text_parametrised(rules, Default::default(), true, false) } fn get_blocker_engine_default(extra_rules: impl IntoIterator>) -> Engine { - let rules = rules_from_lists([ + let mut rules = rules_from_lists([ "data/easylist.to/easylist/easylist.txt", "data/easylist.to/easylist/easyprivacy.txt", "data/uBlockOrigin/unbreak.txt", @@ -62,19 +62,27 @@ fn get_blocker_engine_default(extra_rules: impl IntoIterator = HashMap::new(); let mut false_negative_exceptions: HashMap = HashMap::new(); for req in requests { - let request = Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap(); + let request = Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(); let checked = engine.check_network_request(&request); if req.blocked == 1 && !checked.matched { mismatch_expected_match += 1; @@ -281,13 +273,14 @@ fn check_matching_hostnames() { let source_domain = source_host.domain(); let third_party = source_domain != domain; - let request = Request::new(&req.url, &req.sourceUrl, &req.r#type).unwrap(); + let request = Request::new(&req.url, &req.sourceUrl, &req.r#type, "").unwrap(); let preparsed_request = Request::preparsed( &req.url, url_host.hostname(), source_host.hostname(), &req.r#type, third_party, + "", ); let checked = engine.check_network_request(&request); diff --git a/tests/unit/blocker.rs b/tests/unit/blocker.rs index dfd1590c..1f1d4738 100644 --- a/tests/unit/blocker.rs +++ b/tests/unit/blocker.rs @@ -2,7 +2,6 @@ mod blocker_tests { use super::super::*; - use crate::lists::parse_filters; use crate::request::Request; use crate::resources::{Resource, ResourceStorage}; use base64::{engine::Engine as _, prelude::BASE64_STANDARD}; @@ -13,18 +12,17 @@ mod blocker_tests { fn single_slash() { let filters = ["/|"]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let request = Request::new( "https://example.com/test/", "https://example.com", "xmlhttprequest", + "", ) .unwrap(); assert!(blocker.check(&request, &Default::default()).matched); @@ -33,6 +31,7 @@ mod blocker_tests { "https://example.com/test", "https://example.com", "xmlhttprequest", + "", ) .unwrap(); assert!(!blocker.check(&request, &Default::default()).matched); @@ -42,13 +41,11 @@ mod blocker_tests { filters: impl IntoIterator>, requests: &[(Request, bool)], ) { - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); requests.iter().for_each(|(req, expected_result)| { let matched_rule = blocker.check(req, &Default::default()); @@ -75,16 +72,15 @@ mod blocker_tests { "https://imdb-video.media-imdb.com/kBOeI88k1o23eNAi", "https://www.imdb.com/video/13", "media", + "", ) .unwrap(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::in_memory_from_resources([Resource::simple( "noop-0.1s.mp3", crate::resources::MimeType::AudioMp3, @@ -115,16 +111,15 @@ mod blocker_tests { "https://imdb-video.media-imdb.com/kBOeI88k1o23eNAi", "https://www.imdb.com/video/13", "media", + "", ) .unwrap(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::in_memory_from_resources([Resource::simple( "noop-0.1s.mp3", crate::resources::MimeType::AudioMp3, @@ -150,16 +145,19 @@ mod blocker_tests { "||www3.doubleclick.net^$xmlhttprequest,redirect-rule=noop.txt,domain=lineups.fun", ]; - let request = - Request::new("https://www3.doubleclick.net", "https://lineups.fun", "xhr").unwrap(); - - let (network_filters, _) = parse_filters(filters, true, Default::default()); + let request = Request::new( + "https://www3.doubleclick.net", + "https://lineups.fun", + "xhr", + "", + ) + .unwrap(); let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::in_memory_from_resources([Resource::simple( "noop.txt", crate::resources::MimeType::TextPlain, @@ -180,7 +178,7 @@ mod blocker_tests { fn badfilter_does_not_match() { let filters = ["||foo.com$badfilter"]; let url_results = [( - Request::new("https://foo.com", "https://bar.com", "image").unwrap(), + Request::new("https://foo.com", "https://bar.com", "image", "").unwrap(), false, )]; @@ -196,7 +194,7 @@ mod blocker_tests { "||foo.com$domain=foo.com|bar.com", ]; let url_results = [( - Request::new("https://foo.com", "https://bar.com", "image").unwrap(), + Request::new("https://foo.com", "https://bar.com", "image", "").unwrap(), false, )]; @@ -212,7 +210,7 @@ mod blocker_tests { "||foo.com$domain=foo.com|bar.com,image", ]; let url_results = [( - Request::new("https://foo.com", "https://bar.com", "image").unwrap(), + Request::new("https://foo.com", "https://bar.com", "image", "").unwrap(), true, )]; @@ -225,18 +223,17 @@ mod blocker_tests { fn trailing_dot_domain() { let filters = ["||dot.example.com.^", "||test.example.com^"]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let request = Request::new( "https://dot.example.com", "https://dot.example.com", "document", + "", ) .unwrap(); assert!(!blocker.check(&request, &Default::default()).matched); @@ -245,6 +242,7 @@ mod blocker_tests { "https://dot.example.com.", "https://dot.example.com.", "document", + "", ) .unwrap(); assert!(blocker.check(&request, &Default::default()).matched); @@ -253,6 +251,7 @@ mod blocker_tests { "https://test.example.com", "https://test.example.com", "document", + "", ) .unwrap(); assert!(blocker.check(&request, &Default::default()).matched); @@ -261,6 +260,7 @@ mod blocker_tests { "https://test.example.com.", "https://test.example.com.", "document", + "", ) .unwrap(); assert!(blocker.check(&request, &Default::default()).matched); @@ -278,6 +278,7 @@ mod blocker_tests { "https://r.alimc1.top/test.js", "https://minisite.letv.com/", "script", + "", ) .unwrap(), true, @@ -287,6 +288,7 @@ mod blocker_tests { "https://www.baidu.com/test.js", "https://minisite.letv.com/", "script", + "", ) .unwrap(), false, @@ -296,6 +298,7 @@ mod blocker_tests { "https://r.aabb.top/test.js", "https://example.com/", "script", + "", ) .unwrap(), false, @@ -305,19 +308,18 @@ mod blocker_tests { "https://r.aabb.top/test.js", "https://minisite.letv.com/", "script", + "", ) .unwrap(), true, ), ]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::default(); url_results.into_iter().for_each(|(req, expected_result)| { @@ -346,13 +348,11 @@ mod blocker_tests { "^first-party-only^$csp=script-src 'none',1p", ]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: false, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); { // No directives should be returned for requests that are not `document` or `subdocument` content types. @@ -361,7 +361,8 @@ mod blocker_tests { &Request::new( "https://pirateproxy.live/static/custom_ads.js", "https://pirateproxy.live", - "script" + "script", + "" ) .unwrap() ), @@ -372,7 +373,8 @@ mod blocker_tests { &Request::new( "https://pirateproxy.live/static/custom_ads.js", "https://pirateproxy.live", - "image" + "image", + "" ) .unwrap() ), @@ -383,7 +385,8 @@ mod blocker_tests { &Request::new( "https://pirateproxy.live/static/custom_ads.js", "https://pirateproxy.live", - "object" + "object", + "" ) .unwrap() ), @@ -394,14 +397,20 @@ mod blocker_tests { // A single directive should be returned if only one match is present in the engine, for both document and subdocument types assert_eq!( blocker.get_csp_directives( - &Request::new("https://example.com", "https://vidoza.co", "document").unwrap() + &Request::new("https://example.com", "https://vidoza.co", "document", "") + .unwrap() ), Some(String::from("worker-src 'none'")) ); assert_eq!( blocker.get_csp_directives( - &Request::new("https://example.com", "https://vidoza.net", "subdocument") - .unwrap() + &Request::new( + "https://example.com", + "https://vidoza.net", + "subdocument", + "" + ) + .unwrap() ), Some(String::from("worker-src 'none'")) ); @@ -421,7 +430,8 @@ mod blocker_tests { &Request::new( "https://example.com", "https://pirateproxy.live", - "document" + "document", + "" ) .unwrap() ) @@ -431,7 +441,8 @@ mod blocker_tests { &Request::new( "https://example.com", "https://pirateproxy.live", - "subdocument" + "subdocument", + "" ) .unwrap() ) @@ -441,14 +452,19 @@ mod blocker_tests { // A directive with an exception should not be returned assert_eq!( blocker.get_csp_directives( - &Request::new("https://1337x.to", "https://1337x.to", "document").unwrap() + &Request::new("https://1337x.to", "https://1337x.to", "document", "").unwrap() ), Some(String::from("script-src 'self' 'unsafe-inline'")) ); assert_eq!( blocker.get_csp_directives( - &Request::new("https://1337x.to/no-csp", "https://1337x.to", "subdocument") - .unwrap() + &Request::new( + "https://1337x.to/no-csp", + "https://1337x.to", + "subdocument", + "" + ) + .unwrap() ), None ); @@ -460,7 +476,8 @@ mod blocker_tests { &Request::new( "https://example.com/duplicated-directive", "https://flashx.cc", - "document" + "document", + "" ) .unwrap() ), @@ -471,7 +488,8 @@ mod blocker_tests { &Request::new( "https://example.com/duplicated-directive", "https://flashx.cc", - "subdocument" + "subdocument", + "" ) .unwrap() ), @@ -485,7 +503,8 @@ mod blocker_tests { &Request::new( "https://1337x.to/duplicated-directive/disable-all", "https://thepiratebay10.org", - "document" + "document", + "" ) .unwrap() ), @@ -496,7 +515,8 @@ mod blocker_tests { &Request::new( "https://1337x.to/duplicated-directive/disable-all", "https://thepiratebay10.org", - "document" + "document", + "" ) .unwrap() ), @@ -510,7 +530,8 @@ mod blocker_tests { &Request::new( "htps://github.com/first-party-only", "https://example.com", - "subdocument" + "subdocument", + "" ) .unwrap() ), @@ -521,7 +542,8 @@ mod blocker_tests { &Request::new( "https://example.com/first-party-only", "https://example.com", - "document" + "document", + "" ) .unwrap() ), @@ -540,13 +562,11 @@ mod blocker_tests { "$removeparam=testCase,~xhr", ]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::in_memory_from_resources([Resource::simple( "noopjs", crate::resources::MimeType::ApplicationJavascript, @@ -558,6 +578,7 @@ mod blocker_tests { "https://example.com?q=1&test=2#blue", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -573,6 +594,7 @@ mod blocker_tests { "https://example.com?test=2&q=1#blue", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -588,6 +610,7 @@ mod blocker_tests { "https://example.com?test=2#blue", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -599,7 +622,13 @@ mod blocker_tests { assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?q=1#blue", "https://antonok.com", "xhr").unwrap(), + &Request::new( + "https://example.com?q=1#blue", + "https://antonok.com", + "xhr", + "", + ) + .unwrap(), &resources, ); assert_eq!(result.rewritten_url, None); @@ -610,6 +639,7 @@ mod blocker_tests { "https://example.com?q=1&test=2", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -622,6 +652,7 @@ mod blocker_tests { "https://example.com?test=2&q=1", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -630,28 +661,46 @@ mod blocker_tests { assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?test=2", "https://antonok.com", "xhr").unwrap(), + &Request::new( + "https://example.com?test=2", + "https://antonok.com", + "xhr", + "", + ) + .unwrap(), &resources, ); assert_eq!(result.rewritten_url, Some("https://example.com".into())); assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?test=2", "https://antonok.com", "image").unwrap(), + &Request::new( + "https://example.com?test=2", + "https://antonok.com", + "image", + "", + ) + .unwrap(), &resources, ); assert_eq!(result.rewritten_url, None); assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?q=1", "https://antonok.com", "xhr").unwrap(), + &Request::new("https://example.com?q=1", "https://antonok.com", "xhr", "").unwrap(), &resources, ); assert_eq!(result.rewritten_url, None); assert!(!result.matched); let result = blocker.check( - &Request::new("https://example.com?q=fbclid", "https://antonok.com", "xhr").unwrap(), + &Request::new( + "https://example.com?q=fbclid", + "https://antonok.com", + "xhr", + "", + ) + .unwrap(), &resources, ); assert_eq!(result.rewritten_url, None); @@ -662,6 +711,7 @@ mod blocker_tests { "https://example.com?fbclid=10938&q=1&test=2", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -674,6 +724,7 @@ mod blocker_tests { "https://test.com?fbclid=10938&q=1&test=2", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -689,6 +740,7 @@ mod blocker_tests { "https://example.com?q1=1&q2=2&q3=3&test=2&q4=4&q5=5&fbclid=39", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -704,6 +756,7 @@ mod blocker_tests { "https://example.com?q1=1&q1=2&test=2&test=3", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -719,6 +772,7 @@ mod blocker_tests { "https://example.com/script.js?test=2#blue", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -738,6 +792,7 @@ mod blocker_tests { "https://example.com/block/script.js?test=2", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -754,6 +809,7 @@ mod blocker_tests { "https://example.com/Path/?Test=ABC&testcase=AbC&testCase=aBc", "https://antonok.com", "xhr", + "", ) .unwrap(), &resources, @@ -766,6 +822,7 @@ mod blocker_tests { "https://example.com/Path/?Test=ABC&testcase=AbC&testCase=aBc", "https://antonok.com", "image", + "", ) .unwrap(), &resources, @@ -778,6 +835,7 @@ mod blocker_tests { "https://example.com/Path/?Test=ABC&testcase=AbC&testCase=aBc", "https://antonok.com", "subdocument", + "", ) .unwrap(), &resources, @@ -793,6 +851,7 @@ mod blocker_tests { "https://example.com/Path/?Test=ABC&testcase=AbC&testCase=aBc", "https://antonok.com", "document", + "", ) .unwrap(), &resources, @@ -808,6 +867,7 @@ mod blocker_tests { "https://example.com?Test=ABC?123&test=3#&test=4#b", "https://antonok.com", "document", + "", ) .unwrap(), &resources, @@ -823,6 +883,7 @@ mod blocker_tests { "https://example.com?Test=ABC&testCase=5", "https://antonok.com", "document", + "", ) .unwrap(), &resources, @@ -838,6 +899,7 @@ mod blocker_tests { "https://example.com?Test=ABC&testCase=5", "https://antonok.com", "image", + "", ) .unwrap(), &resources, @@ -938,18 +1000,16 @@ mod blocker_tests { .map(|s| format!("*$removeparam={s}")) .collect::>(); - let (network_filters, _) = parse_filters(&filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let resources = ResourceStorage::default(); for (original, expected) in testcases.into_iter() { let result = blocker.check( - &Request::new(original, "https://example.net", "xhr").unwrap(), + &Request::new(original, "https://example.net", "xhr", "").unwrap(), &resources, ); let expected = if original == expected { @@ -968,19 +1028,18 @@ mod blocker_tests { fn test_removeparam_same_tokens() { let filters = ["$removeparam=example1_", "$removeparam=example1-"]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); let result = blocker.check( &Request::new( "https://example.com?example1_=1&example1-=2", "https://example.com", "xhr", + "", ) .unwrap(), &Default::default(), @@ -1000,13 +1059,11 @@ mod blocker_tests { "@@^exceptc20^$redirect-rule=c:20", ]; - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options = BlockerOptions { enable_optimizations: true, }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new(filters, &blocker_options); fn simple_resource(identifier: &str) -> Resource { Resource::simple( identifier, @@ -1035,6 +1092,7 @@ mod blocker_tests { "https://example.net/test", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1047,6 +1105,7 @@ mod blocker_tests { "https://example.net/test.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1059,6 +1118,7 @@ mod blocker_tests { "https://example.com/test.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1071,6 +1131,7 @@ mod blocker_tests { "https://example.com/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1083,6 +1144,7 @@ mod blocker_tests { "https://example.com/exceptc20/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1095,6 +1157,7 @@ mod blocker_tests { "https://example.com/exceptb10/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1107,6 +1170,7 @@ mod blocker_tests { "https://example.com/exceptc20/exceptb10/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1119,6 +1183,7 @@ mod blocker_tests { "https://example.com/exceptc20/exceptb10/excepta/text.txt", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1131,6 +1196,7 @@ mod blocker_tests { "https://example.com/exceptc20/exceptb10/text", "https://example.com", "xmlhttprequest", + "", ) .unwrap(), &resources, @@ -1157,18 +1223,16 @@ mod blocker_tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let mut blocker = Blocker::new(network_filters, &blocker_options); + let mut blocker = Blocker::new(filters, &blocker_options); let resources = Default::default(); blocker.enable_tags(&["stuff"]); assert_eq!( @@ -1210,18 +1274,16 @@ mod blocker_tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let mut blocker = Blocker::new(network_filters, &blocker_options); + let mut blocker = Blocker::new(filters, &blocker_options); let resources = Default::default(); blocker.enable_tags(&["stuff"]); blocker.enable_tags(&["brian"]); @@ -1264,18 +1326,16 @@ mod blocker_tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); - let (network_filters, _) = parse_filters(filters, true, Default::default()); - let blocker_options: BlockerOptions = BlockerOptions { enable_optimizations: false, // optimizations will reduce number of rules }; - let mut blocker = Blocker::new(network_filters, &blocker_options); + let mut blocker = Blocker::new(filters, &blocker_options); let resources = Default::default(); blocker.enable_tags(&["brian", "stuff"]); assert_eq!( @@ -1310,12 +1370,7 @@ mod blocker_tests { enable_optimizations: true, }; - let mut filter_set = crate::lists::FilterSet::new(true); - filter_set - .add_filter("@@*ad_banner.png", Default::default()) - .unwrap(); - - let blocker = Blocker::new(filter_set.network_filters, &blocker_options); + let blocker = Blocker::new(["@@*ad_banner.png"], &blocker_options); let resources = Default::default(); @@ -1323,6 +1378,7 @@ mod blocker_tests { "http://example.com/ad_banner.png", "https://example.com", "other", + "", ) .unwrap(); @@ -1337,15 +1393,10 @@ mod blocker_tests { enable_optimizations: true, }; - let mut filter_set = crate::lists::FilterSet::new(true); - filter_set - .add_filter("@@||example.com$generichide", Default::default()) - .unwrap(); - - let blocker = Blocker::new(filter_set.network_filters, &blocker_options); + let blocker = Blocker::new(["@@||example.com$generichide\n"], &blocker_options); assert!(blocker.check_generic_hide( - &Request::new("https://example.com", "https://example.com", "other").unwrap() + &Request::new("https://example.com", "https://example.com", "other", "").unwrap() )); } } @@ -1356,13 +1407,16 @@ mod placeholder_string_tests { #[test] fn test_constant_placeholder_string() { let mut filter_set = crate::lists::FilterSet::new(false); - filter_set - .add_filter("||example.com^", Default::default()) - .unwrap(); - let engine = crate::Engine::from_filter_set(filter_set, true); + filter_set.add_filter_list("||example.com^\n".to_string(), Default::default()); + let engine = crate::Engine::new_with_filter_set(filter_set, true); let block = engine.check_network_request( - &crate::request::Request::new("https://example.com", "https://example.com", "document") - .unwrap(), + &crate::request::Request::new( + "https://example.com", + "https://example.com", + "document", + "", + ) + .unwrap(), ); assert_eq!( block.filter, @@ -1416,7 +1470,7 @@ mod legacy_rule_parsing_tests { }; // differences in counts explained by hashset size underreporting as detailed in the next two cases const EASY_PRIVACY: ListCounts = ListCounts { - filters: 55395 - 818, // total - exceptions + filters: 55396 - 818, // total - exceptions cosmetic_filters: 32, exceptions: 818, duplicates: 2, @@ -1472,7 +1526,7 @@ mod legacy_rule_parsing_tests { let rules = rules_from_lists(rule_lists); let (network_filters, cosmetic_filters) = parse_filters( - rules, + rules.lines(), true, ParseOptions { format, @@ -1498,7 +1552,7 @@ mod legacy_rule_parsing_tests { enable_optimizations: false, // optimizations will reduce number of rules }; - let blocker = Blocker::new(network_filters, &blocker_options); + let blocker = Blocker::new_with_list_text(rules, &blocker_options); // Some filters in the filter_map are pointed at by multiple tokens, increasing the total number of items assert!( diff --git a/tests/unit/cosmetic_filter_cache.rs b/tests/unit/cosmetic_filter_cache.rs index 7b937f46..cb544336 100644 --- a/tests/unit/cosmetic_filter_cache.rs +++ b/tests/unit/cosmetic_filter_cache.rs @@ -82,18 +82,12 @@ mod cosmetic_cache_tests { use crate::resources::Resource; use base64::{engine::Engine as _, prelude::BASE64_STANDARD}; - fn cache_from_rules(rules: Vec<&str>) -> CosmeticFilterCache { - let parsed_rules = rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(); - - CosmeticFilterCache::from_rules(parsed_rules) - } - #[test] fn exceptions() { - let cfcache = cache_from_rules(vec!["~example.com##.item", "sub.example.com#@#.item2"]); + let cfcache = CosmeticFilterCache::from_rules(vec![ + "~example.com##.item", + "sub.example.com#@#.item2", + ]); let resources = ResourceStorage::default(); let out = cfcache.hostname_cosmetic_resources(&resources, "test.com", false); @@ -111,7 +105,7 @@ mod cosmetic_cache_tests { #[test] fn exceptions2() { - let cfcache = cache_from_rules(vec!["example.com,~sub.example.com##.item"]); + let cfcache = CosmeticFilterCache::from_rules(vec!["example.com,~sub.example.com##.item"]); let resources = ResourceStorage::default(); let out = cfcache.hostname_cosmetic_resources(&resources, "test.com", false); @@ -130,7 +124,7 @@ mod cosmetic_cache_tests { #[test] fn style_exceptions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##.element:style(background: #fff)", "sub.test.example.com#@#.element:style(background: #fff)", "a1.sub.example.com##.element", @@ -195,7 +189,7 @@ mod cosmetic_cache_tests { fn script_exceptions() { use crate::resources::{MimeType, ResourceType}; - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##+js(set-constant.js, atob, trueFunc)", "sub.test.example.com#@#+js(set-constant.js, atob, trueFunc)", "cosmetic.net##+js(nowebrtc.js)", @@ -259,7 +253,7 @@ mod cosmetic_cache_tests { #[test] fn remove_exceptions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##.element:remove()", "sub.test.example.com#@#.element:remove()", "a1.sub.example.com##.element", @@ -308,7 +302,7 @@ mod cosmetic_cache_tests { #[test] fn remove_attr_exceptions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##.element:remove-attr(style)", "sub.test.example.com#@#.element:remove-attr(style)", "a1.sub.example.com##.element", @@ -371,7 +365,7 @@ mod cosmetic_cache_tests { #[test] fn remove_class_exceptions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com,~sub.example.com##.element:remove-class(overlay)", "sub.test.example.com#@#.element:remove-class(overlay)", "a1.sub.example.com##.element", @@ -435,7 +429,7 @@ mod cosmetic_cache_tests { #[test] #[cfg(feature = "css-validation")] fn procedural_actions() { - let cfcache = cache_from_rules(vec![ + let cfcache = CosmeticFilterCache::from_rules(vec![ "example.com##div:has(video):remove()", "example.com##div:has-text(Ad):remove()", "example.com##div:has-text(Sponsored) > p", @@ -501,12 +495,7 @@ mod cosmetic_cache_tests { "##.children .including #simple-id", "##a.a-class", ]; - let cfcache = CosmeticFilterCache::from_rules( - rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(), - ); + let cfcache = CosmeticFilterCache::from_rules(rules); let out = cfcache.hidden_class_id_selectors(["with"], EMPTY, &HashSet::default()); assert_eq!(out, Vec::::new()); @@ -564,12 +553,7 @@ mod cosmetic_cache_tests { "example.*#@#.a-class", "~test.com###test-element", ]; - let cfcache = CosmeticFilterCache::from_rules( - rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(), - ); + let cfcache = CosmeticFilterCache::from_rules(rules); let resources = ResourceStorage::default(); let exceptions = cfcache .hostname_cosmetic_resources(&resources, "example.co.uk", false) @@ -624,12 +608,7 @@ mod cosmetic_cache_tests { "example.com#@#div > p", "~example.com##a[href=\"notbad.com\"]", ]; - let cfcache = CosmeticFilterCache::from_rules( - rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(), - ); + let cfcache = CosmeticFilterCache::from_rules(rules); let resources = ResourceStorage::default(); let hide_selectors = cfcache @@ -659,12 +638,7 @@ mod cosmetic_cache_tests { "toolforge.org##+js(abort-on-property-read, noAdBlockers)", "github.io##div.adToBlock", ]; - let cfcache = CosmeticFilterCache::from_rules( - rules - .iter() - .map(|r| CosmeticFilter::parse(r, false, Default::default()).unwrap()) - .collect::>(), - ); + let cfcache = CosmeticFilterCache::from_rules(rules); let resources = ResourceStorage::in_memory_from_resources([Resource { name: "abort-on-property-read.js".into(), aliases: vec!["aopr".to_string()], diff --git a/tests/unit/engine.rs b/tests/unit/engine.rs index ce31d490..d1444f5b 100644 --- a/tests/unit/engine.rs +++ b/tests/unit/engine.rs @@ -21,12 +21,12 @@ mod tests { ("https://brave.com/about", true), ]; - let mut engine = Engine::from_rules(filters, Default::default()); + let mut engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); engine.enable_tags(&["stuff"]); engine.enable_tags(&["brian"]); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -55,12 +55,12 @@ mod tests { ("https://brave.com/about", true), ]; - let mut engine = Engine::from_rules(filters, Default::default()); + let mut engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); engine.enable_tags(&["brian", "stuff"]); engine.disable_tags(&["stuff"]); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -87,10 +87,10 @@ mod tests { ("https://brianbondy.com/advert", true), ]; - let engine = Engine::from_rules(filters, Default::default()); + let engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -117,11 +117,11 @@ mod tests { ("https://brianbondy.com/advert", false), ]; - let mut engine = Engine::from_rules(filters, Default::default()); + let mut engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); engine.enable_tags(&["brian", "stuff"]); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -150,7 +150,7 @@ mod tests { ("https://brave.com/about", false), ]; - let mut engine = Engine::from_rules(filters, Default::default()); + let mut engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); engine.enable_tags(&["stuff"]); engine.enable_tags(&["brian"]); let serialized = engine.serialize(); @@ -159,7 +159,7 @@ mod tests { deserialized_engine.deserialize(&serialized).unwrap(); url_results.into_iter().for_each(|(url, expected_result)| { - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = deserialized_engine.check_network_request(&request); if expected_result { assert!(matched_rule.matched, "Expected match for {url}"); @@ -181,26 +181,27 @@ mod tests { #[test] fn deserialization_generate_simple() { - let mut engine = Engine::from_rules(["ad-banner"], Default::default()); + let mut engine = Engine::new_with_list_text("ad-banner", Default::default()); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 10945714988765761881; + const EXPECTED_HASH: u64 = 9901869967784992350; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } #[test] fn deserialization_generate_tags() { - let mut engine = Engine::from_rules(["ad-banner$tag=abc"], Default::default()); + let mut engine = Engine::new_with_list_text("ad-banner$tag=abc", Default::default()); engine.use_tags(&["abc"]); let data = engine.serialize().to_vec(); - const EXPECTED_HASH: u64 = 4608037684406751718; + const EXPECTED_HASH: u64 = 2168748959804448596; assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}"); engine.deserialize(&data).unwrap(); } #[test] fn deserialization_generate_resources() { - let mut engine = Engine::from_rules(["ad-banner$redirect=nooptext"], Default::default()); + let mut engine = + Engine::new_with_list_text("ad-banner$redirect=nooptext", Default::default()); engine.use_resources([ Resource::simple("nooptext", MimeType::TextPlain, ""), @@ -214,15 +215,16 @@ mod tests { #[test] fn deserialization_brave_list() { - let rules = rules_from_lists(&["data/brave/brave-main-list.txt"]); - let mut engine = Engine::from_rules_parametrised(rules, Default::default(), false, true); + let rules = rules_from_lists(["data/brave/brave-main-list.txt"]); + let mut engine = + Engine::new_with_list_text_parametrised(rules, Default::default(), false, true); let data = engine.serialize().to_vec(); #[cfg(feature = "debug-info")] { let debug_info = engine.get_debug_info(); - let low_bound = 9_000_000; - let high_bound = 9_500_000; + let low_bound = 8_700_000; + let high_bound = 9_200_000; assert!( debug_info.flatbuffer_size >= low_bound, "Expected size >= {} bytes, got {}", @@ -237,9 +239,9 @@ mod tests { ); } let expected_hash: u64 = if cfg!(feature = "css-validation") { - 18006544876284176656 + 14819081262113618355 } else { - 7678096075090154250 + 16823792549182954096 }; assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}"); @@ -249,8 +251,8 @@ mod tests { #[test] fn redirect_resource_insertion_works() { - let mut engine = Engine::from_rules( - ["ad-banner$redirect=nooptext", "script.js$redirect=noop.js"], + let mut engine = Engine::new_with_list_text( + ["ad-banner$redirect=nooptext", "script.js$redirect=noop.js"].join("\n"), Default::default(), ); @@ -268,7 +270,7 @@ mod tests { engine.use_resources(resources); let url = "http://example.com/ad-banner.gif"; - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); assert!(matched_rule.matched, "Expected match for {url}"); assert_eq!( @@ -278,7 +280,7 @@ mod tests { ); let url = "http://example.com/script.js"; - let request = Request::new(url, "", "").unwrap(); + let request = Request::new(url, "", "", "").unwrap(); let matched_rule = engine.check_network_request(&request); assert!(matched_rule.matched, "Expected match for {url}"); assert_eq!( @@ -295,12 +297,12 @@ mod tests { fn document() { let filters = ["||example.com$document", "@@||sub.example.com$document"]; - let engine = Engine::from_rules_debug(filters, Default::default()); + let engine = Engine::new_with_list_text(filters.join("\n"), Default::default()); assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched @@ -308,7 +310,8 @@ mod tests { assert!( !engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "script").unwrap() + &Request::new("https://example.com", "https://example.com", "script", "") + .unwrap() ) .matched ); @@ -317,7 +320,8 @@ mod tests { &Request::new( "https://sub.example.com", "https://sub.example.com", - "document" + "document", + "" ) .unwrap() ) @@ -328,11 +332,11 @@ mod tests { #[test] fn implicit_all() { { - let engine = Engine::from_rules_debug(["||example.com^"], Default::default()); + let engine = Engine::new_with_list_text("||example.com^", Default::default()); assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched @@ -340,55 +344,55 @@ mod tests { } { let engine = - Engine::from_rules_debug(["||example.com^$first-party"], Default::default()); + Engine::new_with_list_text("||example.com^$first-party", Default::default()); assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched ); } { - let engine = Engine::from_rules_debug(["||example.com^$script"], Default::default()); + let engine = Engine::new_with_list_text("||example.com^$script", Default::default()); assert!( !engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched ); } { - let engine = Engine::from_rules_debug(["||example.com^$~script"], Default::default()); + let engine = Engine::new_with_list_text("||example.com^$~script", Default::default()); assert!( !engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched ); } { - let engine = Engine::from_rules_debug( - ["||example.com^$document", "@@||example.com^$generichide"], + let engine = Engine::new_with_list_text( + ["||example.com^$document", "@@||example.com^$generichide"].join("\n"), Default::default(), ); assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched ); } { - let engine = Engine::from_rules_debug( - ["example.com"], + let engine = Engine::new_with_list_text( + "example.com", ParseOptions { format: FilterFormat::Hosts, ..Default::default() @@ -397,21 +401,22 @@ mod tests { assert!( engine .check_network_request( - &Request::new("https://example.com", "https://example.com", "document") + &Request::new("https://example.com", "https://example.com", "document", "") .unwrap() ) .matched ); } { - let engine = Engine::from_rules_debug(["||example.com/path"], Default::default()); + let engine = Engine::new_with_list_text("||example.com/path", Default::default()); assert!( !engine .check_network_request( &Request::new( "https://example.com/path", "https://example.com/path", - "document" + "document", + "" ) .unwrap() ) @@ -419,14 +424,15 @@ mod tests { ); } { - let engine = Engine::from_rules_debug(["||example.com/path^"], Default::default()); + let engine = Engine::new_with_list_text("||example.com/path^", Default::default()); assert!( !engine .check_network_request( &Request::new( "https://example.com/path", "https://example.com/path", - "document" + "document", + "" ) .unwrap() ) @@ -437,8 +443,8 @@ mod tests { #[test] fn explicit_all() { - let engine = Engine::from_rules_debug( - ["*$all,domain=rarvinzp.click|ytrqcxat.click"], + let engine = Engine::new_with_list_text( + "*$all,domain=rarvinzp.click|ytrqcxat.click", Default::default(), ); for content_type in [ @@ -454,7 +460,8 @@ mod tests { &Request::new( "https://example.com", "https://rarvinzp.click", - content_type + content_type, + "" ) .unwrap() ) @@ -472,7 +479,8 @@ mod tests { "example.com##.block", "@@||example2.com/test.html$generichide", "example2.com##.block", - ]; + ] + .join("\n"); let url_results = [ ("https://example.com", vec![".block"], true), ("https://example.com/test.html", vec![".block"], true), @@ -484,7 +492,7 @@ mod tests { ("https://example2.com/test.html", vec![".block"], true), ]; - let engine = Engine::from_rules(filters, Default::default()); + let engine = Engine::new_with_list_text(filters, Default::default()); url_results .into_iter() @@ -508,7 +516,7 @@ mod tests { "||addthis.com^$important,3p,domain=~missingkids.com|~missingkids.org|~sainsburys.jobs|~sitecore.com|~amd.com", "||addthis.com/*/addthis_widget.js$script,redirect=addthis.com/addthis_widget.js", ], Default::default()); - let mut engine = Engine::from_filter_set(filter_set, false); + let mut engine = Engine::new_with_filter_set(filter_set, false); engine.use_resources([Resource::simple( "addthis.com/addthis_widget.js", @@ -516,7 +524,7 @@ mod tests { "window.addthis = undefined", )]); - let request = Request::new("https://s7.addthis.com/js/250/addthis_widget.js?pub=resto", "https://www.rhmodern.com/catalog/product/product.jsp?productId=prod14970086&categoryId=cat7150028", "script").unwrap(); + let request = Request::new("https://s7.addthis.com/js/250/addthis_widget.js?pub=resto", "https://www.rhmodern.com/catalog/product/product.jsp?productId=prod14970086&categoryId=cat7150028", "script", "").unwrap(); let result = engine.check_network_request(&request); assert!(result.redirect.is_some()); @@ -526,61 +534,57 @@ mod tests { fn check_match_case_regex_filtering() { { // match case without regex is discarded - let engine = Engine::from_rules_debug(["ad.png$match-case"], Default::default()); - let request = - Request::new("https://example.com/ad.png", "https://example.com", "image").unwrap(); + let engine = Engine::new_with_list_text("ad.png$match-case", Default::default()); + let request = Request::new( + "https://example.com/ad.png", + "https://example.com", + "image", + "", + ) + .unwrap(); assert!(!engine.check_network_request(&request).matched); } { // /^https:\/\/[0-9a-z]{3,}\.[-a-z]{10,}\.(?:li[fv]e|top|xyz)\/[a-z]{8}\/\?utm_campaign=\w{40,}/$doc,match-case,domain=life|live|top|xyz - let engine = Engine::from_rules_debug( - [ - r#"/^https:\/\/[0-9a-z]{3,}\.[-a-z]{10,}\.(?:li[fv]e|top|xyz)\/[a-z]{8}\/\?utm_campaign=\w{40,}/$doc,match-case,domain=life|live|top|xyz"#, - ], + let engine = Engine::new_with_list_text( + r#"/^https:\/\/[0-9a-z]{3,}\.[-a-z]{10,}\.(?:li[fv]e|top|xyz)\/[a-z]{8}\/\?utm_campaign=\w{40,}/$doc,match-case,domain=life|live|top|xyz"#, Default::default(), ); - let request = Request::new("https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "document").unwrap(); + let request = Request::new("https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "https://www.exampleaaa.xyz/testtest/?utm_campaign=aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd", "document", "").unwrap(); assert!(engine.check_network_request(&request).matched); } // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/((?!www)[a-z]{3,}|\d{2})?\.?[-0-9a-z]{6,}\.[a-z]{2,6}\/(?:[a-z]{6,8}\/)?\/?\?u=[0-9a-z]{7}&o=[0-9a-z]{7}/$doc,frame,match-case,domain=buzz|com|de|fun|guru|info|life|live|mobi|online|pw|site|space|top|us|xyz let engine = Engine::from_rules_debug([r#"/^https?:\/\/((?!www)[a-z]{3,}|\d{2})?\.?[-0-9a-z]{6,}\.[a-z]{2,6}\/(?:[a-z]{6,8}\/)?\/?\?u=[0-9a-z]{7}&o=[0-9a-z]{7}/$doc,frame,match-case,domain=buzz|com|de|fun|guru|info|life|live|mobi|online|pw|site|space|top|us|xyz"#], Default::default()); - let request = Request::new("https://example.com/aaaaaa/?u=aaaaaaa&o=bbbbbbb", - "https://example.com/aaaaaa/?u=aaaaaaa&o=bbbbbbb", - "document").unwrap(); + let request = Request::new("https://example.com/aaaaaa/?u=aaaaaaa&o=bbbbbbb", "https://example.com/aaaaaa/?u=aaaaaaa&o=bbbbbbb", "document", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https:\/\/(?:www\d\.)?[-a-z]{6,}\.(?:com|info|net|org)\/(?=[-_a-zA-Z]{0,42}\d)(?=[-_0-9a-z]{0,42}[A-Z])[-_0-9a-zA-Z]{43}\/\?cid=[-_0-9a-zA-Z]{16,36}(?:&qs\d=\S+)?&sid=[_0-9a-f]{1,32}$/$doc,match-case,domain=com|info|net|org let engine = Engine::from_rules_debug([r#"/^https:\/\/(?:www\d\.)?[-a-z]{6,}\.(?:com|info|net|org)\/(?=[-_a-zA-Z]{0,42}\d)(?=[-_0-9a-z]{0,42}[A-Z])[-_0-9a-zA-Z]{43}\/\?cid=[-_0-9a-zA-Z]{16,36}(?:&qs\d=\S+)?&sid=[_0-9a-f]{1,32}$/$doc,match-case,domain=com|info|net|org"#], Default::default()); - let request = Request::new("https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?cid=aaaaaaaaaabbbbbb&qs5=\n&sid=a", - "https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?cid=aaaaaaaaaabbbbbb&qs5=\n&sid=a", - "document").unwrap(); + let request = Request::new("https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?cid=aaaaaaaaaabbbbbb&qs5=\n&sid=a", "https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?cid=aaaaaaaaaabbbbbb&qs5=\n&sid=a", "document", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https:\/\/(?:www\d\.)?[-a-z]{6,}\.(?:com|info|net|org)\/(?=[-_a-zA-Z]{0,42}\d)(?=[-_0-9a-z]{0,42}[A-Z])[-_0-9a-zA-Z]{43}\/\?sid=[_0-9a-f]{1,32}(?:&qs\d=\S+)?&cid=[-_0-9a-zA-Z]{16,36}$/$doc,match-case,domain=com|info|net|org let engine = Engine::from_rules_debug([r#"/^https:\/\/(?:www\d\.)?[-a-z]{6,}\.(?:com|info|net|org)\/(?=[-_a-zA-Z]{0,42}\d)(?=[-_0-9a-z]{0,42}[A-Z])[-_0-9a-zA-Z]{43}\/\?cid=[-_0-9a-zA-Z]{16,36}(?:&qs\d=\S+)?&sid=[_0-9a-f]{1,32}$/$doc,match-case,domain=com|info|net|org"#], Default::default()); - let request = Request::new("https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?sid=1&qs1=\n&cid=aaaaaaaaaabbbbbb", - "https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?sid=1&qs1=\n&cid=aaaaaaaaaabbbbbb", - "document").unwrap(); + let request = Request::new("https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?sid=1&qs1=\n&cid=aaaaaaaaaabbbbbb", "https://www3.example.com/aaaaaaaaaabbbbbbbbbbccccccccccddddddddddAA5/?sid=1&qs1=\n&cid=aaaaaaaaaabbbbbb", "document", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ { // /^http:\/\/[a-z]{5}\.[a-z]{5}\.com\/[a-z]{10}\.apk$/$doc,match-case,domain=com - let engine = Engine::from_rules_debug( - [ - r#"/^http:\/\/[a-z]{5}\.[a-z]{5}\.com\/[a-z]{10}\.apk$/$doc,match-case,domain=com"#, - ], + let engine = Engine::new_with_list_text( + r#"/^http:\/\/[a-z]{5}\.[a-z]{5}\.com\/[a-z]{10}\.apk$/$doc,match-case,domain=com"#, Default::default(), ); let request = Request::new( "http://abcde.abcde.com/aaaaabbbbb.apk", "http://abcde.abcde.com/aaaaabbbbb.apk", "document", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched); @@ -589,48 +593,41 @@ mod tests { /*{ // /\/[A-Z]\/[-0-9a-z]{5,}\.com\/(?:[0-9a-f]{2}\/){3}[0-9a-f]{32}\.js$/$script,1p,match-case let engine = Engine::from_rules_debug([r#"/\/[A-Z]\/[-0-9a-z]{5,}\.com\/(?:[0-9a-f]{2}\/){3}[0-9a-f]{32}\.js$/$script,1p,match-case"#], Default::default()); - let request = Request::new("/A/aaaaa.com/aa/bb/cc/aaaaaaaabbbbbbbbccccccccdddddddd.js", - "/A/aaaaa.com/aa/bb/cc/aaaaaaaabbbbbbbbccccccccdddddddd.js", - "script").unwrap(); + let request = Request::new("/A/aaaaa.com/aa/bb/cc/aaaaaaaabbbbbbbbccccccccdddddddd.js", "/A/aaaaa.com/aa/bb/cc/aaaaaaaabbbbbbbbccccccccdddddddd.js", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/(?:[a-z]{2}\.)?[0-9a-z]{7,16}\.com\/[a-z](?=[a-z]{0,25}[0-9A-Z])[0-9a-zA-Z]{3,26}\/(?:[1-5]\d{4}|[3-9]\d{3})\??(?:_=\d+|v=\d)?$/$frame,script,xhr,popup,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/(?:[a-z]{2}\.)?[0-9a-z]{7,16}\.com\/[a-z](?=[a-z]{0,25}[0-9A-Z])[0-9a-zA-Z]{3,26}\/(?:[1-5]\d{4}|[3-9]\d{3})\??(?:_=\d+|v=\d)?$/$frame,script,xhr,popup,3p,match-case"#], Default::default()); - let request = Request::new("https://aa.example.com/aAaaa/12222", - "https://aa.example.net/aAaaa/12222", - "frame").unwrap(); + let request = Request::new("https://aa.example.com/aAaaa/12222", "https://aa.example.net/aAaaa/12222", "frame", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/(?:[a-z]{2}\.)?[0-9a-z]{7,16}\.website\/[a-z](?=[a-z]{0,25}[0-9A-Z])[0-9a-zA-Z]{3,26}\/(?:[1-5]\d{4}|[3-9]\d{3})\??(?:_=\d+|v=\d)?$/$frame,script,xhr,popup,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/(?:[a-z]{2}\.)?[0-9a-z]{7,16}\.website\/[a-z](?=[a-z]{0,25}[0-9A-Z])[0-9a-zA-Z]{3,26}\/(?:[1-5]\d{4}|[3-9]\d{3})\??(?:_=\d+|v=\d)?$/$frame,script,xhr,popup,3p,match-case"#], Default::default()); - let request = Request::new("https://aa.example.website/aAaaa/12222", - "https://aa.example.website/aAaaa/12222", - "frame").unwrap(); + let request = Request::new("https://aa.example.website/aAaaa/12222", "https://aa.example.website/aAaaa/12222", "frame", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/[a-z]{8,15}\.top(\/(?:\d{1,5}|0NaN|articles?|browse|index|movie|news|pages?|static|view|web|wiki)){1,4}(?:\.html|\/)$/$frame,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[a-z]{8,15}\.top(\/(?:\d{1,5}|0NaN|articles?|browse|index|movie|news|pages?|static|view|web|wiki)){1,4}(?:\.html|\/)$/$frame,3p,match-case"#], Default::default()); - let request = Request::new("https://examples.top/articles.html", - "https://examples.top/articles.html", - "frame").unwrap(); + let request = Request::new("https://examples.top/articles.html", "https://examples.top/articles.html", "frame", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ { // /^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.json$/$xhr,3p,match-case - let engine = Engine::from_rules_debug( - [r#"/^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.json$/$xhr,3p,match-case"#], + let engine = Engine::new_with_list_text( + r#"/^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.json$/$xhr,3p,match-case"#, Default::default(), ); let request = Request::new( "https://examples.top/abcd.json", "https://examples.com/abcd.json", "xhr", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched); @@ -639,39 +636,34 @@ mod tests { /*{ // /^https?:\/\/[a-z]{8,15}\.top\/[-a-z]{4,}\.css\?aHR0c[\/0-9a-zA-Z]{33,}=?=?$/$css,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[a-z]{8,15}\.top\/[-a-z]{4,}\.css\?aHR0c[\/0-9a-zA-Z]{33,}=?=?$/$css,3p,match-case"#], Default::default()); - let request = Request::new("https://examples.top/abcd.css?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", - "https://examples.com/abcd.css?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", - "stylesheet").unwrap(); + let request = Request::new("https://examples.top/abcd.css?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", "https://examples.com/abcd.css?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", "stylesheet", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - inferring unescaped `$` inside regex pattern /*{ // /^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.png\?aHR0c[\/0-9a-zA-Z]{33,}=?=?$/$image,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[a-z]{8,15}\.top\/[a-z]{4,}\.png\?aHR0c[\/0-9a-zA-Z]{33,}=?=?$/$image,3p,match-case"#], Default::default()); - let request = Request::new("https://examples.top/abcd.png?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", - "https://examples.com/abcd.png?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", - "image").unwrap(); + let request = Request::new("https://examples.top/abcd.png?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", "https://examples.com/abcd.png?aHR0c/aaaaaaaaaaAAAAAAAAAA000000000012==", "image", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - because of non-supported look around operator in rust regex https://github.com/rust-lang/regex/issues/127#issuecomment-154713666 /*{ // /^https?:\/\/[a-z]{8,15}\.xyz(\/(?:\d{1,5}|0NaN|articles?|browse|index|movie|news|pages?|static|view|web|wiki)){1,4}(?:\.html|\/)$/$frame,3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[a-z]{8,15}\.xyz(\/(?:\d{1,5}|0NaN|articles?|browse|index|movie|news|pages?|static|view|web|wiki)){1,4}(?:\.html|\/)$/$frame,3p,match-case"#], Default::default()); - let request = Request::new("https://examples.xyz/articles.html", - "https://examples.xyz/articles.html", - "frame").unwrap(); + let request = Request::new("https://examples.xyz/articles.html", "https://examples.xyz/articles.html", "frame", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ { // /^https?:\/\/cdn\.[a-z]{4,6}\.xyz\/app\.js$/$script,3p,match-case - let engine = Engine::from_rules_debug( - [r#"/^https?:\/\/cdn\.[a-z]{4,6}\.xyz\/app\.js$/$script,3p,match-case"#], + let engine = Engine::new_with_list_text( + r#"/^https?:\/\/cdn\.[a-z]{4,6}\.xyz\/app\.js$/$script,3p,match-case"#, Default::default(), ); let request = Request::new( "https://cdn.abcde.xyz/app.js", "https://cdn.abcde.com/app.js", "script", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched); @@ -680,23 +672,20 @@ mod tests { /*{ // /^https:\/\/a\.[-0-9a-z]{4,16}\.(?:club|com?|cyou|info|net|ru|site|top?|xxx|xyz)\/(?=[a-z]{0,6}[0-9A-Z])[0-9a-zA-Z]{7}\.js$/$script,match-case let engine = Engine::from_rules_debug([r#"/^https:\/\/a\.[-0-9a-z]{4,16}\.(?:club|com?|cyou|info|net|ru|site|top?|xxx|xyz)\/(?=[a-z]{0,6}[0-9A-Z])[0-9a-zA-Z]{7}\.js$/$script,match-case"#], Default::default()); - let request = Request::new("https://a.abcd.club/aaaaaaA.js", - "https://a.abcd.club/aaaaaaA.js", - "script").unwrap(); + let request = Request::new("https://a.abcd.club/aaaaaaA.js", "https://a.abcd.club/aaaaaaA.js", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ { // /^https:\/\/cdn\.jsdelivr\.net\/npm\/[-a-z_]{4,22}@latest\/dist\/script\.min\.js$/$script,3p,match-case - let engine = Engine::from_rules_debug( - [ - r#"/^https:\/\/cdn\.jsdelivr\.net\/npm\/[-a-z_]{4,22}@latest\/dist\/script\.min\.js$/$script,3p,match-case"#, - ], + let engine = Engine::new_with_list_text( + r#"/^https:\/\/cdn\.jsdelivr\.net\/npm\/[-a-z_]{4,22}@latest\/dist\/script\.min\.js$/$script,3p,match-case"#, Default::default(), ); let request = Request::new( "https://cdn.jsdelivr.net/npm/abcd@latest/dist/script.min.js", "https://cdn.jsdelivr.com/npm/abcd@latest/dist/script.min.js", "script", + "", ) .unwrap(); assert!(engine.check_network_request(&request).matched); @@ -705,25 +694,19 @@ mod tests { /*{ // /^https?:\/\/[-.0-9a-z]+\/script\.js$/$script,1p,strict3p,match-case let engine = Engine::from_rules_debug([r#"/^https?:\/\/[-.0-9a-z]+\/script\.js$/$script,1p,strict3p,match-case"#], Default::default()); - let request = Request::new("https://www.example.com/script.js", - "https://www.abc.com/script.js", - "script").unwrap(); + let request = Request::new("https://www.example.com/script.js", "https://www.abc.com/script.js", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - unicode not supported in network filter /*{ let engine = Engine::from_rules_debug([r#"/tesT߶/$domain=example.com"#], Default::default()); - let request = Request::new("https://example.com/tesT߶", - "https://example.com", - "script").unwrap(); + let request = Request::new("https://example.com/tesT߶", "https://example.com", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ // fails - unicode not supported in network filter /*{ let engine = Engine::from_rules_debug([r#"/tesT߶/$domain=example.com"#], Default::default()); - let request = Request::new("https://example-tesT߶.com/tesT", - "https://example.com", - "script").unwrap(); + let request = Request::new("https://example-tesT߶.com/tesT", "https://example.com", "script", "").unwrap(); assert!(engine.check_network_request(&request).matched); }*/ } @@ -790,7 +773,7 @@ mod tests { }, ); - let mut engine = Engine::from_filter_set(filter_set, true); + let mut engine = Engine::new_with_filter_set(filter_set, true); engine.use_resources(resources); fn wrap_try(scriptlet_content: &str) -> String { @@ -879,7 +862,7 @@ mod tests { r#"example.com##+js(trusted-set-local-storage-item, "test"test, 3)"#, ], Default::default()); - let mut engine = Engine::from_filter_set(filter_set, true); + let mut engine = Engine::new_with_filter_set(filter_set, true); engine.use_resources(resources); assert_eq!(engine.url_cosmetic_resources("https://dailymail.co.uk").injected_script, r#"function trustedSetLocalStorageItem(key = '', value = '') { setLocalStorageItemFn('local', true, key, value); } @@ -895,4 +878,62 @@ trustedSetLocalStorageItem("mol.ads.cmp.tcf.cache", "{\"getTCData\":{\"cmpId\":2 "" ); } + + #[test] + fn method_option_blocks_post_xhr_only() { + let engine = Engine::new_with_list_text( + "||perplexity.ai/rest/metrics/collect^$xhr,1p,method=post", + Default::default(), + ); + let url = "https://perplexity.ai/rest/metrics/collect?foo=bar"; + let source = "https://perplexity.ai/page"; + + let post = Request::new(url, source, "xhr", "post").unwrap(); + assert!( + engine.check_network_request(&post).matched, + "POST xhr 1p should match" + ); + + let get = Request::new(url, source, "xhr", "get").unwrap(); + assert!( + !engine.check_network_request(&get).matched, + "GET xhr should not match method=post rule" + ); + + let post_3p = Request::new(url, "https://other.com/page", "xhr", "post").unwrap(); + assert!( + !engine.check_network_request(&post_3p).matched, + "POST xhr 3p should not match 1p rule" + ); + + let post_no_method = Request::new(url, source, "xhr", "").unwrap(); + assert!( + !engine.check_network_request(&post_no_method).matched, + "missing method should not match method=post rule" + ); + } + + #[test] + fn method_option_exception_head_get() { + let engine = Engine::new_with_list_text( + "||tracker.example.com^$xhr\n@@*$xhr,method=head|get,domain=app.axenthost.com,3p", + Default::default(), + ); + let url = "https://tracker.example.com/pixel"; + let source = "https://app.axenthost.com/page"; + + for method in ["get", "head"] { + let request = Request::new(url, source, "xhr", method).unwrap(); + assert!( + !engine.check_network_request(&request).matched, + "{method} xhr should be excepted" + ); + } + + let post = Request::new(url, source, "xhr", "post").unwrap(); + assert!( + engine.check_network_request(&post).matched, + "POST xhr should still be blocked" + ); + } } diff --git a/tests/unit/filters/abstract_network.rs b/tests/unit/filters/abstract_network.rs new file mode 100644 index 00000000..6d84d822 --- /dev/null +++ b/tests/unit/filters/abstract_network.rs @@ -0,0 +1,42 @@ +use super::*; + +#[test] +fn parse_domain_option_values_plain_and_entity() { + let values = parse_domain_option_values("google.*|gstatic.com|~example.it").unwrap(); + assert_eq!(values.len(), 3); + assert_eq!( + values[0], + ParsedDomainValue { + included: true, + kind: DomainValueKind::Entity, + value: "google", + raw: "google.*", + } + ); + assert_eq!( + values[1], + ParsedDomainValue { + included: true, + kind: DomainValueKind::Plain, + value: "gstatic.com", + raw: "gstatic.com", + } + ); + assert_eq!( + values[2], + ParsedDomainValue { + included: false, + kind: DomainValueKind::Plain, + value: "example.it", + raw: "example.it", + } + ); +} + +#[test] +fn parse_domain_option_values_strips_regex() { + assert!(parse_domain_option_values("/^foo/").is_err()); + let values = parse_domain_option_values("/^foo/|bar.com").unwrap(); + assert_eq!(values.len(), 1); + assert_eq!(values[0].value, "bar.com"); +} diff --git a/tests/unit/filters/network.rs b/tests/unit/filters/network.rs index af9a5e35..e2de6b52 100644 --- a/tests/unit/filters/network.rs +++ b/tests/unit/filters/network.rs @@ -35,18 +35,24 @@ mod parse_tests { from_websocket: bool, from_xml_http_request: bool, from_document: bool, + from_get: bool, + from_head: bool, + from_post: bool, match_case: bool, third_party: bool, } - impl From<&NetworkFilter> for NetworkFilterBreakdown { + impl From<&NetworkFilter<'_>> for NetworkFilterBreakdown { fn from(filter: &NetworkFilter) -> NetworkFilterBreakdown { NetworkFilterBreakdown { filter: filter.filter.string_view(), - hostname: filter.hostname.as_ref().cloned(), + hostname: filter.hostname.as_ref().map(|host| host.to_string()), opt_domains: filter.opt_domains.as_ref().cloned(), opt_not_domains: filter.opt_not_domains.as_ref().cloned(), - modifier_option: filter.modifier_option.as_ref().cloned(), + modifier_option: filter + .modifier_option + .as_ref() + .map(|value| value.to_string()), // filter type is_exception: filter.is_exception(), @@ -73,6 +79,9 @@ mod parse_tests { from_websocket: filter.mask.contains(NetworkFilterMask::FROM_WEBSOCKET), from_xml_http_request: filter.mask.contains(NetworkFilterMask::FROM_XMLHTTPREQUEST), from_document: filter.mask.contains(NetworkFilterMask::FROM_DOCUMENT), + from_get: filter.mask.contains(NetworkFilterMask::FROM_GET), + from_head: filter.mask.contains(NetworkFilterMask::FROM_HEAD), + from_post: filter.mask.contains(NetworkFilterMask::FROM_POST), match_case: filter.match_case(), third_party: filter.third_party(), } @@ -112,11 +121,25 @@ mod parse_tests { from_websocket: true, from_xml_http_request: true, from_document: false, + from_get: false, + from_head: false, + from_post: false, match_case: false, third_party: true, } } + fn foo_method_breakdown(get: bool, head: bool, post: bool) -> NetworkFilterBreakdown { + let mut expected = default_network_filter_breakdown(); + expected.hostname = Some(String::from("foo")); + expected.is_hostname_anchor = true; + expected.is_plain = true; + expected.from_get = get; + expected.from_head = head; + expected.from_post = post; + expected + } + #[test] // pattern fn parses_plain_pattern() { @@ -587,7 +610,7 @@ mod parse_tests { NetworkFilter::parse(r#"||foo.com$csp=self bar """#, true, Default::default()) .unwrap(); assert!(filter.is_csp()); - assert_eq!(filter.modifier_option, Some(String::from(r#"self bar """#))); + assert_eq!(filter.modifier_option, Some(r#"self bar """#)); } { // parses empty CSP @@ -707,6 +730,62 @@ mod parse_tests { } } + #[test] + fn parses_to() { + { + let filter = + NetworkFilter::parse("||foo.com$to=bar.com", true, Default::default()).unwrap(); + assert_eq!(filter.opt_domains, None); + assert_eq!( + filter.opt_to_domains, + Some(vec![utils::fast_hash("bar.com")]) + ); + assert_eq!(filter.opt_not_to_domains, None); + assert_eq!(filter.opt_to_entities, None); + assert_eq!(filter.opt_not_to_entities, None); + } + { + let filter = NetworkFilter::parse( + "||foo.com$to=google.*|gstatic.com", + true, + Default::default(), + ) + .unwrap(); + assert_eq!( + filter.opt_to_domains, + Some(vec![utils::fast_hash("gstatic.com")]) + ); + assert_eq!( + filter.opt_to_entities, + Some(vec![utils::fast_hash("google")]) + ); + assert_eq!(filter.opt_not_to_domains, None); + assert_eq!(filter.opt_not_to_entities, None); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=~example.it", true, Default::default()).unwrap(); + assert_eq!(filter.opt_to_domains, None); + assert_eq!( + filter.opt_not_to_domains, + Some(vec![utils::fast_hash("example.it")]) + ); + } + { + let filter = NetworkFilter::parse("||foo.com$to=/^foo/", true, Default::default()); + assert_eq!(filter.err(), Some(NetworkFilterError::NoSupportedDomains)); + } + { + let filter = + NetworkFilter::parse("||foo.com$to=/^foo/|bar.com", true, Default::default()) + .unwrap(); + assert_eq!( + filter.opt_to_domains, + Some(vec![utils::fast_hash("bar.com")]) + ); + } + } + #[test] fn parses_redirects() { // parses redirect @@ -714,12 +793,12 @@ mod parse_tests { let filter = NetworkFilter::parse("||foo.com$redirect=bar.js", true, Default::default()) .unwrap(); - assert_eq!(filter.modifier_option, Some(String::from("bar.js"))); + assert_eq!(filter.modifier_option, Some("bar.js")); } { let filter = NetworkFilter::parse("$redirect=bar.js", true, Default::default()).unwrap(); - assert_eq!(filter.modifier_option, Some(String::from("bar.js"))); + assert_eq!(filter.modifier_option, Some("bar.js")); } // parses ~redirect { @@ -794,7 +873,7 @@ mod parse_tests { NetworkFilter::parse("||foo.com^$removeparam=test", true, Default::default()) .unwrap(); assert!(filter.is_removeparam()); - assert_eq!(filter.modifier_option, Some("test".into())); + assert_eq!(filter.modifier_option, Some("test")); } } @@ -974,7 +1053,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("example.com", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||example.com^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||example.com^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("example.com".to_string()); defaults.is_plain = true; @@ -986,7 +1065,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("www.example.com", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||www.example.com^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||www.example.com^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("www.example.com".to_string()); defaults.is_plain = true; @@ -998,7 +1077,10 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("malware.example.com", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||malware.example.com^"); + assert_eq!( + filter.raw_line.as_deref().unwrap(), + "||malware.example.com^" + ); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("malware.example.com".to_string()); defaults.is_plain = true; @@ -1011,7 +1093,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("Example.COM", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||Example.COM^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||Example.COM^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("example.com".to_string()); defaults.is_plain = true; @@ -1024,7 +1106,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("WWW.Example.COM", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||WWW.Example.COM^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||WWW.Example.COM^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("www.example.com".to_string()); defaults.is_plain = true; @@ -1037,7 +1119,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("münchen.de", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||münchen.de^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||münchen.de^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("xn--mnchen-3ya.de".to_string()); defaults.is_plain = true; @@ -1050,7 +1132,7 @@ mod parse_tests { { let filter = NetworkFilter::parse_hosts_style("www.münchen.de", true).unwrap(); assert!(filter.raw_line.is_some()); - assert_eq!(*filter.raw_line.clone().unwrap(), "||www.münchen.de^"); + assert_eq!(filter.raw_line.as_deref().unwrap(), "||www.münchen.de^"); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some("www.xn--mnchen-3ya.de".to_string()); defaults.is_plain = true; @@ -1074,9 +1156,9 @@ mod parse_tests { let options = vec!["genericblock", "inline-script", "popunder", "popup", "woot"]; for option in options { - let filter = - NetworkFilter::parse(&format!("||foo.com${option}"), true, Default::default()); - assert!(filter.err().is_some()); + let rule = format!("||foo.com${option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()); + assert!(filter.is_err()); } } @@ -1134,9 +1216,8 @@ mod parse_tests { for option in options { // positive { - let filter = - NetworkFilter::parse(&format!("||foo.com${option}"), true, Default::default()) - .unwrap(); + let rule = format!("||foo.com${option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; @@ -1148,12 +1229,8 @@ mod parse_tests { } { - let filter = NetworkFilter::parse( - &format!("||foo.com$object,{option}"), - true, - Default::default(), - ) - .unwrap(); + let rule = format!("||foo.com$object,{option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; @@ -1166,12 +1243,8 @@ mod parse_tests { } { - let filter = NetworkFilter::parse( - &format!("||foo.com$domain=bar.com,{option}"), - true, - Default::default(), - ) - .unwrap(); + let rule = format!("||foo.com$domain=bar.com,{option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; @@ -1185,9 +1258,8 @@ mod parse_tests { // negative { - let filter = - NetworkFilter::parse(&format!("||foo.com$~{option}"), true, Default::default()) - .unwrap(); + let rule = format!("||foo.com$~{option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; @@ -1199,12 +1271,8 @@ mod parse_tests { } { - let filter = NetworkFilter::parse( - &format!("||foo.com${option},~{option}"), - true, - Default::default(), - ) - .unwrap(); + let rule = format!("||foo.com${option},~{option}"); + let filter = NetworkFilter::parse(&rule, true, Default::default()).unwrap(); let mut defaults = default_network_filter_breakdown(); defaults.hostname = Some(String::from("foo.com")); defaults.is_hostname_anchor = true; @@ -1252,9 +1320,102 @@ mod parse_tests { let rule = "||some.primewire.c*/sw$script,1p"; let filter = NetworkFilter::parse(rule, true, ParseOptions::default()).unwrap(); let mut tokens_buffer = utils::TokensBuffer::default(); + assert_eq!(filter.get_tokens(&mut tokens_buffer), FilterTokens::Other); assert_eq!( - filter.get_tokens(&mut tokens_buffer), - FilterTokens::Other(&[utils::fast_hash("some"), utils::fast_hash("primewire")]) + tokens_buffer.as_slice(), + &[utils::fast_hash("some"), utils::fast_hash("primewire")] ); } + + #[test] + fn handles_method_options() { + const CASES: &[(&str, bool, bool, bool)] = &[ + ("||foo$method=post", false, false, true), + ("||foo$method=post|get", true, false, true), + ("||foo$method=head|get", true, true, false), + ("||foo$method=~get", false, true, true), + ("||foo$method=POST", false, false, true), + ("||foo$method=post|put", false, false, true), + ]; + + for (rule, get, head, post) in CASES { + let filter = NetworkFilter::parse(rule, true, Default::default()).unwrap(); + assert_eq!( + foo_method_breakdown(*get, *head, *post), + NetworkFilterBreakdown::from(&filter), + "rule: {rule}", + ); + } + + const RULES: &[&str] = &[ + "||lemonde.fr/*?s=$xhr,method=post", + "||perplexity.ai/rest/metrics/collect^$xhr,1p,method=post", + "||apmplus.volces.com/monitor_web/collect$xhr,method=post", + "@@||www.realclear*/esm/assets/js/analytics/chartbeat.js?v=$xhr,1p,method=get,domain=com|org", + "||pagead2.googlesyndication.com^$3p,xhr,method=head,redirect-rule=noop.js,domain=photopea.com", + "||void.nicopr.fr/rec$method=POST", + ]; + + for filter_text in RULES { + assert!( + NetworkFilter::parse(filter_text, true, Default::default()).is_ok(), + "failed to parse: {filter_text}" + ); + } + + { + let filter = NetworkFilter::parse( + "||perplexity.ai/rest/metrics/collect^$xhr,1p,method=post", + true, + Default::default(), + ) + .unwrap(); + assert!(filter.mask.contains(NetworkFilterMask::FROM_POST)); + assert!(filter.mask.contains(NetworkFilterMask::FROM_XMLHTTPREQUEST)); + assert!(!filter.mask.contains(NetworkFilterMask::FROM_NETWORK_TYPES)); + assert!(filter.first_party()); + assert!(!filter.third_party()); + } + + { + let filter = NetworkFilter::parse( + "@@*$xhr,method=head|get,domain=app.axenthost.com,3p", + true, + Default::default(), + ) + .unwrap(); + assert!(filter.is_exception()); + assert!(filter.mask.contains(NetworkFilterMask::FROM_HEAD)); + assert!(filter.mask.contains(NetworkFilterMask::FROM_GET)); + assert!(!filter.mask.contains(NetworkFilterMask::FROM_POST)); + } + + const INVALID_RULES: &[&str] = &[ + "||foo$method=put", + "||foo$method=connect|delete", + "||foo$method=", + "||foo$~method=post", + ]; + + for filter_text in INVALID_RULES { + assert!( + NetworkFilter::parse(filter_text, true, Default::default()).is_err(), + "expected parse error: {filter_text}" + ); + } + + assert!(NetworkFilter::parse("@@||foo^$generichide", true, Default::default()).is_ok()); + + for filter_text in [ + "@@||foo^$generichide,method=post", + "@@||foo^$ghide,method=get", + "@@||foo^$generichide,method=head|get", + ] { + assert_eq!( + NetworkFilter::parse(filter_text, true, Default::default()).err(), + Some(NetworkFilterError::MethodWithGenerichide), + "expected MethodWithGenerichide: {filter_text}", + ); + } + } } diff --git a/tests/unit/filters/network_matchers.rs b/tests/unit/filters/network_matchers.rs index 2a000f46..178dc02f 100644 --- a/tests/unit/filters/network_matchers.rs +++ b/tests/unit/filters/network_matchers.rs @@ -96,7 +96,7 @@ mod match_tests { fn filter_match_url(filter: &str, url: &str, matching: bool) { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); - let request = request::Request::new(url, "https://example.com", "other").unwrap(); + let request = request::Request::new(url, "https://example.com", "other", "").unwrap(); assert!( network_filter.matches_test(&request) == matching, @@ -106,7 +106,7 @@ mod match_tests { fn hosts_filter_match_url(filter: &str, url: &str, matching: bool) { let network_filter = NetworkFilter::parse_hosts_style(filter, true).unwrap(); - let request = request::Request::new(url, "https://example.com", "other").unwrap(); + let request = request::Request::new(url, "https://example.com", "other", "").unwrap(); assert!( network_filter.matches_test(&request) == matching, @@ -275,7 +275,7 @@ mod match_tests { let url = "https://0914.global.ssl.fastly.net/ad2/script/x.js?cb=1549980040838"; let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let request = - request::Request::new(url, "https://www.gamespot.com/metro-exodus/", "script") + request::Request::new(url, "https://www.gamespot.com/metro-exodus/", "script", "") .unwrap(); assert!( network_filter.matches_test(&request), @@ -290,6 +290,7 @@ mod match_tests { url, "https://www1.swatchseries.to/serie/roswell_new_mexico", "script", + "", ) .unwrap(); assert!( @@ -319,33 +320,43 @@ mod match_tests { NetworkFilter::parse("|ws://$domain=4shared.com", true, Default::default()).unwrap(); assert!(network_filter.matches_test( - &request::Request::new("ws://example.com", "https://4shared.com", "websocket").unwrap() + &request::Request::new("ws://example.com", "https://4shared.com", "websocket", "") + .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("wss://example.com", "https://4shared.com", "websocket") + &request::Request::new("wss://example.com", "https://4shared.com", "websocket", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.com", "https://4shared.com", "script").unwrap() + &request::Request::new("http://example.com", "https://4shared.com", "script", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("https://example.com", "https://4shared.com", "script").unwrap() + &request::Request::new("https://example.com", "https://4shared.com", "script", "") + .unwrap() )); // The `ws://` and `wss://` protocols should be used, rather than the resource type. assert!(network_filter.matches_test( - &request::Request::new("ws://example.com", "https://4shared.com", "script").unwrap() + &request::Request::new("ws://example.com", "https://4shared.com", "script", "") + .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("wss://example.com", "https://4shared.com", "script").unwrap() + &request::Request::new("wss://example.com", "https://4shared.com", "script", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.com", "https://4shared.com", "websocket") + &request::Request::new("http://example.com", "https://4shared.com", "websocket", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("https://example.com", "https://4shared.com", "websocket") - .unwrap() + &request::Request::new( + "https://example.com", + "https://4shared.com", + "websocket", + "" + ) + .unwrap() )); } @@ -371,6 +382,28 @@ mod match_tests { .collect::>() }); + let map_opt_hashes = |domains: Option>, mapping: &mut HashMap| { + domains.map(|domains| { + domains + .iter() + .map(|domain| { + mapping.insert(*domain, *domain as u32); + *domain as u32 + }) + .collect::>() + }) + }; + let opt_to_domains = map_opt_hashes(filter.opt_to_domains.clone(), &mut mapping); + let opt_not_to_domains = map_opt_hashes(filter.opt_not_to_domains.clone(), &mut mapping); + let opt_to_entities = map_opt_hashes(filter.opt_to_entities.clone(), &mut mapping); + let opt_not_to_entities = map_opt_hashes(filter.opt_not_to_entities.clone(), &mut mapping); + let to_options = crate::filters::fb_network::ToOptionsFlags::from_bucket_parts( + opt_to_domains.is_some(), + opt_not_to_domains.is_some(), + opt_to_entities.is_some(), + opt_not_to_entities.is_some(), + ); + super::super::check_options(filter.mask, request) && super::super::check_included_domains_mapped( opt_domains.as_deref(), @@ -382,6 +415,22 @@ mod match_tests { request, &mapping, ) + && super::super::check_included_to_options_mapped( + to_options, + opt_to_domains.as_deref(), + opt_to_entities.as_deref(), + request, + &mapping, + crate::filters::filter_data_context::ToRuleCapability::all(), + ) + && super::super::check_excluded_to_options_mapped( + to_options, + opt_not_to_domains.as_deref(), + opt_not_to_entities.as_deref(), + request, + &mapping, + crate::filters::filter_data_context::ToRuleCapability::all(), + ) } #[test] @@ -391,19 +440,19 @@ mod match_tests { { let network_filter = NetworkFilter::parse("||foo$image", true, Default::default()).unwrap(); - let request = request::Request::new("https://foo.com/bar", "", "image").unwrap(); + let request = request::Request::new("https://foo.com/bar", "", "image", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$image", true, Default::default()).unwrap(); - let request = request::Request::new("https://foo.com/bar", "", "script").unwrap(); + let request = request::Request::new("https://foo.com/bar", "", "script", "").unwrap(); assert!(!check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$~image", true, Default::default()).unwrap(); - let request = request::Request::new("https://foo.com/bar", "", "script").unwrap(); + let request = request::Request::new("https://foo.com/bar", "", "script", "").unwrap(); assert!(check_options(&network_filter, &request)); } @@ -412,14 +461,14 @@ mod match_tests { let network_filter = NetworkFilter::parse("||foo$~third-party", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://baz.foo.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://baz.foo.com", "", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$~third-party", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://baz.bar.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://baz.bar.com", "", "").unwrap(); assert!(!check_options(&network_filter, &request)); } @@ -428,14 +477,14 @@ mod match_tests { let network_filter = NetworkFilter::parse("||foo$~first-party", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://baz.bar.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://baz.bar.com", "", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$~first-party", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://baz.foo.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://baz.foo.com", "", "").unwrap(); assert!(!check_options(&network_filter, &request)); } @@ -444,14 +493,14 @@ mod match_tests { let network_filter = NetworkFilter::parse("||foo$domain=foo.com", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://foo.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://foo.com", "", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$domain=foo.com", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://bar.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://bar.com", "", "").unwrap(); assert!(!check_options(&network_filter, &request)); } @@ -460,16 +509,217 @@ mod match_tests { let network_filter = NetworkFilter::parse("||foo$domain=~bar.com", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://foo.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://foo.com", "", "").unwrap(); assert!(check_options(&network_filter, &request)); } { let network_filter = NetworkFilter::parse("||foo$domain=~bar.com", true, Default::default()).unwrap(); let request = - request::Request::new("https://foo.com/bar", "http://bar.com", "").unwrap(); + request::Request::new("https://foo.com/bar", "http://bar.com", "", "").unwrap(); + assert!(!check_options(&network_filter, &request)); + } + + // opt-to-domain (destination hostname) + { + let network_filter = + NetworkFilter::parse("||foo$to=gstatic.com", true, Default::default()).unwrap(); + let request = + request::Request::new("https://cdn.gstatic.com/foo", "https://example.com", "", "") + .unwrap(); + assert!(check_options(&network_filter, &request)); + } + { + let network_filter = + NetworkFilter::parse("||foo$to=gstatic.com", true, Default::default()).unwrap(); + let request = + request::Request::new("https://foo.com/bar", "https://example.com", "", "") + .unwrap(); + assert!(!check_options(&network_filter, &request)); + } + + // opt-not-to-domain + { + let network_filter = + NetworkFilter::parse("||foo$to=~example.it", true, Default::default()).unwrap(); + let request = + request::Request::new("https://foo.com/bar", "https://example.com", "", "") + .unwrap(); + assert!(check_options(&network_filter, &request)); + } + { + let network_filter = + NetworkFilter::parse("||foo$to=~example.it", true, Default::default()).unwrap(); + let request = + request::Request::new("https://example.it/bar", "https://example.com", "", "") + .unwrap(); assert!(!check_options(&network_filter, &request)); } + + // opt-to-entity + { + let network_filter = + NetworkFilter::parse("||foo$to=google.*", true, Default::default()).unwrap(); + let request = request::Request::new( + "https://www.google.co.uk/foo", + "https://example.com", + "", + "", + ) + .unwrap(); + assert!(check_options(&network_filter, &request)); + } + { + let network_filter = + NetworkFilter::parse("||foo$to=google.*", true, Default::default()).unwrap(); + let request = + request::Request::new("https://foo.com/bar", "https://example.com", "", "") + .unwrap(); + assert!(!check_options(&network_filter, &request)); + } + } + + #[test] + fn check_to_option_matches_works() { + fn to_filter_match( + filter: &str, + url: &str, + source: &str, + request_type: &str, + matching: bool, + ) { + let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); + let request = request::Request::new(url, source, request_type, "").unwrap(); + assert!( + network_filter.matches_test(&request) == matching, + "Expected match={matching} for {filter} on {url} (source={source})" + ); + } + + to_filter_match( + "*$script,from=beforeitsnews.com,to=google.*|gstatic.com", + "https://cdn.gstatic.com/foo.js", + "https://beforeitsnews.com/article", + "script", + true, + ); + to_filter_match( + "*$script,from=beforeitsnews.com,to=google.*|gstatic.com", + "https://www.google.co.uk/foo.js", + "https://beforeitsnews.com/article", + "script", + true, + ); + to_filter_match( + "*$script,from=beforeitsnews.com,to=google.*|gstatic.com", + "https://example.com/foo.js", + "https://beforeitsnews.com/article", + "script", + false, + ); + to_filter_match( + "*$script,from=beforeitsnews.com,to=google.*|gstatic.com", + "https://cdn.gstatic.com/foo.js", + "https://other.com/article", + "script", + false, + ); + + to_filter_match( + "||it^$3p,to=~example.it", + "https://tracker.foo.it/path", + "https://example.com/page", + "script", + true, + ); + to_filter_match( + "||it^$3p,to=~example.it", + "https://example.it/path", + "https://example.com/page", + "script", + false, + ); + to_filter_match( + "||it^$3p,to=~example.it", + "https://tracker.foo.it/path", + "https://tracker.foo.it/page", + "script", + false, + ); + } + + #[test] + fn check_method_options_works() { + let url = "https://foo.com/bar"; + + { + let filter = + NetworkFilter::parse("||foo$method=post", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); + assert!(check_options(&filter, &request)); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); + assert!(!check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$method=post|get", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); + assert!(check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$method=head|get", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "head").unwrap(); + assert!(check_options(&filter, &request)); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); + assert!(!check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$method=~get", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); + assert!(!check_options(&filter, &request)); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); + assert!(check_options(&filter, &request)); + let request = request::Request::new(url, "", "xhr", "head").unwrap(); + assert!(check_options(&filter, &request)); + let request = request::Request::new(url, "", "xhr", "").unwrap(); + assert!(!check_options(&filter, &request)); + } + + { + let filter = NetworkFilter::parse("||foo", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); + assert!(check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$xhr,method=post", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "post").unwrap(); + assert!(check_options(&filter, &request)); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); + assert!(!check_options(&filter, &request)); + let request = request::Request::new(url, "", "script", "post").unwrap(); + assert!(!check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$xhr,method=get", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "get").unwrap(); + assert!(check_options(&filter, &request)); + } + + { + let filter = + NetworkFilter::parse("||foo$xhr,method=head", true, Default::default()).unwrap(); + let request = request::Request::new(url, "", "xhr", "head").unwrap(); + assert!(check_options(&filter, &request)); + } } #[test] @@ -482,27 +732,34 @@ mod match_tests { ) .unwrap(); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://example.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://example.com", "", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.example.com", "") + &request::Request::new("http://example.net/adv", "http://foo.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( &request::Request::new( "http://example.net/adv", "http://subfoo.foo.example.com", + "", "" ) .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.example.com", "") + &request::Request::new("http://example.net/adv", "http://bar.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://anotherexample.com", "") - .unwrap() + &request::Request::new( + "http://example.net/adv", + "http://anotherexample.com", + "", + "" + ) + .unwrap() )); } { @@ -513,27 +770,34 @@ mod match_tests { ) .unwrap(); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://example.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://example.com", "", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.example.com", "") + &request::Request::new("http://example.net/adv", "http://foo.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( &request::Request::new( "http://example.net/adv", "http://subfoo.foo.example.com", + "", "" ) .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.example.com", "") + &request::Request::new("http://example.net/adv", "http://bar.example.com", "", "") .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://anotherexample.com", "") - .unwrap() + &request::Request::new( + "http://example.net/adv", + "http://anotherexample.com", + "", + "" + ) + .unwrap() )); } { @@ -544,27 +808,34 @@ mod match_tests { ) .unwrap(); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://example.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://example.com", "", "") + .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.example.com", "") + &request::Request::new("http://example.net/adv", "http://foo.example.com", "", "") .unwrap() )); assert!(network_filter.matches_test( &request::Request::new( "http://example.net/adv", "http://subfoo.foo.example.com", + "", "" ) .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.example.com", "") + &request::Request::new("http://example.net/adv", "http://bar.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://anotherexample.com", "") - .unwrap() + &request::Request::new( + "http://example.net/adv", + "http://anotherexample.com", + "", + "" + ) + .unwrap() )); } { @@ -575,47 +846,54 @@ mod match_tests { ) .unwrap(); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://example.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://example.com", "", "") + .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.example.com", "") + &request::Request::new("http://example.net/adv", "http://foo.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( &request::Request::new( "http://example.net/adv", "http://subfoo.foo.example.com", + "", "" ) .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.example.com", "") + &request::Request::new("http://example.net/adv", "http://bar.example.com", "", "") .unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://anotherexample.com", "") - .unwrap() + &request::Request::new( + "http://example.net/adv", + "http://anotherexample.com", + "", + "" + ) + .unwrap() )); } { let network_filter = NetworkFilter::parse("adv$domain=com|~foo.com", true, Default::default()).unwrap(); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://com", "", "").unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://foo.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://foo.com", "", "").unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://subfoo.foo.com", "") + &request::Request::new("http://example.net/adv", "http://subfoo.foo.com", "", "") .unwrap() )); assert!(network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://bar.com", "").unwrap() + &request::Request::new("http://example.net/adv", "http://bar.com", "", "").unwrap() )); assert!(!network_filter.matches_test( - &request::Request::new("http://example.net/adv", "http://co.uk", "").unwrap() + &request::Request::new("http://example.net/adv", "http://co.uk", "", "").unwrap() )); } } @@ -655,7 +933,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://bit.ly/bar/"; let source = "http://123movies.com"; - let request = request::Request::new(url, source, "").unwrap(); + let request = request::Request::new(url, source, "", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -667,7 +945,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://data.foo.com/9VjjrjU9Or2aqkb8PDiqTBnULPgeI48WmYEHkYer"; let source = "http://123movies.com"; - let request = request::Request::new(url, source, "xmlhttprequest").unwrap(); + let request = request::Request::new(url, source, "xmlhttprequest", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -679,7 +957,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://hello.club/123.css"; let source = "http://123movies.com"; - let request = request::Request::new(url, source, "stylesheet").unwrap(); + let request = request::Request::new(url, source, "stylesheet", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -693,10 +971,10 @@ mod match_tests { { use crate::Engine; let filter = r#"/^https?:\/\/([0-9a-z\-]+\.)?(9anime|animeland|animenova|animeplus|animetoon|animewow|gamestorrent|goodanime|gogoanime|igg-games|kimcartoon|memecenter|readcomiconline|toonget|toonova|watchcartoononline)\.[a-z]{2,4}\/(?!([Ee]xternal|[Ii]mages|[Ss]cripts|[Uu]ploads|ac|ajax|assets|combined|content|cov|cover|(img\/bg)|(img\/icon)|inc|jwplayer|player|playlist-cat-rss|static|thumbs|wp-content|wp-includes)\/)(.*)/$image,other,script,~third-party,xmlhttprequest,domain=~animeland.hu"#; - let engine = Engine::from_rules(vec![filter], Default::default()); + let engine = Engine::new_with_list_text(filter, Default::default()); let url = "https://9anime.to/watch/episode-1"; let source = "https://9anime.to"; - let request = request::Request::new(url, source, "script").unwrap(); + let request = request::Request::new(url, source, "script", "").unwrap(); assert_eq!( engine .get_debug_info() @@ -726,7 +1004,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://example.com/ad.js"; let source = "http://auth.wi-fi.ru"; - let request = request::Request::new(url, source, "script").unwrap(); + let request = request::Request::new(url, source, "script", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -737,7 +1015,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://example.com/ad.js"; let source = "http://auth.wi-fi.ru"; - let request = request::Request::new(url, source, "script").unwrap(); + let request = request::Request::new(url, source, "script", "").unwrap(); assert!( network_filter.matches_test(&request), "Expected match for {filter} on {url}" @@ -752,7 +1030,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl="; let source = "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl="; - let request = request::Request::new(url, source, "document").unwrap(); + let request = request::Request::new(url, source, "document", "").unwrap(); assert!(!request.is_third_party); assert!( network_filter.matches_test(&request), @@ -764,7 +1042,7 @@ mod match_tests { let network_filter = NetworkFilter::parse(filter, true, Default::default()).unwrap(); let url = "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl="; let source = "https://www.google.com/aclk?sa=l&ai=DChcSEwioqMfq5ovjAhVvte0KHXBYDKoYABAJGgJkZw&sig=AOD64_0IL5OYOIkZA7qWOBt0yRmKL4hKJw&ctype=5&q=&ved=0ahUKEwjQ88Hq5ovjAhXYiVwKHWAgB5gQww8IXg&adurl="; - let request = request::Request::new(url, source, "main_frame").unwrap(); + let request = request::Request::new(url, source, "main_frame", "").unwrap(); assert!(!request.is_third_party); assert!( network_filter.matches_test(&request), diff --git a/tests/unit/flatbuffers/mod.rs b/tests/unit/flatbuffers/mod.rs new file mode 100644 index 00000000..57f15853 --- /dev/null +++ b/tests/unit/flatbuffers/mod.rs @@ -0,0 +1,16 @@ +use crate::filters::flatbuffer_generated::fb::NetworkFilterArgs; +use crate::filters::network::NetworkFilterMask; + +/// If this ever changes, confirm that the default value hardcoded in `fb_network_filter.fbs` +/// continues to track the most common mask. +#[test] +fn default_network_filter_mask() { + assert_eq!( + NetworkFilterArgs::default().mask, + (NetworkFilterMask::DEFAULT_OPTIONS + | NetworkFilterMask::IS_HOSTNAME_ANCHOR + | NetworkFilterMask::IS_RIGHT_ANCHOR + | NetworkFilterMask::FROM_DOCUMENT) + .bits() + ); +} diff --git a/tests/unit/lists.rs b/tests/unit/lists.rs index b67a26d2..4b183922 100644 --- a/tests/unit/lists.rs +++ b/tests/unit/lists.rs @@ -197,11 +197,8 @@ mod tests { #[test] fn parse_filter_failed_fuzz_4() { // \\##+js(,\xdd\x8d - let parsed = parse_filter( - &String::from_utf8(vec![92, 35, 35, 43, 106, 115, 40, 44, 221, 141]).unwrap(), - true, - Default::default(), - ); + let rule = String::from_utf8(vec![92, 35, 35, 43, 106, 115, 40, 44, 221, 141]).unwrap(); + let parsed = parse_filter(&rule, true, Default::default()); #[cfg(feature = "css-validation")] assert!(parsed.is_err()); #[cfg(not(feature = "css-validation"))] @@ -290,10 +287,11 @@ mod tests { "! Version: 20220411", "", "! => https://austinhuang.me/0131-block-list/list.txt", - ]; + ] + .join("\n"); let mut filter_set = FilterSet::new(false); - let metadata = filter_set.add_filters(list, ParseOptions::default()); + let metadata = filter_set.add_filter_list(list, ParseOptions::default()); assert_eq!(metadata.title, Some("0131 Block List".to_string())); assert_eq!( @@ -319,10 +317,11 @@ mod tests { "! Last modified: 09/03/2021", "! Expires: 7 days (update frequency)", "! Homepage: https://www.haopro.net/", - ]; + ] + .join("\n"); let mut filter_set = FilterSet::new(false); - let metadata = filter_set.add_filters(list, ParseOptions::default()); + let metadata = filter_set.add_filter_list(list, ParseOptions::default()); assert_eq!(metadata.title, Some("ABPVN Advanced".to_string())); assert_eq!( @@ -392,18 +391,18 @@ mod tests { { let input = "example.com##.selector"; let result = parse_filter(input, true, Default::default()); - assert!(matches!(result, Ok(ParsedFilter::Cosmetic(..)))); + assert!(matches!(result, Ok(ParsedLine::Cosmetic(..)))); } { let input = "9gag.com#?#article:-abp-has(.promoted)"; let result = parse_filter(input, true, Default::default()); - assert!(matches!(result, Ok(ParsedFilter::Cosmetic(..)))); + assert!(matches!(result, Ok(ParsedLine::Cosmetic(..)))); } #[cfg(feature = "css-validation")] { let input = "sportowefakty.wp.pl#@?#body > [class]:not([id]):matches-css(position: fixed):matches-css(top: 0px)"; let result = parse_filter(input, true, Default::default()); - assert!(matches!(result, Ok(ParsedFilter::Cosmetic(..)))); + assert!(matches!(result, Ok(ParsedLine::Cosmetic(..)))); } { let input = diff --git a/tests/unit/network_filter_list.rs b/tests/unit/network_filter_list.rs index 10a8c5e7..7670ee64 100644 --- a/tests/unit/network_filter_list.rs +++ b/tests/unit/network_filter_list.rs @@ -191,7 +191,7 @@ mod tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); @@ -226,7 +226,7 @@ mod tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); @@ -254,7 +254,7 @@ mod tests { let request_expectations: Vec<_> = url_results .into_iter() .map(|(url, expected_result)| { - let request = Request::new(url, "https://example.com", "other").unwrap(); + let request = Request::new(url, "https://example.com", "other", "").unwrap(); (request, expected_result) }) .collect(); @@ -271,15 +271,11 @@ mod tests { let url_results = [ ( - Request::new("https://bit.ly/bar/", "http://123movies.com", "").unwrap(), + Request::new("https://bit.ly/bar/", "http://123movies.com", "", "").unwrap(), true, ), ( - Request::new( - "https://data.foo.com/9VjjrjU9Or2aqkb8PDiqTBnULPgeI48WmYEHkYer", - "http://123movies.com", - "xmlhttprequest", - ) + Request::new("https://data.foo.com/9VjjrjU9Or2aqkb8PDiqTBnULPgeI48WmYEHkYer", "http://123movies.com", "xmlhttprequest", "") .unwrap(), true, ), diff --git a/tests/unit/optimizer.rs b/tests/unit/optimizer.rs index d4779d20..7a62b3ce 100644 --- a/tests/unit/optimizer.rs +++ b/tests/unit/optimizer.rs @@ -22,6 +22,7 @@ mod optimization_tests_pattern_group { ("https://example.com/".to_string() + url_path).as_str(), "https://google.com", "", + "", ) .unwrap(), ); @@ -132,6 +133,7 @@ mod optimization_tests_pattern_group { &Request::new( "https://example.com/v1/pixel?", "https://my.leadpages.net", + "", "" ) .unwrap() @@ -148,6 +150,7 @@ mod optimization_tests_pattern_group { &Request::new( "https://example.com/analytics/v1/foobar", "https://foo.leadpages.net", + "", "" ) .unwrap() @@ -194,21 +197,13 @@ mod optimization_tests_pattern_group { assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://google.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://google.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://foo.leadpages.net", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://foo.leadpages.net", "", "") .unwrap() ) == false ); @@ -259,41 +254,25 @@ mod optimization_tests_pattern_group { assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://google.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://google.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://example.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://example.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://exampletwo.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://exampletwo.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://foo.leadpages.net", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://foo.leadpages.net", "", "") .unwrap() ) == false ); @@ -320,6 +299,7 @@ mod optimization_tests_pattern_group { ("https://example.com/".to_string() + url_path).as_str(), "https://google.com", "", + "", ) .unwrap(), ); @@ -430,6 +410,7 @@ mod optimization_tests_pattern_group { &Request::new( "https://example.com/v1/pixel?", "https://my.leadpages.net", + "", "" ) .unwrap() @@ -446,6 +427,7 @@ mod optimization_tests_pattern_group { &Request::new( "https://example.com/analytics/v1/foobar", "https://foo.leadpages.net", + "", "" ) .unwrap() @@ -492,21 +474,13 @@ mod optimization_tests_union_domain { assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://google.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://google.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://foo.leadpages.net", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://foo.leadpages.net", "", "") .unwrap() ) == false ); @@ -557,41 +531,25 @@ mod optimization_tests_union_domain { assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://google.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://google.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://example.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://example.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://exampletwo.com", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://exampletwo.com", "", "") .unwrap() ) == true ); assert!( filter.matches_test( - &Request::new( - "https://example.com/analytics-v1/foobar", - "https://foo.leadpages.net", - "" - ) + &Request::new("https://example.com/analytics-v1/foobar", "https://foo.leadpages.net", "", "") .unwrap() ) == false ); diff --git a/tests/unit/regex_manager.rs b/tests/unit/regex_manager.rs index 739781dd..bb842fc9 100644 --- a/tests/unit/regex_manager.rs +++ b/tests/unit/regex_manager.rs @@ -7,11 +7,11 @@ mod tests { use mock_instant::thread_local::MockClock; fn make_engine(line: &str) -> Engine { - Engine::from_rules(vec![line], Default::default()) + Engine::new_with_list_text(line, Default::default()) } fn make_request(url: &str) -> request::Request { - request::Request::new(url, "https://example.com", "other").unwrap() + request::Request::new(url, "https://example.com", "other", "").unwrap() } fn get_active_regex_count(regex_manager: &RegexManager) -> usize { diff --git a/tests/unit/request.rs b/tests/unit/request.rs index d1486e08..6fff6771 100644 --- a/tests/unit/request.rs +++ b/tests/unit/request.rs @@ -18,9 +18,11 @@ mod tests { url, schema, hostname, + domain, source_hostname, third_party, url.to_string(), + None, ) } @@ -147,12 +149,64 @@ mod tests { ) } + #[test] + fn destination_hostname_hashes_works() { + let request = build_request( + "script", + "https://cdn.gstatic.com/foo.js", + "https", + "cdn.gstatic.com", + "gstatic.com", + "example.com", + "example.com", + ); + assert!(!request.destination_hashes_initialized()); + assert_eq!( + request.destination_suffix_hashes().unwrap(), + tokenize(&["cdn.gstatic.com", "gstatic.com", "com"], &[]).as_slice(), + ); + } + + #[test] + fn destination_entity_hashes_works() { + let request = build_request( + "script", + "https://www.google.co.uk/foo.js", + "https", + "www.google.co.uk", + "google.co.uk", + "example.com", + "example.com", + ); + assert!(!request.destination_hashes_initialized()); + let entity_hashes = request.destination_entity_hashes().unwrap(); + assert!(entity_hashes.contains(&utils::fast_hash("google"))); + } + + #[test] + fn destination_suffix_hashes_shared_with_source() { + let request = build_request( + "script", + "https://example.com/foo.js", + "https", + "example.com", + "example.com", + "example.com", + "example.com", + ); + assert_eq!( + request.destination_suffix_hashes().unwrap(), + request.source_hostname_hashes.as_ref().unwrap().as_slice(), + ); + } + #[test] fn parses_urls() { let parsed = Request::new( "https://subdomain.example.com/ad", "https://example.com/", "document", + "", ) .unwrap(); assert!(parsed.is_https); @@ -177,14 +231,43 @@ mod tests { "subdomain.example.com/ad", "https://example.com/", "document", + "", ); assert_eq!(bad_url.err(), Some(RequestError::HostnameParseError)); } + #[test] + fn parse_method_works() { + assert_eq!( + Request::new("https://example.com", "https://example.com", "xhr", "get") + .unwrap() + .method, + Some(RequestMethod::Get), + ); + assert_eq!( + Request::new("https://example.com", "https://example.com", "xhr", "PUT") + .unwrap() + .method, + Some(RequestMethod::Put), + ); + assert_eq!( + Request::new("https://example.com", "https://example.com", "xhr", "TRACE") + .unwrap() + .method, + Some(RequestMethod::Other), + ); + assert_eq!( + Request::new("https://example.com", "https://example.com", "xhr", "") + .unwrap() + .method, + None, + ); + } + #[test] fn fuzzing_errors() { { - let parsed = Request::new("https://߶", "https://example.com", "other"); + let parsed = Request::new("https://߶", "https://example.com", "other", ""); assert!(parsed.is_ok()); } { @@ -192,6 +275,7 @@ mod tests { &format!("https://{}", std::str::from_utf8(&[9, 9, 64]).unwrap()), "https://example.com", "other", + "", ); assert!(parsed.is_err()); } diff --git a/tests/unit/resources/resource_storage.rs b/tests/unit/resources/resource_storage.rs index 2052705e..c73842e4 100644 --- a/tests/unit/resources/resource_storage.rs +++ b/tests/unit/resources/resource_storage.rs @@ -834,14 +834,18 @@ mod shared_storage_tests { let shared_storage = Rc::new(in_memory_storage); - let mut engine1 = - crate::Engine::from_rules(["example1.com##+js(test-scriptlet)"], Default::default()); + let mut engine1 = crate::Engine::new_with_list_text( + "example1.com##+js(test-scriptlet)", + Default::default(), + ); engine1.use_resource_storage(BraveCoreResourceStorage { shared_storage: Rc::clone(&shared_storage), }); - let mut engine2 = - crate::Engine::from_rules(["example2.com##+js(test-scriptlet)"], Default::default()); + let mut engine2 = crate::Engine::new_with_list_text( + "example2.com##+js(test-scriptlet)", + Default::default(), + ); engine2.use_resource_storage(BraveCoreResourceStorage { shared_storage: Rc::clone(&shared_storage), });