Skip to content

Commit 10ae8bd

Browse files
authored
fix(dlp): scan non-utf8 request/response body (#9)
* fix(dlp): scan non-utf8 request/response body * style: update comments * style: update comments
1 parent 95f670d commit 10ae8bd

2 files changed

Lines changed: 135 additions & 112 deletions

File tree

‎src/dlp.rs‎

Lines changed: 94 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use regex::Regex;
1+
use regex::bytes::Regex;
22
use tracing::{debug, trace};
33

44
use crate::config::{DlpAction, DlpPattern};
@@ -16,13 +16,13 @@ pub struct DlpScanner {
1616
scan_responses: bool,
1717
}
1818

19-
/// Result of scanning text for sensitive data.
19+
/// Result of scanning bytes for sensitive data.
2020
#[derive(Debug, Clone)]
2121
pub struct ScanResult {
2222
/// Pattern names that matched with action=block.
2323
pub blocked: Vec<String>,
24-
/// The text with action=redact patterns replaced.
25-
pub redacted_text: String,
24+
/// The bytes with action=redact patterns replaced.
25+
pub redacted: Vec<u8>,
2626
/// Whether any redaction was applied.
2727
pub was_redacted: bool,
2828
}
@@ -57,14 +57,15 @@ impl DlpScanner {
5757
})
5858
}
5959

60-
/// Scans the input text for sensitive data.
60+
/// Scans the input body for sensitive data.
6161
/// Returns a list of pattern names that matched with action=block (without including the actual sensitive data).
62-
pub fn scan(&self, text: &str) -> Vec<String> {
63-
trace!(text_len = text.len(), "Scanning text for block patterns");
62+
#[cfg(test)]
63+
pub fn scan(&self, body: &[u8]) -> Vec<String> {
64+
trace!(body_len = body.len(), "Scanning body for block patterns");
6465
let matches: Vec<String> = self
6566
.patterns
6667
.iter()
67-
.filter(|p| p.action == DlpAction::Block && p.regex.is_match(text))
68+
.filter(|p| p.action == DlpAction::Block && p.regex.is_match(body))
6869
.map(|p| {
6970
debug!(pattern = %p.name, "Block pattern matched");
7071
p.name.clone()
@@ -74,24 +75,24 @@ impl DlpScanner {
7475
matches
7576
}
7677

77-
/// Scans text and applies both block detection and redaction.
78-
/// Returns blocked pattern names and redacted text.
79-
pub fn scan_and_redact(&self, text: &str) -> ScanResult {
78+
/// Scans body and applies both block detection and redaction.
79+
/// Returns blocked pattern names and redacted body.
80+
pub fn scan_and_redact(&self, body: &[u8]) -> ScanResult {
8081
trace!(
81-
text_len = text.len(),
82-
"Scanning text for block+redact patterns"
82+
body_len = body.len(),
83+
"Scanning body for block+redact patterns"
8384
);
8485
let blocked: Vec<String> = self
8586
.patterns
8687
.iter()
87-
.filter(|p| p.action == DlpAction::Block && p.regex.is_match(text))
88+
.filter(|p| p.action == DlpAction::Block && p.regex.is_match(body))
8889
.map(|p| {
8990
debug!(pattern = %p.name, "Block pattern matched in request");
9091
p.name.clone()
9192
})
9293
.collect();
9394

94-
let mut redacted = text.to_string();
95+
let mut redacted = body.to_vec();
9596
let mut was_redacted = false;
9697

9798
for p in &self.patterns {
@@ -100,8 +101,8 @@ impl DlpScanner {
100101
let replacement = format!("[REDACTED:{}]", p.name);
101102
redacted = p
102103
.regex
103-
.replace_all(&redacted, replacement.as_str())
104-
.to_string();
104+
.replace_all(&redacted, replacement.as_bytes())
105+
.to_vec();
105106
was_redacted = true;
106107
}
107108
}
@@ -113,16 +114,16 @@ impl DlpScanner {
113114

114115
ScanResult {
115116
blocked,
116-
redacted_text: redacted,
117+
redacted,
117118
was_redacted,
118119
}
119120
}
120121

121-
/// Redacts all sensitive data (both block and redact patterns) from text.
122+
/// Redacts all sensitive data (both block and redact patterns) from body.
122123
/// Used for response scanning where we want to redact rather than block.
123-
pub fn redact_all(&self, text: &str) -> (String, Vec<String>) {
124-
trace!(text_len = text.len(), "Redacting all patterns from text");
125-
let mut redacted = text.to_string();
124+
pub fn redact_all(&self, body: &[u8]) -> (Vec<u8>, Vec<String>) {
125+
trace!(body_len = body.len(), "Redacting all patterns from body");
126+
let mut redacted = body.to_vec();
126127
let mut redacted_names = Vec::new();
127128

128129
for p in &self.patterns {
@@ -131,8 +132,8 @@ impl DlpScanner {
131132
let replacement = format!("[REDACTED:{}]", p.name);
132133
redacted = p
133134
.regex
134-
.replace_all(&redacted, replacement.as_str())
135-
.to_string();
135+
.replace_all(&redacted, replacement.as_bytes())
136+
.to_vec();
136137
redacted_names.push(p.name.clone());
137138
}
138139
}
@@ -192,38 +193,44 @@ mod tests {
192193
]
193194
}
194195

196+
fn subslice(haystack: &[u8], needle: &[u8]) -> bool {
197+
haystack
198+
.windows(needle.len())
199+
.any(|window| window == needle)
200+
}
201+
195202
#[test]
196203
fn test_detect_credit_card() {
197204
let scanner = DlpScanner::new(&default_patterns()).unwrap();
198-
let matches = scanner.scan("My card is 4111 1111 1111 1111 please charge it");
205+
let matches = scanner.scan(b"My card is 4111 1111 1111 1111 please charge it");
199206
assert!(matches.contains(&"credit_card".to_string()));
200207
}
201208

202209
#[test]
203210
fn test_detect_ssn() {
204211
let scanner = DlpScanner::new(&default_patterns()).unwrap();
205-
let matches = scanner.scan("My SSN is 123-45-6789");
212+
let matches = scanner.scan(b"My SSN is 123-45-6789");
206213
assert!(matches.contains(&"ssn".to_string()));
207214
}
208215

209216
#[test]
210217
fn test_detect_email() {
211218
let scanner = DlpScanner::new(&default_patterns()).unwrap();
212-
let matches = scanner.scan("Contact me at user@example.com");
219+
let matches = scanner.scan(b"Contact me at user@example.com");
213220
assert!(matches.contains(&"email".to_string()));
214221
}
215222

216223
#[test]
217224
fn test_no_sensitive_data() {
218225
let scanner = DlpScanner::new(&default_patterns()).unwrap();
219-
let matches = scanner.scan("Tell me about the weather today");
226+
let matches = scanner.scan(b"Tell me about the weather today");
220227
assert!(matches.is_empty());
221228
}
222229

223230
#[test]
224231
fn test_multiple_detections() {
225232
let scanner = DlpScanner::new(&default_patterns()).unwrap();
226-
let matches = scanner.scan("Card: 4111111111111111, SSN: 123-45-6789, email: a@b.com");
233+
let matches = scanner.scan(b"Card: 4111111111111111, SSN: 123-45-6789, email: a@b.com");
227234
assert!(matches.contains(&"credit_card".to_string()));
228235
assert!(matches.contains(&"ssn".to_string()));
229236
assert!(matches.contains(&"email".to_string()));
@@ -232,7 +239,7 @@ mod tests {
232239
#[test]
233240
fn test_empty_patterns() {
234241
let scanner = DlpScanner::new(&[]).unwrap();
235-
let matches = scanner.scan("4111111111111111");
242+
let matches = scanner.scan(b"4111111111111111");
236243
assert!(matches.is_empty());
237244
}
238245

@@ -241,78 +248,78 @@ mod tests {
241248
#[test]
242249
fn test_redact_email() {
243250
let scanner = DlpScanner::new(&mixed_patterns()).unwrap();
244-
let result = scanner.scan_and_redact("Contact me at user@example.com");
251+
let result = scanner.scan_and_redact(b"Contact me at user@example.com");
245252
assert!(result.blocked.is_empty());
246253
assert!(result.was_redacted);
247-
assert!(result.redacted_text.contains("[REDACTED:email]"));
248-
assert!(!result.redacted_text.contains("user@example.com"));
254+
assert!(subslice(&result.redacted, b"[REDACTED:email]"));
255+
assert!(!subslice(&result.redacted, b"user@example.com"));
249256
}
250257

251258
#[test]
252259
fn test_redact_phone() {
253260
let scanner = DlpScanner::new(&mixed_patterns()).unwrap();
254-
let result = scanner.scan_and_redact("Call me at 555-123-4567");
261+
let result = scanner.scan_and_redact(b"Call me at 555-123-4567");
255262
assert!(result.blocked.is_empty());
256263
assert!(result.was_redacted);
257-
assert!(result.redacted_text.contains("[REDACTED:phone_number]"));
258-
assert!(!result.redacted_text.contains("555-123-4567"));
264+
assert!(subslice(&result.redacted, b"[REDACTED:phone_number]"));
265+
assert!(!subslice(&result.redacted, b"555-123-4567"));
259266
}
260267

261268
#[test]
262269
fn test_block_ssn_and_redact_email() {
263270
let scanner = DlpScanner::new(&mixed_patterns()).unwrap();
264-
let result = scanner.scan_and_redact("SSN: 123-45-6789, email: user@example.com");
271+
let result = scanner.scan_and_redact(b"SSN: 123-45-6789, email: user@example.com");
265272
assert!(result.blocked.contains(&"ssn".to_string()));
266273
assert!(result.was_redacted);
267-
assert!(result.redacted_text.contains("[REDACTED:email]"));
274+
assert!(subslice(&result.redacted, b"[REDACTED:email]"));
268275
}
269276

270277
#[test]
271278
fn test_scan_only_returns_block_patterns() {
272279
let scanner = DlpScanner::new(&mixed_patterns()).unwrap();
273280
// Email is action=redact, so scan() should NOT return it
274-
let matches = scanner.scan("Contact me at user@example.com");
281+
let matches = scanner.scan(b"Contact me at user@example.com");
275282
assert!(!matches.contains(&"email".to_string()));
276283
}
277284

278285
#[test]
279286
fn test_scan_returns_block_patterns() {
280287
let scanner = DlpScanner::new(&mixed_patterns()).unwrap();
281-
let matches = scanner.scan("My SSN is 123-45-6789");
288+
let matches = scanner.scan(b"My SSN is 123-45-6789");
282289
assert!(matches.contains(&"ssn".to_string()));
283290
}
284291

285292
#[test]
286293
fn test_redact_all_replaces_everything() {
287294
let scanner = DlpScanner::new(&mixed_patterns()).unwrap();
288295
let (redacted, names) =
289-
scanner.redact_all("SSN: 123-45-6789, email: user@example.com, phone: 555-123-4567");
296+
scanner.redact_all(b"SSN: 123-45-6789, email: user@example.com, phone: 555-123-4567");
290297
assert!(names.contains(&"ssn".to_string()));
291298
assert!(names.contains(&"email".to_string()));
292299
assert!(names.contains(&"phone_number".to_string()));
293-
assert!(redacted.contains("[REDACTED:ssn]"));
294-
assert!(redacted.contains("[REDACTED:email]"));
295-
assert!(redacted.contains("[REDACTED:phone_number]"));
296-
assert!(!redacted.contains("123-45-6789"));
297-
assert!(!redacted.contains("user@example.com"));
298-
assert!(!redacted.contains("555-123-4567"));
300+
assert!(subslice(&redacted, b"[REDACTED:ssn]"));
301+
assert!(subslice(&redacted, b"[REDACTED:email]"));
302+
assert!(subslice(&redacted, b"[REDACTED:phone_number]"));
303+
assert!(!subslice(&redacted, b"123-45-6789"));
304+
assert!(!subslice(&redacted, b"user@example.com"));
305+
assert!(!subslice(&redacted, b"555-123-4567"));
299306
}
300307

301308
#[test]
302-
fn test_redact_all_clean_text() {
309+
fn test_redact_all_clean_bytes() {
303310
let scanner = DlpScanner::new(&mixed_patterns()).unwrap();
304-
let (redacted, names) = scanner.redact_all("Hello, how are you?");
311+
let (redacted, names) = scanner.redact_all(b"Hello, how are you?");
305312
assert!(names.is_empty());
306-
assert_eq!(redacted, "Hello, how are you?");
313+
assert_eq!(redacted, b"Hello, how are you?");
307314
}
308315

309316
#[test]
310317
fn test_no_redaction_when_clean() {
311318
let scanner = DlpScanner::new(&mixed_patterns()).unwrap();
312-
let result = scanner.scan_and_redact("Hello world");
319+
let result = scanner.scan_and_redact(b"Hello world");
313320
assert!(result.blocked.is_empty());
314321
assert!(!result.was_redacted);
315-
assert_eq!(result.redacted_text, "Hello world");
322+
assert_eq!(result.redacted, b"Hello world");
316323
}
317324

318325
#[test]
@@ -331,12 +338,9 @@ mod tests {
331338
action: DlpAction::Redact,
332339
}];
333340
let scanner = DlpScanner::new(&patterns).unwrap();
334-
let result = scanner.scan_and_redact("a@b.com and c@d.com");
341+
let result = scanner.scan_and_redact(b"a@b.com and c@d.com");
335342
assert!(result.was_redacted);
336-
assert_eq!(
337-
result.redacted_text,
338-
"[REDACTED:email] and [REDACTED:email]"
339-
);
343+
assert_eq!(result.redacted, b"[REDACTED:email] and [REDACTED:email]");
340344
}
341345

342346
#[test]
@@ -349,18 +353,45 @@ mod tests {
349353
let scanner = DlpScanner::new(&patterns).unwrap();
350354

351355
// Standard US format
352-
let (redacted, names) = scanner.redact_all("Call 555-123-4567");
356+
let (redacted, names) = scanner.redact_all(b"Call 555-123-4567");
353357
assert!(names.contains(&"phone_number".to_string()));
354-
assert!(!redacted.contains("555-123-4567"));
358+
assert!(!subslice(&redacted, b"555-123-4567"));
355359

356360
// With parentheses
357-
let (redacted, names) = scanner.redact_all("Call (555) 123-4567");
361+
let (redacted, names) = scanner.redact_all(b"Call (555) 123-4567");
358362
assert!(names.contains(&"phone_number".to_string()));
359-
assert!(!redacted.contains("(555) 123-4567"));
363+
assert!(!subslice(&redacted, b"(555) 123-4567"));
360364

361365
// With +1 prefix
362-
let (redacted, names) = scanner.redact_all("Call +1-555-123-4567");
366+
let (redacted, names) = scanner.redact_all(b"Call +1-555-123-4567");
367+
assert!(names.contains(&"phone_number".to_string()));
368+
assert!(!subslice(&redacted, b"+1-555-123-4567"));
369+
}
370+
371+
#[test]
372+
fn test_non_utf8_input() {
373+
let patterns = vec![DlpPattern {
374+
name: "phone_number".to_string(),
375+
regex: r"\b(?:\+?1[-.\s]?)?(?:\(?\d{3}\)?[-.\s]?)?\d{3}[-.\s]?\d{4}\b".to_string(),
376+
action: DlpAction::Redact,
377+
}];
378+
let scanner = DlpScanner::new(&patterns).unwrap();
379+
let input = b"Call \xFF\xFE\xFD 555-123-4567";
380+
std::str::from_utf8(input.as_slice()).unwrap_err(); // Confirm it's not valid UTF-8
381+
382+
let result = scanner.scan_and_redact(input);
383+
assert!(result.was_redacted);
384+
assert!(result.blocked.is_empty());
385+
assert_eq!(
386+
&result.redacted,
387+
b"Call \xFF\xFE\xFD [REDACTED:phone_number]"
388+
);
389+
390+
let (redacted, names) = scanner.redact_all(input);
363391
assert!(names.contains(&"phone_number".to_string()));
364-
assert!(!redacted.contains("+1-555-123-4567"));
392+
assert!(subslice(
393+
&redacted,
394+
b"Call \xFF\xFE\xFD [REDACTED:phone_number]"
395+
));
365396
}
366397
}

0 commit comments

Comments
 (0)