diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java index 49ac24547296..79ff3094431d 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.KeywordAttribute; /** * A TokenFilter which applies a Pattern to each token in the stream, replacing match occurrences @@ -35,7 +36,9 @@ public final class PatternReplaceFilter extends TokenFilter { private final String replacement; private final boolean all; + private final boolean ignoreKeywords; private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class); private final Matcher m; /** @@ -47,12 +50,15 @@ public final class PatternReplaceFilter extends TokenFilter { * Note that this is not the literal string that will be used, '$' and '\' have special * meaning. * @param all if true, all matches will be replaced otherwise just the first match. + * @param ignoreKeywords if true, tokens with KeywordAttribute set to true will not be processed * @see Matcher#quoteReplacement */ - public PatternReplaceFilter(TokenStream in, Pattern p, String replacement, boolean all) { + public PatternReplaceFilter( + TokenStream in, Pattern p, String replacement, boolean all, boolean ignoreKeywords) { super(in); this.replacement = (null == replacement) ? "" : replacement; this.all = all; + this.ignoreKeywords = ignoreKeywords; this.m = p.matcher(termAtt); } @@ -60,6 +66,10 @@ public PatternReplaceFilter(TokenStream in, Pattern p, String replacement, boole public boolean incrementToken() throws IOException { if (!input.incrementToken()) return false; + if (ignoreKeywords && keywordAtt.isKeyword()) { + return true; + } + m.reset(); if (m.find()) { // replaceAll/replaceFirst will reset() this previous find. diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java index 3672209af75e..1c6e4ab476e4 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java @@ -46,6 +46,7 @@ public class PatternReplaceFilterFactory extends TokenFilterFactory { final Pattern pattern; final String replacement; final boolean replaceAll; + final boolean ignoreKeywords; /** Creates a new PatternReplaceFilterFactory */ public PatternReplaceFilterFactory(Map args) { @@ -53,6 +54,7 @@ public PatternReplaceFilterFactory(Map args) { pattern = getPattern(args, "pattern"); replacement = get(args, "replacement"); replaceAll = "all".equals(get(args, "replace", Arrays.asList("all", "first"), "all")); + ignoreKeywords = getBoolean(args, "ignoreKeywords", false); if (!args.isEmpty()) { throw new IllegalArgumentException("Unknown parameters: " + args); } @@ -65,6 +67,6 @@ public PatternReplaceFilterFactory() { @Override public PatternReplaceFilter create(TokenStream input) { - return new PatternReplaceFilter(input, pattern, replacement, replaceAll); + return new PatternReplaceFilter(input, pattern, replacement, replaceAll, ignoreKeywords); } } diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java index c45b13889004..c66d4aa56dd0 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java @@ -22,6 +22,8 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; +import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase; import org.apache.lucene.tests.analysis.MockTokenizer; @@ -30,7 +32,8 @@ public class TestPatternReplaceFilter extends BaseTokenStreamTestCase { public void testReplaceAll() throws Exception { String input = "aabfooaabfooabfoob ab caaaaaaaaab"; TokenStream ts = - new PatternReplaceFilter(whitespaceMockTokenizer(input), Pattern.compile("a*b"), "-", true); + new PatternReplaceFilter( + whitespaceMockTokenizer(input), Pattern.compile("a*b"), "-", true, false); assertTokenStreamContents(ts, new String[] {"-foo-foo-foo-", "-", "c-"}); } @@ -38,7 +41,7 @@ public void testReplaceFirst() throws Exception { String input = "aabfooaabfooabfoob ab caaaaaaaaab"; TokenStream ts = new PatternReplaceFilter( - whitespaceMockTokenizer(input), Pattern.compile("a*b"), "-", false); + whitespaceMockTokenizer(input), Pattern.compile("a*b"), "-", false, false); assertTokenStreamContents(ts, new String[] {"-fooaabfooabfoob", "-", "c-"}); } @@ -46,7 +49,7 @@ public void testStripFirst() throws Exception { String input = "aabfooaabfooabfoob ab caaaaaaaaab"; TokenStream ts = new PatternReplaceFilter( - whitespaceMockTokenizer(input), Pattern.compile("a*b"), null, false); + whitespaceMockTokenizer(input), Pattern.compile("a*b"), null, false, false); assertTokenStreamContents(ts, new String[] {"fooaabfooabfoob", "", "c"}); } @@ -54,7 +57,7 @@ public void testStripAll() throws Exception { String input = "aabfooaabfooabfoob ab caaaaaaaaab"; TokenStream ts = new PatternReplaceFilter( - whitespaceMockTokenizer(input), Pattern.compile("a*b"), null, true); + whitespaceMockTokenizer(input), Pattern.compile("a*b"), null, true, false); assertTokenStreamContents(ts, new String[] {"foofoofoo", "", "c"}); } @@ -62,7 +65,7 @@ public void testReplaceAllWithBackRef() throws Exception { String input = "aabfooaabfooabfoob ab caaaaaaaaab"; TokenStream ts = new PatternReplaceFilter( - whitespaceMockTokenizer(input), Pattern.compile("(a*)b"), "$1\\$", true); + whitespaceMockTokenizer(input), Pattern.compile("(a*)b"), "$1\\$", true, false); assertTokenStreamContents(ts, new String[] {"aa$fooaa$fooa$foo$", "a$", "caaaaaaaaa$"}); } @@ -74,7 +77,7 @@ public void testRandomStrings() throws Exception { protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); TokenStream filter = - new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", false); + new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", false, false); return new TokenStreamComponents(tokenizer, filter); } }; @@ -87,7 +90,7 @@ protected TokenStreamComponents createComponents(String fieldName) { protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); TokenStream filter = - new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true); + new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true, false); return new TokenStreamComponents(tokenizer, filter); } }; @@ -102,10 +105,58 @@ public void testEmptyTerm() throws IOException { protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new KeywordTokenizer(); return new TokenStreamComponents( - tokenizer, new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true)); + tokenizer, + new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true, false)); } }; checkOneTerm(a, "", ""); a.close(); } + + public void testKeywordFilter() throws Exception { + assertAnalyzesTo( + keywordTestAnalyzer(true), + "banana kappa alpha", + new String[] {"bXnXnX", "kappa", "XlphX"}, // kappa unchanged, others modified + new int[] {0, 7, 13}, + new int[] {6, 12, 18}, + null, + new int[] {1, 1, 1}, + null, + false); + + assertAnalyzesTo( + keywordTestAnalyzer(false), + "banana kappa alpha", + new String[] {"bXnXnX", "kXppX", "XlphX"}, // all terms modified + new int[] {0, 7, 13}, + new int[] {6, 12, 18}, + null, + new int[] {1, 1, 1}, + null, + false); + } + + private Analyzer keywordTestAnalyzer(boolean ignoreKeywords) throws Exception { + return new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); + KeywordMarkerFilter keywordFilter = + new KeywordMarkerFilter(tokenizer) { + private final CharTermAttribute term = addAttribute(CharTermAttribute.class); + + @Override + public boolean isKeyword() { + // Mark terms starting with 'k' as keywords + return term.toString().startsWith("k"); + } + }; + return new TokenStreamComponents( + tokenizer, + new PatternReplaceFilter( + keywordFilter, Pattern.compile("a"), "X", true, ignoreKeywords)); + } + }; + } }