Pattern replace keywords

lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java

@@ -22,6 +22,7 @@
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
 
 /**
  * A TokenFilter which applies a Pattern to each token in the stream, replacing match occurrences
@@ -35,7 +36,9 @@
 public final class PatternReplaceFilter extends TokenFilter {
   private final String replacement;
   private final boolean all;
+  private final boolean ignoreKeywords;
   private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
   private final Matcher m;
 
   /**
@@ -47,19 +50,26 @@ public final class PatternReplaceFilter extends TokenFilter {
    *     Note that this is not the literal string that will be used, '$' and '\' have special
    *     meaning.
    * @param all if true, all matches will be replaced otherwise just the first match.
+   * @param ignoreKeywords if true, tokens with KeywordAttribute set to true will not be processed
    * @see Matcher#quoteReplacement
    */
-  public PatternReplaceFilter(TokenStream in, Pattern p, String replacement, boolean all) {
+  public PatternReplaceFilter(
+      TokenStream in, Pattern p, String replacement, boolean all, boolean ignoreKeywords) {
     super(in);
     this.replacement = (null == replacement) ? "" : replacement;
     this.all = all;
+    this.ignoreKeywords = ignoreKeywords;
     this.m = p.matcher(termAtt);
   }
 
   @Override
   public boolean incrementToken() throws IOException {
     if (!input.incrementToken()) return false;
 
+    if (ignoreKeywords && keywordAtt.isKeyword()) {
+      return true;
+    }
+
     m.reset();
     if (m.find()) {
       // replaceAll/replaceFirst will reset() this previous find.

lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java

@@ -46,13 +46,15 @@ public class PatternReplaceFilterFactory extends TokenFilterFactory {
   final Pattern pattern;
   final String replacement;
   final boolean replaceAll;
+  final boolean ignoreKeywords;
 
   /** Creates a new PatternReplaceFilterFactory */
   public PatternReplaceFilterFactory(Map<String, String> args) {
     super(args);
     pattern = getPattern(args, "pattern");
     replacement = get(args, "replacement");
     replaceAll = "all".equals(get(args, "replace", Arrays.asList("all", "first"), "all"));
+    ignoreKeywords = getBoolean(args, "ignoreKeywords", false);
     if (!args.isEmpty()) {
       throw new IllegalArgumentException("Unknown parameters: " + args);
     }
@@ -65,6 +67,6 @@ public PatternReplaceFilterFactory() {
 
   @Override
   public PatternReplaceFilter create(TokenStream input) {
-    return new PatternReplaceFilter(input, pattern, replacement, replaceAll);
+    return new PatternReplaceFilter(input, pattern, replacement, replaceAll, ignoreKeywords);
   }
 }

lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java

@@ -22,6 +22,8 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.tests.analysis.MockTokenizer;
 
@@ -30,39 +32,40 @@ public class TestPatternReplaceFilter extends BaseTokenStreamTestCase {
   public void testReplaceAll() throws Exception {
     String input = "aabfooaabfooabfoob ab caaaaaaaaab";
     TokenStream ts =
-        new PatternReplaceFilter(whitespaceMockTokenizer(input), Pattern.compile("a*b"), "-", true);
+        new PatternReplaceFilter(
+            whitespaceMockTokenizer(input), Pattern.compile("a*b"), "-", true, false);
     assertTokenStreamContents(ts, new String[] {"-foo-foo-foo-", "-", "c-"});
   }
 
   public void testReplaceFirst() throws Exception {
     String input = "aabfooaabfooabfoob ab caaaaaaaaab";
     TokenStream ts =
         new PatternReplaceFilter(
-            whitespaceMockTokenizer(input), Pattern.compile("a*b"), "-", false);
+            whitespaceMockTokenizer(input), Pattern.compile("a*b"), "-", false, false);
     assertTokenStreamContents(ts, new String[] {"-fooaabfooabfoob", "-", "c-"});
   }
 
   public void testStripFirst() throws Exception {
     String input = "aabfooaabfooabfoob ab caaaaaaaaab";
     TokenStream ts =
         new PatternReplaceFilter(
-            whitespaceMockTokenizer(input), Pattern.compile("a*b"), null, false);
+            whitespaceMockTokenizer(input), Pattern.compile("a*b"), null, false, false);
     assertTokenStreamContents(ts, new String[] {"fooaabfooabfoob", "", "c"});
   }
 
   public void testStripAll() throws Exception {
     String input = "aabfooaabfooabfoob ab caaaaaaaaab";
     TokenStream ts =
         new PatternReplaceFilter(
-            whitespaceMockTokenizer(input), Pattern.compile("a*b"), null, true);
+            whitespaceMockTokenizer(input), Pattern.compile("a*b"), null, true, false);
     assertTokenStreamContents(ts, new String[] {"foofoofoo", "", "c"});
   }
 
   public void testReplaceAllWithBackRef() throws Exception {
     String input = "aabfooaabfooabfoob ab caaaaaaaaab";
     TokenStream ts =
         new PatternReplaceFilter(
-            whitespaceMockTokenizer(input), Pattern.compile("(a*)b"), "$1\\$", true);
+            whitespaceMockTokenizer(input), Pattern.compile("(a*)b"), "$1\\$", true, false);
     assertTokenStreamContents(ts, new String[] {"aa$fooaa$fooa$foo$", "a$", "caaaaaaaaa$"});
   }
 
@@ -74,7 +77,7 @@ public void testRandomStrings() throws Exception {
           protected TokenStreamComponents createComponents(String fieldName) {
             Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
             TokenStream filter =
-                new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", false);
+                new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", false, false);
             return new TokenStreamComponents(tokenizer, filter);
           }
         };
@@ -87,7 +90,7 @@ protected TokenStreamComponents createComponents(String fieldName) {
           protected TokenStreamComponents createComponents(String fieldName) {
             Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
             TokenStream filter =
-                new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true);
+                new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true, false);
             return new TokenStreamComponents(tokenizer, filter);
           }
         };
@@ -102,10 +105,58 @@ public void testEmptyTerm() throws IOException {
           protected TokenStreamComponents createComponents(String fieldName) {
             Tokenizer tokenizer = new KeywordTokenizer();
             return new TokenStreamComponents(
-                tokenizer, new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true));
+                tokenizer,
+                new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true, false));
           }
         };
     checkOneTerm(a, "", "");
     a.close();
   }
+
+  public void testKeywordFilter() throws Exception {
+    assertAnalyzesTo(
+        keywordTestAnalyzer(true),
+        "banana kappa alpha",
+        new String[] {"bXnXnX", "kappa", "XlphX"}, // kappa unchanged, others modified
+        new int[] {0, 7, 13},
+        new int[] {6, 12, 18},
+        null,
+        new int[] {1, 1, 1},
+        null,
+        false);
+
+    assertAnalyzesTo(
+        keywordTestAnalyzer(false),
+        "banana kappa alpha",
+        new String[] {"bXnXnX", "kXppX", "XlphX"}, // all terms modified
+        new int[] {0, 7, 13},
+        new int[] {6, 12, 18},
+        null,
+        new int[] {1, 1, 1},
+        null,
+        false);
+  }
+
+  private Analyzer keywordTestAnalyzer(boolean ignoreKeywords) throws Exception {
+    return new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName) {
+        Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+        KeywordMarkerFilter keywordFilter =
+            new KeywordMarkerFilter(tokenizer) {
+              private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
+
+              @Override
+              public boolean isKeyword() {
+                // Mark terms starting with 'k' as keywords
+                return term.toString().startsWith("k");
+              }
+            };
+        return new TokenStreamComponents(
+            tokenizer,
+            new PatternReplaceFilter(
+                keywordFilter, Pattern.compile("a"), "X", true, ignoreKeywords));
+      }
+    };
+  }
 }