diff --git a/changelog/unreleased/SOLR-16341-fix-blank-file-zip-handling.yml b/changelog/unreleased/SOLR-16341-fix-blank-file-zip-handling.yml new file mode 100644 index 000000000000..69fd1515ce73 --- /dev/null +++ b/changelog/unreleased/SOLR-16341-fix-blank-file-zip-handling.yml @@ -0,0 +1,8 @@ + +title: Support blank/zero-byte files in configset zip uploads +type: fixed +authors: + - name: Eric Pugh +links: + - name: SOLR-16341 + url: https://issues.apache.org/jira/browse/SOLR-16341 diff --git a/changelog/unreleased/SOLR-18189.yml b/changelog/unreleased/SOLR-18189.yml new file mode 100644 index 000000000000..638acdf99508 --- /dev/null +++ b/changelog/unreleased/SOLR-18189.yml @@ -0,0 +1,8 @@ +title: New ContentHashVersionProcessor to avoid index churn when adding same-content documents. +type: added +authors: + - name: Francois Huaulme + - name: David Smiley +links: + - name: SOLR-18189 + url: https://issues.apache.org/jira/browse/SOLR-18189 diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java index 6728b17ef103..bb9ca94c761a 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java @@ -22,11 +22,15 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; +import java.util.Enumeration; import java.util.List; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; import org.apache.solr.client.api.endpoint.ConfigsetsApi; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; @@ -85,22 +89,41 @@ public SolrJerseyResponse uploadConfigSet( filesToDelete = new ArrayList<>(); } - try (ZipInputStream zis = new ZipInputStream(requestBody, StandardCharsets.UTF_8)) { - boolean hasEntry = false; - ZipEntry zipEntry; - while ((zipEntry = zis.getNextEntry()) != null) { - hasEntry = true; - String filePath = zipEntry.getName(); - filesToDelete.remove(filePath); - if (!zipEntry.isDirectory()) { - configSetService.uploadFileToConfig(configSetName, filePath, zis.readAllBytes(), true); + // Write the request body to a temp file so we can use ZipFile, which reads the central + // directory and correctly handles entries that use the STORED method with an EXT (data + // descriptor) flag — a combination that ZipInputStream cannot process. This allows + // zero-byte files (e.g. created with `touch`) to be included in the uploaded configset. + final Path tempZip = Files.createTempFile("solr-configset-upload-", ".zip"); + try { + Files.copy(requestBody, tempZip, StandardCopyOption.REPLACE_EXISTING); + try (ZipFile zipFile = new ZipFile(tempZip.toFile())) { + boolean hasEntry = false; + Enumeration entries = zipFile.entries(); + while (entries.hasMoreElements()) { + ZipEntry zipEntry = entries.nextElement(); + hasEntry = true; + String filePath = zipEntry.getName(); + filesToDelete.remove(filePath); + if (!zipEntry.isDirectory()) { + try (InputStream entryStream = zipFile.getInputStream(zipEntry)) { + configSetService.uploadFileToConfig( + configSetName, filePath, entryStream.readAllBytes(), true); + } + } } - } - if (!hasEntry) { + if (!hasEntry) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Either empty zipped data, or non-zipped data was uploaded. In order to upload a configSet, you must zip a non-empty directory to upload."); + } + } catch (ZipException e) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "Either empty zipped data, or non-zipped data was uploaded. In order to upload a configSet, you must zip a non-empty directory to upload."); + "Failed to read the uploaded zip file: " + e.getMessage(), + e); } + } finally { + Files.deleteIfExists(tempZip); } deleteUnusedFiles(configSetService, configSetName, filesToDelete); diff --git a/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessor.java new file mode 100644 index 000000000000..ec59596ab8a2 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessor.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.component.RealTimeGetComponent; +import org.apache.solr.handler.component.RealTimeGetComponent.Resolution; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.update.AddUpdateCommand; + +/** + * An implementation of {@link UpdateRequestProcessor} which computes a hash of field values, and + * uses this hash to reject/accept document updates. + * + * + * + *

Depending on {#dropSameDocuments} value, this processor may drop or accept document updates. + * This implementation can be used for monitoring or dropping no-op updates (updates that do not + * change the Solr document content). + * + *

Note: the hash is computed using {@link Lookup3Signature} and must be stored in a field with + * docValues enabled for retrieval. + * + * @see Lookup3Signature + */ +public class ContentHashVersionProcessor extends UpdateRequestProcessor { + private final SchemaField hashField; + private final SolrQueryResponse rsp; + private final SolrCore core; + private final Predicate includedFields; // Matcher for included fields in hash + private final Predicate excludedFields; // Matcher for excluded fields from hash + private boolean dropSameDocuments; + private int sameCount = 0; + private int differentCount = 0; + + public ContentHashVersionProcessor( + Predicate hashIncludedFields, + Predicate hashExcludedFields, + String hashFieldName, + boolean dropSameDocuments, + SolrQueryRequest req, + SolrQueryResponse rsp, + UpdateRequestProcessor next) { + super(next); + this.core = req.getCore(); + + IndexSchema schema = core.getLatestSchema(); + this.hashField = schema.getField(hashFieldName); + this.dropSameDocuments = dropSameDocuments; + this.rsp = rsp; + this.includedFields = hashIncludedFields; + this.excludedFields = hashExcludedFields; + } + + @Override + public void processAdd(AddUpdateCommand cmd) throws IOException { + SolrInputDocument newDoc = cmd.getSolrInputDocument(); + byte[] newHash = computeDocHash(newDoc); + newDoc.setField(hashField.getName(), newHash); + + if (!isHashAcceptable(cmd.getIndexedId(), newHash)) { + return; + } + super.processAdd(cmd); + } + + @Override + public void finish() throws IOException { + try { + super.finish(); + } finally { + if (sameCount + differentCount > 0) { + if (dropSameDocuments) { + rsp.addToLog("contentHash.duplicatesDropped", sameCount); + } else { + rsp.addToLog("contentHash.duplicatesDetected", sameCount); + } + } + } + } + + private boolean isHashAcceptable(BytesRef indexedDocId, byte[] newHash) throws IOException { + assert null != indexedDocId; + + Optional oldDocHash = getOldDocHash(indexedDocId); + if (oldDocHash.isPresent()) { + if (Arrays.equals(newHash, oldDocHash.get())) { + sameCount++; + return !dropSameDocuments; + } else { + differentCount++; + return true; + } + } + return true; // Doc not found + } + + /** Retrieves the hash value from the old document identified by the given ID. */ + private Optional getOldDocHash(BytesRef indexedDocId) throws IOException { + SolrInputDocument oldDoc = + RealTimeGetComponent.getInputDocument( + core, indexedDocId, indexedDocId, null, Set.of(hashField.getName()), Resolution.DOC); + if (oldDoc == null) { + return Optional.empty(); + } + Object o = oldDoc.getFieldValue(hashField.getName()); + if (o instanceof byte[] bytes) { + return Optional.of(bytes); + } else if (o instanceof ByteBuffer buf) { + byte[] bytes = new byte[buf.remaining()]; + buf.duplicate().get(bytes); + return Optional.of(bytes); + } + return Optional.empty(); + } + + byte[] computeDocHash(SolrInputDocument doc) { + final Signature sig = new Lookup3Signature(); + + // Stream field names, filter, sort, and process in a single pass + doc.values().stream() + .filter(includedFields) // Keep fields that match 'included fields' matcher + .filter(excludedFields.negate()) // Exclude fields that match 'excluded fields' matcher + .sorted( + Comparator.comparing( + SolrInputField + ::getName)) // Sort to ensure consistent field order across different doc field + // orders + .forEach( + inputField -> { + sig.add(inputField.getName()); + Object o = inputField.getValue(); + if (o instanceof Collection) { + for (Object oo : (Collection) o) { + sig.add(String.valueOf(oo)); + } + } else { + sig.add(String.valueOf(o)); + } + }); + + return sig.getSignature(); + } +} diff --git a/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessorFactory.java new file mode 100644 index 000000000000..479ce33e987e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessorFactory.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputField; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; + +/** + * Factory for {@link ContentHashVersionProcessor} instances. + * + *

This processor computes a hash from the document content and sees if an existing (indexed) + * document also has this hash, stored in a designated field. If so, it drops the document to avoid + * needless index churn. Alternatively, it can be configured to merely log the fact. + * + *

Configuration

+ * + * + * + *

Configuration Example

+ * + * 
+ * <processor class="solr.ContentHashVersionProcessorFactory">
+ *   <str name="hashField">content_hash</str>
+ *   <str name="includeFields">title,body,author</str>
+ *   <str name="excludeFields">timestamp,version</str>
+ *   <str name="hashCompareStrategy">drop</str>
+ * </processor>
+ *
+ * + *

Important Considerations

+ * + * + * + *

Monitoring

+ * + *

The processor logs duplicate statistics in the response: + * + *

+ * + * @see ContentHashVersionProcessor + * @see Lookup3Signature + */ +public class ContentHashVersionProcessorFactory extends UpdateRequestProcessorFactory { + private static final char SEPARATOR = ','; // Separator for included/excluded fields + private List includeFields = List.of("*"); // Included fields defaults to 'all' + private List excludeFields = new ArrayList<>(); + private String hashField; // Must be explicitly configured + private boolean dropSameDocuments = true; + + public ContentHashVersionProcessorFactory() {} + + @Override + public void init(NamedList args) { + Object tmp = args.remove("includeFields"); + if (tmp != null) { + if (!(tmp instanceof String)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "'includeFields' must be configured as a "); + } + // Include fields support comma separated list of fields (e.g. "field1,field2,field3"). + // Also supports "*" to include all fields + this.includeFields = + StrUtils.splitSmart((String) tmp, SEPARATOR).stream() + .map(String::trim) + .collect(Collectors.toList()); + } + tmp = args.remove("hashField"); + if (tmp == null) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "'hashField' is required and must be explicitly configured"); + } + if (!(tmp instanceof String)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "'hashField' must be configured as a "); + } + this.hashField = (String) tmp; + + tmp = args.remove("excludeFields"); + if (tmp != null) { + if (!(tmp instanceof String)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "'excludeFields' must be configured as a "); + } + if ("*".equals(((String) tmp).trim())) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, "'excludeFields' can't exclude all fields."); + } + // Exclude fields support comma separated list of fields (e.g. + // "excluded_field1,excluded_field2"). + // Also supports "*" to exclude all fields + this.excludeFields = + StrUtils.splitSmart((String) tmp, SEPARATOR).stream() + .map(String::trim) + .collect(Collectors.toList()); + } + excludeFields.add(hashField); // Hash field name is excluded from hash computation + + tmp = args.remove("hashCompareStrategy"); + if (tmp != null) { + if (!(tmp instanceof String)) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "'hashCompareStrategy' must be configured as a "); + } + String value = ((String) tmp).toLowerCase(Locale.ROOT); + if ("drop".equalsIgnoreCase(value)) { + dropSameDocuments = true; + } else if ("log".equalsIgnoreCase(value)) { + dropSameDocuments = false; + } else { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Value '" + + value + + "' is unsupported for 'hashCompareStrategy', only 'drop' and 'log' are supported."); + } + } + + super.init(args); + } + + @Override + public UpdateRequestProcessor getInstance( + SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) { + return new ContentHashVersionProcessor( + buildFieldMatcher(includeFields), + buildFieldMatcher(excludeFields), + hashField, + dropSameDocuments, + req, + rsp, + next); + } + + public String getHashField() { + return hashField; + } + + public List getIncludeFields() { + return includeFields; + } + + public List getExcludeFields() { + return excludeFields; + } + + public boolean dropSameDocuments() { + return dropSameDocuments; + } + + static Predicate buildFieldMatcher(List fieldNames) { + return inputField -> { + for (String currentFieldName : fieldNames) { + if ("*".equals(currentFieldName)) { + return true; + } + final String fieldName = inputField.getName(); + if (fieldName.equals(currentFieldName)) { + return true; + } + if (currentFieldName.length() > 1 + && currentFieldName.endsWith("*") + && fieldName.startsWith(currentFieldName.substring(0, currentFieldName.length() - 1))) { + return true; + } + } + return false; + }; + } +} diff --git a/solr/core/src/test-files/solr/collection1/conf/schema16.xml b/solr/core/src/test-files/solr/collection1/conf/schema16.xml new file mode 100644 index 000000000000..f34bfed2e4c0 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema16.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + _id + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-contenthashversion.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-contenthashversion.xml new file mode 100644 index 000000000000..1308c63a9727 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-contenthashversion.xml @@ -0,0 +1,65 @@ + + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + ${solr.data.dir:} + + + + + + ${solr.ulog.dir:} + + + + + + _hash_ + _id + + + + + + + _hash_ + _id + log + + + + + + + _hash_ + _id + drop + + + + + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java index 6736af93b686..29c17139d2a4 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java @@ -25,6 +25,7 @@ import jakarta.servlet.http.HttpServletRequestWrapper; import jakarta.servlet.http.HttpServletResponse; import java.io.ByteArrayInputStream; +import java.io.DataOutputStream; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -1002,6 +1003,32 @@ public void testUploadWithForbiddenContent() throws Exception { assertEquals(400, res); } + @Test + public void testUploadWithBlankFile() throws Exception { + // Uploads a zip containing a blank (0-byte) file using STORED method with an EXT descriptor. + // Java's ZipInputStream cannot read this format, but ZipFile can. + // Verifies the upload succeeds and the empty file is stored in the configset. + final String configSetName = "blank-file-configset"; + final String suffix = "-suffix"; + final Path zipFile = createTempZipWithStoredEntryAndExtDescriptor(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(cluster.getZkServer().getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .withConnTimeOut(45000, TimeUnit.MILLISECONDS) + .build()) { + long res = uploadGivenConfigSet(zipFile, configSetName, suffix, null, true, false, true); + assertEquals("Upload of configset with blank file should succeed", 0L, res); + assertTrue( + "blank.txt should have been uploaded to the configset", + zkClient.exists("/configs/" + configSetName + suffix + "/blank.txt")); + assertArrayEquals( + "blank.txt in configset should be empty", + new byte[0], + zkClient.getData("/configs/" + configSetName + suffix + "/blank.txt", null, null)); + } + } + @Test public void testGetFile() throws Exception { String configSetName = "regular"; @@ -1331,6 +1358,99 @@ private Path createTempZipFileWithForbiddenContent(String resourcePath) { } } + /** + * Creates a zip file (in the temp directory) containing an empty file entry that uses the STORED + * compression method with the EXT descriptor flag set. Some zip tools produce this format for + * empty (0-byte) files, e.g., when using {@code touch conf/blank.txt} followed by {@code zip -r + * ...}. Java's {@link java.util.zip.ZipInputStream} cannot read this combination, but {@link + * java.util.zip.ZipFile} handles it correctly by reading from the central directory. + */ + private Path createTempZipWithStoredEntryAndExtDescriptor() throws IOException { + final Path zipFile = createTempFile("configset-blank", "zip"); + // Build a valid ZIP file manually with one STORED entry that has the EXT (data descriptor) + // flag set (flag bit 3 = 0x08). Java's ZipInputStream rejects this combination. + // All multi-byte fields are little-endian. + byte[] fileName = "blank.txt".getBytes(UTF_8); + int fileNameLen = fileName.length; // 9 + + // Offsets for computing central directory offset + // Local file header size: 30 + fileNameLen + int localHeaderSize = 30 + fileNameLen; + // Data descriptor size: 16 (with signature) + int dataDescriptorSize = 16; + // Central directory header size: 46 + fileNameLen + int centralDirHeaderSize = 46 + fileNameLen; + int centralDirOffset = localHeaderSize + dataDescriptorSize; // = 55 + + try (DataOutputStream dos = new DataOutputStream(Files.newOutputStream(zipFile))) { + // --- Local file header --- + dos.write(new byte[] {0x50, 0x4b, 0x03, 0x04}); // signature PK\x03\x04 + dos.write(new byte[] {0x14, 0x00}); // version needed = 20 + dos.write(new byte[] {0x08, 0x00}); // flag: bit 3 (data descriptor / EXT) + dos.write(new byte[] {0x00, 0x00}); // compression method: STORED + dos.write(new byte[] {0x00, 0x00}); // last mod time + dos.write(new byte[] {0x00, 0x00}); // last mod date + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // CRC-32 (0, deferred to data descriptor) + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // compressed size (deferred) + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // uncompressed size (deferred) + dos.write(new byte[] {(byte) fileNameLen, 0x00}); // file name length + dos.write(new byte[] {0x00, 0x00}); // extra field length + dos.write(fileName); // file name "blank.txt" + // (no file data — the file is empty) + + // --- Data descriptor (EXT record) --- + dos.write(new byte[] {0x50, 0x4b, 0x07, 0x08}); // signature PK\x07\x08 + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // CRC-32 (0 for empty file) + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // compressed size + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // uncompressed size + + // --- Central directory header --- + dos.write(new byte[] {0x50, 0x4b, 0x01, 0x02}); // signature PK\x01\x02 + dos.write(new byte[] {0x14, 0x00}); // version made by + dos.write(new byte[] {0x14, 0x00}); // version needed + dos.write(new byte[] {0x08, 0x00}); // flag (same as local header) + dos.write(new byte[] {0x00, 0x00}); // compression method: STORED + dos.write(new byte[] {0x00, 0x00}); // last mod time + dos.write(new byte[] {0x00, 0x00}); // last mod date + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // CRC-32 + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // compressed size + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // uncompressed size + dos.write(new byte[] {(byte) fileNameLen, 0x00}); // file name length + dos.write(new byte[] {0x00, 0x00}); // extra field length + dos.write(new byte[] {0x00, 0x00}); // file comment length + dos.write(new byte[] {0x00, 0x00}); // disk number start + dos.write(new byte[] {0x00, 0x00}); // internal file attributes + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // external file attributes + dos.write(new byte[] {0x00, 0x00, 0x00, 0x00}); // local header relative offset (= 0) + dos.write(fileName); // file name "blank.txt" + + // --- End of central directory record --- + dos.write(new byte[] {0x50, 0x4b, 0x05, 0x06}); // signature PK\x05\x06 + dos.write(new byte[] {0x00, 0x00}); // disk number + dos.write(new byte[] {0x00, 0x00}); // disk with start of central directory + dos.write(new byte[] {0x01, 0x00}); // entries on this disk + dos.write(new byte[] {0x01, 0x00}); // total entries + // size of central directory + dos.write( + new byte[] { + (byte) (centralDirHeaderSize & 0xFF), + (byte) ((centralDirHeaderSize >> 8) & 0xFF), + (byte) ((centralDirHeaderSize >> 16) & 0xFF), + (byte) ((centralDirHeaderSize >> 24) & 0xFF) + }); + // offset of central directory + dos.write( + new byte[] { + (byte) (centralDirOffset & 0xFF), + (byte) ((centralDirOffset >> 8) & 0xFF), + (byte) ((centralDirOffset >> 16) & 0xFF), + (byte) ((centralDirOffset >> 24) & 0xFF) + }); + dos.write(new byte[] {0x00, 0x00}); // comment length + } + return zipFile; + } + private static void zipWithForbiddenContent(Path directory, Path zipfile) throws IOException { OutputStream out = Files.newOutputStream(zipfile); assertTrue(Files.isDirectory(directory)); diff --git a/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorFactoryTest.java new file mode 100644 index 000000000000..b278838de9b7 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorFactoryTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.update.processor; + +import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ContentHashVersionProcessorFactoryTest { + + @BeforeClass + public static void beforeClass() throws Exception { + assumeWorkingMockito(); + } + + @Test + public void shouldHaveSensibleDefaultValues() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + assertEquals(List.of("*"), factory.getIncludeFields()); + assertTrue(factory.dropSameDocuments()); + } + + @Test + public void shouldInitWithHashFieldName() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + args.add("hashField", "_hash_field_"); + factory.init(args); + + assertEquals("_hash_field_", factory.getHashField()); + } + + @Test + public void shouldInitWithAllField() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + args.add("hashField", "content_hash"); + args.add("includeFields", "*"); + factory.init(args); + + assertEquals(1, factory.getIncludeFields().size()); + assertEquals("*", factory.getIncludeFields().getFirst()); + } + + @Test + public void shouldInitWithIncludedFields() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + args.add("hashField", "content_hash"); + args.add("includeFields", " field1,field2 , field3 "); + factory.init(args); + + assertEquals(3, factory.getIncludeFields().size()); + assertEquals(List.of("field1", "field2", "field3"), factory.getIncludeFields()); + } + + @Test + public void shouldInitWithExcludedFields() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + args.add("hashField", "content_hash"); + args.add("excludeFields", " field1,field2 , field3 "); + factory.init(args); + + assertEquals(4, factory.getExcludeFields().size()); + assertEquals(List.of("field1", "field2", "field3", "content_hash"), factory.getExcludeFields()); + } + + @Test + public void shouldSelectDropStrategy() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + args.add("hashField", "content_hash"); + args.add("hashCompareStrategy", "drop"); + factory.init(args); + + assertTrue(factory.dropSameDocuments()); + } + + @Test + public void shouldSelectLogStrategy() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + args.add("hashField", "content_hash"); + args.add("hashCompareStrategy", "log"); + factory.init(args); + + assertFalse(factory.dropSameDocuments()); + } + + @Test(expected = SolrException.class) + public void shouldSelectUnsupportedStrategy() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + args.add("hashField", "content_hash"); + args.add("hashCompareStrategy", "unsupported value"); + factory.init(args); + } + + @Test(expected = SolrException.class) + public void shouldRejectExcludeAllFields() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + args.add("hashField", "content_hash"); + args.add("excludeFields", "*"); + factory.init(args); + } + + @Test(expected = SolrException.class) + public void shouldRequireExplicitHashFieldName() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + // Intentionally not setting hashField + factory.init(args); + } + + @Test + public void shouldAutoExcludeHashFieldFromHashComputation() { + ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory(); + NamedList args = new NamedList<>(); + args.add("hashField", "my_hash_field"); + args.add("excludeFields", "field1,field2"); + factory.init(args); + + // Hash field should be automatically added to excludeFields + assertEquals(3, factory.getExcludeFields().size()); + assertTrue( + "Should contain explicitly excluded field1", factory.getExcludeFields().contains("field1")); + assertTrue( + "Should contain explicitly excluded field2", factory.getExcludeFields().contains("field2")); + assertTrue( + "Should auto-exclude hash field name", + factory.getExcludeFields().contains("my_hash_field")); + } +} diff --git a/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorTest.java new file mode 100644 index 000000000000..4a3764ed6a76 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorTest.java @@ -0,0 +1,452 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.update.processor; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.UUID; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.BinaryField; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ContentHashVersionProcessorTest extends UpdateProcessorTestBase { + + private static final String ID_FIELD = "_id"; + private static final String FIRST_FIELD = "field1"; + private static final String SECOND_FIELD = "field2"; + private static final String THIRD_FIELD = "docField3"; + private static final String FOURTH_FIELD = "field4"; + + private static final String INITIAL_DOC_ID = "1"; + private static final String INITIAL_FIELD1_VALUE = "Initial values used to compute initial hash"; + private static final String INITIAL_FIELD2_VALUE = + "This a constant value for testing include/exclude fields"; + private static final String INITIAL_DOC = + adoc( + ID_FIELD, INITIAL_DOC_ID, + FIRST_FIELD, INITIAL_FIELD1_VALUE, + SECOND_FIELD, INITIAL_FIELD2_VALUE); + private String initialDocHash; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeWorkingMockito(); + initCore("solrconfig-contenthashversion.xml", "schema16.xml"); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + assertU(delQ("*:*")); + addDoc(INITIAL_DOC, "contenthashversion-default"); + assertU(commit()); + + // Query for the document and extract _hash_ field value + initialDocHash = getHashFieldValue(INITIAL_DOC_ID); + } + + private static String getHashFieldValue(String docId) throws Exception { + String response = h.query(req("q", ID_FIELD + ":" + docId, "fl", "_hash_")); + + // Parse XML response to extract _hash_ field value + // Response format: value + String hashPattern = ""; + int startIdx = response.indexOf(hashPattern); + if (startIdx == -1) { + fail("Hash field not found in document " + docId); + } + startIdx += hashPattern.length(); + int endIdx = response.indexOf("", startIdx); + if (endIdx == -1) { + fail("Hash field closing tag not found"); + } + return response.substring(startIdx, endIdx); + } + + private ContentHashVersionProcessor getContentHashVersionProcessor( + List includedFields, List excludedFields) { + final SolrQueryRequest req = mock(SolrQueryRequest.class); + final SolrCore solrCore = mock(SolrCore.class); + final IndexSchema indexSchema = mock(IndexSchema.class); + when(indexSchema.getField("_hash_")).thenReturn(new SchemaField("_hash_", new BinaryField())); + + when(solrCore.getLatestSchema()).thenReturn(indexSchema); + when(req.getCore()).thenReturn(solrCore); + return new ContentHashVersionProcessor( + ContentHashVersionProcessorFactory.buildFieldMatcher(includedFields), + ContentHashVersionProcessorFactory.buildFieldMatcher(excludedFields), + "_hash_", + false, + req, + mock(SolrQueryResponse.class), + mock(UpdateRequestProcessor.class)); + } + + @Test + public void shouldUseExcludedFieldsWildcard() { + // Given + ContentHashVersionProcessor processor = + getContentHashVersionProcessor(List.of("*"), List.of("field*")); + + // Given (doc for update) + SolrInputDocument inputDocument = + doc( + f(ID_FIELD, "0000000001"), + f(FIRST_FIELD, UUID.randomUUID().toString()), + f(SECOND_FIELD, UUID.randomUUID().toString()), + f(THIRD_FIELD, "constant to have a constant hash"), + f(FOURTH_FIELD, UUID.randomUUID().toString())); + + // Then (only ID and THIRD_FIELD is used in hash, other fields contain random values) + assertArrayEquals( + Base64.getDecoder().decode("bwE8Zjq0aOs="), + processor.computeDocHash(inputDocument)); // Hash if only ID field was used + } + + @Test + public void shouldUseIncludedFieldsWildcard() { + // Given + ContentHashVersionProcessor processor = + getContentHashVersionProcessor(List.of("field*"), List.of(THIRD_FIELD)); + + // Given (doc for update) + SolrInputDocument inputDocument = + doc( + f(ID_FIELD, "0000000001"), + f(FIRST_FIELD, "constant to have a constant hash for field1"), + f(SECOND_FIELD, "constant to have a constant hash for field2"), + f(THIRD_FIELD, UUID.randomUUID().toString()), + f(FOURTH_FIELD, "constant to have a constant hash for field4")); + + // Then + assertArrayEquals( + Base64.getDecoder().decode("PozPs2qZQtw="), processor.computeDocHash(inputDocument)); + } + + @Test + public void shouldUseIncludedFieldsWildcard2() { + // Given (variant of previous shouldUseIncludedFieldsWildcard, without the excludedField config) + ContentHashVersionProcessor processor = + getContentHashVersionProcessor(List.of("field*"), List.of()); + + // Given (doc for update) + SolrInputDocument inputDocument = + doc( + f(ID_FIELD, "0000000001"), + f(FIRST_FIELD, "constant to have a constant hash for field1"), + f(SECOND_FIELD, "constant to have a constant hash for field2"), + f(THIRD_FIELD, UUID.randomUUID().toString()), + f(FOURTH_FIELD, "constant to have a constant hash for field4")); + + // Then + assertArrayEquals( + Base64.getDecoder().decode("PozPs2qZQtw="), processor.computeDocHash(inputDocument)); + } + + @Test + public void shouldDedupIncludedFields() { + // Given (processor to include field1 and field2 only) + ContentHashVersionProcessor processorWithDuplicatedFieldName = + getContentHashVersionProcessor(List.of(FIRST_FIELD, FIRST_FIELD, SECOND_FIELD), List.of()); + ContentHashVersionProcessor processorWithWildcard = + getContentHashVersionProcessor( + List.of( // Also change order of config (test reorder of field names) + SECOND_FIELD, FIRST_FIELD, "field1*"), + List.of()); + + // Given (doc for update) + SolrInputDocument inputDocument = + doc( + f(ID_FIELD, "0000000001"), + f(FIRST_FIELD, "constant to have a constant hash for field1"), + f(SECOND_FIELD, "constant to have a constant hash for field2"), + f(THIRD_FIELD, UUID.randomUUID().toString()), + f(FOURTH_FIELD, "constant to have a constant hash for field4")); + + // Then + assertArrayEquals( + Base64.getDecoder().decode("XavrOYGlkXM="), + processorWithDuplicatedFieldName.computeDocHash(inputDocument)); + assertArrayEquals( + Base64.getDecoder().decode("XavrOYGlkXM="), + processorWithWildcard.computeDocHash(inputDocument)); + } + + @Test + public void shouldCreateSignatureForNewDoc() throws Exception { + // When (update) + final String newDocId = UUID.randomUUID().toString(); + assertU( + adoc( + ID_FIELD, newDocId, + FIRST_FIELD, INITIAL_FIELD1_VALUE, + SECOND_FIELD, INITIAL_FIELD2_VALUE)); + assertU(commit()); + + // Then + final String hashFieldValueForNewDoc = getHashFieldValue(newDocId); + assertEquals(initialDocHash, hashFieldValueForNewDoc); + } + + @Test + public void shouldAddToResponseLog() throws Exception { + // Given (command to update existing doc) + final String newDocId = UUID.randomUUID().toString(); + final SolrQueryResponse update1 = + addDoc( + adoc( + ID_FIELD, newDocId, + FIRST_FIELD, INITIAL_FIELD1_VALUE, + SECOND_FIELD, INITIAL_FIELD2_VALUE), + "contenthashversion-default"); + final SolrQueryResponse update2 = + addDoc( + adoc( + ID_FIELD, newDocId, + FIRST_FIELD, "This is a doc with values", + SECOND_FIELD, "that differs from stored doc, so it's considered new"), + "contenthashversion-default"); + assertU(commit()); + + // Then + assertResponse(update1, -1, -1); + assertResponse(update2, 0, -1); + } + + @Test + public void shouldKeepDuplicateDocumentsInLogMode() throws Exception { + // Given: Use log chain which detects but does NOT drop duplicates + final String docId = UUID.randomUUID().toString(); + + // When: Add a document + addDoc( + adoc( + ID_FIELD, docId, + FIRST_FIELD, "original value", + SECOND_FIELD, "original value 2"), + "contenthashversion-log"); + assertU(commit()); + String originalHash = getHashFieldValue(docId); + + // When: Try to add the same content again (duplicate) + SolrQueryResponse duplicateResponse = + addDoc( + adoc( + ID_FIELD, docId, + FIRST_FIELD, "original value", + SECOND_FIELD, "original value 2"), + "contenthashversion-log"); + assertU(commit()); + + // Then: Response should show duplicate was detected but NOT dropped + assertResponse(duplicateResponse, -1, 1); + + // Then: Document should still exist in index + assertQ(req("q", ID_FIELD + ":" + docId), "//result[@numFound='1']"); + + // Then: Document hash should remain unchanged (duplicate was processed) + String currentHash = getHashFieldValue(docId); + assertEquals("Hash should remain unchanged for duplicate", originalHash, currentHash); + + // When: Update with different content + SolrQueryResponse changedResponse = + addDoc( + adoc( + ID_FIELD, docId, + FIRST_FIELD, "changed value", + SECOND_FIELD, "changed value 2"), + "contenthashversion-log"); + assertU(commit()); + + // Then: Response should show content changed + assertResponse(changedResponse, -1, 0); + + // Then: Hash should be updated + String newHash = getHashFieldValue(docId); + assertNotEquals("Hash should change for different content", originalHash, newHash); + } + + @Test + public void shouldExcludeFieldsUpdateSignatureForNewDoc() throws Exception { + // Given (update using URP chain WITHOUT drop doc (log mode)) + final String newDocId = UUID.randomUUID().toString(); + addDoc( + adoc( + ID_FIELD, newDocId, + FIRST_FIELD, INITIAL_FIELD1_VALUE, + SECOND_FIELD, INITIAL_FIELD2_VALUE), + "contenthashversion-default"); + assertU(commit()); + + // Then + final String hashFieldValue = getHashFieldValue(newDocId); + assertEquals(initialDocHash, hashFieldValue); + } + + @Test + public void shouldCommitWithDropModeEnabled() throws Exception { + // Initial document already exists from setUp() + // When: Try to add the same document again (duplicate content) using URP chain WITH drop doc + // (drop mode) + SolrQueryResponse solrQueryResponse = + addDoc( + adoc( + ID_FIELD, INITIAL_DOC_ID, + FIRST_FIELD, INITIAL_FIELD1_VALUE, + SECOND_FIELD, INITIAL_FIELD2_VALUE), + "contenthashversion-drop"); + assertU(commit()); + + // Then: Verify response shows duplicate was dropped + assertResponse(solrQueryResponse, 1, -1); + + // Then: Verify document was NOT actually added/updated (still only 1 doc in index) + assertQ(req("q", "*:*"), "//result[@numFound='1']"); + + // Verify the document still has the original hash + String currentHash = getHashFieldValue(INITIAL_DOC_ID); + assertEquals("Document hash should not have changed", initialDocHash, currentHash); + } + + @Test + public void shouldHandleDocumentWithOnlyIdField() { + // Given: Document with only ID field (no other fields to hash) + ContentHashVersionProcessor processor = + getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD)); + + // When: Compute hash for document with only ID + SolrInputDocument doc = doc(f(ID_FIELD, "only-id-doc")); + + // Then: Should compute hash (even if empty field set) + byte[] hash = processor.computeDocHash(doc); + assertNotNull("Hash should not be null for ID-only document", hash); + assertTrue("Hash should not be empty", hash.length > 0); + } + + @Test + public void shouldHandleMultiValueFields() { + // Given: Processor that includes multi-value fields + ContentHashVersionProcessor processor = + getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD)); + + // When: Document with multi-value field + SolrInputDocument doc1 = doc(f(ID_FIELD, "doc1"), f(FIRST_FIELD, "value1", "value2", "value3")); + + // Then: Should compute consistent hash + byte[] hash1 = processor.computeDocHash(doc1); + assertNotNull(hash1); + + // Same values in same order should produce same hash + SolrInputDocument doc2 = doc(f(ID_FIELD, "doc2"), f(FIRST_FIELD, "value1", "value2", "value3")); + byte[] hash2 = processor.computeDocHash(doc2); + assertArrayEquals("Same multi-value field should produce same hash", hash1, hash2); + + // Different order should produce different hash (collection order matters) + SolrInputDocument doc3 = doc(f(ID_FIELD, "doc3"), f(FIRST_FIELD, "value3", "value1", "value2")); + byte[] hash3 = processor.computeDocHash(doc3); + assertFalse("Different order should produce different hash", Arrays.equals(hash1, hash3)); + } + + @Test + public void shouldHandleNullFieldValues() { + // Given: Processor that handles null values + ContentHashVersionProcessor processor = + getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD)); + + // When: Document with null field value (represented as "null" string) + SolrInputDocument doc = doc(f(ID_FIELD, "null-doc"), f(FIRST_FIELD, (Object) null)); + + // Then: Should compute hash without error + byte[] hash = processor.computeDocHash(doc); + assertNotNull("Should handle null values", hash); + assertTrue("Hash should not be empty", hash.length > 0); + } + + @Test + public void shouldProduceSameHashRegardlessOfFieldOrder() { + // Given: Documents with same fields in different order + ContentHashVersionProcessor processor = + getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD)); + + // When: Create docs with fields in different order + SolrInputDocument doc1 = + doc( + f(ID_FIELD, "doc1"), + f(FIRST_FIELD, "value1"), + f(SECOND_FIELD, "value2"), + f(THIRD_FIELD, "value3")); + + SolrInputDocument doc2 = + doc( + f(ID_FIELD, "doc2"), + f(THIRD_FIELD, "value3"), + f(FIRST_FIELD, "value1"), + f(SECOND_FIELD, "value2")); + + // Then: Hashes should be identical (fields are sorted before hashing) + byte[] hash1 = processor.computeDocHash(doc1); + byte[] hash2 = processor.computeDocHash(doc2); + assertArrayEquals("Hash should be same regardless of field order", hash1, hash2); + } + + @Test + public void shouldHandleEmptyFieldValues() { + // Given: Document with empty string values + ContentHashVersionProcessor processor = + getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD)); + + SolrInputDocument doc1 = doc(f(ID_FIELD, "empty-doc"), f(FIRST_FIELD, ""), f(SECOND_FIELD, "")); + + // When: Compute hash + byte[] hash1 = processor.computeDocHash(doc1); + + // Then: Should produce valid hash + assertNotNull("Should handle empty values", hash1); + assertTrue("Hash should not be empty", hash1.length > 0); + + // Empty strings should produce different hash than no fields + SolrInputDocument doc2 = doc(f(ID_FIELD, "empty-doc")); + byte[] hash2 = processor.computeDocHash(doc2); + assertFalse("Empty string fields should differ from no fields", Arrays.equals(hash1, hash2)); + } + + private static void assertResponse( + SolrQueryResponse solrQueryResponse, int droppedDocCount, int duplicateDocCount) { + if (droppedDocCount >= 0) { + assertNotNull(solrQueryResponse.getToLog().get("contentHash.duplicatesDropped")); + int droppedDocs = (int) solrQueryResponse.getToLog().get("contentHash.duplicatesDropped"); + assertEquals(droppedDocCount, droppedDocs); + } + if (duplicateDocCount >= 0) { + assertNotNull(solrQueryResponse.getToLog().get("contentHash.duplicatesDetected")); + int duplicateDocs = (int) solrQueryResponse.getToLog().get("contentHash.duplicatesDetected"); + assertEquals(duplicateDocCount, duplicateDocs); + } + } +} diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc index 674b0b49360e..4d67da152419 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc @@ -410,6 +410,12 @@ When using any of these factories, please consult the {solr-javadocs}/core/org/a {solr-javadocs}/core/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactory.html[UniqFieldsUpdateProcessorFactory]:: Removes duplicate values found in fields matching the specified conditions. +{solr-javadocs}/core/org/apache/solr/update/processor/ContentHashVersionProcessorFactory.html[ContentHashVersionProcessorFactory]:: Removes duplicate documents detected using a configurable hash of field values. +[WARNING] +==== +This URP should be configured to exclude fields that are updated with xref:indexing-guide:partial-document-updates.adoc[in-place updates] from the content hash calculation, as these fields are updated independently and should not affect duplicate detection. +==== + === Update Processor Factories That Can Be Loaded as Plugins These processors are included in Solr releases as "module", and require additional jars loaded at runtime. diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index ca7677a7e787..b450cff5bd71 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -1159,7 +1159,8 @@ public static String adoc(SolrInputDocument sdoc) { return out.toString(); } - public static void addDoc(String doc, String updateRequestProcessorChain) throws Exception { + public static SolrQueryResponse addDoc(String doc, String updateRequestProcessorChain) + throws Exception { Map params = new HashMap<>(); MultiMapSolrParams mmparams = new MultiMapSolrParams(params); params.put(UpdateParams.UPDATE_CHAIN, new String[] {updateRequestProcessorChain}); @@ -1168,8 +1169,11 @@ public static void addDoc(String doc, String updateRequestProcessorChain) throws UpdateRequestHandler handler = new UpdateRequestHandler(); handler.init(null); req.setContentStreams(List.of(new ContentStreamBase.StringStream(doc))); - handler.handleRequestBody(req, new SolrQueryResponse()); + final SolrQueryResponse rsp = new SolrQueryResponse(); + handler.handleRequestBody(req, rsp); req.close(); + + return rsp; } /**