diff --git a/.gitignore b/.gitignore index d9536745bb4..4e52444dd99 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,4 @@ atlassian-ide-plugin.xml derby.log .java-version oak-shaded-guava/dependency-reduced-pom.xml -.DS_Store -.claude/settings.local.json pom.xml.versionsBackup diff --git a/oak-auth-ldap/pom.xml b/oak-auth-ldap/pom.xml index b1882bd2f60..d746530a985 100644 --- a/oak-auth-ldap/pom.xml +++ b/oak-auth-ldap/pom.xml @@ -94,7 +94,7 @@ org.apache.mina mina-core - 2.1.12 + 2.1.15 org.apache.servicemix.bundles diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureDataStoreService.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureDataStoreService.java index ad63513f47a..7ba968f9a8e 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureDataStoreService.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AbstractAzureDataStoreService.java @@ -33,6 +33,9 @@ import org.osgi.service.component.ComponentContext; /** + * @deprecated Use {@link AzureDataStoreWrapper} instead. + */ +@Deprecated(since = "2.3", forRemoval = true) * Kept for binary compatibility with existing callers. Use {@link AzureDataStoreWrapper} instead. * * @deprecated diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreWrapper.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreWrapper.java index 47239fb0807..a8057fc35db 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreWrapper.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureDataStoreWrapper.java @@ -78,6 +78,9 @@ public class AzureDataStoreWrapper extends AbstractDataStoreService { private static final Logger log = LoggerFactory.getLogger(AzureDataStoreWrapper.class); + // Intentionally set to the legacy v8 PID rather than this class's own FQN. + // Existing OSGi configurations reference the v8 PID, so reusing it here means + // no config migration is required when switching to this wrapper. public static final String NAME = "org.apache.jackrabbit.oak.plugins.blob.datastore.AzureDataStore"; // Same name for now; kept as separate constants so they can diverge if the sources need different keys later. @@ -189,7 +192,8 @@ protected void setStatisticsProvider(StatisticsProvider statisticsProvider) { @Override protected String[] getDescription() { - return new String[]{"type=AzureBlob"}; + String sdkVersion = (activeImpl instanceof AzureDataStoreV12) ? "v12" : "v8"; + return new String[]{"type=AzureBlob", "sdkVersion=" + sdkVersion}; } // -- Inner DelegatingDataStore (returned from createDataStore) ------- diff --git a/oak-doc/src/site/markdown/osgi_config.md b/oak-doc/src/site/markdown/osgi_config.md index 89259d9fb02..6ee3e47fd0d 100644 --- a/oak-doc/src/site/markdown/osgi_config.md +++ b/oak-doc/src/site/markdown/osgi_config.md @@ -229,8 +229,9 @@ blobTrackSnapshotIntervalInSecs (long) - 0 deletions will be visible to other cluster nodes or repositories connected to the shared DatStore after this. This should be less than the blobGcMaxAgeInSecs parameter above and the frequency of blob gc. See [Blob tracker][blobtracker]. -A value of `0` (the default since Oak 2.4.0) disables blob ID tracking entirely for the Segment Node Store. To enable tracking, -set this to a positive value (e.g. `43200` for 12 hours). Note that DocumentNodeStore defaults to `43200`. +A value of `0` disables blob ID tracking entirely. This has been the default for both SegmentNodeStore and +DocumentNodeStore since Oak 2.4.0. +To enable tracking, set this to a positive value (e.g. `43200` for 12 hours). #### DocumentNodeStore @@ -262,7 +263,7 @@ docChildrenCachePercentage | 0 (was 3 until 1.5.6) | Percentage of `cache` alloc cacheSegmentCount | 16 | The number of segments in the LIRS cache | 1.0.15, 1.2.3, 1.3.0 cacheStackMoveDistance | 16 | The delay to move entries to the head of the queue in the LIRS cache | 1.0.15, 1.2.3, 1.3.0 sharedDSRepoId | "" | Custom SharedDataStore repositoryId. Used when custom blobstore configured. Should be unique among the repositories sharing the datastore. | 1.2.11 -blobTrackSnapshotIntervalInSecs | 43200 (12 hrs) | The blob ids cached/tracked locally are synchronized with the DataStore at this interval. Any additions and deletions will be visible to other cluster nodes or repositories connected to the shared DatStore after this. This should be less than the blobGcMaxAgeInSecs parameter above and the frequency of blob gc. See [Blob tracker][blobtracker]. | 1.5.6 +blobTrackSnapshotIntervalInSecs | 0 | The blob ids cached/tracked locally are synchronized with the DataStore at this interval. Any additions and deletions will be visible to other cluster nodes or repositories connected to the shared DatStore after this. This should be less than the blobGcMaxAgeInSecs parameter above and the frequency of blob gc. A value of `0` disables blob ID tracking (default since Oak 2.4.0). See [Blob tracker][blobtracker]. | 1.5.6 updateLimit | 100000 | The number of updates kept in memory until changes are written to a branch in the DocumentStore | 1.7.0 leaseCheckMode | STRICT | The lease check mode. `STRICT` is the default and will stop the DocumentNodeStore as soon as the lease expires. `LENIENT` will give the background lease update a chance to renew the lease even when the lease expired. This mode is only recommended for development, e.g. when debugging an application and the lease may expire when the JVM is stopped at a breakpoint. | 1.9.6 documentStoreType | MONGO | Set to "RDB" for `RDBDocumentStore`; will require a configured Sling DataSource called `oak`. | 1.0 diff --git a/oak-doc/src/site/markdown/plugins/blobstore.md b/oak-doc/src/site/markdown/plugins/blobstore.md index 85773f6df47..e9e190d4c50 100644 --- a/oak-doc/src/site/markdown/plugins/blobstore.md +++ b/oak-doc/src/site/markdown/plugins/blobstore.md @@ -237,9 +237,11 @@ configured NodeStore services. **Default behavior by NodeStore:** * **SegmentNodeStore** (`SegmentNodeStoreService` / `SegmentNodeStoreFactory`): blob ID tracking is **disabled by - default** (`blobTrackSnapshotIntervalInSecs = 0`). To enable it, set a positive value such as `43200` (12 hours). -* **DocumentNodeStore**: blob ID tracking is **enabled by default** with a 12-hour sync interval - (`blobTrackSnapshotIntervalInSecs = 43200`). + default** (`blobTrackSnapshotIntervalInSecs = 0`, default since Oak 2.4.0). To enable it, set a positive value such as + `43200` (12 hours). +* **DocumentNodeStore**: blob ID tracking is **disabled by default** + (`blobTrackSnapshotIntervalInSecs = 0`, default since Oak 2.4.0). To enable it, set a positive value such as `43200` + (12 hours). When tracking is disabled (interval = 0), blob IDs are not cached locally and the warnings described below do not apply. The full blob ID list is always retrieved directly from the DataStore during GC. diff --git a/oak-http/src/main/java/org/apache/jackrabbit/oak/http/AuthorizationField.java b/oak-http/src/main/java/org/apache/jackrabbit/oak/http/AuthorizationField.java index c572e4a3edd..13eda1a1d9b 100644 --- a/oak-http/src/main/java/org/apache/jackrabbit/oak/http/AuthorizationField.java +++ b/oak-http/src/main/java/org/apache/jackrabbit/oak/http/AuthorizationField.java @@ -16,11 +16,12 @@ */ package org.apache.jackrabbit.oak.http; -import org.apache.jackrabbit.util.Base64; - import javax.jcr.SimpleCredentials; import javax.security.auth.login.LoginException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Enumeration; +import java.util.Locale; import java.util.NoSuchElementException; public class AuthorizationField { @@ -44,11 +45,30 @@ public static SimpleCredentials valueOf(Enumeration values) throws Login return parseCredentials(field); } - private static SimpleCredentials parseCredentials(String fieldValue) throws LoginException { - if (fieldValue.startsWith("Basic ")) { - String[] basic = - Base64.decode(fieldValue.substring("Basic ".length())).split(":"); - return new SimpleCredentials(basic[0], basic[1].toCharArray()); + private static SimpleCredentials parseCredentials(String rawFieldValue) throws LoginException { + boolean hasControls = rawFieldValue.chars().anyMatch(c -> c < ' '); + if (hasControls) { + throw new LoginException("Control characters are not allowed"); + } + + String fieldValue = rawFieldValue.trim().replaceAll(" +", " "); + + if (fieldValue.toLowerCase(Locale.ENGLISH).startsWith("basic ")) { + String token68 = fieldValue.substring("basic ".length()); + try { + String decoded = new String(Base64.getDecoder().decode(token68), StandardCharsets.UTF_8); + int colon = decoded.indexOf(':'); + if (colon < 0) { + throw new LoginException( + "Malformed Basic credentials: missing ':' separator"); + } + String userId = decoded.substring(0, colon); + String password = decoded.substring(colon + 1); + + return new SimpleCredentials(userId, password.toCharArray()); + } catch (IllegalArgumentException ex) { + throw new LoginException(ex.getMessage()); + } } else { throw new LoginException("Only Basic Authentication supported"); } diff --git a/oak-http/src/test/java/org/apache/jackrabbit/oak/http/AuthorizationFieldTest.java b/oak-http/src/test/java/org/apache/jackrabbit/oak/http/AuthorizationFieldTest.java index e2a3ab214d3..4579db6c567 100644 --- a/oak-http/src/test/java/org/apache/jackrabbit/oak/http/AuthorizationFieldTest.java +++ b/oak-http/src/test/java/org/apache/jackrabbit/oak/http/AuthorizationFieldTest.java @@ -47,12 +47,10 @@ public void testValid() throws LoginException { assertEquals("bar", new String(credentials.getPassword())); } - @Test(expected = ArrayIndexOutOfBoundsException.class) // BUG: OAK-12259 + @Test(expected = LoginException.class) public void testInvalidBase64() throws LoginException { String b64 = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); - SimpleCredentials credentials = AuthorizationField.valueOf(Collections.enumeration(List.of("Basic dksjdkj" + b64))); - assertEquals("foo", credentials.getUserID()); - assertEquals("bar", new String(credentials.getPassword())); + AuthorizationField.valueOf(Collections.enumeration(List.of("Basic dksjdkj" + b64))); } @Test(expected = LoginException.class) @@ -64,9 +62,67 @@ public void testNoScheme() throws LoginException { @Test public void testColonInPassword() throws LoginException { String b64 = "Basic " + Base64.getEncoder().encodeToString("foo:bar:qux".getBytes(StandardCharsets.UTF_8)); - SimpleCredentials credentials =AuthorizationField.valueOf(Collections.enumeration(List.of(b64))); + SimpleCredentials credentials = AuthorizationField.valueOf(Collections.enumeration(List.of(b64))); + assertEquals("foo", credentials.getUserID()); + assertEquals("bar:qux", new String(credentials.getPassword())); + } + + @Test(expected = LoginException.class) + public void testMissingColon() throws LoginException { + String b64 = "Basic " + Base64.getEncoder().encodeToString("foobarqux".getBytes(StandardCharsets.UTF_8)); + AuthorizationField.valueOf(Collections.enumeration(List.of(b64))); + } + + @Test + public void testSchemeCase() throws LoginException { + String b64 = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); + SimpleCredentials credentials = AuthorizationField.valueOf(Collections.enumeration(List.of("BaSiC " + b64))); + assertEquals("foo", credentials.getUserID()); + assertEquals("bar", new String(credentials.getPassword())); + } + + @Test + public void testMoreWhitespace() throws LoginException { + String b64 = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); + SimpleCredentials credentials = AuthorizationField.valueOf(Collections.enumeration(List.of("Basic " + b64))); + assertEquals("foo", credentials.getUserID()); + assertEquals("bar", new String(credentials.getPassword())); + } + + @Test(expected = LoginException.class) + public void testNonSpWhitespace() throws LoginException { + String b64 = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); + AuthorizationField.valueOf(Collections.enumeration(List.of("Basic \t " + b64))); + } + + @Test(expected = LoginException.class) + public void testBrokenBase64() throws LoginException { + String b64 = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); + // insert a single SP into the base64 sequence + b64 = b64.substring(0,5) + " " + b64.substring(5); + AuthorizationField.valueOf(Collections.enumeration(List.of("Basic " + b64))); + } + + @Test(expected = LoginException.class) + public void testMoreBrokenBase64() throws LoginException { + String b64 = Base64.getEncoder().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); + b64 = b64.substring(0,5) + "=" + b64.substring(5); + AuthorizationField.valueOf(Collections.enumeration(List.of("Basic " + b64))); + } + + @Test + public void testMoreNonAscii() throws LoginException { + String b64 = Base64.getEncoder().encodeToString("test:123\u00a3".getBytes(StandardCharsets.UTF_8)); + SimpleCredentials credentials = AuthorizationField.valueOf(Collections.enumeration(List.of("Basic " + b64))); + assertEquals("test", credentials.getUserID()); + assertEquals("123\u00a3", new String(credentials.getPassword())); + } + + @Test + public void testBasic64NoPadding() throws LoginException { + String b64 = Base64.getEncoder().withoutPadding().encodeToString("foo:bar".getBytes(StandardCharsets.UTF_8)); + SimpleCredentials credentials = AuthorizationField.valueOf(Collections.enumeration(List.of("Basic " + b64))); assertEquals("foo", credentials.getUserID()); - // BUG: OAK-12259 assertEquals("bar", new String(credentials.getPassword())); } } \ No newline at end of file diff --git a/oak-it-osgi/pom.xml b/oak-it-osgi/pom.xml index 11a7092aeb1..26a4020ae44 100644 --- a/oak-it-osgi/pom.xml +++ b/oak-it-osgi/pom.xml @@ -198,7 +198,7 @@ org.testcontainers - elasticsearch + testcontainers-elasticsearch ${testcontainers.version} test diff --git a/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/DocumentBlobTrackerRegistrationTest.java b/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/DocumentBlobTrackerRegistrationTest.java index 4e6a59b6032..bc31f731a75 100644 --- a/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/DocumentBlobTrackerRegistrationTest.java +++ b/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/DocumentBlobTrackerRegistrationTest.java @@ -63,6 +63,9 @@ protected void registerNodeStoreService() { properties.put("repository.home", repoHome); properties.put("mongouri", MongoUtils.URL); properties.put("db", MongoUtils.DB); + // Explicitly enable blob ID tracking (default is 0/disabled since Oak 2.4.0). + // This test verifies tracker reinitialization which requires an active tracker. + properties.put("blobTrackSnapshotIntervalInSecs", 3600L); MockOsgi.setConfigForPid(context.bundleContext(), DocumentNodeStoreService.class.getName(), properties); service = context.registerInjectActivateService(new DocumentNodeStoreService()); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java index ec2c5990d47..12144f43429 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java @@ -384,6 +384,9 @@ private void activate(BundleContext bundleContext, Configuration config) throws oakRegs.add(whiteboard.register(FeatureToggle.class, new FeatureToggle(FulltextIndexEditor.FT_OAK_12193, FulltextIndexEditor.FT_OAK_12193_DISABLE), emptyMap())); + oakRegs.add(whiteboard.register(FeatureToggle.class, + new FeatureToggle(FulltextIndexEditor.FT_OAK_12244, FulltextIndexEditor.FT_OAK_12244_DISABLE), + emptyMap())); initializeIndexDir(bundleContext, config); initializeExtractedTextCache(bundleContext, config, statisticsProvider); tracker = createTracker(bundleContext, config); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java index 60e63d4e0a4..2bb7f6e5f1e 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java @@ -266,7 +266,7 @@ private void processDocs(String indexPath, Iterable docs, boolean doc doc.markProcessed(); } if (doc.delete) { - writer.deleteDocuments(doc.docPath); + writer.deleteDocumentTree(doc.docPath); } else { writer.updateDocument(doc.docPath, doc.doc); } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java index e98e78ef8c5..5f0a7ccc6e9 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LocalIndexWriterFactory.java @@ -56,12 +56,18 @@ public void updateDocument(String path, Iterable doc) } @Override - public void deleteDocuments(String path) throws IOException { + public void deleteDocumentTree(String path) throws IOException { //Hybrid index logic drops the deletes. So no use to //add them to the list //addLuceneDoc(LuceneDoc.forDelete(definition.getIndexPathFromConfig(), path)); } + @Override + public void deleteDocument(String path) throws IOException { + //Hybrid index logic drops the deletes. So no use to + //add them to the list + } + @Override public boolean close(long timestamp) throws IOException { documentHolder.done(indexPath); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java index c70755d4b67..5c9ae9f145a 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java @@ -411,7 +411,12 @@ public void updateDocument(String path, Iterable doc) } @Override - public void deleteDocuments(String path) throws IOException { + public void deleteDocumentTree(String path) throws IOException { + //Do not delete documents. Query side would handle it + } + + @Override + public void deleteDocument(String path) throws IOException { //Do not delete documents. Query side would handle it } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java index 79a7790cbf0..96a62098103 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java @@ -104,11 +104,16 @@ public void updateDocument(String path, Iterable doc) } @Override - public void deleteDocuments(String path) throws IOException { + public void deleteDocumentTree(String path) throws IOException { getWriter().deleteDocuments(newPathTerm(path)); getWriter().deleteDocuments(new PrefixQuery(newPathTerm(path + "/"))); } + @Override + public void deleteDocument(String path) throws IOException { + getWriter().deleteDocuments(newPathTerm(path)); + } + void deleteAll() throws IOException { getWriter().deleteAll(); indexUpdated = true; diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterPool.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterPool.java index 56ab1c6f05a..d2cefdcb839 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterPool.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterPool.java @@ -123,17 +123,31 @@ public void execute() throws IOException { } } - private static class DeleteOperation extends Operation { + private static class DeleteTreeOperation extends Operation { private final String path; - DeleteOperation(LuceneIndexWriter delegate, String path) { + DeleteTreeOperation(LuceneIndexWriter delegate, String path) { super(delegate); this.path = path; } @Override public void execute() throws IOException { - delegate.deleteDocuments(path); + delegate.deleteDocumentTree(path); + } + } + + private static class DeleteDocumentOperation extends Operation { + private final String path; + + DeleteDocumentOperation(LuceneIndexWriter delegate, String path) { + super(delegate); + this.path = path; + } + + @Override + public void execute() throws IOException { + delegate.deleteDocument(path); } } @@ -280,10 +294,16 @@ public void updateDocument(LuceneIndexWriter writer, String path, Iterable doc) } @Override - public void deleteDocuments(String path) throws IOException { + public void deleteDocumentTree(String path) throws IOException { Mount mount = mountInfoProvider.getMountByPath(path); - getWriter(mount).deleteDocuments(path); + getWriter(mount).deleteDocumentTree(path); //In case of default mount look for other mounts with roots under this path //Note that one mount cannot be part of another mount @@ -76,6 +76,13 @@ public void deleteDocuments(String path) throws IOException { } } + @Override + public void deleteDocument(String path) throws IOException { + // Single-document delete: no mount-under-path cleanup needed + Mount mount = mountInfoProvider.getMountByPath(path); + getWriter(mount).deleteDocument(path); + } + @Override public boolean close(long timestamp) throws IOException { // explicitly get writers for mounts which haven't got writers even at close. diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/PooledLuceneIndexWriter.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/PooledLuceneIndexWriter.java index 73e27e5ff62..208a7e2b112 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/PooledLuceneIndexWriter.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/PooledLuceneIndexWriter.java @@ -51,8 +51,14 @@ public void updateDocument(String path, Iterable doc) } @Override - public void deleteDocuments(String path) throws IOException { - writerPool.deleteDocuments(delegateWriter, path); + public void deleteDocumentTree(String path) throws IOException { + writerPool.deleteDocumentTree(delegateWriter, path); + deleteCount++; + } + + @Override + public void deleteDocument(String path) throws IOException { + writerPool.deleteDocument(delegateWriter, path); deleteCount++; } diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor2Test.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor2Test.java index a3cd595a2ef..8ccffdc9a10 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor2Test.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor2Test.java @@ -21,10 +21,13 @@ import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.IndexCommitCallback; import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; @@ -50,13 +53,29 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; +import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexEditor; +import org.junit.After; +import org.junit.Before; + public class LuceneIndexEditor2Test { + @Before + public void resetToggles() { + FulltextIndexEditor.FT_OAK_12244_DISABLE.set(false); + } + + @After + public void restoreToggles() { + FulltextIndexEditor.FT_OAK_12244_DISABLE.set(false); + } + private final NodeState root = INITIAL_CONTENT; private NodeState before = root; private final IndexUpdateCallback updateCallback = mock(IndexUpdateCallback.class); @@ -181,6 +200,114 @@ public void relativeProperties() throws Exception{ propCallback.reset(); } + @Test + public void nodeGainsMixinTriggersIndexUpdate() throws Exception { + LuceneIndexDefinitionBuilder defnb = new LuceneIndexDefinitionBuilder(); + defnb.indexRule("mix:title").property("jcr:title").propertyIndex(); + + NodeState defnState = defnb.build(); + IndexDefinition defn = new IndexDefinition(root, defnState, indexPath); + LuceneIndexEditorContext ctx = newContext(defnState.builder(), defn, true); + EditorHook hook = createHook(ctx); + + updateBefore(defnb); + + // Commit 1: node exists without the mixin — must not be indexed + NodeBuilder builder = before.builder(); + builder.child("a").setProperty("jcr:title", "hello"); + before = hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + assertFalse("Node without mixin should not be indexed", writer.docs.containsKey("/a")); + + // Commit 2: mixin added to existing node — must be indexed + builder = before.builder(); + builder.child("a").setProperty(JcrConstants.JCR_MIXINTYPES, List.of("mix:title"), Type.NAMES); + hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + assertTrue("Node after gaining mixin should be added to index", writer.docs.containsKey("/a")); + } + + @Test + public void nodeLosesMixinTriggersDocumentDeletion() throws Exception { + LuceneIndexDefinitionBuilder defnb = new LuceneIndexDefinitionBuilder(); + defnb.indexRule("mix:title").property("jcr:title").propertyIndex(); + + NodeState defnState = defnb.build(); + IndexDefinition defn = new IndexDefinition(root, defnState, indexPath); + LuceneIndexEditorContext ctx = newContext(defnState.builder(), defn, true); + EditorHook hook = createHook(ctx); + + updateBefore(defnb); + + // Commit 1: node with mixin — must be indexed + NodeBuilder builder = before.builder(); + builder.child("a") + .setProperty(JcrConstants.JCR_MIXINTYPES, List.of("mix:title"), Type.NAMES) + .setProperty("jcr:title", "hello"); + before = hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + assertTrue("Node with mixin should be indexed", writer.docs.containsKey("/a")); + + // Commit 2: mixin removed — existing index document must be deleted + builder = before.builder(); + builder.child("a").removeProperty(JcrConstants.JCR_MIXINTYPES); + hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + assertTrue("Removing mixin should trigger deleteDocument for the node", writer.deletedPaths.contains("/a")); + } + + @Test + public void nodeGainsMixinDoesNotTriggerIndexUpdateWhenToggleDisabled() throws Exception { + FulltextIndexEditor.FT_OAK_12244_DISABLE.set(true); + + LuceneIndexDefinitionBuilder defnb = new LuceneIndexDefinitionBuilder(); + defnb.indexRule("mix:title").property("jcr:title").propertyIndex(); + + NodeState defnState = defnb.build(); + IndexDefinition defn = new IndexDefinition(root, defnState, indexPath); + LuceneIndexEditorContext ctx = newContext(defnState.builder(), defn, true); + EditorHook hook = createHook(ctx); + + updateBefore(defnb); + + // Commit 1: node exists without the mixin + NodeBuilder builder = before.builder(); + builder.child("a").setProperty("jcr:title", "hello"); + before = hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + assertFalse("Node without mixin should not be indexed", writer.docs.containsKey("/a")); + + // Commit 2: mixin added — with toggle disabled, node must not be indexed + builder = before.builder(); + builder.child("a").setProperty(JcrConstants.JCR_MIXINTYPES, List.of("mix:title"), Type.NAMES); + hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + assertFalse("Mixin tracking disabled: node gaining mixin should not be indexed", writer.docs.containsKey("/a")); + } + + @Test + public void nodeLosesMixinDoesNotTriggerDocumentDeletionWhenToggleDisabled() throws Exception { + FulltextIndexEditor.FT_OAK_12244_DISABLE.set(true); + + LuceneIndexDefinitionBuilder defnb = new LuceneIndexDefinitionBuilder(); + defnb.indexRule("mix:title").property("jcr:title").propertyIndex(); + + NodeState defnState = defnb.build(); + IndexDefinition defn = new IndexDefinition(root, defnState, indexPath); + LuceneIndexEditorContext ctx = newContext(defnState.builder(), defn, true); + EditorHook hook = createHook(ctx); + + updateBefore(defnb); + + // Commit 1: node with mixin — indexed because it's a new node + NodeBuilder builder = before.builder(); + builder.child("a") + .setProperty(JcrConstants.JCR_MIXINTYPES, List.of("mix:title"), Type.NAMES) + .setProperty("jcr:title", "hello"); + before = hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + assertTrue("Node with mixin should be indexed", writer.docs.containsKey("/a")); + + // Commit 2: mixin removed — with toggle disabled, stale document must not be deleted + builder = before.builder(); + builder.child("a").removeProperty(JcrConstants.JCR_MIXINTYPES); + hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + assertFalse("Mixin tracking disabled: removing mixin should not trigger deleteDocument", writer.deletedPaths.contains("/a")); + } + private void updateBefore(LuceneIndexDefinitionBuilder defnb) { NodeBuilder builder = before.builder(); NodeBuilder cb = TestUtil.child(builder, PathUtils.getParentPath(indexPath)); @@ -290,7 +417,12 @@ public void updateDocument(String path, Iterable doc) } @Override - public void deleteDocuments(String path) { + public void deleteDocumentTree(String path) { + deletedPaths.add(path); + } + + @Override + public void deleteDocument(String path) { deletedPaths.add(path); } diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterPoolTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterPoolTest.java index 2d94694ae79..472d8d06cb7 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterPoolTest.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterPoolTest.java @@ -78,7 +78,13 @@ public void updateDocument(String path, Iterable doc) } @Override - public void deleteDocuments(String path) { + public void deleteDocumentTree(String path) { + delay(); + deletedPaths.add(path); + } + + @Override + public void deleteDocument(String path) { delay(); deletedPaths.add(path); } @@ -106,7 +112,7 @@ public void testSingleWriter() throws IOException { TestWriter writer = new TestWriter(); Document doc = TestUtil.newDoc("value"); indexWriterPool.updateDocument(writer, "test", doc); - indexWriterPool.deleteDocuments(writer, "test"); + indexWriterPool.deleteDocumentTree(writer, "test"); boolean closeResult = indexWriterPool.closeWriter(writer, 30); indexWriterPool.close(); @@ -163,7 +169,7 @@ public void testCloseWriterPoolWithoutClosingWriters() throws IOException { TestWriter writer = new TestWriter(100); Document doc = TestUtil.newDoc("value"); indexWriterPool.updateDocument(writer, "test", doc); - indexWriterPool.deleteDocuments(writer, "test-deletion"); + indexWriterPool.deleteDocumentTree(writer, "test-deletion"); indexWriterPool.close(); assertEquals(Map.of("test", doc), writer.docs); diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java index b3438faa8c0..436a382e684 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java @@ -209,14 +209,14 @@ public void deletes() throws Exception{ assertEquals(2, numDocs(defaultMount)); writer = factory.newInstance(defn, builder, null, true); - writer.deleteDocuments("/libs/config"); + writer.deleteDocumentTree("/libs/config"); writer.close(0); assertEquals(1, numDocs(fooMount)); assertEquals(2, numDocs(defaultMount)); writer = factory.newInstance(defn, builder, null, true); - writer.deleteDocuments("/content"); + writer.deleteDocumentTree("/content"); writer.close(0); assertEquals(1, numDocs(fooMount)); @@ -240,7 +240,7 @@ public void deleteIncludingMount() throws Exception{ assertEquals(2, numDocs(defaultMount)); writer = factory.newInstance(defn, builder, null, true); - writer.deleteDocuments("/content"); + writer.deleteDocumentTree("/content"); writer.close(0); assertEquals(0, numDocs(fooMount)); diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml index 96507f6217b..c266b80cf34 100644 --- a/oak-parent/pom.xml +++ b/oak-parent/pom.xml @@ -23,7 +23,7 @@ org.apache apache - 37 + 39 @@ -65,7 +65,7 @@ 1.28.5 10.16.1.1 2.19.4 - 1.21.4 + 2.0.3 4.14.0 2.6.17 3.0.25 @@ -664,12 +664,12 @@ commons-io commons-io - 2.20.0 + 2.21.0 commons-codec commons-codec - 1.19.0 + 1.20.0 org.apache.commons diff --git a/oak-run-commons/pom.xml b/oak-run-commons/pom.xml index 6b1fabfa514..812bb463950 100644 --- a/oak-run-commons/pom.xml +++ b/oak-run-commons/pom.xml @@ -212,13 +212,13 @@ org.testcontainers - toxiproxy + testcontainers-toxiproxy ${testcontainers.version} test org.testcontainers - mongodb + testcontainers-mongodb ${testcontainers.version} test diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoConnectionFailureIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoConnectionFailureIT.java index 8abd3a10148..0c7b1a8fa65 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoConnectionFailureIT.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoConnectionFailureIT.java @@ -35,6 +35,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.stats.Clock; import org.jetbrains.annotations.NotNull; +import org.junit.After; import org.junit.Assert; import org.junit.Assume; import org.junit.Before; @@ -90,17 +91,9 @@ public static Collection parameters() { final boolean parallelDump; final boolean testUpdateContent; - @Rule - public final Network network = Network.newNetwork(); - @Rule - public final MongoDBContainer mongoDBContainer = new MongoDBContainer(MongoDockerRule.getDockerImageName()) - .withNetwork(network) - .withNetworkAliases("mongo") - .withExposedPorts(MONGODB_DEFAULT_PORT); - @Rule - public final ToxiproxyContainer toxiproxy = new ToxiproxyContainer(TOXIPROXY_IMAGE) - .withStartupAttempts(3) - .withNetwork(network); + private Network network; + private MongoDBContainer mongoDBContainer; + private ToxiproxyContainer toxiproxy; @Rule public final DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); @Rule @@ -125,6 +118,16 @@ public static void setup() throws IOException { @Before public void before() throws Exception { + network = Network.newNetwork(); + mongoDBContainer = new MongoDBContainer(MongoDockerRule.getDockerImageName()) + .withNetwork(network) + .withNetworkAliases("mongo") + .withExposedPorts(MONGODB_DEFAULT_PORT); + mongoDBContainer.start(); + toxiproxy = new ToxiproxyContainer(TOXIPROXY_IMAGE) + .withStartupAttempts(3) + .withNetwork(network); + toxiproxy.start(); ToxiproxyClient toxiproxyClient = new ToxiproxyClient(toxiproxy.getHost(), toxiproxy.getControlPort()); // For the logic under test this.proxy = toxiproxyClient.createProxy("mongo", "0.0.0.0:8666", "mongo:" + MONGODB_DEFAULT_PORT); @@ -142,6 +145,19 @@ public void before() throws Exception { } } + @After + public void after() { + if (toxiproxy != null) { + toxiproxy.stop(); + } + if (mongoDBContainer != null) { + mongoDBContainer.stop(); + } + if (network != null) { + network.close(); + } + } + @Test public void mongoDisconnectTest() throws Exception { // Testcontainers' MongoDBContainer starts a Mongo replica set of one node. In this configuration we cannot diff --git a/oak-run-elastic/pom.xml b/oak-run-elastic/pom.xml index f5db86d1df6..c874828178e 100644 --- a/oak-run-elastic/pom.xml +++ b/oak-run-elastic/pom.xml @@ -256,7 +256,7 @@ org.testcontainers - elasticsearch + testcontainers-elasticsearch ${testcontainers.version} test diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IncrementalStoreTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IncrementalStoreTest.java index fce85289064..0048281d029 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IncrementalStoreTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IncrementalStoreTest.java @@ -23,6 +23,7 @@ import com.mongodb.ConnectionString; import com.mongodb.client.MongoDatabase; import org.apache.commons.collections4.set.ListOrderedSet; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.commons.Compression; @@ -71,7 +72,6 @@ import org.junit.Test; import org.junit.contrib.java.lang.system.RestoreSystemProperties; import org.junit.rules.TemporaryFolder; -import org.testcontainers.shaded.org.apache.commons.io.FileUtils; import java.io.BufferedReader; import java.io.File; diff --git a/oak-search-elastic/pom.xml b/oak-search-elastic/pom.xml index 9aa1ac10186..2bb76d77bed 100644 --- a/oak-search-elastic/pom.xml +++ b/oak-search-elastic/pom.xml @@ -278,13 +278,13 @@ org.testcontainers - elasticsearch + testcontainers-elasticsearch ${testcontainers.version} test org.testcontainers - toxiproxy + testcontainers-toxiproxy ${testcontainers.version} test diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexProviderService.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexProviderService.java index 89e64b2693e..5ecac5e4078 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexProviderService.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexProviderService.java @@ -63,6 +63,8 @@ import java.util.Hashtable; import java.util.List; +import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexEditor; + import static java.util.Collections.emptyMap; import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.registerMBean; @@ -228,6 +230,9 @@ private void activate(BundleContext bundleContext, Config config) { oakRegs.add(whiteboard.register(FeatureToggle.class, new FeatureToggle(ElasticConnection.FT_OAK_12234, ElasticConnection.FT_OAK_12234_DISABLE), emptyMap())); + oakRegs.add(whiteboard.register(FeatureToggle.class, + new FeatureToggle(FulltextIndexEditor.FT_OAK_12244, FulltextIndexEditor.FT_OAK_12244_DISABLE), + emptyMap())); oakRegs.add(whiteboard.register(FeatureToggle.class, new FeatureToggle(ElasticIndexStatistics.FT_OAK_12248, ElasticIndexStatistics.FT_OAK_12248_ENABLE), emptyMap())); diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java index 893dd792f7a..9e71388e428 100644 --- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java +++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriter.java @@ -166,7 +166,7 @@ public void updateDocument(String path, ElasticDocument doc) throws IOException } @Override - public void deleteDocuments(String path) throws IOException { + public void deleteDocumentTree(String path) throws IOException { retryPolicy.withRetries(() -> bulkProcessorHandler.delete(indexName, ElasticIndexUtils.idFromPath(path))); if (!ElasticIndexEditorProvider.FT_OAK_12206_DISABLE.get()) { // Delete all descendants: mirrors Lucene's PrefixQuery on the path term. @@ -187,6 +187,12 @@ public void deleteDocuments(String path) throws IOException { } } + @Override + public void deleteDocument(String path) throws IOException { + // Exact-document delete: no descendant sweep + retryPolicy.withRetries(() -> bulkProcessorHandler.delete(indexName, ElasticIndexUtils.idFromPath(path))); + } + @Override public boolean close(long timestamp) throws IOException { boolean updateStatus = bulkProcessorHandler.flushIndex(indexName); diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestServer.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestServer.java index cf694cb6739..81812e3933d 100644 --- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestServer.java +++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticTestServer.java @@ -25,13 +25,14 @@ import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.Network; import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.utility.MountableFile; import co.elastic.clients.transport.Version; import java.io.IOException; import java.time.Duration; +import java.util.concurrent.TimeUnit; public class ElasticTestServer implements AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(ElasticTestServer.class); @@ -39,6 +40,7 @@ public class ElasticTestServer implements AutoCloseable { private static final ElasticTestServer SERVER = new ElasticTestServer(); private static volatile ElasticsearchContainer CONTAINER; + private static volatile Network network; private ElasticTestServer() { } @@ -62,23 +64,26 @@ public static synchronized ElasticsearchContainer getESTestServer() { return CONTAINER; } - @SuppressWarnings("resource") private synchronized void setup() { String esDockerImageVersion = ELASTIC_DOCKER_IMAGE_VERSION != null ? ELASTIC_DOCKER_IMAGE_VERSION : Version.VERSION.toString(); LOG.info("Elasticsearch test Docker image version: {}.", esDockerImageVersion); checkIfDockerClientAvailable(); - Network network = Network.newNetwork(); + network = Network.newNetwork(); CONTAINER = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:" + esDockerImageVersion) .withEnv("ES_JAVA_OPTS", "-Xms1g -Xmx1g") - .withCopyFileToContainer( - MountableFile.forClasspathResource("elasticsearch.yml"), - "/usr/share/elasticsearch/config/elasticsearch.yml") + .withEnv("network.host", "0.0.0.0") + .withEnv("ingest.geoip.downloader.enabled", "false") + .withEnv("xpack.security.enabled", "false") + .withEnv("xpack.security.http.ssl.enabled", "false") + .withEnv("action.destructive_requires_name", "false") .withNetwork(network) .withNetworkAliases("elasticsearch") + .waitingFor(Wait.forHttp("/").forPort(9200).forStatusCode(200)) // Default is 30 seconds, which might not be enough on environments with limited resources or network latency .withStartupTimeout(Duration.ofMinutes(3)) .withStartupAttempts(3); CONTAINER.start(); + verifyConnectivity(); Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(LOG).withSeparateOutputStreams(); CONTAINER.followOutput(logConsumer); @@ -86,15 +91,14 @@ private synchronized void setup() { @Override public void close() { - if (this == SERVER) { - // Closed with a shutdown hook - return; - } - if (CONTAINER != null) { CONTAINER.stop(); } CONTAINER = null; + if (network != null) { + network.close(); + } + network = null; } private void checkIfDockerClientAvailable() { @@ -108,6 +112,31 @@ private void checkIfDockerClientAvailable() { assumeNotNull(client); } + private void verifyConnectivity() { + // Ensure the container is actually reachable before tests proceed. + try (ElasticConnection connection = ElasticConnection.newBuilder() + .withIndexPrefix("elastic_test_bootstrap") + .withConnectionParameters(ElasticConnection.DEFAULT_SCHEME, CONTAINER.getHost(), + CONTAINER.getMappedPort(ElasticConnection.DEFAULT_PORT)) + .build()) { + long deadline = System.nanoTime() + TimeUnit.MINUTES.toNanos(1); + while (System.nanoTime() < deadline) { + if (connection.isAvailable()) { + return; + } + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for Elasticsearch readiness", e); + } + } + throw new IllegalStateException("Elasticsearch test container started but did not become reachable via HTTP"); + } catch (IOException e) { + LOG.debug("Error closing bootstrap Elastic connection", e); + } + } + /** * Launches an Elasticsearch Test Server to re-use among several test executions. */ diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriterTest.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriterTest.java index e644f2a456f..ee6c9651fb9 100644 --- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriterTest.java +++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexWriterTest.java @@ -111,7 +111,7 @@ public void singleUpdateDocument() throws IOException { @Test public void singleDeleteDocument() throws IOException { - indexWriter.deleteDocuments("/bar"); + indexWriter.deleteDocumentTree("/bar"); ArgumentCaptor idCaptor = ArgumentCaptor.forClass(String.class); verify(bulkProcessorHandlerMock).delete(eq(indexAlias), idCaptor.capture()); @@ -127,8 +127,8 @@ public void singleDeleteDocument() throws IOException { public void multiRequests() throws IOException { indexWriter.updateDocument("/foo", new ElasticDocument("/foo")); indexWriter.updateDocument("/bar", new ElasticDocument("/bar")); - indexWriter.deleteDocuments("/foo"); - indexWriter.deleteDocuments("/bar"); + indexWriter.deleteDocumentTree("/foo"); + indexWriter.deleteDocumentTree("/bar"); verify(bulkProcessorHandlerMock, times(2)).index(eq(indexAlias), anyString(), any(ElasticDocument.class)); verify(bulkProcessorHandlerMock, times(2)).delete(eq(indexAlias), anyString()); @@ -182,7 +182,7 @@ public void splitLargeString() { @Test public void ft_oak_12206_toggleShouldBeRemoved() { // Time-bombed: if this test fails, the feature toggle FT_OAK-12206 and its guard in - // ElasticIndexWriter#deleteDocuments should be removed — the fix has been in production long enough. + // ElasticIndexWriter#deleteDocumentTree should be removed — the fix has been in production long enough. assertTrue("Feature toggle " + ElasticIndexEditorProvider.FT_OAK_12206 + " is overdue for removal", LocalDate.now().isBefore(LocalDate.of(2027, 5, 6))); } diff --git a/oak-search-elastic/src/test/resources/elasticsearch.yml b/oak-search-elastic/src/test/resources/elasticsearch.yml deleted file mode 100644 index c5b2dc75884..00000000000 --- a/oak-search-elastic/src/test/resources/elasticsearch.yml +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -network.host: 0.0.0.0 -ingest.geoip.downloader.enabled: false -xpack.security.enabled: false - -# In ES 8.0, by default it is no longer possible to use wildcards to delete several indexes in a single operation. -# This is used by the tests for cleanup, so we must set this to true explicitly. -# https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html -action.destructive_requires_name: false \ No newline at end of file diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java index da7a5d29a7c..13985c2f076 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditor.java @@ -35,10 +35,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.JcrConstants; + import java.io.IOException; import java.util.ArrayList; import java.util.BitSet; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -52,7 +55,7 @@ public class FulltextIndexEditor implements IndexEditor, Aggregate.AggregateR /** * Feature toggle name for OAK-12193. - * When this toggle is active (default), deleteDocuments calls are skipped for + * When this toggle is active (default), deleteDocumentTree calls are skipped for * removed subtrees that do not contain any node matching the index definition's * indexing rules. This avoids accumulating large numbers of buffered deletes in * the Lucene writer's DocumentsWriterDeleteQueue during delete-heavy async @@ -62,13 +65,30 @@ public class FulltextIndexEditor implements IndexEditor, Aggregate.AggregateR /** * Kill switch for the OAK-12193 filtered delete behavior. Set to {@code true} to - * revert to the legacy behavior of always issuing a deleteDocuments call for every + * revert to the legacy behavior of always issuing a deleteDocumentTree call for every * removed subtree regardless of whether the index could have indexed its nodes. * Default is {@code false} (filtered behavior active). Wired to the * {@link #FT_OAK_12193} feature toggle at runtime. */ public static final AtomicBoolean FT_OAK_12193_DISABLE = new AtomicBoolean(false); + /** + * Feature toggle name for OAK-12244. + * When active (default), nodes that gain or lose a mixin type at runtime are + * correctly added to or removed from the fulltext index, even when + * {@code jcr:mixinTypes} is not listed in the rule's property definitions. + */ + public static final String FT_OAK_12244 = "FT_OAK-12244"; + + /** + * Kill switch for the OAK-12244 mixin-transition tracking. Set to {@code true} to + * revert to the pre-OAK-12244 behavior where mixin additions and removals do not + * trigger index updates unless an indexed property also changed. + * Default is {@code false} (mixin-transition tracking active). Wired to the + * {@link #FT_OAK_12244} feature toggle at runtime. + */ + public static final AtomicBoolean FT_OAK_12244_DISABLE = new AtomicBoolean(false); + private static final List EMPTY_AGGREGATE_MATCHER_LIST = List.of(); private final FulltextIndexEditorContext context; @@ -81,6 +101,8 @@ public class FulltextIndexEditor implements IndexEditor, Aggregate.AggregateR private boolean propertiesChanged = false; + private boolean wasIndexable = false; + private final List propertiesModified = new ArrayList<>(); /* @@ -143,17 +165,55 @@ public void enter(NodeState before, NodeState after) { if (indexingRule != null) { currentMatchers = indexingRule.getAggregate().createMatchers(this); } + + if (context.isTypeChangeTrackingEnabled() && before.exists()) { + if (hasNodeTypeChange(before, after)) { + wasIndexable = getDefinition().getApplicableIndexingRule(before) != null; + } else { + // Types unchanged: before and after match the same rule + wasIndexable = indexingRule != null; + } + } } } @Override public void leave(NodeState before, NodeState after) throws CommitFailedException { - if (propertiesChanged || !before.exists()) { - if (addOrUpdate(path, after, before.exists())) { - long indexed = context.incIndexedNodes(); - if (indexed % 1000 == 0) { - log.debug("[{}] => Indexed {} nodes...", getIndexName(), indexed); + if (context.isTypeChangeTrackingEnabled()) { + // OAK-12244: act on rule-gained / rule-lost transitions + boolean toBeDeleted = wasIndexable && !isIndexable(); + boolean toBeAdded = !wasIndexable && isIndexable(); + boolean toBeUpdated = wasIndexable && isIndexable() && propertiesChanged; + + if (toBeDeleted) { + try { + // Exact-document delete: children that still carry the mixin must not be evicted + context.getWriter().deleteDocument(path); + context.indexUpdate(); + } catch (IOException e) { + CommitFailedException ce = new CommitFailedException("Fulltext", 5, + "Failed to delete stale index document for " + path + + " in index " + context.getIndexingContext().getIndexPath(), e); + context.getIndexingContext().indexUpdateFailed(ce); + throw ce; + } + } else if (toBeAdded || toBeUpdated) { + if (addOrUpdate(path, after, before.exists())) { + long indexed = context.incIndexedNodes(); + if (indexed % 1000 == 0) { + log.debug("[{}] => Indexed {} nodes...", getIndexName(), indexed); + } + } + } + } else { + // pre-OAK-12244: only property changes and new nodes trigger indexing + if (propertiesChanged || !before.exists()) { + if (addOrUpdate(path, after, before.exists())) { + long indexed = context.incIndexedNodes(); + if (indexed % 1000 == 0) { + log.debug("[{}] => Indexed {} nodes...", getIndexName(), indexed); + } } } } @@ -243,14 +303,14 @@ public Editor childNodeDeleted(String name, NodeState before) } if (!FT_OAK_12193_DISABLE.get()) { - // OAK-12193: skip the deleteDocuments call when no node in the removed subtree - // could have been indexed. Legacy behavior would route a deleteDocuments call + // OAK-12193: skip the deleteDocumentTree call when no node in the removed subtree + // could have been indexed. Legacy behavior would route a deleteDocumentTree call // for every removed subtree regardless, accumulating buffered deletes in the // Lucene writer during delete-heavy async cycles. if (!isDeleted && subtreeHasIndexableNode(context.getDefinition(), before)) { try { FulltextIndexWriter writer = context.getWriter(); - writer.deleteDocuments(childPath); + writer.deleteDocumentTree(childPath); this.context.indexUpdate(); } catch (IOException e) { CommitFailedException ce = new CommitFailedException("Fulltext", 5, "Failed to remove the index entries of" @@ -265,7 +325,7 @@ public Editor childNodeDeleted(String name, NodeState before) try { FulltextIndexWriter writer = context.getWriter(); // Remove all index entries in the removed subtree - writer.deleteDocuments(childPath); + writer.deleteDocumentTree(childPath); this.context.indexUpdate(); } catch (IOException e) { CommitFailedException ce = new CommitFailedException("Fulltext", 5, "Failed to remove the index entries of" @@ -461,6 +521,13 @@ private boolean isIndexable() { return indexingRule != null; } + private static boolean hasNodeTypeChange(NodeState before, NodeState after) { + return !Objects.equals(before.getProperty(JcrConstants.JCR_MIXINTYPES), + after.getProperty(JcrConstants.JCR_MIXINTYPES)) + || !Objects.equals(before.getProperty(JcrConstants.JCR_PRIMARYTYPE), + after.getProperty(JcrConstants.JCR_PRIMARYTYPE)); + } + private String getIndexName() { return context.getDefinition().getIndexName(); } diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java index b8771332286..38427b6bf83 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexEditorContext.java @@ -89,6 +89,8 @@ public abstract class FulltextIndexEditorContext { private final boolean indexDefnRewritten; + private final boolean typeChangeTrackingEnabled; + private FulltextBinaryTextExtractor textExtractor; private PropertyUpdateCallback propertyUpdateCallback; @@ -109,6 +111,7 @@ protected FulltextIndexEditorContext(NodeState root, NodeBuilder definition, this.updateCallback = updateCallback; this.extractedTextCache = extractedTextCache; this.asyncIndexing = asyncIndexing; + this.typeChangeTrackingEnabled = !FulltextIndexEditor.FT_OAK_12244_DISABLE.get(); if (this.definition.isOfOldFormat()) { indexDefnRewritten = true; IndexDefinition.updateDefinition(definition, indexingContext.getIndexPath()); @@ -121,6 +124,10 @@ protected FulltextIndexEditorContext(NodeState root, NodeBuilder definition, public abstract DocumentMaker newDocumentMaker(IndexDefinition.IndexingRule rule, String path); + public boolean isTypeChangeTrackingEnabled() { + return typeChangeTrackingEnabled; + } + protected FulltextBinaryTextExtractor createBinaryTextExtractor(ExtractedTextCache extractedTextCache, IndexDefinition definition, boolean reindex) { return new FulltextBinaryTextExtractor(extractedTextCache, definition, reindex); diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java index bc19febc66a..b936b7f48b6 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextIndexWriter.java @@ -36,11 +36,24 @@ public interface FulltextIndexWriter { void updateDocument(String path, D doc) throws IOException; /** - * Deletes documents which are same or child of given path + * Deletes the document at the given path and all descendant documents. + * Use this when a node is physically removed from the repository. * - * @param path path whose children need to be deleted + * @param path path of the node whose document and all descendants need to be deleted */ - void deleteDocuments(String path) throws IOException; + void deleteDocumentTree(String path) throws IOException; + + /** + * Deletes only the document at the given path, leaving descendant documents untouched. + * Use this when a node loses indexability at runtime (e.g. mixin removed) while its + * children may still be indexable. + * + *

Default implementation falls back to {@link #deleteDocumentTree} for implementations + * that do not differentiate; override in backends that support exact-document deletion. + * + * @param path path of the node whose document needs to be deleted + */ + void deleteDocument(String path) throws IOException; /** * Closes the underlying resources. diff --git a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/PropertyIndexCommonTest.java b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/PropertyIndexCommonTest.java index 1d8d24e4788..bb96e14f16c 100644 --- a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/PropertyIndexCommonTest.java +++ b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/PropertyIndexCommonTest.java @@ -520,6 +520,82 @@ public void indexingBasedOnMixinWithInheritance() throws Exception { }); } + @Test + public void nodeGainsMixinAppearsInMixinBasedIndex() throws Exception { + indexOptions.setIndex( + root, + "test1", + indexOptions.createIndex(indexOptions.createIndexDefinitionBuilder(), "mix:title", false, "jcr:title") + ); + root.commit(); + + Tree test = root.getTree("/").addChild("test"); + Tree a = test.addChild("a"); + a.setProperty("jcr:title", "hello"); + root.commit(); + + String query = "select [jcr:path] from [mix:title] where [jcr:title] = 'hello'"; + assertEventually(() -> assertQuery(query, List.of())); + + a = root.getTree("/test/a"); + a.setProperty(JcrConstants.JCR_MIXINTYPES, List.of("mix:title"), Type.NAMES); + root.commit(); + + assertEventually(() -> assertQuery(query, List.of("/test/a"))); + } + + @Test + public void nodeLosesMixinDisappearsFromMixinBasedIndex() throws Exception { + indexOptions.setIndex( + root, + "test1", + indexOptions.createIndex(indexOptions.createIndexDefinitionBuilder(), "mix:title", false, "jcr:title") + ); + root.commit(); + + Tree test = root.getTree("/").addChild("test"); + Tree a = createNodeWithMixinType(test, "a", "mix:title"); + a.setProperty("jcr:title", "hello"); + root.commit(); + + String query = "select [jcr:path] from [mix:title] where [jcr:title] = 'hello'"; + assertEventually(() -> assertQuery(query, List.of("/test/a"))); + + a = root.getTree("/test/a"); + a.removeProperty(JcrConstants.JCR_MIXINTYPES); + root.commit(); + + assertEventually(() -> assertQuery(query, List.of())); + } + + @Test + public void parentLosesMixinDoesNotCascadeDeleteChildWithSameMixin() throws Exception { + indexOptions.setIndex( + root, + "test1", + indexOptions.createIndex(indexOptions.createIndexDefinitionBuilder(), "mix:title", false, "jcr:title") + ); + root.commit(); + + Tree test = root.getTree("/").addChild("test"); + Tree a = createNodeWithMixinType(test, "a", "mix:title"); + a.setProperty("jcr:title", "parent"); + Tree child = createNodeWithMixinType(a, "child", "mix:title"); + child.setProperty("jcr:title", "childTitle"); + root.commit(); + + String childQuery = "select [jcr:path] from [mix:title] where [jcr:title] = 'childTitle'"; + assertEventually(() -> assertQuery(childQuery, List.of("/test/a/child"))); + + // Remove mixin from parent only — child still has mix:title + a = root.getTree("/test/a"); + a.removeProperty(JcrConstants.JCR_MIXINTYPES); + root.commit(); + + // Child must survive — deleteDocumentTree on the parent must not cascade to it + assertEventually(() -> assertQuery(childQuery, List.of("/test/a/child"))); + } + @Test public void indexingBasedOnMixinAndRelativeProps() throws Exception { indexOptions.setIndex( diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/persistentcache/PersistentDiskCache.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/persistentcache/PersistentDiskCache.java index 25687239380..239cc9148f9 100644 --- a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/persistentcache/PersistentDiskCache.java +++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/persistentcache/PersistentDiskCache.java @@ -111,7 +111,8 @@ public PersistentDiskCache(File directory, int cacheMaxSizeMB, DiskCacheIOMonito () -> maxCacheSizeBytes, () -> Long.valueOf(directory.listFiles().length), () -> FileUtils.sizeOfDirectory(directory), - () -> evictionCount.get()); + () -> evictionCount.get(), + () -> discardCount.get()); } @Override diff --git a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/persistentcache/PersistentRedisCache.java b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/persistentcache/PersistentRedisCache.java index 204484e23c1..754e657fc56 100644 --- a/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/persistentcache/PersistentRedisCache.java +++ b/oak-segment-remote/src/main/java/org/apache/jackrabbit/oak/segment/remote/persistentcache/PersistentRedisCache.java @@ -73,7 +73,7 @@ public PersistentRedisCache(String redisHost, int redisPort, int redisExpireSeco this.redisPool = new JedisPool(jedisPoolConfig, redisHost, redisPort, redisConnectionTimeout, redisSocketTimeout, null, redisDBIndex, null); this.segmentCacheStats = new SegmentCacheStats(NAME, this::getRedisMaxMemory, this::getCacheElementCount, - this::getCurrentWeight, this::getNumberOfEvictedKeys); + this::getCurrentWeight, this::getNumberOfEvictedKeys, () -> discardCount.get()); } private long getCacheElementCount() { diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/AbstractPersistentCache.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/AbstractPersistentCache.java index e60ae43867c..f1641ffb531 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/AbstractPersistentCache.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/AbstractPersistentCache.java @@ -22,6 +22,7 @@ import org.apache.jackrabbit.oak.cache.AbstractCacheStats; import org.apache.jackrabbit.oak.commons.Buffer; +import org.apache.jackrabbit.oak.commons.log.LogSilencer; import org.apache.jackrabbit.oak.commons.time.Stopwatch; import org.apache.jackrabbit.oak.segment.spi.RepositoryNotReachableException; import org.jetbrains.annotations.NotNull; @@ -31,27 +32,53 @@ import java.io.Closeable; import java.util.Set; import java.util.UUID; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; public abstract class AbstractPersistentCache implements PersistentCache, Closeable { private static final Logger logger = LoggerFactory.getLogger(AbstractPersistentCache.class); public static final int THREADS = Integer.getInteger("oak.segment.cache.threads", 10); + public static final int WRITE_QUEUE_SIZE = Integer.getInteger("oak.segment.cache.writeQueueSize", 100); protected ExecutorService executor; protected AtomicLong cacheSize = new AtomicLong(0); protected PersistentCache nextCache; protected final Set writesPending; + protected AtomicLong discardCount = new AtomicLong(); protected SegmentCacheStats segmentCacheStats; + private final LogSilencer writeQueueFullSilencer = new LogSilencer(); + public AbstractPersistentCache() { - executor = Executors.newFixedThreadPool(THREADS); + AtomicInteger threadCount = new AtomicInteger(0); + ThreadFactory threadFactory = r -> { + Thread t = new Thread(r, "segment-cache-writer-" + threadCount.incrementAndGet()); + t.setDaemon(true); + return t; + }; + BlockingQueue writeQueue = new LinkedBlockingQueue<>(WRITE_QUEUE_SIZE); + executor = new ThreadPoolExecutor( + THREADS, THREADS, + 0L, TimeUnit.MILLISECONDS, + writeQueue, + threadFactory, + (r, e) -> { + discardCount.incrementAndGet(); + if (!writeQueueFullSilencer.silence("writeQueueFull")) { + logger.warn("Segment write task discarded: write queue full (capacity={}, totalDiscarded={}){}", + WRITE_QUEUE_SIZE, discardCount.get(), LogSilencer.SILENCING_POSTFIX); + } + }); writesPending = ConcurrentHashMap.newKeySet(); } @@ -148,4 +175,8 @@ public void close() { public int getWritesPending() { return writesPending.size(); } + + public long getWriteDiscardCount() { + return discardCount.get(); + } } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/SegmentCacheStats.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/SegmentCacheStats.java index 541e31aa82c..a29148852ed 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/SegmentCacheStats.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/SegmentCacheStats.java @@ -54,16 +54,21 @@ public class SegmentCacheStats extends AbstractCacheStats { @NotNull final AtomicLong missCount = new AtomicLong(); + @NotNull + private final Supplier writeDiscardCountSupplier; + public SegmentCacheStats(@NotNull String name, @NotNull Supplier maximumWeight, @NotNull Supplier elementCount, @NotNull Supplier currentWeight, - @NotNull Supplier evictionCount) { + @NotNull Supplier evictionCount, + @NotNull Supplier writeDiscardCount) { super(name); this.maximumWeight = maximumWeight; this.elementCount = requireNonNull(elementCount); this.currentWeight = requireNonNull(currentWeight); this.evictionCount = evictionCount; + this.writeDiscardCountSupplier = requireNonNull(writeDiscardCount); } @Override @@ -78,6 +83,10 @@ protected CacheStats getCurrentStats() { ); } + public long getWriteDiscardCount() { + return writeDiscardCountSupplier.get(); + } + @Override public long getElementCount() { return elementCount.get(); diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/package-info.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/package-info.java index 0ad0171f1b9..96eb50916cf 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/package-info.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/package-info.java @@ -15,7 +15,7 @@ * limitations under the License. */ @Internal(since = "1.0.0") -@Version("6.1.0") +@Version("7.0.0") package org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache; import org.apache.jackrabbit.oak.commons.annotations.Internal; diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/MemoryPersistentCache.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/MemoryPersistentCache.java index 896a98c8e5b..db0135f476b 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/MemoryPersistentCache.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/MemoryPersistentCache.java @@ -38,7 +38,8 @@ public MemoryPersistentCache(boolean throwException) { () -> null, () -> null, () -> null, - () -> null); + () -> null, + () -> 0L); } @Override diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/PersistentCacheStatsTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/PersistentCacheStatsTest.java index df2013c26f1..7aa5bcedd9d 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/PersistentCacheStatsTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/spi/persistence/persistentcache/PersistentCacheStatsTest.java @@ -170,7 +170,7 @@ class PersistentCacheImpl extends AbstractPersistentCache { HashMap segments = new HashMap<>(); public PersistentCacheImpl() { - segmentCacheStats = new SegmentCacheStats("stats", () -> maximumWeight, () -> elementCount.get(), () -> currentWeight.get(), () -> evictionCount.get()); + segmentCacheStats = new SegmentCacheStats("stats", () -> maximumWeight, () -> elementCount.get(), () -> currentWeight.get(), () -> evictionCount.get(), () -> 0L); } long maximumWeight = Long.MAX_VALUE; @@ -179,7 +179,7 @@ public PersistentCacheImpl() { AtomicLong evictionCount = new AtomicLong(); void AbstractPersistentCache() { - segmentCacheStats = new SegmentCacheStats("stats", () -> maximumWeight, () -> elementCount.get(), () -> currentWeight.get(), () -> evictionCount.get()); + segmentCacheStats = new SegmentCacheStats("stats", () -> maximumWeight, () -> elementCount.get(), () -> currentWeight.get(), () -> evictionCount.get(), () -> 0L); } @Override diff --git a/oak-store-document/pom.xml b/oak-store-document/pom.xml index 951ed303be9..ba354966267 100644 --- a/oak-store-document/pom.xml +++ b/oak-store-document/pom.xml @@ -348,13 +348,13 @@ org.testcontainers - toxiproxy + testcontainers-toxiproxy ${testcontainers.version} test org.testcontainers - mongodb + testcontainers-mongodb ${testcontainers.version} test diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java index d73fd4db263..5ff80c26344 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java @@ -195,7 +195,7 @@ public class DocumentNodeStoreService { /** * Default interval for taking snapshots of locally tracked blob ids. */ - static final long DEFAULT_BLOB_SNAPSHOT_INTERVAL = 12 * 60 * 60; + static final long DEFAULT_BLOB_SNAPSHOT_INTERVAL = 0L; /** * Feature toggle name to enable prefetch operation in DocumentStore diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java index 078a4afacce..8619f3daff9 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java @@ -35,7 +35,6 @@ import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; import org.testcontainers.containers.MongoDBContainer; import org.testcontainers.containers.Network; @@ -59,19 +58,11 @@ public class LeaseUpdateSocketTimeoutIT { private static final int LEASE_SO_TIMEOUT = 50; - @Rule - public Network network = Network.newNetwork(); + private Network network; - @Rule - public MongoDBContainer mongoDBContainer = new MongoDBContainer(MongoDockerRule.getDockerImageName()) - .withNetwork(network) - .withNetworkAliases("mongo") - .withExposedPorts(MONGODB_DEFAULT_PORT); + private MongoDBContainer mongoDBContainer; - @Rule - public ToxiproxyContainer tp = new ToxiproxyContainer(TOXIPROXY_IMAGE) - .withStartupAttempts(3) - .withNetwork(network); + private ToxiproxyContainer tp; private Proxy proxy; @@ -89,6 +80,16 @@ public static void dockerAvailable() { @Before public void before() throws Exception { + network = Network.newNetwork(); + mongoDBContainer = new MongoDBContainer(MongoDockerRule.getDockerImageName()) + .withNetwork(network) + .withNetworkAliases("mongo") + .withExposedPorts(MONGODB_DEFAULT_PORT); + mongoDBContainer.start(); + tp = new ToxiproxyContainer(TOXIPROXY_IMAGE) + .withStartupAttempts(3) + .withNetwork(network); + tp.start(); clock = new Clock.Virtual(); clock.waitUntil(System.currentTimeMillis()); setClusterNodeInfoClock(clock); @@ -103,7 +104,18 @@ public void before() throws Exception { @After public void after() { - store.dispose(); + if (store != null) { + store.dispose(); + } + if (tp != null) { + tp.stop(); + } + if (mongoDBContainer != null) { + mongoDBContainer.stop(); + } + if (network != null) { + network.close(); + } TestUtils.resetClusterNodeInfoClockToDefault(); }