diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java index 866c5ecbba6..9651f292188 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java @@ -39,7 +39,6 @@ import java.sql.Timestamp; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -154,6 +153,9 @@ public String apply(@Nullable String input) { private Clock clock; + @Nullable + private File lastGcSummaryFile; + /** * Creates an instance of MarkSweepGarbageCollector * @@ -331,6 +333,7 @@ public OperationsStatsMBean getConsistencyOperationStats() { */ protected void markAndSweep(boolean markOnly, boolean forceBlobRetrieve) throws Exception { statsCollector.start(); + lastGcSummaryFile = null; boolean threw = true; GarbageCollectorFileState fs = new GarbageCollectorFileState(root); Stopwatch sw = Stopwatch.createStarted(); @@ -545,6 +548,20 @@ protected long sweep(GarbageCollectorFileState fs, long markStart, boolean force if (checkConsistencyAfterGc) { BlobCollectionType.get(blobStore).checkConsistencyAfterGC(blobStore, fs, consistencyStats); } + + // Copy the gc summary file out of the gcworkdir so it outlives handleRemoves. + // The persistent copy is exposed via getLastGcSummaryFile() for callers that + // need to act on the sweep result after collectGarbage() returns. + File gcSummary = new File(root, fs.getGarbage().getName()); + if (fs.getGarbage().exists() && fs.getGarbage().length() > 0) { + try { + copyFile(fs.getGarbage(), gcSummary); + lastGcSummaryFile = gcSummary; + } catch (IOException e) { + LOG.warn("Could not persist gc summary to [{}]", gcSummary, e); + } + } + BlobCollectionType.get(blobStore).handleRemoves(blobStore, fs.getGarbage(), fs.getMarkedRefs()); if(count != deleted) { @@ -809,6 +826,16 @@ public void setClock(Clock clock) { this.clock = clock; } + /** + * Returns the gc summary file written during the most recent sweep, or {@code null} if no + * sweep has run yet or the last sweep deleted nothing. The file persists beyond the lifetime + * of the internal working directory and can be used by callers to act on the GC result. + */ + @Nullable + public File getLastGcSummaryFile() { + return lastGcSummaryFile; + } + /** * BlobIdRetriever class to retrieve all blob ids. */ @@ -1067,13 +1094,19 @@ long sweepInternal(GarbageCollectableBlobStore blobStore, List ids, LOG.trace("Blob ids to be deleted {}", ids); for (String id : ids) { try { - long deleted = blobStore.countDeleteChunks(new ArrayList<>(Arrays.asList(id)), maxModified); - if (deleted != 1) { - LOG.debug("Blob [{}] not deleted", id); - } else { + long deleted = blobStore.countDeleteChunk(id, maxModified); + if (deleted == 1) { exceptionQueue.add(id); totalDeleted += 1; + } else if (deleted < 0) { + LOG.info("Blob [{}] no longer exists in data store; removing from tracker state", id); + exceptionQueue.add(id); + } else { + LOG.debug("Blob [{}] not deleted", id); } + } catch (DataStoreException e) { + LOG.info("Blob [{}] not found in data store; removing from tracker state", id); + exceptionQueue.add(id); } catch (Exception e) { LOG.warn("Error occurred while deleting blob with id [{}]", id, e); } diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java index 6a44a2c855f..1913958b7b5 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java @@ -174,7 +174,7 @@ public DataRecord getRecordIfStored(DataIdentifier identifier) throws DataStoreE delegate.getRecordIfStored(identifier); long elapsed = System.nanoTime() - start; - stats.getRecordIfStoredCalled(elapsed, TimeUnit.NANOSECONDS, rec.getLength()); + stats.getRecordIfStoredCalled(elapsed, TimeUnit.NANOSECONDS, rec != null ? rec.getLength() : 0); stats.getRecordIfStoredCompleted(identifier.toString()); return rec; @@ -509,42 +509,54 @@ public boolean deleteChunks(List chunkIds, long maxLastModifiedTime) thr return (chunkIds.size() == countDeleteChunks(chunkIds, maxLastModifiedTime)); } + @Override + public long countDeleteChunk(String chunkId, long maxLastModifiedTime) throws Exception { + if (!(delegate instanceof MultiDataStoreAware)) { + return 0; + } + long start = System.nanoTime(); + String blobId = extractBlobId(chunkId); + DataIdentifier identifier = new DataIdentifier(blobId); + try { + DataRecord dataRecord = getRecordIfStored(identifier); + if (dataRecord == null) { + log.info("Blob [{}] not found in data store", blobId); + return -1; + } + boolean eligible = (maxLastModifiedTime <= 0) || dataRecord.getLastModified() <= maxLastModifiedTime; + log.trace("Deleting blob [{}] with last modified date [{}] : [{}]", blobId, + dataRecord.getLastModified(), eligible); + if (eligible) { + ((MultiDataStoreAware) delegate).deleteRecord(identifier); + } + stats.deleted(blobId, System.nanoTime() - start, TimeUnit.NANOSECONDS); + stats.deleteCompleted(blobId); + return eligible ? 1 : 0; + } catch (Exception e) { + stats.deleteFailed(); + throw e; + } + } + @Override public long countDeleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception { - int count = 0; + long count = 0; if (delegate instanceof MultiDataStoreAware) { - try { - List deleted = new ArrayList<>(512); - for (String chunkId : chunkIds) { - long start = System.nanoTime(); - - String blobId = extractBlobId(chunkId); - DataIdentifier identifier = new DataIdentifier(blobId); - DataRecord dataRecord = getRecordForId(identifier); - boolean success = (maxLastModifiedTime <= 0) - || dataRecord.getLastModified() <= maxLastModifiedTime; - log.trace("Deleting blob [{}] with last modified date [{}] : [{}]", blobId, - dataRecord.getLastModified(), success); - if (success) { - ((MultiDataStoreAware) delegate).deleteRecord(identifier); - deleted.add(blobId); - count++; - if (count % 512 == 0) { - log.info("Deleted blobs {}", deleted); - deleted.clear(); - } + List deleted = new ArrayList<>(512); + for (String chunkId : chunkIds) { + long result = countDeleteChunk(chunkId, maxLastModifiedTime); + if (result == 1) { + deleted.add(extractBlobId(chunkId)); + count++; + if (count % 512 == 0) { + log.info("Deleted blobs {}", deleted); + deleted.clear(); } - - stats.deleted(blobId, System.nanoTime() - start, TimeUnit.NANOSECONDS); - stats.deleteCompleted(blobId); - } - if (!deleted.isEmpty()) { - log.info("Deleted blobs {}", deleted); } + // result == -1 means ghost blob — skip silently; caller cleans tracker state } - catch (Exception e) { - stats.deleteFailed(); - throw e; + if (!deleted.isEmpty()) { + log.info("Deleted blobs {}", deleted); } } return count; diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java index a560b5e1452..92d0114c5e7 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java @@ -45,7 +45,10 @@ import static org.apache.jackrabbit.oak.plugins.blob.SharedDataStore.Type.SHARED; import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -139,6 +142,27 @@ private void setupSharedDataRecords(final String refRepoId, final String repoRep when(blobStore.getAllMetadataRecords(SharedDataStoreUtils.SharedStoreRecordType.MARKED_START_MARKER.getType())).thenReturn(refs); } + @Test + public void sweepShouldNotFailWhenGhostBlobCountDeleteChunkReturnsMinusOne() throws Exception { + setupSharedDataRecords("REPO1", "REPO1"); + when(blobStore.getAllChunkIds(0L)).thenReturn(List.of("ghost-blob-id").iterator()); + when(blobStore.countDeleteChunk(anyString(), anyLong())).thenReturn(-1L); + collector.markAndSweep(false, true); + assertEquals(0L, collector.getOperationStats().numDeleted()); + assertEquals(0L, collector.getOperationStats().getFailureCount()); + } + + @Test + public void sweepShouldNotFailWhenGhostBlobCountDeleteChunkThrowsDataStoreException() throws Exception { + setupSharedDataRecords("REPO1", "REPO1"); + when(blobStore.getAllChunkIds(0L)).thenReturn(List.of("ghost-blob-id").iterator()); + when(blobStore.countDeleteChunk(anyString(), anyLong())) + .thenThrow(new DataStoreException("Record does not exist")); + collector.markAndSweep(false, true); + assertEquals(0L, collector.getOperationStats().numDeleted()); + assertEquals(0L, collector.getOperationStats().getFailureCount()); + } + private interface MockGarbageCollectableSharedDataStore extends GarbageCollectableBlobStore, SharedDataStore { } } diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java index 94fb6b77df9..50ef250c03c 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java @@ -54,6 +54,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class DataStoreBlobStoreTest extends AbstractBlobStoreTest { @@ -245,6 +247,35 @@ public void testEmptyIdentifier() throws Exception { public void testGarbageCollection() throws Exception { } + @Test + public void testCountDeleteChunkReturnsMinusOneForGhostBlob() throws Exception { + DataIdentifier ghostId = new DataIdentifier("ghostblobid"); + + OakFileDataStore mockedDS = mock(OakFileDataStore.class); + when(mockedDS.getMinRecordLength()).thenReturn(0); + when(mockedDS.getRecordIfStored(ghostId)).thenReturn(null); + try (DataStoreBlobStore ds = new DataStoreBlobStore(mockedDS)) { + long result = ds.countDeleteChunk("ghostblobid", 0L); + + assertEquals(-1, result); + // deleteRecord must NOT have been called for a ghost blob + verify(mockedDS, never()).deleteRecord(ghostId); + } + } + + + @Test + public void testGetRecordIfStoredReturnsNull() throws DataStoreException { + DataIdentifier missingId = new DataIdentifier("missingblob"); + + DataStore mockedDS = mock(DataStore.class); + when(mockedDS.getMinRecordLength()).thenReturn(0); + when(mockedDS.getRecordIfStored(missingId)).thenReturn(null); + try (DataStoreBlobStore ds = new DataStoreBlobStore(mockedDS)) { + assertNull(ds.getRecordIfStored(missingId)); + } + } + @After @Override public void tearDown() throws Exception { diff --git a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java index ce7c0555f7c..ac978ffd27a 100644 --- a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java +++ b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java @@ -110,6 +110,32 @@ public interface GarbageCollectableBlobStore extends BlobStore { */ long countDeleteChunks(List chunkIds, long maxLastModifiedTime) throws Exception; + /** + * Deletes a single blob with the given id. + * + *

Return values: + *

    + *
  • {@code 1} — blob was found and successfully deleted
    • + *
  • {@code 0} — blob was found but skipped (too recent per {@code maxLastModifiedTime})
    • + *
  • {@code -1} — blob was not found in the data store (ghost blob — e.g. already + * removed by offline GC); the caller should remove the id from any tracker state to + * avoid repeated warnings on future GC runs
    • + *
+ * + *

The default implementation delegates to {@link #countDeleteChunks(List, long)} with a + * single-element list. Implementations that can distinguish ghost blobs from ordinary + * deletion failures (e.g. {@code DataStoreBlobStore}) should override this method. + * + * @param chunkId the chunk id to delete + * @param maxLastModifiedTime the max last modified time to consider for retrieval, + * with the special value '0' meaning no filtering by time + * @return the deletion outcome: {@code 1} deleted, {@code 0} skipped, {@code -1} ghost blob + * @throws Exception the exception + */ + default long countDeleteChunk(String chunkId, long maxLastModifiedTime) throws Exception { + return countDeleteChunks(List.of(chunkId), maxLastModifiedTime); + } + /** * Resolve chunks stored in the blob store from the given Id. * This will not return any chunks stored in-line in the id. diff --git a/oak-run/README.md b/oak-run/README.md index 00ec262fcbf..2c87c056146 100644 --- a/oak-run/README.md +++ b/oak-run/README.md @@ -635,6 +635,7 @@ Maintenance commands for the DataStore: [--out-dir ] \ [--work-dir ] \ [--max-age ] \ + [--repository-home ] \ [--verbose] \ [--verboseRootPath] \ [--useDirListing] \ @@ -683,7 +684,16 @@ The following options are available: --sweep-only-refs-past-retention - Sweep only if the earliest references from all repositories are past the retention period which is govered by the max-age parameter. Boolean (Optional). Defaults to False. Only applicable for --collect-garbage --check-consistency-gc - Performs a consistency check immediately after the GC. - Boolean (Optional). Defaults to False. Only applicable for --collect-garbage + Boolean (Optional). Defaults to False. Only applicable for --collect-garbage + --repository-home - Repository home of the running Oak instance, i.e. the parent of the + 'blobids' BlobIdTracker directory (the tracker lives at /blobids). + Optional: when omitted it is derived from the segment store path (its parent), + which matches a standard deployment. After --collect-garbage (non mark-only) the + BlobIdTracker is reconciled automatically: blob IDs deleted by this run are removed + from the local tracker files and the DataStore snapshot, so online GC no longer + warns about (or fails to clean up) already-deleted blobs on subsequent runs. + Only applicable to a segment NodeStore backed by a FileDataStore (SharedDataStore); + pass this explicitly for a document NodeStore or a non-standard layout. Note: Note: When using --export-metrics the following additional jars have to be downloaded to support Prometheus Pushgatway diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java index 24f6547a564..18f9755d2fe 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCommand.java @@ -44,6 +44,9 @@ import org.apache.commons.io.LineIterator; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.jackrabbit.oak.spi.blob.data.DataRecord; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker; +import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; @@ -72,6 +75,7 @@ import org.apache.jackrabbit.oak.segment.SegmentBlobReferenceRetriever; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; +import org.apache.jackrabbit.oak.spi.blob.data.DataStore; import org.apache.jackrabbit.oak.spi.cluster.ClusterRepositoryInfo; import org.apache.jackrabbit.oak.spi.state.AbstractChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; @@ -267,6 +271,9 @@ private void execute(NodeStoreFixture fixture, DataStoreOptions dataStoreOpts, O } } else if (dataStoreOpts.collectGarbage()) { collector.collectGarbage(dataStoreOpts.markOnly()); + if (!dataStoreOpts.markOnly()) { + reconcileTrackerState(fixture, dataStoreOpts, opts, collector.getLastGcSummaryFile()); + } } } } @@ -360,6 +367,95 @@ private static MarkSweepGarbageCollector getCollector(NodeStoreFixture fixture, return collector; } + /** + * After an offline GC run, reconciles the BlobIdTracker state used by online GC so that the blob IDs + * deleted by this run are removed from the local tracker files and the DataStore snapshot. This prevents + * online GC from repeatedly warning about (and failing to clean up) blobs that no longer exist in the + * data store. + * + *

The tracker is rooted at {@code /blobids} (see {@link BlobIdTracker#build}). The + * repository home is taken from {@code --repository-home} when supplied, otherwise it is derived from the + * segment store path (its parent). Only applicable to a segment NodeStore backed by a FileDataStore + * (SharedDataStore) that maintains a BlobIdTracker; other configurations are skipped gracefully. + */ + private static void reconcileTrackerState(NodeStoreFixture fixture, DataStoreOptions dataStoreOpts, + Options opts, @Nullable File gcFile) { + if (gcFile == null) { + log.info("No blobs deleted by this GC run; nothing to reconcile in tracker state"); + return; + } + + BlobStore blobStore = fixture.getBlobStore(); + if (!(blobStore instanceof DataStoreBlobStore)) { + log.debug("Skipping tracker reconciliation: blob store is not a DataStoreBlobStore (got {})", + blobStore == null ? "null" : blobStore.getClass().getName()); + return; + } + DataStore dataStore = ((DataStoreBlobStore) blobStore).getDataStore(); + if (!(dataStore instanceof SharedDataStore)) { + log.debug("Skipping tracker reconciliation: underlying DataStore does not implement SharedDataStore (got {})", + dataStore == null ? "null" : dataStore.getClass().getName()); + return; + } + + File repositoryHome = resolveRepositoryHome(dataStoreOpts, opts); + if (repositoryHome == null) { + return; + } + // BlobIdTracker.build() appends the "blobids" sub-directory to the repository home; only reconcile when + // that directory actually exists, otherwise we would silently create an empty, unrelated tracker. + File trackerDir = new File(repositoryHome, "blobids"); + if (!trackerDir.isDirectory()) { + log.warn("Skipping tracker reconciliation: no BlobIdTracker directory at [{}]. " + + "Pass --repository-home if the repository home is not the parent of the segment store.", trackerDir); + return; + } + + String repositoryId = ClusterRepositoryInfo.getId(fixture.getStore()); + if (repositoryId == null) { + log.warn("Cannot reconcile tracker state: repository id is null"); + return; + } + + log.info("Reconciling BlobIdTracker state at [{}] using gc file [{}]", trackerDir, gcFile); + try (BlobIdTracker tracker = BlobIdTracker.build(repositoryHome.getAbsolutePath(), repositoryId, 1, + (SharedDataStore) dataStore); + LineIterator lines = FileUtils.lineIterator(gcFile, StandardCharsets.UTF_8.name())) { + tracker.remove(lines); + log.info("BlobIdTracker state reconciled successfully; {} removed from tracker", + gcFile.getName()); + } catch (IOException e) { + log.error("Failed to reconcile BlobIdTracker state", e); + } + } + + /** + * Resolves the repository home that hosts the BlobIdTracker ({@code /blobids}). Uses the + * {@code --repository-home} value when supplied, otherwise derives it from the segment store path (its + * parent). Returns {@code null} (with a warning) when it cannot be determined, e.g. for a document + * NodeStore where the repository home is not part of the connection string. + */ + @Nullable + private static File resolveRepositoryHome(DataStoreOptions dataStoreOpts, Options opts) { + File override = dataStoreOpts.getRepositoryHome(); + if (override != null) { + return override; + } + CommonOptions common = opts.getCommonOpts(); + if (common.isDocument()) { + log.warn("Cannot derive repository home for a document NodeStore; " + + "pass --repository-home to reconcile the BlobIdTracker after offline GC"); + return null; + } + File parent = new File(common.getStoreArg()).getAbsoluteFile().getParentFile(); + if (parent == null) { + log.warn("Cannot derive repository home from store path [{}]; pass --repository-home", + common.getStoreArg()); + return null; + } + return parent; + } + private static BlobReferenceRetriever getRetriever(NodeStoreFixture fixture, DataStoreOptions dataStoreOpts, Options opts) { BlobReferenceRetriever retriever; diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java index 2c4d7554490..5867c41d9df 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreOptions.java @@ -30,6 +30,7 @@ import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.run.cli.OptionsBean; import org.apache.jackrabbit.oak.run.cli.OptionsBeanFactory; +import org.jetbrains.annotations.Nullable; public class DataStoreOptions implements OptionsBean { @@ -56,6 +57,7 @@ public class DataStoreOptions implements OptionsBean { private OptionSpec exportMetrics; private static final String DELIM = ","; private OptionSpec sweepIfRefsPastRetention; + private final OptionSpec repositoryHomeOpt; public DataStoreOptions(OptionParser parser) { collectGarbage = parser.accepts("collect-garbage", @@ -116,6 +118,14 @@ public DataStoreOptions(OptionParser parser) { exportMetrics = parser.accepts("export-metrics", "type, URI to export the metrics and optional metadata all delimeted by semi-colon(;)").withRequiredArg(); + repositoryHomeOpt = parser.accepts("repository-home", + "Repository home of the running Oak instance, i.e. the parent of the 'blobids' BlobIdTracker directory " + + "(e.g. where the tracker lives at /blobids). Optional: when omitted " + + "it is derived from the segment store path (its parent). After --collect-garbage the BlobIdTracker is " + + "reconciled so online GC does not warn about blobs already removed by this run.") + .availableIf(collectGarbage) + .withRequiredArg().ofType(File.class); + //Set of options which define action actionOpts = Set.of(collectGarbage, consistencyCheck, idOp, refOp, metadataOp); operationNames = collectionOperationNames(actionOpts); @@ -252,4 +262,9 @@ public boolean isUseDirListing() { public boolean sweepIfRefsPastRetention() { return options.has(sweepIfRefsPastRetention) && sweepIfRefsPastRetention.value(options) ; } + + @Nullable + public File getRepositoryHome() { + return options.has(repositoryHomeOpt) ? repositoryHomeOpt.value(options) : null; + } } diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java index 9d29d893224..3214c82cb0d 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java @@ -26,6 +26,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Date; import java.util.Dictionary; import java.util.Enumeration; @@ -43,10 +44,12 @@ import java.util.stream.StreamSupport; import ch.qos.logback.classic.Level; +import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.lang3.StringUtils; import joptsimple.OptionException; import org.apache.commons.io.FileUtils; import org.apache.felix.cm.file.ConfigurationHandler; +import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore; import org.apache.jackrabbit.oak.spi.blob.data.DataStore; import org.apache.jackrabbit.oak.spi.blob.data.DataStoreException; import org.apache.jackrabbit.oak.api.Blob; @@ -59,8 +62,9 @@ import org.apache.jackrabbit.oak.commons.collections.IteratorUtils; import org.apache.jackrabbit.oak.commons.collections.SetUtils; import org.apache.jackrabbit.oak.commons.junit.LogCustomizer; -import org.apache.jackrabbit.oak.plugins.blob.MemoryBlobStoreNodeStore; import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector; +import org.apache.jackrabbit.oak.plugins.blob.MemoryBlobStoreNodeStore; +import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker; import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore; import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore; import org.apache.jackrabbit.oak.plugins.document.DocumentMK; @@ -609,6 +613,72 @@ public void gc() throws Exception { testGc(dump, data, 0, false, false); } + + @Test + public void gcReconcilesTrackerStateUsingDerivedRepositoryHome() throws Exception { + Assume.assumeTrue(storeFixture instanceof StoreFixture.SegmentStoreFixture); + Assume.assumeTrue(blobFixture.getType() == Type.FDS); + + File dump = temporaryFolder.newFolder(); + + // The BlobIdTracker of a live instance lives at /blobids, where the repository + // home is the parent of the segment store. The reconcile derives this from the store path, so the + // tracker dir must sit next to the segment store - no --blobids-path/--repository-home needed. + File repositoryHome = new File(storeFixture.getConnectionString()).getAbsoluteFile().getParentFile(); + + // Prepare: 5 blobs added, 3 deleted from nodestore (so 3 are GC candidates), 0 missing from DS + Data data = prepareData(storeFixture, blobFixture, 5, 3, 0); + // Read repoId before closing the store — ClusterRepositoryInfo reads from the segment store + String repoId = org.apache.jackrabbit.oak.spi.cluster.ClusterRepositoryInfo.getId(store); + storeFixture.close(); + + // Pre-populate the tracker with all 5 blob IDs to simulate the state of a live oak + // instance whose tracker still has the IDs that will be deleted by the offline GC run. + SharedDataStore sharedDS = + (SharedDataStore) setupDataStore.getDataStore(); + try (BlobIdTracker preTracker = BlobIdTracker.build(repositoryHome.getAbsolutePath(), repoId, 1, sharedDS)) { + preTracker.add(data.added.iterator()); + } + + // Run offline GC; reconcileTrackerState runs automatically after a (non mark-only) collect-garbage, + // deriving the repository home from the segment store path. + List argsList = new ArrayList<>(Arrays.asList( + "--collect-garbage", "false", + "--max-age", "0", + "--" + getOption(blobFixture.getType()), blobFixture.getConfigPath(), + storeFixture.getConnectionString(), + "--out-dir", dump.getAbsolutePath(), + "--work-dir", temporaryFolder.newFolder().getAbsolutePath())); + if (!StringUtils.isEmpty(additionalParams)) { + argsList.addAll(Arrays.stream(additionalParams.split(" ")).toList()); + } + DataStoreCommand cmd = new DataStoreCommand(); + cmd.execute(argsList.toArray(new String[0])); + + // Assert: gc summary file was created in outDir + Collection gcFiles = org.apache.commons.io.FileUtils.listFiles( + dump, + FileFilterUtils.prefixFileFilter("gc-"), + FileFilterUtils.trueFileFilter()); + assertFalse("A gc-* summary file must exist in outDir after GC", gcFiles.isEmpty()); + + // Assert: the reconcile targeted the live tracker dir, not a nested /blobids/blobids + assertFalse("Reconcile must not create a nested blobids/blobids directory", + new File(repositoryHome, "blobids/blobids").exists()); + + // Assert: deleted blob IDs are no longer in the tracker state + BlobIdTracker postTracker = BlobIdTracker.build(repositoryHome.getAbsolutePath(), repoId, 1, sharedDS); + Set remainingIds; + try { + remainingIds = SetUtils.toSet(postTracker.get()); + } finally { + postTracker.close(); + } + for (String deletedId : data.deleted) { + assertFalse("Deleted blob [" + deletedId + "] must have been removed from tracker state", + remainingIds.contains(deletedId)); + } + } /* Command should throw and exception if --verboseRootPath specified with --collect-garbage