Skip to content

Commit b04d6b4

Browse files
leaves12138leaves12138
committed
[core] Unify manifest sidecar enablement after writer integration
1 parent 711c47d commit b04d6b4

7 files changed

Lines changed: 43 additions & 66 deletions

File tree

‎docs/docs/concepts/spec/manifest.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,8 @@ Java and PyPaimon writers generate sidecars after closing newly written manifest
7575
the `_EXTRA_FILES` reference only after the sidecar closes successfully. Sidecars follow their
7676
owning manifests through failed-commit cleanup, snapshot/tag/changelog retention and orphan
7777
cleanup. Scans use block coverage before applying normal entry filters and reconciling ADD/DELETE
78-
entries. The `manifest.sidecar.write` and `manifest.sidecar.read` table options control these
79-
paths independently; each inherits `manifest-sort.enabled` when unset.
78+
entries. The `manifest.sidecar.enabled` table option controls both paths and inherits
79+
`manifest-sort.enabled` when unset.
8080

8181
The low-level utility includes construction, validation, block selection and optional caching.
8282
`build` reads the completed physical manifest and returns sidecar bytes; it does not itself

‎paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ public ManifestSidecar.Selection selectBlocks(
381381
@Nullable RowRangeIndex query,
382382
@Nullable PartitionPredicate partitionFilter,
383383
@Nullable BucketFilter bucketFilter) {
384-
return !options.manifestSidecarReadEnabled()
384+
return !options.manifestSidecarEnabled()
385385
|| (query == null && partitionFilter == null && bucketFilter == null)
386386
? null
387387
: ManifestSidecar.read(

‎paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java‎

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1526,8 +1526,7 @@ void testDisabledSidecarsDoNotConstrainManifestTargetSize() {
15261526
for (String target : new String[] {"1 bytes", "1 gb"}) {
15271527
Options options = new Options();
15281528
options.set(CoreOptions.MANIFEST_SORT_ENABLED, true);
1529-
options.set(CoreOptions.MANIFEST_SIDECAR_READ, false);
1530-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, false);
1529+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, false);
15311530
options.setString(CoreOptions.MANIFEST_TARGET_FILE_SIZE.key(), target);
15321531
ManifestFile manifestFile =
15331532
createManifestFileFactory(
@@ -1549,8 +1548,7 @@ void testSidecarPayloadsFollowTableMetadata() throws Exception {
15491548
for (boolean dataEvolution : new boolean[] {false, true}) {
15501549
for (int bucket : new int[] {-1, 4}) {
15511550
Options options = new Options();
1552-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true);
1553-
options.set(CoreOptions.MANIFEST_SIDECAR_READ, true);
1551+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
15541552
options.set(CoreOptions.DATA_EVOLUTION_ENABLED, dataEvolution);
15551553
options.set(CoreOptions.BUCKET, bucket);
15561554
String tablePath =
@@ -1613,8 +1611,7 @@ void testSidecarPayloadsFollowTableMetadata() throws Exception {
16131611
void testSidecarCacheUsesExplicitPathsAndSharesTheManifestCache() throws Exception {
16141612
Options options = new Options();
16151613
options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
1616-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true);
1617-
options.set(CoreOptions.MANIFEST_SIDECAR_READ, true);
1614+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
16181615
RecordingFileIO io = new RecordingFileIO();
16191616
SegmentsCache<Path> cache =
16201617
new SegmentsCache<>(1024, MemorySize.ofMebiBytes(1), Long.MAX_VALUE, null, false);
@@ -1672,8 +1669,7 @@ void testSidecarCacheUsesExplicitPathsAndSharesTheManifestCache() throws Excepti
16721669
void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception {
16731670
Options options = new Options();
16741671
options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
1675-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true);
1676-
options.set(CoreOptions.MANIFEST_SIDECAR_READ, true);
1672+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
16771673
RecordingFileIO fileIO = new RecordingFileIO();
16781674
SegmentsCache<Path> cache =
16791675
new SegmentsCache<>(1024, MemorySize.ofMebiBytes(16), Long.MAX_VALUE);
@@ -1830,8 +1826,7 @@ void testReadsOnlySelectedBlocksAndPreservesPhysicalOrdinals() throws Exception
18301826
void testScannerPreservesDeletesAndColumnGroups() {
18311827
Options options = new Options();
18321828
options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
1833-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true);
1834-
options.set(CoreOptions.MANIFEST_SIDECAR_READ, true);
1829+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
18351830
RecordingFileIO fileIO = new RecordingFileIO();
18361831
ManifestFile.Factory factory =
18371832
createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO);
@@ -1898,7 +1893,7 @@ void testScannerPreservesDeletesAndColumnGroups() {
18981893
@Test
18991894
void testSidecarWriteFailureAbortsAllRollingOutputs() {
19001895
Options options = new Options();
1901-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true);
1896+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
19021897
AtomicInteger indexes = new AtomicInteger();
19031898
FileIO failing =
19041899
new LocalFileIO() {
@@ -1939,8 +1934,7 @@ public PositionOutputStream newOutputStream(Path path, boolean overwrite)
19391934
void testBucketOnlyPlanningAndRawRewriteUseNullableBucketPayload() throws Exception {
19401935
Options options = new Options();
19411936
options.set(CoreOptions.BUCKET, 4);
1942-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true);
1943-
options.set(CoreOptions.MANIFEST_SIDECAR_READ, true);
1937+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
19441938
RecordingFileIO io = new RecordingFileIO();
19451939
ManifestFile.Factory factory =
19461940
createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io);
@@ -1991,8 +1985,7 @@ void testBucketOnlyPlanningAndRawRewriteUseNullableBucketPayload() throws Except
19911985
@Test
19921986
void testPartitionOnlyPlanningUsesBlocksWithoutRowIds() {
19931987
Options options = new Options();
1994-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true);
1995-
options.set(CoreOptions.MANIFEST_SIDECAR_READ, true);
1988+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
19961989
RecordingFileIO io = new RecordingFileIO();
19971990
ManifestFile.Factory factory =
19981991
createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io);
@@ -2041,8 +2034,7 @@ void testPartitionOnlyPlanningUsesBlocksWithoutRowIds() {
20412034
void testUnknownRowIdKeepsPartitionIndexAndNoQueryDoesNotReadSidecar() {
20422035
Options options = new Options();
20432036
options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
2044-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true);
2045-
options.set(CoreOptions.MANIFEST_SIDECAR_READ, true);
2037+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
20462038
RecordingFileIO fileIO = new RecordingFileIO();
20472039
ManifestFile manifests =
20482040
createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO)
@@ -2064,8 +2056,7 @@ void testUnknownRowIdKeepsPartitionIndexAndNoQueryDoesNotReadSidecar() {
20642056
void testExplicitIndexReferenceAndNullDoesNotProbe() throws Exception {
20652057
Options options = new Options();
20662058
options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
2067-
options.set(CoreOptions.MANIFEST_SIDECAR_WRITE, true);
2068-
options.set(CoreOptions.MANIFEST_SIDECAR_READ, true);
2059+
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
20692060
RecordingFileIO io = new RecordingFileIO();
20702061
ManifestFile manifests =
20712062
createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io).create();
@@ -2149,6 +2140,7 @@ void testCommitCleanerDeletesExplicitIndexReferences() throws Exception {
21492140

21502141
/** Observes actual file access without adding counters to production readers. */
21512142
private static final class RecordingFileIO extends LocalFileIO {
2143+
21522144
private final List<Path> opened = Collections.synchronizedList(new ArrayList<>());
21532145
private final List<Long> seeks = Collections.synchronizedList(new ArrayList<>());
21542146
private final AtomicLong bytes = new AtomicLong();

‎paimon-python/pypaimon/common/options/core_options.py‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -301,18 +301,11 @@ class CoreOptions:
301301
.with_description("The parallelism for scanning manifest files.")
302302
)
303303

304-
MANIFEST_SIDECAR_WRITE: ConfigOption[bool] = (
305-
ConfigOptions.key("manifest.sidecar.write")
304+
MANIFEST_SIDECAR_ENABLED: ConfigOption[bool] = (
305+
ConfigOptions.key("manifest.sidecar.enabled")
306306
.boolean_type()
307307
.no_default_value()
308-
.with_description("Write manifest sidecars. Defaults to manifest-sort.enabled when unset.")
309-
)
310-
311-
MANIFEST_SIDECAR_READ: ConfigOption[bool] = (
312-
ConfigOptions.key("manifest.sidecar.read")
313-
.boolean_type()
314-
.no_default_value()
315-
.with_description("Read manifest sidecars. Defaults to manifest-sort.enabled when unset.")
308+
.with_description("Enable manifest sidecars. Defaults to manifest-sort.enabled when unset.")
316309
)
317310

318311
MANIFEST_SORT_ENABLED: ConfigOption[bool] = (
@@ -1277,12 +1270,8 @@ def manifest_target_size(self, default=None):
12771270
default = MemorySize.of_bytes(default) if isinstance(default, int) else MemorySize.parse(default)
12781271
return self.options.get(CoreOptions.MANIFEST_TARGET_FILE_SIZE, default).get_bytes()
12791272

1280-
def manifest_sidecar_write_enabled(self):
1281-
enabled = self.options.get(CoreOptions.MANIFEST_SIDECAR_WRITE)
1282-
return self.manifest_sort_enabled() if enabled is None else enabled
1283-
1284-
def manifest_sidecar_read_enabled(self):
1285-
enabled = self.options.get(CoreOptions.MANIFEST_SIDECAR_READ)
1273+
def manifest_sidecar_enabled(self):
1274+
enabled = self.options.get(CoreOptions.MANIFEST_SIDECAR_ENABLED)
12861275
return self.manifest_sort_enabled() if enabled is None else enabled
12871276

12881277
def manifest_sort_enabled(self):

‎paimon-python/pypaimon/manifest/manifest_file_manager.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -139,13 +139,13 @@ def read_entries_parallel(self, manifest_files: List[ManifestFileMeta], manifest
139139
) -> List[ManifestEntry]:
140140

141141
settings = self._sidecar_settings()
142-
query = Query(row_ranges) if settings.read and row_ranges is not None else None
142+
query = Query(row_ranges) if settings.enabled and row_ranges is not None else None
143143

144144
def _process_single_manifest(manifest_file: ManifestFileMeta):
145145
path = f"{self.manifest_path}/{manifest_file.file_name}"
146146
selected = None
147-
if settings.read and (query is not None or partition_filter is not None
148-
or early_entry_filter is not None):
147+
if settings.enabled and (
148+
query is not None or partition_filter is not None or early_entry_filter is not None):
149149
selected = read_sidecar(self.file_io, path, manifest_file, query,
150150
partition_filter, self.partition_keys_fields, early_entry_filter)
151151
if selected is not None and not selected.blocks:
@@ -451,7 +451,7 @@ def _flush(self, file_name: str, avro_bytes: bytes, entries) -> ManifestFileMeta
451451
with self.file_io.new_output_stream(manifest_path) as output_stream:
452452
output_stream.write(avro_bytes)
453453
settings = self._sidecar_settings()
454-
if settings.write:
454+
if settings.enabled:
455455
data = build_from_entries(avro_bytes, entries, settings)
456456
if data is not None:
457457
sidecar_file_name = file_name + SUFFIX

‎paimon-python/pypaimon/manifest/manifest_sidecar.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,13 @@
4444

4545
@dataclass(frozen=True)
4646
class Settings:
47-
write: bool = False
48-
read: bool = False
47+
enabled: bool = False
4948
row_id_enabled: bool = True
5049
bucket_enabled: bool = True
5150

5251
@classmethod
5352
def from_options(cls, options):
54-
return cls(options.manifest_sidecar_write_enabled(), options.manifest_sidecar_read_enabled(),
53+
return cls(options.manifest_sidecar_enabled(),
5554
options.data_evolution_enabled(), options.bucket() != -1)
5655

5756

0 commit comments

Comments
 (0)