Skip to content

Commit 76d952e

Browse files
committed
fix: reject Spark writes with per-partition buckets
1 parent 45d5844 commit 76d952e

5 files changed

Lines changed: 51 additions & 28 deletions

File tree

‎docs/docs/maintenance/rescale-bucket.md‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,8 @@ Please note that
7575
:::
7676

7777
:::warning
78-
Per-partition bucket counts are currently supported by the **Flink** engine only. The **Spark** writer
79-
still derives the bucket from the single table-level bucket count, so when partitions have different
80-
bucket counts (for example, after changing the table-level `bucket` while existing partitions keep
81-
their previous count), Spark may route rows to buckets that do not belong to the partition and corrupt
82-
the per-partition layout. Until Spark support is added, use Flink to write to tables that have
83-
per-partition bucket counts, or perform a full-table rescale so every partition shares the same bucket
84-
count before writing with Spark.
78+
Per-partition bucket counts are currently supported by the **Flink** engine only. Spark rejects writes
79+
to a table with `'bucket.per-partition-count-enabled' = 'true'`; use Flink to write such a table.
8580
:::
8681
- **Unpartitioned tables** require a full rescale before writing. If you change the bucket number and attempt
8782
to write without reorganizing the data first, a `RuntimeException` will be thrown:
@@ -190,4 +185,4 @@ and the job's latency keeps increasing. To improve the data freshness, users can
190185
DATE_FORMAT(gmt_create, 'yyyy-MM-dd') AS dt
191186
FROM raw_orders
192187
WHERE order_status = 'verified';
193-
```
188+
```

‎paimon-core/src/main/java/org/apache/paimon/manifest/PartitionEntry.java‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,21 @@ public PartitionEntry merge(PartitionEntry entry) {
8888

8989
// Use the totalBuckets from the most recently created file. This correctly handles
9090
// the case where a partition has been overwritten with a different bucket count: the
91-
// newer files carry the new totalBuckets, and their creation time is always later.
92-
// When timestamps are equal (e.g., two files written in the same millisecond with
93-
// different bucket counts), we take the larger totalBuckets value. This makes merge
94-
// commutative and associative — a.merge(b) == b.merge(a)
95-
int newTotalBuckets =
96-
newer.lastFileCreationTime == older.lastFileCreationTime
97-
? Math.max(newer.totalBuckets, older.totalBuckets)
98-
: newer.totalBuckets;
91+
// newer files carry the new totalBuckets. A creation timestamp does not provide an
92+
// ordering when the files were created in the same millisecond. In that case, choosing
93+
// either bucket count (for example, the larger one) can silently discard a rescale.
94+
if (newer.lastFileCreationTime == older.lastFileCreationTime
95+
&& newer.totalBuckets != older.totalBuckets) {
96+
throw new IllegalStateException(
97+
String.format(
98+
"Cannot determine the bucket layout for partition %s: files created at %s "
99+
+ "have conflicting bucket counts (%s and %s).",
100+
partition,
101+
newer.lastFileCreationTime,
102+
newer.totalBuckets,
103+
older.totalBuckets));
104+
}
105+
int newTotalBuckets = newer.totalBuckets;
99106

100107
return new PartitionEntry(
101108
partition,

‎paimon-core/src/test/java/org/apache/paimon/manifest/PartitionEntryTest.java‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.junit.jupiter.api.Test;
2424

2525
import static org.assertj.core.api.Assertions.assertThat;
26+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
2627

2728
/** Tests for {@link PartitionEntry#merge(PartitionEntry)}. */
2829
public class PartitionEntryTest {
@@ -103,21 +104,19 @@ public void testMergeWithDeleteEntryPreservesNewerTotalBuckets() {
103104
}
104105

105106
@Test
106-
public void testMergeWithEqualCreationTimeIsCommutative() {
107-
// When creation times are equal, merge must be commutative: a.merge(b) == b.merge(a).
108-
// The tie-break takes the larger totalBuckets so that the parallel, non-deterministic
109-
// aggregation in readPartitionEntries() always produces the same result regardless of
110-
// manifest processing order.
107+
public void testMergeWithEqualCreationTimeAndConflictingBucketsFails() {
108+
// Creation time alone cannot order layouts written in the same millisecond. Choosing one
109+
// of them (for example, the larger bucket count) can silently retain the old layout when
110+
// a partition is downscaled, so reject the ambiguous metadata instead.
111111
PartitionEntry a = entry(1, 2, 1000L);
112112
PartitionEntry b = entry(1, 4, 1000L);
113113

114-
PartitionEntry ab = a.merge(b);
115-
PartitionEntry ba = b.merge(a);
116-
117-
assertThat(ab.totalBuckets()).isEqualTo(4); // max(2, 4) = 4
118-
assertThat(ba.totalBuckets()).isEqualTo(4); // max(4, 2) = 4, commutative
119-
assertThat(ab.fileCount()).isEqualTo(2);
120-
assertThat(ba.fileCount()).isEqualTo(2);
114+
assertThatThrownBy(() -> a.merge(b))
115+
.isInstanceOf(IllegalStateException.class)
116+
.hasMessageContaining("Cannot determine the bucket layout");
117+
assertThatThrownBy(() -> b.merge(a))
118+
.isInstanceOf(IllegalStateException.class)
119+
.hasMessageContaining("Cannot determine the bucket layout");
121120
}
122121

123122
@Test

‎paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonSparkWriter.scala‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,12 @@ case class PaimonSparkWriter(
124124
}
125125

126126
def write(data: DataFrame): Seq[CommitMessage] = {
127+
if (coreOptions.bucketPerPartitionCountEnabled()) {
128+
throw new UnsupportedOperationException(
129+
"Spark does not support writing tables with per-partition bucket counts. " +
130+
"Use Flink to write this table or disable 'bucket.per-partition-count-enabled'.")
131+
}
132+
127133
val sparkSession = data.sparkSession
128134
val uriReaderFactory = uriReaderFactoryForBlobDescriptor
129135
import sparkSession.implicits._

‎paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSinkTest.scala‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,22 @@ class PaimonSinkTest extends PaimonSparkTestBase with StreamTest {
3636

3737
import testImplicits._
3838

39+
test("Paimon Sink: reject per-partition bucket counts") {
40+
spark.sql("""CREATE TABLE per_partition_buckets (id INT, pt STRING)
41+
|PARTITIONED BY (pt)
42+
|TBLPROPERTIES (
43+
| 'bucket' = '2',
44+
| 'bucket.per-partition-count-enabled' = 'true')
45+
|""".stripMargin)
46+
47+
val exception = intercept[UnsupportedOperationException] {
48+
spark.sql("INSERT INTO per_partition_buckets VALUES (1, 'p1')")
49+
}
50+
assert(
51+
exception.getMessage.contains(
52+
"Spark does not support writing tables with per-partition bucket counts"))
53+
}
54+
3955
test("Paimon Sink: forEachBatch") {
4056
failAfter(streamingTimeout) {
4157
withTempDir {

0 commit comments

Comments
 (0)