[python] Support newer PyArrow versions

.github/workflows/ci-python.yml

@@ -152,7 +152,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        pyarrow: ['22.0.0', '23.0.0']
+        pyarrow: ['20.0.0', '21.0.0', '22.0.0', '23.0.0']
     steps:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v5
@@ -162,7 +162,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install -e ./paimon-python --group './paimon-python/pyproject.toml:test'
-          # Exercise the compatibility path before widening the package's Arrow range.
+          # Exercise each supported PyArrow version.
           python -m pip install --no-deps 'pyarrow==${{ matrix.pyarrow }}'
       - name: Test S3 compatibility
         working-directory: paimon-python

docs/docs/pypaimon/catalogs.mdx

@@ -48,7 +48,7 @@ catalog_options = {
 catalog = CatalogFactory.create(catalog_options)
 ```
 
-For an S3 warehouse, pass S3 authentication options with the filesystem catalog options:
+For an S3 warehouse, install `pypaimon[s3]` and pass S3 authentication options with the filesystem catalog options:
 
 ```python
 from pypaimon import CatalogFactory

paimon-python/pypaimon/filesystem/pyarrow_file_io.py

@@ -663,7 +663,14 @@ def _get_s3_delete_client(self):
         if self._s3_delete_client is not None:
             return self._s3_delete_client
 
-        import boto3
+        try:
+            import boto3
+        except ModuleNotFoundError as error:
+            if error.name != "boto3":
+                raise
+            raise ImportError(
+                "S3/OSS batch deletion requires pypaimon[s3] or pypaimon[oss]."
+            ) from error
         from botocore.config import Config
 
         if self._is_oss:

paimon-python/pypaimon/tests/s3_delete_compatibility_test.py

@@ -24,6 +24,7 @@
 import hashlib
 import os
 import socketserver
+import sys
 import threading
 import unittest
 from http.server import BaseHTTPRequestHandler, HTTPServer
@@ -572,6 +573,13 @@ def test_pickle_discards_cached_delete_client(self):
             restored.__setstate__(state)
         self.assertIsNone(restored._s3_delete_client)
 
+    def test_missing_boto3_reports_optional_extra(self):
+        file_io = self._new_file_io()
+        file_io._s3_delete_client = None
+        with mock.patch.dict(sys.modules, {"boto3": None}):
+            with self.assertRaisesRegex(ImportError, r"pypaimon\[s3\]"):
+                file_io._get_s3_delete_client()
+
     def test_jindo_and_native_s3_do_not_use_delete_fallback(self):
         file_io = self._new_file_io()
         file_io._pyarrow_gte_22 = True

paimon-python/pypaimon/tests/table_update_test.py

@@ -37,6 +37,17 @@
 from pypaimon.write.write_builder import BatchWriteBuilder, StreamWriteBuilder
 
 
+@pytest.mark.parametrize("chunked", [False, True])
+def test_predicate_assignment_rejects_reordered_struct_fields(chunked):
+    values = pa.array([{'a': 1, 'b': 2}])
+    if chunked:
+        values = pa.chunked_array([values])
+    target = pa.struct([('b', pa.int32()), ('a', pa.int32())])
+
+    with pytest.raises(ValueError, match="struct fields must match in order"):
+        BatchTableUpdate._assignment_to_array(values, target, 1)
+
+
 # ======================================================================
 # Shared base for batch & stream table-update tests
 # ======================================================================

paimon-python/pypaimon/write/table_update.py

@@ -533,6 +533,9 @@ def _assignment_to_array(
                 f"{len(array)} != {row_count}."
             )
         if array.type != data_type:
+            if pa.types.is_struct(array.type) and pa.types.is_struct(data_type):
+                if array.type.names != data_type.names:
+                    raise ValueError("Assignment struct fields must match in order.")
             if isinstance(array, pa.ChunkedArray):
                 array = pa.chunked_array(
                     [chunk.cast(data_type) for chunk in array.chunks],

paimon-python/pyproject.toml

@@ -63,7 +63,7 @@ dependencies = [
     "polars>=1.32,<2; python_version >= '3.9'",
     "pyarrow>=6,<7; python_version < '3.7'",
     "pyarrow>=7,<13; python_version >= '3.7' and python_version < '3.8'",
-    "pyarrow>=16,!=19.0.0,<20; python_version >= '3.8'",
+    "pyarrow>=16,!=19.0.0,<24; python_version >= '3.8'",
     "pyroaring<=0.3.3; python_version < '3.7'",
     "pyroaring<=0.4.5; python_version == '3.7'",
     "pyroaring>=1.0.0; python_version >= '3.8'",
@@ -94,7 +94,6 @@ hdf5 = [
 rosbag = [
     "rosbags>=0.11.5,<0.12; python_version >= '3.10'",
 ]
-# datasets 4.1+ may require PyArrow 21+, outside the supported core range.
 # Pandas 2.2.2+ supports NumPy 2 selected by the media dependencies.
 lerobot = [
     "datasets>=4,<4.1; python_version >= '3.10'",
@@ -118,10 +117,14 @@ daft = [
     "daft>=0.7.6; python_version >= '3.10'",
 ]
 oss = [
+    "boto3>=1.34,<2; python_version >= '3.10'",
     "oss2>=2.18,<3",
     "ossfs>=2021.8; python_version < '3.8'",
     "ossfs>=2023; python_version >= '3.8'",
 ]
+s3 = [
+    "boto3>=1.34,<2; python_version >= '3.10'",
+]
 jindo = [
     "oss2>=2.18,<3",
     "pyjindosdk>=6.10.4",