diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index 95962575454a..ac18d0ce8b09 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -152,7 +152,7 @@ jobs: strategy: fail-fast: false matrix: - pyarrow: ['22.0.0', '23.0.0'] + pyarrow: ['20.0.0', '21.0.0', '22.0.0', '23.0.0'] steps: - uses: actions/checkout@v6 - uses: actions/setup-python@v5 @@ -162,7 +162,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -e ./paimon-python --group './paimon-python/pyproject.toml:test' - # Exercise the compatibility path before widening the package's Arrow range. + # Exercise each supported PyArrow version. python -m pip install --no-deps 'pyarrow==${{ matrix.pyarrow }}' - name: Test S3 compatibility working-directory: paimon-python diff --git a/docs/docs/pypaimon/catalogs.mdx b/docs/docs/pypaimon/catalogs.mdx index e5354c76968b..47f491a06f17 100644 --- a/docs/docs/pypaimon/catalogs.mdx +++ b/docs/docs/pypaimon/catalogs.mdx @@ -48,7 +48,7 @@ catalog_options = { catalog = CatalogFactory.create(catalog_options) ``` -For an S3 warehouse, pass S3 authentication options with the filesystem catalog options: +For an S3 warehouse, install `pypaimon[s3]` and pass S3 authentication options with the filesystem catalog options: ```python from pypaimon import CatalogFactory diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py index 1a4362aedb2b..b73c0735e342 100644 --- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py +++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py @@ -663,7 +663,14 @@ def _get_s3_delete_client(self): if self._s3_delete_client is not None: return self._s3_delete_client - import boto3 + try: + import boto3 + except ModuleNotFoundError as error: + if error.name != "boto3": + raise + raise ImportError( + "S3/OSS batch deletion requires pypaimon[s3] or pypaimon[oss]." + ) from error from botocore.config import Config if self._is_oss: diff --git a/paimon-python/pypaimon/tests/s3_delete_compatibility_test.py b/paimon-python/pypaimon/tests/s3_delete_compatibility_test.py index f5dd6d08ec97..f782c8230504 100644 --- a/paimon-python/pypaimon/tests/s3_delete_compatibility_test.py +++ b/paimon-python/pypaimon/tests/s3_delete_compatibility_test.py @@ -24,6 +24,7 @@ import hashlib import os import socketserver +import sys import threading import unittest from http.server import BaseHTTPRequestHandler, HTTPServer @@ -572,6 +573,13 @@ def test_pickle_discards_cached_delete_client(self): restored.__setstate__(state) self.assertIsNone(restored._s3_delete_client) + def test_missing_boto3_reports_optional_extra(self): + file_io = self._new_file_io() + file_io._s3_delete_client = None + with mock.patch.dict(sys.modules, {"boto3": None}): + with self.assertRaisesRegex(ImportError, r"pypaimon\[s3\]"): + file_io._get_s3_delete_client() + def test_jindo_and_native_s3_do_not_use_delete_fallback(self): file_io = self._new_file_io() file_io._pyarrow_gte_22 = True diff --git a/paimon-python/pypaimon/tests/table_update_test.py b/paimon-python/pypaimon/tests/table_update_test.py index 6af386546b25..57fae7308546 100644 --- a/paimon-python/pypaimon/tests/table_update_test.py +++ b/paimon-python/pypaimon/tests/table_update_test.py @@ -37,6 +37,17 @@ from pypaimon.write.write_builder import BatchWriteBuilder, StreamWriteBuilder +@pytest.mark.parametrize("chunked", [False, True]) +def test_predicate_assignment_rejects_reordered_struct_fields(chunked): + values = pa.array([{'a': 1, 'b': 2}]) + if chunked: + values = pa.chunked_array([values]) + target = pa.struct([('b', pa.int32()), ('a', pa.int32())]) + + with pytest.raises(ValueError, match="struct fields must match in order"): + BatchTableUpdate._assignment_to_array(values, target, 1) + + # ====================================================================== # Shared base for batch & stream table-update tests # ====================================================================== diff --git a/paimon-python/pypaimon/write/table_update.py b/paimon-python/pypaimon/write/table_update.py index 6c9bb6b839a4..34d9fee28daa 100644 --- a/paimon-python/pypaimon/write/table_update.py +++ b/paimon-python/pypaimon/write/table_update.py @@ -533,6 +533,9 @@ def _assignment_to_array( f"{len(array)} != {row_count}." ) if array.type != data_type: + if pa.types.is_struct(array.type) and pa.types.is_struct(data_type): + if array.type.names != data_type.names: + raise ValueError("Assignment struct fields must match in order.") if isinstance(array, pa.ChunkedArray): array = pa.chunked_array( [chunk.cast(data_type) for chunk in array.chunks], diff --git a/paimon-python/pyproject.toml b/paimon-python/pyproject.toml index c4a4ecb893bd..e90765d7982a 100644 --- a/paimon-python/pyproject.toml +++ b/paimon-python/pyproject.toml @@ -63,7 +63,7 @@ dependencies = [ "polars>=1.32,<2; python_version >= '3.9'", "pyarrow>=6,<7; python_version < '3.7'", "pyarrow>=7,<13; python_version >= '3.7' and python_version < '3.8'", - "pyarrow>=16,!=19.0.0,<20; python_version >= '3.8'", + "pyarrow>=16,!=19.0.0,<24; python_version >= '3.8'", "pyroaring<=0.3.3; python_version < '3.7'", "pyroaring<=0.4.5; python_version == '3.7'", "pyroaring>=1.0.0; python_version >= '3.8'", @@ -94,7 +94,6 @@ hdf5 = [ rosbag = [ "rosbags>=0.11.5,<0.12; python_version >= '3.10'", ] -# datasets 4.1+ may require PyArrow 21+, outside the supported core range. # Pandas 2.2.2+ supports NumPy 2 selected by the media dependencies. lerobot = [ "datasets>=4,<4.1; python_version >= '3.10'", @@ -118,10 +117,14 @@ daft = [ "daft>=0.7.6; python_version >= '3.10'", ] oss = [ + "boto3>=1.34,<2; python_version >= '3.10'", "oss2>=2.18,<3", "ossfs>=2021.8; python_version < '3.8'", "ossfs>=2023; python_version >= '3.8'", ] +s3 = [ + "boto3>=1.34,<2; python_version >= '3.10'", +] jindo = [ "oss2>=2.18,<3", "pyjindosdk>=6.10.4",