Skip to content
CI

feat(python): expose core vector search and scored results #3897

Sign in to view logs

feat(python): expose core vector search and scored results

feat(python): expose core vector search and scored results #3897

Workflow file for this run

.github/workflows/ci.yml at 2a2692f
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: CI
on:
push:
branches:
- main
- release-*
pull_request:
branches:
- main
- release-*
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
cancel-in-progress: true
jobs:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Detect release license inputs
id: release_license_inputs
shell: bash
env:
EVENT_NAME: ${{ github.event_name }}
BEFORE_SHA: ${{ github.event.before }}
BASE_REF: ${{ github.base_ref }}
run: |
set -euo pipefail
if [[ "$EVENT_NAME" == "pull_request" ]]; then
base="origin/$BASE_REF"
elif [[ -n "$BEFORE_SHA" && ! "$BEFORE_SHA" =~ ^0+$ ]] && \
git cat-file -e "$BEFORE_SHA^{commit}"; then
base="$BEFORE_SHA"
else
echo "changed=true" >> "$GITHUB_OUTPUT"
exit 0
fi
if git diff --quiet "$base" HEAD -- \
.gitattributes \
LICENSE \
NOTICE \
':(glob)**/LICENSE' \
':(glob)**/LICENSE.*' \
':(glob)**/NOTICE' \
':(glob)**/NOTICE.*' \
Cargo.lock \
about.hbs \
about.toml \
bindings/go/LICENSE \
bindings/go/NOTICE \
':(glob)bindings/go/THIRD-PARTY-LICENSES.*.html' \
bindings/python/licenses \
bindings/python/THIRD-PARTY-LICENSES.html \
scripts/release_licenses.py \
third-party-licenses \
':(glob)**/Cargo.toml'; then
echo "changed=false" >> "$GITHUB_OUTPUT"
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi
- name: Validate .asf.yaml
run: python3 scripts/validate_asf_yaml.py
- name: Check License Header
uses: apache/skywalking-eyes/header@v0.9.0
- name: Test release artifact verifiers
run: python3 -m unittest discover -s scripts/tests -p 'test_*.py'
- name: Install cargo-deny
uses: taiki-e/install-action@742a3317eac7bd62f91cd888b4eead5e784ba833 # v2.87.1
with:
tool: cargo-deny@0.19.6
- name: Install cargo-about
if: steps.release_license_inputs.outputs.changed == 'true'
uses: taiki-e/install-action@742a3317eac7bd62f91cd888b4eead5e784ba833 # v2.87.1
with:
tool: cargo-about@0.9.1
- name: Fetch locked dependencies
run: cargo fetch --locked
- name: Check dependency policy
run: cargo deny --locked --all-features check --warn unmaintained advisories licenses
- name: Verify dependency reports
run: python3 scripts/dependencies.py verify
- name: Verify release license files
if: steps.release_license_inputs.outputs.changed == 'true'
run: python3 scripts/release_licenses.py --check
# Keep formatting and deny-warnings lints reproducible across stable
# releases. The build matrix independently checks the current toolchain.
- name: Install formatting and lint toolchain
run: rustup toolchain install 1.98.0 --profile minimal --component rustfmt --component clippy
- name: Format
run: cargo +1.98.0 fmt --all -- --check
- name: Clippy
run: cargo +1.98.0 clippy --locked --all-targets --workspace --features fulltext,vortex -- -D warnings
msrv:
name: msrv (${{ matrix.host }})
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-latest
host: x86_64-unknown-linux-gnu
- runner: macos-15
host: aarch64-apple-darwin
env:
PYO3_PYTHON: python3.11
steps:
- uses: actions/checkout@v7
- uses: actions/setup-python@v6
with:
python-version: "3.11"
- name: Install Rust 1.94.0
run: rustup toolchain install 1.94.0 --profile minimal
- name: Validate MSRV host
env:
EXPECTED_HOST: ${{ matrix.host }}
run: |
rustc +1.94.0 -vV | grep -F "host: ${EXPECTED_HOST}"
- name: Validate workspace MSRV metadata
run: |
cargo +1.94.0 metadata --locked --format-version 1 --no-deps |
jq -e '.packages | all(.rust_version == "1.94.0")' >/dev/null
- name: Check workspace at MSRV
run: cargo +1.94.0 check --locked --workspace --all-targets --all-features
build:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
steps:
- uses: actions/checkout@v7
- name: Build
run: cargo build --locked --features fulltext,vortex
unit:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
steps:
- uses: actions/checkout@v7
- name: Limit Windows test build memory
if: runner.os == 'Windows'
shell: pwsh
run: |
# Full-feature test binaries can exhaust the runner's paging file.
"CARGO_BUILD_JOBS=2" >> $env:GITHUB_ENV
"CARGO_PROFILE_TEST_DEBUG=line-tables-only" >> $env:GITHUB_ENV
- name: Test
run: cargo test --locked -p paimon --all-targets --features fulltext,vortex
env:
RUST_LOG: DEBUG
RUST_BACKTRACE: full
# The e2e tests round-trip through a real FileSystemCatalog, which has
# known Windows path-compatibility issues (the `paimon` filesystem
# catalog tests are likewise gated with `#[cfg(not(windows))]`).
- name: Test paimon-rest-server
if: runner.os != 'Windows'
run: cargo test --locked -p paimon-rest-server --all-targets
env:
RUST_LOG: DEBUG
RUST_BACKTRACE: full
integration:
name: integration (${{ matrix.suite }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- suite: rust
needs_docker: true
needs_lumina: false
needs_uv: false
- suite: datafusion
needs_docker: true
needs_lumina: true
needs_uv: false
- suite: lumina
needs_docker: false
needs_lumina: true
needs_uv: false
- suite: python
needs_docker: true
needs_lumina: false
needs_uv: true
- suite: go
needs_docker: true
needs_lumina: false
needs_uv: false
- suite: jindo
needs_docker: false
needs_lumina: false
needs_uv: false
- suite: oss-cpp
needs_docker: false
needs_lumina: false
needs_uv: false
env:
RUST_LOG: DEBUG
RUST_BACKTRACE: full
steps:
- uses: actions/checkout@v7
- name: Rust Cache
uses: actions/cache@v6
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: integration-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
integration-${{ runner.os }}-
- name: Start Docker containers
if: matrix.needs_docker == true
run: make docker-up
- name: Rust Integration Test
if: matrix.suite == 'rust'
run: cargo test --locked -p paimon-integration-tests --all-targets
- name: Install lumina native library
if: matrix.needs_lumina == true
run: |
pip install lumina-data
echo "LUMINA_LIB_PATH=$(python3 -c 'import lumina_data; print(lumina_data.__path__[0])')/lib/liblumina_py.so" >> $GITHUB_ENV
- name: Core Lumina Native Build Test
if: matrix.suite == 'lumina'
run: >
cargo test --locked -p paimon --lib
table::lumina_index_build_builder::tests::test_execute_writes_lumina_index_manifest
--features fulltext,vortex
-- --ignored --exact
# The test filter selects tests, not targets, so without `--test` cargo links all 26
# integration binaries with `vortex` enabled just to run one ignored test in one of them.
- name: DataFusion Lumina Build Query E2E Test
if: matrix.suite == 'lumina'
run: >
cargo test --locked -p paimon-datafusion --test read_tables
--features vortex
vector_search_tests::test_lumina_build_then_vector_search_query
-- --ignored --exact
- name: DataFusion Integration Test
if: matrix.suite == 'datafusion'
env:
# The partition-count DV fixture exceeds Rust's default 2 MiB test-thread stack.
RUST_MIN_STACK: 16777216
run: cargo test --locked -p paimon-datafusion --all-targets
- name: DataFusion Vortex Integration Test
if: matrix.suite == 'datafusion'
run: cargo test --locked -p paimon-datafusion --features vortex --test vortex_tables
- name: Install uv
if: matrix.needs_uv == true
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d
with:
version: "0.9.3"
enable-cache: true
- name: Python Binding Integration Test
if: matrix.suite == 'python'
working-directory: bindings/python
run: make install && make test
- name: Go Integration Test
if: matrix.suite == 'go'
working-directory: bindings/go
run: make test
- uses: actions/setup-python@v6
if: matrix.suite == 'jindo'
with:
python-version: "3.11"
- name: Install Jindo SDK
if: matrix.suite == 'jindo'
run: |
python -m pip install --only-binary=:all: --no-deps pyjindosdk==6.10.4
echo "PAIMON_JINDO_SDK_LIBRARY=$(python -c 'import pathlib, pyjindo; print(pathlib.Path(pyjindo.__path__[0]) / "libjindosdk_python.so")')" >> "$GITHUB_ENV"
- name: Jindo SDK Integration Test
if: matrix.suite == 'jindo'
run: >
cargo test --locked -p paimon --features storage-jindo --lib
io::storage_jindo::tests::test_real_sdk_io_and_failure_mapping
-- --ignored --exact
- name: Stop Docker containers
if: ${{ always() && matrix.needs_docker }}
run: make docker-down
- name: Checkout OSS C++ SDK
if: matrix.suite == 'oss-cpp'
uses: actions/checkout@v7
with:
repository: aliyun/alibabacloud-oss-cpp-sdk-v2
ref: c13a82038b0696f99c32f4e86b623ffcc9e62140
path: .oss-cpp-sdk
- name: Build OSS C++ bridge
if: matrix.suite == 'oss-cpp'
run: |
sudo apt-get update
sudo apt-get install -y libcurl4-openssl-dev libssl-dev
cmake -S .oss-cpp-sdk -B build/oss-sdk -DCMAKE_BUILD_TYPE=Release \
-DUSE_SYSTEM_CURL=ON -DUSE_SYSTEM_OPENSSL=ON -DBUILD_SHARED_LIBS=ON \
-DCMAKE_CXX_FLAGS=-Wno-error=maybe-uninitialized \
-DCMAKE_INSTALL_PREFIX="$PWD/build/oss-install"
cmake --build build/oss-sdk -j 2
cmake --install build/oss-sdk
cmake -S integrations/oss-cpp -B build/oss-cpp \
-DCMAKE_PREFIX_PATH="$PWD/build/oss-install"
cmake --build build/oss-cpp -j 2
echo "OSS_CPP_BRIDGE_LIBRARY=$PWD/build/oss-cpp/liboss_cpp_bridge.so" >> "$GITHUB_ENV"
- name: OSS C++ SDK Integration Test
if: matrix.suite == 'oss-cpp'
run: |
cargo test --locked -p paimon --features storage-oss-cpp,storage-jindo --lib \
io::storage_oss_cpp -- --include-ignored
rustup toolchain install 1.98.0 --profile minimal --component clippy
cargo +1.98.0 clippy --locked -p paimon --features storage-oss-cpp --lib --tests -- -D warnings