diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index e9de2c3b9..1f898ab06 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -7,8 +7,8 @@ on: - vdbbench_* jobs: - build: - name: Run Python Tests + unit-test: + name: Hermetic unit tests strategy: matrix: python-version: [3.11, 3.12] @@ -30,12 +30,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[test]" + python -m pip install -e ".[test]" - name: Run coding checks run: | make lint - - name: Test with pytest + - name: Run hermetic unit tests run: | - make unittest + make unit-test diff --git a/Makefile b/Makefile index ef8207c55..62bb6f130 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,10 @@ -unittest: - PYTHONPATH=`pwd` python3 -m pytest tests/test_dataset.py::TestDataSet::test_download_small -svv +unit-test: + python -m pytest --disable-socket tests/unit + +unittest: unit-test + +e2e-test: + python -m pytest tests/e2e -svv format: PYTHONPATH=`pwd` python3 -m black vectordb_bench diff --git a/pyproject.toml b/pyproject.toml index 223291721..09ecc99ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ test = [ "black", "ruff", "pytest", + "pytest-socket", ] restful = [ "flask" ] qdrant = [ "qdrant-client" ] diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..e54bcdd51 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,42 @@ +# Test layout + +The test suite is being separated by execution requirements. + +## Hermetic unit tests + +`tests/unit/` is the allowlisted local suite. Tests in this directory must be +deterministic and must not use network sockets, credentials, live databases, +containers, or downloaded datasets. + +Install only the project and test extra, then run: + +```shell +pip install -e ".[test]" +make unit-test +``` + +`make unittest` is a compatibility alias. The unit target uses +`pytest-socket` to disable network access and is the suite run by pull-request +CI. + +## Online E2E tests + +`tests/e2e/` contains tests that download datasets or require external +services. Run them explicitly with: + +```shell +make e2e-test +``` + +The dataset E2E tests require network access to S3 and/or Aliyun OSS and can +download large Cohere or LAION datasets. Other E2E tests may require provider +SDK extras, credentials, containers, or a locally running database; document +those prerequisites with the test when migrating it. + +## Legacy tests + +Test modules directly under `tests/` are legacy and are not yet classified. +They are intentionally outside both Make targets: do not assume they are +hermetic, and do not move them into `tests/unit/` until they pass with only +`.[test]` and with network sockets disabled. New tests must go directly into +`tests/unit/` or `tests/e2e/`. diff --git a/tests/test_dataset.py b/tests/e2e/test_dataset.py similarity index 54% rename from tests/test_dataset.py rename to tests/e2e/test_dataset.py index d4ccb283d..bb5088a78 100644 --- a/tests/test_dataset.py +++ b/tests/e2e/test_dataset.py @@ -1,54 +1,38 @@ -from vectordb_bench.backend.dataset import Dataset import logging +import os +import time + import pytest -from pydantic import ValidationError -from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.dataset import Dataset log = logging.getLogger("vectordb_bench") +pytestmark = pytest.mark.integration -class TestDataSet: - def test_iter_dataset(self): - for ds in Dataset: - log.info(ds) - - def test_cohere(self): - cohere = Dataset.COHERE.get(100_000) - log.info(cohere) - assert cohere.name == "Cohere" - assert cohere.size == 100_000 - assert cohere.label == "SMALL" - assert cohere.dim == 768 - - def test_cohere_error(self): - with pytest.raises(ValidationError): - Dataset.COHERE.get(9999) +class TestDataSet: def test_iter_cohere(self): cohere_10m = Dataset.COHERE.manager(10_000_000) cohere_10m.prepare() - import time before = time.time() - for i in cohere_10m: - log.debug(i.head(1)) + for batch in cohere_10m: + log.debug(batch.head(1)) - dur_iter = time.time() - before - log.warning(f"iter through cohere_10m cost={dur_iter/60}min") + duration = time.time() - before + log.warning("iter through cohere_10m cost=%smin", duration / 60) - # pytest -sv tests/test_dataset.py::TestDataSet::test_iter_laion def test_iter_laion(self): laion_100m = Dataset.LAION.manager(100_000_000) - from vectordb_bench.backend.data_source import DatasetSource laion_100m.prepare(source=DatasetSource.AliyunOSS) - import time before = time.time() - for i in laion_100m: - log.debug(i.head(1)) + for batch in laion_100m: + log.debug(batch.head(1)) - dur_iter = time.time() - before - log.warning(f"iter through laion_100m cost={dur_iter/60}min") + duration = time.time() - before + log.warning("iter through laion_100m cost=%smin", duration / 60) def test_download_small(self): openai_50k = Dataset.OPENAI.manager(50_000) @@ -60,8 +44,6 @@ def test_download_small(self): ] file_path = openai_50k.data_dir.joinpath("test.parquet") - import os - DatasetSource.S3.reader().read( openai_50k.data.dir_name.lower(), files=files, @@ -74,4 +56,3 @@ def test_download_small(self): files=files, local_ds_root=openai_50k.data_dir, ) - diff --git a/tests/test_adbpg.py b/tests/unit/test_adbpg.py similarity index 100% rename from tests/test_adbpg.py rename to tests/unit/test_adbpg.py diff --git a/tests/test_aws_opensearch_cli.py b/tests/unit/test_aws_opensearch_cli.py similarity index 100% rename from tests/test_aws_opensearch_cli.py rename to tests/unit/test_aws_opensearch_cli.py diff --git a/tests/test_case_runner_reuse.py b/tests/unit/test_case_runner_reuse.py similarity index 100% rename from tests/test_case_runner_reuse.py rename to tests/unit/test_case_runner_reuse.py diff --git a/tests/test_cli_note.py b/tests/unit/test_cli_note.py similarity index 100% rename from tests/test_cli_note.py rename to tests/unit/test_cli_note.py diff --git a/tests/test_cloud_cold_latency_case.py b/tests/unit/test_cloud_cold_latency_case.py similarity index 100% rename from tests/test_cloud_cold_latency_case.py rename to tests/unit/test_cloud_cold_latency_case.py diff --git a/tests/test_cloud_payload_case.py b/tests/unit/test_cloud_payload_case.py similarity index 100% rename from tests/test_cloud_payload_case.py rename to tests/unit/test_cloud_payload_case.py diff --git a/tests/test_cloud_payload_search.py b/tests/unit/test_cloud_payload_search.py similarity index 100% rename from tests/test_cloud_payload_search.py rename to tests/unit/test_cloud_payload_search.py diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py new file mode 100644 index 000000000..0a513f875 --- /dev/null +++ b/tests/unit/test_dataset.py @@ -0,0 +1,27 @@ +import logging + +import pytest +from pydantic import ValidationError + +from vectordb_bench.backend.dataset import Dataset + + +log = logging.getLogger("vectordb_bench") + + +class TestDataSet: + def test_iter_dataset(self): + for dataset in Dataset: + log.info(dataset) + + def test_cohere(self): + cohere = Dataset.COHERE.get(100_000) + log.info(cohere) + assert cohere.name == "Cohere" + assert cohere.size == 100_000 + assert cohere.label == "SMALL" + assert cohere.dim == 768 + + def test_cohere_error(self): + with pytest.raises(ValidationError): + Dataset.COHERE.get(9999) diff --git a/tests/test_db_client_resolution.py b/tests/unit/test_db_client_resolution.py similarity index 100% rename from tests/test_db_client_resolution.py rename to tests/unit/test_db_client_resolution.py diff --git a/tests/test_fts_cases.py b/tests/unit/test_fts_cases.py similarity index 100% rename from tests/test_fts_cases.py rename to tests/unit/test_fts_cases.py diff --git a/tests/test_fts_cli_user_control.py b/tests/unit/test_fts_cli_user_control.py similarity index 100% rename from tests/test_fts_cli_user_control.py rename to tests/unit/test_fts_cli_user_control.py diff --git a/tests/test_fts_dataset.py b/tests/unit/test_fts_dataset.py similarity index 100% rename from tests/test_fts_dataset.py rename to tests/unit/test_fts_dataset.py diff --git a/tests/test_fts_filter_runner.py b/tests/unit/test_fts_filter_runner.py similarity index 100% rename from tests/test_fts_filter_runner.py rename to tests/unit/test_fts_filter_runner.py diff --git a/tests/test_fts_format_results.py b/tests/unit/test_fts_format_results.py similarity index 100% rename from tests/test_fts_format_results.py rename to tests/unit/test_fts_format_results.py diff --git a/tests/test_fts_metrics.py b/tests/unit/test_fts_metrics.py similarity index 100% rename from tests/test_fts_metrics.py rename to tests/unit/test_fts_metrics.py diff --git a/tests/test_lancedb_config.py b/tests/unit/test_lancedb_config.py similarity index 100% rename from tests/test_lancedb_config.py rename to tests/unit/test_lancedb_config.py diff --git a/tests/test_milvus_zilliz_cli.py b/tests/unit/test_milvus_zilliz_cli.py similarity index 100% rename from tests/test_milvus_zilliz_cli.py rename to tests/unit/test_milvus_zilliz_cli.py diff --git a/tests/test_oss_opensearch_fts.py b/tests/unit/test_oss_opensearch_fts.py similarity index 100% rename from tests/test_oss_opensearch_fts.py rename to tests/unit/test_oss_opensearch_fts.py diff --git a/tests/test_utils.py b/tests/unit/test_utils.py similarity index 100% rename from tests/test_utils.py rename to tests/unit/test_utils.py