From 85890457ff6122457f8bf825ecea169f4e1902f9 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 15 Sep 2026 13:47:54 -0300 Subject: [PATCH] fix test matrix, add py315, make deps fully opt --- .github/workflows/deploy-docs.yml | 6 +- .github/workflows/pypi.yml | 2 +- .../workflows/{tests.yml => tests-core.yml} | 23 +++-- .../{codecov_lints.yml => tests-full.yml} | 12 +-- .pre-commit-config.yaml | 2 +- erddapy/xarray_erddap.py | 78 ++++++++------- pyproject.toml | 20 ++-- tests/{download_test_data.py => conftest.py} | 6 +- tests/test_multiple_server_search.py | 11 ++- tests/test_netcdf_handling.py | 15 ++- tests/test_to_objects.py | 99 +++++++++++++++---- tests/test_xarray_backend.py | 14 +++ 12 files changed, 196 insertions(+), 92 deletions(-) rename .github/workflows/{tests.yml => tests-core.yml} (59%) rename .github/workflows/{codecov_lints.yml => tests-full.yml} (72%) rename tests/{download_test_data.py => conftest.py} (90%) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 08ace54e..71fc83fe 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -17,9 +17,6 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - defaults: - run: - shell: bash -l {0} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -34,7 +31,8 @@ jobs: - name: Install erddapy run: | - pip install -e ".[dev,extras,docs]" + pip install --group docs + pip install -e . - name: Install pandoc uses: pandoc/actions/setup@86321b6dd4675f5014c611e05088e10d4939e09e # v1.1.1 diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 294c9e09..7a8c67c3 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -1,4 +1,4 @@ -name: Publish to PyPI +name: PyPI on: release: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests-core.yml similarity index 59% rename from .github/workflows/tests.yml rename to .github/workflows/tests-core.yml index 4599b022..c58d8868 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests-core.yml @@ -1,4 +1,4 @@ -name: Full Tests +name: Core Tests # no permissions by default permissions: {} @@ -13,15 +13,15 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [ "3.11", "3.14" ] + python-version: [ "3.11", "3.15-dev" ] os: [ ubuntu-latest ] include: - os: windows-latest - python-version: "3.14" + python-version: "3.15-dev" - os: macos-latest - python-version: "3.14" + python-version: "3.15-dev" - os: macos-15-intel - python-version: "3.14" + python-version: "3.15-dev" fail-fast: false permissions: actions: none @@ -32,12 +32,17 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install erddapy run: | - pip install -e ".[dev,extras,docs]" + pip install --group dev + pip install -e . - name: Full Tests # Use the --record-mode=rewrite if you need to re-record the cassetts. - run: > - pushd tests && python download_test_data.py && popd - && python -m pytest -rxs tests + run: | + python -m pytest -rxs tests diff --git a/.github/workflows/codecov_lints.yml b/.github/workflows/tests-full.yml similarity index 72% rename from .github/workflows/codecov_lints.yml rename to .github/workflows/tests-full.yml index 2ca18de7..417f8e0a 100644 --- a/.github/workflows/codecov_lints.yml +++ b/.github/workflows/tests-full.yml @@ -1,4 +1,4 @@ -name: Code coverage report +name: Full Tests # no permissions by default permissions: {} @@ -23,13 +23,13 @@ jobs: - name: Install erddapy run: | - pip install -e ".[dev,extras,docs]" + pip install --group docs + pip install -e . - name: Run tests with coverage - run: > - pushd tests && python download_test_data.py && popd - && python -m pytest -rxs --cov=erddapy tests + run: | + python -m pytest -rxs --cov=erddapy tests - name: Run linters - run: > + run: | prek run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 489d0ea2..b221aa28 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -54,7 +54,7 @@ repos: - id: add-trailing-comma - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.7 + rev: v0.16.8 hooks: - id: ruff args: ["--fix", "--show-fixes"] diff --git a/erddapy/xarray_erddap.py b/erddapy/xarray_erddap.py index a21234f9..c51ed2e1 100644 --- a/erddapy/xarray_erddap.py +++ b/erddapy/xarray_erddap.py @@ -3,49 +3,55 @@ import urllib.parse from collections.abc import Iterable -import xarray as xr -from xarray.backends.common import T_PathFileOrDataStore +try: + import xarray as xr + from xarray.backends.common import T_PathFileOrDataStore + + XARRAY_IMPORT = True +except ImportError: + XARRAY_IMPORT = False from erddapy.core.interfaces import to_xarray from erddapy.core.url import _is_netcdf, _is_url +if XARRAY_IMPORT: -def _make_opendap(url: str) -> str: - parts = urllib.parse.urlparse(url) - opendap_url = urllib.parse.urlunparse( - [parts.scheme, parts.netloc, parts.path, "", "", ""], - ) - return opendap_url.split(".nc")[0] - - -class ERDDAPyBackendEntrypoint(xr.backends.BackendEntrypoint): - """Erddapy backend entrypoint for xarray.""" + def _make_opendap(url: str) -> str: + parts = urllib.parse.urlparse(url) + opendap_url = urllib.parse.urlunparse( + [parts.scheme, parts.netloc, parts.path, "", "", ""], + ) + return opendap_url.split(".nc")[0] - def open_dataset( - self, - filename_or_obj: T_PathFileOrDataStore, - *, - drop_variables: str | Iterable[str] | None = None, # noqa: ARG002 - ) -> xr.Dataset: - """Open ERDDAP URLs as xarray datasets.""" - return open_erddap_dataset(filename_or_obj) + class ERDDAPyBackendEntrypoint(xr.backends.BackendEntrypoint): + """Erddapy backend entrypoint for xarray.""" - open_dataset_parameters = ("filename_or_obj", "drop_variables") + def open_dataset( + self, + filename_or_obj: T_PathFileOrDataStore, + *, + drop_variables: str | Iterable[str] | None = None, # noqa: ARG002 + ) -> xr.Dataset: + """Open ERDDAP URLs as xarray datasets.""" + return open_erddap_dataset(filename_or_obj) - description = "Load ERDDAP URLs in xarray." + open_dataset_parameters = ("filename_or_obj", "drop_variables") + description = "Load ERDDAP URLs in xarray." -def open_erddap_dataset(filename_or_obj: T_PathFileOrDataStore) -> xr.Dataset: - """Open an ERDDAP URL with a netcdf-like response as an xarray object.""" - if not _is_url(filename_or_obj): - msg = f"Expected an ERDDAP URL, got {filename_or_obj!r}." - raise ValueError(msg) - - url = str(filename_or_obj) - if _is_netcdf(url): - response = "nc" - else: - filename_or_obj = _make_opendap(url) - response = "opendap" - - return to_xarray(url, response=response) + def open_erddap_dataset( + filename_or_obj: T_PathFileOrDataStore, + ) -> xr.Dataset: + """Open an ERDDAP URL response as an xarray object.""" + if not _is_url(filename_or_obj): + msg = f"Expected an ERDDAP URL, got {filename_or_obj!r}." + raise ValueError(msg) + + url = str(filename_or_obj) + if _is_netcdf(url): + response = "nc" + else: + filename_or_obj = _make_opendap(url) + response = "opendap" + + return to_xarray(url, response=response) diff --git a/pyproject.toml b/pyproject.toml index 0b607b45..7c8c4f25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,17 +25,19 @@ dynamic = [ "version" ] dependencies = [ "pandas", "requests", - "xarray", ] -optional-dependencies.dev = [ - "dask", +entry-points."xarray.backends".erddapy = "erddapy.xarray_erddap:ERDDAPyBackendEntrypoint" + +[dependency-groups] +dev = [ "pooch", "prek", "pytest", "pytest-cov", "pytest-recording", + "xarray", ] -optional-dependencies.docs = [ +docs = [ "cartopy>=0.21", "geopandas", "joblib", @@ -44,16 +46,22 @@ optional-dependencies.docs = [ "nbsphinx", "regionmask>=0.9", "sphinx", + { include-group = "dev" }, + { include-group = "objects" }, + { include-group = "extras" }, ] -optional-dependencies.extras = [ +extras = [ "joblib", +] +objects = [ + "dask", "netcdf4", "scitools-iris>=3.3", + "xarray", ] urls.documentation = "https://ioos.github.io/erddapy" urls.homepage = "https://github.com/ioos/erddapy" urls.repository = "https://github.com/ioos/erddapy" -entry-points."xarray.backends".erddapy = "erddapy.xarray_erddap:ERDDAPyBackendEntrypoint" [tool.setuptools] packages = [ "erddapy" ] diff --git a/tests/download_test_data.py b/tests/conftest.py similarity index 90% rename from tests/download_test_data.py rename to tests/conftest.py index 84d67e72..69603780 100644 --- a/tests/download_test_data.py +++ b/tests/conftest.py @@ -4,8 +4,10 @@ from pathlib import Path import pooch +import pytest +@pytest.fixture(scope="session", autouse=True) def download_test_data() -> None: """Fetch cassettes yaml data from cassettes test release.""" url = "https://github.com/ioos/erddapy/releases/download" @@ -19,7 +21,3 @@ def download_test_data() -> None: here = Path(__file__).resolve().parent with zipfile.ZipFile(fname, "r") as zip_ref: zip_ref.extractall(here) - - -if __name__ == "__main__": - download_test_data() diff --git a/tests/test_multiple_server_search.py b/tests/test_multiple_server_search.py index 9af67ceb..c7637fc1 100644 --- a/tests/test_multiple_server_search.py +++ b/tests/test_multiple_server_search.py @@ -6,6 +6,13 @@ from erddapy.multiple_server_search import fetch_results, search_servers +try: + import joblib # noqa: F401 + + JOBLIB_INSTALLED = True +except ImportError: + JOBLIB_INSTALLED = False + @pytest.mark.web @pytest.mark.vcr @@ -37,7 +44,7 @@ def test_fetch_no_results(): @pytest.mark.web @pytest.mark.skipif( - sys.platform in ("win32", "darwin"), + (sys.platform in ("win32", "darwin") or not JOBLIB_INSTALLED), reason="run only on linux to avoid extra load on the server", ) def test_search_awesome_erddap_servers_true(): @@ -82,7 +89,7 @@ def servers_list(): @pytest.mark.web @pytest.mark.skipif( - (sys.platform in ("win32", "darwin") or sys.version_info < (3, 10)), + (sys.platform in ("win32", "darwin") or not JOBLIB_INSTALLED), reason="run only on linux and latest to avoid extra load on the server", ) def test_search_servers_with_a_list_parallel_true(servers_list): diff --git a/tests/test_netcdf_handling.py b/tests/test_netcdf_handling.py index 52054fc0..3ad2d70b 100644 --- a/tests/test_netcdf_handling.py +++ b/tests/test_netcdf_handling.py @@ -3,15 +3,26 @@ import platform from pathlib import Path +try: + import netCDF4 # noqa: F401 + + NETCDF4_INSTALLED = True +except ImportError: + NETCDF4_INSTALLED = False + import pytest from erddapy.core.netcdf import _nc_dataset, _tempnc from erddapy.core.url import urlopen -@pytest.mark.web # For some reason we cannot use vcr with requests with in_memory # (also all the to_objects that uses in_memory). +@pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test__nc_dataset_in_memory_https(): """Test loading a netcdf dataset in-memory.""" from netCDF4 import Dataset # noqa: PLC0415 @@ -25,7 +36,7 @@ def test__nc_dataset_in_memory_https(): @pytest.mark.web @pytest.mark.vcr @pytest.mark.skipif( - platform.system().lower() == "windows", + (platform.system().lower() == "windows" or not NETCDF4_INSTALLED), reason="does not remove the file on windows", ) def test__tempnc(): diff --git a/tests/test_to_objects.py b/tests/test_to_objects.py index 38c3e7aa..a4ca8839 100644 --- a/tests/test_to_objects.py +++ b/tests/test_to_objects.py @@ -2,16 +2,33 @@ import sys -import dask -import iris +try: + import dask + + # netcdf-c is not thread safe and iris doesn't limit that. + dask.config.set(scheduler="single-threaded") +except ImportError: + pass + +try: + import iris + + IRIS_INSTALLED = True +except ImportError: + IRIS_INSTALLED = False + import pytest import requests import xarray as xr -from erddapy import ERDDAP +try: + import netCDF4 # noqa: F401 + + NETCDF4_INSTALLED = True +except ImportError: + NETCDF4_INSTALLED = False -# netcdf-c is not thread safe and iris doesn't limit that. -dask.config.set(scheduler="single-threaded") +from erddapy import ERDDAP @pytest.fixture @@ -34,30 +51,30 @@ def gliders(): @pytest.fixture -def ncei(): +def ioos(): """Instantiate ERDDAP class for testing.""" return ERDDAP( - server="https://www.ncei.noaa.gov/erddap/", + server="https://erddap.ioos.us/erddap/", response="htmlTable", ) @pytest.fixture -def dataset_griddap(ncei): +def dataset_griddap(ioos): """Load griddap data for testing.""" - ncei.dataset_id = "AEC_gomex_satellite_climo" - ncei.protocol = "griddap" - ncei.griddap_initialize() - return ncei + ioos.dataset_id = "etopo5_EDDGridCopy" + ioos.protocol = "griddap" + ioos.griddap_initialize() + return ioos @pytest.fixture -def dataset_opendap(ncei): +def dataset_opendap(ioos): """Load griddap data with OPeNDAP response for testing.""" - ncei.dataset_id = "AEC_gomex_satellite_climo" - ncei.protocol = "griddap" - ncei.response = "opendap" - return ncei + ioos.dataset_id = "etopo5_EDDGridCopy" + ioos.protocol = "griddap" + ioos.response = "opendap" + return ioos @pytest.fixture @@ -132,6 +149,10 @@ def test_to_pandas_requests_kwargs(dataset_tabledap): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_to_xarray_tabledap(dataset_tabledap): """Test converting tabledap to an xarray Dataset.""" ds = dataset_tabledap.to_xarray() @@ -144,6 +165,10 @@ def test_to_xarray_tabledap(dataset_tabledap): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_to_xarray_cannot_be_quoted(): """Test dataset that failed when quoted.""" e = ERDDAP(server="https://erddap.aoos.org/erddap/", protocol="tabledap") @@ -157,6 +182,10 @@ def test_to_xarray_cannot_be_quoted(): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_to_xarray_requests_kwargs(dataset_tabledap): """Test converting tabledap to an xarray Dataset with manual timeout.""" ds = dataset_tabledap.to_xarray(requests_kwargs={"timeout": 30}) @@ -169,6 +198,10 @@ def test_to_xarray_requests_kwargs(dataset_tabledap): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_to_xarray_griddap(dataset_griddap): """Test converting griddap to an xarray Dataset.""" ds = dataset_griddap.to_xarray() @@ -176,6 +209,10 @@ def test_to_xarray_griddap(dataset_griddap): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_to_xarray_opendap(dataset_opendap): """Test converting griddap to xarray with the OPeNDAP response.""" ds = dataset_opendap.to_xarray() @@ -183,6 +220,10 @@ def test_to_xarray_opendap(dataset_opendap): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_to_xarray_opendap_griddap_initialize(dataset_opendap): """Test converting griddap aftert calling griddap_initialize.""" dataset_opendap.griddap_initialize() @@ -192,8 +233,8 @@ def test_to_xarray_opendap_griddap_initialize(dataset_opendap): @pytest.mark.web @pytest.mark.skipif( - (sys.platform in ("win32", "darwin")), - reason="run this test until we figure out a way to mock it.", + (sys.platform in ("win32", "darwin") or not IRIS_INSTALLED), + reason="Optional deps are tested in coverage and oldest Python only.", ) def test_to_iris_tabledap(dataset_tabledap): """Test converting tabledap to an iris cube.""" @@ -209,8 +250,8 @@ def test_to_iris_tabledap(dataset_tabledap): @pytest.mark.web @pytest.mark.skipif( - (sys.platform in ("win32", "darwin")), - reason="run this test until we figure out a way to mock it.", + (sys.platform in ("win32", "darwin") or not IRIS_INSTALLED), + reason="Optional deps are tested in coverage and oldest Python only.", ) def test_to_iris_griddap(dataset_griddap): """Test converting griddap to an iris cube.""" @@ -219,6 +260,10 @@ def test_to_iris_griddap(dataset_griddap): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_download_file(dataset_tabledap): """Test file download of tabledap with defined variable and constraints.""" fn = dataset_tabledap.download_file("nc") @@ -231,6 +276,10 @@ def test_download_file(dataset_tabledap): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_download_file_variables_only(dataset_tabledap): """Test direct download of tabledap dataset with undefined constraints.""" dataset_tabledap.constraints = {} @@ -244,6 +293,10 @@ def test_download_file_variables_only(dataset_tabledap): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_download_file_constraints_only(dataset_tabledap): """Test direct download of tabledap dataset with undefined variables.""" dataset_tabledap.variables = [] @@ -257,6 +310,10 @@ def test_download_file_constraints_only(dataset_tabledap): @pytest.mark.web +@pytest.mark.skipif( + not NETCDF4_INSTALLED, + reason="Optional deps are tested in coverage and oldest Python only.", +) def test_download_file_undefined_query(dataset_tabledap): """Test direct download of tabledap dataset with undefined query.""" dataset_tabledap.variables = [] diff --git a/tests/test_xarray_backend.py b/tests/test_xarray_backend.py index c35909c2..21e85e64 100644 --- a/tests/test_xarray_backend.py +++ b/tests/test_xarray_backend.py @@ -5,6 +5,20 @@ import pytest import xarray as xr +try: + import netCDF4 # noqa: F401 + + NETCDF4_INSTALLED = True +except ImportError: + NETCDF4_INSTALLED = False + + +if not NETCDF4_INSTALLED: + pytest.skip( + "Optional deps are tested in coverage and oldest Python only.", + allow_module_level=True, + ) + def test_load_backend(): """Check if the backend exists and is the correct one."""