From fab535fb2582ec1cc77409ffc773f5251b7ffdfd Mon Sep 17 00:00:00 2001 From: ctuguinay Date: Mon, 10 Aug 2026 10:38:02 -0700 Subject: [PATCH 1/4] fix merge conflicts --- echopype/tests/echodata/test_echodata.py | 23 ++++++++++++++++++++++- echopype/utils/io.py | 7 ++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/echopype/tests/echodata/test_echodata.py b/echopype/tests/echodata/test_echodata.py index 0bc2a9e87..1d7341f58 100644 --- a/echopype/tests/echodata/test_echodata.py +++ b/echopype/tests/echodata/test_echodata.py @@ -4,9 +4,10 @@ import fsspec from pathlib import Path import shutil +import warnings from xarray import DataTree -from zarr.errors import GroupNotFoundError +from zarr.errors import GroupNotFoundError, ZarrUserWarning import echopype from echopype.echodata import EchoData @@ -850,3 +851,23 @@ def test_echodata_delete(caplog, ek60_path): # Check that it doesn't exist assert not os.path.exists(temp_zarr_path) + + +@pytest.mark.unit +def test_to_zarr_no_consolidated_metadata_warning(): + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always", ZarrUserWarning) + ed = open_raw( + "echopype/test_data/ek60/ncei-wcsd/" + "Summer2017-D20170719-T211347.raw", + sonar_model="EK60", + ) + ed.to_zarr(overwrite=True) + + assert not any( + isinstance(w.message, ZarrUserWarning) + and "Consolidated metadata is currently not part in the Zarr format 3 specification" + in str(w.message) + for w in caught_warnings + ) + diff --git a/echopype/utils/io.py b/echopype/utils/io.py index 10d14ad2c..b8395d92d 100644 --- a/echopype/utils/io.py +++ b/echopype/utils/io.py @@ -76,7 +76,12 @@ def save_file(ds, path, mode, engine, group=None, compression_settings=None, **k for var, enc in encoding.items(): if isinstance(ds[var].data, DaskArray): ds[var] = ds[var].chunk(enc.get("chunks", {})) - ds.to_zarr(store=path.root, mode=mode, group=group, encoding=encoding, **kwargs) + # Set consolidated to False to avoid warning about consolidated metadata not being supported + # by Zarr 3 + # TODO remove consolidated=False when Zarr 3 supports consolidated metadata + ds.to_zarr( + store=path.root, mode=mode, group=group, consolidated=False, encoding=encoding, **kwargs + ) else: raise ValueError(f"{engine} is not a supported save format") From 4e1348879ade028f8139e2fee1a3c249f7beabe8 Mon Sep 17 00:00:00 2001 From: ctuguinay Date: Mon, 10 Aug 2026 10:51:14 -0700 Subject: [PATCH 2/4] fix path --- echopype/tests/echodata/test_echodata.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/echopype/tests/echodata/test_echodata.py b/echopype/tests/echodata/test_echodata.py index 1d7341f58..296f932ab 100644 --- a/echopype/tests/echodata/test_echodata.py +++ b/echopype/tests/echodata/test_echodata.py @@ -854,12 +854,11 @@ def test_echodata_delete(caplog, ek60_path): @pytest.mark.unit -def test_to_zarr_no_consolidated_metadata_warning(): +def test_to_zarr_no_consolidated_metadata_warning(test_path): with warnings.catch_warnings(record=True) as caught_warnings: warnings.simplefilter("always", ZarrUserWarning) ed = open_raw( - "echopype/test_data/ek60/ncei-wcsd/" - "Summer2017-D20170719-T211347.raw", + test_path["EK60"] / "ncei-wcsd/SH1701/TEST-D20170114-T202932.raw", sonar_model="EK60", ) ed.to_zarr(overwrite=True) From 4705c5aea0f41eb0aca06c1040504a7bf447a058 Mon Sep 17 00:00:00 2001 From: ctuguinay Date: Mon, 10 Aug 2026 11:56:37 -0700 Subject: [PATCH 3/4] separate dask chunk encoding which is in dictionary form from zarr chunk encoding which is in list form --- echopype/tests/echodata/test_echodata.py | 25 +++++++++++++++++++++++- echopype/utils/coding.py | 17 ++++++++++------ echopype/utils/io.py | 10 ++++++++-- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/echopype/tests/echodata/test_echodata.py b/echopype/tests/echodata/test_echodata.py index 296f932ab..ac4fc04e4 100644 --- a/echopype/tests/echodata/test_echodata.py +++ b/echopype/tests/echodata/test_echodata.py @@ -855,6 +855,12 @@ def test_echodata_delete(caplog, ek60_path): @pytest.mark.unit def test_to_zarr_no_consolidated_metadata_warning(test_path): + """ + This test checks that we explicitly use `consolidated=False`, otherwise + xarray will attempt to write consolidated metadata but Zarr v3 does not + support this yet. + TODO remove this test once Zarr v3 supports consolidated metadata. + """ with warnings.catch_warnings(record=True) as caught_warnings: warnings.simplefilter("always", ZarrUserWarning) ed = open_raw( @@ -862,7 +868,6 @@ def test_to_zarr_no_consolidated_metadata_warning(test_path): sonar_model="EK60", ) ed.to_zarr(overwrite=True) - assert not any( isinstance(w.message, ZarrUserWarning) and "Consolidated metadata is currently not part in the Zarr format 3 specification" @@ -870,3 +875,21 @@ def test_to_zarr_no_consolidated_metadata_warning(test_path): for w in caught_warnings ) + +@pytest.mark.unit +def test_to_zarr_no_deprecated_chunk_warning(test_path): + """ + This test checks that dask rechunking receives dictionary and not tuple. + """ + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.simplefilter("always") + ed = open_raw( + test_path["EK60"] / "ncei-wcsd/SH1701/TEST-D20170114-T202932.raw", + sonar_model="EK60", + ) + ed.to_zarr(overwrite=True) + assert not any( + isinstance(w.message, FutureWarning) + and "Supplying chunks as dimension-order tuples" in str(w.message) + for w in caught_warnings + ) diff --git a/echopype/utils/coding.py b/echopype/utils/coding.py index 0295d9046..730effbe6 100644 --- a/echopype/utils/coding.py +++ b/echopype/utils/coding.py @@ -3,6 +3,7 @@ import numpy as np import xarray as xr +from dask.array import Array as DaskArray from dask.array.core import auto_chunks from dask.utils import parse_bytes from xarray import coding @@ -217,17 +218,21 @@ def set_zarr_encodings( # tolerance then no need to rechunk if chunk_diff < chunk_size_tolerance: rechunk = False - chunks = existing_chunks + dask_formatted_chunks = existing_chunks if rechunk: # Use dask auto chunk to determine the optimal chunk # spread for optimal chunk size - chunks = _get_dask_auto_chunk(val, chunk_size=chunk_size) - # Dask expects chunk dictionary but Zarr expects list-like iterable of - # values in encoding - chunks = [*chunks.values()] + dask_formatted_chunks = _get_dask_auto_chunk(val, chunk_size=chunk_size) - encoding[name]["chunks"] = chunks + # Dask expects chunk dictionary but Zarr expects list-like iterable of + # values in encoding + zarr_formatted_chunks = [*dask_formatted_chunks.values()] + + encoding[name]["chunks"] = zarr_formatted_chunks + # Add temporary dask formatted_chunks to encoding for use in dask rechunking + if isinstance(ds[name].data, DaskArray): + encoding[name]["dask_formatted_chunks"] = dask_formatted_chunks if PREFERRED_CHUNKS in encoding[name]: # Remove 'preferred_chunks', use chunks only instead encoding[name].pop(PREFERRED_CHUNKS) diff --git a/echopype/utils/io.py b/echopype/utils/io.py index b8395d92d..13ccba32e 100644 --- a/echopype/utils/io.py +++ b/echopype/utils/io.py @@ -72,10 +72,16 @@ def save_file(ds, path, mode, engine, group=None, compression_settings=None, **k if engine == "netcdf4": ds.to_netcdf(path=path, mode=mode, group=group, encoding=encoding, engine=engine, **kwargs) elif engine == "zarr": - # Ensure that encoding and chunks match + # Dask chunk according to dask formatted chunk encoding. As the zarr chunks are inherited + # from the dask formatted chunks, we avoid any potential chunking mismatch between the two. + # If we do not do this, we will need to call `ds.to_zrar(...,align_chunks=True,...)`, which + # also "rechunks the Dask array to align with Zarr chunks before writing" + # (https://docs.xarray.dev/en/latest/generated/xarray.Dataset.to_zarr.html). for var, enc in encoding.items(): if isinstance(ds[var].data, DaskArray): - ds[var] = ds[var].chunk(enc.get("chunks", {})) + ds[var] = ds[var].chunk(enc.get("dask_formatted_chunks", {})) + # # Remove dask_formatted_chunks from encoding + encoding[var].pop("dask_formatted_chunks") # Set consolidated to False to avoid warning about consolidated metadata not being supported # by Zarr 3 # TODO remove consolidated=False when Zarr 3 supports consolidated metadata From f7d8ed466ccc8c9ba662276d4022c1f2a7b34956 Mon Sep 17 00:00:00 2001 From: ctuguinay Date: Mon, 10 Aug 2026 12:01:46 -0700 Subject: [PATCH 4/4] reword --- echopype/utils/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/echopype/utils/io.py b/echopype/utils/io.py index 13ccba32e..99168f007 100644 --- a/echopype/utils/io.py +++ b/echopype/utils/io.py @@ -72,8 +72,8 @@ def save_file(ds, path, mode, engine, group=None, compression_settings=None, **k if engine == "netcdf4": ds.to_netcdf(path=path, mode=mode, group=group, encoding=encoding, engine=engine, **kwargs) elif engine == "zarr": - # Dask chunk according to dask formatted chunk encoding. As the zarr chunks are inherited - # from the dask formatted chunks, we avoid any potential chunking mismatch between the two. + # Rechunk Dask array according to dask formatted chunk encoding. The zarr chunks are based + # on the dask formatted chunks, so we avoid any potential chunking mismatch between them. # If we do not do this, we will need to call `ds.to_zrar(...,align_chunks=True,...)`, which # also "rechunks the Dask array to align with Zarr chunks before writing" # (https://docs.xarray.dev/en/latest/generated/xarray.Dataset.to_zarr.html).