From 9fabc826ecc6b6dffa91160df182d3339ffce048 Mon Sep 17 00:00:00 2001 From: JbutlerPDX Date: Thu, 8 Oct 2026 11:06:46 -0700 Subject: [PATCH 1/8] fix: Reworking snowtool download command --- CHANGELOG.md | 24 +++ src/snowtool/cli/download.py | 153 ++++++++++-------- src/snowtool/snowdb/config.py | 6 + src/snowtool/snowdb/db.py | 19 +++ .../{api/models => snowdb}/downloads.py | 41 ++++- tests/cli/test_download.py | 124 ++++++++++++++ 6 files changed, 297 insertions(+), 70 deletions(-) rename src/snowtool/{api/models => snowdb}/downloads.py (79%) create mode 100644 tests/cli/test_download.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 77e68da9..4db2fb28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,35 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +> **Live databases (raw download staging):** `snowtool download` previously +> wrote to a hardcoded `/d/projects/gisdata//unprocessed/`. It now +> stages under `downloads//` in the snowdb root, unless the root +> config's new `downloads` map says otherwise. Already-staged +> files are not moved. + +### Added + + - The root config (`snowdb_conf.json`) has a new map for `downloads`: A per-source + directory for `snowtool download`, resolved like any other config path. A source absent + from the map defaults to `downloads/` in the snowdb root. Existing configs without + the key keep working and accept default values + ### Changed + - The dataset URL models have been moved from `snowtool.api.models.downloads` to + `snowtool.snowdb.downloads`. Nothing in the API served the models, so it was moved + for less confusing import structures + - `snowtool download` no longer writes to a hardcoded destination root. See above notes + about live databases ### Removed ### Fixed + - `snowtool download` was not able to fetch anything as the request was never opened + in streaming mode. This would raise an AssertionError about the stream mode not being + enabled before any bytes reached the disk. + - `snowtool download`'s destination was a hardcoded absolute path, rather than where a user would intend + - `snowtool download` will no longer leave a partial '.part' file in the event of an OSError. Only + HTTP and Transport failures would clean the path ### Security diff --git a/src/snowtool/cli/download.py b/src/snowtool/cli/download.py index dcbb327d..7dcd7246 100644 --- a/src/snowtool/cli/download.py +++ b/src/snowtool/cli/download.py @@ -58,17 +58,20 @@ from curl_cffi import requests from curl_cffi.requests.exceptions import HTTPError, RequestException -from snowtool.api.models.downloads import ( +from snowtool.cli._context import config_option, pass_snowdb +from snowtool.cli._dates import DATE +from snowtool.snowdb import diagnostics +from snowtool.snowdb.db import SnowDb +from snowtool.snowdb.downloads import ( BaseUrl, + Downloader, DownloadResult, INSTARRUrls, SNODASUrl, SWANNUrl, ) -from snowtool.cli._context import config_option, pass_snowdb -from snowtool.cli._dates import DATE -from snowtool.snowdb import diagnostics -from snowtool.snowdb.db import SnowDb + +CHUNK_SIZE: int = 1024 * 1024 DEFAULT_TIMEOUT_SECONDS: int = 60 """ @@ -83,6 +86,8 @@ ), } +# NOTE: hand-maintained alongside the dataset registry in snowdb/datasets/. +# A dataset registered there but missing here will KeyError in `download`. SOURCE_MODELS: dict[str, type[BaseUrl]] = { 'swann': SWANNUrl, 'instarr': INSTARRUrls, @@ -90,47 +95,77 @@ } -def _get_file(url: str, dest: Path) -> DownloadResult: - """ - _get_file requests the file from the specified source - (http or ftp), and writes it to the desired destination +def _get_file( + url: str, + dest: Path, + session: Downloader | None = None, +) -> DownloadResult: + """Fetch one file over HTTP(S) or FTP and write it into ``dest``. + + The body is streamed to a sibling ``.part`` file and renamed into place + only once it is complete, so an interrupted download never leaves a + truncated file that later runs would mistake for a good one. Args: - url (str): url for requested file - dest (Path): _description_ + url: Fully-qualified URL of the file to fetch. + dest: Directory to write into. The filename is taken from ``url``. + session: Transport to fetch through. Defaults to a new + ``curl_cffi`` session, closed before returning; pass one in to + reuse a connection across several files, or to inject a fake. Returns: - DownloadResult: _description_ + A ``DownloadResult`` whose ``status`` is ``True`` on success, or + ``False`` with the failure in ``detail``. """ filename = Path(posixpath.basename(urlparse(url).path)) dest = dest / filename dest.parent.mkdir(parents=True, exist_ok=True) tmp_dest = dest.with_suffix(dest.suffix + '.part') + + owns_session = session is None + session = session if session is not None else requests.Session() try: - response = requests.get(url, impersonate='chrome') - response.raise_for_status() - with tmp_dest.open('wb') as f: - for chunk in response.iter_content(1024 * 1024): - f.write(chunk) + response = session.get(url, impersonate='chrome', stream=True) + try: + response.raise_for_status() + with tmp_dest.open('wb') as f: + for chunk in response.iter_content(CHUNK_SIZE): + f.write(chunk) + finally: + response.close() tmp_dest.rename(dest) - status = True - detail = '' - except HTTPError as e: - tmp_dest.unlink(missing_ok=True) - status = False - detail = str(e) - except RequestException as e: + except (HTTPError, RequestException, OSError) as e: tmp_dest.unlink(missing_ok=True) - status = False - detail = str(e) - except OSError as e: - status = False - detail = str(e) + return DownloadResult(status=False, detail=str(e)) + finally: + if owns_session: + session.close() - return DownloadResult( - status=status, - detail=detail, - ) + return DownloadResult(status=True, detail='') + + +def _download_sources( + sources: list[str], + dates: list[date], + snowdb: SnowDb, + session: Downloader, +) -> None: + """Fetch every file each source publishes for each date, logging outcomes.""" + for source_iter in sources: + for d in dates: + model = SOURCE_MODELS[source_iter]._for_date(d) + download_root = snowdb.download_root(source_iter) + for url, dest in model._iter_downloads(): + result = _get_file(url, download_root / dest, session) + if result.status: + logger.info('[%s] %s downloaded', source_iter, d) + else: + logger.warning( + '[%s] %s failed: %s', + source_iter, + d, + result.detail, + ) logger = logging.getLogger(__name__) @@ -144,7 +179,10 @@ def download() -> None: @download.command('date') @click.argument('date', type=click.DateTime(formats=['%Y-%m-%d']), required=True) @click.option('--source', '-s', type=str, multiple=True) +@config_option +@pass_snowdb def download_dates( + snowdb: SnowDb, date: datetime, source: tuple[str, ...] | None, ) -> None: @@ -159,19 +197,11 @@ def download_dates( Defaults to ['snodas', 'instarr', 'swann'] """ sources = list(source) if source else ['snodas', 'instarr', 'swann'] - for source_iter in sources: - model = SOURCE_MODELS[source_iter]._for_date(date.date()) - for url, dest in model._iter_downloads(): - result = _get_file(url, dest) - if result.status: - logger.info('[%s] %s downloaded', source_iter, date.date()) - else: - logger.warning( - '[%s] %s failed: %s', - source_iter, - date.date(), - result.detail, - ) + session = requests.Session() + try: + _download_sources(sources, [date.date()], snowdb, session) + finally: + session.close() @download.command('retry') @@ -203,20 +233,17 @@ def retry_download( retry_end = end if end else date.today() # noqa: DTZ011 sources = list(source) if source else ['snodas', 'instarr', 'swann'] - for source_iter in sources: - ds = snowdb.registered_dataset(source_iter) - missing = diagnostics.missing_dates(ds, start=retry_start, end=retry_end) - - if not missing: - logging.info('[%s] has no missing dates in range', source_iter) - continue - - click.echo(f'[{source_iter}] {len(missing)} missing date(s) found.') - for d in missing: - model = SOURCE_MODELS[source_iter]._for_date(d) - for url, dest in model._iter_downloads(): - result = _get_file(url, Path(dest)) - if result.status: - logger.info('[%s] %s downloaded', source_iter, d) - else: - logger.warning('[%s] %s failed: %s', source_iter, d, result.detail) + session = requests.Session() + try: + for source_iter in sources: + ds = snowdb.registered_dataset(source_iter) + missing = diagnostics.missing_dates(ds, start=retry_start, end=retry_end) + + if not missing: + logger.info('[%s] has no missing dates in range', source_iter) + continue + + click.echo(f'[{source_iter}] {len(missing)} missing date(s) found.') + _download_sources([source_iter], list(missing), snowdb, session) + finally: + session.close() diff --git a/src/snowtool/snowdb/config.py b/src/snowtool/snowdb/config.py index edd2b017..16b6fcc5 100644 --- a/src/snowtool/snowdb/config.py +++ b/src/snowtool/snowdb/config.py @@ -50,6 +50,7 @@ # (``data//``). The single name the layout, the read binder, and the # inline-dataset resolution convention all share. DATA_DIRNAME = 'data' +DOWNLOADS_DIRNAME = 'downloads' def resolve_path( @@ -332,6 +333,11 @@ class RootConfig(ResourceModel): # (3DEP for terrain, the MRLC bundle for land cover). sources: dict[str, Path] = Field(default_factory=dict) + # Per-source raw-download staging roots (source name -> directory; absolute, + # or relative to this config). A source absent here places downloads under + # ``downloads//`` in the root. + downloads: dict[str, Path] = Field(default_factory=dict) + # Where this config lives on disk: set when loaded/saved, ``None`` when built # in code. It is the base every relative link resolves against, so a config # without it cannot resolve relative paths (only absolute links / inline diff --git a/src/snowtool/snowdb/db.py b/src/snowtool/snowdb/db.py index b7e0cdad..aaad9e67 100644 --- a/src/snowtool/snowdb/db.py +++ b/src/snowtool/snowdb/db.py @@ -40,6 +40,7 @@ from snowtool.snowdb.config import ( CONFIG_FILENAME, DATA_DIRNAME, + DOWNLOADS_DIRNAME, DatasetConfig, InlineDatasetLink, RootConfig, @@ -169,6 +170,24 @@ def __init__( self._index: PourpointIndex | None = None self._index_mtime: int | None = None + # The download path for unproccessed data files that are to be ingested + # into Snowdb. Declared per source in the root configs "downloads" map + # and resolved like any other config path; an absent source defaults to + # "downloads/"" in the root + def download_root(self: Self, source: str) -> Path: + + configured = self.config.downloads.get(source) + if configured is not None: + return resolve_path(configured, root=self.root) + if self.root is None: + raise SnowDbConfigError( + None, + f'Cannot resolve the download root for {source}: This config' + " has no location. Give an absolute path in the config's" + ' "downloads" map, or save the config first.', + ) + return self.root / DOWNLOADS_DIRNAME / source + def bind_dataset_from_file( self: Self, name: str, diff --git a/src/snowtool/api/models/downloads.py b/src/snowtool/snowdb/downloads.py similarity index 79% rename from src/snowtool/api/models/downloads.py rename to src/snowtool/snowdb/downloads.py index 85f96219..dd3aa85e 100644 --- a/src/snowtool/api/models/downloads.py +++ b/src/snowtool/snowdb/downloads.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from datetime import date from pathlib import Path -from typing import ClassVar, Self +from typing import ClassVar, Protocol, Self @dataclass @@ -17,11 +17,9 @@ class DownloadResult: class BaseUrl(ABC): - BASE_DEST: ClassVar[str] = '/d/projects/gisdata/{source}/unprocessed/{import_path}' - @classmethod - def _build_dest(cls, source: str, import_path: str | Path) -> Path: - return Path(cls.BASE_DEST.format(source=source, import_path=import_path)) + def _build_dest(cls, import_path: str | Path) -> Path: + return Path(import_path) @classmethod @abstractmethod @@ -59,7 +57,6 @@ def _for_date(cls, target_date: date) -> INSTARRUrls: day=f'{target_date.day:02d}', ) dest = cls._build_dest( - 'instarr', f'{tile}/{target_date.year}/{target_date.month:02d}/', ) tile_downloads[download_url] = dest @@ -103,7 +100,9 @@ def _for_date(cls, target_date: date) -> SWANNUrl: ) return cls( url=download_url, - dest=cls._build_dest('swann', f'{wy!s}/{target_date.month:02d}/'), + dest=cls._build_dest( + f'{wy!s}/{target_date.month:02d}/', + ), ) def _iter_downloads(self) -> Iterator[tuple[str, Path]]: @@ -140,3 +139,31 @@ def _for_date(cls, target_date: date) -> SNODASUrl: def _iter_downloads(self) -> Iterator[tuple[str, Path]]: yield self.url, self.dest + + +class StreamingResponse(Protocol): + """The slice of a streamed response ``_get_file`` consumes.""" + + def raise_for_status(self) -> object: ... + + def iter_content(self, chunk_size: int) -> Iterator[bytes]: ... + + def close(self) -> None: ... + + +class Downloader(Protocol): + """The slice of ``curl_cffi.requests.Session`` that ``_get_file`` needs. + + Declared as a protocol so tests can inject a fake transport through the + ``session`` seam rather than patching the module or reaching the network. + """ + + def get( + self, + url: str, + *, + impersonate: str, + stream: bool, + ) -> StreamingResponse: ... + + def close(self) -> None: ... diff --git a/tests/cli/test_download.py b/tests/cli/test_download.py new file mode 100644 index 00000000..a25c7c81 --- /dev/null +++ b/tests/cli/test_download.py @@ -0,0 +1,124 @@ +"""``_get_file``'s write path, driven through an injected fake transport. + +The real transport is ``curl_cffi``; these tests pass a ``FakeSession`` +through ``_get_file``'s ``session`` seam instead, so the chunk-write, +atomic-rename, and failure-cleanup behavior is exercised with no network. +``test_download_network.py`` covers the real transport. +""" + +import pytest + +from curl_cffi.requests.exceptions import HTTPError, RequestException + +from snowtool.cli.download import _get_file + +URL = 'https://example.invalid/data/SNODAS_20260925.tar' +FILENAME = 'SNODAS_20260925.tar' +PART = FILENAME + '.part' + + +class FakeResponse: + """A streamed response that yields fixed chunks, or raises.""" + + def __init__(self, chunks=(), raises=None): + self._chunks = chunks + self._raises = raises + self.closed = False + + def raise_for_status(self): + if self._raises is not None: + raise self._raises + + def iter_content(self, chunk_size): + yield from self._chunks + + def close(self): + self.closed = True + + +class FakeSession: + """Records how it was called and hands back a prepared response.""" + + def __init__(self, response): + self._response = response + self.calls = [] + self.closed = False + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + return self._response + + def close(self): + self.closed = True + + +def test_streams_chunks_to_dest(tmp_path): + response = FakeResponse(chunks=[b'abc', b'de']) + session = FakeSession(response) + + result = _get_file(URL, tmp_path, session) + + assert result.status is True + assert result.detail == '' + assert (tmp_path / FILENAME).read_bytes() == b'abcde' + assert not (tmp_path / PART).exists() + assert response.closed is True + + +def test_requests_stream_mode(tmp_path): + """The regression guard: without ``stream=True`` iter_content asserts.""" + session = FakeSession(FakeResponse(chunks=[b'x'])) + + _get_file(URL, tmp_path, session) + + url, kwargs = session.calls[0] + assert url == URL + assert kwargs['stream'] is True + + +def test_filename_is_taken_from_the_url(tmp_path): + session = FakeSession(FakeResponse(chunks=[b'x'])) + + _get_file('https://example.invalid/a/b/UA_SWE_v1.nc', tmp_path, session) + + assert (tmp_path / 'UA_SWE_v1.nc').is_file() + + +def test_creates_missing_destination_directories(tmp_path): + session = FakeSession(FakeResponse(chunks=[b'x'])) + nested = tmp_path / '2026' / '09' + + result = _get_file(URL, nested, session) + + assert result.status is True + assert (nested / FILENAME).is_file() + + +@pytest.mark.parametrize( + 'error', + [ + HTTPError('HTTP Error 404: '), + RequestException('connection reset'), + OSError('No space left on device'), + ], +) +def test_failure_cleans_up_the_part_file(tmp_path, error): + response = FakeResponse(raises=error) + session = FakeSession(response) + + result = _get_file(URL, tmp_path, session) + + assert result.status is False + assert str(error) in result.detail + assert not (tmp_path / FILENAME).exists() + assert not (tmp_path / PART).exists() + assert response.closed is True + + +def test_an_injected_session_is_left_open(tmp_path): + """A caller-supplied session is reused across files, so _get_file keeps it.""" + session = FakeSession(FakeResponse(chunks=[b'x'])) + + _get_file(URL, tmp_path, session) + + assert session.closed is False From 56c86d9cf3da5c9e97a3ab4fda793f210ee9cc43 Mon Sep 17 00:00:00 2001 From: JbutlerPDX Date: Thu, 8 Oct 2026 16:30:21 -0700 Subject: [PATCH 2/8] fix: Correctly format SWANN url with water year --- src/snowtool/snowdb/downloads.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/snowtool/snowdb/downloads.py b/src/snowtool/snowdb/downloads.py index dd3aa85e..4ebf759b 100644 --- a/src/snowtool/snowdb/downloads.py +++ b/src/snowtool/snowdb/downloads.py @@ -74,7 +74,7 @@ class SWANNUrl(BaseUrl): dest: Path BASE_URL: ClassVar[str] = ( - 'https://climate.arizona.edu/data/UA_SWE/DailyData_800m/WY{year}/UA_SWE_Depth_800m_v1_{year}{month}{day}_early.nc' + 'https://climate.arizona.edu/data/UA_SWE/DailyData_800m/WY{water_year}/UA_SWE_Depth_800m_v1_{year}{month}{day}_early.nc' ) @staticmethod @@ -94,7 +94,8 @@ def _water_year(d: date) -> int: def _for_date(cls, target_date: date) -> SWANNUrl: wy = cls._water_year(target_date) download_url = cls.BASE_URL.format( - year=wy, + water_year=wy, + year=target_date.year, month=f'{target_date.month:02d}', day=f'{target_date.day:02d}', ) From 2594228f99f988054f6d0507f8834821860b714a Mon Sep 17 00:00:00 2001 From: JbutlerPDX Date: Thu, 8 Oct 2026 16:30:56 -0700 Subject: [PATCH 3/8] feat: Add new test for SWANNs water year directories --- tests/cli/test_download.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/cli/test_download.py b/tests/cli/test_download.py index a25c7c81..eaa1944d 100644 --- a/tests/cli/test_download.py +++ b/tests/cli/test_download.py @@ -6,11 +6,14 @@ ``test_download_network.py`` covers the real transport. """ +from datetime import date + import pytest from curl_cffi.requests.exceptions import HTTPError, RequestException from snowtool.cli.download import _get_file +from snowtool.snowdb.downloads import SWANNUrl URL = 'https://example.invalid/data/SNODAS_20260925.tar' FILENAME = 'SNODAS_20260925.tar' @@ -122,3 +125,17 @@ def test_an_injected_session_is_left_open(tmp_path): _get_file(URL, tmp_path, session) assert session.closed is False + + +@pytest.mark.parametrize( + ('target', 'expected_wy'), + [ + (date(2026, 9, 30), 2026, '20260930'), # last day of WY2026 + (date(2026, 10, 1), 2027, '20261001'), # first day of WY2027 + (date(2026, 10, 5), 2027, '20261005'), + (date(2027, 1, 15), 2027, '20270115'), + ], +) +def test_swann_correct_water_year(target, expected_wy): + url = SWANNUrl._for_date(target).url + assert f'/WY{expected_wy}/' in url From 1f4b02e05a713b5508c54ef30084d2dc01cdcfac Mon Sep 17 00:00:00 2001 From: JbutlerPDX Date: Thu, 8 Oct 2026 16:47:27 -0700 Subject: [PATCH 4/8] fix: Fix issue where download failed out on INSTARR --- src/snowtool/cli/download.py | 68 ++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/src/snowtool/cli/download.py b/src/snowtool/cli/download.py index 7dcd7246..c96cd4cc 100644 --- a/src/snowtool/cli/download.py +++ b/src/snowtool/cli/download.py @@ -48,10 +48,13 @@ import logging import posixpath +import shutil from datetime import date, datetime, timedelta from pathlib import Path +from urllib.error import URLError from urllib.parse import urlparse +from urllib.request import urlopen import click @@ -74,17 +77,7 @@ CHUNK_SIZE: int = 1024 * 1024 DEFAULT_TIMEOUT_SECONDS: int = 60 -""" -Some servers (e.g. climate.arizona.edu) reject requests with no/generic -User-Agent headers. Need to identify as a real browser to avoid spurious 403s. -""" -REQUEST_HEADERS: dict[str, str] = { - 'User-Agent': ( - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' - 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'Chrome/124.0.0.0 Safari/537.36' - ), -} + # NOTE: hand-maintained alongside the dataset registry in snowdb/datasets/. # A dataset registered there but missing here will KeyError in `download`. @@ -95,6 +88,27 @@ } +def _fetch_ftp(url: str, tmp_dest: Path) -> None: + """Stream an ``ftp://`` URL to ``tmp_dest`` using the standard library.""" + with ( + urlopen(url, timeout=DEFAULT_TIMEOUT_SECONDS) as response, # noqa: S310 + tmp_dest.open('wb') as f, + ): + shutil.copyfileobj(response, f, CHUNK_SIZE) + + +def _fetch_http(url: str, tmp_dest: Path, session: Downloader) -> None: + """Stream an ``http(s)://`` URL to ``tmp_dest`` through ``session``.""" + response = session.get(url, impersonate='chrome', stream=True) + try: + response.raise_for_status() + with tmp_dest.open('wb') as f: + for chunk in response.iter_content(CHUNK_SIZE): + f.write(chunk) + finally: + response.close() + + def _get_file( url: str, dest: Path, @@ -102,16 +116,11 @@ def _get_file( ) -> DownloadResult: """Fetch one file over HTTP(S) or FTP and write it into ``dest``. - The body is streamed to a sibling ``.part`` file and renamed into place - only once it is complete, so an interrupted download never leaves a - truncated file that later runs would mistake for a good one. - Args: url: Fully-qualified URL of the file to fetch. dest: Directory to write into. The filename is taken from ``url``. - session: Transport to fetch through. Defaults to a new - ``curl_cffi`` session, closed before returning; pass one in to - reuse a connection across several files, or to inject a fake. + session: Transport for HTTP(S) fetches. Defaults to a new ``curl_cffi`` + session, closed before returning. Unused for ``ftp://``. Returns: A ``DownloadResult`` whose ``status`` is ``True`` on success, or @@ -122,23 +131,20 @@ def _get_file( dest.parent.mkdir(parents=True, exist_ok=True) tmp_dest = dest.with_suffix(dest.suffix + '.part') - owns_session = session is None - session = session if session is not None else requests.Session() + is_ftp = urlparse(url).scheme == 'ftp' + owns_session = session is None and not is_ftp try: - response = session.get(url, impersonate='chrome', stream=True) - try: - response.raise_for_status() - with tmp_dest.open('wb') as f: - for chunk in response.iter_content(CHUNK_SIZE): - f.write(chunk) - finally: - response.close() + if is_ftp: + _fetch_ftp(url, tmp_dest) + else: + session = session if session is not None else requests.Session() + _fetch_http(url, tmp_dest, session) tmp_dest.rename(dest) - except (HTTPError, RequestException, OSError) as e: + except (HTTPError, RequestException, URLError, OSError) as e: tmp_dest.unlink(missing_ok=True) return DownloadResult(status=False, detail=str(e)) finally: - if owns_session: + if owns_session and session is not None: session.close() return DownloadResult(status=True, detail='') @@ -152,9 +158,9 @@ def _download_sources( ) -> None: """Fetch every file each source publishes for each date, logging outcomes.""" for source_iter in sources: + download_root = snowdb.download_root(source_iter) for d in dates: model = SOURCE_MODELS[source_iter]._for_date(d) - download_root = snowdb.download_root(source_iter) for url, dest in model._iter_downloads(): result = _get_file(url, download_root / dest, session) if result.status: From 09d61e0c34fdbe2467b430ae33015203f345e466 Mon Sep 17 00:00:00 2001 From: JbutlerPDX Date: Thu, 8 Oct 2026 16:51:57 -0700 Subject: [PATCH 5/8] fix: Removed unneeded target in download test --- tests/cli/test_download.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/cli/test_download.py b/tests/cli/test_download.py index eaa1944d..1cad5aad 100644 --- a/tests/cli/test_download.py +++ b/tests/cli/test_download.py @@ -130,10 +130,10 @@ def test_an_injected_session_is_left_open(tmp_path): @pytest.mark.parametrize( ('target', 'expected_wy'), [ - (date(2026, 9, 30), 2026, '20260930'), # last day of WY2026 - (date(2026, 10, 1), 2027, '20261001'), # first day of WY2027 - (date(2026, 10, 5), 2027, '20261005'), - (date(2027, 1, 15), 2027, '20270115'), + (date(2026, 9, 30), 2026), # last day of WY2026 + (date(2026, 10, 1), 2027), # first day of WY2027 + (date(2026, 10, 5), 2027), + (date(2027, 1, 15), 2027), ], ) def test_swann_correct_water_year(target, expected_wy): From f13aea2e8e04adbba158045c1791ecd6f66a00f1 Mon Sep 17 00:00:00 2001 From: JbutlerPDX Date: Thu, 8 Oct 2026 17:27:29 -0700 Subject: [PATCH 6/8] fix: Duplicate variable in config.py --- src/snowtool/snowdb/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/snowtool/snowdb/config.py b/src/snowtool/snowdb/config.py index 2d9d7928..4b61fa81 100644 --- a/src/snowtool/snowdb/config.py +++ b/src/snowtool/snowdb/config.py @@ -50,7 +50,6 @@ # (``data//``). The single name the layout, the read binder, and the # inline-dataset resolution convention all share. DATA_DIRNAME = 'data' -DOWNLOADS_DIRNAME = 'downloads' # The conventional subdirectory of the root holding every dataset's # raw data files (``downloads//``). From f76a637c4e0860c35c8c15131b0d74f0cb481ae3 Mon Sep 17 00:00:00 2001 From: JbutlerPDX Date: Thu, 8 Oct 2026 17:30:47 -0700 Subject: [PATCH 7/8] docs: Update changelog --- CHANGELOG.md | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae17e1c5..3c0a475b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,43 +7,23 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] -<<<<<<< HEAD -> **Live databases (raw download staging):** `snowtool download` previously -> wrote to a hardcoded `/d/projects/gisdata//unprocessed/`. It now -> stages under `downloads//` in the snowdb root, unless the root -> config's new `downloads` map says otherwise. Already-staged -> files are not moved. - ### Added - - The root config (`snowdb_conf.json`) has a new map for `downloads`: A per-source - directory for `snowtool download`, resolved like any other config path. A source absent - from the map defaults to `downloads/` in the snowdb root. Existing configs without - the key keep working and accept default values - -======= -### Added - ->>>>>>> main ### Changed - - The dataset URL models have been moved from `snowtool.api.models.downloads` to - `snowtool.snowdb.downloads`. Nothing in the API served the models, so it was moved - for less confusing import structures - - `snowtool download` no longer writes to a hardcoded destination root. See above notes - about live databases ### Removed ### Fixed - - `snowtool download` was not able to fetch anything as the request was never opened - in streaming mode. This would raise an AssertionError about the stream mode not being - enabled before any bytes reached the disk. - - `snowtool download`'s destination was a hardcoded absolute path, rather than where a user would intend - - `snowtool download` will no longer leave a partial '.part' file in the event of an OSError. Only - HTTP and Transport failures would clean the path ### Security +## [v0.8.1] - 2026-10-08 + +### Fixed + - `snowtool download` was breaking when attempting to download data + from INSTARR's FTP server. This was an issue with the curl_cffi package not + supporting FTP protocol. Created helper function to download files over FTP + ## [v0.8.0] - 2026-10-08 From 9ff822af2a9dd79153309dfbeb38670b2b1e8b44 Mon Sep 17 00:00:00 2001 From: JbutlerPDX Date: Thu, 8 Oct 2026 20:04:31 -0700 Subject: [PATCH 8/8] docs: Update changelog --- CHANGELOG.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0a475b..b9f35b01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,13 +24,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. from INSTARR's FTP server. This was an issue with the curl_cffi package not supporting FTP protocol. Created helper function to download files over FTP + - `snowtool download` requested the wrong SWANN file for any date in and after + October. The URL template used one `{year}` placeholder for both the water year + and the filename's calendar date, so a request for 2026-10-05 would resolve to + `WY2027/UA_SWE_Depth_800m_v1_20271005_early.nc` and return a 404. Fixed by creating + a separate placeholder for water year in building the request URL ## [v0.8.0] - 2026-10-08 > **Live databases (raw download staging):** `snowtool download` previously > wrote to a hardcoded `/d/projects/gisdata//unprocessed/`. It now > stages under `downloads//` in the snowdb root, unless the root -> config's new `downloads` map says otherwise. Decalre the current path +> config's new `downloads` map says otherwise. Declare the current path > in the map before upgrading. Already-staged > files are not moved. @@ -737,7 +742,9 @@ output. Initial release 🎉 -[Unreleased]: https://github.com/PSU-CSAR/snowtool/compare/v0.7.0...HEAD +[Unreleased]: https://github.com/PSU-CSAR/snowtool/compare/v0.8.1...HEAD +[v0.8.1]: https://github.com/PSU-CSAR/snowtool/compare/v0.8.0...v0.8.1 +[v0.8.0]: https://github.com/PSU-CSAR/snowtool/compare/v0.7.0...v0.8.0 [v0.7.0]: https://github.com/PSU-CSAR/snowtool/compare/v0.6.0...v0.7.0 [v0.6.0]: https://github.com/PSU-CSAR/snowtool/compare/v0.5.1...v0.6.0 [v0.5.1]: https://github.com/PSU-CSAR/snowtool/compare/v0.5.0...v0.5.1