diff --git a/CHANGELOG.md b/CHANGELOG.md index e02826b..3c0a475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Security +## [v0.8.1] - 2026-10-08 + +### Fixed + - `snowtool download` was breaking when attempting to download data + from INSTARR's FTP server. This was an issue with the curl_cffi package not + supporting FTP protocol. Created helper function to download files over FTP + ## [v0.8.0] - 2026-10-08 diff --git a/src/snowtool/cli/download.py b/src/snowtool/cli/download.py index cd1bd3f..55179e4 100644 --- a/src/snowtool/cli/download.py +++ b/src/snowtool/cli/download.py @@ -48,10 +48,13 @@ import logging import posixpath +import shutil from datetime import date, datetime, timedelta from pathlib import Path +from urllib.error import URLError from urllib.parse import urlparse +from urllib.request import urlopen import click @@ -74,17 +77,6 @@ CHUNK_SIZE: int = 1024 * 1024 DEFAULT_TIMEOUT_SECONDS: int = 60 -""" -Some servers (e.g. climate.arizona.edu) reject requests with no/generic -User-Agent headers. Need to identify as a real browser to avoid spurious 403s. -""" -REQUEST_HEADERS: dict[str, str] = { - 'User-Agent': ( - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' - 'AppleWebKit/537.36 (KHTML, like Gecko) ' - 'Chrome/124.0.0.0 Safari/537.36' - ), -} # NOTE: hand-maintained alongside the dataset registry in snowdb/datasets/. # A dataset registered there but missing here will KeyError in `download`. @@ -95,6 +87,27 @@ } +def _fetch_ftp(url: str, tmp_dest: Path) -> None: + """Stream an ``ftp://`` URL to ``tmp_dest`` using the standard library.""" + with ( + urlopen(url, timeout=DEFAULT_TIMEOUT_SECONDS) as response, # noqa: S310 + tmp_dest.open('wb') as f, + ): + shutil.copyfileobj(response, f, CHUNK_SIZE) + + +def _fetch_http(url: str, tmp_dest: Path, session: Downloader) -> None: + """Stream an ``http(s)://`` URL to ``tmp_dest`` through ``session``.""" + response = session.get(url, impersonate='chrome', stream=True) + try: + response.raise_for_status() + with tmp_dest.open('wb') as f: + for chunk in response.iter_content(CHUNK_SIZE): + f.write(chunk) + finally: + response.close() + + def _get_file( url: str, dest: Path, @@ -102,16 +115,11 @@ def _get_file( ) -> DownloadResult: """Fetch one file over HTTP(S) or FTP and write it into ``dest``. - The body is streamed to a sibling ``.part`` file and renamed into place - only once it is complete, so an interrupted download never leaves a - truncated file that later runs would mistake for a good one. - Args: url: Fully-qualified URL of the file to fetch. dest: Directory to write into. The filename is taken from ``url``. - session: Transport to fetch through. Defaults to a new - ``curl_cffi`` session, closed before returning; pass one in to - reuse a connection across several files, or to inject a fake. + session: Transport for HTTP(S) fetches. Defaults to a new ``curl_cffi`` + session, closed before returning. Unused for ``ftp://``. Returns: A ``DownloadResult`` whose ``status`` is ``True`` on success, or @@ -122,23 +130,20 @@ def _get_file( dest.parent.mkdir(parents=True, exist_ok=True) tmp_dest = dest.with_suffix(dest.suffix + '.part') - owns_session = session is None - session = session if session is not None else requests.Session() + is_ftp = urlparse(url).scheme == 'ftp' + owns_session = session is None and not is_ftp try: - response = session.get(url, impersonate='chrome', stream=True) - try: - response.raise_for_status() - with tmp_dest.open('wb') as f: - for chunk in response.iter_content(CHUNK_SIZE): - f.write(chunk) - finally: - response.close() + if is_ftp: + _fetch_ftp(url, tmp_dest) + else: + session = session if session is not None else requests.Session() + _fetch_http(url, tmp_dest, session) tmp_dest.rename(dest) - except (HTTPError, RequestException, OSError) as e: + except (HTTPError, RequestException, URLError, OSError) as e: tmp_dest.unlink(missing_ok=True) return DownloadResult(status=False, detail=str(e)) finally: - if owns_session: + if owns_session and session is not None: session.close() return DownloadResult(status=True, detail='') diff --git a/src/snowtool/snowdb/downloads.py b/src/snowtool/snowdb/downloads.py index 83d0159..9fa9cde 100644 --- a/src/snowtool/snowdb/downloads.py +++ b/src/snowtool/snowdb/downloads.py @@ -74,7 +74,7 @@ class SWANNUrl(BaseUrl): dest: Path BASE_URL: ClassVar[str] = ( - 'https://climate.arizona.edu/data/UA_SWE/DailyData_800m/WY{year}/UA_SWE_Depth_800m_v1_{year}{month}{day}_early.nc' + 'https://climate.arizona.edu/data/UA_SWE/DailyData_800m/WY{water_year}/UA_SWE_Depth_800m_v1_{year}{month}{day}_early.nc' ) @staticmethod @@ -94,7 +94,8 @@ def _water_year(d: date) -> int: def _for_date(cls, target_date: date) -> SWANNUrl: wy = cls._water_year(target_date) download_url = cls.BASE_URL.format( - year=wy, + water_year=wy, + year=target_date.year, month=f'{target_date.month:02d}', day=f'{target_date.day:02d}', ) diff --git a/tests/cli/test_download.py b/tests/cli/test_download.py index a25c7c8..1cad5aa 100644 --- a/tests/cli/test_download.py +++ b/tests/cli/test_download.py @@ -6,11 +6,14 @@ ``test_download_network.py`` covers the real transport. """ +from datetime import date + import pytest from curl_cffi.requests.exceptions import HTTPError, RequestException from snowtool.cli.download import _get_file +from snowtool.snowdb.downloads import SWANNUrl URL = 'https://example.invalid/data/SNODAS_20260925.tar' FILENAME = 'SNODAS_20260925.tar' @@ -122,3 +125,17 @@ def test_an_injected_session_is_left_open(tmp_path): _get_file(URL, tmp_path, session) assert session.closed is False + + +@pytest.mark.parametrize( + ('target', 'expected_wy'), + [ + (date(2026, 9, 30), 2026), # last day of WY2026 + (date(2026, 10, 1), 2027), # first day of WY2027 + (date(2026, 10, 5), 2027), + (date(2027, 1, 15), 2027), + ], +) +def test_swann_correct_water_year(target, expected_wy): + url = SWANNUrl._for_date(target).url + assert f'/WY{expected_wy}/' in url