Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.

### Security

## [v0.8.1] - 2026-10-08

### Fixed
- `snowtool download` was breaking when attempting to download data
from INSTARR's FTP server. This was an issue with the curl_cffi package not
supporting FTP protocol. Created helper function to download files over FTP


## [v0.8.0] - 2026-10-08

Expand Down
65 changes: 35 additions & 30 deletions src/snowtool/cli/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,13 @@

import logging
import posixpath
import shutil

from datetime import date, datetime, timedelta
from pathlib import Path
from urllib.error import URLError
from urllib.parse import urlparse
from urllib.request import urlopen

import click

Expand All @@ -74,17 +77,6 @@
CHUNK_SIZE: int = 1024 * 1024

DEFAULT_TIMEOUT_SECONDS: int = 60
"""
Some servers (e.g. climate.arizona.edu) reject requests with no/generic
User-Agent headers. Need to identify as a real browser to avoid spurious 403s.
"""
REQUEST_HEADERS: dict[str, str] = {
'User-Agent': (
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/124.0.0.0 Safari/537.36'
),
}

# NOTE: hand-maintained alongside the dataset registry in snowdb/datasets/.
# A dataset registered there but missing here will KeyError in `download`.
Expand All @@ -95,23 +87,39 @@
}


def _fetch_ftp(url: str, tmp_dest: Path) -> None:
"""Stream an ``ftp://`` URL to ``tmp_dest`` using the standard library."""
with (
urlopen(url, timeout=DEFAULT_TIMEOUT_SECONDS) as response, # noqa: S310
tmp_dest.open('wb') as f,
):
shutil.copyfileobj(response, f, CHUNK_SIZE)


def _fetch_http(url: str, tmp_dest: Path, session: Downloader) -> None:
"""Stream an ``http(s)://`` URL to ``tmp_dest`` through ``session``."""
response = session.get(url, impersonate='chrome', stream=True)
try:
response.raise_for_status()
with tmp_dest.open('wb') as f:
for chunk in response.iter_content(CHUNK_SIZE):
f.write(chunk)
finally:
response.close()


def _get_file(
url: str,
dest: Path,
session: Downloader | None = None,
) -> DownloadResult:
"""Fetch one file over HTTP(S) or FTP and write it into ``dest``.

The body is streamed to a sibling ``.part`` file and renamed into place
only once it is complete, so an interrupted download never leaves a
truncated file that later runs would mistake for a good one.

Args:
url: Fully-qualified URL of the file to fetch.
dest: Directory to write into. The filename is taken from ``url``.
session: Transport to fetch through. Defaults to a new
``curl_cffi`` session, closed before returning; pass one in to
reuse a connection across several files, or to inject a fake.
session: Transport for HTTP(S) fetches. Defaults to a new ``curl_cffi``
session, closed before returning. Unused for ``ftp://``.

Returns:
A ``DownloadResult`` whose ``status`` is ``True`` on success, or
Expand All @@ -122,23 +130,20 @@ def _get_file(
dest.parent.mkdir(parents=True, exist_ok=True)
tmp_dest = dest.with_suffix(dest.suffix + '.part')

owns_session = session is None
session = session if session is not None else requests.Session()
is_ftp = urlparse(url).scheme == 'ftp'
owns_session = session is None and not is_ftp
try:
response = session.get(url, impersonate='chrome', stream=True)
try:
response.raise_for_status()
with tmp_dest.open('wb') as f:
for chunk in response.iter_content(CHUNK_SIZE):
f.write(chunk)
finally:
response.close()
if is_ftp:
_fetch_ftp(url, tmp_dest)
else:
session = session if session is not None else requests.Session()
_fetch_http(url, tmp_dest, session)
tmp_dest.rename(dest)
except (HTTPError, RequestException, OSError) as e:
except (HTTPError, RequestException, URLError, OSError) as e:
tmp_dest.unlink(missing_ok=True)
return DownloadResult(status=False, detail=str(e))
finally:
if owns_session:
if owns_session and session is not None:
session.close()

return DownloadResult(status=True, detail='')
Expand Down
5 changes: 3 additions & 2 deletions src/snowtool/snowdb/downloads.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ class SWANNUrl(BaseUrl):
dest: Path

BASE_URL: ClassVar[str] = (
'https://climate.arizona.edu/data/UA_SWE/DailyData_800m/WY{year}/UA_SWE_Depth_800m_v1_{year}{month}{day}_early.nc'
'https://climate.arizona.edu/data/UA_SWE/DailyData_800m/WY{water_year}/UA_SWE_Depth_800m_v1_{year}{month}{day}_early.nc'
)

@staticmethod
Expand All @@ -94,7 +94,8 @@ def _water_year(d: date) -> int:
def _for_date(cls, target_date: date) -> SWANNUrl:
wy = cls._water_year(target_date)
download_url = cls.BASE_URL.format(
year=wy,
water_year=wy,
year=target_date.year,
month=f'{target_date.month:02d}',
day=f'{target_date.day:02d}',
)
Expand Down
17 changes: 17 additions & 0 deletions tests/cli/test_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@
``test_download_network.py`` covers the real transport.
"""

from datetime import date

import pytest

from curl_cffi.requests.exceptions import HTTPError, RequestException

from snowtool.cli.download import _get_file
from snowtool.snowdb.downloads import SWANNUrl

URL = 'https://example.invalid/data/SNODAS_20260925.tar'
FILENAME = 'SNODAS_20260925.tar'
Expand Down Expand Up @@ -122,3 +125,17 @@ def test_an_injected_session_is_left_open(tmp_path):
_get_file(URL, tmp_path, session)

assert session.closed is False


@pytest.mark.parametrize(
('target', 'expected_wy'),
[
(date(2026, 9, 30), 2026), # last day of WY2026
(date(2026, 10, 1), 2027), # first day of WY2027
(date(2026, 10, 5), 2027),
(date(2027, 1, 15), 2027),
],
)
def test_swann_correct_water_year(target, expected_wy):
url = SWANNUrl._for_date(target).url
assert f'/WY{expected_wy}/' in url
Loading