diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7818bef..441d5c7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,3 +13,14 @@ updates: github-actions: patterns: - '*' + cooldown: + default-days: 7 + + - package-ecosystem: "pre-commit" + directory: "/" + schedule: + interval: "daily" + labels: + - "Bot" + cooldown: + default-days: 7 diff --git a/.github/workflows/pre-commit-checks.yml b/.github/workflows/pre-commit-checks.yml new file mode 100644 index 0000000..c94fec5 --- /dev/null +++ b/.github/workflows/pre-commit-checks.yml @@ -0,0 +1,31 @@ +name: pre-commit + +on: + pull_request: + push: + +# Deny all permissions by default +permissions: {} + +jobs: + run: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.x" + + - name: Install + shell: bash -l {0} + run: | + python -m pip install prek + + - name: Run linters + run: > + prek run --all-files diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 2bea62e..352968e 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -14,13 +14,24 @@ defaults: shell: bash jobs: - packages: + pypi-publish: + name: Upload release to PyPI runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/thredds_crawler/ + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Should be enough for setuptools-scm + fetch-depth: 100 + persist-credentials: false - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.x" @@ -48,7 +59,4 @@ jobs: - name: Publish a Python distribution to PyPI if: success() && github.event_name == 'release' - uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4 - with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6c82225..2cfffcb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,37 +3,44 @@ name: Full Tests on: pull_request: push: - branches: [main] + branches: [ main ] + +# no permissions by default +permissions: {} jobs: run: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ] - os: [windows-latest, ubuntu-latest, macos-latest] + python-version: [ "3.11", "3.15-dev" ] + os: [ ubuntu-latest ] + include: + - os: windows-latest + python-version: "3.15-dev" + - os: macos-latest + python-version: "3.15-dev" + - os: macos-15-intel + python-version: "3.15-dev" fail-fast: false - defaults: - run: - shell: bash -l {0} steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Should be enough for setuptools-scm + fetch-depth: 100 + persist-credentials: false + - - name: Setup Micromamba Python ${{ matrix.python-version }} - uses: mamba-org/setup-micromamba@b09ef9b599704322748535812ca03efb2625677b # v2.0.5 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - environment-name: TEST - init-shell: bash - create-args: >- - python=${{ matrix.python-version }} - --file requirements.txt - --file requirements-dev.txt - --channel conda-forge + python-version: "3.x" - name: Install thredds_crawler run: | - python -m pip install -e . --no-deps --force-reinstall + python -m pip install --group dev + python -m pip install -e . - name: Full Tests run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d582df..e1fa234 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: check-ast @@ -14,7 +14,7 @@ repos: files: requirements-dev.txt - repo: https://github.com/keewis/blackdoc - rev: v0.3.9 + rev: v0.4.6 hooks: - id: blackdoc @@ -26,7 +26,7 @@ repos: args: [--config=pyproject.toml] - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.3 hooks: - id: codespell exclude: > @@ -37,29 +37,32 @@ repos: - --ignore-words-list=pres - repo: https://github.com/asottile/add-trailing-comma - rev: v3.1.0 + rev: v4.0.0 hooks: - id: add-trailing-comma - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.4 + rev: v0.16.8 hooks: - id: ruff args: ["--fix", "--show-fixes"] - id: ruff-format - repo: https://github.com/tox-dev/pyproject-fmt - rev: v2.5.0 + rev: v2.29.4 hooks: - id: pyproject-fmt +- repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v1.30.1 + hooks: + - id: zizmor + ci: autofix_commit_msg: | [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci autofix_prs: false - autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' - autoupdate_schedule: monthly skip: [] submodules: false diff --git a/README.md b/README.md index 660eaa2..e50897d 100644 --- a/README.md +++ b/README.md @@ -63,12 +63,12 @@ from thredds_crawler.crawl import Crawl print(Crawl.SKIPS) [ - ".*files.*", - ".*Individual Files.*", - ".*File_Access.*", - ".*Forecast Model Run.*", - ".*Constant Forecast Offset.*", - ".*Constant Forecast Date.*" + ".*files.*", + ".*Individual Files.*", + ".*File_Access.*", + ".*Forecast Model Run.*", + ".*Constant Forecast Offset.*", + ".*Constant Forecast Date.*", ] ``` @@ -162,13 +162,9 @@ You can pass an auth parameter as needed. It needs to be a [requests compatible ```python from thredds_crawler.crawl import Crawl + auth = ("user", "password") -c = Crawl( - "http://tds.maracoos.org/thredds/MODIS.xml", - select=[".*-Agg"], - skip=Crawl.SKIPS, - auth=auth -) +c = Crawl("http://tds.maracoos.org/thredds/MODIS.xml", select=[".*-Agg"], skip=Crawl.SKIPS, auth=auth) ``` @@ -254,15 +250,15 @@ c = Crawl("http://tds.maracoos.org/thredds/MODIS.xml", select=[".*-Agg"]) urls = [s.get("url") for d in c.datasets for s in d.services if s.get("service").lower() == "opendap"] print(urls) [ - "http://tds.maracoos.org/thredds/dodsC/MODIS-Agg.nc", - "http://tds.maracoos.org/thredds/dodsC/MODIS-2009-Agg.nc", - "http://tds.maracoos.org/thredds/dodsC/MODIS-2010-Agg.nc", - "http://tds.maracoos.org/thredds/dodsC/MODIS-2011-Agg.nc", - "http://tds.maracoos.org/thredds/dodsC/MODIS-2012-Agg.nc", - "http://tds.maracoos.org/thredds/dodsC/MODIS-2013-Agg.nc", - "http://tds.maracoos.org/thredds/dodsC/MODIS-One-Agg.nc", - "http://tds.maracoos.org/thredds/dodsC/MODIS-Three-Agg.nc", - "http://tds.maracoos.org/thredds/dodsC/MODIS-Seven-Agg.nc" + "http://tds.maracoos.org/thredds/dodsC/MODIS-Agg.nc", + "http://tds.maracoos.org/thredds/dodsC/MODIS-2009-Agg.nc", + "http://tds.maracoos.org/thredds/dodsC/MODIS-2010-Agg.nc", + "http://tds.maracoos.org/thredds/dodsC/MODIS-2011-Agg.nc", + "http://tds.maracoos.org/thredds/dodsC/MODIS-2012-Agg.nc", + "http://tds.maracoos.org/thredds/dodsC/MODIS-2013-Agg.nc", + "http://tds.maracoos.org/thredds/dodsC/MODIS-One-Agg.nc", + "http://tds.maracoos.org/thredds/dodsC/MODIS-Three-Agg.nc", + "http://tds.maracoos.org/thredds/dodsC/MODIS-Seven-Agg.nc", ] ``` @@ -273,10 +269,7 @@ This isn"t necessarialy the size on disk, because it does not account for `missi ```python from thredds_crawler.crawl import Crawl -c = Crawl( - "http://thredds.axiomalaska.com/thredds/catalogs/cencoos.html", - select=["MB_.*"] -) +c = Crawl("http://thredds.axiomalaska.com/thredds/catalogs/cencoos.html", select=["MB_.*"]) sizes = [d.size for d in c.datasets] print(sizes) [29247.410283999998, 72166.289680000002] @@ -316,7 +309,7 @@ import logging import logging.handlers logger = logging.getLogger("thredds_crawler") -fh = logging.handlers.RotatingFileHandler("/var/log/iso_harvest/iso_harvest.log", maxBytes=1024*1024*10, backupCount=5) +fh = logging.handlers.RotatingFileHandler("/var/log/iso_harvest/iso_harvest.log", maxBytes=1024 * 1024 * 10, backupCount=5) fh.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) @@ -330,25 +323,25 @@ logger.setLevel(logging.DEBUG) SAVE_DIR = "/srv/http/iso" THREDDS_SERVERS = { - "aoos": "http://thredds.axiomalaska.com/thredds/catalogs/aoos.html", - "cencoos": "http://thredds.axiomalaska.com/thredds/catalogs/cencoos.html", - "maracoos" : "http://tds.maracoos.org/thredds/catalog.html", - "glos": "http://tds.glos.us/thredds/catalog.html" + "aoos": "http://thredds.axiomalaska.com/thredds/catalogs/aoos.html", + "cencoos": "http://thredds.axiomalaska.com/thredds/catalogs/cencoos.html", + "maracoos": "http://tds.maracoos.org/thredds/catalog.html", + "glos": "http://tds.glos.us/thredds/catalog.html", } for subfolder, thredds_url in THREDDS_SERVERS.items(): - logger.info("Crawling %s (%s)" % (subfolder, thredds_url)) - crawler = Crawl(thredds_url, debug=True) - isos = [(d.id, s.get("url")) for d in crawler.datasets for s in d.services if s.get("service").lower() == "iso"] - filefolder = os.path.join(SAVE_DIR, subfolder) - if not os.path.exists(filefolder): - os.makedirs(filefolder) - for iso in isos: - try: - filename = iso[0].replace("/", "_") + ".iso.xml" - filepath = os.path.join(filefolder, filename) - logger.info("Downloading/Saving %s" % filepath) - urllib.urlretrieve(iso[1], filepath) - except BaseException: - logger.exception("Error!") + logger.info("Crawling %s (%s)" % (subfolder, thredds_url)) + crawler = Crawl(thredds_url, debug=True) + isos = [(d.id, s.get("url")) for d in crawler.datasets for s in d.services if s.get("service").lower() == "iso"] + filefolder = os.path.join(SAVE_DIR, subfolder) + if not os.path.exists(filefolder): + os.makedirs(filefolder) + for iso in isos: + try: + filename = iso[0].replace("/", "_") + ".iso.xml" + filepath = os.path.join(filefolder, filename) + logger.info("Downloading/Saving %s" % filepath) + urllib.urlretrieve(iso[1], filepath) + except BaseException: + logger.exception("Error!") ``` diff --git a/pyproject.toml b/pyproject.toml index 66a6480..715d0ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,6 @@ build-backend = "setuptools.build_meta" requires = [ "setuptools>=42", "setuptools-scm", - "wheel", ] [project] @@ -14,19 +13,31 @@ license = { file = "LICENSE.txt" } maintainers = [ { name = "Kyle Wilcox", email = "kyle@axiomdatascience.com" }, ] -requires-python = ">=3.9" +requires-python = ">=3.11" classifiers = [ "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", ] dynamic = [ - "dependencies", "version", ] +dependencies = [ + "lxml", + "python-dateutil", + "pytz", + "requests", +] + +[dependency-groups] +dev = [ + "netcdf4", + "prek", + "pytest", +] urls.documentation = "https://ioos.github.io/thredds_crawler" urls.homepage = "https://github.com/ioos/thredds_crawler" urls.repository = "https://github.com/ioos/thredds_crawler" @@ -36,43 +47,41 @@ packages = [ "thredds_crawler", ] include-package-data = true - -[tool.setuptools.dynamic] -dependencies = { file = [ +dynamic.dependencies = { file = [ "requirements.txt", ] } [tool.setuptools_scm] +tag_regex = "^(?Pv)?(?P[^\\+]+)(?P.*)?$" write_to = "thredds_crawler/_version.py" write_to_template = "__version__ = '{version}'" -tag_regex = "^(?Pv)?(?P[^\\+]+)(?P.*)?$" - -[tool.check-manifest] -ignore = [ - ".coveragerc", - "docs", - "docs/*", -] - -[tool.pytest.ini_options] -filterwarnings = [ - "error:::thredds_crawler.*", - "ignore::UserWarning", - "ignore::RuntimeWarning", -] [tool.interrogate] +fail-under = 85 ignore-init-method = true ignore-init-module = false ignore-magic = false ignore-semiprivate = false ignore-private = false ignore-module = false -fail-under = 85 exclude = [ "docs", "tests", ] +color = true verbose = 1 quiet = false -color = true + +[tool.check-manifest] +ignore = [ + ".coveragerc", + "docs", + "docs/*", +] + +[tool.pytest] +ini_options.filterwarnings = [ + "error:::thredds_crawler.*", + "ignore::UserWarning", + "ignore::RuntimeWarning", +] diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index c1d3485..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -netcdf4 -pytest diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 2fa04b9..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -lxml -python-dateutil -pytz -requests