modernize CIs

.github/dependabot.yml

@@ -13,3 +13,14 @@ updates:
       github-actions:
         patterns:
           - '*'
+    cooldown:
+      default-days: 7
+
+  - package-ecosystem: "pre-commit"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+      - "Bot"
+    cooldown:
+      default-days: 7

.github/workflows/pre-commit-checks.yml

@@ -0,0 +1,31 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+
+# Deny all permissions by default
+permissions: {}
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      with:
+        persist-credentials: false
+
+    - name: Set up Python
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
+      with:
+        python-version: "3.x"
+
+    - name: Install
+      shell: bash -l {0}
+      run: |
+        python -m pip install prek
+
+    - name: Run linters
+      run: >
+        prek run --all-files

.github/workflows/pypi.yml

@@ -14,13 +14,24 @@ defaults:
     shell: bash
 
 jobs:
-  packages:
+  pypi-publish:
+    name: Upload release to PyPI
     runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/thredds_crawler/
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      with:
+        # Should be enough for setuptools-scm
+        fetch-depth: 100
+        persist-credentials: false
 
     - name: Set up Python
-      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
         python-version: "3.x"
 
@@ -48,7 +59,4 @@ jobs:
 
     - name: Publish a Python distribution to PyPI
       if: success() && github.event_name == 'release'
-      uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc  # v1.12.4
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_PASSWORD }}
+      uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33  # v1.14.2

.github/workflows/tests.yml

@@ -3,37 +3,44 @@ name: Full Tests
 on:
   pull_request:
   push:
-    branches: [main]
+    branches: [ main ]
+
+# no permissions by default
+permissions: {}
 
 jobs:
   run:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ]
-        os: [windows-latest, ubuntu-latest, macos-latest]
+        python-version: [ "3.11", "3.15-dev" ]
+        os: [ ubuntu-latest ]
+        include:
+          - os: windows-latest
+            python-version: "3.15-dev"
+          - os: macos-latest
+            python-version: "3.15-dev"
+          - os: macos-15-intel
+            python-version: "3.15-dev"
       fail-fast: false
-    defaults:
-      run:
-        shell: bash -l {0}
 
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      with:
+        # Should be enough for setuptools-scm
+        fetch-depth: 100
+        persist-credentials: false
+
 
-    - name: Setup Micromamba Python ${{ matrix.python-version }}
-      uses: mamba-org/setup-micromamba@b09ef9b599704322748535812ca03efb2625677b  # v2.0.5
+    - name: Set up Python
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
       with:
-        environment-name: TEST
-        init-shell: bash
-        create-args: >-
-          python=${{ matrix.python-version }}
-          --file requirements.txt
-          --file requirements-dev.txt
-          --channel conda-forge
+        python-version: "3.x"
 
     - name: Install thredds_crawler
       run: |
-        python -m pip install -e . --no-deps --force-reinstall
+        python -m pip install --group dev
+        python -m pip install -e .
 
     - name: Full Tests
       run: |

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
+  rev: v6.0.0
   hooks:
     - id: trailing-whitespace
     - id: check-ast
@@ -14,7 +14,7 @@ repos:
       files: requirements-dev.txt
 
 - repo: https://github.com/keewis/blackdoc
-  rev: v0.3.9
+  rev: v0.4.6
   hooks:
     - id: blackdoc
 
@@ -26,7 +26,7 @@ repos:
       args: [--config=pyproject.toml]
 
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.4.1
+  rev: v2.4.3
   hooks:
     - id: codespell
       exclude: >
@@ -37,29 +37,32 @@ repos:
         - --ignore-words-list=pres
 
 - repo: https://github.com/asottile/add-trailing-comma
-  rev: v3.1.0
+  rev: v4.0.0
   hooks:
     - id: add-trailing-comma
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.9.4
+  rev: v0.16.8
   hooks:
     - id: ruff
       args: ["--fix", "--show-fixes"]
     - id: ruff-format
 
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: v2.5.0
+  rev: v2.29.4
   hooks:
     - id: pyproject-fmt
 
+- repo: https://github.com/woodruffw/zizmor-pre-commit
+  rev: v1.30.1
+  hooks:
+    - id: zizmor
+
 ci:
     autofix_commit_msg: |
         [pre-commit.ci] auto fixes from pre-commit.com hooks
 
         for more information, see https://pre-commit.ci
     autofix_prs: false
-    autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
-    autoupdate_schedule: monthly
     skip: []
     submodules: false

README.md

@@ -63,12 +63,12 @@ from thredds_crawler.crawl import Crawl
 
 print(Crawl.SKIPS)
 [
-  ".*files.*",
-  ".*Individual Files.*",
-  ".*File_Access.*",
-  ".*Forecast Model Run.*",
-  ".*Constant Forecast Offset.*",
-  ".*Constant Forecast Date.*"
+    ".*files.*",
+    ".*Individual Files.*",
+    ".*File_Access.*",
+    ".*Forecast Model Run.*",
+    ".*Constant Forecast Offset.*",
+    ".*Constant Forecast Date.*",
 ]
 ```
 
@@ -162,13 +162,9 @@ You can pass an auth parameter as needed. It needs to be a [requests compatible
 
 ```python
 from thredds_crawler.crawl import Crawl
+
 auth = ("user", "password")
-c = Crawl(
-  "http://tds.maracoos.org/thredds/MODIS.xml",
-  select=[".*-Agg"],
-  skip=Crawl.SKIPS,
-  auth=auth
-)
+c = Crawl("http://tds.maracoos.org/thredds/MODIS.xml", select=[".*-Agg"], skip=Crawl.SKIPS, auth=auth)
 ```
 
 
@@ -254,15 +250,15 @@ c = Crawl("http://tds.maracoos.org/thredds/MODIS.xml", select=[".*-Agg"])
 urls = [s.get("url") for d in c.datasets for s in d.services if s.get("service").lower() == "opendap"]
 print(urls)
 [
-  "http://tds.maracoos.org/thredds/dodsC/MODIS-Agg.nc",
-  "http://tds.maracoos.org/thredds/dodsC/MODIS-2009-Agg.nc",
-  "http://tds.maracoos.org/thredds/dodsC/MODIS-2010-Agg.nc",
-  "http://tds.maracoos.org/thredds/dodsC/MODIS-2011-Agg.nc",
-  "http://tds.maracoos.org/thredds/dodsC/MODIS-2012-Agg.nc",
-  "http://tds.maracoos.org/thredds/dodsC/MODIS-2013-Agg.nc",
-  "http://tds.maracoos.org/thredds/dodsC/MODIS-One-Agg.nc",
-  "http://tds.maracoos.org/thredds/dodsC/MODIS-Three-Agg.nc",
-  "http://tds.maracoos.org/thredds/dodsC/MODIS-Seven-Agg.nc"
+    "http://tds.maracoos.org/thredds/dodsC/MODIS-Agg.nc",
+    "http://tds.maracoos.org/thredds/dodsC/MODIS-2009-Agg.nc",
+    "http://tds.maracoos.org/thredds/dodsC/MODIS-2010-Agg.nc",
+    "http://tds.maracoos.org/thredds/dodsC/MODIS-2011-Agg.nc",
+    "http://tds.maracoos.org/thredds/dodsC/MODIS-2012-Agg.nc",
+    "http://tds.maracoos.org/thredds/dodsC/MODIS-2013-Agg.nc",
+    "http://tds.maracoos.org/thredds/dodsC/MODIS-One-Agg.nc",
+    "http://tds.maracoos.org/thredds/dodsC/MODIS-Three-Agg.nc",
+    "http://tds.maracoos.org/thredds/dodsC/MODIS-Seven-Agg.nc",
 ]
 ```
 
@@ -273,10 +269,7 @@ This isn"t necessarialy the size on disk, because it does not account for `missi
 ```python
 from thredds_crawler.crawl import Crawl
 
-c = Crawl(
-  "http://thredds.axiomalaska.com/thredds/catalogs/cencoos.html",
-  select=["MB_.*"]
-)
+c = Crawl("http://thredds.axiomalaska.com/thredds/catalogs/cencoos.html", select=["MB_.*"])
 sizes = [d.size for d in c.datasets]
 print(sizes)
 [29247.410283999998, 72166.289680000002]
@@ -316,7 +309,7 @@ import logging
 import logging.handlers
 
 logger = logging.getLogger("thredds_crawler")
-fh = logging.handlers.RotatingFileHandler("/var/log/iso_harvest/iso_harvest.log", maxBytes=1024*1024*10, backupCount=5)
+fh = logging.handlers.RotatingFileHandler("/var/log/iso_harvest/iso_harvest.log", maxBytes=1024 * 1024 * 10, backupCount=5)
 fh.setLevel(logging.DEBUG)
 ch = logging.StreamHandler()
 ch.setLevel(logging.DEBUG)
@@ -330,25 +323,25 @@ logger.setLevel(logging.DEBUG)
 SAVE_DIR = "/srv/http/iso"
 
 THREDDS_SERVERS = {
-    "aoos":      "http://thredds.axiomalaska.com/thredds/catalogs/aoos.html",
-    "cencoos":   "http://thredds.axiomalaska.com/thredds/catalogs/cencoos.html",
-    "maracoos" : "http://tds.maracoos.org/thredds/catalog.html",
-    "glos":      "http://tds.glos.us/thredds/catalog.html"
+    "aoos": "http://thredds.axiomalaska.com/thredds/catalogs/aoos.html",
+    "cencoos": "http://thredds.axiomalaska.com/thredds/catalogs/cencoos.html",
+    "maracoos": "http://tds.maracoos.org/thredds/catalog.html",
+    "glos": "http://tds.glos.us/thredds/catalog.html",
 }
 
 for subfolder, thredds_url in THREDDS_SERVERS.items():
-  logger.info("Crawling %s (%s)" % (subfolder, thredds_url))
-  crawler = Crawl(thredds_url, debug=True)
-  isos = [(d.id, s.get("url")) for d in crawler.datasets for s in d.services if s.get("service").lower() == "iso"]
-  filefolder = os.path.join(SAVE_DIR, subfolder)
-  if not os.path.exists(filefolder):
-    os.makedirs(filefolder)
-  for iso in isos:
-    try:
-      filename = iso[0].replace("/", "_") + ".iso.xml"
-      filepath = os.path.join(filefolder, filename)
-      logger.info("Downloading/Saving %s" % filepath)
-      urllib.urlretrieve(iso[1], filepath)
-    except BaseException:
-      logger.exception("Error!")
+    logger.info("Crawling %s (%s)" % (subfolder, thredds_url))
+    crawler = Crawl(thredds_url, debug=True)
+    isos = [(d.id, s.get("url")) for d in crawler.datasets for s in d.services if s.get("service").lower() == "iso"]
+    filefolder = os.path.join(SAVE_DIR, subfolder)
+    if not os.path.exists(filefolder):
+        os.makedirs(filefolder)
+    for iso in isos:
+        try:
+            filename = iso[0].replace("/", "_") + ".iso.xml"
+            filepath = os.path.join(filefolder, filename)
+            logger.info("Downloading/Saving %s" % filepath)
+            urllib.urlretrieve(iso[1], filepath)
+        except BaseException:
+            logger.exception("Error!")
 ```