diff --git a/.github/PULL_REQUEST_TEMPLATE/new_sr_method.md b/.github/PULL_REQUEST_TEMPLATE/new_sr_method.md index 6b0c60aea..7db8058e5 100644 --- a/.github/PULL_REQUEST_TEMPLATE/new_sr_method.md +++ b/.github/PULL_REQUEST_TEMPLATE/new_sr_method.md @@ -7,22 +7,42 @@ If you need help with the PR, feel free to tag @srbench-comp and we'll respond a --> ## Submission Checklist +A submission spans **two** directories, using the same name in both. See +[CONTRIBUTING.md](../../CONTRIBUTING.md#where-your-files-go) for the details. + - [ ] title of this PR is meaningful, i.e. "adding method X" -- [ ] A folder has been added to `algorithms/` with a meaningful name corresponding to your method name. -- [ ] The added folder includes these elements: - - [ ] `metadata.yml` (**required**): A file describing your submission, following the descriptions in (algorithms/feat/metadata.yml). - - [ ] `regressor.py` (**required**): a Python file that defines your method, named appropriately. See [algorithms/feat/regressor.py][regressor] for complete documentation. - `regressor.py` contains: - - [ ] `est`: a sklearn-compatible `Regressor` object. - - [ ] `model(est, X=None)`: a function that returns a [**sympy-compatible**](https://www.sympy.org) string specifying the final model. It can optionally take the training data as an input argument. - - [ ] `eval_kwargs` *(optional)*: a dictionary that can specify method-specific arguments to `evaluate_model.py`. - - [ ] `LICENSE` *(optional)* A license file - - [ ] `environment.yml` *(optional)*: a [conda environment file](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-from-an-environment-yml-file) that specifies dependencies for your submission. - It will be used to update the baseline environment (`environment.yml` in the root directory). - To the extent possible, conda should be used to specify the dependencies you need. - If your method is part of conda, great! You can just put that in here and leave `install.sh` blank. - - [ ] `requirements.txt` *(optional)*: a pypi requirements file. The script will run `pip install -r requirements.txt` if this file is found, before proceeding. - - [ ] `install.sh` *(optional)*: a bash script that installs your method **without sudo permissions**. -- [ ] I did not include source code; instead I used `install.sh` to pull it from a stable source repository. - -- [ ] I locally tested that `bash local_ci.sh [method-folder-name]` runs successfully without error. \ No newline at end of file + +**`algorithms//`** — how to install your method + +- [ ] `metadata.yml` (**required**): A file describing your submission, following the descriptions in [algorithms/feat/metadata.yml][metadata]. `name`, `authors`, `email`, `description` and `url` are filled in. +- [ ] `LICENSE` *(optional)* A license file +- [ ] `environment.yml` *(optional)*: a [conda environment file](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-from-an-environment-yml-file) that specifies dependencies for your submission. + It will be used to update the baseline environment (`base_environment.yml` in the root directory). + To the extent possible, conda should be used to specify the dependencies you need. + If your method is part of conda, great! You can just put that in here and leave `install.sh` blank. +- [ ] `requirements.txt` *(optional)*: a pypi requirements file. The script will run `pip install -r requirements.txt` if this file is found, before proceeding. +- [ ] `install.sh` *(optional)*: a bash script that installs your method **without sudo permissions**. +- [ ] I did not include source code; instead I used `install.sh` to pull it from a stable source repository. + +**`experiment/methods//`** — how to call your method + +- [ ] `regressor.py` (**required**): a Python file that defines your method. See [experiment/methods/feat/regressor.py][regressor] for complete documentation. + `regressor.py` contains: + - [ ] `est`: a sklearn-compatible `Regressor` object. + - [ ] `model(est, X=None)`: a function that returns a [**sympy-compatible**](https://www.sympy.org) string specifying the final model. It can optionally take the training data as an input argument. + - [ ] `eval_kwargs` *(optional)*: a dictionary that can specify method-specific arguments to `evaluate_model.py`. +- [ ] `__init__.py` (**required**): an empty file, so the harness can import your method. + +**Checks** + +- [ ] `python scripts/check_method_layout.py` passes. +- [ ] I locally tested my method with: + ``` + bash scripts/make_docker_compose_file.sh + docker compose build base + docker compose build + docker compose run --rm bash test.sh + ``` + +[metadata]: https://github.com/cavalab/srbench/blob/master/algorithms/feat/metadata.yml +[regressor]: https://github.com/cavalab/srbench/blob/master/experiment/methods/feat/regressor.py \ No newline at end of file diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index bbce4ecc1..006abc10f 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -33,6 +33,19 @@ on: - '.github/workflows/**' jobs: + # Fast structural check. The build-and-test matrix below is generated from + # `ls algorithms/`, so a method added only under experiment/methods/ would + # otherwise get a full set of green checks without ever being built. + validate-layout: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: pip install pyyaml + - run: python scripts/check_method_layout.py + list-algs: runs-on: ubuntu-latest outputs: @@ -128,7 +141,8 @@ jobs: build-and-test: runs-on: ubuntu-latest - needs: + needs: + - validate-layout - check-changes - print-changes - list-algs @@ -139,7 +153,10 @@ jobs: matrix: alg: ${{ fromJson(needs.list-algs.outputs.matrix) }} fail-fast: false - if: always() + # always() keeps the matrix running even when check-changes reports no + # changes, but a layout failure means the images would be built from a + # method that cannot be imported -- don't spend ~27 docker builds on that. + if: always() && needs.validate-layout.result == 'success' steps: - uses: actions/checkout@v4 - name: Check if algorithm has changed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 42c9d0f31..033acbbb6 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,25 +26,76 @@ You can leverage this code base and previous experimental results to do so. - An open-source method with a [scikit-learn compatible API](https://scikit-learn.org/stable/developers/develop.html) - Your method should be compatible with **Python 3.7 or higher** to ensure compatibility with conda-forge. - If your method uses a random seed, it should have a `random_state` attribute that can be set. -- Methods must have their own folders in the `algorithms` directory (e.g., `algorithms/feat`). -This folder should contain: - 1. `metadata.yml` (**required**): A file describing your submission, following the descriptions in [algorithms/feat/metadata.yml][metadata]. - 2. `regressor.py` (**required**): a Python file that defines your method, named appropriately. See [algorithms/feat/regressor.py][regressor] for complete documentation. - It should contain: - - `est`: a sklearn-compatible `Regressor` object. - - `model(est, X=None)`: a function that returns a [**sympy-compatible**](https://www.sympy.org) string specifying the final model. It can optionally take the training data as an input argument. See [guidance below](###-returning-a-sympy-compatible-model-string). - - `eval_kwargs` (optional): a dictionary that can specify method-specific arguments to `evaluate_model.py`. - - We expect your algorithm to have a `max_time` parameter that lets us control the maximum execution time in seconds. When running the experiments in a cluster, we will give extra time to compensate for the overhead of initializing everything, and the maximum time considered is just the fit process. A signal `signal.SIGALRM` will be sent to your process if `fit(X, y)` exceeds the maximum time, and you can implement strategies to handle this signal. One idea is to store a random initial solution as the best and update it during the execution to ensure the `evaluate_model.py` script will find an equation to work on. - 3. `LICENSE` *(optional)* A license file - 4. `environment.yml` *(optional)*: a [conda environment file](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-from-an-environment-yml-file) that specifies dependencies for your submission. - It will be used to update the baseline environment (`environment.yml` in the root directory). - To the extent possible, conda should be used to specify the dependencies you need. - If your method is part of conda, great! You can just put that in here and leave `install.sh` blank. - 5. `requirements.txt` *(optional)*: a pypi requirements file. The script will run `pip install -r requirements.txt` if this file is found, before proceeding. - 5. `install.sh` *(optional)*: a bash script that installs your method. + +### Where your files go + +A submission spans **two** directories, and both are required. +This is the single most common thing to get wrong, so it is worth reading closely. + +``` +algorithms// # how to INSTALL your method +├── metadata.yml # required +├── environment.yml # optional +├── requirements.txt # optional +├── install.sh # optional +├── Dockerfile # optional +└── LICENSE # optional + +experiment/methods// # how to CALL your method +├── regressor.py # required +└── __init__.py # required (an empty file) +``` + +The split follows from how the benchmark runs. +`algorithms//` is copied into your Docker image when it is built, so it holds everything needed to *install* your method. +`experiment/` is mounted into the running container, so `regressor.py` is read at *run* time and is never baked into the image. + +Use the **same directory name** in both places. +Note that `metadata.yml` belongs with the install files in `algorithms/`, not next to `regressor.py`. + +You can check your layout before opening a PR: + +```bash +python scripts/check_method_layout.py +``` + +CI runs this same check, and it will fail your PR if anything is out of place. + +#### `algorithms//` + + 1. `metadata.yml` (**required**): A file describing your submission, following the descriptions in [algorithms/feat/metadata.yml][metadata]. Please fill in `name`, `authors`, `email`, `description` and `url`. + 2. `LICENSE` *(optional)* A license file + 3. `environment.yml` *(optional)*: a [conda environment file](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-from-an-environment-yml-file) that specifies dependencies for your submission. + It will be used to update the baseline environment (`base_environment.yml` in the root directory). + To the extent possible, conda should be used to specify the dependencies you need. + If your method is part of conda, great! You can just put that in here and leave `install.sh` blank. + 4. `requirements.txt` *(optional)*: a pypi requirements file. The script will run `pip install -r requirements.txt` if this file is found, before proceeding. + 5. `install.sh` *(optional)*: a bash script that installs your method. **Note: scripts should not require sudo permissions. The library and include paths should be directed to conda environment; the environmental variable `$CONDA_PREFIX` specifies the path to the environment. 6. `Dockerfile` *(optional)*: we will try to dockerize all algorithms. You can optionally have a `Dockerfile` inside your `algorithms/your-submission` folder to describe specific images for running your algorithm. If no file is provided, it will use `alg-Dockerfile` for your container. You can specify the image as you like, as long as you have as minimal dependences the python packages described in `base_environment.yml`, as they are used to run the experiment scripts. See [this example](algorithms/tir/Dockerfile) in case you want to use a custom image. *Notice that there is a workflow to build the docker images and push them to dockerhub*. - 7. **do not include your source code**. use `install.sh` to pull it from a stable source repository. + 7. **do not include your source code**. use `install.sh` to pull it from a stable source repository. + +#### `experiment/methods//` + + 1. `regressor.py` (**required**): a Python file that defines your method. See [experiment/methods/feat/regressor.py][regressor] for complete documentation. + It should contain: + - `est`: a sklearn-compatible `Regressor` object. + - `model(est, X=None)`: a function that returns a [**sympy-compatible**](https://www.sympy.org) string specifying the final model. It can optionally take the training data as an input argument. See [guidance below](#model-compatibility-with-sympy). + - `eval_kwargs` (optional): a dictionary that can specify method-specific arguments to `evaluate_model.py`. Only these keys are recognized: `test_params`, `max_train_samples`, `scale_x`, `scale_y`, `pre_train`, `use_dataframe`. + - We expect your algorithm to have a `max_time` parameter that lets us control the maximum execution time in seconds. When running the experiments in a cluster, we will give extra time to compensate for the overhead of initializing everything, and the maximum time considered is just the fit process. A signal `signal.SIGALRM` will be sent to your process if `fit(X, y)` exceeds the maximum time, and you can implement strategies to handle this signal. One idea is to store a random initial solution as the best and update it during the execution to ensure the `evaluate_model.py` script will find an equation to work on. + - The harness looks for the time limit under any of these attribute names: `max_time`, `timeout_in_seconds`, `timeout`, `stop_time`, `time_limit`. If your estimator exposes none of them, it will simply be killed when it runs long. + 2. `__init__.py` (**required**): an empty file, so the harness can import your method. + +### Testing your submission locally + +Build your image and run the same tests CI runs: + +```bash +bash scripts/make_docker_compose_file.sh # regenerates docker-compose.yml +docker compose build base # the shared base image, needed once +docker compose build +docker compose run --rm bash test.sh +``` ### model compatibility with sympy @@ -64,3 +115,7 @@ def model(est, X=None): ``` 2. The operators/functions in the model are available in [sympy's function set](https://docs.sympy.org/latest/modules/functions/index.html). + +[metadata]: https://github.com/cavalab/srbench/blob/master/algorithms/feat/metadata.yml +[regressor]: https://github.com/cavalab/srbench/blob/master/experiment/methods/feat/regressor.py + diff --git a/README.md b/README.md index 07cc19153..30fd0e9b8 100755 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/cavalab/srbench.git A detailed guide on how to reproduce the experiments by yourself is provided in [`docs/user_guide.md`](./docs/user_guide.md). -Once you get all the results, you nee to collate the results using the `collate` scripts in [`./postprocessing/scripts`](./postprocessing/scripts/collate_experiments_results.py) +Once you get all the results, you need to collate them using the `collate` scripts in [`./postprocessing/`](./postprocessing/): [`collate_blackbox_results.py`](./postprocessing/collate_blackbox_results.py) and [`collate_groundtruth_results.py`](./postprocessing/collate_groundtruth_results.py). # References diff --git a/algorithms/eql/metadata.yml b/algorithms/eql/metadata.yml index 10ae0e82a..578d60abc 100644 --- a/algorithms/eql/metadata.yml +++ b/algorithms/eql/metadata.yml @@ -3,7 +3,7 @@ authors: # the participants email: alessandro.simon@tuebingen.mpg.de name: EQL # name of the submission /method description: | # anything you'd like here to describe the method. -Implementation of the Equation Learner architecture as described in -'Learning equations for extrapolation and control' (Sahoo et. al) + Implementation of the Equation Learner architecture as described in + 'Learning equations for extrapolation and control' (Sahoo et. al) url: https://al.is.mpg.de/research_projects/symbolic-regression-and-equation-learning # a link to the project diff --git a/docs/user_guide.md b/docs/user_guide.md index ee72da9aa..c4e85338b 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -7,7 +7,7 @@ should check out the [v2.0 release](https://github.com/cavalab/srbench/releases/ ### Local install -We have provided a [conda environment](../base_environment.yml), [configuration script](configure.sh), and [installation script](../scripts/install_algorithm.sh) that should make installation straightforward. +We have provided a [conda environment](../base_environment.yml), [configuration script](../configure.sh), and [installation script](../scripts/install_algorithm.sh) that should make installation straightforward. The installation script is the same used internally when building the docker images. We've currently tested this on Ubuntu and CentOS. @@ -23,7 +23,7 @@ conda config --set solver libmamba 1. Install the conda environment naming it `srbench`: ```bash -conda env create -f environment.yml -n srbench +conda env create -f base_environment.yml -n srbench conda activate srbench ``` diff --git a/experiment/methods/gplearn/regressor.py b/experiment/methods/gplearn/regressor.py index 99fa7a329..2b7c3564f 100644 --- a/experiment/methods/gplearn/regressor.py +++ b/experiment/methods/gplearn/regressor.py @@ -37,4 +37,4 @@ def model(est, X=None): def complexity(est): #TODO: check - return len(re.split('\(|,',model(est))) + return len(re.split(r'\(|,',model(est))) diff --git a/experiment/methods/xgboost/__init__.py b/experiment/methods/xgboost/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/local_ci.sh b/local_ci.sh index 0309af676..134cab89a 100644 --- a/local_ci.sh +++ b/local_ci.sh @@ -1,67 +1,36 @@ -SUBNAME=$1 -SUBFOLDER="algorithms/$1" -echo "testing $SUBFOLDER" -# SUBFOLDER=official_competitors/$SUBNAME -SUBENV=srbench-$SUBNAME -# update base env -# mamba env update -n srbench -f environment.yml +#!/bin/bash +# Build and test one algorithm the same way CI does. +# +# bash local_ci.sh +# +# must match a directory in both algorithms/ and +# experiment/methods/. Run from the repository root. +set -euo pipefail -# install method -cd $SUBFOLDER -pwd -echo "Installing dependencies for ${SUBNAME}" -echo "........................................" -echo "Copying base environment" -echo "........................................" -conda create --name $SUBENV --clone srbench -if [ -e environment.yml ] ; then - echo "Installing conda dependencies" - echo "........................................" - mamba env update -n $SUBENV -f environment.yml -fi -if [ -e requirements.txt ] ; then - echo "Installing pip dependencies" - echo "........................................" - mamba run -n $SUBENV pip install -r requirements.txt +SUBNAME="${1:-}" +if [ -z "$SUBNAME" ]; then + echo "usage: bash local_ci.sh " >&2 + exit 1 fi -eval "$(conda shell.bash hook)" -conda init bash -conda activate $SUBENV -if test -f "install.sh" ; then -echo "running install.sh..." -echo "........................................" -bash install.sh -else -echo "::warning::No install.sh file found in ${SUBFOLDER}. Assuming the method is a conda package specified in environment.yml." +if [ ! -d "algorithms/${SUBNAME}" ] || [ ! -d "experiment/methods/${SUBNAME}" ]; then + echo "error: ${SUBNAME} needs a directory in BOTH algorithms/ and experiment/methods/" >&2 + echo "see CONTRIBUTING.md for the expected layout" >&2 + exit 1 fi -# Copy files and environment -echo "Copying files and environment to experiment/methods ..." -echo "........................................" -cd ../../ -mkdir -p experiment/methods/$SUBNAME -cp $SUBFOLDER/regressor.py experiment/methods/$SUBNAME/ -cp $SUBFOLDER/metadata.yml experiment/methods/$SUBNAME/ -touch experiment/methods/$SUBNAME/__init__.py +echo "==> checking method layout" +python scripts/check_method_layout.py + +echo "==> regenerating docker-compose.yml" +bash scripts/make_docker_compose_file.sh -# export env -echo "Exporting environment" -conda env export -n $SUBENV > $SUBFOLDER/environment.lock.yml +# Algorithm images are FROM srbench/base, so the base image has to exist first. +echo "==> building base image" +docker compose build base -# Test Method -cd experiment -pwd -ls -echo "activating conda env $SUBENV..." -echo "........................................" -conda activate $SUBENV -conda env list -conda info -python -m pytest -v test_algorithm.py --ml $SUBNAME -python -m pytest -v test_evaluate_model.py --ml $SUBNAME +echo "==> building ${SUBNAME}" +docker compose build "${SUBNAME}" -# Store Competitor -# cd .. -# rsync -avz --exclude=".git" submission/$SUBNAME official_competitors/ -# rm -rf submission/$SUBNAME +echo "==> testing ${SUBNAME}" +docker compose run --rm "${SUBNAME}" bash test.sh diff --git a/scripts/check_method_layout.py b/scripts/check_method_layout.py new file mode 100644 index 000000000..affb53578 --- /dev/null +++ b/scripts/check_method_layout.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Validate that each benchmarked method is wired up correctly. + +SRBench splits every method across two directories: + + algorithms// install spec, baked into the docker image at build time + experiment/methods// regressor.py, bind-mounted over /srbench at run time + +Both are required. A method that lands in only one of them can still show a full +set of green checks -- the `build-and-test` matrix is generated from `ls algorithms/`, +so a submission that touches only `experiment/methods/` never gets a job at all. +This script closes that gap. + +Run it locally before opening a PR: + + python scripts/check_method_layout.py + +Exit status is 0 when every check passes, 1 otherwise. Warnings never fail the run. +""" + +from __future__ import annotations + +import ast +import os +import sys + +try: + import yaml +except ImportError: # pragma: no cover - only hit outside the srbench env + yaml = None + +ALG_DIR = "algorithms" +METHOD_DIR = os.path.join("experiment", "methods") + +# Directories under experiment/methods/ that intentionally have no algorithms/ +# counterpart: 2021-era estimator variants, sklearn baselines, and the tuned +# estimators used by `analyze.py -tuned`. They are not containerized and are not +# part of the current benchmark roster. Do not add new entries here -- a new +# method needs both directories. See open issue #161 on retiring these. +LEGACY_METHOD_DIRS = { + "afp_ehc", + "afp_fe", + "experimental", + "geneticengine_1p1", + "geneticengine_hc", + "geneticengine_rs", + "sklearn_adaboost", + "sklearn_lasso", + "sklearn_linear", + "sklearn_mlp", + "sklearn_randomforest", + "sklearn_ridge", + "sklearn_sgd", + "tuned", +} + +# metadata.yml predates any validation and 10 of these files are empty. Existing +# methods are grandfathered so CI stays green; new methods must fill it in. +# Shrinking this set is a good standalone cleanup PR. +METADATA_GRANDFATHERED = { + "bsr", + "eplex", + "ffx", + "gplearn", + "itea", + "lightgbm", + "nesymres", + "sklearn", + "tir", + "xgboost", +} + +REQUIRED_METADATA_KEYS = ("name", "authors", "email", "description", "url") + +errors: list[str] = [] +warnings: list[str] = [] + + +def error(method: str, msg: str) -> None: + errors.append(f"{method}: {msg}") + + +def warn(method: str, msg: str) -> None: + warnings.append(f"{method}: {msg}") + + +def top_level_names(path: str) -> set[str] | None: + """Names bound at module scope, without importing (imports would need deps).""" + try: + tree = ast.parse(open(path, encoding="utf-8").read()) + except SyntaxError as exc: + error(path, f"regressor.py is not valid Python: {exc}") + return None + + names: set[str] = set() + for node in tree.body: + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + names.add(target.id) + elif isinstance(node, ast.AnnAssign): + # `est: RegressorMixin = FeatRegressor(...)` -- the common style here + if isinstance(node.target, ast.Name): + names.add(node.target.id) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(node.name) + elif isinstance(node, ast.Import): + for alias in node.names: + names.add((alias.asname or alias.name).split(".")[0]) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + names.add(alias.asname or alias.name) + return names + + +def check_algorithm(name: str) -> None: + """Every algorithms// needs a matching runnable method.""" + alg_path = os.path.join(ALG_DIR, name) + + metadata = os.path.join(alg_path, "metadata.yml") + if not os.path.isfile(metadata): + error(name, f"missing {metadata}") + elif yaml is not None: + try: + parsed = yaml.safe_load(open(metadata, encoding="utf-8")) + except yaml.YAMLError as exc: + error(name, f"{metadata} is not valid YAML: {exc}") + parsed = None + if parsed is not None or name not in METADATA_GRANDFATHERED: + missing = [k for k in REQUIRED_METADATA_KEYS if k not in (parsed or {})] + if missing and name not in METADATA_GRANDFATHERED: + error(name, f"{metadata} is missing required key(s): {', '.join(missing)}") + elif missing: + warn(name, f"{metadata} is incomplete (missing {', '.join(missing)})") + + method_path = os.path.join(METHOD_DIR, name) + regressor = os.path.join(method_path, "regressor.py") + if not os.path.isdir(method_path): + error( + name, + f"has {alg_path}/ but no {method_path}/ -- regressor.py lives under " + f"{METHOD_DIR}//, not in {ALG_DIR}//", + ) + return + if not os.path.isfile(regressor): + error(name, f"missing {regressor}") + return + + if not os.path.isfile(os.path.join(method_path, "__init__.py")): + error(name, f"missing {method_path}/__init__.py (an empty file is fine)") + + names = top_level_names(regressor) + if names is None: + return + for required in ("est", "model"): + if required not in names: + error(name, f"{regressor} does not define `{required}` at module level") + + if os.path.isfile(os.path.join(alg_path, "regressor.py")): + warn(name, f"{alg_path}/regressor.py is ignored; the harness imports {regressor}") + if os.path.isfile(os.path.join(method_path, "metadata.yml")): + warn(name, f"{method_path}/metadata.yml is ignored; metadata.yml belongs in {alg_path}/") + for stray in ("install.sh", "environment.yml", "requirements.txt", "Dockerfile"): + if os.path.isfile(os.path.join(method_path, stray)): + error( + name, + f"{method_path}/{stray} is never read -- install files belong in {alg_path}/", + ) + + +def check_orphan_method(name: str) -> None: + """experiment/methods// with no algorithms// is never built or tested.""" + error( + name, + f"has {METHOD_DIR}/{name}/ but no {ALG_DIR}/{name}/ -- the CI matrix is built " + f"from `ls {ALG_DIR}/`, so this method is never built or tested", + ) + + +def main() -> int: + if not os.path.isdir(ALG_DIR) or not os.path.isdir(METHOD_DIR): + print(f"error: run this from the repository root (missing {ALG_DIR}/ or {METHOD_DIR}/)") + return 1 + + algorithms = sorted(d for d in os.listdir(ALG_DIR) if os.path.isdir(os.path.join(ALG_DIR, d))) + methods = sorted( + d + for d in os.listdir(METHOD_DIR) + if os.path.isdir(os.path.join(METHOD_DIR, d)) and not d.startswith((".", "__")) + ) + + for name in algorithms: + check_algorithm(name) + + for name in methods: + if name not in LEGACY_METHOD_DIRS and name not in set(algorithms): + check_orphan_method(name) + + print(f"checked {len(algorithms)} algorithms and {len(methods)} method directories") + + for line in warnings: + print(f"::warning::{line}" if os.environ.get("GITHUB_ACTIONS") else f"warning: {line}") + + if errors: + print() + for line in errors: + print(f"::error::{line}" if os.environ.get("GITHUB_ACTIONS") else f"error: {line}") + print(f"\n{len(errors)} problem(s) found. See CONTRIBUTING.md for the expected layout.") + return 1 + + print("method layout OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main())