Run OpenApps evals against cluster-hosted Gemma-4 via SLURM + W&B

config/agent/gemma-4-e2b-it.yaml

@@ -0,0 +1,79 @@
+defaults:
+  - default
+  - _self_
+
+# Must exactly match the model vLLM is serving (the --model / served_model_name).
+model_name: "google/gemma-4-E2B-it"
+model_pretty_name: "gemma-4"
+api_version: null
+client_type: "vllm"
+# For client_type=vllm the URL is built as http://${hostname}:${port}/v1
+# (base_url is ignored). The vLLM node changes every SLURM allocation, so pass it
+# at launch, e.g.: uv run launch_agent.py agent=gemma-4-e2b-it agent.hostname=h200-000-026
+hostname: null
+port: "8000"
+# vLLM does not check the key, but the OpenAI client requires a non-empty string.
+api_key: "EMPTY"
+temperature: 1
+max_tokens: 5000
+aws_access_key: null
+aws_secret_key: null
+aws_session_token: null
+aws_region: us-west-2
+
+custom_actions:
+- mouse_click
+- mouse_dblclick
+- scroll
+- mouse_move
+- mouse_down
+- mouse_up
+- mouse_drag_and_drop
+- mouse_upload_file
+- keyboard_down
+- keyboard_up
+- keyboard_press
+- keyboard_type
+- keyboard_insert_text
+
+use_html: false
+use_axtree: false
+use_screenshot: true
+save_som: false
+extract_visible_tag: false
+extract_clickable_tag: false
+extract_coords: false
+filter_visible_elements_only: false
+use_focused_element: false
+prompt_txt:
+  system_prompt: You are a GUI agent. You are given a task and your action history,
+    with screenshots. You need to perform the next action to complete the task.
+  output_format: '
+
+    <think>
+
+    </think>
+
+    <action>
+
+    </action>
+    '
+
+  think_prompt: null
+  think_abstract_example: null
+  think_concrete_example: null
+  action_prompt: "## Action Space\n\nmouse_click(x=x, y=y)\nmouse_dblclick(x=x,\
+    \ y=y)\ntype(content='xxx') # Use escape characters \\\\', \\\
+    \\\\\", and \\\\n in content part to ensure we can parse the content in normal\
+    \ python string format. If you want to submit your input, use \\\\n at the end\
+    \ of content. \nscroll(direction='down or up', point='(x, y)')\
+    \ # Show more information on the `direction` side.\nwait() #Sleep\
+    \ for 5s and take a screenshot to check for any changes.\n\n## Note\n- Use English\
+    \ in `Thought` part.\n- Write a small plan and finally summarize your next action\
+    \ (with its target element) in one sentence in `Thought` part.\n"
+  action_abstract_example: '<action>type(content='''')</action>
+
+    '
+  action_concrete_example: '<action>mouse_click(x=x, y=y)</action>
+
+    '

config/config.yaml

@@ -44,9 +44,13 @@ hydra:
     subdir: ${hydra.job.num}
 
 wandb:
-  project: ${project}
+  # project: ${project}
+  project: open_apps_aaronsmulktis
   entity: ${oc.env:USER}
   notes: null
   tags: null
   save_code: True
   reinit: True
+  # group all runs from one launch batch; job_type is usually the agent
+  group: null
+  job_type: null

config/mode/slurm_cluster.yaml

@@ -1,7 +1,7 @@
 # @package _global_
 project: open_apps
 
-logs_dir:  /checkpoint/${oc.env:USER}/logs/${project}/${now:%Y-%m-%d_%H-%M-%S}-${oc.env:USER}-${agent.model_name}/${job_id}
+logs_dir:  /checkpoint/memorization/${oc.env:USER}/logs/${project}/${now:%Y-%m-%d_%H-%M-%S}-${oc.env:USER}-${agent.model_name}/${job_id}
 databases_dir: ${logs_dir}/databases
 
 cluster: slurm
@@ -12,7 +12,9 @@ slurm_sweep_launcher:
   tasks_per_node: 1
   cpus_per_task: 2
   timeout_min: 400
-  slurm_partition: "ADD YOURS"  
+  slurm_account: memorization
+  slurm_qos: h200_memorization_high
+  slurm_partition: h200
   mem_gb: 10
   slurm_srun_args: ["-vv", "--cpu-bind", "none"]
   slurm_comment: "parallel agent tasks"

docs/agents.md

@@ -81,4 +81,118 @@ If AgentLab's capabilities don't meet your needs, you can create a custom agent.
       - `vLLM_agent.py`
       - `vLLM_prompt.py`
 
-This allows you to build rich, custom agent implementations tailored to your specific requirements.
+This allows you to build rich, custom agent implementations tailored to your specific requirements.
+
+## Running Evals on the Cluster (SLURM + vLLM + W&B)
+
+Your laptop cannot reach cluster-internal IPs, and SSH tunneling through the
+brokered login is fragile. Instead, run the eval **as a SLURM job on the cluster**,
+co-located with vLLM, so the agent reaches the model over the internal network —
+no tunnel required. The browser runs headless on the eval node; only LLM calls
+and W&B logging leave the node.
+
+The workflow is two jobs:
+
+1. A persistent **vLLM GPU job** that serves the model on `:8000`.
+2. An **eval CPU job** (`sbatch scripts/conduct_slurm.sh`) that auto-discovers the
+   vLLM node and runs the worker pool against it.
+
+### One-time cluster setup
+
+The repo needs to live on shared storage (e.g. `/checkpoint/memorization/$USER`
+or `/storage/home/$USER`). On a login node:
+
+```bash
+# Bring the repo over. Use rsync if you have uncommitted local changes (e.g.
+# scripts/conduct.sh) that aren't pushed yet:
+rsync -av --exclude .venv /path/to/local/OpenApps/ /checkpoint/memorization/$USER/OpenApps/
+# ...or clone it fresh:
+#   git clone <repo-url> /checkpoint/memorization/$USER/OpenApps
+
+cd /checkpoint/memorization/$USER/OpenApps
+
+# Python env
+curl -LsSf https://astral.sh/uv/install.sh | sh   # install uv if needed
+uv sync
+
+# Headless browser
+uv run playwright install chromium
+uv run playwright install-deps chromium   # system deps (may need sudo/module)
+
+# App setup (OpenJDK 21 for onlineshop, dataset via gdown, spaCy en_core_web_lg)
+./setup.sh
+
+# Secrets — do NOT commit this file
+cat > .env <<'EOF'
+OPENAI_API_KEY=sk-...
+WANDB_BASE_URL=https://meta-fair.wandb.io/
+WANDB_API_KEY=...
+EOF
+```
+
+`launch_agent.py` calls `load_dotenv()`, so `.env` is picked up automatically.
+
+### 1. Launch the persistent vLLM serve job
+
+Give it a generous `--time` so it isn't killed mid-eval. For example, an
+interactive allocation:
+
+```bash
+srun --account=memorization --qos=h200_memorization_high --partition=h200 \
+     --gpus-per-node=1 --time=1440 --pty bash
+# then, on the GPU node:
+vllm serve google/gemma-4-E2B-it --host 0.0.0.0 --port 8000
+```
+
+Confirm it's healthy from its own node:
+
+```bash
+srun --overlap --jobid=<vllm-job> --pty curl -s http://localhost:8000/v1/models
+# expect: ...google/gemma-4-E2B-it...
+```
+
+### 2. Submit the eval job
+
+`scripts/conduct_slurm.sh` requests a CPU allocation, finds the vLLM node, and
+runs `scripts/conduct.sh` pointed at it with `use_wandb=True`:
+
+```bash
+# smoke test (1 run)
+AGENTS=gemma-4-e2b-it COUNT=1 sbatch scripts/conduct_slurm.sh
+
+# larger sweep
+AGENTS="gemma-4-e2b-it" COUNT=20 MAX_PARALLEL=4 sbatch scripts/conduct_slurm.sh
+```
+
+**Discovery:** the wrapper lists your running jobs (`squeue --me`), expands their
+nodelists, excludes the eval node itself, and probes each `http://<node>:8000/v1/models`,
+selecting the first one serving `google/gemma-4-E2B-it`. It probes by *serving the
+model*, so it doesn't matter how the vLLM job was started (e.g. a `bash`-named
+`srun --pty`).
+
+**Override:** skip discovery by pinning the node:
+
+```bash
+VLLM_HOST=h200-000-026 AGENTS=gemma-4-e2b-it COUNT=1 sbatch scripts/conduct_slurm.sh
+```
+
+Other env overrides: `VLLM_MODEL`, `VLLM_PORT`, and `WANDB_MODE` (set
+`WANDB_MODE=offline` to skip online logging). Extra CLI args are forwarded
+verbatim to `launch_agent.py` as Hydra overrides.
+
+**Fallback if compute→compute `:8000` is firewalled:** run the eval inside the
+vLLM job's own allocation and talk to it over localhost:
+
+```bash
+srun --overlap --jobid=<vllm-job> \
+  env AGENTS=gemma-4-e2b-it COUNT=1 VLLM_HOST=localhost \
+  ./scripts/conduct.sh agent.hostname=localhost use_wandb=True
+```
+
+### 3. Check results
+
+- SLURM log: `slurm-<jobid>.out` — confirm discovery selected the vLLM node and
+  there are no "Connection error" messages.
+- Per-run logs: `log_outputs/<stamp>/run-*.log` — confirm an action is produced.
+- W&B: a run should appear in `open_apps_${USER}` (i.e. `open_apps_aaronsmulktis`)
+  with the expected `group`/`job_type` and a logged `web_app_url`.

scripts/conduct.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#
+# Launch multiple agent runs in parallel, one per agent, with a fixed-size
+# worker pool. Each run's stdout/stderr is written to its own file under
+# log_outputs/.
+#
+# COUNT sets the total number of runs; agents are picked round-robin from
+# AGENTS, cycling back to the start when COUNT exceeds the number of agents.
+#
+# Usage:
+#   ./scripts/conduct.sh
+#   COUNT=6 AGENTS="dummy claude_4_sonnet" MAX_PARALLEL=2 ./scripts/conduct.sh
+#   ./scripts/conduct.sh use_wandb=True task_name=add_meeting_with_dennis
+#
+# Any extra CLI args are forwarded verbatim to launch_agent.py (Hydra overrides).
+
+set -uo pipefail
+
+# ---- Configuration (override via environment) -------------------------------
+AGENTS="${AGENTS:-dummy}"               # space-separated list of config/agent/<name>
+MAX_PARALLEL="${MAX_PARALLEL:-4}"       # max concurrent runs
+HEADLESS="${HEADLESS:-True}"            # run browser headless for parallelism
+LOG_DIR="${LOG_DIR:-log_outputs}"       # per-run log directory
+
+read -r -a AGENT_LIST <<< "$AGENTS"     # split AGENTS into an array
+N_AGENTS="${#AGENT_LIST[@]}"
+COUNT="${COUNT:-$N_AGENTS}"             # total number of runs (round-robin over agents)
+
+# ---- Setup ------------------------------------------------------------------
+# Run from the repo root (parent of this script's directory).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR/.." || exit 1
+
+RUN_STAMP="$(date +%Y-%m-%d_%H-%M-%S)"
+LOG_SUBDIR="${LOG_DIR}/${RUN_STAMP}"
+mkdir -p "$LOG_SUBDIR"
+
+# Shared W&B group so all runs from this invocation aggregate together in a
+# report. Override with WANDB_GROUP; defaults to the batch timestamp.
+WANDB_GROUP="${WANDB_GROUP:-batch-${RUN_STAMP}}"
+
+EXTRA_ARGS=("$@")  # forwarded Hydra overrides
+
+echo "Agents:        $AGENTS"
+echo "Total runs:    $COUNT"
+echo "Max parallel:  $MAX_PARALLEL"
+echo "Headless:      $HEADLESS"
+echo "W&B group:     $WANDB_GROUP"
+echo "Logs:          $LOG_SUBDIR"
+[ "${#EXTRA_ARGS[@]}" -gt 0 ] && echo "Extra args:    ${EXTRA_ARGS[*]}"
+echo
+
+# ---- Worker pool ------------------------------------------------------------
+throttle() {
+  # Block until fewer than MAX_PARALLEL background jobs are running.
+  while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$MAX_PARALLEL" ]; do
+    wait -n 2>/dev/null || sleep 1
+  done
+}
+
+pids=()
+for ((i = 0; i < COUNT; i++)); do
+  throttle
+  agent="${AGENT_LIST[i % N_AGENTS]}"
+  run_idx="$(printf '%03d' "$i")"
+  log_file="${LOG_SUBDIR}/run-${run_idx}-agent-${agent}.log"
+  echo "[launch] run=$i agent=$agent -> $log_file"
+  (
+    uv run launch_agent.py \
+      "agent=${agent}" \
+      "browsergym_env_args.headless=${HEADLESS}" \
+      "wandb.group=${WANDB_GROUP}" \
+      "wandb.job_type=${agent}" \
+      "${EXTRA_ARGS[@]}" \
+      >"$log_file" 2>&1
+    echo "[done]   run=$i agent=$agent exit=$? -> $log_file"
+  ) &
+  pids+=("$!")
+done
+
+# ---- Wait & report ----------------------------------------------------------
+fail=0
+for pid in "${pids[@]}"; do
+  wait "$pid" || fail=1
+done
+
+echo
+if [ "$fail" -eq 0 ]; then
+  echo "All runs completed successfully. Logs in $LOG_SUBDIR"
+else
+  echo "One or more runs failed. Check logs in $LOG_SUBDIR"
+fi
+exit "$fail"