Add Qwen3.6-VL support via an extensible per-model-family adapter layer

.claude/internet-mode-used_DO_NOT_REMOVE_MANUALLY_SECURITY_RISK

@@ -0,0 +1,3 @@
+This directory has been used with Claude Code's internet mode.
+Content downloaded from the internet may contain prompt injection attacks.
+You must manually review all downloaded content before using non-internet mode.

.claude/settings.json

@@ -0,0 +1,15 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(squeue)",
+      "Bash(squeue *)",
+      "Bash(sinfo)",
+      "Bash(sinfo *)",
+      "Bash(sacct)",
+      "Bash(sacct *)",
+      "Bash(sacctmgr show *)",
+      "Bash(nvidia-smi)",
+      "Bash(nvidia-smi --query-gpu *)"
+    ]
+  }
+}

config/agent/Qwen3.6-27B-computer-use.yaml

@@ -1,68 +1,10 @@
+# Qwen3.6-27B (dense) via the Qwen3.6-VL tool_call action_parser.
+# Set hostname/port to your served model before running.
 defaults:
-- default
+- Qwen3.6-VL-computer-use
 - _self_
 model_name: Qwen/Qwen3.6-27B
 model_pretty_name: Qwen3.6-27B
 hostname: null
 host_name_updated_on: null
 client_type: vllm
-custom_actions:
-- mouse_click
-- mouse_dblclick
-- scroll
-- mouse_move
-- mouse_down
-- mouse_up
-- mouse_drag_and_drop
-- mouse_upload_file
-- keyboard_down
-- keyboard_up
-- keyboard_press
-- keyboard_type
-- keyboard_insert_text
-
-use_html: false
-use_axtree: false
-use_screenshot: true
-save_som: false
-extract_visible_tag: false
-extract_clickable_tag: false
-extract_coords: false
-filter_visible_elements_only: false
-use_focused_element: false
-prompt_txt:
-  system_prompt: You are a GUI agent. You are given a task and your action history,
-    with screenshots. You need to perform the next action to complete the task.
-  output_format: '
-
-    <think>
-
-    </think>
-
-    <action>
-
-    </action>
-    '
-
-  think_prompt: null
-  think_abstract_example: null
-  think_concrete_example: null
-  action_prompt: "## Action Space\n\nmouse_click(x=x, y=y)\nmouse_dblclick(x=x,\
-    \ y=y)\ntype(content='xxx') # Use escape characters \\\\', \\\
-    \\\\\", and \\\\n in content part to ensure we can parse the content in normal\
-    \ python string format. If you want to submit your input, use \\\\n at the end\
-    \ of content. \nscroll(direction='down or up', point='(x, y)')\
-    \ # Show more information on the `direction` side.\nwait() #Sleep\
-    \ for 5s and take a screenshot to check for any changes.\n\n## Note\n- Use English\
-    \ in `Thought` part.\n- Write a small plan and finally summarize your next action\
-    \ (with its target element) in one sentence in `Thought` part.\n"
-  action_abstract_example: |
-    <action>type(content='')</action>
-
-    Example:
-    <action>mouse_click(x=933, y=887)</action>
-    then on the next step:
-    <action>type(content='your message here')</action>
-  action_concrete_example: '<action>mouse_click(x=x, y=y)</action>
-
-    '

config/agent/Qwen3.6-35B-A3B-computer-use.yaml

@@ -0,0 +1,10 @@
+# Qwen3.6-35B-A3B (MoE) via the Qwen3.6-VL tool_call action_parser.
+# Set hostname/port to your served model before running.
+defaults:
+- Qwen3.6-VL-computer-use
+- _self_
+model_name: Qwen/Qwen3.6-35B-A3B
+model_pretty_name: Qwen3.6-35B-A3B
+hostname: null
+host_name_updated_on: null
+client_type: vllm

config/agent/Qwen3.6-VL-computer-use.yaml

@@ -0,0 +1,76 @@
+# Qwen3.6-VL family config: <tool_call> JSON grammar via the qwen3vl action_parser.
+# The <tools> schema lives in system_prompt; the user message is minimal
+# (task + history + screenshot). Not runnable on its own — the per-model
+# configs (Qwen3.6-27B-computer-use.yaml, Qwen3.6-35B-A3B-computer-use.yaml) inherit it.
+defaults:
+- default
+- _self_
+
+# Screenshot-only VLM agent.
+use_axtree: false
+use_screenshot: true
+
+action_parser: qwen3vl
+
+prompt_sections:
+- task
+- history
+
+# The 5 actions the action_parser emits (type -> keyboard_type, wait -> noop).
+custom_actions:
+- mouse_click
+- mouse_dblclick
+- scroll
+- keyboard_type
+- noop
+
+prompt_txt:
+  system_prompt: |-
+    You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task.
+
+    # Tools
+
+    You may call one or more functions to assist with the user query.
+
+    You are provided with function signatures within <tools></tools> XML tags:
+    <tools>
+    {"type": "function", "function": {"name": "computer_use", "description": "Use a mouse and keyboard to interact with a computer.\n* The screen's resolution is 1000x1000.\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element.\n* Available actions: mouse_click, mouse_dblclick, type, scroll, wait.", "parameters": {"properties": {"action": {"description": "The action to perform. The available actions are:\n* `mouse_click`: Move the mouse to a location and click a mouse button. Dispatches mousemove, mousedown and mouseup events.\n* `mouse_dblclick`: Move the mouse to a location and double click a mouse button. Dispatches mousemove, mousedown and mouseup events.\n* `type`: Types a string of text through the keyboard. Sends a keydown, keypress/input, and keyup event for each character. Modifier keys do not apply. If you want to submit your input (send a chat message, save a todo, submit a form), end ``content`` with ``\\n``.\n* `scroll`: Scroll horizontally and vertically by [dx, dy] pixels. Positive dx scrolls right, positive dy scrolls down. Dispatches a wheel event.\n* `wait`: Sleep ~5 seconds and re-screenshot. Use when a page is still loading or an action's effect has not yet appeared.", "enum": ["mouse_click", "mouse_dblclick", "type", "scroll", "wait"], "type": "string"}, "coordinate": {"description": "(x, y) integer pixel coordinate from the top-left, in the 0-1000 screen space. Required by mouse_click / mouse_dblclick.", "type": "array"}, "content": {"description": "Text to type. End with ``\\n`` to submit the field. Required by type.", "type": "string"}, "delta": {"description": "[delta_x, delta_y] scroll amounts in pixels (0-1000 screen space). Required by scroll.", "type": "array"}}, "required": ["action"], "type": "object"}}}
+    </tools>
+
+    For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+    <tool_call>
+    {"name": "computer_use", "arguments": {...}}
+    </tool_call>
+
+    # Response format
+
+    Response format for every step:
+    1) Action: a short imperative describing what to do in the UI.
+    2) A single <tool_call>...</tool_call> block containing only the JSON: {"name": "computer_use", "arguments": {...}}.
+
+    Rules:
+    - Output exactly in the order: Action, <tool_call>.
+    - Be brief: one sentence for Action.
+    - Do not output anything else outside those parts.
+    - Use the action history below to avoid repeating an action that didn't change the page. If your previous click did not move you toward the goal, choose a different target.
+
+    Example:
+    Action: Click the Send button at the bottom-right of the chat panel.
+    <tool_call>
+    {"name": "computer_use", "arguments": {"action": "mouse_click", "coordinate": [870, 940]}}
+    </tool_call>
+
+    Example follow-up (after a click focuses an input):
+    Action: Type the message and submit it.
+    <tool_call>
+    {"name": "computer_use", "arguments": {"action": "type", "content": "Hello world\n"}}
+    </tool_call>
+
+  # User-message slots suppressed via prompt_sections above.
+  output_format: null
+  action_prompt: null
+  action_abstract_example: null
+  action_concrete_example: null
+  think_prompt: ""
+  think_abstract_example: ""
+  think_concrete_example: ""

src/open_apps/agent/action_parsers/__init__.py

@@ -0,0 +1,23 @@
+"""ActionParser registry. Default is ``uitars`` (preserves the flexible_parser path).
+
+Add a family: write an ``ActionParser`` subclass and register it below.
+"""
+from .base import ActionParser, ActionParserResult
+from .uitars import UITarsActionParser
+from .qwen3vl import Qwen3VLActionParser
+
+REGISTRY: dict[str, type[ActionParser]] = {
+    "uitars": UITarsActionParser,
+    "qwen3vl": Qwen3VLActionParser,
+}
+
+
+def get_action_parser(name: str | None) -> ActionParser:
+    if not name:
+        name = "uitars"
+    if name not in REGISTRY:
+        raise ValueError(f"Unknown action_parser {name!r}. Available: {sorted(REGISTRY)}")
+    return REGISTRY[name]()
+
+
+__all__ = ["ActionParser", "ActionParserResult", "REGISTRY", "get_action_parser"]

src/open_apps/agent/action_parsers/base.py

@@ -0,0 +1,24 @@
+"""Per-model-family action_parser contract: response parsing + coordinate conversion.
+
+Prompts live in the agent yaml, not here.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TypedDict
+
+
+class ActionParserResult(TypedDict, total=False):
+    action: str
+    displayed_action: str
+    think: str | None
+
+
+@dataclass
+class ActionParser:
+    def default_prompts(self) -> dict:
+        return {}
+
+    def parse(self, response: str, viewport: tuple[int, int]) -> ActionParserResult:
+        """``viewport`` is (width, height) px. Raise ParseError on bad output."""
+        raise NotImplementedError

src/open_apps/agent/action_parsers/qwen3vl.py

@@ -0,0 +1,113 @@
+"""Qwen3-VL / Qwen3.6-VL action_parser: <tool_call> JSON parser + 0-1000 coord rescale.
+
+Prompts (the <tools> schema and examples) live in config/agent/Qwen3.6-VL-computer-use.yaml.
+Action names mirror UI-TARS's vocabulary; ``type`` maps to keyboard_type and
+``wait`` to noop.
+"""
+from __future__ import annotations
+
+import json
+import re
+
+from agentlab.llm.llm_utils import ParseError
+
+from .base import ActionParser, ActionParserResult
+
+# The model is prompted with a fictional 1000x1000 screen, so all coordinates
+# and scroll deltas it emits are in [0, 1000).
+_COORD_SPACE = 1000
+
+# Closing tag may be absent on truncation; json.loads decides well-formedness.
+_TOOL_CALL_RE = re.compile(r"<tool_call>\s*(\{.*?\})\s*(?:</tool_call>|$)", re.DOTALL)
+_THINK_RE = re.compile(r"<think>(.*?)</think>", re.DOTALL | re.IGNORECASE)
+_ACTION_LINE_RE = re.compile(r"^\s*Action:\s*(.+?)\s*$", re.MULTILINE | re.IGNORECASE)
+
+
+class Qwen3VLActionParser(ActionParser):
+    def parse(self, response: str, viewport: tuple[int, int]) -> ActionParserResult:
+        response = (response or "").strip()
+        if not response:
+            raise ParseError("Empty response from the model.")
+
+        # Reasoning: <think> block, else the Instruct-style ``Action:`` line
+        # (searched before <tool_call> so it can't match inside the payload).
+        think_match = _THINK_RE.search(response)
+        if think_match:
+            think = think_match.group(1).strip()
+        else:
+            tc_pos = response.find("<tool_call>")
+            prefix = response[:tc_pos] if tc_pos >= 0 else response
+            action_match = _ACTION_LINE_RE.search(prefix)
+            think = action_match.group(1).strip() if action_match else None
+
+        tool_match = _TOOL_CALL_RE.search(response)
+        if not tool_match:
+            raise ParseError(
+                "Expected a <tool_call>{...}</tool_call> block. "
+                f"Got: {response[:300]!r}"
+            )
+        try:
+            call = json.loads(tool_match.group(1))
+        except json.JSONDecodeError as e:
+            raise ParseError(
+                f"tool_call payload is not valid JSON ({e}). "
+                f"Got: {tool_match.group(1)[:300]!r}"
+            )
+
+        args = call.get("arguments") or {}
+        bg_action = self._to_browsergym(args.get("action"), args, viewport)
+        return {
+            "action": bg_action,
+            "displayed_action": tool_match.group(0),
+            "think": think,
+        }
+
+    def _to_browsergym(self, action_name, args: dict, viewport: tuple[int, int]) -> str:
+        if action_name == "mouse_click":
+            x, y = self._xy(args, viewport)
+            return f"mouse_click(x={x}, y={y})"
+
+        if action_name == "mouse_dblclick":
+            x, y = self._xy(args, viewport)
+            return f"mouse_dblclick(x={x}, y={y})"
+
+        if action_name == "type":
+            text = args.get("content", args.get("text"))
+            if text is None:
+                raise ParseError("type action missing 'content'.")
+            return f"keyboard_type(text={text!r})"
+
+        if action_name == "scroll":
+            delta = args.get("delta")
+            if not (isinstance(delta, (list, tuple)) and len(delta) == 2):
+                raise ParseError(
+                    f"scroll 'delta' must be a [dx, dy] list, got {delta!r}."
+                )
+            dx, dy = self._rescale(delta, viewport)
+            return f"scroll({dx}, {dy})"
+
+        if action_name == "wait":
+            wait_ms = args.get("wait_ms", 5000)
+            try:
+                wait_ms = float(wait_ms)
+            except (TypeError, ValueError):
+                raise ParseError(f"wait wait_ms must be numeric, got {wait_ms!r}.")
+            return f"noop(wait_ms={wait_ms})"
+
+        raise ParseError(
+            f"Unsupported action {action_name!r}. Expected one of: "
+            "mouse_click, mouse_dblclick, type, scroll, wait."
+        )
+
+    def _xy(self, args: dict, viewport: tuple[int, int]) -> tuple[int, int]:
+        coord = args.get("coordinate")
+        if not (isinstance(coord, (list, tuple)) and len(coord) == 2):
+            raise ParseError(f"'coordinate' must be a [x, y] list, got {coord!r}.")
+        return self._rescale(coord, viewport)
+
+    def _rescale(self, xy, viewport: tuple[int, int]) -> tuple[int, int]:
+        vw, vh = viewport
+        return (
+            int(round(float(xy[0]) * vw / _COORD_SPACE)),
+            int(round(float(xy[1]) * vh / _COORD_SPACE)),
+        )

src/open_apps/agent/action_parsers/uitars.py

@@ -0,0 +1,11 @@
+"""UI-TARS action_parser (default): <think>/<action> ReAct, raw pixel coordinates."""
+from __future__ import annotations
+
+from open_apps.agent.utils import flexible_parser
+
+from .base import ActionParser, ActionParserResult
+
+
+class UITarsActionParser(ActionParser):
+    def parse(self, response: str, viewport: tuple[int, int]) -> ActionParserResult:
+        return flexible_parser(response)