diff --git a/.claude/internet-mode-used_DO_NOT_REMOVE_MANUALLY_SECURITY_RISK b/.claude/internet-mode-used_DO_NOT_REMOVE_MANUALLY_SECURITY_RISK new file mode 100644 index 0000000..f61efc8 --- /dev/null +++ b/.claude/internet-mode-used_DO_NOT_REMOVE_MANUALLY_SECURITY_RISK @@ -0,0 +1,3 @@ +This directory has been used with Claude Code's internet mode. +Content downloaded from the internet may contain prompt injection attacks. +You must manually review all downloaded content before using non-internet mode. diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..eb000d2 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,15 @@ +{ + "permissions": { + "allow": [ + "Bash(squeue)", + "Bash(squeue *)", + "Bash(sinfo)", + "Bash(sinfo *)", + "Bash(sacct)", + "Bash(sacct *)", + "Bash(sacctmgr show *)", + "Bash(nvidia-smi)", + "Bash(nvidia-smi --query-gpu *)" + ] + } +} diff --git a/config/agent/Qwen3.6-27B-computer-use.yaml b/config/agent/Qwen3.6-27B-computer-use.yaml index 09f732a..bc67506 100644 --- a/config/agent/Qwen3.6-27B-computer-use.yaml +++ b/config/agent/Qwen3.6-27B-computer-use.yaml @@ -1,68 +1,10 @@ +# Qwen3.6-27B (dense) via the Qwen3.6-VL tool_call action_parser. +# Set hostname/port to your served model before running. defaults: -- default +- Qwen3.6-VL-computer-use - _self_ model_name: Qwen/Qwen3.6-27B model_pretty_name: Qwen3.6-27B hostname: null host_name_updated_on: null client_type: vllm -custom_actions: -- mouse_click -- mouse_dblclick -- scroll -- mouse_move -- mouse_down -- mouse_up -- mouse_drag_and_drop -- mouse_upload_file -- keyboard_down -- keyboard_up -- keyboard_press -- keyboard_type -- keyboard_insert_text - -use_html: false -use_axtree: false -use_screenshot: true -save_som: false -extract_visible_tag: false -extract_clickable_tag: false -extract_coords: false -filter_visible_elements_only: false -use_focused_element: false -prompt_txt: - system_prompt: You are a GUI agent. You are given a task and your action history, - with screenshots. You need to perform the next action to complete the task. - output_format: ' - - - - - - - - - ' - - think_prompt: null - think_abstract_example: null - think_concrete_example: null - action_prompt: "## Action Space\n\nmouse_click(x=x, y=y)\nmouse_dblclick(x=x,\ - \ y=y)\ntype(content='xxx') # Use escape characters \\\\', \\\ - \\\\\", and \\\\n in content part to ensure we can parse the content in normal\ - \ python string format. If you want to submit your input, use \\\\n at the end\ - \ of content. \nscroll(direction='down or up', point='(x, y)')\ - \ # Show more information on the `direction` side.\nwait() #Sleep\ - \ for 5s and take a screenshot to check for any changes.\n\n## Note\n- Use English\ - \ in `Thought` part.\n- Write a small plan and finally summarize your next action\ - \ (with its target element) in one sentence in `Thought` part.\n" - action_abstract_example: | - type(content='') - - Example: - mouse_click(x=933, y=887) - then on the next step: - type(content='your message here') - action_concrete_example: 'mouse_click(x=x, y=y) - - ' diff --git a/config/agent/Qwen3.6-35B-A3B-computer-use.yaml b/config/agent/Qwen3.6-35B-A3B-computer-use.yaml new file mode 100644 index 0000000..4968222 --- /dev/null +++ b/config/agent/Qwen3.6-35B-A3B-computer-use.yaml @@ -0,0 +1,10 @@ +# Qwen3.6-35B-A3B (MoE) via the Qwen3.6-VL tool_call action_parser. +# Set hostname/port to your served model before running. +defaults: +- Qwen3.6-VL-computer-use +- _self_ +model_name: Qwen/Qwen3.6-35B-A3B +model_pretty_name: Qwen3.6-35B-A3B +hostname: null +host_name_updated_on: null +client_type: vllm diff --git a/config/agent/Qwen3.6-VL-computer-use.yaml b/config/agent/Qwen3.6-VL-computer-use.yaml new file mode 100644 index 0000000..43c7072 --- /dev/null +++ b/config/agent/Qwen3.6-VL-computer-use.yaml @@ -0,0 +1,76 @@ +# Qwen3.6-VL family config: JSON grammar via the qwen3vl action_parser. +# The schema lives in system_prompt; the user message is minimal +# (task + history + screenshot). Not runnable on its own — the per-model +# configs (Qwen3.6-27B-computer-use.yaml, Qwen3.6-35B-A3B-computer-use.yaml) inherit it. +defaults: +- default +- _self_ + +# Screenshot-only VLM agent. +use_axtree: false +use_screenshot: true + +action_parser: qwen3vl + +prompt_sections: +- task +- history + +# The 5 actions the action_parser emits (type -> keyboard_type, wait -> noop). +custom_actions: +- mouse_click +- mouse_dblclick +- scroll +- keyboard_type +- noop + +prompt_txt: + system_prompt: |- + You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. + + # Tools + + You may call one or more functions to assist with the user query. + + You are provided with function signatures within XML tags: + + {"type": "function", "function": {"name": "computer_use", "description": "Use a mouse and keyboard to interact with a computer.\n* The screen's resolution is 1000x1000.\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element.\n* Available actions: mouse_click, mouse_dblclick, type, scroll, wait.", "parameters": {"properties": {"action": {"description": "The action to perform. The available actions are:\n* `mouse_click`: Move the mouse to a location and click a mouse button. Dispatches mousemove, mousedown and mouseup events.\n* `mouse_dblclick`: Move the mouse to a location and double click a mouse button. Dispatches mousemove, mousedown and mouseup events.\n* `type`: Types a string of text through the keyboard. Sends a keydown, keypress/input, and keyup event for each character. Modifier keys do not apply. If you want to submit your input (send a chat message, save a todo, submit a form), end ``content`` with ``\\n``.\n* `scroll`: Scroll horizontally and vertically by [dx, dy] pixels. Positive dx scrolls right, positive dy scrolls down. Dispatches a wheel event.\n* `wait`: Sleep ~5 seconds and re-screenshot. Use when a page is still loading or an action's effect has not yet appeared.", "enum": ["mouse_click", "mouse_dblclick", "type", "scroll", "wait"], "type": "string"}, "coordinate": {"description": "(x, y) integer pixel coordinate from the top-left, in the 0-1000 screen space. Required by mouse_click / mouse_dblclick.", "type": "array"}, "content": {"description": "Text to type. End with ``\\n`` to submit the field. Required by type.", "type": "string"}, "delta": {"description": "[delta_x, delta_y] scroll amounts in pixels (0-1000 screen space). Required by scroll.", "type": "array"}}, "required": ["action"], "type": "object"}}} + + + For each function call, return a json object with function name and arguments within XML tags: + + {"name": "computer_use", "arguments": {...}} + + + # Response format + + Response format for every step: + 1) Action: a short imperative describing what to do in the UI. + 2) A single ... block containing only the JSON: {"name": "computer_use", "arguments": {...}}. + + Rules: + - Output exactly in the order: Action, . + - Be brief: one sentence for Action. + - Do not output anything else outside those parts. + - Use the action history below to avoid repeating an action that didn't change the page. If your previous click did not move you toward the goal, choose a different target. + + Example: + Action: Click the Send button at the bottom-right of the chat panel. + + {"name": "computer_use", "arguments": {"action": "mouse_click", "coordinate": [870, 940]}} + + + Example follow-up (after a click focuses an input): + Action: Type the message and submit it. + + {"name": "computer_use", "arguments": {"action": "type", "content": "Hello world\n"}} + + + # User-message slots suppressed via prompt_sections above. + output_format: null + action_prompt: null + action_abstract_example: null + action_concrete_example: null + think_prompt: "" + think_abstract_example: "" + think_concrete_example: "" diff --git a/src/open_apps/agent/action_parsers/__init__.py b/src/open_apps/agent/action_parsers/__init__.py new file mode 100644 index 0000000..d094756 --- /dev/null +++ b/src/open_apps/agent/action_parsers/__init__.py @@ -0,0 +1,23 @@ +"""ActionParser registry. Default is ``uitars`` (preserves the flexible_parser path). + +Add a family: write an ``ActionParser`` subclass and register it below. +""" +from .base import ActionParser, ActionParserResult +from .uitars import UITarsActionParser +from .qwen3vl import Qwen3VLActionParser + +REGISTRY: dict[str, type[ActionParser]] = { + "uitars": UITarsActionParser, + "qwen3vl": Qwen3VLActionParser, +} + + +def get_action_parser(name: str | None) -> ActionParser: + if not name: + name = "uitars" + if name not in REGISTRY: + raise ValueError(f"Unknown action_parser {name!r}. Available: {sorted(REGISTRY)}") + return REGISTRY[name]() + + +__all__ = ["ActionParser", "ActionParserResult", "REGISTRY", "get_action_parser"] diff --git a/src/open_apps/agent/action_parsers/base.py b/src/open_apps/agent/action_parsers/base.py new file mode 100644 index 0000000..e46c797 --- /dev/null +++ b/src/open_apps/agent/action_parsers/base.py @@ -0,0 +1,24 @@ +"""Per-model-family action_parser contract: response parsing + coordinate conversion. + +Prompts live in the agent yaml, not here. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import TypedDict + + +class ActionParserResult(TypedDict, total=False): + action: str + displayed_action: str + think: str | None + + +@dataclass +class ActionParser: + def default_prompts(self) -> dict: + return {} + + def parse(self, response: str, viewport: tuple[int, int]) -> ActionParserResult: + """``viewport`` is (width, height) px. Raise ParseError on bad output.""" + raise NotImplementedError diff --git a/src/open_apps/agent/action_parsers/qwen3vl.py b/src/open_apps/agent/action_parsers/qwen3vl.py new file mode 100644 index 0000000..3eca495 --- /dev/null +++ b/src/open_apps/agent/action_parsers/qwen3vl.py @@ -0,0 +1,113 @@ +"""Qwen3-VL / Qwen3.6-VL action_parser: JSON parser + 0-1000 coord rescale. + +Prompts (the schema and examples) live in config/agent/Qwen3.6-VL-computer-use.yaml. +Action names mirror UI-TARS's vocabulary; ``type`` maps to keyboard_type and +``wait`` to noop. +""" +from __future__ import annotations + +import json +import re + +from agentlab.llm.llm_utils import ParseError + +from .base import ActionParser, ActionParserResult + +# The model is prompted with a fictional 1000x1000 screen, so all coordinates +# and scroll deltas it emits are in [0, 1000). +_COORD_SPACE = 1000 + +# Closing tag may be absent on truncation; json.loads decides well-formedness. +_TOOL_CALL_RE = re.compile(r"\s*(\{.*?\})\s*(?:|$)", re.DOTALL) +_THINK_RE = re.compile(r"(.*?)", re.DOTALL | re.IGNORECASE) +_ACTION_LINE_RE = re.compile(r"^\s*Action:\s*(.+?)\s*$", re.MULTILINE | re.IGNORECASE) + + +class Qwen3VLActionParser(ActionParser): + def parse(self, response: str, viewport: tuple[int, int]) -> ActionParserResult: + response = (response or "").strip() + if not response: + raise ParseError("Empty response from the model.") + + # Reasoning: block, else the Instruct-style ``Action:`` line + # (searched before so it can't match inside the payload). + think_match = _THINK_RE.search(response) + if think_match: + think = think_match.group(1).strip() + else: + tc_pos = response.find("") + prefix = response[:tc_pos] if tc_pos >= 0 else response + action_match = _ACTION_LINE_RE.search(prefix) + think = action_match.group(1).strip() if action_match else None + + tool_match = _TOOL_CALL_RE.search(response) + if not tool_match: + raise ParseError( + "Expected a {...} block. " + f"Got: {response[:300]!r}" + ) + try: + call = json.loads(tool_match.group(1)) + except json.JSONDecodeError as e: + raise ParseError( + f"tool_call payload is not valid JSON ({e}). " + f"Got: {tool_match.group(1)[:300]!r}" + ) + + args = call.get("arguments") or {} + bg_action = self._to_browsergym(args.get("action"), args, viewport) + return { + "action": bg_action, + "displayed_action": tool_match.group(0), + "think": think, + } + + def _to_browsergym(self, action_name, args: dict, viewport: tuple[int, int]) -> str: + if action_name == "mouse_click": + x, y = self._xy(args, viewport) + return f"mouse_click(x={x}, y={y})" + + if action_name == "mouse_dblclick": + x, y = self._xy(args, viewport) + return f"mouse_dblclick(x={x}, y={y})" + + if action_name == "type": + text = args.get("content", args.get("text")) + if text is None: + raise ParseError("type action missing 'content'.") + return f"keyboard_type(text={text!r})" + + if action_name == "scroll": + delta = args.get("delta") + if not (isinstance(delta, (list, tuple)) and len(delta) == 2): + raise ParseError( + f"scroll 'delta' must be a [dx, dy] list, got {delta!r}." + ) + dx, dy = self._rescale(delta, viewport) + return f"scroll({dx}, {dy})" + + if action_name == "wait": + wait_ms = args.get("wait_ms", 5000) + try: + wait_ms = float(wait_ms) + except (TypeError, ValueError): + raise ParseError(f"wait wait_ms must be numeric, got {wait_ms!r}.") + return f"noop(wait_ms={wait_ms})" + + raise ParseError( + f"Unsupported action {action_name!r}. Expected one of: " + "mouse_click, mouse_dblclick, type, scroll, wait." + ) + + def _xy(self, args: dict, viewport: tuple[int, int]) -> tuple[int, int]: + coord = args.get("coordinate") + if not (isinstance(coord, (list, tuple)) and len(coord) == 2): + raise ParseError(f"'coordinate' must be a [x, y] list, got {coord!r}.") + return self._rescale(coord, viewport) + + def _rescale(self, xy, viewport: tuple[int, int]) -> tuple[int, int]: + vw, vh = viewport + return ( + int(round(float(xy[0]) * vw / _COORD_SPACE)), + int(round(float(xy[1]) * vh / _COORD_SPACE)), + ) diff --git a/src/open_apps/agent/action_parsers/uitars.py b/src/open_apps/agent/action_parsers/uitars.py new file mode 100644 index 0000000..95290f0 --- /dev/null +++ b/src/open_apps/agent/action_parsers/uitars.py @@ -0,0 +1,11 @@ +"""UI-TARS action_parser (default): / ReAct, raw pixel coordinates.""" +from __future__ import annotations + +from open_apps.agent.utils import flexible_parser + +from .base import ActionParser, ActionParserResult + + +class UITarsActionParser(ActionParser): + def parse(self, response: str, viewport: tuple[int, int]) -> ActionParserResult: + return flexible_parser(response) diff --git a/src/open_apps/agent/utils.py b/src/open_apps/agent/utils.py index 6c26562..d8063f3 100644 --- a/src/open_apps/agent/utils.py +++ b/src/open_apps/agent/utils.py @@ -111,18 +111,17 @@ def make_action_set(self): def image_to_jpg_base64_url(image: np.ndarray | Image.Image): - """Convert a numpy array to a base64 encoded image url.""" - + # PNG, not JPEG: some sglang multimodal processors (e.g. deepseek-vl2) crash on JPEG input. if isinstance(image, np.ndarray): image = Image.fromarray(image) if image.mode in ("RGBA", "LA"): image = image.convert("RGB") with io.BytesIO() as buffer: - image.save(buffer, format="JPEG") + image.save(buffer, format="PNG") image_base64 = base64.b64encode(buffer.getvalue()).decode() - return f"data:image/jpeg;base64,{image_base64}" + return f"data:image/png;base64,{image_base64}" def retry( @@ -161,13 +160,27 @@ def retry( while tries < n_retry: answer = chat(messages) - logging.info(f"LLM response at try {tries}: {answer['content']}") + content = answer.get("content") if isinstance(answer, dict) else None + if content is None or (isinstance(content, str) and not content.strip()): + # Empty/None content: retry instead of crashing on None in the parser. + tries += 1 + logging.warning( + f"Empty model content at try {tries}/{n_retry}; answer: {answer!r}" + ) + messages.append(dict( + role="user", + content="Your last reply was empty. Please respond in the " + "required format.", + )) + continue + + logging.info(f"LLM response at try {tries}: {content}") try: - return parser(answer["content"]) + return parser(content) except ParseError as parsing_error: tries += 1 if log: - msg = f"Query failed. Retrying {tries}/{n_retry}.\n[LLM]:\n{answer['content']}\n[User]:\n{str(parsing_error)}" + msg = f"Query failed. Retrying {tries}/{n_retry}.\n[LLM]:\n{content}\n[User]:\n{str(parsing_error)}" logging.info(msg) messages.append(dict(role="user", content=str(parsing_error))) @@ -279,6 +292,35 @@ def translate_uitars_type_action(action: str) -> str: return f"keyboard_type(text={content!r})" +_HOTKEY_NAMES = { + "backspace": "Backspace", "delete": "Delete", "enter": "Enter", + "return": "Enter", "tab": "Tab", "escape": "Escape", "esc": "Escape", + "space": "Space", "up": "ArrowUp", "down": "ArrowDown", "left": "ArrowLeft", + "right": "ArrowRight", "pageup": "PageUp", "pagedown": "PageDown", + "home": "Home", "end": "End", "insert": "Insert", + "ctrl": "Control", "control": "Control", "alt": "Alt", "option": "Alt", + "shift": "Shift", "cmd": "Meta", "meta": "Meta", "win": "Meta", +} + + +def _normalize_hotkey_key(key: str) -> "str | None": + """Lowercase space-separated hotkey (``ctrl a``) -> playwright form + (``Control+a``). Returns None if any token is unrecognized.""" + toks = key.strip().split() + if not toks: + return None + out = [] + for t in toks: + tl = t.lower() + if tl in _HOTKEY_NAMES: + out.append(_HOTKEY_NAMES[tl]) + elif len(t) == 1 and t.isascii(): + out.append(t) + else: + return None + return "+".join(out) + + def uitars_parser(result): "Translates UITARS actions to browser gym actions" # note karenu: I am not sure if the translation is perfect @@ -334,11 +376,12 @@ def uitars_parser(result): result["action"] = ( f"mouse_click(x={int(coords[0])}, y={int(coords[1])}, button='right')" ) - # hotkey(key='ctrl alt e') -> keyboard_press(key=key_comb) + # hotkey(key='ctrl alt e') -> keyboard_press(key='Control+Alt+e') if result["action"].startswith("hotkey(key="): key_comb = re.findall(r"hotkey\(key='(.*?)'\)", result["action"]) if key_comb: - result["action"] = f"keyboard_press(key='{key_comb[0]}')" + key = _normalize_hotkey_key(key_comb[0]) or key_comb[0] + result["action"] = f"keyboard_press(key='{key}')" return result diff --git a/src/open_apps/agent/vLLM_agent.py b/src/open_apps/agent/vLLM_agent.py index e49a339..f40eff1 100644 --- a/src/open_apps/agent/vLLM_agent.py +++ b/src/open_apps/agent/vLLM_agent.py @@ -18,6 +18,7 @@ ) from .utils import CustomActionSetArgs, retry, save_som_coordinates from .vLLM_prompt import VllmMainPrompt, PromptFlags +from .action_parsers import get_action_parser from anthropic import AnthropicBedrock from openai import AzureOpenAI, OpenAI @@ -226,6 +227,12 @@ class AgentArgs(AgentLabAgentArgs): aws_session_token: str = None aws_region: str = "us-west-2" base_url: str = None + # Per-model-family action_parser (parser + coordinate space). See + # open_apps.agent.action_parsers. Default preserves the flexible_parser path. + action_parser: str = "uitars" + # User-message sections to render; None = legacy default. See + # VllmMainPrompt._SECTION_RENDERERS. + prompt_sections: list[str] = None def make_flags(self) -> PromptFlags: return PromptFlags( @@ -288,6 +295,8 @@ def make_agent(self) -> Agent: flags=self.make_flags(), prompt_txt=self.prompt_txt, save_dir=self.save_dir, + action_parser_name=self.action_parser, + prompt_sections=self.prompt_sections, ) @@ -299,6 +308,8 @@ def __init__( prompt_txt: dict, max_retry: int = 3, save_dir: str = None, + action_parser_name: str = "uitars", + prompt_sections: list[str] | None = None, ): logging.info("Initializing vllmAgent with flags: %s", asdict(flags)) self.chat_llm = chat_model_args.make_model() @@ -307,6 +318,10 @@ def __init__( self.flags = flags self.action_set = flags.action.action_set.make_action_set() self._obs_preprocessor = dp.make_obs_preprocessor(flags.obs) + self.action_parser = get_action_parser(action_parser_name) + self.prompt_sections = ( + list(prompt_sections) if prompt_sections is not None else None + ) self.prompt_txt = prompt_txt self.save_dir = Path(save_dir) if save_dir is not None else None self.reset(seed=None) @@ -327,6 +342,8 @@ def get_action(self, obs: Any): flags=self.flags, prompt_txt=self.prompt_txt, # pass the flags to the prompt client_type=self.chat_model_args.client_type, + action_parser=self.action_parser, + prompt_sections=self.prompt_sections, ) system_prompt = SystemMessage( diff --git a/src/open_apps/agent/vLLM_prompt.py b/src/open_apps/agent/vLLM_prompt.py index 7e4e294..2691234 100644 --- a/src/open_apps/agent/vLLM_prompt.py +++ b/src/open_apps/agent/vLLM_prompt.py @@ -21,14 +21,14 @@ from open_apps.agent.utils import flexible_parser def image_to_jpg_base64_url(image: np.ndarray | Image.Image): - """Convert a numpy array to a base64 encoded image url.""" - + # PNG, not JPEG: some sglang multimodal processors (e.g. deepseek-vl2) + # crash on JPEG input. if isinstance(image, np.ndarray): image = Image.fromarray(image) if image.mode in ("RGBA", "LA"): image = image.convert("RGB") buffered = io.BytesIO() - image.save(buffered, format="JPEG") + image.save(buffered, format="PNG") return base64.standard_b64encode(buffered.getvalue()).decode("utf-8") class HumanMessage(_HumanMessage): @@ -37,16 +37,15 @@ def __init__(self, content, client_type): self.client_type = client_type def add_image(self, image: np.ndarray | Image.Image | str, detail: str = None): - + if not isinstance(image, str): image = image_to_jpg_base64_url(image) - if self.client_type == "aws": - self["content"].append({"type": "image", "source": {"type": 'base64', "media_type": "image/jpeg", "data": image}}) + self["content"].append({"type": "image", "source": {"type": 'base64', "media_type": "image/png", "data": image}}) return - image = f"data:image/jpeg;base64,{image}" + image = f"data:image/png;base64,{image}" if detail: self.add_content("image_url", {"url": image, "detail": detail}) else: @@ -244,18 +243,29 @@ def __init__( thoughts: list[str], flags: PromptFlags, prompt_txt: dict, - client_type: str = "vllm" + client_type: str = "vllm", + action_parser=None, + prompt_sections: list[str] | None = None, ) -> None: super().__init__() self.flags = flags self.history = History(actions, thoughts) obs = obs_history[-1] - + self.goal = obs["goal_object"] - + self.obs = Observation(obs_history[-1], self.flags.obs) - + self.prompt_txt = prompt_txt + self.action_parser = action_parser + self.prompt_sections = prompt_sections + + # Viewport (width, height) for coord rescaling; screenshot is (H, W, 3). + screenshot = obs.get("screenshot") + if screenshot is not None and hasattr(screenshot, "shape"): + self.viewport = (int(screenshot.shape[1]), int(screenshot.shape[0])) + else: + self.viewport = (1920, 1080) self.action_prompt = ActionPrompt(action_set, action_flags=flags.action, concrete_ex_txt=prompt_txt.get("action_concrete_example"), @@ -269,16 +279,16 @@ def __init__( self.client_type = client_type - @property - def _prompt(self) -> HumanMessage: - # todo: maybe surface out the ordering of the elements in the prompt - - prompt = HumanMessage(f""" + # Section renderers: each returns the text for one user-message section + # ("" = render nothing). ``_prompt`` composes them per prompt_sections. + + def _render_task(self) -> str: + return f""" # User Instructions -{self.goal[0]['text']}""", self.client_type) - - prompt.add_text( - f""" +{self.goal[0]['text']}""" + + def _render_header_block(self) -> str: + return f""" ## Output Format {self.prompt_txt.output_format} \ {self.obs.prompt}\ @@ -288,11 +298,15 @@ def _prompt(self) -> HumanMessage: """ - ) - if self.flags.use_abstract_example: - prompt.add_text( - f""" + def _render_history_only(self) -> str: + # History alone, for action_parsers that suppress header_block (e.g. qwen3vl). + return self.history._prompt + + def _render_abstract_example(self) -> str: + if not self.flags.use_abstract_example: + return "" + return f""" # Abstract Example Here is an abstract version of the answer with description of the content of @@ -302,13 +316,13 @@ def _prompt(self) -> HumanMessage: {self.action_prompt.abstract_ex}\ -Do not output anything except the thought and action. +Do not output anything except the thought and action. """ - ) - if self.flags.use_concrete_example: - prompt.add_text( - f""" + def _render_concrete_example(self) -> str: + if not self.flags.use_concrete_example: + return "" + return f""" # Concrete Example Here is a concrete example of how to format your answer. @@ -317,9 +331,50 @@ def _prompt(self) -> HumanMessage: {self.action_prompt.concrete_ex}\ It is very important that you follow the format above. """ - ) + + _SECTION_RENDERERS = { + "task": "_render_task", + "header_block": "_render_header_block", + "history": "_render_history_only", + "abstract_example": "_render_abstract_example", + "concrete_example": "_render_concrete_example", + } + + # Legacy order, used when prompt_sections is unset. + _DEFAULT_SECTIONS = ( + "task", + "header_block", + "abstract_example", + "concrete_example", + ) + + @property + def _prompt(self) -> HumanMessage: + sections = self.prompt_sections or self._DEFAULT_SECTIONS + rendered = [] + for name in sections: + renderer_name = self._SECTION_RENDERERS.get(name) + if renderer_name is None: + raise ValueError( + f"Unknown prompt section {name!r}. " + f"Known: {sorted(self._SECTION_RENDERERS)}" + ) + text = getattr(self, renderer_name)() + if text: + rendered.append(text) + + # HumanMessage requires an initial text; keep one so the screenshot + # can attach even if every section was suppressed. + if not rendered: + rendered = [""] + + prompt = HumanMessage(rendered[0], self.client_type) + for text in rendered[1:]: + prompt.add_text(text) return self.obs.add_screenshot(prompt) - + def _parse_answer(self, text_answer): + if self.action_parser is not None: + return self.action_parser.parse(text_answer, viewport=self.viewport) return flexible_parser(text_answer) diff --git a/tests/test_action_parsers.py b/tests/test_action_parsers.py new file mode 100644 index 0000000..959d0c2 --- /dev/null +++ b/tests/test_action_parsers.py @@ -0,0 +1,188 @@ +"""Tests for per-model-family action_parsers (registry + qwen3vl parser + uitars).""" +from __future__ import annotations + +import json + +import pytest +from agentlab.llm.llm_utils import ParseError + +from open_apps.agent.action_parsers import REGISTRY, get_action_parser +from open_apps.agent.action_parsers.qwen3vl import Qwen3VLActionParser + + +VIEWPORT = (1920, 1080) + + +# --------------------------------------------------------------------------- +# Registry plumbing +# --------------------------------------------------------------------------- + +def test_registry_lists_known_action_parsers(): + assert set(REGISTRY) == {"uitars", "qwen3vl"} + + +def test_get_action_parser_defaults_to_uitars_when_none(): + assert type(get_action_parser(None)).__name__ == "UITarsActionParser" + + +def test_get_action_parser_raises_on_unknown_name(): + with pytest.raises(ValueError, match="Unknown action_parser 'bogus'"): + get_action_parser("bogus") + + +# --------------------------------------------------------------------------- +# qwen3vl: prompts live in yaml, not the action_parser +# --------------------------------------------------------------------------- + +def test_qwen_action_parser_supplies_no_prompt_defaults(): + assert Qwen3VLActionParser().default_prompts() == {} + + +# --------------------------------------------------------------------------- +# qwen3vl parser: every supported action, end-to-end +# --------------------------------------------------------------------------- + +def _qwen_response(action: str, **args) -> str: + """The envelope shape Qwen-VL emits when it issues a tool call.""" + payload = {"name": "computer_use", "arguments": {"action": action, **args}} + return f"\n{json.dumps(payload)}\n" + + +@pytest.fixture +def qwen(): + return Qwen3VLActionParser() + + +def test_qwen_parses_left_click_and_rescales_to_viewport(qwen): + # 0-1000: (870, 940) on 1920x1080 -> (round(870*1920/1000)=1670, + # round(940*1080/1000)=1015). + response = ( + "The Send button is bottom-right.\n" + + _qwen_response("mouse_click", coordinate=[870, 940]) + ) + out = qwen.parse(response, viewport=VIEWPORT) + assert out["action"] == "mouse_click(x=1670, y=1015)" + assert out["think"] == "The Send button is bottom-right." + assert "" in out["displayed_action"] + + +def test_qwen_captures_action_line_as_think(qwen): + # Instruct variants use ``Action: ...`` instead of . + response = ( + "Action: Click the Send button at the bottom-right of the chat panel.\n" + + _qwen_response("mouse_click", coordinate=[870, 940]) + ) + out = qwen.parse(response, viewport=VIEWPORT) + assert out["think"] == "Click the Send button at the bottom-right of the chat panel." + assert out["action"] == "mouse_click(x=1670, y=1015)" + + +def test_qwen_think_block_wins_over_action_line(qwen): + response = ( + "structured plan\n" + "Action: short plan\n" + + _qwen_response("mouse_click", coordinate=[100, 200]) + ) + assert qwen.parse(response, viewport=VIEWPORT)["think"] == "structured plan" + + +def test_qwen_parses_type_action_with_content_kwarg(qwen): + response = _qwen_response("type", content="hello world\n") + out = qwen.parse(response, viewport=VIEWPORT) + assert out["action"] == "keyboard_type(text='hello world\\n')" + assert out["think"] is None + + +def test_qwen_type_action_accepts_text_alias(qwen): + response = _qwen_response("type", text="hi\n") + assert qwen.parse(response, viewport=VIEWPORT)["action"] == "keyboard_type(text='hi\\n')" + + +def test_qwen_parses_scroll_down_as_positive_delta(qwen): + # delta [0, 500] in 0-1000 -> (0, round(500*1080/1000)=540). + response = _qwen_response("scroll", delta=[0, 500]) + assert qwen.parse(response, viewport=VIEWPORT)["action"] == "scroll(0, 540)" + + +def test_qwen_parses_scroll_up_signs(qwen): + response = _qwen_response("scroll", delta=[0, -500]) + assert qwen.parse(response, viewport=VIEWPORT)["action"] == "scroll(0, -540)" + + +def test_qwen_parses_mouse_dblclick(qwen): + # 100/1000*1920=192; 200/1000*1080=216. + response = _qwen_response("mouse_dblclick", coordinate=[100, 200]) + assert qwen.parse(response, viewport=VIEWPORT)["action"] == "mouse_dblclick(x=192, y=216)" + + +def test_qwen_wait_maps_to_noop_with_5s_default(qwen): + assert qwen.parse(_qwen_response("wait"), viewport=VIEWPORT)["action"] == "noop(wait_ms=5000.0)" + out = qwen.parse(_qwen_response("wait", wait_ms=1000), viewport=VIEWPORT) + assert out["action"] == "noop(wait_ms=1000.0)" + + +# --------------------------------------------------------------------------- +# qwen3vl parser: robustness +# --------------------------------------------------------------------------- + +def test_qwen_handles_truncated_closing_tag(qwen): + # vLLM sometimes cuts off mid-tag; the regex allows a missing . + response = ( + "\n" + '{"name": "computer_use", "arguments": {"action": "mouse_click", "coordinate": [100, 200]}}' + ) + assert qwen.parse(response, viewport=VIEWPORT)["action"] == "mouse_click(x=192, y=216)" + + +def test_qwen_raises_on_empty_response(qwen): + with pytest.raises(ParseError, match="Empty response"): + qwen.parse("", viewport=VIEWPORT) + + +def test_qwen_raises_when_tool_call_block_missing(qwen): + with pytest.raises(ParseError, match="Expected a "): + qwen.parse("I think I should click somewhere.", viewport=VIEWPORT) + + +def test_qwen_raises_on_malformed_json(qwen): + with pytest.raises(ParseError, match="not valid JSON"): + qwen.parse("{not valid json}", viewport=VIEWPORT) + + +def test_qwen_raises_on_unsupported_action(qwen): + with pytest.raises(ParseError, match="Unsupported action 'teleport'"): + qwen.parse(_qwen_response("teleport", coordinate=[1, 2]), viewport=VIEWPORT) + + +def test_qwen_raises_when_mouse_click_missing_coordinate(qwen): + response = ( + '{"name": "computer_use", ' + '"arguments": {"action": "mouse_click"}}' + ) + with pytest.raises(ParseError, match="must be a \\[x, y\\] list"): + qwen.parse(response, viewport=VIEWPORT) + + +def test_qwen_raises_when_type_missing_content(qwen): + response = ( + '{"name": "computer_use", ' + '"arguments": {"action": "type"}}' + ) + with pytest.raises(ParseError, match="missing 'content'"): + qwen.parse(response, viewport=VIEWPORT) + + +# --------------------------------------------------------------------------- +# uitars action_parser: default path must be untouched +# --------------------------------------------------------------------------- + +def test_uitars_action_parser_parses_native_action_syntax(): + a = get_action_parser("uitars") + response = ( + "Click the Submit button." + "click(point='100 200')" + ) + out = a.parse(response, viewport=VIEWPORT) + assert out["action"] == "mouse_click(x=100, y=200)" + assert out["think"] == "Click the Submit button." + assert out["displayed_action"] == "click(point='100 200')"