diff --git a/src/open_apps/agent/utils.py b/src/open_apps/agent/utils.py index e1f3fda..6c26562 100644 --- a/src/open_apps/agent/utils.py +++ b/src/open_apps/agent/utils.py @@ -303,22 +303,26 @@ def uitars_parser(result): # type(content=text) -> keyboard_type(text=text) if result["action"].startswith("type(content="): result["action"] = translate_uitars_type_action(result["action"]) - # scroll(direction='down', point='(906,509)') -> scroll(dx, dy) - if result["action"].startswith("scroll(direction=d"): - direction = re.findall( - r"scroll\(direction='(.*?)', point='\((\d+),(\d+)\)'\)", result["action"] + # scroll(direction='down'|'up'|'left'|'right', point='(x, y)') -> scroll(dx, dy) + if result["action"].startswith("scroll("): + dir_match = re.search( + r"direction\s*=\s*['\"]?(down|up|left|right)", + result["action"], + re.IGNORECASE, ) - if direction: - result["action"] = f"scroll({int(direction[0][1])}, {int(direction[0][2])})" - # scroll(direction='up', point='(906,509)') -> scroll(dx, dy) - if result["action"].startswith("scroll(direction=u"): - direction = re.findall( - r"scroll\(direction='(.*?)', point='\((\d+),(\d+)\)'\)", result["action"] - ) - if direction: - result["action"] = ( - f"scroll({-int(direction[0][1])}, {-int(direction[0][2])})" - ) + nums = re.findall(r"-?\d+", result["action"]) + if dir_match and len(nums) >= 2: + direction = dir_match.group(1).lower() + x, y = int(nums[0]), int(nums[1]) + if direction == "down": + dx, dy = 0, y + elif direction == "up": + dx, dy = 0, -y + elif direction == "right": + dx, dy = x, 0 + else: + dx, dy = -x, 0 + result["action"] = f"scroll({dx}, {dy})" # right_single(point='(531,256)') -> mouse_click(x, y, button='right') if result["action"].startswith("right_single(point="): coords = re.findall(r"\d+", result["action"]) diff --git a/tests/test_uitars_parser.py b/tests/test_uitars_parser.py new file mode 100644 index 0000000..60f5a01 --- /dev/null +++ b/tests/test_uitars_parser.py @@ -0,0 +1,30 @@ +from open_apps.agent.utils import flexible_parser + + +def _action(native: str) -> str: + return flexible_parser(f"t{native}")["action"] + + +def test_scroll_down_translates_to_positive_dy(): + assert _action("scroll(direction='down', point='(612,455)')") == "scroll(0, 455)" + + +def test_scroll_up_translates_to_negative_dy(): + assert _action("scroll(direction='up', point='(1920,536)')") == "scroll(0, -536)" + + +def test_scroll_left_and_right_move_along_x_axis(): + assert _action("scroll(direction='right', point='(300,400)')") == "scroll(300, 0)" + assert _action("scroll(direction='left', point='(300,400)')") == "scroll(-300, 0)" + + +def test_scroll_tolerates_whitespace_in_point(): + assert _action("scroll(direction='down', point='(612, 455)')") == "scroll(0, 455)" + + +def test_click_point_regression(): + assert _action("click(point='(100,200)')") == "mouse_click(x=100, y=200)" + + +def test_type_regression(): + assert _action("type(content='hello\\n')") == "keyboard_type(text='hello\\n')"