From 2a5d22c6be9616f6e27037d1abeb4f8cc17c5476 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Sat, 18 Apr 2026 16:49:41 +0800 Subject: [PATCH] Add clipboard, hotkeys, triggers, cron, REST, CLI, plugins, i18n MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every new feature ships with a headless Python API, an AC_* executor command (where applicable), and a GUI affordance — keeping the facade Qt-free. New modules: clipboard, hotkey daemon, trigger engine, cron scheduling, REST API server, plugin loader, subcommand CLI, script variables, OCR, recording editor, window manager, watcher. GUI: menu bar with File/View/Tools/Language/Help, closable tabs with a View → Tabs restore menu, new Hotkeys/Triggers/Plugins tabs, socket-server tab split into TCP + REST sections, and live language switching across English / Traditional Chinese / Simplified Chinese / Japanese. CLAUDE.md now requires every feature to expose both headless and GUI access paths, verified by an import-time PySide6 leak check. Docs: new /new_features/ page (Eng + Zh), updated CLI page to document the python -m je_auto_control.cli subcommand interface. Tests: 40 new headless unit tests (cron, hotkey parse, plugin loader, trigger engine, REST server, CLI, clipboard round-trip). --- CLAUDE.md | 21 ++ docs/source/Eng/doc/cli/cli_doc.rst | 75 +++- .../Eng/doc/new_features/new_features_doc.rst | 211 +++++++++++ docs/source/Eng/eng_index.rst | 1 + .../Zh/doc/new_features/new_features_doc.rst | 200 ++++++++++ docs/source/Zh/zh_index.rst | 1 + je_auto_control/__init__.py | 88 ++++- je_auto_control/cli.py | 137 +++++++ je_auto_control/gui/_shell_report_tabs.py | 159 ++++++++ je_auto_control/gui/hotkeys_tab.py | 107 ++++++ .../gui/language_wrapper/english.py | 25 ++ .../gui/language_wrapper/japanese.py | 42 +++ .../multi_language_wrapper.py | 91 ++++- .../language_wrapper/simplified_chinese.py | 42 +++ .../language_wrapper/traditional_chinese.py | 25 ++ je_auto_control/gui/live_hud_tab.py | 78 ++++ je_auto_control/gui/main_widget.py | 318 ++++++++-------- je_auto_control/gui/main_window.py | 188 ++++++++-- je_auto_control/gui/plugins_tab.py | 61 +++ je_auto_control/gui/recording_editor_tab.py | 188 ++++++++++ je_auto_control/gui/scheduler_tab.py | 111 ++++++ .../gui/script_builder/__init__.py | 4 + .../gui/script_builder/builder_tab.py | 138 +++++++ .../gui/script_builder/command_schema.py | 346 ++++++++++++++++++ .../gui/script_builder/step_form_view.py | 246 +++++++++++++ .../gui/script_builder/step_list_view.py | 168 +++++++++ .../gui/script_builder/step_model.py | 77 ++++ je_auto_control/gui/selector/__init__.py | 5 + .../gui/selector/region_overlay.py | 107 ++++++ .../gui/selector/region_selector.py | 17 + .../gui/selector/template_cropper.py | 50 +++ je_auto_control/gui/socket_server_tab.py | 141 +++++++ je_auto_control/gui/triggers_tab.py | 227 ++++++++++++ je_auto_control/gui/window_tab.py | 95 +++++ je_auto_control/utils/clipboard/__init__.py | 4 + je_auto_control/utils/clipboard/clipboard.py | 160 ++++++++ .../utils/executor/action_executor.py | 143 ++++++-- .../utils/executor/action_schema.py | 58 +++ .../utils/executor/flow_control.py | 192 ++++++++++ je_auto_control/utils/hotkey/__init__.py | 6 + je_auto_control/utils/hotkey/hotkey_daemon.py | 211 +++++++++++ je_auto_control/utils/ocr/__init__.py | 10 + je_auto_control/utils/ocr/ocr_engine.py | 159 ++++++++ .../utils/plugin_loader/__init__.py | 8 + .../utils/plugin_loader/plugin_loader.py | 77 ++++ .../utils/recording_edit/__init__.py | 10 + .../utils/recording_edit/editor.py | 82 +++++ je_auto_control/utils/rest_api/__init__.py | 6 + je_auto_control/utils/rest_api/rest_server.py | 135 +++++++ je_auto_control/utils/scheduler/__init__.py | 6 + je_auto_control/utils/scheduler/cron.py | 109 ++++++ je_auto_control/utils/scheduler/scheduler.py | 179 +++++++++ je_auto_control/utils/script_vars/__init__.py | 6 + .../utils/script_vars/interpolate.py | 63 ++++ je_auto_control/utils/triggers/__init__.py | 10 + .../utils/triggers/trigger_engine.py | 193 ++++++++++ je_auto_control/utils/watcher/__init__.py | 6 + je_auto_control/utils/watcher/watcher.py | 72 ++++ .../wrapper/auto_control_window.py | 93 +++++ .../flow_control/test_flow_control.py | 149 ++++++++ .../unit_test/flow_control/test_step_model.py | 39 ++ test/unit_test/headless/__init__.py | 0 test/unit_test/headless/test_cli.py | 53 +++ test/unit_test/headless/test_clipboard.py | 32 ++ test/unit_test/headless/test_cron.py | 66 ++++ test/unit_test/headless/test_hotkey_parse.py | 52 +++ test/unit_test/headless/test_interpolate.py | 31 ++ test/unit_test/headless/test_ocr_engine.py | 30 ++ test/unit_test/headless/test_plugin_loader.py | 84 +++++ .../unit_test/headless/test_recording_edit.py | 56 +++ test/unit_test/headless/test_rest_server.py | 61 +++ test/unit_test/headless/test_scheduler.py | 67 ++++ .../unit_test/headless/test_trigger_engine.py | 81 ++++ test/unit_test/headless/test_watcher.py | 58 +++ 74 files changed, 6374 insertions(+), 273 deletions(-) create mode 100644 docs/source/Eng/doc/new_features/new_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/new_features_doc.rst create mode 100644 je_auto_control/cli.py create mode 100644 je_auto_control/gui/_shell_report_tabs.py create mode 100644 je_auto_control/gui/hotkeys_tab.py create mode 100644 je_auto_control/gui/language_wrapper/japanese.py create mode 100644 je_auto_control/gui/language_wrapper/simplified_chinese.py create mode 100644 je_auto_control/gui/live_hud_tab.py create mode 100644 je_auto_control/gui/plugins_tab.py create mode 100644 je_auto_control/gui/recording_editor_tab.py create mode 100644 je_auto_control/gui/scheduler_tab.py create mode 100644 je_auto_control/gui/script_builder/__init__.py create mode 100644 je_auto_control/gui/script_builder/builder_tab.py create mode 100644 je_auto_control/gui/script_builder/command_schema.py create mode 100644 je_auto_control/gui/script_builder/step_form_view.py create mode 100644 je_auto_control/gui/script_builder/step_list_view.py create mode 100644 je_auto_control/gui/script_builder/step_model.py create mode 100644 je_auto_control/gui/selector/__init__.py create mode 100644 je_auto_control/gui/selector/region_overlay.py create mode 100644 je_auto_control/gui/selector/region_selector.py create mode 100644 je_auto_control/gui/selector/template_cropper.py create mode 100644 je_auto_control/gui/socket_server_tab.py create mode 100644 je_auto_control/gui/triggers_tab.py create mode 100644 je_auto_control/gui/window_tab.py create mode 100644 je_auto_control/utils/clipboard/__init__.py create mode 100644 je_auto_control/utils/clipboard/clipboard.py create mode 100644 je_auto_control/utils/executor/action_schema.py create mode 100644 je_auto_control/utils/executor/flow_control.py create mode 100644 je_auto_control/utils/hotkey/__init__.py create mode 100644 je_auto_control/utils/hotkey/hotkey_daemon.py create mode 100644 je_auto_control/utils/ocr/__init__.py create mode 100644 je_auto_control/utils/ocr/ocr_engine.py create mode 100644 je_auto_control/utils/plugin_loader/__init__.py create mode 100644 je_auto_control/utils/plugin_loader/plugin_loader.py create mode 100644 je_auto_control/utils/recording_edit/__init__.py create mode 100644 je_auto_control/utils/recording_edit/editor.py create mode 100644 je_auto_control/utils/rest_api/__init__.py create mode 100644 je_auto_control/utils/rest_api/rest_server.py create mode 100644 je_auto_control/utils/scheduler/__init__.py create mode 100644 je_auto_control/utils/scheduler/cron.py create mode 100644 je_auto_control/utils/scheduler/scheduler.py create mode 100644 je_auto_control/utils/script_vars/__init__.py create mode 100644 je_auto_control/utils/script_vars/interpolate.py create mode 100644 je_auto_control/utils/triggers/__init__.py create mode 100644 je_auto_control/utils/triggers/trigger_engine.py create mode 100644 je_auto_control/utils/watcher/__init__.py create mode 100644 je_auto_control/utils/watcher/watcher.py create mode 100644 je_auto_control/wrapper/auto_control_window.py create mode 100644 test/unit_test/flow_control/test_flow_control.py create mode 100644 test/unit_test/flow_control/test_step_model.py create mode 100644 test/unit_test/headless/__init__.py create mode 100644 test/unit_test/headless/test_cli.py create mode 100644 test/unit_test/headless/test_clipboard.py create mode 100644 test/unit_test/headless/test_cron.py create mode 100644 test/unit_test/headless/test_hotkey_parse.py create mode 100644 test/unit_test/headless/test_interpolate.py create mode 100644 test/unit_test/headless/test_ocr_engine.py create mode 100644 test/unit_test/headless/test_plugin_loader.py create mode 100644 test/unit_test/headless/test_recording_edit.py create mode 100644 test/unit_test/headless/test_rest_server.py create mode 100644 test/unit_test/headless/test_scheduler.py create mode 100644 test/unit_test/headless/test_trigger_engine.py create mode 100644 test/unit_test/headless/test_watcher.py diff --git a/CLAUDE.md b/CLAUDE.md index fe37703..87f4898 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,6 +74,27 @@ python -m pytest test/integrated_test/ python -m build ``` +## Feature Delivery Rules + +### Every feature must ship both a headless API and a GUI surface + +No feature is complete unless it can be driven entirely without the GUI **and** has a corresponding GUI affordance. Concretely: + +- **Headless core in `utils/` or `wrapper/`**: all business logic lives in a module with zero `PySide6` imports. Users must be able to `import je_auto_control` and call the feature without ever instantiating a Qt class. +- **Re-export from the package facade**: add the public functions / classes to `je_auto_control/__init__.py` and its `__all__` so `import je_auto_control as ac; ac.(...)` works out of the box. +- **Executor command coverage**: wire an `AC_*` command into `utils/executor/action_executor.py` so the feature is usable from JSON action files, the socket server, the scheduler, and the visual script builder — all without Python glue. +- **GUI tab or control is a thin wrapper**: the Qt widget must only translate user input into calls on the headless core. It must not contain business logic that would be unreachable headlessly. +- **The top-level package stays Qt-free**: `import je_auto_control` MUST NOT import `PySide6`. The GUI entry point is loaded lazily inside `start_autocontrol_gui()`. Verify with: + + ```python + import sys, je_auto_control # noqa + assert not any("PySide6" in m for m in sys.modules) + ``` + +- **Tests cover the headless path**: at least one unit test in `test/unit_test/` must exercise the feature through its non-GUI API with no Qt imports. + +Features that are inherently interactive (e.g. region picking with the mouse, template cropping) still count as GUI-only — but they must accept programmatic equivalents (e.g. `screenshot(screen_region=[...])` with explicit coordinates) so scripts can replay the same effect headlessly. + ## Coding Standards ### Security First diff --git a/docs/source/Eng/doc/cli/cli_doc.rst b/docs/source/Eng/doc/cli/cli_doc.rst index a2a709e..18d3fe8 100644 --- a/docs/source/Eng/doc/cli/cli_doc.rst +++ b/docs/source/Eng/doc/cli/cli_doc.rst @@ -2,47 +2,90 @@ Command-Line Interface ======================= -AutoControl can be used directly from the command line to execute automation scripts. +Two CLI entry points are provided: -Execute a Single Action File -============================= +- ``python -m je_auto_control`` — legacy flag-style runner for one-off + execute / create-project operations. Also launches the GUI when called + with no arguments. +- ``python -m je_auto_control.cli`` — subcommand-based runner for running + scripts, listing scheduler jobs, and starting the socket / REST servers. + +Subcommand CLI (``python -m je_auto_control.cli``) +================================================== + +Run a script +------------ .. code-block:: bash - python -m je_auto_control --execute_file "path/to/actions.json" + python -m je_auto_control.cli run script.json + python -m je_auto_control.cli run script.json --var count=10 --var name=alice + python -m je_auto_control.cli run script.json --dry-run + +``--var name=value`` is parsed as JSON when the value parses, otherwise +it is treated as a plain string. ``--dry-run`` records every action +through the executor without invoking any side effects. + +List scheduler jobs +------------------- + +.. code-block:: bash + + python -m je_auto_control.cli list-jobs + +Start the TCP socket server +--------------------------- + +.. code-block:: bash + + python -m je_auto_control.cli start-server --host 127.0.0.1 --port 9938 - # Short form +Start the REST API server +------------------------- + +.. code-block:: bash + + python -m je_auto_control.cli start-rest --host 127.0.0.1 --port 9939 + +Endpoints: ``GET /health``, ``GET /jobs``, ``POST /execute`` with +``{"actions": [...]}``. + +Legacy flag-style CLI (``python -m je_auto_control``) +===================================================== + +Execute a single action file +---------------------------- + +.. code-block:: bash + + python -m je_auto_control --execute_file "path/to/actions.json" python -m je_auto_control -e "path/to/actions.json" -Execute All Files in a Directory -================================ +Execute all files in a directory +-------------------------------- .. code-block:: bash python -m je_auto_control --execute_dir "path/to/action_files/" - - # Short form python -m je_auto_control -d "path/to/action_files/" -Execute a JSON String Directly -============================== +Execute a JSON string directly +------------------------------ .. code-block:: bash python -m je_auto_control --execute_str '[["AC_screenshot", {"file_path": "test.png"}]]' -Create a Project Template -========================= +Create a project template +------------------------- .. code-block:: bash python -m je_auto_control --create_project "path/to/my_project" - - # Short form python -m je_auto_control -c "path/to/my_project" Launch the GUI -============== +-------------- .. code-block:: bash diff --git a/docs/source/Eng/doc/new_features/new_features_doc.rst b/docs/source/Eng/doc/new_features/new_features_doc.rst new file mode 100644 index 0000000..29fb41e --- /dev/null +++ b/docs/source/Eng/doc/new_features/new_features_doc.rst @@ -0,0 +1,211 @@ +===================== +New Features (2026-04) +===================== + +This page documents the April 2026 additions to AutoControl. Every new +feature ships with a headless Python API **and** a GUI affordance, and is +wired into the executor so it works from JSON scripts, the socket server, +the REST API, and the CLI without any Python glue. + +.. contents:: + :local: + :depth: 2 + + +Clipboard +========= + +Headless:: + + import je_auto_control as ac + ac.set_clipboard("hello") + text = ac.get_clipboard() + +Action-JSON commands:: + + [["AC_clipboard_set", {"text": "hello"}]] + [["AC_clipboard_get", {}]] + +Backends: Windows (Win32 via ``ctypes``), macOS (``pbcopy``/``pbpaste``), +Linux (``xclip`` or ``xsel``). A ``RuntimeError`` is raised if no backend +is available. + + +Dry-run / step-debug executor +============================= + +Run an action list through the executor without invoking any side effects +— useful for validating JSON scripts:: + + from je_auto_control.utils.executor.action_executor import executor + record = executor.execute_action(actions, dry_run=True) + +``step_callback`` lets you observe each action before it runs:: + + executor.execute_action(actions, step_callback=lambda a: print(a)) + +From the CLI:: + + python -m je_auto_control.cli run script.json --dry-run + + +Global hotkey daemon (Windows) +============================== + +Bind OS-level hotkeys to action-JSON scripts:: + + from je_auto_control import default_hotkey_daemon + default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") + default_hotkey_daemon.start() + +Supported modifiers: ``ctrl``, ``alt``, ``shift``, ``win`` / ``super`` / +``meta``. Keys: letters, digits, ``f1`` … ``f12``, arrows, ``space``, +``enter``, ``tab``, ``escape``, ``home``, ``end``, ``insert``, ``delete``, +``pageup``, ``pagedown``. + +macOS and Linux currently raise ``NotImplementedError`` on +``start()`` — the Strategy-pattern interface is in place so backends can +be added later. + +GUI: **Hotkeys** tab (bind/unbind, start/stop daemon, live fired count). + + +Event triggers +============== + +Poll-based triggers fire an action script when a screen/state change is +detected:: + + from je_auto_control import default_trigger_engine, ImageAppearsTrigger + default_trigger_engine.add(ImageAppearsTrigger( + trigger_id="", script_path="scripts/click_ok.json", + image_path="templates/ok_button.png", threshold=0.85, + repeat=True, + )) + default_trigger_engine.start() + +Available trigger types: + +- ``ImageAppearsTrigger`` — template match on the current screen +- ``WindowAppearsTrigger`` — title substring match +- ``PixelColorTrigger`` — pixel color within tolerance +- ``FilePathTrigger`` — mtime change on a path + +GUI: **Triggers** tab (add/remove/start/stop, live fired count). + + +Cron scheduling +=============== + +Five-field cron (``minute hour day-of-month month day-of-week``) with +``*``, comma-lists, ``*/step``, and ``start-stop`` ranges:: + + from je_auto_control import default_scheduler + job = default_scheduler.add_cron_job( + script_path="scripts/daily.json", + cron_expression="0 9 * * 1-5", # 09:00 on weekdays + ) + default_scheduler.start() + +Interval and cron jobs coexist in the same scheduler; ``job.is_cron`` +tells them apart. GUI: **Scheduler** tab has cron/interval radio. + + +Plugin loader +============= + +A plugin file is any ``.py`` defining top-level callables whose names +start with ``AC_``. Each one becomes a new executor command:: + + # my_plugins/greeting.py + def AC_greet(args=None): + return f"hello, {args['name']}" + +:: + + from je_auto_control import ( + load_plugin_directory, register_plugin_commands, + ) + commands = load_plugin_directory("my_plugins/") + register_plugin_commands(commands) + + # Now usable from JSON: + # [["AC_greet", {"name": "world"}]] + +GUI: **Plugins** tab (browse directory, one-click register). + +.. warning:: + Plugin files execute arbitrary Python. Only load from directories + under your own control. + + +REST API server +=============== + +A stdlib-only HTTP server that exposes the executor and scheduler:: + + from je_auto_control import start_rest_api_server + server = start_rest_api_server(host="127.0.0.1", port=9939) + +Endpoints: + +- ``GET /health`` — liveness probe +- ``GET /jobs`` — scheduler job list +- ``POST /execute`` with body ``{"actions": [...]}`` — run actions + +GUI: **Socket Server** tab now has a separate REST section with its own +host/port and a ``0.0.0.0`` opt-in. + +.. note:: + Defaults to ``127.0.0.1`` per CLAUDE.md policy. Bind to ``0.0.0.0`` + only when you have authenticated the network boundary. + + +CLI runner +========== + +A thin subcommand-based CLI over the headless APIs:: + + python -m je_auto_control.cli run script.json + python -m je_auto_control.cli run script.json --var name=alice --dry-run + python -m je_auto_control.cli list-jobs + python -m je_auto_control.cli start-server --port 9938 + python -m je_auto_control.cli start-rest --port 9939 + +``--var name=value`` is parsed as JSON when possible (so ``count=10`` +becomes an int), otherwise treated as a string. + + +Multi-language GUI (i18n) +========================= + +Live language switching via the **Language** menu. Built-in packs: + +- English +- Traditional Chinese (繁體中文) +- Simplified Chinese (简体中文) +- Japanese (日本語) + +Register additional languages at runtime:: + + from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, + ) + language_wrapper.register_language("French", {"menu_file": "Fichier", ...}) + +Missing keys fall through to the English default, so a feature ships +with usable labels even before its translations land. + + +Closable tabs + menu bar +======================== + +The main window is now a ``QMainWindow`` with: + +- **File** → Open Script, Exit +- **View → Tabs** → checkable entries for every tab (restore closed tabs) +- **Tools** → Start hotkey daemon / scheduler / trigger engine +- **Language** → select a registered language pack +- **Help** → About + +Close any tab with its ✕ button; re-open it via *View → Tabs*. diff --git a/docs/source/Eng/eng_index.rst b/docs/source/Eng/eng_index.rst index 6e3b5b4..ad0a638 100644 --- a/docs/source/Eng/eng_index.rst +++ b/docs/source/Eng/eng_index.rst @@ -22,3 +22,4 @@ Comprehensive guides for all AutoControl features. doc/critical_exit/critical_exit_doc doc/cli/cli_doc doc/create_project/create_project_doc + doc/new_features/new_features_doc diff --git a/docs/source/Zh/doc/new_features/new_features_doc.rst b/docs/source/Zh/doc/new_features/new_features_doc.rst new file mode 100644 index 0000000..155a128 --- /dev/null +++ b/docs/source/Zh/doc/new_features/new_features_doc.rst @@ -0,0 +1,200 @@ +===================== +新功能 (2026-04) +===================== + +本頁說明 2026 年 4 月加入的新功能。每一項新功能都同時提供 **無 GUI 的 +Python API** 與 **對應的 GUI 介面**，並已連接到 executor，可從 JSON 腳 +本、socket server、REST API 及 CLI 直接使用，無需撰寫額外 Python 程式 +碼。 + +.. contents:: + :local: + :depth: 2 + + +剪貼簿 (Clipboard) +================== + +程式碼呼叫:: + + import je_auto_control as ac + ac.set_clipboard("hello") + text = ac.get_clipboard() + +Action-JSON 指令:: + + [["AC_clipboard_set", {"text": "hello"}]] + [["AC_clipboard_get", {}]] + +平台後端：Windows (ctypes + Win32)、macOS (``pbcopy`` / ``pbpaste``)、 +Linux (``xclip`` 或 ``xsel``)。若無可用後端會拋出 ``RuntimeError``。 + + +試跑 / 逐步除錯 (Dry-run / Step Debug) +====================================== + +無副作用執行 action 列表，方便驗證 JSON 腳本:: + + from je_auto_control.utils.executor.action_executor import executor + record = executor.execute_action(actions, dry_run=True) + +``step_callback`` 可在每一條 action 執行前收到通知:: + + executor.execute_action(actions, step_callback=lambda a: print(a)) + +CLI:: + + python -m je_auto_control.cli run script.json --dry-run + + +全域熱鍵 (Windows) +================== + +綁定系統級熱鍵到 action-JSON 腳本:: + + from je_auto_control import default_hotkey_daemon + default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") + default_hotkey_daemon.start() + +支援的修飾鍵：``ctrl``、``alt``、``shift``、``win`` / ``super`` / +``meta``。按鍵：英文字母、數字、``f1``–``f12``、方向鍵、``space``、 +``enter``、``tab``、``escape`` 等。 + +macOS 與 Linux 目前在 ``start()`` 會拋出 ``NotImplementedError`` ── +採 Strategy pattern，未來可擴充對應後端。 + +GUI：**全域熱鍵** 分頁。 + + +事件觸發器 (Triggers) +===================== + +輪詢式觸發器，偵測到畫面或狀態變化時執行腳本:: + + from je_auto_control import default_trigger_engine, ImageAppearsTrigger + default_trigger_engine.add(ImageAppearsTrigger( + trigger_id="", script_path="scripts/click_ok.json", + image_path="templates/ok_button.png", threshold=0.85, + repeat=True, + )) + default_trigger_engine.start() + +可用類型： + +- ``ImageAppearsTrigger`` — 螢幕上偵測模板影像 +- ``WindowAppearsTrigger`` — 視窗標題包含子字串 +- ``PixelColorTrigger`` — 某座標像素顏色在容差內 +- ``FilePathTrigger`` — 監看檔案 mtime 變化 + +GUI：**事件觸發器** 分頁。 + + +Cron 排程 +========= + +五欄位 cron (``minute hour day-of-month month day-of-week``)，支援 +``*``、逗號列表、``*/step`` 步進、``start-stop`` 範圍:: + + from je_auto_control import default_scheduler + job = default_scheduler.add_cron_job( + script_path="scripts/daily.json", + cron_expression="0 9 * * 1-5", # 週一到週五 09:00 + ) + default_scheduler.start() + +interval 與 cron 排程可同時存在，可由 ``job.is_cron`` 判斷類型。 +GUI：**排程器** 分頁新增 cron/interval 切換。 + + +外掛載入器 (Plugin Loader) +========================== + +外掛檔案為任何定義頂層 ``AC_*`` callable 的 ``.py``，每個都會成為新的 +executor 指令:: + + # my_plugins/greeting.py + def AC_greet(args=None): + return f"hello, {args['name']}" + +:: + + from je_auto_control import ( + load_plugin_directory, register_plugin_commands, + ) + commands = load_plugin_directory("my_plugins/") + register_plugin_commands(commands) + +GUI：**外掛** 分頁，可選擇目錄一鍵載入。 + +.. warning:: + 外掛檔案會直接執行任意 Python 程式。請務必只載入自己信任的目錄。 + + +REST API 伺服器 +=============== + +純 stdlib HTTP server，公開 executor 與 scheduler:: + + from je_auto_control import start_rest_api_server + server = start_rest_api_server(host="127.0.0.1", port=9939) + +端點： + +- ``GET /health`` +- ``GET /jobs`` +- ``POST /execute`` (body: ``{"actions": [...]}``) + +GUI：**Socket 伺服器** 分頁新增獨立的 REST 區塊。 + +.. note:: + 預設綁定 ``127.0.0.1`` (符合 CLAUDE.md 規範)。只有在網路邊界已做好 + 身分驗證時才綁定 ``0.0.0.0``。 + + +CLI 子指令介面 +============== + +以 headless API 為基礎的輕量 CLI:: + + python -m je_auto_control.cli run script.json + python -m je_auto_control.cli run script.json --var name=alice --dry-run + python -m je_auto_control.cli list-jobs + python -m je_auto_control.cli start-server --port 9938 + python -m je_auto_control.cli start-rest --port 9939 + +``--var name=value`` 優先以 JSON 解析 (``count=10`` 會變 int)，失敗時 +當作字串。 + + +GUI 多語系 +========== + +可透過 **Language** 選單即時切換。內建語言包： + +- English +- 繁體中文 (Traditional Chinese) +- 简体中文 (Simplified Chinese) +- 日本語 (Japanese) + +執行期可註冊額外語言:: + + from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, + ) + language_wrapper.register_language("French", {"menu_file": "Fichier", ...}) + +缺少的 key 會退回英文，讓新功能未翻譯前仍可正常顯示。 + + +可關閉分頁 + 選單列 +=================== + +主視窗改為 ``QMainWindow`` + 選單列： + +- **File** → 開啟腳本 / 結束 +- **View → Tabs** → 每個分頁可勾選顯示或隱藏 +- **Tools** → 啟動熱鍵 / 排程 / 觸發器服務 +- **Language** → 切換語言 +- **Help** → 關於 + +點擊分頁的 ✕ 即可關閉，之後可從 *View → Tabs* 恢復。 diff --git a/docs/source/Zh/zh_index.rst b/docs/source/Zh/zh_index.rst index 4b5a22a..c355a79 100644 --- a/docs/source/Zh/zh_index.rst +++ b/docs/source/Zh/zh_index.rst @@ -22,3 +22,4 @@ AutoControl 所有功能的完整使用指南。 doc/critical_exit/critical_exit_doc doc/cli/cli_doc doc/create_project/create_project_doc + doc/new_features/new_features_doc diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 25148f9..51f5192 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -35,8 +35,54 @@ add_command_to_executor # executor from je_auto_control.utils.executor.action_executor import execute_action +from je_auto_control.utils.executor.action_executor import \ + execute_action_with_vars from je_auto_control.utils.executor.action_executor import execute_files from je_auto_control.utils.executor.action_executor import executor +# Clipboard (headless) +from je_auto_control.utils.clipboard.clipboard import ( + get_clipboard, set_clipboard, +) +# Hotkey daemon (headless) +from je_auto_control.utils.hotkey.hotkey_daemon import ( + HotkeyBinding, HotkeyDaemon, default_hotkey_daemon, +) +# OCR (headless) +from je_auto_control.utils.ocr.ocr_engine import ( + TextMatch, click_text, find_text_matches, locate_text_center, + set_tesseract_cmd, wait_for_text, +) +# Plugin loader (headless) +from je_auto_control.utils.plugin_loader.plugin_loader import ( + discover_plugin_commands, load_plugin_directory, load_plugin_file, + register_plugin_commands, +) +# REST API (headless) +from je_auto_control.utils.rest_api.rest_server import ( + RestApiServer, start_rest_api_server, +) +# Triggers (headless) +from je_auto_control.utils.triggers.trigger_engine import ( + FilePathTrigger, ImageAppearsTrigger, PixelColorTrigger, TriggerEngine, + WindowAppearsTrigger, default_trigger_engine, +) +# Recording editor (headless helpers) +from je_auto_control.utils.recording_edit.editor import ( + adjust_delays, filter_actions, insert_action, remove_action, + scale_coordinates, trim_actions, +) +# Scheduler (headless) +from je_auto_control.utils.scheduler.scheduler import ( + ScheduledJob, Scheduler, default_scheduler, +) +# Script variables (headless) +from je_auto_control.utils.script_vars.interpolate import ( + interpolate_actions, interpolate_value, load_vars_from_json, +) +# Watchers (headless) +from je_auto_control.utils.watcher.watcher import ( + LogTail, MouseWatcher, PixelWatcher, +) # file process from je_auto_control.utils.file_process.get_dir_file_list import \ get_dir_files_as_list @@ -106,8 +152,17 @@ from je_auto_control.wrapper.auto_control_screen import screen_size from je_auto_control.wrapper.auto_control_screen import screenshot from je_auto_control.wrapper.auto_control_screen import get_pixel -# GUI -from je_auto_control.gui import start_autocontrol_gui +# Cross-platform window manager (headless) +from je_auto_control.wrapper.auto_control_window import ( + close_window_by_title, find_window, focus_window, list_windows, + show_window_by_title, wait_for_window, +) + + +def start_autocontrol_gui(*args, **kwargs): + """Launch the GUI (imports PySide6 lazily so headless usage stays Qt-free).""" + from je_auto_control.gui import start_autocontrol_gui as _impl + return _impl(*args, **kwargs) __all__ = [ "click_mouse", "mouse_keys_table", "get_mouse_position", "press_mouse", "release_mouse", @@ -120,7 +175,36 @@ "AutoControlScreenException", "ImageNotFoundException", "AutoControlJsonActionException", "AutoControlRecordException", "AutoControlActionNullException", "AutoControlActionException", "record", "stop_record", "read_action_json", "write_action_json", "execute_action", "execute_files", "executor", + "execute_action_with_vars", "add_command_to_executor", "test_record_instance", "pil_screenshot", + # OCR + "TextMatch", "find_text_matches", "locate_text_center", "wait_for_text", + "click_text", "set_tesseract_cmd", + # Recording editor + "trim_actions", "insert_action", "remove_action", "filter_actions", + "adjust_delays", "scale_coordinates", + # Scheduler + "Scheduler", "ScheduledJob", "default_scheduler", + # Script variables + "interpolate_actions", "interpolate_value", "load_vars_from_json", + # Watchers + "MouseWatcher", "PixelWatcher", "LogTail", + # Window manager + "list_windows", "find_window", "focus_window", "wait_for_window", + "close_window_by_title", "show_window_by_title", + # Clipboard + "get_clipboard", "set_clipboard", + # Hotkey daemon + "HotkeyDaemon", "HotkeyBinding", "default_hotkey_daemon", + # Plugin loader + "load_plugin_file", "load_plugin_directory", "discover_plugin_commands", + "register_plugin_commands", + # REST API + "RestApiServer", "start_rest_api_server", + # Triggers + "TriggerEngine", "default_trigger_engine", + "ImageAppearsTrigger", "WindowAppearsTrigger", + "PixelColorTrigger", "FilePathTrigger", "generate_html", "generate_html_report", "generate_json", "generate_json_report", "generate_xml", "generate_xml_report", "get_dir_files_as_list", "create_project_dir", "start_autocontrol_socket_server", "callback_executor", "package_manager", "ShellManager", "default_shell_manager", diff --git a/je_auto_control/cli.py b/je_auto_control/cli.py new file mode 100644 index 0000000..2468aa8 --- /dev/null +++ b/je_auto_control/cli.py @@ -0,0 +1,137 @@ +"""Command-line entry point. + +Usage:: + + python -m je_auto_control.cli run script.json [--var x=10 --var y=20] + python -m je_auto_control.cli list-jobs + python -m je_auto_control.cli start-server --port 9938 + python -m je_auto_control.cli start-rest --port 9939 + +The CLI is a thin wrapper around the headless APIs so every feature works +without ever importing PySide6. +""" +import argparse +import json +import signal +import sys +import time +from typing import Dict, List, Optional, Sequence + + +def _parse_vars(pairs: Optional[Sequence[str]]) -> Dict[str, object]: + """Parse ``--var name=value`` entries into a dict (JSON value when parseable).""" + resolved: Dict[str, object] = {} + for raw in pairs or []: + if "=" not in raw: + raise SystemExit(f"--var must be name=value; got {raw!r}") + name, value = raw.split("=", 1) + try: + resolved[name.strip()] = json.loads(value) + except ValueError: + resolved[name.strip()] = value + return resolved + + +def cmd_run(args: argparse.Namespace) -> int: + from je_auto_control.utils.executor.action_executor import ( + execute_action, execute_action_with_vars, + ) + from je_auto_control.utils.json.json_file import read_action_json + actions = read_action_json(args.script) + variables = _parse_vars(args.var) + if args.dry_run: + from je_auto_control.utils.executor.action_executor import executor + if variables: + from je_auto_control.utils.script_vars.interpolate import ( + interpolate_actions, + ) + actions = interpolate_actions(actions, variables) + result = executor.execute_action(actions, dry_run=True) + elif variables: + result = execute_action_with_vars(actions, variables) + else: + result = execute_action(actions) + json.dump(result, sys.stdout, indent=2, default=str, ensure_ascii=False) + sys.stdout.write("\n") + return 0 + + +def cmd_list_jobs(_: argparse.Namespace) -> int: + from je_auto_control.utils.scheduler.scheduler import default_scheduler + jobs = default_scheduler.list_jobs() + for job in jobs: + kind = "cron" if job.is_cron else f"{job.interval_seconds}s" + sys.stdout.write( + f"{job.job_id}\t{kind}\truns={job.runs}\t{job.script_path}\n" + ) + return 0 + + +def cmd_start_server(args: argparse.Namespace) -> int: + from je_auto_control.utils.socket_server.auto_control_socket_server import ( + start_autocontrol_socket_server, + ) + server = start_autocontrol_socket_server(args.host, args.port) + sys.stdout.write(f"Socket server listening on {args.host}:{args.port}\n") + _run_until_signal(server.shutdown) + server.server_close() + return 0 + + +def cmd_start_rest(args: argparse.Namespace) -> int: + from je_auto_control.utils.rest_api.rest_server import start_rest_api_server + server = start_rest_api_server(args.host, args.port) + sys.stdout.write(f"REST API listening on {server.address[0]}:{server.address[1]}\n") + _run_until_signal(server.stop) + return 0 + + +def _run_until_signal(shutdown: callable) -> None: + stopping = {"flag": False} + + def _handler(_signum, _frame): + stopping["flag"] = True + + signal.signal(signal.SIGINT, _handler) + try: + while not stopping["flag"]: + time.sleep(0.25) + finally: + shutdown() + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="je_auto_control") + sub = parser.add_subparsers(dest="command", required=True) + + p_run = sub.add_parser("run", help="Execute an action JSON file") + p_run.add_argument("script") + p_run.add_argument("--var", action="append", + help="name=value override; may be repeated") + p_run.add_argument("--dry-run", action="store_true", + help="record actions without calling them") + p_run.set_defaults(func=cmd_run) + + p_jobs = sub.add_parser("list-jobs", help="List scheduler jobs") + p_jobs.set_defaults(func=cmd_list_jobs) + + p_srv = sub.add_parser("start-server", help="Start the TCP socket server") + p_srv.add_argument("--host", default="127.0.0.1") + p_srv.add_argument("--port", type=int, default=9938) + p_srv.set_defaults(func=cmd_start_server) + + p_rest = sub.add_parser("start-rest", help="Start the REST API server") + p_rest.add_argument("--host", default="127.0.0.1") + p_rest.add_argument("--port", type=int, default=9939) + p_rest.set_defaults(func=cmd_start_rest) + return parser + + +def main(argv: Optional[List[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/je_auto_control/gui/_shell_report_tabs.py b/je_auto_control/gui/_shell_report_tabs.py new file mode 100644 index 0000000..5f509a5 --- /dev/null +++ b/je_auto_control/gui/_shell_report_tabs.py @@ -0,0 +1,159 @@ +"""Shell-command and Report-generation tab builders (extracted mixin).""" +from PySide6.QtWidgets import ( + QFileDialog, QGroupBox, QHBoxLayout, QLabel, QLineEdit, QPushButton, + QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.utils.generate_report.generate_html_report import generate_html_report +from je_auto_control.utils.generate_report.generate_json_report import generate_json_report +from je_auto_control.utils.generate_report.generate_xml_report import generate_xml_report +from je_auto_control.utils.shell_process.shell_exec import ShellManager +from je_auto_control.utils.start_exe.start_another_process import start_exe +from je_auto_control.utils.test_record.record_test_class import test_record_instance + + +def _t(key: str) -> str: + return language_wrapper.translate(key, key) + + +class ShellReportTabsMixin: + """Provides shell-command and report-generation tab builders/handlers.""" + + def _build_shell_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + shell_group = QGroupBox(_t("shell_command_label")) + sg = QVBoxLayout() + self.shell_input = QLineEdit() + self.shell_input.setPlaceholderText("echo hello") + self.shell_exec_btn = QPushButton(_t("execute_shell")) + self.shell_exec_btn.clicked.connect(self._execute_shell) + sh = QHBoxLayout() + sh.addWidget(self.shell_input) + sh.addWidget(self.shell_exec_btn) + sg.addLayout(sh) + shell_group.setLayout(sg) + layout.addWidget(shell_group) + + exe_group = QGroupBox(_t("start_exe_label")) + eg = QHBoxLayout() + self.exe_path_input = QLineEdit() + self.exe_browse_btn = QPushButton(_t("browse")) + self.exe_browse_btn.clicked.connect(self._browse_exe) + self.exe_start_btn = QPushButton(_t("start_exe")) + self.exe_start_btn.clicked.connect(self._start_exe) + eg.addWidget(self.exe_path_input) + eg.addWidget(self.exe_browse_btn) + eg.addWidget(self.exe_start_btn) + exe_group.setLayout(eg) + layout.addWidget(exe_group) + + layout.addWidget(QLabel(_t("shell_output"))) + self.shell_output_text = QTextEdit() + self.shell_output_text.setReadOnly(True) + layout.addWidget(self.shell_output_text) + tab.setLayout(layout) + return tab + + def _execute_shell(self): + try: + cmd = self.shell_input.text() + if not cmd: + return + mgr = ShellManager() + mgr.exec_shell(cmd) + self.shell_output_text.setText( + f"Executed: {cmd}\n(Check console for output)" + ) + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.shell_output_text.setText(f"Error: {error}") + + def _browse_exe(self): + path, _ = QFileDialog.getOpenFileName( + self, _t("start_exe_label"), "", "Executable (*.exe);;All (*)", + ) + if path: + self.exe_path_input.setText(path) + + def _start_exe(self): + try: + path = self.exe_path_input.text() + if not path: + return + start_exe(path) + self.shell_output_text.setText(f"Started: {path}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.shell_output_text.setText(f"Error: {error}") + + def _build_report_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + tr_group = QGroupBox(_t("test_record_status")) + tr_h = QHBoxLayout() + self.tr_enable_btn = QPushButton(_t("enable_test_record")) + self.tr_enable_btn.clicked.connect(lambda: self._set_test_record(True)) + self.tr_disable_btn = QPushButton(_t("disable_test_record")) + self.tr_disable_btn.clicked.connect(lambda: self._set_test_record(False)) + self.tr_status_label = QLabel("OFF") + tr_h.addWidget(self.tr_enable_btn) + tr_h.addWidget(self.tr_disable_btn) + tr_h.addWidget(self.tr_status_label) + tr_group.setLayout(tr_h) + layout.addWidget(tr_group) + + name_h = QHBoxLayout() + name_h.addWidget(QLabel(_t("report_name"))) + self.report_name_input = QLineEdit("autocontrol_report") + name_h.addWidget(self.report_name_input) + layout.addLayout(name_h) + + btn_h = QHBoxLayout() + self.html_report_btn = QPushButton(_t("generate_html_report")) + self.html_report_btn.clicked.connect(self._gen_html) + self.json_report_btn = QPushButton(_t("generate_json_report")) + self.json_report_btn.clicked.connect(self._gen_json) + self.xml_report_btn = QPushButton(_t("generate_xml_report")) + self.xml_report_btn.clicked.connect(self._gen_xml) + btn_h.addWidget(self.html_report_btn) + btn_h.addWidget(self.json_report_btn) + btn_h.addWidget(self.xml_report_btn) + layout.addLayout(btn_h) + + layout.addWidget(QLabel(_t("report_result"))) + self.report_result_text = QTextEdit() + self.report_result_text.setReadOnly(True) + layout.addWidget(self.report_result_text) + layout.addStretch() + tab.setLayout(layout) + return tab + + def _set_test_record(self, enable: bool): + test_record_instance.set_record_enable(enable) + self.tr_status_label.setText("ON" if enable else "OFF") + + def _gen_html(self): + try: + name = self.report_name_input.text() or "autocontrol_report" + generate_html_report(name) + self.report_result_text.setText(f"HTML report generated: {name}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.report_result_text.setText(f"Error: {error}") + + def _gen_json(self): + try: + name = self.report_name_input.text() or "autocontrol_report" + generate_json_report(name) + self.report_result_text.setText(f"JSON report generated: {name}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.report_result_text.setText(f"Error: {error}") + + def _gen_xml(self): + try: + name = self.report_name_input.text() or "autocontrol_report" + generate_xml_report(name) + self.report_result_text.setText(f"XML report generated: {name}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.report_result_text.setText(f"Error: {error}") diff --git a/je_auto_control/gui/hotkeys_tab.py b/je_auto_control/gui/hotkeys_tab.py new file mode 100644 index 0000000..a3c7746 --- /dev/null +++ b/je_auto_control/gui/hotkeys_tab.py @@ -0,0 +1,107 @@ +"""Hotkeys tab: bind global hotkeys to action JSON files.""" +from typing import Optional + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import ( + QFileDialog, QHBoxLayout, QLabel, QLineEdit, QMessageBox, QPushButton, + QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, +) + +from je_auto_control.utils.hotkey.hotkey_daemon import default_hotkey_daemon + + +class HotkeysTab(QWidget): + """Add / remove hotkey bindings and toggle the daemon.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._combo_input = QLineEdit() + self._combo_input.setPlaceholderText("ctrl+alt+1") + self._script_input = QLineEdit() + self._status = QLabel("Daemon stopped") + self._table = QTableWidget(0, 4) + self._table.setHorizontalHeaderLabels( + ["ID", "Combo", "Script", "Fired"] + ) + self._timer = QTimer(self) + self._timer.setInterval(1000) + self._timer.timeout.connect(self._refresh) + self._build_layout() + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + form = QHBoxLayout() + form.addWidget(QLabel("Combo:")) + form.addWidget(self._combo_input) + form.addWidget(QLabel("Script:")) + form.addWidget(self._script_input, stretch=1) + browse = QPushButton("Browse") + browse.clicked.connect(self._browse) + form.addWidget(browse) + add = QPushButton("Bind") + add.clicked.connect(self._on_bind) + form.addWidget(add) + root.addLayout(form) + + root.addWidget(self._table, stretch=1) + + ctl = QHBoxLayout() + for label, handler in ( + ("Remove selected", self._on_remove), + ("Start daemon", self._on_start), + ("Stop daemon", self._on_stop), + ): + btn = QPushButton(label) + btn.clicked.connect(handler) + ctl.addWidget(btn) + ctl.addStretch() + root.addLayout(ctl) + root.addWidget(self._status) + + def _browse(self) -> None: + path, _ = QFileDialog.getOpenFileName(self, "Select script", "", "JSON (*.json)") + if path: + self._script_input.setText(path) + + def _on_bind(self) -> None: + combo = self._combo_input.text().strip() + script = self._script_input.text().strip() + if not combo or not script: + QMessageBox.warning(self, "Error", "Combo and script path are required") + return + try: + default_hotkey_daemon.bind(combo, script) + except ValueError as error: + QMessageBox.warning(self, "Error", str(error)) + return + self._refresh() + + def _on_remove(self) -> None: + row = self._table.currentRow() + if row < 0: + return + bid = self._table.item(row, 0).text() + default_hotkey_daemon.unbind(bid) + self._refresh() + + def _on_start(self) -> None: + try: + default_hotkey_daemon.start() + except NotImplementedError as error: + QMessageBox.warning(self, "Error", str(error)) + return + self._timer.start() + self._status.setText("Daemon running") + + def _on_stop(self) -> None: + default_hotkey_daemon.stop() + self._timer.stop() + self._status.setText("Daemon stopped") + + def _refresh(self) -> None: + bindings = default_hotkey_daemon.list_bindings() + self._table.setRowCount(len(bindings)) + for row, binding in enumerate(bindings): + for col, value in enumerate((binding.binding_id, binding.combo, + binding.script_path, str(binding.fired))): + self._table.setItem(row, col, QTableWidgetItem(value)) diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 0d6c3ee..450e15b 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -8,6 +8,15 @@ "tab_image_detect": "Image Detection", "tab_record": "Record / Playback", "tab_script": "Script Executor", + "tab_script_builder": "Script Builder", + "tab_recording_editor": "Recording Editor", + "tab_window_manager": "Windows", + "tab_scheduler": "Scheduler", + "tab_socket_server": "Socket Server", + "tab_live_hud": "Live HUD", + "tab_hotkeys": "Hotkeys", + "tab_triggers": "Triggers", + "tab_plugins": "Plugins", "tab_screen_record": "Screen Recording", "tab_shell": "Shell Command", "tab_report": "Report", @@ -45,6 +54,8 @@ "file_path_label": "File Path:", "browse": "Browse", "region_label": "Region (x1, y1, x2, y2):", + "pick_region": "Pick Region", + "crop_template": "Crop Template", "get_pixel_label": "Get Pixel Color", "pixel_x": "X:", "pixel_y": "Y:", @@ -109,4 +120,18 @@ # Language "language_label": "Language:", + + # Menu bar + "menu_file": "File", + "menu_file_open_script": "Open Script...", + "menu_file_exit": "Exit", + "menu_view": "View", + "menu_view_tabs": "Tabs", + "menu_tools": "Tools", + "menu_tools_start_hotkeys": "Start hotkey daemon", + "menu_tools_start_scheduler": "Start scheduler", + "menu_tools_start_triggers": "Start trigger engine", + "menu_language": "Language", + "menu_help": "Help", + "menu_help_about": "About AutoControlGUI", } diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py new file mode 100644 index 0000000..095dd1b --- /dev/null +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -0,0 +1,42 @@ +japanese_word_dict = { + "application_name": "AutoControlGUI", + + # Tabs + "tab_auto_click": "自動クリック", + "tab_screenshot": "スクリーンショット", + "tab_image_detect": "画像検出", + "tab_record": "記録 / 再生", + "tab_script": "スクリプト実行", + "tab_script_builder": "スクリプトビルダー", + "tab_recording_editor": "記録エディタ", + "tab_window_manager": "ウィンドウ管理", + "tab_scheduler": "スケジューラー", + "tab_socket_server": "ソケットサーバー", + "tab_live_hud": "ライブ HUD", + "tab_hotkeys": "ホットキー", + "tab_triggers": "トリガー", + "tab_plugins": "プラグイン", + "tab_screen_record": "画面録画", + "tab_shell": "シェル", + "tab_report": "レポート", + + # Menu bar + "menu_file": "ファイル", + "menu_file_open_script": "スクリプトを開く...", + "menu_file_exit": "終了", + "menu_view": "表示", + "menu_view_tabs": "タブ", + "menu_tools": "ツール", + "menu_tools_start_hotkeys": "ホットキーデーモン開始", + "menu_tools_start_scheduler": "スケジューラー開始", + "menu_tools_start_triggers": "トリガーエンジン開始", + "menu_language": "言語", + "menu_help": "ヘルプ", + "menu_help_about": "AutoControlGUI について", + + # Status/messages + "language_label": "言語:", + "browse": "参照", + "start": "開始", + "stop": "停止", +} diff --git a/je_auto_control/gui/language_wrapper/multi_language_wrapper.py b/je_auto_control/gui/language_wrapper/multi_language_wrapper.py index bc8a1df..5ef0aba 100644 --- a/je_auto_control/gui/language_wrapper/multi_language_wrapper.py +++ b/je_auto_control/gui/language_wrapper/multi_language_wrapper.py @@ -1,30 +1,83 @@ +"""Runtime language switcher + registry. + +Languages are registered by a display name (shown in the menu) mapped to a +dict of key → translation. Missing keys fall through to the English default +so new features degrade gracefully before their translations land. +""" +from typing import Dict, List + from je_auto_control.gui.language_wrapper.english import english_word_dict -from je_auto_control.gui.language_wrapper.traditional_chinese import traditional_chinese_word_dict +from je_auto_control.gui.language_wrapper.japanese import japanese_word_dict +from je_auto_control.gui.language_wrapper.simplified_chinese import ( + simplified_chinese_word_dict, +) +from je_auto_control.gui.language_wrapper.traditional_chinese import ( + traditional_chinese_word_dict, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger +class LanguageWrapper: + """Holds the active language and exposes lookups + change notifications.""" -class LanguageWrapper(object): - - def __init__( - self - ): + def __init__(self) -> None: autocontrol_logger.info("Init LanguageWrapper") - self.language: str = "English" - self.choose_language_dict = { + self._registry: Dict[str, dict] = { "English": english_word_dict, - "Traditional_Chinese": traditional_chinese_word_dict + "Traditional_Chinese": traditional_chinese_word_dict, + "Simplified_Chinese": simplified_chinese_word_dict, + "Japanese": japanese_word_dict, } - self.language_word_dict: dict = self.choose_language_dict.get(self.language) - - def reset_language(self, language) -> None: - autocontrol_logger.info(f"LanguageWrapper reset_language language: {language}") - if language in [ - "English", - "Traditional_Chinese" - ]: - self.language = language - self.language_word_dict = self.choose_language_dict.get(self.language) + self.language: str = "English" + self._listeners: List[callable] = [] + self.language_word_dict: dict = self._merged(self.language) + + @property + def available_languages(self) -> List[str]: + """Sorted list of registered language names.""" + return sorted(self._registry.keys()) + + def register_language(self, name: str, words: dict) -> None: + """Add or replace a language dict at runtime (plugin-friendly).""" + self._registry[name] = dict(words) + if name == self.language: + self.language_word_dict = self._merged(name) + + def reset_language(self, language: str) -> None: + if language not in self._registry: + autocontrol_logger.warning("unknown language: %s", language) + return + self.language = language + self.language_word_dict = self._merged(language) + for listener in list(self._listeners): + try: + listener(language) + except (OSError, RuntimeError) as error: + autocontrol_logger.error("language listener failed: %r", error) + + def add_listener(self, callback) -> None: + """Register ``callback(language)`` to fire when the language changes.""" + if callback not in self._listeners: + self._listeners.append(callback) + + def remove_listener(self, callback) -> None: + if callback in self._listeners: + self._listeners.remove(callback) + + def translate(self, key: str, default: str = "") -> str: + """Look up ``key`` in the active language, with English/default fallback.""" + value = self.language_word_dict.get(key) + if value is not None: + return value + fallback = english_word_dict.get(key) + if fallback is not None: + return fallback + return default or key + + def _merged(self, language: str) -> dict: + merged = dict(english_word_dict) + merged.update(self._registry.get(language, {})) + return merged language_wrapper = LanguageWrapper() diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py new file mode 100644 index 0000000..ee6b3af --- /dev/null +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -0,0 +1,42 @@ +simplified_chinese_word_dict = { + "application_name": "AutoControlGUI", + + # Tabs + "tab_auto_click": "自动点击", + "tab_screenshot": "屏幕截图", + "tab_image_detect": "图像检测", + "tab_record": "录制 / 回放", + "tab_script": "脚本执行", + "tab_script_builder": "脚本编辑器", + "tab_recording_editor": "录制编辑器", + "tab_window_manager": "窗口管理", + "tab_scheduler": "调度器", + "tab_socket_server": "Socket 服务器", + "tab_live_hud": "实时监看", + "tab_hotkeys": "全局热键", + "tab_triggers": "事件触发器", + "tab_plugins": "插件", + "tab_screen_record": "屏幕录像", + "tab_shell": "Shell 命令", + "tab_report": "报告生成", + + # Menu bar + "menu_file": "文件", + "menu_file_open_script": "打开脚本...", + "menu_file_exit": "退出", + "menu_view": "视图", + "menu_view_tabs": "分页", + "menu_tools": "工具", + "menu_tools_start_hotkeys": "启动热键守护进程", + "menu_tools_start_scheduler": "启动调度器", + "menu_tools_start_triggers": "启动触发器引擎", + "menu_language": "语言", + "menu_help": "帮助", + "menu_help_about": "关于 AutoControlGUI", + + # Status/messages + "language_label": "语言：", + "browse": "浏览", + "start": "开始", + "stop": "停止", +} diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index 866674b..e672be1 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -8,6 +8,15 @@ "tab_image_detect": "影像偵測", "tab_record": "錄製 / 回放", "tab_script": "腳本執行", + "tab_script_builder": "腳本編輯器", + "tab_recording_editor": "錄製後製", + "tab_window_manager": "視窗管理", + "tab_scheduler": "排程器", + "tab_socket_server": "Socket 伺服器", + "tab_live_hud": "即時監看", + "tab_hotkeys": "全域熱鍵", + "tab_triggers": "事件觸發器", + "tab_plugins": "外掛", "tab_screen_record": "螢幕錄影", "tab_shell": "Shell 命令", "tab_report": "報告產生", @@ -45,6 +54,8 @@ "file_path_label": "檔案路徑：", "browse": "瀏覽", "region_label": "區域 (x1, y1, x2, y2)：", + "pick_region": "框選區域", + "crop_template": "框選模板", "get_pixel_label": "取得像素顏色", "pixel_x": "X：", "pixel_y": "Y：", @@ -109,4 +120,18 @@ # Language "language_label": "語言：", + + # Menu bar + "menu_file": "檔案", + "menu_file_open_script": "開啟腳本...", + "menu_file_exit": "結束", + "menu_view": "檢視", + "menu_view_tabs": "分頁", + "menu_tools": "工具", + "menu_tools_start_hotkeys": "啟動熱鍵服務", + "menu_tools_start_scheduler": "啟動排程器", + "menu_tools_start_triggers": "啟動觸發引擎", + "menu_language": "語言", + "menu_help": "說明", + "menu_help_about": "關於 AutoControlGUI", } diff --git a/je_auto_control/gui/live_hud_tab.py b/je_auto_control/gui/live_hud_tab.py new file mode 100644 index 0000000..fb1d71a --- /dev/null +++ b/je_auto_control/gui/live_hud_tab.py @@ -0,0 +1,78 @@ +"""Live HUD: mouse position, pixel colour under cursor and log tail.""" +from typing import Optional + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import ( + QGroupBox, QHBoxLayout, QLabel, QPushButton, QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.watcher.watcher import LogTail, MouseWatcher, PixelWatcher + + +class LiveHUDTab(QWidget): + """Poll watchers and render a readable HUD.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._mouse = MouseWatcher() + self._pixel = PixelWatcher() + self._log_tail = LogTail(capacity=400) + self._pos_label = QLabel("Mouse: --") + self._color_label = QLabel("Pixel: --") + self._log_view = QTextEdit() + self._log_view.setReadOnly(True) + self._timer = QTimer(self) + self._timer.setInterval(250) + self._timer.timeout.connect(self._tick) + self._build_layout() + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + status_group = QGroupBox("Watchers") + status_layout = QVBoxLayout() + status_layout.addWidget(self._pos_label) + status_layout.addWidget(self._color_label) + status_group.setLayout(status_layout) + root.addWidget(status_group) + + ctl = QHBoxLayout() + start_btn = QPushButton("Start HUD") + start_btn.clicked.connect(self._start) + stop_btn = QPushButton("Stop HUD") + stop_btn.clicked.connect(self._stop) + clear_btn = QPushButton("Clear log") + clear_btn.clicked.connect(self._log_view.clear) + for btn in (start_btn, stop_btn, clear_btn): + ctl.addWidget(btn) + ctl.addStretch() + root.addLayout(ctl) + + root.addWidget(QLabel("Recent log:")) + root.addWidget(self._log_view, stretch=1) + + def _start(self) -> None: + self._log_tail.attach(autocontrol_logger) + self._timer.start() + + def _stop(self) -> None: + self._timer.stop() + self._log_tail.detach(autocontrol_logger) + + def _tick(self) -> None: + try: + x, y = self._mouse.sample() + except RuntimeError as error: + self._pos_label.setText(f"Mouse: {error}") + return + self._pos_label.setText(f"Mouse: ({x}, {y})") + rgb = self._pixel.sample(x, y) + self._color_label.setText(f"Pixel: {rgb}" if rgb is not None else "Pixel: n/a") + lines = self._log_tail.snapshot() + self._log_view.setPlainText("\n".join(lines)) + scrollbar = self._log_view.verticalScrollBar() + scrollbar.setValue(scrollbar.maximum()) + + def closeEvent(self, event) -> None: # noqa: N802 # reason: Qt override + self._stop() + super().closeEvent(event) diff --git a/je_auto_control/gui/main_widget.py b/je_auto_control/gui/main_widget.py index 4527d2f..05d4847 100644 --- a/je_auto_control/gui/main_widget.py +++ b/je_auto_control/gui/main_widget.py @@ -1,15 +1,27 @@ import json +from dataclasses import dataclass from PySide6.QtCore import QTimer, Signal, QObject from PySide6.QtGui import QIntValidator, QDoubleValidator, QKeyEvent, Qt from PySide6.QtWidgets import ( - QWidget, QLineEdit, QComboBox, QPushButton, QVBoxLayout, QLabel, + QWidget, QLineEdit, QPushButton, QVBoxLayout, QLabel, QGridLayout, QHBoxLayout, QMessageBox, QTabWidget, QTextEdit, QFileDialog, QCheckBox, QGroupBox ) from je_auto_control.gui._auto_click_tab import AutoClickTabMixin +from je_auto_control.gui._shell_report_tabs import ShellReportTabsMixin +from je_auto_control.gui.hotkeys_tab import HotkeysTab from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.gui.live_hud_tab import LiveHUDTab +from je_auto_control.gui.plugins_tab import PluginsTab +from je_auto_control.gui.recording_editor_tab import RecordingEditorTab +from je_auto_control.gui.scheduler_tab import SchedulerTab +from je_auto_control.gui.script_builder import ScriptBuilderTab +from je_auto_control.gui.selector import crop_template_to_file, open_region_selector +from je_auto_control.gui.socket_server_tab import SocketServerTab +from je_auto_control.gui.triggers_tab import TriggersTab +from je_auto_control.gui.window_tab import WindowManagerTab from je_auto_control.wrapper.auto_control_screen import screen_size, screenshot, get_pixel from je_auto_control.wrapper.auto_control_image import locate_all_image, locate_image_center, locate_and_click from je_auto_control.wrapper.auto_control_record import record, stop_record @@ -17,17 +29,11 @@ from je_auto_control.utils.json.json_file import read_action_json, write_action_json from je_auto_control.utils.file_process.get_dir_file_list import get_dir_files_as_list from je_auto_control.utils.cv2_utils.screen_record import ScreenRecorder -from je_auto_control.utils.shell_process.shell_exec import ShellManager -from je_auto_control.utils.start_exe.start_another_process import start_exe -from je_auto_control.utils.generate_report.generate_html_report import generate_html_report -from je_auto_control.utils.generate_report.generate_json_report import generate_json_report -from je_auto_control.utils.generate_report.generate_xml_report import generate_xml_report -from je_auto_control.utils.test_record.record_test_class import test_record_instance def _t(key: str) -> str: """language_wrapper shorthand""" - return language_wrapper.language_word_dict.get(key, key) + return language_wrapper.translate(key, key) class _WorkerSignals(QObject): @@ -35,35 +41,136 @@ class _WorkerSignals(QObject): error = Signal(str) +@dataclass +class _TabEntry: + key: str + title_key: str + widget: QWidget + + # ============================================================================= # Main Widget # ============================================================================= -class AutoControlGUIWidget(AutoClickTabMixin, QWidget): +class AutoControlGUIWidget(AutoClickTabMixin, ShellReportTabsMixin, QWidget): + """Owns the QTabWidget and exposes show/hide/list APIs for the menu bar.""" + + tabs_changed = Signal() def __init__(self, parent=None): super().__init__(parent) layout = QVBoxLayout() + self._tab_entries: list = [] + self.tabs = QTabWidget() - self.tabs.addTab(self._build_auto_click_tab(), _t("tab_auto_click")) - self.tabs.addTab(self._build_screenshot_tab(), _t("tab_screenshot")) - self.tabs.addTab(self._build_image_detect_tab(), _t("tab_image_detect")) - self.tabs.addTab(self._build_record_tab(), _t("tab_record")) - self.tabs.addTab(self._build_script_tab(), _t("tab_script")) - self.tabs.addTab(self._build_screen_record_tab(), _t("tab_screen_record")) - self.tabs.addTab(self._build_shell_tab(), _t("tab_shell")) - self.tabs.addTab(self._build_report_tab(), _t("tab_report")) + self.tabs.setTabsClosable(True) + self.tabs.tabCloseRequested.connect(self._on_tab_close_requested) + + self._add_tab("auto_click", "tab_auto_click", self._build_auto_click_tab()) + self._add_tab("screenshot", "tab_screenshot", self._build_screenshot_tab()) + self._add_tab("image_detect", "tab_image_detect", self._build_image_detect_tab()) + self._add_tab("record", "tab_record", self._build_record_tab()) + self._add_tab("script", "tab_script", self._build_script_tab()) + self._add_tab("script_builder", "tab_script_builder", ScriptBuilderTab()) + self._add_tab("recording_editor", "tab_recording_editor", RecordingEditorTab()) + self._add_tab("window_manager", "tab_window_manager", WindowManagerTab()) + self._add_tab("scheduler", "tab_scheduler", SchedulerTab()) + self._add_tab("socket_server", "tab_socket_server", SocketServerTab()) + self._add_tab("live_hud", "tab_live_hud", LiveHUDTab()) + self._add_tab("screen_record", "tab_screen_record", self._build_screen_record_tab()) + self._add_tab("shell", "tab_shell", self._build_shell_tab()) + self._add_tab("report", "tab_report", self._build_report_tab()) + self._add_tab("hotkeys", "tab_hotkeys", HotkeysTab()) + self._add_tab("triggers", "tab_triggers", TriggersTab()) + self._add_tab("plugins", "tab_plugins", PluginsTab()) layout.addWidget(self.tabs) self.setLayout(layout) - # shared state self.timer = QTimer() self.repeat_count = 0 self.repeat_max = 0 self.screen_recorder = ScreenRecorder() self._record_data = [] + # --- tab registry API ---------------------------------------------------- + + def _add_tab(self, key: str, title_key: str, widget: QWidget) -> None: + self._tab_entries.append(_TabEntry(key=key, title_key=title_key, widget=widget)) + self.tabs.addTab(widget, language_wrapper.translate(title_key, title_key)) + + def _find_entry(self, key: str): + for entry in self._tab_entries: + if entry.key == key: + return entry + return None + + def list_registered_tabs(self) -> list: + """Return metadata for the View → Tabs menu.""" + return [ + { + "key": entry.key, + "title": language_wrapper.translate(entry.title_key, entry.title_key), + "visible": self.tabs.indexOf(entry.widget) != -1, + } + for entry in self._tab_entries + ] + + def show_tab(self, key: str) -> None: + entry = self._find_entry(key) + if entry is None or self.tabs.indexOf(entry.widget) != -1: + return + target_index = 0 + for candidate in self._tab_entries: + if candidate.key == key: + break + if self.tabs.indexOf(candidate.widget) != -1: + target_index += 1 + title = language_wrapper.translate(entry.title_key, entry.title_key) + self.tabs.insertTab(target_index, entry.widget, title) + self.tabs.setCurrentWidget(entry.widget) + self.tabs_changed.emit() + + def hide_tab(self, key: str) -> None: + entry = self._find_entry(key) + if entry is None: + return + index = self.tabs.indexOf(entry.widget) + if index != -1: + self.tabs.removeTab(index) + self.tabs_changed.emit() + + def _on_tab_close_requested(self, index: int) -> None: + widget = self.tabs.widget(index) + for entry in self._tab_entries: + if entry.widget is widget: + self.hide_tab(entry.key) + return + + def retranslate(self) -> None: + """Relabel visible tabs after a language change.""" + for entry in self._tab_entries: + index = self.tabs.indexOf(entry.widget) + if index != -1: + self.tabs.setTabText( + index, language_wrapper.translate(entry.title_key, entry.title_key), + ) + + def open_script_file(self, path: str) -> None: + """Load a JSON script into the Script Executor tab and focus it.""" + entry = self._find_entry("script") + if entry is not None and self.tabs.indexOf(entry.widget) == -1: + self.show_tab("script") + self.script_path_input.setText(path) + try: + data = read_action_json(path) + self.script_editor.setText(json.dumps(data, indent=2, ensure_ascii=False)) + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.script_result_text.setText(f"Error loading: {error}") + return + if entry is not None: + self.tabs.setCurrentWidget(entry.widget) + # ========================================================================= # Tab 2: Screenshot # ========================================================================= @@ -95,7 +202,10 @@ def _build_screenshot_tab(self) -> QWidget: ss_grid.addWidget(QLabel(_t("region_label")), 1, 0) self.ss_region_input = QLineEdit() self.ss_region_input.setPlaceholderText("0, 0, 800, 600") - ss_grid.addWidget(self.ss_region_input, 1, 1, 1, 2) + ss_grid.addWidget(self.ss_region_input, 1, 1) + self.ss_pick_region_btn = QPushButton(_t("pick_region")) + self.ss_pick_region_btn.clicked.connect(self._pick_ss_region) + ss_grid.addWidget(self.ss_pick_region_btn, 1, 2) btn_h = QHBoxLayout() self.ss_take_btn = QPushButton(_t("take_screenshot")) @@ -144,6 +254,13 @@ def _browse_ss_path(self): if path: self.ss_path_input.setText(path) + def _pick_ss_region(self): + region = open_region_selector(self) + if region is None: + return + x, y, w, h = region + self.ss_region_input.setText(f"{x}, {y}, {x + w}, {y + h}") + def _take_screenshot(self): try: path = self.ss_path_input.text() or None @@ -179,6 +296,9 @@ def _build_image_detect_tab(self) -> QWidget: self.img_browse_btn = QPushButton(_t("browse")) self.img_browse_btn.clicked.connect(self._browse_img) grid.addWidget(self.img_browse_btn, 0, 2) + self.img_crop_btn = QPushButton(_t("crop_template")) + self.img_crop_btn.clicked.connect(self._crop_template) + grid.addWidget(self.img_crop_btn, 0, 3) grid.addWidget(QLabel(_t("threshold_label")), 1, 0) self.threshold_input = QLineEdit("0.8") @@ -213,6 +333,21 @@ def _browse_img(self): if path: self.img_path_input.setText(path) + def _crop_template(self): + save_path, _ = QFileDialog.getSaveFileName( + self, _t("crop_template"), "", "PNG (*.png)" + ) + if not save_path: + return + try: + region = crop_template_to_file(save_path, self) + if region is None: + return + self.img_path_input.setText(save_path) + self.detect_result_text.setText(f"Template saved: {save_path} region={region}") + except (OSError, ValueError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + def _get_detect_params(self): path = self.img_path_input.text() if not path: @@ -507,151 +642,6 @@ def _stop_screen_record(self): except (OSError, ValueError, TypeError, RuntimeError) as error: QMessageBox.warning(self, "Error", str(error)) - # ========================================================================= - # Tab 7: Shell Command - # ========================================================================= - def _build_shell_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Shell command - shell_group = QGroupBox(_t("shell_command_label")) - sg = QVBoxLayout() - self.shell_input = QLineEdit() - self.shell_input.setPlaceholderText("echo hello") - self.shell_exec_btn = QPushButton(_t("execute_shell")) - self.shell_exec_btn.clicked.connect(self._execute_shell) - sh = QHBoxLayout() - sh.addWidget(self.shell_input) - sh.addWidget(self.shell_exec_btn) - sg.addLayout(sh) - shell_group.setLayout(sg) - layout.addWidget(shell_group) - - # Start exe - exe_group = QGroupBox(_t("start_exe_label")) - eg = QHBoxLayout() - self.exe_path_input = QLineEdit() - self.exe_browse_btn = QPushButton(_t("browse")) - self.exe_browse_btn.clicked.connect(self._browse_exe) - self.exe_start_btn = QPushButton(_t("start_exe")) - self.exe_start_btn.clicked.connect(self._start_exe) - eg.addWidget(self.exe_path_input) - eg.addWidget(self.exe_browse_btn) - eg.addWidget(self.exe_start_btn) - exe_group.setLayout(eg) - layout.addWidget(exe_group) - - layout.addWidget(QLabel(_t("shell_output"))) - self.shell_output_text = QTextEdit() - self.shell_output_text.setReadOnly(True) - layout.addWidget(self.shell_output_text) - tab.setLayout(layout) - return tab - - def _execute_shell(self): - try: - cmd = self.shell_input.text() - if not cmd: - return - mgr = ShellManager() - mgr.exec_shell(cmd) - self.shell_output_text.setText(f"Executed: {cmd}\n(Check console for output)") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.shell_output_text.setText(f"Error: {error}") - - def _browse_exe(self): - path, _ = QFileDialog.getOpenFileName(self, _t("start_exe_label"), "", "Executable (*.exe);;All (*)") - if path: - self.exe_path_input.setText(path) - - def _start_exe(self): - try: - path = self.exe_path_input.text() - if not path: - return - start_exe(path) - self.shell_output_text.setText(f"Started: {path}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.shell_output_text.setText(f"Error: {error}") - - # ========================================================================= - # Tab 8: Report - # ========================================================================= - def _build_report_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Test record toggle - tr_group = QGroupBox(_t("test_record_status")) - tr_h = QHBoxLayout() - self.tr_enable_btn = QPushButton(_t("enable_test_record")) - self.tr_enable_btn.clicked.connect(lambda: self._set_test_record(True)) - self.tr_disable_btn = QPushButton(_t("disable_test_record")) - self.tr_disable_btn.clicked.connect(lambda: self._set_test_record(False)) - self.tr_status_label = QLabel("OFF") - tr_h.addWidget(self.tr_enable_btn) - tr_h.addWidget(self.tr_disable_btn) - tr_h.addWidget(self.tr_status_label) - tr_group.setLayout(tr_h) - layout.addWidget(tr_group) - - # Report name - name_h = QHBoxLayout() - name_h.addWidget(QLabel(_t("report_name"))) - self.report_name_input = QLineEdit("autocontrol_report") - name_h.addWidget(self.report_name_input) - layout.addLayout(name_h) - - # Generate buttons - btn_h = QHBoxLayout() - self.html_report_btn = QPushButton(_t("generate_html_report")) - self.html_report_btn.clicked.connect(self._gen_html) - self.json_report_btn = QPushButton(_t("generate_json_report")) - self.json_report_btn.clicked.connect(self._gen_json) - self.xml_report_btn = QPushButton(_t("generate_xml_report")) - self.xml_report_btn.clicked.connect(self._gen_xml) - btn_h.addWidget(self.html_report_btn) - btn_h.addWidget(self.json_report_btn) - btn_h.addWidget(self.xml_report_btn) - layout.addLayout(btn_h) - - layout.addWidget(QLabel(_t("report_result"))) - self.report_result_text = QTextEdit() - self.report_result_text.setReadOnly(True) - layout.addWidget(self.report_result_text) - layout.addStretch() - tab.setLayout(layout) - return tab - - def _set_test_record(self, enable: bool): - test_record_instance.set_record_enable(enable) - self.tr_status_label.setText("ON" if enable else "OFF") - - def _gen_html(self): - try: - name = self.report_name_input.text() or "autocontrol_report" - generate_html_report(name) - self.report_result_text.setText(f"HTML report generated: {name}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.report_result_text.setText(f"Error: {error}") - - def _gen_json(self): - try: - name = self.report_name_input.text() or "autocontrol_report" - generate_json_report(name) - self.report_result_text.setText(f"JSON report generated: {name}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.report_result_text.setText(f"Error: {error}") - - def _gen_xml(self): - try: - name = self.report_name_input.text() or "autocontrol_report" - generate_xml_report(name) - self.report_result_text.setText(f"XML report generated: {name}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.report_result_text.setText(f"Error: {error}") - # ========================================================================= # Global keyboard shortcut: Ctrl+4 to stop # ========================================================================= diff --git a/je_auto_control/gui/main_window.py b/je_auto_control/gui/main_window.py index 4774afc..3d2a7d5 100644 --- a/je_auto_control/gui/main_window.py +++ b/je_auto_control/gui/main_window.py @@ -1,71 +1,181 @@ +"""Top-level window with menu bar, closable tabs, and live language switching.""" import sys -from PySide6.QtWidgets import QMainWindow, QApplication, QComboBox, QLabel, QHBoxLayout, QWidget +from PySide6.QtGui import QAction, QActionGroup +from PySide6.QtWidgets import ( + QApplication, QFileDialog, QMainWindow, QMenu, QMessageBox, +) from qt_material import QtStyleTools -from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.gui.language_wrapper.multi_language_wrapper import ( + language_wrapper, +) from je_auto_control.gui.main_widget import AutoControlGUIWidget +def _t(key: str, default: str = "") -> str: + return language_wrapper.translate(key, default or key) + + class AutoControlGUIUI(QMainWindow, QtStyleTools): + """Main window: menu bar + AutoControlGUIWidget (which owns the tabs).""" - def __init__(self): + def __init__(self) -> None: super().__init__() - - # Application ID for Windows taskbar - self.app_id = language_wrapper.language_word_dict.get("application_name") + self.app_id = _t("application_name", "AutoControlGUI") if sys.platform in ["win32", "cygwin", "msys"]: from ctypes import windll windll.shell32.SetCurrentProcessExplicitAppUserModelID(self.app_id) - # Style - self.setStyleSheet( - "font-size: 12pt;" - "font-family: 'Lato';" - ) + self.setStyleSheet("font-size: 12pt; font-family: 'Lato';") self.apply_stylesheet(self, "dark_amber.xml") - # Window title and size - self.setWindowTitle(language_wrapper.language_word_dict.get("application_name", "AutoControlGUI")) - self.resize(900, 700) + self.setWindowTitle(_t("application_name", "AutoControlGUI")) + self.resize(1000, 760) - # Central widget - self.auto_control_gui_widget = AutoControlGUIWidget() + self.auto_control_gui_widget = AutoControlGUIWidget(parent=self) self.setCentralWidget(self.auto_control_gui_widget) - # Menu bar with language switcher + self._view_menu: QMenu = None + self._tab_actions: list = [] self._build_menu_bar() + self.auto_control_gui_widget.tabs_changed.connect(self._rebuild_tabs_menu) + language_wrapper.add_listener(self._on_language_changed) + + # --- menu construction --------------------------------------------------- + + def _build_menu_bar(self) -> None: + bar = self.menuBar() + bar.clear() + bar.addMenu(self._build_file_menu()) + bar.addMenu(self._build_view_menu()) + bar.addMenu(self._build_tools_menu()) + bar.addMenu(self._build_language_menu()) + bar.addMenu(self._build_help_menu()) + + def _build_file_menu(self) -> QMenu: + menu = QMenu(_t("menu_file", "File"), self) + open_action = QAction(_t("menu_file_open_script", "Open Script..."), self) + open_action.triggered.connect(self._on_open_script) + menu.addAction(open_action) + menu.addSeparator() + exit_action = QAction(_t("menu_file_exit", "Exit"), self) + exit_action.triggered.connect(self.close) + menu.addAction(exit_action) + return menu + + def _build_view_menu(self) -> QMenu: + menu = QMenu(_t("menu_view", "View"), self) + tabs_menu = menu.addMenu(_t("menu_view_tabs", "Tabs")) + self._view_menu = tabs_menu + self._rebuild_tabs_menu() + return menu + + def _rebuild_tabs_menu(self) -> None: + if self._view_menu is None: + return + self._view_menu.clear() + self._tab_actions = [] + for entry in self.auto_control_gui_widget.list_registered_tabs(): + action = QAction(entry["title"], self, checkable=True) + action.setChecked(entry["visible"]) + action.setData(entry["key"]) + action.toggled.connect(self._on_tab_action_toggled) + self._view_menu.addAction(action) + self._tab_actions.append(action) + + def _on_tab_action_toggled(self, checked: bool) -> None: + action = self.sender() + if not isinstance(action, QAction): + return + key = action.data() + if checked: + self.auto_control_gui_widget.show_tab(key) + else: + self.auto_control_gui_widget.hide_tab(key) + + def _build_tools_menu(self) -> QMenu: + menu = QMenu(_t("menu_tools", "Tools"), self) + menu.addAction( + _t("menu_tools_start_hotkeys", "Start hotkey daemon"), + self._start_hotkeys, + ) + menu.addAction( + _t("menu_tools_start_scheduler", "Start scheduler"), + self._start_scheduler, + ) + menu.addAction( + _t("menu_tools_start_triggers", "Start trigger engine"), + self._start_triggers, + ) + return menu + + def _build_language_menu(self) -> QMenu: + menu = QMenu(_t("menu_language", "Language"), self) + group = QActionGroup(menu) + group.setExclusive(True) + for lang in language_wrapper.available_languages: + action = QAction(lang.replace("_", " "), menu, checkable=True) + action.setData(lang) + action.setChecked(lang == language_wrapper.language) + action.triggered.connect(self._on_language_selected) + group.addAction(action) + menu.addAction(action) + return menu + + def _build_help_menu(self) -> QMenu: + menu = QMenu(_t("menu_help", "Help"), self) + menu.addAction( + _t("menu_help_about", "About AutoControlGUI"), self._on_about, + ) + return menu - def _build_menu_bar(self): - menu_bar = self.menuBar() + # --- actions ------------------------------------------------------------- - # Language selector in menu bar - lang_widget = QWidget() - lang_layout = QHBoxLayout(lang_widget) - lang_layout.setContentsMargins(4, 0, 4, 0) + def _on_open_script(self) -> None: + path, _ = QFileDialog.getOpenFileName( + self, _t("menu_file_open_script", "Open Script"), "", "JSON (*.json)", + ) + if path: + self.auto_control_gui_widget.open_script_file(path) + + def _on_language_selected(self) -> None: + action = self.sender() + if not isinstance(action, QAction): + return + language_wrapper.reset_language(action.data()) + + def _on_language_changed(self, _language: str) -> None: + self.setWindowTitle(_t("application_name", "AutoControlGUI")) + self.auto_control_gui_widget.retranslate() + self._build_menu_bar() - lang_label = QLabel(language_wrapper.language_word_dict.get("language_label", "Language:")) - self.lang_combo = QComboBox() - self.lang_combo.addItems(["English", "Traditional_Chinese"]) - self.lang_combo.setCurrentText(language_wrapper.language) - self.lang_combo.currentTextChanged.connect(self._on_language_changed) + def _on_about(self) -> None: + QMessageBox.about( + self, _t("menu_help_about", "About"), + "AutoControlGUI — cross-platform automation framework.", + ) - lang_layout.addWidget(lang_label) - lang_layout.addWidget(self.lang_combo) + def _start_hotkeys(self) -> None: + from je_auto_control.utils.hotkey.hotkey_daemon import default_hotkey_daemon + try: + default_hotkey_daemon.start() + except NotImplementedError as error: + QMessageBox.warning(self, "Error", str(error)) - menu_bar.setCornerWidget(lang_widget) + def _start_scheduler(self) -> None: + from je_auto_control.utils.scheduler.scheduler import default_scheduler + default_scheduler.start() - def _on_language_changed(self, language: str): - language_wrapper.reset_language(language) - # Rebuild UI with new language - self.setWindowTitle(language_wrapper.language_word_dict.get("application_name", "AutoControlGUI")) - self.auto_control_gui_widget = AutoControlGUIWidget() - self.setCentralWidget(self.auto_control_gui_widget) - self._build_menu_bar() + def _start_triggers(self) -> None: + from je_auto_control.utils.triggers.trigger_engine import ( + default_trigger_engine, + ) + default_trigger_engine.start() if "__main__" == __name__: app = QApplication(sys.argv) window = AutoControlGUIUI() window.show() - sys.exit(app.exec()) \ No newline at end of file + sys.exit(app.exec()) diff --git a/je_auto_control/gui/plugins_tab.py b/je_auto_control/gui/plugins_tab.py new file mode 100644 index 0000000..0a89c34 --- /dev/null +++ b/je_auto_control/gui/plugins_tab.py @@ -0,0 +1,61 @@ +"""Plugins tab: load extra AC_ commands from a user directory.""" +from typing import Optional + +from PySide6.QtWidgets import ( + QFileDialog, QHBoxLayout, QLabel, QLineEdit, QListWidget, QMessageBox, + QPushButton, QVBoxLayout, QWidget, +) + +from je_auto_control.utils.plugin_loader.plugin_loader import ( + load_plugin_directory, register_plugin_commands, +) + + +class PluginsTab(QWidget): + """Pick a directory of plugins, register their ``AC_*`` callables.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._dir_input = QLineEdit() + self._list = QListWidget() + self._status = QLabel("No plugins loaded") + self._build_layout() + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + form = QHBoxLayout() + form.addWidget(QLabel("Plugin dir:")) + form.addWidget(self._dir_input, stretch=1) + browse = QPushButton("Browse") + browse.clicked.connect(self._browse) + form.addWidget(browse) + load = QPushButton("Load + register") + load.clicked.connect(self._on_load) + form.addWidget(load) + root.addLayout(form) + root.addWidget(QLabel("Registered commands:")) + root.addWidget(self._list, stretch=1) + root.addWidget(self._status) + + def _browse(self) -> None: + path = QFileDialog.getExistingDirectory(self, "Plugin directory") + if path: + self._dir_input.setText(path) + + def _on_load(self) -> None: + path = self._dir_input.text().strip() + if not path: + return + try: + commands = load_plugin_directory(path) + except (OSError, NotADirectoryError) as error: + QMessageBox.warning(self, "Error", str(error)) + return + if not commands: + self._status.setText(f"No AC_* callables found in {path}") + return + registered = register_plugin_commands(commands) + self._list.clear() + for name in registered: + self._list.addItem(name) + self._status.setText(f"Registered {len(registered)} commands from {path}") diff --git a/je_auto_control/gui/recording_editor_tab.py b/je_auto_control/gui/recording_editor_tab.py new file mode 100644 index 0000000..599d898 --- /dev/null +++ b/je_auto_control/gui/recording_editor_tab.py @@ -0,0 +1,188 @@ +"""Recording Editor tab: trim, filter and rescale recorded action lists.""" +import json +from typing import Optional + +from PySide6.QtWidgets import ( + QFileDialog, QHBoxLayout, QLabel, QLineEdit, QListWidget, QMessageBox, + QPushButton, QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.utils.json.json_file import read_action_json, write_action_json +from je_auto_control.utils.recording_edit.editor import ( + adjust_delays, filter_actions, remove_action, scale_coordinates, trim_actions, +) + + +class RecordingEditorTab(QWidget): + """Load a recording JSON and apply non-destructive edits.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._actions: list = [] + self._path_input = QLineEdit() + self._list = QListWidget() + self._preview = QTextEdit() + self._preview.setReadOnly(True) + self._status = QLabel("") + self._trim_start = QLineEdit("0") + self._trim_end = QLineEdit("") + self._delay_factor = QLineEdit("1.0") + self._delay_clamp = QLineEdit("0") + self._scale_x = QLineEdit("1.0") + self._scale_y = QLineEdit("1.0") + self._build_layout() + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + top = QHBoxLayout() + top.addWidget(QLabel("File:")) + top.addWidget(self._path_input, stretch=1) + for label, handler in ( + ("Browse", self._browse), + ("Load", self._load), + ("Save As", self._save_as), + ): + btn = QPushButton(label) + btn.clicked.connect(handler) + top.addWidget(btn) + root.addLayout(top) + + root.addWidget(self._list, stretch=1) + + ops1 = QHBoxLayout() + ops1.addWidget(QLabel("Trim start:")) + ops1.addWidget(self._trim_start) + ops1.addWidget(QLabel("end:")) + ops1.addWidget(self._trim_end) + trim_btn = QPushButton("Apply trim") + trim_btn.clicked.connect(self._apply_trim) + ops1.addWidget(trim_btn) + remove_btn = QPushButton("Remove selected") + remove_btn.clicked.connect(self._remove_selected) + ops1.addWidget(remove_btn) + ops1.addStretch() + root.addLayout(ops1) + + ops2 = QHBoxLayout() + ops2.addWidget(QLabel("Delay x")) + ops2.addWidget(self._delay_factor) + ops2.addWidget(QLabel("floor ms:")) + ops2.addWidget(self._delay_clamp) + delay_btn = QPushButton("Apply delays") + delay_btn.clicked.connect(self._apply_delays) + ops2.addWidget(delay_btn) + ops2.addWidget(QLabel("Scale x:")) + ops2.addWidget(self._scale_x) + ops2.addWidget(QLabel("y:")) + ops2.addWidget(self._scale_y) + scale_btn = QPushButton("Apply scale") + scale_btn.clicked.connect(self._apply_scale) + ops2.addWidget(scale_btn) + ops2.addStretch() + root.addLayout(ops2) + + ops3 = QHBoxLayout() + keep_mouse = QPushButton("Keep mouse only") + keep_mouse.clicked.connect(lambda: self._filter_prefix("AC_mouse")) + keep_keyboard = QPushButton("Keep keyboard only") + keep_keyboard.clicked.connect( + lambda: self._filter_prefix(("AC_type_keyboard", "AC_press_keyboard_key", + "AC_release_keyboard_key", "AC_hotkey", "AC_write")) + ) + ops3.addWidget(keep_mouse) + ops3.addWidget(keep_keyboard) + ops3.addStretch() + root.addLayout(ops3) + + root.addWidget(QLabel("Preview:")) + root.addWidget(self._preview, stretch=1) + root.addWidget(self._status) + + def _browse(self) -> None: + path, _ = QFileDialog.getOpenFileName(self, "Open recording", "", "JSON (*.json)") + if path: + self._path_input.setText(path) + + def _load(self) -> None: + path = self._path_input.text().strip() + if not path: + return + try: + self._actions = read_action_json(path) + except (OSError, ValueError) as error: + QMessageBox.warning(self, "Error", str(error)) + return + self._refresh() + + def _save_as(self) -> None: + if not self._actions: + return + path, _ = QFileDialog.getSaveFileName(self, "Save recording", "", "JSON (*.json)") + if not path: + return + try: + write_action_json(path, self._actions) + except (OSError, ValueError) as error: + QMessageBox.warning(self, "Error", str(error)) + return + self._status.setText(f"Saved to {path}") + + def _refresh(self) -> None: + self._list.clear() + for idx, action in enumerate(self._actions): + self._list.addItem(f"{idx}: {action[0]}") + self._preview.setPlainText(json.dumps(self._actions, indent=2, ensure_ascii=False)) + self._status.setText(f"{len(self._actions)} actions") + + def _apply_trim(self) -> None: + try: + start = int(self._trim_start.text() or "0") + end_text = self._trim_end.text().strip() + end = int(end_text) if end_text else None + except ValueError: + QMessageBox.warning(self, "Error", "Trim indices must be integers") + return + self._actions = trim_actions(self._actions, start, end) + self._refresh() + + def _remove_selected(self) -> None: + row = self._list.currentRow() + if row < 0: + return + try: + self._actions = remove_action(self._actions, row) + except IndexError as error: + QMessageBox.warning(self, "Error", str(error)) + return + self._refresh() + + def _apply_delays(self) -> None: + try: + factor = float(self._delay_factor.text() or "1.0") + clamp = int(self._delay_clamp.text() or "0") + except ValueError: + QMessageBox.warning(self, "Error", "Factor/clamp must be numeric") + return + self._actions = adjust_delays(self._actions, factor=factor, clamp_ms=clamp) + self._refresh() + + def _apply_scale(self) -> None: + try: + fx = float(self._scale_x.text() or "1.0") + fy = float(self._scale_y.text() or "1.0") + except ValueError: + QMessageBox.warning(self, "Error", "Scale factors must be numeric") + return + self._actions = scale_coordinates(self._actions, fx, fy) + self._refresh() + + def _filter_prefix(self, prefix) -> None: + def keep(action: list) -> bool: + if not (isinstance(action, list) and action): + return False + name = action[0] + if isinstance(prefix, str): + return name.startswith(prefix) + return name in prefix + self._actions = filter_actions(self._actions, keep) + self._refresh() diff --git a/je_auto_control/gui/scheduler_tab.py b/je_auto_control/gui/scheduler_tab.py new file mode 100644 index 0000000..af75ff2 --- /dev/null +++ b/je_auto_control/gui/scheduler_tab.py @@ -0,0 +1,111 @@ +"""Scheduler tab: register interval-based action JSON runs.""" +from typing import Optional + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import ( + QCheckBox, QFileDialog, QHBoxLayout, QLabel, QLineEdit, QMessageBox, + QPushButton, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, +) + +from je_auto_control.utils.scheduler import default_scheduler + + +class SchedulerTab(QWidget): + """Add / remove / start / stop scheduler jobs.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._path_input = QLineEdit() + self._interval_input = QLineEdit("60") + self._repeat_check = QCheckBox("Repeat") + self._repeat_check.setChecked(True) + self._table = QTableWidget(0, 5) + self._table.setHorizontalHeaderLabels( + ["Job ID", "Script", "Interval (s)", "Runs", "Enabled"] + ) + self._status = QLabel("Scheduler stopped") + self._timer = QTimer(self) + self._timer.setInterval(1000) + self._timer.timeout.connect(self._refresh_table) + self._build_layout() + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + form = QHBoxLayout() + form.addWidget(QLabel("Script:")) + form.addWidget(self._path_input, stretch=1) + browse = QPushButton("Browse") + browse.clicked.connect(self._browse) + form.addWidget(browse) + form.addWidget(QLabel("Every (s):")) + form.addWidget(self._interval_input) + form.addWidget(self._repeat_check) + add_btn = QPushButton("Add") + add_btn.clicked.connect(self._on_add) + form.addWidget(add_btn) + root.addLayout(form) + + root.addWidget(self._table, stretch=1) + + ctl = QHBoxLayout() + for label, handler in ( + ("Remove selected", self._on_remove), + ("Start scheduler", self._on_start), + ("Stop scheduler", self._on_stop), + ): + btn = QPushButton(label) + btn.clicked.connect(handler) + ctl.addWidget(btn) + ctl.addStretch() + root.addLayout(ctl) + root.addWidget(self._status) + + def _browse(self) -> None: + path, _ = QFileDialog.getOpenFileName(self, "Select script", "", "JSON (*.json)") + if path: + self._path_input.setText(path) + + def _on_add(self) -> None: + path = self._path_input.text().strip() + if not path: + QMessageBox.warning(self, "Error", "Script path is required") + return + try: + interval = float(self._interval_input.text() or "60") + except ValueError: + QMessageBox.warning(self, "Error", "Interval must be a number") + return + default_scheduler.add_job( + script_path=path, + interval_seconds=interval, + repeat=self._repeat_check.isChecked(), + ) + self._refresh_table() + + def _on_remove(self) -> None: + row = self._table.currentRow() + if row < 0: + return + job_id = self._table.item(row, 0).text() + default_scheduler.remove_job(job_id) + self._refresh_table() + + def _on_start(self) -> None: + default_scheduler.start() + self._timer.start() + self._status.setText("Scheduler running") + + def _on_stop(self) -> None: + default_scheduler.stop() + self._timer.stop() + self._status.setText("Scheduler stopped") + + def _refresh_table(self) -> None: + jobs = default_scheduler.list_jobs() + self._table.setRowCount(len(jobs)) + for row, job in enumerate(jobs): + for col, value in enumerate(( + job.job_id, job.script_path, f"{job.interval_seconds:g}", + str(job.runs), "Yes" if job.enabled else "No", + )): + self._table.setItem(row, col, QTableWidgetItem(value)) diff --git a/je_auto_control/gui/script_builder/__init__.py b/je_auto_control/gui/script_builder/__init__.py new file mode 100644 index 0000000..9a431fa --- /dev/null +++ b/je_auto_control/gui/script_builder/__init__.py @@ -0,0 +1,4 @@ +"""Visual script editor for composing AC_* action lists.""" +from je_auto_control.gui.script_builder.builder_tab import ScriptBuilderTab + +__all__ = ["ScriptBuilderTab"] diff --git a/je_auto_control/gui/script_builder/builder_tab.py b/je_auto_control/gui/script_builder/builder_tab.py new file mode 100644 index 0000000..12ca1ef --- /dev/null +++ b/je_auto_control/gui/script_builder/builder_tab.py @@ -0,0 +1,138 @@ +"""Composite widget that ties the step tree and form into a Script Builder tab.""" +import json +from typing import Optional + +from PySide6.QtCore import Qt +from PySide6.QtGui import QAction +from PySide6.QtWidgets import ( + QFileDialog, QHBoxLayout, QMenu, QMessageBox, QPushButton, QSplitter, + QTextEdit, QToolButton, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.script_builder.command_schema import ( + CATEGORIES, COMMAND_SPECS, specs_in_category, +) +from je_auto_control.gui.script_builder.step_form_view import StepFormView +from je_auto_control.gui.script_builder.step_list_view import StepTreeView +from je_auto_control.gui.script_builder.step_model import ( + Step, actions_to_steps, steps_to_actions, +) +from je_auto_control.utils.executor.action_executor import execute_action +from je_auto_control.utils.json.json_file import read_action_json, write_action_json + + +class ScriptBuilderTab(QWidget): + """Visual editor for composing AC_* scripts.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._tree = StepTreeView() + self._form = StepFormView() + self._result = QTextEdit() + self._result.setReadOnly(True) + self._result.setMaximumHeight(140) + self._build_layout() + self._wire_signals() + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + root.addLayout(self._build_toolbar()) + splitter = QSplitter(Qt.Orientation.Horizontal) + splitter.addWidget(self._tree) + splitter.addWidget(self._form) + splitter.setSizes([320, 480]) + root.addWidget(splitter, stretch=1) + root.addWidget(self._result) + + def _build_toolbar(self) -> QHBoxLayout: + bar = QHBoxLayout() + bar.addWidget(self._add_button()) + for label, handler in ( + ("Delete", self._on_delete), + ("Up", lambda: self._tree.move_selected(-1)), + ("Down", lambda: self._tree.move_selected(1)), + ): + btn = QPushButton(label) + btn.clicked.connect(handler) + bar.addWidget(btn) + bar.addStretch() + for label, handler in ( + ("Load JSON", self._on_load), + ("Save JSON", self._on_save), + ("Run", self._on_run), + ): + btn = QPushButton(label) + btn.clicked.connect(handler) + bar.addWidget(btn) + return bar + + def _add_button(self) -> QToolButton: + button = QToolButton() + button.setText("Add Step") + button.setPopupMode(QToolButton.ToolButtonPopupMode.InstantPopup) + menu = QMenu(button) + for category in CATEGORIES: + submenu = menu.addMenu(category) + for spec in specs_in_category(category): + action = QAction(spec.label, submenu) + action.triggered.connect( + lambda _checked=False, cmd=spec.command: self._add_step_from_command(cmd) + ) + submenu.addAction(action) + button.setMenu(menu) + return button + + def _wire_signals(self) -> None: + self._tree.selected_step_changed.connect(self._form.load_step) + self._form.step_changed.connect(self._tree.refresh_current_label) + + def _add_step_from_command(self, command: str) -> None: + spec = COMMAND_SPECS.get(command) + if spec is None: + return + defaults = { + f.name: f.default for f in spec.fields + if f.default is not None and not f.optional + } + step = Step(command=command, params=defaults) + self._tree.add_step(step) + + def _on_delete(self) -> None: + self._tree.remove_selected() + self._form.load_step(None) + + def _on_save(self) -> None: + path, _ = QFileDialog.getSaveFileName(self, "Save script", "", "JSON (*.json)") + if not path: + return + try: + actions = steps_to_actions(self._tree.root_steps()) + write_action_json(path, actions) + self._result.setPlainText(f"Saved: {path}") + except (OSError, ValueError, TypeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _on_load(self) -> None: + path, _ = QFileDialog.getOpenFileName(self, "Load script", "", "JSON (*.json)") + if not path: + return + try: + actions = read_action_json(path) + self._tree.load_steps(actions_to_steps(actions)) + self._form.load_step(None) + self._result.setPlainText(f"Loaded: {path}") + except (OSError, ValueError, TypeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _on_run(self) -> None: + try: + actions = steps_to_actions(self._tree.root_steps()) + if not actions: + QMessageBox.information(self, "Info", "No steps to run") + return + result = execute_action(actions) + self._result.setPlainText( + json.dumps(result, indent=2, default=str, ensure_ascii=False) + ) + except (OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py new file mode 100644 index 0000000..004354f --- /dev/null +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -0,0 +1,346 @@ +"""Schema definitions for AC_* commands used by the visual script editor. + +Each entry describes: +- category (for grouping in the Add menu) +- display label +- parameter fields (name, type, optional, default, choices) +- optional nested-body keys (for flow control) +""" +from dataclasses import dataclass +from enum import Enum +from typing import List, Mapping, Optional, Sequence, Tuple + + +class FieldType(str, Enum): + STRING = "string" + INT = "int" + FLOAT = "float" + BOOL = "bool" + ENUM = "enum" + FILE_PATH = "file_path" + RGB = "rgb" + + +@dataclass(frozen=True) +class FieldSpec: + name: str + field_type: FieldType + optional: bool = False + default: Optional[object] = None + choices: Sequence[str] = () + min_value: Optional[float] = None + max_value: Optional[float] = None + placeholder: str = "" + + +@dataclass(frozen=True) +class CommandSpec: + command: str + category: str + label: str + fields: Tuple[FieldSpec, ...] = () + body_keys: Tuple[str, ...] = () + description: str = "" + + +_MOUSE_BUTTONS = ("mouse_left", "mouse_right", "mouse_middle") + + +def _build_specs() -> List[CommandSpec]: + specs: List[CommandSpec] = [] + _add_mouse_specs(specs) + _add_keyboard_specs(specs) + _add_screen_specs(specs) + _add_image_specs(specs) + _add_ocr_specs(specs) + _add_window_specs(specs) + _add_flow_specs(specs) + _add_misc_specs(specs) + return specs + + +def _add_mouse_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_click_mouse", "Mouse", "Click Mouse", + fields=( + FieldSpec("mouse_keycode", FieldType.ENUM, choices=_MOUSE_BUTTONS, + default="mouse_left"), + FieldSpec("x", FieldType.INT, optional=True), + FieldSpec("y", FieldType.INT, optional=True), + FieldSpec("times", FieldType.INT, optional=True, default=1, min_value=1), + ), + )) + specs.append(CommandSpec( + "AC_set_mouse_position", "Mouse", "Move Mouse To", + fields=( + FieldSpec("x", FieldType.INT, default=0), + FieldSpec("y", FieldType.INT, default=0), + ), + )) + specs.append(CommandSpec( + "AC_press_mouse", "Mouse", "Press Mouse Button", + fields=( + FieldSpec("mouse_keycode", FieldType.ENUM, choices=_MOUSE_BUTTONS, + default="mouse_left"), + FieldSpec("x", FieldType.INT, optional=True), + FieldSpec("y", FieldType.INT, optional=True), + ), + )) + specs.append(CommandSpec( + "AC_release_mouse", "Mouse", "Release Mouse Button", + fields=( + FieldSpec("mouse_keycode", FieldType.ENUM, choices=_MOUSE_BUTTONS, + default="mouse_left"), + FieldSpec("x", FieldType.INT, optional=True), + FieldSpec("y", FieldType.INT, optional=True), + ), + )) + specs.append(CommandSpec( + "AC_mouse_scroll", "Mouse", "Scroll Wheel", + fields=( + FieldSpec("scroll_value", FieldType.INT, default=1), + FieldSpec("x", FieldType.INT, optional=True), + FieldSpec("y", FieldType.INT, optional=True), + ), + )) + specs.append(CommandSpec( + "AC_get_mouse_position", "Mouse", "Get Mouse Position" + )) + + +def _add_keyboard_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_type_keyboard", "Keyboard", "Type Key", + fields=( + FieldSpec("keycode", FieldType.STRING, placeholder="e.g. a, enter, 65"), + ), + )) + specs.append(CommandSpec( + "AC_press_keyboard_key", "Keyboard", "Press Key", + fields=( + FieldSpec("keycode", FieldType.STRING, placeholder="e.g. shift"), + ), + )) + specs.append(CommandSpec( + "AC_release_keyboard_key", "Keyboard", "Release Key", + fields=( + FieldSpec("keycode", FieldType.STRING, placeholder="e.g. shift"), + ), + )) + specs.append(CommandSpec( + "AC_write", "Keyboard", "Write Text", + fields=( + FieldSpec("write_string", FieldType.STRING, placeholder="Hello, world"), + ), + )) + specs.append(CommandSpec( + "AC_hotkey", "Keyboard", "Hotkey", + fields=( + FieldSpec("key_code_list", FieldType.STRING, + placeholder="ctrl, shift, s"), + ), + )) + + +def _add_screen_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_screenshot", "Screen", "Screenshot", + fields=( + FieldSpec("file_path", FieldType.FILE_PATH, optional=True), + FieldSpec("screen_region", FieldType.STRING, optional=True, + placeholder="0,0,800,600"), + ), + )) + specs.append(CommandSpec("AC_screen_size", "Screen", "Get Screen Size")) + + +def _add_image_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_locate_image_center", "Image", "Locate Image", + fields=( + FieldSpec("image", FieldType.FILE_PATH), + FieldSpec("detect_threshold", FieldType.FLOAT, optional=True, + default=0.8, min_value=0.0, max_value=1.0), + FieldSpec("draw_image", FieldType.BOOL, optional=True, default=False), + ), + )) + specs.append(CommandSpec( + "AC_locate_and_click", "Image", "Locate & Click", + fields=( + FieldSpec("image", FieldType.FILE_PATH), + FieldSpec("mouse_keycode", FieldType.ENUM, choices=_MOUSE_BUTTONS, + default="mouse_left"), + FieldSpec("detect_threshold", FieldType.FLOAT, optional=True, + default=0.8, min_value=0.0, max_value=1.0), + FieldSpec("draw_image", FieldType.BOOL, optional=True, default=False), + ), + )) + specs.append(CommandSpec( + "AC_locate_all_image", "Image", "Locate All", + fields=( + FieldSpec("image", FieldType.FILE_PATH), + FieldSpec("detect_threshold", FieldType.FLOAT, optional=True, + default=0.8, min_value=0.0, max_value=1.0), + ), + )) + + +def _add_ocr_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_locate_text", "OCR", "Locate Text", + fields=( + FieldSpec("target", FieldType.STRING), + FieldSpec("lang", FieldType.STRING, optional=True, default="eng"), + FieldSpec("min_confidence", FieldType.FLOAT, optional=True, + default=60.0, min_value=0.0, max_value=100.0), + ), + )) + specs.append(CommandSpec( + "AC_wait_text", "OCR", "Wait for Text", + fields=( + FieldSpec("target", FieldType.STRING), + FieldSpec("lang", FieldType.STRING, optional=True, default="eng"), + FieldSpec("timeout", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll", FieldType.FLOAT, optional=True, default=0.5), + FieldSpec("min_confidence", FieldType.FLOAT, optional=True, + default=60.0, min_value=0.0, max_value=100.0), + ), + )) + specs.append(CommandSpec( + "AC_click_text", "OCR", "Click Text", + fields=( + FieldSpec("target", FieldType.STRING), + FieldSpec("mouse_keycode", FieldType.ENUM, + choices=_MOUSE_BUTTONS, default="mouse_left"), + FieldSpec("lang", FieldType.STRING, optional=True, default="eng"), + FieldSpec("min_confidence", FieldType.FLOAT, optional=True, + default=60.0, min_value=0.0, max_value=100.0), + ), + )) + + +def _add_window_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_focus_window", "Window", "Focus Window", + fields=( + FieldSpec("title_substring", FieldType.STRING), + FieldSpec("case_sensitive", FieldType.BOOL, optional=True, default=False), + ), + )) + specs.append(CommandSpec( + "AC_wait_window", "Window", "Wait for Window", + fields=( + FieldSpec("title_substring", FieldType.STRING), + FieldSpec("timeout", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll", FieldType.FLOAT, optional=True, default=0.5), + ), + )) + specs.append(CommandSpec( + "AC_close_window", "Window", "Close Window", + fields=(FieldSpec("title_substring", FieldType.STRING),), + )) + specs.append(CommandSpec("AC_list_windows", "Window", "List Windows")) + + +def _add_flow_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_sleep", "Flow", "Sleep", + fields=( + FieldSpec("seconds", FieldType.FLOAT, default=1.0, min_value=0.0), + ), + )) + specs.append(CommandSpec( + "AC_wait_image", "Flow", "Wait for Image", + fields=( + FieldSpec("image", FieldType.FILE_PATH), + FieldSpec("threshold", FieldType.FLOAT, optional=True, default=0.8, + min_value=0.0, max_value=1.0), + FieldSpec("timeout", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll", FieldType.FLOAT, optional=True, default=0.2, + min_value=0.01), + ), + )) + specs.append(CommandSpec( + "AC_wait_pixel", "Flow", "Wait for Pixel", + fields=( + FieldSpec("x", FieldType.INT), + FieldSpec("y", FieldType.INT), + FieldSpec("rgb", FieldType.RGB, placeholder="255,255,255"), + FieldSpec("tolerance", FieldType.INT, optional=True, default=0, + min_value=0, max_value=255), + FieldSpec("timeout", FieldType.FLOAT, optional=True, default=10.0), + FieldSpec("poll", FieldType.FLOAT, optional=True, default=0.2, + min_value=0.01), + ), + )) + specs.append(CommandSpec( + "AC_loop", "Flow", "Loop (N times)", + fields=( + FieldSpec("times", FieldType.INT, default=3, min_value=1), + ), + body_keys=("body",), + )) + specs.append(CommandSpec( + "AC_while_image", "Flow", "While Image Visible", + fields=( + FieldSpec("image", FieldType.FILE_PATH), + FieldSpec("threshold", FieldType.FLOAT, optional=True, default=0.8, + min_value=0.0, max_value=1.0), + FieldSpec("max_iter", FieldType.INT, optional=True, default=100, + min_value=1), + ), + body_keys=("body",), + )) + specs.append(CommandSpec( + "AC_if_image_found", "Flow", "If Image Found", + fields=( + FieldSpec("image", FieldType.FILE_PATH), + FieldSpec("threshold", FieldType.FLOAT, optional=True, default=0.8, + min_value=0.0, max_value=1.0), + ), + body_keys=("then", "else"), + )) + specs.append(CommandSpec( + "AC_if_pixel", "Flow", "If Pixel Matches", + fields=( + FieldSpec("x", FieldType.INT), + FieldSpec("y", FieldType.INT), + FieldSpec("rgb", FieldType.RGB, placeholder="255,255,255"), + FieldSpec("tolerance", FieldType.INT, optional=True, default=0, + min_value=0, max_value=255), + ), + body_keys=("then", "else"), + )) + specs.append(CommandSpec( + "AC_retry", "Flow", "Retry on Failure", + fields=( + FieldSpec("max_attempts", FieldType.INT, optional=True, default=3, + min_value=1), + FieldSpec("backoff", FieldType.FLOAT, optional=True, default=0.5, + min_value=0.0), + ), + body_keys=("body",), + )) + specs.append(CommandSpec("AC_break", "Flow", "Break Loop")) + specs.append(CommandSpec("AC_continue", "Flow", "Continue Loop")) + + +def _add_misc_specs(specs: List[CommandSpec]) -> None: + specs.append(CommandSpec( + "AC_shell_command", "Shell", "Shell Command", + fields=(FieldSpec("shell_command", FieldType.STRING),), + )) + specs.append(CommandSpec( + "AC_execute_process", "Shell", "Start Executable", + fields=(FieldSpec("program_path", FieldType.FILE_PATH),), + )) + + +_SPECS: Tuple[CommandSpec, ...] = tuple(_build_specs()) +COMMAND_SPECS: Mapping[str, CommandSpec] = {spec.command: spec for spec in _SPECS} +CATEGORIES: Tuple[str, ...] = tuple(dict.fromkeys(spec.category for spec in _SPECS)) + + +def specs_in_category(category: str) -> List[CommandSpec]: + """Return all specs belonging to ``category`` in declaration order.""" + return [spec for spec in _SPECS if spec.category == category] diff --git a/je_auto_control/gui/script_builder/step_form_view.py b/je_auto_control/gui/script_builder/step_form_view.py new file mode 100644 index 0000000..4a0a98b --- /dev/null +++ b/je_auto_control/gui/script_builder/step_form_view.py @@ -0,0 +1,246 @@ +"""Schema-driven form for editing a Step's parameters.""" +from typing import Any, Callable, Dict, Optional + +from PySide6.QtCore import Signal +from PySide6.QtGui import QDoubleValidator, QIntValidator +from PySide6.QtWidgets import ( + QCheckBox, QComboBox, QFileDialog, QFormLayout, QHBoxLayout, QLabel, + QLineEdit, QPushButton, QWidget, +) + +from je_auto_control.gui.script_builder.command_schema import ( + COMMAND_SPECS, CommandSpec, FieldSpec, FieldType, +) +from je_auto_control.gui.script_builder.step_model import Step + + +_EDITOR_BUILDERS: Dict[FieldType, Callable[["StepFormView", FieldSpec], QWidget]] + + +class StepFormView(QWidget): + """Right-pane editor for a single Step.""" + + step_changed = Signal() + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._step: Optional[Step] = None + self._editors: Dict[str, QWidget] = {} + self._layout = QFormLayout(self) + self._title = QLabel("No step selected") + self._layout.addRow(self._title) + + def load_step(self, step: Optional[Step]) -> None: + """Populate the form with fields for ``step``.""" + self._clear() + self._step = step + if step is None: + self._title.setText("No step selected") + return + spec = COMMAND_SPECS.get(step.command) + if spec is None: + self._title.setText(f"Unknown command: {step.command}") + return + self._title.setText(f"{spec.label} ({spec.command})") + for field_spec in spec.fields: + editor = self._build_editor(field_spec) + self._editors[field_spec.name] = editor + self._layout.addRow(self._field_label(field_spec), editor) + self._populate_from_step(spec, step) + + def _clear(self) -> None: + while self._layout.rowCount() > 1: + self._layout.removeRow(1) + self._editors.clear() + + @staticmethod + def _field_label(field_spec: FieldSpec) -> str: + suffix = "" if not field_spec.optional else " (optional)" + return f"{field_spec.name}{suffix}" + + def _build_editor(self, spec: FieldSpec) -> QWidget: + builder = _EDITOR_BUILDERS.get(spec.field_type, StepFormView._build_string) + return builder(self, spec) + + def _build_string(self, spec: FieldSpec) -> QWidget: + editor = QLineEdit() + editor.setPlaceholderText(spec.placeholder) + editor.textChanged.connect(self._commit_field) + return editor + + def _build_int(self, spec: FieldSpec) -> QWidget: + editor = QLineEdit() + validator = QIntValidator() + if spec.min_value is not None: + validator.setBottom(int(spec.min_value)) + if spec.max_value is not None: + validator.setTop(int(spec.max_value)) + editor.setValidator(validator) + editor.setPlaceholderText(spec.placeholder) + editor.textChanged.connect(self._commit_field) + return editor + + def _build_float(self, spec: FieldSpec) -> QWidget: + editor = QLineEdit() + low = -1e9 if spec.min_value is None else float(spec.min_value) + high = 1e9 if spec.max_value is None else float(spec.max_value) + editor.setValidator(QDoubleValidator(low, high, 4)) + editor.setPlaceholderText(spec.placeholder) + editor.textChanged.connect(self._commit_field) + return editor + + def _build_bool(self, spec: FieldSpec) -> QWidget: + del spec + editor = QCheckBox() + editor.toggled.connect(self._commit_field) + return editor + + def _build_enum(self, spec: FieldSpec) -> QWidget: + editor = QComboBox() + for choice in spec.choices: + editor.addItem(choice) + editor.currentTextChanged.connect(self._commit_field) + return editor + + def _build_file(self, spec: FieldSpec) -> QWidget: + container = QWidget() + row = QHBoxLayout(container) + row.setContentsMargins(0, 0, 0, 0) + line = QLineEdit() + line.setPlaceholderText(spec.placeholder) + browse = QPushButton("...") + browse.setMaximumWidth(30) + browse.clicked.connect(lambda: self._browse_file(line)) + line.textChanged.connect(self._commit_field) + row.addWidget(line) + row.addWidget(browse) + container.setProperty("line_edit", line) + return container + + def _build_rgb(self, spec: FieldSpec) -> QWidget: + editor = QLineEdit() + editor.setPlaceholderText(spec.placeholder or "R,G,B") + editor.textChanged.connect(self._commit_field) + return editor + + def _browse_file(self, target: QLineEdit) -> None: + path, _ = QFileDialog.getOpenFileName(self, "Select file") + if path: + target.setText(path) + + def _populate_from_step(self, spec: CommandSpec, step: Step) -> None: + for field_spec in spec.fields: + editor = self._editors[field_spec.name] + value = step.params.get(field_spec.name, field_spec.default) + _set_editor_value(editor, field_spec, value) + + def _commit_field(self) -> None: + if self._step is None: + return + spec = COMMAND_SPECS.get(self._step.command) + if spec is None: + return + new_params: Dict[str, Any] = {} + for field_spec in spec.fields: + editor = self._editors.get(field_spec.name) + if editor is None: + continue + value = _read_editor_value(editor, field_spec) + if value is None and field_spec.optional: + continue + new_params[field_spec.name] = value + self._step.params = new_params + self.step_changed.emit() + + +def _file_edit(editor: QWidget) -> Optional[QLineEdit]: + line = editor.property("line_edit") + return line if isinstance(line, QLineEdit) else None + + +def _set_text_value(editor: QWidget, value: Any) -> None: + editor.setText("" if value is None else str(value)) + + +def _set_rgb_value(editor: QWidget, value: Any) -> None: + if isinstance(value, (list, tuple)): + editor.setText(",".join(str(int(v)) for v in value)) + else: + _set_text_value(editor, value) + + +def _set_file_value(editor: QWidget, value: Any) -> None: + line = _file_edit(editor) + if line is not None: + _set_text_value(line, value) + + +_SETTERS = { + FieldType.STRING: _set_text_value, + FieldType.INT: _set_text_value, + FieldType.FLOAT: _set_text_value, + FieldType.BOOL: lambda e, v: e.setChecked(bool(v)), + FieldType.ENUM: lambda e, v: e.setCurrentText(str(v) if v is not None else ""), + FieldType.FILE_PATH: _set_file_value, + FieldType.RGB: _set_rgb_value, +} + + +def _set_editor_value(editor: QWidget, spec: FieldSpec, value: Any) -> None: + setter = _SETTERS.get(spec.field_type) + if setter is not None: + setter(editor, value) + + +def _read_string(editor: QWidget) -> Any: + return editor.text() or None + + +def _read_int(editor: QWidget) -> Any: + text = editor.text().strip() + return int(text) if text else None + + +def _read_float(editor: QWidget) -> Any: + text = editor.text().strip() + return float(text) if text else None + + +def _read_file(editor: QWidget) -> Any: + line = _file_edit(editor) + return (line.text() or None) if line is not None else None + + +def _read_rgb(editor: QWidget) -> Any: + text = editor.text().strip() + if not text: + return None + parts = [p.strip() for p in text.split(",") if p.strip()] + return [int(p) for p in parts] + + +_READERS = { + FieldType.STRING: _read_string, + FieldType.INT: _read_int, + FieldType.FLOAT: _read_float, + FieldType.BOOL: lambda e: bool(e.isChecked()), + FieldType.ENUM: lambda e: e.currentText() or None, + FieldType.FILE_PATH: _read_file, + FieldType.RGB: _read_rgb, +} + + +def _read_editor_value(editor: QWidget, spec: FieldSpec) -> Any: + reader = _READERS.get(spec.field_type) + return reader(editor) if reader is not None else None + + +_EDITOR_BUILDERS = { + FieldType.STRING: StepFormView._build_string, + FieldType.INT: StepFormView._build_int, + FieldType.FLOAT: StepFormView._build_float, + FieldType.BOOL: StepFormView._build_bool, + FieldType.ENUM: StepFormView._build_enum, + FieldType.FILE_PATH: StepFormView._build_file, + FieldType.RGB: StepFormView._build_rgb, +} diff --git a/je_auto_control/gui/script_builder/step_list_view.py b/je_auto_control/gui/script_builder/step_list_view.py new file mode 100644 index 0000000..2289715 --- /dev/null +++ b/je_auto_control/gui/script_builder/step_list_view.py @@ -0,0 +1,168 @@ +"""Tree view of script steps with nested bodies for flow-control commands.""" +from typing import List, Optional, Tuple + +from PySide6.QtCore import Qt, Signal +from PySide6.QtWidgets import QTreeWidget, QTreeWidgetItem, QWidget + +from je_auto_control.gui.script_builder.command_schema import COMMAND_SPECS +from je_auto_control.gui.script_builder.step_model import Step + + +ROLE_STEP = Qt.ItemDataRole.UserRole + 1 +ROLE_BODY_KEY = Qt.ItemDataRole.UserRole + 2 + + +class StepTreeView(QTreeWidget): + """Tree of steps. + + Each top-level item is a Step. Items whose command has ``body_keys`` get + one child per body key; the body-key items host the nested Step children. + """ + + selected_step_changed = Signal(object) # emits Optional[Step] + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self.setHeaderLabels(["Step"]) + self.setColumnCount(1) + self.setDragDropMode(QTreeWidget.DragDropMode.InternalMove) + self.setSelectionMode(QTreeWidget.SelectionMode.SingleSelection) + self._roots: List[Step] = [] + self.itemSelectionChanged.connect(self._emit_selection) + + def load_steps(self, steps: List[Step]) -> None: + """Rebuild the tree from a list of root Steps.""" + self.clear() + self._roots = list(steps) + for step in self._roots: + self.addTopLevelItem(_build_item(step)) + self.expandAll() + + def root_steps(self) -> List[Step]: + """Return the current root Step list (kept in sync via refresh).""" + return list(self._roots) + + def refresh_current_label(self) -> None: + """Update the label of the currently selected item after edits.""" + item = self.currentItem() + if item is None: + return + step = item.data(0, ROLE_STEP) + if isinstance(step, Step): + item.setText(0, step.label) + + def add_step(self, step: Step) -> None: + """Append a new Step at the most appropriate location.""" + parent_item, body_key = self._selected_body_target() + if parent_item is None or body_key is None: + self._roots.append(step) + self.addTopLevelItem(_build_item(step)) + return + parent_step: Step = parent_item.data(0, ROLE_STEP) + parent_step.bodies.setdefault(body_key, []).append(step) + body_item = _find_body_item(parent_item, body_key) + if body_item is None: + body_item = _add_body_node(parent_item, body_key) + body_item.addChild(_build_item(step)) + body_item.setExpanded(True) + + def remove_selected(self) -> None: + """Remove the currently selected Step (not body-key nodes).""" + item = self.currentItem() + if item is None or item.data(0, ROLE_STEP) is None: + return + step = item.data(0, ROLE_STEP) + parent = item.parent() + if parent is None: + self._roots.remove(step) + index = self.indexOfTopLevelItem(item) + self.takeTopLevelItem(index) + return + body_key = parent.data(0, ROLE_BODY_KEY) + grandparent_step: Step = parent.parent().data(0, ROLE_STEP) + siblings = grandparent_step.bodies.get(body_key, []) + if step in siblings: + siblings.remove(step) + parent.removeChild(item) + + def move_selected(self, offset: int) -> None: + """Shift the selected Step up (offset=-1) or down (+1) in its list.""" + item = self.currentItem() + if item is None or item.data(0, ROLE_STEP) is None: + return + parent = item.parent() + siblings = self._sibling_list(parent) + if siblings is None: + return + step = item.data(0, ROLE_STEP) + idx = siblings.index(step) + new_idx = idx + offset + if not 0 <= new_idx < len(siblings): + return + siblings[idx], siblings[new_idx] = siblings[new_idx], siblings[idx] + if parent is None: + self.takeTopLevelItem(self.indexOfTopLevelItem(item)) + self.insertTopLevelItem(new_idx, item) + else: + parent.removeChild(item) + parent.insertChild(new_idx, item) + self.setCurrentItem(item) + + def _sibling_list(self, parent_item: Optional[QTreeWidgetItem] + ) -> Optional[List[Step]]: + if parent_item is None: + return self._roots + body_key = parent_item.data(0, ROLE_BODY_KEY) + if body_key is None: + return None + grandparent = parent_item.parent() + if grandparent is None: + return None + grandparent_step: Step = grandparent.data(0, ROLE_STEP) + return grandparent_step.bodies.setdefault(body_key, []) + + def _selected_body_target(self) -> Tuple[Optional[QTreeWidgetItem], Optional[str]]: + item = self.currentItem() + if item is None: + return None, None + body_key = item.data(0, ROLE_BODY_KEY) + if body_key is not None: + return item.parent(), body_key + step = item.data(0, ROLE_STEP) + if isinstance(step, Step) and step.bodies: + first_key = next(iter(step.bodies)) + return item, first_key + return None, None + + def _emit_selection(self) -> None: + item = self.currentItem() + step = item.data(0, ROLE_STEP) if item is not None else None + self.selected_step_changed.emit(step if isinstance(step, Step) else None) + + +def _build_item(step: Step) -> QTreeWidgetItem: + item = QTreeWidgetItem([step.label]) + item.setData(0, ROLE_STEP, step) + spec = COMMAND_SPECS.get(step.command) + if spec is not None: + for body_key in spec.body_keys: + body_item = _add_body_node(item, body_key) + for child_step in step.bodies.get(body_key, []): + body_item.addChild(_build_item(child_step)) + return item + + +def _add_body_node(parent: QTreeWidgetItem, body_key: str) -> QTreeWidgetItem: + node = QTreeWidgetItem([f"[{body_key}]"]) + node.setData(0, ROLE_BODY_KEY, body_key) + node.setFlags(node.flags() & ~Qt.ItemFlag.ItemIsDragEnabled) + parent.addChild(node) + return node + + +def _find_body_item(parent: QTreeWidgetItem, body_key: str) -> Optional[QTreeWidgetItem]: + for i in range(parent.childCount()): + child = parent.child(i) + if child.data(0, ROLE_BODY_KEY) == body_key: + return child + return None diff --git a/je_auto_control/gui/script_builder/step_model.py b/je_auto_control/gui/script_builder/step_model.py new file mode 100644 index 0000000..58590c3 --- /dev/null +++ b/je_auto_control/gui/script_builder/step_model.py @@ -0,0 +1,77 @@ +"""Step data model and (de)serialisation between the tree view and AC JSON.""" +from dataclasses import dataclass, field +from typing import Any, Dict, List, Mapping, Tuple + +from je_auto_control.gui.script_builder.command_schema import COMMAND_SPECS + + +@dataclass +class Step: + """A single node in the script tree. + + ``bodies`` maps body key (``body``, ``then``, ``else``) to child steps, + mirroring the flow-control structure in the executor. + """ + command: str + params: Dict[str, Any] = field(default_factory=dict) + bodies: Dict[str, List["Step"]] = field(default_factory=dict) + + @property + def label(self) -> str: + """Human-readable label derived from the command and key params.""" + spec = COMMAND_SPECS.get(self.command) + base = spec.label if spec else self.command + detail = _summarise_params(self.params) + return f"{base} {detail}" if detail else base + + +def step_to_action(step: Step) -> list: + """Convert a Step to the executor's action list entry.""" + params: Dict[str, Any] = dict(step.params) + for body_key, children in step.bodies.items(): + params[body_key] = [step_to_action(child) for child in children] + if not params: + return [step.command] + return [step.command, params] + + +def action_to_step(action: list) -> Step: + """Convert a single action entry back to a Step.""" + if not action or not isinstance(action[0], str): + raise ValueError(f"Invalid action: {action!r}") + command = action[0] + raw_params: Mapping[str, Any] = action[1] if len(action) > 1 and isinstance(action[1], dict) else {} + spec = COMMAND_SPECS.get(command) + body_keys: Tuple[str, ...] = spec.body_keys if spec else () + + params: Dict[str, Any] = {} + bodies: Dict[str, List[Step]] = {} + for key, value in raw_params.items(): + if key in body_keys and isinstance(value, list): + bodies[key] = [action_to_step(child) for child in value] + else: + params[key] = value + return Step(command=command, params=params, bodies=bodies) + + +def actions_to_steps(actions: list) -> List[Step]: + """Convert a flat action list to a list of Steps.""" + return [action_to_step(entry) for entry in actions] + + +def steps_to_actions(steps: List[Step]) -> list: + """Convert a list of Steps back to an AC action list.""" + return [step_to_action(step) for step in steps] + + +def _summarise_params(params: Mapping[str, Any]) -> str: + """Produce a compact one-line summary of param values.""" + if not params: + return "" + parts = [] + for key, value in list(params.items())[:3]: + text = str(value) + if len(text) > 24: + text = text[:21] + "..." + parts.append(f"{key}={text}") + return ", ".join(parts) diff --git a/je_auto_control/gui/selector/__init__.py b/je_auto_control/gui/selector/__init__.py new file mode 100644 index 0000000..c980615 --- /dev/null +++ b/je_auto_control/gui/selector/__init__.py @@ -0,0 +1,5 @@ +"""Region and template selector tools for the AutoControlGUI.""" +from je_auto_control.gui.selector.region_selector import open_region_selector +from je_auto_control.gui.selector.template_cropper import crop_template_to_file + +__all__ = ["open_region_selector", "crop_template_to_file"] diff --git a/je_auto_control/gui/selector/region_overlay.py b/je_auto_control/gui/selector/region_overlay.py new file mode 100644 index 0000000..7579258 --- /dev/null +++ b/je_auto_control/gui/selector/region_overlay.py @@ -0,0 +1,107 @@ +"""Full-screen translucent overlay for drawing a selection rectangle.""" +from typing import Optional, Tuple + +from PySide6.QtCore import QPoint, QRect, Qt, Signal +from PySide6.QtGui import QColor, QKeyEvent, QMouseEvent, QPainter, QPen +from PySide6.QtWidgets import QApplication, QWidget + + +class RegionOverlay(QWidget): + """Frameless full-screen widget for selecting a rectangular region.""" + + region_selected = Signal(int, int, int, int) + cancelled = Signal() + + def __init__(self) -> None: + super().__init__( + None, + Qt.WindowType.FramelessWindowHint + | Qt.WindowType.WindowStaysOnTopHint + | Qt.WindowType.Tool, + ) + self.setAttribute(Qt.WidgetAttribute.WA_TranslucentBackground) + self.setCursor(Qt.CursorShape.CrossCursor) + virtual = self._virtual_geometry() + self.setGeometry(virtual) + self._virtual_origin = virtual.topLeft() + self._origin: Optional[QPoint] = None + self._current: Optional[QPoint] = None + + @staticmethod + def _virtual_geometry() -> QRect: + screens = QApplication.screens() + geom = screens[0].geometry() + for screen in screens[1:]: + geom = geom.united(screen.geometry()) + return geom + + def _rect(self) -> QRect: + if self._origin is None or self._current is None: + return QRect() + return QRect(self._origin, self._current).normalized() + + def paintEvent(self, event) -> None: # noqa: N802 Qt override + del event + painter = QPainter(self) + painter.fillRect(self.rect(), QColor(0, 0, 0, 90)) + rect = self._rect() + if rect.isEmpty(): + return + painter.setCompositionMode(QPainter.CompositionMode.CompositionMode_Clear) + painter.fillRect(rect, Qt.GlobalColor.transparent) + painter.setCompositionMode(QPainter.CompositionMode.CompositionMode_SourceOver) + pen = QPen(QColor(255, 180, 0), 2) + painter.setPen(pen) + painter.drawRect(rect) + + def mousePressEvent(self, event: QMouseEvent) -> None: # noqa: N802 + if event.button() == Qt.MouseButton.LeftButton: + self._origin = event.position().toPoint() + self._current = self._origin + self.update() + + def mouseMoveEvent(self, event: QMouseEvent) -> None: # noqa: N802 + if self._origin is not None: + self._current = event.position().toPoint() + self.update() + + def mouseReleaseEvent(self, event: QMouseEvent) -> None: # noqa: N802 + if event.button() != Qt.MouseButton.LeftButton or self._origin is None: + return + self._current = event.position().toPoint() + rect = self._rect() + self.close() + if rect.width() < 2 or rect.height() < 2: + self.cancelled.emit() + return + screen_x = rect.x() + self._virtual_origin.x() + screen_y = rect.y() + self._virtual_origin.y() + self.region_selected.emit(screen_x, screen_y, rect.width(), rect.height()) + + def keyPressEvent(self, event: QKeyEvent) -> None: # noqa: N802 + if event.key() == Qt.Key.Key_Escape: + self.close() + self.cancelled.emit() + else: + super().keyPressEvent(event) + + +def pick_region_blocking(parent: Optional[QWidget] = None + ) -> Optional[Tuple[int, int, int, int]]: + """Open overlay and block until the user selects a region or cancels.""" + del parent + overlay = RegionOverlay() + result: dict = {"region": None} + + def on_selected(x: int, y: int, w: int, h: int) -> None: + result["region"] = (x, y, w, h) + + overlay.region_selected.connect(on_selected) + overlay.cancelled.connect(lambda: None) + overlay.showFullScreen() + overlay.activateWindow() + overlay.raise_() + # Spin the local event loop until the overlay closes. + while overlay.isVisible(): + QApplication.processEvents() + return result["region"] diff --git a/je_auto_control/gui/selector/region_selector.py b/je_auto_control/gui/selector/region_selector.py new file mode 100644 index 0000000..cd59db6 --- /dev/null +++ b/je_auto_control/gui/selector/region_selector.py @@ -0,0 +1,17 @@ +"""Public API for drag-to-select region picking.""" +from typing import Optional, Tuple + +from PySide6.QtWidgets import QWidget + +from je_auto_control.gui.selector.region_overlay import pick_region_blocking + + +def open_region_selector(parent: Optional[QWidget] = None + ) -> Optional[Tuple[int, int, int, int]]: + """ + Display a full-screen overlay and return the chosen (x, y, w, h) region. + + :param parent: optional parent widget (currently unused; the overlay is top-level) + :return: (x, y, width, height) in virtual-desktop coordinates, or None if cancelled + """ + return pick_region_blocking(parent) diff --git a/je_auto_control/gui/selector/template_cropper.py b/je_auto_control/gui/selector/template_cropper.py new file mode 100644 index 0000000..35e0642 --- /dev/null +++ b/je_auto_control/gui/selector/template_cropper.py @@ -0,0 +1,50 @@ +"""Crop a selected screen region and save it as a template image.""" +import os +from typing import Optional, Tuple + +import cv2 +import numpy as np +from PIL import ImageGrab +from PySide6.QtWidgets import QWidget + +from je_auto_control.gui.selector.region_selector import open_region_selector +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +def _capture_region(region: Tuple[int, int, int, int]) -> np.ndarray: + """Grab the given (x, y, w, h) region as a BGR numpy array.""" + x, y, w, h = region + pil_image = ImageGrab.grab(bbox=(x, y, x + w, y + h), all_screens=True) + return cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR) + + +def _validate_output_path(raw_path: str) -> str: + """Reject empty paths and ensure the target directory exists.""" + if not raw_path: + raise ValueError("Output path is empty") + resolved = os.path.realpath(raw_path) + parent_dir = os.path.dirname(resolved) + if not parent_dir or not os.path.isdir(parent_dir): + raise ValueError(f"Target directory does not exist: {parent_dir}") + return resolved + + +def crop_template_to_file(save_path: str, + parent: Optional[QWidget] = None + ) -> Optional[Tuple[int, int, int, int]]: + """ + Prompt the user to drag-select a region and save it as a template PNG. + + :param save_path: absolute or relative path for the output PNG + :param parent: optional parent widget passed to the overlay + :return: selected (x, y, w, h) or None if cancelled + """ + region = open_region_selector(parent) + if region is None: + return None + resolved = _validate_output_path(save_path) + frame = _capture_region(region) + if not cv2.imwrite(resolved, frame): + raise OSError(f"cv2.imwrite failed: {resolved}") + autocontrol_logger.info("template cropped: region=%s path=%s", region, resolved) + return region diff --git a/je_auto_control/gui/socket_server_tab.py b/je_auto_control/gui/socket_server_tab.py new file mode 100644 index 0000000..dce68ed --- /dev/null +++ b/je_auto_control/gui/socket_server_tab.py @@ -0,0 +1,141 @@ +"""Socket + REST server control panel.""" +from typing import Optional + +from PySide6.QtGui import QIntValidator +from PySide6.QtWidgets import ( + QCheckBox, QGroupBox, QHBoxLayout, QLabel, QLineEdit, QMessageBox, + QPushButton, QVBoxLayout, QWidget, +) + +from je_auto_control.utils.rest_api.rest_server import RestApiServer +from je_auto_control.utils.socket_server.auto_control_socket_server import ( + start_autocontrol_socket_server, +) + + +class SocketServerTab(QWidget): + """Start / stop the TCP socket server and the REST API server.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._tcp_server = None + self._rest_server: Optional[RestApiServer] = None + self._tcp_host = QLineEdit("127.0.0.1") + self._tcp_port = QLineEdit("9938") + self._tcp_port.setValidator(QIntValidator(1, 65535)) + self._tcp_any = QCheckBox("Bind TCP to 0.0.0.0 (exposes to network)") + self._tcp_status = QLabel("TCP stopped") + self._tcp_start_btn = QPushButton("Start TCP") + self._tcp_stop_btn = QPushButton("Stop TCP") + self._tcp_stop_btn.setEnabled(False) + + self._rest_host = QLineEdit("127.0.0.1") + self._rest_port = QLineEdit("9939") + self._rest_port.setValidator(QIntValidator(1, 65535)) + self._rest_any = QCheckBox("Bind REST to 0.0.0.0 (exposes to network)") + self._rest_status = QLabel("REST stopped") + self._rest_start_btn = QPushButton("Start REST") + self._rest_stop_btn = QPushButton("Stop REST") + self._rest_stop_btn.setEnabled(False) + + self._build_layout() + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + + tcp_group = QGroupBox("TCP socket server") + tcp_layout = QVBoxLayout(tcp_group) + tcp_form = QHBoxLayout() + tcp_form.addWidget(QLabel("Host:")) + tcp_form.addWidget(self._tcp_host) + tcp_form.addWidget(QLabel("Port:")) + tcp_form.addWidget(self._tcp_port) + tcp_layout.addLayout(tcp_form) + tcp_layout.addWidget(self._tcp_any) + tcp_btns = QHBoxLayout() + self._tcp_start_btn.clicked.connect(self._start_tcp) + self._tcp_stop_btn.clicked.connect(self._stop_tcp) + tcp_btns.addWidget(self._tcp_start_btn) + tcp_btns.addWidget(self._tcp_stop_btn) + tcp_btns.addStretch() + tcp_layout.addLayout(tcp_btns) + tcp_layout.addWidget(self._tcp_status) + root.addWidget(tcp_group) + + rest_group = QGroupBox("REST API server") + rest_layout = QVBoxLayout(rest_group) + rest_form = QHBoxLayout() + rest_form.addWidget(QLabel("Host:")) + rest_form.addWidget(self._rest_host) + rest_form.addWidget(QLabel("Port:")) + rest_form.addWidget(self._rest_port) + rest_layout.addLayout(rest_form) + rest_layout.addWidget(self._rest_any) + rest_btns = QHBoxLayout() + self._rest_start_btn.clicked.connect(self._start_rest) + self._rest_stop_btn.clicked.connect(self._stop_rest) + rest_btns.addWidget(self._rest_start_btn) + rest_btns.addWidget(self._rest_stop_btn) + rest_btns.addStretch() + rest_layout.addLayout(rest_btns) + rest_layout.addWidget(self._rest_status) + root.addWidget(rest_group) + + root.addStretch() + + def _resolved_host(self, input_field: QLineEdit, any_addr: QCheckBox) -> str: + if any_addr.isChecked(): + return "0.0.0.0" # noqa: S104 # nosec B104 # reason: explicit opt-in via checkbox + return input_field.text().strip() or "127.0.0.1" + + def _start_tcp(self) -> None: + if self._tcp_server is not None: + return + host = self._resolved_host(self._tcp_host, self._tcp_any) + try: + port = int(self._tcp_port.text() or "9938") + self._tcp_server = start_autocontrol_socket_server(host, port) + except (OSError, ValueError) as error: + QMessageBox.warning(self, "Error", str(error)) + return + self._tcp_status.setText(f"Listening on {host}:{port}") + self._tcp_start_btn.setEnabled(False) + self._tcp_stop_btn.setEnabled(True) + + def _stop_tcp(self) -> None: + if self._tcp_server is None: + return + try: + self._tcp_server.shutdown() + self._tcp_server.server_close() + except OSError as error: + QMessageBox.warning(self, "Error", str(error)) + self._tcp_server = None + self._tcp_status.setText("TCP stopped") + self._tcp_start_btn.setEnabled(True) + self._tcp_stop_btn.setEnabled(False) + + def _start_rest(self) -> None: + if self._rest_server is not None: + return + host = self._resolved_host(self._rest_host, self._rest_any) + try: + port = int(self._rest_port.text() or "9939") + self._rest_server = RestApiServer(host=host, port=port) + self._rest_server.start() + except (OSError, ValueError) as error: + QMessageBox.warning(self, "Error", str(error)) + self._rest_server = None + return + self._rest_status.setText(f"Listening on {host}:{port}") + self._rest_start_btn.setEnabled(False) + self._rest_stop_btn.setEnabled(True) + + def _stop_rest(self) -> None: + if self._rest_server is None: + return + self._rest_server.stop() + self._rest_server = None + self._rest_status.setText("REST stopped") + self._rest_start_btn.setEnabled(True) + self._rest_stop_btn.setEnabled(False) diff --git a/je_auto_control/gui/triggers_tab.py b/je_auto_control/gui/triggers_tab.py new file mode 100644 index 0000000..78cd216 --- /dev/null +++ b/je_auto_control/gui/triggers_tab.py @@ -0,0 +1,227 @@ +"""Triggers tab: image / window / pixel / file event watchers.""" +from typing import Optional + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import ( + QCheckBox, QComboBox, QFileDialog, QHBoxLayout, QLabel, QLineEdit, + QMessageBox, QPushButton, QStackedWidget, QTableWidget, QTableWidgetItem, + QVBoxLayout, QWidget, +) + +from je_auto_control.utils.triggers.trigger_engine import ( + FilePathTrigger, ImageAppearsTrigger, PixelColorTrigger, + WindowAppearsTrigger, default_trigger_engine, +) + + +class TriggersTab(QWidget): + """Build triggers, run the engine, inspect the table.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._script_input = QLineEdit() + self._repeat_check = QCheckBox("Repeat") + self._repeat_check.setChecked(True) + self._type_combo = QComboBox() + self._type_combo.addItems(["Image appears", "Window appears", + "Pixel matches", "File changed"]) + self._stack = QStackedWidget() + self._image_widgets = self._build_image_form() + self._window_widgets = self._build_window_form() + self._pixel_widgets = self._build_pixel_form() + self._file_widgets = self._build_file_form() + self._status = QLabel("Engine stopped") + self._table = QTableWidget(0, 5) + self._table.setHorizontalHeaderLabels( + ["ID", "Type", "Detail", "Fired", "Enabled"] + ) + self._timer = QTimer(self) + self._timer.setInterval(1000) + self._timer.timeout.connect(self._refresh) + self._build_layout() + self._type_combo.currentIndexChanged.connect(self._stack.setCurrentIndex) + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + form_top = QHBoxLayout() + form_top.addWidget(QLabel("Script:")) + form_top.addWidget(self._script_input, stretch=1) + browse = QPushButton("Browse") + browse.clicked.connect(self._browse_script) + form_top.addWidget(browse) + form_top.addWidget(self._repeat_check) + form_top.addWidget(QLabel("Type:")) + form_top.addWidget(self._type_combo) + add_btn = QPushButton("Add trigger") + add_btn.clicked.connect(self._on_add) + form_top.addWidget(add_btn) + root.addLayout(form_top) + root.addWidget(self._stack) + root.addWidget(self._table, stretch=1) + + ctl = QHBoxLayout() + for label, handler in ( + ("Remove selected", self._on_remove), + ("Start engine", self._on_start), + ("Stop engine", self._on_stop), + ): + btn = QPushButton(label) + btn.clicked.connect(handler) + ctl.addWidget(btn) + ctl.addStretch() + root.addLayout(ctl) + root.addWidget(self._status) + + def _build_image_form(self) -> dict: + widget = QWidget() + layout = QHBoxLayout(widget) + path_input = QLineEdit() + threshold_input = QLineEdit("0.8") + browse = QPushButton("Browse") + browse.clicked.connect(lambda: self._browse_image(path_input)) + layout.addWidget(QLabel("Image:")) + layout.addWidget(path_input, stretch=1) + layout.addWidget(browse) + layout.addWidget(QLabel("Threshold:")) + layout.addWidget(threshold_input) + self._stack.addWidget(widget) + return {"path": path_input, "threshold": threshold_input} + + def _build_window_form(self) -> dict: + widget = QWidget() + layout = QHBoxLayout(widget) + title_input = QLineEdit() + layout.addWidget(QLabel("Title contains:")) + layout.addWidget(title_input, stretch=1) + self._stack.addWidget(widget) + return {"title": title_input} + + def _build_pixel_form(self) -> dict: + widget = QWidget() + layout = QHBoxLayout(widget) + x_input = QLineEdit("0") + y_input = QLineEdit("0") + r_input = QLineEdit("0") + g_input = QLineEdit("0") + b_input = QLineEdit("0") + tol_input = QLineEdit("8") + for label, field in (("X:", x_input), ("Y:", y_input), + ("R:", r_input), ("G:", g_input), ("B:", b_input), + ("±:", tol_input)): + layout.addWidget(QLabel(label)) + layout.addWidget(field) + self._stack.addWidget(widget) + return {"x": x_input, "y": y_input, "r": r_input, + "g": g_input, "b": b_input, "tol": tol_input} + + def _build_file_form(self) -> dict: + widget = QWidget() + layout = QHBoxLayout(widget) + path_input = QLineEdit() + browse = QPushButton("Browse") + browse.clicked.connect(lambda: self._browse_watch(path_input)) + layout.addWidget(QLabel("Watch path:")) + layout.addWidget(path_input, stretch=1) + layout.addWidget(browse) + self._stack.addWidget(widget) + return {"path": path_input} + + def _browse_script(self) -> None: + path, _ = QFileDialog.getOpenFileName(self, "Select script", "", "JSON (*.json)") + if path: + self._script_input.setText(path) + + def _browse_image(self, target: QLineEdit) -> None: + path, _ = QFileDialog.getOpenFileName(self, "Select image", "", + "Images (*.png *.jpg *.bmp)") + if path: + target.setText(path) + + def _browse_watch(self, target: QLineEdit) -> None: + path, _ = QFileDialog.getOpenFileName(self, "Select file to watch", "") + if path: + target.setText(path) + + def _on_add(self) -> None: + script = self._script_input.text().strip() + if not script: + QMessageBox.warning(self, "Error", "Script path is required") + return + try: + trigger = self._build_trigger(script) + except ValueError as error: + QMessageBox.warning(self, "Error", str(error)) + return + default_trigger_engine.add(trigger) + self._refresh() + + def _build_trigger(self, script: str): + idx = self._type_combo.currentIndex() + common = {"trigger_id": "", "script_path": script, + "repeat": self._repeat_check.isChecked()} + if idx == 0: + return ImageAppearsTrigger( + image_path=self._image_widgets["path"].text().strip(), + threshold=float(self._image_widgets["threshold"].text() or "0.8"), + **common, + ) + if idx == 1: + return WindowAppearsTrigger( + title_substring=self._window_widgets["title"].text().strip(), + **common, + ) + if idx == 2: + w = self._pixel_widgets + return PixelColorTrigger( + x=int(w["x"].text() or "0"), y=int(w["y"].text() or "0"), + target_rgb=(int(w["r"].text() or "0"), + int(w["g"].text() or "0"), + int(w["b"].text() or "0")), + tolerance=int(w["tol"].text() or "8"), + **common, + ) + return FilePathTrigger( + watch_path=self._file_widgets["path"].text().strip(), + **common, + ) + + def _on_remove(self) -> None: + row = self._table.currentRow() + if row < 0: + return + tid = self._table.item(row, 0).text() + default_trigger_engine.remove(tid) + self._refresh() + + def _on_start(self) -> None: + default_trigger_engine.start() + self._timer.start() + self._status.setText("Engine running") + + def _on_stop(self) -> None: + default_trigger_engine.stop() + self._timer.stop() + self._status.setText("Engine stopped") + + def _refresh(self) -> None: + triggers = default_trigger_engine.list_triggers() + self._table.setRowCount(len(triggers)) + for row, trigger in enumerate(triggers): + detail = _describe(trigger) + for col, value in enumerate(( + trigger.trigger_id, type(trigger).__name__, detail, + str(trigger.fired), "Yes" if trigger.enabled else "No", + )): + self._table.setItem(row, col, QTableWidgetItem(value)) + + +def _describe(trigger) -> str: + if isinstance(trigger, ImageAppearsTrigger): + return f"img={trigger.image_path} th={trigger.threshold}" + if isinstance(trigger, WindowAppearsTrigger): + return f"title~{trigger.title_substring!r}" + if isinstance(trigger, PixelColorTrigger): + return f"({trigger.x},{trigger.y})={trigger.target_rgb} ±{trigger.tolerance}" + if isinstance(trigger, FilePathTrigger): + return f"watch={trigger.watch_path}" + return "?" diff --git a/je_auto_control/gui/window_tab.py b/je_auto_control/gui/window_tab.py new file mode 100644 index 0000000..ac5852e --- /dev/null +++ b/je_auto_control/gui/window_tab.py @@ -0,0 +1,95 @@ +"""Window Manager tab: list, focus, close windows.""" +from typing import Optional + +from PySide6.QtCore import QTimer +from PySide6.QtWidgets import ( + QHBoxLayout, QLabel, QLineEdit, QMessageBox, QPushButton, QTableWidget, + QTableWidgetItem, QVBoxLayout, QWidget, +) + +from je_auto_control.wrapper.auto_control_window import ( + close_window_by_title, focus_window, list_windows, +) + + +class WindowManagerTab(QWidget): + """Browse top-level windows and trigger focus / close actions.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._table = QTableWidget(0, 2) + self._table.setHorizontalHeaderLabels(["HWND", "Title"]) + self._filter = QLineEdit() + self._filter.setPlaceholderText("Filter by title substring") + self._filter.textChanged.connect(self._apply_filter) + self._status = QLabel("") + self._build_layout() + self.refresh() + + def _build_layout(self) -> None: + root = QVBoxLayout(self) + top = QHBoxLayout() + refresh = QPushButton("Refresh") + refresh.clicked.connect(self.refresh) + top.addWidget(refresh) + top.addWidget(self._filter, stretch=1) + root.addLayout(top) + root.addWidget(self._table, stretch=1) + actions = QHBoxLayout() + for label, handler in ( + ("Focus selected", self._on_focus), + ("Close selected", self._on_close), + ): + btn = QPushButton(label) + btn.clicked.connect(handler) + actions.addWidget(btn) + actions.addStretch() + root.addLayout(actions) + root.addWidget(self._status) + + def refresh(self) -> None: + try: + windows = list_windows() + except NotImplementedError as error: + self._status.setText(str(error)) + self._table.setRowCount(0) + return + self._table.setRowCount(len(windows)) + for row, (hwnd, title) in enumerate(windows): + self._table.setItem(row, 0, QTableWidgetItem(str(hwnd))) + self._table.setItem(row, 1, QTableWidgetItem(title)) + self._status.setText(f"{len(windows)} windows") + QTimer.singleShot(0, self._apply_filter) + + def _apply_filter(self) -> None: + needle = self._filter.text().strip().lower() + for row in range(self._table.rowCount()): + item = self._table.item(row, 1) + visible = not needle or (item is not None and needle in item.text().lower()) + self._table.setRowHidden(row, not visible) + + def _selected_title(self) -> Optional[str]: + row = self._table.currentRow() + if row < 0: + return None + item = self._table.item(row, 1) + return item.text() if item is not None else None + + def _on_focus(self) -> None: + title = self._selected_title() + if not title: + return + try: + focus_window(title, case_sensitive=True) + except (NotImplementedError, RuntimeError, OSError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _on_close(self) -> None: + title = self._selected_title() + if not title: + return + try: + close_window_by_title(title, case_sensitive=True) + self.refresh() + except (NotImplementedError, RuntimeError, OSError) as error: + QMessageBox.warning(self, "Error", str(error)) diff --git a/je_auto_control/utils/clipboard/__init__.py b/je_auto_control/utils/clipboard/__init__.py new file mode 100644 index 0000000..004d613 --- /dev/null +++ b/je_auto_control/utils/clipboard/__init__.py @@ -0,0 +1,4 @@ +"""Cross-platform headless clipboard access.""" +from je_auto_control.utils.clipboard.clipboard import get_clipboard, set_clipboard + +__all__ = ["get_clipboard", "set_clipboard"] diff --git a/je_auto_control/utils/clipboard/clipboard.py b/je_auto_control/utils/clipboard/clipboard.py new file mode 100644 index 0000000..ca90a59 --- /dev/null +++ b/je_auto_control/utils/clipboard/clipboard.py @@ -0,0 +1,160 @@ +"""Headless cross-platform text clipboard. + +Windows uses Win32 clipboard API via ctypes. +macOS shells out to pbcopy / pbpaste. +Linux shells out to xclip or xsel (whichever is available). + +All functions raise ``RuntimeError`` if the platform backend is missing so +callers can degrade gracefully. +""" +import shutil +import subprocess +import sys +from typing import Optional + + +def get_clipboard() -> str: + """Return the current clipboard text (empty string if empty).""" + if sys.platform.startswith("win"): + return _win_get() + if sys.platform == "darwin": + return _mac_get() + return _linux_get() + + +def set_clipboard(text: str) -> None: + """Replace clipboard contents with ``text``.""" + if not isinstance(text, str): + raise TypeError("set_clipboard expects a str") + if sys.platform.startswith("win"): + _win_set(text) + return + if sys.platform == "darwin": + _mac_set(text) + return + _linux_set(text) + + +# === Windows backend ========================================================= + +def _win_get() -> str: + import ctypes + from ctypes import wintypes + + user32 = ctypes.WinDLL("user32", use_last_error=True) + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + cf_unicodetext = 13 + + user32.OpenClipboard.argtypes = [wintypes.HWND] + user32.OpenClipboard.restype = wintypes.BOOL + user32.GetClipboardData.argtypes = [wintypes.UINT] + user32.GetClipboardData.restype = wintypes.HANDLE + user32.CloseClipboard.restype = wintypes.BOOL + kernel32.GlobalLock.argtypes = [wintypes.HGLOBAL] + kernel32.GlobalLock.restype = ctypes.c_void_p + kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL] + + if not user32.OpenClipboard(None): + raise RuntimeError("OpenClipboard failed") + try: + handle = user32.GetClipboardData(cf_unicodetext) + if not handle: + return "" + pointer = kernel32.GlobalLock(handle) + if not pointer: + return "" + try: + return ctypes.wstring_at(pointer) + finally: + kernel32.GlobalUnlock(handle) + finally: + user32.CloseClipboard() + + +def _win_set(text: str) -> None: + import ctypes + from ctypes import wintypes + + user32 = ctypes.WinDLL("user32", use_last_error=True) + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + cf_unicodetext = 13 + gmem_moveable = 0x0002 + + user32.OpenClipboard.argtypes = [wintypes.HWND] + user32.OpenClipboard.restype = wintypes.BOOL + user32.EmptyClipboard.restype = wintypes.BOOL + user32.SetClipboardData.argtypes = [wintypes.UINT, wintypes.HANDLE] + user32.SetClipboardData.restype = wintypes.HANDLE + user32.CloseClipboard.restype = wintypes.BOOL + kernel32.GlobalAlloc.argtypes = [wintypes.UINT, ctypes.c_size_t] + kernel32.GlobalAlloc.restype = wintypes.HGLOBAL + kernel32.GlobalLock.argtypes = [wintypes.HGLOBAL] + kernel32.GlobalLock.restype = ctypes.c_void_p + kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL] + + data = ctypes.create_unicode_buffer(text) + size = ctypes.sizeof(data) + handle = kernel32.GlobalAlloc(gmem_moveable, size) + if not handle: + raise RuntimeError("GlobalAlloc failed") + pointer = kernel32.GlobalLock(handle) + if not pointer: + raise RuntimeError("GlobalLock failed") + ctypes.memmove(pointer, ctypes.addressof(data), size) + kernel32.GlobalUnlock(handle) + if not user32.OpenClipboard(None): + raise RuntimeError("OpenClipboard failed") + try: + user32.EmptyClipboard() + if not user32.SetClipboardData(cf_unicodetext, handle): + raise RuntimeError("SetClipboardData failed") + finally: + user32.CloseClipboard() + + +# === macOS backend =========================================================== + +def _mac_get() -> str: + result = subprocess.run( + ["pbpaste"], capture_output=True, check=True, timeout=5, + ) + return result.stdout.decode("utf-8", errors="replace") + + +def _mac_set(text: str) -> None: + subprocess.run( + ["pbcopy"], input=text.encode("utf-8"), + check=True, timeout=5, + ) + + +# === Linux backend =========================================================== + +def _linux_cmd() -> Optional[list]: + if shutil.which("xclip"): + return ["xclip", "-selection", "clipboard"] + if shutil.which("xsel"): + return ["xsel", "--clipboard"] + return None + + +def _linux_get() -> str: + cmd = _linux_cmd() + if cmd is None: + raise RuntimeError("Install xclip or xsel for Linux clipboard support") + read_cmd = cmd + ["-o"] if cmd[0] == "xclip" else cmd + ["--output"] + result = subprocess.run( + read_cmd, capture_output=True, check=True, timeout=5, + ) + return result.stdout.decode("utf-8", errors="replace") + + +def _linux_set(text: str) -> None: + cmd = _linux_cmd() + if cmd is None: + raise RuntimeError("Install xclip or xsel for Linux clipboard support") + write_cmd = cmd + ["-i"] if cmd[0] == "xclip" else cmd + ["--input"] + subprocess.run( + write_cmd, input=text.encode("utf-8"), + check=True, timeout=5, + ) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 59ba6e9..c810d47 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -1,5 +1,5 @@ import types -from typing import Any, Dict, List, Union +from typing import Any, Callable, Dict, List, Optional, Union from je_auto_control.utils.exception.exception_tags import ( action_is_null_error_message, add_command_exception_error_message, @@ -9,6 +9,19 @@ AutoControlActionException, AutoControlAddCommandException, AutoControlActionNullException ) +from je_auto_control.utils.clipboard.clipboard import ( + get_clipboard, set_clipboard, +) +from je_auto_control.utils.executor.action_schema import validate_actions +from je_auto_control.utils.executor.flow_control import ( + BLOCK_COMMANDS, LoopBreak, LoopContinue, +) +from je_auto_control.utils.ocr.ocr_engine import ( + click_text as ocr_click_text, + locate_text_center as ocr_locate_text_center, + wait_for_text as ocr_wait_for_text, +) +from je_auto_control.utils.script_vars.interpolate import interpolate_actions from je_auto_control.utils.generate_report.generate_html_report import generate_html, generate_html_report from je_auto_control.utils.generate_report.generate_json_report import generate_json, generate_json_report from je_auto_control.utils.generate_report.generate_xml_report import generate_xml, generate_xml_report @@ -30,6 +43,9 @@ ) from je_auto_control.wrapper.auto_control_record import record, stop_record from je_auto_control.wrapper.auto_control_screen import screenshot, screen_size +from je_auto_control.wrapper.auto_control_window import ( + close_window_by_title, focus_window, list_windows, wait_for_window, +) class Executor: @@ -39,9 +55,11 @@ class Executor: - 提供 event_dict 對應字串名稱到函式 - 支援滑鼠、鍵盤、螢幕、影像辨識、報告生成等功能 - 可執行 action list 或 action file + - 支援流程控制指令 (AC_loop, AC_if_image_found 等) """ def __init__(self): + self._block_commands = BLOCK_COMMANDS # 事件字典，對應字串名稱到函式 self.event_dict: dict = { # Mouse 滑鼠相關 @@ -103,65 +121,117 @@ def __init__(self): # Process "AC_execute_process": start_exe, + + # OCR + "AC_locate_text": ocr_locate_text_center, + "AC_wait_text": ocr_wait_for_text, + "AC_click_text": ocr_click_text, + + # Window management + "AC_list_windows": list_windows, + "AC_focus_window": focus_window, + "AC_wait_window": wait_for_window, + "AC_close_window": close_window_by_title, + + # Clipboard + "AC_clipboard_get": get_clipboard, + "AC_clipboard_set": set_clipboard, } + def known_commands(self) -> set: + """Return the set of all command names the executor recognises.""" + return set(self.event_dict.keys()) | set(self._block_commands.keys()) + def _execute_event(self, action: list) -> Any: """ 執行單一事件 Execute a single event """ - event = self.event_dict.get(action[0]) + name = action[0] + block_handler = self._block_commands.get(name) + if block_handler is not None: + args = action[1] if len(action) == 2 else {} + if not isinstance(args, dict): + raise AutoControlActionException( + f"{name} requires a dict of arguments" + ) + return block_handler(self, args) + + event = self.event_dict.get(name) if event is None: - raise AutoControlActionException(f"Unknown action: {action[0]}") + raise AutoControlActionException(f"Unknown action: {name}") if len(action) == 2: if isinstance(action[1], dict): return event(**action[1]) - else: - return event(*action[1]) - elif len(action) == 1: + return event(*action[1]) + if len(action) == 1: return event() - else: - raise AutoControlActionException(cant_execute_action_error_message + " " + str(action)) - - def execute_action(self, action_list: Union[list, dict]) -> Dict[str, str]: + raise AutoControlActionException(cant_execute_action_error_message + " " + str(action)) + + def execute_action(self, action_list: Union[list, dict], + raise_on_error: bool = False, + _validated: bool = False, + dry_run: bool = False, + step_callback: Optional[Callable[[list], None]] = None, + ) -> Dict[str, str]: """ 執行 action list Execute all actions in action list :param action_list: list 或 dict (包含 auto_control key) + :param raise_on_error: 若為 True，遇到錯誤立即拋出 (流程控制用) + :param _validated: 內部用；子呼叫已驗證過時避免重複驗證 + :param dry_run: 若為 True，只記錄將執行的動作，不實際呼叫。 + :param step_callback: 每個 action 開始前呼叫此 hook（偵錯用）。 :return: 執行紀錄字典 """ autocontrol_logger.info(f"execute_action, action_list: {action_list}") + action_list = self._unwrap_action_list(action_list) + if not _validated: + validate_actions(action_list, self.known_commands()) + + execute_record_dict: Dict[str, Any] = {} + for action in action_list: + if step_callback is not None: + step_callback(action) + if dry_run: + execute_record_dict["dry-run: " + str(action)] = "(not executed)" + continue + self._run_one_action(action, execute_record_dict, raise_on_error) + + for key, value in execute_record_dict.items(): + autocontrol_logger.info("%s -> %s", key, value) + return execute_record_dict + @staticmethod + def _unwrap_action_list(action_list: Union[list, dict]) -> list: + """Normalise the ``action_list`` argument or raise on invalid input.""" if isinstance(action_list, dict): action_list = action_list.get("auto_control") if action_list is None: raise AutoControlActionNullException(executor_list_error_message) - if not isinstance(action_list, list) or len(action_list) == 0: raise AutoControlActionNullException(action_is_null_error_message) - - execute_record_dict = {} - - for action in action_list: - try: - event_response = self._execute_event(action) - execute_record = "execute: " + str(action) - execute_record_dict[execute_record] = event_response - except (AutoControlActionException, OSError, RuntimeError, AttributeError, TypeError, ValueError) as error: - autocontrol_logger.info( - f"execute_action failed, action: {action}, error: {repr(error)}" - ) - record_action_to_list("AC_execute_action", None, repr(error)) - execute_record = "execute: " + str(action) - execute_record_dict[execute_record] = repr(error) - - # 輸出執行結果 Log results - for key, value in execute_record_dict.items(): - autocontrol_logger.info("%s -> %s", key, value) - - return execute_record_dict + return action_list + + def _run_one_action(self, action: list, record: Dict[str, Any], + raise_on_error: bool) -> None: + """Execute a single action, recording the result or raising.""" + key = "execute: " + str(action) + try: + record[key] = self._execute_event(action) + except (LoopBreak, LoopContinue): + raise + except (AutoControlActionException, OSError, RuntimeError, + AttributeError, TypeError, ValueError) as error: + if raise_on_error: + raise + autocontrol_logger.info( + f"execute_action failed, action: {action}, error: {repr(error)}" + ) + record_action_to_list("AC_execute_action", None, repr(error)) + record[key] = repr(error) def execute_files(self, execute_files_list: list) -> List[Dict[str, str]]: """ @@ -202,4 +272,11 @@ def execute_action(action_list: list) -> Dict[str, str]: def execute_files(execute_files_list: list) -> List[Dict[str, str]]: - return executor.execute_files(execute_files_list) \ No newline at end of file + return executor.execute_files(execute_files_list) + + +def execute_action_with_vars(action_list: list, variables: dict + ) -> Dict[str, str]: + """Interpolate ``${name}`` placeholders with ``variables`` and execute.""" + resolved = interpolate_actions(action_list, variables) + return executor.execute_action(resolved) diff --git a/je_auto_control/utils/executor/action_schema.py b/je_auto_control/utils/executor/action_schema.py new file mode 100644 index 0000000..a5a33c0 --- /dev/null +++ b/je_auto_control/utils/executor/action_schema.py @@ -0,0 +1,58 @@ +""" +Structural validation for action lists. + +Validates the outer shape (``[name]`` / ``[name, params]``), that names are in the +executor allowlist, and that flow-control nested bodies are themselves valid lists. +""" +from typing import Any, Iterable, Set + +from je_auto_control.utils.exception.exceptions import AutoControlActionException + + +FLOW_BODY_KEYS = { + "AC_if_image_found": ("then", "else"), + "AC_if_pixel": ("then", "else"), + "AC_loop": ("body",), + "AC_while_image": ("body",), + "AC_retry": ("body",), +} + + +def validate_actions(actions: Any, known_commands: Iterable[str]) -> None: + """Validate an action list recursively; raise on the first problem.""" + known = set(known_commands) + _validate_list(actions, known, trail="root") + + +def _validate_list(actions: Any, known: Set[str], trail: str) -> None: + if not isinstance(actions, list): + raise AutoControlActionException( + f"{trail}: action list must be a list, got {type(actions).__name__}" + ) + for idx, action in enumerate(actions): + _validate_single(action, known, f"{trail}[{idx}]") + + +def _validate_single(action: Any, known: Set[str], trail: str) -> None: + if not isinstance(action, list) or not 1 <= len(action) <= 2: + raise AutoControlActionException( + f"{trail}: must be [name] or [name, params]" + ) + name = action[0] + if not isinstance(name, str) or name not in known: + raise AutoControlActionException(f"{trail}: unknown command {name!r}") + if len(action) == 2 and not isinstance(action[1], (dict, list)): + raise AutoControlActionException( + f"{trail}: params must be dict or list" + ) + _validate_nested_bodies(name, action, known, trail) + + +def _validate_nested_bodies(name: str, action: list, known: Set[str], trail: str) -> None: + if name not in FLOW_BODY_KEYS or len(action) < 2 or not isinstance(action[1], dict): + return + for body_key in FLOW_BODY_KEYS[name]: + body = action[1].get(body_key) + if body is None: + continue + _validate_list(body, known, f"{trail}.{body_key}") diff --git a/je_auto_control/utils/executor/flow_control.py b/je_auto_control/utils/executor/flow_control.py new file mode 100644 index 0000000..2af34d2 --- /dev/null +++ b/je_auto_control/utils/executor/flow_control.py @@ -0,0 +1,192 @@ +""" +Flow-control block commands for the action executor. + +These commands receive the owning executor and a dict of arguments; +they may execute nested action lists (``body`` / ``then`` / ``else``) +by delegating back to ``executor.execute_action``. +""" +import time +from typing import Any, Callable, Dict, Mapping, Optional, Sequence + +from je_auto_control.utils.exception.exceptions import ( + AutoControlActionException, ImageNotFoundException, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.wrapper.auto_control_image import locate_image_center +from je_auto_control.wrapper.auto_control_screen import get_pixel + + +class LoopBreak(Exception): + """Internal signal raised by AC_break; caught only by loop handlers.""" + + +class LoopContinue(Exception): + """Internal signal raised by AC_continue; caught only by loop handlers.""" + + +def _image_present(image: str, threshold: float) -> bool: + """Return True when the template image is detected on screen.""" + try: + locate_image_center(image, threshold) + return True + except (ImageNotFoundException, OSError, RuntimeError, ValueError, TypeError): + return False + + +def _pixel_matches(x: int, y: int, rgb: Sequence[int], tolerance: int) -> bool: + """Return True when the pixel at (x, y) matches rgb within tolerance.""" + color = get_pixel(x, y) + if color is None or len(color) < 3 or len(rgb) < 3: + return False + return all(abs(int(color[i]) - int(rgb[i])) <= tolerance for i in range(3)) + + +def _run_branch(executor: Any, body: Optional[list]) -> Any: + """Execute a nested branch only when it is a non-empty list.""" + if not body: + return None + return executor.execute_action(body, _validated=True) + + +def _run_strict(executor: Any, body: list) -> Any: + """Execute a nested body, re-raising the first error.""" + return executor.execute_action(body, raise_on_error=True, _validated=True) + + +def exec_if_image_found(executor: Any, args: Mapping[str, Any]) -> Any: + """Run ``then`` when the image is present, else run ``else``.""" + image = args["image"] + threshold = float(args.get("threshold", 0.8)) + key = "then" if _image_present(image, threshold) else "else" + return _run_branch(executor, args.get(key)) + + +def exec_if_pixel(executor: Any, args: Mapping[str, Any]) -> Any: + """Run ``then`` when pixel matches, else run ``else``.""" + matched = _pixel_matches( + int(args["x"]), int(args["y"]), + list(args["rgb"]), int(args.get("tolerance", 0)), + ) + key = "then" if matched else "else" + return _run_branch(executor, args.get(key)) + + +def exec_wait_image(executor: Any, args: Mapping[str, Any]) -> bool: + """Poll for the image until timeout; raise on timeout.""" + del executor + image = args["image"] + threshold = float(args.get("threshold", 0.8)) + timeout = float(args.get("timeout", 10.0)) + poll = max(float(args.get("poll", 0.2)), 0.01) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if _image_present(image, threshold): + return True + time.sleep(poll) + raise AutoControlActionException(f"AC_wait_image timeout: {image}") + + +def exec_wait_pixel(executor: Any, args: Mapping[str, Any]) -> bool: + """Poll for a matching pixel until timeout; raise on timeout.""" + del executor + x, y = int(args["x"]), int(args["y"]) + rgb = list(args["rgb"]) + tolerance = int(args.get("tolerance", 0)) + timeout = float(args.get("timeout", 10.0)) + poll = max(float(args.get("poll", 0.2)), 0.01) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if _pixel_matches(x, y, rgb, tolerance): + return True + time.sleep(poll) + raise AutoControlActionException(f"AC_wait_pixel timeout at ({x},{y})") + + +def exec_sleep(executor: Any, args: Mapping[str, Any]) -> None: + """Sleep for ``seconds``.""" + del executor + time.sleep(float(args["seconds"])) + + +def exec_loop(executor: Any, args: Mapping[str, Any]) -> int: + """Execute ``body`` a fixed number of times; honour break/continue.""" + times = int(args["times"]) + body = args.get("body") or [] + completed = 0 + for _ in range(times): + try: + executor.execute_action(body, _validated=True) + except LoopContinue: + completed += 1 + continue + except LoopBreak: + break + completed += 1 + return completed + + +def exec_while_image(executor: Any, args: Mapping[str, Any]) -> int: + """Execute ``body`` while the image is present, up to ``max_iter``.""" + image = args["image"] + threshold = float(args.get("threshold", 0.8)) + max_iter = int(args.get("max_iter", 100)) + body = args.get("body") or [] + iterations = 0 + while iterations < max_iter and _image_present(image, threshold): + try: + executor.execute_action(body, _validated=True) + except LoopContinue: + pass + except LoopBreak: + break + iterations += 1 + return iterations + + +def exec_retry(executor: Any, args: Mapping[str, Any]) -> Any: + """Execute ``body`` with retries; raise after exhausting attempts.""" + max_attempts = max(int(args.get("max_attempts", 3)), 1) + backoff = float(args.get("backoff", 0.5)) + body = args.get("body") or [] + last_error: Optional[BaseException] = None + for attempt in range(max_attempts): + try: + return _run_strict(executor, body) + except (AutoControlActionException, OSError, RuntimeError, + AttributeError, TypeError, ValueError) as error: + last_error = error + autocontrol_logger.info( + "AC_retry attempt %d/%d failed: %s", + attempt + 1, max_attempts, repr(error) + ) + if attempt + 1 < max_attempts: + time.sleep(backoff * (2 ** attempt)) + raise AutoControlActionException( + f"AC_retry exhausted after {max_attempts} attempts" + ) from last_error + + +def exec_break(executor: Any, args: Mapping[str, Any]) -> None: + """Signal the innermost loop to stop.""" + del executor, args + raise LoopBreak() + + +def exec_continue(executor: Any, args: Mapping[str, Any]) -> None: + """Signal the innermost loop to advance to the next iteration.""" + del executor, args + raise LoopContinue() + + +BLOCK_COMMANDS: Dict[str, Callable[[Any, Mapping[str, Any]], Any]] = { + "AC_if_image_found": exec_if_image_found, + "AC_if_pixel": exec_if_pixel, + "AC_wait_image": exec_wait_image, + "AC_wait_pixel": exec_wait_pixel, + "AC_sleep": exec_sleep, + "AC_loop": exec_loop, + "AC_while_image": exec_while_image, + "AC_retry": exec_retry, + "AC_break": exec_break, + "AC_continue": exec_continue, +} diff --git a/je_auto_control/utils/hotkey/__init__.py b/je_auto_control/utils/hotkey/__init__.py new file mode 100644 index 0000000..a2634a9 --- /dev/null +++ b/je_auto_control/utils/hotkey/__init__.py @@ -0,0 +1,6 @@ +"""Global hotkey daemon — binds OS-level hotkeys to action JSON files.""" +from je_auto_control.utils.hotkey.hotkey_daemon import ( + HotkeyDaemon, HotkeyBinding, default_hotkey_daemon, +) + +__all__ = ["HotkeyDaemon", "HotkeyBinding", "default_hotkey_daemon"] diff --git a/je_auto_control/utils/hotkey/hotkey_daemon.py b/je_auto_control/utils/hotkey/hotkey_daemon.py new file mode 100644 index 0000000..574f664 --- /dev/null +++ b/je_auto_control/utils/hotkey/hotkey_daemon.py @@ -0,0 +1,211 @@ +"""Global hotkey daemon. + +Windows implementation uses ``RegisterHotKey`` + a dedicated message pump +thread. macOS / Linux raise ``NotImplementedError`` for now — the Strategy +pattern keeps the public API stable so backends can be added later. + +Usage:: + + from je_auto_control import default_hotkey_daemon + default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") + default_hotkey_daemon.start() +""" +import sys +import threading +import uuid +from dataclasses import dataclass +from typing import Callable, Dict, List, Optional, Tuple + +from je_auto_control.utils.json.json_file import read_action_json +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +MOD_ALT = 0x0001 +MOD_CONTROL = 0x0002 +MOD_SHIFT = 0x0004 +MOD_WIN = 0x0008 +MOD_NOREPEAT = 0x4000 + +_MODIFIER_MAP = { + "ctrl": MOD_CONTROL, "control": MOD_CONTROL, + "alt": MOD_ALT, + "shift": MOD_SHIFT, + "win": MOD_WIN, "super": MOD_WIN, "meta": MOD_WIN, +} + + +@dataclass +class HotkeyBinding: + """One registered hotkey → script binding.""" + binding_id: str + combo: str + script_path: str + enabled: bool = True + fired: int = 0 + + +def parse_combo(combo: str) -> Tuple[int, int]: + """Parse ``"ctrl+alt+1"`` into ``(modifiers, virtual_key_code)``.""" + if not combo or not combo.strip(): + raise ValueError("hotkey combo is empty") + parts = [p.strip().lower() for p in combo.split("+") if p.strip()] + if not parts: + raise ValueError(f"invalid hotkey combo: {combo!r}") + modifiers = 0 + key_part: Optional[str] = None + for part in parts: + if part in _MODIFIER_MAP: + modifiers |= _MODIFIER_MAP[part] + else: + if key_part is not None: + raise ValueError(f"hotkey {combo!r} has multiple non-modifier keys") + key_part = part + if key_part is None: + raise ValueError(f"hotkey {combo!r} is missing a primary key") + return modifiers | MOD_NOREPEAT, _key_to_vk(key_part) + + +def _key_to_vk(key: str) -> int: + if len(key) == 1: + return ord(key.upper()) + table = { + "f1": 0x70, "f2": 0x71, "f3": 0x72, "f4": 0x73, "f5": 0x74, + "f6": 0x75, "f7": 0x76, "f8": 0x77, "f9": 0x78, "f10": 0x79, + "f11": 0x7A, "f12": 0x7B, + "space": 0x20, "enter": 0x0D, "return": 0x0D, + "tab": 0x09, "escape": 0x1B, "esc": 0x1B, + "left": 0x25, "up": 0x26, "right": 0x27, "down": 0x28, + "home": 0x24, "end": 0x23, "insert": 0x2D, "delete": 0x2E, + "pageup": 0x21, "pagedown": 0x22, + } + lowered = key.lower() + if lowered in table: + return table[lowered] + raise ValueError(f"unsupported hotkey key: {key!r}") + + +class HotkeyDaemon: + """Register OS-level hotkeys and run their action JSON on trigger.""" + + def __init__(self, + executor: Optional[Callable[[list], object]] = None) -> None: + from je_auto_control.utils.executor.action_executor import execute_action + self._execute = executor or execute_action + self._bindings: Dict[str, HotkeyBinding] = {} + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + self._stop = threading.Event() + self._pending_register: List[HotkeyBinding] = [] + self._id_counter = 100 + self._registered_ids: Dict[str, int] = {} + + def bind(self, combo: str, script_path: str, + binding_id: Optional[str] = None) -> HotkeyBinding: + """Register a hotkey → script binding. Safe to call before/after start.""" + parse_combo(combo) + bid = binding_id or uuid.uuid4().hex[:8] + binding = HotkeyBinding( + binding_id=bid, combo=combo, script_path=script_path, + ) + with self._lock: + self._bindings[bid] = binding + self._pending_register.append(binding) + return binding + + def unbind(self, binding_id: str) -> bool: + with self._lock: + return self._bindings.pop(binding_id, None) is not None + + def list_bindings(self) -> List[HotkeyBinding]: + with self._lock: + return list(self._bindings.values()) + + def start(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + if not sys.platform.startswith("win"): + raise NotImplementedError( + "HotkeyDaemon currently supports Windows only" + ) + self._stop.clear() + self._thread = threading.Thread( + target=self._run_win, daemon=True, name="AutoControlHotkey", + ) + self._thread.start() + + def stop(self, timeout: float = 2.0) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=timeout) + self._thread = None + + # --- Windows backend ----------------------------------------------------- + + def _run_win(self) -> None: + import ctypes + from ctypes import wintypes + + user32 = ctypes.WinDLL("user32", use_last_error=True) + user32.RegisterHotKey.argtypes = [ + wintypes.HWND, ctypes.c_int, wintypes.UINT, wintypes.UINT, + ] + user32.RegisterHotKey.restype = wintypes.BOOL + user32.UnregisterHotKey.argtypes = [wintypes.HWND, ctypes.c_int] + user32.PeekMessageW.argtypes = [ + ctypes.POINTER(wintypes.MSG), wintypes.HWND, + wintypes.UINT, wintypes.UINT, wintypes.UINT, + ] + user32.PeekMessageW.restype = wintypes.BOOL + + msg = wintypes.MSG() + wm_hotkey = 0x0312 + pm_remove = 0x0001 + + self._drain_pending(user32) + while not self._stop.is_set(): + self._drain_pending(user32) + while user32.PeekMessageW(ctypes.byref(msg), None, 0, 0, pm_remove): + if msg.message == wm_hotkey: + self._handle_win_hotkey(msg.wParam) + self._stop.wait(0.05) + + # cleanup registrations + for registered_id in list(self._registered_ids.values()): + user32.UnregisterHotKey(None, registered_id) + self._registered_ids.clear() + + def _drain_pending(self, user32) -> None: + with self._lock: + pending = list(self._pending_register) + self._pending_register.clear() + for binding in pending: + modifiers, vk = parse_combo(binding.combo) + self._id_counter += 1 + reg_id = self._id_counter + if user32.RegisterHotKey(None, reg_id, modifiers, vk): + self._registered_ids[binding.binding_id] = reg_id + else: + autocontrol_logger.error( + "RegisterHotKey failed for %s (%s)", + binding.combo, binding.binding_id, + ) + + def _handle_win_hotkey(self, registered_id: int) -> None: + match: Optional[HotkeyBinding] = None + with self._lock: + for bid, reg_id in self._registered_ids.items(): + if reg_id == registered_id: + match = self._bindings.get(bid) + break + if match is None or not match.enabled: + return + try: + actions = read_action_json(match.script_path) + self._execute(actions) + except (OSError, ValueError, RuntimeError) as error: + autocontrol_logger.error("hotkey %s failed: %r", + match.combo, error) + match.fired += 1 + + +default_hotkey_daemon = HotkeyDaemon() diff --git a/je_auto_control/utils/ocr/__init__.py b/je_auto_control/utils/ocr/__init__.py new file mode 100644 index 0000000..ba3c00c --- /dev/null +++ b/je_auto_control/utils/ocr/__init__.py @@ -0,0 +1,10 @@ +"""OCR helpers for locating and interacting with on-screen text.""" +from je_auto_control.utils.ocr.ocr_engine import ( + TextMatch, click_text, find_text_matches, locate_text_center, + set_tesseract_cmd, wait_for_text, +) + +__all__ = [ + "TextMatch", "click_text", "find_text_matches", "locate_text_center", + "set_tesseract_cmd", "wait_for_text", +] diff --git a/je_auto_control/utils/ocr/ocr_engine.py b/je_auto_control/utils/ocr/ocr_engine.py new file mode 100644 index 0000000..e3f9992 --- /dev/null +++ b/je_auto_control/utils/ocr/ocr_engine.py @@ -0,0 +1,159 @@ +"""Headless OCR wrapper using ``pytesseract``. + +Text search can be restricted to a region to reduce CPU cost. The Tesseract +binary is loaded lazily; if it is missing, a clear ``RuntimeError`` is raised +rather than ``ImportError`` so callers can degrade gracefully. +""" +import time +from dataclasses import dataclass +from typing import List, Optional, Sequence, Tuple, Union + +from je_auto_control.utils.exception.exceptions import AutoControlActionException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +_pytesseract = None +_image_grab = None + + +def _load_backend(): + """Import pytesseract + PIL.ImageGrab lazily; raise helpful error if missing.""" + global _pytesseract, _image_grab + if _pytesseract is not None: + return _pytesseract, _image_grab + try: + import pytesseract as pt + from PIL import ImageGrab + except ImportError as error: + raise RuntimeError( + "OCR requires 'pytesseract' and a Tesseract binary. " + "Install with: pip install pytesseract" + ) from error + _pytesseract = pt + _image_grab = ImageGrab + return pt, ImageGrab + + +@dataclass(frozen=True) +class TextMatch: + """One OCR hit with absolute screen coordinates.""" + text: str + x: int + y: int + width: int + height: int + confidence: float + + @property + def center(self) -> Tuple[int, int]: + return self.x + self.width // 2, self.y + self.height // 2 + + +def set_tesseract_cmd(path: str) -> None: + """Override the Tesseract executable path (useful on Windows).""" + pt, _ = _load_backend() + pt.pytesseract.tesseract_cmd = path + + +def _grab(region: Optional[Sequence[int]]): + _, image_grab = _load_backend() + if region is None: + return image_grab.grab(all_screens=True), 0, 0 + x, y, w, h = region + bbox = (int(x), int(y), int(x) + int(w), int(y) + int(h)) + return image_grab.grab(bbox=bbox, all_screens=True), int(x), int(y) + + +def _parse_matches(data: dict, offset_x: int, offset_y: int, + min_confidence: float) -> List[TextMatch]: + """Convert ``image_to_data`` dict into TextMatch records.""" + matches: List[TextMatch] = [] + count = len(data.get("text", [])) + for i in range(count): + text = (data["text"][i] or "").strip() + if not text: + continue + try: + conf = float(data["conf"][i]) + except (TypeError, ValueError): + conf = -1.0 + if conf < min_confidence: + continue + matches.append(TextMatch( + text=text, + x=int(data["left"][i]) + offset_x, + y=int(data["top"][i]) + offset_y, + width=int(data["width"][i]), + height=int(data["height"][i]), + confidence=conf, + )) + return matches + + +def find_text_matches(target: str, + lang: str = "eng", + region: Optional[Sequence[int]] = None, + min_confidence: float = 60.0, + case_sensitive: bool = False) -> List[TextMatch]: + """Return every on-screen match for ``target`` as TextMatch records.""" + pt, _ = _load_backend() + frame, offset_x, offset_y = _grab(region) + try: + data = pt.image_to_data(frame, lang=lang, output_type=pt.Output.DICT) + except (OSError, RuntimeError) as error: + raise RuntimeError( + "Tesseract binary not found. Install it and/or call set_tesseract_cmd()." + ) from error + + needle = target if case_sensitive else target.lower() + matches = _parse_matches(data, offset_x, offset_y, min_confidence) + return [m for m in matches + if (m.text if case_sensitive else m.text.lower()) == needle + or needle in (m.text if case_sensitive else m.text.lower())] + + +def locate_text_center(target: str, + lang: str = "eng", + region: Optional[Sequence[int]] = None, + min_confidence: float = 60.0, + case_sensitive: bool = False) -> Tuple[int, int]: + """Return the centre (x, y) of the first match; raise if not found.""" + hits = find_text_matches(target, lang, region, min_confidence, case_sensitive) + if not hits: + raise AutoControlActionException(f"OCR: text not found: {target!r}") + return hits[0].center + + +def wait_for_text(target: str, + lang: str = "eng", + region: Optional[Sequence[int]] = None, + timeout: float = 10.0, + poll: float = 0.5, + min_confidence: float = 60.0, + case_sensitive: bool = False) -> Tuple[int, int]: + """Poll until ``target`` appears on screen; raise on timeout.""" + poll = max(0.05, float(poll)) + deadline = time.monotonic() + float(timeout) + while time.monotonic() < deadline: + try: + return locate_text_center(target, lang, region, min_confidence, + case_sensitive) + except AutoControlActionException: + time.sleep(poll) + raise AutoControlActionException(f"OCR: wait_for_text timeout: {target!r}") + + +def click_text(target: str, + mouse_keycode: Union[int, str] = "mouse_left", + lang: str = "eng", + region: Optional[Sequence[int]] = None, + min_confidence: float = 60.0, + case_sensitive: bool = False) -> Tuple[int, int]: + """Locate ``target`` text and click its centre.""" + # Import here to avoid circular import when executor loads this module. + from je_auto_control.wrapper.auto_control_mouse import click_mouse, set_mouse_position + center = locate_text_center(target, lang, region, min_confidence, case_sensitive) + set_mouse_position(*center) + click_mouse(mouse_keycode) + autocontrol_logger.info("click_text %r @ %s", target, center) + return center diff --git a/je_auto_control/utils/plugin_loader/__init__.py b/je_auto_control/utils/plugin_loader/__init__.py new file mode 100644 index 0000000..cc990ce --- /dev/null +++ b/je_auto_control/utils/plugin_loader/__init__.py @@ -0,0 +1,8 @@ +"""Discover external Python plugins and register their AC_ callables.""" +from je_auto_control.utils.plugin_loader.plugin_loader import ( + discover_plugin_commands, load_plugin_directory, load_plugin_file, +) + +__all__ = [ + "discover_plugin_commands", "load_plugin_directory", "load_plugin_file", +] diff --git a/je_auto_control/utils/plugin_loader/plugin_loader.py b/je_auto_control/utils/plugin_loader/plugin_loader.py new file mode 100644 index 0000000..456282d --- /dev/null +++ b/je_auto_control/utils/plugin_loader/plugin_loader.py @@ -0,0 +1,77 @@ +"""Load ``AC_*`` callables from user-supplied Python plugin files. + +A plugin file is any ``*.py`` containing top-level functions whose names +start with ``AC_``. Each such callable is registered into the executor's +``event_dict`` under its function name, so it becomes usable from JSON +action files and the socket/REST servers without any further plumbing. + +Security: plugin files execute arbitrary Python — only load from a trusted +directory under the user's control. +""" +import importlib.util +import os +import pathlib +import uuid +from types import ModuleType +from typing import Dict, List + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +def load_plugin_file(path: str) -> Dict[str, callable]: + """Import ``path`` and return a mapping of ``AC_*`` callables it defines.""" + resolved = os.path.realpath(path) + if not os.path.isfile(resolved): + raise FileNotFoundError(f"plugin file not found: {resolved}") + module = _import_isolated_module(resolved) + return discover_plugin_commands(module) + + +def load_plugin_directory(directory: str) -> Dict[str, callable]: + """Load every ``*.py`` in ``directory`` and merge their AC_* callables.""" + root = pathlib.Path(os.path.realpath(directory)) + if not root.is_dir(): + raise NotADirectoryError(f"plugin directory not found: {root}") + merged: Dict[str, callable] = {} + for file_path in sorted(root.glob("*.py")): + if file_path.name.startswith("_"): + continue + try: + commands = load_plugin_file(str(file_path)) + except (OSError, ImportError, SyntaxError) as error: + autocontrol_logger.error("plugin %s failed to load: %r", + file_path, error) + continue + merged.update(commands) + return merged + + +def discover_plugin_commands(module: ModuleType) -> Dict[str, callable]: + """Return every ``AC_*`` callable defined on ``module``.""" + commands: Dict[str, callable] = {} + for attr_name in dir(module): + if not attr_name.startswith("AC_"): + continue + attr = getattr(module, attr_name) + if callable(attr): + commands[attr_name] = attr + return commands + + +def register_plugin_commands(commands: Dict[str, callable]) -> List[str]: + """Register ``commands`` into the global executor and return their names.""" + from je_auto_control.utils.executor.action_executor import executor + for name, func in commands.items(): + executor.event_dict[name] = func + return sorted(commands.keys()) + + +def _import_isolated_module(file_path: str) -> ModuleType: + """Import a .py file without touching ``sys.modules`` namespace collisions.""" + module_name = f"je_auto_control_plugin_{uuid.uuid4().hex}" + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load plugin spec for {file_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/je_auto_control/utils/recording_edit/__init__.py b/je_auto_control/utils/recording_edit/__init__.py new file mode 100644 index 0000000..f4ec9ab --- /dev/null +++ b/je_auto_control/utils/recording_edit/__init__.py @@ -0,0 +1,10 @@ +"""Helpers for editing recorded action lists without re-recording.""" +from je_auto_control.utils.recording_edit.editor import ( + adjust_delays, filter_actions, insert_action, remove_action, + scale_coordinates, trim_actions, +) + +__all__ = [ + "adjust_delays", "filter_actions", "insert_action", "remove_action", + "scale_coordinates", "trim_actions", +] diff --git a/je_auto_control/utils/recording_edit/editor.py b/je_auto_control/utils/recording_edit/editor.py new file mode 100644 index 0000000..0314132 --- /dev/null +++ b/je_auto_control/utils/recording_edit/editor.py @@ -0,0 +1,82 @@ +"""Pure-Python helpers for editing recorded action lists. + +All functions return new lists rather than mutating the input so callers can +preserve the original recording. +""" +from typing import Callable, List + + +def trim_actions(actions: List[list], start: int = 0, end: int = None + ) -> List[list]: + """Return a slice ``actions[start:end]`` (end=None means to the end).""" + return list(actions[start:end]) + + +def insert_action(actions: List[list], index: int, new_action: list + ) -> List[list]: + """Return a new list with ``new_action`` inserted at ``index``.""" + result = list(actions) + if index < 0 or index > len(result): + raise IndexError(f"insert_action: index {index} out of range") + result.insert(index, new_action) + return result + + +def remove_action(actions: List[list], index: int) -> List[list]: + """Return a new list with the entry at ``index`` removed.""" + if index < 0 or index >= len(actions): + raise IndexError(f"remove_action: index {index} out of range") + return actions[:index] + actions[index + 1:] + + +def filter_actions(actions: List[list], + predicate: Callable[[list], bool]) -> List[list]: + """Return only actions for which ``predicate(action)`` is true.""" + return [action for action in actions if predicate(action)] + + +def adjust_delays(actions: List[list], factor: float = 1.0, + clamp_ms: int = 0) -> List[list]: + """Scale every ``AC_sleep`` delay by ``factor`` (and clamp to ``clamp_ms``). + + :param factor: multiplier for ``seconds`` values (<1 speeds up). + :param clamp_ms: floor for each resulting delay, in milliseconds. + """ + floor_seconds = max(0.0, float(clamp_ms) / 1000.0) + adjusted: List[list] = [] + for action in actions: + if _is_sleep(action): + original = float(action[1].get("seconds", 0.0)) + new_seconds = max(floor_seconds, original * float(factor)) + params = dict(action[1]) + params["seconds"] = new_seconds + adjusted.append([action[0], params]) + else: + adjusted.append(action) + return adjusted + + +def scale_coordinates(actions: List[list], + x_factor: float, y_factor: float) -> List[list]: + """Multiply every ``x`` / ``y`` parameter; useful when replaying on a new resolution.""" + scaled: List[list] = [] + for action in actions: + if len(action) < 2 or not isinstance(action[1], dict): + scaled.append(action) + continue + params = dict(action[1]) + if "x" in params and isinstance(params["x"], (int, float)): + params["x"] = int(round(params["x"] * x_factor)) + if "y" in params and isinstance(params["y"], (int, float)): + params["y"] = int(round(params["y"] * y_factor)) + scaled.append([action[0], params]) + return scaled + + +def _is_sleep(action: list) -> bool: + return ( + isinstance(action, list) + and len(action) == 2 + and action[0] == "AC_sleep" + and isinstance(action[1], dict) + ) diff --git a/je_auto_control/utils/rest_api/__init__.py b/je_auto_control/utils/rest_api/__init__.py new file mode 100644 index 0000000..38e6653 --- /dev/null +++ b/je_auto_control/utils/rest_api/__init__.py @@ -0,0 +1,6 @@ +"""Stdlib-based REST server mirroring the TCP socket server.""" +from je_auto_control.utils.rest_api.rest_server import ( + RestApiServer, start_rest_api_server, +) + +__all__ = ["RestApiServer", "start_rest_api_server"] diff --git a/je_auto_control/utils/rest_api/rest_server.py b/je_auto_control/utils/rest_api/rest_server.py new file mode 100644 index 0000000..bd59f07 --- /dev/null +++ b/je_auto_control/utils/rest_api/rest_server.py @@ -0,0 +1,135 @@ +"""Simple REST API server using stdlib ``http.server``. + +Endpoints:: + + GET /health → {"status": "ok"} + POST /execute body=JSON → {"result": } + GET /jobs → list of scheduler jobs + +The server defaults to ``127.0.0.1`` and the caller must opt into binding +to ``0.0.0.0`` — matching the policy in CLAUDE.md. +""" +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, Optional, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +class _JSONHandler(BaseHTTPRequestHandler): + """Dispatch HTTP calls into executor / scheduler primitives.""" + + server_version = "AutoControlREST/1.0" + + # Suppress default stderr access logs — route through the project logger. + def log_message(self, fmt: str, *args: Any) -> None: + autocontrol_logger.info("rest-api %s - %s", + self.address_string(), fmt % args) + + def do_GET(self) -> None: # noqa: N802 # reason: stdlib API + if self.path == "/health": + self._send_json({"status": "ok"}) + return + if self.path == "/jobs": + self._send_json({"jobs": self._serialize_jobs()}) + return + self._send_json({"error": f"unknown path: {self.path}"}, status=404) + + def do_POST(self) -> None: # noqa: N802 # reason: stdlib API + if self.path != "/execute": + self._send_json({"error": f"unknown path: {self.path}"}, status=404) + return + payload = self._read_json_body() + if payload is None: + return + actions = payload.get("actions") if isinstance(payload, dict) else None + if actions is None: + self._send_json({"error": "missing 'actions' field"}, status=400) + return + try: + from je_auto_control.utils.executor.action_executor import execute_action + result = execute_action(actions) + except (OSError, RuntimeError, ValueError, TypeError) as error: + self._send_json({"error": repr(error)}, status=500) + return + self._send_json({"result": result}, default=str) + + # --- helpers ------------------------------------------------------------- + + def _read_json_body(self) -> Optional[Any]: + length = int(self.headers.get("Content-Length", "0") or "0") + if length <= 0 or length > 1_000_000: + self._send_json({"error": "invalid Content-Length"}, status=400) + return None + raw = self.rfile.read(length) + try: + return json.loads(raw.decode("utf-8")) + except (ValueError, UnicodeDecodeError) as error: + self._send_json({"error": f"invalid JSON: {error}"}, status=400) + return None + + def _send_json(self, payload: Dict[str, Any], status: int = 200, + default=None) -> None: + body = json.dumps(payload, ensure_ascii=False, default=default).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + @staticmethod + def _serialize_jobs() -> list: + from je_auto_control.utils.scheduler.scheduler import default_scheduler + return [ + { + "job_id": job.job_id, "script_path": job.script_path, + "interval_seconds": job.interval_seconds, + "is_cron": job.is_cron, "repeat": job.repeat, + "runs": job.runs, "enabled": job.enabled, + } + for job in default_scheduler.list_jobs() + ] + + +class RestApiServer: + """Thin wrapper that owns the HTTP server + its background thread.""" + + def __init__(self, host: str = "127.0.0.1", port: int = 9939) -> None: + self._address: Tuple[str, int] = (host, port) + self._server: Optional[ThreadingHTTPServer] = None + self._thread: Optional[threading.Thread] = None + + @property + def address(self) -> Tuple[str, int]: + return self._address + + def start(self) -> None: + if self._server is not None: + return + self._server = ThreadingHTTPServer(self._address, _JSONHandler) + self._address = self._server.server_address[:2] + self._thread = threading.Thread( + target=self._server.serve_forever, daemon=True, + name="AutoControlREST", + ) + self._thread.start() + autocontrol_logger.info("REST API listening on %s:%d", *self._address) + + def stop(self, timeout: float = 2.0) -> None: + if self._server is None: + return + self._server.shutdown() + self._server.server_close() + if self._thread is not None: + self._thread.join(timeout=timeout) + self._server = None + self._thread = None + + +def start_rest_api_server(host: str = "127.0.0.1", + port: int = 9939) -> RestApiServer: + """Start and return a ``RestApiServer``; convenience wrapper.""" + server = RestApiServer(host=host, port=port) + server.start() + return server diff --git a/je_auto_control/utils/scheduler/__init__.py b/je_auto_control/utils/scheduler/__init__.py new file mode 100644 index 0000000..4513ce9 --- /dev/null +++ b/je_auto_control/utils/scheduler/__init__.py @@ -0,0 +1,6 @@ +"""Simple interval-based scheduler for action JSON scripts.""" +from je_auto_control.utils.scheduler.scheduler import ( + ScheduledJob, Scheduler, default_scheduler, +) + +__all__ = ["ScheduledJob", "Scheduler", "default_scheduler"] diff --git a/je_auto_control/utils/scheduler/cron.py b/je_auto_control/utils/scheduler/cron.py new file mode 100644 index 0000000..899f17c --- /dev/null +++ b/je_auto_control/utils/scheduler/cron.py @@ -0,0 +1,109 @@ +"""Minimal cron-expression parser (5-field: minute hour dom month dow). + +Supports ``*``, comma-lists (``1,5,10``), step values (``*/5``) and ranges +(``1-4``). Enough for scheduling most automation jobs without pulling in +``croniter`` as a dependency. + +``dow``: 0=Sun, 6=Sat to match standard cron. +""" +import datetime as _dt +from dataclasses import dataclass +from typing import List, Set + + +_FIELD_BOUNDS = ( + (0, 59), # minute + (0, 23), # hour + (1, 31), # day of month + (1, 12), # month + (0, 6), # day of week (0=Sun) +) + + +@dataclass(frozen=True) +class CronExpression: + """Parsed five-field cron expression. + + Each slot is the set of allowed integers for that field. + """ + minutes: Set[int] + hours: Set[int] + days_of_month: Set[int] + months: Set[int] + days_of_week: Set[int] + + def matches(self, moment: _dt.datetime) -> bool: + """Return ``True`` if ``moment`` satisfies every slot.""" + # Python weekday(): Mon=0..Sun=6 → cron dow: Sun=0..Sat=6 + cron_dow = (moment.weekday() + 1) % 7 + return ( + moment.minute in self.minutes + and moment.hour in self.hours + and moment.day in self.days_of_month + and moment.month in self.months + and cron_dow in self.days_of_week + ) + + +def parse_cron(expression: str) -> CronExpression: + """Parse a five-field cron expression; raise ``ValueError`` on failure.""" + fields = expression.strip().split() + if len(fields) != 5: + raise ValueError( + f"cron expression must have 5 fields; got {len(fields)}: {expression!r}" + ) + slots = [ + _parse_field(fields[i], _FIELD_BOUNDS[i][0], _FIELD_BOUNDS[i][1]) + for i in range(5) + ] + return CronExpression( + minutes=slots[0], hours=slots[1], days_of_month=slots[2], + months=slots[3], days_of_week=slots[4], + ) + + +def next_match(expression: CronExpression, + after: _dt.datetime) -> _dt.datetime: + """Return the next ``datetime`` (minute-resolution) matching ``expression``. + + Searches up to one year ahead to keep the runtime bounded. + """ + moment = (after + _dt.timedelta(minutes=1)).replace(second=0, microsecond=0) + limit = moment + _dt.timedelta(days=366) + while moment < limit: + if expression.matches(moment): + return moment + moment += _dt.timedelta(minutes=1) + raise ValueError("cron expression has no match within 366 days") + + +def _parse_field(raw: str, lo: int, hi: int) -> Set[int]: + values: Set[int] = set() + for piece in raw.split(","): + values.update(_expand_piece(piece, lo, hi)) + return values + + +def _expand_piece(piece: str, lo: int, hi: int) -> List[int]: + step = 1 + if "/" in piece: + base, step_str = piece.split("/", 1) + step = int(step_str) + if step <= 0: + raise ValueError(f"cron step must be positive: {piece!r}") + else: + base = piece + + if base == "*": + start, stop = lo, hi + elif "-" in base: + start_str, stop_str = base.split("-", 1) + start, stop = int(start_str), int(stop_str) + else: + start = stop = int(base) + + if start < lo or stop > hi or start > stop: + raise ValueError( + f"cron field {piece!r} out of range [{lo}, {hi}]" + ) + return list(range(start, stop + 1, step)) diff --git a/je_auto_control/utils/scheduler/scheduler.py b/je_auto_control/utils/scheduler/scheduler.py new file mode 100644 index 0000000..df7db6e --- /dev/null +++ b/je_auto_control/utils/scheduler/scheduler.py @@ -0,0 +1,179 @@ +"""Thread-based scheduler for repeated or delayed execution of action JSON files. + +Not a full cron — intentionally minimal: one-shot (run after N seconds) and +repeating (run every N seconds, optionally with a maximum run count). +""" +import datetime as _dt +import threading +import time +import uuid +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + +from je_auto_control.utils.json.json_file import read_action_json +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.scheduler.cron import ( + CronExpression, next_match, parse_cron, +) + + +@dataclass +class ScheduledJob: + """One scheduled execution entry. + + Either ``interval_seconds`` OR ``cron_expression`` drives firing — never both. + + :param job_id: unique identifier; auto-generated if empty. + :param script_path: path to an action JSON file to execute. + :param interval_seconds: delay before first run + between repeats (interval mode). + :param cron_expression: parsed cron rule (cron mode); ``None`` for interval jobs. + :param repeat: if False, run once then remove the job (interval mode only). + :param max_runs: optional cap on total runs (None = unlimited). + :param runs: number of times this job has executed. + :param enabled: paused jobs stay registered but skip firing. + :param next_run_ts: monotonic deadline (interval) or wall-clock epoch (cron). + """ + job_id: str + script_path: str + interval_seconds: float = 0.0 + cron_expression: Optional[CronExpression] = None + repeat: bool = True + max_runs: Optional[int] = None + runs: int = 0 + enabled: bool = True + next_run_ts: float = field(default=0.0) + + @property + def is_cron(self) -> bool: + return self.cron_expression is not None + + +class Scheduler: + """Thread-safe scheduler that polls jobs on a background thread.""" + + def __init__(self, executor: Optional[Callable[[list], object]] = None, + tick_seconds: float = 0.5) -> None: + from je_auto_control.utils.executor.action_executor import execute_action + self._execute = executor or execute_action + self._tick = max(0.1, float(tick_seconds)) + self._jobs: Dict[str, ScheduledJob] = {} + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + self._stop = threading.Event() + + def add_job(self, script_path: str, interval_seconds: float, + repeat: bool = True, max_runs: Optional[int] = None, + job_id: Optional[str] = None) -> ScheduledJob: + """Register and schedule a new interval job; return the record.""" + jid = job_id or uuid.uuid4().hex[:8] + now = time.monotonic() + interval = max(0.1, float(interval_seconds)) + job = ScheduledJob( + job_id=jid, script_path=script_path, + interval_seconds=interval, + repeat=repeat, max_runs=max_runs, + next_run_ts=now + interval, + ) + with self._lock: + self._jobs[jid] = job + autocontrol_logger.info("scheduler add_job %s %s", jid, script_path) + return job + + def add_cron_job(self, script_path: str, cron_expression: str, + max_runs: Optional[int] = None, + job_id: Optional[str] = None) -> ScheduledJob: + """Register a cron-driven job (5-field expression).""" + expression = parse_cron(cron_expression) + jid = job_id or uuid.uuid4().hex[:8] + now_wall = _dt.datetime.now() + next_at = next_match(expression, now_wall) + job = ScheduledJob( + job_id=jid, script_path=script_path, + interval_seconds=0.0, + cron_expression=expression, + repeat=True, max_runs=max_runs, + next_run_ts=next_at.timestamp(), + ) + with self._lock: + self._jobs[jid] = job + autocontrol_logger.info("scheduler add_cron_job %s %r -> %s", + jid, cron_expression, next_at.isoformat()) + return job + + def remove_job(self, job_id: str) -> bool: + with self._lock: + return self._jobs.pop(job_id, None) is not None + + def set_enabled(self, job_id: str, enabled: bool) -> bool: + with self._lock: + job = self._jobs.get(job_id) + if job is None: + return False + job.enabled = bool(enabled) + return True + + def list_jobs(self) -> List[ScheduledJob]: + with self._lock: + return list(self._jobs.values()) + + def start(self) -> None: + """Start the polling thread if it is not already running.""" + if self._thread is not None and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True, + name="AutoControlScheduler") + self._thread.start() + + def stop(self, timeout: float = 2.0) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=timeout) + self._thread = None + + def _run(self) -> None: + while not self._stop.is_set(): + self._tick_once() + self._stop.wait(self._tick) + + def _tick_once(self) -> None: + now_mono = time.monotonic() + now_wall = time.time() + due: List[ScheduledJob] = [] + with self._lock: + for job in self._jobs.values(): + if not job.enabled: + continue + deadline_now = now_wall if job.is_cron else now_mono + if deadline_now >= job.next_run_ts: + due.append(job) + for job in due: + self._fire(job, now_mono, now_wall) + + def _fire(self, job: ScheduledJob, now_mono: float, now_wall: float) -> None: + try: + actions = read_action_json(job.script_path) + self._execute(actions) + except (OSError, ValueError, RuntimeError) as error: + autocontrol_logger.error("scheduler job %s failed: %r", + job.job_id, error) + with self._lock: + live = self._jobs.get(job.job_id) + if live is None: + return + live.runs += 1 + if live.max_runs is not None and live.runs >= live.max_runs: + self._jobs.pop(job.job_id, None) + return + if live.is_cron: + next_dt = next_match(live.cron_expression, + _dt.datetime.fromtimestamp(now_wall)) + live.next_run_ts = next_dt.timestamp() + return + if not live.repeat: + self._jobs.pop(job.job_id, None) + return + live.next_run_ts = now_mono + live.interval_seconds + + +default_scheduler = Scheduler() diff --git a/je_auto_control/utils/script_vars/__init__.py b/je_auto_control/utils/script_vars/__init__.py new file mode 100644 index 0000000..7ca26c7 --- /dev/null +++ b/je_auto_control/utils/script_vars/__init__.py @@ -0,0 +1,6 @@ +"""Variable interpolation for action JSON scripts.""" +from je_auto_control.utils.script_vars.interpolate import ( + interpolate_actions, interpolate_value, load_vars_from_json, +) + +__all__ = ["interpolate_actions", "interpolate_value", "load_vars_from_json"] diff --git a/je_auto_control/utils/script_vars/interpolate.py b/je_auto_control/utils/script_vars/interpolate.py new file mode 100644 index 0000000..baba464 --- /dev/null +++ b/je_auto_control/utils/script_vars/interpolate.py @@ -0,0 +1,63 @@ +"""Substitute ``${var}`` placeholders in action JSON before execution. + +A placeholder that exactly matches ``${name}`` is replaced by the raw value +(preserving type — int stays int). A placeholder embedded in a larger string +falls back to string substitution, e.g. ``"x=${x}"`` → ``"x=42"``. + +Unknown variables raise ``ValueError`` so mistakes fail fast rather than +silently executing with wrong values. +""" +import json +import re +from pathlib import Path +from typing import Any, Mapping, MutableMapping + +_PLACEHOLDER = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") + + +def interpolate_value(value: Any, variables: Mapping[str, Any]) -> Any: + """Recursively interpolate placeholders inside ``value``.""" + if isinstance(value, str): + return _interpolate_string(value, variables) + if isinstance(value, list): + return [interpolate_value(item, variables) for item in value] + if isinstance(value, dict): + return {k: interpolate_value(v, variables) for k, v in value.items()} + return value + + +def interpolate_actions(actions: list, variables: Mapping[str, Any]) -> list: + """Return a new action list with placeholders substituted.""" + return interpolate_value(actions, variables) + + +def _interpolate_string(text: str, variables: Mapping[str, Any]) -> Any: + exact = _PLACEHOLDER.fullmatch(text) + if exact is not None: + return _lookup(exact.group(1), variables) + return _PLACEHOLDER.sub( + lambda m: str(_lookup(m.group(1), variables)), text + ) + + +def _lookup(name: str, variables: Mapping[str, Any]) -> Any: + if name not in variables: + raise ValueError(f"Unknown variable: ${{{name}}}") + return variables[name] + + +def load_vars_from_json(path: str, + into: MutableMapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """Load a flat JSON object as a variable bag.""" + with open(Path(path), encoding="utf-8") as file: + data = json.load(file) + if not isinstance(data, dict): + raise ValueError(f"{path}: expected a JSON object of variables") + if into is None: + into = {} + for key, value in data.items(): + if not isinstance(key, str): + raise ValueError(f"{path}: variable names must be strings") + into[key] = value + return into diff --git a/je_auto_control/utils/triggers/__init__.py b/je_auto_control/utils/triggers/__init__.py new file mode 100644 index 0000000..2bd894b --- /dev/null +++ b/je_auto_control/utils/triggers/__init__.py @@ -0,0 +1,10 @@ +"""Event-driven trigger engine (image / window / pixel / file watchers).""" +from je_auto_control.utils.triggers.trigger_engine import ( + FilePathTrigger, ImageAppearsTrigger, PixelColorTrigger, TriggerEngine, + WindowAppearsTrigger, default_trigger_engine, +) + +__all__ = [ + "FilePathTrigger", "ImageAppearsTrigger", "PixelColorTrigger", + "TriggerEngine", "WindowAppearsTrigger", "default_trigger_engine", +] diff --git a/je_auto_control/utils/triggers/trigger_engine.py b/je_auto_control/utils/triggers/trigger_engine.py new file mode 100644 index 0000000..c9a116c --- /dev/null +++ b/je_auto_control/utils/triggers/trigger_engine.py @@ -0,0 +1,193 @@ +"""Poll-based trigger engine. + +Each trigger implements ``is_fired()`` and an action JSON path to execute +when the condition is met. A single background thread polls all active +triggers and invokes the executor when any fires. + +Triggers fire at most once by default — set ``repeat=True`` to re-arm +after each firing. +""" +import os +import threading +import time +import uuid +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional, Tuple + +from je_auto_control.utils.json.json_file import read_action_json +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +@dataclass +class _TriggerBase: + """Shared fields / behaviour for all triggers.""" + trigger_id: str + script_path: str + repeat: bool = False + enabled: bool = True + fired: int = 0 + cooldown_seconds: float = 0.5 + _last_fire: float = field(default=0.0) + + def is_fired(self) -> bool: # pragma: no cover - abstract + raise NotImplementedError + + def should_poll(self, now: float) -> bool: + return self.enabled and (now - self._last_fire) >= self.cooldown_seconds + + +@dataclass +class ImageAppearsTrigger(_TriggerBase): + """Fire when the template ``image_path`` is found on screen.""" + image_path: str = "" + threshold: float = 0.8 + + def is_fired(self) -> bool: + from je_auto_control.wrapper.auto_control_image import locate_image_center + try: + result = locate_image_center(self.image_path, self.threshold, False) + except (OSError, RuntimeError, ValueError): + return False + return result is not None + + +@dataclass +class WindowAppearsTrigger(_TriggerBase): + """Fire when any open window title contains ``title_substring``.""" + title_substring: str = "" + case_sensitive: bool = False + + def is_fired(self) -> bool: + from je_auto_control.wrapper.auto_control_window import find_window + try: + return find_window(self.title_substring, + case_sensitive=self.case_sensitive) is not None + except (NotImplementedError, OSError, RuntimeError): + return False + + +@dataclass +class PixelColorTrigger(_TriggerBase): + """Fire when pixel at ``(x, y)`` matches ``target_rgb`` within tolerance.""" + x: int = 0 + y: int = 0 + target_rgb: Tuple[int, int, int] = (0, 0, 0) + tolerance: int = 8 + + def is_fired(self) -> bool: + from je_auto_control.wrapper.auto_control_screen import get_pixel + try: + raw = get_pixel(int(self.x), int(self.y)) + except (OSError, RuntimeError, ValueError, TypeError): + return False + if raw is None or len(raw) < 3: + return False + return all(abs(int(raw[i]) - int(self.target_rgb[i])) <= self.tolerance + for i in range(3)) + + +@dataclass +class FilePathTrigger(_TriggerBase): + """Fire when ``watch_path`` mtime changes (created or modified).""" + watch_path: str = "" + _baseline: Optional[float] = None + + def is_fired(self) -> bool: + try: + mtime = os.path.getmtime(self.watch_path) + except OSError: + return False + if self._baseline is None: + self._baseline = mtime + return False + if mtime > self._baseline: + self._baseline = mtime + return True + return False + + +class TriggerEngine: + """Polls registered triggers on a background thread.""" + + def __init__(self, executor: Optional[Callable[[list], object]] = None, + tick_seconds: float = 0.25) -> None: + from je_auto_control.utils.executor.action_executor import execute_action + self._execute = executor or execute_action + self._tick = max(0.05, float(tick_seconds)) + self._triggers: Dict[str, _TriggerBase] = {} + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + self._stop = threading.Event() + + def add(self, trigger: _TriggerBase) -> _TriggerBase: + """Register ``trigger``; assigns an id when missing.""" + if not trigger.trigger_id: + trigger.trigger_id = uuid.uuid4().hex[:8] + with self._lock: + self._triggers[trigger.trigger_id] = trigger + return trigger + + def remove(self, trigger_id: str) -> bool: + with self._lock: + return self._triggers.pop(trigger_id, None) is not None + + def list_triggers(self) -> List[_TriggerBase]: + with self._lock: + return list(self._triggers.values()) + + def set_enabled(self, trigger_id: str, enabled: bool) -> bool: + with self._lock: + trigger = self._triggers.get(trigger_id) + if trigger is None: + return False + trigger.enabled = bool(enabled) + return True + + def start(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread( + target=self._run, daemon=True, name="AutoControlTriggers", + ) + self._thread.start() + + def stop(self, timeout: float = 2.0) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=timeout) + self._thread = None + + def _run(self) -> None: + while not self._stop.is_set(): + self._poll_once() + self._stop.wait(self._tick) + + def _poll_once(self) -> None: + now = time.monotonic() + with self._lock: + candidates = [t for t in self._triggers.values() + if t.should_poll(now)] + for trigger in candidates: + if not trigger.is_fired(): + continue + self._fire(trigger, now) + + def _fire(self, trigger: _TriggerBase, now: float) -> None: + try: + actions = read_action_json(trigger.script_path) + self._execute(actions) + except (OSError, ValueError, RuntimeError) as error: + autocontrol_logger.error("trigger %s failed: %r", + trigger.trigger_id, error) + with self._lock: + live = self._triggers.get(trigger.trigger_id) + if live is None: + return + live.fired += 1 + live._last_fire = now + if not live.repeat: + self._triggers.pop(trigger.trigger_id, None) + + +default_trigger_engine = TriggerEngine() diff --git a/je_auto_control/utils/watcher/__init__.py b/je_auto_control/utils/watcher/__init__.py new file mode 100644 index 0000000..f423592 --- /dev/null +++ b/je_auto_control/utils/watcher/__init__.py @@ -0,0 +1,6 @@ +"""Headless polling primitives for mouse position, pixel colour, and log tail.""" +from je_auto_control.utils.watcher.watcher import ( + LogTail, MouseWatcher, PixelWatcher, +) + +__all__ = ["LogTail", "MouseWatcher", "PixelWatcher"] diff --git a/je_auto_control/utils/watcher/watcher.py b/je_auto_control/utils/watcher/watcher.py new file mode 100644 index 0000000..8211458 --- /dev/null +++ b/je_auto_control/utils/watcher/watcher.py @@ -0,0 +1,72 @@ +"""Poll-based observers usable from a GUI timer or a standalone thread. + +Each watcher exposes a ``sample()`` method that returns the latest reading +without side effects, so the same primitive can drive a HUD or a headless +monitor script. +""" +import collections +import logging +import threading +from typing import Deque, List, Optional, Tuple + + +class MouseWatcher: + """Sample the current mouse position on demand.""" + + def sample(self) -> Tuple[int, int]: + """Return the current ``(x, y)``; raise ``RuntimeError`` on failure.""" + from je_auto_control.wrapper.auto_control_mouse import get_mouse_position + try: + x, y = get_mouse_position() + except (OSError, RuntimeError, ValueError, TypeError) as error: + raise RuntimeError(f"MouseWatcher.sample failed: {error!r}") from error + return int(x), int(y) + + +class PixelWatcher: + """Sample the pixel colour at an arbitrary coordinate.""" + + def sample(self, x: int, y: int) -> Optional[Tuple[int, int, int]]: + """Return ``(r, g, b)`` at ``(x, y)``, or ``None`` on failure.""" + from je_auto_control.wrapper.auto_control_screen import get_pixel + try: + raw = get_pixel(int(x), int(y)) + except (OSError, RuntimeError, ValueError, TypeError): + return None + if raw is None or len(raw) < 3: + return None + return int(raw[0]), int(raw[1]), int(raw[2]) + + +class LogTail(logging.Handler): + """A ring buffer of recent log messages, suitable for live display.""" + + def __init__(self, capacity: int = 200, + level: int = logging.INFO) -> None: + super().__init__(level=level) + self._buffer: Deque[str] = collections.deque(maxlen=max(10, int(capacity))) + self._lock = threading.Lock() + self.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) + + def emit(self, record: logging.LogRecord) -> None: + try: + text = self.format(record) + except (ValueError, TypeError): + text = record.getMessage() + with self._lock: + self._buffer.append(text) + + def snapshot(self) -> List[str]: + """Return a copy of the current buffer (oldest first).""" + with self._lock: + return list(self._buffer) + + def attach(self, logger: logging.Logger) -> None: + """Install this tail on ``logger`` (idempotent).""" + if self not in logger.handlers: + logger.addHandler(self) + + def detach(self, logger: logging.Logger) -> None: + """Remove this tail from ``logger``.""" + if self in logger.handlers: + logger.removeHandler(self) diff --git a/je_auto_control/wrapper/auto_control_window.py b/je_auto_control/wrapper/auto_control_window.py new file mode 100644 index 0000000..dcd4fcb --- /dev/null +++ b/je_auto_control/wrapper/auto_control_window.py @@ -0,0 +1,93 @@ +"""Cross-platform window management facade. + +On Windows, delegates to ``windows_window_manage`` (Win32 API). +On macOS / Linux, operations raise a clear ``NotImplementedError``. +""" +import sys +import time +from typing import List, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlActionException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +_IS_WINDOWS = sys.platform in ("win32", "cygwin", "msys") + + +def _require_windows() -> None: + if not _IS_WINDOWS: + raise NotImplementedError( + f"Window management is only implemented on Windows (got {sys.platform})" + ) + + +def list_windows() -> List[Tuple[int, str]]: + """Return a list of ``(hwnd, title)`` for every visible top-level window.""" + _require_windows() + from je_auto_control.windows.window import windows_window_manage as wm + return wm.get_all_window_hwnd() + + +def find_window(title_substring: str, + case_sensitive: bool = False) -> Optional[Tuple[int, str]]: + """Return the first window whose title contains ``title_substring``.""" + needle = title_substring if case_sensitive else title_substring.lower() + for hwnd, title in list_windows(): + haystack = title if case_sensitive else title.lower() + if needle in haystack: + return hwnd, title + return None + + +def focus_window(title_substring: str, case_sensitive: bool = False) -> int: + """Bring the first matching window to the foreground; return its hwnd.""" + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + raise AutoControlActionException( + f"focus_window: no window matches {title_substring!r}" + ) + hwnd, title = hit + from je_auto_control.windows.window import windows_window_manage as wm + wm.set_foreground_window(hwnd) + autocontrol_logger.info("focused window hwnd=%s title=%r", hwnd, title) + return hwnd + + +def wait_for_window(title_substring: str, + timeout: float = 10.0, + poll: float = 0.5, + case_sensitive: bool = False) -> int: + """Poll until a window with the given title appears; return its hwnd.""" + _require_windows() + poll = max(0.05, float(poll)) + deadline = time.monotonic() + float(timeout) + while time.monotonic() < deadline: + hit = find_window(title_substring, case_sensitive) + if hit is not None: + return hit[0] + time.sleep(poll) + raise AutoControlActionException( + f"wait_for_window timeout: {title_substring!r}" + ) + + +def close_window_by_title(title_substring: str, case_sensitive: bool = False) -> bool: + """Minimise the first matching window.""" + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return False + from je_auto_control.windows.window import windows_window_manage as wm + return wm.close_window(hit[0]) + + +def show_window_by_title(title_substring: str, cmd_show: int = 1, + case_sensitive: bool = False) -> bool: + """Show or restore a window (``cmd_show`` follows Win32 ShowWindow).""" + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return False + from je_auto_control.windows.window import windows_window_manage as wm + wm.show_window(hit[0], int(cmd_show)) + return True diff --git a/test/unit_test/flow_control/test_flow_control.py b/test/unit_test/flow_control/test_flow_control.py new file mode 100644 index 0000000..1ed1bc1 --- /dev/null +++ b/test/unit_test/flow_control/test_flow_control.py @@ -0,0 +1,149 @@ +"""Unit tests for flow-control executor commands and schema validation.""" +import time + +import pytest + +from je_auto_control.utils.exception.exceptions import ( + AutoControlActionException, +) +from je_auto_control.utils.executor.action_executor import Executor +from je_auto_control.utils.executor.action_schema import validate_actions + + +@pytest.fixture() +def executor_with_hooks(): + """Return a fresh Executor plus a mutable counter for assertions.""" + executor = Executor() + state = {"count": 0, "last_error": None} + + def noop(): + state["count"] += 1 + return state["count"] + + executor.event_dict["AC_noop"] = noop + return executor, state + + +def test_ac_loop_runs_body_exact_times(executor_with_hooks): + ex, state = executor_with_hooks + ex.execute_action([["AC_loop", {"times": 4, "body": [["AC_noop"]]}]]) + assert state["count"] == 4 + + +def test_ac_break_exits_loop_early(executor_with_hooks): + ex, state = executor_with_hooks + ex.execute_action([["AC_loop", {"times": 10, + "body": [["AC_noop"], ["AC_break"]]}]]) + assert state["count"] == 1 + + +def test_ac_continue_does_not_stop_loop(executor_with_hooks): + ex, state = executor_with_hooks + ex.execute_action([["AC_loop", {"times": 3, + "body": [["AC_continue"], ["AC_noop"]]}]]) + assert state["count"] == 0 + + +def test_ac_sleep_delays(executor_with_hooks): + ex, _ = executor_with_hooks + start = time.monotonic() + ex.execute_action([["AC_sleep", {"seconds": 0.05}]]) + assert time.monotonic() - start >= 0.04 + + +def test_ac_wait_image_times_out(monkeypatch, executor_with_hooks): + ex, _ = executor_with_hooks + from je_auto_control.utils.executor import flow_control + + monkeypatch.setattr(flow_control, "_image_present", lambda *a, **k: False) + result = ex.execute_action([ + ["AC_wait_image", {"image": "x.png", "timeout": 0.1, "poll": 0.01}], + ]) + # error is captured in the record dict, not raised + assert any("timeout" in repr(v).lower() for v in result.values()) + + +def test_ac_retry_succeeds_after_failures(executor_with_hooks): + ex, state = executor_with_hooks + attempts = {"n": 0} + + def flaky(): + attempts["n"] += 1 + if attempts["n"] < 3: + raise RuntimeError("not yet") + return "ok" + + ex.event_dict["AC_flaky"] = flaky + ex.execute_action([["AC_retry", { + "max_attempts": 5, "backoff": 0.001, "body": [["AC_flaky"]] + }]]) + assert attempts["n"] == 3 + + +def test_ac_retry_exhausts_and_records_error(executor_with_hooks): + ex, _ = executor_with_hooks + + def always_fail(): + raise RuntimeError("boom") + + ex.event_dict["AC_failer"] = always_fail + result = ex.execute_action([["AC_retry", { + "max_attempts": 2, "backoff": 0.001, "body": [["AC_failer"]] + }]]) + assert any("exhausted" in repr(v) for v in result.values()) + + +def test_schema_rejects_unknown_command(executor_with_hooks): + ex, _ = executor_with_hooks + with pytest.raises(AutoControlActionException): + ex.execute_action([["AC_does_not_exist", {}]]) + + +def test_schema_validates_nested_body(executor_with_hooks): + ex, _ = executor_with_hooks + with pytest.raises(AutoControlActionException): + ex.execute_action([["AC_loop", { + "times": 1, "body": [["AC_nonexistent"]] + }]]) + + +def test_validate_actions_accepts_valid_payload(): + validate_actions( + [["AC_loop", {"times": 1, "body": []}]], + {"AC_loop"}, + ) + + +def test_validate_actions_rejects_non_list(): + with pytest.raises(AutoControlActionException): + validate_actions("not a list", {"AC_loop"}) + + +def test_validate_actions_rejects_wrong_body_type(): + with pytest.raises(AutoControlActionException): + validate_actions( + [["AC_loop", {"times": 1, "body": "not a list"}]], + {"AC_loop"}, + ) + + +def test_if_image_found_selects_then_branch(monkeypatch, executor_with_hooks): + ex, state = executor_with_hooks + from je_auto_control.utils.executor import flow_control + + monkeypatch.setattr(flow_control, "_image_present", lambda *a, **k: True) + ex.execute_action([["AC_if_image_found", { + "image": "x.png", "then": [["AC_noop"]], "else": [] + }]]) + assert state["count"] == 1 + + +def test_if_image_found_selects_else_branch(monkeypatch, executor_with_hooks): + ex, state = executor_with_hooks + from je_auto_control.utils.executor import flow_control + + monkeypatch.setattr(flow_control, "_image_present", lambda *a, **k: False) + ex.execute_action([["AC_if_image_found", { + "image": "x.png", "then": [], "else": [["AC_noop"]] + }]]) + assert state["count"] == 1 diff --git a/test/unit_test/flow_control/test_step_model.py b/test/unit_test/flow_control/test_step_model.py new file mode 100644 index 0000000..02b514e --- /dev/null +++ b/test/unit_test/flow_control/test_step_model.py @@ -0,0 +1,39 @@ +"""Tests for Script Builder step model (de)serialisation.""" +from je_auto_control.gui.script_builder.step_model import ( + actions_to_steps, steps_to_actions, +) + + +def test_flat_action_roundtrip(): + actions = [ + ["AC_click_mouse", {"mouse_keycode": "mouse_left", "x": 10, "y": 20}], + ["AC_sleep", {"seconds": 0.5}], + ] + assert steps_to_actions(actions_to_steps(actions)) == actions + + +def test_nested_flow_roundtrip(): + actions = [ + ["AC_loop", {"times": 3, "body": [ + ["AC_type_keyboard", {"keycode": "a"}], + ]}], + ["AC_if_image_found", {"image": "x.png", "then": [ + ["AC_sleep", {"seconds": 1}], + ], "else": [ + ["AC_break"], + ]}], + ] + assert steps_to_actions(actions_to_steps(actions)) == actions + + +def test_step_label_includes_params(): + [step] = actions_to_steps([ + ["AC_click_mouse", {"mouse_keycode": "mouse_left", "x": 100}] + ]) + assert "Click Mouse" in step.label + assert "mouse_keycode=" in step.label + + +def test_no_params_serialises_to_name_only(): + actions = [["AC_break"]] + assert steps_to_actions(actions_to_steps(actions)) == actions diff --git a/test/unit_test/headless/__init__.py b/test/unit_test/headless/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/unit_test/headless/test_cli.py b/test/unit_test/headless/test_cli.py new file mode 100644 index 0000000..c297492 --- /dev/null +++ b/test/unit_test/headless/test_cli.py @@ -0,0 +1,53 @@ +"""Tests for the CLI entry point (argument parsing + dry-run execution).""" +import json + +import pytest + +from je_auto_control.cli import _parse_vars, build_parser, main + + +def test_parse_vars_json_values(): + result = _parse_vars(["count=10", "name=\"alice\"", "active=true"]) + assert result == {"count": 10, "name": "alice", "active": True} + + +def test_parse_vars_falls_back_to_string(): + result = _parse_vars(["path=C:/Users/Name"]) + assert result == {"path": "C:/Users/Name"} + + +def test_parse_vars_rejects_missing_equals(): + with pytest.raises(SystemExit): + _parse_vars(["nope"]) + + +def test_parse_vars_accepts_none(): + assert _parse_vars(None) == {} + + +def test_build_parser_requires_subcommand(): + parser = build_parser() + with pytest.raises(SystemExit): + parser.parse_args([]) + + +def test_run_dry_run_does_not_invoke_actions(tmp_path, capsys): + script = tmp_path / "s.json" + script.write_text( + json.dumps([["AC_click_mouse", {"mouse_keycode": "mouse_left"}]]), + encoding="utf-8", + ) + rc = main(["run", str(script), "--dry-run"]) + assert rc == 0 + out = capsys.readouterr().out + payload = json.loads(out) + assert any("dry-run" in key for key in payload) + + +def test_list_jobs_prints_nothing_when_empty(capsys): + rc = main(["list-jobs"]) + assert rc == 0 + # default_scheduler is shared; we just check that output is tab-separated + out = capsys.readouterr().out + for line in out.splitlines(): + assert "\t" in line diff --git a/test/unit_test/headless/test_clipboard.py b/test/unit_test/headless/test_clipboard.py new file mode 100644 index 0000000..e58f040 --- /dev/null +++ b/test/unit_test/headless/test_clipboard.py @@ -0,0 +1,32 @@ +"""Round-trip tests for the headless clipboard.""" +import shutil +import sys + +import pytest + +from je_auto_control.utils.clipboard.clipboard import get_clipboard, set_clipboard + + +def _clipboard_available() -> bool: + if sys.platform.startswith("win"): + return True + if sys.platform == "darwin": + return shutil.which("pbcopy") is not None + return shutil.which("xclip") is not None or shutil.which("xsel") is not None + + +pytestmark = pytest.mark.skipif( + not _clipboard_available(), + reason="no clipboard backend available on this host", +) + + +def test_set_and_get_roundtrip(): + payload = "AutoControl clipboard 測試 🎯" + set_clipboard(payload) + assert get_clipboard() == payload + + +def test_set_clipboard_rejects_non_string(): + with pytest.raises(TypeError): + set_clipboard(123) diff --git a/test/unit_test/headless/test_cron.py b/test/unit_test/headless/test_cron.py new file mode 100644 index 0000000..835c975 --- /dev/null +++ b/test/unit_test/headless/test_cron.py @@ -0,0 +1,66 @@ +"""Tests for the cron-expression parser.""" +import datetime as dt + +import pytest + +from je_auto_control.utils.scheduler.cron import next_match, parse_cron + + +def test_parse_all_stars_matches_every_minute(): + expr = parse_cron("* * * * *") + now = dt.datetime(2026, 4, 18, 12, 30) + assert expr.matches(now) + assert expr.matches(now.replace(minute=0)) + assert expr.matches(now.replace(hour=0, minute=0)) + + +def test_parse_comma_list_and_step(): + expr = parse_cron("0,30 * * * *") + assert 0 in expr.minutes and 30 in expr.minutes + assert 15 not in expr.minutes + + every_five = parse_cron("*/5 * * * *") + assert 0 in every_five.minutes and 55 in every_five.minutes + assert 7 not in every_five.minutes + + +def test_parse_range(): + expr = parse_cron("* 9-17 * * *") + assert expr.hours == set(range(9, 18)) + + +def test_parse_rejects_wrong_field_count(): + with pytest.raises(ValueError): + parse_cron("* * *") + + +def test_parse_rejects_out_of_range(): + with pytest.raises(ValueError): + parse_cron("60 * * * *") + + +def test_parse_rejects_zero_step(): + with pytest.raises(ValueError): + parse_cron("*/0 * * * *") + + +def test_next_match_rolls_to_next_hour(): + expr = parse_cron("0 * * * *") # top of every hour + after = dt.datetime(2026, 4, 18, 12, 30) + nxt = next_match(expr, after) + assert nxt == dt.datetime(2026, 4, 18, 13, 0) + + +def test_next_match_honours_day_of_week(): + # 2026-04-18 is a Saturday (cron dow=6); Sunday is dow=0. + expr = parse_cron("0 9 * * 0") + after = dt.datetime(2026, 4, 18, 8, 0) + nxt = next_match(expr, after) + assert nxt == dt.datetime(2026, 4, 19, 9, 0) + + +def test_next_match_minute_resolution_skips_same_minute(): + expr = parse_cron("30 12 * * *") + after = dt.datetime(2026, 4, 18, 12, 30, 45) + nxt = next_match(expr, after) + assert nxt == dt.datetime(2026, 4, 19, 12, 30) diff --git a/test/unit_test/headless/test_hotkey_parse.py b/test/unit_test/headless/test_hotkey_parse.py new file mode 100644 index 0000000..501adaf --- /dev/null +++ b/test/unit_test/headless/test_hotkey_parse.py @@ -0,0 +1,52 @@ +"""Tests for the pure-logic parts of the hotkey daemon.""" +import pytest + +from je_auto_control.utils.hotkey.hotkey_daemon import ( + MOD_ALT, MOD_CONTROL, MOD_NOREPEAT, MOD_SHIFT, MOD_WIN, parse_combo, +) + + +def test_parse_single_modifier_and_letter(): + modifiers, vk = parse_combo("ctrl+alt+1") + assert modifiers & MOD_CONTROL + assert modifiers & MOD_ALT + assert modifiers & MOD_NOREPEAT + assert vk == ord("1") + + +def test_parse_is_case_and_whitespace_insensitive(): + modifiers, vk = parse_combo(" SHIFT + A ") + assert modifiers & MOD_SHIFT + assert vk == ord("A") + + +def test_parse_win_aliases(): + mods_win, _ = parse_combo("win+a") + mods_super, _ = parse_combo("super+a") + mods_meta, _ = parse_combo("meta+a") + assert mods_win == mods_super == mods_meta + + +def test_parse_function_key(): + _, vk = parse_combo("ctrl+f5") + assert vk == 0x74 + + +def test_parse_rejects_empty(): + with pytest.raises(ValueError): + parse_combo("") + + +def test_parse_rejects_two_primary_keys(): + with pytest.raises(ValueError): + parse_combo("a+b") + + +def test_parse_rejects_modifier_only(): + with pytest.raises(ValueError): + parse_combo("ctrl+alt") + + +def test_parse_rejects_unknown_key_name(): + with pytest.raises(ValueError): + parse_combo("ctrl+blorp") diff --git a/test/unit_test/headless/test_interpolate.py b/test/unit_test/headless/test_interpolate.py new file mode 100644 index 0000000..21318e7 --- /dev/null +++ b/test/unit_test/headless/test_interpolate.py @@ -0,0 +1,31 @@ +"""Tests for ${var} interpolation in action lists.""" +import pytest + +from je_auto_control.utils.script_vars.interpolate import ( + interpolate_actions, interpolate_value, +) + + +def test_exact_placeholder_preserves_type(): + assert interpolate_value("${x}", {"x": 42}) == 42 + assert interpolate_value("${x}", {"x": [1, 2]}) == [1, 2] + + +def test_embedded_placeholder_coerces_to_string(): + assert interpolate_value("x=${x}!", {"x": 42}) == "x=42!" + + +def test_unknown_variable_raises(): + with pytest.raises(ValueError): + interpolate_value("${missing}", {}) + + +def test_nested_actions_interpolated(): + actions = [["AC_click_mouse", {"x": "${px}", "y": "${py}"}]] + resolved = interpolate_actions(actions, {"px": 10, "py": 20}) + assert resolved == [["AC_click_mouse", {"x": 10, "y": 20}]] + + +def test_non_placeholder_passes_through(): + assert interpolate_value(123, {}) == 123 + assert interpolate_value("plain text", {}) == "plain text" diff --git a/test/unit_test/headless/test_ocr_engine.py b/test/unit_test/headless/test_ocr_engine.py new file mode 100644 index 0000000..e52b292 --- /dev/null +++ b/test/unit_test/headless/test_ocr_engine.py @@ -0,0 +1,30 @@ +"""Tests for the OCR parser logic (no real Tesseract binary required).""" +from je_auto_control.utils.ocr.ocr_engine import TextMatch, _parse_matches + + +def _sample_data(): + return { + "text": ["", "hello", "world", "low_conf"], + "conf": ["-1", "95.0", "88.0", "10.0"], + "left": [0, 10, 100, 50], + "top": [0, 20, 30, 40], + "width": [0, 40, 60, 30], + "height": [0, 15, 15, 15], + } + + +def test_parse_matches_skips_blank_and_low_conf(): + matches = _parse_matches(_sample_data(), 0, 0, min_confidence=60.0) + assert [m.text for m in matches] == ["hello", "world"] + + +def test_parse_matches_applies_offsets(): + matches = _parse_matches(_sample_data(), 100, 200, min_confidence=60.0) + # hello starts at (10, 20) -> (110, 220) + assert matches[0].x == 110 + assert matches[0].y == 220 + + +def test_text_match_center_is_midpoint(): + match = TextMatch(text="x", x=10, y=20, width=30, height=40, confidence=90.0) + assert match.center == (25, 40) diff --git a/test/unit_test/headless/test_plugin_loader.py b/test/unit_test/headless/test_plugin_loader.py new file mode 100644 index 0000000..4537ed0 --- /dev/null +++ b/test/unit_test/headless/test_plugin_loader.py @@ -0,0 +1,84 @@ +"""Tests for the plugin loader.""" +import pytest + +from je_auto_control.utils.plugin_loader.plugin_loader import ( + discover_plugin_commands, load_plugin_directory, load_plugin_file, + register_plugin_commands, +) + + +PLUGIN_SOURCE = """ +from_ac_prefix = "not a callable" + +def AC_hello(ctx=None): + return "hello" + +def AC_echo(args=None): + return args + +def not_exported(): + return "hidden" + +AC_non_callable = 42 +""" + + +def _write_plugin(tmp_path, name, body=PLUGIN_SOURCE): + file_path = tmp_path / name + file_path.write_text(body, encoding="utf-8") + return str(file_path) + + +def test_load_plugin_file_returns_ac_callables(tmp_path): + path = _write_plugin(tmp_path, "sample.py") + commands = load_plugin_file(path) + assert set(commands.keys()) == {"AC_hello", "AC_echo"} + assert commands["AC_hello"]() == "hello" + assert commands["AC_echo"]({"x": 1}) == {"x": 1} + + +def test_load_plugin_directory_merges_files_and_skips_underscore(tmp_path): + _write_plugin(tmp_path, "a.py", "def AC_a():\n return 'a'\n") + _write_plugin(tmp_path, "b.py", "def AC_b():\n return 'b'\n") + _write_plugin(tmp_path, "_private.py", "def AC_hidden():\n return 'x'\n") + commands = load_plugin_directory(str(tmp_path)) + assert set(commands.keys()) == {"AC_a", "AC_b"} + + +def test_load_plugin_file_raises_for_missing_file(tmp_path): + with pytest.raises(FileNotFoundError): + load_plugin_file(str(tmp_path / "missing.py")) + + +def test_load_plugin_directory_raises_when_not_a_dir(tmp_path): + path = tmp_path / "not_a_dir.py" + path.write_text("", encoding="utf-8") + with pytest.raises(NotADirectoryError): + load_plugin_directory(str(path)) + + +def test_register_plugin_commands_adds_and_removes_cleanly(tmp_path): + from je_auto_control.utils.executor.action_executor import executor + path = _write_plugin(tmp_path, "reg.py") + commands = load_plugin_file(path) + names = register_plugin_commands(commands) + try: + assert "AC_hello" in executor.event_dict + assert "AC_echo" in executor.event_dict + assert names == sorted(["AC_hello", "AC_echo"]) + finally: + for name in names: + executor.event_dict.pop(name, None) + + +def test_discover_ignores_non_callable_ac_attribute(): + class Module: + AC_value = 42 + + @staticmethod + def AC_run(): + return 1 + + found = discover_plugin_commands(Module) + assert "AC_value" not in found + assert "AC_run" in found diff --git a/test/unit_test/headless/test_recording_edit.py b/test/unit_test/headless/test_recording_edit.py new file mode 100644 index 0000000..deafddb --- /dev/null +++ b/test/unit_test/headless/test_recording_edit.py @@ -0,0 +1,56 @@ +"""Tests for the non-destructive recording editor helpers.""" +import pytest + +from je_auto_control.utils.recording_edit.editor import ( + adjust_delays, filter_actions, insert_action, remove_action, + scale_coordinates, trim_actions, +) + + +def _sample(): + return [ + ["AC_click_mouse", {"x": 100, "y": 200}], + ["AC_sleep", {"seconds": 1.0}], + ["AC_type_keyboard", {"keycode": "a"}], + ["AC_sleep", {"seconds": 0.2}], + ] + + +def test_trim_slices_returns_copy(): + actions = _sample() + result = trim_actions(actions, 1, 3) + assert len(result) == 2 + assert result[0][0] == "AC_sleep" + result.append(["AC_noop"]) + assert len(actions) == 4 + + +def test_insert_rejects_out_of_range(): + with pytest.raises(IndexError): + insert_action(_sample(), 99, ["AC_noop"]) + + +def test_remove_drops_index(): + result = remove_action(_sample(), 0) + assert result[0][0] == "AC_sleep" + + +def test_filter_keeps_predicate_matches(): + result = filter_actions(_sample(), + lambda action: action[0] != "AC_sleep") + assert [action[0] for action in result] == ["AC_click_mouse", "AC_type_keyboard"] + + +def test_adjust_delays_scales_and_clamps(): + result = adjust_delays(_sample(), factor=0.5, clamp_ms=300) + # 1.0 * 0.5 = 0.5 (above clamp of 0.3) + # 0.2 * 0.5 = 0.1 (clamped up to 0.3) + assert result[1][1]["seconds"] == pytest.approx(0.5) + assert result[3][1]["seconds"] == pytest.approx(0.3) + + +def test_scale_coordinates_multiplies_xy(): + result = scale_coordinates(_sample(), 2.0, 3.0) + assert result[0][1] == {"x": 200, "y": 600} + # non-coordinate actions untouched + assert result[2] == ["AC_type_keyboard", {"keycode": "a"}] diff --git a/test/unit_test/headless/test_rest_server.py b/test/unit_test/headless/test_rest_server.py new file mode 100644 index 0000000..8f37634 --- /dev/null +++ b/test/unit_test/headless/test_rest_server.py @@ -0,0 +1,61 @@ +"""Tests for the REST API server.""" +import json +import urllib.error +import urllib.request + +import pytest + +from je_auto_control.utils.rest_api.rest_server import RestApiServer + + +@pytest.fixture() +def rest_server(): + server = RestApiServer(host="127.0.0.1", port=0) + server.start() + yield server + server.stop(timeout=1.0) + + +def _request(server, path, method="GET", body=None): + host, port = server.address + url = f"http://{host}:{port}{path}" + data = None + headers = {} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(req, timeout=3) as response: # nosec B310 # reason: localhost test server + return response.status, json.loads(response.read().decode("utf-8")) + + +def test_health_endpoint(rest_server): + status, payload = _request(rest_server, "/health") + assert status == 200 + assert payload == {"status": "ok"} + + +def test_jobs_endpoint_returns_list(rest_server): + status, payload = _request(rest_server, "/jobs") + assert status == 200 + assert isinstance(payload.get("jobs"), list) + + +def test_execute_rejects_missing_actions(rest_server): + try: + _request(rest_server, "/execute", method="POST", body={}) + except urllib.error.HTTPError as error: + assert error.code == 400 + payload = json.loads(error.read().decode("utf-8")) + assert "actions" in payload.get("error", "") + else: + pytest.fail("expected 400 response") + + +def test_unknown_path_returns_404(rest_server): + try: + _request(rest_server, "/nope") + except urllib.error.HTTPError as error: + assert error.code == 404 + else: + pytest.fail("expected 404 response") diff --git a/test/unit_test/headless/test_scheduler.py b/test/unit_test/headless/test_scheduler.py new file mode 100644 index 0000000..5b8dd4a --- /dev/null +++ b/test/unit_test/headless/test_scheduler.py @@ -0,0 +1,67 @@ +"""Tests for the Scheduler headless module.""" +import time + +from je_auto_control.utils.scheduler.scheduler import Scheduler + + +def test_add_and_remove_job(): + calls = [] + sched = Scheduler(executor=lambda actions: calls.append(actions)) + job = sched.add_job("script.json", interval_seconds=5.0) + assert job.script_path == "script.json" + assert len(sched.list_jobs()) == 1 + assert sched.remove_job(job.job_id) is True + assert sched.list_jobs() == [] + + +def test_set_enabled_toggles_flag(): + sched = Scheduler(executor=lambda actions: None) + job = sched.add_job("s.json", 10.0) + assert sched.set_enabled(job.job_id, False) is True + assert sched.list_jobs()[0].enabled is False + assert sched.set_enabled("no-such-job", True) is False + + +def test_job_fires_and_updates_runs(monkeypatch): + executed = [] + sched = Scheduler( + executor=lambda actions: executed.append(actions), + tick_seconds=0.1, + ) + monkeypatch.setattr( + "je_auto_control.utils.scheduler.scheduler.read_action_json", + lambda path: [["AC_noop"]], + ) + job = sched.add_job("fake.json", interval_seconds=0.1, repeat=False) + sched.start() + try: + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and sched.list_jobs(): + time.sleep(0.05) + finally: + sched.stop(timeout=1.0) + assert executed, "executor should have been called at least once" + # Non-repeating job is removed after firing. + assert all(j.job_id != job.job_id for j in sched.list_jobs()) + + +def test_max_runs_cap(monkeypatch): + executed = [] + sched = Scheduler( + executor=lambda actions: executed.append(1), + tick_seconds=0.05, + ) + monkeypatch.setattr( + "je_auto_control.utils.scheduler.scheduler.read_action_json", + lambda path: [["AC_noop"]], + ) + sched.add_job("fake.json", interval_seconds=0.1, + repeat=True, max_runs=2) + sched.start() + try: + deadline = time.monotonic() + 3.0 + while time.monotonic() < deadline and sched.list_jobs(): + time.sleep(0.05) + finally: + sched.stop(timeout=1.0) + assert len(executed) == 2 diff --git a/test/unit_test/headless/test_trigger_engine.py b/test/unit_test/headless/test_trigger_engine.py new file mode 100644 index 0000000..8edfddd --- /dev/null +++ b/test/unit_test/headless/test_trigger_engine.py @@ -0,0 +1,81 @@ +"""Tests for the poll-based trigger engine (FilePath + engine plumbing).""" +import json +import os +import time + +from je_auto_control.utils.triggers.trigger_engine import ( + FilePathTrigger, TriggerEngine, +) + + +def _write_actions(path, actions): + path.write_text(json.dumps(actions), encoding="utf-8") + return str(path) + + +def test_file_path_trigger_fires_on_mtime_change(tmp_path): + watched = tmp_path / "sentinel.txt" + watched.write_text("v1", encoding="utf-8") + trigger = FilePathTrigger( + trigger_id="f1", script_path="unused.json", + watch_path=str(watched), + ) + assert trigger.is_fired() is False # baseline capture + # Force a mtime change by bumping the file's mtime forward. + later = watched.stat().st_mtime + 2 + os.utime(str(watched), (later, later)) + assert trigger.is_fired() is True + # Subsequent call with no change does not re-fire. + assert trigger.is_fired() is False + + +def test_engine_runs_trigger_once_when_non_repeat(tmp_path): + script_path = _write_actions(tmp_path / "s.json", [["AC_noop"]]) + watched = tmp_path / "w.txt" + watched.write_text("v1", encoding="utf-8") + + calls = [] + engine = TriggerEngine( + executor=lambda actions: calls.append(actions), + tick_seconds=0.05, + ) + trigger = FilePathTrigger( + trigger_id="", script_path=script_path, + watch_path=str(watched), repeat=False, + ) + engine.add(trigger) + engine.start() + try: + # First poll captures baseline; bump mtime to force firing. + time.sleep(0.1) + later = watched.stat().st_mtime + 2 + os.utime(str(watched), (later, later)) + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and not engine.list_triggers() == []: + time.sleep(0.05) + if calls: + break + finally: + engine.stop(timeout=1.0) + assert calls, "executor should have been invoked" + # Non-repeating trigger should be removed after firing. + assert all(t.trigger_id != trigger.trigger_id for t in engine.list_triggers()) + + +def test_engine_set_enabled_suppresses_polling(tmp_path): + watched = tmp_path / "w.txt" + watched.write_text("v1", encoding="utf-8") + engine = TriggerEngine(executor=lambda actions: None, tick_seconds=0.05) + trigger = FilePathTrigger( + trigger_id="t1", script_path="unused.json", + watch_path=str(watched), repeat=True, + ) + engine.add(trigger) + assert engine.set_enabled("t1", False) is True + assert engine.list_triggers()[0].enabled is False + assert engine.set_enabled("missing", True) is False + + +def test_engine_remove_returns_false_for_missing(): + engine = TriggerEngine(executor=lambda actions: None) + assert engine.remove("nope") is False diff --git a/test/unit_test/headless/test_watcher.py b/test/unit_test/headless/test_watcher.py new file mode 100644 index 0000000..8721436 --- /dev/null +++ b/test/unit_test/headless/test_watcher.py @@ -0,0 +1,58 @@ +"""Tests for the watcher module (MouseWatcher, PixelWatcher, LogTail).""" +import logging + +import pytest + +from je_auto_control.utils.watcher.watcher import ( + LogTail, MouseWatcher, PixelWatcher, +) + + +def test_log_tail_buffers_messages(): + tail = LogTail(capacity=10) + logger = logging.getLogger("je_auto_control.test_log_tail") + logger.setLevel(logging.INFO) + tail.attach(logger) + try: + for i in range(30): + logger.info("msg %d", i) + finally: + tail.detach(logger) + snap = tail.snapshot() + assert len(snap) == 10 + assert "msg 29" in snap[-1] + assert "msg 20" in snap[0] + + +def test_log_tail_attach_detach_idempotent(): + tail = LogTail() + logger = logging.getLogger("je_auto_control.test_attach") + tail.attach(logger) + tail.attach(logger) + assert logger.handlers.count(tail) == 1 + tail.detach(logger) + tail.detach(logger) + assert tail not in logger.handlers + + +def test_mouse_watcher_reports_failure(monkeypatch): + def broken(): + raise OSError("nope") + import je_auto_control.wrapper.auto_control_mouse as mouse_mod + monkeypatch.setattr(mouse_mod, "get_mouse_position", broken) + with pytest.raises(RuntimeError): + MouseWatcher().sample() + + +def test_pixel_watcher_returns_none_on_error(monkeypatch): + import je_auto_control.wrapper.auto_control_screen as screen_mod + monkeypatch.setattr(screen_mod, "get_pixel", + lambda x, y: (_ for _ in ()).throw(OSError("nope"))) + assert PixelWatcher().sample(0, 0) is None + + +def test_pixel_watcher_normalises_rgb(monkeypatch): + import je_auto_control.wrapper.auto_control_screen as screen_mod + monkeypatch.setattr(screen_mod, "get_pixel", + lambda x, y: (12, 34, 56, 255)) + assert PixelWatcher().sample(1, 1) == (12, 34, 56)