From 322aa0adbb80ad120cc33ef1a291d81bc803ae7a Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 25 Sep 2026 19:51:05 +0800 Subject: [PATCH 01/47] Decode Windows console output in its code page under Python 3.15's default UTF-8 mode, where the locale default says UTF-8, and skip distributions without metadata in the SBOM, which 3.15 raises for --- CHANGELOG.md | 6 ++ architecture_explore.md | 34 +++++------ .../Eng/doc/new_features/v4_features_doc.rst | 3 +- .../Zh/doc/new_features/v4_features_doc.rst | 2 +- docs/updates/2026-09-d.md | 17 ++++++ docs/updates/README.md | 3 +- .../utils/executor/flow_data_commands.py | 5 +- .../mcp_server/tools/_handlers_system.py | 8 +-- .../utils/remote_desktop/host_service.py | 4 ++ je_auto_control/utils/sbom/sbom.py | 28 ++++++++- .../utils/shell_process/shell_exec.py | 18 +++++- .../headless/test_console_encoding.py | 59 +++++++++++++++++++ .../headless/test_sbom_missing_metadata.py | 51 ++++++++++++++++ 13 files changed, 207 insertions(+), 31 deletions(-) create mode 100644 test/unit_test/headless/test_console_encoding.py create mode 100644 test/unit_test/headless/test_sbom_missing_metadata.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a9a8e1b8e..a0a11e277 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -427,6 +427,12 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's ### Fixed +- Python 3.15 turns UTF-8 mode on by default. `AC_shell_to_var`, the MCP + `shell_command` tool, `ShellManager` and the remote host's `status` still + decode a Windows console program's output in its code page there, + instead of as UTF-8: `sc query` raised, and the others returned + replacement characters. The SBOM skips a distribution that has no + metadata instead of failing on 3.15, or listing it as `unknown` before. - The Flow Editor opens action files saved with a BOM, keeps a wrapped file's other keys on save, and writes atomically. - The region selector (template cropping, OCR / screenshot / WebRTC regions) diff --git a/architecture_explore.md b/architecture_explore.md index 63987baf6..e7ec46e90 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,059 | -| 程式碼總行數 | 154,389 | +| 程式碼總行數 | 154,430 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 775 | | 套件門面 `__all__` 公開名稱數 | 1,244 | @@ -272,7 +272,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 14,553 行。 +> 24 個套件、約 14,552 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -283,7 +283,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/dag/` | 536 | 跨主機 DAG 編排器（圖模型 + runner） | | `utils/decision_table/` | 112 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | | `utils/deterministic/` | 116 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | -| `utils/executor/` | 9,504 | **核心**。`Executor` 指令分派表（775 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/executor/` | 9,503 | **核心**。`Executor` 指令分派表（775 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | | `utils/flow_debugger/` | 155 | action list 的單步除錯器與追蹤器 | | `utils/input_macro/` | 451 | 定時輸入事件：錄製結果的整形（`timeline`／`InputRecorder`，Windows 與 macOS 共用）、重播與宣告式輸入序列 DSL | | `utils/json/` | 99 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | @@ -303,7 +303,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.2 框架基礎設施 -> 14 個套件、約 3,014 行。 +> 14 個套件、約 3,030 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -319,7 +319,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/package_manager/` | 101 | 動態載入套件並把 executor 注入其中 | | `utils/path_guard/` | 114 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | | `utils/platform_id/` | 62 | 作業系統家族的單一判定點。`sys.platform` 原本在一百多處跟字面清單比對，而那些清單都沒有 BSD；`is_x11_unix()` 問的是「這是不是 X11 unix」，這才是守衛一直想問的問題 | -| `utils/shell_process/` | 263 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | +| `utils/shell_process/` | 279 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | | `utils/start_exe/` | 36 | 啟動另一個執行檔行程 | ### 5.4.3 排程、觸發與背景監看 @@ -514,14 +514,14 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 19,332 行。 +> 6 個套件、約 19,336 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/admin/` | 418 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | | `utils/config_sync/` | 332 | 透過訊令伺服器做跨機器設定同步 | | `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | -| `utils/remote_desktop/` | 12,912 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/remote_desktop/` | 12,916 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | | `utils/usb/` | 4,524 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | | `utils/usbip/` | 1,008 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | @@ -630,7 +630,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.14 安全、機密與合規 -> 13 個套件、約 2,913 行。 +> 13 個套件、約 2,935 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -641,7 +641,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/provenance/` | 126 | SLSA 建置來源證明（in-toto v1） | | `utils/rbac/` | 299 | 角色型存取控制：使用者、角色與權杖驗證（尚未接到 REST／MCP） | | `utils/redaction/` | 508 | 截圖遮蔽層：規則偵測 + 政策 + 協調器（上傳 VLM 前先遮） | -| `utils/sbom/` | 148 | SBOM（CycloneDX）產生 | +| `utils/sbom/` | 170 | SBOM（CycloneDX）產生 | | `utils/secret_ref/` | 143 | URI scheme 形式的值參照解析 | | `utils/secrets/` | 360 | 加密機密儲存庫，供 `${secrets.NAME}` 解析 | | `utils/secrets_scan/` | 138 | 掃描 action JSON／資料中應入庫卻硬編碼的機密 | @@ -696,13 +696,13 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 上表以子套件為單位；以下把行數最大的幾個子系統展開到檔案層。 -#### `utils/executor/`（9,504 行）— 執行核心 +#### `utils/executor/`（9,503 行）— 執行核心 | 檔案 | 行數 | 職責 | | --- | ---: | --- | | `action_executor.py` | 8,308 | `Executor` 類別與 `event_dict` 分派表（775 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | | `flow_control.py` | 643 | 真正的流程控制：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`／`AC_get_var`／`AC_inc_var`）。`LoopBreak`／`LoopContinue` 以例外實作。34 個區塊指令的分派表 `BLOCK_COMMANDS` 也在這裡，含下一列匯入的資料來源指令。 | -| `flow_data_commands.py` | 272 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | +| `flow_data_commands.py` | 271 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | | `action_schema.py` | 159 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | | `action_redaction.py` | 83 | 記錄與紀錄鍵用的遮蔽：`AC_secret_*` 的參數（金庫通行碼、機密值）在寫進 log、當成結果紀錄的鍵之前換成 `***`，巢狀在區塊指令裡的也一樣。 | | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | @@ -745,7 +745,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `rate_limit.py` | 48 | 工具呼叫的 token bucket 限流。 | | `__main__.py` | 92 | `je_auto_control_mcp` console script 進入點。 | -#### `utils/remote_desktop/`（12,912 行／56 檔） +#### `utils/remote_desktop/`（12,916 行／56 檔） 三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 @@ -755,7 +755,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `webrtc_viewer.py` | 677 | WebRTC 檢視端：接收視訊並送出輸入。 | | `host.py` | 669 | TCP 主機：接受迴圈、TLS 包裝、連線／認證握手、音訊與剪貼簿廣播、檔案推送、單次 token。 | | `viewer.py` | 634 | TCP 檢視端。 | -| `host_service.py` | 558 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | +| `host_service.py` | 562 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | | `host_client.py` | 453 | TCP 主機的每連線處理器：一個檢視端一個實例,擁有它的認證交換、sender／audio／receiver 三條執行緒,以及入站訊息的路由表。 | | `registry.py` | 370 | `AC_remote_*` 指令使用的行程級單例。 | | `webrtc_transport.py` | 411 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | @@ -1070,8 +1070,8 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | --- | ---: | ---: | | `gui/` | 94 | 27,525 | | `utils/mcp_server/` | 35 | 18,798 | -| `utils/remote_desktop/` | 56 | 12,912 | -| `utils/executor/` | 7 | 9,504 | +| `utils/remote_desktop/` | 56 | 12,916 | +| `utils/executor/` | 7 | 9,503 | | `utils/usb/` | 17 | 4,524 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,410 | | `utils/accessibility/` | 14 | 3,117 | @@ -1088,6 +1088,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 925 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 846 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 679 | 54,959 | -| **總計** | **1,053** | **154,324** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 679 | 54,997 | +| **總計** | **1,053** | **154,365** | diff --git a/docs/source/Eng/doc/new_features/v4_features_doc.rst b/docs/source/Eng/doc/new_features/v4_features_doc.rst index c1b868985..378ba0571 100644 --- a/docs/source/Eng/doc/new_features/v4_features_doc.rst +++ b/docs/source/Eng/doc/new_features/v4_features_doc.rst @@ -74,7 +74,8 @@ Flow control & variables assertion DSL. * **Read into a variable** — bind external data into the flow scope for later ``${var}`` use: ``AC_ocr_to_var`` (region text), ``AC_shell_to_var`` - (command stdout, decoded with ``encoding`` -- default the locale's; a + (command stdout, decoded with ``encoding`` -- default the locale's code + page, also under Python 3.15's UTF-8 mode; a timeout ends the command and everything it started, and a ``.bat`` / ``.cmd`` argument holding cmd syntax is refused), ``AC_read_file_to_var`` (file text; UTF-8 with or without a byte-order mark unless ``encoding`` diff --git a/docs/source/Zh/doc/new_features/v4_features_doc.rst b/docs/source/Zh/doc/new_features/v4_features_doc.rst index 98e89d2df..2a010e192 100644 --- a/docs/source/Zh/doc/new_features/v4_features_doc.rst +++ b/docs/source/Zh/doc/new_features/v4_features_doc.rst @@ -64,7 +64,7 @@ Builder 項目。視覺與視窗功能的 geometry / IO 操作皆可注入,因 ``AC_assert_duration`` 在區塊耗時超過預算時判失敗——銜接 profiler 與 斷言 DSL 的延遲回歸守門。 * **讀進變數** — 把外部資料綁進流程範圍供後續 ``${var}`` 使用: - ``AC_ocr_to_var``(區域文字)、``AC_shell_to_var``(命令 stdout,以 ``encoding`` 解碼,預設為系統地區設定的編碼; + ``AC_ocr_to_var``(區域文字)、``AC_shell_to_var``(命令 stdout,以 ``encoding`` 解碼,預設為系統地區設定的字碼頁,Python 3.15 的 UTF-8 模式下也是; 逾時會結束該命令及它啟動的所有程序,``.bat`` / ``.cmd`` 的參數含 cmd 語法時會拒絕)、 ``AC_read_file_to_var``(檔案文字;除非以 ``encoding`` 指定,否則讀 UTF-8,有無 BOM 皆可)、``AC_http_to_var``(GET body 或 dotted JSON path)、``AC_now_to_var``(strftime)、``AC_random_to_var`` diff --git a/docs/updates/2026-09-d.md b/docs/updates/2026-09-d.md index c1c577303..1f526c9da 100644 --- a/docs/updates/2026-09-d.md +++ b/docs/updates/2026-09-d.md @@ -297,3 +297,20 @@ This is the second half of the remote desktop GUI audit (the first is U-20260925 - **Files**: - Code: `utils/mcp_server/{_subscriptions,_stateless,_http_stateless,http_transport,server}.py`. - Docs: the MCP server doc (Eng/Zh), `CHANGELOG.md`, `Progress.md` (item removed), and `architecture_explore.md` (row, line counts). + +## U-20260925-51 · 2026-09-25 · Python 3.15 readiness: Windows console output is decoded in its code page under the default UTF-8 mode, and the SBOM skips distributions that have no metadata · #bugfix #compat + +- **Why**: Python 3.15.0 is due on 2026-10-01 (rc2 is out). Its What's New lists two changes that reach this package. +- **UTF-8 mode by default (PEP 686)**: + - In UTF-8 mode, `locale.getpreferredencoding(False)` answers `utf-8`, and `subprocess.run(text=True)` decodes UTF-8. `cmd`, `sc` and `ipconfig` still write the ANSI code page (cp950 on Traditional Chinese Windows). + - `shell_exec.py` already records the symptom from an earlier fix: `磁碟區` came back as replacement characters. + - Four readers relied on the locale default: `AC_shell_to_var` (`flow_data_commands.py`), the MCP `shell_command` tool (`_handlers_system.py`), `ShellManager.program_encoding`, and the remote host's `status` command (`sc query` with `text=True`). The last one raised `UnicodeDecodeError`, which its `except (OSError, SubprocessError)` did not catch. + - The new `shell_exec.console_encoding()` returns `locale.getencoding()` (3.11+, ignores UTF-8 mode) on Windows, and the preferred encoding elsewhere. That is the same answer as before on 3.10–3.14, where UTF-8 mode is off. `sc query` now passes it with `errors="replace"`. +- **`importlib.metadata`**: on 3.15 `Distribution.metadata` raises `MetadataNotFound` (a `FileNotFoundError`) for a distribution without a METADATA file, such as a directory left by an interrupted uninstall. `build_sbom` caught only `KeyError` / `AttributeError`, so one such leftover ended the whole SBOM. On 3.14 it was listed as `unknown` at version `0`, with a `DeprecationWarning` for the implicit `None`. + - `sbom._name` reads the name with `.get` and treats a missing file as no name. Distributions without one are skipped, in the full inventory and in a dependency closure. +- **Tests**: + - `test_console_encoding.py` (new, 5) simulates UTF-8 mode on Windows on any interpreter. 4 of 5 fail on the old code. + - `test_sbom_missing_metadata.py` (new, 2) runs with warnings as errors and simulates both the 3.15 and the 3.14 shape. Both fail on the old code. +- **Files**: + - Code: `utils/shell_process/shell_exec.py`, `utils/executor/flow_data_commands.py`, `utils/mcp_server/tools/_handlers_system.py`, `utils/remote_desktop/host_service.py`, `utils/sbom/sbom.py`. + - Docs: the v4 features doc (Eng/Zh, `AC_shell_to_var`'s default encoding), `CHANGELOG.md`, `architecture_explore.md` (line counts). diff --git a/docs/updates/README.md b/docs/updates/README.md index b4ec0c408..0bcba3910 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260925-51 | 2026-09-25 | Python 3.15 readiness: Windows console output is decoded in its code page under the default UTF-8 mode, and the SBOM skips distributions that have no metadata | #bugfix #compat | [2026-09-d](2026-09-d.md) | | U-20260925-48 | 2026-09-25 | MCP 2026-07-28 subscriptions/listen over stdio and HTTP: an acknowledgement of what the server will send, notifications tagged with the subscription id, cancellation by notifications/cancelled or a closed stream, and a completion answer when the server ends it; the 2026-07-28 Progress item is done | #done #mcp | [2026-09-d](2026-09-d.md) | | U-20260925-47 | 2026-09-25 | The MCP server's subscription handlers move from server.py into _subscriptions.py, before subscriptions/listen joins them; no behaviour change | #refactor #mcp | [2026-09-d](2026-09-d.md) | | U-20260925-46 | 2026-09-25 | MCP 2026-07-28 over Streamable HTTP: a stateless POST mirrors its body into MCP-Protocol-Version, Mcp-Method and Mcp-Name, a disagreeing header is 400 HeaderMismatch, version and metadata errors are 400 and unknown methods 404, and no session is kept | #feature #mcp | [2026-09-d](2026-09-d.md) | @@ -290,7 +291,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09-d.md](2026-09-d.md) | 2026-09 | 16 | +| [2026-09-d.md](2026-09-d.md) | 2026-09 | 17 | | [2026-09-c.md](2026-09-c.md) | 2026-09 | 38 | | [2026-09-b.md](2026-09-b.md) | 2026-09 | 68 | | [2026-09.md](2026-09.md) | 2026-09 | 79 | diff --git a/je_auto_control/utils/executor/flow_data_commands.py b/je_auto_control/utils/executor/flow_data_commands.py index 74d97fa59..8af0a3749 100644 --- a/je_auto_control/utils/executor/flow_data_commands.py +++ b/je_auto_control/utils/executor/flow_data_commands.py @@ -26,10 +26,9 @@ def exec_shell_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: which is what a console program writes on Windows) and bound under ``var`` (default ``shell_output``) for later ``${var}`` use. """ - import locale import subprocess # nosec B404 — argv list only, no shell from je_auto_control.utils.shell_process.shell_exec import ( - command_args, refuse_batch_metacharacters, run_captured, + command_args, console_encoding, refuse_batch_metacharacters, run_captured, ) command = args.get("command", args.get("shell_command")) if command is None or command == "" or command == []: @@ -39,7 +38,7 @@ def exec_shell_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: # As AC_shell_command does: cmd.exe re-parses a .bat's arguments, so a # ${var} holding "x&ver" ran a second command. refuse_batch_metacharacters(argv) - encoding = str(args.get("encoding") or locale.getpreferredencoding(False)) + encoding = str(args.get("encoding") or console_encoding()) timeout_s = float(args.get("timeout", 30.0)) try: completed = run_captured(argv, timeout_s) diff --git a/je_auto_control/utils/mcp_server/tools/_handlers_system.py b/je_auto_control/utils/mcp_server/tools/_handlers_system.py index 5968370b5..b995e26ff 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers_system.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers_system.py @@ -257,15 +257,15 @@ def shell_command(command: str, timeout: float = 30.0 protects against the command injection classes Bandit B602 / B605 cover. """ - import locale - - from je_auto_control.utils.shell_process.shell_exec import command_args, run_captured + from je_auto_control.utils.shell_process.shell_exec import ( + command_args, console_encoding, run_captured, + ) if not command or not command.strip(): raise ValueError("command must be a non-empty string") proc = run_captured(command_args(command), float(timeout)) # Bytes, decoded leniently: strict decoding failed in the reader thread # on output the locale's code page cannot read, and stdout came back None. - encoding = locale.getpreferredencoding(False) + encoding = console_encoding() return { "exit_code": int(proc.returncode), "stdout": proc.stdout.decode(encoding, errors="replace"), diff --git a/je_auto_control/utils/remote_desktop/host_service.py b/je_auto_control/utils/remote_desktop/host_service.py index 58063e043..5d3d2ebf3 100644 --- a/je_auto_control/utils/remote_desktop/host_service.py +++ b/je_auto_control/utils/remote_desktop/host_service.py @@ -258,10 +258,14 @@ def _print_status() -> int: print(f"No config at {_default_config_path()} — run 'configure' or 'init'.") if sys.platform == "win32": import subprocess # nosec B404 # reason: only invoke fixed sc query argv + from je_auto_control.utils.shell_process.shell_exec import console_encoding try: + # sc writes the code page; text=True alone decodes UTF-8 from + # Python 3.15 on and raised on a localised Windows. result = subprocess.run( # nosec B603 B607 # reason: fixed argv list, no shell ["sc", "query", "JeAutoControlRemoteHost"], capture_output=True, text=True, timeout=5, check=False, + encoding=console_encoding(), errors="replace", ) if result.returncode == 0: print("Windows service status:") diff --git a/je_auto_control/utils/sbom/sbom.py b/je_auto_control/utils/sbom/sbom.py index a1642eac9..7a634f8d9 100644 --- a/je_auto_control/utils/sbom/sbom.py +++ b/je_auto_control/utils/sbom/sbom.py @@ -33,8 +33,23 @@ def _purl(name: str, version: str) -> str: return f"pkg:pypi/{normalized}@{urllib.parse.quote(version, safe='.-_~!')}" +def _name(dist: "metadata.Distribution") -> Optional[str]: + """The distribution's name; ``None`` when it has no metadata to read it from. + + A directory left behind by an interrupted uninstall has none. Python 3.15 + raises ``MetadataNotFound`` (a ``FileNotFoundError``) for it, which ended + the whole SBOM; earlier versions return empty metadata, which listed an + ``unknown`` component at version ``0``. + """ + try: + name = dist.metadata.get("Name") # type: ignore[attr-defined] # reason: Message.get + except FileNotFoundError: + return None + return str(name) if name else None + + def _component(dist: "metadata.Distribution") -> Dict[str, Any]: - name = dist.metadata["Name"] or "unknown" + name = _name(dist) or "unknown" version = dist.version or "0" component: Dict[str, Any] = { "type": "library", "name": name, "version": version, @@ -54,10 +69,15 @@ def _component(dist: "metadata.Distribution") -> Dict[str, Any]: def _iter_distributions(root: Optional[str]): - """Yield distributions: all installed, or the closure of ``root``.""" + """Yield distributions that have metadata: all installed, or the closure of ``root``.""" if root is None: - yield from metadata.distributions() + yield from (dist for dist in metadata.distributions() if _name(dist) is not None) return + yield from _closure(root) + + +def _closure(root: str): + """Yield ``root`` and every distribution it requires here, each once.""" seen: Set[str] = set() queue = [root] while queue: @@ -72,6 +92,8 @@ def _iter_distributions(root: Optional[str]): dist = metadata.distribution(name) except metadata.PackageNotFoundError: continue + if _name(dist) is None: + continue yield dist for req in (dist.requires or []): if _applies(req): diff --git a/je_auto_control/utils/shell_process/shell_exec.py b/je_auto_control/utils/shell_process/shell_exec.py index c66abf83b..7e8b900f3 100644 --- a/je_auto_control/utils/shell_process/shell_exec.py +++ b/je_auto_control/utils/shell_process/shell_exec.py @@ -12,6 +12,22 @@ from je_auto_control.utils.logging.logging_instance import autocontrol_logger +def console_encoding() -> str: + """The encoding a console program writes in: on Windows the ANSI code page. + + ``locale.getpreferredencoding(False)`` answers ``utf-8`` in UTF-8 mode, + which Python 3.15 turns on by default (PEP 686), while ``cmd``, ``sc`` or + ``ipconfig`` keep writing the code page (cp950 on Traditional Chinese + Windows), so their output came back as replacement characters. + ``locale.getencoding`` (3.11+) ignores UTF-8 mode. Elsewhere the preferred + encoding stays: a UTF-8 locale gives the same answer either way. + """ + getencoding = getattr(locale, "getencoding", None) + if sys.platform == "win32" and getencoding is not None: + return getencoding() + return locale.getpreferredencoding(False) + + def command_args(shell_command: Union[str, List[str]]) -> Union[str, List[str]]: """What to hand ``subprocess`` for ``shell_command``, never through a shell. @@ -133,7 +149,7 @@ def __init__(self, shell_encoding: Optional[str] = None, program_buffer: int = 1 self.process: Union[subprocess.Popen, None] = None self.run_output_queue: queue.Queue = queue.Queue() self.run_error_queue: queue.Queue = queue.Queue() - self.program_encoding: str = shell_encoding or locale.getpreferredencoding(False) + self.program_encoding: str = shell_encoding or console_encoding() self.program_buffer: int = program_buffer def exec_shell(self, shell_command: Union[str, List[str], None] = None, *, diff --git a/test/unit_test/headless/test_console_encoding.py b/test/unit_test/headless/test_console_encoding.py new file mode 100644 index 000000000..542b23b40 --- /dev/null +++ b/test/unit_test/headless/test_console_encoding.py @@ -0,0 +1,59 @@ +"""Console output is decoded in the code page it is written in, on Python 3.15 too. + +Python 3.15 turns UTF-8 mode on by default (PEP 686), and in UTF-8 mode +``locale.getpreferredencoding(False)`` answers ``utf-8`` while ``cmd`` and +``sc`` keep writing the ANSI code page. The tests simulate that mode on any +interpreter: the preferred encoding says UTF-8, the locale says cp950. +No subprocess is started. +""" +import locale +import subprocess # nosec B404 # reason: only CompletedProcess is built, nothing runs +import sys +import types + +import pytest + +from je_auto_control.utils.shell_process import shell_exec + +_VOLUME = "磁碟區" + + +@pytest.fixture() +def utf8_mode_on_windows(monkeypatch): + monkeypatch.setattr(sys, "platform", "win32") + monkeypatch.setattr(locale, "getpreferredencoding", lambda do_setlocale=True: "utf-8") + monkeypatch.setattr(locale, "getencoding", lambda: "cp950", raising=False) + + +def _console_writes(monkeypatch, module): + def run_captured(argv, timeout_s): + return subprocess.CompletedProcess(argv, 0, _VOLUME.encode("cp950"), b"") # nosemgrep # reason: builds a result, runs nothing + monkeypatch.setattr(module, "run_captured", run_captured) + + +def test_windows_reads_the_code_page_in_utf8_mode(utf8_mode_on_windows): + assert shell_exec.console_encoding() == "cp950" + + +def test_other_platforms_keep_the_preferred_encoding(monkeypatch): + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(locale, "getpreferredencoding", lambda do_setlocale=True: "utf-8") + monkeypatch.setattr(locale, "getencoding", lambda: "ANSI_X3.4-1968", raising=False) + assert shell_exec.console_encoding() == "utf-8" + + +def test_shell_to_var_decodes_the_code_page(utf8_mode_on_windows, monkeypatch): + from je_auto_control.utils.executor.flow_data_commands import exec_shell_to_var + _console_writes(monkeypatch, shell_exec) + executor = types.SimpleNamespace(variables=types.SimpleNamespace(set=lambda name, value: None)) + assert exec_shell_to_var(executor, {"command": ["vol"]})["output"] == _VOLUME + + +def test_the_mcp_shell_tool_decodes_the_code_page(utf8_mode_on_windows, monkeypatch): + from je_auto_control.utils.mcp_server.tools._handlers_system import shell_command + _console_writes(monkeypatch, shell_exec) + assert shell_command("vol")["stdout"] == _VOLUME + + +def test_shell_manager_defaults_to_the_code_page(utf8_mode_on_windows): + assert shell_exec.ShellManager().program_encoding == "cp950" diff --git a/test/unit_test/headless/test_sbom_missing_metadata.py b/test/unit_test/headless/test_sbom_missing_metadata.py new file mode 100644 index 000000000..91e51a24e --- /dev/null +++ b/test/unit_test/headless/test_sbom_missing_metadata.py @@ -0,0 +1,51 @@ +"""The SBOM skips a distribution with no metadata instead of failing or inventing one. + +A directory left by an interrupted uninstall is a distribution without a +METADATA file. Python 3.15 raises ``MetadataNotFound`` (a +``FileNotFoundError``) when its metadata is read, which ended the whole SBOM; +3.14 returns empty metadata whose missing keys are deprecated, and the SBOM +listed it as ``unknown`` at version ``0``. Both shapes are simulated here. +""" +import warnings +from importlib import metadata + +from je_auto_control.utils.sbom import sbom + + +class _Dist(metadata.Distribution): + def __init__(self, text): + self._text = text + + def read_text(self, filename): + return self._text if filename == "METADATA" else None + + def locate_file(self, path): + return path + + +class _Python315Leftover(_Dist): + @property + def metadata(self): + raise FileNotFoundError("No package metadata was found.") + + +def _dists(): + return [_Dist("Name: good\nVersion: 1.0\n"), _Python315Leftover(None), _Dist(None)] + + +def test_every_installed_distribution_without_metadata_is_skipped(monkeypatch): + monkeypatch.setattr(sbom.metadata, "distributions", _dists) + with warnings.catch_warnings(): + warnings.simplefilter("error") + components = sbom.build_sbom(None)["components"] + assert [(c["name"], c["version"]) for c in components] == [("good", "1.0")] + + +def test_a_dependency_without_metadata_is_skipped(monkeypatch): + by_name = {"root": _Dist("Name: root\nVersion: 2.0\nRequires-Dist: gone\nRequires-Dist: old\n"), + "gone": _Python315Leftover(None), "old": _Dist(None)} + monkeypatch.setattr(sbom.metadata, "distribution", by_name.__getitem__) + with warnings.catch_warnings(): + warnings.simplefilter("error") + components = sbom.build_sbom("root")["components"] + assert [c["name"] for c in components] == ["root"] From b4d1adf617fab18ed38ce5497312618e974bbaac Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 25 Sep 2026 18:48:00 +0800 Subject: [PATCH 02/47] Map remote desktop input and the broadcast cursor through the captured frame's origin, so clicks land where the viewer clicked on a second monitor, a region, or a virtual desktop reaching above the primary screen --- CHANGELOG.md | 4 + architecture_explore.md | 24 ++--- docs/updates/2026-09-d.md | 13 +++ docs/updates/README.md | 2 + je_auto_control/utils/remote_desktop/host.py | 10 +- .../utils/remote_desktop/host_capture.py | 38 ++++++-- .../utils/remote_desktop/input_dispatch.py | 26 ++++- .../utils/remote_desktop/multi_viewer.py | 14 ++- .../utils/remote_desktop/webrtc_host.py | 10 +- .../utils/remote_desktop/webrtc_transport.py | 10 ++ .../headless/test_rd_capture_origin.py | 94 +++++++++++++++++++ 11 files changed, 216 insertions(+), 29 deletions(-) create mode 100644 test/unit_test/headless/test_rd_capture_origin.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a0a11e277..fa9ddcc52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -433,6 +433,10 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's instead of as UTF-8: `sc query` raised, and the others returned replacement characters. The SBOM skips a distribution that has no metadata instead of failing on 3.15, or listing it as `unknown` before. +- Remote desktop hosts map viewer input and the broadcast cursor through + the captured frame's origin, so clicks land correctly on a second monitor, + a capture region, or a virtual desktop that extends above or left of the + primary screen. `dispatch_input` takes an optional `origin`. - The Flow Editor opens action files saved with a BOM, keeps a wrapped file's other keys on save, and writes atomically. - The region selector (template cropping, OCR / screenshot / WebRTC regions) diff --git a/architecture_explore.md b/architecture_explore.md index e7ec46e90..a54ef57a1 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,059 | -| 程式碼總行數 | 154,430 | +| 程式碼總行數 | 154,504 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 775 | | 套件門面 `__all__` 公開名稱數 | 1,244 | @@ -514,14 +514,14 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 19,336 行。 +> 6 個套件、約 19,410 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/admin/` | 418 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | | `utils/config_sync/` | 332 | 透過訊令伺服器做跨機器設定同步 | | `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | -| `utils/remote_desktop/` | 12,916 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/remote_desktop/` | 12,990 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | | `utils/usb/` | 4,524 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | | `utils/usbip/` | 1,008 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | @@ -745,24 +745,24 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `rate_limit.py` | 48 | 工具呼叫的 token bucket 限流。 | | `__main__.py` | 92 | `je_auto_control_mcp` console script 進入點。 | -#### `utils/remote_desktop/`（12,916 行／56 檔） +#### `utils/remote_desktop/`（12,990 行／56 檔） 三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `webrtc_host.py` | 716 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入;session 生命週期、DataChannel 接線、檔案收發。 | +| `webrtc_host.py` | 720 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入;session 生命週期、DataChannel 接線、檔案收發。 | | `webrtc_viewer.py` | 677 | WebRTC 檢視端：接收視訊並送出輸入。 | -| `host.py` | 669 | TCP 主機：接受迴圈、TLS 包裝、連線／認證握手、音訊與剪貼簿廣播、檔案推送、單次 token。 | +| `host.py` | 673 | TCP 主機：接受迴圈、TLS 包裝、連線／認證握手、音訊與剪貼簿廣播、檔案推送、單次 token。 | | `viewer.py` | 634 | TCP 檢視端。 | | `host_service.py` | 562 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | | `host_client.py` | 453 | TCP 主機的每連線處理器：一個檢視端一個實例,擁有它的認證交換、sender／audio／receiver 三條執行緒,以及入站訊息的路由表。 | | `registry.py` | 370 | `AC_remote_*` 指令使用的行程級單例。 | -| `webrtc_transport.py` | 411 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | -| `multi_viewer.py` | 339 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | +| `webrtc_transport.py` | 421 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | +| `multi_viewer.py` | 349 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | | `signaling_server.py` | 427 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | | `audit_log.py` | 355 | SQLite 雜湊鏈稽核記錄。 | -| `host_capture.py` | 297 | TCP 主機的影格與游標產生：螢幕列舉、監視器索引轉擷取區域、預設 JPEG／游標 provider,以及 `FrameProductionMixin`（游標輪詢、擷取迴圈、上線編碼）。 | +| `host_capture.py` | 323 | TCP 主機的影格與游標產生：螢幕列舉、監視器索引轉擷取區域、預設 JPEG／游標 provider,以及 `FrameProductionMixin`（游標輪詢、擷取迴圈、上線編碼）。 | | `ws_protocol.py` | 318 | 最小 RFC 6455 WebSocket 框架與握手。 | | `file_transfer.py` | 371 | 分塊檔案傳輸。 | | `relay.py` | 315 | NAT 穿透失敗時的 TCP 中繼。 | @@ -784,7 +784,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `signaling_client.py` | 164 | 純標準庫的訊令用戶端。 | | `trust_list.py` | 139 | 自動接受的檢視端信任清單。 | | `webrtc_inspector.py` | 138 | 行程級的 `StatsSnapshot` 滾動視窗。 | -| `input_dispatch.py` | 141 | 在主機端套用輸入訊息。 | +| `input_dispatch.py` | 161 | 在主機端套用輸入訊息。 | | `session_recorder.py` | 139 | 以 PyAV 把 WebRTC 影格錄成 mp4。 | | `totp.py` | 146 | RFC 6238 TOTP（零外部相依）。 | | `file_sync.py` | 141 | 輪詢式資料夾鏡像。 | @@ -1070,7 +1070,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | --- | ---: | ---: | | `gui/` | 94 | 27,525 | | `utils/mcp_server/` | 35 | 18,798 | -| `utils/remote_desktop/` | 56 | 12,916 | +| `utils/remote_desktop/` | 56 | 12,990 | | `utils/executor/` | 7 | 9,503 | | `utils/usb/` | 17 | 4,524 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,410 | @@ -1089,5 +1089,5 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 846 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 679 | 54,997 | -| **總計** | **1,053** | **154,365** | +| **總計** | **1,053** | **154,439** | diff --git a/docs/updates/2026-09-d.md b/docs/updates/2026-09-d.md index 1f526c9da..7e4afb5b6 100644 --- a/docs/updates/2026-09-d.md +++ b/docs/updates/2026-09-d.md @@ -314,3 +314,16 @@ This is the second half of the remote desktop GUI audit (the first is U-20260925 - **Files**: - Code: `utils/shell_process/shell_exec.py`, `utils/executor/flow_data_commands.py`, `utils/mcp_server/tools/_handlers_system.py`, `utils/remote_desktop/host_service.py`, `utils/sbom/sbom.py`. - Docs: the v4 features doc (Eng/Zh, `AC_shell_to_var`'s default encoding), `CHANGELOG.md`, `architecture_explore.md` (line counts). +## U-20260925-49 · 2026-09-25 · Remote desktop input lands where the viewer clicked: both hosts dispatch viewer coordinates relative to the captured frame's origin, and the legacy host broadcasts its cursor in frame coordinates · #bugfix #remote-desktop #multi-monitor + +- **Found by**: following the region-selector fix (U-20260925-44) through the hosts. The WebRTC track already draws its cursor at `cursor - monitor.left/top`, so frame coordinates are relative to the capture origin. No host added that origin back before moving the pointer. +- **Bugs**: + - Every host passed the viewer's frame coordinates to `set_mouse_position` as screen positions. A click on a frame that does not start at (0, 0) landed that far off: a second monitor, a `region`, or the legacy host's default full capture, whose frame starts at the virtual desktop's corner. On this machine that corner is (0, -164), because a monitor sits above the primary one. + - The legacy host broadcast `get_mouse_position()` as is, so the viewer drew the remote cursor off by the same amount. +- **Change**: + - `dispatch_input(message, origin=(x, y))` adds the origin to absolute `x` / `y`. `dispatcher_at(origin_of)` asks for the origin on every message, because the captured monitor can change mid-session. + - `RemoteDesktopHost` takes the origin from its region, or from the virtual desktop via `capture_origin()` (`mss` monitor 0), and reports the default cursor relative to it. A custom frame provider or dispatcher is left as it was. + - `ScreenVideoTrack.capture_origin` records the last resolved monitor's or region's corner. `WebRTCDesktopHost` dispatches by its own track's origin, and `MultiViewerHost` by the shared track's, which its relayed sessions cannot see. + - If `mss` cannot read the layout (`ScreenShotError`, no display), the origin falls back to (0, 0) and the host still starts. +- **Tests**: `test_rd_capture_origin.py` (new, 6) fails 6/6 on the old code. The 582 remote-desktop tests touching these hosts pass. +- **Files**: `utils/remote_desktop/{input_dispatch,host,host_capture,webrtc_transport,webrtc_host,multi_viewer}.py`, and `architecture_explore.md` (line counts). diff --git a/docs/updates/README.md b/docs/updates/README.md index 0bcba3910..40557cf01 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260925-49 | 2026-09-25 | Remote desktop input lands where the viewer clicked: both hosts dispatch viewer coordinates relative to the captured frame's origin, and the legacy host broadcasts its cursor in frame coordinates | #bugfix #remote-desktop #multi-monitor | [2026-09-d](2026-09-d.md) | | U-20260925-51 | 2026-09-25 | Python 3.15 readiness: Windows console output is decoded in its code page under the default UTF-8 mode, and the SBOM skips distributions that have no metadata | #bugfix #compat | [2026-09-d](2026-09-d.md) | | U-20260925-48 | 2026-09-25 | MCP 2026-07-28 subscriptions/listen over stdio and HTTP: an acknowledgement of what the server will send, notifications tagged with the subscription id, cancellation by notifications/cancelled or a closed stream, and a completion answer when the server ends it; the 2026-07-28 Progress item is done | #done #mcp | [2026-09-d](2026-09-d.md) | | U-20260925-47 | 2026-09-25 | The MCP server's subscription handlers move from server.py into _subscriptions.py, before subscriptions/listen joins them; no behaviour change | #refactor #mcp | [2026-09-d](2026-09-d.md) | @@ -291,6 +292,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| +| [2026-09-d.md](2026-09-d.md) | 2026-09 | 14 | | [2026-09-d.md](2026-09-d.md) | 2026-09 | 17 | | [2026-09-c.md](2026-09-c.md) | 2026-09 | 38 | | [2026-09-b.md](2026-09-b.md) | 2026-09 | 68 | diff --git a/je_auto_control/utils/remote_desktop/host.py b/je_auto_control/utils/remote_desktop/host.py index d4eaca466..dd6180538 100644 --- a/je_auto_control/utils/remote_desktop/host.py +++ b/je_auto_control/utils/remote_desktop/host.py @@ -4,6 +4,7 @@ import ssl import threading import time +from functools import partial from typing import Any, Callable, List, Mapping, Optional, Sequence from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -44,7 +45,7 @@ from je_auto_control.utils.remote_desktop.host_capture import ( CursorProvider, FrameProductionMixin, FrameProvider, _DEFAULT_QUALITY, _default_frame_provider, - _resolve_cursor_provider, _resolve_monitor_region, + _resolve_cursor_provider, _resolve_monitor_region, capture_origin, ) from je_auto_control.utils.remote_desktop.host_client import ( _ClientHandler, @@ -134,14 +135,17 @@ def __init__( self._frame_provider: FrameProvider = ( frame_provider or _default_frame_provider(region, int(quality)) ) - self._dispatch: InputDispatcher = input_dispatcher or dispatch_input + # Viewer input and the broadcast cursor are in frame coordinates; the + # default frame starts at the region's or the virtual desktop's corner. + origin = capture_origin(region) if frame_provider is None else (0, 0) + self._dispatch: InputDispatcher = input_dispatcher or partial(dispatch_input, origin=origin) self._file_receiver: Optional[FileReceiver] = None self._audio_config = audio_config self._audio_capture_override = audio_capture self._audio_capture: Optional[AudioCapture] = None self._on_pending_viewer = on_pending_viewer self._cursor_provider: Optional[CursorProvider] = _resolve_cursor_provider( - cursor_provider, enable_cursor_broadcast, + cursor_provider, enable_cursor_broadcast, origin, ) self._cursor_thread: Optional[threading.Thread] = None # Latest broadcast cursor payload, kept so newly-authenticated diff --git a/je_auto_control/utils/remote_desktop/host_capture.py b/je_auto_control/utils/remote_desktop/host_capture.py index 1db0348e0..0b57c3c76 100644 --- a/je_auto_control/utils/remote_desktop/host_capture.py +++ b/je_auto_control/utils/remote_desktop/host_capture.py @@ -12,7 +12,7 @@ import time from io import BytesIO from typing import ( - TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, + TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Tuple, ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -88,15 +88,38 @@ def _resolve_monitor_region( int(mon["width"]), int(mon["height"]), ) +def capture_origin(region: Optional[Sequence[int]]) -> Tuple[int, int]: + """Screen position of the default frame provider's top-left pixel. + + The region's corner, or for a full capture the virtual desktop's, which + is negative when a monitor sits above or left of the primary one. + ``(0, 0)`` when ``mss`` is unavailable to say. + """ + if region is not None: + return int(region[0]), int(region[1]) + try: + monitors = list_host_monitors() + except Exception as error: # noqa: BLE001 # reason: mss raises its own ScreenShotError (no display, no permission); the host still starts, as it did before the origin was read + autocontrol_logger.warning("remote_desktop: monitor layout unavailable, input unshifted: %r", error) + return 0, 0 + if not monitors: + return 0, 0 + return monitors[0]["left"], monitors[0]["top"] + + def _resolve_cursor_provider( explicit: Optional[CursorProvider], - enabled: bool) -> Optional[CursorProvider]: - """Pick the cursor provider — explicit > default > disabled.""" + enabled: bool, origin: Tuple[int, int] = (0, 0)) -> Optional[CursorProvider]: + """Pick the cursor provider — explicit > default > disabled. + + The default one reports the pointer relative to ``origin``, the frame's + top-left: the viewer draws it in frame coordinates. + """ if explicit is not None: return explicit - return _default_cursor_provider() if enabled else None + return _default_cursor_provider(origin) if enabled else None -def _default_cursor_provider() -> CursorProvider: +def _default_cursor_provider(origin: Tuple[int, int] = (0, 0)) -> CursorProvider: """Build a cursor-position poller using the project's mouse wrapper. The wrapper is imported lazily inside the closure so importing this @@ -112,9 +135,12 @@ def provide() -> Optional[Sequence[int]]: except ImportError: return None try: - return get_mouse_position() + position = get_mouse_position() except (OSError, RuntimeError, AttributeError): return None + if position is None: + return None + return int(position[0]) - origin[0], int(position[1]) - origin[1] return provide def _default_frame_provider(region: Optional[Sequence[int]] = None, diff --git a/je_auto_control/utils/remote_desktop/input_dispatch.py b/je_auto_control/utils/remote_desktop/input_dispatch.py index 49ef07f8c..a3d7dcac0 100644 --- a/je_auto_control/utils/remote_desktop/input_dispatch.py +++ b/je_auto_control/utils/remote_desktop/input_dispatch.py @@ -7,7 +7,7 @@ can be imported on non-host systems (e.g. inside the viewer process) without pulling in OS-specific backends. """ -from typing import Any, Callable, Dict, Mapping +from typing import Any, Callable, Dict, Mapping, Tuple from je_auto_control.utils.exception.exceptions import AutoControlException @@ -47,8 +47,15 @@ def _import_wrappers(): } -def dispatch_input(message: Mapping[str, Any]) -> Any: - """Validate ``message`` and call the matching wrapper function.""" +def dispatch_input(message: Mapping[str, Any], *, origin: Tuple[int, int] = (0, 0)) -> Any: + """Validate ``message`` and call the matching wrapper function. + + ``x`` / ``y`` are positions in the frame the viewer sees; ``origin`` is + the screen position of that frame's top-left pixel, added before the + pointer moves. Without it a click on a captured monitor or region other + than one starting at (0, 0) -- or on a virtual desktop reaching above or + left of the primary screen -- landed that far off. + """ if not isinstance(message, Mapping): raise InputDispatchError( f"input message must be a mapping, got {type(message).__name__}" @@ -60,6 +67,9 @@ def dispatch_input(message: Mapping[str, Any]) -> Any: return None wrappers = _import_wrappers() try: + if "x" in message and "y" in message and tuple(origin) != (0, 0): + message = {**message, "x": int(message["x"]) + int(origin[0]), + "y": int(message["y"]) + int(origin[1])} return _APPLIERS[action](message, wrappers) except (KeyError, TypeError, ValueError, OverflowError) as error: # A missing field (KeyError), a non-numeric or infinite coordinate @@ -128,6 +138,16 @@ def _apply_type(message: Mapping[str, Any], wrappers: Dict[str, Any]) -> Any: return wrappers["write"](text) +def dispatcher_at(origin_of: Callable[[], Tuple[int, int]]) -> InputDispatcher: + """An :data:`InputDispatcher` that asks ``origin_of()`` for the capture origin per message. + + Hosts pass a callable, not a value: the captured monitor can change mid-session. + """ + def dispatch(message: Mapping[str, Any]) -> Any: + return dispatch_input(message, origin=origin_of()) + return dispatch + + _APPLIERS: Dict[str, Callable[[Mapping[str, Any], Dict[str, Any]], Any]] = { "mouse_move": _apply_mouse_move, "mouse_move_relative": _apply_mouse_move_relative, diff --git a/je_auto_control/utils/remote_desktop/multi_viewer.py b/je_auto_control/utils/remote_desktop/multi_viewer.py index 5232a0b31..5842eae94 100644 --- a/je_auto_control/utils/remote_desktop/multi_viewer.py +++ b/je_auto_control/utils/remote_desktop/multi_viewer.py @@ -21,7 +21,7 @@ ) from exc from je_auto_control.utils.logging.logging_instance import autocontrol_logger -from je_auto_control.utils.remote_desktop.input_dispatch import dispatch_input +from je_auto_control.utils.remote_desktop.input_dispatch import dispatcher_at from je_auto_control.utils.remote_desktop.permissions import SessionPermissions from je_auto_control.utils.remote_desktop.trust_list import TrustList from je_auto_control.utils.remote_desktop.webrtc_host import WebRTCDesktopHost @@ -50,6 +50,11 @@ def __init__(self, config: WebRTCConfig) -> None: def subscribe(self): return self._relay.subscribe(self._track) + @property + def capture_origin(self) -> Tuple[int, int]: + """The shared track's frame origin on screen.""" + return self._track.capture_origin + def stop(self) -> None: self._track.stop() @@ -84,7 +89,8 @@ def __init__(self, *, token: str, permissions if permissions is not None else SessionPermissions.from_read_only(read_only) ) - self._dispatch = input_dispatcher or dispatch_input + # The sessions relay one shared track, so they map input by its origin. + self._dispatch = input_dispatcher or dispatcher_at(self._capture_origin) self._ip_whitelist = list(ip_whitelist) if ip_whitelist else [] self._on_annotation = on_annotation self._on_session_state = on_session_state @@ -95,6 +101,10 @@ def __init__(self, *, token: str, self._source: Optional[_ScreenSource] = None self._lock = threading.Lock() + def _capture_origin(self) -> Tuple[int, int]: + source = self._source + return source.capture_origin if source is not None else (0, 0) + # --- session lifecycle -------------------------------------------------- def create_session_offer(self) -> Tuple[str, str]: diff --git a/je_auto_control/utils/remote_desktop/webrtc_host.py b/je_auto_control/utils/remote_desktop/webrtc_host.py index e86925e0a..043b9b30f 100644 --- a/je_auto_control/utils/remote_desktop/webrtc_host.py +++ b/je_auto_control/utils/remote_desktop/webrtc_host.py @@ -17,11 +17,11 @@ import asyncio import json import threading -from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional +from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Tuple from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop.audit_log import default_audit_log -from je_auto_control.utils.remote_desktop.input_dispatch import dispatch_input +from je_auto_control.utils.remote_desktop.input_dispatch import dispatcher_at from je_auto_control.utils.remote_desktop.permissions import SessionPermissions from je_auto_control.utils.remote_desktop.rate_limit import ( RateLimitConfig, RateLimiter, @@ -93,7 +93,7 @@ def __init__(self, *, token: str, # NOSONAR python:S107 self._on_state_change = on_state_change self._on_authenticated = on_authenticated self._on_pending_viewer = on_pending_viewer - self._dispatch = input_dispatcher or dispatch_input + self._dispatch = input_dispatcher or dispatcher_at(self._capture_origin) self._offer_consent = offer_consent or (lambda peer: True) self._trust_list = trust_list # permissions argument wins; otherwise derive from read_only shorthand @@ -690,6 +690,10 @@ def read_only(self) -> bool: def permissions(self) -> SessionPermissions: return self._permissions + def _capture_origin(self) -> Tuple[int, int]: + """Where this host's own screen track starts; (0, 0) for a relayed track.""" + return getattr(self._video_track, "capture_origin", (0, 0)) + def _dispatch_input_safely(self, payload: Any) -> None: if not isinstance(payload, dict): return diff --git a/je_auto_control/utils/remote_desktop/webrtc_transport.py b/je_auto_control/utils/remote_desktop/webrtc_transport.py index 6852d84ee..650cfff33 100644 --- a/je_auto_control/utils/remote_desktop/webrtc_transport.py +++ b/je_auto_control/utils/remote_desktop/webrtc_transport.py @@ -276,6 +276,10 @@ def __init__(self, monitor_index: int = 1, fps: int = 24, self._region = region self._show_cursor = show_cursor self._monitor: Optional[dict] = None + # The last resolved monitor's corner, kept across set_target_monitor + # so input in between still maps somewhere sensible. + self._origin: Tuple[int, int] = ( + (int(region[0]), int(region[1])) if region is not None else (0, 0)) self._executor = ThreadPoolExecutor( max_workers=1, thread_name_prefix="rd-capture", ) @@ -287,6 +291,11 @@ def __init__(self, monitor_index: int = 1, fps: int = 24, def fps(self) -> int: return self._fps + @property + def capture_origin(self) -> Tuple[int, int]: + """Screen position of the frame's top-left pixel, for mapping viewer input.""" + return self._origin + def set_target_fps(self, fps: int) -> None: """Tune capture rate at runtime; clamped to 1..60. Used by the adaptive-bitrate controller — fps is aiortc's only reliable lever @@ -321,6 +330,7 @@ def _resolve(self) -> dict: sct = mss_grabber() _capture_local.sct = sct self._monitor = _resolve_monitor(sct, self._monitor_index) + self._origin = (int(self._monitor.get("left", 0)), int(self._monitor.get("top", 0))) return self._monitor def _timestamp(self) -> Tuple[int, fractions.Fraction]: diff --git a/test/unit_test/headless/test_rd_capture_origin.py b/test/unit_test/headless/test_rd_capture_origin.py new file mode 100644 index 000000000..2e1c612c8 --- /dev/null +++ b/test/unit_test/headless/test_rd_capture_origin.py @@ -0,0 +1,94 @@ +"""Remote input and the broadcast cursor use the captured frame's coordinates. + +A viewer clicks in the frame it sees; the hosts passed those coordinates to +set_mouse_position as screen positions. A frame whose top-left is not (0, 0) +-- a second monitor, a region, or a virtual desktop reaching above the +primary screen (the legacy host's default) -- put every click that far off, +and the legacy host broadcast the cursor in screen coordinates. +""" +import types + +import pytest + +from je_auto_control.utils.remote_desktop import host_capture, input_dispatch +from je_auto_control.utils.remote_desktop.input_dispatch import InputDispatchError, dispatch_input + +_ORIGIN = (1920, -164) + + +@pytest.fixture() +def calls(monkeypatch): + recorded = [] + + def make(name): + return lambda *args, **kwargs: recorded.append((name, args)) + + fake = {name: make(name) for name in ( + "click_mouse", "mouse_scroll", "press_mouse", "release_mouse", "set_mouse_position", + "press_keyboard_key", "release_keyboard_key", "write")} + fake["get_mouse_position"] = lambda: (1930, -144) + monkeypatch.setattr(input_dispatch, "_import_wrappers", lambda: fake) + return recorded + + +def test_absolute_positions_are_moved_by_the_origin(calls): + dispatch_input({"action": "mouse_move", "x": 10, "y": 20}, origin=_ORIGIN) + dispatch_input({"action": "mouse_click", "x": 10, "y": 20}, origin=_ORIGIN) + dispatch_input({"action": "mouse_scroll", "x": 10, "y": 20, "amount": 1}, origin=_ORIGIN) + dispatch_input({"action": "mouse_move_relative", "dx": 5, "dy": 5}, origin=_ORIGIN) + assert calls == [("set_mouse_position", (1930, -144)), ("set_mouse_position", (1930, -144)), + ("click_mouse", ("mouse_left",)), ("mouse_scroll", (1, 1930, -144)), + ("set_mouse_position", (1935, -139))] + + +def test_a_bad_coordinate_is_still_a_dispatch_error(calls): + with pytest.raises(InputDispatchError): + dispatch_input({"action": "mouse_move", "x": "left", "y": 20}, origin=_ORIGIN) + assert calls == [] + + +def test_the_capture_origin_is_the_region_or_the_virtual_desktop(monkeypatch): + assert host_capture.capture_origin([1920, -164, 1920, 1080]) == _ORIGIN + monkeypatch.setattr(host_capture, "list_host_monitors", + lambda: [{"left": 0, "top": -164}, {"left": 0, "top": 0}]) + assert host_capture.capture_origin(None) == (0, -164) + monkeypatch.setattr(host_capture, "list_host_monitors", lambda: []) + assert host_capture.capture_origin(None) == (0, 0) + + +def test_the_legacy_host_maps_input_and_the_cursor_through_its_region(calls, monkeypatch): + from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost + from je_auto_control.wrapper import auto_control_mouse + monkeypatch.setattr(auto_control_mouse, "get_mouse_position", lambda: (1930, -144)) + host = RemoteDesktopHost(token="t", region=[1920, -164, 1920, 1080]) + host._dispatch({"action": "mouse_move", "x": 10, "y": 20}) + assert calls == [("set_mouse_position", (1930, -144))] + assert host._cursor_provider() == (10, 20) + + +def test_the_webrtc_hosts_dispatch_by_their_track_origin(calls): + pytest.importorskip("aiortc") + pytest.importorskip("av") + from je_auto_control.utils.remote_desktop.multi_viewer import MultiViewerHost + from je_auto_control.utils.remote_desktop.webrtc_host import WebRTCDesktopHost + from je_auto_control.utils.remote_desktop.webrtc_transport import ScreenVideoTrack + track = ScreenVideoTrack(region=[1920, -164, 1920, 1080]) + single = WebRTCDesktopHost(token="t") + single._video_track = track + single._dispatch({"action": "mouse_move", "x": 10, "y": 20}) + multi = MultiViewerHost(token="t") + multi._source = types.SimpleNamespace(capture_origin=track.capture_origin) + multi._dispatch({"action": "mouse_move", "x": 1, "y": 2}) + assert calls == [("set_mouse_position", (1930, -144)), ("set_mouse_position", (1921, -162))] + track.stop() + + +def test_a_host_without_a_display_still_starts(monkeypatch): + class ScreenShotError(Exception): + """mss's own error type, which is no OSError.""" + + def no_display(): + raise ScreenShotError("$DISPLAY not set") + + monkeypatch.setattr(host_capture, "list_host_monitors", no_display) + assert host_capture.capture_origin(None) == (0, 0) From 5d0dbd9af3050b049408335013bfe65a9350e768 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 25 Sep 2026 19:35:46 +0800 Subject: [PATCH 03/47] Stamp host annotations with the frame's screen origin and draw them on that screen in its logical pixels, through shared conversions in gui/_screen_geometry.py --- CHANGELOG.md | 2 + architecture_explore.md | 27 +++++---- docs/updates/2026-09-d.md | 10 ++++ docs/updates/README.md | 2 + je_auto_control/gui/_screen_geometry.py | 37 ++++++++++++ .../gui/remote_desktop/annotation_overlay.py | 46 ++++++++++++-- .../gui/selector/region_overlay.py | 16 +---- .../utils/remote_desktop/multi_viewer.py | 10 +++- .../utils/remote_desktop/webrtc_host.py | 4 +- .../headless/test_rd_annotation_screens.py | 60 +++++++++++++++++++ .../headless/test_webrtc_host_channels.py | 3 +- 11 files changed, 183 insertions(+), 34 deletions(-) create mode 100644 je_auto_control/gui/_screen_geometry.py create mode 100644 test/unit_test/headless/test_rd_annotation_screens.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fa9ddcc52..8e99be1a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -433,6 +433,8 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's instead of as UTF-8: `sc query` raised, and the others returned replacement characters. The SBOM skips a distribution that has no metadata instead of failing on 3.15, or listing it as `unknown` before. +- WebRTC host annotations are drawn at the viewer's position on the + captured screen, including other monitors and scaled displays. - Remote desktop hosts map viewer input and the broadcast cursor through the captured frame's origin, so clicks land correctly on a second monitor, a capture region, or a virtual desktop that extends above or left of the diff --git a/architecture_explore.md b/architecture_explore.md index a54ef57a1..bb1481018 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -19,8 +19,8 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,059 | -| 程式碼總行數 | 154,504 | +| Python 模組總數（含周邊子專案） | 1,060 | +| 程式碼總行數 | 154,577 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 775 | | 套件門面 `__all__` 公開名稱數 | 1,244 | @@ -514,14 +514,14 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 19,410 行。 +> 6 個套件、約 19,420 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/admin/` | 418 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | | `utils/config_sync/` | 332 | 透過訊令伺服器做跨機器設定同步 | | `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | -| `utils/remote_desktop/` | 12,990 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/remote_desktop/` | 13,000 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | | `utils/usb/` | 4,524 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | | `utils/usbip/` | 1,008 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | @@ -745,13 +745,13 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `rate_limit.py` | 48 | 工具呼叫的 token bucket 限流。 | | `__main__.py` | 92 | `je_auto_control_mcp` console script 進入點。 | -#### `utils/remote_desktop/`（12,990 行／56 檔） +#### `utils/remote_desktop/`（13,000 行／56 檔） 三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `webrtc_host.py` | 720 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入;session 生命週期、DataChannel 接線、檔案收發。 | +| `webrtc_host.py` | 722 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入;session 生命週期、DataChannel 接線、檔案收發。 | | `webrtc_viewer.py` | 677 | WebRTC 檢視端：接收視訊並送出輸入。 | | `host.py` | 673 | TCP 主機：接受迴圈、TLS 包裝、連線／認證握手、音訊與剪貼簿廣播、檔案推送、單次 token。 | | `viewer.py` | 634 | TCP 檢視端。 | @@ -759,7 +759,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `host_client.py` | 453 | TCP 主機的每連線處理器：一個檢視端一個實例,擁有它的認證交換、sender／audio／receiver 三條執行緒,以及入站訊息的路由表。 | | `registry.py` | 370 | `AC_remote_*` 指令使用的行程級單例。 | | `webrtc_transport.py` | 421 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | -| `multi_viewer.py` | 349 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | +| `multi_viewer.py` | 357 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | | `signaling_server.py` | 427 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | | `audit_log.py` | 355 | SQLite 雜湊鏈稽核記錄。 | | `host_capture.py` | 323 | TCP 主機的影格與游標產生：螢幕列舉、監視器索引轉擷取區域、預設 JPEG／游標 provider,以及 `FrameProductionMixin`（游標輪詢、擷取迴圈、上線編碼）。 | @@ -890,10 +890,11 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `_report_tab.py` | 88 | 報表分頁 mixin。 | | `_i18n_helpers.py` | 66 | 需要即時語言切換的分頁共用的翻譯註冊 mixin。 | | `_validators.py` | 29 | `int_validator()`／`double_validator()`：以 C locale 驗證的數字輸入框 validator，接受的正是 `int()`／`float()` 讀得懂的寫法（預設 locale 在法文、德文下只收小數逗號）。所有數字 `QLineEdit` 都用它。 | +| `_screen_geometry.py` | 37 | Qt 邏輯座標與截圖用的原生像素互轉：`native_region()`、`screen_at_native()`、`logical_point()`（每個螢幕的左上角在兩者相同，螢幕內依 device pixel ratio 縮放）。區域選取與主機端標註覆蓋層都用它。 | | `_daemon_thread.py` | 79 | `DaemonThread`：`QThread` 的替代品，保留遠端桌面 worker 用到的介面（`start`／`run`／`isRunning`／`wait`／`requestInterruption`／`started`／`finished`），但 `run()` 跑在 daemon `threading.Thread` 上，刪除物件或程式結束都不會銷毀執行中的執行緒。 | | `_worker_thread.py` | 192 | `start_worker()`：在 daemon `threading.Thread` 上執行 `QObject` worker 的 `run()`（沒有 `QThread` 可被銷毀），並經由分頁擁有的中繼物件回報結果（回呼一律在 GUI 執行緒；worker 沒處理的例外也送到 `on_fail`）；worker 留在模組登錄表直到 GUI 執行緒看到它結束，回傳 `WorkerHandle`（`isRunning()`）；程式結束時先呼叫 worker 的 `request_stop()`，最多等 10 秒，仍在跑的隨行程結束。 | | `language_wrapper/` | 5,031 | 四語系字典（英／日／簡中／繁中）+ `multi_language_wrapper` 執行期切換器與監聽註冊表。 | -| `selector/` | 227 | 拖曳選取螢幕區域的半透明全螢幕覆蓋層與樣板裁切工具（互動式，但都有對應的程式化 API）。 | +| `selector/` | 215 | 拖曳選取螢幕區域的半透明全螢幕覆蓋層與樣板裁切工具（互動式，但都有對應的程式化 API）。 | > **分頁指令一律走 Actions 選單**：分頁本身只放輸入、表格與結果檢視，指令由視窗層選單暴露。 > 核心分頁在 `main_widget.py` 註冊時宣告動作；功能分頁實作 `menu_actions()`（目前 40 個檔案有此 hook）。 @@ -952,7 +953,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | diagnostics | `diagnostics_tab.py` | 91 | 執行子系統檢查並顯示結果。 | | report | `_report_tab.py` | 81 | 產生 HTML／JSON／XML 報表。 | -#### 遠端桌面 GUI（`gui/remote_desktop/`，19 檔／6,608 行） +#### 遠端桌面 GUI（`gui/remote_desktop/`，19 檔／6,646 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -970,7 +971,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `_helpers.py` | 249 | 面板共用輔助：翻譯、Qt→AC 鍵滑鼠對應、TLS context、狀態徽章、指紋與時間格式化。 | | `remote_screen_window.py` | 140 | 檢視端的彈出視窗。 | | `tray_icon.py` | 106 | WebRTC 主機的系統匣圖示。 | -| `annotation_overlay.py` | 136 | 主機端標註的透明最上層覆蓋。 | +| `annotation_overlay.py` | 174 | 主機端標註的透明最上層覆蓋。 | | `sparkline.py` | 77 | WebRTC 統計面板的迷你走勢圖。 | | `blanking_overlay.py` | 71 | 遠端連線期間的隱私遮蔽全螢幕覆蓋。 | | `viewer_screen_window.py` | 46 | 顯示連入檢視端分享畫面的彈出視窗。 | @@ -1068,9 +1069,9 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | -| `gui/` | 94 | 27,525 | +| `gui/` | 95 | 27,588 | | `utils/mcp_server/` | 35 | 18,798 | -| `utils/remote_desktop/` | 56 | 12,990 | +| `utils/remote_desktop/` | 56 | 13,000 | | `utils/executor/` | 7 | 9,503 | | `utils/usb/` | 17 | 4,524 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,410 | @@ -1089,5 +1090,5 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 846 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 679 | 54,997 | -| **總計** | **1,053** | **154,439** | +| **總計** | **1,054** | **154,512** | diff --git a/docs/updates/2026-09-d.md b/docs/updates/2026-09-d.md index 7e4afb5b6..c634a7a3a 100644 --- a/docs/updates/2026-09-d.md +++ b/docs/updates/2026-09-d.md @@ -327,3 +327,13 @@ This is the second half of the remote desktop GUI audit (the first is U-20260925 - If `mss` cannot read the layout (`ScreenShotError`, no display), the origin falls back to (0, 0) and the host still starts. - **Tests**: `test_rd_capture_origin.py` (new, 6) fails 6/6 on the old code. The 582 remote-desktop tests touching these hosts pass. - **Files**: `utils/remote_desktop/{input_dispatch,host,host_capture,webrtc_transport,webrtc_host,multi_viewer}.py`, and `architecture_explore.md` (line counts). + +## U-20260925-50 · 2026-09-25 · Host annotations are drawn where the viewer drew them: the hosts stamp each event with the frame's screen origin, and the overlay moves to that screen and converts native pixels to its logical ones · #bugfix #remote-desktop #hidpi + +- **Bug**: the viewer draws in frame pixels. `HostAnnotationOverlay` covered the primary screen and painted those pixels as its own logical ones. So strokes were off by the frame's origin (another monitor or a region), and on a scaled screen by the device pixel ratio as well. +- **Change**: + - `WebRTCDesktopHost` stamps every annotation with `screen_origin` from its track, overwriting any value the viewer sent. `MultiViewerHost` stamps it with the shared track's origin, because its sessions relay that track. + - The overlay moves to the screen holding the point (and clears strokes left on the previous screen), then converts native to logical pixels. + - The conversions are shared in a new `gui/_screen_geometry.py`: `native_region`, moved from the region selector, plus `screen_at_native` and `logical_point`. +- **Tests**: `test_rd_annotation_screens.py` (new, 3) runs the overlay in a child process on the offscreen platform with the two-screen layout used for U-20260925-44. A point at frame (10, 20) with origin (1920, -164) lands at (8, 16) on the 125% screen. The test also checks that both hosts overwrite a viewer-sent origin. All 3 fail on the old code. +- **Files**: `gui/_screen_geometry.py` (new), `gui/remote_desktop/annotation_overlay.py`, `gui/selector/region_overlay.py`, `utils/remote_desktop/{webrtc_host,multi_viewer}.py`, and `architecture_explore.md` (a row for the new module, and line counts). diff --git a/docs/updates/README.md b/docs/updates/README.md index 40557cf01..8355317ee 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260925-50 | 2026-09-25 | Host annotations are drawn where the viewer drew them: the hosts stamp each event with the frame's screen origin, and the overlay moves to that screen and converts native pixels to its logical ones | #bugfix #remote-desktop #hidpi | [2026-09-d](2026-09-d.md) | | U-20260925-49 | 2026-09-25 | Remote desktop input lands where the viewer clicked: both hosts dispatch viewer coordinates relative to the captured frame's origin, and the legacy host broadcasts its cursor in frame coordinates | #bugfix #remote-desktop #multi-monitor | [2026-09-d](2026-09-d.md) | | U-20260925-51 | 2026-09-25 | Python 3.15 readiness: Windows console output is decoded in its code page under the default UTF-8 mode, and the SBOM skips distributions that have no metadata | #bugfix #compat | [2026-09-d](2026-09-d.md) | | U-20260925-48 | 2026-09-25 | MCP 2026-07-28 subscriptions/listen over stdio and HTTP: an acknowledgement of what the server will send, notifications tagged with the subscription id, cancellation by notifications/cancelled or a closed stream, and a completion answer when the server ends it; the 2026-07-28 Progress item is done | #done #mcp | [2026-09-d](2026-09-d.md) | @@ -292,6 +293,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| +| [2026-09-d.md](2026-09-d.md) | 2026-09 | 15 | | [2026-09-d.md](2026-09-d.md) | 2026-09 | 14 | | [2026-09-d.md](2026-09-d.md) | 2026-09 | 17 | | [2026-09-c.md](2026-09-c.md) | 2026-09 | 38 | diff --git a/je_auto_control/gui/_screen_geometry.py b/je_auto_control/gui/_screen_geometry.py new file mode 100644 index 000000000..4f430cb78 --- /dev/null +++ b/je_auto_control/gui/_screen_geometry.py @@ -0,0 +1,37 @@ +"""Convert between Qt's logical coordinates and the native pixels screenshots use. + +Qt keeps each screen's top-left corner the same in both and scales within the +screen by its device pixel ratio. Measured on Windows: a 125% screen at +(1920, -164) is 1536x864 in Qt and 1920x1080 natively. +""" +from typing import Optional, Tuple + +from PySide6.QtCore import QPointF, QRect +from PySide6.QtGui import QGuiApplication, QScreen + +Region = Tuple[int, int, int, int] + + +def native_region(screen: QScreen, rect: QRect) -> Region: + """``rect`` (logical pixels, relative to ``screen``'s corner) as native (x, y, w, h).""" + origin = screen.geometry().topLeft() + ratio = screen.devicePixelRatio() + return (origin.x() + round(rect.x() * ratio), origin.y() + round(rect.y() * ratio), + round(rect.width() * ratio), round(rect.height() * ratio)) + + +def screen_at_native(x: float, y: float) -> Optional[QScreen]: + """The screen whose native rectangle holds (x, y), or ``None``.""" + for screen in QGuiApplication.screens(): + geometry, ratio = screen.geometry(), screen.devicePixelRatio() + if (geometry.x() <= x < geometry.x() + geometry.width() * ratio + and geometry.y() <= y < geometry.y() + geometry.height() * ratio): + return screen + return None + + +def logical_point(screen: QScreen, x: float, y: float) -> QPointF: + """Native (x, y) on ``screen`` as a global logical point.""" + origin = screen.geometry().topLeft() + ratio = screen.devicePixelRatio() + return QPointF(origin.x() + (x - origin.x()) / ratio, origin.y() + (y - origin.y()) / ratio) diff --git a/je_auto_control/gui/remote_desktop/annotation_overlay.py b/je_auto_control/gui/remote_desktop/annotation_overlay.py index a82562a94..51899ac51 100644 --- a/je_auto_control/gui/remote_desktop/annotation_overlay.py +++ b/je_auto_control/gui/remote_desktop/annotation_overlay.py @@ -4,6 +4,10 @@ ``WebRTCDesktopHost.on_annotation`` and paints them on a click-through fullscreen window over the host's screen — so the host user sees the same annotations the viewer is drawing in real time. + +The viewer draws in frame pixels; the host stamps each event with the frame's +``screen_origin``, and the overlay moves to the screen holding the point and +converts native pixels to that screen's logical ones. """ from __future__ import annotations @@ -14,6 +18,8 @@ from PySide6.QtGui import QColor, QGuiApplication, QPainter, QPen, QPolygonF from PySide6.QtWidgets import QWidget +from je_auto_control.gui._screen_geometry import logical_point, screen_at_native + # A viewer could send strokes and points without end, each one repainting # every earlier one: the host's memory and paint time grew with it. _MAX_STROKES = 200 @@ -31,6 +37,15 @@ def _point(event: dict) -> Optional[Tuple[float, float]]: return (x, y) if math.isfinite(x) and math.isfinite(y) else None +def _origin(value) -> Tuple[int, int]: + """The host-stamped frame origin; (0, 0) when missing or malformed.""" + try: + x, y = value + return int(x), int(y) + except (TypeError, ValueError, OverflowError): + return 0, 0 + + def _width(value) -> int: """A pen width between 1 and ``_MAX_WIDTH``.""" try: @@ -54,10 +69,10 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: self.setAttribute(Qt.WidgetAttribute.WA_ShowWithoutActivating) self._strokes: List[dict] = [] self._current: Optional[dict] = None - # Cover the primary screen (multi-monitor case: caller can move/resize) - screen = QGuiApplication.primaryScreen() - if screen is not None: - self.setGeometry(screen.geometry()) + # The primary screen until a point lands on another one. + self._target = QGuiApplication.primaryScreen() + if self._target is not None: + self.setGeometry(self._target.geometry()) def show_overlay(self) -> None: if not self.isVisible(): @@ -79,12 +94,35 @@ def apply(self, event: dict) -> None: point = _point(event) if point is None: return + point = self._local_point(point, _origin(event.get("screen_origin"))) if action == "point": self.add_point(*point) return self.begin_stroke(*point, color=str(event.get("color") or "#ff0000"), width=_width(event.get("width"))) + def _local_point(self, point: Tuple[float, float], + origin: Tuple[int, int]) -> Tuple[float, float]: + """A frame point as a position in this overlay. + + Frame pixels used to be drawn as the overlay's logical pixels: off by + the frame's origin on screen, and on a scaled screen by its ratio. + """ + native_x, native_y = point[0] + origin[0], point[1] + origin[1] + screen = screen_at_native(native_x, native_y) or self._target + if screen is None: + return point + if screen is not self._target: + # Strokes on the old screen would be drawn at the new one's positions. + self.hide() + self.clear() + self._target = screen + self.setScreen(screen) + self.setGeometry(screen.geometry()) + logical = logical_point(screen, native_x, native_y) + corner = self.geometry().topLeft() + return logical.x() - corner.x(), logical.y() - corner.y() + def begin_stroke(self, x: float, y: float, *, color: str = "#ff0000", width: int = _DEFAULT_WIDTH) -> None: self._current = { diff --git a/je_auto_control/gui/selector/region_overlay.py b/je_auto_control/gui/selector/region_overlay.py index a37d95f7f..2e4956436 100644 --- a/je_auto_control/gui/selector/region_overlay.py +++ b/je_auto_control/gui/selector/region_overlay.py @@ -5,25 +5,13 @@ result was still offset by the virtual desktop's origin, and Qt's logical coordinates are not the native pixels screenshots use on a scaled screen. """ -from typing import List, Optional, Tuple +from typing import List, Optional from PySide6.QtCore import QEventLoop, QPoint, QRect, Qt, Signal from PySide6.QtGui import QColor, QKeyEvent, QMouseEvent, QPainter, QPen, QScreen from PySide6.QtWidgets import QApplication, QWidget -Region = Tuple[int, int, int, int] - - -def native_region(screen: QScreen, rect: QRect) -> Region: - """``rect`` (in logical pixels, relative to ``screen``) as native (x, y, w, h). - - Qt keeps a screen's top-left corner the same in logical and native - coordinates and scales within the screen by its device pixel ratio. - """ - origin = screen.geometry().topLeft() - ratio = screen.devicePixelRatio() - return (origin.x() + round(rect.x() * ratio), origin.y() + round(rect.y() * ratio), - round(rect.width() * ratio), round(rect.height() * ratio)) +from je_auto_control.gui._screen_geometry import Region, native_region class RegionOverlay(QWidget): diff --git a/je_auto_control/utils/remote_desktop/multi_viewer.py b/je_auto_control/utils/remote_desktop/multi_viewer.py index 5842eae94..9c5455e7f 100644 --- a/je_auto_control/utils/remote_desktop/multi_viewer.py +++ b/je_auto_control/utils/remote_desktop/multi_viewer.py @@ -105,6 +105,14 @@ def _capture_origin(self) -> Tuple[int, int]: source = self._source return source.capture_origin if source is not None else (0, 0) + def _annotate(self, data: dict) -> None: + """Pass a session's annotation on with the shared track's origin. + + A session relays the shared track, so its own origin is (0, 0). + """ + if self._on_annotation is not None: + self._on_annotation({**data, "screen_origin": self._capture_origin()}) + # --- session lifecycle -------------------------------------------------- def create_session_offer(self) -> Tuple[str, str]: @@ -120,7 +128,7 @@ def create_session_offer(self) -> Tuple[str, str]: permissions=self._permissions, input_dispatcher=self._dispatch, ip_whitelist=self._ip_whitelist, - on_annotation=self._on_annotation, + on_annotation=self._annotate if self._on_annotation is not None else None, external_video_track=self._source.subscribe(), on_state_change=self._wrap_state_callback(session_id), on_authenticated=self._wrap_auth_callback(session_id), diff --git a/je_auto_control/utils/remote_desktop/webrtc_host.py b/je_auto_control/utils/remote_desktop/webrtc_host.py index 043b9b30f..33e5ed911 100644 --- a/je_auto_control/utils/remote_desktop/webrtc_host.py +++ b/je_auto_control/utils/remote_desktop/webrtc_host.py @@ -557,7 +557,9 @@ def _handle_annotate(self, data: dict) -> None: if self._on_annotation is None: return try: - self._on_annotation(dict(data)) + # Where the frame the viewer drew on starts on screen; a value the + # viewer sent is overwritten. + self._on_annotation({**data, "screen_origin": self._capture_origin()}) except (RuntimeError, OSError) as error: autocontrol_logger.debug("annotation cb: %r", error) diff --git a/test/unit_test/headless/test_rd_annotation_screens.py b/test/unit_test/headless/test_rd_annotation_screens.py new file mode 100644 index 000000000..71d71f04f --- /dev/null +++ b/test/unit_test/headless/test_rd_annotation_screens.py @@ -0,0 +1,60 @@ +"""Host annotations land where the viewer drew them, on any screen and scale. + +The viewer draws in frame pixels. The overlay drew them as its own logical +pixels on the primary screen: off by the frame's origin (another monitor, a +region), and on a scaled screen by its device pixel ratio. +""" +import types + +import pytest + +pytest.importorskip("PySide6.QtWidgets", exc_type=ImportError) + +from headless._exit_probe import run_probe # noqa: E402 + +_PROBE = r''' +import json, os, sys, tempfile +from pathlib import Path +work = Path(tempfile.mkdtemp()) +(work / "screens.json").write_text(json.dumps({"screens": [ + {"name": "primary", "x": 0, "y": 0, "width": 1920, "height": 1080}, + {"name": "scaled", "x": 1920, "y": -164, "width": 1536, "height": 864, "dpr": 1.25}]}), + encoding="utf-8") +os.chdir(work) # platform options are colon-separated: no drive letter in the path +os.environ["QT_QPA_PLATFORM"] = "offscreen:configfile=screens.json" +from PySide6.QtWidgets import QApplication +app = QApplication([]) +from je_auto_control.gui.remote_desktop.annotation_overlay import HostAnnotationOverlay +overlay = HostAnnotationOverlay() +origin = [1920, -164] if sys.argv[1] == "scaled" else [0, 0] +overlay.apply({"action": "begin", "x": 10, "y": 20, "screen_origin": origin}) +print(overlay._target.name(), overlay._strokes[-1]["points"][0], flush=True) +''' + + +@pytest.mark.parametrize("screen, expected", [ + ("primary", "primary (10.0, 20.0)"), + # native (1930, -144) on the 125% screen: 1920 + 10 / 1.25, -164 + 20 / 1.25, less its corner + ("scaled", "scaled (8.0, 16.0)"), +]) +def test_a_point_is_drawn_on_its_screen_in_its_pixels(screen, expected): + done = run_probe(_PROBE, screen) + assert done.returncode == 0, done.stderr[-2000:] + assert done.stdout.strip().splitlines()[-1] == expected + + +def test_the_hosts_stamp_the_frame_origin_over_the_viewers(): + pytest.importorskip("aiortc") + pytest.importorskip("av") + from je_auto_control.utils.remote_desktop.multi_viewer import MultiViewerHost + from je_auto_control.utils.remote_desktop.webrtc_host import WebRTCDesktopHost + from je_auto_control.utils.remote_desktop.webrtc_transport import ScreenVideoTrack + seen = [] + single = WebRTCDesktopHost(token="t", on_annotation=seen.append) + single._video_track = ScreenVideoTrack(region=[1920, -164, 1920, 1080]) + single._handle_annotate({"type": "annotate", "action": "begin", "x": 1, "y": 2, "screen_origin": [5, 5]}) + multi = MultiViewerHost(token="t", on_annotation=seen.append) + multi._source = types.SimpleNamespace(capture_origin=(100, 200)) + multi._annotate({"action": "point", "x": 1, "y": 2, "screen_origin": [5, 5]}) + assert [event["screen_origin"] for event in seen] == [(1920, -164), (100, 200)] + single._video_track.stop() diff --git a/test/unit_test/headless/test_webrtc_host_channels.py b/test/unit_test/headless/test_webrtc_host_channels.py index e76d7eb0a..ea83df0c9 100644 --- a/test/unit_test/headless/test_webrtc_host_channels.py +++ b/test/unit_test/headless/test_webrtc_host_channels.py @@ -265,7 +265,8 @@ def test_an_annotation_reaches_the_gui_callback(): host._wire_control_channel(channel) host._authenticated = True _deliver(host, {"type": "annotate", "shape": "arrow"}) - assert seen == [{"type": "annotate", "shape": "arrow"}] + # The viewer's fields, plus where the host's frame starts on screen. + assert seen == [{"type": "annotate", "shape": "arrow", "screen_origin": (0, 0)}] def test_an_annotation_with_no_listener_is_dropped(host): From 734a418e28598783da9078d0f4a78b957d508f9c Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 25 Sep 2026 19:59:46 +0800 Subject: [PATCH 04/47] Scale screen-coordinate conversions by the device pixel ratio everywhere but macOS, where captures and the pointer take points --- architecture_explore.md | 8 ++--- docs/updates/2026-09-d.md | 7 +++++ docs/updates/README.md | 7 ++--- je_auto_control/gui/_screen_geometry.py | 29 ++++++++++++++----- .../headless/test_region_selector_screens.py | 6 +++- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/architecture_explore.md b/architecture_explore.md index bb1481018..5c67e209c 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,060 | -| 程式碼總行數 | 154,577 | +| 程式碼總行數 | 154,592 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 775 | | 套件門面 `__all__` 公開名稱數 | 1,244 | @@ -890,7 +890,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `_report_tab.py` | 88 | 報表分頁 mixin。 | | `_i18n_helpers.py` | 66 | 需要即時語言切換的分頁共用的翻譯註冊 mixin。 | | `_validators.py` | 29 | `int_validator()`／`double_validator()`：以 C locale 驗證的數字輸入框 validator，接受的正是 `int()`／`float()` 讀得懂的寫法（預設 locale 在法文、德文下只收小數逗號）。所有數字 `QLineEdit` 都用它。 | -| `_screen_geometry.py` | 37 | Qt 邏輯座標與截圖用的原生像素互轉：`native_region()`、`screen_at_native()`、`logical_point()`（每個螢幕的左上角在兩者相同，螢幕內依 device pixel ratio 縮放）。區域選取與主機端標註覆蓋層都用它。 | +| `_screen_geometry.py` | 52 | Qt 邏輯座標與截圖用的原生像素互轉：`native_region()`、`screen_at_native()`、`logical_point()`（每個螢幕的左上角在兩者相同，螢幕內依 device pixel ratio 縮放）。區域選取與主機端標註覆蓋層都用它。 | | `_daemon_thread.py` | 79 | `DaemonThread`：`QThread` 的替代品，保留遠端桌面 worker 用到的介面（`start`／`run`／`isRunning`／`wait`／`requestInterruption`／`started`／`finished`），但 `run()` 跑在 daemon `threading.Thread` 上，刪除物件或程式結束都不會銷毀執行中的執行緒。 | | `_worker_thread.py` | 192 | `start_worker()`：在 daemon `threading.Thread` 上執行 `QObject` worker 的 `run()`（沒有 `QThread` 可被銷毀），並經由分頁擁有的中繼物件回報結果（回呼一律在 GUI 執行緒；worker 沒處理的例外也送到 `on_fail`）；worker 留在模組登錄表直到 GUI 執行緒看到它結束，回傳 `WorkerHandle`（`isRunning()`）；程式結束時先呼叫 worker 的 `request_stop()`，最多等 10 秒，仍在跑的隨行程結束。 | | `language_wrapper/` | 5,031 | 四語系字典（英／日／簡中／繁中）+ `multi_language_wrapper` 執行期切換器與監聽註冊表。 | @@ -1069,7 +1069,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | -| `gui/` | 95 | 27,588 | +| `gui/` | 95 | 27,603 | | `utils/mcp_server/` | 35 | 18,798 | | `utils/remote_desktop/` | 56 | 13,000 | | `utils/executor/` | 7 | 9,503 | @@ -1090,5 +1090,5 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 846 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 679 | 54,997 | -| **總計** | **1,054** | **154,512** | +| **總計** | **1,054** | **154,527** | diff --git a/docs/updates/2026-09-d.md b/docs/updates/2026-09-d.md index c634a7a3a..e5adeaa6a 100644 --- a/docs/updates/2026-09-d.md +++ b/docs/updates/2026-09-d.md @@ -337,3 +337,10 @@ This is the second half of the remote desktop GUI audit (the first is U-20260925 - The conversions are shared in a new `gui/_screen_geometry.py`: `native_region`, moved from the region selector, plus `screen_at_native` and `logical_point`. - **Tests**: `test_rd_annotation_screens.py` (new, 3) runs the overlay in a child process on the offscreen platform with the two-screen layout used for U-20260925-44. A point at frame (10, 20) with origin (1920, -164) lands at (8, 16) on the 125% screen. The test also checks that both hosts overwrite a viewer-sent origin. All 3 fail on the old code. - **Files**: `gui/_screen_geometry.py` (new), `gui/remote_desktop/annotation_overlay.py`, `gui/selector/region_overlay.py`, `utils/remote_desktop/{webrtc_host,multi_viewer}.py`, and `architecture_explore.md` (a row for the new module, and line counts). + +## U-20260925-52 · 2026-09-25 · Screen-coordinate conversions no longer scale by the device pixel ratio on macOS, where captures and the pointer take points; and measured: importing the package makes a Windows process system-DPI-aware · #bugfix #hidpi #macos + +- **Bug**: U-20260925-44 (region selector) and U-20260925-50 (host annotations) convert Qt's logical pixels to screen coordinates by multiplying by the device pixel ratio. On macOS, `mss`, `screencapture` and Quartz events all take points, which is Qt's logical unit, so a Retina display (ratio 2) doubled every selected region. +- **Change**: `gui/_screen_geometry.capture_ratio(screen)` is 1.0 on macOS and the device pixel ratio elsewhere. `native_region`, `screen_at_native` and `logical_point` all use it. `test_region_selector_screens.py` gains a case that runs the two-screen probe with `sys.platform` set to `darwin`: `(2020, -64, 201, 101)`, unscaled. +- **Measured on Windows**: `import je_auto_control` makes the process system-DPI-aware, through `SetProcessDPIAware()` in `windows/screen/win32_screen.py` (`GetAwarenessFromDpiAwarenessContext` = 1). Qt then reports the 125% screen as `(1920, -164, 1536, 864)` at ratio 1.0, which is the same virtualised space as `mss` and the cursor. So in the GUI the conversions reduce to the origin offset, which still fixes the 164-pixel error measured on this machine. The ratio applies in per-monitor-aware processes and on Linux with Qt scaling. The awareness call is in a module Jeffrey_RPA uses and was not changed. +- **Files**: `gui/_screen_geometry.py`, `test/unit_test/headless/test_region_selector_screens.py`, and `architecture_explore.md` (line counts). diff --git a/docs/updates/README.md b/docs/updates/README.md index 8355317ee..c74868ac9 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,9 +58,10 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260925-52 | 2026-09-25 | Screen-coordinate conversions no longer scale by the device pixel ratio on macOS, where captures and the pointer take points; and measured: importing the package makes a Windows process system-DPI-aware | #bugfix #hidpi #macos | [2026-09-d](2026-09-d.md) | +| U-20260925-51 | 2026-09-25 | Python 3.15 readiness: Windows console output is decoded in its code page under the default UTF-8 mode, and the SBOM skips distributions that have no metadata | #bugfix #compat | [2026-09-d](2026-09-d.md) | | U-20260925-50 | 2026-09-25 | Host annotations are drawn where the viewer drew them: the hosts stamp each event with the frame's screen origin, and the overlay moves to that screen and converts native pixels to its logical ones | #bugfix #remote-desktop #hidpi | [2026-09-d](2026-09-d.md) | | U-20260925-49 | 2026-09-25 | Remote desktop input lands where the viewer clicked: both hosts dispatch viewer coordinates relative to the captured frame's origin, and the legacy host broadcasts its cursor in frame coordinates | #bugfix #remote-desktop #multi-monitor | [2026-09-d](2026-09-d.md) | -| U-20260925-51 | 2026-09-25 | Python 3.15 readiness: Windows console output is decoded in its code page under the default UTF-8 mode, and the SBOM skips distributions that have no metadata | #bugfix #compat | [2026-09-d](2026-09-d.md) | | U-20260925-48 | 2026-09-25 | MCP 2026-07-28 subscriptions/listen over stdio and HTTP: an acknowledgement of what the server will send, notifications tagged with the subscription id, cancellation by notifications/cancelled or a closed stream, and a completion answer when the server ends it; the 2026-07-28 Progress item is done | #done #mcp | [2026-09-d](2026-09-d.md) | | U-20260925-47 | 2026-09-25 | The MCP server's subscription handlers move from server.py into _subscriptions.py, before subscriptions/listen joins them; no behaviour change | #refactor #mcp | [2026-09-d](2026-09-d.md) | | U-20260925-46 | 2026-09-25 | MCP 2026-07-28 over Streamable HTTP: a stateless POST mirrors its body into MCP-Protocol-Version, Mcp-Method and Mcp-Name, a disagreeing header is 400 HeaderMismatch, version and metadata errors are 400 and unknown methods 404, and no session is kept | #feature #mcp | [2026-09-d](2026-09-d.md) | @@ -293,9 +294,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| -| [2026-09-d.md](2026-09-d.md) | 2026-09 | 15 | -| [2026-09-d.md](2026-09-d.md) | 2026-09 | 14 | -| [2026-09-d.md](2026-09-d.md) | 2026-09 | 17 | +| [2026-09-d.md](2026-09-d.md) | 2026-09 | 20 | | [2026-09-c.md](2026-09-c.md) | 2026-09 | 38 | | [2026-09-b.md](2026-09-b.md) | 2026-09 | 68 | | [2026-09.md](2026-09.md) | 2026-09 | 79 | diff --git a/je_auto_control/gui/_screen_geometry.py b/je_auto_control/gui/_screen_geometry.py index 4f430cb78..ba600b161 100644 --- a/je_auto_control/gui/_screen_geometry.py +++ b/je_auto_control/gui/_screen_geometry.py @@ -1,9 +1,14 @@ -"""Convert between Qt's logical coordinates and the native pixels screenshots use. +"""Convert between Qt's logical coordinates and the screen coordinates captures use. Qt keeps each screen's top-left corner the same in both and scales within the -screen by its device pixel ratio. Measured on Windows: a 125% screen at -(1920, -164) is 1536x864 in Qt and 1920x1080 natively. +screen by its device pixel ratio. Measured on Windows in a per-monitor-aware +process: a 125% screen at (1920, -164) is 1536x864 in Qt and 1920x1080 +natively. (``import je_auto_control`` makes a process system-DPI-aware, and +then Qt reports a ratio of 1.0 there, in the same virtualised space as the +cursor and ``mss``.) On macOS the capture APIs and the pointer take points, +which is Qt's logical unit, so no scaling applies. """ +import sys from typing import Optional, Tuple from PySide6.QtCore import QPointF, QRect @@ -12,10 +17,20 @@ Region = Tuple[int, int, int, int] +def capture_ratio(screen: QScreen) -> float: + """Screen-coordinate units per Qt logical pixel on ``screen``. + + The device pixel ratio, except on macOS: ``mss``, ``screencapture`` and + Quartz events all take points there, and scaling a Retina screen by 2 + doubled every region. + """ + return 1.0 if sys.platform == "darwin" else screen.devicePixelRatio() + + def native_region(screen: QScreen, rect: QRect) -> Region: - """``rect`` (logical pixels, relative to ``screen``'s corner) as native (x, y, w, h).""" + """``rect`` (logical pixels, relative to ``screen``'s corner) in screen coordinates (x, y, w, h).""" origin = screen.geometry().topLeft() - ratio = screen.devicePixelRatio() + ratio = capture_ratio(screen) return (origin.x() + round(rect.x() * ratio), origin.y() + round(rect.y() * ratio), round(rect.width() * ratio), round(rect.height() * ratio)) @@ -23,7 +38,7 @@ def native_region(screen: QScreen, rect: QRect) -> Region: def screen_at_native(x: float, y: float) -> Optional[QScreen]: """The screen whose native rectangle holds (x, y), or ``None``.""" for screen in QGuiApplication.screens(): - geometry, ratio = screen.geometry(), screen.devicePixelRatio() + geometry, ratio = screen.geometry(), capture_ratio(screen) if (geometry.x() <= x < geometry.x() + geometry.width() * ratio and geometry.y() <= y < geometry.y() + geometry.height() * ratio): return screen @@ -33,5 +48,5 @@ def screen_at_native(x: float, y: float) -> Optional[QScreen]: def logical_point(screen: QScreen, x: float, y: float) -> QPointF: """Native (x, y) on ``screen`` as a global logical point.""" origin = screen.geometry().topLeft() - ratio = screen.devicePixelRatio() + ratio = capture_ratio(screen) return QPointF(origin.x() + (x - origin.x()) / ratio, origin.y() + (y - origin.y()) / ratio) diff --git a/test/unit_test/headless/test_region_selector_screens.py b/test/unit_test/headless/test_region_selector_screens.py index e06dfaf5d..18427abfd 100644 --- a/test/unit_test/headless/test_region_selector_screens.py +++ b/test/unit_test/headless/test_region_selector_screens.py @@ -26,6 +26,8 @@ from PySide6.QtWidgets import QApplication app = QApplication([]) from je_auto_control.gui.selector.region_overlay import RegionOverlay, pick_region_blocking +if sys.argv[1].endswith("-mac"): + sys.platform = "darwin" # captures and the pointer take points there def drag(): overlays = [w for w in QApplication.topLevelWidgets() if isinstance(w, RegionOverlay) and w.isVisible()] @@ -33,7 +35,7 @@ def drag(): overlays[0].close() # closed without a selection return for overlay in overlays: - if overlay.screen().name() == sys.argv[1]: + if overlay.screen().name() == sys.argv[1].split("-")[0]: QTest.mousePress(overlay, Qt.MouseButton.LeftButton, pos=QPoint(100, 100)) QTest.mouseRelease(overlay, Qt.MouseButton.LeftButton, pos=QPoint(300, 200)) return @@ -49,6 +51,8 @@ def drag(): ("primary", "(100, 100, 201, 101)"), # (1920 + 100 * 1.25, -164 + 100 * 1.25, 201 * 1.25, 101 * 1.25), rounded ("scaled", "(2045, -39, 251, 126)"), + # macOS: Qt's logical pixels are the points captures take, so no scaling + ("scaled-mac", "(2020, -64, 201, 101)"), ]) def test_a_drag_answers_in_native_pixels_on_its_screen(screen, region): done = run_probe(_PROBE, screen) From 5d848a5ba2a940766f77d99f58592df62ef3f688 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 25 Sep 2026 20:15:31 +0800 Subject: [PATCH 05/47] Keep semantic replay on the recorded point for unnamed or off-screen anchors and anchor on controls rather than windows, treat a failed heal as a failed step, contain OpenCV errors in video motion and preprocessing, keep 16-bit scans visible, build the HTML report in linear time, and unescape D-Bus addresses --- CHANGELOG.md | 9 ++ architecture_explore.md | 24 +-- docs/updates/2026-09-d.md | 19 +++ docs/updates/README.md | 2 + .../utils/dbus_client/session_bus.py | 21 ++- .../generate_report/generate_html_report.py | 35 ++--- je_auto_control/utils/media_assert/media.py | 66 +++++--- .../utils/preprocess/preprocess.py | 39 ++++- .../utils/semantic_recording/enrich.py | 16 +- .../utils/semantic_recording/replay.py | 12 +- .../utils/semantic_recording/self_healing.py | 18 ++- .../headless/test_recording_media_audit.py | 143 ++++++++++++++++++ 12 files changed, 345 insertions(+), 59 deletions(-) create mode 100644 test/unit_test/headless/test_recording_media_audit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e99be1a1..33a427229 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -439,6 +439,15 @@ it shipped into a `## [x.y.z] - date` section of their own; the tag's the captured frame's origin, so clicks land correctly on a second monitor, a capture region, or a virtual desktop that extends above or left of the primary screen. `dispatch_input` takes an optional `origin`. +- Semantic replay keeps the recorded point for an unnamed or off-screen + anchor, and recording anchors on the clicked control, not its window. +- A failed self-healing lookup is a failed step instead of aborting replay. +- Video motion checks clip their region and stream frames; preprocessing + handles 16-bit, single-channel and float images; OpenCV errors in both are + recorded as step failures. +- HTML report generation is linear in the number of records. +- D-Bus socket addresses are unescaped, and unmarshallable values raise + `DBusError`. - The Flow Editor opens action files saved with a BOM, keeps a wrapped file's other keys on save, and writes atomically. - The region selector (template cropping, OCR / screenshot / WebRTC regions) diff --git a/architecture_explore.md b/architecture_explore.md index 5c67e209c..fbde3e822 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,060 | -| 程式碼總行數 | 154,592 | +| 程式碼總行數 | 154,705 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 775 | | 套件門面 `__all__` 公開名稱數 | 1,244 | @@ -303,7 +303,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.2 框架基礎設施 -> 14 個套件、約 3,030 行。 +> 14 個套件、約 3,041 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -311,7 +311,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/config_bundle/` | 424 | 使用者設定的單檔匯出／匯入 | | `utils/critical_exit/` | 132 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | | `utils/diagnostics/` | 330 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | -| `utils/dbus_client/` | 703 | 只用標準函式庫的 D-Bus session bus 客戶端。原本在 `linux_wayland/` 為 portal 交握而寫，AT-SPI 無障礙後端成為第二個使用者後搬到這裡（`utils/` 在分層上在各 OS 套件之上） | +| `utils/dbus_client/` | 714 | 只用標準函式庫的 D-Bus session bus 客戶端。原本在 `linux_wayland/` 為 portal 交握而寫，AT-SPI 無障礙後端成為第二個使用者後搬到這裡（`utils/` 在分層上在各 OS 套件之上） | | `utils/exception/` | 213 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | | `utils/failure_bundle/` | 219 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | | `utils/file_process/` | 40 | 目錄檔案列舉（`execute_dir` 的後端） | @@ -371,7 +371,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.5 影像辨識與畫面分析 -> 37 個套件、約 5,782 行。 +> 37 個套件、約 5,819 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -398,7 +398,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/monitor_layout/` | 320 | 多螢幕／虛擬桌面幾何（在哪個螢幕、位置、重映射）＋ `logical_frame` 以滑鼠座標空間擷取畫面 | | `utils/motion_regions/` | 73 | 兩影格間的局部變化／活動偵測（absdiff） | | `utils/perceptual_diff/` | 196 | 感知式（YIQ）影像差異，抑制反鋸齒邊緣誤報 | -| `utils/preprocess/` | 219 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | +| `utils/preprocess/` | 256 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | | `utils/qr/` | 59 | 從影像或螢幕區域解碼 QR code（OpenCV） | | `utils/rotated_match/` | 166 | 容忍旋轉與縮放的樣板比對（尺度空間 × 角度掃描） | | `utils/saliency/` | 114 | 頻譜殘差視覺顯著性：顯著圖與排序後的顯著區域 | @@ -464,7 +464,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.8 元素定位、自我修復與智慧等待 -> 23 個套件、約 4,316 行。 +> 23 個套件、約 4,354 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -488,7 +488,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/screen_state/` | 191 | 語義畫面狀態：快照／差異與結構化畫面描述 | | `utils/scroll_find/` | 103 | 捲動直到目標影像／文字可見 | | `utils/self_healing/` | 359 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | -| `utils/semantic_recording/` | 460 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | +| `utils/semantic_recording/` | 498 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | | `utils/settle_detector/` | 79 | 以純函式介面判定 UI 是否已靜止 | | `utils/smart_waits/` | 674 | 智慧等待：以影格差異取代 `time.sleep` | @@ -558,7 +558,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.12 報表、可觀測性與測試治理 -> 34 個套件、約 7,506 行。 +> 34 個套件、約 7,533 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -573,8 +573,8 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/failure_signature/` | 76 | 把錯誤訊息正規化成穩定的 SHA-256 失敗簽章並分群 | | `utils/flake_cluster/` | 103 | 以共同失敗 Jaccard 相似度為易碎測試分群 | | `utils/flakiness/` | 150 | 以執行歷史分析不穩定測試 | -| `utils/generate_report/` | 293 | HTML／JSON／XML 三種報表產生器（Template Method） | -| `utils/media_assert/` | 242 | 媒體斷言：音訊活動與影片動態檢查 | +| `utils/generate_report/` | 294 | HTML／JSON／XML 三種報表產生器（Template Method） | +| `utils/media_assert/` | 268 | 媒體斷言：音訊活動與影片動態檢查 | | `utils/observability/` | 710 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | | `utils/otlp_export/` | 109 | OTLP/JSON span 匯出 | | `utils/percentiles/` | 119 | 可合併的串流延遲摘要與精確百分位數 | @@ -1089,6 +1089,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 925 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 846 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 679 | 54,997 | -| **總計** | **1,054** | **154,527** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 679 | 55,110 | +| **總計** | **1,054** | **154,640** | diff --git a/docs/updates/2026-09-d.md b/docs/updates/2026-09-d.md index e5adeaa6a..b3f376c73 100644 --- a/docs/updates/2026-09-d.md +++ b/docs/updates/2026-09-d.md @@ -344,3 +344,22 @@ This is the second half of the remote desktop GUI audit (the first is U-20260925 - **Change**: `gui/_screen_geometry.capture_ratio(screen)` is 1.0 on macOS and the device pixel ratio elsewhere. `native_region`, `screen_at_native` and `logical_point` all use it. `test_region_selector_screens.py` gains a case that runs the two-screen probe with `sys.platform` set to `darwin`: `(2020, -64, 201, 101)`, unscaled. - **Measured on Windows**: `import je_auto_control` makes the process system-DPI-aware, through `SetProcessDPIAware()` in `windows/screen/win32_screen.py` (`GetAwarenessFromDpiAwarenessContext` = 1). Qt then reports the 125% screen as `(1920, -164, 1536, 864)` at ratio 1.0, which is the same virtualised space as `mss` and the cursor. So in the GUI the conversions reduce to the origin offset, which still fixes the 164-pixel error measured on this machine. The ratio applies in per-monitor-aware processes and on Linux with Qt scaling. The awareness call is in a module Jeffrey_RPA uses and was not changed. - **Files**: `gui/_screen_geometry.py`, `test/unit_test/headless/test_region_selector_screens.py`, and `architecture_explore.md` (line counts). +## U-20260925-55 · 2026-09-25 · Semantic replay no longer moves clicks to the wrong control, a failed heal is a failed step, OpenCV errors from video regions and odd images stay in the executor, 16-bit scans are not blackened, the HTML report is linear, and D-Bus unescapes addresses and rejects values it cannot marshal · #bugfix #audit #recording #vision + +A read-only audit agent found these in five `utils/` subpackages, each reproduced with a probe and re-checked here against the current tree. `test_recording_media_audit.py` (new, 8) fails 8/8 on the old code, and the 318 existing tests for these modules pass. + +- **Semantic replay**, high (wrong clicks): + - `_default_a11y_locator` turned a recorded empty name into "any name", so a click on an unnamed icon was moved to the app's Close button. An unnamed anchor now does not relocate. + - A match with no rectangle (a hidden tab page) relocated the click to (0, 0). It now keeps the recorded point. + - Enrichment walked only the default 200 elements, which starts at the window, so a control further down was never seen, the window became the anchor, and replay clicked the window's centre. It now walks `SCAN_LIMIT` elements and never anchors on a container role (window, frame, dialog, pane). +- **Self-healing**: `_heal` ran outside the step's error handling. A VLM request error, or a returned NaN, escaped `replay()` and lost the steps already run. A failed heal is now a failed step, with `last_error` "heal failed: ...". +- **Video motion**: + - An off-frame, negative or zero-width `region` sliced an empty frame, and `cv2.error`, which is not in the executor's catch list, aborted the whole script. The region is now clipped, one with nothing left is a `ValueError`, and OpenCV errors are contained. + - Every frame of the segment was kept in memory (about 37 GB for ten minutes of 1080p30). The differences are now summed as the frames are read. +- **Preprocessing**: + - Images are converted to uint8 on input. A 16-bit PNG used to stay 16-bit, so Otsu wrote 0 and 255 into a uint16 image (black) and adaptive thresholding raised. + - `(H, W, 1)` arrays and float images now work. + - `block_size < 3` and `grid < 1` are `ValueError`, every public function turns `cv2.error` into `AutoControlScreenException`, and a comma-separated `steps` string is split (`"deskew"` was walked letter by letter). +- **HTML report**: each record was appended to the whole report so far, which is quadratic (42 s for 16,000 records). The tables are now joined once; `make_html_table` keeps its signature. +- **D-Bus**: address values are percent-unescaped (`/run/bus-for-%3A0` is `/run/bus-for-:0`). Out-of-range numbers (`u` = -1, `y` = 300, which used to go out as 44) and truncated signatures (`"(i"`, `"a"`) raise `DBusError` instead of `struct.error` / `IndexError`. +- **Files**: `utils/{semantic_recording/{replay,enrich,self_healing},media_assert/media,preprocess/preprocess,generate_report/generate_html_report,dbus_client/session_bus}.py`, and `architecture_explore.md` (line counts). diff --git a/docs/updates/README.md b/docs/updates/README.md index c74868ac9..c7efe3071 100644 --- a/docs/updates/README.md +++ b/docs/updates/README.md @@ -58,6 +58,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | ID | Date | Title | Tags | Batch | |---|---|---|---|---| +| U-20260925-55 | 2026-09-25 | Semantic replay no longer moves clicks to the wrong control, a failed heal is a failed step, OpenCV errors from video regions and odd images stay in the executor, 16-bit scans are not blackened, the HTML report is linear, and D-Bus unescapes addresses and rejects values it cannot marshal | #bugfix #audit #recording #vision | [2026-09-d](2026-09-d.md) | | U-20260925-52 | 2026-09-25 | Screen-coordinate conversions no longer scale by the device pixel ratio on macOS, where captures and the pointer take points; and measured: importing the package makes a Windows process system-DPI-aware | #bugfix #hidpi #macos | [2026-09-d](2026-09-d.md) | | U-20260925-51 | 2026-09-25 | Python 3.15 readiness: Windows console output is decoded in its code page under the default UTF-8 mode, and the SBOM skips distributions that have no metadata | #bugfix #compat | [2026-09-d](2026-09-d.md) | | U-20260925-50 | 2026-09-25 | Host annotations are drawn where the viewer drew them: the hosts stamp each event with the frame's screen origin, and the overlay moves to that screen and converts native pixels to its logical ones | #bugfix #remote-desktop #hidpi | [2026-09-d](2026-09-d.md) | @@ -294,6 +295,7 @@ In the same commit: delete the item from `Progress.md`, add a `#done` entry here | File | Period | Entries | |---|---|---:| +| [2026-09-d.md](2026-09-d.md) | 2026-09 | 17 | | [2026-09-d.md](2026-09-d.md) | 2026-09 | 20 | | [2026-09-c.md](2026-09-c.md) | 2026-09 | 38 | | [2026-09-b.md](2026-09-b.md) | 2026-09 | 68 | diff --git a/je_auto_control/utils/dbus_client/session_bus.py b/je_auto_control/utils/dbus_client/session_bus.py index 92e57145b..5e78191ee 100644 --- a/je_auto_control/utils/dbus_client/session_bus.py +++ b/je_auto_control/utils/dbus_client/session_bus.py @@ -33,6 +33,7 @@ import socket import struct import time +import urllib.parse from typing import Any, Dict, List, Optional, Tuple from je_auto_control.utils.exception.exceptions import AutoControlException @@ -52,6 +53,8 @@ #: primary one is at a negative coordinate. Without it the accessibility #: backend could read a tree but not where anything was. _FIXED = { + # "y" is written here too, range-checked: byte() masks, so 300 went out as 44. + "y": (" None: """Write one fixed-width number, aligned to its own width.""" fmt, width = _FIXED[code] self.align(width) - self.raw(struct.pack(fmt, float(value) if fmt == " None: encoded = value.encode("utf-8") @@ -171,9 +178,13 @@ def done(self) -> bool: return self.index >= len(self.text) def peek(self) -> str: + if self.index >= len(self.text): + raise DBusError(f"signature {self.text!r} ends inside a type") return self.text[self.index] def take(self) -> str: + if self.index >= len(self.text): + raise DBusError(f"signature {self.text!r} ends inside a type") code = self.text[self.index] self.index += 1 return code @@ -194,9 +205,7 @@ def take_complete(self) -> str: def _write_value(writer: _Writer, reader: _SignatureReader, value: Any) -> None: code = reader.take() - if code == "y": - writer.byte(int(value)) - elif code == "b": + if code == "b": writer.uint32(1 if value else 0) elif code in _FIXED: writer.fixed(code, value) @@ -425,7 +434,9 @@ def _socket_target(address: str) -> Tuple[str, bool]: fields = [part.split("=", 1) for part in candidate[len("unix:"):].split(",") if "=" in part] - options = dict(fields) + # Values are percent-escaped (the D-Bus spec): /run/bus-for-%3A0 is + # the socket /run/bus-for-:0. + options = {key: urllib.parse.unquote(value) for key, value in fields} if "path" in options: return options["path"], False if "abstract" in options: diff --git a/je_auto_control/utils/generate_report/generate_html_report.py b/je_auto_control/utils/generate_report/generate_html_report.py index 6946e87a9..db0d4dc96 100644 --- a/je_auto_control/utils/generate_report/generate_html_report.py +++ b/je_auto_control/utils/generate_report/generate_html_report.py @@ -113,16 +113,18 @@ def make_html_table(event_str: str, record_data: dict, table_head: str) -> str: # AC_write of "