From 615eba244de01ffac0e7d0ba700d6c937b19fa9d Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sat, 28 Dec 2024 16:31:51 -0500 Subject: [PATCH 01/31] oa_pynput -> pynput; handle linux --- openadapt/build.py | 6 +++-- openadapt/build_utils.py | 4 +++- openadapt/capture/__init__.py | 22 ++++++++++++------- openadapt/models.py | 6 ++--- openadapt/playback.py | 2 +- openadapt/plotting.py | 4 +++- openadapt/record.py | 12 +++++----- openadapt/strategies/base.py | 2 +- openadapt/utils.py | 7 ++++-- openadapt/window/__init__.py | 3 ++- poetry.lock | 41 ++++++++++++++++------------------- pyproject.toml | 2 +- 12 files changed, 62 insertions(+), 49 deletions(-) diff --git a/openadapt/build.py b/openadapt/build.py index 1b0a82a45..1fefe8a9c 100644 --- a/openadapt/build.py +++ b/openadapt/build.py @@ -16,7 +16,7 @@ import urllib.request import gradio_client -import oa_pynput +import pynput import pycocotools import pydicom import pyqttoast @@ -34,7 +34,7 @@ def build_pyinstaller() -> None: """Build the application using PyInstaller.""" additional_packages_to_install = [ - oa_pynput, + pynput, pydicom, spacy_alignments, gradio_client, @@ -275,6 +275,8 @@ def main() -> None: create_macos_dmg() elif sys.platform == "win32": create_windows_installer() + else: + print(f"WARNING: openadapt.build is not yet supported on {sys.platform=}") if __name__ == "__main__": diff --git a/openadapt/build_utils.py b/openadapt/build_utils.py index 0cde71576..0d7228ad3 100644 --- a/openadapt/build_utils.py +++ b/openadapt/build_utils.py @@ -17,13 +17,15 @@ def get_root_dir_path() -> pathlib.Path: if not path.exists(): path.mkdir(parents=True, exist_ok=True) return path - else: + elif sys.platform == "win32": # if windows, get the path to the %APPDATA% directory and set the path # for all user preferences path = pathlib.Path.home() / "AppData" / "Roaming" / "openadapt" if not path.exists(): path.mkdir(parents=True, exist_ok=True) return path + else: + print(f"WARNING: openadapt.build_utils is not yet supported on {sys.platform=}") def is_running_from_executable() -> bool: diff --git a/openadapt/capture/__init__.py b/openadapt/capture/__init__.py index 756bed34f..91ae7b98c 100644 --- a/openadapt/capture/__init__.py +++ b/openadapt/capture/__init__.py @@ -1,5 +1,7 @@ """Capture the screen, audio, and camera as a video on macOS and Windows. +TODO: support Linux + Module: capture.py """ @@ -7,12 +9,13 @@ if sys.platform == "darwin": from . import _macos as impl + device = impl.Capture() elif sys.platform == "win32": from . import _windows as impl + device = impl.Capture() else: - raise Exception(f"Unsupported platform: {sys.platform}") - -device = impl.Capture() + print(f"WARNING: openadapt.capture is not yet supported on {sys.platform=}") + device = None def get_capture() -> impl.Capture: @@ -26,19 +29,22 @@ def get_capture() -> impl.Capture: def start(audio: bool = False, camera: bool = False) -> None: """Start the capture.""" - device.start(audio=audio, camera=camera) + if device: + device.start(audio=audio, camera=camera) def stop() -> None: """Stop the capture.""" - device.stop() + if device: + device.stop() def test() -> None: """Test the capture.""" - device.start() - input("Press enter to stop") - device.stop() + if device: + device.start() + input("Press enter to stop") + device.stop() if __name__ in ("__main__", "capture"): diff --git a/openadapt/models.py b/openadapt/models.py index b2286b812..2f42f17fb 100644 --- a/openadapt/models.py +++ b/openadapt/models.py @@ -9,7 +9,7 @@ import sys from bs4 import BeautifulSoup -from oa_pynput import keyboard +from pynput import keyboard from PIL import Image, ImageChops import numpy as np import sqlalchemy as sa @@ -649,9 +649,9 @@ def to_prompt_dict( "title", "help", ] - if sys.platform == "win32": + if sys.platform != "darwin": logger.warning( - "key_suffixes have not yet been defined on Windows." + "key_suffixes have not yet been defined on {sys.platform=}." "You can help by uncommenting the lines below and pasting " "the contents of the window_dict into a new GitHub Issue." ) diff --git a/openadapt/playback.py b/openadapt/playback.py index c909ee535..388b709b5 100644 --- a/openadapt/playback.py +++ b/openadapt/playback.py @@ -1,6 +1,6 @@ """Utilities for playing back ActionEvents.""" -from oa_pynput import keyboard, mouse +from pynput import keyboard, mouse from openadapt.common import KEY_EVENTS, MOUSE_EVENTS from openadapt.custom_logger import logger diff --git a/openadapt/plotting.py b/openadapt/plotting.py index 03c6a5b0c..06eacb612 100644 --- a/openadapt/plotting.py +++ b/openadapt/plotting.py @@ -435,8 +435,10 @@ def plot_performance( if view_file: if sys.platform == "darwin": os.system(f"open {fpath}") - else: + elif sys.platform == "win32": os.system(f"start {fpath}") + else: + os.system(f"xdg-open {fpath}") else: plt.savefig(BytesIO(), format="png") # save fig to void if view_file: diff --git a/openadapt/record.py b/openadapt/record.py index eef25c7c8..8cc4b59e2 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -20,7 +20,7 @@ import time import tracemalloc -from oa_pynput import keyboard, mouse +from pynput import keyboard, mouse from pympler import tracker import av @@ -577,7 +577,7 @@ def trigger_action_event( event_q.put(Event(utils.get_timestamp(), "action", action_event_args)) -def on_move(event_q: queue.Queue, x: int, y: int, injected: bool) -> None: +def on_move(event_q: queue.Queue, x: int, y: int, injected: bool = False) -> None: """Handles the 'move' event. Args: @@ -603,7 +603,7 @@ def on_click( y: int, button: mouse.Button, pressed: bool, - injected: bool, + injected: bool = False, ) -> None: """Handles the 'click' event. @@ -638,7 +638,7 @@ def on_scroll( y: int, dx: int, dy: int, - injected: bool, + injected: bool = False, ) -> None: """Handles the 'scroll' event. @@ -949,7 +949,7 @@ def read_keyboard_events( def on_press( event_q: queue.Queue, key: keyboard.Key | keyboard.KeyCode, - injected: bool, + injected: bool = False, ) -> None: """Event handler for key press events. @@ -1000,7 +1000,7 @@ def on_press( def on_release( event_q: queue.Queue, key: keyboard.Key | keyboard.KeyCode, - injected: bool, + injected: bool = False, ) -> None: """Event handler for key release events. diff --git a/openadapt/strategies/base.py b/openadapt/strategies/base.py index de114331b..98fd6dc71 100644 --- a/openadapt/strategies/base.py +++ b/openadapt/strategies/base.py @@ -4,7 +4,7 @@ from pprint import pformat import time -from oa_pynput import keyboard, mouse +from pynput import keyboard, mouse import numpy as np from openadapt import adapters, models, playback, utils diff --git a/openadapt/utils.py b/openadapt/utils.py index 9fbfd720c..7b8a48e1e 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -53,6 +53,9 @@ # TODO: move to constants.py EMPTY = (None, [], {}, "") SCT = mss.mss() +# TODO: determine programmatically in Linux +DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS = 0.5 +DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS = 5 _logger_lock = threading.Lock() _start_time = None @@ -232,7 +235,7 @@ def get_double_click_interval_seconds() -> float: return windll.user32.GetDoubleClickTime() / 1000 else: - raise Exception(f"Unsupported {sys.platform=}") + return DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS def get_double_click_distance_pixels() -> int: @@ -260,7 +263,7 @@ def get_double_click_distance_pixels() -> int: logger.warning(f"{x=} != {y=}") return max(x, y) else: - raise Exception(f"Unsupported {sys.platform=}") + return DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS def get_monitor_dims() -> tuple[int, int]: diff --git a/openadapt/window/__init__.py b/openadapt/window/__init__.py index fe0cb9e9f..8f88d7ae6 100644 --- a/openadapt/window/__init__.py +++ b/openadapt/window/__init__.py @@ -15,7 +15,8 @@ # TODO: implement Linux from . import _windows as impl else: - raise Exception(f"Unsupported platform: {sys.platform}") + impl = None + logger.warning(f"openadapt.window is not yet supported on {sys.platform=}") def get_active_window_data( diff --git a/poetry.lock b/poetry.lock index 9cf91a77a..5c56d682f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -4155,24 +4155,6 @@ dev = ["black", "pre-commit", "tox"] doc = ["m2r2", "sphinx"] test = ["pytest", "pytest-cov"] -[[package]] -name = "oa-pynput" -version = "1.7.7" -description = "Monitor and control user input devices" -optional = false -python-versions = "*" -files = [ - {file = "oa_pynput-1.7.7-py2.py3-none-any.whl", hash = "sha256:c1ee3d910d108fb216ddcac0ee01ab7b928f9a9307a47afda5b9a69fbb9da5a7"}, - {file = "oa_pynput-1.7.7.tar.gz", hash = "sha256:d20e2e93fee874dadc634b127e4a6278653fdf9165affe6eaf466e4e4807b9e8"}, -] - -[package.dependencies] -evdev = {version = ">=1.3", markers = "sys_platform in \"linux\""} -pyobjc-framework-ApplicationServices = {version = ">=8.0", markers = "sys_platform == \"darwin\""} -pyobjc-framework-Quartz = {version = ">=8.0", markers = "sys_platform == \"darwin\""} -python-xlib = {version = ">=0.17", markers = "sys_platform in \"linux\""} -six = "*" - [[package]] name = "onnxruntime" version = "1.20.0" @@ -5047,7 +5029,6 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs optional = false python-versions = ">=3.8" files = [ - {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, ] @@ -5058,7 +5039,6 @@ description = "A collection of ASN.1-based protocols modules" optional = false python-versions = ">=3.8" files = [ - {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"}, {file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"}, ] @@ -5572,6 +5552,23 @@ cffi = ">=1.4.1" docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] +[[package]] +name = "pynput" +version = "1.7.7" +description = "Monitor and control user input devices" +optional = false +python-versions = "*" +files = [ + {file = "pynput-1.7.7-py2.py3-none-any.whl", hash = "sha256:afc43f651684c98818de048abc76adf9f2d3d797083cb07c1f82be764a2d44cb"}, +] + +[package.dependencies] +evdev = {version = ">=1.3", markers = "sys_platform in \"linux\""} +pyobjc-framework-ApplicationServices = {version = ">=8.0", markers = "sys_platform == \"darwin\""} +pyobjc-framework-Quartz = {version = ">=8.0", markers = "sys_platform == \"darwin\""} +python-xlib = {version = ">=0.17", markers = "sys_platform in \"linux\""} +six = "*" + [[package]] name = "pyobjc-core" version = "10.3.1" @@ -9124,4 +9121,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "3.10.x" -content-hash = "fbb9a8c0ac03708a131f06d1d3f7086d7718dacbf03d199b70e2df76e23640dd" +content-hash = "c308a565dc79a85f2da6bce428d5af53fb03c8e531114a6bfa00299a613ce803" diff --git a/pyproject.toml b/pyproject.toml index 009fa7b44..87e154c77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,6 @@ pyobjc-framework-avfoundation = { version = "^9.2", markers = "sys_platform == ' fastapi = "^0.111.1" screen-recorder-sdk = { version = "^1.3.0", markers = "sys_platform == 'win32'" } pyaudio = { version = "^0.2.13", markers = "sys_platform == 'win32'" } -oa-pynput = "^1.7.7" oa-atomacos = { version = "3.2.0", markers = "sys_platform == 'darwin'" } presidio-image-redactor = "^0.0.48" pywebview = "^4.2.2" @@ -109,6 +108,7 @@ tokencost = "^0.1.12" numba = "^0.60.0" llvmlite = "^0.43.0" ell-ai = "^0.0.14" +pynput = "^1.7.7" [tool.pytest.ini_options] filterwarnings = [ # suppress warnings starting from "setuptools>=67.3" From a4ae541c6f245239ac2f2d37df1fa9df4aadab14 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 17:07:04 -0500 Subject: [PATCH 02/31] fix --- openadapt/window/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/window/__init__.py b/openadapt/window/__init__.py index 8f88d7ae6..2935983d3 100644 --- a/openadapt/window/__init__.py +++ b/openadapt/window/__init__.py @@ -11,7 +11,7 @@ if sys.platform == "darwin": from . import _macos as impl -elif sys.platform in ("win32", "linux"): +elif sys.platform == "win32": # TODO: implement Linux from . import _windows as impl else: From ac9753f38d86f94e93fc7792fd7f5a90685cf124 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 17:08:54 -0500 Subject: [PATCH 03/31] if not impl return None --- openadapt/window/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openadapt/window/__init__.py b/openadapt/window/__init__.py index 2935983d3..d9b8f583b 100644 --- a/openadapt/window/__init__.py +++ b/openadapt/window/__init__.py @@ -59,7 +59,10 @@ def get_active_window_state(read_window_data: bool) -> dict | None: dict or None: A dictionary containing the state of the active window, or None if the state is not available. """ + if not impl: + return None # TODO: save window identifier (a window's title can change, or + # multiple windows can have the same title) try: return impl.get_active_window_state(read_window_data) except Exception as exc: @@ -78,6 +81,8 @@ def get_active_element_state(x: int, y: int) -> dict | None: dict or None: A dictionary containing the state of the active element, or None if the state is not available. """ + if not impl: + return None try: return impl.get_active_element_state(x, y) except Exception as exc: From e6aa792ac5d5146ee0d91f500666324efcd15ebf Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 17:45:33 -0500 Subject: [PATCH 04/31] add window/_linux.py --- openadapt/window/__init__.py | 3 +- openadapt/window/_linux.py | 150 +++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 openadapt/window/_linux.py diff --git a/openadapt/window/__init__.py b/openadapt/window/__init__.py index d9b8f583b..08a3097cc 100644 --- a/openadapt/window/__init__.py +++ b/openadapt/window/__init__.py @@ -12,8 +12,9 @@ if sys.platform == "darwin": from . import _macos as impl elif sys.platform == "win32": - # TODO: implement Linux from . import _windows as impl +elif sys.platform.startswith("linux"): + from . import _linux as impl else: impl = None logger.warning(f"openadapt.window is not yet supported on {sys.platform=}") diff --git a/openadapt/window/_linux.py b/openadapt/window/_linux.py new file mode 100644 index 000000000..6f7be6e26 --- /dev/null +++ b/openadapt/window/_linux.py @@ -0,0 +1,150 @@ +import xcffib +import xcffib.xproto +import pickle +import time + +from openadapt.custom_logger import logger + + +def get_active_window_meta() -> dict | None: + """Retrieve metadata of the active window using X server directly. + + Returns: + dict or None: A dictionary containing metadata of the active window. + """ + try: + # Connect to the X server + conn = xcffib.connect() + root = conn.get_setup().roots[0].root + + # Get the _NET_ACTIVE_WINDOW atom + atom = conn.core.InternAtom( + False, len("_NET_ACTIVE_WINDOW"), "_NET_ACTIVE_WINDOW" + ).reply().atom + + # Fetch the active window ID + active_window = conn.core.GetProperty( + False, root, atom, xcffib.xproto.Atom.WINDOW, 0, 1 + ).reply() + if not active_window.value_len: + return None + + window_id = int.from_bytes(active_window.value, byteorder="little") + + # Get window geometry + geom = conn.core.GetGeometry(window_id).reply() + + return { + "window_id": window_id, + "x": geom.x, + "y": geom.y, + "width": geom.width, + "height": geom.height, + "title": get_window_title(conn, window_id), + } + except Exception as exc: + logger.warning(f"Failed to retrieve active window metadata: {exc}") + return None + + +def get_window_title(conn: xcffib.Connection, window_id: int) -> str: + """Retrieve the title of a given window. + + Args: + conn (xcffib.Connection): X server connection. + window_id (int): The ID of the window. + + Returns: + str: The title of the window, or an empty string if unavailable. + """ + try: + atom = conn.core.InternAtom( + False, len("_NET_WM_NAME"), "_NET_WM_NAME" + ).reply().atom + title_property = conn.core.GetProperty( + False, window_id, atom, xcffib.xproto.Atom.STRING, 0, 1024 + ).reply() + if title_property.value_len > 0: + return title_property.value.decode("utf-8") + except Exception as exc: + logger.warning(f"Failed to retrieve window title: {exc}") + return "" + + +def get_active_window_state(read_window_data: bool) -> dict | None: + """Get the state of the active window. + + Args: + read_window_data (bool): Whether to include detailed data about the window. + + Returns: + dict or None: A dictionary containing the state of the active window. + """ + meta = get_active_window_meta() + if not meta: + return None + + if read_window_data: + data = get_window_data(meta) + else: + data = {} + + state = { + "title": meta.get("title", ""), + "left": meta.get("x", 0), + "top": meta.get("y", 0), + "width": meta.get("width", 0), + "height": meta.get("height", 0), + "window_id": meta.get("window_id", 0), + "meta": meta, + "data": data, + } + try: + pickle.dumps(state, protocol=pickle.HIGHEST_PROTOCOL) + except Exception as exc: + logger.warning(f"{exc=}") + state.pop("data") + return state + + +def get_window_data(meta: dict) -> dict: + """Retrieve detailed data for the active window. + + Args: + meta (dict): Metadata of the active window. + + Returns: + dict: Detailed data of the window. + """ + # Placeholder for additional detailed data retrieval. + return {} + + +def get_active_element_state(x: int, y: int) -> dict | None: + """Get the state of the active element at the specified coordinates. + + Args: + x (int): The x-coordinate of the element. + y (int): The y-coordinate of the element. + + Returns: + dict or None: A dictionary containing the state of the active element. + """ + # Placeholder: Implement element-level state retrieval if necessary. + return {"x": x, "y": y, "state": "placeholder"} + + +def main() -> None: + """Test function for retrieving and inspecting the state of the active window.""" + time.sleep(1) + + state = get_active_window_state(read_window_data=True) + print(state) + pickle.dumps(state, protocol=pickle.HIGHEST_PROTOCOL) + import ipdb + + ipdb.set_trace() # noqa: E702 + + +if __name__ == "__main__": + main() From c1d36d631d43b687bdff2df56fbf19cbad3a630c Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 17:48:05 -0500 Subject: [PATCH 05/31] global X server connection --- openadapt/window/_linux.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/openadapt/window/_linux.py b/openadapt/window/_linux.py index 6f7be6e26..5ffb21307 100644 --- a/openadapt/window/_linux.py +++ b/openadapt/window/_linux.py @@ -5,16 +5,30 @@ from openadapt.custom_logger import logger +# Global X server connection +_conn = None + + +def get_x_server_connection() -> xcffib.Connection: + """Get or create a global connection to the X server. + + Returns: + xcffib.Connection: A global connection object. + """ + global _conn + if _conn is None: + _conn = xcffib.connect() + return _conn + def get_active_window_meta() -> dict | None: - """Retrieve metadata of the active window using X server directly. + """Retrieve metadata of the active window using a persistent X server connection. Returns: dict or None: A dictionary containing metadata of the active window. """ try: - # Connect to the X server - conn = xcffib.connect() + conn = get_x_server_connection() root = conn.get_setup().roots[0].root # Get the _NET_ACTIVE_WINDOW atom From b87f404596d2993c805f8112e2a78ffd6ad7aa2a Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 17:50:13 -0500 Subject: [PATCH 06/31] byte order native --- openadapt/window/_linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/window/_linux.py b/openadapt/window/_linux.py index 5ffb21307..258b0ef7f 100644 --- a/openadapt/window/_linux.py +++ b/openadapt/window/_linux.py @@ -43,7 +43,7 @@ def get_active_window_meta() -> dict | None: if not active_window.value_len: return None - window_id = int.from_bytes(active_window.value, byteorder="little") + window_id = int.from_bytes(active_window.value, byteorder="native") # Get window geometry geom = conn.core.GetGeometry(window_id).reply() From 6ad10272d10bbbdfa8f00becf20812dc9125fa68 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 18:00:59 -0500 Subject: [PATCH 07/31] window_id_bytes --- openadapt/window/_linux.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openadapt/window/_linux.py b/openadapt/window/_linux.py index 258b0ef7f..dca28153a 100644 --- a/openadapt/window/_linux.py +++ b/openadapt/window/_linux.py @@ -32,9 +32,7 @@ def get_active_window_meta() -> dict | None: root = conn.get_setup().roots[0].root # Get the _NET_ACTIVE_WINDOW atom - atom = conn.core.InternAtom( - False, len("_NET_ACTIVE_WINDOW"), "_NET_ACTIVE_WINDOW" - ).reply().atom + atom = conn.core.InternAtom(False, len("_NET_ACTIVE_WINDOW"), "_NET_ACTIVE_WINDOW").reply().atom # Fetch the active window ID active_window = conn.core.GetProperty( @@ -43,7 +41,9 @@ def get_active_window_meta() -> dict | None: if not active_window.value_len: return None - window_id = int.from_bytes(active_window.value, byteorder="native") + # Convert the value to a proper bytes object + window_id_bytes = b"".join(active_window.value[:4]) # Concatenate bytes + window_id = int.from_bytes(window_id_bytes, byteorder="little") # Get window geometry geom = conn.core.GetGeometry(window_id).reply() From d47b90f0e0c726e895a5888535d1cc98f00b79b7 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 18:13:42 -0500 Subject: [PATCH 08/31] get_sct() --- openadapt/utils.py | 16 ++++++++++++---- openadapt/window/_linux.py | 6 ++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index 7b8a48e1e..699fb16bf 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -52,11 +52,11 @@ # TODO: move to constants.py EMPTY = (None, [], {}, "") -SCT = mss.mss() # TODO: determine programmatically in Linux DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS = 0.5 DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS = 5 +_sct = None _logger_lock = threading.Lock() _start_time = None _start_perf_counter = None @@ -266,6 +266,13 @@ def get_double_click_distance_pixels() -> int: return DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS +def get_sct() -> Any: + global _sct + if _sct is None: + _sct = mss.mss() + return _sct + + def get_monitor_dims() -> tuple[int, int]: """Get the dimensions of the monitor. @@ -273,7 +280,7 @@ def get_monitor_dims() -> tuple[int, int]: tuple[int, int]: The width and height of the monitor. """ # TODO XXX: replace with get_screenshot().size and remove get_scale_ratios? - monitor = SCT.monitors[0] + monitor = get_sct().monitors[0] monitor_width = monitor["width"] monitor_height = monitor["height"] return monitor_width, monitor_height @@ -423,8 +430,9 @@ def take_screenshot() -> Image.Image: PIL.Image: The screenshot image. """ # monitor 0 is all in one - monitor = SCT.monitors[0] - sct_img = SCT.grab(monitor) + sct = get_sct() + monitor = sct.monitors[0] + sct_img = sct.grab(monitor) image = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") return image diff --git a/openadapt/window/_linux.py b/openadapt/window/_linux.py index dca28153a..8be244137 100644 --- a/openadapt/window/_linux.py +++ b/openadapt/window/_linux.py @@ -32,7 +32,9 @@ def get_active_window_meta() -> dict | None: root = conn.get_setup().roots[0].root # Get the _NET_ACTIVE_WINDOW atom - atom = conn.core.InternAtom(False, len("_NET_ACTIVE_WINDOW"), "_NET_ACTIVE_WINDOW").reply().atom + atom = conn.core.InternAtom( + False, len("_NET_ACTIVE_WINDOW"), "_NET_ACTIVE_WINDOW" + ).reply().atom # Fetch the active window ID active_window = conn.core.GetProperty( @@ -42,7 +44,7 @@ def get_active_window_meta() -> dict | None: return None # Convert the value to a proper bytes object - window_id_bytes = b"".join(active_window.value[:4]) # Concatenate bytes + window_id_bytes = b"".join(active_window.value) # Concatenate bytes window_id = int.from_bytes(window_id_bytes, byteorder="little") # Get window geometry From d2c4d1472ac2bcb42333dbe9f119fc266268bc37 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 18:23:55 -0500 Subject: [PATCH 09/31] get_thread_local_sct --- openadapt/utils.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index 699fb16bf..079cd95f8 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -56,11 +56,13 @@ DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS = 0.5 DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS = 5 -_sct = None _logger_lock = threading.Lock() _start_time = None _start_perf_counter = None +# Thread-local storage to ensure thread-safe reuse of `mss` instances +_thread_local = threading.local() + def configure_logging(logger: logger, log_level: str) -> None: """Configure the logging settings for OpenAdapt. @@ -266,11 +268,11 @@ def get_double_click_distance_pixels() -> int: return DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS -def get_sct() -> Any: - global _sct - if _sct is None: - _sct = mss.mss() - return _sct +def get_thread_local_sct() -> mss.mss: + """Retrieve or create the `mss` instance for the current thread.""" + if not hasattr(_thread_local, "sct"): + _thread_local.sct = mss.mss() + return _thread_local.sct def get_monitor_dims() -> tuple[int, int]: @@ -280,7 +282,7 @@ def get_monitor_dims() -> tuple[int, int]: tuple[int, int]: The width and height of the monitor. """ # TODO XXX: replace with get_screenshot().size and remove get_scale_ratios? - monitor = get_sct().monitors[0] + monitor = get_thread_local_sct().monitors[0] monitor_width = monitor["width"] monitor_height = monitor["height"] return monitor_width, monitor_height @@ -430,7 +432,7 @@ def take_screenshot() -> Image.Image: PIL.Image: The screenshot image. """ # monitor 0 is all in one - sct = get_sct() + sct = get_thread_local_sct() monitor = sct.monitors[0] sct_img = sct.grab(monitor) image = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") From 6b103e385ac64acbbb2909a03bdefaf72fae8dfe Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 18:35:42 -0500 Subject: [PATCH 10/31] Fallback to WM_NAME --- openadapt/window/_linux.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/openadapt/window/_linux.py b/openadapt/window/_linux.py index 8be244137..dd8c14837 100644 --- a/openadapt/window/_linux.py +++ b/openadapt/window/_linux.py @@ -74,14 +74,27 @@ def get_window_title(conn: xcffib.Connection, window_id: int) -> str: str: The title of the window, or an empty string if unavailable. """ try: - atom = conn.core.InternAtom( + # Attempt to fetch _NET_WM_NAME + atom_net_wm_name = conn.core.InternAtom( False, len("_NET_WM_NAME"), "_NET_WM_NAME" ).reply().atom title_property = conn.core.GetProperty( - False, window_id, atom, xcffib.xproto.Atom.STRING, 0, 1024 + False, window_id, atom_net_wm_name, xcffib.xproto.Atom.STRING, 0, 1024 + ).reply() + if title_property.value_len > 0: + title_bytes = bytes(title_property.value) # Convert to proper bytes + return title_bytes.decode("utf-8") + + # Fallback to WM_NAME + atom_wm_name = conn.core.InternAtom( + False, len("WM_NAME"), "WM_NAME" + ).reply().atom + title_property = conn.core.GetProperty( + False, window_id, atom_wm_name, xcffib.xproto.Atom.STRING, 0, 1024 ).reply() if title_property.value_len > 0: - return title_property.value.decode("utf-8") + title_bytes = bytes(title_property.value) # Convert to proper bytes + return title_bytes.decode("utf-8") except Exception as exc: logger.warning(f"Failed to retrieve window title: {exc}") return "" From 34790c89e889c986c29896d51fe455e366e49c21 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 29 Dec 2024 18:37:04 -0500 Subject: [PATCH 11/31] convert bytes with join --- openadapt/window/_linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openadapt/window/_linux.py b/openadapt/window/_linux.py index dd8c14837..2fb4fb0d2 100644 --- a/openadapt/window/_linux.py +++ b/openadapt/window/_linux.py @@ -82,7 +82,7 @@ def get_window_title(conn: xcffib.Connection, window_id: int) -> str: False, window_id, atom_net_wm_name, xcffib.xproto.Atom.STRING, 0, 1024 ).reply() if title_property.value_len > 0: - title_bytes = bytes(title_property.value) # Convert to proper bytes + title_bytes = b"".join(title_property.value) # Convert using b"".join() return title_bytes.decode("utf-8") # Fallback to WM_NAME @@ -93,7 +93,7 @@ def get_window_title(conn: xcffib.Connection, window_id: int) -> str: False, window_id, atom_wm_name, xcffib.xproto.Atom.STRING, 0, 1024 ).reply() if title_property.value_len > 0: - title_bytes = bytes(title_property.value) # Convert to proper bytes + title_bytes = b"".join(title_property.value) # Convert using b"".join() return title_bytes.decode("utf-8") except Exception as exc: logger.warning(f"Failed to retrieve window title: {exc}") From 87e11944934bea0fc297203c4f2d155537f18cfe Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 12:54:10 -0500 Subject: [PATCH 12/31] task_started_events --- openadapt/record.py | 109 +++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 57 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 8cc4b59e2..39ce4f5ca 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -138,7 +138,7 @@ def process_events( perf_q: sq.SynchronizedQueue, recording: Recording, terminate_processing: multiprocessing.Event, - started_counter: multiprocessing.Value, + started_event: threading.Event, num_screen_events: multiprocessing.Value, num_action_events: multiprocessing.Value, num_window_events: multiprocessing.Value, @@ -157,7 +157,7 @@ def process_events( perf_q: A queue for collecting performance data. recording: The recording object. terminate_processing: An event to signal the termination of the process. - started_counter: Value to increment once started. + started_event: Event to set once started. num_screen_events: A counter for the number of screen events. num_action_events: A counter for the number of action events. num_window_events: A counter for the number of window events. @@ -177,8 +177,7 @@ def process_events( while not terminate_processing.is_set() or not event_q.empty(): event = event_q.get() if not started: - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() started = True logger.trace(f"{event=}") assert event.type in EVENT_TYPES, event @@ -371,7 +370,7 @@ def write_events( perf_q: sq.SynchronizedQueue, recording: Recording, terminate_processing: multiprocessing.Event, - started_counter: multiprocessing.Value, + started_event: multiprocessing.Event, pre_callback: Callable[[float], dict] | None = None, post_callback: Callable[[dict], None] | None = None, ) -> None: @@ -385,7 +384,7 @@ def write_events( perf_q: A queue for collecting performance data. recording: The recording object. terminate_processing: An event to signal the termination of the process. - started_counter: Value to increment once started. + started_event: Event to increment once started. pre_callback: Optional function to call before main loop. Takes recording timestamp as only argument, returns a state dict. post_callback: Optional function to call after main loop. Takes state dict as @@ -422,8 +421,7 @@ def write_events( for _ in range(num_processed): progress.update() if not started: - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() started = True try: event = write_q.get_nowait() @@ -705,7 +703,7 @@ def read_screen_events( event_q: queue.Queue, terminate_processing: multiprocessing.Event, recording: Recording, - started_counter: multiprocessing.Value, + started_event: threading.Event, # TODO: throttle # max_cpu_percent: float = 50.0, # Maximum allowed CPU percent # max_memory_percent: float = 50.0, # Maximum allowed memory percent @@ -717,7 +715,7 @@ def read_screen_events( event_q: A queue for adding screen events. terminate_processing: An event to signal the termination of the process. recording: The recording object. - started_counter: Value to increment once started. + started_event: Event to set once started. """ utils.set_start_time(recording.timestamp) @@ -729,8 +727,7 @@ def read_screen_events( logger.warning("Screenshot was None") continue if not started: - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() started = True event_q.put(Event(utils.get_timestamp(), "screen", screenshot)) logger.info("Done") @@ -741,7 +738,7 @@ def read_window_events( event_q: queue.Queue, terminate_processing: multiprocessing.Event, recording: Recording, - started_counter: multiprocessing.Value, + started_event: threading.Event, ) -> None: """Read window events and add them to the event queue. @@ -749,7 +746,7 @@ def read_window_events( event_q: A queue for adding window events. terminate_processing: An event to signal the termination of the process. recording: The recording object. - started_counter: Value to increment once started. + started_event: Event to set once started. """ utils.set_start_time(recording.timestamp) @@ -762,8 +759,7 @@ def read_window_events( continue if not started: - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() started = True if window_data["title"] != prev_window_data.get("title") or window_data[ @@ -796,7 +792,7 @@ def performance_stats_writer( perf_q: sq.SynchronizedQueue, recording: Recording, terminate_processing: multiprocessing.Event, - started_counter: multiprocessing.Value, + started_event: multiprocessing.Event, ) -> None: """Write performance stats to the database. @@ -806,7 +802,7 @@ def performance_stats_writer( perf_q: A queue for collecting performance data. recording: The recording object. terminate_processing: An event to signal the termination of the process. - started_counter: Value to increment once started. + started_event: Event to set once started. """ utils.set_start_time(recording.timestamp) @@ -816,8 +812,7 @@ def performance_stats_writer( session = crud.get_new_session(read_and_write=True) while not terminate_processing.is_set() or not perf_q.empty(): if not started: - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() started = True try: event_type, start_time, end_time = perf_q.get_nowait() @@ -838,7 +833,7 @@ def memory_writer( recording: Recording, terminate_processing: multiprocessing.Event, record_pid: int, - started_counter: multiprocessing.Value, + started_event: multiprocessing.Event, ) -> None: """Writes memory usage statistics to the database. @@ -847,7 +842,7 @@ def memory_writer( terminate_processing (multiprocessing.Event): The event used to terminate the process. record_pid (int): The process ID to monitor memory usage for. - started_counter: Value to increment once started. + started_event: Event to set once started. Returns: None @@ -862,8 +857,7 @@ def memory_writer( session = crud.get_new_session(read_and_write=True) while not terminate_processing.is_set(): if not started: - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() started = True memory_usage_bytes = 0 @@ -928,7 +922,7 @@ def read_keyboard_events( event_q: queue.Queue, terminate_processing: multiprocessing.Event, recording: Recording, - started_counter: multiprocessing.Value, + started_event: threading.Event, ) -> None: """Reads keyboard events and adds them to the event queue. @@ -937,7 +931,7 @@ def read_keyboard_events( terminate_processing (multiprocessing.Event): The event to signal termination of event reading. recording (Recording): The recording object. - started_counter: Value to increment once started. + started_event: Event to set once started. Returns: None @@ -1028,8 +1022,7 @@ def on_release( # NOTE: listener may not have actually started by now # TODO: handle race condition, e.g. by sending synthetic events from main thread - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() terminate_processing.wait() keyboard_listener.stop() @@ -1039,7 +1032,7 @@ def read_mouse_events( event_q: queue.Queue, terminate_processing: multiprocessing.Event, recording: Recording, - started_counter: multiprocessing.Value, + started_event: threading.Event, ) -> None: """Reads mouse events and adds them to the event queue. @@ -1047,7 +1040,7 @@ def read_mouse_events( event_q: The event queue to add the mouse events to. terminate_processing: The event to signal termination of event reading. recording: The recording object. - started_counter: Value to increment once started. + started_event: Event to set once started. Returns: None @@ -1063,8 +1056,7 @@ def read_mouse_events( # NOTE: listener may not have actually started by now # TODO: handle race condition, e.g. by sending synthetic events from main thread - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() terminate_processing.wait() mouse_listener.stop() @@ -1073,14 +1065,14 @@ def read_mouse_events( def record_audio( recording: Recording, terminate_processing: multiprocessing.Event, - started_counter: multiprocessing.Value, + started_event: multiprocessing.Event, ) -> None: """Record audio narration during the recording and store data in database. Args: recording: The recording object. terminate_processing: An event to signal the termination of the process. - started_counter: Value to increment once started. + started_event: Event to set once started. """ utils.configure_logging(logger, LOG_LEVEL) utils.set_start_time(recording.timestamp) @@ -1110,8 +1102,7 @@ def audio_callback( # NOTE: listener may not have actually started by now # TODO: handle race condition, e.g. by sending synthetic events from main thread - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() terminate_processing.wait() audio_stream.stop() @@ -1222,7 +1213,7 @@ def run_browser_event_server( event_q: queue.Queue, terminate_processing: Event, recording: Recording, - started_counter: multiprocessing.Value, + started_event: threading.Event, ) -> None: """Run the browser event server. @@ -1230,7 +1221,7 @@ def run_browser_event_server( event_q: A queue for adding browser events. terminate_processing: An event to signal the termination of the process. recording: The recording object. - started_counter: Value to increment once started. + started_event: Event to set once started. Returns: None @@ -1253,8 +1244,7 @@ def run_server() -> None: ) as server: ws_server_instance = server logger.info("WebSocket server started") - with started_counter.get_lock(): - started_counter.value += 1 + started_event.set() server.serve_forever() # Start the server in a separate thread @@ -1327,12 +1317,12 @@ def record( perf_q = sq.SynchronizedQueue() if terminate_processing is None: terminate_processing = multiprocessing.Event() - started_counter = multiprocessing.Value("i", 0) task_by_name = {} + task_started_events = {} window_event_reader = threading.Thread( target=read_window_events, - args=(event_q, terminate_processing, recording, started_counter), + args=(event_q, terminate_processing, recording, task_started_events.setdefault('window_event_reader', threading.Event())), ) window_event_reader.start() task_by_name["window_event_reader"] = window_event_reader @@ -1340,28 +1330,28 @@ def record( if config.RECORD_BROWSER_EVENTS: browser_event_reader = threading.Thread( target=run_browser_event_server, - args=(event_q, terminate_processing, recording, started_counter), + args=(event_q, terminate_processing, recording, task_started_events.setdefault('browser_event_reader', threading.Event())), ) browser_event_reader.start() task_by_name["browser_event_reader"] = browser_event_reader screen_event_reader = threading.Thread( target=read_screen_events, - args=(event_q, terminate_processing, recording, started_counter), + args=(event_q, terminate_processing, recording, task_started_events.setdefault('screen_event_reader', threading.Event())), ) screen_event_reader.start() task_by_name["screen_event_reader"] = screen_event_reader keyboard_event_reader = threading.Thread( target=read_keyboard_events, - args=(event_q, terminate_processing, recording, started_counter), + args=(event_q, terminate_processing, recording, task_started_events.setdefault('keyboard_event_reader', threading.Event())), ) keyboard_event_reader.start() task_by_name["keyboard_event_reader"] = keyboard_event_reader mouse_event_reader = threading.Thread( target=read_mouse_events, - args=(event_q, terminate_processing, recording, started_counter), + args=(event_q, terminate_processing, recording, task_started_events.setdefault('mouse_event_reader', threading.Event())), ) mouse_event_reader.start() task_by_name["mouse_event_reader"] = mouse_event_reader @@ -1384,7 +1374,7 @@ def record( perf_q, recording, terminate_processing, - started_counter, + task_started_events.setdefault('event_processor', threading.Event()), num_screen_events, num_action_events, num_window_events, @@ -1405,7 +1395,7 @@ def record( perf_q, recording, terminate_processing, - started_counter, + task_started_events.setdefault('screen_event_writer', multiprocessing.Event()), ), ) screen_event_writer.start() @@ -1422,7 +1412,7 @@ def record( perf_q, recording, terminate_processing, - started_counter, + task_started_events.setdefault('browser_event_writer', multiprocessing.Event()), ), ) browser_event_writer.start() @@ -1438,7 +1428,7 @@ def record( perf_q, recording, terminate_processing, - started_counter, + task_started_events.setdefault('action_event_writer', multiprocessing.Event()), ), ) action_event_writer.start() @@ -1454,7 +1444,7 @@ def record( perf_q, recording, terminate_processing, - started_counter, + task_started_events.setdefault('window_event_writer', multiprocessing.Event()), ), ) window_event_writer.start() @@ -1471,7 +1461,7 @@ def record( perf_q, recording, terminate_processing, - started_counter, + task_started_events.setdefault('video_writer', multiprocessing.Event()), video_pre_callback, video_post_callback, ), @@ -1485,7 +1475,7 @@ def record( args=( recording, terminate_processing, - started_counter, + task_started_events.setdefault('audio_event_writer', multiprocessing.Event()), ), ) audio_recorder.start() @@ -1498,7 +1488,7 @@ def record( perf_q, recording, terminate_perf_event, - started_counter, + task_started_events.setdefault('perf_stats_writer', multiprocessing.Event()), ), ) perf_stats_writer.start() @@ -1512,7 +1502,7 @@ def record( recording, terminate_perf_event, record_pid, - started_counter, + task_started_events.setdefault('mem_writer', multiprocessing.Event()), ), ) mem_writer.start() @@ -1530,9 +1520,14 @@ def record( expected_starts = len(task_by_name) logger.info(f"{expected_starts=}") while True: - if started_counter.value >= expected_starts: + started_tasks = sum(event.is_set() for event in task_started_events.values()) + if started_tasks >= expected_starts: break - time.sleep(0.1) # Sleep to reduce busy waiting + waiting_for = [task for task, event in task_started_events.items() if not event.is_set()] + logger.info(f"Waiting for tasks to start: {waiting_for}") + logger.info(f"Started tasks: {started_tasks}/{expected_starts}") + time.sleep(1) # Sleep to reduce busy waiting + for _ in range(5): logger.info("*" * 40) logger.info("All readers and writers have started. Waiting for input events...") From 4b20fcb8646ed17d19412adb80440d78016baf57 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 13:06:24 -0500 Subject: [PATCH 13/31] thread_local -> process_local --- openadapt/utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index 079cd95f8..d5d6f8f47 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -60,8 +60,8 @@ _start_time = None _start_perf_counter = None -# Thread-local storage to ensure thread-safe reuse of `mss` instances -_thread_local = threading.local() +# Process-local storage for MSS instances +_process_local = threading.local() def configure_logging(logger: logger, log_level: str) -> None: @@ -268,11 +268,11 @@ def get_double_click_distance_pixels() -> int: return DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS -def get_thread_local_sct() -> mss.mss: +def get_process_local_sct() -> mss.mss: """Retrieve or create the `mss` instance for the current thread.""" - if not hasattr(_thread_local, "sct"): - _thread_local.sct = mss.mss() - return _thread_local.sct + if not hasattr(_process_local, "sct"): + _process_local.sct = mss.mss() + return _process_local.sct def get_monitor_dims() -> tuple[int, int]: @@ -282,7 +282,7 @@ def get_monitor_dims() -> tuple[int, int]: tuple[int, int]: The width and height of the monitor. """ # TODO XXX: replace with get_screenshot().size and remove get_scale_ratios? - monitor = get_thread_local_sct().monitors[0] + monitor = get_process_local_sct().monitors[0] monitor_width = monitor["width"] monitor_height = monitor["height"] return monitor_width, monitor_height @@ -432,7 +432,7 @@ def take_screenshot() -> Image.Image: PIL.Image: The screenshot image. """ # monitor 0 is all in one - sct = get_thread_local_sct() + sct = get_process_local_sct() monitor = sct.monitors[0] sct_img = sct.grab(monitor) image = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") From 3ec5da6e15ec90ef7aff9aabf0b9e872c7688d96 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 13:08:05 -0500 Subject: [PATCH 14/31] threading.local -> multiprocessing.local --- openadapt/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index d5d6f8f47..635e3a610 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -11,6 +11,7 @@ import base64 import importlib.metadata import inspect +import multiprocessing import os import sys import threading @@ -61,7 +62,7 @@ _start_perf_counter = None # Process-local storage for MSS instances -_process_local = threading.local() +_process_local = multiprocessing.local() def configure_logging(logger: logger, log_level: str) -> None: From 8dc38f6316ea940defa80131b61622025a3de370 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 13:14:12 -0500 Subject: [PATCH 15/31] multiprocessing_utils; xcffib --- openadapt/utils.py | 4 ++-- poetry.lock | 32 +++++++++++++++++++++++++++++++- pyproject.toml | 2 ++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index 635e3a610..b2c5e1f2a 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -11,7 +11,6 @@ import base64 import importlib.metadata import inspect -import multiprocessing import os import sys import threading @@ -21,6 +20,7 @@ from jinja2 import Environment, FileSystemLoader from PIL import Image, ImageEnhance from posthog import Posthog +import multiprocessing_utils from openadapt.build_utils import is_running_from_executable, redirect_stdout_stderr from openadapt.custom_logger import logger @@ -62,7 +62,7 @@ _start_perf_counter = None # Process-local storage for MSS instances -_process_local = multiprocessing.local() +_process_local = multiprocessing_utils.local() def configure_logging(logger: logger, log_level: str) -> None: diff --git a/poetry.lock b/poetry.lock index 5c56d682f..7adb445b4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3915,6 +3915,23 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} +[[package]] +name = "multiprocessing-utils" +version = "0.4" +description = "Multiprocessing utils (shared locks and thread locals)" +optional = false +python-versions = "*" +files = [ + {file = "multiprocessing_utils-0.4-py2.py3-none-any.whl", hash = "sha256:c232e0bbc6ba753ca7a0df5d49b0cc4e26454635d4f373f5133c551aec8c27ee"}, + {file = "multiprocessing_utils-0.4.tar.gz", hash = "sha256:43281d5e017d9b3f3e6114762c21f10c2a2b0392837c5096380e6c413ae79b6c"}, +] + +[package.dependencies] +six = "*" + +[package.extras] +test = ["tox"] + [[package]] name = "murmurhash" version = "1.0.10" @@ -8968,6 +8985,19 @@ files = [ {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, ] +[[package]] +name = "xcffib" +version = "1.5.0" +description = "A drop in replacement for xpyb, an XCB python binding" +optional = false +python-versions = "*" +files = [ + {file = "xcffib-1.5.0.tar.gz", hash = "sha256:a95c9465f2f97b4fcede606bd1e08407a32df71cb760fd57bfe53677db691acc"}, +] + +[package.dependencies] +cffi = ">=1.1.0" + [[package]] name = "yarl" version = "1.17.1" @@ -9121,4 +9151,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "3.10.x" -content-hash = "c308a565dc79a85f2da6bce428d5af53fb03c8e531114a6bfa00299a613ce803" +content-hash = "5ee4ca5a50f3fc9e3e2b43028b9b6dcb8318e58bf0e4cf01a4df32d9d5c425c2" diff --git a/pyproject.toml b/pyproject.toml index 87e154c77..bd6410f9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ alembic = "1.8.1" black = "^24.8.0" pygetwindow = { version = "<0.0.5", markers = "sys_platform == 'win32'" } pywin32 = { version = "306", markers = "sys_platform == 'win32'" } +xcffib = { version = "1.5.0", markers = "sys_platform == 'linux'" } ascii-magic = "2.3.0" bokeh = "2.4.3" clipboard = "0.0.4" @@ -109,6 +110,7 @@ numba = "^0.60.0" llvmlite = "^0.43.0" ell-ai = "^0.0.14" pynput = "^1.7.7" +multiprocessing-utils = "^0.4" [tool.pytest.ini_options] filterwarnings = [ # suppress warnings starting from "setuptools>=67.3" From 4fa4d4e76e0c0bf4a263860f4223668805457dab Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 13:35:01 -0500 Subject: [PATCH 16/31] global monitor_width/monitor_height --- openadapt/record.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 39ce4f5ca..3c9615c38 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -65,6 +65,9 @@ stop_sequence_detected = False ws_server_instance = None +# TODO XXX replace with utils.get_monitor_dims() once fixed +monitor_width, monitor_height = utils.take_screenshot().size + def collect_stats(performance_snapshots: list[tracemalloc.Snapshot]) -> None: """Collects and appends performance snapshots using tracemalloc. @@ -458,10 +461,8 @@ def video_pre_callback(db: crud.SaSession, recording: Recording) -> dict[str, An dict[str, Any]: The updated state. """ video_file_path = video.get_video_file_path(recording.timestamp) - # TODO XXX replace with utils.get_monitor_dims() once fixed - width, height = utils.take_screenshot().size video_container, video_stream, video_start_timestamp = ( - video.initialize_video_writer(video_file_path, width, height) + video.initialize_video_writer(video_file_path, monitor_width, monitor_height) ) crud.update_video_start_time(db, recording, video_start_timestamp) return { From 246294173923e6c2ed287d56556e89fa89d9a319 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 13:58:02 -0500 Subject: [PATCH 17/31] black --- openadapt/record.py | 71 ++++++++++++++++++++++++++++++-------- openadapt/window/_linux.py | 22 +++++++----- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 3c9615c38..4740da8c9 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -1323,7 +1323,12 @@ def record( window_event_reader = threading.Thread( target=read_window_events, - args=(event_q, terminate_processing, recording, task_started_events.setdefault('window_event_reader', threading.Event())), + args=( + event_q, + terminate_processing, + recording, + task_started_events.setdefault("window_event_reader", threading.Event()), + ), ) window_event_reader.start() task_by_name["window_event_reader"] = window_event_reader @@ -1331,28 +1336,50 @@ def record( if config.RECORD_BROWSER_EVENTS: browser_event_reader = threading.Thread( target=run_browser_event_server, - args=(event_q, terminate_processing, recording, task_started_events.setdefault('browser_event_reader', threading.Event())), + args=( + event_q, + terminate_processing, + recording, + task_started_events.setdefault( + "browser_event_reader", threading.Event() + ), + ), ) browser_event_reader.start() task_by_name["browser_event_reader"] = browser_event_reader screen_event_reader = threading.Thread( target=read_screen_events, - args=(event_q, terminate_processing, recording, task_started_events.setdefault('screen_event_reader', threading.Event())), + args=( + event_q, + terminate_processing, + recording, + task_started_events.setdefault("screen_event_reader", threading.Event()), + ), ) screen_event_reader.start() task_by_name["screen_event_reader"] = screen_event_reader keyboard_event_reader = threading.Thread( target=read_keyboard_events, - args=(event_q, terminate_processing, recording, task_started_events.setdefault('keyboard_event_reader', threading.Event())), + args=( + event_q, + terminate_processing, + recording, + task_started_events.setdefault("keyboard_event_reader", threading.Event()), + ), ) keyboard_event_reader.start() task_by_name["keyboard_event_reader"] = keyboard_event_reader mouse_event_reader = threading.Thread( target=read_mouse_events, - args=(event_q, terminate_processing, recording, task_started_events.setdefault('mouse_event_reader', threading.Event())), + args=( + event_q, + terminate_processing, + recording, + task_started_events.setdefault("mouse_event_reader", threading.Event()), + ), ) mouse_event_reader.start() task_by_name["mouse_event_reader"] = mouse_event_reader @@ -1375,7 +1402,7 @@ def record( perf_q, recording, terminate_processing, - task_started_events.setdefault('event_processor', threading.Event()), + task_started_events.setdefault("event_processor", threading.Event()), num_screen_events, num_action_events, num_window_events, @@ -1396,7 +1423,9 @@ def record( perf_q, recording, terminate_processing, - task_started_events.setdefault('screen_event_writer', multiprocessing.Event()), + task_started_events.setdefault( + "screen_event_writer", multiprocessing.Event() + ), ), ) screen_event_writer.start() @@ -1413,7 +1442,9 @@ def record( perf_q, recording, terminate_processing, - task_started_events.setdefault('browser_event_writer', multiprocessing.Event()), + task_started_events.setdefault( + "browser_event_writer", multiprocessing.Event() + ), ), ) browser_event_writer.start() @@ -1429,7 +1460,9 @@ def record( perf_q, recording, terminate_processing, - task_started_events.setdefault('action_event_writer', multiprocessing.Event()), + task_started_events.setdefault( + "action_event_writer", multiprocessing.Event() + ), ), ) action_event_writer.start() @@ -1445,7 +1478,9 @@ def record( perf_q, recording, terminate_processing, - task_started_events.setdefault('window_event_writer', multiprocessing.Event()), + task_started_events.setdefault( + "window_event_writer", multiprocessing.Event() + ), ), ) window_event_writer.start() @@ -1462,7 +1497,7 @@ def record( perf_q, recording, terminate_processing, - task_started_events.setdefault('video_writer', multiprocessing.Event()), + task_started_events.setdefault("video_writer", multiprocessing.Event()), video_pre_callback, video_post_callback, ), @@ -1476,7 +1511,9 @@ def record( args=( recording, terminate_processing, - task_started_events.setdefault('audio_event_writer', multiprocessing.Event()), + task_started_events.setdefault( + "audio_event_writer", multiprocessing.Event() + ), ), ) audio_recorder.start() @@ -1489,7 +1526,9 @@ def record( perf_q, recording, terminate_perf_event, - task_started_events.setdefault('perf_stats_writer', multiprocessing.Event()), + task_started_events.setdefault( + "perf_stats_writer", multiprocessing.Event() + ), ), ) perf_stats_writer.start() @@ -1503,7 +1542,7 @@ def record( recording, terminate_perf_event, record_pid, - task_started_events.setdefault('mem_writer', multiprocessing.Event()), + task_started_events.setdefault("mem_writer", multiprocessing.Event()), ), ) mem_writer.start() @@ -1524,7 +1563,9 @@ def record( started_tasks = sum(event.is_set() for event in task_started_events.values()) if started_tasks >= expected_starts: break - waiting_for = [task for task, event in task_started_events.items() if not event.is_set()] + waiting_for = [ + task for task, event in task_started_events.items() if not event.is_set() + ] logger.info(f"Waiting for tasks to start: {waiting_for}") logger.info(f"Started tasks: {started_tasks}/{expected_starts}") time.sleep(1) # Sleep to reduce busy waiting diff --git a/openadapt/window/_linux.py b/openadapt/window/_linux.py index 2fb4fb0d2..3ad5568c6 100644 --- a/openadapt/window/_linux.py +++ b/openadapt/window/_linux.py @@ -32,9 +32,11 @@ def get_active_window_meta() -> dict | None: root = conn.get_setup().roots[0].root # Get the _NET_ACTIVE_WINDOW atom - atom = conn.core.InternAtom( - False, len("_NET_ACTIVE_WINDOW"), "_NET_ACTIVE_WINDOW" - ).reply().atom + atom = ( + conn.core.InternAtom(False, len("_NET_ACTIVE_WINDOW"), "_NET_ACTIVE_WINDOW") + .reply() + .atom + ) # Fetch the active window ID active_window = conn.core.GetProperty( @@ -75,9 +77,11 @@ def get_window_title(conn: xcffib.Connection, window_id: int) -> str: """ try: # Attempt to fetch _NET_WM_NAME - atom_net_wm_name = conn.core.InternAtom( - False, len("_NET_WM_NAME"), "_NET_WM_NAME" - ).reply().atom + atom_net_wm_name = ( + conn.core.InternAtom(False, len("_NET_WM_NAME"), "_NET_WM_NAME") + .reply() + .atom + ) title_property = conn.core.GetProperty( False, window_id, atom_net_wm_name, xcffib.xproto.Atom.STRING, 0, 1024 ).reply() @@ -86,9 +90,9 @@ def get_window_title(conn: xcffib.Connection, window_id: int) -> str: return title_bytes.decode("utf-8") # Fallback to WM_NAME - atom_wm_name = conn.core.InternAtom( - False, len("WM_NAME"), "WM_NAME" - ).reply().atom + atom_wm_name = ( + conn.core.InternAtom(False, len("WM_NAME"), "WM_NAME").reply().atom + ) title_property = conn.core.GetProperty( False, window_id, atom_wm_name, xcffib.xproto.Atom.STRING, 0, 1024 ).reply() From b38df7850224325082226caa47f658cb7b8f6474 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 13:58:16 -0500 Subject: [PATCH 18/31] add capture._linux --- openadapt/capture/__init__.py | 8 ++- openadapt/capture/_linux.py | 119 ++++++++++++++++++++++++++++++++++ openadapt/capture/_macos.py | 5 +- openadapt/capture/_windows.py | 6 +- 4 files changed, 127 insertions(+), 11 deletions(-) create mode 100644 openadapt/capture/_linux.py diff --git a/openadapt/capture/__init__.py b/openadapt/capture/__init__.py index 91ae7b98c..dd572551a 100644 --- a/openadapt/capture/__init__.py +++ b/openadapt/capture/__init__.py @@ -9,13 +9,15 @@ if sys.platform == "darwin": from . import _macos as impl - device = impl.Capture() elif sys.platform == "win32": from . import _windows as impl - device = impl.Capture() +elif sys.platform == "linux": + from . import _linux as impl else: print(f"WARNING: openadapt.capture is not yet supported on {sys.platform=}") - device = None + impl = None + +device = impl.Capture() if impl else None def get_capture() -> impl.Capture: diff --git a/openadapt/capture/_linux.py b/openadapt/capture/_linux.py new file mode 100644 index 000000000..8da5f4a77 --- /dev/null +++ b/openadapt/capture/_linux.py @@ -0,0 +1,119 @@ +"""Allows for capturing the screen and audio on Linux. + +usage: see bottom of file +""" + +import os +import subprocess +from datetime import datetime +from sys import platform +import wave +import pyaudio + +from openadapt.config import CAPTURE_DIR_PATH + + +class Capture: + """Capture the screen, audio, and camera on Linux.""" + + def __init__(self) -> None: + """Initialize the capture object.""" + assert platform == "linux", platform + + self.is_recording = False + self.audio_out = None + self.video_out = None + self.audio_stream = None + self.audio_frames = [] + + # Initialize PyAudio + self.audio = pyaudio.PyAudio() + + def start(self, audio: bool = True, camera: bool = False) -> None: + """Start capturing the screen, audio, and camera. + + Args: + audio (bool, optional): Whether to capture audio (default: True). + camera (bool, optional): Whether to capture the camera (default: False). + """ + if self.is_recording: + raise RuntimeError("Recording is already in progress") + + self.is_recording = True + capture_dir = CAPTURE_DIR_PATH + if not os.path.exists(capture_dir): + os.mkdir(capture_dir) + + # Start video capture using ffmpeg + video_filename = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + ".mp4" + self.video_out = os.path.join(capture_dir, video_filename) + self._start_video_capture() + + # Start audio capture + if audio: + audio_filename = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + ".wav" + self.audio_out = os.path.join(capture_dir, audio_filename) + self._start_audio_capture() + + def _start_video_capture(self) -> None: + """Start capturing the screen using ffmpeg.""" + cmd = [ + "ffmpeg", + "-f", "x11grab", # Capture X11 display + "-video_size", "1920x1080", # Set resolution + "-framerate", "30", # Set frame rate + "-i", ":0.0", # Capture from display 0 + "-c:v", "libx264", # Video codec + "-preset", "ultrafast", # Speed/quality tradeoff + "-y", self.video_out # Output file + ] + self.video_proc = subprocess.Popen(cmd) + + def _start_audio_capture(self) -> None: + """Start capturing audio using PyAudio.""" + self.audio_stream = self.audio.open( + format=pyaudio.paInt16, + channels=2, + rate=44100, + input=True, + frames_per_buffer=1024, + stream_callback=self._audio_callback + ) + self.audio_frames = [] + self.audio_stream.start_stream() + + def _audio_callback(self, in_data: bytes, frame_count: int, time_info: dict, status: int) -> tuple: + """Callback function to process audio data.""" + self.audio_frames.append(in_data) + return (None, pyaudio.paContinue) + + def stop(self) -> None: + """Stop capturing the screen, audio, and camera.""" + if self.is_recording: + # Stop the video capture + self.video_proc.terminate() + + # Stop audio capture + if self.audio_stream: + self.audio_stream.stop_stream() + self.audio_stream.close() + self.audio.terminate() + self.save_audio() + + self.is_recording = False + + def save_audio(self) -> None: + """Save the captured audio to a WAV file.""" + if self.audio_out: + with wave.open(self.audio_out, "wb") as wf: + wf.setnchannels(2) + wf.setsampwidth(self.audio.get_sample_size(pyaudio.paInt16)) + wf.setframerate(44100) + wf.writeframes(b"".join(self.audio_frames)) + + +if __name__ == "__main__": + capture = Capture() + capture.start(audio=True, camera=False) + input("Press enter to stop") + capture.stop() diff --git a/openadapt/capture/_macos.py b/openadapt/capture/_macos.py index 65529910c..d2c00e7d4 100644 --- a/openadapt/capture/_macos.py +++ b/openadapt/capture/_macos.py @@ -23,10 +23,7 @@ class Capture: def __init__(self) -> None: """Initialize the capture object.""" - if platform != "darwin": - raise NotImplementedError( - "This is the macOS implementation, please use the Windows version" - ) + assert platform == "darwin", platform objc.options.structs_indexable = True diff --git a/openadapt/capture/_windows.py b/openadapt/capture/_windows.py index ab400c950..ad09e48b1 100644 --- a/openadapt/capture/_windows.py +++ b/openadapt/capture/_windows.py @@ -21,10 +21,8 @@ def __init__(self, pid: int = 0) -> None: pid (int, optional): The process ID of the window to capture. Defaults to 0 (the entire screen) """ - if platform != "win32": - raise NotImplementedError( - "This is the Windows implementation, please use the macOS version" - ) + assert platform == "win32", platform + self.is_recording = False self.video_out = None self.audio_out = None From 1be7c884d8bb4fe00681e7faf305ddafb2b1f47d Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 14:17:40 -0500 Subject: [PATCH 19/31] capture._linux.get_screen_resolution --- openadapt/capture/_linux.py | 61 +++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/openadapt/capture/_linux.py b/openadapt/capture/_linux.py index 8da5f4a77..443b541f4 100644 --- a/openadapt/capture/_linux.py +++ b/openadapt/capture/_linux.py @@ -1,14 +1,9 @@ -"""Allows for capturing the screen and audio on Linux. - -usage: see bottom of file -""" - -import os import subprocess +import os from datetime import datetime from sys import platform -import wave import pyaudio +import wave from openadapt.config import CAPTURE_DIR_PATH @@ -18,7 +13,8 @@ class Capture: def __init__(self) -> None: """Initialize the capture object.""" - assert platform == "linux", platform + if platform != "linux": + assert platform == "linux", platform self.is_recording = False self.audio_out = None @@ -29,6 +25,19 @@ def __init__(self) -> None: # Initialize PyAudio self.audio = pyaudio.PyAudio() + def get_screen_resolution(self) -> tuple: + """Get the screen resolution dynamically using xrandr.""" + try: + # Get screen resolution using xrandr + output = subprocess.check_output( + "xrandr | grep '*' | awk '{print $1}'", shell=True + ) + resolution = output.decode("utf-8").strip() + width, height = resolution.split("x") + return int(width), int(height) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to get screen resolution: {e}") + def start(self, audio: bool = True, camera: bool = False) -> None: """Start capturing the screen, audio, and camera. @@ -44,10 +53,13 @@ def start(self, audio: bool = True, camera: bool = False) -> None: if not os.path.exists(capture_dir): os.mkdir(capture_dir) + # Get the screen resolution dynamically + screen_width, screen_height = self.get_screen_resolution() + # Start video capture using ffmpeg video_filename = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + ".mp4" self.video_out = os.path.join(capture_dir, video_filename) - self._start_video_capture() + self._start_video_capture(screen_width, screen_height) # Start audio capture if audio: @@ -55,17 +67,24 @@ def start(self, audio: bool = True, camera: bool = False) -> None: self.audio_out = os.path.join(capture_dir, audio_filename) self._start_audio_capture() - def _start_video_capture(self) -> None: - """Start capturing the screen using ffmpeg.""" + def _start_video_capture(self, width: int, height: int) -> None: + """Start capturing the screen using ffmpeg with the dynamic resolution.""" cmd = [ "ffmpeg", - "-f", "x11grab", # Capture X11 display - "-video_size", "1920x1080", # Set resolution - "-framerate", "30", # Set frame rate - "-i", ":0.0", # Capture from display 0 - "-c:v", "libx264", # Video codec - "-preset", "ultrafast", # Speed/quality tradeoff - "-y", self.video_out # Output file + "-f", + "x11grab", # Capture X11 display + "-video_size", + f"{width}x{height}", # Use dynamic screen resolution + "-framerate", + "30", # Set frame rate + "-i", + ":0.0", # Capture from display 0 + "-c:v", + "libx264", # Video codec + "-preset", + "ultrafast", # Speed/quality tradeoff + "-y", + self.video_out, # Output file ] self.video_proc = subprocess.Popen(cmd) @@ -77,12 +96,14 @@ def _start_audio_capture(self) -> None: rate=44100, input=True, frames_per_buffer=1024, - stream_callback=self._audio_callback + stream_callback=self._audio_callback, ) self.audio_frames = [] self.audio_stream.start_stream() - def _audio_callback(self, in_data: bytes, frame_count: int, time_info: dict, status: int) -> tuple: + def _audio_callback( + self, in_data: bytes, frame_count: int, time_info: dict, status: int + ) -> tuple: """Callback function to process audio data.""" self.audio_frames.append(in_data) return (None, pyaudio.paContinue) From ee10863d27e0d5427391e7015f1c8a7eeca689a7 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 14:31:41 -0500 Subject: [PATCH 20/31] cleanup --- openadapt/capture/__init__.py | 2 -- openadapt/window/__init__.py | 7 +------ openadapt/window/_linux.py | 2 +- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/openadapt/capture/__init__.py b/openadapt/capture/__init__.py index dd572551a..279f84844 100644 --- a/openadapt/capture/__init__.py +++ b/openadapt/capture/__init__.py @@ -1,7 +1,5 @@ """Capture the screen, audio, and camera as a video on macOS and Windows. -TODO: support Linux - Module: capture.py """ diff --git a/openadapt/window/__init__.py b/openadapt/window/__init__.py index 08a3097cc..f146d367a 100644 --- a/openadapt/window/__init__.py +++ b/openadapt/window/__init__.py @@ -16,8 +16,7 @@ elif sys.platform.startswith("linux"): from . import _linux as impl else: - impl = None - logger.warning(f"openadapt.window is not yet supported on {sys.platform=}") + raise Exception(f"Unsupported platform: {sys.platform}") def get_active_window_data( @@ -60,8 +59,6 @@ def get_active_window_state(read_window_data: bool) -> dict | None: dict or None: A dictionary containing the state of the active window, or None if the state is not available. """ - if not impl: - return None # TODO: save window identifier (a window's title can change, or # multiple windows can have the same title) try: @@ -82,8 +79,6 @@ def get_active_element_state(x: int, y: int) -> dict | None: dict or None: A dictionary containing the state of the active element, or None if the state is not available. """ - if not impl: - return None try: return impl.get_active_element_state(x, y) except Exception as exc: diff --git a/openadapt/window/_linux.py b/openadapt/window/_linux.py index 3ad5568c6..dbf329dc1 100644 --- a/openadapt/window/_linux.py +++ b/openadapt/window/_linux.py @@ -149,7 +149,7 @@ def get_window_data(meta: dict) -> dict: Returns: dict: Detailed data of the window. """ - # Placeholder for additional detailed data retrieval. + # TODO: implement, e.g. with pyatspi return {} From 9a5f66e8aa421bf26350279f2824cdb5932a8c48 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 14:39:51 -0500 Subject: [PATCH 21/31] get_double_click_interval_seconds/pixels on linux --- openadapt/utils.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index b2c5e1f2a..fdfa8604f 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -12,6 +12,7 @@ import importlib.metadata import inspect import os +import subprocess import sys import threading import time @@ -237,8 +238,21 @@ def get_double_click_interval_seconds() -> float: from ctypes import windll return windll.user32.GetDoubleClickTime() / 1000 - else: + elif sys.platform.startswith("linux"): + try: + output = subprocess.check_output( + ["xprop", "-root", "_NET_DOUBLE_CLICK_TIME"], stderr=subprocess.DEVNULL + ).decode() + if "_NET_DOUBLE_CLICK_TIME(CARDINAL)" in output: + interval_ms = int(output.split()[-1]) + return interval_ms / 1000 + except (subprocess.CalledProcessError, ValueError, FileNotFoundError): + logger.warning( + "Failed to fetch double click interval. Falling back to default." + ) return DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS + else: + raise Exception(f"Unsupported platform: {sys.platform}") def get_double_click_distance_pixels() -> int: @@ -265,8 +279,24 @@ def get_double_click_distance_pixels() -> int: if x != y: logger.warning(f"{x=} != {y=}") return max(x, y) - else: + elif sys.platform.startswith("linux"): + try: + output = subprocess.check_output( + ["xdpyinfo"], stderr=subprocess.DEVNULL + ).decode() + for line in output.splitlines(): + if "resolution:" in line: + dpi = int(line.split()[1].split("x")[0]) # Get horizontal DPI + # Estimate double-click distance as 4mm converted to pixels. + # 1 inch = 25.4mm, so dpi / 25.4 = pixels per mm. + return int(dpi / 25.4 * 4) + except (subprocess.CalledProcessError, ValueError, FileNotFoundError): + logger.warning( + "Failed to fetch double click distance. Falling back to default." + ) return DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS + else: + raise Exception(f"Unsupported platform: {sys.platform}") def get_process_local_sct() -> mss.mss: From c65d66f58680e8f3967b0b444b1a754560d1aa08 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 15:00:02 -0500 Subject: [PATCH 22/31] gnome/kde cmd --- openadapt/utils.py | 50 +++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index fdfa8604f..aecfd0030 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -221,6 +221,22 @@ def override_double_click_interval_seconds( get_double_click_interval_seconds.override_value = override_value +def get_linux_setting(gnome_command, kde_command, default_value): + """Try to get a setting from GNOME or KDE, falling back to a default value.""" + try: + # Try GNOME first + output = subprocess.check_output(gnome_command, shell=True).decode().strip() + return int(output) + except (subprocess.CalledProcessError, ValueError): + try: + # If GNOME fails, try KDE + output = subprocess.check_output(kde_command, shell=True).decode().strip() + return int(output) + except (subprocess.CalledProcessError, ValueError): + # If both fail, return the default value + return default_value + + def get_double_click_interval_seconds() -> float: """Get the double click interval in seconds. @@ -239,18 +255,10 @@ def get_double_click_interval_seconds() -> float: return windll.user32.GetDoubleClickTime() / 1000 elif sys.platform.startswith("linux"): - try: - output = subprocess.check_output( - ["xprop", "-root", "_NET_DOUBLE_CLICK_TIME"], stderr=subprocess.DEVNULL - ).decode() - if "_NET_DOUBLE_CLICK_TIME(CARDINAL)" in output: - interval_ms = int(output.split()[-1]) - return interval_ms / 1000 - except (subprocess.CalledProcessError, ValueError, FileNotFoundError): - logger.warning( - "Failed to fetch double click interval. Falling back to default." - ) - return DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS + gnome_cmd = "gsettings get org.gnome.desktop.peripherals.mouse double-click" + kde_cmd = "kreadconfig5 --group KDE --key DoubleClickInterval" + value = get_linux_setting(gnome_cmd, kde_cmd, DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS * 1000) + return value / 1000 # Convert from milliseconds to seconds else: raise Exception(f"Unsupported platform: {sys.platform}") @@ -280,21 +288,9 @@ def get_double_click_distance_pixels() -> int: logger.warning(f"{x=} != {y=}") return max(x, y) elif sys.platform.startswith("linux"): - try: - output = subprocess.check_output( - ["xdpyinfo"], stderr=subprocess.DEVNULL - ).decode() - for line in output.splitlines(): - if "resolution:" in line: - dpi = int(line.split()[1].split("x")[0]) # Get horizontal DPI - # Estimate double-click distance as 4mm converted to pixels. - # 1 inch = 25.4mm, so dpi / 25.4 = pixels per mm. - return int(dpi / 25.4 * 4) - except (subprocess.CalledProcessError, ValueError, FileNotFoundError): - logger.warning( - "Failed to fetch double click distance. Falling back to default." - ) - return DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS + gnome_cmd = "gsettings get org.gnome.desktop.peripherals.mouse double-click-distance" + kde_cmd = "kreadconfig5 --group KDE --key DoubleClickDistance" + return get_linux_setting(gnome_cmd, kde_cmd, DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS) else: raise Exception(f"Unsupported platform: {sys.platform}") From b908ff7f0674ad46554b86b9d772cd6708e7d4d4 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 15:00:17 -0500 Subject: [PATCH 23/31] black --- openadapt/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index aecfd0030..dd4d80c1c 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -257,7 +257,9 @@ def get_double_click_interval_seconds() -> float: elif sys.platform.startswith("linux"): gnome_cmd = "gsettings get org.gnome.desktop.peripherals.mouse double-click" kde_cmd = "kreadconfig5 --group KDE --key DoubleClickInterval" - value = get_linux_setting(gnome_cmd, kde_cmd, DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS * 1000) + value = get_linux_setting( + gnome_cmd, kde_cmd, DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS * 1000 + ) return value / 1000 # Convert from milliseconds to seconds else: raise Exception(f"Unsupported platform: {sys.platform}") @@ -288,9 +290,13 @@ def get_double_click_distance_pixels() -> int: logger.warning(f"{x=} != {y=}") return max(x, y) elif sys.platform.startswith("linux"): - gnome_cmd = "gsettings get org.gnome.desktop.peripherals.mouse double-click-distance" + gnome_cmd = ( + "gsettings get org.gnome.desktop.peripherals.mouse double-click-distance" + ) kde_cmd = "kreadconfig5 --group KDE --key DoubleClickDistance" - return get_linux_setting(gnome_cmd, kde_cmd, DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS) + return get_linux_setting( + gnome_cmd, kde_cmd, DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS + ) else: raise Exception(f"Unsupported platform: {sys.platform}") From 02dd31cbb23d8cc0b9a642bd21e0bde55248fbcd Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 15:02:32 -0500 Subject: [PATCH 24/31] fix gnome_cmd --- openadapt/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index dd4d80c1c..d0ba0128f 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -290,9 +290,7 @@ def get_double_click_distance_pixels() -> int: logger.warning(f"{x=} != {y=}") return max(x, y) elif sys.platform.startswith("linux"): - gnome_cmd = ( - "gsettings get org.gnome.desktop.peripherals.mouse double-click-distance" - ) + gnome_cmd = "gsettings get org.gnome.desktop.peripherals.mouse double-click" kde_cmd = "kreadconfig5 --group KDE --key DoubleClickDistance" return get_linux_setting( gnome_cmd, kde_cmd, DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS From 0968b39d10257a6ca17943d6a9c942324a572cca Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 15:35:23 -0500 Subject: [PATCH 25/31] more robust get_double_click_distance_pixels on linux --- openadapt/utils.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index d0ba0128f..6ac7c8115 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -12,6 +12,7 @@ import importlib.metadata import inspect import os +import re import subprocess import sys import threading @@ -265,6 +266,25 @@ def get_double_click_interval_seconds() -> float: raise Exception(f"Unsupported platform: {sys.platform}") +def get_linux_device_id(device_name: str) -> int | None: + """Get the device ID for a device containing the given name. + + Args: + device_name (str): The name to search for in device listings. + + Returns: + Optional[int]: The device ID if found, None otherwise. + """ + try: + output = subprocess.check_output(["xinput", "list"], text=True) + match = re.search(f"\\b{re.escape(device_name)}\\b.*?id=(\\d+)", output) + if match: + return int(match.group(1)) + except (subprocess.CalledProcessError, FileNotFoundError): + pass + return None + + def get_double_click_distance_pixels() -> int: """Get the double click distance in pixels. @@ -290,11 +310,20 @@ def get_double_click_distance_pixels() -> int: logger.warning(f"{x=} != {y=}") return max(x, y) elif sys.platform.startswith("linux"): - gnome_cmd = "gsettings get org.gnome.desktop.peripherals.mouse double-click" - kde_cmd = "kreadconfig5 --group KDE --key DoubleClickDistance" - return get_linux_setting( - gnome_cmd, kde_cmd, DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS - ) + device_id = get_linux_device_id("Mouse") + if device_id is not None: + try: + output = subprocess.check_output( + ["xinput", "list-props", str(device_id)], text=True + ) + match = re.search( + r"libinput Scrolling Pixel Distance \\((\\d+)\\):\\s+(\\d+)", + output, + ) + if match: + return int(match.group(2)) + except (subprocess.CalledProcessError, FileNotFoundError): + pass else: raise Exception(f"Unsupported platform: {sys.platform}") From 4e3f05a3260e5e7ee29f8c7304a84e3115ac5ea3 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 15:38:29 -0500 Subject: [PATCH 26/31] fix regex --- openadapt/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index 6ac7c8115..f374c852a 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -317,7 +317,7 @@ def get_double_click_distance_pixels() -> int: ["xinput", "list-props", str(device_id)], text=True ) match = re.search( - r"libinput Scrolling Pixel Distance \\((\\d+)\\):\\s+(\\d+)", + r"libinput Scrolling Pixel Distance \((\d+)\):\s+(\d+)", output, ) if match: From ec90949ba0b838ae55bfb76adf8b878033578f86 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 15:45:39 -0500 Subject: [PATCH 27/31] get_xinput_property --- openadapt/utils.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index f374c852a..9b7528709 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -285,6 +285,28 @@ def get_linux_device_id(device_name: str) -> int | None: return None +def get_xinput_property(device_id: int, property_name: str) -> int | None: + """Get a specific property value from xinput for a given device. + + Args: + device_id (int): The ID of the device. + property_name (str): The name of the property to search for. + + Returns: + Optional[int]: The property value if found, None otherwise. + """ + try: + output = subprocess.check_output( + ["xinput", "list-props", str(device_id)], text=True + ) + match = re.search(rf"{re.escape(property_name)} \\(\\d+\\):\\s+(\\d+)", output) + if match: + return int(match.group(2)) + except (subprocess.CalledProcessError, FileNotFoundError): + pass + return None + + def get_double_click_distance_pixels() -> int: """Get the double click distance in pixels. @@ -312,18 +334,10 @@ def get_double_click_distance_pixels() -> int: elif sys.platform.startswith("linux"): device_id = get_linux_device_id("Mouse") if device_id is not None: - try: - output = subprocess.check_output( - ["xinput", "list-props", str(device_id)], text=True - ) - match = re.search( - r"libinput Scrolling Pixel Distance \((\d+)\):\s+(\d+)", - output, - ) - if match: - return int(match.group(2)) - except (subprocess.CalledProcessError, FileNotFoundError): - pass + value = get_xinput_property(device_id, "libinput Scrolling Pixel Distance") + if value is not None: + return value + return DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS else: raise Exception(f"Unsupported platform: {sys.platform}") From 3aa844d193690f2417ae97755f0cd2313b584369 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 15:53:50 -0500 Subject: [PATCH 28/31] fix regex --- openadapt/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index 9b7528709..24b338b0f 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -299,7 +299,7 @@ def get_xinput_property(device_id: int, property_name: str) -> int | None: output = subprocess.check_output( ["xinput", "list-props", str(device_id)], text=True ) - match = re.search(rf"{re.escape(property_name)} \\(\\d+\\):\\s+(\\d+)", output) + match = re.search(rf"{property_name} \((\d+)\):\s+(\d+)", output) if match: return int(match.group(2)) except (subprocess.CalledProcessError, FileNotFoundError): From d3b61f7e13240bc6b664373c678f11e09147192c Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 15:54:55 -0500 Subject: [PATCH 29/31] flake8 --- openadapt/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index 24b338b0f..1af004d05 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -222,7 +222,7 @@ def override_double_click_interval_seconds( get_double_click_interval_seconds.override_value = override_value -def get_linux_setting(gnome_command, kde_command, default_value): +def get_linux_setting(gnome_command: str, kde_command: str, default_value: int) -> int: """Try to get a setting from GNOME or KDE, falling back to a default value.""" try: # Try GNOME first From 29df1baa57d20299bafc0f87b45ada7c1f26a9e4 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 16:13:36 -0500 Subject: [PATCH 30/31] cleanup --- openadapt/capture/__init__.py | 20 ++++++++------------ openadapt/capture/_linux.py | 2 +- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/openadapt/capture/__init__.py b/openadapt/capture/__init__.py index 279f84844..243c00c48 100644 --- a/openadapt/capture/__init__.py +++ b/openadapt/capture/__init__.py @@ -9,13 +9,12 @@ from . import _macos as impl elif sys.platform == "win32": from . import _windows as impl -elif sys.platform == "linux": +elif sys.platform.startswith("linux"): from . import _linux as impl else: - print(f"WARNING: openadapt.capture is not yet supported on {sys.platform=}") - impl = None + raise Exception(f"Unsupported platform: {sys.platform}") -device = impl.Capture() if impl else None +device = impl.Capture() def get_capture() -> impl.Capture: @@ -29,22 +28,19 @@ def get_capture() -> impl.Capture: def start(audio: bool = False, camera: bool = False) -> None: """Start the capture.""" - if device: - device.start(audio=audio, camera=camera) + device.start(audio=audio, camera=camera) def stop() -> None: """Stop the capture.""" - if device: - device.stop() + device.stop() def test() -> None: """Test the capture.""" - if device: - device.start() - input("Press enter to stop") - device.stop() + device.start() + input("Press enter to stop") + device.stop() if __name__ in ("__main__", "capture"): diff --git a/openadapt/capture/_linux.py b/openadapt/capture/_linux.py index 443b541f4..20475f182 100644 --- a/openadapt/capture/_linux.py +++ b/openadapt/capture/_linux.py @@ -13,7 +13,7 @@ class Capture: def __init__(self) -> None: """Initialize the capture object.""" - if platform != "linux": + if not platform.startswith("linux"): assert platform == "linux", platform self.is_recording = False From 34fc8ed522581743f207cc0162db9dd7ae2ce627 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Mon, 30 Dec 2024 16:15:51 -0500 Subject: [PATCH 31/31] update todo --- openadapt/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/utils.py b/openadapt/utils.py index 1af004d05..a457dc2a5 100644 --- a/openadapt/utils.py +++ b/openadapt/utils.py @@ -55,7 +55,7 @@ # TODO: move to constants.py EMPTY = (None, [], {}, "") -# TODO: determine programmatically in Linux +# TODO: move to config.py DEFAULT_DOUBLE_CLICK_INTERVAL_SECONDS = 0.5 DEFAULT_DOUBLE_CLICK_DISTANCE_PIXELS = 5