#!/usr/bin/env python3 """Interact with the in-app browser / preview pane in the Hermes desktop GUI (click, type, scroll). Elements are addressed by legible refs from ``action="elements"`` (``btn-sign-in``); a ref survives re-renders and only a navigation retires it, so the renderer answers with a *delta* instead of re-sending the inventory. Round-trips through the gateway's blocking-prompt bridge (``preview.act.request`` / ``preview.act.respond``); this module is schema + a thin dispatcher over the platform-injected callback. ``desktop_ui`` toolset: desktop-sourced sessions only. """ from typing import Callable, Optional from tools.desktop_ui import passthrough_json from tools.registry import registry, tool_error ACTIONS = ("elements", "click", "hover", "type", "scroll", "press", "strobe", "back", "forward", "reload") SCROLL_TO = ("top", "bottom") # Verbs that need something to act on — a ref from the last inventory, or a # raw CSS selector. `scroll` is deliberately absent: bare, it scrolls the page. NEEDS_TARGET = ("click", "hover", "type", "press") def drive_preview_tool( action: str = "", ref: Optional[str] = None, selector: Optional[str] = None, text: Optional[str] = None, key: Optional[str] = None, submit: Optional[bool] = None, amount: Optional[int] = None, to: Optional[str] = None, limit: Optional[int] = None, full: Optional[bool] = None, allow_shortcut: Optional[bool] = None, callback: Optional[Callable] = None) -> str: """Dispatch one interaction to the desktop renderer and return its outcome.""" if callback is None: return tool_error("drive_preview is only available in the Hermes desktop app.") verb = (action or "").strip().lower() if verb not in ACTIONS: return tool_error(f"action must be one of: {', '.join(ACTIONS)}.") if verb in NEEDS_TARGET and not (ref or selector): return tool_error(f"{verb} needs a ref from action='elements' (e.g. 'btn-sign-in') or a CSS selector.") if verb == "type" and text is None: return tool_error("type needs the text to enter.") if verb == "press" and not key: return tool_error("press needs a key, e.g. 'Enter' or 'Escape'.") if to is not None and to not in SCROLL_TO: return tool_error(f"to must be one of: {', '.join(SCROLL_TO)}.") try: fields = ( ("action", verb), ("ref", ref), ("selector", selector), ("text", text), ("key", key), ("submit", submit), ("full", full), ("to", to), ("allow_shortcut", allow_shortcut), ("amount", None if amount is None else int(amount)), ("max", None if limit is None else int(limit)), ) except (TypeError, ValueError): return tool_error("amount and max must be integers.") try: raw = callback({name: val for name, val in fields if val is not None}) except Exception as exc: return tool_error(f"Failed to act on the in-app browser: {exc}") if not raw: return tool_error( "No GUI window answered with a page: no preview tab is open. " "Open a page with open_preview first. If the pane IS open, the desktop app " "may be older than this backend — its bridge-unavailable error names that case.") return passthrough_json(raw) ACT_PREVIEW_SCHEMA = { "name": "drive_preview", # Response-shape teaching kept only where skipping it wastes calls (delta # semantics, rebound refs, strobe's burst): a model that doesn't know them # re-reads pages or loops strobe. # See #95681. "description": ( "Use the web page open in the desktop preview pane (the one " "`desktop_preview` opens): log in, fill forms, click through flows. ALWAYS " "start with action='elements' — it inventories clickable/typable " "things as refs ('btn-sign-in') with role/label/value; act by ref, " "not guessed selectors. Refs survive re-renders and only die on " "navigation (you'll be told they're stale — call elements again). " "After the first full inventory, actions answer with a DELTA: " "'added' in full, 'changed' as ref + moved fields, 'removed'/" "'rebound' as ref lists ('rebound' needs nothing from you — the ref " "already follows the rebuilt element). Anything unmentioned is " "unchanged; do not re-read to check. Input is real (pointer travels, " "hover menus open). Actions: elements, click, hover (park the " "pointer — opens dropdowns before clicking in), type (submit=true " "also presses Enter), scroll, press, strobe (visual flourish only — " "one call runs a multi-second burst; never loop it), back/forward/" "reload. Moves draw live and fade; annotate_preview leaves a lasting " "mark. A printable press on body/html is refused unless allow_shortcut " "is true. Page text only: desktop_preview action=read. Separate automated " "browser: browser_* tools." ), "parameters": { "type": "object", "properties": { "action": { "type": "string", "enum": list(ACTIONS), "description": "Start with 'elements'.", }, "ref": { "type": "string", "description": "Element ref from an earlier elements call.", }, "selector": { "type": "string", "description": "CSS selector fallback. Prefer ref.", }, "text": {"type": "string", "description": "type: the text."}, "submit": { "type": "boolean", "description": "type: press Enter + submit the form after.", }, "key": { "type": "string", "description": "press: key name ('Enter', 'Escape', 'ArrowDown').", }, "allow_shortcut": { "type": "boolean", "description": "press: allow a printable key on body/html. Off by default.", }, "amount": { "type": "integer", "description": "scroll: pixels (negative = up; default ~one screen).", }, "to": { "type": "string", "enum": list(SCROLL_TO), "description": "scroll: jump to top/bottom instead.", }, "max": { "type": "integer", "description": "elements: cap the inventory.", }, "full": { "type": "boolean", "description": "elements: full re-read instead of a delta. Rarely needed.", }, }, "required": ["action"], }, } registry.register( name="drive_preview", toolset="desktop_ui", schema=ACT_PREVIEW_SCHEMA, handler=lambda args, **kw: drive_preview_tool( action=args.get("action", ""), limit=args.get("max"), callback=kw.get("callback"), **{k: args.get(k) for k in ( "ref", "selector", "text", "key", "submit", "amount", "to", "full", "allow_shortcut")}, ), emoji="🖱️")