"""Regression tests for #52374 — raw clarify tool-call JSON must never leak into the chat as a tool-progress bubble. The adapter's ``send_clarify`` is the user-facing rendering of a clarify prompt (interactive buttons, or the numbered-text fallback). The gateway's tool-progress callback used to also render a progress bubble for the ``clarify`` tool.started event — in verbose mode that bubble contains the raw tool-call args JSON (``{"question": ..., "choices": [...]}``), and because the progress queue drains on a background task the JSON landed right underneath the rendered interactive prompt on Slack. """ import importlib import sys import threading import types import pytest from gateway.config import Platform, PlatformConfig from gateway.platforms.base import BasePlatformAdapter, SendResult from gateway.session import SessionSource class ProgressCaptureAdapter(BasePlatformAdapter): """Records every send so the test can assert nothing leaked.""" def __init__(self, platform=Platform.SLACK): super().__init__(PlatformConfig(enabled=True, token="***"), platform) self.sent = [] self.edits = [] # Set on the first send/edit. The fake agent (on an executor thread) # waits on it, so the turn cannot end before the progress task has # rendered its first bubble. self.delivered = threading.Event() def _record(self, content): self.delivered.set() async def connect(self, *, is_reconnect: bool = False) -> bool: return True async def disconnect(self) -> None: return None async def send(self, chat_id, content, reply_to=None, metadata=None) -> SendResult: self.sent.append({"chat_id": chat_id, "content": content}) self._record(content) return SendResult(success=True, message_id="m-1") async def edit_message(self, chat_id, message_id, content) -> SendResult: self.edits.append({"chat_id": chat_id, "message_id": message_id, "content": content}) self._record(content) return SendResult(success=True, message_id=message_id) async def send_typing(self, chat_id, metadata=None) -> None: return None async def stop_typing(self, chat_id) -> None: return None async def get_chat_info(self, chat_id: str): return {"id": chat_id} class ClarifyThenToolAgent: """Emits a clarify tool.started (with raw args) then a normal tool, and returns only once the progress task has delivered its first bubble. Both events are queued before that wait, and the queue drains FIFO, so the first bubble is the clarify one if clarify leaks and the terminal one if not. A timeout means nothing drained and fails the test. """ adapter = None def __init__(self, **kwargs): self.tool_progress_callback = kwargs.get("tool_progress_callback") self.tools = [] def run_conversation(self, message, conversation_history=None, task_id=None): cb = self.tool_progress_callback if cb is not None: cb( "tool.started", "clarify", "Which environment?", {"question": "Which environment?", "choices": ["staging", "production"]}, ) cb("tool.started", "terminal", "pwd", {}) if not type(self).adapter.delivered.wait(timeout=5.0): raise AssertionError("progress task never delivered a bubble") return {"final_response": "done", "messages": [], "api_calls": 1} def _make_runner(adapter): gateway_run = importlib.import_module("gateway.run") GatewayRunner = gateway_run.GatewayRunner runner = object.__new__(GatewayRunner) runner.adapters = {adapter.platform: adapter} runner._voice_mode = {} runner._prefill_messages = [] runner._ephemeral_system_prompt = "" runner._reasoning_config = None runner._provider_routing = {} runner._fallback_model = None runner._session_db = None runner._running_agents = {} runner._session_run_generation = {} runner.hooks = types.SimpleNamespace(loaded_hooks=False) runner.config = types.SimpleNamespace( thread_sessions_per_user=False, group_sessions_per_user=False, stt_enabled=False, ) return runner def _install_fakes(monkeypatch, mode): monkeypatch.setenv("HERMES_TOOL_PROGRESS_MODE", mode) fake_dotenv = types.ModuleType("dotenv") fake_dotenv.load_dotenv = lambda *a, **k: None monkeypatch.setitem(sys.modules, "dotenv", fake_dotenv) fake_run_agent = types.ModuleType("run_agent") fake_run_agent.AIAgent = ClarifyThenToolAgent monkeypatch.setitem(sys.modules, "run_agent", fake_run_agent) import tools.terminal_tool # noqa: F401 — register terminal emoji gateway_run = importlib.import_module("gateway.run") monkeypatch.setattr(gateway_run, "_resolve_runtime_agent_kwargs", lambda: {"api_key": "***"}) return gateway_run @pytest.mark.parametrize("mode", ["verbose", "all"]) @pytest.mark.asyncio async def test_clarify_tool_never_renders_progress_bubble(monkeypatch, tmp_path, mode): """No progress bubble for clarify — in any mode, especially verbose. Verbose mode used to dump the raw args JSON (``{"question": ..., "choices": [...]}``) into the chat right under the interactive prompt (#52374). """ adapter = ProgressCaptureAdapter() runner = _make_runner(adapter) monkeypatch.setattr(ClarifyThenToolAgent, "adapter", adapter) gateway_run = _install_fakes(monkeypatch, mode) monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path) source = SessionSource(platform=Platform.SLACK, chat_id="C1", chat_type="dm") result = await runner._run_agent( message="hello", context_prompt="", history=[], source=source, session_id="sess-clarify-leak", session_key="agent:main:slack:dm:C1", ) assert result["final_response"] == "done" all_content = "\n".join( [m["content"] for m in adapter.sent] + [e["content"] for e in adapter.edits] ) # Raw clarify args JSON must not leak anywhere. assert '"question"' not in all_content assert '"choices"' not in all_content assert "Which environment?" not in all_content # No clarify progress line at all (verb "Asking" / tool name). assert "clarify" not in all_content assert "Asking" not in all_content # The unrelated terminal tool still renders progress normally (and proves # the no-leak asserts above ran against a drained queue). assert "pwd" in all_content