"""Outer-loop exception handler for the conversation turn loop: phase-aware classification (interpreter shutdown, deterministic local post-processing bug vs API-path failure), unanswered tool_call error results, and the per-turn error cap that stops the loop from spinning until the budget is gone. Nothing here imports ``agent.conversation_loop`` at module level (cycle); loop-internal constants resolve lazily. """ from __future__ import annotations from dataclasses import dataclass import logging import os import sys from typing import Any from agent.message_metadata import append_message from agent.turn_failure_copy import short_detail, site_copy logger = logging.getLogger("agent.conversation_loop") @dataclass class OuterErrorVerdict: """``action`` is ``"break"`` (turn ends with ``final_response``/``turn_exit_reason`` set) or ``"fallthrough"`` (retry the iteration). The other fields are the loop locals the handler rebinds.""" action: str _outer_error_count: Any _turn_exit_reason: Any failed: Any final_response: Any def handle_outer_loop_error( agent: Any, *, e: Any, _outer_error_count: Any, api_call_count: Any, messages: Any, conversation_history: Any, _turn_exit_reason: Any, failed: Any, final_response: Any, ) -> OuterErrorVerdict: """Handle an exception that escaped the response-processing block. Shutdown and local-processing errors are deterministic and end the turn; API-path errors retry until ``min(_MAX_OUTER_LOOP_ERRORS, max_iterations)`` escaped exceptions. The assistant message is never appended here: a prefill/interim assistant may already be the tail (assistant→assistant); ``finalize_turn`` appends only when safe.""" from agent.conversation_loop import ( _API_CALL_MODULES, _LOCAL_PROCESSING_MODULES, _MAX_OUTER_LOOP_ERRORS, _is_interpreter_shutdown_error, _ra, ) def _verdict(action: str) -> OuterErrorVerdict: return OuterErrorVerdict( action=action, _outer_error_count=_outer_error_count, _turn_exit_reason=_turn_exit_reason, failed=failed, final_response=final_response, ) # Count every escaped exception before classification so permanent failures # terminate even with an unlimited turn budget. _outer_error_count += 1 # Interpreter shutdown makes every executor op raise: break. # Phase-aware error classification. The huge outer try/except spans both the actual API request and all # local post-processing of the returned assistant message. Deterministic local bugs (e.g. passing a # multimodal content list into a regex helper after a vision turn or context compaction) should not be # retried: they will fail identically on every iteration and only burn the iteration budget. We classify # an error as local by inspecting the traceback: if the exception propagated through any of the known # local post-processing helpers and never entered the interruptible API-call helpers, it is almost # certainly a local processing bug. (#66267) Interpreter shutdown: if the process is tearing down, every # executor-backed operation (API call, tool dispatch, memory sync) raises ``RuntimeError: cannot # schedule new futures after interpreter shutdown``. Retrying is pointless — the executor is gone for # good — and each retry just spams another traceback. Break immediately so the turn exits cleanly. # (#93217) if sys.is_finalizing() or _is_interpreter_shutdown_error(e): error_msg = f"Interpreter is shutting down — cannot continue (API call #{api_call_count}): {e}" try: agent._safe_print(f"❌ {error_msg}", diagnostic=True) except (OSError, ValueError): pass logger.warning(error_msg) # Best-effort persist — the dying executor may raise the same error; don't let # it mask the shutdown exit. finalize_turn retries. try: agent._persist_session(messages, conversation_history) except Exception: pass _turn_exit_reason = "interpreter_shutdown" failed = True _sid = getattr(agent, "session_id", None) final_response = site_copy( "interpreter_shutdown", resume=f" (CLI: `hermes --resume {_sid}`)" if _sid else "", ) return _verdict("break") # Deterministic local post-processing bugs (traceback via local helpers, never API # helpers) aren't retried. tb_module_names: set[str] = set() _tb = e.__traceback__ while _tb is not None: tb_module_names.add(os.path.splitext(os.path.basename(_tb.tb_frame.f_code.co_filename))[0]) _tb = _tb.tb_next _is_local_processing_error = bool(tb_module_names & _LOCAL_PROCESSING_MODULES) and not ( tb_module_names & _API_CALL_MODULES ) if _is_local_processing_error: error_msg = f"Error during local message processing after API call #{api_call_count}: {str(e)}" else: error_msg = f"Error during API call #{api_call_count}: {str(e)}" # Honor the _vprint contract: suppress_status_output silences hard failures; # quiet_mode -q still shows them. Traceback is logged below. if getattr(agent, "suppress_status_output", False): logger.error(error_msg) else: try: agent._safe_print(f"❌ {error_msg}", diagnostic=True) except (OSError, ValueError): logger.error(error_msg) # ERROR level with traceback so outer-loop failures land in agent.log AND errors.log. logger.exception("Outer loop error in API call #%d", api_call_count) # An appended assistant tool_calls message needs a role="tool" result per # tool_call_id; fill in error results for unanswered ones. for idx in range(len(messages) - 1, -1, -1): msg = messages[idx] if not isinstance(msg, dict): break if msg.get("role") == "tool": continue if msg.get("role") == "assistant" and msg.get("tool_calls"): answered_ids = { m["tool_call_id"] for m in messages[idx + 1:] if isinstance(m, dict) and m.get("role") == "tool" } for tc in msg["tool_calls"]: if tc and isinstance(tc, dict) and tc["id"] not in answered_ids: append_message(messages, { "role": "tool", "name": _ra().AIAgent._get_tool_call_name_static(tc), "tool_call_id": tc["id"], "content": f"Error executing tool: {error_msg}", }) break # Non-tool errors are already printed; a synthetic message would pollute history # and risk breaking role alternation. # Local errors are deterministic: stop early instead of retrying until the budget is # gone; a small per-turn cap prevents infinite spinning. _outer_error_cap = min(_MAX_OUTER_LOOP_ERRORS, max(1, agent.max_iterations)) if ( _is_local_processing_error or api_call_count >= agent.max_iterations - 1 or _outer_error_count >= _outer_error_cap ): # finalize_turn stamps failure_reason=loop_error from the exit reason so the desktop # card stops reading these as "unknown"/retryable. The deterministic local bug keeps # ``failed`` as it was (an incomplete, not failed, turn): flipping it made every such # child run a strike against the kanban dispatcher breaker. detail = short_detail(e) if _is_local_processing_error: _turn_exit_reason = f"local_processing_error({error_msg[:80]})" final_response = site_copy("local_processing_error", detail=detail) elif _outer_error_count >= _outer_error_cap: failed = True _turn_exit_reason = f"repeated_outer_errors({error_msg[:80]})" final_response = site_copy("loop_error", detail=detail) else: _turn_exit_reason = f"error_near_max_iterations({error_msg[:80]})" final_response = site_copy("loop_error", detail=detail) return _verdict("break") return _verdict("fallthrough")