)j 8dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ej dkZ ddl mZmZmZddlmZddlmZmZddlmZmZmZmZddlmZejeZed z Z d Z!d Z"d Z#d Z$dZ%d Z&dZ'dZ(de)de*fdZ+eGddZ,GddZ-e-Z.de/ddfdZ0ddl1m2Z2m3Z3ddddgd d!d"dd#d$dd%d$d&d'd(d)d&d*d$d&d+d(d)d,d-gd.d/Z4d0Z5e2j6dd1e4e5d23dS)4a Process Registry -- In-memory registry for managed background processes. Tracks processes spawned via terminal(background=true), providing: - Output buffering (rolling 200KB window) - Status polling and log retrieval - Blocking wait with interrupt support - Process killing - Crash recovery via JSON checkpoint file - Session-scoped tracking for gateway reset protection Background processes execute THROUGH the environment interface -- nothing runs on the host machine unless TERMINAL_ENV=local. For Docker, Singularity, Modal, Daytona, and SSH backends, the command runs inside the sandbox. Usage: from tools.process_registry import process_registry # Spawn a background process (called from terminal_tool) session = process_registry.spawn(env, "pytest -v", task_id="task_123") # Poll for status result = process_registry.poll(session.id) # Block until done result = process_registry.wait(session.id, timeout=300) # Kill it process_registry.kill(session.id) NWindows) _find_shell_resolve_safe_cwd_sanitize_subprocess_env)windows_hide_flags) dataclassfield)AnyDictListOptional)get_hermes_homezprocesses.jsoni@ i@ secondsreturnctdt|}|dkr|dSt|d\}}|dkr|d|dSt|d\}}|d|dS)Nr<szm zh m)maxintdivmod)rrminssecshourss ;/home/ubuntu/.hermes/hermes-agent/tools/process_registry.pyformat_uptime_shortr!Ns As7||A2vvwww2JD$ byy!!$!!!!r""KE4  t   cFeZdZUdZeed<eed<dZeed<dZeed<dZe e ed<dZ e e j ed <dZeed <dZe eed <d Zeed <dZeed<dZe e ed<dZeed<eZe ed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZeed<dZ eed<dZ!e ed<dZ"eed<e#e$Z%e&eed <e#dd!Z'e ed"<e#dd!Z(e ed#<e#dd!Z)eed$<e#d d!Z*eed%<e#d d!Z+eed&<e#dd!Z,eed'<e#dd!Z-e ed(<e#e.j/Z0e.j/ed)<e#dd!Z1e e.j2ed*<e#dd!Z3eed+<dS),ProcessSessionz3A tracked background process with output buffering.idcommandtask_id session_keyNpidprocessenv_refcwd started_atFexited exit_code output_buffermax_output_charsdetachedhost pid_scopewatcher_platformwatcher_chat_idwatcher_user_idwatcher_user_namewatcher_thread_idwatcher_message_idrwatcher_intervalnotify_on_complete)default_factorywatch_patterns)defaultrepr _watch_hits_watch_suppressed_watch_disabled_watch_last_emit_at_watch_cooldown_until_watch_strike_candidate_watch_consecutive_strikes_lock_reader_thread_pty)4__name__ __module__ __qualname____doc__str__annotations__r(r)r*r rr+ subprocessPopenr,r r-r/floatr0boolr1r2MAX_OUTPUT_CHARSr3r4r6r7r8r9r:r;r<r=r>r listr@r rCrDrErFrGrHrI threadingLockrJrKThreadrLr"r r$r$Ys== GGG LLLGSKC#*.GXj& '...GSC#JFD#Ix}###M3,c,,,HdIscOSOSss    c$$$$ %d ; ; ;NDI;;;uQU333K333"U15999s999!E%e<<AAA16t%1P1P1PNHY-.PPPd///D#/////r"r$ceZdZdZdZdZededefdZde dedd fd Z d e de fd Z ed eede fdZdee dee fdZed edd fdZededefdZ dAdededededede de fdZ dBdedededededede fdZde fdZde ded ed!ed"ef d#Zde fd$Zde fd%Zd&ede fd'ZdCd)Zd&edee fd*ZdDd,Zd&edefd-Z dEd&ed0ed1edefd2Z!dFd&ededefd3Z"d&edefd4Z#d&ed5edefd6Z$dGd&ed5edefd7Z%d&edefd8Z&defd9Z'dFdede(fd:Z)dede fd;Z*dede fd<Z+dFdedefd=Z,d>Z-d?Z.defd@Z/d S)HProcessRegistrya$ In-memory registry of running and finished background processes. Thread-safe. Accessed from: - Executor threads (terminal_tool, process tool handlers) - Gateway asyncio loop (watcher tasks, session reset checks) - Cleanup thread (sandbox reaping coordination) )z'bash: cannot set terminal process groupz"bash: no job control in this shellzno job control in this shellz!cannot set terminal process groupz)tcsetattr: Inappropriate ioctl for devicec(i|_i|_tj|_g|_ddl}||_t|_ tj|_ d|_ d|_ d|_d|_dS)Nrr.)_running _finishedrYrZrJpending_watchersqueueQueuecompletion_queueset_completion_consumed_global_watch_lock_global_watch_window_start_global_watch_window_hits_global_watch_tripped_until$_global_watch_suppressed_during_trip)self _queue_mods r __init__zProcessRegistry.__init__s35 46^%% 79 #"""2<2B2B2D2D*-! #,."2"214'./&25(9:111r"textrc|dratfdtjDr<dr%tfdtjDz5ProcessRegistry._clean_shell_noise..s,cc%EU1X-ccccccr"r)splitanyr^_SHELL_NOISE_SUBSTRINGSpopjoin)rprvs @r _clean_shell_noisez"ProcessRegistry._clean_shell_noises 4   cccc?;bccccc  IIaLLL cccc?;bccccc yyr"sessionnew_textNc|jr|jrdS|jrdSg}d}|D]=}|jD]3}||vr-||||}n4>|sdSt j}d}|j5|jri||jkr^|xj t|z c_ |j s7d|_ |xj dz c_ |j tkrd|_d|_d}d} nR|jr|j sd|_ d|_ ||_|t z|_|xjdz c_|j } d|_ d} dddn #1swxYwY| rv|rr|j|j|j|jd|j |j|j|j|j|j|jd|jdtd t d d dSd |dd } t| dkr | dddz} ||sdS|j|j|j|jd|| | |j|j|j|j|j|jd dS)uzScan new output for watch patterns and queue notifications. Called from reader threads with new_text being the freshly-read chunk. Per-session rate limit: at most ONE watch-match notification per WATCH_MIN_INTERVAL_SECONDS. Any match arriving inside the cooldown window is dropped and counts as ONE strike for that window. After WATCH_STRIKE_LIMIT consecutive strike windows, watch_patterns is disabled for this session and the session is promoted to notify_on_complete semantics — one notification when the process actually exits, no more mid-process spam. NFTrwatch_disabledz$Watch patterns disabled for process u — z7 consecutive rate-limit windows triggered (min spacing zms). Falling back to notify_on_complete semantics; you'll get exactly one notification when the process exits.) session_idr)r&type suppressedplatformchat_iduser_id user_name thread_id message_idmessagerriz ...(truncated) watch_match) rr)r&rpatternoutputrrrrrrr)r@rEr0 splitlinesappendrstriptimerJrGrDlenrHrIWATCH_STRIKE_LIMITr>rFWATCH_MIN_INTERVAL_SECONDSrCreputr%r)r&r7r8r9r:r;r<r|_global_watch_admit) rmr~r matched_linesmatched_patternlinepatnowshould_disable return_earlyrrs r _check_watch_patternsz%ProcessRegistry._check_watch_patternss% )@  F >  F ''))  D-  $;;!((777&.*-E    Fikk ]% %% % , %w7T1T1T))S-?-??))6 .6:G366!;669=OOO26/6:2)-# 1;#;;:;G627//2+036P0P-##q(##$6 ,-)$ K% %% %% %% %% %% %% %% %% %% %% %% %% %% %% %N   %))")*#*#6&,")"; ' 8&6&6!(!:!(!:")"<LwzLL-LL(BLLL++( F="-.. v;;  ETE]%77F'',,  F !!!*".!&$0.. 2 2!4# #      sCEE#&E#rc |j5|jrJ||jkr?|j}d|_d|_||_d|_|dkrdddd|d|ddddddd }nd}nd}|jr ||jkr|xjd z c_d }d}ni||jz t kr||_d|_|jt kr$|tz|_|xjd z c_|}d }n|xjd z c_d}d }dddn #1swxYwY||j ||>|j dddd d t dt dtddddddd |S)a1Return True if this watch_match event is allowed through the global breaker. Semantics: - If we're currently in a cooldown period, drop the event and count it. - Otherwise, slide the rolling window and check the global cap. - If the cap is exceeded, trip the breaker for WATCH_GLOBAL_COOLDOWN_SECONDS and emit ONE summary event so the agent/user sees "N notifications were suppressed" instead of getting them individually. - When the cooldown ends, emit a release summary and reset counters. r.rr'watch_overflow_releasedz%Watch-pattern notifications resumed. z1 match event(s) were suppressed during the flood.) rr)r&rrrrrrrrNrFTwatch_overflow_trippedzWatch-pattern overflow: >z notifications in zCs across all processes. Suppressing further watch_match events for zs.) rr)r&rrrrrrr) rhrkrlrirjWATCH_GLOBAL_WINDOW_SECONDSWATCH_GLOBAL_MAX_PER_WINDOWWATCH_GLOBAL_COOLDOWN_SECONDSrer)rmrr release_msgadmittrip_nows r rz#ProcessRegistry._global_watch_admit?sb $3 !3 !/ #C4;[4[4[!F 360<=925/12.>>')')#% 9&0])]]]%'#%#%%'%'##KK"#'KK" / !C$:Z4Z4Z99Q>9988 /T /F``. This is the documented Microsoft primitive for tree-kill and matches the existing convention in ``gateway.status.terminate_pid``. We can't reuse the POSIX psutil path on Windows because: 1. Windows doesn't maintain a Unix-style process tree — ``psutil.Process.children(recursive=True)`` walks PPID links that go stale when intermediate processes exit, so enumeration is best-effort and misses orphaned descendants. 2. ``psutil.Process.terminate()`` on Windows is ``TerminateProcess()`` which kills only the target handle and is a hard kill — there is no Windows equivalent of a SIGTERM that cascades through a process group. (See the warning in ``gateway/status.py::terminate_pid``: "os.kill with SIGTERM is not equivalent to a tree-killing hard stop" on Windows.) Headless Chromium has no GUI window, so the softer ``taskkill /T`` without ``/F`` won't reach it either. ``psutil`` is a hard dependency (see ``pyproject.toml``); the bare-``os.kill`` fallback covers OSError / PermissionError on POSIX and a missing ``taskkill.exe`` on Windows (effectively unreachable on real Windows installs, but cheap insurance). taskkillz/PIDz/Tz/FTr)capture_outputrptimeout creationflagsstdinNr recursive) _IS_WINDOWSrSrunrQrDEVNULLFileNotFoundErrorTimeoutExpiredOSErroroskillsignalSIGTERMProcessLookupErrorPermissionErrorpsutilProcesschildren terminate NoSuchProcess)r*rparentchilds r _terminate_host_pidz#ProcessRegistry._terminate_host_pids@   S4>#'"4"6"6$, &z'@'J   GC0000!3_ED  F  ^^C((F488  OO%%%%+D        #    FF)    V^,,,,,,/A     sAA B))B B) B# B)"B##B)(B)3-D!C65D6 DDDD E<)E<:EE83E<7E88E<envcDt|dd}t|r |}t|tr,|dr|dpdSn2#t $r%}td|Yd}~nd}~wwxYwdS)zEReturn the writable sandbox temp dir for env-backed background tasks. get_temp_dirN/z*Could not resolve environment temp dir: %sz/tmp) getattrcallable isinstancerQ startswithr Exceptionloggerdebug)rrtemp_direxcs r _env_temp_dirzProcessRegistry._env_temp_dirssND99 L ! ! P P'<>>h,,71D1DS1I1I7#??3//636 P P P I3OOOOOOOO PvsA A.. B8BBr'Fr&r-r(r)env_varsuse_ptyc tdtjjdd|||t |pt jtj}|ri trddl m }nddl m }t} tt j|} d| d<|| d d |g|j| d } | j|_| |_t'j|j|fd d|j} | |_| |j5|||j|j<dddn #1swxYwY||S#t:$rt<dYn1t@$r%} t<d| Yd} ~ nd} ~ wwxYwt} tt j|}d|d<trdtCini}tEj#| d d |gfd |j|ddtDj$tDj%tDj&trdn t j'd |}||_(|j|_ t'j|j)|fd d|j} | |_| |j5|||j|j<dddn #1swxYwY|n#t@$r ts tUtVdtVj,}t j-t j.|j|nE#t^t`tbf$r|2YnwxYw|2n#t@$rYnwxYw |3dn#t@$rYnwxYwwxYw|S)aw Spawn a background process locally. Only for TERMINAL_ENV=local. Other backends use spawn_via_env(). Args: use_pty: If True, use a pseudo-terminal via ptyprocess for interactive CLI tools (Codex, Claude Code, Python REPL). Falls back to subprocess.Popen if ptyprocess is not installed. proc_N )r%r&r(r)r-r/r) PtyProcess1PYTHONUNBUFFEREDz-liczset +m; )rx)r-r dimensionsTzproc-pty-reader-targetargsdaemonnamez3ptyprocess not installed, falling back to pipe modez0PTY spawn failed (%s), falling back to pipe moderutf-8replace) rpr-rencodingerrorsstdoutstderrr preexec_fnz proc-reader-SIGKILLr)4r$uuiduuid4hexrrgetcwdrrwinptyr ptyprocessrrenvironspawnr-r*rLrYr[_pty_reader_loopr%rKstartrJ_prune_if_neededr`_write_checkpoint ImportErrorrwarningrrrSrTPIPESTDOUTrsetsidr+ _reader_looprrrkillpggetpgidrrrrwait)rmr&r-r(r)rrr~_PtyProcessCls user_shellpty_envpty_procreaderebg_env _popen_kwargsproc kill_signals r spawn_localzProcessRegistry.spawn_locals&!.tz||',..#!#"455y{{     ( V& VHCCCCCCCGGGGGG(]] 22:xHH.1*+)//)=G)=)=> ( 0 'l ' #)0!8GJ88  *0& Z88))+++07DM'*-888888888888888&&((( V V VTUUUUU V V VQSTUUUUUUUU V !]] *"*h??%(!"CNV*<*>*>??TV  !5G!5!5 6   ?$$*9tt       h # %(Z0GJ00 F &,G " LLNNN 4 4%%''',3 gj) 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4  " " $ $ $ $    " $&-fi&P&P  "*TX"6"6 DDDD.I$$$ $IIKKK      ! $$$$     ' *s,CE=,$E E=E  E=#E $E==$G# G,G  GAL$K>2 L>LLLL O>)O1A M>=O>+N,)O+N,,OO> OO>OO>O,+O>, O96O>8O99O>rrc tdtjjdd||||t j|d}||}|d|jd} |d|jd} |d|jd } tj|} tj|} tj| }tj| }tj| }d | d | d |d |d|d| } | ||d}| dd }| D]@}| }| rt||_nA|jnd|_t| dd|_|jdkrd|_| dd |_n/#t&$r"}d|_d|_d||_Yd}~nd}~wwxYw|jsEt)j|j||| | | fdd|j}||_||j5||js||j|j<dddn #1swxYwY|js||S)a# Spawn a background process through a non-local environment backend. For Docker/Singularity/Modal/Daytona/SSH: runs the command inside the sandbox using the environment's execute() interface. We wrap the command to capture the in-sandbox PID and redirect output to a log file inside the sandbox, then poll the log via subsequent execute() calls. This is less capable than local spawn (no live stdout pipe, no stdin), but it ensures the command runs in the correct sandbox context. rNrsandbox)r%r&r(r)r-r/r,r6z /hermes_bg_z.logz.pidz.exitz mkdir -p z && ( nohup bash -lc z > z$ 2>&1; rc=$?; printf '%s\n' "$rc" > z ) & echo $! > z && cat F)rrewrite_compound_backgroundrr'T returncoderzFailed to start: z proc-poller-r)r$rrrrrr%shlexquoteexecutegetstriprisdigitrr*r0r1r2rrYr[_env_poller_looprKrrJrr`r)rmrr&r-r(r)rr~rlog_pathpid_path exit_pathquoted_commandquoted_temp_dirquoted_log_pathquoted_pid_pathquoted_exit_path bg_commandresultrrrrs r spawn_via_envzProcessRegistry.spawn_via_envs(!.tz||',..#y{{    %%c**;;7:;;;;;7:;;;==GJ=== W--+h//+h//+h// ;y11 D D D . D D3B D D/? D D) D D3B D D  <[[,1!F ZZ"--3355F))++  zz||<<>>"%d))GKE {"!%$' <(D(D$E$E!$))(*G%(. 8R(@(@(F(F(H(H% < < <!GN "G $;$;$;G ! ! ! ! ! ! < ~ %,sHh B0GJ00 F &,G " LLNNN Z 4 4  ! ! # # #> 4,3 gj) 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 ~ %  " " $ $ $s+?D H H6H11H6 +KK Kcd} |jjd}|sn|r||}d}|j5|xj|z c_t |j|jkr|j|j d|_dddn #1swxYwY|||n2#t$r%}t d|Yd}~nd}~wwxYw |j dn2#t$r%}t d|Yd}~nd}~wwxYwd|_ |jj|_||dS# |j dn2#t$r%}t d|Yd}~nd}~wwxYwd|_ |jj|_||wxYw) z:Background thread: read stdout from a local Popen process.TFNzProcess stdout reader ended: %srrz$Process wait timed out or failed: %s)r+rreadr}rJr2rr3rrrrr r0rr1r)rmr~ first_chunkchunkrs r rzProcessRegistry._reader_loops  , ;.33D99( 33E::E"'K]bb))U2))7011G4LLL070EwG_F_F`F`0a-bbbbbbbbbbbbbbb**7E::: ; ? ? ? LL:A > > > > > > > > ? H$$Q$//// H H H CQGGGGGGGG H!GN ' :G   " "7 + + + + +  H$$Q$//// H H H CQGGGGGGGG H!GN ' :G   " "7 + + + +sAB>AB B>BB>"B#B>=E.> C-C(#E.(C--E.1D D<D77D<.G,0F  G, F;F61G,6F;;1G,r!r"r#cNtj|}tj|}tj|}d} |js^tjd |d|dd} | dd} | rt| | kr | | d nd} t| } |j5| |_ t|j |j kr|j |j d |_ d d d n #1swxYwY| r| || |d |d d } | dd }|r| d  dkr|d|dd }|dd } t| d  |_n#t t"f$r d |_YnwxYwd|_||d Sn4#t&$r'd|_d |_||Yd SwxYw|j\d Sd S)zBBackground thread: poll a sandbox log file for non-local backends.rzcat 2>/dev/nullrrrr'Nzkill -0 "$(cat z# 2>/dev/null)" 2>/dev/null; echo $?rr0T)rrr0rsleeprrrrJr2r3rrrrr1 ValueError IndexErrorrr)rmr~rr!r"r#r&r'r(prev_output_lenr* new_outputdeltacheck check_output exit_resultexit_strs r r z ProcessRegistry._env_poller_loop s> +h//+h// ;y11.+  JqMMM) %IO%I%I%ISUVV#ZZ"55  C>  L$;$;$=$=b$A$G$G$I$IS$P$P"%++=/=== !#.##K +x<<BBDDH/,/0C0C0E0Eb0I0O0O0Q0Q,R,R))& 3///,.)))/%)GN**7333F   !%$&!&&w///  M.+ + + + + s\A(I'?D I'DI'DCI'+>H*)I'*II'II''-JJc|j} |r |d}|rt|tr|n|dd}|j5|xj|z c_t|j|j kr|j|j d|_dddn #1swxYwY| ||n#t$rYn#t$rYnwxYw|n2#t$r%}td|Yd}~nd}~wwxYw |n2#t$r%}td|Yd}~nd}~wwxYwd|_t#|d r|jnd |_||dS) z2Background thread: read output from a PTY process.r-rrrNzPTY stdout reader ended: %sz PTY wait timed out or failed: %sT exitstatusr)rLisaliver.rrQdecoderJr2rr3rEOFErrorrrrr r0hasattrrAr1r)rmr~ptyr0rprs r rz ProcessRegistry._pty_reader_loop@s^l ;++--  HHTNNEB(25#(>(>kuuELLQXajLDkDk$]jj#11T911"7#899G2 C >CC CC D C8*D, C85D7C88D D?D::D?E F"FFc |j5|j|jddu}||j|j<dddn #1swxYwY||re|jr`ddlm}|j r||j ddnd}|j d|j|j |j |j|ddSdSdS)uMove a session from running to finished. Idempotent: if the session was already moved (e.g. kill_process raced with the reader thread), the second call is a no-op — no duplicate completion notification is enqueued. Nr strip_ansi0r' completion)rrr)r&r1r)rJr`r{r%rarr>tools.ansi_striprIr2rerr)r&r1)rmr~ was_runningrI output_tails r rz!ProcessRegistry._move_to_finished_sOZ 1 1-++GJ==TIK)0DN7: & 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1     75  3 3 3 3 3 3GNG\d**W%:566%BCCCbdK  ! % %$%j&2"?$.% ''         s2AA  A rc||jvS)zJCheck if a completion notification was already consumed via wait/poll/log.)rg)rmrs r is_completion_consumedz&ProcessRegistry.is_completion_consumed|sT666r"list[tuple[dict, str]]cg}|js |j}n#t$rYnwxYw|dd}|ddkr||rt |}|r|||f|j|S)zPop all pending notification events and return formatted pairs. Returns a list of (raw_event, formatted_text) tuples. Skips completion events that were already consumed via wait/poll/log. rr'rrK)reempty get_nowaitrrrPformat_process_notificationr)rmresultsevt_evt_sidrps r drain_notificationsz#ProcessRegistry.drain_notificationss '--// , +6688    ww|R00Hwwv,..43N3Nx3X3X..s33D ,T{+++'--// ,s7 AAc|j5|j|p|j|}dddn #1swxYwY||S)z*Get a session by ID (running or finished).N)rJr`rrar)rmrr~s r rzProcessRegistry.gets Z V Vm'' 33Ut~7I7I*7U7UG V V V V V V V V V V V V V V V--g666s5A  A A r$c||jrdSt|dd}|dS |}n#t$rYdSwxYw|dSd}t|dd}|atsY ddl}|}|||j}|||j|tj z | } | r.t| tr| n| dd}n#tt t"f$rYnwxYw |||j|nB#t$rYn6wxYw# |||j|w#t$rYwwxYwxYwn8#t$r+} t$d |j| Yd} ~ nd} ~ wwxYw|j5|rG|xj|z c_t/|j|jkr|j|j d|_d |_||_dddn #1swxYwYt$d |j|||dS) u$Reconcile session.exited against the real child process state. The reader thread (`_reader_loop`) sets `session.exited = True` only in its `finally` block, which runs when `stdout.read()` returns EOF. If the direct `Popen` child has exited but a descendant process (e.g. a daemon spawned by `hermes update` restarting the gateway) is still holding the stdout pipe open, the reader blocks forever and poll() keeps returning "running" indefinitely (issue #17327 — 74 polls over 7 minutes on Feishu). This helper closes that window: when `session.exited` is still False but the direct child's `Popen.poll()` reports an exit code, drain any readable bytes non-blocking and flip `session.exited`. The orphaned reader thread remains stuck on its blocking `read()` but is a daemon thread and will be reaped with the process. Safe no-op on sessions without a local `Popen` (env/PTY), already- exited sessions, and detached-recovered sessions. Nr+r'rrrrr@z$Non-blocking drain failed for %s: %sTzxReconciled session %s: direct child exited with code %s but reader was still blocked (orphaned pipe). Flipped to exited.)r0rpollrrfcntlfilenoF_GETFLF_SETFLr O_NONBLOCKr.rrQrCBlockingIOErrorrr6rrr%rJr2rr3r1infor) rmr~rrcdrainedrr]fdflagsr0rs r _reconcile_local_exitz%ProcessRegistry._reconcile_local_exits!( ?gn? Fw 400 < F BB    FF  : F x..  k  T ]]__ B 66 B ur}/DEEE "KKMMEo+5eS+A+A"n%%u||T[dm|GnGn'*=D B u====$ B u====$ T T T CWZQRSSSSSSSS T] # # ^%%0%%w,--0HHH,3,A7C[B[B\B\,]G)!GN "G   # # # # # # # # # # # # # # #   D J   w'''''s7 AA+AF AD E D'$E&D''E+EF  EF EF FE76F7 FFFFF G!F;;G AH..H25H2cddlm}||}|dd|dS|||j5|jr||jddnd}dddn #1swxYwY|j|j|jrd nd |j ttj |j z |d }|jr$|j |d <|j||jr d |d<d|d<|S)z9Check status and get new output for a background process.rrHN not_foundNo process with ID statuserrorr'r0running)rr&rmr*uptime_secondsoutput_previewr1Tr4z=Process recovered after restart -- output history unavailablenote)rLrIrrhrJr2r%r&r0r*rrr/r1rgaddr4)rmrrIr~rrr*s r r\zProcessRegistry.polls//////((:&& ?)4V*4V4VWW W ""7+++ ] h hJQJ_gZZ(=eff(EFFFegN h h h h h h h h h h h h h h h"*").?hhi;!$)++0B"BCC,    > 6")"3F;   % ) )* 5 5 5   ]!%F: \F6N s"A00A47A4roffsetlimitcddlm}||}|dd|dS|j5||j}dddn #1swxYwY|}t |}|dkr|dkr || d} n ||||z} |j|jrdndd | |t | d d } |jr|j || S) z;Read the full output log with optional pagination by lines.rrHNrjrkrlr0rprrz lines)rrmr total_linesshowing) rLrIrrJr2rrr%r0r|rgrt) rmrrvrwrIr~ full_outputrvryselectedr*s r read_logzProcessRegistry.read_logsr//////((:&& ?)4V*4V4VWW W ] < <$*W%:;;K < < < < < < < < < < < < < < <&&((%jj  Q;;5199eVWW~HHVFUN23H"*").?hhiii))&h--///    > 6  % ) )* 5 5 5 sA  AAcbddlm}ddlm} t t jdd}n#ttf$rd}YnwxYw|}|}d}|r||kr |} d|d |d }n|p|} | |} | d d |d Stj | z} tj | kr| | } | | | jrD|j|d| j|| jddd} |r|| d<| S|r%d|| jdddd} |r|| d<| Stjdtj | kd|| jddd} |r|| d<n d| d| d<| S)aY Block until a process exits, timeout, or interrupt. Args: session_id: The process to wait for. timeout: Max seconds to block. Falls back to TERMINAL_TIMEOUT config. Returns: dict with status ("exited", "timeout", "interrupted", "not_found") and output snapshot. rrH)is_interruptedTERMINAL_TIMEOUT180NzRequested wait of z%s was clamped to configured limit of rrjrkrlr0rJ)rmr1r timeout_note interruptedroz+User sent a new message -- wait interrupted)rmrrsrr)rmrzWaited zs, process still running)rLrItools.interruptrrrgetenvr6 TypeErrorrr monotonicrrhr0rgrtr1r2r5) rmrrrI_is_interrupteddefault_timeout max_timeoutrequested_timeoutreffective_timeoutr~deadliner*s r r zProcessRegistry.wait s 0/////EEEEEE "!"),>"F"FGGOOI& " " "!OOO "% #  A!2[!@!@ + 9%699*5999 L !2 @[ ((:&& ?)4V*4V4VWW W>##&77n))44W==G  & &w / / /~ )--j999&!(!2(j)>uvv)FGG  :-9F>*    +(j)>uvv)FGGI  :-9F>* JqMMM7n))<  j!6uvv!>??    [%1F> " "%Z/@%Z%Z%ZF> " s"1AAcd||}|dd|dS|jr d|jdS |jrZ |jdn#t $r/|jr$tj|jtj YnwxYw|j r tr|j nd dl } ||j j}|d D]'} |#|j$rY$wxYw|n#|j$rYnwxYwn#t"t$f$r|j YnwxYw|jr-|jr&|jd |jd d n|jr|jdkr|jr||jsL|j5d|_d|_dddn #1swxYwY||d|jdS||jndddSd|_d|_|||d|jdS#t $r}dt;|dcYd}~Sd}~wwxYw)zKill a background process.Nrjrkrlalready_exited)rmr1T)forcerrzkill r3rrr5rnzkRecovered process cannot be killed after restart because its original runtime handle is no longer availableikilled)rmr)rr0r1rLrrr*rrrrr+rrrrrrrr,rr4r6rrJrrrr%rQ)rmrr~rrrrs r kill_processzProcessRegistry.kill_processms((:&& ?)4V*4V4VWW W > *$.  7 8|/ =L***6666 ==={= V^<<<=( +" !113333%  !%+^^GO4G%H%HF)/4)H)H))!)$)OO$5$5$5$5'-';!)!)!)$(D!)",,....%3!!! D!*O<+++O((*****+ W[ ''(I (I(I(IST'UUUU! g&76&A&Agk&A..w{;; 11)-,0)111111111111111**7333"2%,%6((5555&M"GN #G   " "7 + + +  " " $ $ $&gjAA A 8 8 8%A77 7 7 7 7 7 7 8sJ AJ 5B J B J %E7D39D D3 DD3DD32E3 E=E?EEJ *E2/J 1E22A1J #G>2 J >HJ H!J (J AJ J/J*$J/*J/datac.||}|dd|dS|jrdddSt|dr|jr tr:t |t r|dnt|}n,t |tr| dn|}|j |d t|d S#t$r}d t|dcYd}~Sd}~wwxYw|j r |j jsd d dS |j j ||j jd t|d S#t$r}d t|dcYd}~Sd}~wwxYw) zASend raw data to a running process's stdin (no newline appended).NrjrkrlrProcess has already finishedrLrok)rm bytes_writtenrn?Process stdin not available (non-local backend or stdin closed))rr0rErLrrbytesrCrQencodewriterrr+rflush)rmrrr~pty_datars r write_stdinzProcessRegistry.write_stdins((:&& ?)4V*4V4VWW W > Y.9WXX X 7F # # < < <W7A$7N7N]t{{7333TWX\T]T]HH7A$7L7LVt{{7333RVH ""8,,,"&TCCC < < <")CFF;;;;;;;; < sgo&; s%0qrr r 8 O ! ' ' - - - O ! ' ' ) ) )"SYY?? ? 8 8 8%A77 7 7 7 7 7 7 8s>BC D'C>8D>DAE.. F8F FFc4|||dzS)zGSend data + newline to a running process's stdin (like pressing Enter).rr)r)rmrrs r submit_stdinzProcessRegistry.submit_stdins D4K888r"c||}|dd|dS|jrdddSt|drO|jrH |jdd d S#t $r}d t |dcYd}~Sd}~wwxYw|jr |jjsd d dS |jj dd d S#t $r}d t |dcYd}~Sd}~wwxYw)zGClose a running process's stdin / send EOF without killing the process.NrjrkrlrrrLrzEOF sent)rmrrnrz stdin closed) rr0rErLsendeofrrQr+rclose)rmrr~rs r close_stdinzProcessRegistry.close_stdinsb((:&& ?)4V*4V4VWW W > Y.9WXX X 7F # # <  < < $$&&&"&:>>> < < <")CFF;;;;;;;; < sgo&; s%0qrr r 8 O ! ' ' ) ) )"~>> > 8 8 8%A77 7 7 7 7 7 7 8s<A"" B,B=BB$"C C-C("C-(C-cN t|jS#t$rYdSwxYw)akReturn the count of currently-running background processes. Cheap O(1) read of the running dict, suitable for status-bar polling on every render tick. CPython dict ``len()`` is atomic; callers do not need to hold ``self._lock``. Reflects ``_running`` only: sessions are moved to ``_finished`` when their subprocess exits. r)rr`r)rms r count_runningzProcessRegistry.count_runnings9 t}%% %   11 s  $$c j5tjtjz}dddn #1swxYwYfd|D}rfd|D}g}|D]}|j|jdd|j|jtj dtj |j ttj |j z |jrdnd|jr|jddnd d }|jr |j|d <|jrd |d <|||S)z1List all running and recently-finished processes.Nc:g|]}|Sr\)r)rtrrms r z1ProcessRegistry.list_sessions..s'PPPa66q99PPPr"c*g|]}|jk |Sr\r(rtrr(s r rz1ProcessRegistry.list_sessions..s%LLL!qyG7K7KA7K7K7Kr"ruz%Y-%m-%dT%H:%M:%Sr0rpi8r')rr&r-r*r/rqrmrrr1Tr4)rJrXr`valuesrar%r&r-r*rstrftime localtimer/rr0r2r1r4r)rmr( all_sessionsr*rentrys`` r list_sessionszProcessRegistry.list_sessionss Z X X 4 4 6 677$t~?T?T?V?V:W:WWL X X X X X X X X X X X X X X XQPPP<PPP  MLLLL|LLLL ! !Ad9TcT?uu"m,?PQP\A]A]^^"%dikkAL&@"A"A&'h=((I<=O"S!/$%%"8"8QS  Ex 1%&[k"z )$(j! MM%  sAA$$A(+A(ch|j5t|j}dddn #1swxYwY|D]}|||j5t fd|jDcdddS#1swxYwYdS)zK|]}|jko|j VdSNr(r0rs r rwz7ProcessRegistry.has_active_processes.."sG W$5QXr"rJrXr`rrry)rmr(sessionsr~s ` r has_active_processesz$ProcessRegistry.has_active_processesP Z 4 4DM002233H 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4  4 4G  * *7 3 3 3 3 Z  --//                  !'<AA(2B''B+.B+ch|j5t|j}dddn #1swxYwY|D]}|||j5t fd|jDcdddS#1swxYwYdS)z>Check if there are active processes for a gateway session key.Nc3>K|]}|jko|j VdSr)r)r0)rtrr)s r rwz9ProcessRegistry.has_active_for_session..0sG ,=QXr"r)rmr)rr~s ` r has_active_for_sessionz&ProcessRegistry.has_active_for_session'rrc |j5fd|jD}dddn #1swxYwYd}|D]8}||j}|ddvr|dz }9|S)zQKill all running processes, optionally filtered by task_id. Returns count killed.c<g|]} |jk|j|Srrrs r rz,ProcessRegistry.kill_all..8s;OqyG';';QX';';';';r"Nrrm>rrr)rJr`rrr%r)rmr(targetsrr~r*s ` r kill_allzProcessRegistry.kill_all5s Z  =//11G                  G&&wz22Fzz(##'CCC!  s &;??cVtjfdjD}|D]$}j|=j|%t jt jz}|tkrBjr;tjfd}j|=j|j j z}j|z }|rxj|zc_dSdS)zGRemove oldest finished sessions if over MAX_PROCESSES. Must hold _lock.c@g|]\}}|jz tk|Sr\)r/FINISHED_TTL_SECONDS)rtsidrrs r rz4ProcessRegistry._prune_if_needed..Js:   Cal"&::: :::r"c(j|jSr)rar/)rrms r z2ProcessRegistry._prune_if_needed..UsDN3)>>)G3  /  % % . % % % % / /r"cd |j5g}|jD]}|js|id|jd|jd|jd|jd|j d|j d|j d|j d |j d |jd |jd |jd |jd|jd|jd|jd|j dddn #1swxYwYddlm}|t0|dS#t2$r(}t4d|dYd}~dSd}~wwxYw)z=Write running process metadata to checkpoint file atomically.rr&r*r6r-r/r(r)r7r8r9r:r;r<r=r>r@Nr)atomic_json_writez#Failed to write checkpoint file: %sT)exc_info)rJr`rr0rr%r&r*r6r-r/r(r)r7r8r9r:r;r<r=r>r@utilsrCHECKPOINT_PATHrrr)rmentriesrrrs r rz!ProcessRegistry._write_checkpointcs  R  --//A8((!$(%qy("15(( ( "15 ( )!, (&qy(*1=(/0B(.q/@(.q/@(01D(01D(1!2F(/0B( 1!2F!("-a.>#(               2 0 / / / / /  ow 7 7 7 7 7 R R R LL>DL Q Q Q Q Q Q Q Q Q Rs;C=CC C=CC= C!C== D/D**D/cZtsdS tjtd}n#t $rYdSwxYwd}|D]}|d}|s|dd}|dkr:td|dd d d ||q| |}|r,td"id |d d|dd d|ddd|ddd|d|d|dd|dtj ddd|ddd|ddd|ddd|ddd|ddd|ddd|ddd|ddd|dg}|j 5||j |j<d d d n #1swxYwY|dz }td |jd d ||jdkrW|j|j|j|j|j|j|j|j|j|j|jd! ||S)#z On gateway startup, probe PIDs from checkpoint file. Returns the number of processes recovered as detached. rr)rr*r6r5z=Skipping recovery for non-host process: %s (pid=%s, scope=%s)r&unknownNrr%rr(r'r)r-r/r4Tr7r8r9r:r;r<r=r>Fr@rz'Recovered detached process: %s (pid=%d)) rcheck_intervalr)rrrrrrr>r\)rexistsjsonloads read_textrrrrcrr$rrJr`r%r&r=rbrr)r7r8r9r:r;r<r>r)rmr recoveredrr*r6aliver~s r recover_from_checkpointz'ProcessRegistry.recover_from_checkpoints %%'' 1 j!:!:G!:!L!LMMGG   11  < < E))E""C  +v66IF"" SIIi33CRC8  ++C00E' (\**!IIi;;;"IIi444!& - < < <    (i  %((( %yyty{{CCC"T&+YY/A2%F%F%F%*II.?$D$D$D%*II.?$D$D$D',ii0CR&H&H&H',ii0CR&H&H&H(-yy1Er'J'J'J &+YY/A1%E%E%E!"(-yy1Eu'M'M'M#$$)99-=r#B#B#B%(Z8807DM'*-888888888888888Q  EwWZXZWZG[]`aaa+a//)00&-j*1*B'.':$+$<#*#:#*#:%,%>%,%>&-&@.5.H 2 2       s#-A AAI::I> I> )Nr'r'NF)Nr'r'r)rrQ)r~r$rN)rrur)r')0rMrNrOrPrzro staticmethodrQr}r$rrUrVrr rrrrr rdictrr+rr rrrPrYrrhr\r}r rrrrrrXrrrrrrrr\r"r r^r^s;;;8     \ ~ ^~ s~ t~ ~ ~ ~ @UuUUUUUn   $   \ .1IhWeNf&?????\?F 3 3   \ GGGG G  G  GG GGGGZ^^ ^^ ^  ^  ^^ ^^^^D,N,,,,83%3,/3;>3JM3Z]3333j(((((>:777777(7c7h~&>7777 F(F(F(F(Pst>3d>KKsKSKDKKKKZD8sD8tD8D8D8D8L8c888888>99s9#9t99998c8d8888. s    SD> C D     # $       s    "///:RRRBOOOOOOOr"r^rWz str | Nonec  |dd}|dd}|dd}|dkrd|dd d S|d krb|d d }|dd }|dd}d|d|d|d|}|r |d|dz }|d z }|S|dd }|dd }d|d|d|d|d S)zFormat a process notification event into a [IMPORTANT: ...] message. Handles completion events (notify_on_complete), watch pattern matches, and watch disabled events from the unified completion_queue. rrKrrr&rz [IMPORTANT: rr']rr?rrrz[IMPORTANT: Background process z matched watch pattern "z ". Command: z Matched output: z (z/ earlier matches were suppressed by rate limit)r1z completed (exit code z ). Command: z Output: )r) rWevt_type_sid_cmd_pat_out_suprp_exits r rUrUs wwv|,,H 77< + +D 779i ( (D###7cggi447777=  wwy#&&wwx$$ww|Q'' 'd ' '# ' ' ' '!% ' '  P O$OOO OD   GGK % %E 778R D $         r")registry tool_errorr+afManage background processes started with terminal(background=true). Actions: 'list' (show all), 'poll' (check status + new output), 'log' (full output with pagination), 'wait' (block until done or timeout), 'kill' (terminate), 'write' (send raw stdin data without newline), 'submit' (send data + Enter, for answering prompts), 'close' (close stdin/send EOF).objectstring)rXr\logr rrsubmitrz)Action to perform on background processes)renum descriptionz]Process session ID (from terminal background output). Required for all actions except 'list'.)rrz@Text to send to process stdin (for 'write' and 'submit' actions)integerzJMax seconds to block for 'wait' action. Returns partial output on timeout.r)rrminimumz6Line offset for 'log' action (default: last 200 lines)z$Max lines to return for 'log' action)actionrrrrvrwr)r propertiesrequired)rr parametersc r|d}|dd}|d#t|ddnd}|dkr1tjdt|id S|d vr|st d |S|d kr.tjt|d S|d krYtjt||dd|ddd S|dkrCtjt ||dd S|dkr.tjt |d S|dkrPtjt |t|ddd S|dkrPtjt |t|ddd S|dkr.tjt |d St d|dS)Nr(rr'rrX processesrF) ensure_ascii>rrr\r rrrzsession_id is required for r\rrvrrwru)rvrwr rrrrrrrzUnknown process action: z8. Use: list, poll, log, wait, kill, write, submit, close)rrQrdumpsprocess_registryrrr\r}r rrrr)rkwr(rrs r _handle_processr2sffYG XXh # #F48HH\4J4J4VTXXlB//000\^J z;(8(F(Fw(F(W(WXglmmmm N N N FDFDDEE E V  :.33J??eTTT T u__:.77488Ha#8#8RU@V@V8XXfkmmm m v  :.33JQZH[H[3\\kpqqq q v  :.;;JGGV[\\\ \ w  :.:::s488TZ\^K_K_G`G`aapuvvv v x  :.;;JDHHU[]_L`L`HaHabbqvwww w w  :.:::FFUZ[[[ [ qqqq r rrr"terminalu⚙️)rtoolsetschemahandleremoji)7rPrloggingrrrrrSrYrrsystemrtools.environments.localrrrhermes_cli._subprocess_compatr dataclassesrr typingr r r r hermes_cli.configr getLoggerrMrrrWrrrrrrrrrQr!r$r^rrrUtools.registryrrPROCESS_SCHEMArregisterr\r"r rs0>    ho9, ]]]]]]]]]]<<<<<<((((((((,,,,,,,,,,,,------  8 $ $"/##&66  !  " ,0,0,0,0,0,0,0 ,0^JJJJJJJJ\*#?$$#T#l####R0///////  _![[[J !~ !a "k "W "E/  :J?  **Zsss:      r"