KiG dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ej dkZ ddl mZmZddlmZmZddlmZmZmZmZddlmZejeZedz Zd Zd Zd Z eGd d Z!GddZ"e"Z#ddl$m%Z%ddddgdddddddddddddddddd ddd!d"gd#d$Z&d%Z'e%j(dd&e&e'd'(dS))a Process Registry -- In-memory registry for managed background processes. Tracks processes spawned via terminal(background=true), providing: - Output buffering (rolling 200KB window) - Status polling and log retrieval - Blocking wait with interrupt support - Process killing - Crash recovery via JSON checkpoint file - Session-scoped tracking for gateway reset protection Background processes execute THROUGH the environment interface -- nothing runs on the host machine unless TERMINAL_ENV=local. For Docker, Singularity, Modal, Daytona, and SSH backends, the command runs inside the sandbox. Usage: from tools.process_registry import process_registry # Spawn a background process (called from terminal_tool) session = process_registry.spawn(env, "pytest -v", task_id="task_123") # Poll for status result = process_registry.poll(session.id) # Block until done result = process_registry.wait(session.id, timeout=300) # Kill it process_registry.kill(session.id) NWindows) _find_shell_sanitize_subprocess_env) dataclassfield)AnyDictListOptional)get_hermes_homezprocesses.jsoni@ i@ceZdZUdZeed<eed<dZeed<dZeed<dZe e ed<dZ e e j ed <dZeed <dZe eed <d Zeed <dZeed<dZe e ed<dZeed<eZe ed<dZeed<dZeed<dZeed<dZeed<dZe ed<eej Z!ej ed<eddZ"e ej#ed<eddZ$eed<dS)ProcessSessionz3A tracked background process with output buffering.idcommandtask_id session_keyNpidprocessenv_refcwdg started_atFexited exit_code output_buffermax_output_charsdetachedwatcher_platformwatcher_chat_idwatcher_thread_idrwatcher_interval)default_factory_lock)defaultrepr_reader_thread_pty)%__name__ __module__ __qualname____doc__str__annotations__rrrr intr subprocessPopenrrrrfloatrboolrrMAX_OUTPUT_CHARSrrrr r!r"r threadingLockr$r'Threadr(3/home/ubuntu/hermes-agent/tools/process_registry.pyrr>s== GGG LLLGSKC#*.GXj& '...GSC#JFD#Ix}###M3,c,,,HdcOSsc!E).AAAE9>AAA16t%1P1P1PNHY-.PPPd///D#/////r9rceZdZdZdZdZededefdZ d1d ed ed ed ede de de fdZ d2de d ed ed ed edede fdZde fdZde de dedefdZde fdZde fdZdedee fdZdede fdZd3ded!ed"ede fd#Zd4dedede fd$Zdede fd%Zded&ede fd'Zd5ded&ede fd(Zd4d edefd)Zd ede fd*Zd ede fd+Zd4d edefd,Z d-Z!d.Z"d/Z#defd0Z$dS)6ProcessRegistrya$ In-memory registry of running and finished background processes. Thread-safe. Accessed from: - Executor threads (terminal_tool, process tool handlers) - Gateway asyncio loop (watcher tasks, session reset checks) - Cleanup thread (sandbox reaping coordination) )z'bash: cannot set terminal process groupz"bash: no job control in this shellzno job control in this shellz!cannot set terminal process groupz)tcsetattr: Inappropriate ioctl for devicec`i|_i|_tj|_g|_dSN)_running _finishedr5r6r$pending_watchersselfs r:__init__zProcessRegistry.__init__ks/35 46^%% 79r9textreturnc|dratfdtjDr<dr%tfdtjDz5ProcessRegistry._clean_shell_noise..ws,cc%EU1X-ccccccr9r)splitanyr<_SHELL_NOISE_SUBSTRINGSpopjoin)rErLs @r:_clean_shell_noisez"ProcessRegistry._clean_shell_noisess 4   cccc?;bccccc  IIaLLL cccc?;bccccc yyr9NrFrrrrenv_varsuse_ptyc tdtjjdd||||pt jt j}|rf trddlm }nddl m }t} ttj |} d| d<|| d |g|j| d } | j|_| |_t%j|j|fd d |j} | |_| |j5|||j|j<dddn #1swxYwY||S#t8$rt:dYn1t>$r%} t:d| Yd} ~ nd} ~ wwxYwt} ttj |}d|d<tAj!| d |gd |j|ddt@j"t@j#t@j"trdn tj$ }||_%|j|_t%j|j&|fd d|j} | |_| |j5|||j|j<dddn #1swxYwY||S)aw Spawn a background process locally. Only for TERMINAL_ENV=local. Other backends use spawn_via_env(). Args: use_pty: If True, use a pseudo-terminal via ptyprocess for interactive CLI tools (Codex, Claude Code, Python REPL). Falls back to subprocess.Popen if ptyprocess is not installed. proc_N )rrrrrrr) PtyProcess1PYTHONUNBUFFEREDz-lic)x)renv dimensionsTzproc-pty-reader-targetargsdaemonnamez3ptyprocess not installed, falling back to pipe modez0PTY spawn failed (%s), falling back to pipe modeutf-8replace) rErr^encodingerrorsstdoutstderrstdin preexec_fnz proc-reader-)'ruuiduuid4hexosgetcwdtime _IS_WINDOWSwinptyrY ptyprocessrrenvironspawnrrr(r5r7_pty_reader_looprr'startr$_prune_if_neededr?_write_checkpoint ImportErrorloggerwarning Exceptionr0r1PIPESTDOUTsetsidr _reader_loop)rCrrrrrTrUsession_PtyProcessCls user_shellpty_envpty_procreaderebg_envprocs r: spawn_localzProcessRegistry.spawn_local}s&!.tz||',..#"ry{{y{{     ( V& VHCCCCCCCGGGGGG(]] 22:xHH.1*+)//1 ( 0 'l ' #)0!8GJ88  *0& Z88))+++07DM'*-888888888888888&&((( V V VTUUUUU V V VQSTUUUUUUUU V !]] *"*h??%(!"  ) ?$/*9tt     h !$, ,,     "(  Z 0 0  ! ! # # #(/DM'* % 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0    sUB=E-$E  E- EE-EE--$G GF<<G#$KKK r^timeoutc tdtjjdd||||t j|}d|jd}d|jd} t j|} d| d |d | d | } || | } | d d } | D]@}| }| rt||_nAn/#t$r"}d|_d|_d||_Yd}~nd}~wwxYw|jsDt'j|j|||| fdd|j}||_||j5|||j|j<dddn #1swxYwY||S)a# Spawn a background process through a non-local environment backend. For Docker/Singularity/Modal/Daytona/SSH: runs the command inside the sandbox using the environment's execute() interface. We wrap the command to capture the in-sandbox PID and redirect output to a log file inside the sandbox, then poll the log via subsequent execute() calls. This is less capable than local spawn (no live stdout pipe, no stdin), but it ensures the command runs in the correct sandbox context. rWNrX)rrrrrrrz/tmp/hermes_bg_z.logz.pidznohup bash -c z > z 2>&1 & echo $! > z && cat routputrTzFailed to start: z proc-poller-r`)rrmrnrorrrshlexquoteexecutegetstrip splitlinesisdigitr/rrrrrr5r7_env_poller_loopr'ryr$rzr?r{)rCr^rrrrrrlog_pathpid_pathquoted_command bg_commandresultrlinerrs r: spawn_via_envzProcessRegistry.spawn_via_envs(!.tz||',..#y{{   6WZ5555WZ555W-- 6^ 6 6 6 6! 6 6+3 6 6  <[[W[==FZZ"--3355F))++  zz||<<>>"%d))GKE < < <!GN "G $;$;$;G ! ! ! ! ! ! < ~ %,sHh70GJ00 F &,G " LLNNN Z 0 0  ! ! # # #(/DM'* % 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0    s+BD E!D>>E$GG G rcd} |jjd}|sn|r||}d}|j5|xj|z c_t |j|jkr|j|j d|_dddn #1swxYwYn2#t$r%}t d|Yd}~nd}~wwxYw |j dn2#t$r%}t d|Yd}~nd}~wwxYwd|_ |jj |_||dS) z:Background thread: read stdout from a local Popen process.TFNzProcess stdout reader ended: %srz$Process wait timed out or failed: %s)rrireadrSr$rlenrrr}debugwaitr returncoder_move_to_finished)rCr first_chunkchunkrs r:rzProcessRegistry._reader_loop6s  ? b.33D99( 33E::E"'K]bb))U2))7011G4LLL070EwG_F_F`F`0a-bbbbbbbbbbbbbbb b ? ? ? LL:A > > > > > > > > ? D O  + + + + D D D LL? C C C C C C C C D#O6 w'''''sUAB(AB B(BB("B#B(( C2CCC77 D&D!!D&rrcF|jstjd |d|dd}|dd}|r]|j5||_t|j|jkr|j|j d|_dddn #1swxYwY|d |d d }|dd }|r| d  d kr|d|d d } | dd } t| d  |_ n#ttf$r d |_ YnwxYwd|_||dSn4#t $r'd|_d |_ ||YdSwxYw|jdSdS)zBBackground thread: poll a sandbox log file for non-local backends.zcat 2>/dev/nullrrrrNzkill -0 $(cat z" 2>/dev/null) 2>/dev/null; echo $?rr0z wait $(cat T)rrrsleeprrr$rrrrrr/r ValueError IndexErrorrr) rCrr^rrr new_outputcheck check_output exit_resultexit_strs r:rz ProcessRegistry._env_poller_loopQs.&  JqMMM$ %BH%B%B%BBOO#ZZ"55 f ff0:-w4558PPP4;4I7KcJcJdJd4eG1fffffffffffffff  QXQQQ$ %yy266<<>>  L$;$;$=$=b$A$G$G$I$IS$P$P"%++RhRRR !#.##K +x<<BBDDH/,/0C0C0E0Eb0I0O0O0Q0Q,R,R))& 3///,.)))/%)GN**7333F   !%$&!&&w///  C.& & & & & sZ:G#?B# G##B''G#*B'+B;G#'>F&%G#&G>G#GG##-HHc`|j} |r |d}|rt|tr|n|dd}|j5|xj|z c_t|j|j kr|j|j d|_dddn #1swxYwYn#t$rYn#t$rYnwxYw|n2#t$r%}t d|Yd}~nd}~wwxYw |n2#t$r%}t d|Yd}~nd}~wwxYwd|_t!|d r|jnd |_||dS) z2Background thread: read output from a PTY process.rrerf)rhNzPTY stdout reader ended: %sz PTY wait timed out or failed: %sT exitstatusr)r(isaliver isinstancer-decoder$rrrEOFErrorrr}rrrhasattrrrr)rCrptyrrErs r:rxz ProcessRegistry._pty_reader_loop}sHl ;++--  HHTNNEj(25#(>(>kuuELLQXajLDkDk$]jj#11T911"7#899G2 C >CC CC C: C"C: C"C:!C""C:: D)D$$D)-E E1 E,,E1c|j5|j|jd||j|j<dddn #1swxYwY|dS)z(Move a session from running to finished.N)r$r?rQrr@r{)rCrs r:rz!ProcessRegistry._move_to_finisheds Z 1 1 M  gj$ / / /)0DN7: & 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1      s0AA A session_idc|j5|j|p|j|cdddS#1swxYwYdS)z*Get a session by ID (running or finished).N)r$r?rr@)rCrs r:rzProcessRegistry.gets Z S S=$$Z00RDN4F4Fz4R4R S S S S S S S S S S S S S S S S S Ss4A  A A cddlm}||}|dd|dS|j5|jr||jddnd}dddn #1swxYwY|j|j|jrd nd |jttj |j z |d }|jr |j |d <|j r d |d<d|d<|S)z9Check status and get new output for a background process.r strip_ansiN not_foundNo process with ID statuserrorrrrunning)rrrruptime_secondsoutput_previewrTrz=Process recovered after restart -- output history unavailablenote)tools.ansi_striprrr$rrrrrr/rrrrr)rCrrrrrs r:pollzProcessRegistry.polls\//////((:&& ?)4V*4V4VWW W ] h hJQJ_gZZ(=eff(EFFFegN h h h h h h h h h h h h h h h"*").?hhi;!$)++0B"BCC,    > 4")"3F;    ]!%F: \F6N s"AA"Aroffsetlimitcddlm}||}|dd|dS|j5||j}dddn #1swxYwY|}t |}|dkr|dkr || d} n ||||z} |j|jrdndd | |t | d d S) z;Read the full output log with optional pagination by lines.rrNrrrrrrHz lines)rrr total_linesshowing) rrrr$rrrrrrR) rCrrrrr full_outputrLrselecteds r:read_logzProcessRegistry.read_logsM//////((:&& ?)4V*4V4VWW W ] < <$*W%:;;K < < < < < < < < < < < < < < <&&((%jj  Q;;5199eVWW~HHVFUN23H"*").?hhiii))&h--///    sA  AAcddlm}ddlm}t t jdd}|}|}d}|r||kr |} d|d|d }n|p|} ||} | d d |d Stj | z} tj | kr| j r*d | j || j ddd} |r|| d<| S| r%d|| j dddd} |r|| d<| Stjdtj | kd|| j ddd} |r|| d<n d| d| d<| S)aY Block until a process exits, timeout, or interrupt. Args: session_id: The process to wait for. timeout: Max seconds to block. Falls back to TERMINAL_TIMEOUT config. Returns: dict with status ("exited", "timeout", "interrupted", "not_found") and output snapshot. rr)_interrupt_eventTERMINAL_TIMEOUT180NzRequested wait of z%s was clamped to configured limit of srrrri0)rrr timeout_note interruptedrz+User sent a new message -- wait interrupted)rrrr)rrzWaited zs, process still running)rrtools.terminal_toolrr/rpgetenvrrr monotonicrrris_setr) rCrrrrdefault_timeout max_timeoutrequested_timeoutreffective_timeoutrdeadliners r:rzProcessRegistry.waits 0/////888888bi(:EBBCC% #  A!2[!@!@ + 9%699*5999 L !2 @[ ((:&& ?)4V*4V4VWW W>##&77n))~ &!(!2(j)>uvv)FGG  :-9F>* &&(( +(j)>uvv)FGGI  :-9F>* JqMMM+n))0  j!6uvv!>??    [%1F> " "%Z/@%Z%Z%ZF> " r9c||}|dd|dS|jr d|jdS |jrY |jdn#t $r.|jr$tj|jtj YnwxYw|j r tr|j n;tj tj|j jtj nc#tt f$r|j Yn7wxYw|jr,|jr%|jd |jd d d|_d |_|||d|jdS#t $r}dt-|dcYd}~Sd}~wwxYw)zKill a background process.Nrrralready_exited)rrT)forcezkill rrrikilled)rrr)rrrr( terminaterrrpkillsignalSIGTERMrrskillpggetpgidProcessLookupErrorPermissionErrorrrrr{rr-)rCrrrs r: kill_processzProcessRegistry.kill_process#s((:&& ?)4V*4V4VWW W > *$.   8| V=L***6666 ==={= V^<<<= V+"S113333 "*W_-@"A"A6>RRR*O<+++O((*****+ VW[ V''(I (I(I(IST'UUU!GN #G   " "7 + + +  " " $ $ $&gjAA A 8 8 8%A77 7 7 7 7 7 7 8sfFAF5B FB FAC76F7*D$!F#D$$A6F G%F<6G<Gdatac||}|dd|dS|jrdddSt|dr|jr t |t r|dn|}|j|d t|d S#t$r}d t |dcYd}~Sd}~wwxYw|j r |j j sd d dS |j j ||j j d t|d S#t$r}d t |dcYd}~Sd}~wwxYw) zASend raw data to a running process's stdin (no newline appended).NrrrrzProcess has already finishedr(reok)r bytes_writtenrz?Process stdin not available (non-local backend or stdin closed)) rrrr(rr-encodewriterrrrkflush)rCrrrpty_datars r: write_stdinzProcessRegistry.write_stdinLs((:&& ?)4V*4V4VWW W > Y.9WXX X 7F # # <  < <3=dC3H3HR4;;w///d ""8,,,"&TCCC < < <")CFF;;;;;;;; < sgo&; s%0qrr r 8 O ! ' ' - - - O ! ' ' ) ) )"SYY?? ? 8 8 8%A77 7 7 7 7 7 7 8s>AB C&B=7C=CAD-- E7EEEc4|||dzS)zGSend data + newline to a running process's stdin (like pressing Enter).rH)r)rCrrs r: submit_stdinzProcessRegistry.submit_stdings D4K888r9c |j5t|jt|jz}dddn #1swxYwYrfd|D}g}|D]}|j|jdd|j|jtj dtj |j ttj |j z |jrdnd|jr|jddndd }|jr |j|d <|jrd |d <|||S) z1List all running and recently-finished processes.Nc*g|]}|jk |Sr8rrJrrs r: z1ProcessRegistry.list_sessions..qs%LLL!qyG7K7KA7K7K7Kr9rz%Y-%m-%dT%H:%M:%Srri8r)rrrrrrrrrTr)r$listr?valuesr@rrrrrrstrftime localtimerr/rrrrappend)rCr all_sessionsrrentrys ` r: list_sessionszProcessRegistry.list_sessionsks Z X X 4 4 6 677$t~?T?T?V?V:W:WWL X X X X X X X X X X X X X X X  MLLLL|LLLL ! !Ad9TcT?uu"m,?PQP\A]A]^^"%dikkAL&@"A"A&'h=((I<=O"S!/$%%"8"8QS  Ex 1%&[k"z )$(j! MM%  sAA##A'*A'c|j5tfd|jDcdddS#1swxYwYdS)zK|]}|jko|j VdSr>rrr s r:rMz7ProcessRegistry.has_active_processes..sG W$5QXr9Nr$rOr?r )rCrs `r:has_active_processesz$ProcessRegistry.has_active_processes Z  --//                  2AA A c|j5tfd|jDcdddS#1swxYwYdS)z>Check if there are active processes for a gateway session key.c3>K|]}|jko|j VdSr>)rr)rJrrs r:rMz9ProcessRegistry.has_active_for_session..sG ,=QXr9Nr)rCrs `r:has_active_for_sessionz&ProcessRegistry.has_active_for_sessionrrc |j5fd|jD}dddn #1swxYwYd}|D]8}||j}|ddvr|dz }9|S)zQKill all running processes, optionally filtered by task_id. Returns count killed.c<g|]} |jk|j|Sr>rr s r:r z,ProcessRegistry.kill_all..s;OqyG';';QX';';';';r9Nrr)rrr)r$r?r rrr)rCrtargetsrrrs ` r:kill_allzProcessRegistry.kill_alls Z  =//11G                  G&&wz22Fzz(##'CCC!  s &;??cTtjfdjD}|D] }j|= tjtjz}|t kr*jr%t jfd}j|=dSdSdS)zGRemove oldest finished sessions if over MAX_PROCESSES. Must hold _lock.c@g|]\}}|jz tk|Sr8)rFINISHED_TTL_SECONDS)rJsidrnows r:r z4ProcessRegistry._prune_if_needed..s:   Cal"&::: :::r9c(j|jSr>)r@r)r$rCs r:z2ProcessRegistry._prune_if_needed..sDN3DL Q Q Q Q Q Q Q Q Q Rs;B8A>B B8BB8BB88 C*C%%C*ctsdS tjtd}n#t $rYdSwxYwd}|D]}|d}|sd} tj|dd}n#ttf$rYnwxYw|rt|d|dd |d d |d d ||d |dtj d|dd |dd |dd |dd }|j 5||j|j<dddn #1swxYwY|dz }t d|jdd||jdkr?|j|j|j|j|j|j|jd ddlm}|tgn4#t $r'}t d|dYd}~nd}~wwxYw|S)z On gateway startup, probe PIDs from checkpoint file. Returns the number of processes recovered as detached. rre)rgrFTrrunknownrrrrrrr r!r") rrrrrrrrrr r!r"Nrz'Recovered detached process: %s (pid=%d)<)rcheck_intervalrplatformchat_id thread_idr2z#Could not clear checkpoint file: %sr4)r7existsjsonloads read_textrrrprrrrrrr$r?rr}inforr"rArrrr r!r6r3r) rCr8 recoveredrraliverr3rs r:recover_from_checkpointz'ProcessRegistry.recover_from_checkpoints" %%'' 1 j!:!:G!:!L!LMMGG   11  * * E))E""C E Q&8     (\*!IIi;;!IIi44 % - < < %(($yyty{{CC!%*YY/A2%F%F$)II.?$D$D&+ii0CR&H&H%*YY/A1%E%E   Z8807DM'*-888888888888888Q  EwWZXZWZG[]`aaa+a//)00&-j*1*B'.':$+$<#*#:%,%> 22 R / / / / / /  or 2 2 2 2 R R R LL>DL Q Q Q Q Q Q Q Q RsM-A AA>BB*)B*F88F< ?F< >I J JJ)NrrNF)Nrrr)rrr>)r)%r)r*r+r,rPrD staticmethodr-rSdictr3rrrr/rrrrxrr rrrrrrrr rrrr rzr0r{rGr8r9r:r<r<Ys999     \ nnnn n  n  nn nnnnhEE EE E  E  EE EEEER(N((((6*%*,/*;>*JM****X(((((<!!!!!ScSh~&>SSSS st4  3   d    8BBsBSBDBBBBH'8s'8t'8'8'8'8R8c888888699s9#9t9999SD:CD#$   s    "***"$$$RRR6BBBBBBBr9r<)registryraFManage background processes started with terminal(background=true). Actions: 'list' (show all), 'poll' (check status + new output), 'log' (full output with pagination), 'wait' (block until done or timeout), 'kill' (terminate), 'write' (send raw stdin data without newline), 'submit' (send data + Enter, for answering prompts).objectstring)r rlogrrrsubmitz)Action to perform on background processes)typeenum descriptionz]Process session ID (from terminal background output). Required for all actions except 'list'.)rOrQz@Text to send to process stdin (for 'write' and 'submit' actions)integerzJMax seconds to block for 'wait' action. Returns partial output on timeout.r)rOrQminimumz6Line offset for 'log' action (default: last 200 lines)z$Max lines to return for 'log' action)actionrrrrrrT)rO propertiesrequired)rdrQ parametersc Hddl}|d}|dd}|d#t|ddnd}|dkr2|dt|id S|d vr|s|d d |id S|dkr/|t|d S|dkrZ|t||dd|ddd S|dkrD|t||dd S|dkr/|t |d S|dkrQ|t |t|ddd S|dkrQ|t |t|ddd S|d d|did S)NrrrTrrr  processesrF) ensure_ascii)rrMrrrrNrzsession_id is required for rrMrrr)rrrrrrrrrNzUnknown process action: z1. Use: list, poll, log, wait, kill, write, submit) rArr-dumpsprocess_registryrrrrrrr)rbkw_jsonrrTrs r:_handle_processr_WsffYG XXh # #F48HH\4J4J4VTXXlB//000\^J {{K)9)G)GPW)G)X)XYhm{nnn E E E f;;)Ov)O)OP_d;ee e V  ;;/44Z@@u;UU U u__;;/88488Ha#8#8RU@V@V 9 X Xfkmm m v  ;;/44ZR[I\I\4]]lq;rr r v  ;;/<ros>    ho9, JJJJJJJJ((((((((,,,,,,,,,,,,------  8 $ $"/##&66  0000000 04E E E E E E E E R#?$$ $#####  ?!RRRJ !~ !a "k "W "E/  :J?  **ZLLL8      r9