+
    i8                     f   R t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	t	^ RI
Ht ^ RIHtHtHt ^ RIHt ^ RIHt ^ RIHt ]P,                  ! ]4      t]! 4       R,          tR R	 ltR
 R ltR R ltR R ltR R ltR R lt]P@                  ! 4       t!RR R llt" ! R R]4      t#R# )a  Singularity/Apptainer persistent container environment.

Security-hardened with --containall, --no-home, capability dropping.
Supports configurable resource limits and optional filesystem persistence
via writable overlay directories that survive across sessions.
NPath)AnyDictOptional)get_hermes_home)BaseEnvironment)is_interruptedzsingularity_snapshots.jsonc                $    V ^8  d   QhR\         /#    returnstr)formats   ";/home/ubuntu/hermes-agent/tools/environments/singularity.py__annotate__r      s      c     c                     \         P                  ! R4      '       d   R# \         P                  ! R4      '       d   R# \        R4      h)zLocate the apptainer or singularity CLI binary.

Returns the executable name (``"apptainer"`` or ``"singularity"``).
Raises ``RuntimeError`` with install instructions if neither is found.
	apptainersingularityzNeither 'apptainer' nor 'singularity' was found in PATH. Install Apptainer (https://apptainer.org/docs/admin/main/installation.html) or Singularity and ensure the CLI is available.)shutilwhichRuntimeError r   r   _find_singularity_executabler      s:     ||K  ||M""
	: r   c                $    V ^8  d   QhR\         /# r   r   )r   s   "r   r   r   .   s      s r   c                 v   \        4       p  \        P                  ! V R.RR^
R7      pTP                  ^ 8w  d@   TP                  P                  4       R,          p\	        RT  R	TP                   R
T 24      hT #   \         d    \	        RT  R24      h\        P
                   d    \	        RT  R24      hi ; i)zPreflight check: resolve the executable and verify it responds.

Returns the executable name on success.
Raises ``RuntimeError`` with an actionable message on failure.
versionTcapture_outputtexttimeoutz:Singularity backend selected but the resolved executable 'z1' could not be executed. Check your installation.'z6 version' timed out. The runtime may be misconfigured.:N   Nz version' failed (exit code z): )	r   
subprocessrunFileNotFoundErrorr   TimeoutExpired
returncodestderrstrip)exeresultr*   s      r   _ensure_singularity_availabler.   .   s     '
(C
)	
  A$$&t,u01B1B0C3vhO
 	
 J!  
H N> >
 	
 $$ 
uJK
 	

s   A: :,B8'B8c                F    V ^8  d   QhR\         \        \        3,          /# r   r   r   )r   s   "r   r   r   P   s      c3h r   c                      \         P                  4       '       d*    \        P                  ! \         P	                  4       4      # / #   \
         d     / # i ; iN)_SNAPSHOT_STOREexistsjsonloads	read_text	Exceptionr   r   r   _load_snapshotsr9   P   sN    	::o779:: I  	I	s   'A AAc                J    V ^8  d   QhR\         \        \        3,          RR/# )r   datar   Nr0   )r   s   "r   r   r   Y   s"     ; ;$sCx. ;T ;r   c                     \         P                  P                  R R R7       \         P                  \        P
                  ! V ^R7      4       R# )Tparentsexist_ok)indentN)r3   parentmkdir
write_textr5   dumps)r;   s   &r   _save_snapshotsrE   Y   s4       =tzz$q9:r   c                $    V ^8  d   QhR\         /# r   r   )r   s   "r   r   r   b   s      $ r   c                    \         P                  ! R4      p V '       d!   \        V 4      pVP                  RRR7       V# ^ RIHp V! 4       R,          p\        R4      pVP                  4       '       d|   \         P                  ! V\         P                  4      '       dQ   V\         P                  ! RR4      ,          R	,          pVP                  RRR7       \        P                  R
V4       V# VP                  RRR7       V# )a  Get the best directory for Singularity sandboxes.

Resolution order:
  1. TERMINAL_SCRATCH_DIR (explicit override)
  2. TERMINAL_SANDBOX_DIR / singularity (shared sandbox root)
  3. /scratch (common on HPC clusters)
  4. ~/.hermes/sandboxes/singularity (fallback)
TERMINAL_SCRATCH_DIRTr=   )get_sandbox_dirr   z/scratchUSERhermeszhermes-agentz Using /scratch for sandboxes: %s)osgetenvr   rB   tools.environments.baserI   r4   accessW_OKloggerinfo)custom_scratchscratch_pathrI   sandboxscratchuser_scratchs         r   _get_scratch_dirrX   b   s     YY56NN+4$77-/G:G~~BIIgrww7768!<<~M4$76EMM$M.Nr   c                $    V ^8  d   QhR\         /# r   r   )r   s   "r   r   r      s     
 
$ 
r   c                     \         P                  ! R4      p V '       d!   \        V 4      pVP                  RRR7       V# \	        4       pVR,          pVP                  RRR7       V# )z1Get the Apptainer cache directory for SIF images.APPTAINER_CACHEDIRTr=   z
.apptainer)rL   rM   r   rB   rX   )	cache_dir
cache_pathrV   s      r   _get_apptainer_cache_dirr^      sa    		./I)_
5 G<'JTD1r   c                <    V ^8  d   QhR\         R\         R\         /# )r   image
executabler   r   )r   s   "r   r   r      s!     3 3S 3c 3C 3r   c           
        V P                  R4      '       d"   \        V 4      P                  4       '       d   V # V P                  R4      '       g   V # V P	                  RR4      P	                  RR4      P	                  RR4      p\        4       pW2 R2,          pVP                  4       '       d   \        V4      # \        ;_uu_ 4        VP                  4       '       d   \        V4      uuRRR4       # \        P                  R4       \        P                  R	V 4       \        P                  R
V4       VR,          pVP                  RRR7       \        P                  P                  4       p\        V4      VR&   \        V4      VR&    \        P                  ! VR\        V4      V .RRRVR7      pVP                   ^ 8w  dI   \        P#                  R4       \        P#                  RVP$                  R,          4       V uuRRR4       # \        P                  R4       \        V4      uuRRR4       #   \        P&                   dK    \        P#                  R4       TP                  4       '       d   TP)                  4        T u uuRRR4       # \*         d-   p\        P#                  RT4       T u Rp?uuRRR4       # Rp?ii ; i  + '       g   i     R# ; i)zGet or build a SIF image from a docker:// URL.

Returns the path unchanged if it's already a .sif file.
For docker:// URLs, checks the cache and builds if needed.
z.sifz	docker:// /-:Nz&Building SIF image (one-time setup)...z  Source: %sz  Target: %stmpTr=   APPTAINER_TMPDIRr[   buildiX  )r    r!   r"   envz/SIF build failed, falling back to docker:// URLz  Error: %s:Ni  NzSIF image built successfullyz2SIF build timed out, falling back to docker:// URLz2SIF build error: %s, falling back to docker:// URL)endswithr   r4   
startswithreplacer^   r   _sif_build_lockrQ   rR   rB   rL   environcopyr%   r&   r)   warningr*   r(   unlinkr8   )	r`   ra   
image_namer\   sif_pathtmp_dirrj   r-   es	   &&       r   _get_or_build_sifrw      s+    ~~f$u+"4"4"6"6K(({B/77SAII#sSJ(*I\..H8}	??x= 
 	<=NE*NH-e#dT2jjoo"%g,$'	N !	^^Wc(mU;#$F   A%PQ}fmmD.AB/ 
0 KK67x=3 
4 (( 	NNOP  !L= 
>  	NNOQRSLC 
>	? 
s\   !K4BKA6H7H77?K7K	KKKK6K7KKKK	c                   t   a a ] tR t^t oRtRV3R lV 3R llltR tRRRRR/V3R lR	 llltR
 tRt	Vt
V ;t# )SingularityEnvironmenta  Hardened Singularity/Apptainer container with resource limits and persistence.

Security: --containall (isolated PID/IPC/mount namespaces, no host home mount),
--no-home, writable-tmpfs for scratch space. The container cannot see or modify
the host filesystem outside of explicitly bound paths.

Persistence: when enabled, the writable overlay directory is preserved across
sessions so installed packages and files survive cleanup/restore.
c                J   < V ^8  d   QhRS[ RS[ RS[RS[RS[RS[RS[RS[ /# )	r   r`   cwdr"   cpumemorydiskpersistent_filesystemtask_id)r   intfloatbool)r   __classdict__s   "r   r   #SingularityEnvironment.__annotate__   s[        	
     $ r   c	                
  < \         S
V `  W#R 7       \        4       V n        \	        WP                  4      V n        R\        P                  ! 4       P                  R,           2V n	        RV n
        Wpn        Wn        RV n        W@n        WPn        V P                  '       dS   \!        4       R,          p	V	P#                  RRR7       V	RV 2,          V n        V P                  P#                  RRR7       V P%                  4        R# )	)r{   r"   hermes_:N   NFNzhermes-overlaysTr=   zoverlay-)super__init__r.   ra   rw   r`   uuiduuid4hexinstance_id_instance_started_persistent_task_id_overlay_dir_cpu_memoryrX   rB   _start_instance)selfr`   r{   r"   r|   r}   r~   r   r   overlay_base	__class__s   &&&&&&&&& r   r   SingularityEnvironment.__init__   s     	S279&uoo>
$TZZ\%5%5c%:$;<!&0,0 	 +-0AALtd; ,'/C CD##D4#@r   c                >   V P                   R R.pVP                  RR.4       V P                  '       d:   V P                  '       d(   VP                  R\	        V P                  4      .4       MVP                  R4        ^ RIHpHp V! 4        FO  pVP                  RVR,           R	VR
,           R2.4       \        P                  RVR,          VR
,          4       KQ  	  V! 4        FO  pVP                  RVR,           R	VR
,           R2.4       \        P                  RVR,          VR
,          4       KQ  	  V P                  ^ 8  d!   VP                  RV P                   R2.4       V P                  ^ 8  d'   VP                  R\	        V P                  4      .4       VP                  \	        V P                  4      V P                   .4        \"        P$                  ! VRR^xR7      pVP&                  ^ 8w  d   \)        RVP*                   24      hRV n        \        P                  RV P                   V P                  4       R#   \         d"   p\        P                  RT4        Rp?EL7Rp?ii ; i  \"        P.                   d    \)        R4      hi ; i)instancestartz--containallz	--no-homez	--overlayz--writable-tmpfs)get_credential_file_mountsget_skills_directory_mountz--bind	host_pathrf   container_pathz:roz(Singularity: binding credential %s -> %sz(Singularity: binding skills dir %s -> %sz8Singularity: could not load credential/skills mounts: %sNz--memoryMz--cpusTr   zFailed to start instance: z/Singularity instance %s started (persistent=%s)zInstance start timed out)ra   extendr   r   r   appendtools.credential_filesr   r   rQ   rR   r8   debugr   r   r`   r   r%   r&   r)   r   r*   r   r(   )r   cmdr   r   mount_entryskills_mountrv   r-   s   &       r   r   &SingularityEnvironment._start_instance   sA   
G4 	

NK01  1 1 1JJS):):%;<=JJ)*	Xe9;

H[)A(B!KP`DaCbbe&fgh>, 01  < !; <

Hk)B(C1\RbEcDddg&hij> - !12 != <<!JJ
t||nA$67899q=JJ#dii.12

C

OT%5%567	;^^C4QTUF  A%"%?#OPP%)D"KKI(($*:*:<!  	XLLSUVWW	X$ (( 	;9::	;s&    B<I A5I; I8I33I8;!Jr"   N
stdin_datac          
      T   < V ^8  d   QhRS[ RS[ RS[R,          RS[ R,          RS[/# )r   commandr{   r"   Nr   r   )r   r   dict)r   r   s   "r   r   r   )  sI     OT OTs OT OTtOT$JOT26OTr   c               J  aa V P                   '       g   R RRR/# T;'       g    V P                  pT;'       g    V P                  pV P                  V4      w  rxVe   Ve
   W,           p	M	Ve   Tp	MTp	VR8X  d	   RV 2pRpM<VP	                  R4      '       d&   R\
        P                  ! VR	,          4       R
V 2pRpV P                  RRVRV P                   2RRV.p
 ^ RI	p. o\        P                  ! T
\        P                  \        P                  V	'       d   \        P                  M\        P                  RR7      oV	'       d7    SP                  P!                  V	4       SP                  P#                  4        VV3R lp\&        P(                  ! VRR7      pVP+                  4        VP-                  4       V,           pSP/                  4       f   \1        4       '       dR   SP3                  4         SP5                  ^R7       VP;                  ^R7       R RP;                  S4      R,           R^/# VP-                  4       V8  d4   SP9                  4        VP;                  ^R7       V P=                  V4      # VP?                  R4       K  VP;                  ^R7       R RP;                  S4      RSP@                  /#   \$         d     ELSi ; i  \        P6                   d    SP9                  4         Li ; i  \$         d   pR RT 2R^/u Rp?# Rp?ii ; i)outputzInstance not startedr)   N~zcd ~ && z/tmpz~/zcd ~/:r   NNz && execz--pwdzinstance://bashz-cT)stdoutr*   stdinr!   c                  v   <  SP                    F  p SP                  V 4       K  	  R #   \         d     R # i ; ir2   )r   r   r8   )line_output_chunksprocs    r   _drain.SingularityEnvironment.execute.<locals>._drainX  s6     $&--d3 !,  s   $) 88)targetdaemon)r"   rc   z
[Command interrupted]g?zSingularity execution error: )!r   r"   r{   _prepare_commandrl   shlexquotera   r   timer%   PopenPIPESTDOUTDEVNULLr   writecloser8   	threadingThreadr   	monotonicpollr	   	terminatewaitr(   killjoin_timeout_resultsleepr)   )r   r   r{   r"   r   effective_timeoutwork_direxec_command
sudo_stdineffective_stdinr   _timer   readerdeadlinerv   r   r   s   &&&$$           @@r   executeSingularityEnvironment.execute)  s    %%%4lBGG#33t||??$((#'#8#8#A  !j&<(5O#(O(O s?%l^4LH  &&"5;;x|#<"=T,PLHT--./t\+0	T N##!z/@/@)8jooj>P>P	D JJ$$_5JJ$$& %%VDAFLLN(+<<H))+%!##NN$$		!	, KKK* "''."9<U"U$c  ??$x/IIKKKK*//0ABBC KKK"bggn5|T__UUC ! & &44 $		$  	T =aSA<QRSS	Ts{   !<L -L 5K A8L ;K -L ;AL AL KL KL %L>L  LL L"LL"L"c                   V P                   '       d[    \        P                  ! V P                  RRV P                  .RR^R7       \
        P                  RV P                  4       RV n         V P                  '       dM   V P                  '       d9   \        4       p\        V P                  4      W P                  &   \        V4       R# R# R#   \         d,   p\
        P                  RT P                  T4        Rp?LRp?ii ; i)	zMStop the instance. If persistent, the overlay dir survives for next creation.r   stopTr   zSingularity instance %s stoppedz*Failed to stop Singularity instance %s: %sNF)r   r%   r&   ra   r   rQ   rR   r8   rq   r   r   r9   r   r   rE   )r   rv   	snapshotss   &  r   cleanupSingularityEnvironment.cleanupz  s    !!!b__j&$:J:JK#'dB =t?O?OP &+D"  1 1 1')I'*4+<+<'=Imm$I& !2  bKTM]M]_`aabs   AC D!C>>D)	r   r   r   r   r   r   ra   r`   r   )r   <       r   r   Fdefault)rc   )__name__
__module____qualname____firstlineno____doc__r   r   r   r   __static_attributes____classdictcell____classcell__)r   r   s   @@r   ry   ry      sH      B2;hOT&*OT)-OT OTb' 'r   ry   )r   )$r   r5   loggingrL   r   r   r%   tempfiler   r   pathlibr   typingr   r   r   hermes_constantsr   rN   r   tools.interruptr	   	getLoggerr   rQ   r3   r   r.   r9   rE   rX   r^   Lockrn   rw   ry   r   r   r   <module>r      s      	        & & , 3 *			8	$!#&BB"D;:
 .."3tB'_ B'r   