
    je                     P   U d Z ddlmZ ddlZddlZddlZddlmZ ddl	m
Z
mZmZmZmZmZmZmZmZ ddlmZ ddlmZmZmZ dZej        Zej        Zej        Zej        Zej        Zej        Zej         Z ej!        Z!ej"        Z"ej#        Z#ej$        Z$ej%        Z%ej&        Z& G d	 d
e
          Z' ej(        d          Z)i Z*eee+e+f         e'f         e,d<   dddZ-d Z.ddddZ/d Z0d Z1dZ2 e0de2z   dz             Z3 e0de2z   dz             Z4 e0d          Z5 e0d          Z6d Z7 G d de
          Z8 G d de8          Z9 G d  d!          Z:d" Z;d# Z<d$d%d&e=d'ee+e+f         d(ee+e+f         d)ee9         fd*Z>d+ Z?dd$d,d-d.ee=         d'ee+e+f         d/ee+         d(ee+e+f         d)ee9         f
d0Z@d1 ZAeBd2k    rhejC        d3         ZD eEeD          5 ZFeFG                                ZHddd           n# 1 swxY w Y    e>eH ed4          5          D ]ZI eJeI           dS dS )6a  
This tokenizer has been copied from the ``tokenize.py`` standard library
tokenizer. The reason was simple: The standard library tokenizer fails
if the indentation is not right. To make it possible to do error recovery the
    tokenizer needed to be rewritten.

Basically this is a stripped down version of the standard library module, so
you can read the documentation there. Additionally we included some speed and
memory optimizations here.
    )absolute_importN)BOM_UTF8)	
NamedTupleTupleIteratorIterableListDictPatternSetAny)PythonTokenTypes)split_linesPythonVersionInfoparse_version_stringu   􏿿c                       e Zd ZU eed<   ee         ed<   ee         ed<   eeef         ed<   eed<   eeef         ed<   ee         ed<   dS )	TokenCollectionpseudo_tokensingle_quotedtriple_quotedendpats
whitespacefstring_pattern_mapalways_break_tokensN)__name__
__module____qualname__r   __annotations__r   strr
        \/home/ubuntu/.hermes/hermes-agent/venv/lib/python3.11/site-packages/parso/python/tokenize.pyr   r   +   s         s8s8#w,c3h'''S!!!!!r!   r   zutf-8_token_collection_cacheFcapturec                 R    |rJ d}| s|dz  }|d                     |          z   dz   S )N(z?:|))join)r%   choiceskwargsstarts       r"   groupr.   :   s?    :E 388G$$$s**r!   c                      t          |  dz   S )N?)r.   )r+   s    r"   mayber1   C   s    '?S  r!   include_fstringonly_fstringc                     d }g d}dh}| rddg}|r|}t                      }n||z  }n|rt                      S |D ]7}t          j        |          D ] }|                     ||                     !8|S )Nc              3   n   K   t          j        d | D              D ]}d                    |          V  d S )Nc                 :    g | ]}||                                 fS r    )upper).0cs     r"   
<listcomp>zI_all_string_prefixes.<locals>.different_case_versions.<locals>.<listcomp>J   s$    %E%E%Eq!''))n%E%E%Er!    )
_itertoolsproductr*   )prefixss     r"   different_case_versionsz5_all_string_prefixes.<locals>.different_case_versionsI   sO      #%E%Ef%E%E%EF 	 	A''!**	 	r!   )brubrr<   ffr)setr=   permutationsupdate)r3   r4   rA   valid_string_prefixesresultrF   r?   ts           r"   _all_string_prefixesrN   H   s       211TF $K 	'$%!UUFF!Q&!!	 uu ( 6 6(00 	6 	6A MM11!445555	6 Mr!   c                 @    t          j        | t           j                  S N)recompileUNICODE)exprs    r"   _compilerU   e   s    :dBJ'''r!   c                     	 t           t          |                    S # t          $ r+ t          |           xt           t          |           <   }|cY S w xY wrP   )r#   tupleKeyError_create_token_collection)version_inforL   s     r"   _get_token_collectionr[   i   s`    &u\':':;;   $\22	3l 3 34vs    2AAz"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z(?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c                    d}t          |          }d}dt          z   dz   }d}d}d}d}t          ||||          }	d	}
t          d
d          t          |
          z   }d|
z   }t          ||          }t          d|dz             }t          |||	          }t	                      }t          | }t          t	          d           }t	          dd          }t          | }d}d}d}d}t          |dz   |dz             }t          ddddddd          }d}g d }| d!k    r|                    d"d#           n|                    d"d$           t          | }t          |||          }t          |d%z   t          d&d'          z   |d(z   t          d)d'          z             }||g} d*}!|r|                     |t          |! z              t          d+g| R  }"t          |d,          t          |"||||d,          z   }#i }$|D ]V}%t          |          |$|%d&z   <   t          |          |$|%d)z   <   t          |          |$|%dz   <   t          |          |$|%dz   <   Wt                      }&t                      }'i }(|D ]<})d-D ]}*|&                    |)|*z              d.D ]}*|'                    |)|*z              =|D ]})|!D ]
}*|*|(|)|*z   <   d/}+t          |#          },t          |,|&|'|$||(t          |+                    S )0Nz[ \f\t]*z	#[^\r\n]*u   ([A-Za-z_0-9-z]+)z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]T)r3   r2   z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""'''"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?~z[][(){}])z\.\.\.z\r\n?z\nz[;.,@])      r   z:=?:z'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*'z\\(?:\r\n?|\n)z"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*")rd   rc   r^   r]   z\\(?:\r\n?|\n)|\Zr$   )rd   rc   )r^   r]   );importclassdeftryexceptfinallywhilewithreturncontinuebreakdelpassglobalassertnonlocal)
rU   MAX_UNICODEr.   r1   rN   insertappendrH   addr   )-rZ   
Whitespacer   CommentName	Hexnumber	Binnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberpossible_prefixesStringPrefixStringPrefixWithFfstring_prefixesFStringStartSingleDoubleSingle3Double3TripleOperatorBracketspecial_argsSpecialFunnyContStrpseudo_extra_pool
all_quotesPseudoExtrasPseudoTokenr   _prefixr   r   r   rM   quoteALWAYS_BREAK_TOKENSpseudo_token_compileds-                                                r"   rY   rY   ~   s    J*%%JG ;.6D*I#I$I2IiIyAAI,HC-/ /16xAJ#h.H
H--K/w1FGGJ:{I66F -..+,L3DIIIJ+DtTTT*+L !F F+G+G$u,.?%.GHHF
 Ye+ H
 G:::LvAu%%%%As###\"G(GW--E L#EEC!2334 #EEC!23345 5G !&)'J D  z0B!BCCC-B0ABBBL
D111lFE7D$GGGHK G$ 5 5!)&!1!1#!)&!1!1##+G#4#4% #+G#4#4%   EEMEEM ) ) 	) 	)Ea%i((((! 	) 	)Ea%i((((	)  3 3 	3 	3E-2E	**	3S %[11}mW'-@)A)A  r!   c                   t    e Zd ZU eed<   eed<   eeef         ed<   eed<   edeeef         fd            Z	dS )Tokentypestring	start_posr?   rn   c                     t          | j                  }t          |          dk    r"| j        d         t          |          z   dz
  dfS | j        d         | j        d         t          | j                  z   fS N   r   )r   r   lenr   )selfliness     r"   end_poszToken.end_pos   sj    DK((u::>>>!$s5zz1A5q88>!$dnQ&7#dk:J:J&JJJr!   N)
r   r   r   r   r   r   r   intpropertyr   r    r!   r"   r   r      s|         
KKKS#XKKKKsCx K K K XK K Kr!   r   c                       e Zd Zd ZdS )PythonTokenc                 H    d|                      | j        j                  z  S )Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r   )_replacer   namer   s    r"   __repr__zPythonToken.__repr__   s$    H49>223 	4r!   N)r   r   r   r   r    r!   r"   r   r      s#        4 4 4 4 4r!   r   c                   2    e Zd Zd Zd Zd Zd Zd Zd ZdS )FStringNodec                 L    || _         d| _        d| _        d | _        d| _        d S )Nr   r<   )r   parentheses_countprevious_lineslast_string_start_posformat_spec_count)r   r   s     r"   __init__zFStringNode.__init__   s1    
!" *." "#r!   c                 &    | xj         dz  c_         d S )Nr   )r   r   	characters     r"   open_parentheseszFStringNode.open_parentheses	  s    !#r!   c                 N    | xj         dz  c_         | j         dk    r	d| _        d S d S r   r   r   r   s     r"   close_parentheseszFStringNode.close_parentheses  s:    !#!Q&&%&D""" '&r!   c                 2    t          | j                  dk    S )Nr`   )r   r   r   s    r"   allow_multilinezFStringNode.allow_multiline  s    4:!##r!   c                 "    | j         | j        k    S rP   r   r   s    r"   
is_in_exprzFStringNode.is_in_expr  s    %(>>>r!   c                 :    |                                   o| j        S rP   )r   r   r   s    r"   is_in_format_speczFStringNode.is_in_format_spec  s    ??$$$?)??r!   N)	r   r   r   r   r   r   r   r   r   r    r!   r"   r   r      ss        # # #$ $ $' ' '$ $ $? ? ?@ @ @ @ @r!   r   c           
         t          |           D ]\  }}|                                }t          |          t          |          z
  }|                    |j                  rYt          t          |j        |||z   f||d |         z             }	d}|j        rJ | |d = |	dt          |j                  |z   fc S d |dfS )Nr?   r<   r   )	enumeratelstripr   
startswithr   r   FSTRING_ENDr   )
fstring_stackr   line_nrcolumnadditional_prefixfstring_stack_indexnodelstripped_string
len_lstriptokens
             r"   _close_fstring_if_necessaryr     s    %.}%=%= ; ;!T!==??[[3'7#8#88
&&tz22 
	;
&:-.()<<	  E !#****1223"c$*oo
:::::
	; "A%%r!   c                    |d         }|                                 }|                                r|rt          }nt          }n|rt          }nt
          }|                    ||          }|	|j        |fS |j        s	||f|_        |	                    d          }	|D ]T}
| |
j
                                     |	          }|0|	                    d          d t          |
j
                            }	U|}|t          |	          z  }|	                    d          s|	                    d          r|xj        |	z  c_        d}	n
|j        |	z   }	|	|fS )Nr   
r<   )r   r   fstring_format_spec_multi_linefstring_format_spec_single_linefstring_string_multi_linefstring_string_single_linematchr   r   r.   r   r   endswith)r   r   linelnumpostosr   regexr   r   fstring_stack_node	end_matchnew_poss                r"   _find_fstring_stringr   .  sw   

C))++O
 	/ 	42EE3EE 	/-EE.EKKc""E}!3&& 0%)3K![[^^F+ I I.45;;FCC	 __Q''(G#.@.F*G*G)G(GHFGs6{{G t - 5 5 -f$#f,7?r!   )r   r   )r   coderZ   r   rn   c                H    t          | d          }t          |||          S )z0Generate tokens from a the source code (string).T)keepends)rZ   r   )r   tokenize_lines)r   rZ   r   r   s       r"   tokenizer   V  s+     t,,,E%liPPPPr!   c                       fd}|S )zL
    A small helper function to help debug the tokenize_lines function.
    c               ?   H   K    | i |D ]}t          |           |V  d S rP   )print)argsr,   r   funcs      r"   wrapperz_print_tokens.<locals>.wrapperb  sE      T4*6** 	 	E%LLLKKKK	 	r!   r    )r   r   s   ` r"   _print_tokensr   ^  s#        
 Nr!   T)indentsr   is_first_tokenr   r   c          
   #     ./K   ./fd}t          |          \  }}}}	}
}}d}dgd}d}d}d}d}d}|d         dz
  .g }| D ]}.dz  .d}t          |          }|r]|                    t                    r t          }|dd         }t          |          }d|d         z  |z   }|d         }||d         z  }d	}|r`|                    |          }|r>|                    d          }t          t          ||d|         z   ||          V  d}d}n||z   }||z   }||k     r|r|d
         }|                                sGt          |	||.|          \  }}|r't          t          ||j        d          V  d|_        d||k    rn||d         }t          ||.||          \  }}}||z  }||V  |r|} |D ]}!|!j        }"|	|"                             ||          }#|#k|#                    d          }$t          |$          t          |"          z
  |z   t          |           k     r#|d|         |$dt          |"                    z   } |                    | |          }%n|                    ||          }%|%rb||%                    d          z   }d}|%                    d          \  }&}.|&f/|%                    d          }'|'dk    r|sJ |}n?|'d         }(nH|
                    ||          })||)                                         }(|)                                }&.|&f/|rc|(dvr_|(dk    s|%Wd	}|dk    rO|sM|&}*|*d
         k    r.t          t"          d/d          V                      |*            ||*          E d{V  |%s|
                    ||          })|r+|dk    r%|s# ||)                                          E d{V  |)                                }d	}t          t&          ||         .|f||)                    d          z             V  d}|dz  }|(|v s|(dk    r'|'dk    r!|'dk    rt          t(          |'/|          V  n|%                    d          |'|v rO|s|rKg |dd<   d}t+          j        d|d|&                   }+|+# ||+                                          E d{V  |'                                rt          t.          |'/|          V  nt1          |'/|          E d{V  n|(dv rZt3          d |D                       r|                                 |s"|dk    r|st          t6          |'/|          V  n||'z   }d}n|(dk    ro|'                    d          s|'                    d          rJ |r:|d
                                         r t          t&          |(/|          V  |&dz   }n:||'z   }n3|'|v rj|	|'         }|                    ||          }|r:|                    d          }||&|         }'t          t          |'/|          V  nՉ/}||&d         }|}n|(|v s|'dd         |v s|'dd         |v r|'d
         dv r].|&f}|	                    |(          p5|	                    |'d                   p|	                    |'d                   }||&d         }|}nIt          t          |'/|          V  n'|'|v rB|                    t=          ||'                              t          t>          |'/|          V  n|(dk    r||&d         dv r||||&d         z   z  }n|'dv r$|r|d
                              |'           n|dz  }n}|'dv r&|r|d
         !                    |'           n[|r|dz  }nS|'                    d          r>|r<|d
         j"        |d
         j#        z
  dk    r|d
         xj#        dz  c_#        d}'|&dz   }t          tH          |'/|          V  ||k     |rEt          t&          |||          V  |                    d          s|                    d          rd}|r3|d
         }|j        r$t          t          |j        |j        d          V  .|f},dd         D ]/}-%                                 t          tL          d|,d          V  0t          tN          d|,|          V  dS )a)  
    A heavily modified Python standard library tokenizer.

    Additionally to the default information, yields also the prefix of each
    token. This idea comes from lib2to3. The prefix contains all information
    that is irrelevant for the parser like newlines in parentheses or comments.
    c              3      K   | d         k     ri| d         k    r"t          t          d| fd          V  | d<   d S                                  t          t          dd          V  | d         k     gd S d S )Nr   r<   )r   ERROR_DEDENTpopDEDENT)r-   r   r   sposs    r"   dedent_if_necessaryz+tokenize_lines.<locals>.dedent_if_necessaryz  s      gbk!!wr{""!,T5M2FFFFF#KKMMMfb$33333 gbk!!!!!!r!   r   N
0123456789r<   Tr   ^Fr   r      z
#\.z...r`   z	[ \f\t]*$z
c              3   @   K   | ]}|                                  V  d S rP   )r   )r9   rF   s     r"   	<genexpr>z!tokenize_lines.<locals>.<genexpr>  s1      FF11,,...FFFFFFr!   #r   r   )z\
z\
z\z([{z)]}rb   )(r[   r   r   BOM_UTF8_STRINGr   endr   STRINGr   r   FSTRING_STRINGr   r   r   r   r.   spanINDENTrx   
ERRORTOKENNUMBERrQ   isidentifierNAME_split_illegal_unicode_nameanyclearNEWLINEr   getr   FSTRING_STARTr   r   r   r   OPr   r   	ENDMARKER)0r   rZ   r   r   r   r  r   r   r   r   r   r   r   paren_levelmax_numcharscontstrnew_liner?   r   r   r   r   endprogendmatchcontstr_startcontliner   r   restfstring_end_tokenquote_lengthstring_liner   r   r   end_match_stringpseudomatchr-   r   initialr   indent_startmr   indentr   r  s0     `                                           @@r"   r   r   k  s     4 4 4 4 4 4 4 	l++2L-0K#DHG HFQ<!D')M U; U;	4yy 	#// !$3!ABBx4yy 1%,DA,CIaL D"N 	}}T**H 
ll1oo!Gd4C4j0!6+ + + + + !D.#d?Djj #B'~~'' "6wtUY[^"_"_KFC 	!)*F5 $&      .0* d{{CDDzE`!%F FB!#4l |#$0++++  <"*7 V V&.4E ' 4 4T3 ? ?I ,+4??1+=+=(/003u::=Cc+FVFVVV*.tt*7G#e**7U*UK*00cBB*00s;; %*[->->q-A-AA$&!(--a00
se}#))!,,B;;!MM6(.% ("((s33uyy{{+		e} AG7224;K^ !##M##(L#gbk11)&"dB?????|44422<@@@@@@@@@ "((s33 @q 0 0 022599;;?????????iikk !S	D#;%A6     %'!q8##35C<<EUNN!&%v>>>>>>""1%%1///]/k/')M!!!$"#KtFUF|<<A}#6#6quuww#?#????????%%'' P%dE4@@@@@@:5$OOOOOOOOOOF""FFFFFFF *!''))) 7K1$4$4]$4%gudFCCCCCC(.%C >>$//Lt8L8LLLL  7]2%6%A%A%C%C 7%j'4HHHHH!)CC(.%%-''!%."==s33 ",,q//C sOE%feT6BBBBBB$(M"566lG#H M))"1"I.."1"I..9&& %)%KM&{{733  8w{{587L7L  8")++eAh"7"7 "566lG#H%feT6BBBBBB---$$[1DU1K%L%LMMM!-fEEEEEEDT%&&\5O%O%O!Vd566l%::!E>>$ )%b)::5AAAA#q(e^^$ -%b);;EBBBB& -'1,K%%c** $} $)"-?'+=>ABC C ""%771<77E!)C!"eT6:::::m Djjp  *g}fEEEEED!! 	W%5%5d%;%; 	H 	B 	 2)       DjG !""+ 3 3&"gr222222
iW.?
@
@@@@@@r!   c              #     	K   	fd}dd|	t          |           D ]\  }}r@|                                r& |            V  |dd|d         |d         |z   f	A|z  G|z   }|                                r|cr! |            V  d|d         |d         |z   f	|dr |            V  d S d S )Nc                  D    t          rt          nt                     S rP   )r   r  r  )found
is_illegalr   r?   s   r"   create_tokenz1_split_illegal_unicode_name.<locals>.create_token  s     =::uc6RRRr!   r<   Fr   r   T)r   r  )
r   r   r?   r4  ichar	new_foundr2  r3  r   s
     `    @@@r"   r  r    sW     S S S S S S S S EJ
CU## " "4 	"  "" "lnn$$$"
lIaL1$44I%%'' "! 9&,..(((F#A,	!q(88C!

 lnn r!   __main__r   z3.10)rZ   )K__doc__
__future__r   sysrQ   	itertoolsr=   codecsr   typingr   r   r   r   r	   r
   r   r   r   parso.python.tokenr   parso.utilsr   r   r   rv   r  r  r  r  r  r  r   r  r  r   r  r  r   r   decoder  r#   r   r   r.   r1   rN   rU   r[   unicode_character_namer   r   r   r   rY   r   r   r   r   r   r   r   r   r   r  r   argvpathopenrF   readr   r   r   r    r!   r"   <module>rG     sC  	 	 	 ' & & & & & 



 				                                0 / / / / / L L L L L L L L L L 		 		 

"		 		 &	(
, .!0*" " " " "j " " " "(/'**BD eCHo>? D D D " + + + + +! ! !
 -2     :( ( (   ? %X2234   %H225KK   #+(+K"L"L !))!4!4 i i iXK K K K KJ K K K4 4 4 4 4% 4 4 4@ @ @ @ @ @ @ @:& & &$% % %R OUQ Q Q
Q %c3hQ<A#s(OQkQ Q Q Q	 	 	" !'ZA ZA ZAC=ZA S/ZA #Y	ZA
 S#XZA kZA ZA ZA ZAz  B z8A;D	d qvvxx               $-A-A&-I-IJJJ  e 
 s   G55G9<G9