
    Mhd                        U d Z ddlmZ ddlZddlZddlZddlmZ ddl	m
Z
mZmZmZmZmZmZmZ ddlmZ ddlmZmZmZ dZej        Zej        Zej        Zej        Zej        Zej        Zej        Zej         Z ej!        Z!ej"        Z"ej#        Z#ej$        Z$ej%        Z% G d	 d
e
          Z& ej'        d          Z(i Z)eee&f         e*d<   dddZ+d Z,ddddZ-d Z.d Z/dZ0 e.de0z   dz             Z1 e.de0z   dz             Z2 e.d          Z3 e.d          Z4d Z5 G d de
          Z6 G d de6          Z7 G d  d!          Z8d" Z9d# Z:d$d%d&e;d'ed(ee<e<f         d)ee7         fd*Z=d+ Z>dd$d,d-d.ee;         d'ed/ee<         d(ee<e<f         d)ee7         f
d0Z?d1 Z@eAd2k    rhejB        d3         ZC eDeC          5 ZEeEF                                ZGddd           n# 1 swxY w Y    e=eG ed4          5          D ]ZH eIeH           dS dS )6a  
This tokenizer has been copied from the ``tokenize.py`` standard library
tokenizer. The reason was simple: The standard library tokenizer fails
if the indentation is not right. To make it possible to do error recovery the
    tokenizer needed to be rewritten.

Basically this is a stripped down version of the standard library module, so
you can read the documentation there. Additionally we included some speed and
memory optimizations here.
    )absolute_importN)BOM_UTF8)
NamedTupleTupleIteratorIterableListDictPatternSet)PythonTokenTypes)split_linesPythonVersionInfoparse_version_stringu   􏿿c                       e Zd ZU eed<   ee         ed<   ee         ed<   eeef         ed<   eed<   eeef         ed<   ee         ed<   dS )	TokenCollectionpseudo_tokensingle_quotedtriple_quotedendpats
whitespacefstring_pattern_mapalways_break_tokensN)	__name__
__module____qualname__r   __annotations__r   strr
   r        U/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/parso/python/tokenize.pyr   r   +   s         s8s8#w,c3h'''s#####r    r   zutf-8_token_collection_cacheFcapturec                 R    |rJ d}| s|dz  }|d                     |          z   dz   S )N(z?:|))join)r$   choiceskwargsstarts       r!   groupr-   :   s@    E 388G$$$s**r    c                      t          |  dz   S )N?)r-   )r*   s    r!   mayber0   C   s    '?S  r    include_fstringonly_fstringc                     d }g d}dh}| rddg}|r|}t                      }n||z  }n|rt                      S |D ]7}t          j        |          D ] }|                     ||                     !8|S )Nc              3   n   K   t          j        d | D              D ]}d                    |          V  d S )Nc                 :    g | ]}||                                 fS r   )upper).0cs     r!   
<listcomp>zI_all_string_prefixes.<locals>.different_case_versions.<locals>.<listcomp>J   s$    %E%E%Eq!''))n%E%E%Er     )
_itertoolsproductr)   )prefixss     r!   different_case_versionsz5_all_string_prefixes.<locals>.different_case_versionsI   sO      #%E%Ef%E%E%EF 	 	A''!**	 	r    )brubrr;   ffr)setr<   permutationsupdate)r2   r3   r@   valid_string_prefixesresultrE   r>   ts           r!   _all_string_prefixesrM   H   s       211TF $K 	'$%!UUFF!Q&!!	 uu ( 6 6(00 	6 	6A MM11!445555	6 Mr    c                 @    t          j        | t           j                  S N)recompileUNICODE)exprs    r!   _compilerT   e   s    :dBJ'''r    c                     	 t           t          |                    S # t          $ r+ t          |           xt           t          |           <   }|cY S w xY wrO   )r"   tupleKeyError_create_token_collection)version_inforK   s     r!   _get_token_collectionrZ   i   s`    &u\':':;;   $\22	3l 3 34vs    2AAz"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z(?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c                    d}t          |          }d}dt          z   dz   }d}d}d}d}t          ||||          }	d	}
t          d
d          t          |
          z   }d|
z   }t          ||          }t          d|dz             }t          |||	          }t	                      }t          | }t          t	          d           }t	          dd          }t          | }d}d}d}d}t          |dz   |dz             }t          ddddddd          }d}g d }| d!k    r|                    d"d#           n|                    d"d$           t          | }t          |||          }t          |d%z   t          d&d'          z   |d(z   t          d)d'          z             }||g} d*}!|r|                     |t          |! z              t          d+g| R  }"t          |d,          t          |"||||d,          z   }#i }$|D ]V}%t          |          |$|%d&z   <   t          |          |$|%d)z   <   t          |          |$|%dz   <   t          |          |$|%dz   <   Wt                      }&t                      }'i }(|D ]<})d-D ]}*|&                    |)|*z              d.D ]}*|'                    |)|*z              =|D ]})|!D ]
}*|*|(|)|*z   <   d/}+t          |#          },t          |,|&|'|$||(t          |+                    S )0Nz[ \f\t]*z	#[^\r\n]*u   ([A-Za-z_0-9-z]+)z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]T)r2   r1   z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""'''"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?~z[][(){}])z\.\.\.z\r\n?z\nz[;.,@])      r   z:=?:z'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*'z\\(?:\r\n?|\n)z"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*")rc   rb   r]   r\   z\\(?:\r\n?|\n)|\Zr#   )rc   rb   )r]   r\   );importclassdeftryexceptfinallywhilewithreturncontinuebreakdelpassglobalassertnonlocal)
rT   MAX_UNICODEr-   r0   rM   insertappendrG   addr   )-rY   
Whitespacer   CommentName	Hexnumber	Binnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberpossible_prefixesStringPrefixStringPrefixWithFfstring_prefixesFStringStartSingleDoubleSingle3Double3TripleOperatorBracketspecial_argsSpecialFunnyContStrpseudo_extra_pool
all_quotesPseudoExtrasPseudoTokenr   _prefixr   r   r   rL   quoteALWAYS_BREAK_TOKENSpseudo_token_compileds-                                                r!   rX   rX   ~   s    J*%%JG ;.6D*I#I$I2IiIyAAI,HC-/ /16xAJ#h.H
H--K/w1FGGJ:{I66F -..+,L3DIIIJ+DtTTT*+L !F F+G+G$u,.?%.GHHF
 Ye+ H
 G:::LvAu%%%%As###\"G(GW--E L#EEC!2334 #EEC!23345 5G !&)'J D  z0B!BCCC-B0ABBBL
D111lFE7D$GGGHK G$ 5 5!)&!1!1#!)&!1!1##+G#4#4% #+G#4#4%   EEMEEM ) ) 	) 	)Ea%i((((! 	) 	)Ea%i((((	)  3 3 	3 	3E-2E	**	3S %[11}mW'-@)A)A  r    c                   t    e Zd ZU eed<   eed<   eeef         ed<   eed<   edeeef         fd            Z	dS )Tokentypestring	start_posr>   rm   c                     t          | j                  }t          |          dk    r"| j        d         t          |          z   dz
  dfS | j        d         | j        d         t          | j                  z   fS N   r   )r   r   lenr   )selfliness     r!   end_poszToken.end_pos   sj    DK((u::>>>!$s5zz1A5q88>!$dnQ&7#dk:J:J&JJJr    N)
r   r   r   r   r   r   r   intpropertyr   r   r    r!   r   r      s|         
KKKS#XKKKKsCx K K K XK K Kr    r   c                       e Zd Zd ZdS )PythonTokenc                 H    d|                      | j        j                  z  S )Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r   )_replacer   namer   s    r!   __repr__zPythonToken.__repr__   s$    H49>223 	4r    N)r   r   r   r   r   r    r!   r   r      s#        4 4 4 4 4r    r   c                   2    e Zd Zd Zd Zd Zd Zd Zd ZdS )FStringNodec                 L    || _         d| _        d| _        d | _        d| _        d S )Nr   r;   )r   parentheses_countprevious_lineslast_string_start_posformat_spec_count)r   r   s     r!   __init__zFStringNode.__init__   s1    
!" %)" "#r    c                 &    | xj         dz  c_         d S )Nr   )r   r   	characters     r!   open_parentheseszFStringNode.open_parentheses	  s    !#r    c                 N    | xj         dz  c_         | j         dk    r	d| _        d S d S r   r   r   r   s     r!   close_parentheseszFStringNode.close_parentheses  s:    !#!Q&&%&D""" '&r    c                 2    t          | j                  dk    S )Nr_   )r   r   r   s    r!   allow_multilinezFStringNode.allow_multiline  s    4:!##r    c                 "    | j         | j        k    S rO   r   r   s    r!   
is_in_exprzFStringNode.is_in_expr  s    %(>>>r    c                 :    |                                   o| j        S rO   )r   r   r   s    r!   is_in_format_speczFStringNode.is_in_format_spec  s    ??$$$?)??r    N)	r   r   r   r   r   r   r   r   r   r   r    r!   r   r      ss        # # #$ $ $' ' '$ $ $? ? ?@ @ @ @ @r    r   c           
         t          |           D ]\  }}|                                }t          |          t          |          z
  }|                    |j                  rYt          t          |j        |||z   f||d |         z             }	d}|j        rJ | |d = |	dt          |j                  |z   fc S d |dfS )Nr>   r;   r   )	enumeratelstripr   
startswithr   r   FSTRING_ENDr   )
fstring_stackr   line_nrcolumnadditional_prefixfstring_stack_indexnodelstripped_string
len_lstriptokens
             r!   _close_fstring_if_necessaryr     s    %.}%=%= ; ;!T!==??[[3'7#8#88
&&tz22 
	;
&:-.()<<	  E !#****1223"c$*oo
:::::
	; "A%%r    c                    |d         }|                                 }|                                r|rt          }nt          }n|rt          }nt
          }|                    ||          }|	|j        |fS |j        s	||f|_        |	                    d          }	|D ]T}
| |
j
                                     |	          }|0|	                    d          d t          |
j
                            }	U|}|t          |	          z  }|	                    d          s|	                    d          r|xj        |	z  c_        d}	n
|j        |	z   }	|	|fS )Nr   
r;   )r   r   fstring_format_spec_multi_linefstring_format_spec_single_linefstring_string_multi_linefstring_string_single_linematchr   r   r-   r   r   endswith)r   r   linelnumpostosr   regexr   r   fstring_stack_node	end_matchnew_poss                r!   _find_fstring_stringr   .  sw   

C))++O
 	/ 	42EE3EE 	/-EE.EKKc""E}!3&& 0%)3K![[^^F+ I I.45;;FCC	 __Q''(G#.@.F*G*G)G(GHFGs6{{G t - 5 5 -f$#f,7?r    )r   r   )r   coderY   r   rm   c                H    t          | d          }t          |||          S )z0Generate tokens from a the source code (string).T)keepends)rY   r   )r   tokenize_lines)r   rY   r   r   s       r!   tokenizer   V  s+     t,,,E%liPPPPr    c                       fd}|S )zL
    A small helper function to help debug the tokenize_lines function.
    c               ?   H   K    | i |D ]}t          |           |V  d S rO   )print)argsr+   r   funcs      r!   wrapperz_print_tokens.<locals>.wrapperb  sE      T4*6** 	 	E%LLLKKKK	 	r    r   )r   r   s   ` r!   _print_tokensr   ^  s#        
 Nr    T)indentsr   is_first_tokenr   r   c          
   #     ./K   ./fd}t          |          \  }}}}	}
}}d}dgd}d}d}d}d}d}|d         dz
  .g }| D ]}.dz  .d}t          |          }|r]|                    t                    r t          }|dd         }t          |          }d|d         z  |z   }|d         }||d         z  }d	}|r`|                    |          }|r>|                    d          }t          t          ||d|         z   ||          V  d}d}n||z   }||z   }||k     r|r|d
         }|                                sGt          |	||.|          \  }}|r't          t          ||j        d          V  d|_        d||k    rn||d         }t          ||.||          \  }}}||z  }||V  |r|} |D ]}!|!j        }"|	|"                             ||          }#|#k|#                    d          }$t          |$          t          |"          z
  |z   t          |           k     r#|d|         |$dt          |"                    z   } |                    | |          }%n|                    ||          }%|%rb||%                    d          z   }d}|%                    d          \  }&}.|&f/|%                    d          }'|'dk    r|sJ |}n?|'d         }(nH|
                    ||          })||)                                         }(|)                                }&.|&f/|rc|(dvr_|(dk    s|%Wd	}|dk    rO|sM|&}*|*d
         k    r.t          t"          d/d          V                      |*            ||*          E d{V  |%s|
                    ||          })|r+|dk    r%|s# ||)                                          E d{V  |)                                }d	}t          t&          ||         .|f||)                    d          z             V  d}|dz  }|(|v s|(dk    r'|'dk    r!|'dk    rt          t(          |'/|          V  n|%                    d          |'|v rO|s|rKg |dd<   d}t+          j        d|d|&                   }+|+# ||+                                          E d{V  |'                                rt          t.          |'/|          V  nt1          |'/|          E d{V  n|(dv rZt3          d |D                       r|                                 |s"|dk    r|st          t6          |'/|          V  n||'z   }d}n|(dk    ro|'                    d          s|'                    d          rJ |r:|d
                                         r t          t&          |(/|          V  |&dz   }n:||'z   }n3|'|v rj|	|'         }|                    ||          }|r:|                    d          }||&|         }'t          t          |'/|          V  nՉ/}||&d         }|}n|(|v s|'dd         |v s|'dd         |v r|'d
         dv r].|&f}|	                    |(          p5|	                    |'d                   p|	                    |'d                   }||&d         }|}nIt          t          |'/|          V  n'|'|v rB|                    t=          ||'                              t          t>          |'/|          V  n|(dk    r||&d         dv r||||&d         z   z  }n|'dv r$|r|d
                              |'           n|dz  }n}|'dv r&|r|d
         !                    |'           n[|r|dz  }nS|'                    d          r>|r<|d
         j"        |d
         j#        z
  dk    r|d
         xj#        dz  c_#        d}'|&dz   }t          tH          |'/|          V  ||k     |rEt          t&          |||          V  |                    d          s|                    d          rd}|r3|d
         }|j        r$t          t          |j        |j        d          V  .|f},dd         D ]/}-%                                 t          tL          d|,d          V  0t          tN          d|,|          V  dS )a)  
    A heavily modified Python standard library tokenizer.

    Additionally to the default information, yields also the prefix of each
    token. This idea comes from lib2to3. The prefix contains all information
    that is irrelevant for the parser like newlines in parentheses or comments.
    c              3      K   | d         k     ri| d         k    r"t          t          d| fd          V  | d<   d S                                  t          t          dd          V  | d         k     gd S d S )Nr   r;   )r   ERROR_DEDENTpopDEDENT)r,   r   r   sposs    r!   dedent_if_necessaryz+tokenize_lines.<locals>.dedent_if_necessaryz  s      gbk!!wr{""!,T5M2FFFFF#KKMMMfb$33333 gbk!!!!!!r    r   N
0123456789r;   Tr   ^Fr   r      z
#\.z...r_   z	[ \f\t]*$z
c              3   @   K   | ]}|                                  V  d S rO   )r   )r8   rE   s     r!   	<genexpr>z!tokenize_lines.<locals>.<genexpr>  s1      FF11,,...FFFFFFr    #r   r   )z\
z\
z\z([{z)]}ra   )(rZ   r   r   BOM_UTF8_STRINGr   endr   STRINGr   r   FSTRING_STRINGr   r   r   r   r-   spanINDENTrw   
ERRORTOKENNUMBERrP   isidentifierNAME_split_illegal_unicode_nameanyclearNEWLINEr   getr   FSTRING_STARTr   r   r   r   OPr   r   	ENDMARKER)0r   rY   r   r   r   r  r   r   r   r   r   r   r   paren_levelmax_numcharscontstrnew_liner>   r   r   r   r   endprogendmatchcontstr_startcontliner   r   restfstring_end_tokenquote_lengthstring_liner   r   r   end_match_stringpseudomatchr,   r   initialr   indent_startmr   indentr   r   s0     `                                           @@r!   r   r   k  s     4 4 4 4 4 4 4 	l++2L-0K#DHG HFQ<!D')M U; U;	4yy 	#// !$3!ABBx4yy 1%,DA,CIaL D"N 	}}T**H 
ll1oo!Gd4C4j0!6+ + + + + !D.#d?Djj #B'~~'' "6wtUY[^"_"_KFC 	!)*F5 $&      .0* d{{CDDzE`!%F FB!#4l |#$0++++  <"*7 V V&.4E ' 4 4T3 ? ?I ,+4??1+=+=(/003u::=Cc+FVFVVV*.tt*7G#e**7U*UK*00cBB*00s;; %*[->->q-A-AA$&!(--a00
se}#))!,,B;;!MMM(.% ("((s33uyy{{+		e} AG7224;K^ !##M##(L#gbk11)&"dB?????|44422<@@@@@@@@@ "((s33 @q 0 0 022599;;?????????iikk !S	D#;%A6     %'!q8##35C<<EUNN!&%v>>>>>>""1%%1///]/k/')M!!!$"#KtFUF|<<A}#6#6quuww#?#????????%%'' P%dE4@@@@@@:5$OOOOOOOOOOF""FFFFFFF *!''))) 7K1$4$4]$4%gudFCCCCCC(.%C >>$//Lt8L8LLLL  7]2%6%A%A%C%C 7%j'4HHHHH!)CC(.%%-''!%."==s33 ",,q//C sOE%feT6BBBBBB$(M"566lG#H M))"1"I.."1"I..9&& %)%KM&{{733  8w{{587L7L  8")++eAh"7"7 "566lG#H%feT6BBBBBB---$$[1DU1K%L%LMMM!-fEEEEEEDT%&&\5O%O%O!Vd566l%::!E>>$ )%b)::5AAAA#q(e^^$ -%b);;EBBBB& -'1,K%%c** $} $)"-?'+=>ABC C ""%771<77E!)C!"eT6:::::m Djjp  *g}fEEEEED!! 	W%5%5d%;%; 	H 	B 	 2)       DjG !""+ 3 3&"gr222222
iW.?
@
@@@@@@r    c              #     	K   	fd}dd|	t          |           D ]\  }}r@|                                r& |            V  |dd|d         |d         |z   f	A|z  G|z   }|                                r|cr! |            V  d|d         |d         |z   f	|dr |            V  d S d S )Nc                  D    t          rt          nt                     S rO   )r   r  r  )found
is_illegalr   r>   s   r!   create_tokenz1_split_illegal_unicode_name.<locals>.create_token  s     =::uc6RRRr    r;   Fr   r   T)r   r  )
r   r   r>   r3  ichar	new_foundr1  r2  r   s
     `    @@@r!   r  r    sW     S S S S S S S S EJ
CU## " "4 	"  "" "lnn$$$"
lIaL1$44I%%'' "! 9&,..(((F#A,	!q(88C!

 lnn r    __main__r   z3.10)rY   )J__doc__
__future__r   sysrP   	itertoolsr<   codecsr   typingr   r   r   r   r	   r
   r   r   parso.python.tokenr   parso.utilsr   r   r   ru   r  r  r  r  r  r  r   r  r  r   r  r  r   r   decoder
  r"   r   r-   r0   rM   rT   rZ   unicode_character_namer   r   r   r   rX   r   r   r   r   r   r   r   r   r   r   r  r   argvpathopenrE   readr   r   r   r   r    r!   <module>rF     s"  	 	 	 ' & & & & & 



 				                              0 / / / / / L L L L L L L L L L 		 		 

"		 		 &	(
, .!0*$ $ $ $ $j $ $ $ "(/'**DF /@A F F F " + + + + +! ! !
 -2     :( ( (   ? %X2234   %H225KK   #+(+K"L"L !))!4!4 i i iXK K K K KJ K K K4 4 4 4 4% 4 4 4@ @ @ @ @ @ @ @:& & &$% % %R QWQ Q Q
Q 1Q>CCHoQkQ Q Q Q	 	 	" !'ZA ZA ZAC=ZA $ZA #Y	ZA
 S#XZA kZA ZA ZA ZAz  B z8A;D	d qvvxx               $-A-A&-I-IJJJ  e 
 s   :GG"G