o
0iAe @ sR d Z dZddlmZ ddlmZmZ ddlZddl Z ddl
m Z ddl Z
ddlZddlZddlT ddlmZ ed ejZed
ejZddlZejg d Z[G d d
d
ed
dZdd Zdd Zdd ZdZ dZ e ede ee Z dZ dZ dZ!dZ"d Z#ee e!e"e#Z$d Z%ed d ee% Z&d e% Z'ee&e'Z(ed!e(d" Z)ee)e(e$Z*d#d$ Z+e j,d%d& Z-ee+ Z.d'Z/d(Z0d)Z1d*Z2ee.d+ e.d, Z3ee.d- e.d. Z4ee5ej6e7ed/d0 Z8ed1e8Z9ee*e9e4e Z:e e: Z;ee.d2 ed3d e.d4 ed5d Zi Z?e+ D ]Z@e/e?e@d3 < e0e?e@d5 < e1e?e@d+ < e2e?e@d, < qeA ZBeA ZCe+ D ]%ZDeDd5 eDd3 fD ]ZEeBFeE qHeDd, eDd+ fD ]ZEeCFeE qYq>d7ZGG d8d9 d9eHZIG d:d; d;eHZJG dd? ZLd@dA ZMdBdC ZNdDdE ZdFdG ZOdHdI ZPdJdK ZQdLdM ZReSdNkreR dS dS )OzKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord )open)lookupBOM_UTF8N)
TextIOWrapper)*)EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$))tokenizegenerate_tokensdetect_encoding
untokenize TokenInfoc @ s e Zd Zdd Zedd ZdS )r c C s$ d| j t| j f }d| j|d S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))type)r
tok_name_replace)selfannotated_type r //opt/alt/python310/lib64/python3.10/tokenize.py__repr__/ s
zTokenInfo.__repr__c C s$ | j tkr| jtv rt| j S | j S N)r
OPstringr r r r r
exact_type4 s
zTokenInfo.exact_typeN)__name__
__module__ __qualname__r propertyr r r r r r . s r ztype string start end linec G s dd | d S )N(|))joinchoicesr r r group; s r$ c G t | d S )Nr r$ r" r r r any< r' c G r% )N?r& r" r r r maybe= r( r* z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c C sV g d} dh}| D ] }t |D ]}t jdd |D D ]
}|d| q qq |S )N)brufbrfr c S s g | ]}|| fqS r )upper).0cr r r
_ s z(_all_string_prefixes..)
_itertools permutationsproductaddr! )_valid_string_prefixesresultprefixtr- r r r _all_string_prefixesT s r> c C s t | t jS r )recompileUNICODE)exprr r r _compilec s rC z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"T)reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z
\\\r?\n|\Z c @ e Zd ZdS )
TokenErrorNr r r r r r r rI r( rI c @ rH )StopTokenizingNrJ r r r r rK r( rK c @ s, e Zd Zdd Zdd Zdd Zdd Zd S )
Untokenizerc C s g | _ d| _d| _d | _d S )N r )tokensprev_rowprev_colencodingr r r r __init__ s
zUntokenizer.__init__c C s |\}}|| j k s|| j kr || jk r td||| j | j|| j }|r1| jd| d| _|| j }|rB| jd| d S d S )Nz+start ({},{}) precedes previous end ({},{})\
r )rO rP
ValueErrorformatrN append)r startrowcol
row_offset
col_offsetr r r add_whitespace s
zUntokenizer.add_whitespacec C s, t |}g }d}|D ]}t|dkr| || nv|\}}}} }
|tkr)|| _q
|tkr/ na|tkr9|| q
|tkrH| | \| _
| _ q
|t t
fv rQd}n |rn|rn|d } |d t| krl| j| t| | _ d}| | | j| | \| _
| _ |t t
fv r| j
d7 _
d| _ q
d| jS )NF TrM r r1 )iterlencompatENCODINGrQ ENDMARKERINDENTrW DEDENTpoprO rP NEWLINENLrN r] r! ) r iterableitindents startliner= tok_typetokenrX endlineindentr r r r sH
zUntokenizer.untokenizec
C s g }| j j}|d ttfv }d}t|g|D ]W}|d d \}} |tkr)| | _q|tt fv r3| d7 } |t
kr@|r=d| } d}nd}|t krL|| q|t krU|
q|ttfv r^d}n |rj|rj||d d}|| qd S )Nr Fr^ rT Tr_ )rN rW rh ri r6 chainrc rQ NAMENUMBERSTRINGre rf rg )
r ro rj rl toks_appendrm
prevstringtoktoknumtokvalr r r rb s:
zUntokenizer.compatN)r r r rR r] r rb r r r r rL s
%rL c C s, t }|| }|jd ur||j}|S r )rL r rQ encode)rj utoutr r r r s
r c C sJ | d d dd}|dks|drdS |dv s!|dr#dS | S ) N _-utf-8zutf-8-)zlatin-1
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r )lowerreplace
startswith)orig_encencr r r _get_normal_name s r c s zj jW n ty d Y nw d d }d}fdd} fdd}| }|tr8d |dd }d }|s>|g fS ||}|rI||gfS t|sS||gfS | }|s]||gfS ||}|ri|||gfS |||gfS )
NFr c s z W S t y Y dS w N )
StopIterationr readliner r read_or_stopA s
z%detect_encoding..read_or_stopc s z| d}W n ty d}d urd|}t|w t|}|s(d S t|d}zt|}W n t yR d u rHd| }t|d|}t|w rp|dkrld u rcd}t|d}t||d 7 }|S )
Nr z'invalid or missing encoding declarationz {} for {!r}rM zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
decodeUnicodeDecodeErrorrV SyntaxError cookie_rematchr r$ r LookupError)rq line_stringmsgr rQ codec) bom_foundfilenamer r find_cookieG s@
z$detect_encoding..find_cookieT utf-8-sig)__self__nameAttributeErrorr r blank_rer )r rQ defaultr r firstsecondr )r r r r r
) s< &
r
c C sP t | d}zt|j\}}|d t||dd}d|_|W S | )Nrbr T)line_bufferingr, )
_builtin_openr
r seekr modeclose)r bufferrQ linestextr r r r s
r c C s8 t | \}}td}t|t| d|}t|j|S r )r
r6 repeatrs r` _tokenize__next__)r rQ consumedemptyrl_genr r r r s
r c c sl d } }}d}d\}}d }dg} |d ur%|dkr d}t t|dddV d}
d} z| }
| } W n ty< d} Y nw |d urF| |} |d
7 }dt| } }
|r| sZtd ||| }|r|d } }t t|| d | |||f|| V d\}}d }n|r| d d d
kr| dd dkrt t || ||t| f|V d}d }q)|| }|| }q)|dkr|s| sn)d}| |
k r| | dkr|d
7 }n| | dkr|t
d
t
}n
| | dkrd}nn| d
7 } | |
k s| |
krn| | dv r8| | dkr$| | d d}t t ||| f|| t| f| V | t|7 } t t
| | d || f|t| f| V q)|| d krU| | t t| d | |df|| f| V || d k r|| vrjtdd|| | f| d d } t td|| f|| f| V || d k s\n | std|dfd}| |
k rtt| | }|r|d
\}}||f||f|}}} ||krq| || | | }}||v s|dkr|dkr|dkrt t|||| V n|dv r|dkrt t
|||| V nt t|||| V n|dkrt t |||| V n|tv rItt| }|| | }|r<|d} | || }t t|||| f| V n||f}| |d }| }n|tv s`|d d tv s`|d d tv r|d d kr||f}tt|pt|d
pt|d }| |d d
}}| }nXt t|||| V nI| rt t |||| V n:|d krd
}n2|d v r|d
7 }n |d!v r|d
8 }t t |||| V nt t | | || f|| d
f| V | d
7 } | |
k sq*|
r|
d dvr|
dst td|d
t|
f|d
t|
d
fdV | d
d D ]}t td|df|dfdV qt t d|df|dfdV d S )"Nr
0123456789)r1 r r r )r r r1 r TrM zEOF in multi-line stringrS z\
rT z#
#z
r_ z3unindent does not match any outer indentation levelz
zEOF in multi-line statement.z...r^ r
\z([{z)]})!r rc r r ra rI r rp rv
ERRORTOKENtabsizerstripCOMMENTri rW re IndentationErrorrf rC PseudoTokenspanru rh
triple_quotedendpats
single_quotedget isidentifierrt r stripr rd )r rQ lnumparenlev continuednumcharscontstrneedcontcontlinerl last_linerq posmaxstrstartendprogendmatchrp column
comment_token pseudomatchrX sposeposro initialrr r r r r s*
$
"
$#. r c C s t | d S r )r r r r r r f s
r c
s2 dd l } dd d fdd }| jdd}|jdd d
d d |jd
ddddd | }zQ|jrP|j}t|d}tt|j}W d n1 sJw Y n d}t t
j jd }|D ] }|j }|j
rf|j}d|j|j } td| t| |jf q[W d S ty }
z |
jd dd \} } ||
jd || | f W Y d }
~
d S d }
~
w ty }
z|
jd \} } ||
jd || | f W Y d }
~
d S d }
~
w ty }
z ||
| W Y d }
~
d S d }
~
w ty }
z ||
W Y d }
~
d S d }
~
w ty td Y d S ty }
z d|
d }
~
ww ) Nr c S s t j| t jd d S )Nr )sysstderrwrite)messager r r perrorr s zmain..perrorc sR |r|f| | f } d| n|r d|| f n d| t d d S )Nz%s:%d:%d: error: %sz
%s: error: %sz error: %srM )r exit)r r locationargsr r r errorv s zmain..errorzpython -m tokenize)progr r) z filename.pyz'the file to tokenize; defaults to stdin)destnargsmetavarhelpz-ez--exactexact
store_truez(display token names using the exact type)r actionr r zz %d,%d-%d,%d:z%-20s%-15s%-15rrM r z interrupted
zunexpected error: %s)NN)argparseArgumentParser add_argument
parse_argsr r listr r r r stdinr
r r rX rp printr r r r rI r OSErrorKeyboardInterrupt Exception)
r r parserr r r. rN ro
token_type token_rangeerrrq r r r r mainn sf $$ r __main__)T
__author__ __credits__builtinsr r codecsr r collections functoolsior itertoolsr6 r? r ro r r@ ASCIIr r __all__
namedtupler r$ r' r*
WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumberr> lru_cacherC StringPrefixSingleDoubleSingle3Double3TripleStringmapescapesortedSpecialFunny
PlainTokenTokenContStr PseudoExtrasr r _prefixsetr r r= r- r9 r r rI rK rL r r r
r r r r r r r r r s
_ ] 8
=