1jt U d Z ddlZddlZddlmZ ddlmZ ddlm Z ddl
m Z ddl
mZ dd lmZ ej r
ddlZdd
lmZ ed Zej ej d f ed
< ej d Z ej d Z ej dej Z ej dej ej z Z! ej dej ej z Z" e d Z# e d Z$ e d Z% e d Z& e d Z' e d Z( e d Z) e d Z* e d Z+ e d Z, e d Z- e d Z. e d Z/ e d Z0 e d! Z1 e d" Z2 e d# Z3 e d$ Z4 e d% Z5 e d& Z6 e d' Z7 e d( Z8 e d) Z9 e d* Z: e d+ Z; e d, Z< e d- Z= e d. Z> e d/ Z? e d0 Z@ e d1 ZA e d2 ZB e d3 ZC e d4 ZD e d5 ZE e d6 ZF e d7 ZG e d8 ZH e d9 ZI e d: ZJ e d; ZK e d< ZL e d= ZM e d> ZN e d? ZO e d@ ZP e dA ZQ e dB ZR e dC ZSi dDe#dEe;dFe'dGe*dHe3dIe2dJe6dKe<dLe.dMe8dNe/dOe9dPe-dQe7dRe)dSe4dTe+e,e0e1e$e(e%e5e&e:dU ZTdV eTU D ZV eWeT eWeV k s
J dW ej dNdXX dY eYeTdZ [ D dO ZZ e[eIeKeJe=eNeOePg Z\ e[e=eQeKePg Z]d\e^d]e^fd^Z_d_d`d]e^fdaZ`dbe^d]e^fdcZadde^d]ebfdeZcdfdgd]ejd ej e^e^f fdhZe G di dj Zf G dk d`ejg Zh G dl dm Zi G dn do ZjdvdpZk G dq drel Zm G ds dtejg Zn G du d ZodS )wzImplements a Jinja / Python combination lexer. The ``Lexer`` class
is used to do some preprocessing. It filters out invalid operators like
the bitshift operators we don't allow in templates. It separates
template code and python code in expressions.
N) literal_eval)deque)intern )pattern)TemplateSyntaxError)LRUCache) Environment2 Lexer _lexer_cachez\s+z (\r\n|\r|\n)z7('([^'\\]*(?:\\.[^'\\]*)*)'|"([^"\\]*(?:\\.[^"\\]*)*)")z
(
0b(_?[0-1])+ # binary
|
0o(_?[0-7])+ # octal
|
0x(_?[\da-f])+ # hex
|
[1-9](_?\d)* # decimal
|
0(_?0)* # decimal zero
)
z
(?) z>=rY s 888daQ888 zoperators droppedrP c # > K | ]}t j | V d S N)reescaperU xs rX ra s* PP!1 PPPPPPrZ c " t | S r\ )lenr` s rX re s Aw rZ )key
token_typereturnc | t v r
t | S t dt dt dt dt
dt dt dt dt dt d t d
t d i
| | S ) Nzbegin of commentzend of commentr6 zbegin of statement blockzend of statement blockzbegin of print statementzend of print statementzbegin of line statementzend of line statementztemplate data / textzend of template)reverse_operatorsTOKEN_COMMENT_BEGINTOKEN_COMMENT_END
TOKEN_COMMENTTOKEN_LINECOMMENTTOKEN_BLOCK_BEGINTOKEN_BLOCK_ENDTOKEN_VARIABLE_BEGINTOKEN_VARIABLE_ENDTOKEN_LINESTATEMENT_BEGINTOKEN_LINESTATEMENT_END
TOKEN_DATA TOKEN_EOFget)rg s rX _describe_token_typerx s| &&& ,, /+y951 8 4!#< !8*$
c*j!!
"rZ tokenTokenc X | j t k r| j S t | j S )z#Returns a description of the token.)type
TOKEN_NAMEvaluerx )ry s rX describe_tokenr s' zZ {
+ ++rZ exprc z d| v r'| dd \ }}|t k r|S n| }t | S )z0Like `describe_token` but for token expressions.rO r )splitr} rx )r r| r~ s rX describe_token_exprr sL
d{{jja(( e : L % %%rZ r~ c P t t | S )zsCount the number of newline characters in the string. This is
useful for extensions that filter a stream.
)rc
newline_refindall)r~ s rX count_newlinesr s z!!%(( ) ))rZ environmentr
c f t j }t | j t || j ft | j t || j ft | j t || j fg}| j @|
t | j t d || j z f | j @|
t | j t d || j z f d t |d D S )zACompiles all the rules from the environment into a list of rules.Nz ^[ \t\v]*z(?:^|(?<=\S))[^\S\r\n]*c " g | ] }|d d
S )r NrT r_ s rX
z!compile_rules.. s 7 7 7aAabbE 7 7 7rZ T)reverse)r] r^ rc comment_start_stringrk block_start_stringro variable_start_stringrq line_statement_prefixappendrs line_comment_prefixTOKEN_LINECOMMENT_BEGINsorted)r eruless rX
compile_rulesr sC
A
0 1 1
Ak. / /
. / /
Ak, - -
1 2 2
Ak/ 0 0
E$ (4
K566) qq!B C CC
&2
K344'*QQ{/N-O-OO
8 76%666 7 7 77rZ c T e Zd ZdZefdedej e ddfdZde dedd fd
Z
dS ) FailurezjClass that raises a `TemplateSyntaxError` if called.
Used by the `Lexer` to specify known errors.
messageclsrh Nc " || _ || _ d S r\ )r error_class)selfr r s rX __init__zFailure.__init__ s rZ linenofilenamez te.NoReturnc : | | j || r\ )r r )r r r s rX __call__zFailure.__call__ s t|VX>>>rZ ) __name__
__module__ __qualname____doc__r strtTyper intr rT rZ rX r r s
@S !"(;!<
?s ?c ?m ? ? ? ? ? ?rZ r c Z e Zd ZU eed< eed< eed< defdZdedefdZdedefd Z d
S ) rz r r| r~ rh c t | S r\ )r r s rX __str__z
Token.__str__ s d###rZ r c t | j |k rdS d|v r&| dd | j | j gk S dS )zTest a token against a token expression. This can either be a
token type or ``'token_type:token_value'``. This can only test
against string values and types.
TrO r F)r| r r~ r r s rX testz
Token.test sF 9 4 $;;::c1%%$)TZ)@@ @urZ iterablec : t fd|D S )z(Test against multiple token expressions.c 3 B K | ]} | V d S r\ )r )rU r r s rX ra z!Token.test_any..$ s- 88t499T??888888rZ )any)r r s ` rX test_anyzToken.test_any" s& 8888x888888rZ N)
r r r r __annotations__r r boolr r rT rZ rX rz rz s KKK
III JJJ$ $ $ $ $
9# 9$ 9 9 9 9 9 9rZ c . e Zd ZdZd dZd
dZdefdZdS ) TokenStreamIteratorz`The iterator for tokenstreams. Iterate over the stream
until the eof token is reached.
stream TokenStreamrh Nc || _ d S r\ )r )r r s rX r z TokenStreamIterator.__init__, s
rZ c | S r\ rT r s rX __iter__z TokenStreamIterator.__iter__/ s rZ c | j j }|j t u r | j t
t
| j |S r\ )r currentr| rv close
StopIterationnextr ry s rX __next__z TokenStreamIterator.__next__2 sH # : " " K T[ rZ )r r rh N)rh r )r r r r r r rz r rT rZ rX r r ' sa % rZ r c $ e Zd ZdZdej e dej e dej e fdZ de
fdZ de fdZ
ede fd Zd
edd fd Zdefd
Zddedd fdZdedej e fdZdede fdZdefdZddZdedefdZd S )r zA token stream is an iterable that yields :class:`Token`\s. The
parser however does not iterate over it but calls :meth:`next` to go
one token ahead. The current active token is stored as :attr:`current`.
generatorr+ r c t | | _ t | _ || _ || _ d| _ t dt d | _ t | d S )NFr ) iter_iterr _pushedr+ r closedrz
TOKEN_INITIALr r )r r r+ r s rX r zTokenStream.__init__C sS )__
*/''
Q
r22 T
rZ rh c t | S r\ )r r s rX r zTokenStream.__iter__Q s "4(((rZ c P t | j p| j j t uS r\ )r r r r| rv r s rX __bool__zTokenStream.__bool__T s! DL!!GT\%6i%GGrZ c | S )z Are we at the end of the stream?rT r s rX eoszTokenStream.eosW s xrZ ry Nc : | j | dS )z Push a token back to the stream.N)r r r s rX pushzTokenStream.push\ s E"""""rZ c j t | }| j }| | || _ |S )zLook at the next token.)r r r )r old_tokenresults rX lookzTokenStream.look` s2 JJ &
rZ r nc H t | D ]}t | dS )zGot n tokens ahead.N)ranger )r r _s rX skipzTokenStream.skiph s. q A JJJJ rZ r c X | j | rt | S dS )zqPerform the token test and return the token if it matched.
Otherwise the return value is `None`.
N)r r r r s rX next_ifzTokenStream.next_ifm s- < T " " :: trZ c 0 | | duS )z8Like :meth:`next_if` but only returns `True` or `False`.N)r r s rX skip_ifzTokenStream.skip_ifv s ||D!!--rZ c | j }| j r | j | _ nR| j j t ur? t | j | _ n$# t $ r | Y nw xY w|S )z|Go one token ahead and return the old one.
Use the built-in :func:`next` instead of calling this directly.
) r r popleftr| rv r r r r )r rvs rX r zTokenStream.__next__z s
\ < <//11DLL
\
i
/
/
#DJ / /
s A A=<A=c t | j j t d | _ t d | _ d| _ dS )zClose the stream.r rT TN)rz r r rv r r r r s rX r zTokenStream.close s2 T\ 0)R@@ "XX
rZ c h | j | st | }| j j t u r*t d|d| j j | j | j t d|dt | j | j j | j | j t | S )z}Expect a given token type and return it. This accepts the same
argument as :meth:`jinja2.lexer.Token.test`.
z%unexpected end of template, expected rN zexpected token z, got ) r r r r| rv r r r+ r r r r s rX expectzTokenStream.expect s | && &t,,D| I--)EDEEEL'IM &P$PPt|0L0LPP #
DzzrZ )r )rh N)r r r r r Iterablerz Optionalr r r r r r propertyr r r r r r r r r r rT rZ rX r r = s
:e$ jo *S/ )- ) ) ) )H$ H H H H T X#% #D # # # #e c $
C AJu$5 .C .D . . . .% " 3 5 rZ r c | j | j | j | j | j | j | j | j | j | j | j
| j f }t
| }|t | xt |< }|S )z(Return a lexer which is probably cached.)r block_end_stringr variable_end_stringr comment_end_stringr r trim_blocks
lstrip_blocksnewline_sequencekeep_trailing_newliner
rw r )r rf lexers rX get_lexerr s &$)'(&)' !$)
C
S ! !E }$)+$6$66 SE LrZ c & e Zd ZdZdZ fdZ xZS )OptionalLStripzWA special tuple for marking a point in the state that can have
lstrip applied.
rT c H t | | S r\ )super__new__)r memberskwargs __class__s rX r zOptionalLStrip.__new__ s wwsG,,,rZ )r r r r __slots__r
__classcell__)r s @rX r r sI I- - - - - - - - -rZ r c e Zd ZU ej e ed< ej eej edf ej e f ed< ej
e ed< dS )_Ruler .tokenscommandN) r r r r Patternr r UnionTupler r rT rZ rX r r sb
Ys^
GCc*AGG,<< ====
Z_ rZ r c e Zd ZdZddZdedefdZ dd ed
ej e d ej e d ej e de f
d
Z
ddej ej e
eef d
ej e d ej e dej e fdZ dd ed
ej e d ej e d ej e dej ej e
eef f
dZdS )r a
Class that implements a lexer for a given environment. Automatically
created by the environment class, usually you don't have to do that.
Note that the lexer is not automatically bound to an environment.
Multiple environments can share the same lexer.
r r
rh Nc t j }dt dt j t fd}t t t d t t t d t t t d t t t d t t t d t t t" d g}t% | } ||j } ||j } ||j } ||j } |j rdnd}
|j r |d nd | _ |j | _ |j | _ d| d| d | d
} d | gd |D z } d
t |d| d t; t< d d t |d t< d gt> t |d| d| d | |
d
t@ tB fd t |d tE d fd gtF t |d| d| d | |
d tH d g|z tJ t |d| d | tL d g|z tN t |d| d| d| d | |
d
t; t< tP d t |d tE d fd gtR t |d tT d g|z tV t |d tX tZ fd gi| _. d S ) Nr` rh c Z t j | t j t j z S r\ )r] compileMSrd s rX czLexer.__init__..c s :a-- -rZ z\n?r z[^ \t]z(?Pz(\-|\+|)\s*raw\s*(?:\-z\s*|z))rP c & g | ]\ }}d | d| dS )z(?P.
s0 Q Q Q$!Q 5Q 5 5 5 5 5 Q Q QrZ rootz(.*?)(?:rH #bygroupz.+z (.*?)((?:\+z|\-#popz(.)zMissing end of comment tagz(?:\+z\-z (.*?)((?:z(\-|\+|))\s*endraw\s*(?:\+z Missing end of raw directivez \s*(\n|$)z(.*?)()(?=\n|$))/r] r^ r r r r
whitespace_reTOKEN_WHITESPACEfloat_re TOKEN_FLOAT
integer_re
TOKEN_INTEGERname_rer} string_re TOKEN_STRING operator_reTOKEN_OPERATORr r r r r r r lstrip_unless_rer r joinr ru rk rm rl r ro rp rq rr TOKEN_RAW_BEGIN
TOKEN_RAW_ENDrs rt r rn TOKEN_LINECOMMENT_ENDr )
r r r r tag_rulesroot_tag_rulesblock_start_re block_end_recomment_end_revariable_end_reblock_suffix_re root_raw_re
root_parts_res
rX r zLexer.__init__ sJ I . .3 . . . .
-!14 8 8 (K . . *mT 2 2 ':t , , )\4 0 0 +~t 4 4
$
'{33;9::q566 ;9::!K ;<< %0$;C&& 1<0I S) t + < %0%F"
8n
8
8!
8
8'3
8
8
8
M Q Q. Q Q Q Q
A2-22233":z:: aaggz400
AA~ A A. A A +A-<A A A #$56
aaii'*F"G"G!I4PP
"
A> > >, > > )>+:> > > $
!ABOBB BBCC& # #
A?^ ? ?!-? ?2>? ? )?+:? ? ?
#:}== aaii'*H"I"I!KTRR
&aa oo'>GG( (
$A())&(=> &F2
rZ r~ c B t | j | S )z`Replace all newlines with the configured sequence in strings
and template data.
)r r& r )r r~ s rX _normalize_newlineszLexer._normalize_newlinesV s ~~d 3U;;;rZ sourcer+ r statec ~ | |||| }t | ||| || S )z:Calls tokeniter + tokenize and wraps it in a token stream.) tokeniterr wrap)r r0 r+ r r1 r s rX tokenizezLexer.tokenize\ s= h>>499VT8<AD
E
A
EE c #
K t | ddd }| j s|d dk r|d= d | }d}d}dg}|,|dk r&|d v s
J d
| |d z | j |d } t
| }
g } | j } d}
d } | D ] \ }}}| || }| | r|t t t fv r9t |t r| }t |t r|d }t! d
|ddd D }|dk rL| }|t
| d d }
|g|dd }n|dk ry| w| t* sK| d dz }|dk s|r+| || s|d| g|dd }t1 | D ]\ }}|j t4 u r ||| |dk rb| D ](\ }}|!|||fV || d z
} n)t9 |d || } | s |t: vr||| fV || d |
z z
}d}
n| } |t> k r| dk r| d n| dk r| d nk| dk r| d nO| dv rK| stA d| d||| | ! } | | k rtA d| d| d||| | s |t: vr||| fV || d z
}| dd dk }|" } ||d k r|! ns|dk rX| D ] \ }}|| | n t9 |d n| | | j |d } n| |k rt9 |d | } n&||
k rdS tA d || d | ||| L)!a This method tokenizes the text and returns the tokens in a
generator. Use this method if you just want to tokenize a template.
.. versionchanged:: 3.0
Only ``\n``, ``\r\n`` and ``\r`` are treated as line
breaks.
N r8 r
r r r )variableblockz
invalid state_beginTc 3 K | ]}||V d S r\ rT )rU gs rX ra z"Lexer.tokeniter.. s" )S)SQ]!]]]])S)SrZ r@ r? r z= wanted to resolve the token dynamically but no group matchedrI rJ rG rH rE rF )rJ rH rF z unexpected ''z
', expected 'r zA wanted to resolve the new state dynamically but no group matchedz* yielded empty string without stack changezunexpected char z at )#r r r r! r r rc r matchrr rp rt
isinstancetuplegroupsr r rstripcount groupdictrw rq rfindsearch enumerater r items RuntimeErrorignore_if_emptygroupr r popend) r r0 r+ r r1 linesposr stack statetokens
source_lengthbalancing_stackr newlines_stripped
line_startingregexr new_statemrQ text
strip_signstrippedl_posidxry rf r~ r< expected_oppos2s rX r3 zLexer.tokeniter sb ((1-) eBi2oob 5!! & 1111?111 LL) * * *jr+ F
') 0
e ,7 c
c
(vyKK,, 9 # v&#+2 ( (
fe,, ^/ XXZZF!&.99 I &ay &*)S)SVADqD\)S)S)S%S%S
%,,'+{{}}H04S]]__0E0K0KD0Q0Q -&.%<%'>tU'K'K !I.26E6l-HVABBZ-HF&/&7&7 2 2
U ?g55"'%"9"9 9 #j 0 0./kkmm.A.A.C.C " "
U#(#4*0#u*<$<$<$<$*ekk$.?.?$?F$)E $5
'3', %< %< %<'" '" !" %* $*#;D # :uO'C'C&,eT&9 9 9 9 "djj&6&69J&J JF01 - -52< 7799D // 3;; + 2 23 7 7 7 7 !S[[ + 2 23 7 7 7 7 !S[[ + 2 23 7 7 7 7 !_ 4 4#2 "&9$:4$:$:$:FD('" '" !" +:*=*=*?*?K *d 2 2&9$T4$T$Tk$T$T$T$*$($, '" '" !" 3v_ < < $fd 2222djj . ..F ! "##$ 6
uuww ( F** "j00*+++--*=*=*?*? JC $ 0 % S 1 1 1 % 1 #/#( !9 !9 !9# # !& Y///"&*U2Y"7KK S[[& NNN
-''F *?vc{??#??x Ge rZ )r r
rh N)NNN)NN)r r r r r r r/ r r r r5 r r r Iteratorrz r4 r3 rT rZ rX r r s w
w
w
w
r< < < < < < !%$(!% N N N jo N *S/ N
z# N
N N N N !%$( 4. 4.
173S=124. jo4. *S/ 4.
E 4. 4. 4. 4.t %)!% H HH joH *S/ H
z# H
AGCcM* +
H H H H H HrZ )r r
rh r )pr r] typingr astr collectionsr sysr _identifierr r
exceptionsr utilsr
TYPE_CHECKINGtyping_extensionster r
r
MutableMappingr r r r r r r
IGNORECASEVERBOSEr r TOKEN_ADD TOKEN_ASSIGN TOKEN_COLON TOKEN_COMMA TOKEN_DIV TOKEN_DOTTOKEN_EQTOKEN_FLOORDIVTOKEN_GT
TOKEN_GTEQ TOKEN_LBRACETOKEN_LBRACKET TOKEN_LPARENTOKEN_LT
TOKEN_LTEQ TOKEN_MOD TOKEN_MULTOKEN_NE
TOKEN_PIPE TOKEN_POW TOKEN_RBRACETOKEN_RBRACKET TOKEN_RPARENTOKEN_SEMICOLON TOKEN_SUB TOKEN_TILDEr r r r} r r ro rp rq rr r" r# rk rl rm rs rt r r$ rn ru r rv rB rX rj rc r! r r frozensetr; rZ r rx r r r r Listr r
NamedTuplerz r r r rP r r r rT rZ rX r s1
+ + + + + + + + + + + + ? )""""(((((( 4<8B<< a qw /0 ? ? ?
6""
RZ
(
(
BJBBD
RZ MBJ
2: MBJ
F5MM vh fWoo fWoo F5MM F5MM 6$<<
## 6$<<
VF^^
vh
##vh 6$<<
VF^^
F5MM F5MM 6$<<
VF^^
F5MM vh
##vh &%% F5MM fWoo 6,'' fWoo y!!
VF^^
vh
##F=)) &%% v .// VN++ &%%y!!
f_-- F=)) y!!
"F#8 9 9 &!455 &!455 011 F=))
VF^^
y!!
F5MM
.
)
(
(!
" #
$
5
: 98ioo&7&7888
s9~~ ./////1D///bjSPPvvi=N=N'O'O'OPPPPPSSS
)z=2CD
"S "S " " " "(,' ,c , , , ,
&c
&c
&
&
&
&*# *# * * * *&8} &8S8I1J &8 &8 &8 &8R ? ? ? ? ? ? ? ? 9 9 9 9 9AL 9 9 98 ,j j j j j j j jZ 0
-
-
-
-
-U
-
-
- AL P P P P P P P P P PrZ