֦i$ d dl Z d dlZd dlZd dlZdgZ e j dd Z G d d Z G d d Z G d d Z
y)
NRobotFileParser RequestRatezrequests secondsc X e Zd Z ddZd Zd Zd Zd Zd Zd Z d Z
d Z d
Z d Z
d Zy
)r c z g | _ g | _ d | _ d| _ d| _ | j | d| _ y )NFr )entriessitemaps
default_entry disallow_all allow_allset_url last_checkedselfurls +/usr/lib64/python3.12/urllib/robotparser.py__init__zRobotFileParser.__init__ s;
! ! S c | j S N)r
r s r mtimezRobotFileParser.mtime& s r c 8 dd l }|j | _ y )Nr )timer
)r r s r modifiedzRobotFileParser.modified/ s IIKr c r || _ t j j | dd \ | _ | _ y )N )r urllibparseurlparsehostpathr s r r zRobotFileParser.set_url7 s0 : %|| 4 4S 9!A > 49r c t j j | j }|j }| j |j
d j y # t j j $ rU}|j dv rd| _ n%|j dk\ r|j dk rd| _ |j Y d }~y d }~ww xY w)Nzutf-8)i i Ti i )r requesturlopenr readr decode
splitlineserror HTTPErrorcoder
r close)r frawerrs r r& zRobotFileParser.read< s B
9&&txx0A &&(C JJszz'*557 8 ||%% xx:%$(!S SXX^!% IIKK s )A+ + C A CC c d|j v r| j || _ y y | j j | y N*)
useragentsr r append)r entrys r
_add_entryzRobotFileParser._add_entryJ s= %"" "!!)%*" *
LL &r c d}t }| j |D ] }|s4|dk( r
t }d}n"|dk( r | j | t }d}|j d }|dk\ r|d | }|j }|sh|j dd }t
| dk( s|d j j |d< t j j |d j |d< |d dk( rB|dk( r| j | t }|j j |d d}*|d dk( r3|dk7 s9|j j t |d d d}e|d d k( r3|dk7 st|j j t |d d
d}|d d k( r?|dk7 s|d j j rt! |d |_ d}|d d k( r|dk7 s|d j d
}t
| dk( rk|d j j rJ|d j j r)t% t! |d t! |d |_ d}|d dk( s| j( j |d |dk( r| j | y y )Nr r #:z
user-agentdisallowFallowTz crawl-delayz request-rate/sitemap)Entryr r6 findstripsplitlenlowerr r unquoter3 r4 rulelinesRuleLineisdigitintdelayr req_rater )r linesstater5 lineinumberss r r zRobotFileParser.parseS s
DA: !GE EaZOOE* !GE E #AAvBQx::>
\\**6<<+?+?+DE
ll%%r"Z__ j . .
0C0C'E Fll %C\\E * s++ " %%//4 4r c | j sy | j D ]! }|j | s|j c S | j r| j j S y r )r r rX rJ r r rZ r5 s r crawl_delayzRobotFileParser.crawl_delay sT zz|\\E * {{" " %%++ +r c | j sy | j D ]! }|j | s|j c S | j r| j j S y r )r r rX rK r r^ s r request_ratez RobotFileParser.request_rate sT zz|\\E * ~~% " %%.. .r c 4 | j sy | j S r )r r s r site_mapszRobotFileParser.site_maps s }}}} r c | j }| j || j gz }dj t t | S )Nz
)r r joinmapstr)r r s r __str__zRobotFileParser.__str__ s@ ,, ) !3!3 44G{{3sG,--r N)rR )__name__
__module__ __qualname__r r r r r& r6 r r\ r_ ra rc rh r r r r sE
!(?
9 'G#R:
.r c e Zd Z d Zd Zd Zy)rG c |dk( r|sd}t j j t j j | }t j j | | _ || _ y )NrR T)r r rS r rW r" rY )r r" rY s r r zRuleLine.__init__ sP 2:i I||&&v||'<'>zTADIIMMr N)ri rj rk r rX rh rl r r rG rG s 1#BNr rG c & e Zd Z d Zd Zd Zd Zy)r? c < g | _ g | _ d | _ d | _ y r )r3 rF rJ rK r s r r zEntry.__init__ s
r c g }| j D ] }|j d| | j |j d| j | j 7| j }|j d|j d|j
|j
t t | j dj | S )Nz User-agent: z
Crawl-delay: zRequest-rate: r=
) r3 r4 rJ rK requestssecondsextendrf rg rF re )r retagentrates r rh z
Entry.__str__ s __E JJ eW- . % :: ! JJ tzzl3 4 == $==D JJ
a ~F G
3sDNN+,yy~ r c |j d d j }| j D ] }|dk( r y|j }||v s y y)Nr= r r2 TF)rB rD r3 )r rZ r} s r rX zEntry.applies_to sT @ OOC(+113 __E|KKME !
% r c f | j D ]! }|j | s|j c S y)NT)rF rX rY )r rq rN s r rY zEntry.allowance s2 % NNDx(~~% # r N)ri rj rk r rh rX rY rl r r r? r? s I
r r? ) collections urllib.errorr urllib.parseurllib.request__all__
namedtupler r rG r? rl r r r sX
$k$$]4FG . .DN N$( (r