§
    (¶ÕiÐ$  ã                   ó˜   — d Z ddlZddlZddlZdgZ ej        dd¦  «        Z G d„ d¦  «        Z G d„ d¦  «        Z	 G d	„ d
¦  «        Z
dS )a%   robotparser.py

    Copyright (C) 2000  Bastian Kleineidam

    You can choose between two licenses when using this package:
    1) GNU GPLv2
    2) PSF license for Python 2.2

    The robots.txt Exclusion Protocol is implemented as specified in
    http://www.robotstxt.org/norobots-rfc.txt
é    NÚRobotFileParserÚRequestRatezrequests secondsc                   ó\   — e Zd ZdZdd„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d	„ Z
d
„ Zd„ Zd„ Zd„ Zd„ ZdS )r   zs This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

    Ú c                 ó„   — g | _         g | _        d | _        d| _        d| _        |                      |¦  «         d| _        d S )NFr   )ÚentriesÚsitemapsÚdefault_entryÚdisallow_allÚ	allow_allÚset_urlÚlast_checked©ÚselfÚurls     ú)/usr/lib/python3.11/urllib/robotparser.pyÚ__init__zRobotFileParser.__init__   sG   € ØˆŒØˆŒØ!ˆÔØ!ˆÔØˆŒØ�Š�SÑÔÐØˆÔÐÐó    c                 ó   — | j         S )z·Returns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        )r   ©r   s    r   ÚmtimezRobotFileParser.mtime%   s   € ð Ô Ð r   c                 ó@   — ddl }|                      ¦   «         | _        dS )zYSets the time the robots.txt file was last fetched to the
        current time.

        r   N)Útimer   )r   r   s     r   ÚmodifiedzRobotFileParser.modified.   s#   € ð
 	ˆˆˆØ ŸIšI™KœKˆÔÐÐr   c                 ó|   — || _         t          j                             |¦  «        dd…         \  | _        | _        dS )z,Sets the URL referring to a robots.txt file.é   é   N)r   ÚurllibÚparseÚurlparseÚhostÚpathr   s     r   r   zRobotFileParser.set_url6   s4   € àˆŒÝ%œ|×4Ò4°SÑ9Ô9¸!¸A¸#Ô>ÑˆŒ	�4”9�9�9r   c                 ó°  — 	 t           j                             | j        ¦  «        }|                     ¦   «         }|                      |                     d¦  «                             ¦   «         ¦  «         dS # t           j        j	        $ rK}|j
        dv rd| _        n)|j
        dk    r|j
        dk     rd| _        Y d}~dS Y d}~dS Y d}~dS Y d}~dS d}~ww xY w)z4Reads the robots.txt URL and feeds it to the parser.zutf-8)i‘  i“  Ti�  iô  N)r   ÚrequestÚurlopenr   Úreadr   ÚdecodeÚ
splitlinesÚerrorÚ	HTTPErrorÚcoder   r   )r   ÚfÚrawÚerrs       r   r&   zRobotFileParser.read;   só   € ð		9Ý”×&Ò& t¤xÑ0Ô0ˆAð —&’&‘(”(ˆCØ�JŠJ�s—z’z 'Ñ*Ô*×5Ò5Ñ7Ô7Ñ8Ô8Ð8Ð8Ð8øõ Œ|Ô%ð 	&ð 	&ð 	&ØŒx˜:Ð%Ð%Ø$(�Ô!Ð!Ø”˜S’� S¤X°¢^ ^Ø!%�”������ð !�����ð "Ð!Ð!Ð!Ð!Ð!Ø%3 ^ ^ ^ ^ ^øøøøð	&øøøs   ‚$A6 Á6CÂ
.CÃCc                 óp   — d|j         v r| j        €	|| _        d S d S | j                             |¦  «         d S ©NÚ*)Ú
useragentsr
   r   Úappend)r   Úentrys     r   Ú
_add_entryzRobotFileParser._add_entryH   sM   € Ø�%Ô"Ð"Ð"àÔ!Ð)à%*�Ô"Ð"Ð"ð *Ð)ð ŒL×Ò Ñ&Ô&Ð&Ð&Ð&r   c                 ó<  — d}t          ¦   «         }|                      ¦   «          |D �]V}|sB|dk    rt          ¦   «         }d}n+|dk    r%|                      |¦  «         t          ¦   «         }d}|                     d¦  «        }|dk    r
|d|…         }|                     ¦   «         }|sŒƒ|                     dd¦  «        }t          |¦  «        dk    �r©|d                              ¦   «                              ¦   «         |d<   t          j	         
                    |d                              ¦   «         ¦  «        |d<   |d         dk    rM|dk    r#|                      |¦  «         t          ¦   «         }|j                             |d         ¦  «         d}�Œo|d         dk    r8|dk    r0|j                             t          |d         d	¦  «        ¦  «         d}�Œ³|d         d
k    r8|dk    r0|j                             t          |d         d¦  «        ¦  «         d}�Œ÷|d         dk    rP|dk    rH|d                              ¦   «                              ¦   «         rt!          |d         ¦  «        |_        d}�ŒS|d         dk    rË|dk    rÃ|d                              d¦  «        }t          |¦  «        dk    r“|d                              ¦   «                              ¦   «         rg|d                              ¦   «                              ¦   «         r;t%          t!          |d         ¦  «        t!          |d         ¦  «        ¦  «        |_        d}�Œ*|d         dk    r | j                             |d         ¦  «         �ŒX|dk    r|                      |¦  «         dS dS )z”Parse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        r   r   é   ú#Nú:z
user-agentÚdisallowFÚallowTzcrawl-delayzrequest-rateú/Úsitemap)ÚEntryr   r5   ÚfindÚstripÚsplitÚlenÚlowerr   r   Úunquoter2   r3   Ú	rulelinesÚRuleLineÚisdigitÚintÚdelayr   Úreq_rater	   )r   ÚlinesÚstater4   ÚlineÚiÚnumberss          r   r   zRobotFileParser.parseQ   sJ  € ð ˆÝ‘”ˆà�Š‰ŒˆØð 7	2ñ 7	2ˆDØð Ø˜A’:�:Ý!™GœG�EØ�E�EØ˜a’Z�ZØ—O’O EÑ*Ô*Ð*Ý!™GœG�EØ�Eà—	’	˜#‘”ˆAØ�AŠvˆvØ˜B˜Q˜B”x�Ø—:’:‘<”<ˆDØð ØØ—:’:˜c 1Ñ%Ô%ˆDÝ�4‰yŒy˜AŠ~‰~Ø˜qœ'Ÿ-š-™/œ/×/Ò/Ñ1Ô1��Q‘Ý œ,×.Ò.¨t°A¬w¯}ª}©¬Ñ?Ô?��Q‘Ø˜”7˜lÒ*Ð*Ø ’z�zØŸš¨Ñ.Ô.Ð.Ý %¡¤˜ØÔ$×+Ò+¨D°¬GÑ4Ô4Ð4Ø�E‘EØ˜!”W 
Ò*Ð*Ø ’z�zØœ×.Ò.­x¸¸Q¼ÀÑ/GÔ/GÑHÔHÐHØ !˜ùØ˜!”W Ò'Ð'Ø ’z�zØœ×.Ò.­x¸¸Q¼ÀÑ/FÔ/FÑGÔGÐGØ !˜ùØ˜!”W Ò-Ð-Ø ’z�zð   œ7Ÿ=š=™?œ?×2Ò2Ñ4Ô4ð 7Ý*-¨d°1¬g©,¬,˜EœKØ !˜ùØ˜!”W Ò.Ð.Ø ’z�zØ"& q¤'§-¢-°Ñ"4Ô"4˜å ™LœL¨AÒ-Ð-°'¸!´*×2BÒ2BÑ2DÔ2D×2LÒ2LÑ2NÔ2NÐ-Ø '¨¤
× 0Ò 0Ñ 2Ô 2× :Ò :Ñ <Ô <ð .å-8½¸WÀQ¼Z¹¼Í#ÈgÐVWÌjÉ/Ì/Ñ-ZÔ-Z˜EœNØ !˜ùØ˜!”W 	Ò)Ð)ð
 ”M×(Ò(¨¨a¬Ñ1Ô1Ð1ùØ�AŠ:ˆ:Ø�OŠO˜EÑ"Ô"Ð"Ð"Ð"ð ˆ:r   c                 ó  — | j         rdS | j        rdS | j        sdS t          j                             t          j                             |¦  «        ¦  «        }t          j                             dd|j        |j	        |j
        |j        f¦  «        }t          j                             |¦  «        }|sd}| j        D ].}|                     |¦  «        r|                     |¦  «        c S Œ/| j        r| j                             |¦  «        S dS )z=using the parsed robots.txt decide if useragent can fetch urlFTr   r<   )r   r   r   r   r   r    rD   Ú
urlunparser"   ÚparamsÚqueryÚfragmentÚquoter   Ú
applies_toÚ	allowancer
   )r   Ú	useragentr   Ú
parsed_urlr4   s        r   Ú	can_fetchzRobotFileParser.can_fetchš   s   € àÔð 	Ø�5ØŒ>ð 	Ø�4ð
 Ô ð 	Ø�5õ ”\×*Ò*­6¬<×+?Ò+?ÀÑ+DÔ+DÑEÔEˆ
ÝŒl×%Ò% r¨"¨Z¬_ØÔ˜jÔ.°
Ô0Cð'Eñ Fô FˆåŒl× Ò  Ñ%Ô%ˆØð 	ØˆCØ”\ð 	,ð 	,ˆEØ×Ò 	Ñ*Ô*ð ,Ø—’ sÑ+Ô+Ð+Ð+Ð+ð,ð Ôð 	5ØÔ%×/Ò/°Ñ4Ô4Ð4àˆtr   c                 ó¨   — |                       ¦   «         sd S | j        D ] }|                     |¦  «        r	|j        c S Œ!| j        r| j        j        S d S ©N)r   r   rV   rI   r
   ©r   rX   r4   s      r   Úcrawl_delayzRobotFileParser.crawl_delay·   sm   € Ø�zŠz‰|Œ|ð 	Ø�4Ø”\ð 	#ð 	#ˆEØ×Ò 	Ñ*Ô*ð #Ø”{Ð"Ð"Ð"ð#àÔð 	,ØÔ%Ô+Ð+Øˆtr   c                 ó¨   — |                       ¦   «         sd S | j        D ] }|                     |¦  «        r	|j        c S Œ!| j        r| j        j        S d S r\   )r   r   rV   rJ   r
   r]   s      r   Úrequest_ratezRobotFileParser.request_rateÁ   sm   € Ø�zŠz‰|Œ|ð 	Ø�4Ø”\ð 	&ð 	&ˆEØ×Ò 	Ñ*Ô*ð &Ø”~Ð%Ð%Ð%ð&àÔð 	/ØÔ%Ô.Ð.Øˆtr   c                 ó"   — | j         sd S | j         S r\   )r	   r   s    r   Ú	site_mapszRobotFileParser.site_mapsË   s   € ØŒ}ð 	Ø�4ØŒ}Ðr   c                 ó„   — | j         }| j        �|| j        gz   }d                     t          t          |¦  «        ¦  «        S )Nz

)r   r
   ÚjoinÚmapÚstr)r   r   s     r   Ú__str__zRobotFileParser.__str__Ð   s>   € Ø”,ˆØÔÐ)Ø Ô!3Ð 4Ñ4ˆGØ�{Š{�3�s GÑ,Ô,Ñ-Ô-Ð-r   N)r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r&   r5   r   rZ   r^   r`   rb   rg   © r   r   r   r      sÛ   € € € € € ðð ð
ð ð ð ð!ð !ð !ð(ð (ð (ð?ð ?ð ?ð
9ð 9ð 9ð'ð 'ð 'ðG#ð G#ð G#ðRð ð ð:ð ð ðð ð ðð ð ð
.ð .ð .ð .ð .r   c                   ó$   — e Zd ZdZd„ Zd„ Zd„ ZdS )rF   zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.c                 óè   — |dk    r|sd}t           j                             t           j                             |¦  «        ¦  «        }t           j                             |¦  «        | _        || _        d S )Nr   T)r   r   rQ   r    rU   r"   rW   )r   r"   rW   s      r   r   zRuleLine.__init__Ú   s[   € Ø�2Š:ˆ:˜iˆ:àˆIÝŒ|×&Ò&¥v¤|×'<Ò'<¸TÑ'BÔ'BÑCÔCˆÝ”L×&Ò& tÑ,Ô,ˆŒ	Ø"ˆŒˆˆr   c                 óL   — | j         dk    p|                     | j         ¦  «        S r0   )r"   Ú
startswith)r   Úfilenames     r   rV   zRuleLine.applies_toâ   s$   € ØŒy˜CÒÐA 8×#6Ò#6°t´yÑ#AÔ#AÐAr   c                 ó.   — | j         rdnddz   | j        z   S )NÚAllowÚDisallowz: )rW   r"   r   s    r   rg   zRuleLine.__str__å   s   € Øœ>Ð9��¨z¸TÑAÀDÄIÑMÐMr   N)rh   ri   rj   rk   r   rV   rg   rl   r   r   rF   rF   ×   sS   € € € € € ð1ð 1ð#ð #ð #ðBð Bð BðNð Nð Nð Nð Nr   rF   c                   ó*   — e Zd ZdZd„ Zd„ Zd„ Zd„ ZdS )r>   z?An entry has one or more user-agents and zero or more rulelinesc                 ó>   — g | _         g | _        d | _        d | _        d S r\   )r2   rE   rI   rJ   r   s    r   r   zEntry.__init__ë   s"   € ØˆŒØˆŒØˆŒ
ØˆŒˆˆr   c                 ó|  — g }| j         D ]}|                     d|› �¦  «         Œ| j        �|                     d| j        › �¦  «         | j        �,| j        }|                     d|j        › d|j        › �¦  «         |                     t          t          | j	        ¦  «        ¦  «         d 
                    |¦  «        S )NzUser-agent: zCrawl-delay: zRequest-rate: r<   ú
)r2   r3   rI   rJ   ÚrequestsÚsecondsÚextendre   rf   rE   rd   )r   ÚretÚagentÚrates       r   rg   zEntry.__str__ñ   s¿   € ØˆØ”_ð 	/ð 	/ˆEØ�JŠJÐ- eÐ-Ð-Ñ.Ô.Ð.Ð.ØŒ:Ð!Ø�JŠJÐ3 t¤zÐ3Ð3Ñ4Ô4Ð4ØŒ=Ð$Ø”=ˆDØ�JŠJÐF¨¬ÐFÐF¸¼ÐFÐFÑGÔGÐGØ�
Š
•3•s˜DœNÑ+Ô+Ñ,Ô,Ð,Ø�yŠy˜‰~Œ~Ðr   c                 ó¼   — |                      d¦  «        d                              ¦   «         }| j        D ]&}|dk    r dS |                     ¦   «         }||v r dS Œ'dS )z2check if this entry applies to the specified agentr<   r   r1   TF)rA   rC   r2   )r   rX   r}   s      r   rV   zEntry.applies_toý   sp   € ð —O’O CÑ(Ô(¨Ô+×1Ò1Ñ3Ô3ˆ	Ø”_ð 	ð 	ˆEØ˜Š|ˆ|à�t�tØ—K’K‘M”MˆEØ˜	Ð!Ð!Ø�t�tð "àˆur   c                 óV   — | j         D ] }|                     |¦  «        r	|j        c S Œ!dS )zZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT)rE   rV   rW   )r   rq   rM   s      r   rW   zEntry.allowance
  sA   € ð ”Nð 	&ð 	&ˆDØ�Š˜xÑ(Ô(ð &Ø”~Ð%Ð%Ð%ð&àˆtr   N)rh   ri   rj   rk   r   rg   rV   rW   rl   r   r   r>   r>   é   sV   € € € € € ØIÐIðð ð ð
ð 
ð 
ðð ð ðð ð ð ð r   r>   )rk   ÚcollectionsÚurllib.parser   Úurllib.requestÚ__all__Ú
namedtupler   r   rF   r>   rl   r   r   ú<module>r†      så   ðð
ð 
ð Ð Ð Ð Ø Ð Ð Ð Ø Ð Ð Ð àÐ
€à$ˆkÔ$ ]Ð4FÑGÔG€ð~.ð ~.ð ~.ð ~.ð ~.ñ ~.ô ~.ð ~.ðBNð Nð Nð Nð Nñ Nô Nð Nð$(ð (ð (ð (ð (ñ (ô (ð (ð (ð (r   