
    4pf                         d dl mZmZmZ d dlmZ 	 d dlmZ d dlm	Z
mZ e
e_	        ee_        dgZ G d de          Z G d de          Z G d	 d
e          ZdS )    )absolute_importdivisionunicode_literalsstr)urllib)parserequestRobotFileParserc                   J    e Zd ZdZddZd Zd Zd Zd Zd Z	d	 Z
d
 Zd ZdS )r   zs This class provides a set of methods to read, parse and answer
    questions about a single robots.txt file.

     c                 v    g | _         d | _        d| _        d| _        |                     |           d| _        d S )NFr   )entriesdefault_entrydisallow_all	allow_allset_urllast_checkedselfurls     c/var/www/html/nettyfy-visnx/env/lib/python3.11/site-packages/future/backports/urllib/robotparser.py__init__zRobotFileParser.__init__   s@    !!S    c                     | j         S )zReturns the time the robots.txt file was last fetched.

        This is useful for long-running web spiders that need to
        check for new robots.txt files periodically.

        )r   r   s    r   mtimezRobotFileParser.mtime&   s       r   c                 @    ddl }|                                 | _        dS )zYSets the time the robots.txt file was last fetched to the
        current time.

        r   N)timer   )r   r   s     r   modifiedzRobotFileParser.modified/   s#    
 	 IIKKr   c                 |    || _         t          j                            |          dd         \  | _        | _        dS )z,Sets the URL referring to a robots.txt file.      N)r   r   r	   urlparsehostpathr   s     r   r   zRobotFileParser.set_url7   s4    %|44S99!A#>	4999r   c                    	 t           j                            | j                  }|                                }|                     |                    d                                                     dS # t           j        j	        $ r:}|j
        dv rd| _        n|j
        dk    rd| _        Y d}~dS Y d}~dS Y d}~dS d}~ww xY w)z4Reads the robots.txt URL and feeds it to the parser.zutf-8)i  i  Ti  N)r   r
   urlopenr   readr	   decode
splitlineserror	HTTPErrorcoder   r   )r   frawerrs       r   r)   zRobotFileParser.read<   s    		9&&tx00A &&((CJJszz'**557788888 |% 	& 	& 	&x:%%$(!!S!% ! "!!!!!	&s   $A6 6C
#B??Cc                 p    d|j         v r| j        	|| _        d S d S | j                            |           d S N*)
useragentsr   r   append)r   entrys     r   
_add_entryzRobotFileParser._add_entryI   sM    %"""!)%*""" *) L&&&&&r   c                 R   d}t                      }|D ]}|sB|dk    rt                      }d}n+|dk    r%|                     |           t                      }d}|                    d          }|dk    r
|d|         }|                                }|s|                    dd          }t          |          dk    rH|d                                                                         |d<   t          j        	                    |d                                                   |d<   |d         dk    rM|dk    r#|                     |           t                      }|j
                            |d                    d}o|d         dk    r8|dk    r0|j                            t          |d         d	                     d}|d         d
k    r6|dk    r0|j                            t          |d         d                     d}|dk    r|                     |           dS dS )zParse the input lines from a robots.txt file.

        We allow that a user-agent: line is not preceded by
        one or more blank lines.
        r   r"      #N:z
user-agentdisallowFallowT)Entryr8   findstripsplitlenlowerr   r	   unquoter5   r6   	rulelinesRuleLine)r   linesstater7   lineis         r   r	   zRobotFileParser.parseR   s     !	" !	"D A::!GGEEEaZZOOE***!GGEE		#AAvvBQBx::<<D ::c1%%D4yyA~~q'--////11Q ,..tAw}}??Q7l**zz... %$++DG444EE!W
**zz..xQ/G/GHHH !!W''zz..xQ/F/FGGG !A::OOE""""" :r   c                    | j         rdS | j        rdS t          j                            t          j                            |                    }t          j                            dd|j        |j        |j	        |j
        f          }t          j                            |          }|sd}| j        D ].}|                    |          r|                    |          c S /| j        r| j                            |          S dS )z=using the parsed robots.txt decide if useragent can fetch urlFTr   /)r   r   r   r	   r$   rE   
urlunparser&   paramsqueryfragmentquoter   
applies_to	allowancer   )r   	useragentr   
parsed_urlr7   s        r   	can_fetchzRobotFileParser.can_fetch   s    	5> 	4 \**6<+?+?+D+DEE
l%%r"Z_j.
0C'E F Fl  %% 	C\ 	, 	,E	** ,s+++++,  	5%//444tr   c                 J    d                     d | j        D                       S )Nr   c                 2    g | ]}t          |          d z   S )
r   ).0r7   s     r   
<listcomp>z+RobotFileParser.__str__.<locals>.<listcomp>   s#    DDDeE

T)DDDr   )joinr   r   s    r   __str__zRobotFileParser.__str__   s%    wwDDt|DDDEEEr   N)r   )__name__
__module____qualname____doc__r   r   r    r   r)   r8   r	   rW   r^    r   r   r   r      s         
   ! ! !( ( (? ? ?
9 9 9' ' '0# 0# 0#f  .F F F F Fr   c                   $    e Zd ZdZd Zd Zd ZdS )rG   zoA rule line is a single "Allow:" (allowance==True) or "Disallow:"
       (allowance==False) followed by a path.c                 p    |dk    r|sd}t           j                            |          | _        || _        d S )Nr   T)r   r	   rR   r&   rT   )r   r&   rT   s      r   r   zRuleLine.__init__   s6    2::i:IL&&t,,	"r   c                 L    | j         dk    p|                    | j                   S r3   )r&   
startswith)r   filenames     r   rS   zRuleLine.applies_to   s$    yCA8#6#6ty#A#AAr   c                 .    | j         rdpddz   | j        z   S )NAllowDisallowz: )rT   r&   r   s    r   r^   zRuleLine.__str__   s     *78jD@49LLr   N)r_   r`   ra   rb   r   rS   r^   rc   r   r   rG   rG      sS        1 1# # #B B BM M M M Mr   rG   c                   *    e Zd ZdZd Zd Zd Zd ZdS )r?   z?An entry has one or more user-agents and zero or more rulelinesc                 "    g | _         g | _        d S )N)r5   rF   r   s    r   r   zEntry.__init__   s    r   c                     g }| j         D ]}|                    d|dg           | j        D ]&}|                    t          |          dg           'd                    |          S )NzUser-agent: rZ   r   )r5   extendrF   r   r]   )r   retagentrJ   s       r   r^   zEntry.__str__   su    _ 	6 	6EJJt45555N 	* 	*DJJD		4())))wws||r   c                     |                     d          d                                         }| j        D ]&}|dk    r dS |                                }||v r dS 'dS )z2check if this entry applies to the specified agentrM   r   r4   TF)rB   rD   r5   )r   rU   rq   s      r   rS   zEntry.applies_to   sp     OOC((+1133	_ 	 	E||ttKKMME	!!tt "ur   c                 V    | j         D ] }|                    |          r	|j        c S !dS )zZPreconditions:
        - our agent applies to this entry
        - filename is URL decodedT)rF   rS   rT   )r   rh   rJ   s      r   rT   zEntry.allowance   sA     N 	& 	&Dx(( &~%%%&tr   N)r_   r`   ra   rb   r   r^   rS   rT   rc   r   r   r?   r?      sV        II          r   r?   N)
__future__r   r   r   future.builtinsr   future.backportsr   future.backports.urllibr	   _parser
   _request__all__objectr   rG   r?   rc   r   r   <module>r|      s   B B B B B B B B B B      
 $ # # # # # H H H H H H H H 
EF EF EF EF EFf EF EF EFPM M M M Mv M M M"" " " " "F " " " " "r   