
    @jRC                        g d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z
ddlZ
ddlZddlZ	 ddlZd Zd Zn# e$ r ddlZd Zd ZY nw xY w ej        e          Zg dZ ej        dd	                    e          z   d
z   d	                    e          z   dz             Zej         G d de
j        j                              Zej         G d d                      Zej         G d d                      Z ej         G d d                      Z!ej         G d de
j        j                              Z" G d de
j        j#                  Z$dS ))VKontaktePostPhotoPhotoVariantVideoUserVKontakteUserScraper    Nc                 *    t          j        |           S N)zoneinfoZoneInfoss    a/home/ubuntu/.hermes/hermes-agent/venv/lib/python3.11/site-packages/snscrape/modules/vkontakte.py	_timezoner      s    		1		    c                 (    t          j         |d| i|S )Ntzinfo)datetimetzargskwargss      r   _localised_datetimer      s    		D	82	8	8	88r   c                 *    t          j        |           S r
   )pytztimezoner   s    r   r   r      s    	q		r   c                 J    |                      t          j        |i |          S r
   )localizer   r   s      r   r   r      s$    	X&777	8	88r   )JanFebMarAprMayJunJulAugSepOctNovDecz5^(?P<date>today|yesterday|(?P<day1>\d+)\s+(?P<month1>|z#)(\s+(?P<year1>\d{4}))?|(?P<month2>z^)\s+(?P<day2>\d+),\s+(?P<year2>\d{4}))\s+at\s+(?P<hour>\d+):(?P<minute>\d+)\s+(?P<ampm>[ap]m)$c                   4   e Zd ZU eed<   ej        ej        ej        ej	        f                  ed<   eed<   dZ
ej        ej        e                  ed<   dZej        ej        d                  ed<   dZej        d         ed	<   dZej        d          ed
<   d ZdS )r   urldatecontentNoutlinksr   photosr   video
quotedPostc                     | j         S r
   )r-   selfs    r   __str__zVKontaktePost.__str__4   s	    	/r   )__name__
__module____qualname__str__annotations__typingOptionalUnionr   r.   r0   Listr1   r2   r3   r7    r   r   r   r   *   s         	v|H$5x}$DEFFFF
/36?6;s+,33315W-	.555#' '''04V__-444    r   r   c                   T    e Zd ZU ej        d         ed<   dZej        e         ed<   dS )r   r   variantsNr-   )	r8   r9   r:   r=   r@   r<   r-   r>   r;   rA   r   r   r   r   8   s<         ;~&&&&!foc!!!!!r   r   c                   .    e Zd ZU eed<   eed<   eed<   dS )r   r-   widthheightNr8   r9   r:   r;   r<   intrA   r   r   r   r   >   s+         		r   r   c                   B    e Zd ZU eed<   eed<   eed<   eed<   eed<   dS )r   idlistdurationr-   thumbUrlNrG   rA   r   r   r   r   E   s=         
	r   r   c                      e Zd ZU eed<   eed<   eed<   dZej        e         ed<   dZ	ej        ej
        e                  ed<   dZej        ej        j                 ed<   dZej        ej        j                 ed<   dZej        ej        j                 ed	<   dZej        ej        j                 ed
<   dZej        ej        j                 ed<   ej                            dd d          Zej                            dd d          Zej                            dd d          Zej                            dd d          Zej                            dd d          Zd ZdS )r   usernamenameverifiedNdescriptionwebsites	followerspostsr1   tags	followingfollowersGranularityc                     | j         j        S r
   )rT   granularityr5   s    r   <lambda>zUser.<lambda>[       _c_m_y r   zfollowers.granularitypostsGranularityc                     | j         j        S r
   )rU   rZ   r5   s    r   r[   zUser.<lambda>\   s    W[WaWm r   zposts.granularityphotosGranularityc                     | j         j        S r
   )r1   rZ   r5   s    r   r[   zUser.<lambda>]   s    Y]YdYp r   zphotos.granularitytagsGranularityc                     | j         j        S r
   )rV   rZ   r5   s    r   r[   zUser.<lambda>^   s    UYU^Uj r   ztags.granularityfollowingGranularityc                     | j         j        S r
   )rW   rZ   r5   s    r   r[   zUser.<lambda>_   r\   r   zfollowing.granularityc                     d| j          S )Nhttps://vk.com/)rO   r5   s    r   r7   zUser.__str__a   s    	*4=	*	**r   )r8   r9   r:   r;   r<   boolrR   r=   r>   rS   r@   rT   snscrapebaseIntWithGranularityrU   r1   rV   rW   _DeprecatedPropertyrX   r]   r_   ra   rc   r7   rA   r   r   r   r   N   s        
%)foc")))/36?6;s+,333@DFOHM<=DDD<@89@@@=A9	:AAA;?vx}78???@DFOHM<=DDD 99:PRyRy  |S  T  TM556HJmJm  pC  D  D]667JLpLp  sG  H  H=445FHjHjl~ 99:PRyRy  |S  T  T+ + + + +r   r   c                        e Zd ZdZ fdZd Zd Zd ZddZd Z	d	 Z
d
 Zd Zd Zed             Zed             Z xZS )r   zvkontakte-userc                      t                      j        di | || _        d| j         | _        ddd| _        d | _        d | _        d S )Nrf   zDMozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0zen-US,en;q=0.5)z
User-AgentzAccept-LanguagerA   )super__init__	_username_baseUrl_headers_initialPage_initialPageSoup)r6   rO   r   	__class__s      r   ro   zVKontakteUserScraper.__init__h   sh    %''V$.4DN44$-!g  }M  N  N$-$$r   c                     |rz|                     dd                              d          rQ|d                             dd          }|dk    rd }t          j                            |d         d|                   S d S )Nhref /away.php?to=&   )get
startswithfindurllibparseunquote)r6   aends      r   _away_a_to_urlz#VKontakteUserScraper._away_a_to_urlp   sz     4155''88 4	
6R	 	 3	Rii
C
,

qyc2
3
33	r   c                 T    d|j         v o|j         d                             d          S )Nz
aria-labelphoto)attrsr~   )r6   r   s     r   is_photozVKontakteUserScraper.is_photoy   s)    		 	NQW\%:%E%Eg%N%NNr   c           	         |sd S d|j         v rBt          j                            t          |d                   t          j        j                  S t                              |j                  x}rt          d          }|
                    d          dv rgt          j                            |          }|
                    d          dk    r|t          j        d          z  }|j        |j        |j        }}}nt          |
                    d	          p9|
                    d
          p$t          j                            |          j                  }t                               |
                    d          p|
                    d                    dz   }t          |
                    d          p|
                    d                    }t          |
                    d                    }|dk    r|dz  }|
                    d          dk    r|dz  }t          |
                    d                    }	t%          ||||||	          S t'          j        dd                    t                     z   dz   |j                  x}rt          j        t          |
                    d                    t                               |
                    d                    dz   t          |
                    d                              S |j        dvr$t,                              d|j                   d S d S )NtimezEurope/Moscowr.   )today	yesterday)r   r      )daysyear1year2month1month2day1day2hour   ampmpmminutez^(?P<day>\d+)\s+(?P<month>r+   z)\s+(?P<year>\d{4})$yearmonthday)r2   r   zCould not parse date string: )r   r   fromtimestamprH   r   utc_datePatternmatchtextr   groupnow	timedeltar   r   r   _monthsindexr   rejoinr.   _loggerwarning)
r6   dateSpanr   r   r.   r   r   r   r   r   s
             r   _date_span_to_datez'VKontakteUserScraper._date_span_to_date|   s   	 
$x~


)
)#hv.>*?*?ARAV
W
WW!!(-000e B/""2kk&333  b ))D{{6k))	Xq))))Ty$*dh%DDu{{7##bu{{7';';bx?P?T?TZ\?T?]?]?bccDMM%++h//H5;;x3H3HIIAME
ekk&!!8U[[%8%8
9
9C
ekk&!!
"
"4
bjjBJDkk&T!!BJDH%%&&6
b$sD&
A
AAx58I8IILccemerssse t
-EKK//00'--G@T@T2U2UXY2Y[^_d_j_jkp_q_q[r[r
s
ss],,,
??D8=DDEEEEE -,r   Fc                     |                     d|sdnd          }|s4t                              dt          |          d d                    d S t          j                             j        |d                   }|                    d          s,|rk|                    d	          s|                    d
          rAd|v r=|d         dk    r1|	                    dd          d         
                    d          dv sJ |s-|                     dd                               dd          }n,|                     dd                               dd          }|                     dd          }|r" fd|                    d          D             ng }|                     dd          x}rI|                     dd          x}	r0                     |	          x}
r|
|vr|                    |
           d }d }|s|                     dd          n|                     dd          x}rD|s3|j        j        dk    r#d|j        j        v rd|j        j        d         v sg }|                    dd          D ]}                     |          s'd|j        vrt                              d|            ?d|j        v rt%          |d         |d          t'          |d!                   d"|d          |d#         |d#                              d$          d%z   x}|d#                              d&|                   '          }d(|j        vs@|d(                             d)          r%d*|d(         vs|d(                             d+          s t                              d,| d-           4|d(         |d(                              d*          d.         }t+          j        |          d/ d0                                         D             }|D ]}d0         |                             d1          s'd0         d2          d0         |          d0         |<   | dd0                  d3                             d1          s3d0         d2          d0                  d3          d0                  d3<   t1          d4 d0                                         D                       sRt3          fd5|D                       r7t3          fd6|D                       rt3          fd7|D                       s t                              d,| d8           g }|D ]}| d|                    t5          d9d0                  d3         vrd0                  d3          d9nd0                  d3         d0                  d         d0                  d:         ;                     d|j        v rM|d                             d<          r2|d         d=d          
                    d>          d?k    rd"|d          nd }|                    t7          ||@                     |                     dd          x}r                     |dAB          nd }t;          |                     |          |r|j        nd |pd |pd |pd |C          S )DNr   	post_linkpublished_by_dateclass_zSkipping post without link:    rw   zhttps://vk.com/wallzhttps://vk.com/videozhttps://vk.com/photo_r|   r   
0123456789)rx   z?reply=div	post_datespanrel_datecopy_post_datewall_post_textc                 B    g | ]}                     |          xS rA   )r   ).0r   hr6   s     r   
<listcomp>z:VKontakteUserScraper._post_div_to_item.<locals>.<listcomp>   s1    PPPA8K8KA8N8N3N1PaPPPr   
media_linkmedia_link__title	wall_textpage_post_sized_thumbsclass
copy_quotepage_post_thumb_wrapz
data-videoz/Skipping non-photo and non-video thumb wrap on z	data-listzdata-durationzhttps://vk.comstylezbackground-image: url(   ))rJ   rK   rL   r-   rM   onclickzreturn showPhoto('z{"temp":z	}, event)zPhoto thumb wrap on z' has no or unexpected onclick, skippingic                 r    g | ]4}t          |          d k    dt          |          cxk    rdk    /n n|5S )r   a   z   )lenordr   ks     r   r   z:VKontakteUserScraper._post_div_to_item.<locals>.<listcomp>   sR    bbbac!ffkkbTWXYTZTZNaNaNaNa^aNaNaNaNaNaNaNaNar   tempzhttps://ri   r   c              3      K   | ]}|d vV  	dS )>	   wxyzw_x_y_z_ri   NrA   r   s     r   	<genexpr>z9VKontakteUserScraper._post_div_to_item.<locals>.<genexpr>   s(      
j
jQ1HH
j
j
j
j
j
jr   c              3      K   | ]C}d          |         d          | d         d         d          | d         d         dz   fv V  DdS )r   r   r   .jpgNrA   r   r   photoObjs     r   r   z9VKontakteUserScraper._post_div_to_item.<locals>.<genexpr>   s        G  Gpqx"x'71'@'CXfEUYZV]V]V]E^_`EadjEj&kk  G  G  G  G  G  Gr   c              3   z   K   | ]5}d          |                              d          odd          |         v V  6dS )r   zhttps://sunz.userapi.com/N)r~   r   s     r   r   z9VKontakteUserScraper._post_div_to_item.<locals>.<genexpr>   sk        C  Clmx"--m<<gT\]cTdefTgAg  C  C  C  C  C  Cr   c              3      K   | ]v}t          d          | dx                   dk    oMt          d                   d         t                    o&t          d                   d         t                    V  wdS )r   r      r      N)r   
isinstancerH   )r   r   r   r   s     r   r   z9VKontakteUserScraper._post_div_to_item.<locals>.<genexpr>   s        p  p  Z[s8F#a777]R455:  Uz(SYJZ[]J^_`Jacf?g?g  Ulvw  AG  xH  IK  xL  MN  xO  QT  mU  mU  p  p  p  p  p  pr   z( has unexpected data structure, skippingr   r   )r-   rE   rF   z/photo   z0123456789-_rx   )rC   r-   T)isCopy)r-   r.   r/   r0   r1   r2   r3   ) r   r   r   r;   r   r   urljoinrq   r~   rsplitstripfind_allr   appendparentrP   r   r   r   rH   endswithjsonloadskeysanyallr   r   _post_div_to_itemr   r   r   )r6   postr   postLinkr-   r   textDivr0   mediaLinkDiv
mediaLinkArw   r1   r2   	thumbsDivr   begin	photoDatasingleLetterKeysr   photoVariantsphotoUrlquoteDivr3   r   r   r   s   `                      @@@r   r   z&VKontakteUserScraper._post_div_to_item   s	   YYs%W[[DWYXX(	 

??E#d))DSD/EEFFF	6T]HV,<==#
...
/
/  wF  wOe@f@f  wjmjxjx  zP  kQ  kQ  w  Y\  `c  Yc  Yc  hk  ln  ho  sv  hv  hv  {~  {E  {E  FI  KL  {M  {M  NO  {P  {V  {V  Wc  {d  {d  hw  {w  {w  {w  
w	 biii4499&:9VV88ii(8i99>>sM`>aa8IIe&6I77'T[cPPPPP))#..PPPPac(iii===l  %%c4G%HHHj!!*---d 
??4&
%BHRDIIekI:::dXXY^  jBX  C  C  Ci *C*C").%77GyGWG]<]<]bnr{  sC  sI  JQ  sR  cR  cR6s-CDD 'C 'Cq== AG ; ;__LsLLMMMqw\?{^Q'(((QvY((7ajoo6N&O&ORT&TTUXYZaXbXgXghkmrXsXsst  U q|'>'>?S'T'TXbjklujvXvXv~  AJ  K  T  T  U`  a  aXv__XCXXXYYY)Qy\..z::R?@Iz)$$Hbb8F#3#8#8#:#:bbb W WVQ**:66 O'/7N&9I!9LNNhvq'''RVR #..z:: W#+F#3F#; VXf=Mb=QRS=T V Vhvr1

j
jRZ[aRbRgRgRiRi
j
j
jjj   G  G  G  G  vF  G  G  G  G  G  C  C  C  C  rB  C  C  C  C  C   p  p  p  p  p  _o  p  p  p  p  p __YCYYYZZZM Y Y'''R,QW_ghn_opr_stu_vQvQv0@0DQ0G-M-M-M-M  }E  FL  }M  NP  }Q  RS  }T  ^f  gm  ^n  oq  ^r  st  ^u  @H  IO  @P  QS  @T  UV  @W  X  X  X  Y  Y  Y  Y/5/@/@QvYEYEYZbEcEc/@hijphqrsrtrthuh{h{  }K  iL  iL  PR  iR  iR+&	+++  X\H
MM%=AAABBBBOSyyY^iuyOvOvCv8  Bt%%h%>>>  ~B*	
""8,,%/gll44^tMT
 
 
 r   c              #   l   K   |                     dd          D ]}|                     |          V  d S )Nr   r   r   )r   r   )r6   soupr   s      r   _soup_to_itemsz#VKontakteUserScraper._soup_to_items   sL      mmEFm33 & &d				%	%%%%%& &r   c                 ^   | j         t                              d           |                     | j        | j                  }|j        dvr't          j        	                    d|j                   |t          j        |j        d|j                  c| _         | _        | j         | j        fS )NzRetrieving initial data)headers)r     Got status code lxml)from_encoding)rs   r   info_getrq   rr   status_coderh   ri   ScraperExceptionbs4BeautifulSoupr/   encodingrt   )r6   rs     r   _initial_pagez"VKontakteUserScraper._initial_page   s    	
<<)***yy$-y881m:%%
-
(
()KAM)K)K
L
LL./1B19ffgfp1q1q1q+4d+		D1	11r   c              #      K                                     \  }}|j        dk    rt                              d           d S |                    dd          rt                              d           d S |                    dd          x}r!t                              |j                   d S |                    dd	          }|st                              d
           d S |j        d                             d          d         }d|j        d         v r4t          |j        d                             d          d                   }nd}t          j        d           fd} ||          E d {V  d}t          j        dd          D ]t}	                     |||	          }
|
                    d          r d S |
                    d          s|
dk    rt                              d|	 d           t!          |dz   |	dz             D ]}                     |||          }|                    d          r nm|                    d          s2|dk    rLt"          j                            d|d d         d           |t)          j        |d           !          E d {V  !t"          j                            d|
d d         d          |	}t)          j        |
d           } ||          E d {V  vd S )"Nr  zWall does not existr   profile_closed_wall_dummyr   zPrivate profileh5profile_deleted_textr   zWall has no postszdata-post-idr   r   
post_fixedr   rJ   r   rx     )maxlenc              3      K                        |           D ]M}t          |j                            dd          d                   }|vr|V                      |           Nd S )Nr   r   )r  rH   r-   r   r   )r  itempostIDlast1000PostIDsr6   s      r   _process_soupz5VKontakteUserScraper.get_items.<locals>._process_soup  sy      ""4(( $ $ta((+,,F_$$ZZZF###	$ $r   
   )startstepz!<div class="page_block no_posts">z<div id="postz"\/blank.php?block=119910902"zEncountered geoblock on offset z>, trying to work around the block but might be missing contentzGot an unknown response: r   z...r  )r  )r  r  r   r   r   r   r
  r   splitrH   collectionsdeque	itertoolscount_get_wall_offsetr~   rangerh   ri   r  r  r  )r6   r  r  profileDeleted
newestPostownerIDfixedPostIDr  lastWorkingOffsetoffsetrU   geoblockOffsetgeoPostsr  s   `            @r   	get_itemszVKontakteUserScraper.get_items   s       '!T]c
??()))	6	YYu:Y;; 

??$%%%	6		$1G	HHHn 

??>&'''	6yyy00*	 

<<#$$$	6^,22377:'Z%g...Z%d+11#66q9::;;;%t444/$ $ $ $ $ $ ]4         2666 " "f  gv>>5:;; 
	EE


?
+
+ Y000__}v}}}~~~ !2Q!6DD 	K 	K&&{G^LLh			@	A	A u  11 _
6
6
6]++,]QURUQU,],],]^^^c&7&&I&IJJJJJJJJJJJ
-
(
()WU4C4[)W)W)W
X
XX

E6
*
*4mD!!!!!!!!!!-" "r   c                 |   | j                                         }d|d<   t                              d|            |                     dddd|fd|fd	d
|fdd|fg|          }|j        dk    r't          j                            d|j                   |	                                d         d         d         }|S )NXMLHttpRequestzX-Requested-WithzRetrieving page offset zhttps://vk.com/al_wall.php)actget_wall)alr   fixedr.  )	onlyCachefalseowner_id)typeownwall_start_from)datar  r   r  payloadr   r   )
rr   copyr   r
  _postr  rh   ri   r  r   )r6   r,  r+  r.  r  r  rU   s          r   r'  z%VKontakteUserScraper._get_wall_offset2  s    M  ' 0'
	,,111222
jj Wk,BXvDVXnq{  ~E  qF  HW  Zk  ms  Yt  u   !
 ]c		'	'(J1=(J(J	K	KK
&&((9
a
 
#%	,r   c           
         |                                  \  }}|j        dk    rd S i }|j                            dd          d         |d<   |                    dd          }|j        |d<   t          |                    d	d
                    |d<   |                    d	d          x}r
|j        |d<   |                    d	d          x}r=g }|                    d	ddg          D ]}d|d         v r&|                    d	d          }	|	r|	j        dk    r2n#|d         dk    r
|j        |d<   |d         dk    rV|                    d          D ]}
|
d                             d          s$t          
                    d|
d                    A|                    t          j                            |
d                             dd          d                             dd          d                              |r||d<   d t           d!t"          j        t&          t&          f         fd"}|                    d	d#          x}r|                    dd$          D ]v}
 ||
                    d	d%          j                  \  }}|
                    d	d&          j        }|d'v r| d }|d(v r#t(          j                            ||          ||<   w|                    d	d)          x}rr|                    d	d*          x}rY|                    d+d,          j        d-k    r9t)          j        j         ||                    d+d.          j                   |d/<   |                    d	d0          x}rr|                    d	d*          x}rY|                    d+d,          j        d1k    r9t)          j        j         ||                    d+d.          j                   |d2<   t/          d3i |S )4Nr   /r   rO   h1	page_namer   rP   r   page_verifiedrQ   page_current_info)rJ   rR   page_info_wrapprofile_info_rowgroup_info_rowr   fl_lzWebsite:titleDescriptionWebsiter   rw   ry   zSkipping odd website link: =rz   r   rS   r   returnc                    |                      d          rt          | d d                   dz  dfS |                      d          r_| d d         }d}d| v r.|dt          |                    d          d                   z  z  }t          t	          |          dz            |fS t          |                     d	d
                    dfS )NKr|   r  Mi@B .r  r   ,rx   )r   rH   r   r"  floatreplace)r   baseNum	precisions      r   	parse_numz3VKontakteUserScraper._get_entity.<locals>.parse_numb  s    jjoo 	&q"v;;t##	

3 &fGI
axxB#gmmC0034445YuW~~'(()33qyyb!!""A%%r   counts_modulepage_counterr&  label)followerr   r   tag)rT   rU   r1   rV   profile_idols
header_topr   header_label	Followingheader_countrW   public_followers	FollowersrT   rA   )r  r  r-   r   r   r   rg   r   r~   r   r   r   r   r   r   r"  r;   r=   TuplerH   rh   ri   rj   r   )r6   r  r  r   nameH1descriptionDivinfoDivrS   rowDivlabelDivr   rZ  	countsDivr&  rZ   r]  idolsDivtopDivfollowersDivs                      r   _get_entityz VKontakteUserScraper._get_entityA  sw     '!T]c	6&u||C++A.&99TK900&;&.FKKKHHII&		%.A	BBBn /).6-5'7888g "8!!%3EGW2X!YY X XvVG_,,EF33X 33 4 	w=(($kf]w9$$__S!! X XfI  11 ooAAfIAABBB__V\))!F)//#q*A*A!*D*J*J3PQ*R*RST*UVVWWWW	X
  "!F:
&3 
&6<S1 
& 
& 
& 
& 99U_9===i Js^<< J Jq"166%'6#B#B#GHHE;FF57F++0E444[[[U888]55e[IIVE]))E)888h zu|<<<v z&++f_m+BnBnBs  xC  CC  CC"-:IIfkkRXcqkFrFrFw<x<xyF; ii,>i???l z""5<"@@@v zfkkRXcqkFrFrFw  |G  GG  GG"-:IIfkkRXcqkFrFrFw<x<xyF;	r   c                 p    |                     dt          j                            d          d           d S )NrO   zA VK username)r;  help)add_argumentrh   utilsnonempty_string_arg)cls	subparsers     r   _cli_setup_parserz&VKontakteUserScraper._cli_setup_parser  s4    HN,N,Nz,Z,Zcrsssssr   c                 8    |                      ||j                  S r
   )_cli_constructrO   )rw  r   s     r   _cli_from_argsz#VKontakteUserScraper._cli_from_args  s    			D$-	0	00r   )F)r8   r9   r:   rP   ro   r   r   r   r   r  r  r1  r'  rq  classmethodry  r|  __classcell__)ru   s   @r   r   r   e   s             O O OF F F>H H H HT& & &2 2 2=" =" ="~  ? ? ?B t t +t 1 1 +1 1 1 1 1r   r   )%__all__r  r#  dataclassesr   r%  r   loggingr   snscrape.baserh   snscrape.utilsr=   urllib.parser   r   r   r   ImportErrorr   	getLoggerr8   r   r   compiler   r   	dataclassri   Itemr   r   r   r   r   Scraperr   rA   r   r   <module>r     s  
]
]
] 


                				             9  9 9 9 9  9 9 9  9 9 9 9 99 '
H
%
%
^
^
^rz BDGHHWDUDUVY22 58HHW4E4EFIVV W W 
 
 
 
 
HM& 
 
 
 " " " " " " " "
                 + + + + +8= + + +,c1 c1 c1 c1 c18=0 c1 c1 c1 c1 c1s   A AA