
    @j>                        g d Z ddlZddlZddlZddlZddlZddlZddlZddl	Zddl
Z
ddlZ ej        e          Zej         G d dej        j                              Zej         G d dej        j                              Z G d dej        j                  Z G d	 d
e          Z G d de          Z G d de          Z G d de          ZdS ))FacebookPostUserFacebookUserScraperFacebookCommunityScraperFacebookGroupScraper    Nc                       e Zd ZU eed<   eed<   ej        ed<   ej        e         ed<   eed<   e	j
                            dd d          Zd Zd	S )
r   cleanUrldirtyUrldatecontentoutlinks
outlinksssc                 6    d                     | j                  S )N )joinr   selfs    `/home/ubuntu/.hermes/hermes-agent/venv/lib/python3.11/site-packages/snscrape/modules/facebook.py<lambda>zFacebookPost.<lambda>   s    388TXTaKbKb     c                     | j         S N)r	   r   s    r   __str__zFacebookPost.__str__   s
    	r   N)__name__
__module____qualname__str__annotations__datetimetypingOptionallistsnscrapebase_DeprecatedPropertyr   r    r   r   r   r      s         
/#
m//>b>bdnoo    r   r   c                      e Zd ZU eed<   eed<   eed<   eed<   dZej	        e
j                 ed<   dZej	        e         ed<   dZej	        e         ed<   dZej	        e         ed	<   dZej	        e         ed
<   dZej	        e         ed<   dZej	        e         ed<   dZej	        e         ed<   dZej	        ej        e                  ed<   d ZdS )r   usernamepageIdnameverifiedNcreated	pageOwnerlikes	followerscheckinsaddressphonewebkeywordsc                     d| j          dS Nhttps://www.facebook.com//)r(   r   s    r   r   zUser.__str__1   s    	5T]	5	5	55r   )r   r   r   r   r   intboolr,   r    r!   r   r   r-   r.   r/   r0   r1   r2   r3   r4   Listr   r&   r   r   r   r   !   s!        	
+/&/(-
(///#'FOC '''#####'FOC '''"&6?3&&&!%&/#
%%%####!foc!!!/36?6;s+,3336 6 6 6 6r   r   c                        e Zd Zd Zd Zd ZdS )_FacebookCommonScraperc           	         t           j                            |          }|j        dk    rwt           j                            |j                  }|j        |j        |j        t           j                            d|d         d         fd|d         d         ff          df}n|j        dk    rht           j                            |j                  }|j        |j        |j        t           j                            d|d         d         ff          df}n!|j        dk    rt           j                            |j                  }|d	         d         }|	                    d
          
                    d          rd|                    dd          d          }|j        |j        |j        t           j                            d	|ff          df}ne|j                            d          d         dk    s]|j                            d          sC|j                            d          s)|j                            d          ddd         ddgk    r|j        |j        |j        ddf}n|j                            d          d         dv r|j                            d          dk    r|j        |j        |j        ddf}nv|j                            d          dk    rT|j                            d          }|                    d           |j        |j        d                    |          ddf}n|S |S t           j                            |          S )Nz/permalink.php
story_fbidr   id z
/photo.phpfbid/media/set/set
0123456789z.a.za..   r8      posts/events//notes/   groups	permalink)photosvideos      )urllibparseurlparsepathparse_qsqueryschemenetloc	urlencoderstripendswithrsplitsplit
startswithcountpopr   
urlunsplit)r   r
   uqcleansetVal	pathcompss          r   
_clean_urlz!_FacebookCommonScraper._clean_url6   s   lH%%!V|QW%%1Hah(>(>qQ]_`Oa@beiklmqkrstkudv?w(x(xz|}55v|QW%%1Hah(>(>6ST@V?X(Y(Y[]^55v|QW%%1eHQK6mmL!!**511 -,&--Q''*,,FHah(>(>?Q(R(RTVW55v||Cw&&!&*;*;J*G*G&16K\K\]fKgKg&klkqkwkwx{k|k|}~  @A  BC  ~C  lD  IQ  S^  H_  l_  l_HahB/55v||C333 fll31XqxR0EE	SQS!!IMM!Xqx)!4!4b"=EEO ?		 	 	'	''r   c                     |dk    rFt          fddD                       s)dk    r
d|v rd|v rdS                     d          rdS d	S d
S |dk    rt          j        d          sd	S d
S d S )Nuserc              3       K   | ]}|v V  	d S r   r&   ).0xhrefs     r   	<genexpr>z6_FacebookCommonScraper._is_odd_link.<locals>.<genexpr>^   s?        N  NAa4i  N  N  N  N  N  Nr   )z/posts/z/photos/z/videos/z/permalink.php?rJ   rK   z/photo.php?rC   #z	new photozto the album)TFz /business/help/788160621327601/?)TT)FNgroupz#^/groups/[^/]+/permalink/\d+/(\?|$))anyr`   rematch)r   ro   	entryTextmodes    `  r   _is_odd_linkz#_FacebookCommonScraper._is_odd_link[   s    	V^^
  N  N  N  N  "M  N  N  N  N  N s{{{i//Ni4O4OK	;	<	< KJ
+w
(94
@
@ :
+ r   c           	   #     K   d }|                     dd          D ]}|j        }d}|r/|j        dk    rd|j        v rd|j        d         v rd}n	|j        }|/|r@|                    dd          }|                    dd	          }	|	s(|s&t
                              d
| d|j                   |	r|r|d         dk    r	|	d         }
n
|r|d         }
|                     |
|j        |          \  }}|r |rt
                              d|
            t          j
                            ||
          }|                     |          }t          j                            t          |                    dd          d                   t          j        j                  }|                    dd          x}r|j        }nd }g }|                     d          D ]}|                    d          s|                    d          }
|
                    d          sDt          j
                            t          j
                            |
          j                  }d|vst1          |d                   dk    rt
                              d|
            |d         d         }|                    d          s|                    d          r||vr|                    |           t5          |||||          V  d S )Ndiv_5pcrclass_FclassTa_5pcqz_17z-zIgnoring link-less entry after : ro   rq   zIgnoring odd link: abbr_5ptzz
data-utime_5pbxzhttps://l.facebook.com/l.php?rd   rG   zIgnoring odd outlink: r   zhttp://zhttps://)r	   r
   r   r   r   )find_allparentr*   attrsfind_loggerwarningtextrx   rS   rT   urljoinri   r   fromtimestampr9   timezoneutchas_attrgetr`   rW   rU   rX   lenappendr   )r   soupbaseUrlrw   r	   entryr   isNestedentryA	mediaSetAro   oddLinkwarnr
   r   
contentDivr   r   r   rX   outlinks                        r   _soup_to_itemsz%_FacebookCommonScraper._soup_to_itemsm   sQ     (}}UW}55 /u /ueL68	 {e6< 7 7Gv|T[G\<\<\X
]F	 
 
  JJsWJ--6zz#z009
 F OOPhPP%*PPQQQ  6&>S#8#8VDD &>D$$T5:t<<=7D  3__1411222l""7D118ooh''8


)
)#ejj'j.R.RS_.`*a*ackctcx
y
y4ZZZ888z oGGG8NN3  q::f 55==D??:;; L!!&,"7"7"="="CDDE
%3uSz??a//__4d44555CjmG)$$ (:(::(F(F 7ZbKbKb__W	htW^ks	t	t	ttttt_/u /ur   N)r   r   r   ri   rx   r   r&   r   r   r=   r=   5   sG        #( #( #(J  $1u 1u 1u 1u 1ur   r=   c                   V     e Zd Z fdZd Zd Zed             Zed             Z xZ	S ) _FacebookUserAndCommunityScraperc                 x     t                      j        di | || _        ddd| _        d | _        d | _        d S )NzBMozilla/5.0 (X11; Linux i686; rv:78.0) Gecko/20100101 Firefox/78.0en-US,en;q=0.5z
User-AgentzAccept-Languager&   )super__init__	_username_headers_initialPage_initialPageSoup)r   r(   kwargs	__class__s      r   r   z)_FacebookUserAndCommunityScraper.__init__   sW    %''V$.!e  {K  L  L$-$$r   c                 N   | j         t                              d           |                     | j        | j                  }|j        dvr't          j        	                    d|j                   || _         t          j        |j        d          | _        | j         | j        fS )NzRetrieving initial dataheaders)     Got status code lxml)r   r   info_get_baseUrlr   status_coder#   r$   ScraperExceptionbs4BeautifulSoupr   r   )r   rs     r   _initial_pagez._FacebookUserAndCommunityScraper._initial_page   s    	
<<)***yy$-y881m:%%
-
(
()KAM)K)K
L
LL4,QVV<<4		D1	11r   c              #   :  K   t          j        d          }t          j        d          }|                                 \  }}|j        dk    rt                              d           d S |                     || j        d          E d {V  |                    d|          x}rt          	                    d           | 
                    t          j                            | j        |                    d	                    d
z   | j                  }|j        dk    r't           j                            d|j                   t'          j        |                    d|j                            }d|v sJ t/          |d                   dk    sJ t/          |d         d                   dk    sJ |d         d         d         dk    sJ |d         d          d            |d         d         d         dv sJ |d         d         d         dk    sJ d|d         d         d         v sJ t1          j        |d         d         d         d         d          }|                     || j        d          E d {V  |                    d|          x}d S d S )Nz&^/pages_reaction_units/more/\?page_id=^for \(;;\);r   zUser does not existrk   r   )ajaxifyzRetrieving next pager   z&__a=1r   r   r   rA   domopsrG   r   rL   replacez is not "replace")z/#www_pages_reaction_see_more_unitwww_pages_homez8#www_pages_reaction_see_more_unitwww_pages_community_tabrH   F__htmlrR   r   )rt   compiler   r   r   r   r   r   r   r   r   rS   rT   r   r   r   r#   r$   r   jsonloadssubr   r   r   r   )r   nextPageLinkPatternspuriousForLoopPatternr   r   nextPageLinkresponses          r   	get_itemsz*_FacebookUserAndCommunityScraper.get_items   s     
#LMM:o66  '!T]c
??()))	6  t}f=========32EFF	F ?
<<&''' yy%%dm\5E5Ei5P5PQQT\\hlhuyvv1ms
-
(
()KAM)K)K
L
LLj/33B??@@8
h




hx 
!
!Q
&
&
&
&
hx #
$
$
)
)
)
)
8
Q

"i
/
/
/HX4Fq4I1\1\1\
/
/
/
8
Q

"  'V  V  V  V  V
8
Q

"e
+
+
+
+
hx(+A.
.
.
.
.

HX.q1!4X>
G
G4!!$v>>>>>>>>># 32EFF	F ? ? ? ? ?r   c                 p    |                     dt          j                            d          d           d S )Nr(   zA Facebook username or user IDtypehelpadd_argumentr#   utilsnonempty_string_argcls	subparsers     r   _cli_setup_parserz2_FacebookUserAndCommunityScraper._cli_setup_parser   sG    HN,N,Nz,Z,Z  dD  E  E  E  E  Er   c                 8    |                      ||j                  S r   )_cli_constructr(   r   argss     r   _cli_from_argsz/_FacebookUserAndCommunityScraper._cli_from_args   s    			D$-	0	00r   )
r   r   r   r   r   r   classmethodr   r   __classcell__r   s   @r   r   r      s            2 2 2? ? ?: E E +E 1 1 +1 1 1 1 1r   r   c                   (     e Zd ZdZ fdZd Z xZS )r   zfacebook-userc                 Z     t                      j        |i | d| j         d| _        d S r6   r   r   r   r   r   r   r   r   s      r   r   zFacebookUserScraper.__init__   s7    %''D#F###?dn???$---r   c           	      x   i }t          j        d          }t          j        d          }t          j        d          }g d}t          j        dd                    |          z   dz             }|                                 \  }}|j        dk    rd S |                    |j                  }	|                    |	                    d	                    }
|
                    d
          |d<   |                    |j                  }t          j	        |                    d
                    }t          j        |d          }|                    dd          j        |d<   t          |                    dd                    |d<   |                    dd          }|j                            d          r|j                            dd
          d
         }|                    |          }|s
J d            t#          j        t'          |                    d                    |                    |                    d
                    d
z   t'          |                    d                              |d<   |j                            d          r$|j                            dd
          d
         |d<   |                    dd          }|                    dd          D ]}|j        }|                    d           rAt'          |                    d!d
          d	                             d"d#                    |d$<   `|                    d%          rAt'          |                    d!d
          d	                             d"d#                    |d&<   |                    d'          r@t'          |                    d!d
          d	                             d"d#                    |d(<   |                    dd)          }|r|                    dd*          D ]}|                    d+d,          }|s|d-         d.k    r6|                    dd/          j        }t          j        d0d1|          |d2<   _|d-         d3k    r |                    dd          j        |d4<   |d-         d5k    r|                    d          D ]}|j        d#k    sd6|j        vs|                    d7          r,|d6         }|                    d8          s
J d9            t4          j                            |                    d:d
          d
                             d;d
          d	                   |d<<   X|d-         d=k    r2|                    dd          j                            d>          |d?<   |                    d@dAB          }|dC                             dD          r|dC                             dE          sJ t'          |dC         dFdG                   |dH<   t;          dIi |S )JNz9"markup":\[\["__markup_a588f507_0_0",\{"__html":(".*?")\}z4<div\s[^>]*(?<=\s)data-key\s*=\s*"tab_home".*?</div>z<a\s[^>]*(?<=\s)href="/([^/]+))JanuaryFebruaryMarchAprilMayJuneJulyAugust	SeptemberOctoberNovemberDecemberz^(|z) (\d+), (\d+)$r   r   rG   r(   r   r   z_64-fr|   r*   _56_fr+   rz   z_61-0zPage created - z - zunexpected created div contentrR   rH   r,   zConfirmed Page Owner: r   r-   _6590_4bl9z people like thisr   ,rA   r.   z people follow thisr/   z
 check-insr0   _u9q_2pi9imgz_3-91srcz<https://static.xx.fbcdn.net/rsrc.php/v3/y5/r/vfXKA62x4Da.png_2wzdz \((\d+,)?\d+(\.\d+)? mi\)
r1   z<https://static.xx.fbcdn.net/rsrc.php/v3/yW/r/mYv88EsODOI.pngr2   z<https://static.xx.fbcdn.net/rsrc.php/v3/yx/r/xVA3lB-GVep.pngro   spanzhttps://l.facebook.com/l.php?u=zunexpected web link=&r3   z<https://static.xx.fbcdn.net/rsrc.php/v3/yl/r/LwDWwC1d0Rx.pngu    · r4   metazal:android:url)propertyr   z
fb://page/z?referrer=app_link
   ir)   r&   )rt   r   r   r   r   searchr   rr   r   r   r   r   r   r:   r`   r_   ru   r   r   r9   indexr   r]   r   r   r   rS   rT   unquoter   )r   r   nameVerifiedMarkupPatternhandleDivPatternhandlePatternmonthscreatedDatePatternr   r   	handleDivhandlenameVerifiedMarkupnameVerifiedSouppageTransparencyContentDivcreatedDateMessmcommunityDivrz   r   aboutDivr   
rawAddressr   dirtyWebandroidUrlMetas                            r   _get_entityzFacebookUserScraper._get_entity   s   & j)effZ WXX*>??- F  F  F&z$&)9)9"9<N"NOO  '!T]c	6%%af--)	 2 233&||A&077??z"4":":1"="=>>&'96BB#((w(??D&.,11#1HHII&#yyyAA$//0ABB e/4::5!DDQG?001
-----!}S__fll1771::6N6NQR6RTWXYX_X_`aXbXbTcTcdd6)$//0HII K38>>tQGGJ6+5733,""57";; E Ec
(4
mm'(( E$**S!,,Q/77R@@AAF7OO+,, Edjja003;;CDDEEF;l## ETZZQ//2::3CCDDF:YYuvY..( O88 O Os
((57(
+
+C 
5zSSS((57(338Z =tZPPVI	UU	U	Uxxx88=VG__	UU	U	Ull3 W W	
2qw..!&&...6h  !BCCZZEZZZCl**8>>#q+A+A!+D+J+J3PQ+R+RST+UVVfUmmW 
UU	U	U((57(;;@FFvNNVJ99V0@9AA.			"	-	-l	;	;xy@Y@b@bcw@x@xxx	x	22c6:;;&	r   )r   r   r   r*   r   r  r   r   s   @r   r   r      sX        @ @ @ @ @D D D D D D Dr   r   c                   "     e Zd ZdZ fdZ xZS )r   zfacebook-communityc                 Z     t                      j        |i | d| j         d| _        d S )Nr7   z/community/r   r   s      r   r   z!FacebookCommunityScraper.__init__*  s7    %''D#F###IdnIII$---r   )r   r   r   r*   r   r   r   s   @r   r   r   '  sF        J J J J J J J J Jr   r   c                   T     e Zd ZdZ fdZd Zed             Zed             Z xZ	S )r   zfacebook-groupc                 H     t                      j        di | || _        d S )Nr&   )r   r   _group)r   rr   r   r   s      r   r   zFacebookGroupScraper.__init__2  s+    %''V$+++r   c              #   t  K   ddd}t          j        d          }t          d          }t          j        d          }d| j         d}|                     ||	          }|j        d
k    rt                              d           d S |j        dk    r't          j	        
                    d|j                   d|j        vrt          j	        
                    d          t          j        |j        d          }dD ]}|j                            |          t          |          z   }	|j        |	|j                            d|	                   }
|                    d|
          }|st          j	        
                    d          t!          |j                  t          j        j        urt          j	        
                    d          t          j        |j        d          }|                     ||d          E d {V  
|                    |j                                      d          |d          x}r|                     d|dd|          }|j        dk    r't          j	        
                    d|j                   t/          j        |                    d|j                            }|d         dk    rd S t          j        |d         d          }|                     ||d          E d {V  |                    |j                                      d          |d          x}d S d S )NzxMozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36r   r   z&"GroupEntstreamPagelet",\{.*?\}(?=,\{)z"GroupEntstreamPagelet",r   z#https://upload.facebook.com/groups/z/?sorting_setting=CHRONOLOGICALr   r   zGroup does not existr   r   +content:{pagelet_group_mall:{container_id:"z:Code container ID marker not found (does the group exist?)r   )r  z-content:{group_mall_after_tti:{container_id:""code)r@   zCode container not foundz)Code container does not contain a commentrr   r   zJhttps://upload.facebook.com/ajax/pagelet/generic.php/GroupEntstreamPageletrG   )data__a)paramsr   rA   payload)rt   r   r   r  r   r   r   r   r#   r$   r   r   r   r   r  r   r   stringelementCommentr   r  rr   r   r   r   )r   r   pageletDataPatternpageletDataPrefixLengthr   r   r   r   codeContainerIdStartcodeContainerIdPoscodeContainerIdcodeContainercodeSoupr  objs                  r   r   zFacebookGroupScraper.get_items6  sZ      V  k{  |  |'z"KLL :;;:o66^$+^^^'
ii7i++!]c
??)***	6}		'	'(J1=(J(J	K	KK2!&@@		'	'(d	e	ee		166	*	*$  	> 	>%9::SAU=V=VVV.cCU1V1VVW?99V/9::=
 E
-
(
()C
D
DD
=  (;;;
-
(
()T
U
UU 4f==8!!(GW========== $**162288;;<S<T<TU	U :yyP1%%   1
 ms
-
(
()KAM)K)K
L
LL	*..r16::	;	;3	)n	E

C	NF
3
34!!$999999999 $**162288;;<S<T<TU	U : : : : :r   c                 p    |                     dt          j                            d          d           d S )Nrr   zA group name or IDr   r   r   s     r   r   z&FacebookGroupScraper._cli_setup_parserg  s4    )K)KG)T)T]qrrrrrr   c                 8    |                      ||j                  S r   )r   rr   r   s     r   r   z#FacebookGroupScraper._cli_from_argsk  s    			D$*	-	--r   )
r   r   r   r*   r   r   r   r   r   r   r   s   @r   r   r   /  s            /: /: /:b s s +s . . +. . . . .r   r   )__all__r   dataclassesr   r   loggingrt   snscrape.baser#   snscrape.utilsr    urllib.parserS   	getLoggerr   r   	dataclassr$   Itemr   r   Scraperr=   r   r   r   r   r&   r   r   <module>r:     s  
m
m
m 


        				              '
H
%
% 
 
 
 
 
8=% 
 
 
 6 6 6 6 68= 6 6 6&iu iu iu iu iuX]2 iu iu iuX51 51 51 51 51'= 51 51 51pK K K K K: K K K\J J J J J? J J J>. >. >. >. >.1 >. >. >. >. >.r   