+ ViSfRt^RIt^RIHtHtHtHt^RIHtH t H t H t H t H t HtHtHtHtHtHt^RIHtHtHtHt^RIHtRRltRR ltR R ltR R ltRRltR#RR/RRlllt RR/RRllt!R#RR/RRlllt"RRlt#R$RR/RRlllt$R$RR/RRlllt%RRlt&RR lt'R!R"lt(R#)%z. Crawling functionality for Firecrawl v2 API. N)OptionalDictAnyList) CrawlRequestCrawlJob CrawlResponseDocumentCrawlParamsRequestCrawlParamsResponseCrawlParamsData WebhookConfigCrawlErrorsResponseActiveCrawlsResponse ActiveCrawlPaginationConfig) HttpClienthandle_response_errorvalidate_scrape_optionsprepare_scrape_options)normalize_document_inputc(V^8dQhR\RR/#)requestreturnN)r)formats"Y/home/ubuntu/hermes-agent/venv/lib/python3.14/site-packages/firecrawl/v2/methods/crawl.py __annotate__rs88\8d8cVP'd!VPP4'g \R4hVPeVP^8:d \R4hVPe\ VP4R#R#)z Validate crawl request parameters. Args: request: CrawlRequest to validate Raises: ValueError: If request is invalid URL cannot be emptyNzLimit must be positive)urlstrip ValueErrorlimitscrape_optionsr)rs&r_validate_crawl_requestr&sm ;;;gkk//11.//}} W]]a%7122) 6 67*rc0V^8dQhR\R\/#)rrr)rdict)rs"rrr&sGGLGTGrcf\V4RVP/pVP'dVPVR&VPe"\ VP4pV'dW!R&VP RRR7pVP RR4VP RR4VP RR4VPeP\VP\4'dVPVR&MVPP RR 7VR&R R R R RRRRRRRRRRRRRRRRRRRRR R!R"R#/pVP4F wrVWS9gK VP V4W&K" VPV4R$V9d8\VR$,\4'dVR$,P4VR$&V#)%z Prepare crawl request for API submission. Args: request: CrawlRequest to prepare Returns: Dictionary ready for API submission r!promptN scrapeOptionsT) exclude_none exclude_unsetr%webhook)r, include_paths includePaths exclude_paths excludePathsmax_discovery_depthmaxDiscoveryDepthsitemapignore_query_parametersignoreQueryParametersdeduplicate_similar_urlsdeduplicateSimilarURLscrawl_entire_domaincrawlEntireDomainallow_external_linksallowExternalLinksallow_subdomainsallowSubdomainsignore_robots_txtignoreRobotsTxtdelaymax_concurrencymaxConcurrencyregex_on_full_urlregexOnFullURLzero_data_retentionzeroDataRetention integration) r&r!r*r%r model_dumppopr. isinstancestritemsupdater")rdata scrape_data request_datafield_mappings snake_case camel_cases& r_prepare_crawl_requestrV&sG$ 7;; D~~~ X),W-C-CD $/ !%%4t%LLUD!Xt$%t," goos + +%ooDO&oo88d8KDO 29!#:"$<2 4-.+-2N$#1"6"6"8  %+// ;D #9  KK D,?!E!E"=1779] KrcrV^8dQhR\\\,,R\\,/#)r data_listr)rrrr )rs"rrrps'htCy&9d8nrc .pT;'g.F>p\V\4'gKVP\R/\ V4B4K@ V#)N)rLr(appendr r)rX documentsdoc_datas& r_parse_crawl_documentsr^psJ "IOOO h % %   XK(@(JK L$ rctV^8dQhR\\\3,R\\\3,/#)r response_datar)rrMr)rs"rrrxs*  S#X 4S> rcrVPR4'g\VPRR44hRVPR4RVPR^4RVPR^4RVPR^4RVPR 4R VPR 4R \VPR .44/#) successerrorUnknown error occurredstatus completedtotal credits_used creditsUsed expires_at expiresAtnextrP)get Exceptionr^)r`s&r_parse_crawl_status_responseroxs   Y ' ' ))'3KLMM -##H-]&&{A6""7A. ))-;m'' 4 !!&)&}'8'8'DE rc<V^8dQhR\R\R\/#rclientrr)rrr)rs"rrrs& N N N\ Nm NrcZ\V4pVPRV4pVP'g \VR4VP 4pVP R4'd0RVP R4RVP R4/p\ R/VB#\VP RR44h) a Start a crawl job for a website. Args: client: HTTP client instance request: CrawlRequest containing URL and options Returns: CrawlResponse with job information Raises: ValueError: If request is invalid Exception: If the crawl operation fails to start z /v2/crawlz start crawlrbidr!rcrdrZ)rVpostokrjsonrmrrn)rrrrRresponser`job_datas&& r start_crawlrzs*'2L{{; 5H ;;;h 6MMOM## -##D) =$$U+  (x(( ))'3KLMMrrequest_timeoutc V^8dQhR\R\R\\,R\\,R\ /#)rrrjob_idpagination_configr{r)rrMrrfloatr)rs"rrrsD>> > > 01> e_ >  >rc 2VPRV 2VR7pVP'g \VR4VP4p\ V4pVR,pV'd VP MRpV'dWVR,'dHV'd)VP e\V4VP 8g\VVR,VVVR7p\VR ,VR ,VR ,VR ,VR ,V'gVR,VR7#RVR7#)a Get the status of a crawl job. Args: client: HTTP client instance job_id: ID of the crawl job pagination_config: Optional configuration for pagination behavior request_timeout: Timeout (in seconds) for each individual HTTP request. When auto-pagination is enabled (default) and there are multiple pages of results, this timeout applies to each page request separately, not to the entire operation Returns: CrawlJob with current status and data Raises: Exception: If the status check fails /v2/crawl/timeoutzget crawl statusrPTrlNr{rerfrgrhrjrerfrgrhrjrlrP) rmrvrrwro auto_paginate max_resultslen_fetch_all_pagesr) rrr}r~r{rxr`payloadr\rs &&&$ rget_crawl_statusrs2zzJvh/zIH ;;;h(:;MMOM*=9GI8I%33dM  ) ) 5  N/;; ;$  FO  +   x +&g^,<($1WV_   8<  rc^V^8dQhR\R\R\\,R\/#)rrrnext_urlr{r)rrMrrr)rs"rrrs4$$ $$e_ $  $rc VPWR7pVP'g \VR4VP4p\ V4p\ VR,VR,VR,VR,VR,VR,VR ,R 7#) au Fetch a single page of crawl results using the provided next URL. Args: client: HTTP client instance next_url: Opaque next URL from a prior crawl status response request_timeout: Timeout (in seconds) for the HTTP request Returns: CrawlJob with the page data and next URL (if any) Raises: Exception: If the request fails or returns an error response rzget crawl status pagererfrgrhrjrlrPr)rmrvrrwror)rrrr{rxr`rs&&$ rget_crawl_status_pagers(zz(z J G  %DNN,cN d 0 -MMO  7 BL %V,,H&3y>[+H   X & -  "s9~'D  #6* a %  " % s F FFc<V^8dQhR\R\R\/#)rrrr}r)rrMbool)rs"rrr]s!666S6T6rcVPRV 24pVP'g \VR4VP4pVP R4R8H#)z Cancel a running crawl job. Args: client: HTTP client instance job_id: ID of the crawl job to cancel Returns: bool: True if the crawl was cancelled, False otherwise Raises: Exception: If the cancellation fails rz cancel crawlre cancelled)deletervrrwrm)rrr}rxr`s&& r cancel_crawlr]sM}}z&23H ;;;h7MMOM   X &+ 55rc V^8dQhR\R\R\R\\,R\\,R\ /#)rrrr} poll_intervalrr{r)rrMintrrr)rs"rrrtsL+"+" +" +"+"c] +" e_ +"+"rc\P!4p\VVVR7pVPR9dV#Ve4\P!4V, V8d\ RV RV R24h\P !V4Kq)a Wait for a crawl job to complete, polling for status updates. Args: client: HTTP client instance job_id: ID of the crawl job poll_interval: Seconds between status checks timeout: Maximum seconds to wait (None for no timeout) request_timeout: Optional timeout (in seconds) for each status request Returns: CrawlJob when job completes Raises: Exception: If the job fails TimeoutError: If timeout is reached rz Crawl job z did not complete within z seconds)rffailedr)rrrre TimeoutErrorsleep)rrr}rrr{r crawl_jobs&&&&$ rwait_for_crawl_completionrts2!J $  +    C C   DNN$4z$AW#LF83LWIU]^_ _ =!rc V^8dQhR\R\R\R\\,R\\,R\ /#)rrrrrrr{r)rrrrrr)rs"rrrsL)) ) ))c] ) e_ ))rc`\W4pVPpVeTMTp\VVVVVR7#)a Start a crawl job and wait for it to complete. Args: client: HTTP client instance request: CrawlRequest containing URL and options poll_interval: Seconds between status checks timeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout) request_timeout: Timeout (in seconds) for each individual HTTP request, including pagination requests when fetching results. If there are multiple pages, each page request gets this timeout Returns: CrawlJob when job completes Raises: ValueError: If request is invalid Exception: If the crawl fails to start or complete TimeoutError: If timeout is reached r)rzrtr)rrrrrr{rr}effective_request_timeouts&&&&$ rcrawlrsF8F,I \\F4C3NT[ %1  rc<V^8dQhR\R\R\/#rq)rr r )rs"rrrs,sNsNsN6HsN_sNrcRVP'd!VPP4'g \R4hVP'd!VPP4'g \R4hRVPRVP/pVP RV4pVP 'g \ VR4VP4pVPR4'EdVPR/4p/pR R R R R RRRRRRRRRRRRRRRRRRR/ pR V9d3VR ,p\V\4'd\R3/VBVR &MWR &VP4FwrW9gK V R8XdWY,eWY,p /p R!R"R#R$R%R&R'R(R)R*R+R,/p V P4FwrW9gK W,W&K R-V 9d8V R-,p\V\4'd^R.IHpV!VR/7V R-&MVV R-&V P4FwppVV 9gKVR-8wgKVV V&K WV &KWY,Wj&K VP4FwppVV9gKVVV&K R0V9d VR0,VR0&\!R3/VB#\#VPR1R244h)4a8 Get crawl parameters from LLM based on URL and prompt. Args: client: HTTP client instance request: CrawlParamsRequest containing URL and prompt Returns: CrawlParamsData containing suggested crawl options Raises: ValueError: If request is invalid Exception: If the operation fails r zPrompt cannot be emptyr!r*z/v2/crawl/params-previewzcrawl params previewrbrPr0r/r2r1r4r3r5r7r6r9r8r;r:r=r<r?r>rDrCr+r%rHrGr. includeTags include_tags excludeTags exclude_tagsonlyMainContentonly_main_contentwaitForwait_forskipTlsVerificationskip_tls_verificationremoveBase64Imagesremove_base64_imagesformats) ScrapeFormats)rrrcrdrZ)r!r"r#r*rurvrrwrmrLr(r rNlisttypesrr rn)rrrrRrxr` params_dataconverted_paramsrS webhook_datarUrTscrape_opts_dataconverted_scrape_optsscrape_field_mappings scrape_camel scrape_snake formats_datarkeyvalues&& rcrawl_params_previewrs ;;;gkk//11.// >>>!5!5!7!7122 w{{'..L {{5|DH ;;;h(>?MMOM###''3  O O !6 y #%> $&@ !6 "8 1 / - !6    #&y1L,--.;.Kl.K +.:+&4&:&:&< "J(0[5L5X'2'>$,.)%~%~)+>!:-/F,.D -)7L6Q6Q6S2 ';BRB`1?7T !$44'7 'B %lD99=?LUa?b1) U&;;y@P9>1#6'?4IZ03>3J$0I'=N&++-JC.((- %.  %*7 *B Y '2!122 ))'3KLMMrc<V^8dQhR\R\R\/#)r http_clientcrawl_idr)rrMr)rs"rrrDs'GG*GG@SGrc rVPRV R24pVP'g \VR4VP4pVPRV4pRVPR.4RVPRVPR.44/p\ R /VB# \ dp\ RT 24hR p?ii;i) z Get errors from a crawl job. Args: http_client: HTTP client for making requests crawl_id: The ID of the crawl job Returns: CrawlErrorsResponse containing errors and robots blocked URLs Raises: Exception: If the request fails rz/errorszcheck crawl errorsrPerrorsrobots_blocked robotsBlockedz'Failed to parse crawl errors response: NrZ)rmrvrrwrrn)rrrxbodyr normalizedes&& rget_crawl_errorsrDsH:W=>H ;;;h(<= G}}((64( gkk(B/ gkk/7;;GWY[;\] #0Z00 GA!EFFGsA"B B6#B11B6c0V^8dQhR\R\/#)rrrr)rr)rs"rrrds f fj f-A frc nVPR4pVP'g \VR4VP4pVPR4'g\ VPRR44hVPR.4p.pVFp\ V\ 4'gKVPRVPR4RVPR VPR44R VPR 4R VPR 4/4K \R VUu.uFp\R/VBNK upR 7#uupi)z Get a list of currently active crawl jobs. Args: client: HTTP client instance Returns: ActiveCrawlsResponse containing a list of active crawl jobs Raises: Exception: If the request fails z/v2/crawl/activezget active crawlsrbrcrdcrawlsrtteam_idteamIdr!optionsT)rbrrZ) rmrvrrwrnrLr(r[rr)rrrxr crawls_innormalized_crawlscncs& rget_active_crawlsrdszz,-H ;;;h(;< ==?D 88I  *BCDD2&I  a    $ $aeeDk155155+;<quuU|155+ &  Rc5dRcBk6GB6GRc5d ee5dsD2 )N)rN))__doc__rtypingrrrrrrrrr r r r r rrrrutilsrrrrutils.normalizerr&rVr^rorzrrrrrrrrrrZrrrs ,, gf68*GT  NF> (, >>B$(, $NH (, HHV6.+" (, +"+"\) (, ))XsNlG@ fr