հj@dZddlmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl mZmZmZmZddlmZddlmZddlmZddlmZmZdd lmZdd lmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5ddl6m7Z7ddl8m9Z9m:Z:m;Z;e jxe=Z>ee?e?fZ@d5dZAGddeBZCd6dZDGddeBZEd7dZF d8 d9dZGd:d ZHGd!d"ZIGd#d$eZJd;d%ZKeKdd-ZO d? d@d.ZPdd/ dAd0ZQGd1d2eZRGd3d4ZSy)BzO The main purpose of this module is to expose LinkCollector.collect_sources(). ) annotationsN)CallableIterableMutableMappingSequence) dataclass) HTMLParser)Values) NamedTupleProtocol)canonicalize_name)Response) RetryError)ConnectionFailedErrorConnectionTimeoutErrorNetworkConnectionErrorProxyConnectionErrorSSLMissingErrorSSLVerificationError)Link) SearchScope) PipSession)raise_for_status)is_archive_fileredact_auth_from_url) url_to_path)vcs)CandidatesFromPage LinkSource build_sourcectjD]6}|jj|s#|t |dvs4|cSy)zgLook for VCS schemes in the URL. Returns the matched VCS scheme, or None if there's no match. z+:N)rschemeslower startswithlen)urlschemes S/var/www/mission/venv/lib/python3.12/site-packages/pip/_internal/index/collector.py_match_vcs_schemer+4sE ++ 99; ! !& )c#f+.>$.FM c eZdZdfd ZxZS)_NotAPIContentcBt|||||_||_yN)super__init__ content_type request_desc)selfr3r4 __class__s r*r2z_NotAPIContent.__init__@s" |4((r,)r3strr4r7returnNone)__name__ __module__ __qualname__r2 __classcell__r6s@r*r.r.?s ))r,r.c|jjdd}|j}|jdryt ||j j )z Check the Content-Type header to ensure the response contains a Simple API Response. Raises `_NotAPIContent` if the content type is not a valid content-type. Content-TypeUnknown)z text/htmlz#application/vnd.pypi.simple.v1+html#application/vnd.pypi.simple.v1+jsonN)headersgetr%r&r.requestmethod)responser3content_type_ls r*_ensure_api_headerrIFs[##'' BL!'')N     x'7'7'>'> ??r,c eZdZy)_NotHTTPN)r:r;r<r,r*rKrK\sr,rKctjj|\}}}}}|dvr t|j |d}t |t |y)z Send a HEAD request to the URL, and ensure the response contains a simple API Response. Raises `_NotHTTP` if the URL is not available for a HEAD request, or `_NotAPIContent` if the content type is not a valid content type. >httphttpsT)allow_redirectsN)urllibparseurlsplitrKheadrrI)r(sessionr)netlocpathqueryfragmentresps r*_ensure_api_responser[`sV-3LL,A,A#,F)FFD% &&j <<T< 2DTtr,Fctt|jr t||tj dt |ddjgdi}|rtj dd|d<|j|| }t|t|tj d t ||jjd d |S) aYAccess an Simple API response with GET, and return the response. This consists of three parts: 1. If the URL looks suspiciously like an archive, send a HEAD first to check the Content-Type is HTML or Simple API, to avoid downloading a large file. Raise `_NotHTTP` if the content type cannot be determined, or `_NotAPIContent` if it is not HTML or a Simple API. 2. Actually perform the request. Raise HTTP exceptions on network failures. 3. Check the Content-Type header to make sure we got a Simple API response, and raise `_NotAPIContent` otherwise. )rUzGetting page %sAcceptz, )rBz*application/vnd.pypi.simple.v1+html; q=0.1ztext/html; q=0.01z"Refreshing package index response.z max-age=0z Cache-Control)rCzFetched page %s as %sr@rA) rrfilenamer[loggerdebugrjoinrDrrIrC)r(rUforce_revalidaterCrZs r*_get_simple_responsercrs"tCy))*S'2 LL"$8$=> $))  G  9:#. ;;sG; ,DTt LLS! 3 Kr,c|rHd|vrDtjj}|d|d<|jd}|r t |Sy)z=Determine if we have any encoding information in our headers.r@z content-typecharsetN)emailmessageMessage get_paramr7)rCmres r*_get_encoding_from_headersrksK>W, MM ! ! ##N3.++i( w<  r,c$eZdZddZddZddZy)CacheablePageContentc.|jsJ||_yr0)cache_link_parsingpager5rps r*r2zCacheablePageContent.__init__s&&&& r,ct|t|xr-|jj|jjk(Sr0) isinstancetyperpr()r5others r*__eq__zCacheablePageContent.__eq__s-%d,P%**..1PPr,c@t|jjSr0)hashrpr(r5s r*__hash__zCacheablePageContent.__hash__sDIIMM""r,N)rp IndexContentr8r9)ruobjectr8bool)r8int)r:r;r<r2rvrzrLr,r*rmrmsQ#r,rmceZdZddZy) ParseLinkscyr0rLrqs r*__call__zParseLinks.__call__sr,Nrpr{r8zIterable[Link])r:r;r<rrLr,r*rrsAr,rcztjdfd tjdfd }|S)z Given a function that parses an Iterable[Link] from an IndexContent, cache the function's result (keyed by CacheablePageContent), unless the IndexContent `page` has `page.cache_link_parsing == False`. c:t|jSr0)listrp)cacheable_pagefns r*wrapperz*with_cached_index_content..wrappersB~**+,,r,c`|jrt|St|Sr0)rormr)rprrs r*wrapper_wrapperz2with_cached_index_content..wrapper_wrappers+  " "/56 6BtH~r,)rrmr8 list[Link])rpr{r8r) functoolscachewraps)rrrs` @r*with_cached_index_contentrs@__--__R r,r{c#TK|jj}|jdr^tj|j }|j dgD])}tj||j}|&|+yt|j}|jxsd}|j|j j||j}|jxs|}|jD]!} tj | ||}||#yw)z\ Parse a Simple API's Index Content, and yield its anchor elements as Link objects. rBfilesNzutf-8)page_urlbase_url)r3r%r&jsonloadscontentrDr from_jsonr(HTMLLinkParserencodingfeeddecoderanchors from_element) rprHdatafilelinkparserrr(ranchors r* parse_linksrs &&,,.N  !FGzz$,,'HHWb) D>>$1D|J    DHH %F}}'H KK ##H-. ((C%#H..  #I <  sD&D(T)frozencPeZdZUdZded<ded<ded<ded<d Zd ed <dd Zy )r{aRepresents one response (or page), along with its URL. :param encoding: the encoding to decode the given content. :param url: the URL from which the HTML was downloaded. :param cache_link_parsing: whether links parsed from this page's url should be cached. PyPI index urls should have this set to False, for example. bytesrr7r3 str | Nonerr(Tr}roc,t|jSr0)rr(rys r*__str__zIndexContent.__str__ s#DHH--r,N)r8r7)r:r;r<__doc____annotations__rorrLr,r*r{r{s/N H##.r,c4eZdZdZdfd ZddZddZxZS)rzf HTMLParser that keeps the first base HREF and a list of all anchor elements' attributes. cPt|d||_d|_g|_y)NT)convert_charrefs)r1r2r(rr)r5r(r6s r*r2zHTMLLinkParser.__init__s( $/$( 46 r,c|dk(r(|j|j|}|||_yy|dk(r%|jjt |yy)Nbasea)rget_hrefrappenddict)r5tagattrshrefs r*handle_starttagzHTMLLinkParser.handle_starttagsT &=T]]2=='D $  CZ LL  U ,r,c*|D]\}}|dk(s |cSy)NrrL)r5rnamevalues r*rzHTMLLinkParser.get_href#s&  KD%v~  r,)r(r7r8r9)rr7rlist[tuple[str, str | None]]r8r9)rrr8r)r:r;r<rr2rrr=r>s@r*rrs 7-r,rc<|tj}|d||y)Nz%Could not fetch URL %s: %s - skipping)r_r`)rreasonmeths r*_handle_get_simple_failr*s  ||| 0$?r,ct|j}t|j|jd||j|S)Nr@)rr(ro)rkrCr{rr()rGrors r*_make_index_contentr4sE*(*:*:;H ( LL-  r,)rbcv|jjddd}t|}|rtj d||y|j drtt jjt|rL|jds|dz }tjj|d}tjd| t||| }t!||j" S#t$$rtj d |Yyt&$r6}tj d ||j(|j*Yd}~yd}~wt,t.f$r}t1||Yd}~yd}~wt2t4f$r5}d |j6}t1||tj8Yd}~yd}~wt:$r#}t1|d|j6Yd}~yd}~wt<$r#}t1|d|j6Yd}~yd}~wt>$r)}t1|tA|j6Yd}~yd}~wwxYw)N#rrzICannot look at %s URL %s because it does not support lookup as web pages.zfile:/z index.htmlz# file: URL is directory, getting %srUrb)roz`Skipping page %s because it looks like an archive, and cannot be checked by a HTTP HEAD request.zSkipping page %s because the %s request got Content-Type: %s. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/htmlz4There was a problem confirming the ssl certificate: )rzconnection error: zproxy connection error: )!r(splitr+r_warningr&osrWisdirrendswithrQrRurljoinr`rcrrorKr.r4r3rrrrrcontextinforrrr7)rrUrbr( vcs_schemerZexcrs r*_get_index_contentrAs ((..a  #C#3'J W    ~~g277==S1A#B||C 3JC ll""3 5 :C@U# 3C <#4Dz'LinkCollector.create..sIs-c2Is)rrno_index)rUr) index_urlextra_index_urlsrr_r`rarrcreater)clsrUoptionssuppress_no_indexrrrlink_collectors r*rzLinkCollector.creates''(7+C+CC   $5 LL&IjII J''-2 "))!!%% '% r,c.|jjSr0)rrrys r*rzLinkCollector.find_linkss  +++r,Nc|jj}d|vxs|duxr t||v}t||j|S)z> Fetch an HTML page containing package links. z:all:Nr)rUrefresh_packager r)r5location package_namerbshould_force_revalidates r*fetch_responsezLinkCollector.fetch_responses_ <<77")-="=#  $ D!,/3CC " LL4  r,ctjfdjjDj }tjfdj Dj }t jtjrwtj||Dcgc]}||jd|j!}}t|ddg|z}t jdj|t!t#|t#|Scc}w)Nc 3hK|])}t|jjdd+yw)Fcandidates_from_pagepage_validator expand_dirro project_nameNr"rUis_secure_originrlocrrr5s r*rz0LinkCollector.collect_sources..sA 4  %9#||<< #()    4 /2c 3hK|])}t|jjdd+yw)TrNrrs r*rz0LinkCollector.collect_sources..sA 5  %9#||<<#')    5 rz* z' location(s) to search for versions of : )rr) collections OrderedDictrget_index_urls_locationsvaluesrr_ isEnabledForloggingDEBUG itertoolschainrr'r`rarr)r5rrindex_url_sourcesfind_links_sourcessliness``` r*collect_sourceszLinkCollector.collect_sourcess2 (33 4 ((AA,O 4   &( )44 5  5   &(    w}} -#);=NO=QVV%7QVVH E u:,##/.3E LL5) *./-.  s>$D<)rUrrrr8r9F)rUrrr rr}r8r)r8z list[str]r0)rrrrr8IndexContent | None)rr7rr r8r) r:r;r<rr2 classmethodrpropertyrrr rLr,r*rrs"   #(    B,,:>  ,6  (, , 1,   , r,r)r(r7r8r)rGrr8r9)r(r7rUrr8r9r )r(r7rUrrbr}r8r)rCResponseHeadersr8r)rrr8rrr0)rrrzstr | ExceptionrzCallable[..., None] | Noner8r9)T)rGrror}r8r{)rrrUrrbr}r8r )Tr __future__rr email.messagerfrrrrr urllib.parserQcollections.abcrrrr dataclassesr html.parserr optparser typingr r pip._vendor.packaging.utilsr pip._vendor.requestsrpip._vendor.requests.exceptionsrpip._internal.exceptionsrrrrrrpip._internal.models.linkr!pip._internal.models.search_scoperpip._internal.network.sessionrpip._internal.network.utilsrpip._internal.utils.filetypesrpip._internal.utils.miscrpip._internal.utils.urlsrpip._internal.vcsrsourcesr r!r" getLoggerr:r_r7rr+ Exceptionr.rIrKr[rcrkrmrrrr{rrrrrrrLr,r*r(s#  HH!" :)6+948990!AA   8 $ c*)Y)@, y *#8 8 88 8v # #BB(8 $...(Z>(,@ @ @ %@ @48  ,0  "# ? ?? ?  ?D,z, v v r,