-PhWtdZddlZddlZddlZddlZddlmZ ddlmZn #e$rdZYnwxYw ddl Z n #e$rdZ YnwxYwdZ ddZ Gd d Z Gd d Z Gd dZGddZdZdZGddZGddeZGddeZGddeZdS)z1 The classes that actually handle the downloads. N parse_url)tqdmFc  tttttd}t |}|d|vr1t d|dd|d|d||d|}|S)a Choose the appropriate downloader for the given URL based on the protocol. Parameters ---------- url : str A URL (including protocol). progressbar : bool or an arbitrary progress bar object If True, will print a progress bar of the download to standard error (stderr). Requires `tqdm `__ to be installed. Alternatively, an arbitrary progress bar object can be passed. See :ref:`custom-progressbar` for details. Returns ------- downloader A downloader class, like :class:`pooch.HTTPDownloader`, :class:`pooch.FTPDownloader`, or :class: `pooch.SFTPDownloader`. Examples -------- >>> downloader = choose_downloader("http://something.com") >>> print(downloader.__class__.__name__) HTTPDownloader >>> downloader = choose_downloader("https://something.com") >>> print(downloader.__class__.__name__) HTTPDownloader >>> downloader = choose_downloader("ftp://something.com") >>> print(downloader.__class__.__name__) FTPDownloader >>> downloader = choose_downloader("doi:DOI/filename.csv") >>> print(downloader.__class__.__name__) DOIDownloader )ftphttpshttpsftpdoiprotocolzUnrecognized URL protocol 'z' in 'z'. Must be one of .) progressbar) FTPDownloaderHTTPDownloaderSFTPDownloader DOIDownloaderr ValueErrorkeys)urlrknown_downloaders parsed_url downloaders Q/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pooch/downloaders.pychoose_downloaderr#sL 3J*%666 :*Z*@ : : : :/4466 : : :   ;":j#9:{SSSJ c$eZdZdZddZ ddZdS) ra Download manager for fetching files over HTTP/HTTPS. When called, downloads the given file URL into the specified local file. Uses the :mod:`requests` library to manage downloads. Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to customize the download of files (for example, to use authentication or print a progress bar). Parameters ---------- progressbar : bool or an arbitrary progress bar object If True, will print a progress bar of the download to standard error (stderr). Requires `tqdm `__ to be installed. Alternatively, an arbitrary progress bar object can be passed. See :ref:`custom-progressbar` for details. chunk_size : int Files are streamed *chunk_size* bytes at a time instead of loading everything into memory at one. Usually doesn't need to be changed. **kwargs All keyword arguments given when creating an instance of this class will be passed to :func:`requests.get`. Examples -------- Download one of the data files from the Pooch repository: >>> import os >>> from pooch import __version__, check_version >>> url = "https://github.com/fatiando/pooch/raw/{}/data/tiny-data.txt" >>> url = url.format(check_version(__version__, fallback="main")) >>> downloader = HTTPDownloader() >>> # Not using with Pooch.fetch so no need to pass an instance of Pooch >>> downloader(url=url, output_file="tiny-data.txt", pooch=None) >>> os.path.exists("tiny-data.txt") True >>> with open("tiny-data.txt") as f: ... print(f.read().strip()) # A tiny data file for test purposes only 1 2 3 4 5 6 >>> os.remove("tiny-data.txt") Authentication can be handled by passing a user name and password to :func:`requests.get`. All arguments provided when creating an instance of the class are forwarded to :func:`requests.get`. We'll use ``auth=(username, password)`` to use basic HTTPS authentication. The https://httpbin.org website allows us to make a fake a login request using whatever username and password we provide to it: >>> user = "doggo" >>> password = "goodboy" >>> # httpbin will ask for the user and password we provide in the URL >>> url = f"https://httpbin.org/basic-auth/{user}/{password}" >>> # Trying without the login credentials causes an error >>> downloader = HTTPDownloader() >>> try: ... downloader(url=url, output_file="tiny-data.txt", pooch=None) ... except Exception: ... print("There was an error!") There was an error! >>> # Pass in the credentials to HTTPDownloader >>> downloader = HTTPDownloader(auth=(user, password)) >>> downloader(url=url, output_file="tiny-data.txt", pooch=None) >>> with open("tiny-data.txt") as f: ... for line in f: ... print(line.rstrip()) { "authenticated": true, "user": "doggo" } >>> os.remove("tiny-data.txt") Fc r||_||_||_|jdurtt ddSdSNT2Missing package 'tqdm' required for progress bars.)kwargsr chunk_sizerrselfrr$r#s r__init__zHTTPDownloader.__init__sE &$  t # # QRR R $ # rcddl}|rR|jdt}|||d}t |jdk}|S|j} | dt}| ddt|d } | rt|d } |j|fd|i| }| | |j } t|jd d} |jdur3t t$jd k} t)| d | ddd}n|jr|j}| |_| D]N}|rJ||||jr||jO|jr=||| || r|n#| r|wwxYwdS)a Download the given URL over HTTP to the given output file. Uses :func:`requests.get`. Parameters ---------- url : str The URL to the file you want to download. output_file : str or file-like object Path (and file name) to which the file will be downloaded. pooch : :class:`~pooch.Pooch` The instance of :class:`~pooch.Pooch` that is calling this method. check_only : bool If True, will only check if a file exists on the server and **without downloading the file**. Will return ``True`` if the file exists and ``False`` otherwise. Returns ------- availability : bool or None If ``check_only==True``, returns a boolean indicating if the file is available on the server. Otherwise, returns ``None``. rNtimeoutT)r)allow_redirectsstreamwritew+b)r$zcontent-lengthwin32OBtotalncolsasciiunit unit_scaleleave)requestsr#getDEFAULT_TIMEOUTheadbool status_codecopypop setdefaulthasattropenraise_for_status iter_contentr$intheadersrsysplatformrr3r-flushupdateresetclose)r&r output_filepooch check_onlyr9r)response availabler#ispathcontentr3 use_asciiprogresschunks r__call__zHTTPDownloader.__call__s}:   kooiAAG}}S'4}PPHX1S899I !!##**Y88(D)))['222  3{E22K( $#x|CCCCFCCH  % % ' ' '++t+GGG(,,-=qAABBE4''!!899 ## ! '+!&  9 99%%e,,,%%''''9!888  !   &&&    $!!### $!!#### $ts E H++INFrF__name__ __module__ __qualname____doc__r'rXrrrrZsXJJXSSSS38VVVVVVrrc0eZdZdZ d dZd d ZdS) ra Download manager for fetching files over FTP. When called, downloads the given file URL into the specified local file. Uses the :mod:`ftplib` module to manage downloads. Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to customize the download of files (for example, to use authentication or print a progress bar). Parameters ---------- port : int Port used for the FTP connection. username : str User name used to login to the server. Only needed if the server requires authentication (i.e., no anonymous FTP). password : str Password used to login to the server. Only needed if the server requires authentication (i.e., no anonymous FTP). Use the empty string to indicate no password is required. account : str Some servers also require an "account" name for authentication. timeout : int Timeout in seconds for ftp socket operations, use None to mean no timeout. progressbar : bool If True, will print a progress bar of the download to standard error (stderr). Requires `tqdm `__ to be installed. **Custom progress bars are not yet supported.** chunk_size : int Files are streamed *chunk_size* bytes at a time instead of loading everything into memory at one. Usually doesn't need to be changed.  anonymousNFrc||_||_||_||_||_||_||_|jdurttddSdSr!) portusernamepasswordaccountr)rr$rr)r&rfrgrhrir)rr$s rr'zFTPDownloader.__init__,sc       &$  t # # QRR R $ # rct|}tj|j}||d|j|rt j|d\}} | |j |j |j || |v} |n#|wxYw| Std } | rt!d | |j |j |j d|d} |jr|d t't(jd k} t-t/||dd | d d d 5fd} || | |jdddn #1swxYwYn"|| j|j|| rn0#|| rwwxYwdS)aZ Download the given URL over FTP to the given output file. Parameters ---------- url : str The URL to the file you want to download. output_file : str or file-like object Path (and file name) to which the file will be downloaded. pooch : :class:`~pooch.Pooch` The instance of :class:`~pooch.Pooch` that is calling this method. check_only : bool If True, will only check if a file exists on the server and **without downloading the file**. Will return ``True`` if the file exists and ``False`` otherwise. Returns ------- availability : bool or None If ``check_only==True``, returns a boolean indicating if the file is available on the server. Otherwise, returns ``None``. r)netloc)hostrfpath)userpasswdacctr-r.zRETR zTYPE Ir/r0r1Tr2cvt||dSz+Update the progress bar and write to outputN)rKlenr-)datarNrVs rcallbackz(FTPDownloader.__call__..callback|s5 D 222#))$/////r) blocksizeN)rftplibFTPr)connectrfosrnsplitloginrgrhrinlstrMrBrCrvoidcmdr=rHrIrrFsize retrbinaryr$r-quit)r&rrNrOrPrr directory file_namerRrScommandrUrvrVs ` @rrXzFTPDownloader.__call__@s0s^^ j... H-DI >>>  #%7==F1C#D#D Iy  t}T] VVV%))<)<<    ['222  3{E22K $ II4=T\I R R R2j022G V H%%% !899 chhz&'9::;;## QQ000000 NN7HNPPPQQQQQQQQQQQQQQQw (9T_UUU HHJJJ $!!### HHJJJ $!!#### $ts=9>C C" B(H<3$G# H<#G''H<*G'+&H<<-I))rbrcrdrdNFrrZr[r`rrrrsh""LSSSS(HHHHHHrrc,eZdZdZ d dZdZdS) ra Download manager for fetching files over SFTP. When called, downloads the given file URL into the specified local file. Requires `paramiko `__ to be installed. Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to customize the download of files (for example, to use authentication or print a progress bar). Parameters ---------- port : int Port used for the SFTP connection. username : str User name used to login to the server. Only needed if the server requires authentication (i.e., no anonymous SFTP). password : str Password used to login to the server. Only needed if the server requires authentication (i.e., no anonymous SFTP). Use the empty string to indicate no password is required. timeout : int Timeout in seconds for sftp socket operations, use None to mean no timeout. progressbar : bool or an arbitrary progress bar object If True, will print a progress bar of the download to standard error (stderr). Requires `tqdm `__ to be installed. rcrdNFc$||_||_||_||_||_||_g}|jrt |dt|d|r"td |dS)Nr"z7Missing package 'paramiko' required for SFTP downloads. ) rfrgrhrir)rrappendparamikorjoin)r&rfrgrhrir)rerrorss rr'zSFTPDownloader.__init__s       &   P MMN O O O   MMS T T T  /SXXf--.. . / /rcr t|}tj|d|jf}d} ||j|jtj|}|j | _ |j r_t||dj}t!t"jdk}t'|d|dd d  |j r> 5 fd } ||d|| dddn #1swxYwYn||d||||dSdS#|||wwxYw) aQ Download the given URL over SFTP to the given output file. The output file must be given as a string (file name/path) and not an open file object! Otherwise, paramiko cannot save to that file. Parameters ---------- url : str The URL to the file you want to download. output_file : str Path (and file name) to which the file will be downloaded. **Cannot be a file object**. pooch : :class:`~pooch.Pooch` The instance of :class:`~pooch.Pooch` that is calling this method. rl)sockN)rgrhrnr/r0r1Tr2ct|_t|jz dSrs)rFr3rKn)currentr3rVs rrvz)SFTPDownloader.__call__..callbacks7),U Ghj,@(A(ABBBBBr)rv)rr Transportrfrzrgrh SFTPClientfrom_transportr) get_channel settimeoutrrFstatst_sizer=rHrIrr:rM) r&rrNrOr connectionr rrUrvrVs @rrXzSFTPDownloader.__call__s"s^^ 'j.BDI-NOOO        N N N&55jAAD,0LD     ) 499Z%788@AA !899 ##  :QQCCCCC HHZ/xHPPPQQQQQQQQQQQQQQQF+[999                s0C F $D3' F 3D77F :D7; F -F6)rrcrdrdNFr[r`rrrrsVD////411111rrc eZdZdZddZdZdS)ra< Download manager for fetching files from Digital Object Identifiers (DOIs). Open-access data repositories often issue Digital Object Identifiers (DOIs) for data which provide a stable link and citation point. The trick is finding out the download URL for a file given the DOI. When called, this downloader uses the repository's public API to find out the download URL from the DOI and file name. It then uses :class:`pooch.HTTPDownloader` to download the URL into the specified local file. Allowing "URL"s to be specified with the DOI instead of the actual HTTP download link. Uses the :mod:`requests` library to manage downloads and interact with the APIs. The **format of the "URL"** is: ``doi:{DOI}/{file name}``. Notice that there are no ``//`` like in HTTP/FTP and you must specify a file name after the DOI (separated by a ``/``). Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to be able to download files given the DOI instead of an HTTP link. Supported repositories: * `figshare `__ * `Zenodo `__ * `Dataverse `__ instances .. attention:: DOIs from other repositories **will not work** since we need to access their particular APIs to find the download links. We welcome suggestions and contributions adding new repositories. Parameters ---------- progressbar : bool or an arbitrary progress bar object If True, will print a progress bar of the download to standard error (stderr). Requires `tqdm `__ to be installed. Alternatively, an arbitrary progress bar object can be passed. See :ref:`custom-progressbar` for details. chunk_size : int Files are streamed *chunk_size* bytes at a time instead of loading everything into memory at one. Usually doesn't need to be changed. **kwargs All keyword arguments given when creating an instance of this class will be passed to :func:`requests.get`. Examples -------- Download one of the data files from the figshare archive of Pooch test data: >>> import os >>> downloader = DOIDownloader() >>> url = "doi:10.6084/m9.figshare.14763051.v1/tiny-data.txt" >>> # Not using with Pooch.fetch so no need to pass an instance of Pooch >>> downloader(url=url, output_file="tiny-data.txt", pooch=None) >>> os.path.exists("tiny-data.txt") True >>> with open("tiny-data.txt") as f: ... print(f.read().strip()) # A tiny data file for test purposes only 1 2 3 4 5 6 >>> os.remove("tiny-data.txt") Same thing but for our Zenodo archive: >>> url = "doi:10.5281/zenodo.4924875/tiny-data.txt" >>> downloader(url=url, output_file="tiny-data.txt", pooch=None) >>> os.path.exists("tiny-data.txt") True >>> with open("tiny-data.txt") as f: ... print(f.read().strip()) # A tiny data file for test purposes only 1 2 3 4 5 6 >>> os.remove("tiny-data.txt") Frc 0||_||_||_dSN)r#rr$r%s rr'zDOIDownloader.__init__Ls &$rc t|}t|d}|d}|ddkr |dd}||}td|j|jd|j}||||dS) a7 Download the given DOI URL over HTTP to the given output file. Uses the repository's API to determine the actual HTTP download URL from the given DOI. Uses :func:`requests.get`. Parameters ---------- url : str The URL to the file you want to download. output_file : str or file-like object Path (and file name) to which the file will be downloaded. pooch : :class:`~pooch.Pooch` The instance of :class:`~pooch.Pooch` that is calling this method. rlrnr/rN)rr$r`)rdoi_to_repository download_urlrrr$r#) r&rrNrOrdata_repositoryrrrs rrXzDOIDownloader.__call__Qs(s^^ +Jx,@AAv& Q<3  !!"" I&33I>> $ (T_  HL     <e44444rNrYr[r`rrrrsDOOb%%%% "5"5"5"5"5rrcddl}|d|t}|j}d|jcxkrdkrnnt d|d|d |S) z Follow a DOI link to resolve the URL of the archive. Parameters ---------- doi : str The DOI of the archive. Returns ------- url : str The URL of the archive in the data repository. rNzhttps://doi.org/rkXzArchive with doi:z not found (see z). Is the DOI correct?)r9r:r;rr>r)r r9rQrs r doi_to_urlrvs OOO||4s44o|NNH ,C h"((((S((((( P P PS P P P    Jrc |ddkr |dd}tttg}t|}d}|D]}||||}|(t |d}t d|d|S)a] Instantiate a data repository instance from a given DOI. This function implements the chain of responsibility dispatch to the correct data repository class. Parameters ---------- doi : str The DOI of the archive. Returns ------- data_repository : DataRepository The data repository object rN) archive_urlr rlzInvalid data repository 'zy'. To request or contribute support for this repository, please open an issue at https://github.com/fatiando/pooch/issues)FigshareRepositoryZenodoRepositoryDataverseRepositoryr initializerr)r repositoriesrrrepo repositorys rrrs* 2w#~~#2#h LS//KO  ""oo'.O {++H5  O  O O O   rc0eZdZedZdZdZdS)DataRepositorycdS) Initialize the data repository if the given URL points to a corresponding repository. Initializes a data repository object. This is done as part of a chain of responsibility. If the class cannot handle the given repository URL, it returns `None`. Otherwise a `DataRepository` instance is returned. Parameters ---------- doi : str The DOI that identifies the repository archive_url : str The resolved URL for the DOI Nr`)clsr rs rrzDataRepository.initializes &trct)n Use the repository API to get the download URL for a file given the archive URL. Parameters ---------- file_name : str The name of the file in the archive that will be downloaded. Returns ------- download_url : str The HTTP URL that can be used to download the file. NotImplementedError)r&rs rrzDataRepository.download_urls "!rct) Populate the registry using the data repository's API Parameters ---------- pooch : Pooch The pooch instance that the registry will be added to. r)r&rOs rpopulate_registryz DataRepository.populate_registrys "!rN)r\r]r^ classmethodrrrr`rrrrsK[("""$ " " " " "rrcfeZdZdZdZedZedZedZ dZ dZ dS) rzhttps://zenodo.org/api/recordsc>||_||_d|_d|_dSr)rr _api_response _api_versionr&r rs rr'zZenodoRepository.__init__s'&! rcTt|}|ddkrdS|||S)rrlz zenodo.orgNrrr rparsed_archive_urls rrzZenodoRepository.initializes9('{33 h '< 7 74s3 $$$rc|jaddl}|jdd}||jd|t |_|jS)zCached API response from ZenodoNrrrrk)rr9rr|r: base_api_urlr;json)r&r9 article_ids r api_responsezZenodoRepository.api_responses|   % OOO)//44R8J!)$33z33'".""dff   !!rc|jxtd|jdDrd|_nLtd|jdDrd|_n td|jd|jd |jS) aj Version of the Zenodo API we are interacting with The versions can either be : - ``"legacy"``: corresponds to the Zenodo API that was supported until 2023-10-12 (before the migration to InvenioRDM). - ``"new"``: corresponds to the new API that went online on 2023-10-13 after the migration to InvenioRDM. The ``"new"`` API breaks backward compatibility with the ``"legacy"`` one and could probably be replaced by an updated version that restores the behaviour of the ``"legacy"`` one. Returns ------- str Nc3K|]}d|vV dS)keyNr`.0files r z/ZenodoRepository.api_version..Cs&HHT5D=HHHHHHrfileslegacyc3K|]}d|vV dS)filenameNr`rs rrz/ZenodoRepository.api_version..Es'OODZ4'OOOOOOrnewz5Couldn't determine the version of the Zenodo API for  (doi:).)rallrrrr r&s r api_versionzZenodoRepository.api_version.s(   $HHT->w-GHHHHH $,!!OOD4Eg4NOOOOO $)!! <'<z1ZenodoRepository.download_url..hsNNN4T%[$NNNrrcg|] }|d S)rr`rs r z1ZenodoRepository.download_url..jsMMM$T*%MMMrFile '' not found in data archive rrlinksr&idzhttps://zenodo.org/records/z/files/z ?download=1)rrrrr )r&rrrrs rrzZenodoRepository.download_urlNs2  x ' 'NN43DW3MNNNEEMM$2CG2LMMME E ! !888#88+/8888   x ' ' +G4V+++Z55555rrcReZdZdZedZdZedZdZ dZ dS)rc0||_||_d|_dSrrr rrs rr'zFigshareRepository.__init__&!rcTt|}|ddkrdS|||S)rrlz figshare.comNrrs rrzFigshareRepository.initializes9('{33 h '> 9 94s3 $$$rc|jd\}}|dd}|ddkrdSt|dd}|S)zi Parse version from the doi Return None if version is not available in the doi. rrrrvNr)r r|rF)r&_suffix last_partversions r_parse_version_from_doiz*FigshareRepository._parse_version_from_dois_HNN3'' 6LL%%b) Q<3  4im$$rc|jddl}|d|jtd}|d}|}|)tjd|jdtd|}nd|d |}||t}| |d |_|jS) z!Cached API response from FigshareNrz)https://api.figshare.com/v2/articles?doi=rkrzThe Figshare DOI 'zv' doesn't specify which version of the repository should be used. Figshare will point to the latest version available.z%https://api.figshare.com/v2/articles/z /versions/r) rr9r:r r;rrwarningswarn UserWarningrD)r&r9articlerrapi_urlrQs rrzFigshareRepository.api_responses)   % OOOllFDHFF'#dffQG!J2244G KKKK O*NN 7!77-477  ||G_|EEH  % % ' ' '!)!9D !!rc d|jD}||vr#td|d|jd|jd||d}|S)rc i|] }|d| S)namer`rs rrz3FigshareRepository.download_url..sBBBftBBBrrrrrr)rrrr )r&rrrs rrzFigshareRepository.download_urlstCB0ABBB E ! !ddd@PddX\X`ddd Y'7 rcL|jD]}d|d|j|d<dS)rr computed_md5rN)rrr&rOrs rrz$FigshareRepository.populate_registrysE) Q QH/Ph~6N/P/PEN8F+ , , Q QrN) r\r]r^r'rrrrrrrr`rrrrs""" %%[%2 ("("X("T. Q Q Q Q QrrceZdZdZedZedZedZej dZdZ dZ dS) rc0||_||_d|_dSrrrs rr'zDataverseRepository.__init__rrc|||}d|jcxkrdkrnndS|||}||_|S)rrrN)_get_api_responser>r)rr rrQrs rrzDataverseRepository.initializesf&((k:: (& , , , , , , , , ,4Sk** "* rcddl}t|}||dd|dd|t}|S)z Perform the actual API request This has been separated into a separate ``classmethod``, as it can be used prior and after the initialization. rNr://rlz-/api/datasets/:persistentId?persistentId=doi:rk)r9rr:r;)rr rr9parsedrQs rrz%DataverseRepository._get_api_response9sl ;''<<j! 4 4fX&6 4 4.1 4 4#    rch|j%||j|j|_|jS)z-Cached API response from a DataVerse instance)rrr rrs rrz DataverseRepository.api_responseLs:   %!%!7!7$*""D !!rc||_dS)zUpdate the cached API responseN)r)r&rQs rrz DataverseRepository.api_responseWs&rc .t|j}|j}d|dddD}||vr#t d|d|jd|jd|d d |d d ||d }|S)rc8i|]}|dd|dS)dataFilerr`rs rrz4DataverseRepository.download_url..ns9     Z ($z*:   rru latestVersionrrrrrrr rlz/api/access/datafile/r)rrrrrr )r&rr rQrrs rrz DataverseRepository.download_url]s4+,,$))++   (9'B    E ! !888#88+/8888  j! ( (fX&6 ( (Y% ( ( rc|jdddD]'}d|dd|j|dd<(dS) rrurrrrmd5rN)rrrrs rrz%DataverseRepository.populate_registry~si)..008I'R  H4x +E244 N8J/ ; < <  rN) r\r]r^r'rrrrrsetterrrr`rrrrs""" [:[$""X"&&& B     rrrZ)r_r{rHrxrutilsrr ImportErrorrr;rrrrrrrrrrrr`rrrs  DDDOOOOHHH4444njjjjjjjjZAAAAAAAAHllllllll^y5y5y5y5y5y5y5y5x8222j2"2"2"2"2"2"2"2"jV5V5V5V5V5~V5V5V5r}Q}Q}Q}Q}Q}Q}Q}Q@vvvvv.vvvvvs!++4>>