
    -PhJ                     z   d Z ddlZddlZddlmZ ddlZ	 ddlZn# e$ r dZY nw xY w	 ddlZn# e$ r dZY nw xY wddl	m
Z
 ddlmZmZmZmZmZmZmZmZmZ ddlmZ dd	lmZmZmZmZmZmZmZmZ  e            Z  e            Z! e            Z" e            Z# e            Z$ej%        &                    edu d
          ej%        '                    de dz   e!g          d                         Z(ej%        &                    edu d          d             Z)d Z*ej%        j+        d             Z,ej%        j+        d             Z-ej%        j+        ej%        '                    dedfedfedfgg d          d                         Z.ej%        j+        ej%        '                    de!e"e$gg d          d                         Z/ej%        j+        d             Z0ej%        j+        d             Z1ej%        j+        ej%        '                    ddd g          d!                         Z2ej%        j+        d"             Z3ej%        j+        ej%        &                    edu d#          d$                         Z4ej%        j+        ej%        &                    edu d#          d%                         Z5ej%        &                    edud&          d'             Z6ej%        &                    edud(          ej%        '                    d)eeeg          d*                         Z7ej%        j+        ej%        &                    edu d
          ej%        '                    d+e efe!efgd,d-g          d.                                     Z8ej%        j+        ej%        &                    edu d
          d/                         Z9ej%        j+        ej%        &                    edu d
          ej%        &                    edu d          d0                                     Z:ej%        j+        d1             Z; G d2 d3          Z<dS )4zK
Test the downloader classes and functions separately from the Pooch core.
    N)TemporaryDirectory   )Pooch)	HTTPDownloaderFTPDownloaderSFTPDownloaderDOIDownloaderchoose_downloaderFigshareRepositoryZenodoRepositoryDataverseRepository
doi_to_url)Unzip   )pooch_test_urlcheck_large_datacheck_tiny_datadata_over_ftppooch_test_figshare_urlpooch_test_zenodo_url pooch_test_zenodo_with_slash_urlpooch_test_dataverse_urlzrequires tqdm)reasonurltiny-data.txtc                 >    t          | d          }|j        du sJ dS )DThe progressbar keyword argument must pass through choose_downloaderTprogressbarNr
   r   r   
downloaders     \/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pooch/tests/test_downloaders.pytest_progressbar_kwarg_passedr$   :   s0     #3D999J!T))))))    zrequires paramikoc                  B    d} t          | d          }|j        du sJ dS )r   /sftp://test.rebex.net/pub/example/pocketftp.pngTr   Nr    r!   s     r#   "test_progressbar_kwarg_passed_sftpr(   H   s5     <C"3D999J!T))))))r%   c                     t          j        t                    5  t          d           ddd           n# 1 swxY w Y   t          j        t                    5  t          d           ddd           dS # 1 swxY w Y   dS )z6Should raise ValueError when protocol is not supportedzhttpup://some-invalid-url.comNzdoii:XXX/XXX/file)pytestraises
ValueErrorr
    r%   r#   test_unsupported_protocolr.   P   s    	z	"	" ; ;9:::; ; ; ; ; ; ; ; ; ; ; ; ; ; ; 
z	"	" / /-.../ / / / / / / / / / / / / / / / / /s   6::A77A;>A;c                      t          j        t                    5 }  t                      ddd           ddd           n# 1 swxY w Y   dt	          | j                  v sJ dS )z/Should fail if data repository is not supportedz%doi:10.21105/joss.01943/file_name.txtN)r   output_filepoochz(Invalid data repository 'joss.theoj.org')r*   r+   r,   r	   strvalueexcs    r#   test_invalid_doi_repositoryr6   Y   s     
z	"	" 
c7TQU	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 6SYGGGGGGs   AAAc                      t          j        t                    5 } t          d           ddd           n# 1 swxY w Y   dt	          | j                  v sJ dS )z#Should fail if the DOI is not foundNOTAREALDOI)doiNzIs the DOI correct?)r*   r+   r,   r   r2   r3   r4   s    r#   test_doi_url_not_foundr:   d   s     
z	"	" &c}%%%%& & & & & & & & & & & & & & & C	NN222222s   7;;zrepository,doiz10.6084/m9.figshare.14763051.v1z10.5281/zenodo.4924875z10.11588/data/TKCFEF)figsharezenodo	dataverse)idsc                    t          j        t                    5 }t          |          }|                     ||          }|                    d           ddd           n# 1 swxY w Y   dt          |j                  v sJ dS )z3Should fail if the file is not found in the archivezbla.txt	file_nameNzFile 'bla.txt' not found)r*   r+   r,   r   
initializedownload_urlr2   r3   )
repositoryr9   r5   r   repos        r#    test_figshare_url_file_not_foundrF   l   s     
z	"	" /coo$$S#..I.../ / / / / / / / / / / / / / / &SY777777s   <A""A&)A&c                     t                      5 }t                      }t          j                            |d          } || dz   |d           t          |           ddd           dS # 1 swxY w Y   dS )zTest the DOI downloaderr   N)r   r	   ospathjoinr   )r   local_storer"   outfiles       r#   test_doi_downloaderrM      s     
		 !"__
',,{O<<
3('4888   	! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   AA**A.1A.c                  L   t                      5 } t          dz   }t                      }t          j                            | d          } |||d            t                      |dd          }d |D             \  }t          |           ddd           dS # 1 swxY w Y   dS )zf
    Test the Zenodo downloader when the path contains a forward slash

    Related to issue #336
    z!santisoler/pooch-test-data-v1.zipztest-data.zipNdownload)actionr1   c                     g | ]}d |v |	S )r   r-   ).0fs     r#   
<listcomp>z>test_zenodo_downloader_with_slash_in_fname.<locals>.<listcomp>   s"    >>>!A)=)=A)=)=)=r%   )r   ZENODOURL_W_SLASHr	   rH   rI   rJ   r   r   )rK   base_urlr"   rL   fnamesfnames         r#   *test_zenodo_downloader_with_slash_in_fnamerY      s     
		 	$'JJ"__
',,{O<<
8Wd+++ 4@@@>>v>>>	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   A=BB Bc                     t           } | d|                     d                   dz   } | dd         }d| d}t                      5 }t                      }t          j                            |d          }t          j        t          |	          5   || dz   |d           ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )
zf
    Test if passing a Figshare url without a version warns about it, but still
    downloads it.
    N./   zThe Figshare DOI 'z#' doesn't specify which version of r   match)
FIGSHAREURLrindexr   r	   rH   rI   rJ   r*   warnsUserWarning)r   r9   warning_msgrK   r"   rL   s         r#   !test_figshare_unspecified_versionrf      sS    C


3
 3
&C
ad)COsOOOK			 ="__
',,{O<<\+[999 	= 	=Js_,gt<<<	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	== = = = = = = = = = = = = = = = = =s7   A
CB-!C-B1	1C4B1	5CC	C	zversion, missing, present)r   7LC08_L2SP_218074_20190114_20200829_02_T1-cropped.tar.gzcropped-before.tar.gz)r   rh   rg   c                 v    d|  }d| d}t          ||          }d |j        D             }||v sJ ||vsJ dS )zG
    Test if setting the version in Figshare DOI works as expected
    z10.6084/m9.figshare.21665630.vhttps://doi.org/r\   c                     g | ]
}|d          S )namer-   )rR   items     r#   rT   z:test_figshare_data_repository_versions.<locals>.<listcomp>   s    @@@$f@@@r%   N)r   api_response)versionmissingpresentr9   r   r;   	filenamess          r#   &test_figshare_data_repository_versionsrs      sn    , 57
4
4C
#S
#
#
#C!#s++H@@(*?@@@Ii)######r%   c                 F   t          | d          5 }t                      5 }t          | j                  }t          j                            |d          } |||d           t          |           ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zTest ftp downloaderr   )portN)r   r   r   server_portrH   rI   rJ   r   )	ftpserverr   rK   r"   rL   s        r#   test_ftp_downloaderrx      s    
y/	2	2 %c!! 	%[&I,ABBBJgll;@@GJsGT***G$$$		% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%% % % % % % % % % % % % % % % % % %s5   BAA>2B>B	BB	BBBzrequires paramiko to run SFTPc                     t                      5 } t          dd          }d}t          j                            | d          } |||d           t          j                            |          sJ 	 ddd           dS # 1 swxY w Y   dS )zTest sftp downloaderdemopasswordusernamer{   r'   pocketftp.pngN)r   r   rH   rI   rJ   exists)rK   r"   r   rL   s       r#   test_sftp_downloaderr      s     
		 '#VjIII
?',,{O<<
3&&&w~~g&&&&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's   A"A??BBc                     t                      5 } t          dd          }d}t          j                            | d          }t          |d          5 }t          j        t                    5   |||d           ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zFDownloader should fail when a file object rather than string is passedrz   r{   r|   r'   r~   wbN)	r   r   rH   rI   rJ   openr*   r+   	TypeError)rK   r"   r   rL   outfile_objs        r#   (test_sftp_downloader_fail_if_file_objectr      sx    
		 3#VjIII
?',,{O<<'4   	3Ky)) 3 3
3T2223 3 3 3 3 3 3 3 3 3 3 3 3 3 3	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3	3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3sZ   AB6B-B;BBBBBB6B"	"B6%B"	&B66B:=B:zparamiko must be missingc                      t          j        t                    5 } t                       ddd           n# 1 swxY w Y   dt	          | j                  v sJ dS )z+test must fail if paramiko is not installedNz
'paramiko')r*   r+   r,   r   r2   r3   r4   s    r#   -test_sftp_downloader_fail_if_paramiko_missingr      s     
z	"	" c              3sy>>))))))s   599ztqdm must be missingr"   c                     t          j        t                    5 } | d           ddd           n# 1 swxY w Y   dt          |j                  v sJ dS )zFMake sure an error is raised if trying to use progressbar without tqdmTr   Nz'tqdm')r*   r+   r,   r2   r3   )r"   r5   s     r#   !test_downloader_progressbar_failsr      s     
z	"	" %c
t$$$$% % % % % % % % % % % % % % %s39~~%%%%%%s   377zurl,downloaderhttpr;   c                     |d          }t                      5 }d}| |z   } t          j                            ||          } || |d           |                                }|j                            d          d                                         }t          |          dk    sJ t          j
        dk    rd	}	nd
}	|dd         |	k    sJ t          |           ddd           dS # 1 swxY w Y   dS )z@Setup a downloader function that prints a progress bar for fetchTr   r   Nr^   O   win32100%|####################A   100%|████████████████████   )r   rH   rI   rJ   
readouterrerrsplitstriplensysplatformr   )
r   r"   capsysrO   rK   rX   rL   capturedprintedprogresss
             r#   test_downloader_progressbarr   	  sH    zd+++H			 !Ek',,{E22gt$$$$$&&,$$T**2.44667||r!!!!<7""2HHZHss|x''''   !! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   CC,,C03C0c                 H   t          |d          5 }t          d|j                  }t                      5 }t          j                            |d          } |||d           |                                 }|j        	                    d          d         
                                }t          |          dk    sJ t          j        dk    rd	}nd
}|dd         |k    sJ t          |           ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zESetup an FTP downloader function that prints a progress bar for fetchr   T)r   ru   Nr   r^   r   r   r   r   r   )r   r   rv   r   rH   rI   rJ   r   r   r   r   r   r   r   r   )	r   rw   r   rO   rK   rL   r   r   r   s	            r#   test_downloader_progressbar_ftpr   &  s    
y/	2	2 %c T	8MNNN!! 	%[gll;@@GHS'4((( ((**Hl((..r288::Gw<<2%%%%|w&&6^ 3B3<8++++G$$$!	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%% % % % % % % % % % % % % % % % % %s5   %DB=C?3D?D	DD	DDDc                    t          ddd          }t                      5 }d}t          j                            |d          } |||d           |                                 }|j                            d          d	                                         }t          |          d
k    sJ t          j        dk    rd}nd}|dd         |k    sJ t          j                            |          sJ 	 ddd           dS # 1 swxY w Y   dS )zFSetup an SFTP downloader function that prints a progress bar for fetchTrz   r{   )r   r}   r{   r'   r~   Nr   r^   r   r   r   r   r   )r   r   rH   rI   rJ   r   r   r   r   r   r   r   r   )r   r"   rK   r   rL   r   r   r   s           r#    test_downloader_progressbar_sftpr   ?  sP   
  D6JWWWJ			 '?',,{O<<
3&&&$$&&,$$T**2.44667||r!!!!<7""2HHZHss|x''''w~~g&&&&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's   CD  DDc                     G d d          } |d          }t          |          }t                      5 }d}t          |z   }t          j                            |d          } |||d           |                                 }|j                            d          d         	                                }	d	}
|	|
k    sJ t          |           ddd           dS # 1 swxY w Y   dS )
zASetup a downloader function with an arbitrary progress bar class.c                   F    e Zd ZdZd Zd Zd Zd Zd Ze	d             Z
dS )	Etest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplayz&A minimalist replacement for tqdm.tqdmc                 "    d| _         || _        d S )Nr   )counttotal)selfr   s     r#   __init__zNtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.__init__^  s    DJDJJJr%   c                 Z    t          | j                  dz   t          | j                  z   S )zrepresent current completionr\   )r2   r   r   r   s    r#   __repr__zNtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.__repr__b  s"    tz??S(3tz??::r%   c                 D    t          d|  t          j        d           dS )zprint self.__repr__ to stderrr    )fileendNprintr   stderrr   s    r#   renderzLtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.renderf  s%    +t++CJB777777r%   c                 <    || _         |                                  dS )zmodify completion and renderN)r   r   )r   is     r#   updatezLtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.updatej  s    DJKKMMMMMr%   c                     d| _         dS )zset counter to 0r   N)r   r   s    r#   resetzKtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.reseto  s    DJJJr%   c                  <    t          dt          j                   dS )zprint a new empty liner   )r   Nr   r-   r%   r#   closezKtest_downloader_arbitrary_progressbar.<locals>.MinimalProgressDisplay.closes  s     "3:&&&&&&r%   N)__name__
__module____qualname____doc__r   r   r   r   r   staticmethodr   r-   r%   r#   MinimalProgressDisplayr   [  s        44	 	 		; 	; 	;	8 	8 	8	 	 	
	 	 	 
	' 	' 
	' 	' 	'r%   r   N)r   r   zlarge-data.txtr   r^   z336/336)r   r   BASEURLrH   rI   rJ   r   r   r   r   r   )r   r   pbarrO   rK   rX   r   rL   r   r   r   s              r#   %test_downloader_arbitrary_progressbarr   W  sQ   ' ' ' ' ' ' ' ': "!---D$///H			 " o',,{,<==gt$$$$$&&,$$T**2.4466("""" 	!!!" " " " " " " " " " " " " " " " " "s   BCC"%C"c                      e Zd ZdZdZde Zde ZdZdZdZ	ddeeed	ed
e	 deidgdZ
ddeeed	ee	deidgdZddeeed	ee	deidd	ed
e	 deidgdZej                            dde
fdefdefg          d             Zej                            dde
fdefg          d             Zej                            de
eg          d             ZdS )TestZenodoAPISupportz0
    Test support for different Zenodo APIs
    i@ z10.0001/zenodo.rj   zmy-file.zipzMhttps://zenodo.org/api/files/513d7033-93a2-4eeb-821c-2fb0bbab0012/my-file.zip 2942bfabb3d05332b66eb128e0842cffz 2021-20-19T08:00:00.000000+00:00z$513d7033-93a2-4eeb-821c-2fb0bbab0012md5:r   )idkeychecksumlinks)createdmodifiedr   r9   doi_urlfiles)r   filenamer   r   zapi_version, api_responselegacynewinvalidc                 t   |                     d| j                                       |           t          | j        | j                  }|                    d          |_        |dk    r|j        |k    sJ dS d}t          j
        t          |          5  |j        }ddd           dS # 1 swxY w Y   dS )z@
        Test if the API version is correctly detected.
        /zenodo.r9   archive_urlr   r   z0Couldn't determine the version of the Zenodo APIr_   N)expect_request
article_idrespond_with_jsonr   r9   r   url_forbase_api_urlapi_versionr*   r+   r,   )r   
httpserverr   rn   r"   msgs         r#   test_api_versionz%TestZenodoAPISupport.test_api_version  s    	!!">T_">">??QQ	
 	
 	
 &$(MMM
","4"4R"8"8
)##)[888888DCz555 5 5(45 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5s   B--B14B1c                 f   |                     d| j                                       |           t          | j        | j                  }|                    d          |_        |                    | j	                  }|dk    r|| j
        k    sJ d	S d| j         d| j	         d}||k    sJ d	S )
zK
        Test if the download url is correct for each API version.
        r   r   r   r@   r   zhttps://zenodo.org/records/z/files/z?download=1N)r   r   r   r   r9   r   r   r   rC   rA   file_url)r   r   r   rn   r"   rC   expected_urls          r#   test_download_urlz&TestZenodoAPISupport.test_download_url  s     	!!">T_">">??QQ	
 	
 	
 &$(MMM
","4"4R"8"8
!...HH(""4=000000G?G G+/>G G G   <//////r%   rn   c                 T   |                     d| j                                       |           t          d|          }t	          | j        | j                  }|                    d          |_        |	                    |           |j
        | j        d| j         ik    sJ dS )zX
        Test if population of registry is correctly done for each API version.
        r   r   )rV   rI   r   r   N)r   r   r   r   r   r9   r   r   r   populate_registryregistryrA   file_checksum)r   r   tmp_pathrn   puppyr"   s         r#   test_populate_registryz+TestZenodoAPISupport.test_populate_registry  s     	!!">T_">">??QQ	
 	
 	
 r111%$(MMM
","4"4R"8"8
$$U+++~$.2M9K2M2M!NNNNNNNr%   N)r   r   r   r   r   r9   r   rA   r   r   legacy_api_responsenew_api_responseinvalid_api_responser*   markparametrizer   r   r   r-   r%   r#   r   r     s#         J
(J
(
(C&&&GIW  7M 66 = 2=22H	 	
 & 66 =%)H	 	
 & 66 =%)H	  = 2=22H	 
 4 [#*+$%,-	
 5 5 5( [#
'	(52B*CD 0 0	 0. [	./ O O	 O O Or%   r   )=r   rH   r   tempfiler   r*   tqdmImportErrorparamikor   r   downloadersr   r   r   r	   r
   r   r   r   r   
processorsr   utilsr   r   r   r   r   r   r   r   r   ra   	ZENODOURLrU   DATAVERSEURLr   skipifr   r$   r(   r.   networkr6   r:   rF   rM   rY   rf   rs   rx   r   r   r   r   r   r   r   r   r   r-   r%   r#   <module>r      s"    
			 



 ' ' ' ' ' ' KKKK   DDDOOOO   HHH      
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 .

%%''!!##	4466 '')) DDL99	/! * *  :9* H$-@AA* * BA*/ / / H H H 3 3 3 	>?	34	45
 	,++   8 8  8 	)\*+++   
! !  !   & = = =$ 	

	
 $ $   $ % % % H$-LMM' ' NM ' H$-LMM3 3 NM 3 HD(1KLL* * ML* D$-CDD~'VWW& & XW ED& DDL99~m <=	   
! !  :9 !, DDL99% % :9 %. DDL99H$-@AA' ' BA :9 '* /" /" /"dWO WO WO WO WO WO WO WO WO WOs    %%. 88