
    z-Ph?                    0"   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZ 	 d dlZn# e$ r dZY nw xY wd dlZd dlZd dlmZ d dlZd dlZd dlmZ d dlZd dlmZ d dlm Z m!Z!m"Z"m#Z#m$Z$ 	 d dl%Z&n# e$ r dZ&Y nw xY w	 d dl'm(Z) n# e$ r dZ)Y nw xY w	 d dl*m+Z, n# e$ r dZ,Y nw xY wej-        j(        Z. G d d          Z/d Z0d	 Z1d
 Z2ej3        d             Z4ej3        d             Z5 ej3        d          d             Z6ej3        d             Z( ej3        ddgddg          d             Z7ej-        j+        d             Z8d Z9ej-        j+        d             Z:ej-        j+        d             Z;ej-        j+        d             Z<ej-        j+        d             Z=ej-        j+        d             Z>ej-        j+        d             Z?ej-        j+        d             Z@d  ZAd! ZBd" ZCej-        D                    d#g d$          d%eEd&eFd'eFfd(            ZGd) ZHd* ZIej-        j+        d+             ZJej-        j+        d,             ZKej-        j+        d-             ZLd. ZMd/ ZNej-        D                    d0 ejO        d1d2          d3d4gg          ej-        D                    d5ddg          ej-        j+        d6                                     ZPej-        j+        d7             ZQej-        j+        ej-        jR        d8                         ZSd9 ZTd: ZUej-        j+        d;             ZVej-        j+        dd<            ZWej-        j+        d=             ZXej-        j%        ej-        j+        d>                         ZYej-        j+        d?             ZZej-        j+        d@             Z[ej-        j+        dA             Z\ej-        j%        ej-        j+        dB                         Z]ej-        j+        dC             Z^ej-        j+        dD             Z_ddEZ`ej-        j%        ej-        j+        dF                         Zaej-        j+        dG             Zbej-        j%        ej-        j+        dH                         Zcej-        j+        dI             Zdej-        j+        dJ             Zeej-        j+        dK             Zfej-        j+        dL             Zgej-        j+        dM             Zhej-        j+        dN             Ziej-        j%        ej-        j+        dO                         Zjej-        j+        ej-        D                    dPdQ dR g          dS                         Zkej-        j+        ej-        D                    dTddg          ej-        D                    dPdU dV g          dW                                     Zlej-        D                    dPdX dY g          dZ             Zmej-        D                    dPd[ d\ g          d]             Znd^ Zod_ Zpej-        j+        ej-        j%        d`                         Zqda Zrdb Zsdc Ztdd Zude Zvdf Zwdg Zxej-        j+        dh             Zydi ZzddjZ{dk Z|dl Z}dm Z~ej-        j+        dn             Zej-        j+        do             Zej-        j+        dp             Zej-        j+        dq             Zej-        j+        dr             Zej-        j+        ds             Zej-        j+        dt             Zej-        j+        du             Zej-        j+        dv             Zej-        j+        dw             Zdx Zdy Zdz Zd{ Zej-        D                    d|ddg          d}             Zd~ Zej-        j+        d             Zej-        j+        d             Zej-        j+        d             Zej-        j+        d             Zd Zd Zej-        j+        ej-        D                    dddg          ej-        D                    dddg          ej-        D                    dTddg          ej-        D                    dg dg dfg dg dfg dg dfg dg dfg dg dfg dg dfg dg dfg          d                                                             Zej-        j%        d             Zej3        d             Zej-        j+        ej-        jR        d                         Zej-        j+        ej-        jR        d                         Zej-        j+        ej-        jR        d                         Zej-        j+        ej-        jR        d                         Zej-        j+        d             Zej-        j+        d             Zej-        j%        d             Zej-        j+        d             Zej-        j+        d             Zej-        j+        d             Zd Zd Zd Zd Zej-        j+        d             Zej-        j+        d             Zd Zej-        j        d             Zej-        j        d             Zd Zej-        j        d             Zej-        j%        d             Zej-        j%        ej-        D                    dg d          d                         Zd Zd Zd Zej-        j%        d             Zej-        j%        d             Zej-        j%        d             Zd Zd Zd Zej-        j%        ej-        D                    dg d          d                         Zd Zej-        j+        ej-        j%        d                         Zej-        j+        ej-        j%        ej-                            ej        dk    d          d                                     Zej-        j+        ej-        j%        d                         Zej-        j+        d             Zej-        j+        ej-        j%        d                         Zd ZdÄ Zej-        j+        ej-        j%        dĄ                         Zej-        j+        ej-        j%        dń                         Zej-        j+        ej-        j%        dƄ                         Zej-        j+        ej-        j%        dǄ                         Zej-        j+        dȄ             Zej-        j+        ej-        j%        dɄ                         Zej-        j+        ej-        j%        dʄ                         Zd˄ Zej-        j%        ej-        j+        d̄                         Zej-        j+        ej-        j%        d̈́                         Zd΄ Z	 ddτZej-        j+        dЄ             Zej-        j+        ej-        j%        dф                         Zd҄ Zdӄ ZdԄ Zej-        j+        ej-        j	        dՄ                         Zdք Zej-        j%        dׄ             ZאddۄZd܄ Zd݄ Zej-        j+        dބ             Zej-        j+        d߄             Zej-        j+        d             Zej-        j+        d             Zej-        j+        ej-        j%        d                         Zej-        j+        ej-        j%        d                         Zd Zd Zd Zd Zd Zej-        j        ej-        j+        d                         Zd Zej-        j+        d             Zej-        j+        d             Zej-        j+        ej-        j%        d                         Zd Zej-        j+        d             Zej-        j+        ej-        jR        d                         ZdZej-        j+        ej-        jR        d                         Zej-        j+        d             Zej-        j(        d             Zej-        j(        d             Zej-        j(        d             Zej-        j(        d             Zej-        j(        d             Zej-        j(        d             Zej-        j(        d             Zej-        D                    dddg          d             Zej-        D                    dddg          d             Zd  Zd Zej-        D                    dd          d             Zd Zej-        j+        d             Zd Zd Z ej-        D                    dddg          d	             Zd
 Zd Zd ZdS (      N)copytreequote)is_threading_enabled)FSProtocolClassProxyHandler_configure_s3_limited_user_filesystem_uri
change_cwdc                       e Zd Zd ZddZdS )TableStreamWrapperc                     || _         d S Ntable)selfr   s     Z/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pyarrow/tests/test_dataset.py__init__zTableStreamWrapper.__init__F   s    


    Nc                 6    | j                             |          S r   )r   __arrow_c_stream__)r   requested_schemas     r   r   z%TableStreamWrapper.__arrow_c_stream__I   s    z,,-=>>>r   r   )__name__
__module____qualname__r   r    r   r   r   r   E   s7          ? ? ? ? ? ?r   r   c           	      N   dd l }dd l} |j         ddd          } |j        d          }|                    g d          }g }t	          |           D ]:}|                    ||t          |          t          |          f           ||z  };t          j	        |g d          S )	Nr   i        )days)greenblueyellowredorange)dateindexvaluecolorcolumns)
datetime	itertools	timedeltacyclerangeappendfloatnextpd	DataFrame)nr,   r-   dayintervalcolorsdatais           r   _generate_datar<   M   s    OOO
(
D!Q
'
'C!x!q)))H__IIIJJFD1XX  S!U1XXtF||4555x<&I&I&IJJJJr   c           
         t          j        t          j        dt          j                              t          j        dt          j                              t          j        dt          j                              t          j        dt          j                              g          }t           j                            | |d          }|	                                S )Nr&   r'   r(   r)   F)schemapreserve_index)
par>   fielddate32int64float64stringTablefrom_pandasreplace_schema_metadata)dfr>   r   s      r   _table_from_pandasrJ   ]   s    Y
%%
"(**%%
"*,,''
")++&&	  F H  F5 IIE((***r   c                 \   |                                  D ]}|                                5 }t          |t          j                  sJ |j        rJ |                                sJ |                                sJ |                                rJ 	 d d d            n# 1 swxY w Y   d S r   )	get_fragmentsopen
isinstancer@   
NativeFileclosedseekablereadablewritable)datasetfragmentnfs      r   +assert_dataset_fragment_convenience_methodsrW   h   s    ))++ % %]]__ 	%b"-00000y   ;;==   ;;==   {{}}$$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%% %s   A(B  B$	'B$	c                     t          j                    } ddg}t          |          D ]\  }}d                    ||          }|                     |           |                     |          5 }t          t          d                    t          t          t          t          d                              t          t          t          t          d                              |gdz  d t          d          D             g}t          j        dt          j                    fdt          j                    fdt          j                    fd	t          j                    fd
t          j        t          j                    t          j                    d          fg          }t          j        ||          }t          j                            |g          }	t)          j        |	|           d d d            n# 1 swxY w Y   | S )Nzsubdir/1/xxxzsubdir/2/yyyz{}/file{}.parquetr   c                 >    g | ]}|d z  t          |d z            dS    abstr).0js     r   
<listcomp>zmockfs.<locals>.<listcomp>   s.    AAA1q1u3q1u::..AAAr   i64f64r`   conststructr\   r>   )fs_MockFileSystem	enumerateformat
create_diropen_output_streamlistr0   mapr2   r`   r@   r>   rC   rD   rE   rg   record_batchrF   from_batchespqwrite_table)
mockfsdirectoriesr;   	directorypathoutr:   r>   batchr   s
             r   ru   ru   s   s   !!F 	K
 "+.. ' '9")))Q77)$$$&&t,, 	'U1XXSa))**SeAhh''((aAAaAAAD Y

#
%	$"(**%2928::BIKK%H%HIIJ   F OD888EH))5'22EN5#&&&%	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	'( Ms   -F G99G=	 G=	c                    	 ddl m}m} ddlm}  |            fdt                      		fd}|                     |d|            | |                    }t          j        	fd            }||fS )	Nr   )LocalFileSystemPyFileSystemr   )r   c                      fd| D             S )Nc                 T    h | ]$}                     t          |                    %S r   )normalize_pathr`   )ra   plocalfss     r   	<setcomp>z6open_logging_fs.<locals>.normalized.<locals>.<setcomp>   s-    >>>1&&s1vv..>>>r   r   )pathsr   s    r   
normalizedz#open_logging_fs.<locals>.normalized   s    >>>>>>>>r   c                                          t          |                    }                    |           | j                            |          S r   )r   r`   add_fsopen_input_file)r   rx   r   openeds     r   r   z(open_logging_fs.<locals>.open_input_file   sD    %%c$ii00

4x''---r   r   c              3      K                                     	 d V              |           k    sJ d S #             |           k    sJ w xY wr   )clear)expected_openedr   r   s    r   assert_opensz%open_logging_fs.<locals>.assert_opens   s      	EEEE:f%%O)D)DDDDDDD::f%%O)D)DDDDDDDDDs	   9 A)	
pyarrow.fsr|   r}   test_fsr   setsetattr
contextlibcontextmanager)
monkeypatchr|   r}   r   r   ri   r   r   r   r   s
          @@@r   open_logging_fsr      s    88888888%%%%%%oG? ? ? ? ? UUF. . . . . . &7III	ll7++	,	,BE E E E E E |r   module)scopec           	      b   | j         j                            d           | j         j                            d           t          d          t	          j                    }t                    fdt          ddz            D             \  }}}|                    d           t                    t          fdt          dd	z            D                       D ]i\  }}d

                    |          }|                    |          5 }t          j        t          |          |           d d d            n# 1 swxY w Y   j|                    d           |                    |j        j        j        |j        g          D ]\  }	} dj
        |	 }
d
                    |
          }|                    |
           |                    |          5 }t          j        t          |          |           d d d            n# 1 swxY w Y   |                    d           |                    |j        j        j        |j        j        j        g          D ]\  }	} dj
        |	 }
d
                    |
          }|                    |
           |                    |          5 }t          j        t          |          |           d d d            n# 1 swxY w Y   |                    d           |                    d          D ]\  }	}d
                    |	          }
d
                    |
          }|                    |
           |                    |          5 }t          j        t          |          |           d d d            n# 1 swxY w Y   |S )Npandasparquet  c                 :    g | ]}j         ||d z  z            S )   iloc)ra   r;   rI   r6   s     r   rc   z!multisourcefs.<locals>.<listcomp>   s,    KKKAbga!Q$h/KKKr   r   r   plainc                 :    g | ]}j         ||d z  z            S 
   r   )ra   r;   df_ar6   s     r   rc   z!multisourcefs.<locals>.<listcomp>   s,    PPPty1QU73PPPr   r   zplain/chunk-{}.parquetr>   zschema/{}/{}z{}/chunk.parquethivezhive/year={}/month={}
hive_colorr)   zhive_color/color={})configpyarrowrequiresr<   ri   rj   lenr0   rm   rk   rl   rn   rs   rt   rJ   groupbyr&   dt	dayofweekr)   yearmonth)requestru   df_bdf_cdf_dr;   chunkrx   ry   partfolderrI   r   r6   s              @@@r   multisourcefsr      s   N##H---N##I...			B!!F 	BAKKKKKq!QT9J9JKKKD$d gD		APPPPPU1aB=O=OPPPQQ ; ;5'..q11&&t,, 	;N-e44c:::	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; h||TY\%;TZ$HII ; ;e&&-!((00&!!!&&t,, 	;N-e44c:::	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; f||TY\%6	8J$KLL ; ;e/(/6!((00&!!!&&t,, 	;N-e44c:::	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; l###||G,, ; ;e&--d33!((00&!!!&&t,, 	;N-e44c:::	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; 	; MsH   #E		E	E	##HH	H	6#K%%K)	,K)	4#N##N'	*N'	c           
         t          j                    }t          j        dd          }t          j        d          }t          j        t          j        t          j        dt          j	                              t          j        dt          j
                              g                    |_        t          j        | |||          }|                                S )NsubdirT	recursivegroupkey)dsParquetFileFormatri   FileSelectorFileSystemFactoryOptionsDirectoryPartitioningr@   r>   rA   int32rE   partitioningFileSystemDatasetFactoryfinish)ru   rl   selectoroptionsfactorys        r   rT   rT      s    !##Fx4888H)(33G3
	HWbhjj))HUBIKK((
 	 	 G )&(FGLLG>>r   TFthreadedserial)paramsidsc                 D    | j          G fdd          } |            S )z]
    Fixture which allows dataset scanning operations to be
    run with/without threads
    c                   H    e Zd Z fdZ fdZd Zd Zd Zd Zd Z	d Z
d	S )
dataset_reader.<locals>.readerc                     | _         d S r   use_threads)r   r   s    r   r   z'dataset_reader.<locals>.reader.__init__  s    *Dr   c                 8    d|v rt          d          |d<   d S )Nr   z9Invalid use of dataset_reader, do not specify use_threads)	Exception)r   kwargsr   s     r   _patch_kwargsz,dataset_reader.<locals>.reader._patch_kwargs
  s7    &&$& & & %0F=!!!r   c                 F    |                      |            |j        di |S Nr   )r   to_tabler   rT   r   s      r   r   z'dataset_reader.<locals>.reader.to_table  s/    v&&&#7#--f---r   c                 F    |                      |            |j        di |S r   )r   
to_batchesr   s      r   r   z)dataset_reader.<locals>.reader.to_batches  /    v&&&%7%/////r   c                 F    |                      |            |j        di |S r   )r   scannerr   s      r   r   z&dataset_reader.<locals>.reader.scanner  s.    v&&&"7?,,V,,,r   c                 H    |                      |            |j        |fi |S r   )r   head)r   rT   num_rowsr   s       r   r   z#dataset_reader.<locals>.reader.head  s0    v&&&7<33F333r   c                 H    |                      |            |j        |fi |S r   )r   take)r   rT   indicesr   s       r   r   z#dataset_reader.<locals>.reader.take!  s0    v&&&7<226222r   c                 F    |                      |            |j        di |S r   )r   
count_rowsr   s      r   r   z)dataset_reader.<locals>.reader.count_rows%  r   r   N)r   r   r   r   r   r   r   r   r   r   r   r   s   r   readerr     s        	+ 	+ 	+ 	+ 	+	0 	0 	0 	0 	0	. 	. 	.	0 	0 	0	- 	- 	-	4 	4 	4	3 	3 	3	0 	0 	0 	0 	0r   r   )param)r   r   r   s     @r   dataset_readerr      sL     -K"0 "0 "0 "0 "0 "0 "0 "0 "0 "0H 688Or   c                 	    t          j        t          j        dt          j                              g          }t	          j                    ddg}d t          dd          D             } fdt          ||          D             }t	          j        d          t	          j        d	          k    }t	          j	        || |
          }t          j	        
                    || ||          }||fD ]-}t          |t          j	                  sJ t          |j        t          j                  sJ |j                            |          sJ t          |j                  t          |          k    sJ t#          |                                          }t          |||          D ](\  }	}
}|	j                            |
          sJ |	j        |k    sJ t          |	j        t          j                  sJ t          |	t          j                  sJ |	j        dgk    sJ |	j        dk    sJ t#          |	                                          }|	j        t1          |          cxk    rdk    sn J t          |d         t          j                  sJ |d         j        |k    sJ |d         j        dgk    sJ |d         j        dk    sJ *t#          |                    t	          j        d          dk                        }t1          |          dk    sJ /t	          j	        ||           }|j                            t	          j        d                    sJ t          j	        
                    ||           }|j                            t	          j        d                    sJ |                                D ]0}	|	j                            t	          j        d                    sJ 1t3          j        t6          d          5  t	          j	        ||           d d d            n# 1 swxY w Y   t3          j        t6          d          5  t	          j	        ||d           d d d            n# 1 swxY w Y   t3          j        t6          d          5  t          j	        
                    |           d d d            d S # 1 swxY w Y   d S )Nrf   subdir/1/xxx/file0.parquetsubdir/2/yyy/file1.parquetc                 >    g | ]}t          j        d           |k    S )r   r   rA   ra   xs     r   rc   z+test_filesystem_dataset.<locals>.<listcomp>3  s'    ===A"(6""a'===r   r   r[   c                 D    g | ]\  }}                     ||          S r   make_fragment)ra   rx   r   file_formatru   s      r   rc   z+test_filesystem_dataset.<locals>.<listcomp>4  s?     ; ; ;T4 **4>> ; ; ;r   leveli9  )r>   rl   
filesystemroot_partition)r>   rl   r   
partitionsr   r   filter   r>   rl   r   Tzincorrect typematch)r>   rl   r   rl   )r@   r>   rA   rC   r   r   r0   zipscalarFileSystemDataset
from_pathsrN   rl   partition_expressionequalsr   filesro   rL   rx   ParquetFileFragment
row_groupsnum_row_groupssplit_by_row_groupr   pytestraises	TypeError)ru   r>   r   r   	fragmentsr   dataset_from_fragmentsdataset_from_pathsrT   rU   	partitionrx   row_group_fragmentsr   s   `            @r   test_filesystem_datasetr  ,  s   Y
"(**%%  F &((K)+GHE==q!===J; ; ; ; ;#&uj#9#9; ; ;IXg&&")D//9N1&.   -88f[Vn 9  
 +,>? # #'2#788888'."*>?????+22>BBBBB7=!!SZZ////..0011	),Y
E)J)J 	> 	>%Hi077	BBBBB=D((((hor/CDDDDDh(>?????&1#----*a////"&x'B'B'D'D"E"E*c2E.F.FKKKK!KKKKKK1!4b6LMMMMM&q).$6666&q)4;;;;&q)8A=====..bhw6G6G16L.MMNN	9~~""""" "&  G '..ry????? "--f[V .  G '..ry?????))++ E E,33BIdOODDDDDD 
y(8	9	9	9 = =
YV<<<= = = = = = = = = = = = = = = 
y(8	9	9	9 C C
Yv$/	C 	C 	C 	CC C C C C C C C C C C C C C C 
y(8	9	9	9 G G
''	+'FFFG G G G G G G G G G G G G G G G G Gs6   P??QQ%R

RR0"SS#&S#c                    t          j        t          j        dt          j                              g          }t	          j                    }dg}t          j                            |||t          j	                              }|
                                 t          j        t                    5  |                     |           d d d            d S # 1 swxY w Y   d S )Nf1znonexistingfile.arrowr   )r@   r>   rA   rC   r   IpcFileFormatr  r  ri   r|   rL   r  r  FileNotFoundErrorr   )r   r>   r   r   rT   s        r   1test_filesystem_dataset_no_filesystem_interactionr  t  s   Y
rxzz""  F "$$K$%E "--f[%'' .  G  
(	)	) ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s   1CCCc                    t          | t          j                  sJ t          | j        t          j                  sJ t	          j        g dt	          j                              }t	          j        g dt	          j                              }|	                    |           D ]r}t          |t          j
                  sJ |                    d                              |          sJ |                    d                              |          sJ s|                    |                                           D ]?}t          |t          j                  sJ t          |j        t          j                  sJ @|                    |           }t          |t          j                  sJ t)          |          dk    sJ t          j        d          dk    }|                     d|          }|                    d	                                          }|d         ddgk    sJ |d
         ddgk    sJ t1          |d	                   ddgk    sJ t1          |d                   ddgk    sJ t          j        d          dk    }|                     d|          }|                    d	                                          }|d         g dk    sJ |d
         g dk    sJ |d	         g dk    sJ |d         g dk    sJ t          j        d          t          j        d
          t          j        d          dk    d}|                     d|          }|                    d                                          }t3          |          g dk    sJ |d         g dk    sJ |d
         g dk    sJ |d         g dk    sJ t5          |            d S )Nr   r   r   r[   r   typer   r   r   rd   T)r   r   r   re         ?r   r   xxxyyy)rg   r^   1)r   r   r   r   )r         @r   r$  )r   r   r   r   )r!  r!  r"  r"  )rd   re   new)r   r+   )
r   r   r   r   r   r   r[   r[   r   r   )
        r&  r   r          @r'        @r(  r$  r$  r%  )
FFTTFFFFTT)rN   r   Datasetr>   r@   SchemaarrayrC   rD   r   RecordBatchcolumnr  r   scan_batchesTaggedRecordBatchrU   Fragmentr   rF   r   rA   sort_by	to_pydictsortedro   rW   )	rT   r   expected_i64expected_f64rz   r   	conditionresult
projections	            r   test_datasetr9    s   grz*****gnbi00000 8OOO"(**===L8OOO"*,,???L**733 4 4%00000||A%%l33333||A%%l333333''00==?? 7 7%!566666%."+666666##G,,EeRX&&&&&u::1$I$yAAF^^G$$..00F%=QF""""%=RH$$$$&/""q!f,,,,&-  UEN2222 ))S0I$yAAF^^G$$..00F%=LLL((((%=0000000'?lll****%=8888888 xxx((C/ J
 $
CCF^^E"",,..F<<0000000%=:::::::%= ; ; ; ; ; ; ;%= 7 7 7 7 7 7 7/88888r   c                 l    |                      dd          }t          |          }|j        dk    sJ d S )N      )fragment_readaheadbatch_readahead   )r   r3   num_columns)rT   r   rz   s      r   test_scanner_optionsrA    s?      B JJGMME!!!!!!r   c                    |                     | t          j                              }t          |t          j                  sJ t          j        t          j                  5  |                     | dg           d d d            n# 1 swxY w Y   |                     | dgt          j                              }|j	        | j
        k    sJ |j        t          j
        dt          j                    fg          k    sJ t          |t          j                  sJ |                                }|                                D ]!}|j
        |j        k    sJ |j        dk    sJ "||                                                                k    sJ |j
        |j        k    sJ t%          |j                  D ]E}t          j        |g          }|                    |          |                    |          k    sJ Ft          j        t          j                  5  |                    t          j        |j        g                     d d d            n# 1 swxY w Y   |j        |                                k    sJ |                     | g dt          j                              }|                                }g d}|j        |k    sJ |                    d          }|d	                                         d
gdz  dgdz  z   k    sJ |d                                         dgdz  dgdz  z   k    sJ |d                                         dgdz  k    sJ |d                                         dgdz  k    sJ d S )N)memory_poolunknownr*   rd   )r+   rC  r   )
__filename__fragment_index__batch_index__last_in_fragmentrF  rE  r   r   r   r   rG  r   rH  T)r   r@   default_memory_poolrN   r   Scannerr  r  ArrowInvaliddataset_schemar>   projected_schemarC   r   r   r@  	to_readerread_allr0   r   r+  r   ArrowIndexErrorr   column_namesr1  	to_pylist)	rT   r   r   r   rz   r;   r   expected_namessorted_tables	            r   test_scannerrU    s<   $$R355 % 7 7Ggrz*****	r	'	' = =w<<<= = = = = = = = = = = = = = = $$Wug131G1I1I % K KG!W^3333#ry5"(**2E1F'G'GGGGGgrz*****E##%% & &|w77777 A%%%%%G%%''00222222<7333335>"" < <(A3--zz'""gll7&;&;;;;;;	r)	*	* 1 1RXu~.//0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 >W//111111$$W 7M 7M 7M 241G1I1I	 % K KG
 E= = =N////==!344L%//11	%&*	%&*	+, , , , *+5577QC!GqcAg<MNNNN(2244b@@@@,-7799dVb[HHHHHHs$   #BBB.IIIc                    t          j                    }t          j                    }t          j        |           	 |                                }t
          j                            |           }|                                }|                                |k    sJ 	 t          j        |           d S # t          j        |           w xY wr   )	r@   rI  system_memory_poolset_memory_poolbytes_allocatedr   rJ  from_datasetr   )rT   old_poolpoolallocated_beforer   _s         r   test_scanner_memory_poolr_    s     %''H  ""Dt%//11*))'22##%%(888888
8$$$$$8$$$$s   A!B4 4C
c                 ~   |                     | d          }|t          j                            g | j                  k    sJ |                     | ddg                                          }|ddgik    sJ |                     | ddgt          j        d          dk                                              }|dddgik    sJ |                     | d	dg                                          }|dt          t          d
                    dz  ik    sJ t          |                                           }|                     ddg                                          }|ddgik    sJ |                     d	dg                                          }|dt          t          d
                    ik    sJ d S )Nr   rh   r   rd   r*   r   r+   r   r[      r   )r   r@   rF   rr   r>   r2  r   rA   ro   r0   r3   rL   )rT   r   r7  rU   s       r   	test_headrc    s     !,,FRX**2gn*EEEEEE  !eW ==GGIIFeaS\!!!!  !eW(*!(; ! = ==FY[[ eaV_$$$$  $ @@JJLLFeT%((^^a/00000G))++,,H]]1ug]..88::FeaS\!!!!]]4%]11;;==FeT%((^^,,,,,,,r   c                 2   t          |                                           }ddgt          j        ddg          fD ]F}|                    |                              |          }|                    ||          |k    sJ Gt          j        t                    5  |                    |t          j        dg                     d d d            n# 1 swxY w Y   ddgt          j        ddg          fD ]D}|                    | |          |                    |                               |          k    sJ Et          j        t                    5  |                    | t          j        dg                     d d d            d S # 1 swxY w Y   d S )Nr   r[   r   r?  r   )	r3   rL   r@   r+  r   r   r  r  
IndexError)rT   r   rU   r   expecteds        r   	test_takerg  (  s    G))++,,HFBHaV,,- B B!**84499'BB""8W55AAAAA	z	"	" 5 5Hbhsmm4445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 FBHaV,,- P P""W !/!8!8!A!A!F!Fw!O!OP P P P P	z	"	" 5 5GRXrd^^4445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5s$   *CCC*FFFc                 L   t          |                                           }|                    |          dk    sJ |                    |t          j        d          dk              dk    sJ |                    |           dk    sJ |                    | t          j        d          dk              dk    sJ |                    | t          j        d          dk              dk    sJ |                    | t          j        d          d	k               d	k    sJ d S )
Nr   rd   r   r   r   r   r   r[   r   )r3   rL   r   r   rA   )rT   r   rU   s      r   test_count_rowsri  8  sI   G))++,,H$$X..!3333$$%A- % / /234 4 4 4 $$W--3333$$))Q. % 0 0345 5 5 5 $$WRXe__5I$JJaOOOO$$WRXe__q5H$IIQNNNNNNr   c                      t           j        t           j        t           j        g} | D ]=}t	          j        t                    5   |             d d d            n# 1 swxY w Y   >d S r   )r   
FileFormatrJ  Partitioningr  r  r  )classesklasss     r   test_abstract_classesro  H  s    



G
   ]9%% 	 	EGGG	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 s   AA	A	c                  d   t          j        t          j        dt          j                              t          j        dt          j                              g          } t
          j        t
          j        t
          j        fD ]B} ||           }t          |t
          j
                  sJ | ||           k    sJ |dk    sJ Ct          j        t          j        dt          j                              t          j        dt          j                              g          } t          j        |           }t          |j                  dk    sJ t          d |j        D                       sJ |                    d          }t          |t
          j                  sJ t          j        d          d	k    t          j        d          d
k    z  }|                    |          sJ t#          j        t           j                  5  |                    d           d d d            n# 1 swxY w Y   |                    d          }t          j        d          d	k    }|                    |          sJ |t          j        | d          k    sJ t          j        t          j        dt          j                              t          j        dt          j                              g          } t          j        | d          }t          |j                  dk    sJ t          d |j        D                       sJ |                    d          }t          j        d          t          j        d          k    t          j        d          t          j        d	          k    z  }|                    |          sJ |                    d          }t          j        d                                          t          j        d          t          j        d	          k    z  }|                    |          sJ dD ]M}t#          j        t           j                  5  |                    |           d d d            n# 1 swxY w Y   N|t          j        | d          k    sJ t          j        t          j        dt          j                              t          j        dt          j                              g          } t          j        |           }t          |j                  dk    sJ t          d |j        D                       sJ |                    d          }t          |t
          j                  sJ t          j        d          d	k    t          j        d          d
k    z  }|                    |          sJ t#          j        t           j                  5  |                    d           d d d            n# 1 swxY w Y   |t          j        | d          k    sJ t          j        t          j        dt          j                              t          j        dt          j        t          j                    t          j                                        g          } t          j        | dt          j        g d          i          }|j        d         J |j        d                                         g dk    sJ |t          j        | d           k    sJ t          j        t          j        t          j        dt          j                              t          j        dt          j        t          j                    t          j                                        g          dt          j        g d          i          }|j        d         J |j        d                                         g dk    sJ t          j        t          j        t9          d                    t          j        d  t9          d          D                       t          j        d!gd"z  d#gd"z  z             gg d$%          }t          j        d&t          j                    fg          }t
          j        t
          j        t
          j        fD ]}t;          j                    5 } ||          }t          j        ||d'|(           t          j         |d'|(          }	|	!                                }
|
                    |          sJ 	 d d d            n# 1 swxY w Y   t;          j                    5 }t          j        |          }t          j        ||d'|(           d }	t#          j        tD          d)*          5  t          j         |d'tG          d          (          }	d d d            n# 1 swxY w Y   |	J 	 d d d            d S # 1 swxY w Y   d S )+Nrd   re   zother objectr   r   r   c              3      K   | ]}|d u V  	d S r   r   r   s     r   	<genexpr>z$test_partitioning.<locals>.<genexpr>e  &      <<QqDy<<<<<<r   z/3/3.14/r[   gQ	@z/prefix/3/aaaz/3/nonesegment_encodingalphabetaxyz)null_fallbackc              3      K   | ]}|d u V  	d S r   r   r   s     r   rr  z$test_partitioning.<locals>.<genexpr>{  rs  r   z/alpha=0/beta=3/r   z/alpha=xyz/beta=3/)z/alpha=one/beta=2/z/alpha=one/z
/beta=two/otherc              3      K   | ]}|d u V  	d S r   r   r   s     r   rr  z$test_partitioning.<locals>.<genexpr>  rs  r   z3_3.14_prefix_3_aaa_)firstsecondthirddictionariesr      c              3   <   K   | ]}t          j                     V  d S r   randomra   r^  s     r   rr  z$test_partitioning.<locals>.<genexpr>  (      %I%I!fmoo%I%I%I%I%I%Ir   r]   r   r^   r  f2r   namesr   ipcrl   r   z,Expected Partitioning or PartitioningFactoryr   )$r@   r>   rA   rC   rD   r   r   HivePartitioningFilenamePartitioningrN   rl  r   r  allparse
Expressionr  r  r  rK  r  is_null
dictionaryint8rE   r+  rR  r   r0   tempfileTemporaryDirectorywrite_datasetrT   r   
ValueErrorint)r>   rn  r   exprrf  
shouldfailr   partitioning_schematempdir	load_backload_back_tables              r   test_partitioningr  S  s   Y


##

%%  F *B,?)+ . .uV}},88888uuV}},,,,~-----Y
"(**%%

%%  F +F33L|())Q....<<,";<<<<<<<<j))DdBM*****!!Q&28E??d+BCH;;x     	r	'	' , ,?+++, , , , , , , , , , , , , , , e$$Dx  A%H;;x     23FVTTTTTTTY
"(**%%
$$  F &vUCCCL|())Q....<<,";<<<<<<<<011D	'		bill	*	&		RYq\\	)	+  ;;x     233D	'			"	"	$	$(8(8BIaLL(H	I  ;;x     I + +
]2?++ 	+ 	+z***	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 2.vWMMMMMMMY
"(**%%

%%  F *622L|())Q....<<,";<<<<<<<<i((DdBM*****!!Q&28E??d+BCH;;x     	r	'	' , ,?+++, , , , , , , , , , , , , , , 226FSSSSSSSY
"(**%%
bgii==>>  F +eRX.J.J.J%K%KL  L $Q'///$Q'1133 8$ 8$ 8$ $ $ $ $23FNNNNNNN*
	HWbhjj))HUBM"'))RY[[AABB
 	 	
 2888899
  L $Q'///$Q'1133 8$ 8$ 8$ $ $ $ $ H
rRX%I%IuRyy%I%I%III
#sebj())+ #""  E
 )fbikk%:$;<<*B,?)+ 	1 	1(** 	1g 5!455LUG$)F F F F
750<> > >I'0022O"))%000000	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 
	$	&	& !'/0CDD
 %L	B 	B 	B 	B	]:!OQ Q Q 	O 	O
75s1vvNNNI	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O 	O     ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   H**H.1H.#RR		R		0XXX;A&e..e2	5e2	A
h%%h<h%h	h%h	h%%h),h)c           
         t          j        t          j        dt          j                              t          j        dt          j                              g          }t          j        |          t          j        |          t          j        |          t          j        |d          t          j        |d          t          j        |dd          g}|D ]0}| 	                    | 
                    |                    |k    sJ 1d S )Nrd   re   rt  ru  ry  )rv  rz  )r@   r>   rA   rC   rD   r   r   r  r  loadsdumps)pickle_moduler>   partsr   s       r   test_partitioning_picklingr    s    Y


##

%%  F
 	 ((
F##
''
 &AAA
@@@
FV5QQQE  F F""=#6#6t#<#<==EEEEEF Fr   z@flavor, expected_defined_partition, expected_undefined_partition))r  )zfoo=A/bar=ant%20bee r  r  )r   )z	A/ant beer  r  )r  )r  z
A_ant bee_)r  r^  flavorexpected_defined_partitionexpected_undefined_partitionc                 R   t          j        dt          j                    fdt          j                    fg          } t          t          |           |          }|                    t          j        d          dk    t          j        d          dk    z            |k    sJ |                    d	                    |                    
                    t          j        d          dk    t          j        d          dk    z            sJ |                    t          j        d          dk    t          j        d          dk    z  t          j        d          dk    t          j        d          dk    z  z            |k    sJ |                    t          j        d          dk    t          j        d          dk    z  t          j        d          dk    t          j        d          dk    z  z            |k    sJ | dk    rft          j        t           j        d	          5  |                    t          j        d          dk               d d d            d S # 1 swxY w Y   d S |                    t          j        d          dk              d
k    sJ d S )Nfoobarrh   zant beeA/r  zDNo partition key for foo but a key was provided subsequently for barr   )zbar=ant%20beer  )r@   r>   rE   getattrr   rl   pcrA   r  joinr  r  r  rK  )r  r  r  r  r   s        r    test_dataset_partitioning_formatr    s    )eRY[[%9E29;;;O$PQQ&72v&&.ABBBL 	RXe__	9bhuooQT>TUVV%	& 	& 	& 	&
 chh'ABBCCJJ	%C	BHUOOy$@A     	huoo*rx#/EF9,%C1GHJ	
 	
 &		& 	& 	& 	& 	huoo*rx#/EF9,%C1GHJ	
 	
 (		( 	( 	( 	( ### ]O(
 
 
 	B 	B "(5//Y">AAA	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B ""RXe__	%ADD I
 
 
 
 
 
 
s   0,I))I-0I-c                     t          j        t          j        g dg dd                    } t          j        d          }t          j        d          }|                     |dz   ||z
  |dz  |                    d          |z  d	
          }t          j        g dg dg dg dd	          }|                    |          sJ d S )Nr   r   r[   )r   r   r   r\   r]   r^   r   r   rD   )za+1zb-aza*2za/br*   r   r[   r   )r   r   )r   r      )      ?r   g      ?)r   rT   r@   r   rA   r   castr  )rT   r]   r^   r7  rf  s        r   $test_expression_arithmetic_operatorsr  ,  s    j						"B"BCCDDG
A
A1u1u1uvvi  1$	' '  F xyyyy   H =="""""""r   c                  0   d dD             \  } }}t          j        |           ddik    sJ t          j        |           t          j        |           k    sJ t          j        | |z  |z            d dD             k    sJ t          j        d          dk    }t          j        |          i k    sJ t          j        | |z            ddik    sJ t          j        d                                          }t          j        |          dd ik    sJ d S )Nc                 >    g | ]}t          j        |          |k    S r   r   ra   fs     r   rc   z'test_partition_keys.<locals>.<listcomp>>  s%    ///Arx{{a///r   abcr]   c                     i | ]}||S r   r   r  s     r   
<dictcomp>z'test_partition_keys.<locals>.<dictcomp>A  s    /D/D/D1/D/D/Dr   dr[   )r   get_partition_keys_get_partition_keysrA   r  )r]   r^   cnopenulls        r   test_partition_keysr  =  s&   /////GAq! ##Sz1111 ##r'=a'@'@@@@@ Q++/D/De/D/D/DDDDD8C==AD &&",,,, T**sCj88888C==  ""D &&3+555555r   c                  8   t          j                    } t          j        ddg          }t          j        d          }| j        t                      k    sJ |j        ddhk    sJ | j        dk    sJ |j        dk    sJ | | k    sJ | |k    sJ | |k    sJ d S )Nr]   r^   dictionary_columnsmscoerce_int96_timestamp_unitns)r   ParquetReadOptionsr  r   r  )opts1opts2opts3s      r   test_parquet_read_optionsr  K  s    !##E!c3Z@@@E!dCCCE#suu,,,,#Sz1111,4444,4444E>>>>E>>>>E>>>>>>r   c                  @   t          j                    } t          j        dh          }t          j        d          }| j        t          j                    k    sJ |j        t          j        dg          k    sJ |j        t          j        d          k    sJ d S )Nr]   r  sr  )r   r   read_optionsr  )pff1pff2pff3s      r   %test_parquet_file_format_read_optionsr  ]  s    !!DC5999DC@@@D 5 7 77777 5# O O OOOOO 5$'!) !) !) ) ) ) ) ) )r   c                  Z   t          j                    } t          j        d          }t          j        dd          }t          j        dd          }t          j        dd	
          }t          j        d          }t          j        ddd          }t          j        d|          }| j        du sJ | j        dk    sJ t                      r| j        du sJ | j        dk    sJ | j	        dk    sJ | j
        du sJ |j        du sJ |j        dk    sJ t                      r|j        du sJ |j        du sJ |j        dk    sJ t                      r|j        du sJ |j        du sJ |j        dk    sJ t                      r|j        du sJ |j        dk    sJ |j	        d	k    sJ |j
        du sJ t                      r|j        du sJ |j        |k    sJ |j        | j        k    sJ | | k    sJ | |k    sJ ||k    sJ ||k    sJ || k    sJ || k    sJ || k    sJ d S )N   buffer_sizei    T)r  use_buffered_streamF)r  
pre_bufferi@ i )thrift_string_size_limitthrift_container_size_limitpage_checksum_verificationrb  )hole_size_limitrange_size_limitlazy)r  cache_optionsi @B )r   ParquetFragmentScanOptionsr@   CacheOptionsr  r  r   r  r  r  r  r  )r  r  r  opts4opts5opts6
cache_optsopts7s           r   test_parquet_scan_optionsr  i  s   )++E)d;;;E)t5 5 5E)eNNNE)!'$*- - -E )#') ) )EdD D DJ)TTTTE$----%%%% (4'''')[8888,	9999+u4444$----%%%% (4''''$,,,,%%%% (4''''$----%%%% )5(((()V3333,6666+t3333 (4''''*,,,,%"55555E>>>>E>>>>E>>>>E>>>>E>>>>E>>>>E>>>>>>r   c                 &   t          j                    t          j                    t          j        t          j                            dd                    t          j        t          j                            ddg                    t          j        t          j                            dd	                    t          j                    t          j        t          j                            dd
                    t          j        t          j                            dd                    g}	 |	                    t          j
                               n# t          $ r Y nw xY wt          g|                    t          j                    t          j        dh          t          j        d          t          j        dddd          g           |D ]0}|                     |                     |                    |k    sJ 1d S )N	T)	delimiterignore_empty_linesr[   r  )	skip_rowsrQ  r  i   )r  
block_sizeignorenewlines_in_valuesunexpected_field_behavior)parse_optionsF   r   r  r]   r  )r  r  {   i  )r  r  r  r  )r   r  CsvFileFormatr@   csvParseOptionsReadOptionsJsonFileFormatjsonr1   OrcFileFormatImportErrorrs   extendr   r  r  )r  formatsr   s      r   test_file_format_picklingr    s:   


,,t@D - F F 	G 	G
bf&8&8ug '9 '/ '/ 	0 	0 	0
bf&8&8E '9 '+ '+ 	, 	, 	,

'..$IQ / S S	T 	T 	T 	rw':':" (; (. (. 	/ 	/ 	/G r'))****    
~ "" SE::: T::: $( ),,/	  	

 
	 
	 
	  T T""=#6#6{#C#CDDSSSSST Ts   9&E   
E-,E-c                    t          j                    t          j        t          j                            d                    t          j        t          j                            d                    t          j                    t          j        t          j                            dd	                    t          j        t          j                            dd
                    g}t          =|
                    t          j        d          t          j        d          g           |D ]0}|                     |                     |                    |k    sJ 1d S )NT)strings_can_be_nullconvert_options   r  r  Ferrorr   i   r  r  r  )r  )r   CsvFragmentScanOptionsr@   r  ConvertOptionsr
  JsonFragmentScanOptionsr  r	  rs   r  r  r  r  )r  r   options      r   #test_fragment_scan_options_picklingr    sr   
!##
!F11d1KK	M 	M 	M
!++u+==	? 	? 	?
"$$
"G  E;B ! D D	E 	E 	",,#,NN	P 	P 	PG 
~)d;;;)T:::
 	 	 	
  J J""=#6#6v#>#>??6IIIIIJ Jr   paths_or_selectorr   r   r   r   r  c                    t          j        t          j        dh          |          }t          j        d          }t          j        t          j        t          j        dt          j                              t          j        dt          j	                              g                    |_
        |j        dk    sJ |j        ddgk    sJ |j        d	u sJ t          j        | |||          }|                                }|                                                    t          j        t          j        d
t          j                              t          j        dt          j                              t          j        dt          j        t          j                    t          j	                                        t          j        dt          j                              t          j        dt          j        t          j                    t          j	                    d                    t          j        dt          j                              t          j        dt          j	                              g          d	          sJ t+          |                                t.                    sJ t+          |                    |          t           j                  sJ |j                            t          j        d                    sJ |                                }t+          |t           j                  sJ |                                }t          j        g dt          j                              }	t          j        g dt          j                              }
t
          j                            t          j        g dt          j                              t          j        d                                 t          j	                                        }t          j        d tC          d          D                       }|"                                }tG          |ddgddg          D ]o\  \  }}}}t          j        |gdz  t          j                              }t          j        |gdz  t          j	                              }t          j        |dz
  gdz  t          j                              }|j$        J |j%        dk    sJ |d                             |	          sJ |d                             |
          sJ |d                             |          sJ |d                             |          sJ |d                             |          sJ |d                             |          sJ |d                             |          sJ q|&                                }t+          |t
          j'                  sJ tQ          |          dk    sJ |j%        dk    sJ d S ) Nr`   r  )r  r  r   r   r   .r^  Frd   re   rf   rg   r\   check_metadataTr  r  z	0 1 2 3 4c                 >    g | ]}|d z  t          |d z            dS rZ   r_   ra   r;   s     r   rc   z+test_filesystem_factory.<locals>.<listcomp>  s>      3  3  3$% '(!e#a!e** = =  3  3  3r   r   r   r   r!  r"  r?  r   r[   r   r  r   ))r   r   r  r   r   r@   r>   rA   r   rE   r   partition_base_dirselector_ignore_prefixesexclude_invalid_filesr   inspectr  rC   rD   r  rg   rN   inspect_schemasro   r   r  r   r  r   r+  DictionaryArrayfrom_arrayssplitr0   r.  r  r  r@  r   rF   r   )ru   r  r  rl   r   r   inspected_schemarT   r   r4  r5  expected_strexpected_structiteratorrz   rU   r   r   expected_groupexpected_keyexpected_constr   s                         r   test_filesystem_factoryr4    s{    !*ugFFF  F
 )(33G3
	HWbhjj))HUBIKK((
 	 	 G %1111+Sz9999(E1111)!67 G ((??##BI


##

%%
bhjj")++>>??
"(**%%
2928::+-9;;&8 &8 9 9 	: 	:
"(**%%
	$$	/ 	% 	%  $ 	 	 	 	 	 g--//66666gnn%566*, , , , ,!((499999nnGgr344444ooG8OOO"(**===L8OOO"*,,???L%11
rxzz222
""$$29;;777 L h  3  3).q 3  3  3 4 4O##%%H),X1vu~)N)N - -%5#5'A+BHJJ???x		<<<519+/

CCC,888 A%%%%Qx|,,,,,Qx|,,,,,Qx|,,,,,Qx~.....Qx/////Qx~.....Qx|,,,,,,EeRX&&&&&u::!!!!!!r   c                    t          j                    }t          j        d| |          }|j        D ]}|                    ||           }|j        dgk    sJ |                    || dg          }||fD ]O}t          |t           j                  sJ |j        |k    sJ t          |j	        t          |                     sJ P|j        dgk    sJ d S )N/plainr   rl   r   r  )r   r   rT   r	  r   r  rN   r
  rx   r   r  )r   parquet_formatrT   rx   rU   row_group_fragmentr  s          r   test_make_fragmentr;  4  s   )++Njm .0 0 0G  
4 
4!//mDD"qc))))+99$FGS : J J./ 	A 	AAa!7888886T>>>>alD,?,?@@@@@@!,33333
4 
4r   c                 6   | \  }}}}}}}t          j                    |g}fd|D             }	t          j        |	|j                                                  }
|
                    |          sJ fdj        D             }fdt          ||          D             }t          j        ||j                  }                                }
|
                    |          sJ d |D             }fdt          ||          D             }t          j        ||j                  }t          j	        t          j        j        d          5  |                                }d	d	d	           n# 1 swxY w Y   d
 |D             }fdt          ||          D             }t          j        ||j                  }t          j	        t          d          5  |                                }d	d	d	           d	S # 1 swxY w Y   d	S )z
    Test passing file_size to make_fragment. Not all FS implementations make use
    of the file size (by implementing an OpenInputFile that takes a FileInfo), but
    s3 does, which is why it's used here.
    c                 <    g | ]}                     |          S r   r   )ra   rx   r   ri   s     r   rc   z0test_make_fragment_with_size.<locals>.<listcomp>T  s9     $ $ $ **444 $ $ $r   )rl   r>   r   c                 N    g | ]!}j                             |          j        "S r   )r   get_file_infosize)ra   r   rT   s     r   rc   z0test_make_fragment_with_size.<locals>.<listcomp>^  s-    RRRq'$22155:RRRr   c                 F    g | ]\  }}                     ||           S )	file_sizer   ra   rx   r@  r   ri   s      r   rc   z0test_make_fragment_with_size.<locals>.<listcomp>_  sH     E E E)tT '44T24NN E E Er   c                     g | ]}d S )r   r   ra   rx   s     r   rc   z0test_make_fragment_with_size.<locals>.<listcomp>h  s    ***Da***r   c                 F    g | ]\  }}                     ||           S rB  r   rD  s      r   rc   z0test_make_fragment_with_size.<locals>.<listcomp>i  H     I I I)tT '44T24NN I I Ir   zParquet file size is 1 bytesr   Nc                     g | ]}d S )r  r   rF  s     r   rc   z0test_make_fragment_with_size.<locals>.<listcomp>t  s    000$g000r   c                 F    g | ]\  }}                     ||           S rB  r   rD  s      r   rc   z0test_make_fragment_with_size.<locals>.<listcomp>u  rH  r   zHTTP status 416)r   r   r  r>   r   r  r	  r  r  r  r   librK  OSError)s3_example_simpler   rx   urihostport
access_key
secret_keyr   r  tbl
sizes_truefragments_with_sizedataset_with_sizesizes_toosmallsizes_toolargerT   r   ri   s                   @@@r   test_make_fragment_with_sizerY  G  sa    @Q<E4S$j*&((KFE$ $ $ $ $"$ $ $I"+elr  G 



C::e SRRRGMRRRJE E E E E-0
-C-CE E E,KQS   



C::e +*E***NI I I I I-0-G-GI I I ,KQS   
w{/7U	V	V	V - -!**,,- - - - - - - - - - - - - - - 10%000NI I I I I-0-G-GI I I ,KQS  
 
w&7	8	8	8 - -!**,,- - - - - - - - - - - - - - - - - -s$   F  FF,HHHc                    t          j        d          }t          j        |                    d                    }t          j                    }|                    |          }t          |	                                t          j
                  sJ t          j        g dg dg dgg d          }|                     |                              |          sJ |                    |                    |                    }|                     |                              |                                          sJ d S )NzT
        alpha,num,animal
        a,12,dog
        b,11,cat
        c,10,rabbit
    utf-8r]   r^   r        r   dogcatrabbitrw  numanimalr  )textwrapdedentr@   	py_bufferencoder   r  r   rN   rM   BufferReaderr   r   r  r  r  )r   r  contentbuffer
csv_formatrU   rf  pickleds           r   "test_make_csv_fragment_from_bufferrp    s9   o  	 	G \'..1122F!##J''//H hmmoor77777x%1113 988: : :H ""8,,33H=====!!-"5"5h"?"?@@G""7++2283D3D3F3FGGGGGGGr   c                 b   d}t          j        |                    d                    }t          j                    }|                    |          }t          |                                t           j                  sJ t          j	        g dg dg dgg d          }| 
                    |                              |          sJ |                    |                    |                    }| 
                    |                              |
                                          sJ d S )Nz{"alpha" : "a", "num": 12, "animal" : "dog"}
{"alpha" : "b", "num": 11, "animal" : "cat"}
{"alpha" : "c", "num": 10, "animal" : "rabbit"}
r[  r\  r]  r`  rd  r  )r@   ri  rj  r   r  r   rN   rM   rk  r   r   r  r  r  )r   r  rl  rm  json_formatrU   rf  ro  s           r   #test_make_json_fragment_from_bufferrs    s*   <G \'..1122F#%%K((00H hmmoor77777x%1113 988: : :H ""8,,33H=====!!-"5"5h"?"?@@G""7++2283D3D3F3FGGGGGGGr   c                 b   t          j        g d          t          j        g d          t          j        g d          g}|d                                         |d         |d                                         g}t          j        t          j        ddg	          d
d          }|t          j                    f||fg}|D ]\  }}t          j        |g d          }t          j                    }t          j	        ||           |
                                }	|                    |	          }
|                     |
                              |          sJ |                    |                    |
                    }|                     |                              |          sJ d S )Nr\  r]  r`  r   r   r   rw  rf  r  Tr  )r  r  r  rd  r  )r@   r+  dictionary_encoder   r   r  r   BufferOutputStreamrs   rt   getvaluer   r   r  r  r  )r   r  arraysdictionary_arraysdictionary_formatcasesformat_r   ry   rm  rU   ro  s               r   &test_make_parquet_fragment_from_bufferr}    s    	!!

)))**F 	q	##%%q	q	##%%
 ,* '2
 
 
 !   
%''(	-.E ! > >'A'A'ABBB#%%
uc"""((00&&x0077>>>>>%%m&9&9(&C&CDD&&w//66u======> >r   c                     t          j        t          d          dgdz  dgdz  dgdz  z   gg d          }t          | dz            }t	          j        ||d	g|
           t          j        |dd|          }||fS )Nr<  r   r]   r   r^   r  r  test_parquet_datasetr   )partition_cols
chunk_sizer   r   )rl   r   r   )r@   r   r0   r`   rs   write_to_datasetr   rT   )r  r  r   r   rx   rT   s         r   _create_dataset_for_fragmentsr    s    H	qA37SEAI	12"""  E
 w//00Dt(.xJH H H HjYV
  G '>r   c                    t          |           \  }}t          |                                          }t          |          dk    sJ |d         }ddg}|j        j        |k    sJ |j                            |j        |j	                  |j        k    sJ |j
                            t          j        d          dk              sJ |                    |          }|j        |k    sJ |                    |                    d                              dd                    sJ |                    ||j                  }|j        g d	k    sJ |                    |                    dd                    sJ |j        |j                            d          k    sJ |                    ||j        t          j        d          dk     
          }|j        g d	k    sJ d S )Nr   r   r  r  r   r]   r   rh   r  )r>   r   )r  ro   rL   r   physical_schemar  rl   r(  rx   r   r  r  r   rA   r   rQ  remove_columnslicer>   remove)r  r   r   rT   r  r  physical_namesr7  s           r   test_fragmentsr    s   27;;NE7 W**,,--Iy>>Q!AD\N"n44448AFAL11Q5FFFFF!((&)9)9S)@AAAAA $$Q''F.0000==,,Q//55a;;<<<<< $$Qw~$>>F"6"6"66666==Q**+++++ 4 4Q 7 77777 $$	'.$!); % = =F"6"6"6666666r   c                    t          j        t          d          dgdz  dgdz  z   gddg          }t          | dz            }t	          j        ||dg	           t          j        t          j        d
g          d          }t          j	        |d|          }|
                    t          j        d          dk              }t          t          |                    dk    sJ d S )Nr<  r   r   r   colr   r  r  r  )r   r  r   r  r   r  r   )r@   r   r0   r`   rs   r  r   r   r>   rT   rL   rA   r   ro   )r  r   rx   r   rT   r  s         r   test_fragments_implicit_castr  	  s     HeAhha1#' 125&/JJJEw//00DtVH====?29&6%788HHHDjidCCCG%%RXf-=-=-B%CCItI1$$$$$$r   c                 6  
 t          |           \  
}	 d
fd	}t          |                                          d         }|j        }|                    |                    |                    }|                    |          |                    |          k    sJ |                    |j        |j	        |j
                  }|                    |                              |                    |                    sJ  ||d           |                    |j        |j	        |j
                  } ||dt          j        d          dk                |                    |j        |j	        |j
                  } ||ddgt          j        d          d	k     
           |                    |j        |j	        |j
                  } ||dt          j        d          dk               d|j                            ddd          z   }	t!          j        t$          |	          5  |                    |j        |j	        |j
                  }|                    |t          j        d          dk               d d d            d S # 1 swxY w Y   d S )Nc                     |                      j        ||          }|r|nj        }|j        |k    sJ  j        |                     |          }|                    |          sJ d S )Nr>   r+   r   )r   r>   rQ  r  selectr  )rU   	row_slicer+   r   actualrQ  rf  r   s          r   assert_yields_projectedz;test_fragments_reconstruct.<locals>.assert_yields_projected  s    ""< # A A")Awwu/A"l22225;	*11,??}}X&&&&&&&r   r   )r  )r   r   )r   r   r  r   r   r'  ra  r   r]   z&No match for FieldRef.Name\(part\) in Fr   NN)r  ro   rL   rl   r  r  r   r   rx   r   r  r  r   rA   r  	to_stringr  r  r  )r  r   r  rT   r  rU   r9  pickled_fragmentnew_fragmentpatternr   s             @r   test_fragments_reconstructr    s\   27;;NE7 6:' ' ' ' ' ' G))++,,Q/H_N %**=+>+>x+H+HII"" +44X>>? ? ? ? "//x*%: 0 < <L ""<0077))+ + + + +L&111 "//x*%: 0 < <L L&$!9KLLLL "//x*%: 0 < <L L&%)F28D>>C3GI I I I "//x*%: 0 < <L L&#%8F#3#3s#:< < < <
 9'11%FFGG	z	1	1	1 N N%33M8.!)!> 4 @ @ 	RXf5E5E5LMMM	N N N N N N N N N N N N N N N N N Ns   ,AJJJc                     t          | d          \  }}t          |                                          d         }t          |                                          }t	          |          |j        cxk    rdk    sn J |                    |d         |j                  }|j        g dk    sJ t	          |          dk    sJ |	                    |
                    dd                    sJ |d         j        J |d         j        dk    sJ |d         j        d         j        dddddddk    sJ t          |                    t          j        d	          dk     
                    d         }t          |                    t          j        d	          dk                         }t	          |          dk    sJ |                    |d         t          j        d	          dk     
          }t	          |          dk    sJ d S )Nr   r  r   rh   r  r   minmaxr  r  r  r   )r  ro   rL   r  r   r  r   r>   rQ  r  r  r  
statisticsr   rA   )r  r   r   rT   rU   r  r7  s          r   !test_fragments_parquet_row_groupsr  T  s8   27qIIINE7G))++,,Q/H x::<<=="##x'>CCCC!CCCCCC$$Aw~ % 7 7F"6"6"66666v;;!==Q**+++++q!,888q!0A5555q!,Q/:""""? ?    
 G))$!1C)DDEEaHHx::28D>>A;MNNOO"##q(((($$Arx~~'9 % ; ;Fv;;!r   c                    t          j        dt          d          i          }t          j        || dz  d           t          j        | dz  d          }t          |                                          d         }|j	        
                    |j        |j        d	d
g          }|j        dk    sJ |                                 |j        dk    sJ t          |j                  dk    sJ d S )Nr]   r<  test.parquetr   row_group_sizer   r  r   r   r[   r8  )r@   r   r0   rs   rt   r   rT   ro   rL   rl   r   rx   r   r  ensure_complete_metadatar   r  )r  r   rT   original_fragmentrU   s        r   %test_fragments_parquet_num_row_groupsr  r  s    Hc588_%%EN5'N21EEEEj>1)DDDGW224455a8 !'55 1 <q6 6  H "a''''%%'''"a''''x"##q((((((r   c                    t          j        t          ddgddg                    }|d                             d          |d<   t	          j        t          j        |          | dz             d	d lm	}  |j	        | dz            }|
                    | |j        d          dk    
          }|j        d	         |                                k                                                                    sJ d S )Nr]   r^   r   r   )col1col2r  categoryztest_filter_dictionary.parquetr   r   )r4   r5   dictastypers   rt   r@   r   pyarrow.datasetrT   r   rA   r   	to_pandasr  )r  r   rI   r   rT   r7  s         r   ,test_fragments_parquet_row_groups_dictionaryr    s     
dc
!Q888	9	9BF"":..BvJN28B<<+K!KLLL      bj#CCDDG$$WXRXf5E5E5L$MMFGAJ&**,,,1133779999999r   c                    |\  }}t          | d|          \  }}t          |                                          d         } ||j        g          5  |                                 d d d            n# 1 swxY w Y   |j        ddgk    sJ  |g           5  |                                 d d d            n# 1 swxY w Y   t          |j        t          j	                  sJ |j
                            |j        |j        ddg          }|j        |j        k    sJ |                                 |j        d         }	|	j        dk    sJ |	j        dk    sJ |	j        J |                    |                    |                    }
 ||j        g          5  |
j        ddgk    sJ |
j        d         }	|	j        dk    sJ |	j        J 	 d d d            d S # 1 swxY w Y   d S )Nr   r  r   r   r   r8  )r  ro   rL   rx   r  r  rN   metadatars   FileMetaDatarl   r   r   idr   r  r  r  )r  r   r  ri   r   r^  rT   rU   r  	row_groupr  s              r   &test_fragments_parquet_ensure_metadatar    s   &B.A"  JAw G))++,,Q/H 
x}o	&	& , ,))+++, , , , , , , , , , , , , , ,1a&(((( 
b		 , ,))+++, , , , , , , , , , , , , , , h'99999 ?00x*1v 1  L "h&99999 ))+++'*I<1""""+++ %**=+>+>|+L+LMM	x}o	&	& 0 0*q!f4444$/2	|q    #////	0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s6   A44A8;A8B::B>B>3GG#&G#c                    |\  }}t          | |          \  }}t          |                                          d         } |g           5  |                    |                    |                    }d d d            n# 1 swxY w Y    ||j        g          5  |j        }	d d d            n# 1 swxY w Y   |	dgk    sJ d S )Nr   r   r   )r  ro   rL   r  r  rx   r  )
r  r   r  ri   r   r^  rT   rU   r  r  s
             r   )test_fragments_parquet_pickle_no_metadatar    sk    'B.w2FFFJAwG))++,,Q/H 
b		 N N(..}/B/B8/L/LMMN N N N N N N N N N N N N N N 
',-	.	. 1 1%0
1 1 1 1 1 1 1 1 1 1 1 1 1 1 1!s$   )BBBB11B58B5c                    t          j        t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j	                              t          j        g dt          j
                              t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j        d                    t          j        g dt          j        d                    t          j        g dt          j        d                    t          j        g dt          j                              t          j        g dt          j                              t          j        g dt          j        d                    t          j        g dt          j        d                    gg d	
          }t+          | dz            }t-          j        |||           |t1          j        |dd          fS )N)TNF)r   r   *   )r   g      $@      E@)r]   Nzr  r  us)r   r   l    jt )booleanr  uint8int16uint16r   uint32rC   uint64r2   doubleutf8binaryts[s]ts[ms]ts[us]rB   date64time32time64r  test_parquet_dataset_all_typesr  r   r   r  )r@   r   r+  bool_r  r  r  r  r   r  rC   r  float32rD   r  r  	timestamprB   r  r  r  r`   rs   r  r   rT   )r  r  r   rx   s       r   _create_dataset_all_typesr    s   HH((("(**55H[[["')),,H[[["(**--H[[["(**--H[[[")++..H[[["(**--H[[[")++..H[[["(**--H[[[")++..H&&&
55H&&&
55H%%%rwyy11H%%%ry{{33H[[[",s"3"344H[[[",t"4"455H[[[",t"4"455H[[[")++..H(((")++66H[[[")C..11H[[[")D//22)	
,
 
 
/- - -E^ w99::D t
;;;;"*T)&IIIIIr   c                    t          |           \  }}t          |                                          d         }dd lfd}fd}fd}j        }j        }t          |                                          }	|	d         j        J |	d         j        d         }
|
j        dk    sJ |
j	        dk    sJ |
j
        i ddd	d
dddd
dddd
dddd
dddd
dddd
dddd
dddd
dddd
dddd
dddd
dddd
dddd
d |d           |d          d
d  |d           |d          d
d! |d           |d          d
d" |d#dd$           |d#d$d%          d
 |d#dd           |d#d$d&          d
 |ddd           |ddd          d
 |dddd           |dddd          d
d'k    sJ d S )(Nr   c                 .     j         ddddd|           S N  r   r   r,   r   r,   s    r   dt_sz.test_parquet_fragment_statistics.<locals>.dt_s  s     ))$1aA>>>r   c           
      6     j         dddddd| dz            S )Nr  r   r   r   r  r  s    r   dt_msz/test_parquet_fragment_statistics.<locals>.dt_ms  s&    **4Aq!Q$GGGr   c           	      0     j         dddddd|           S r  r  r  s    r   dt_usz/test_parquet_fragment_statistics.<locals>.dt_us  s"    **4Aq!QBBBr   r[   r   r  FTr  r  r   r  r  r  r  r   r  rC   r  r2   r   r  r  r  r]   r  r     a   zr  r  r  rB   r  r   r^     )r  r  r  )r  ro   rL   r,   r&   timer  r  r   total_byte_sizer  )r  r   rT   rU   r  r  r  r&   r  r  r  r,   s              @r    test_parquet_fragment_statisticsr    s-    /w77NE7G))++,,Q/HOOO>>>>>GGGGGBBBBB=D=D x::<<==q!,888#A&1!4I""""$t++++ $5..$"%%$ 	2&&$ 	2&&	$
 	!B''$ 	2&&$ 	!B''$ 	2&&$ 	!B''$ 	T**$ 	#d++$ 	C(($ 	$t,,$ 	ab22$ 	%%((559955$  	%%((559955!$" 	$$tQ**44a3D3DEE#$$ $tQ**44a3D3DEE$q!Q--Q2??$q!Q**441a3D3DEE)$ $ $      r   c                 P   t          j        g dg dd          }t          j        || dz  d           t	          j        | dz  d          }t          |                                          d	                                         }|d
         j	        d	         j
        i k    sJ d S )N)r   r   NN)r]   r^   NNr\   r  r   r  r   r  r   r   )r@   r   rs   rt   r   rT   ro   rL   r  r  r  )r  r   rT   r  s       r   &test_parquet_fragment_statistics_nullsr  2  s    H---4J4J4JKKLLEN5'N21EEEEj>1)DDDGW**,,--a0CCEEIQ<"1%0B666666r   c                 `   t          j        g dg dd          d d         }|                    | dz  d           t          j        | dz  d	          }t          |                                          d                                         }|d         j        d         j	        i k    sJ d S )
N)r]   r^   r^   r   r   r  r\   r   r  r   enginer   r  )
r4   r5   
to_parquetr   rT   ro   rL   r  r  r  )r  rI   rT   r  s       r   'test_parquet_empty_row_group_statisticsr  =  s     
OOO)))<<	=	=bqb	ABMM'N*9M===j>1)DDDGW**,,--a0CCEEIQ<"1%0B666666r   c                 0   t          | d          \  }}t          |                                          d         }|j                            t          j        d          dk              sJ t          |                    t          j        d          dk    |j                            }t          |          dk    sJ t          |                    t          j        d          dk    |j                            }t          |          dk    sJ d S )Nr   r  r   r   r]   r   r>   r^   )
r  ro   rL   r  r  r   rA   r  r>   r   )r  r   rT   rU   r  s        r   +test_fragments_parquet_row_groups_predicater  I  s    27qIIINE7G))++,,Q/H(//0@0@C0GHHHHH ##28F+;+;s+B+2> 	$ 	; 	;< < "##q(((( ##28F+;+;s+B+2> 	$ 	; 	;< < "##q((((((r   c                 >   t          | d          \  }}t          |                                          d         }|j        }t          |                                          }|                    |                    |                    }|                    |          |                    |          k    sJ |                    |j	        |j
        |j        dg          }	|                    |	          }
|
                    |                    |d                             sJ |                    |j	        |j
        |j        dh          }	|                    |	|j        ddgt          j        d          dk     	          }
|
j        ddgk    sJ t#          |
          dk    sJ |                    |j	        |j
        |j        dh          }	t%          j        t(          d
          5  |                    |	           d d d            d S # 1 swxY w Y   d S )Nr   r  r   )r  r  r   r  r   r[   r  zreferences row group 2r   )r  ro   rL   rl   r  r  r  r   r   rx   r   r  r  r>   r   rA   rQ  r   r  r  re  )r  r   r  r   rT   rU   r9  r  r  r  r7  s              r   -test_fragments_parquet_row_groups_reconstructr  `  s    37qIIINE7G))++,,Q/H_Nx::<<== %**=+>+>x+H+HII"" +44X>>? ? ? ? "//x*%:3 0  L $$\22F==001DQ1GHHIIIII "//x*%:3 0  L $$U\D&>x~~! % % %F 4.0000v;;! "//x*%:3 0  L 
z)A	B	B	B . .---. . . . . . . . . . . . . . . . . .s   /HHHc                    |\  }}t          | d|          \  }}t          |                                          d         }|                    ddg          } |g           5  |j        dk    sJ |j        ddgk    sJ |j        d         j        J 	 d d d            n# 1 swxY w Y   |                    |          }	|	                                ddgddgdk    sJ |                    g           }|j        dk    sJ |j        g k    sJ |                    ||j	                  }	|	j
        dk    sJ |	                    |d d                   sJ d S )	Nr   r  r   r[   row_group_idsr   r  rh   )r  ro   rL   subsetr  r  r  r   r2  r>   r   r  
r  r   r   ri   r   r   rT   rU   subfragr7  s
             r   !test_fragments_parquet_subset_idsr    s    'B27q>@B B BNE7G))++,,Q/H ooQFo33G	b		 < <%****!aV++++!!$/;;;;< < < < < < < < < < < < < < < $$W--FAq!f!=!===== ooBo//G!Q&&&&####$$WW^$DDF?a==rr#######s   %1B##B'*B'c                    |\  }}t          | d|          \  }}t          |                                          d         }|                    t	          j        d          dk              } |g           5  |j        dk    sJ t          |j                  dk    sJ |j        d         j	        J 	 d d d            n# 1 swxY w Y   |
                    |          }	|	                                g dg ddk    sJ |                    t	          j        d          d	k              }|j        dk    sJ |j        g k    sJ |
                    ||j        
          }	|	j        dk    sJ |	                    |d d                   sJ |                    t	          j        d          dk    |j        
          }|j        dk    sJ d S )Nr   r  r   r  r[   r  )r   r   r   r  r   rh   r   r]   r   )r  ro   rL   r  r   rA   r  r   r  r  r   r2  r>   r   r  r  s
             r   $test_fragments_parquet_subset_filterr    s6    'B27q>@B B BNE7G))++,,Q/H oobhtnn122G	b		 < <%****7%&&!++++!!$/;;;;< < < < < < < < < < < < < < < $$W--F			!C!CCCCC oobhtnnq011G!Q&&&&####$$WW^$DDF?a==rr##### oobhv..#5gnoMMG!Q&&&&&&s   8<CCCc                    t          | d          \  }}t          |                                          d         }t          j        t
                    5  |                    t          j        d          dk    ddg           d d d            n# 1 swxY w Y   t          j        t
                    5  |                                 d d d            d S # 1 swxY w Y   d S )Nr   r  r   r  r   r  )	r  ro   rL   r  r  r  r  r   rA   )r  r^  rT   rU   s       r   %test_fragments_parquet_subset_invalidr     s[   .w1EEEJAwG))++,,Q/H 
z	"	" C C!+Aq6BBBC C C C C C C C C C C C C C C 
z	"	"                   s$   0BBB5CCCc                 l   t          j        g d          }t          j        g d          }t          j        g d          }t           j                            ||gddg          }t           j                            ||gddg          }t          j        d	|i          }t          j        || d
z  d           t          j        | d
z  d          }t          |
                                          d         }|j        dk    sJ |                    t          j        d	d          dk              }	|	j        dk    sJ |                    t          j        d	d          dk              }	|	j        dk    sJ |                    t          j        d	dd          dk              }	|	j        dk    sJ |                    t          j        d	dd          dk              }	|	j        dk    sJ t          j        t           j        d          5  |                    t          j        d	d          dk               d d d            n# 1 swxY w Y   t          j        t"          d          5  |                    t          j        d	d          dk               d d d            d S # 1 swxY w Y   d S )N)r   r   r   r[   )皙?皙?333333?皙?r   r   r[   r   f21f22r  r  r  r  zdata_struct.parquetr   r  r   r  r   r   r   zNo match for FieldRef.Nestedr   f3z)Function 'greater' has no kernel matching)r@   r+  StructArrayr+  r   rs   rt   r   rT   ro   rL   r  r  rA   r  r  rK  NotImplementedError)
r  r  r  r  r  
struct_colr   rT   rU   r  s
             r   0test_fragments_parquet_subset_with_nested_fieldsr    s    
,,,		B
('''
(
(C
(<<<
 
 C		#	#S#Juen	#	E	EB++RHT4L+IIJHeZ())EN5'$99!LLLLj#88KKKGG))++,,Q/H"a''''oobhud33a788G!Q&&&&oobhud33a788G!Q&&&&oobhudE::Q>??G!Q&&&&oobhudE::a?@@G!Q&&&& 
r.L	M	M	M 3 3--12223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 
#N
 
 
 3 3 	--12223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3s$   -I		II/-J))J-0J-c                 :   t          |                                          d         }t          |          dk    st          |          dk    sJ t          |           \  }}t	          j        |d          }t          |                                          d         }t          |          d                    |j                            t          |                              k    sJ | dz  }t          j                            ||           t	          j        |d          }t          |                                          d         }t          |          d	                    |j                            t          |                              k    sJ d S )
Nr   zb<pyarrow.dataset.ParquetFileFragment path=subdir/1/xxx/file0.parquet partition=[key=xxx, group=1]>zb<pyarrow.dataset.ParquetFileFragment path=subdir/1/xxx/file0.parquet partition=[group=1, key=xxx]>r   r  z-<pyarrow.dataset.ParquetFileFragment path={}>data.featherfeatherz/<pyarrow.dataset.FileFragment type=ipc path={}>)ro   rL   repr_create_single_filer   rT   rl   r   r   r`   r@   r  write_feather)r  rT   rU   r   rx   s        r   test_fragments_reprr    s    G))++,,Q/H 	X	(	( 	( 	X	(	( 	( 	( 	( &g..KE4ji000GG))++,,Q/HX7>>--c$ii88	: 	:	: 	: 	: 	: ^#DJUD)))ji000GG))++,,Q/HX9@@--c$ii88	: 	:	: 	: 	: 	: 	: 	:r   ro  c                     | S r   r   r   ms     r   <lambda>r        Q r   c                 R    |                     |                    |                     S r   r  r  r  s     r   r  r        QWWQWWQZZ-@-@ r   c                    t          j        dd          }t          j                    }t          j        d          }t          j                            ddg          } |||          }t          |t          j                  sJ ||_	        t          j
        | |||          }|                                }t          j        dt          j                    fdt          j                    fdt          j                    fd	t          j                    fd
t          j        t          j                    t          j                    d          fdt          j                    fdt          j                    fg          }	|                    |	          sJ t          j                                        }
t          |
t          j                  sJ d S )Nr   Tr   r   r   rd   re   r`   rf   rg   r\   )ri   r   r   r   r   r   discoverrN   PartitioningFactorypartitioning_factoryr   r(  r@   r>   rC   rD   rE   rg   r   r  r  )ru   ro  r  r  rl   r   r   r   r-  expected_schemahive_partitioning_factorys              r   test_partitioning_factoryr#    s    DAAA!##F)(33G3<<gu=MNN"7#7GG*B,BCCCCC#7G )!67 G ((i	

	
			"(**	2928::BIKK@@AAB	"(**		!  O ""?33333 " 3 < < > >/1GHHHHHHHr   infer_dictionaryc                     | S r   r   r  s     r   r  r  @  r  r   c                 R    |                     |                    |                     S r   r  r  s     r   r  r  @  r  r   c                    t          j        dd          }t          j                    }t          j        d          }t          j                            ddg|          } |||          |_        t          j        | |||          }|	                                }	|rt          j        t          j                    t          j                              }
|	                    d          j        |
k    sJ |                                                                                                }|                    d                              d          }t          j        dgd	z  d
gd	z  z                                             }|                    |          sJ |                                                    t          j        d          dk              }|                    d                              d          }|                    dd	          }|                    |          sJ d S |	                    d          j        t          j                    k    sJ d S )Nr   Tr   r   r   r$  r   r!  r   r"  r   )ri   r   r   r   r   r   r  r   r   r(  r@   r  r   rE   rA   r  r   r   combine_chunksr-  r   r+  ru  r  r  )ru   r$  ro  r  r  rl   r   r   r   inferred_schemaexpected_typer   r  rf  s                 r   $test_partitioning_factory_dictionaryr,  =  s!    DAAA!##F)(33G3<<	%+; = = =#*7+?#O#OG )!674 4G oo''O @bhjj")++>>$$U++0MAAAA  ))++::<<e$$**1--8UGaK5'A+566HHJJ}}X&&&&&   ))%E1I)JJe$$**1-->>!Q''}}X&&&&&&&$$U++0BIKK??????r   c                     | S r   r   r  s     r   r  r  b  r  r   c                 R    |                     |                    |                     S r   r  r  s     r   r  r  b  r  r   c                    t          j                    }t          j                    }t	          j        dt	          j                    fg          }t	          j        t	          j        t          d                    g|          }t	          j        dt	          j
        d          fdt	          j                    fg          }t	          j        dt	          j                    fdt	          j                    fg          }t	          j        t          |          t          |          z             }dD ]}	|                    |	           |                    |	dz             5 }
t          j                            |
|          5 }|                    |           |                                 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          j        d	d
          }t          j        d	          }t          j                            |          } | ||          |_        t          j        ||||          }|                                }||k    sJ |                                                    dt          j        d                              t	          j                              i          }|d         d                                         dk    sJ t          j                            ddgd          } | ||          |_        t          j        ||||          }t          |                                                                          }|d         j         !                    t          j        d          dk    t          j        d          dk    z            sJ t          j        |d          } | ||          |_"        t          j        ||||          }t          |                                                                          }|d         j         !                    t          j        d          dk    t          j        d          dk    z            sJ t          j                            |d          } | ||          |_        t          j        ||||          }tG          j$        t          j%        d          5  |                                }d d d            n# 1 swxY w Y   t          j        dd
          }t          j        d          }t          j&                            |          } | ||          |_        t          j        ||||          }|                                }||k    sJ |                                                    dt          j        d                              t	          j                              i          }|d         d                                         dk    sJ t          j&                            d          } | ||          |_        t          j        ||||          }t          |                                                                          }|d         j         !                    t          j        d          dk    t          j        d          dk    z            sJ t          j&        |d          |_"        t          j        ||||          }t          |                                                                          }|d         j         !                    t          j        d          dk    t          j        d          dk    z            sJ t          j&                            |d          } | ||          |_        t          j        ||||          }tG          j$        t          j%        d          5  |                                }d d d            d S # 1 swxY w Y   d S )Nrd   r   rh   r&   r  rE   )z%directory/2021-05-04 00%3A00%3A00/%24z,hive/date=2021-05-04 00%3A00%3A00/string=%24
/0.featherrw   Tr   date_intr*   r   逎`rt  ru  2021-05-04 00%3A00%3A00%24r>   rv  +Could not cast segments for partition fieldr   r   )'ri   rj   r   r  r@   r>   rC   r   r+  r0   r  rE   ro   rm   rn   r  new_filert   closer   r   r   r  r   r   r(  r   r   rA   r  as_pyrL   r  r  r   r  r  rK  r  )ro  r  ru   rl   r>   r   partition_schemastring_partition_schemafull_schemarw   sinkwriterr   r   r   r   r*  r  r  r   s                       r   *test_partitioning_factory_segment_encodingr?  a  s    !!FFY

+,--FHbhuRyy))*6:::Ey
",s##	$x&=>@ @ i
")++	29;; 78: :)DLL40@+A+AABBK  	 	)$$$&&y<'?@@ 	Dv.. &""5)))              	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 {d;;;H)+66G3<< = ! !#*7+?#O#OG )&(FGLLGoo''Ok))))^^&&BHV$$))"(**550&  F !9Q<:----3<<	V = 5 5#*7+?#O#OG )&(FGLLGW^^%%335566IQ<,33	&		6	6	(		u	$	&' ' ' ' ' +&: : :L"7<??G)&(FGLLGW^^%%335566IQ<,33	&		6	6	(		u	$	&' ' ' ' ' 3<<& = : :#*7+?#O#OG )&(FGLLG	rJ
L 
L 
L , ,!//++, , , , , , , , , , , , , , ,
 v666H)&11G.77 8 ! !#*7+?#O#OG )&(FGLLGoo''Ok))))^^&&BHV$$))"(**550&  F !9Q<:----.77 8 ! !#*7+?#O#OG )&(FGLLGW^^%%335566IQ<,33	&		6	6	(		u	$	&' ' ' ' ' .&: : :G)&(FGLLGW^^%%335566IQ<,33	&		6	6	(		u	$	&' ' ' ' ' .77& 8 : :#*7+?#O#OG )&(FGLLG	rJ
L 
L 
L , ,!//++, , , , , , , , , , , , , , , , , ,sZ    !G*F7+G7F;;G>F;?GG	G	(T		TT`55`9<`9c                     | S r   r   r  s     r   r  r    r  r   c                 R    |                     |                    |                     S r   r  r  s     r   r  r    r  r   c                 0   t          j                    }t          j                    }t	          j        dt	          j                    fg          }t	          j        t	          j        t          d                    g|          }t	          j        dt	          j
        d          fdt	          j                    fg          }t	          j        dt	          j                    fdt	          j                    fg          }t	          j        t          |          t          |          z             }t	          j        dt	          j
        d          fdt	          j                    fg          }	t	          j        dt	          j                    fdt	          j                    fg          }
d	}|                    |           |                    |d
z             5 }t          j                            ||          5 }|                    |           |                                 d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          j        dd          }t          j        d          }t          j                            |          } | ||          |_        t          j        ||||          }|                                }||k    sJ |                                                    dt          j        d                              t	          j                              i          }|d         d                                         dk    sJ t          j                            d          } | ||          |_        t          j        ||||          }t          |                                                                          }|d         j         !                    t          j        d          dk    t          j        d          dk    z            sJ t          j        |d          } | ||          |_"        t          j        ||||          }t          |                                                                          }|d         j         !                    t          j        d          dk    t          j        d          dk    z            sJ t          j                            d          } | ||          |_        t          j        ||||          }t          |                                                                          }|d         j         !                    t          j        d          dk    t          j        d          dk    z            sJ t          j        |
d          } | ||          |_"        t          j        ||||          }t          |                                                                          }|d         j         !                    t          j        d          dk    t          j        d          dk    z            sJ t          j                            |	d          } | ||          |_        t          j        ||||          }tG          j$        t          j%        d          5  |                                }d d d            d S # 1 swxY w Y   d S )Nrd   r   rh   ztest'; dater  ztest';[ string'ztest%27%3B%20dateztest%27%3B%5B%20string%27zLhive/test%27%3B%20date=2021-05-04 00%3A00%3A00/test%27%3B%5B%20string%27=%24r0  r   Tr   r1  r*   r   r2  rN  ru  z2021-05-04 00:00:00$rt  r3  r4  r5  r6  r   )&ri   rj   r   r  r@   r>   rC   r   r+  r0   r  rE   ro   rm   rn   r  r7  rt   r8  r   r   r  r  r   r   r(  r   r   rA   r  r9  rL   r  r  r   r  r  rK  )ro  r  ru   rl   r>   r   r:  r;  r<  partition_schema_enstring_partition_schema_enrw   r=  r>  r   r   r   r   r*  r  r  r   s                         r   ;test_partitioning_factory_hive_segment_encoding_key_encodedrF    s[    !!FFY

+,--FHbhuRyy))*6:::Ey
c**	+.?-MNP P i
	%(929;;'GHJ J)DLL40@+A+AABBK)
r|C00	1
%ry{{	3	56 6 "$
ry{{	+
%ry{{	3	5"6 "61I
i   		"	"9|#;	<	< V__T6** 	fu%%%LLNNN	 	 	 	 	 	 	 	 	 	 	 	 	 	 	               v666H)&11G.77 8 ! !#*7+?#O#OG )&(FGLLGoo''Ok))))^^&&BH]++00<<0&  F !9Q<:----.77 8    #*7+?#O#OG )&(FGLLGW^^%%335566IQ<,33	-	 	 $9	9	#	$	$	+	-. . . . . &%9 9 9L"7<??G)&(FGLLGW^^%%335566IQ<,33	-	 	 $9	9	#	$	$	+	-. . . . . .77 8 ! !#*7+?#O#OG )&(FGLLGW^^%%335566IQ<,33	%	&	&*C	C	-	.	.%	7	9: : : : : &"V= = =L"7<??G)&(FGLLGW^^%%335566IQ<,33	%	&	&*C	C	-	.	.%	7	9: : : : : .77"V 8 = =#*7+?#O#OG )&(FGLLG	rJ
L 
L 
L , ,!//++, , , , , , , , , , , , , , , , , ,sH   !I8*H."I.H2	2I5H2	6II	I	)\\\c           
         t          j        g dg dd          }t          j        t          j        t          j        dt          j                              t          j        dt          j                              g                    }t          j        t           j	                  5  t          j
        || d|           d d d            d S # 1 swxY w Y   d S )Nr   yNr   rI  r  r\   r]   r^   r  r  )r@   r   r   r   r>   rA   rE   r  r  rK  r  r  r   r   s      r   /test_dictionary_partitioning_outer_nulls_raisesrL    s   H+++///BBCCE?
	28C--rxRY[[/I/IJKKM MD	r	'	' J J
DIIIIJ J J J J J J J J J J J J J J J J Js   *CCCc                     t          j        g dg dd          }t          j        t                    5  t          j        || d           d d d            d S # 1 swxY w Y   d S )NrH  rJ  r\   zbasename-{i}.arrow)r@   r   r  r  r  r   r  )r  r   s     r   test_positional_keywords_raisesrN  &  s    H+++///BBCCE	y	!	! ? ?
)=>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?s   AA Ac                 0   d}t          j        t          j        d|dz             t          j        |dz             d          }t          j        |d |         | dz  dg           t          j        |d |dz            | dz  dg           t          j        | dz  dg	          }|d         j        dk    sJ t          j        | dz  dd
g	          }|d         j        dk    sJ t          j        | dz  dg	          }|d         j        dk    sJ d S )Ni   r   r   )r   r(   oner   r  twor*   r(   r   )	r@   r   repeatnparangers   r  
read_table
num_chunks)r  
BATCH_SIZEr   s      r   test_read_partition_keys_onlyrX  ,  sU    J HyJN++:>**, , - -E kzk%1 1 1 1 ozA~o%1 1 1 1 M'E/E7;;;E<"a''''M'E/E73CDDDE<"a''''M'E/E7;;;E<"a''''''r   c                 b     t          j                   }t           fd|D                       S )Nc                     g | ]?}t           j                            t           j                            |                    @S r   )osrx   isdirr  )ra   elbasedirs     r   rc   z _has_subdirs.<locals>.<listcomp>I  s7    LLLRbgll7B7788LLLr   )r[  listdirany)r^  elementss   ` r   _has_subdirsrb  G  s5    z'""HLLLL8LLLMMMr   c                 H   t          j        |           D ]}t           j                            | |          }t           j                            |          rKt          j        ||          }t          |          rt          |||           w|                    |           d S r   )	r[  r_  rx   r  r\  	posixpathrb  _do_list_all_dirsr1   )r^  path_so_farr7  r  true_nestednorm_nesteds         r   re  re  L  s    Z   + +gll7A..7==%% 	+#.a88KK(( +!+{FCCCCk***+ +r   c                 ,    g }t          | d|           |S )Nr  )re  )r^  r7  s     r   _list_all_dirsrj  W  s    Fgr6***Mr   c                 h    t          t          |                     }|t          |          k    sJ d S r   )r   rj  )r  expected_directoriesactual_directoriess      r   _check_dataset_directoriesrn  ]  s8    ^G4455%9!:!:::::::r   c           
      p   t          j        g dg dd          }t          j        t          j        t          j        dt          j                              t          j        dt          j                              g                    }t          j        || d|           t          | g d           d S )	NrJ  rH  r\   r]   r^   r  r  )zx/xzy/yr  )	r@   r   r   r   r>   rA   rE   r  rn  rK  s      r   (test_dictionary_partitioning_inner_nullsrp  b  s    H???1A1A1ABBCCE?
	28C--rxRY[[/I/IJKKM MDUGEEEEEw(;(;(;<<<<<r   c           
      t   t          j        g dg dd          }t          j        t          j        t          j        dt          j                              t          j        dt          j                              g          d d          }t          j        || d|           t          | g d	           d S )
N)r   Nr  rH  r\   r]   r^   ry  r  r  )za=x/b=xz	a=xyz/b=yz	a=z/b=xyz)	r@   r   r   r  r>   rA   rE   r  rn  rK  s      r   test_hive_partitioning_nullsrr  j  s    H+++2B2B2BCCDDEry	#ry{{	#	#RXc29;;%?%?@ B  BCGP PDUGEEEEEw(M(M(MNNNNNr   c                  2   t          j        dt          j                    fdt          j                    fg          } ddg}t	          j        |           }t          |t          j                  sJ t	          j        | d          }t          |t          j                  sJ t	          j        |          }t          |t          j                  sJ t          j
        t                    5  t	          j                     d d d            n# 1 swxY w Y   t          j
        t          d          5  t	          j        |            d d d            n# 1 swxY w Y   t          j
        t          d          5  t	          j        | |            d d d            n# 1 swxY w Y   t	          j        | d	
          }t          |t          j                  sJ t	          j        | dd	          }t          |t          j                  sJ t	          j        d	
          }t          |t          j                  sJ t          j
        t                    5  t	          j        |d	
           d d d            n# 1 swxY w Y   t          j
        t          d          5  t	          j        |d	           d d d            n# 1 swxY w Y   t          j
        t                    5  t	          j        | d
           d d d            d S # 1 swxY w Y   d S )Nr   r   inferr  )field_nameszExpected listr   zCannot specify bothr   r  )r  r  zCannot specify 'field_names')ru  r  unsupported)r@   r>   r  r  r   r   rN   r   r  r  r  r  r  )r>   r  r   s      r   test_partitioning_functionrw  r  s6   Y,w		.BCDDFWE ?6""DdB455555?6888DdB233333?u---DdB233333	z	"	"  
              	z	9	9	9 , ,
F++++, , , , , , , , , , , , , , ,	z)>	?	?	? 4 4
F33334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 ?6&111DdB/00000?6GGGDdB233333?&)))DdB233333	z	"	" . .
f----. . . . . . . . . . . . . . .	z)G	H	H	H : :
E&9999: : : : : : : : : : : : : : : 
z	"	" 6 6
}55556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6sl   -DDD3EEE;FF"%F"I;;I?I?!KKK(LLLc                    t          j        t          j        dt          j        t          j                    t          j                                        t          j        dt          j        t          j                    t          j                                        g          }t          j        	                    |          }t          j
        dd| |          }|j        j        |k    sJ |                                }|                    d          j                            |j        d                   sJ |                    d                                          dgd	z  d
gd	z  z   k    sJ |                    d          j                            |j        d                   sJ |                    d                                          dgd	z  dgd	z  z   k    sJ d S )Nr   r   rh   r   r   rl   r   r   r   r   r   r   r!  r"  )r@   r>   rA   r  r  r   rE   r   r   r  rT   r   r   r-  r  r  typesrR  )ru   r>   r   rT   r   s        r   *test_directory_partitioning_dictionary_keyr{    s    Y
"-		28::>>??
bgii==>>  F #,,F,;;DjvD  G &&0000E<<  %,,V\!_=====<<  **,,a1#'0AAAAA<<#**6<?;;;;;<<((**ugkUGaK.GGGGGGGr   c                    t          j        t          j        dt          j        t          j                    t          j                                        t          j        dt          j        t          j                    t          j                                        g          }t          j                            |          }t          j	        dd| |          }|j
        j        |k    sJ |                                }t          t          dd                    }t          t          d	d
                    }|                    d          j                            |j        d                   sJ |                    d          j        D ]7}|j                                        }|                                 ||k    sJ 8|                    d          j                            |j        d	                   sJ |                    d          j        D ]7}|j                                        }|                                 ||k    sJ 8d S )Nr   r   rh   r   r   ry  i  i  r      r   )r@   r>   rA   r  r  r  r   r  r  rT   r   r   ro   r0   r-  r  r  rz  chunksrR  sort)	r   r>   r   rT   r   year_dictionarymonth_dictionaryr   r  s	            r   %test_hive_partitioning_dictionary_keyr    s   Y
rwyy"(**==>>
"-		28::>>??  F ''v'66Djy]  G &&0000E5t,,--OE!RLL))<<$++FLO<<<<<f%%, ) )!++--(((((<<  %,,V\!_=====g&&- * *!++--))))))* *r   c                     |/t          j        t          d          dgdz  dgdz  z   d          }| dz  }t          j        |||           ||fS )	N	   r&  r   r   r   r\   r  r  r@   r   r0   rs   rt   )base_dirr   r  rx   s       r   r  r    s`    }uQxxrdQh".ABBCCn$DN5$~>>>>$;r   c                 8   t          j        t          d          dgdz  dgdz  z   d          }| dz  }t          j        ||           t          j        t          dd          dgdz  dgdz  z   d          }| d	z  }t          j        ||           ||f||ffS )
Nr  r&  r   r   r   r\   ztest1.parquetr  ztest2.parquetr  )r  table1path1table2path2s        r   _create_directory_of_filesr    s    XE!HHB4!8rdQh+>??@@F&EN65!!!XE!RLLtax2$(/BCCDDF&EN65!!!FeU^++r   c                     | |                     |                    |                     fD ]M}| j                            |j                  sJ |                    |                               |          sJ Nd S r   )r  r  r>   r  r   )rT   r   r   picklerr  s        r   _check_datasetr    s    w}}W]]7%;%;<<= > >~$$U\22222&&w//66u======> >r   c                    t          | t          j                  sJ | t          |           | gt          |           gfD ]B}t	          j        | fi |}t          |t          j                  sJ t          ||||           Ct          | j	                  5  t	          j        | j
        fi |}t          |t          j                  sJ t          ||||           d d d            d S # 1 swxY w Y   d S r   )rN   pathlibPathr`   r   rT   r  r  r   parentname)rx   r   r   r  r   r   rT   s          r   _check_dataset_from_pathr    s]   dGL))))) CIIvD		{3 @ @*T,,V,,'2#788888w~w???? 
DK	 	  @ @*TY11&11'2#788888w~w???@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @s   AC))C-0C-c                 N    t          |           \  }}t          ||||           d S r   r  r  r  r   r  r   rx   s        r   test_open_dataset_single_filer    s-    %g..KE4T5.-HHHHHr   c                 R    t          | d          \  }}t          ||||           d S )Nr   r  r  r  s        r   test_deterministic_row_orderr    s4    
 &ga@@@KE4T5.-HHHHHr   c                 v    t          |           \  }}t          j        |          }t          | |||           d S r   )r  r@   concat_tablesr  )r  r   r  tablesr^  r   s         r   test_open_dataset_directoryr  	  s=    *733IFAV$$EWe^]KKKKKr   c                    t          |           \  }\  }}t          j        |          }t          j        ||g          t          j        t          |          t          |          g          g}|fd|D             z  }|D ]O}|j                            |j                  sJ |                    |          }	|	                    |          sJ Pd S )Nc                 `    g | ]*}                                         |                    +S r   r  )ra   r  r  s     r   rc   z3test_open_dataset_list_of_files.<locals>.<listcomp>	  sB       89M//2233  r   )	r  r@   r  r   rT   r`   r>   r  r   )
r  r   r  r  r  r  r   datasetsrT   r7  s
     `       r   test_open_dataset_list_of_filesr  	  s    7@@FNUEV$$E 	
E5>""

CJJE

+,,H     =E   H  $ $~$$U\22222((11}}U######$ $r   c                    t          |           \  }}t          |          }t          j        |          }|j                            |j                  sJ t          j        |t          j                              }|j                            |j                  sJ t          j	        t                    5  t          j        |t          j                               d d d            d S # 1 swxY w Y   d S )Nr  )r  r   r   rT   r>   r  ri   r|   r  r  r  rj   )r  r   rx   fspathdataset1dataset2s         r   #test_open_dataset_filesystem_fspathr  	  s"    &g..KE4T""F z&!!H?!!%,///// z&R-?-A-ABBBH?!!%,///// 
y	!	! < <

6b&8&:&:;;;;< < < < < < < < < < < < < < < < < <s   8(C--C14C1c                 t   | dz  }|                                  t          |          \  }}|                    |          }t          j        |          }t          j        |t          j                              }t          j        t          |          t          |                    }	|	                    |
                    |                    }
|                    |          |                    |          cxk    r3|                    |	          cxk    r|                    |
          k    sn J d S )Nsingle-filer  )mkdirr  relative_tor   rT   ri   r|   r`   r
   r  r  r   )r  r   r  rw   r   rx   relative_pathd1d2d3d4s              r   test_construct_from_single_filer  /	  s[   -'IOO%i00KE4$$Y//M 
D		B	DR%7%9%9	:	:	:B	C&&?93M3M	N	N	NB			]0044	5	5B""2&&.*A*A
+ + J J J J&&r**J J J J.<.E.Eb.I.IJ J J J J J J Jr   c                 X   | dz  }|                                  t          |          \  }}t          j        |          }t          j        |t	          j                              }t          j        |j        t          |                     }|                    |          }	|                    |          }
|                    |          }|	|
cxk    r|k    sn J |||fD ]E}|	                    |
                    |                    }|                    |          |	k    sJ Fd S )Nsingle-directoryr  )r  r  r   rT   ri   r|   r  r
   r   r  r  )r  r   r  rw   r  r   r  r  r  t1t2t3r  restoreds                 r   $test_construct_from_single_directoryr  C	  s0   ,,IOO.y99MFE	I		B	I"*<*>*>	?	?	?B	INw/G/G	H	H	HB		 	 	$	$B		 	 	$	$B		 	 	$	$B>>>>r>>>>>> "b\ 7 7 &&}':':1'='=>>&&x00B666667 7r   c                      dz  }|                                  t          |          \  }} fd|D             }t                     5  t          j        |          }|                    |          }t          |          t          t          t          |                    k    sJ 	 d d d            n# 1 swxY w Y   t          j        |t                               }|                    |          }	t          j        |          }
|                    |
          }t          j        |t          j                              }|                    |          }||	cxk    r|cxk    r|k    sn J d S )Nzlist-of-filesc                 :    g | ]}|                               S r   )r  )ra   r   r  s     r   rc   z5test_construct_from_list_of_files.<locals>.<listcomp>^	  s%    <<<ammG,,<<<r   r  )r  r  r   r   rT   r   r   sumrp   r
   ri   r|   )r  r   rw   r  r   relative_pathsr  r  r  r  r  r  r  t4s   `             r   !test_construct_from_list_of_filesr  W	  s    /)IOO.y99MFE<<<<e<<<N	G		 0 0Z''$$R((2ww#c#v..///////0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 
Nw/G/G	H	H	HB		 	 	$	$B	E		B		 	 	$	$B	Eb&8&:&:	;	;	;B		 	 	$	$BrRs   
AB66B:=B:c                     ddg}t          j        t          d          5  t          j        ||            d d d            d S # 1 swxY w Y   d S )Nr   z!subdir/1/xxx/doesnt-exist.parquetzdoesnt-existr   r  )r  r  r  r   rT   )ru   r	  s     r   -test_construct_from_list_of_mixed_paths_failsr  n	  s     	%+E 
(	?	?	? - -

5V,,,,- - - - - - - - - - - - - - - - - -s   AAAc                     t          j        ddg|           }t          j        d|           }t          j        ||g          }t          |t           j                  sJ t	          t          |                                                    dk    sJ |                                }t	          |          dk    sJ |j        dk    sJ t	          |j	                  dk    sJ |j	        D ]}|j
        ddgk    sJ d S )	Nr   r   r  r   r   r  r   r   )r   rT   rN   UnionDatasetr   ro   rL   r   r@  childrenr	  )ru   r]   r^   rT   r   childs         r   (test_construct_from_mixed_child_datasetsr  y	  s4    	
002>D	F 	F 	FA

8///Aj!Q  Ggr/////tG))++,,--2222Eu::!!!!w  A%%%%! = ={;;= = = = = == =r   c                      t          j        g d          } |                                 }|j        dk    sJ |j        dk    sJ d S )Nr  r  r   )r   rT   r   r   r@  )emptyr   s     r   test_construct_empty_datasetr  	  sR    Jr%(((ENNE>Q!!!!!!r   c            
      8   t          j        g dt          j        dt          j                    fdt          j                    fg                    } t          j        t          d          5  | 	                                 d d d            d S # 1 swxY w Y   d S )Nr  r]   rl   r>   zMultiple matches for .*a.* in r   )
r   rT   r@   r>   rC   rE   r  r  r  r   )r  s    r   *test_construct_dataset_with_invalid_schemar  	  s    Jr%		bhjj	bikk; 1 1   E 
z)I	J	J	J                   s   -BBBc                    t          j        | t          j        d          t          j                              }t          j        | t          j        d          t          j                              }t
          j                            t          j        t          d                    gdg          t
          j                            t          j        t          d                    gdg          }t          j        t          d	          5  t          j        ||g           d d d            n# 1 swxY w Y   d
}t          j        t          |	          5  t          j        g d           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          j        d            d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          j        fdt          d          D                        d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          j        g            d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          j        |g           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          j        dg           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          j        dg           d d d            d S # 1 swxY w Y   d S )Nr6  r  /schemar   r]   r  r^   z"Expected.*FileSystemDatasetFactoryr   zExpected a list of path-like or dataset objects, or a list of batches or tables. The given list contains the following types: intr  zbExpected a path-like, list of path-likes or a list of Datasets instead of the given type: NoneTypezcExpected a path-like, list of path-likes or a list of Datasets instead of the given type: generatorc              3      K   | ]}V  d S r   r   )ra   r^  batch1s     r   rr  z<test_construct_from_invalid_sources_raise.<locals>.<genexpr>	  s#      --qF------r   r[   zEMust provide schema to construct in-memory dataset from an empty listzFItem has schema
b: int64
which does not match expected schema
a: int64z}Expected a list of path-like or dataset objects, or a list of batches or tables. The given list contains the following types:r   zCExpected a list of tables or batches. The given list contains a int)r   r   ri   r   r   r@   r,  r+  r+  r0   r  r  r  rT   r  InMemoryDataset)r   child1child2batch2rf  r  s        @r   )test_construct_from_invalid_sources_raiser  	  s5   (
!!#%%  F
 (
	""#%%  F
 ^''%)))<)<(=cU'KKF^''%)))<)<(=cU'KKF	y(L	M	M	M % %

FF#$$$% % % % % % % % % % % % % % %	 
 
y	1	1	1  

999              	.  
y	1	1	1  

4              	/  
y	1	1	1 / /

----E!HH---.../ / / / / / / / / / / / / / / 	P  
z	2	2	2  
2              	  
y	1	1	1 % %

FF#$$$% % % % % % % % % % % % % % %	J  
y	1	1	1    

FA;                              	N  
y	1	1	1 ( (
FA;'''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (s   D88D<?D< FF
F+GGG4.H..H25H2I77I;>I;KK	K*LLL5MM Mc                 J   t           j                            t          j        t	          d                    gdg          }t           j                            |g          }t          j        g dt          j	        g                     
                                }|t          j        g           k    sJ |||g|gfD ]}t          j        |          }| 
                    |          |k    sJ t          t          |                                                    dk    sJ t          |                                          
                                |k    sJ t           j                            t          |                                                    |k    sJ d S )Nr   r]   r  r  r  r   )r@   r,  r+  r+  r0   rF   rr   r   rT   r>   r   r   r   ro   rL   r3   r   )r   rz   r   dataset_tablesourcerT   s         r   test_construct_in_memoryr  	  s   N&&r(;(;'<SE&JJEH!!5'**EJr%	" ! ! !!) BHRLL((((%%5'2 J J*V$$&&w//588884--//0011Q6666G))++,,55775@@@@x$$T'*<*<*>*>%?%?@@EIIIIIJ Jr   r   c                 0   t           j                            t          j        t	          d                    gdg          t           j                            g          d}fdd ffdd ffdj        ffD ]\  }}t          j	                             |            ||           }|
                                k    sJ t          j        t           j        |	          5  |
                                 d d d            n# 1 swxY w Y   d S )
Nr   r]   r  z#OneShotFragment was already scannedc                  P    t           j                             j         g          S r   )r@   RecordBatchReaderrr   r>   rz   s   r   r  z$test_scan_iterator.<locals>.<lambda>	  s#    R)66ug' ' r   c                  "    t                     S r   )r   r   s   r   r  z$test_scan_iterator.<locals>.<lambda>	  s    '.. r   c                  :     fdt          d          D             S )Nc              3      K   | ]}V  d S r   r   )ra   r^  rz   s     r   rr  z7test_scan_iterator.<locals>.<lambda>.<locals>.<genexpr>	  s#      ..e......r   r   r0   r  s   r   r  z$test_scan_iterator.<locals>.<lambda>	  s"    ....U1XX... r   r>   r   r   )r@   r,  r+  r+  r0   rF   rr   r>   r   rJ  r   r  r  rK  )r   r  r   r>   r   rz   r   s        @@r   test_scan_iteratorr  	  su   N&&r(;(;'<SE&JJEH!!5'**E1E' ' ' '(,.....5....=	   *))GIIf+ * ? ?!!U****]2?%888 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 s   )D

D	D	c                    t          j        t          d          dgdz  dgdz  z   d          }| dz  }|                                 t          d          D ]]}|d	                    |          z  }|                                 t          j        |                    d|z  d          |d
z             ^|                    dt          j	        dgdz  dgdz  z   dgdz  z   t          j
                                        }||fS )Nr  r&  r   r   r   r\   zdataset-partitionedr[   zpart={}r  r   r   r   r   r  )r@   r   r0   r  rl   rs   rt   r  append_columnr+  r   )r^  r   rx   r;   r   
full_tables         r   _create_partitioned_datasetr  
  s
   H5882$(bTAX*=>>??E**DJJLLL1XX C Ci&&q)))


u{{1Q3**D>,ABBBB$$!qA37*aS1W428::FFFH HJ tr   c           
         t          |           \  }}|                    ddg          }t          ||||           t          j        t          |          t          j        d                    }|j                            |j                  sJ t          |           5  t          j        dt          j        d                    }|j                            |j                  sJ 	 d d d            n# 1 swxY w Y   t          j        t          |          d          }|j                            |j                  sJ t          j        t          |          t          j        t          j        dt          j                    fg          d                    }|j                            t          j        dt          j                                        }|j                            |          sJ |                                }|                    dt          j        dgd	z  d
gd	z  z   dgd	z  z   t          j                                        }	|                    |	          sJ d S )Nr]   r^   r   r  r   zdataset-partitioned/r   r   r[   r   r   r  )r  r  r  r   rT   r`   r   r>   r  r   r@   r  r1   rA   r   r  r+  )
r  r   r  r  rx   r   rT   r!  r7  rf  s
             r   'test_open_dataset_partitioned_directoryr  
  s   27;;J sCj))ET5.-HHH jD		v > > >@ @ @G>  !233333 
G		 8 8*3*,/*H*H*HJ J J~$$Z%67777778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 jT888G>  !233333 jD		_I		*+,,V= = => > >G l))"(62799*E*EFFO>  11111F""!qA37*aS1W42799EEEG GH=="""""""s   "AC::C>C>c                    t          |           \  }}t          j        t          |                    }|j                            |j                  sJ t          j        t          |          t          j                              }|j                            |j                  sJ t          |           5  t          j        dt          j                              }d d d            n# 1 swxY w Y   |j                            |j                  sJ t          j
        t                    5  t          j        t          |          t          j                               d d d            d S # 1 swxY w Y   d S )Nr  r  )r  r   rT   r`   r>   r  ri   r|   r   r  r  r  rj   )r  r   rx   r  r  dataset3s         r   test_open_dataset_filesystemr  ;
  s    &g..KE4 z#d))$$H?!!%,///// z#d))0B0D0DEEEH?!!%,///// 
G		 O O:n9K9M9MNNNO O O O O O O O O O O O O O O?!!%,///// 
(	)	) ? ?

3t99);)=)=>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?s$   9(C--C14C125E44E8;E8c                     t          |           \  }}t          j        t          d          5  t	          j        |gd           d d d            d S # 1 swxY w Y   d S )Nz format 'blabla' is not supportedr   blablar  )r  r  r  r  r   rT   )r  r^  rx   s      r   $test_open_dataset_unsupported_formatr  R
  s    !'**GAt	z)K	L	L	L , ,

D6(++++, , , , , , , , , , , , , , , , , ,s   AAAc                 b   t          |           \  }}t          j        |          }t          j        ||g          }t          |t          j                  sJ |                    |                    |                    }|                    |          |                    |          k    sJ d S r   )r  r   rT   rN   r  r  r  r   )r  r   r  r^  rx   rT   unionro  s           r   test_open_union_datasetr  Y
  s    !'**GAtjGJ)**EeR_-----!!-"5"5e"<"<==G""7++~/F/Fu/M/MMMMMMMr   c                     t          j        d| d          }t          j        t          d          5  t          j        |gd           d d d            d S # 1 swxY w Y   d S )Nr6  r   r7  zcannot pass any additionalr   r  )r   rT   r  r  r  )r   r  s     r   .test_open_union_dataset_with_additional_kwargsr  e
  s    JxM)LLLE	z)E	F	F	F . .

E79----. . . . . . . . . . . . . . . . . .s   AAAc                  2   t          j        t                    5  t          j        dd           d d d            n# 1 swxY w Y   t          j        t
          j        d          5  t          j        dd           d d d            d S # 1 swxY w Y   d S )Nzi-am-not-existing.arrowr  r  zcannot be relativer   zfile:i-am-not-existing.arrow)r  r  r  r   rT   r@   rK  r   r   r   #test_open_dataset_non_existing_filer  k
  s(    
(	)	) < <

,U;;;;< < < < < < < < < < < < < < < 
r.B	C	C	C A A

1%@@@@A A A A A A A A A A A A A A A A A As!   =AA(BBBr   rw   r   rz  ry  partition_keysr  BCr  )DEFr  )r   Nr[   )r  Nr  )Nr   r[   c           	         t          j        t          d          dgdz  dgdz  z   d          }d |d         v p	d |d         v }|d	k    r|rd S |d	k    r(t          j                            d
dg          }d}d }nM|r"t          j                            |          }n t          j                                      }d}|r|}nd}| dz  }	|	                                 |\  }
}|
D ]R}|D ]M}|	|                    |p||p|          z  }|                    d           t          j
        ||dz             NSt          j        t          |	          |          }fd}|j                            t          j        d
 ||
d                                                           t          j        d ||d                                       }|j                            |          sJ d S )Nr  r&  r   r   r   r\   r   r   rw   part1part2r(  z{0}/{1})r$  rz  zpart1={0}/part2={1}__HIVE_DEFAULT_PARTITION__rT   T)parentsr  r  c                 @   rat          | t                    rt          j                    nt          j                    }t          j        t          j                    |          S t          | t                    rt          j                    nt          j                    S r   )rN   r`   r@   rE   r   r  )r   
value_typer$  s     r   r+  z/test_partition_discovery.<locals>.expected_type
  sm     	G(23(<(<L"(**J=Z888",S#"6"6F29;;;BHJJFr   )r@   r   r0   r   r   r  r  r  rl   rs   rt   rT   r`   r>   r1   rA   r  )r  r   rz  r$  r  r   has_nullfmt
null_valuebasepath
part_keys1
part_keys2r   r  rx   rT   r+  r!  s      `              r   test_partition_discoveryr  u
  sl   " H5883%!)seai*?@@AAE~a((EDN14E,EH{""x"{""/88g1A 9 C C

 	3.77!1 8  LL .77!1 8 3 3L# 	6&JJ5J"HNN+J
 9 9 	9 	9E

5.J0CDDEDJJtJ$$$N5$"78888		9 jX\BBBGG G G G G l))
--
16677 f
--
16677  
 >  1111111r   c                 J   t          j        t          j        ddgd          t	          d          d          }t          j        |                    dg          j        d          }t          j	        || |d	
           t          j
        | d	t
          j                            d                    }t          j        |d         |d                                         d          }|                                                    |          sJ t!          |                                          d         }|                    |j                                      |d d                   sJ |j        }|                    |                    |                    }|                                                    |          sJ |                    |                    |                    }|                    |j                                      |d d                   sJ |                    |j                                                                      |d d                                                   sJ |j                            |          sJ d S )Nr  r  r   r   r   r  r   r   r  r  r   rl   Tr(  r  r  )r  r   r   rh   )r@   r   rS  rR  r0   r   r   r  r>   r  rT   r  r  ru  r   r  ro   rL   r  r  r  r  )	r  r  r   r   rT   rf  rU   	part_exprr  s	            r   4test_dataset_partitioned_dictionary_type_reconstructr  
  sq    Hbic
A66uRyyIIJJE?5<<118HHHDUG$yIIIIj	(1141HH  G xeeFm&E&E&G&GHH H $$X.....G))++,,Q/HGN33::8BQB<HHHHH-I""=#6#6w#?#?@@H%%h/////""=#6#6x#@#@AAHGN33::8BQB<HHHHHGN33==??FF!       (//	:::::::r   c                 f   ddl m} | d         \  }}}}d                    ||||          }|                    |          \  }} |j        d           t          j        dg di          }	 |j        d          5 }
t          j	        |	|
           d d d            n# 1 swxY w Y   |	|||||||fS )	Nr   
FileSystem
connectionz_s3://{}:{}@mybucket/data.parquet?scheme=http&endpoint_override={}:{}&allow_bucket_creation=Truemybucketr]   r  zmybucket/data.parquet)
r   r  rl   from_urirm   r@   r   rn   rs   rt   )	s3_serverr  rO  rP  rQ  rR  rN  ri   rx   r   ry   s              r   rM  rM  
  s   %%%%%%)2<)@&D$
J	&	
Jd	3	3  ""3''HBBM*Hc999%&&E		6	7	7 #3
uc"""# # # # # # # # # # # # # # # $CtZCCs   <BB"%B"c                    | \  }}}}}}}}t          j        |d          }|                    |                              |          sJ t          j        |d|          }|                    |                              |          sJ d S )Nr   r  rl   r   )r   rT   r   r  )rM  r   r   rx   ri   rN  r^  rT   s           r   test_open_dataset_from_uri_s3r  
  s     (9$E4S!Q1 jY///G""7++22599999 jiB???G""7++2259999999r   c                     | \  }}}}}}}}t          j        d          }|                    |          }t          j        |d|          }	|                    |	                              |          sJ d S )Nr  r   r  )ri   r   r?  r   rT   r   r  )
rM  r   r   rx   r   rN  r^  r   finfosrT   s
             r    test_open_dataset_from_fileinfosr  
  s     0A,E4S!Q1z**H%%h//Fj	jIIIG""7++2259999999r   c           	         | \  }}}}}}}}t          j        d          }ddlm}	m}
 |                    ||dd                    ||          i          }t          j        |d|          }|	                                
                    |          sJ  |
 |	|                    }t          j        |d|          }|	                                
                    |          sJ d S )	Ns3fsr   )FSSpecHandlerr}   endpoint_urlzhttp://{}:{})r   secretclient_kwargsr   r  )r  importorskipr   r!  r}   S3FileSystemrl   r   rT   r   r  )rM  r   rx   r^  rO  rP  rQ  rR  r   r!  r}   ri   rT   s                r   $test_open_dataset_from_uri_s3_fsspecr'    s    =N9E4AtT:zv&&D66666666			N11$==
 
 
 
B jiB???G$$U+++++ 
mmB''	(	(BjiB???G$$U+++++++r   c                    ddl m} | d         \  }}}}d}d}d                    ||||||          }|                    |          \  }	}|dk    sJ  |	j        |           t          j        dg d	i          }
 |	j        |          5 }t          j	        |
|           d d d            n# 1 swxY w Y   t          j        |d
          }|                                                    |
          sJ d                    ||||          }g d}|D ]Z\  }}|                    |          }t          j        ||d
          }|                                                    |
          sJ [t          j        t
          j        d          5  |                    d          }t          j        d|           d d d            n# 1 swxY w Y   d}d}|                    |          }t          j        t"                    5 }t          j        d|           d d d            n# 1 swxY w Y   t%          |j                  |                    d||          k    sJ d}|                    |          }t          j        t"                    5 }t          j        d|           d d d            n# 1 swxY w Y   t%          |j                  |                    d||          k    sJ d S )Nr   r  r  theirbucketnested/folder/data.parquetzOs3://{}:{}@{}/{}?scheme=http&endpoint_override={}:{}&allow_bucket_creation=truez&theirbucket/nested/folder/data.parquetr]   r  r   r  3s3://{}:{}@{{}}?scheme=http&endpoint_override={}:{}))ztheirbucket/nested/folder/z/data.parquet)ztheirbucket/nested/folderdata.parquet)ztheirbucket/nested/folder/data.parquet)ztheirbucket/nestedr-  )r)  z/nested/folder/data.parquet)r)  r*  r7  zMissing bucket namer   r  z'/theirbucket/nested/folder/data.parquetr  zThe path component of the filesystem URI must point to a directory but it has a type: `{}`. The path component is `{}` and the given filesystem URI is `{}`ztheirbucket/doesnt/existr,  NotFoundFile)r   r  rl   r  rm   r@   r   rn   rs   rt   r   rT   r   r  r  r  rK  r  r`   r(   )r  r  rO  rP  rQ  rR  bucketrx   rN  ri   r   ry   rT   templater{  prefixr  excs                     r   -test_open_dataset_from_s3_with_filesystem_urir4  "  s    &%%%%%)2<)@&D$
JF'D&&,f
FD$'
 '
 
 ""3''HB;;;;;BM&Hc999%&&E		t	$	$ #
uc"""# # # # # # # # # # # # # # # jY///G$$U+++++ 	>DD
D$	
 	
 
  E  0 0oof%%*Tc)DDD!!((//////	r.C	D	D	D N Nooc""

<MMMMN N N N N N N N N N N N N N N
	! 
 &D
//$

C	z	"	" 3c

>c22223 3 3 3 3 3 3 3 3 3 3 3 3 3 3sy>>U\\*dC@@@@@@3D
//$

C	z	"	" 3c

>c22223 3 3 3 3 3 3 3 3 3 3 3 3 3 3sy>>U\\&$<<<<<<<<sH   
B,,B03B0,GGGH''H+.H+J44J8;J8c                     t          |           \  }}t          j        d          }|                    d          }t	          j        ||          }|j                            |j                  sJ d S )Nfsspecfiler  )r  r  r%  r   r   rT   r>   r  )r  r   rx   r6  r   rT   s         r   test_open_dataset_from_fsspecr8  f  sn    %g..KE4 **F''Gj'222G>  .......r   c                 `   t          j        d          }t          j        dg di          }| dz  }t	          j        ||           |                    d          }|                    |           d                             d          sJ t          j
                    }t          j        t          j        |                    }|                    ||          }|                    |j                  sJ |                    ||          }|j                            |j                  sJ d S )Nr6  r]   r  r,  r7  r   )r  r%  r@   r   rs   rt   r   lsendswithr   r   ri   r}   r!  r(  r  r>   r   r  )	r  r6  r   rx   	fsspec_fsrl   r   r>   rU   s	            r   test_file_format_inspect_fsspecr=  q  s     **F Hc999%&&E^#DN5$ !!&))I<<  #,,^<<<<< !##F !1)!<!<==J^^D*--F==&&&&&##D*55H#**5<8888888r   c                 \   | dz  }t          j        ddgdz  t          d          d          }t          j        |                    dg          j        d	          }t          j        |||d
           t          j        t          j        dt          j        d          fg          d	          }t          j	        |d
|          }t          j
        d          t          j        d          k    }|                    ||          }|                    d                                          g dk    sJ dd l}t          j
        d           |j        ddd          k    }|                    ||          }|                    d                                          g dk    sJ d S )Ntest_partition_timestamps
2012-01-01z
2012-01-02r   r   )datesr  rA  r   r  r  r  r  r  r   r  )r   r[   r   r?  r  r   i  r   )r@   r   r0   r   r   r  r>   r  r  rT   rA   r4   	Timestampr   r-  rR  r,   )r  r   rx   r   r   rT   r6  r,   s           r   test_filter_timestamprC    s    00DH-1Bii   E
 ?5<<	229&IIIDUDtIFFFF ?29wS0A0A&B%CDD"(* * *DjidCCCG!!BL$>$>>I##GI#>>E<<''))___<<<<OOO!!$5H$5dAq$A$AAI##GI#>>E<<''))___<<<<<<r   c                 l   t          j        dt          j        g dt          j                              i          }t	          | |          \  }}t          j        t          |                    }t          j        d          dk    }t          |
                    ||                    dk    sJ d S )Nr]   )r   r   r   r[   r   r   r  r   r   r[   )r@   r   r+  r  r  r   rT   r`   rA   r   r   )r  r   r   r^  rx   rT   filter_s          r   test_filter_implicit_castrF    s     Hc28$6$6$6RWYYGGGHIIE!'511GAtjT##GhsmmaG~&&ww&??@@AEEEEEEr   c                 6   t          j        dg di          }t          | |          \  }}t          j        t          |                    }|                    |t          j        d          t          j        d           k              }|j	        dk    sJ d S )Nr  )r]   r^   Nr   r   )
r@   r   r  r   rT   r`   r   rA   r  r   )r  r   r   r^  rx   rT   s         r   test_filter_equal_nullrH    s     Hc+++,--E!'511GAtjT##G##48 $  E >Qr   c           	      :   t          j        g dd t          d          D             d t          dd          D             d          }t          | |          \  }}t	          j        t          |                    }t          j        t	          j	        d          t          j
        d	d
g                    }|                    ||          j        dk    sJ t          j        t	          j	        d                    dk    }|                    ||          j        dk    sJ t          j        t	          j	        d          t	          j	        d                    }|                    |d|i          }|d                                         g dk    sJ d S )N)r]   r^   Nr]   r  c                 <    g | ]}t          j         d dd|          S i  r   r  r$  s     r   rc   z2test_filter_compute_expression.<locals>.<listcomp>  s)    AAA1haA..AAAr   r   c                 :    g | ]}t          j         d d|          S rK  r  r$  s     r   rc   z2test_filter_compute_expression.<locals>.<listcomp>  s'    AAAha++AAAr   r   r  r  r  r]   r^   r   r[   r  r   r  r    r*   r  )r@   r   r0   r  r   rT   r`   r  is_inrA   r+  r   r   hourdays_betweenrR  )	r  r   r   r^  rx   rT   rE  r    r7  s	            r   test_filter_compute_expressionrP    sy   H'''AAaAAAAAU1a[[AAA   E
 "'511GAtjT##Ghrx}}bhSz&:&:;;G""77";;DIIIIgbhsmm$$)G""77";;DIIII?28C=="(3--88D$$Wvtn$EEF&>##%%888888r   c                    t          j        | t          j        d          t          j                              }t          j        |g          }t          |                                          dk    sJ t          d |                                D                       sJ |                                d         	                    |
                                          sJ |
                                	                    |
                                          sJ t          |                                t           j                  sJ d S )Nr6  r  r   c              3   J   K   | ]}t          |t          j                  V  d S r   )rN   r@   r*  )ra   r  s     r   rr  z%test_dataset_union.<locals>.<genexpr>  s.      KKAz!RY''KKKKKKr   r   )r   r   ri   r   r   UnionDatasetFactoryr   r)  r  r  r(  rN   r   r)  )r   r  r   s      r   test_dataset_unionrT    s%   'rx00#%%  E $eW--G w&&(())Q....KK1H1H1J1JKKKKKKKK""$$Q'..u}}???????##EMMOO44444gnn&&
3333333r   c                 0	   t          j        d|d          }t          j        d|dddg          }t          j        d|dd	          }|j        |j        cxk    r|j        k    sn J t          j        |||g          }t          |t           j                  sJ d
}t          j        t          |          5  t          j        ||g|           d d d            n# 1 swxY w Y   t          j        dt          j	                    fdt          j
                    fdt          j                    fdt          j                    fdt          j                    fdt          j                    fdt          j                    fg          }|j                            |          sJ |                                j                            |          sJ t          j        ||g          }t          j        dt          j	                    fdt          j
                    fdt          j                    fdt          j                    fdt          j                    fdt          j                    fg          }|j                            |          sJ |                                j                            |          sJ t          j        dt          j                    fdt          j                    fdt          j	                    fg          }t          j        ||g|          }|                                j                            |          sJ t          j        dt          j                    fdt          j                    fdt          j                    fg          }t          j        ||g|          }|                                j                            |          sJ t          j        t#          d          dgdz  dgdz  z   dgg d          }t%          | |          \  }	}
t          j        |
          }t          j        t          j        d          5  t          j        ||g           d d d            d S # 1 swxY w Y   d S )Nr6  r   r7  r  weekr)   r   rl   r   /hiver   z$cannot pass any additional argumentsr   r  r&   r'   r(   r   r   rh   rD  r  r&  r   r   r   	abcdefghj)r&   r(   r'   r  r   zUnable to merge)r   rT   r>   rN   r  r  r  r  r@   rB   rC   rD   rE   r   r  r   r   r0   r  ArrowTypeError)r  r   r  r  child3	assembledmsgr!  r   r^  rx   child4s               r   &test_union_dataset_from_other_datasetsr_    s   Z]9MMMFZ	mI&,g%68 8 8FZM)%+- - -F =FM::::V]::::::
FFF344Ii11111
0C	z	-	-	- ? ?

FF#>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? i		"(**	"*,,	")++			"(**!  O ""?33333&--o>>>>>
FF+,,Ii		"(**	"*,,	")++		"(**!  O ""?33333&--o>>>>>i	"(**	")++	!  O
 
FF+ODDDI&--o>>>>>i	"(**	")++	BIKK !  O
 
FF+ODDDI&--o>>>>> HeAhhqB4!8 3[A5557 7 7E!'777GAtZF	r(0A	B	B	B % %

FF#$$$% % % % % % % % % % % % % % % % % %s$   :CC#&C#'RRRc                     d}t          j        t          |          5  t          j        g d|            d d d            d S # 1 swxY w Y   d S )Nz8points to a directory, but only file paths are supportedr   )r6  r  rX  r  )r  r  IsADirectoryErrorr   rT   )r   r]  s     r   4test_dataset_from_a_list_of_local_directories_raisesrb  %  s    
DC	(	4	4	4 M M

111mLLLLM M M M M M M M M M M M M M M M M Ms   AAAc           
         t          j        t          j        d|           t          j        d|           t          j        d|           g          }t          j        dt          j                    fdt          j                    fdt          j                    fdt          j                    fg          }|j                            |          sJ t          j        t          j        d|           t          j        d|           t          j        d| d	
          g          }t          j        dt          j                    fdt          j                    fdt          j                    fdt          j                    fdt          j	                    fdt          j	                    fg          }|j                            |          sJ d S )Nr6  r  r  rX  r&   r'   r(   r)   r   )r   r   r   r   )
r   rT   r@   r>   rB   rC   rD   rE   r  r   )r   rT   r!  s      r   &test_union_dataset_filesystem_datasetsrd  +  s   j

8666

9777

7}555  G
 i		"(**	"*,,	")++	!  O >  11111 j

8666

9777

7}6JJJ  G
 i		"(**	"*,,	")++		"(**!  O >  1111111r   c                     t          j        g dg dd          }t          j        | dz             d fd	}d }|} ||||j                   |j        }|} |||           t          j        ddg          }t          j        g dg dgd	d
g          } |||           t          j        dg          }t          j        g dgd
g          } |||           t          j        ddg          }t          j        g dt          j        g dd          gd
dg          } |||           t          j        ddg          }t          j        t           dz            |          }t          j        |d
         	                    d          |d	         gd
d	g          } |||           t          j        d
t          j
        t          j                              fdg          }t          j        t           dz            |          }|j                            |          sJ t          j        t          d          5                      |           d d d            d S # 1 swxY w Y   d S )Nr  r  r  r  r\   r,  c                 "   t          j        t          dz            |           }||j                            |          sJ n|j                            |           sJ                     |          }|                    |          sJ d S )Nr,  rh   )r   rT   r`   r>   r  r   )r>   rf  r!  rT   r7  r   r  s        r   r  z-test_specified_schema.<locals>._check_datasetP  s    *S>!9::6JJJ&>((999999>((00000((11}}X&&&&&&&r   )r!  )r^   rD   )r]   rC   r^   r]   r  )r  r   NNNr   r  r  )r]   r   rh   z#Unsupported cast from int64 to listr   r   )r@   r   rs   rt   r>   r+  r   rT   r`   r  list_r   r  r  r  r  r   )r  r   r   r  r>   rf  rT   s   ``     r   test_specified_schemarj  K  s&   H999<<<8899EN5'N2333' ' ' ' ' ' ' FHN68U\BBBB \FHN68$$$ Y(.9::Fxyyy1#sDDDHN68$$$ Y'((FxC5111HN68$$$ Y788Fx"4"4"47CCCE"Cj* * *H N68$$$ Y(89::FjW~566vFFFGxs11s%"Cj* * *H N68$$$ Ybhrxzz2235EFGGFjW~566vFFFG>  (((((	*B
D 
D 
D ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s   I44I8;I8c                 "   | dz  }t          j        dg di          }t          j        ||           t          j        dt          j                    fg          }t          j        t          |          gdz  |          }|j        	                    |          sJ |
                    |          }t          j        t          d          5  |                                }|                                 d d d            d S # 1 swxY w Y   d S )Nr,  r]   r  d   rh   z#Unsupported cast from int64 to nullr   )r@   r   rs   rt   r>   r  r   rT   r`   r  r   r  r  r  rN  rO  )r  r   fnr   r>   rT   r   r   s           r   test_incompatible_schema_hangrn    sI    
>	!BHc999%&&EN5"Ybgii()**Fj#b''S888G>  ((((($$W--G	*B
D 
D 
D  ""$$                 s   )DDDc                    t          j        t          j        g dd          t          j        g dd          d          }t          | dz            }t          j        |          5 }t          j        ||j                  }|                    |                                d                    |	                                 d d d            n# 1 swxY w Y   t          j        |t          j                    	          }|                    |          }|                    |          sJ t          |           d
D ]D}t          j        ||	          }|                    |          }|                    |          sJ Ed S )Nr  r  r  rf  rD   r\   z
test.arrowr   r  )r  arrow)r@   r   r+  r`   output_streamRecordBatchFileWriterr>   write_batchr   r8  r   rT   r  r   r  rW   )	r  r   r   rx   r=  r>  rT   r7  
format_strs	            r   test_ipc_formatru    s   H28IIIF;;;8LLLyAAAC C D DE w%&&D		$		 4)$==5++--a0111              
 jb&6&8&8999G$$W--F==/888& $ $
*T*555((11}}U######$ $s   *ACCCc           	      f   ddl m} t          j        t          j        g dd          t          j        g dd          d          }t          | d	z            }|                    ||           t          j        |t          j	                    
          }t          |                                          }t          |d         t          j                  sJ |                    |          }|                    d           |                    |          sJ t#          |           t          j        |d
          }|                    |          }|                    d           |                    |          sJ |                    |dg          }|                    d           |                    |                    dg                    sJ |                    |dt          j        d          dz  i          }|                    d           |                    t          j        dt          j        g dd          i                    sJ |                    |          dk    sJ |                    |t          j        d          dk              dk    sJ d S )Nr   orcr  r  r  rf  rD   r\   test.orcr  T)fullrx  r^   r*   b2r   )r  r  g333333?r[   r]   r   r   )r   rx  r@   r   r+  r`   rt   r   rT   r  ro   rL   rN   FileFragmentr   validater  rW   r  rA   r   )r  r   rx  r   rx   rT   r  r7  s           r   test_orc_formatr~    s   H28IIIF;;;8LLLyAAAC C D DE w#$$DOOE4   jb&6&8&8999GW**,,--IilBO44444$$W--F
OOO==/888je,,,G$$W--F
OOO==$$Wse$<<F
OOO==se,,-----$$$ 12 %  F OOO==
$I>>>?@@     $$W--2222$$WRXc]]Q5F$GG1LLLLLLr   c                    ddl m} t          j        t          j        g dd          t          j        g dd          d          }t          | d	z            }|                    ||           t          j        |d
          }t          |
                    |                    }t          |          dk    sJ |d         j        dk    sJ |d                             |
                                d                   sJ d S )Nr   rw  r  r  r  rf  rD   r\   ry  rx  r  r   r[   )r   rx  r@   r   r+  r`   rt   r   rT   ro   r   r   r   r  )r  r   rx  r   rx   rT   r7  s          r   test_orc_scan_optionsr    s   H28IIIF;;;8LLLyAAAC C D DE w#$$DOOE4   je,,,G.++G4455Fv;;!!9""""!9E,,..q12222222r   c                      	 ddl m}  d S # t          $ rO t          j        t
          d          5  t          j        dd           d d d            Y d S # 1 swxY w Y   Y d S w xY w)Nr   r  z'not built with support for the ORC filer   r   rx  r  )r  r  r  r  r  r  r   rT   r  s    r   test_orc_format_not_supportedr    s    *11111111 * * *]G
 
 
 	* 	* Js5))))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	**s2   
 %A#AA#A	A#A	A#"A#c            	         t          j        t          d          5  t          j        t          j        dt          d          i          dd           d d d            n# 1 swxY w Y   t          j                    } t          j        t          d          5  | 	                                 d d d            d S # 1 swxY w Y   d S )Nz9Writing datasets not yet implemented for this file formatr   r]   r   rx  z/tmp)rl   r  )
r  r  r  r   r  r@   r   r0   r  make_write_options)ofs    r   +test_orc_writer_not_implemented_for_datasetr    sQ   	I
 
 
 
 
 	Hc599%&&uv	
 	
 	
 	
	
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
			B	I
 
 
     		                                   s#   9A!!A%(A%B<<C C c                 J   t          j        t          j        g dd          t          j        g dd          d          }t          | dz            }|                                                    |d	           t          j        |t          j                    
          }|	                    |          }|
                    |          sJ t          |           t          j        |d
          }|	                    |          }|
                    |          sJ d S )Nr  rC   r  rf  rD   r\   test.csvFr'   r  r  )r@   r   r+  r`   r  to_csvr   rT   r  r   r  rW   )r  r   r   rx   rT   r7  s         r   test_csv_formatr    s   H28IIIG<<<8LLLyAAAC C D DE w#$$D	OOT///jb&6&8&8999G$$W--F==/888je,,,G$$W--F==r   compression)bz2gziplz4zstdc                     t           j                            |          s't          j        d                    |                     t          j        t          j        g dd          t          j        g dd          d          }t          j
                    }|dk    r|nd	}t          | d
| z            }|                    ||          5 }|                                                    d          }|                    |                    d                     d d d            n# 1 swxY w Y   t#          j        |t#          j                              }	|                    |	          }
|
                    |          sJ d S )Nz{} support is not builtr  rC   r  rf  rD   r\   r  gzz	test.csv.r  Fr  r[  r  )r   Codecis_availabler  skiprl   r@   r   r+  ri   r|   r`   rn   r  r  writerj  r   rT   r  r   r  )r  r  r   r   r   suffixrx   r=  csv_strrT   r7  s              r   test_csv_format_compressedr    s    =%%k22 C-44[AABBBH28IIIG<<<8LLLyAAAC C D DE#%%J'611[[tFw-V---..D		&	&t	&	E	E , //##***77

7>>'**+++, , , , , , , , , , , , , , , jb&6&8&8999G$$W--F==s   AD00D47D4c           	         t          | dz            }t          |d          5 }|                    d           d d d            n# 1 swxY w Y   t          j        |d          }|                    |          }|                    t          j        dt          j	        g d          i                    sJ t          j        |t          j
        t          j                            d	          
                    }|                    |          }|                    t          j        dt          j	        ddg          i                    sJ t          j        |t          j
        t          j                            dg          
                    }|                    |          }|                    t          j        dt          j	        g d          i                    sJ d S )Nr  wzskipped
col0
foo
bar
r  r  skipped)col0r  r  r   )r  r  r  r  r  rQ  )r  r  r  r  )r`   rM   r  r   rT   r   r  r@   r   r+  r  r  r
  )r  r   rx   r=  rT   r7  s         r   test_csv_format_optionsr  6  s0   w#$$D	dC 0D

.///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0je,,,G$$W--F==
)RX&<&<&<==>??A A A A A jb&6V''!'44'6 '6 '6 7 7 7G$$W--F==628UEN+C+C"DEEFFFFFjb&6V''eW'=='? '? '? @ @ @G$$W--F==
%"C"C"CDDEFFH H H H H H H   AA	A	c           
      r   t          | dz            }t          |d          5 }|                    d           d d d            n# 1 swxY w Y   t          j        |t          j        t          j                            d                              }|	                    |          }g d}|j
        |k    sJ |                    t          j        t          j        d	g          t          j        d
g          t          j        dg          t          j        d	g          d                    sJ d S )Nr  r  z1,a,true,1
T)autogenerate_column_namesr  r  )f0r  r  r	  r   r]   )r`   rM   r  r   rT   r  r@   r  r
  r   rQ  r  r   r+  )r  r   rx   r=  rT   r7  expected_column_namess          r   (test_csv_format_options_generate_columnsr  K  s{   w#$$D	dC #D

>"""# # # # # # # # # # # # # # # jb&6V''$'GG'I 'I 'I J J JG$$W--F444"77777==1#)+3%)+4&)9)9)+1##8 #8 9 9 : : : : : : :r  c           	         t          | dz            }t          |d          5 }|                    d           d d d            n# 1 swxY w Y   t          j        |d          }t
          j                            dgd          }t          j        |t          j        
                    d	
                    }|                    ||          }|                    t          j        dt          j        g d          i                    sJ t          j        |          }t          j        ||          }|                    |          }|                    t          j        dt          j        g d          i                    sJ t          j                    }|                    ||          }|                    t          j        dt          j        g d          i                    sJ d S )Nr  r  zcol0
foo
spam
MYNULL
r  r  MYNULLT)null_valuesr  r  r  )r  r  )fragment_scan_optionsr  )r  spamNr  )r  r  r  )r`   rM   r  r   rT   r   r  r  r  r@   r
  r   r  r   r+  r  )	r  r   rx   r=  rT   r  r   r7  rn  s	            r   test_csv_fragment_optionsr  [  s.   w#$$D	dC 0D

.///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0je,,,Gk00hZEI 1 K KO''V''5'99; ; ;G $$WG$LLF==6284I4I4I+J+J"KLLMMMMM!/BBBJjj111G$$W--F==6284I4I4I+J+J"KLLMMMMM'))G$$WG$LLF==
&"(#<#<#<==>??A A A A A A Ar  c                    t          j        t          j        g dd          t          j        g dd          d          }t          | dz            }|                                                    d	          d
d                             dd          }t          |d          5 }|                    |           d d d            n# 1 swxY w Y   t          j
        |t          j                              }|                    |          }|                    |          sJ t          |           t          j
        |d          }|                    |          }|                    |          sJ d S )Nr  rC   r  rf  rD   r\   	test.jsonrecordsorientr   r  },{}
{r  r  r  )r@   r   r+  r`   r  to_jsonreplacerM   r  r   rT   r  r   r  rW   r  r   r   rx   ry   r  rT   r7  s           r   test_json_formatr  s  s   H28IIIG<<<8LLLyAAAC C D DE w$%%D
//


#
#9
#
5
5ad
;
C
CE6
R
RC	dC A	               jb&7&9&9:::G$$W--F==/888jf---G$$W--F==s   *CCCc           	         t          j        t          j        g dd          t          j        g dd          d          }t          | dz            }|                                                    d	          d
d                             dd          }t          |d          5 }|                    |           d d d            n# 1 swxY w Y   t          j
        t          d          5  t          j        |t          j        t           j                            d                              }d d d            n# 1 swxY w Y   t          j        |t          j        t           j                            d                              }|                    |          }|                    |          sJ d S Nr  rC   r  rf  rD   r\   r  r  r  r   r  r  r  r  ztry to increase block sizer   r   r  r  r  @   )r@   r   r+  r`   r  r  r  rM   r  r  r  r  r   rT   r  r  r
  r   r  r  s           r   test_json_format_optionsr    s2   H28IIIG<<<8LLLyAAAC C D DE w$%%D
//


#
#9
#
5
5ad
;
C
CE6
R
RC	dC A	               
z9
; 
; 
; = =*T"*;,,,::+< +< +< = = == = = = = = = = = = = = = = =
 jb&7W((B(77'9 '9 '9 : : :G$$W--F==s%   *CCC2AEE
E
c                    t          j        t          j        g dd          t          j        g dd          d          }t          | dz            }|                                                    d	          d
d                             dd          }t          |d          5 }|                    |           d d d            n# 1 swxY w Y   t          j
        t          d          5  t          j        t           j                            d                    }t          j        |t          j        |                    }d d d            n# 1 swxY w Y   t          j        t           j                            d                    }t          j        |t          j        |                    }|                    |          }|                    |          sJ d S r  )r@   r   r+  r`   r  r  r  rM   r  r  r  r  r   r  r  r
  rT   r  r   r  )	r  r   r   rx   ry   r  r   rT   r7  s	            r   test_json_fragment_optionsr    sW   H28IIIG<<<8LLLyAAAC C D DE w$%%D
//


#
#9
#
5
5ad
;
C
CE6
R
RC	dC A	               
z9
; 
; 
; F F,,,,::< < <*T"*;G*D*DEEE	F F F F F F F F F F F F F F F (W((B(779 9 9Gjb&7&@&@AAAG$$W--F==s%   *CCC2AEE!Ec                    t          | dz            }dD ]*\  }}t          |d          5 }|                    |           d d d            n# 1 swxY w Y   t          j        dt          j                    fdt          j                    fg          }t          j        dgdgd|	          }t          j                            |
          }t          j
        |          }	t          j        ||	          }
|
j                            |          sJ |
                                                    |          sJ ,d S )Nr  ))latin-1s   a,b
un,lphant)utf16s    a , b 
 u n ,  l  p h a n t wbr]   r^   un
   éléphantr\   rh   encodingr  r  )r`   rM   r  r@   r>   rE   r   r  r
  r   r  rT   r  r   )r  r   rx   r  
input_rowsr=  r!  expected_tabler  r   dataset_transcodeds              r   test_encodingr    s   w#$$D! D D* $ 	#JJz"""	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# )c29;;%7#ry{{9K$LMM)5#8 #8@OQ Q Q v))8)<<&LAAAZ[AAA!(//@@@@@!**,,33NCCCCCC%D Ds   AA	A	c                 D   t          | dz            }t          |d          5 }|                    d           d d d            n# 1 swxY w Y   t          j        dt          j                    fdt          j                    fg          }t          j        dgdgd|	          }t          j        |d
|          }t          j
        t          j        j        d          5  |                    |           d d d            n# 1 swxY w Y   t          j                            d          }t          j        |          }t          j        ||          }	|	j                            |          sJ |	                                                    |          sJ d S )Nr  r  s   ,b
un,lphant   ér^   r  r  )r  r^   rh   r  r  zinvalid UTF8r   r  r  r  r  )r`   rM   r  r@   r>   rE   r   r   rT   r  r  r   rK  rK  r   r  r
  r  r  )
r  r   rx   r=  r!  r  rT   r  r   r  s
             r   test_column_names_encodingr    s
   w#$$D	dD		 1T

/0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 i$	!4sBIKK6H IJJOXdV%1N4 4<KM M MN jeODDDG	w{/~	F	F	F ) )((() ) ) ) ) ) ) ) ) ) ) ) ) ) ) 6%%y%99L"===KD===$++O<<<<<&&((//???????s#   AA	A	#DD	D	c                    ddl m} t          j        t          j        g dd          t          j        g dd          d          }| d	z  }|                                  ||t          |d
z                       t          j        |t          j	                              }|
                    |          }|                    |          sJ t          |           t          j        |d          }|
                    |          }|                    |          sJ |
                    |ddg          }|j        ddgk    sJ |
                    |ddg          }|j        ddgk    sJ  ||t          |dz            d           t          j        t                     5  |
                    t          j        |d                     d d d            d S # 1 swxY w Y   d S )Nr   )r  r  r  r  rf  rD   r\   feather_datasetr  r  r  r^   r]   r*   zdata1.featherr   version)pyarrow.featherr  r@   r   r+  r  r`   r   rT   r  r   r  rW   rQ  r  r  r  )r  r   r  r   r^  rT   r7  s          r   test_feather_formatr    sg   ------H28IIIF;;;8LLLyAAAC C D DE ))GMMOOOM%W~566777j)9););<<<G$$W--F==/888j333G$$W--F== $$WsCj$AAF3*,,,,$$WsCj$AAF3*,,,, M%W677CCCC	z	"	" G G
79 E E EFFFG G G G G G G G G G G G G G G G G Gs   **G!!G%(G%)r  r  brotlic                    t          j        t          j        dgdz  d          t          j        g ddz  d          d          }t           j                            |          st          j                     | d	z  }|                                 t          j	                    }| d
z  }|                                 t          j
        |t          |dz            ||                    d                      |dk    rt          j        t          d          5  |                    |          }d d d            n# 1 swxY w Y   t          j        t          d          5  t          j        |          }|                    |          }d d d            n# 1 swxY w Y   d S |                    |          }t          j
        |t          |dz            ||           t          j        |t          j	                              }	|                    |	          }
|
                    |          sJ |dz  dz  }|                                j        }|dz  dz  }|                                j        }||k     sJ d S )Nr   ,  r  r  rf  rl  rD   r\   feather_dataset_compressedfeather_dataset_uncompressedz
data.arrowr  rl   file_optionsr  zCompression typer   r  part-0.arrow)r@   r   r+  r  r  r  r  r  r   r  r  r`   r  r  r  rT   r   r  statst_size)r  r  r   r   r^  r   uncompressed_basedirwrite_optionscodecrT   r7  compressed_filecompressed_sizeuncompressed_fileuncompressed_sizes                  r   test_feather_format_compressedr    s3    H28QCG&9998LLL$49EEEG G H HE8  -- 44GMMOOO"$$K"%CC    </00 333EE	    h]:-?@@@ 	) 	)'::' ; ) )M	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) ]:-?@@@ 	N 	NH[))E'::u:MMM	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	22{2KKMGl"##"	    j)9););<<<G$$W--F==,~=O%**,,4O,|;nL)..008.......s$   E  EE&+FF!$F!c                 :   g }t          d          D ]X}t          j        |gdz  d t          d          D             d          }t          j        |t          |           |           Yt          | dz            }t          j        |j        ||           ||fS )zO
    Creates a simple (flat files, no nested partitioning) Parquet dataset
    r   r   c                 4    g | ]}t          j                     S r   r  r  s     r   rc   z2_create_parquet_dataset_simple.<locals>.<listcomp>B  s    0T0T0TQ0T0T0Tr   r  metadata_collector	_metadata)r0   r@   r   rs   r  r`   write_metadatar>   )	root_pathr  r;   r   metadata_paths        r   _create_parquet_dataset_simpler  :  s    
 1XX 
 
b0T0T%PR))0T0T0TUUVV
3y>>6H	
 	
 	
 	
 	
 	K/00Mm-    %r   c                    | dz  }t          |          \  }}t          j        |          }|j                            |j                  sJ t          |j                  dk    sJ |                                }|j        dk    sJ d S )Nr  r   (   )	r  r   parquet_datasetr>   r  r   r	  r   r   )r  r  r  r   rT   r7  s         r   test_parquet_dataset_factoryr  P  s     00I9)DDM5 //G>  .....w}""""F?b      r   win32z'Results in FileNotFoundError on Windows)reasonc                    t          j        d          }| dz  }t          |          \  }}|                    d          }t	          j        t	          j        |                    }t          j        ||          }|j	        
                    |j	                  sJ t          |j                  dk    sJ |                                }|j        dk    sJ d S )Nr6  r  r7  r  r   r  )r  r%  r  r   ri   r}   r!  r   r  r>   r  r   r	  r   r   )	r  r6  r  r  r   r<  r   rT   r7  s	            r   #test_parquet_dataset_factory_fsspecr  \  s      **F 00I9)DDM5 !!&))I !1)!<!<==J :FFFG>  .....w}""""F?b      r   c                    | dz  }t          j        dgdz  t          j                            d          d          }g }t          j        |t          |          |           t          |dz            }t          j        |j	        ||           t          j        |          }|j	                            |j	                  sJ |                                }|j        dk    sJ d S )Nr  r   r   r  r  r  )r@   r   rS  r  randnrs   r  r`   r  r>   r   r  r  r   r   )r  r  r   r  r  rT   r7  s          r   &test_parquet_dataset_factory_roundtripr  t  s     00IHQC"HBIOOB,?,?@@AAEs9~~2D    	K/00Mm-     //G>  .....F?b      r   c                 Z   g }t          d          D ]y}t          j        dt          t          |dz  |dz   dz                      i          }| | dz  }t	          j        |||           |d                             | d           zt          | dz            }t	          j        |j	        ||           t          j        |          }|                                }|                    d                                          }|t          t          dd	                    k    sJ d S )
Nr   r  r   z.parquetr  r  r  r   rl  )r0   r@   r   ro   rs   rt   set_file_pathr`   r  r>   r   r  r   r-  rR  )	r  	metadatasr;   r   
table_pathr  rT   scanned_tablescanned_cols	            r   "test_parquet_dataset_factory_orderr     s/    I 2YY 4 44adQqS"H--../1 1!~~~-

ujYGGGG"##qNNN3333+-..MelM9=== //G$$&&M&&t,,6688K$uQ}}--------r   c                    | dz  }t          |          \  }}t          |                    d                    d                                          t	          j        |          }|j                            |j                  sJ t          |j	                  dk    sJ t          j        t                    5  |                                 d d d            d S # 1 swxY w Y   d S )Ntest_parquet_dataset_invalid	*.parquetr   r   )r  ro   globunlinkr   r  r>   r  r   r	  r  r  r  r   )r  r  r  r   rT   s        r   $test_parquet_dataset_factory_invalidr    s    88I9)DDM5	$	$%%a(//111 //G>  .....w}""""	(	)	)                   s   :CC #C c                    t          t          |                     d                              }t          j        |d                   j                                        }g }|D ]e}t          j        |          j        }|                    t          |
                    |                                |                    |           f| dz  }t          j        |||           |S )Nr  r   r  r  )ro   r3  rglobrs   ParquetFiler>   to_arrow_schemar  r  r`   r  r1   r  )r  parquet_pathsr>   r  rx   r  r  s          r   _create_metadata_filer    s    	 < <==>>M^M!,--4DDFFF , ,>$''0s4#3#3I#>#>??@@@!!(+++++M2D    r   c           
         t          j        t          j        t          d                    t          j        t          j                            d                    t          j        t	          j        ddgd                    gg d          }|                    ddi          }t          j
        |t          |           d	g
           t          |           |fS )Nr  r]   r^   r   r  r  r   r(   r   r  )r@   r   r+  r0   rS  r  r  rR  rH   rs   r  r`   r  )r  r   s     r   #_create_parquet_dataset_partitionedr    s    H
rRXbioob&9&9::
C:r**++- #""  E
 ))5'*:;;Es9~~vhGGGG ++U22r   c                     | dz  }t          |          \  }}t          j        d          }t          j        ||          }|j                            |j                  sJ t          |j                  dk    sJ |                                }|j	        dk    sJ |
                                                    d                              d	          }|
                                }t          j                            ||           d S )
N(test_parquet_dataset_factory_partitionedr   r  r  r   r  r  Tdrop)r  r   r   r  r>   r  r   r	  r   r   r  sort_valuesreset_indexr4   testingassert_frame_equal)r  r  r  r   r   rT   r7  rf  s           r   r  r    s    DDI>yIIM5?&111L \JJJG>  .....w}""""F?b     ++D11==4=HHF  HJ!!&(33333r   c                 0   | dz  }t          |          \  }}t          j        |d          }|j                            |j                  sJ d|j        j        v sJ t          |                                          }d|d         j        j        v sJ d S )N%test_parquet_dataset_factory_metadatar   r     keyr   )	r  r   r  r>   r  r  ro   rL   r  )r  r  r  r   rT   r  s         r   r  r    s     AAI>yIIM5 VDDDG>  .....W^,,,,,W**,,--IYq\1:::::::r   c                 p   |\  }}| dz  }t          |          \  }} ||g          5  t          j        |t          j        d          |          }d d d            n# 1 swxY w Y    |g           5  t	          |                                          }d d d            n# 1 swxY w Y    |g           5  t	          |                    t          j        d          dk                         d d d            n# 1 swxY w Y    |g           5  |d                             t          j        d          dk               d d d            n# 1 swxY w Y    |g           5  |d                                         }	|	d                                          d d d            d S # 1 swxY w Y   d S )N#test_parquet_dataset_lazy_filteringr   r  )r   r   r     r   )	r  r   r  r   ro   rL   rA   r  r  )
r  r   ri   r   r  r  r^  rT   r  rg_fragmentss
             r   r  r    s     'B
 ??I5i@@M1 
}o	&	&  $777                 
b		 2 2..0011	2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 
b		 9 9W""28D>>B#6778889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 
b		 = =!''(;<<<= = = = = = = = = = = = = = = 
b		 3 3 |6688Q002223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3sY   +A  A$'A$6"B$$B(+B(:9C??DD2EEE)5F++F/2F/c                 H   t          j        dg di          }| dz  }|                    |           t          j        |          }|                    |          j        }|                    |dg          j        }d|j        v sJ |                    |d          sJ d S )Nr]   r  r  r*   s   pandasTr!  )	r4   r5   r  r   rT   r   r>   r  r  )r  r   rI   rx   rT   r>   rM  s          r   test_dataset_schema_metadatar    s     
sIII&	'	'B^#DMM$jG$$W--4F%..w.FFM ''''==)$=???????r   c                 T   t          j        dt          j        g dd          i          }t          j        |t          | dz                       t          j        dt          j                    fg          }t          j	        | dz  d|          }|
                    |t          j        d          dk    	          }|d                             |d                             d
                              d                    sJ t          |                                          d         }|
                    |t          j        d          dk    |          }|d                             |d                             d
                              d                    sJ d S )Nr  r  r   r  r,  r   r  r   r   rC   r   r  )r@   r   r+  rs   rt   r`   r>   rC   r   rT   r   rA   r  r  r  ro   rL   )r  r   r   r>   rT   filteredrU   s          r   test_filter_mismatching_schemar"  .  s    HeRXlllAAABCCEN5#g677888 Y

+,--Fj. 6C C CG
 &&wrx7J&KKHE?!!%,"3"3G"<"<"B"B1"E"EFFFFFG))++,,Q/H&&%1,V ' = =HE?!!%,"3"3G"<"<"B"B1"E"EFFFFFFFr   c                    t          j        d                                t          t	          d                    d          }t          | dz            }t          j        ||dg           t          j	        |d          }|
                    |          }|
                    |dg	          }|                    d                              |                    d                    sJ d S )
Nza a b br   r  r9  r   r  r   r  r*   )r@   r   r,  ro   r0   r`   rs   r  r   rT   r   r-  r  )r  r   r   rx   rT   all_cols	part_onlys          r   +test_dataset_project_only_partition_columnsr&  D  s     Hioo//U1XXGGHHEw'((DtVH====jF333G&&w//H''&'BBI??6"")))*:*:6*B*BCCCCCCCr   c           	         t          j        dt          j        g dd          i          }| dz  }|                    |d           t          j        |dt          j        dt          j	                    fg          	          }t          j
        dt          j        g dt          j	                              i          }|                    |                              |          sJ d S )
Nr  rh  objectdtypez(test_dataset_project_null_column.parquetr   r  r   r  )r4   r5   rS  r+  r  r   rT   r@   r>   rC   r   r   r  )r  r   rI   r  rT   rf  s         r    test_dataset_project_null_columnr+  T  s     
ubh'9'9'9JJJK	L	LB<<AMM!IM&&&j9 "	E28::+>*? @ @B B BGx);););RXZZ H HIJJH""7++228<<<<<<<r   c                    ddl m} t          j        g dg dg dd          }|                    || dz             t          j        | dz  d	          }|                    |t          j        d
          t          j        d          	                    dd          t          j        d          dk    d          }t          j        g dt          j
        g dd          g dd          }|                    |          sJ t          j        t          d          5  |                    |d
d
i           d d d            d S # 1 swxY w Y   d S )Nr   r  r  )r   r'  r(  r\  r  r  r  r  r  r  r   Fsafer  r]   )	A_renamedB_as_intC_is_ar*   r  )TFFzExpected an Expressionr   )r   r  r@   r   r  r   rT   r   rA   r  r+  r  r  r  r  )r  r   r  r   rT   r7  rf  s          r   test_dataset_project_columnsr3  b  s   H999<<<oooNNOOE%>!9:::j>1)DDDG$$WXc]]HSMM&&wU&;;(3--3&7 7$  F
 xYYHYYYW555&&&   H
 ==""""" 
y(@	A	A	A = =#s<<<= = = = = = = = = = = = = = = = = =s   %EEEc                    t          |           \  }}t          j        |          }t          |j        t          j                  sJ t          |           \  }}t          j        |          }t          |j        t          j                  sJ t          j        |d          }|j        }|J t          |t          j                  sJ |j        t          j        dt          j
                    fg          k    sJ t          |j                  dk    sJ |j        d         t          j        g dt          j
                              k    sJ t          j        t          j        dt          j
                    fg          d          }t          |t          j                  sJ t          |j                  dk    sJ t          d |j        D                       sJ t          j        ||          }|j        }t          |t          j                  sJ |j        t          j        dt          j
                    fg          k    sJ t          |j                  dk    sJ t          d	 |j        D                       sJ t          j        |d          }t          j        t!          |                                          |j        |j        |j        
          }|j        J | dz  }t)          |          \  }}t          j        |d          }|j        }|J t          |t          j                  sJ |j        t          j        dt          j                    fg          k    sJ t          |j                  dk    sJ t/          |j        d                                                   ddhk    sJ d S )Nr   r  r   r   r   )r   r   r   r  c              3      K   | ]}|d u V  	d S r   r   r   s     r   rr  z6test_dataset_preserved_partitioning.<locals>.<genexpr>  &      44QqDy444444r   c              3      K   | ]}|d u V  	d S r   r   r   s     r   rr  z6test_dataset_preserved_partitioning.<locals>.<genexpr>  r6  r   r   zdata-partitioned-metadatar]   r^   )r  r   rT   rN   r   r   r  r  r>   r@   r   r   r  r+  r  r  ro   rL   rl   r   r  r  rE   r   rR  )	r  r^  rx   rT   r  r   r  r  r  s	            r   #test_dataset_preserved_partitioningr8  z  s    "'**GAtjGg*B,DEEEEE 37;;JjGg*B,DEEEEE jF333GDdB/00000;")fbhjj%9$:;;;;;;t !!Q&&&&Q28IIIrxzz#B#BBBBB ?29vrxzz&:%;<<VLLLDdB/00000t !!Q&&&&44$"344444444jD111GDdB/00000;")fbhjj%9$:;;;;;;t !!Q&&&&44$"344444444 jF333G#W""$$%%gn~'*<  H  ((( 55I:9EEM1 VDDDGDdB/00000;")fbikk%:$;<<<<<<t !!Q&&&& t #--//00S#J>>>>>>r   c                    t          j        t          j        dt          j                              t          j        dt          j        t          j                    t          j                                        g          }t          j        g dt          t          d                    d|          }t          | dz            }t          j        ||dg           t          j        | dz            }|                    d                                          |                    d                                          k    sJ |                    d                              |                    d                    sJ d S )	Nr  r   )NNr]   r]   r   r  rh   r9  r  )r@   r>   rA   rC   r  r   rE   r   ro   r0   r`   rs   r  rU  r-  rR  r  )r  r>   r   rx   actual_tables        r   +test_write_to_dataset_given_null_just_worksr;    sa    Y


##
rxzz29;;??@@  F H444!%((^^- -5;= = =E w'((DtVH=====>!9::L v&&00  	f			'	'	)	)* * * *u%%,,U\\%-@-@AAAAAAAr   c                 v    dd l m}  |j        |  |j        |dfg                    } |j        | |          S )Nr   	ascending)r   )pyarrow.computecomputesort_indicesSortOptionsr   )tabsort_colr  sorted_indicess       r   _sort_tablerE    s\          $R_^R^h%<$=>>@ @ @N273'''r   c                    |p|}t          j        | |d|d           t          |                    d                    }t	          |          t	          |          k    sJ t          j        |d|          }t          |                                |                              t          |                                 |                    sJ d S )Nrp  Frl   r   r   *r  )	r   r  ro   r  r   rT   rE  r   r  )rT   r  expected_filesrC  base_dir_pathr   
file_pathsr  s           r   _check_dataset_roundtriprL    s    !-XMWhw".EC C C C m))#..//Jz??c.111111 zgLB B BH x((**H55<<G$$&&113 3 3 3 3 3 3r   c                    | dz  }|                                  t          |          }t          j        |          }| dz  }|dz  g}t	          |t          |          |d|           | dz  }|dz  g}t	          |||d|           | dz  }|                                  t          |          }t          j        |          }| dz  }|dz  g}t	          |t          |          |d|           d S )Nr  zsingle-file-targetr  r]   zsingle-file-target2r  zsingle-directory-target)r  r  r   rT   rL  r`   r  )r  rw   r^  rT   targetrI  s         r   test_write_datasetrO    s    -'IOOI&&Aj##G ++F~-.NWc&kk>3OOO ,,F~-.NWfnc6JJJ ,,IOO"9--Aj##G00F~-.NWc&kk>3OOOOOr   c                 l   | dz  }t          |          }t          j        d          }t          j        ||          }| dz  }|dz  |dz  dz  |dz  |dz  dz  g}t          j        t	          j        d	t	          j                    fg          d          }t          |t          |          |d
||           | dz  }|dz  |dz  dz  |dz  |dz  dz  g}t          j        t	          j        d	t	          j                    fg                    }t          |t          |          |d
||           d S )Npartitionedr   r  r  zpartitioned-hive-targetpart=ar  part=br   r  partitioned-dir-targetr]   r^   )	r  r   r   rT   r@   r>   rE   rL  r`   )r  rw   r^  r   rT   rN  expected_pathsr  s           r   test_write_dataset_partitionedrV    sz    -'I+I66A?&111Lj>>>G 00F6H,~=6H,~=N /
	FBIKK()**6; ; ;VndF(* * * *
 //Ffsl^3fsl^3N /
	FBIKK()**, ,VndF(* * * * * *r   c                 4    t          j        g dg dd          }t          j        | ddg           t          j         ddg          }|j        } fd|D             }|h dk    sJ |                                }|                    |          sJ d S )	NrH  rJ  r\   r  r^   r  c                     h | ];}t          t          j        |                                        j                  <S r   r`   r  r  r  r  ra   r  r  s     r   r   z6test_write_dataset_with_field_names.<locals>.<setcomp>5  G       =>GLOO''00788  r   >   r   rI  r  r@   r   r   r  rT   r	  r   r  r  r   r  r	  partitioning_dirsr  s   `     r   #test_write_dataset_with_field_namesr_  -  s    H+++///BBCCEUGE#&%) ) ) ) 
75uEEEIOE   BG   ////((**O!!%(((((((r   c                 4    t          j        g dg dd          }t          j        | ddgd           t          j         dd          }|j        } fd	|D             }|h d
k    sJ |                                }|                    |          sJ d S )NrH  rJ  r\   r  r^   r   )rl   r   partitioning_flavorr  c                     h | ];}t          t          j        |                                        j                  <S r   rY  rZ  s     r   r   z;test_write_dataset_with_field_names_hive.<locals>.<setcomp>F  r[  r   >   b=xb=yb=zr\  r]  s   `     r   (test_write_dataset_with_field_names_hiverf  >  s    H+++///BBCCEUGE#&%VE E E E 
75vFFFIOE   BG    5 5 55555((**O!!%(((((((r   c                 L   t          j        g dg dg dd          }t          j        || ddg           t          j        | ddg          }t          j                    5 }t          j        |                    ddg	          |ddg           t          j        |ddg          }|                                }t          |
                                          |                    d
          
                                k    sJ 	 d d d            d S # 1 swxY w Y   d S )NrH  rJ  r  r\  r  r^   r  r  r*   r]   )r@   r   r   r  rT   r  r  r   r   r  r2  drop_columnsr  r   rT   tempdir2r  r  s         r   test_write_dataset_with_scannerrk  O  s   H+++///$99& & ' 'E UGE#&%) ) ) ) jcUCCCG		$	&	& =(
#s<<!%se	E 	E 	E 	E JxSEJJJ	#,,..O--//  ++C00::<<= = = = == = = = = = = = = = = = = = = = = =s   #B(DD Dc           	         	
 t          j                     G fddt                    }t          j         |t          j                                        t          j        t          j        dt          j	                              g          }t          j
        t          j        t          t          d                              g|          	d
d}dd		
fd
}t          j                             |            |d	          t          j         fd          }|                                 	 t'          j                    fd}d}d} |            dk     r3
|k    r
|k    rd	}n$
}t'          j        d            |            dk     3|sJ 	 d                                 |                                 d S # d                                 |                                 w xY w)Nc                       e Zd Z fdZdS )6test_write_dataset_with_backpressure.<locals>.GatingFsc                 d                                      | j                            ||          S )Nr  )waitr   rn   )r   rx   r  consumer_gates      r   rn   zItest_write_dataset_with_backpressure.<locals>.GatingFs.open_output_streamk  s0       8..th.GGGr   N)r   r   r   rn   )rr  s   r   GatingFsrn  j  s5        	H 	H 	H 	H 	H 	H 	Hr   rs  r:   r  rh   r          Tc               3   j   K   k     r)sd S t          j        d           dz   V  k     'd S d S )Ng{Gz?r   )r  sleep)rz   batches_readend
keep_goings   r   counting_generatorz@test_write_dataset_with_backpressure.<locals>.counting_generatorz  s[      S   JtALKKK S      r   r  c                  N    t          j        t                    d           S )Nr   r  )r   r  r`   )	gating_fsr   r  s   r   r  z6test_write_dataset_with_backpressure.<locals>.<lambda>  s+    r'S\\)	K K K r   )rN  c                  0    t          j                      z
  S r   )r  )starts   r   durationz6test_write_dataset_with_backpressure.<locals>.duration  s    9;;&&r   Fr   r  )	threadingEventr   ri   r}   r|   r@   r>   rA   r   rq   r+  ro   r0   r   rJ  rr   Threadr  r  rw  r   r  )r  rs  r>   min_backpressurer{  write_threadr  
last_valuebackpressure_probably_hitrz   rx  rr  ry  r}  rz  r   r  s   `        @@@@@@@@r   $test_write_dataset_with_backpressurer  b  s    O%%M
H H H H H H H< H H H
 );)=)= > >??IY44566F ORXd5+;+;&<&<==>vNNNEL
CJ        j%%V & ? ?G #K K K K K KL L LL !		' 	' 	' 	' 	' 
$)!hjj2oo...:--04-)
JsOOO hjj2oo )((((
 
 
s   A!G ,G?c                    t          j        g dg dd          }t          j        || ddg           t          j        | ddg          }t          j                    5 }t          j        ||ddg           t          j        |ddg          }|                                }t          |	                                          |	                                k    sJ 	 d d d            d S # 1 swxY w Y   d S )NrJ  r  r^   r  r  r^   r  )
r@   r   r   r  rT   r  r  r   r  r2  ri  s         r   test_write_dataset_with_datasetr    se   H???;;<<EUGE#&%) ) ) ) jcUCCCG		$	&	& F(
( %SE	; 	; 	; 	; JxSEJJJ	#,,..O--//00EOO4E4EEEEEEF F F F F F F F F F F F F F F F F Fs    A?C--C14C1c           	      b   | dz  }t          j        g dg dd          }t          j        t          j        t          j        dt          j                              g          d          }d }t          j        |||d	
           t          j        g dg dd          }t          j	        t           j
                  5  t          j        |||d	
           d d d            n# 1 swxY w Y   t          j        ddgi          }|dz  dz  }t          j                            ||           t          j        |||d	d           t          j        g dg dd          }t          j        | d	|                                          } |||           |                                sJ t          j        |||d	d           t          j        g dg dd          }t          j        | d	|                                          } |||           |                                rJ d S )Nr   rJ  r  r  r  r   )r>   r  c                     |                                                      d                              d          }|                                                     d                              d          }|                    |          sJ d S )Nr^   Tr  )r  r  r  r  )r  r  df1df2s       r   compare_tables_ignoring_orderzGtest_write_dataset_existing_data.<locals>.compare_tables_ignoring_order  sv    llnn((--99t9DDllnn((--99t9DDzz#r   r  r  r\  r  r^   ezc=2z	foo.arrowoverwrite_or_ignore)r   rl   existing_data_behavior)r  r   r]   r^   r  )r   r   r   r[   r   r  delete_matching)r   r]   r^   r  r  )r@   r   r   r   r>   rA   rC   r  r  r  rK  r   r  r  rT   r   exists)	r  rw   r   r   r  extra_table
extra_fileoverwrittenreadbacks	            r    test_write_dataset_existing_datar    s   $IH???;;<<E?")	#rxzz	"	"#+% +%-35 5 5L   UILOOOOH???;;<<E 
r	'	' B B
	&25	B 	B 	B 	BB B B B B B B B B B B B B B B (C#<((KU"[0JO!!+z::: UIL!,AC C C C ('''ooo>>@ @Kz'%'35 5 55=XZZ !!(K888 UIL!:KM M M M (!5!5!5LLLIIJJKz'%'35 5 55=XZZ !!(K888  """""""s   C''C+.C+r   r   r   c                 >    fdt          |           D             S )Nc                 :    g | ]}t          j                  S r   )r  randint)ra   r^  r  r  s     r   rc   z._generate_random_int_array.<locals>.<listcomp>  s%    :::FN3$$:::r   r  r@  r  r  s    ``r   _generate_random_int_arrayr    s'    :::::eDkk::::r   c                     g }g }t          |           D ]L}|                    t          |d|                     |                    dt          |          z              Mt	          j        ||          }|S )Nr   r  r  r:   r  )r0   r1   r  r`   r@   rq   )num_of_columnsnum_of_recordsr:   rQ  r;   rq   s         r   _generate_data_and_columnsr    s    DL>"" * *.N343AC C C 	D 	D 	D 	C#a&&L))))?LAAALr   c                     t          t          t          j        |                               d|                               S )Nz**/*.)r   ro   r  r  r  base_directoryr   s     r   _get_num_of_files_generatedr    s9    tGL00556Kk6K6KLLMMNNNr   c                 F   | dz  }dd}d}d}t          ||          }t          j        ||d|           t          j        |          }|z  dz   }t          |          |k    sJ g }t          |          D ]_\  }	}
|t          |
          z  }t          j        |d          }|	                    |
                                j        d	                    `|t          |          k    sJ |t          |          k    sJ t          fd
|D                       sJ d S )Nr   r   r   #   r   )rl   max_rows_per_filemax_rows_per_groupr   r  r   c              3   $   K   | ]
}|k    V  d S r   r   )ra   file_rowcountr  s     r   rr  z7test_write_dataset_max_rows_per_file.<locals>.<genexpr>&  s=       < <   11 < < < < < <r   )r  r   r  r[  r_  r   rk   r`   rT   r1   r   shaper  r  )r  rw   r  r  r  rq   files_in_direxpected_partitionsresult_row_combinationr^  f_filef_pathrT   r  s                @r   $test_write_dataset_max_rows_per_filer    s   $INN-n.<> >L \9Y'8(:< < < < :i((L ),==A | 33333  |,, C C	6S[[(*VI666%%g&6&6&8&8&>q&ABBBB #&<"="=====S!7888888 < < < <$:< < < < < < < < < <r   c                    | dz  }d}d}dg d}fd|D             }|dz  }t          j        ||||d	           t          j        |          }t	          |          D ]\  }}	|t          |	          z  }
t          j        |
d
          }|                                }|                                }t	          |          D ]9\  }}|j	        }|t          |          dz
  k     r||k    r||k    sJ 1||k    sJ :d S )Nr   r  r<  r   )
r   r   r   r   r   r   r   r   r   r   c                 0    g | ]}t          |          S r   )r  )ra   r  r  s     r   rc   z9test_write_dataset_min_rows_per_group.<locals>.<listcomp>3  s:     : : :( 11?A A : : :r   min_rows_groupr   )r:   r  min_rows_per_groupr  rl   r  r   )r   r  r[  r_  rk   r`   rT   r   r   r   r   )r  rw   r  r  record_sizesrecord_batchesdata_sourcer  r^  r  r  rT   r   batchesr  rz   rows_per_batchr  s                    @r   %test_write_dataset_min_rows_per_groupr  *  ss   $IN111L: : : :,8: : :N ..K.;(:(:%' ' ' '
 :k**L|,, < <	6s6{{**VI666  """"$$"7++ 	< 	<IB"^NCLL1$$$%);;;"&888888%);;;;;;	<< <r   c                    | dz  }d}d}d}t          ||          }|dz  }t          j        |||d           t          j        |          }g }|D ]q}	|t          |	          z  }
t          j        |
d          }|                                }|                                }|D ]}|	                    |j
                   r|dd	gk    sJ d S )
Nr   r  r      max_rows_groupr   )r:   r  r  rl   r  r^  )r  r   r  r[  r_  r`   rT   r   r   r1   r   )r  rw   r  r  r  rq   r  r  batched_datar  r  rT   r   r  rz   s                  r   %test_write_dataset_max_rows_per_groupr  N  s   $INN-n.<> >L ..K,(:%' ' ' ' :k**LL 0 0s6{{**VI666  """"$$ 	0 	0E////	0 B8######r   c                    | dz  }d}d}ddg}t          j        g dg dg|          }t          j        g d	g d
g|          }t          j        g dg dg|          }t          j        g dg dg|          }t           j                            ||||g          }	t	          j        t          j        ||         t          j                    fg          d          }
|dz  }t	          j        |	||
|           d } |||||          \  }}||k    sJ |dz  }d}t	          j        |	||
||d            |||||          \  }}||k    sJ d S )Nr   r   r   c1c2)r   r   r[   r   r   r   )r]   r^   r  r  r  r]   r  )r   r  r?  r<  r   r   )r]   r^   r  r  r  r  )r  r   r_  r^  r   r   )r]   r^   r  r  r  r  )r}  r  r  r;  r   r   )r]   r^   r  r  r  r^   r   r  default)r:   r  r   rl   c                     t          | |          }t          t          j                            ||                             }||fS )Nr  )r  r   r@   r?  unique)r  rq   r   col_idnum_of_files_generatednumber_of_partitionss         r   _get_compare_pairz<test_write_dataset_max_open_files.<locals>._get_compare_pair  sN    !<&K"A "A "A"2:#4#4\&5I#J#JKK%';;;r   max_1r[   F)r:   r  r   rl   max_open_filesr   )	r@   rq   rF   rr   r   r   r>   rE   r  )r  rw   r   partition_column_idrQ  record_batch_1record_batch_2record_batch_3record_batch_4r   r   data_source_1r  r  r  data_source_2r  s                    r   !test_write_dataset_max_open_filesr  k  sH   $IK$<L_+>+>+>+I+I+I+K+79 9 9N _+=+=+=+I+I+I+K+79 9 9N _+@+@+@+I+I+I+K+79 9 9N _+A+A+A+I+I+I+K+79 9 9N H!!>>#1>#C D DE ?
	L!45ry{{CDEE  L 	)M%-".{D D D D< < < 
M>;/1 1 10 "%99999 'MN%-".{$2G G G G
 
M>;/1 1 10 "$8888888r   c                    | dz  }t          |          }t          j        |t          j                            d                    }| dz  }|dz  |dz  dz  |dz  |dz  dz  g}t          j        t          j        |j                            d	          g          d	t          j	        ddg          i
          }t          |t          |          |d||           d S )NrQ  Tr(  r  rT  r]   r  r^   r   r  r  )r  r   rT   r  r  r   r@   r>   rA   r+  rL  r`   )r  rw   r^  rT   rN  rU  r   s          r   #test_write_dataset_partitioned_dictr    s    -'I+I66A j(1141HHJ J JG //Ffsl^3fsl^3N ?29V$$.& $' $'bhSz2235 5 5L VndF!# # # # # #r   c                    | dz  }t          |          }t          j        |d          }t          j        t	          j        dt	          j                    fg          d          }| dz  }g fd}t          j        ||d|d	|
           |dz  dz  |dz  dz  h}t          t          t          j                            }||k    sJ | dz  }	t          j        ||	d|d           t          j        |d|          }
t          j        |	d|          }|
                                                    |                                          sJ d S )NrQ  r   r  r   r  partitioned1c                 <                         | j                   d S r   )r1   rx   )written_filepaths_writtens    r   file_visitorz4test_write_dataset_use_threads.<locals>.file_visitor  s     \./////r   r  Trl   r   r   r  rR  part-0.featherrS  partitioned2FrG  r  )r  r   rT   r   r@   r>   rE   r  r   rp   r  r  r   r  )r  rw   r^  rT   r   target1r  rU  paths_written_settarget2result1result2r  s               @r   test_write_dataset_use_threadsr    s    -'I+I66Aj888G?
	FBIKK()**6; ; ;L &GM0 0 0 0 0 |    	(--(--N Cm<<==....&G    jNNNGjNNNG$$W%5%5%7%78888888r   c           	        
 t          j        t          j        t          d                    t          j        d t          d          D                       t          j        dgdz  dgdz  z             gg d          }| dz  }t	          j        ||d	d
           t          |                    d                    }|dz  g}t          |          t          |          k    sJ t	          j	        |d          
                                }|                    |          sJ | dz  }|dz  |dz  dz  |dz  |dz  dz  g}g 
g 
fd}t	          j        t          j        dt          j                    fg          d          }t	          j        ||d
d	||           t          |                    d                    }t          |          t          |          k    sJ d 
D             }|k    sJ t	          j	        |d|          }|
                                                    |          sJ t          
          dk    sJ 
D ]}	t!          j        |	          |v sJ d S )Nr  c              3   <   K   | ]}t          j                     V  d S r   r  r  s     r   rr  z#test_write_table.<locals>.<genexpr>  r  r   r]   r   r^   r  r  singledat_{i}.arrowr  basename_templaterl   rH  zdat_0.arrowr  r  rQ  rR  rS  c                 p                         | j                                        | j                   d S r   )r1   rx   r@  )r  visited_pathsvisited_sizess    r   r  z&test_write_table.<locals>.file_visitor  s7    \.///\./////r   r   r   r  )rl   r  r   r  c                 L    g | ]!}t           j                            |          "S r   )r[  rx   getsizerF  s     r   rc   z$test_write_table.<locals>.<listcomp>  s&    DDDdBGOOD))DDDr   r  r   )r@   r   r+  r0   r   r  ro   r  r   rT   r   r  r   r>   rE   r   r  r  )r  r   r  rK  rU  r7  r  r   actual_sizesvisited_pathr  r  s             @@r   test_write_tabler    s   H
rRX%I%IuRyy%I%I%III
#sebj()) "!!# # #E
 !HUH'6yJ J J J hnnS))**J./Nz??c.111111Z///88::F== &H8X0=@8X0=@N
 MM0 0 0 0 0 0 ?
	FBIKK()**6; ; ;LUHY'6".\K K K K hnnS))**Jz??c.111111DDmDDDLL((((Z\JJJF??##E*****}""""% < <|L))^;;;;;< <r   c           	         t          j        t          j        t          d                    t          j        d t          d          D                       t          j        dgdz  dgdz  z             gg d          }t          j        |gdz            }| d	z  }t          j        ||d
           t          |                    d                    t          |dz  g          k    sJ t          j	        |d          
                                                    |          sJ | dz  }t          j        |g|d
           t          |                    d                    t          |dz  g          k    sJ t          j	        |d          
                                                    |          sJ | dz  }t          j        |                                |d
           t          |                    d                    t          |dz  g          k    sJ t          j	        |d          
                                                    |          sJ | dz  }t          j        ||g|d
           t          |                    d                    t          |dz  g          k    sJ t          j	        |d          
                                                    t          j        |gdz                      sJ d S )Nr   c              3   <   K   | ]}t          j                     V  d S r   r  r  s     r   rr  z6test_write_table_multiple_fragments.<locals>.<genexpr>#  r  r   r]   r   r^   r  r  r   r  r  r  rH  r  r  zsingle-listmultiplezmultiple-table)r@   r   r+  r0   r  r   r  r   r  rT   r   r  r   )r  r   r  s      r   #test_write_table_multiple_fragmentsr  !  s!   H
rRX%I%IuRyy%I%I%III
#cUQY&'' "!!# # #E eWQY''E !HUHY7777x~~c""##sH7G,G+H'I'IIIII:hu---6688??FFFFF &HeWhy9999x~~c""##sH7G,G+H'I'IIIII:hu---6688??FFFFF #HU%%'')DDDDx~~c""##s	$	$%(' (' ' ' ' ':hu---6688??FFFFF ))HeU^Xi@@@@x~~c""##s	$	$%(' (' ' ' ' ':hu---6688??
%##      r   c           	         t          j        t          j        t          d                    t          j        d t          d          D                       t          j        dgdz  dgdz  z             gg d          }| dz  }t	          j        d	 |                                D             ||j        d
d           t	          j        |d          	                                }|
                    |          sJ | dz  }t           j                            |j        |                                          }t	          j        ||d
d           t	          j        |d          	                                }|
                    |          sJ | dz  }t          |          }t	          j        ||d
d           t	          j        |d          	                                }|
                    |          sJ d S )Nr  c              3   <   K   | ]}t          j                     V  d S r   r  r  s     r   rr  z&test_write_iterable.<locals>.<genexpr>G  r  r   r]   r   r^   r  r  inmemory_iterablec              3      K   | ]}|V  d S r   r   )ra   rz   s     r   rr  z&test_write_iterable.<locals>.<genexpr>L  s"      <<e<<<<<<r   r  r  )r>   r  rl   r  inmemory_readerr  inmemory_pycapsule)r@   r   r+  r0   r   r  r   r>   rT   r   r  r  rr   r   )r  r   r  r7  r   streams         r   test_write_iterabler  E  s   H
rRX%I%IuRyy%I%I%III
#sebj()) "!!# # #E
 ,,H<<)9)9););<<<h!L'6uF F F F Z///88::F==**H!..u|/4/?/?/A/AC CFVXQVWWWWZ///88::F==--H&&FVXQVWWWWZ///88::F==r   c           	         t          j        t          j        t          d                    t          j        d t          d          D                       t          j        dgdz  dgdz  z             gg d          }t	          j        |          }| dz  }t	          j        |                    |          |d	
           |                    t	          j        |d
                    }|	                    |          sJ | dz  }t	          j        |                    |dg          |d	
           |                    t	          j        |d
                    }|	                    |
                    dg                    sJ t          j        t          d          5  t	          j        |                    |          ||j        d	           d d d            d S # 1 swxY w Y   d S )Nr  c              3   <   K   | ]}t          j                     V  d S r   r  r  s     r   rr  z%test_write_scanner.<locals>.<genexpr>b  r  r   r]   r   r^   r  r  dataset_from_scannerr  r  r  dataset_from_scanner2r  r*   zCannot specify a schemar   )r>   rl   )r@   r   r+  r0   r   rT   r  r   r   r  r  r  r  r  r>   )r  r   r   rT   r  r7  s         r   test_write_scannerr  `  sV   H
rRX%I%IuRyy%I%I%III
#sebj()) "!!# # #E jG//H^++ 9. . . .$$RZ%G%G%GHHF== 00H^++GdV+DDi1 1 1 1$$RZ%G%G%GHHF==tf--..... 
z)B	C	C	C @ @
//88( %Y	@ 	@ 	@ 	@@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @s   %1G##G'*G'c                 .   t          j        t          j        t          d                    t          j        dgdz  dgdz  z                                             gddg          }t          j        |                    dg          j                  }| dz  }t          j	        ||d	|
           t
          j
                            dgd          }t          j        |d|
                                          }|                    |          sJ d S )Nr  r]   r   r^   r  r   r  rT   r  r  Tr(  r  )r@   r   r+  r0   ru  r   r   r  r>   r  r   r  rT   r   r  )r  r   r   r  partitioning_readr7  s         r   !test_write_table_partitioned_dictr  z  s%    H
r
#sebj());;== V_  E
 ?5<<#9#9#@AAL"Hx	   
 099	4 : ) )Z->  hjj  ==r   c           
         t          j        t          j        t          d          d          t          j        t	          j        ddd                              d                    t          j        t	          j        dd	gd
                    gg d          }| dz  }t          j	        ||d           t          |                    d                    }|dz  g}t          |          t          |          k    sJ t          j        |d                                          }|                    |          sJ dD ]}t          j                    }|                    |          }dt%          |          v sJ | d                    |          z  }t          j	        ||||           t)          j        |dz            }	|dk    rdnd}
|	j        |
k    sJ t          j        |d                                          }|j        }|dk    rM|                    d|                    d                              t          j                                        }|dv rN|                    d|                    d                              t          j        d                              }|                    |          }|                    |          sJ d S )Nr  r  r  r@  zdatetime64[D]r)  zdatetime64[ns]r]   r^   r   r  r  r  r   r  rH  part-0.parquet)1.02.42.6r  z(<pyarrow.dataset.ParquetFileWriteOptionszparquet_dataset_version{0}r  r  r  r   )r  r  r   r  )r@   r   r+  r0   rS  rT  r  rR  r   r  ro   r  r   rT   r   r  r   r  r  rl   rs   read_metadataformat_versionr>   rA   	with_typerC   r  r  )r  r   r  rK  rU  r7  r  rl   optsmetaexpected_versionr>   rf  s                r   test_write_dataset_parquetr    s    H
r***
<?CCCJJ  	 	
C:r**++	
 "!!# # #E **HUHY7777hnnS))**J!112Nz??c.111111Z333<<>>F== ) ' '%''(((999T$ZZGGGG9@@III
dKKKK+; ;<<$+u$4$455%"&66666 HY777@@BBeZZ6<<??#<#<RXZZ#H#HIIFn$$ZZ6<<??#<#<R\$=O=O#P#PQQF::f%%}}X&&&&&&'' 'r   c           	         t          j        t          j        t          d                    t          j        d t          d          D                       t          j        dgdz  dgdz  z             gg d          }| dz  }t	          j        ||d	
           t          |                    d                    }|dz  g}t          |          t          |          k    sJ t	          j	        |d	
          
                                }|                    |          sJ t	          j        t          j                            |j        j                            }|                    d          }| dz  }t	          j        ||||           t	          j	        ||
          
                                }|                    |          sJ d S )Nr  c              3   <   K   | ]}t          j                     V  d S r   r  r  s     r   rr  z)test_write_dataset_csv.<locals>.<genexpr>  r  r   r]   r   r^   )r  r  chr1r  csv_datasetr  r  rH  z
part-0.csvr  r  F)include_headercsv_dataset_noheaderr  )r@   r   r+  r0   r   r  ro   r  r   rT   r   r  r  r   r  r
  r>   r  r  )r  r   r  rK  rU  r7  rl   r  s           r   test_write_dataset_csvr    s   H
rRX%I%IuRyy%I%I%III
#sebj()) "!!# # #E
 &HUHU3333hnnS))**J-.Nz??c.111111Z///88::F== 7;+B+B\' ,C ,) ,) * * *F$$E$::D//HUHV$GGGGZ00099;;F==r   c           	      X   t          j        t          j        t          d                    t          j        d t          d          D                       t          j        dgdz  dgdz  z             gg d          }dfd	}| d
z  }t	          j        ||d|           sJ d S )Nr  c              3   <   K   | ]}t          j                     V  d S r   r  r  s     r   rr  z:test_write_dataset_parquet_file_visitor.<locals>.<genexpr>  r  r   r]   r   r^   r  r  Fc                 B    | j         | j         j        dk    rdd S d S d S )Nr[   T)r  r@  )r  visitor_calleds    r   r  z=test_write_dataset_parquet_file_visitor.<locals>.file_visitor  s4    !-%1Q66!NNN .-66r   r  r   )rl   r  )r@   r   r+  r0   r   r  )r  r   r  r  r"  s       @r   'test_write_dataset_parquet_file_visitorr#    s    H
rRX%I%IuRyy%I%I%III
#sebj()) "!!# # #E
 N" " " " " **HUHY".0 0 0 0 r   c                   	
 d t          d          D             }d t          d          D             }t          j        ||dgdz  dgdz  z   d          }| dz  }t          j        t          j        d	t          j                    fg          d
          }g 	d 
	
fd}t          j        ||d|d|           |dz  dz  |dz  dz  h}t          t          t          j        	                    }||k    sJ 
J 
j        dk    sJ d S )Nc                 $    g | ]}|gd z  D ]}|S r   r   ra   r   items      r   rc   z?test_partition_dataset_parquet_file_visitor.<locals>.<listcomp>  s+    CCCuglCCdtCCCCr   r   c                 *    g | ]}|gd z  D ]}|d z  S r   r   r&  s      r   rc   z?test_partition_dataset_parquet_file_visitor.<locals>.<listcomp>  s/    FFF5%2FFtBwFFFFr   r]   r  r^   r  rQ  r   r   r  c                 X    | j         r| j                             | j                   d S r   )r  r1   rx   )r  r  sample_metadatas    r   r  zAtest_partition_dataset_parquet_file_visitor.<locals>.file_visitor  s3      	4*3O\./////r   r   Tr  rR  r  rS  r   )r0   r@   r   r   r   r>   rE   r  r   rp   r  r  r@  )r  f1_valsf2_valsr   r  r   r  rU  r  r  r*  s            @@r   +test_partition_dataset_parquet_file_visitorr-    st   CCqCCCGFFE!HHFFFGHG7"ebjC52:57 7 8 8E -'I?
	FBIKK()**6; ; ;L MO0 0 0 0 0 0 y|    	H//H//N Cm<<==....&&&&!++++++r   c                    t          j        dt          j        dd          gi          }|d         j        j        dk    sJ t          j        || d           t          j	        | dz            }|d         j        j        dk    sJ d S )Nr]   r@  zEurope/Brussels)tzr   r  r  )
r@   r   r4   rB  r  r/  r   r  rs   rU  )r  r   r7  s      r   (test_write_dataset_arrow_schema_metadatar0    s    
 HcBL:KLLLMNOOE:?!22222UGI6666]7%5566F#;"3333333r   c                     ddl m} t          j        dg di          }|                    ddi          }t          j        || d           |                    | d	z            j        }|j	        ddik    sJ d S )
Nr   r-  r]   r  r     valuer  r  r  )
r   r  r@   r   rH   r   r  rU  r>   r  )r  r  r   r>   s       r   "test_write_dataset_schema_metadatar3     s    Hc999%&&E))68*<==EUGI6666*: :;;BF?vx0000000r   c                     t          j        dg di          }|                    ddi          }t          j        || d           t          j        | dz            j        }|j        ddik    sJ d S )Nr]   r  r  r2  r   r  r  )	r@   r   rH   r   r  rs   rU  r>   r  )r  r   r>   s      r   *test_write_dataset_schema_metadata_parquetr5  ,  s     Hc999%&&E))68*<==EUGI6666]7%5566=F?vx0000000r   c           	      2   | \  }}}}}}}}d                     ||||          }t          j        t          j        t	          d                    t          j        d t	          d          D                       t          j        dgdz  dgdz  z             gg d          }t          j        t          j        d	t          j                    fg          d
          }	t          j	        |d|d|	           t          j
        d|dd
                                          }
|
                    |          sJ |                     d          }t          j	        ||d|	           t          j
        d|dd
                                          }
|
                    |          sJ |                     d          }t          j	        |d|d|	           t          j
        d|dd
                                          }
|
                    |          sJ d S )Nr+  r  c              3   <   K   | ]}t          j                     V  d S r   r  r  s     r   rr  z(test_write_dataset_s3.<locals>.<genexpr>B  r  r   r]   r   r^   r  r  r   r   r  zmybucket/datasetr  rW  r  zmybucket/dataset2r  r  r  zmybucket/dataset3)rl   r@   r   r+  r0   r   r   r>   rE   r  rT   r   r  )rM  r^  ri   rO  rP  rQ  rR  uri_templater   r   r7  rN  s               r   test_write_dataset_s3r9  7  sP    7H3Aq"atZ=DD
D$	0 	0 
 H
rRX%I%IuRyy%I%I%III
#sebj())+ #""  E
 ?29vry{{&;%<==fMMMD !b   
 Zr%f  hjj  == 

1
2
2CUC	EEEEZ5v  hjj  == 

j
)
)Czc)$    Z5v  hjj  ==r   aC  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:PutObject",
                "s3:ListBucket",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]
        }
    ]
}c           	         ddl m} | d         \  }}}}t          | t          dd            |ddd                    ||          d          }t          j        t          j        t          d	                    t          j        d
 t          d	          D                       t          j        dgdz  dgdz  z             gg d          }t          j
        t          j        dt          j                    fg          d          }t          j        |d|dd|d           t          j        d|dd                                          }|                    |          sJ t          j        |d|dd|d           t          j        d|dd                                          }|                    |          sJ t#          j        t&          d          5  t          j        |d|ddd           d d d            n# 1 swxY w Y    |ddd                    ||          dd           }t#          j        t&          d!          5  t          j        |d|ddd           d d d            d S # 1 swxY w Y   d S )"Nr   )r&  r  test_dataset_limited_user
limited123z{}:{}http)rQ  rR  endpoint_overrideschemer  c              3   <   K   | ]}t          j                     V  d S r   r  r  s     r   rr  z1test_write_dataset_s3_put_only.<locals>.<genexpr>  r  r   r]   r   r^   r  r  r   r   r  zexisting-bucketr  Fr  )r   rl   rm   r   r  r  rW  Tz&Bucket 'non-existing-bucket' not foundr   znon-existing-bucket)r   rl   rm   r  limited)rQ  rR  r>  r?  allow_bucket_creationz(Access Denied|ACCESS_DENIED))r   r&  r	   _minio_put_only_policyrl   r@   r   r+  r0   r   r   r>   rE   r  rT   r   r  r  r  rL  )	r  r&  rO  rP  r^  ri   r   r   r7  s	            r   test_write_dataset_s3_put_onlyrD  z  s    (''''' !.D$1y*@:LJ J J	.!..t44	
 
 
B H
rRX%I%IuRyy%I%I%III
#rSEBJ&'') #""  E
 ?29vry{{&;%<==fMMMD  RU4    ZbV  hjj  ==  RT4    ZbV  hjj  == 
wE
G 
G 
G 
 

(R#8	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
!..t44"
 
 
B 
w&E	F	F	F 
 

(R#8	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s$   HHHI66I:=I:c           
         t          j        dd d gi          }t          j        || dz             t          j        t          j        dt          j        t          j                    t          j                                        g          }t          j
                            | dz  g|t          j                    t          j                              }|                    |          }|j        |k    sJ d S )Nr]   r  )r   r>   rl   r   )r@   r   rs   rt   r>   rA   r  r   rE   r   r  r  r   ri   r|   r   )r  r   r   r>   fsdss        r   $test_dataset_null_to_dictionary_castrG    s     HcD$<())EN5'N2333Y
bmBHJJ	<<==  F **'(#%%%''	 +  D ##D))E<6!!!!!!r   c                    t          j        g dg dd          }t          j        || dz  d           t          j        | dz  d          }t          j        g dg dd	          }t          j        || d
z  d           t          j        | d
z  d          }|                    |dd          }|                                t          j        g dg dg dd          k    sJ |                    |ddd          }|                                                    d          t          j        g dg dg dd          k    sJ d S )Nr   r   r  r]   r^   r  colAr  r  r  r  c   r   r   Zr  r  )colBcol3r  rL  rQ  r  r  NrL  r  rR  
full outer)	join_typer   r   r  rN  r]   r^   r  Nr  r  NrP  r@   r   r   r  rT   r  r   r1  r  r  ds1r  ds2r7  s         r   test_dataset_joinr^    s   			  
 
B R46666
*Wt^E
2
2
2C	

  
 
B R46666
*Wt^E
2
2
2CXXc66**F??		   * * ! !     XXc66\XBBF??$$V,,%%%%%%: : 1 1      r   c                    t          j        g dg dd          }t          j        || dz  d           t          j        | dz  d          }t          j        g dg dd	          }t          j        || d
z  d           t          j        | d
z  d          }|                    |d          }|                                t          j        g dg dg dd          k    sJ |                    |ddd          }|                                                    d          t          j        g dg dg dd          k    sJ d S )NrI  rJ  rK  r  r  r  rM  rO  )rL  rR  r  rL  rS  rT  rU  _rrV  right_suffixrW  rX  rY  rZ  r[  s         r   test_dataset_join_unique_keyrc    s   			  
 
B R46666
*Wt^E
2
2
2C	

  
 
B R46666
*Wt^E
2
2
2CXXc6""F??		   * * ! !     XXc6\XMMF??$$V,,%%%%%%: : 1 1      r   c           	          t          j        g dg dg dd          }t          j        || dz  d           t          j        | dz  d          }t          j        g dg d	g d
d          }t          j        || dz  d           t          j        | dz  d          }|                    |ddd          }|                                                    d          t          j        g dg dg dg dg dgg d          k    sJ d S )NrI  r   r  <   rJ  )rL  rQ  colValsr  r  r  rM  rN  r  r   rO  r  rL  rU  r`  ra  rW  )r   r  rf  NrX  )r   r  NrN  rY  )rL  rQ  rg  colB_r	colVals_rr  rZ  r[  s         r   test_dataset_join_collisionsrk    sx   			"??  
 
B
 R46666
*Wt^E
2
2
2C	

"??  
 
B
 R46666
*Wt^E
2
2
2CXXc6\XMMF??$$V,,: @??1A 1A 1A A A A A A Ar   c                 8   t           j                            g dg dd          }t          j        || dz  d           t          j        | dz  d          }t           j                            g dg dg d	d
          }t          j        || dz  d           t          j        | dz  d          }|                    |ddddd          }|                                                    d          t          j	        g dg dg dd          k    sJ d S )N)r   r   r   r  r?  )r]   r^   r]   r^   r  rK  r  r  r  )r   r  r  )r]   r^   g)r   r(  g      @)rQ  rR  colCr  rL  r  r   rQ  rR  onby	toleranceright_onright_by)r   NNNN)rL  r  rn  )
r@   rF   from_pydictr   r  rT   	join_asofr   r1  r   r[  s         r   test_dataset_join_asofrw  5  sg   			)))  
 
B R46666
*Wt^E
2
2
2C			

  
 
B
 R46666
*Wt^E
2
2
2C]]6Q&   F ??$$V,,))),,,: : 1 1      r   c                    t          j        g dg dg dd          }t          j        || dz  d           t          j        | dz  d          }t          j        g dg d	g d
g dd          }t          j        || dz  d           t          j        | dz  d          }|                    |dddgd          }|                                                    d          t          j        g dg dg dg dd          k    sJ d S )NrI  re  r  )rL  rQ  rp  r  r  r  rh  rO  rM  r  )rQ  rg  rL  rp  r  rp  rL  rQ  r   rp  rq  rr  )Nr  NrL  rQ  rp  rg  )r@   r   r   r  rT   rv  r   r1  r[  s         r   "test_dataset_join_asof_multiple_byr{  Q  sl   			ii  
 
B
 R46666
*Wt^E
2
2
2C	"??

ii	  
 
B R46666
*Wt^E
2
2
2C]]&&)Q   F ??$$V,,		ii$$$	: : 1 1      r   c                    t          j        dg di          }t          j        || dz  d           t          j        | dz  d          }t          j        g dg dd          }t          j        || d	z  d           t          j        | d	z  d          }|                    |dg d
          }|                                t          j        g dg dd          k    sJ d S )Nrp  r  r  r  r  rO  r  )rg  rp  r  r   ry  )rP  rP  r  )rp  rg  )r@   r   r   r  rT   rv  r   r[  s         r   test_dataset_join_asof_empty_byr}  o  s*   	iii 
 
B R46666
*Wt^E
2
2
2C	"??ii  
 
B R46666
*Wt^E
2
2
2C]]q   F ??ii"??* * ! !      r   c           
         t          j        g dg dg dg dd          }t          j        || dz  d           t          j        | dz  d          }t          j        g d	g d
g dg dg dd          }t          j        || dz  d           t          j        | dz  d          }d}t          j        t          |          5  |                    |dddgddddg           d d d            d S # 1 swxY w Y   d S )NrI  re  r  rJ  rz  r  r  r  rh  rO  )rl  ru  r  rM  r  )rQ  rg  colUniqrL  rp  r  zXColumns {'colVals'} present in both tables. AsofJoin does not support column collisions.r   rp  rL  rQ  r   ro  )	r@   r   r   r  rT   r  r  r  rv  )r  r  r\  r  r]  r]  s         r   !test_dataset_join_asof_collisionsr    s   			ii"??	  
 
B R46666
*Wt^E
2
2
2C	"??"??

ii  
 
B R46666
*Wt^E
2
2
2C	7  
z	-	-	- 
 
Dff-VV$4 	 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   	 C66C:=C:dstyperi   memc           	         t          j        g dg dd          }|dk    r4t          j        || dz  d           t          j        | dz  d          }n"|dk    rt          j        |          }nt
          |                    t          j        d	          d
k                                   t          j        d          dk              }|dk    rt          j	        nt          j
        }t          ||          sJ |                                t          j        dgdgd          k    sJ |                    d          t          j        dgdgd          k    sJ |                    t          j        d	          dk                                   t          j        d	          dk                                  t          j        d	          dk              }|                                t          j        dgdgd          k    sJ t          j        || dz  d           t          j        | dz  d          }|                                t          j        dgdgd          k    sJ |                    t          j        t          j        ddgddgd                    dd          }|                                                    d          t          j        dd gddgddgd          k    sJ t#          j        t&                    5  |                    d            d d d            n# 1 swxY w Y   t#          j        t(                    5  |                                 d d d            n# 1 swxY w Y   |j                            d          }	|                    t          j        d	          d
k                                   |	          }
|
                                t          j        d	ddgi          k    sJ t#          j        t           j                  5  |                    |	                                           d d d            d S # 1 swxY w Y   d S )Nr   r   r  r<  r]   r^   r  rm  rK  ri   r  r  r  r  rL  r[   r  r]   r   r   r<  r  r   r   r^   r!  r   r  rQ  r  zright outerkeysrV  rQ  )rL  rQ  r  )r@   r   r   r  rT   r  r   r  rA   r  r  rN   r   r   r   r  r1  r  r  r  r  rL   r>   r  replace_schemarK  )r  r  r  r\  r7  rf  r2r!  joinedschema_without_col2	newschemas              r   test_dataset_filterr    su    
$$$  
 
B ~~
Wt^E::::j4666	5jnn!! ZZ((1,--44RXf5E5E5LMMF'-~~r##2;MHfh'''''??* * ! !    
 ;;q>>RX' '       
BHV$$q(	)	)	0	0
1
 
%gRXf-=-=-BgCC ;;==BH& &       VWz1%@@@@z'J.u===H"(, , # #     [[BHRc
. . % %     / /F ??$$V,,D	Rc
: : 1 1     
y	!	!  

4               
z	"	"                 *++A..


1 n())  28A- $ $     
r	'	' > > 	122;;===	> > > > > > > > > > > > > > > > > >s6   L99L= L=M>>NN/(Q$$Q(+Q(c           	      &   t          j        g dg dd          }t          j        g dg dd          }|dk    rgt          j        || dz  d	           t          j        | dz  d	          }t          j        || d
z  d	           t          j        | d
z  d	          }n6|dk    r)t          j        |          }t          j        |          }nt
          t          j        ||f                              t          j        d          dk     t          j        d          dk    z            }|	                                t          j        g dg dd          k    sJ |
                    t          j        t          j        ddgddgd                    dd          }|	                                                    d          t          j        g dg dg dd          k    sJ |                    t          j        d          dk               }|                    t          j        d          dk               }	t          j        t          d          5  t          j        ||	f           d d d            d S # 1 swxY w Y   d S )Nr  r  rK  )r  r   r_  )hr;   lri   r  r  r  r  r  rL  r[   r  )r   r   r  )r]   r^   r  r   r  r]   r^   r  r  z
left outerr  )r   r  N)rL  r  rQ  zcurrently not supportedr   )r@   r   r   r  rT   r  r   r  rA   r   r  r1  r  r  r  )
r  r  r  r  r\  r]  filtered_union_dsr  filtered_ds1filtered_ds2s
             r   test_union_dataset_filterr    s    
$$$  
 
B 
  
 
B ~~
Wt^E::::j4666
Wt^E::::j4666	5jnnjnn!!
C:..55	&		A	"(6"2"2a"78  %%''28		5 5 , ,    
 ##BJrxRc
9 9 0 0 % %  $ . .F ??$$V,,		: : 1 1     ::bhv..233L::bhv..344L	z)B	C	C	C 1 1

L,/0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s   "JJ
J
c                    | dz  }t          |          \  }}t          j        |          }|                                }|j        dk    sJ |                    t          j        d          dk               }|                                j        dk    sJ t          j	        t                    5  |                                 d d d            d S # 1 swxY w Y   d S )Ntest_parquet_dataset_filterr  r  r   r  )r  r   r  r   r   r   r  rA   r  r  r  rL   )r  r  r  r^  rT   r7  filtered_dss          r   r  r  (  s   77I5i@@M1 //GF?b    ..$!!344K!!*b0000	z	"	" $ $!!###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s   0CCCc                    t          j        t          j        t          d                    gdg          }t	          j        |          }dt	          j        d          i}|                    |          }t	          j        || dgd           t          j
        t          d	          5  t	          j        || dgd           d
d
d
           d
S # 1 swxY w Y   d
S )z
    Ensure the projected schema is used to validate partitions for scanner

    https://issues.apache.org/jira/browse/ARROW-17228
    r  original_columnr  renamed_columnr*   r  r  z0'Column original_column does not exist in schemar   N)r@   r   r+  r0   r   rT   rA   r   r  r  r  KeyError)r  r   table_datasetr+   r   s        r   4test_write_dataset_with_scanner_use_projected_schemar  7  sD    HbhuRyy))*3D2EFFFEJu%%M"(#455G ##G#44G(8'9%I I I I 	O	
 	
 	

 

 	W,=+>u	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   ,CCCrl   )r  r   c           
         |dk    rt          j        d           t          j        ddgddgd dddgdd	id gd
ddg dddigd
gd          }t	          j        || dz  |           t	          j        | dz  |          }|                    g d          }|                                dd ddgd d	dd gddddg ddd dgddgk    sJ d S )Nr   zpyarrow.parquetabc123qrs456r   r   buttonr  r  )r  elementvaluesstructsscrollwindow)Nr[   r   fizzbuzz)user_ida.dotted.fieldinteractionr   r  )r  zinteraction.typezinteraction.valueszinteraction.structsr  r*   )r  r  )r  r  r  r  r  )	r  r%  r@   r   r   r  rT   r   rR  )r  rl   r   r\  s       r   test_read_table_nested_columnsr  P  ss   -...H(H!5)*A(q6u~t&<	> 	>h<<ff-=,>	@ 	@&   E UGg-f====
*Ww&v
6
6
6C LL: : :  ; ;E ??dq!f"51148A	O 	Oh,,,$T223q	J 	J!      r   c                 v   ddl m} | dz  }t          j                            t          j        g dt          j                              t          j        g dt          j                              gddg          } |j        ||ddgd	
            |j        |dd	t          j	        t          j
        dt          j                              t          j
        dt          j                              g                                                                                    }||                    d          k    sJ |                    d                                          }t!          t#          |                    }d |D             }t!          t%          j        |                    }||k    sJ d S )Nr   )rT   zslash-writer-xr   r   r[   r   r   )experiment/A/f.csvzexperiment/B/f.csvr  zexperiment/C/k.csvzexperiment/M/i.csvexp_idexp_metar  r   )r:   r  rl   r   ra  )r  rl   r   r>   r   c                 6    g | ]}d t          |d          z   S )z	exp_meta=r  r.  r   rF  s     r   rc   z5test_dataset_partition_with_slash.<locals>.<listcomp>  s*    MMMD[5B#7#7#77MMMr   )r   rT   r@   rF   r+  r+  r   r  r  r>   rA   r   r)  r1  r-  rR  r3  r   r[  r_  )tmpdirr   rx   dt_tablerU  r  encoded_pathsrK  s           r   !test_dataset_partition_with_slashr  l  s   %%%%%%$$Dx##
"(**--
 ( ( ()+	4 	4%5 8@6L	N NH B \"    y"(8RXZZ88(:rwyy99; < <	   hjj!!  z))(333333q!!++--Hc(mm$$HMMHMMMM
4(())JJ&&&&&&r   c                 b   t          j        t          j        dt          j                    d          t          j        dt          j                    d          g          }g dg dg}t           j                            ||          }t          j        || d	z             t          j	        | d	z  d
          }|
                                j                            |          sJ t          j        || dz  d
           t          j	        | dz  d
          }|
                                j                            |          sJ t          j        ||g| dz  d
           t          j	        | dz  d
          }|
                                j                            |          sJ d S )Nr   F)nullablerI  Tr  Nr   Nrh   	nulltest1r   r  	nulltest2	nulltest3)r@   r>   rA   rC   rF   r+  rs   r  r   rT   r   r  r  )r  schema_nullablerx  r   rT   s        r   'test_write_dataset_preserve_nullabilityr    s    i
bhjj5111
bhjj4000!2 3 3O ii)FH   @@Ew4555j;.yAAAG$++O<<<<<UGk1)DDDDj;.yAAAG$++O<<<<<eU^W{%:9MMMMj;.yAAAG$++O<<<<<<<r   c                 z   t          j        t          j        dt          j                    ddi          t          j        dt          j                              g          }t          j        t          j        dt          j                              t          j        dt          j                              g          }g dg dg}t           j                            ||          }t           j                            ||          }t          j        ||g| d	z  d
           t          j        | d	z  d
          }|	                                j        
                    |d          sJ t          j        ||g| dz  d
           t          j        | dz  d
          }|	                                j        
                    |d          sJ t          j        ||g| dz  d
|           t          j        | dz  d
          }|	                                j        
                    |d          sJ d S )Nr   s   foos   barrp  rI  r  r  rh   test1r   r  Tr!  test2test3r  )r@   r>   rA   rC   rF   r+  r   r  rT   r   r  )r  schema_metadataschema_no_metarx  r   table_no_metarT   s          r   *test_write_dataset_preserve_field_metadatar    s&   i
bhjjFF+;<<<
bhjj!!!# $ $O Y
bhjj!!
bhjj!! # $ $N ii)FH   @@EH(((GGM e]+Ww->yQQQQj7*9===G$++OD+QQQQQmU+Ww->yQQQQj7*9===G$++N4+PPPPP mU+Ww->y+- - - -j7*9===G$++OD+QQQQQQQr   c                    dD ]O}dD ]H}t          j        t          j        dt          j                              t          j        dt          j                              g          }g dg dg}t           j                            ||          }t          j                    }| d| z  }t          j        ||d|	                    ||	          d
           t          j
        |d          }|j        D ]W}	t          j        |	          }
|
                    d                              d          }|j        |u sJ |j        ||z  u sJ XJQd S )N)TFr   rI  r  r  rh   write_page_index_r   )write_statisticswrite_page_indexr  )rl   r  r  r  r   )r@   r>   rA   rC   rF   r+  r   r   r  r  rT   r	  rs   r  r  r-  has_offset_indexhas_column_index)r  r  r  r>   rx  r   r   r  r\  r7  r  ccs               r   #test_write_dataset_write_page_indexr    s   ) R R - 	R 	RYbhjj))bhjj)) + , ,F  ii1FH(((??E.00K!G5E!G!GGH (;;%5%5 <   (=	 	 	 	 *Xi888C	 R R+D11''**11!44*.>>>>>*.>AQ.QQQQQQR-	RR Rr   c                    t          j        t          j        g d          t          j        g d          gddg          }|dk    r4t          j        || dz  d	           t          j        | dz  d	          }n"|d
k    rt          j        |          }nt          |                    d                                          	                                g dg ddk    sJ |                    dg                                          	                                g dg ddk    sJ |
                    t          j        d          dk                                   d                                          	                                g dg ddk    sJ t           j                            t          j        g dt          j                              t          j        g d          gddg          }t          j        |          }|                    dg          }|                                	                                }|d         g dk    sJ |d         g dk    sJ |                    dg          }|                                	                                }|d         g dk    sJ |d         g dk    sJ d S )N)r[   r   r   r   r   )r^   r]   r^   r]   r  r  r  r  ri   r  r  r  r  )r]   r]   r^   r^   r  r  )r  r  )r  
descending)r  r^   r^   r]   r]   )r   r   r[   r   r   r   )r]   r]   r^   r  )r   r?  r?  r  r  )r  carr  foobarr]   r^   )r]   r  )r  r?  r?  r   )r  r  r  r  )r]   r=  )r@   r   r+  r   r  rT   r  r1  r   r2  r   r  rA   rF   r+  rC   )r  r  r   r   
sorted_tabsorted_tab_dicts         r   test_dataset_sort_byr    s    H
!!
***++ ! ! !E
 ~~
$u====Z$u555	5Z!!::h((**4466)))!//; ;    
 ::/011::<<FFHH)))!//M M    
 99bhx((1,..66 hjj))! !    H  
RXZZ000
00011" 3Z !  E 
E		B0122J ))++5577O3===00003#B#B#BBBBB/011J ))++5577O3===00003#B#B#BBBBBBBr   c                    t          j        dg di          }t           j                                        }|                    d          }| dz  }t          j        ||||           t          j        d          }t           j                            |          }t          j        ||	                                          }||k    sJ | d
z  }t          ||           t          |                                          }	t          |	          dk    sJ |	d         }
t          |
                                          }|d         |d         k    sJ |d         |d         c|d<   |d<   |
                    |           t          j        d          }t           j                            |          }t          j        ||	                                          }||k    sJ |t          j        dg di          k    sJ t!          j        t$          d          5  t          j        ||	                                          }ddd           dS # 1 swxY w Y   dS )zwCheck that checksum verification works for datasets created with
    ds.write_dataset and read with ds.dataset.to_tabler]   r  T)write_page_checksumcorrect_dir)r:   r  rl   r  r  )default_fragment_scan_optionsr  corrupted_dirr   r      $   F)r   r[   r   r   zCRC checksum verificationr   N)r@   r   rT   r   r  r   r  r  r   r   ro   iterdirr   	bytearray
read_byteswrite_bytesr  r  rL  )r  
table_origpq_write_formatr  original_dir_pathpq_scan_opts_crcpq_read_format_crctable_checkcorrupted_dir_pathcorrupted_file_path_listcorrupted_file_pathbin_datapq_scan_opts_no_crcpq_read_format_no_crctable_corruptr^  s                   r   1test_checksum_write_dataset_read_dataset_to_tabler    s    3-..J j2244O#66  7 " "M  -/""	    4#') ) )55&6 6 8 8*!   hjj  $$$$ !?2 2333
  $$6$>$>$@$@AA'((A----215,7799::H B<8B<''''!)"x|HRL(2, ##H--- 7#(* * *J88&9 9 ; ;J#8: : ::B(** 
 J&&&&BHc<<<%8999999 
w&A	B	B	B  J%
 
 
 (** 	
                 s   !)IIIc                     d} d}t          j        t                    5 }t          j        j                            d           d d d            n# 1 swxY w Y   | t          |j                  v s|t          |j                  v sJ t          j                                        }d}t          j        t          |          5  |                    d           d d d            d S # 1 swxY w Y   d S )NzImake_write_options() should be called on an instance of ParquetFileFormatzqdescriptor 'make_write_options' for 'pyarrow._dataset_parquet.ParquetFileFormat' objects doesn't apply to a 'int'+   z;make_write_options\(\) takes exactly 0 positional argumentsr   )	r  r  r  r@   rT   r   r  r`   r(   )msg_1msg_2excinfopformatr]  s        r   test_make_write_options_errorr  b  sa   -E(E 
y	!	! <W

$77;;;< < < < < < < < < < < < < < <C&&&&%3w}3E3E*E*E*E*Ej**,,G
IC	y	,	,	, ' '""2&&&' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 's#   %AAAC&&C*-C*c                 T   	 dd l m} n$# t          $ r t          j        d           Y nw xY wd}d}|                     |j                            |          |j                            |                                                    }|	                                dddgik    sJ d S )Nr   zsubstrait NOT enableds   
SOhttps://github.com/apache/arrow/blob/main/format/substrait/extension_types.yaml	
u64
	u32

str"i
i64
f64
str
const
struct
a
b
group
key7
:
Z
b
:

:
b
*
bs3  
/functions_comparison.yaml
SOhttps://github.com/apache/arrow/blob/main/format/substrait/extension_types.yamlequal:any1_any1	
u64
	u32

"
 "
("i
i64
f64
str
const
struct
a
b
group
key7
:
Z
b
:

:
b
*
bra  r`   4)
pyarrow.substrait	substraitr  r  r  r   BoundExpressionsfrom_substraitr   r2  )rT   psr8  	filteringr7  s        r   test_scanner_from_substraitr  u  s    -&&&&&&& - - -+,,,,,-WJ
CI __#22:>>"11)<<    hjj  %#s!4444444s   	 **r  r   )r   r   r   (  r   r,   r[  r  rd  r  sysr  rg  r  r  shutilr   urllib.parser   numpyrS  r  r  r   r@   r>  r?  r  pyarrow.csvr  r   ri   pyarrow.jsonpyarrow.libr   pyarrow.tests.utilr   r   r	   r
   r   r   r4   r  rT   r   pyarrow.parquetr   rs   mark
pytestmarkr   r<   rJ   rW   fixtureru   r   r   r   r  r  r9  rA  rU  r_  rc  rg  ri  ro  r  r  parametrizer`   tupler  r  r  r  r  r  r  r  r   r4  r;  s3rY  rp  rs  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r#  r,  r?  rF  rL  rN  rX  rb  re  rj  rn  rp  rr  rw  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rM  r  r  r'  r4  r8  r=  rC  rF  rH  rP  rT  r_  rb  rd  rj  rn  ru  rx  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  skipifplatformr  r  r   r  r  r  r  r  r  r  r"  r&  r+  r3  r8  r;  rE  rL  rO  rV  r_  rf  rk  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r#  r-  r0  r3  r5  r9  rC  rD  rG  r^  rc  rk  rw  r{  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r   <module>r     s  $      				       



                      	BBB                              , , , , , ,, , , , , , , , , , , , , ,   	BBB          	BBB          	BBB
 [ 
? ? ? ? ? ? ? ?K K K + + +% % %   D   @ h0 0  0f    	
 	H  + +	 +\ DG DG DGN) ) ), 89 89 89v " " " -I -I -I` % % %$ - - -, 5 5 5 O O O  A! A! A!HF F F$ F   8
8
 %8
 #(8
 8
 8
 8
v# # #"6 6 6   " ) ) ) 6 6 6r$T $T $TNJ J J2 ,BOH---$$/   t}55E" E"  65 E"P 4 4 4$ 5- 5-  5-pH H H2H H H, "> "> ">J    " 7 7 7@ 	% 	%  	% 9N 9N 9Nx   : ) ) )" 
: 
:  
: %0 %0 %0P   "5J 5J 5J 5Jp ' '  'T 7 7 7 7 7  7 ) ) ), &. &. &.R $ $ $6 ' ' '> 	 	 	  3  3  3F       F  @ @AC CI IC C I< +eT];; @ @AC C@ @C C <; @@  @ @AC Ca, a,C Ca,H  @ @AC CR, R,C CR,jJ J J? ? ? ( (  (2N N N
+ + +  ; ; ;
= = =O O O"6 "6 "6J H H H(* * *8   , , ,> > >@ @ @" I I I
 I I I L L L $ $ $$ < < <& J J J& 7 7 7&      , - - - = = =*" " "  ?( ?( ?(DJ J J  66  76&  " !# !# !#H ? ? ?, , , , N N N. . .A A A +v)>??5$-88+eT];;)__iii YY __ooo&YY			\\???#YY   !\\<<< ,  62 62  <; 98 @? 62r ; ; ;< D D D( 
: 
:  
: : :  : , ,  ,0 ?= ?=  ?=D / / / 9 9 92 = = =6 F F F 	 	 	 9 9 9(4 4 4?% ?% ?%DM M M2 2 2@ 6) 6) 6)r   "$ $ $, #M #M #ML 3 3 3,* * *      "      $  ) ) )        (H H H*: : : A A A0      (      (      ,D D D2@ @ @0G G G@  ) ) )  
,/ ,/  ,/^     , ! !  ! CLG+D  F F! !F F  !( ! !  !2 . . .* 	 	  	  $3 3 3 4 4  4$ 
; 
;  
; 3 3  3N @ @  @  G G G* D D  D 	= 	=  	== = =0 :? :?  :?z B B  B&( ( ( ?C3 3 3 3&  P  P  PF * *  *>) ) )") ) )"= = =& H H  HVF F F" -# -# -#`; ; ; ;	 	 	O O O !< !< !<H  <  <  <F $ $ $8 A9 A9 A9H # #  #0 #9 #9  #9L,< ,< ,<^! ! !H     6@ @ @4     0 (' ('  ('V     4   *  ,  ,  ,F 4 4  4	1 	1 	1 1 1 1 ,  ,   , ^ $ I
 I
  I
X " " "$   <   < A A A6   6   :   . 
 
 
> %$  K> K> K>\ %$  *1 *1 *1Z$ $ $
 
 
2 #566  766"' "' "'J = = =.R R R:R R R@ %$  -C -C -C`F F FR' ' '&5 5 5 5 5sE   ? A	A	B BBB B%$B%)B0 0B:9B: