
    z-Ph-                    ,   d dl Z d dlZd dlmZmZ d dlmZ d dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZmZmZmZmZmZmZm Z m!Z!m"Z" d dl#m$Z$ d Z%d Z&d Z'd7dZ(d Z)d Z*d Z+ G d d          Z,d Z-d Z.d Z/d Z0d Z1 G d de j2                  Z3 G d de3          Z4 G d de4          Z5 G d de4          Z6 G d  d!e3          Z7 G d" d#e7          Z8ej9        j         G d$ d%e7                      Z: G d& d'          Z; G d( d)e;ej<                  Z= G d* d+e;ej<                  Z>d, Z?d- Z@d. ZAd/ ZBej9        C                    d0d1 d2 d3 d4 f          d5             ZDd6 ZEdS )8    N)datedatetime)Decimal)
open_csvread_csvReadOptionsParseOptionsConvertOptionsISO8601	write_csvWriteOptions	CSVWriter
InvalidRow)utilc               #   T   K   t           j        } | E d {V  | D ]}| D ]	}||z   V  
d S N)stringascii_lowercase)lettersfirstseconds      V/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pyarrow/tests/test_csv.pygenerate_col_namesr   /   sf      $G ! ! 	! 	!F&.    	!! !    c              #   V   K   t          d||z  |          D ]}| |||z            V  d S Nr   rangearrnum_colsnum_rowsis       r   
split_rowsr$   8   sL      1h)844 " "!AL.!!!!!" "r   c              #   J   K   t          d|          D ]}| |d |         V  d S r   r   r   s       r   split_columnsr&   >   sC      1h  !+X+ r      
   
Tc           	        
 t          j        d          

fdt          | |z            D             }t          j                    }t          t          j        t                      |                     }|r=|	                    d
                    |                     |	                    |           t          || |          D ]R}|	                    d
                    t          t          |                               |	                    |           S|                                                                }d t!          || |          D             }t"          j                            ||          }	||	fS )N*   c                 <    g | ]}                     d d          S )r     )randint).0_rnds     r   
<listcomp>z#make_random_csv.<locals>.<listcomp>F   s'    
D
D
DA3;;q$
D
D
Dr   ,c                 \    g | ])}t          j        |t          j                               *S ))type)paarrayint64)r/   rows     r   r2   z#make_random_csv.<locals>.<listcomp>P   sC     B B B x"(**--- B B Br   )randomRandomr   ioStringIOlist	itertoolsislicer   writejoinr$   mapstrgetvalueencoder&   r6   Tablefrom_arrays)r!   r"   linesepwrite_namesr    csv	col_namesr9   columnsexpectedr1   s             @r   make_random_csvrO   D   s[   
-

C
D
D
D
Dx(/B)C)C
D
D
DC
+--CY%&8&:&:HEEFFI 		#((9%%&&&		'#x22  		#((3sC==))***		'
,,..


!
!CB B'Xx@@B B BGx##GY77H=r   c                     t          j                    }|                    d                    |                      |                    d           |                                                                S )Nr3   
)r<   r=   rA   rB   rE   rF   )column_namesrK   s     r   make_empty_csvrS   V   sT    
+--CIIchh|$$%%%IIdOOO<<>>  """r   c                      |             }|                                 D ]`\  }}t          ||          |d         k    sJ d|z               |D ]1}t          |||           t          ||          |k    s
J d            2at          j        t
                    5  d|_        ddd           n# 1 swxY w Y   d |                                 D             } | di |}|                                 D ]\  }}t          ||          |k    sJ dS )zD
    Check setting and getting attributes of an *Options class.
    r   zincorrect default value for zfailed setting valueTNc                 &    i | ]\  }}||d          S )    )r/   namevaluess      r   
<dictcomp>z'check_options_class.<locals>.<dictcomp>n   s"    LLLfD&)LLLr   rW   )itemsgetattrsetattrpytestraisesAttributeErrorzzz_non_existent)clsattr_valuesoptsrX   rY   vnon_defaultsvalues           r   check_options_classrh   ]   s    355D#))++ D DftT""fQi///*T1 0// 	D 	DAD$"""4&&!+++-C++++	D 
~	&	& % % $% % % % % % % % % % % % % % % ML8I8I8K8KLLLL3D#))++ , ,etT""e+++++, ,s   B--B14B1c                      | di |}|                     |                    ||j                            }|                                D ]\  }}t	          ||          |k    sJ d S )N)protocolrW   )loadsdumpsHIGHEST_PROTOCOLr[   r\   )rb   picklerrc   rd   new_optsrX   rg   s          r   check_options_class_picklingrp   u   s    3D}}W]]44;4L + N N O OH"((** 0 0ex&&%/////0 0r   c                   &    e Zd Zd Zd Zd Zd ZdS )InvalidRowHandlerc                 "    || _         g | _        d S r   )resultrows)selfrt   s     r   __init__zInvalidRowHandler.__init__~   s    			r   c                 D    | j                             |           | j        S r   )ru   appendrt   )rv   r9   s     r   __call__zInvalidRowHandler.__call__   s     	{r   c                 L    t          |t                    o|j        | j        k    S r   
isinstancerr   rt   rv   others     r   __eq__zInvalidRowHandler.__eq__   s&    5"344 ,+	-r   c                 N    t          |t                     p|j        | j        k    S r   r|   r~   s     r   __ne__zInvalidRowHandler.__ne__   s)    u&7888 ,+	-r   N)__name__
__module____qualname__rw   rz   r   r   rW   r   r   rr   rr   }   sP            - - -- - - - -r   rr   c                     d} d}d t          dd          D             }t          t          || |                    }|g dg dgk    sJ t          t          || |                    }|ddgdd	gd
dgddgddggk    sJ d S )N   r'   c                     g | ]}|S rW   rW   )r/   xs     r   r2   z7test_split_rows_and_columns_utility.<locals>.<listcomp>   s    
#
#
#1
#
#
#r   rV      )rV   r'         r   )         	   r(   r   r   r   r   r   r   r(   )r   r>   r$   r&   )r!   r"   r    ru   rM   s        r   #test_split_rows_and_columns_utilityr      s    HH
#
#eArll
#
#
#C
3(3344D     =h99::G	
A	
A	
A	
A	
B      r   c           
      F   t           } |            }t          |ddgddgg ddggddgddgdd	g
           t          || ddddgddd	           |j        dk    sJ d|_        |j        dk    sJ  |d          }|j        dk    sJ |                                 d}t          j        t          j        |          5   |            }d|_        |                                 d d d            n# 1 swxY w Y   d}t          j        t          j        |          5   |            }d|_	        |                                 d d d            n# 1 swxY w Y   d}t          j        t          j        |          5   |            }d|_
        |                                 d d d            n# 1 swxY w Y   d}t          j        t          j        |          5   |            }d|_        d|_        |                                 d d d            d S # 1 swxY w Y   d S )NTFr   r   abcdutf8utf16   )use_threads	skip_rowsrR   autogenerate_column_namesencodingskip_rows_after_names)rn   r   r   rR   r   r   r   90  i  
block_sizez-ReadOptions: block_size must be at least 1: 0matchz-ReadOptions: skip_rows cannot be negative: -1z9ReadOptions: skip_rows_after_names cannot be negative: -1zTReadOptions: autogenerate_column_names cannot be true when column_names are providedab)r   rh   rp   r   validater^   r_   r6   ArrowInvalidr   r   r   rR   )pickle_modulerb   rd   r   s       r   test_read_optionsr      sN   
C355D$#$a&&(4,%738$-"('!2/0"g7 7 7 7 !m-1+,/3Tl;@*179; ; ; ; ?QDO?e####3$D?d""""MMOOO;E	re	4	4	4  suu              
 <E	re	4	4	4  suu              
 HE	re	4	4	4  suu%'"              
)E	re	4	4	4  suu)-&&	                 sH   ?&C11C58C5&EEE=&F//F36F3-HHHc                    t           }t          d          }t          |ddgddgg dddgddgddgd |g           t          || dddddd|		  	          |                                              |            }d
|_        |                                 d}t          j        t          j	        |          5   |            }d|_        |                                 d d d            n# 1 swxY w Y   t          j        t          j	        |          5   |            }d|_        |                                 d d d            n# 1 swxY w Y   d}t          j        t          j	        |          5   |            }d|_
        |                                 d d d            n# 1 swxY w Y   t          j        t          j	        |          5   |            }d|_
        |                                 d d d            n# 1 swxY w Y   d}t          j        t          j	        |          5   |            }d|_        |                                 d d d            n# 1 swxY w Y   t          j        t          j	        |          5   |            }d|_        |                                 d d d            d S # 1 swxY w Y   d S )Nskipr3   r   Fy)"zFT)	delimiterescape_char
quote_chardouble_quotenewlines_in_valuesignore_empty_linesinvalid_row_handler)rn   r   r   r   r   r   r   r   	z,ParseOptions: delimiter cannot be \\r or \\nr   rQ   z-ParseOptions: quote_char cannot be \\r or \\nz.ParseOptions: escape_char cannot be \\r or \\n)r	   rr   rh   rp   r   r   r^   r_   r6   r   r   r   )r   rb   skip_handlerrd   r   s        r   test_parse_optionsr      s?   
C$V,,LSz%*CL#4#4#4&*E],14=,0%=-1<,@B B B B !m+.-0,1.348495AC C C C CEENN355DDNMMOOO>E	re	4	4	4  suu              
 
re	4	4	4  suu              
 @E	re	4	4	4  suu              
 
re	4	4	4  suu              
 AE	re	4	4	4  suu              
 
re	4	4	4  suu                 sl   7&C))C-0C-&EE
E
3&F%%F),F)&HH	H/&I!!I%(I%&J??KKc                    t           } |            }t          |ddgddgddgddgg ddggddgddgg t          dgg	  	         t          || ddddddgddt          dg	
  
         t	          j        t                    5  d
|_        d d d            n# 1 swxY w Y   |j        dk    sJ d|_        |j        dk    sJ |j	        i k    sJ t          j                    t          j                    d|_	        |j	        t          j                    t          j                    dk    sJ ddd|_	        |j	        t          j                    t          j                    dk    sJ t          j        dt          j                    fdt          j                    fg          }||_	        |j	        t          j                    t          j                    dk    sJ dt          j                    fg|_	        |j	        dt          j                    ik    sJ t	          j        t$          d          5  dd i|_	        d d d            n# 1 swxY w Y   t	          j        t$                    5  d|_	        d d d            n# 1 swxY w Y   t'          |j        t*                    sJ d|j        v sJ d|j        v sJ ddg|_        |j        ddgk    sJ t'          |j        t*                    sJ ddg|_        |j        ddgk    sJ t'          |j        t*                    sJ ddg|_        |j        ddgk    sJ |j        g k    sJ t          g|_        |j        t          gk    sJ  |dt          j                    iddgddgdd gd!t          d"g#          }|j	        dt          j                    ik    sJ |j        ddgk    sJ |j        dd gk    sJ |j        ddgk    sJ |j        d!k    sJ |j        t          d"gk    sJ d S )$NTF.r3   defabcz%y-%m)
check_utf8strings_can_be_nullquoted_strings_can_be_nulldecimal_pointinclude_columnsinclude_missing_columnsauto_dict_encodetimestamp_parsers)	rn   r   r   r   r   r   r   r   r   z..r   韆 )r   cint16null)re   wr   r   r   r   zDataType expectedr    N/AxxxyyyNnnTttFffi  z%Y-%m-%d)column_typesnull_valuestrue_valuesfalse_valuesauto_dict_max_cardinalityr   )r
   rh   r   rp   r^   r_   
ValueErrorr   r   r   r6   r   float32r   schemaint32r   binary	TypeErrorr}   r   r>   r   r   r   )r   rb   rd   r   s       r   test_convert_optionsr     sC   
C355Du"DM$(%=CjeU^,!&124 4 4 4 !] #( %"G,	. 	. 	. 	. 
z	"	" " "!" " " " " " " " " " " " " " " )A----%*D")U2222"""" hjjrz||<<Dbhjjrz|| D DDDDD%F33Dbhjjrwyy A AAAAAYbhjj)C+=>??FDbhjjry{{ C CCCCCry{{+,Dbikk 22222	y(;	<	<	< ( ( $K( ( ( ( ( ( ( ( ( ( ( ( ( ( (	y	!	!                 d&-----!!!!!D$$$$$u~Du~----d&-----u~Du~----d'.....D....!R''''%YD!gY....3S"')),;S$K $K3"):!68 8 8D bgii 00000T{****d++++T{****)S0000!gz%:::::::s6   >BBB0
II
I
*I>>JJc                     t           }  |             }t          | ddgg dg d           |j        dk    sJ d|_        |j        dk    sJ  | d	          }|j        dk    sJ |                                 d
}t	          j        t          j        |          5   |             }d|_        |                                 d d d            d S # 1 swxY w Y   d S )NTF)r3   r   |)needednone	all_valid)include_headerr   quoting_styler   r   i&  )
batch_sizez.WriteOptions: batch_size must be at least 1: 0r   )r   rh   r   r   r^   r_   r6   r   )rb   rd   r   s      r   test_write_optionsr   f  sD   
C355DT5M5E5E5E5557 7 7 7 ?QDO?e####3$D?d""""MMOOO<E	re	4	4	4  suu                 s   &CCCc                       e Zd ZdZej        d             Zeej        d                         Ze	d             Z
d Zd Zej        j        d             Zd Zd	 Zd
S )BaseTestCSVzDCommon tests which are shared by streaming and non streaming readersc                     t           )z
        :param b: bytes to be parsed
        :param kwargs: arguments passed on to open the csv file
        :return: b parsed as a single Table
        NotImplementedErrorrv   r   kwargss      r   
read_byteszBaseTestCSV.read_bytes  s
     "!r   c                     t           )z#Whether this test is multi-threadedr   rv   s    r   r   zBaseTestCSV.use_threads  s
     "!r   c                 T    | j         t          |          k    sJ | j        |k    sJ d S r   )num_columnslenrR   )tablenamess     r   check_nameszBaseTestCSV.check_names  s6     CJJ....!U******r   c                    d}t                      }d|_        |                     ||          }|                     |ddg           |                                ddgdd	gd
k    sJ d|_        |                     ||          }|                     |dd	g           |                                g g dk    sJ d|_        t          j        t          j                  5  |                     ||          }d d d            n# 1 swxY w Y   d}d|_        |                     ||          }|                     |ddg           |                                dgd	gdk    sJ d|_        ddg|_	        |                     ||          }|                     |ddg           |                                g g dk    sJ d S )N   ab,cd
ef,gh
ij,kl
mn,op
rV   read_optionsefghijmnklop)r   r   r   )r  r  r      abcd
,,,,,
ij,kl
mn,op
r'   )r   r  )
r   r   r   r   	to_pydictr^   r_   r6   r   rR   rv   ru   rd   r   s       r   test_header_skip_rowsz!BaseTestCSV.test_header_skip_rows  s^   .}}488t---  ,,%
 %
 
 
 
 

 488t---  %
 %
 
 
 
 

 ]2?++ 	= 	=OODtO<<E	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 .488t---  &&%
 %
 
 
 
 
 !4L488t---  %
 %
 
 
 
 
 
 
s    DDDc                 (	   d}t                      }d|_        |                     ||          }|                     |ddg           |                                ddgdd	gd
k    sJ d|_        |                     ||          }|                     |ddg           |                                g g d
k    sJ d|_        |                     ||          }|                     |ddg           |                                g g d
k    sJ d}d|_        ddg|_        |                     ||          }|                     |ddg           |                                ddgdd	gdk    sJ t                      }d}d|_        t                      }d|_        |                     |||          }|                     |ddg           |                                dgd	gd
k    sJ d|_        d|_        |                     |||          }|                     |ddg           |                                dgd	gd
k    sJ t                      }t          dd          \  }}d|_        t          |          dz  |_        |                     ||          }|j        |j        k    sJ |j        dk    sJ |                                }|                                                                D ]\  }}|dd          ||         k    sJ |                     |||          }|j        |j        k    sJ |j        dk    sJ |                                }|                                                                D ]\  }}|dd          ||         k    sJ t          ddd          \  }}t                      }d|_        d |_        |j        j        |_        |                     |||          }|j        |j        k    sJ |j        |j        |j        z
  |j        z
  k    sJ |                                }|                                                                D ](\  }}||j        |j        z   d          ||         k    sJ )d S )!Nr   rV   r   r   r   r   r  r  r  r   r   r   r   r  r'   f0f1r
  r  s#   ab,cd
"e
f","g

h"
"ij","k
l"
mn,opTr   parse_options   r   r-   r!   r"   i  r   d      F)r!   r"   rJ   %   )   )r   r   r   r   r  rR   r	   r   r   rO   r   r   r"   r[   r   r   )	rv   ru   rd   r   
parse_optsrN   
table_dictrX   rY   s	            r   test_skip_rows_after_namesz&BaseTestCSV.test_skip_rows_after_names  s   .}}%&"488t---  ,,%
 %
 
 
 
 
 &'"488t---  %
 %
 
 
 
 
 &'"488t---  %
 %
 
 
 
 
 .%&"!4L488t---  ,,%
 %
 
 
 
 
 }} =%&"!^^
(,
%4.8   : :t---  &&%
 %
 
 
 
 
 &'"4.8   : :t---  &&%
 %
 
 
 
 
 }} )!dCCCh%("d))b.488|x....~$$$$__&&
$..006688 	4 	4LD&#$$<:d#333333 4.8   : :|x....~$$$$__&&
$..006688 	4 	4LD&#$$<:d#333333 )!c5:< < <h}}%'"$O14.8   : :|x....!DN2*++ + + + __&&
$..006688 	& 	&LD&4>D,FFGGHt$% % % % %	& 	&r   c                 	     fd}t          ddd          \  }}t                      }t          |          dz  |_        t	                      }dt          j                    i|_        |dz   } |d	d
          }t          j	        t
          j
        |          5                       |||           d d d            n# 1 swxY w Y   |dz   } |dd
|          }	t          j	        t
          j
        |	          5                       |||           d d d            n# 1 swxY w Y   d}
||
z   } |dd
|
dd                             d                    }t          j	        t
          j
        |          5                       |||           d d d            n# 1 swxY w Y   d|_        t          j	        t
          j
        |          5                       |||           d d d            n# 1 swxY w Y   t          j	        t
          j
        |	          5                       |||           d d d            n# 1 swxY w Y   t          j	        t
          j
        |          5                       |||           d d d            n# 1 swxY w Y   d|_        t          ddd          \  }}g d|_        |dz   } |d	d          }t          j	        t
          j
        |          5                       |||           d d d            n# 1 swxY w Y   ||
z   } |dd|
dd                             d                    }t          j	        t
          j
        |          5                       |||           d d d            n# 1 swxY w Y   |dz   } |dd          }	|	                    t          |                    }	t          j	        t
          j
        |	          5                       |||           d d d            n# 1 swxY w Y   d|_        t          j	        t
          j
        |          5                       |||           d d d            n# 1 swxY w Y   t          j	        t
          j
        |	          5                       |||           d d d            d S # 1 swxY w Y   d S )Nc                 ^    j         rd}nd                    |          } | j        |g|R  S )Nr   z	Row #{}: )r   format)
msg_formatr9   argsrow_inforv   s       r   
format_msgz@BaseTestCSV.test_row_number_offset_in_errors.<locals>.format_msg.  sC     3&--c22$:$X55555r   r   r  T)rJ   r   r   s   1,2
z{}Expected 4 columns, got 2f   r   r   convert_optionss	   a,b,c,d
zDIn CSV column #0: {}CSV conversion error to int32: invalid value 'a's  this is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long rowthis is a long row,3
z#{}Expected 4 columns, got 2: {} ...r   `   zutf-8/   Fr   r   r   de      )rO   r   r   r   r
   r6   r   r   r^   r_   r   r   decoder   rR   r  r   )rv   r  rK   r0   r   r!  csv_bad_columnsmessage_columnscsv_bad_typemessage_valuelong_rowcsv_bad_columns_longmessage_longs   `            r    test_row_number_offset_in_errorsz,BaseTestCSV.test_row_number_offset_in_errors+  s   	6 	6 	6 	6 	6 !CT:::Q"}}"%c((Q,(**(+RXZZ'8$ 
*$*%BCHH]2?/BBB 	= 	=OOO)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 ^+"
?  ]2?-@@@ 	= 	=OOL)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 <"X~!z"G"*1R4."7"7"@"@B B]2?,??? 	= 	=OO0)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= .0*]2?/BBB 	= 	=OOO)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 ]2?-@@@ 	= 	=OOL)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 ]2?,??? 	= 	=OO0)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 ./* !CU;;;Q$8$8$8!
*$*%BCHH]2?/BBB 	= 	=OOO)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
  #X~!z"G"*1R4."7"7"@"@B B]2?,??? 	= 	=OO0)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 ^+"
?  &,,SXX66]2?-@@@ 	= 	=OOL)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= "$]2?/BBB 	= 	=OOO)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 ]2?-@@@ 	= 	=OOL)5,;  = = =	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=s   B??CC<D!!D%(D%;F  F$'F$G77G;>G;"III2JJJ8LL!$L!5NN!N8PP!$P!Q44Q8;Q8SS	S	c                     d}t                      }t          j        t          d          5                       ||           d d d            n# 1 swxY w Y   t          d          |_                             ||          }|                                ddgdd	gd
k    sJ  fd}t          dd |d          d          t          dd |d          d          g}|j        j	        |k    sJ t          d          |_        t          j        t          d          5                       ||           d d d            n# 1 swxY w Y   t          dd |d          d          g}|j        j	        |k    sJ t          d          |_        |
                    |                    |                    }                     ||          }|                                ddgdd	gd
k    sJ d S )Ns   a,b
c
d,e
f,g,h
i,j
zExpected 2 columns, got 1: cr   r  r   r%  r#   ejr   c                     j         rd n| S r   )r   )r   rv   s    r   row_numz5BaseTestCSV.test_invalid_row_handler.<locals>.row_num  s    +2442r   r'   rV   r   r   r   zf,g,herror)r	   r^   r_   r   r   rr   r   r  r   ru   rk   rl   )rv   r   ru   r  r   r6  expected_rowss   `      r   test_invalid_row_handlerz$BaseTestCSV.test_invalid_row_handler  s   +!^^
]46 6 6 	< 	< OOD
O;;;	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< *;6)B)B
&J??  ss%
 %
 
 
 
 

	3 	3 	3 	3 	3 q!WWQZZ--q!WWQZZ11
 -2mCCCC *;7)C)C
&]46 6 6 	< 	< OOD
O;;;	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< $Aq''!**c::;-2mCCCC *;6)B)B
&"(()<)<Z)H)HII
J??  ss%
 %
 
 
 
 
 
 
s#   AAA$EEEc                    d}ddgddgddgd}t          d	d
          D ]P}|                     |t          d          t          |                                                    }||k    sJ Qt          d	d          D ]Z}t          j        t          d          5  |                     |t          |                     d d d            n# 1 swxY w Y   [d S )Ns   a,b,c
d,e,"f
"
g,h,i
r%  gr3  hzf
r#   r   r   r   r      T)r   r   r  r   r   z#cell values spanning multiple linesr   r   )r   r   r	   r   r  r^   r_   r   )rv   ru   rN   r   r%  s        r   test_chunker_out_of_syncz$BaseTestCSV.test_chunker_out_of_sync  ss    0ss
 

  2,, 	! 	!JLD$I$I$I(J???   A AAJ  =====  2,, 	K 	KJz%JL L L K K{j'I'I'I   K K KK K K K K K K K K K K K K K K	K 	Ks   &CC	C	N)r   r   r   __doc__r   abstractmethodr   propertyr   staticmethodr   r  r  r^   marknumpyr0  r9  r@  rW   r   r   r   r   ~  s        NN" " " " "  X" + + \++
 +
 +
Zg& g& g&R [f= f= f=P)
 )
 )
VK K K K Kr   r   c                   (   e Zd ZdddZd Zd Zd Zd Zd Zd	 Z	d
 Z
d Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd  Z d! Z!d" Z"e#j$        j%        d#             Z&e#j$        j%        d$             Z'd%S )&BaseCSVTableReadT)validate_fullc                    t          | j        t                    sJ |                    dt	                                }| j        |_        t          |g|R i |}|                    |           |S )a=  
        Reads the CSV file into memory using pyarrow's read_csv
        csv The CSV bytes
        args Positional arguments to be forwarded to pyarrow's read_csv
        validate_full Whether or not to fully validate the resulting table
        kwargs Keyword arguments to be forwarded to pyarrow's read_csv
        r   full)r}   r   bool
setdefaultr   r   r   )rv   rK   rI  r  r   r   r   s          r   r   zBaseCSVTableRead.read_csv  sw     $*D11111((GG#'#3 .t...v..M***r   c                 B     | j         t          j        |          fi |S r   )r   r6   	py_bufferr   s      r   r   zBaseCSVTableRead.read_bytes  $    t}R\!__77777r   c                 x   d}dgdgd}t          j        |          }|                     |          }|                                |k    sJ t          j        |                                          }t          j        t                    5  |                     |           d d d            d S # 1 swxY w Y   d S )Ns   a,b
1,2
rV   r'   r   )	r<   BytesIOr   r  r=   r(  r^   r_   r   )rv   dataexpected_databior   sios         r   test_file_objectz!BaseCSVTableRead.test_file_object  s     c,,jc""  M1111k$++--((]9%% 	 	MM#	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   B//B36B3c                     d}|                      |          }t          |t          j                  sJ |                     |g d           |j        dk    sJ d S )N   abc,def,gh
)r   r   r   r   )r   r}   r6   rG   r   r"   rv   ru   r   s      r   test_headerzBaseCSVTableRead.test_header  sf    %%%***** 4 4 4555~""""""r   c                 v    d}dgdgd}|                      |          }|                                |k    sJ d S )Ns   ﻿a,b
1,2
rV   r'   r   r   r  )rv   ru   rU  r   s       r   test_bomzBaseCSVTableRead.test_bom  sJ    ( c,,%%  M111111r   c                    g d}dD ]~}d|fD ]w}|                     |          |z   }|                     |          }t          |                                          dk    sJ |                                g dg ddk    sJ xd S )N)s   a,bs   1,2s   3,4s   56,78)   
      
r   rV   )rV   r   8   )r'   r   N   r   )rB   r   r   
to_batchesr  )rv   ru   line_endingfile_endingrT  r   s         r   test_one_chunkzBaseCSVTableRead.test_one_chunk  s     2112 	 	K #[1  "''--;--5++--..!3333((##- -     		 	r   c                 R   d}t                      }ddg|_        |                     ||          }|                     |ddg           |                                g dg ddk    sJ d|_        |                     ||          }|                     |ddg           |                                d	gd
gdk    sJ d|_        |                     ||          }|                     |ddg           |                                g g dk    sJ d|_        t          j        t          j	                  5  |                     ||          }d d d            n# 1 swxY w Y   d|_        g d|_        t          j        t          j	        d          5  |                     ||          }d d d            n# 1 swxY w Y   d}d|_        ddg|_        |                     ||          }|                     |ddg           |                                dd	gdd
gdk    sJ d S )Nr   r   r   r   r   r   r   r  r   r   r  r  r   r   r   r  r  r   r   r   )r   r   r   Expected 3 columns, got 2r   r  r'   r   r  )
r   rR   r   r   r  r   r^   r_   r6   r   r  s       r   test_header_column_namesz)BaseCSVTableRead.test_header_column_names  s   .}} #J488c
+++  ))))))%
 %
 
 
 
 

 488c
+++  %
 %
 
 
 
 

 488c
+++  %
 %
 
 
 
 

 ]2?++ 	= 	=OODtO<<E	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 +OO]2?!<> > > 	= 	=OODtO<<E	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=
 . #J488c
+++  %
 %
 
 
 
 
 
 
s$   7EE"EF::F>F>c                 (   d}t                      }d|_        |                     ||          }|                     |ddg           |                                g dg ddk    sJ d	|_        |                     ||          }|                     |ddg           |                                d
gdgdk    sJ d|_        t          j        t          j	                  5  |                     ||          }d d d            d S # 1 swxY w Y   d S )Nr   Tr   r
  r  rk  rl  r  r   r  r  r   )
r   r   r   r   r  r   r^   r_   r6   r   r  s       r   %test_header_autogenerate_column_namesz6BaseCSVTableRead.test_header_autogenerate_column_namesF  s   .}})-&488t---  ******%
 %
 
 
 
 

 488t---  &&%
 %
 
 
 
 
 ]2?++ 	= 	=OODtO<<E	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	=s   "DDDc                    d}t                      }dg|_        |                     ||          }|                     |dg           |                                dg dik    sJ ddg|_        |                     ||          }t          j        dt          j                    fdt          j                    fg          }|j        |k    sJ |                                g dg ddk    sJ g d|_        t          j	        t          d	
          5  |                     ||           d d d            d S # 1 swxY w Y   d S )Nr   r   r!  r   r   r  r   r   r  r  )r   r   xxr   yyz9Column 'xx' in include_columns does not exist in CSV filer   )r
   r   r   r   r  r6   r   r   r^   r_   KeyError)rv   ru   r!  r   r   s        r   test_include_columnsz%BaseCSVTableRead.test_include_columns_  s   .(**+/&'oFF'''  $$$%
 
 
 
 

 ,0,'oFFT29;;/!29;;/1 2 2|v%%%%  $$$$$$%
 %
 
 
 
 
 +=*<*<']8">? ? ? 	C 	C OOD/OBBB	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	Cs   D88D<?D<c                    d}t                      }t                      }g d|_        d|_        |                     |||          }t          j        dt          j                    fdt          j                    fdt          j                    fg          }|j        |k    sJ |	                                g dg d	g ddk    sJ ddg|_
        dd
g|_        |                     |||          }t          j        dt          j                    fd
t          j                    fg          }|j        |k    sJ |	                                g dg ddk    sJ t          j                    t          j                    d|_        |                     |||          }t          j        dt          j                    fd
t          j                    fg          }|j        |k    sJ |	                                g dg ddk    sJ d S )Nr   rv  Tr   rw  r   rx  NNNrt  r   rl  )NNNN)rx  r   )s   cds   ghs   kls   op)r   r
   r   r   r   r6   r   r   r   r  rR   r   r   r   rv   ru   r   r!  r   r   s         r   test_include_missing_columnsz-BaseCSVTableRead.test_include_missing_columns|  sw   ."}}(***<*<*<'26/<0?   A AT2799-!29;;/!2799-/ 0 0 |v%%%%  $$$$$$$$$%
 %
 
 
 
 
 &*4L!+/,'<0?   A AT29;;/!2799-/ 0 0|v%%%%  ******%
 %
 
 
 
 
 /1ikk.0hjj(: (:$<0?   A AT29;;/!28::.0 1 1|v%%%%  ...***%
 %
 
 
 
 
 
 
r   c                 6   d}|                      |          }t          j        dt          j                    fdt          j                    fdt          j                    fg          }|j        |k    sJ |                                ddgddgd	d
gdk    sJ d S )Ns   a,b,c
1,2,3
4,5,6
r   r   r   rV   r   r'   r   r   r   r=  )r   r6   r   r8   r  rv   ru   r   r   s       r   test_simple_intsz!BaseCSVTableRead.test_simple_ints  s    '%%S"(**- "(**- "(**-/ 0 0 |v%%%%  QQQ%
 %
 
 
 
 
 
 
r   c                 d   d}|                      |          }t          j        dt          j                    fdt          j                    fdt          j                    fdt          j                    fg          }|j        |k    sJ |                                ddgdd	gd
dgddgdk    sJ d S )Ns    a,b,c,d
1,2,3,0
4.0,-5,foo,True
r   r   r   r%        ?g      @r'   3fooFTr$  )r   r6   r   float64r8   r   bool_r  r  s       r   test_simple_variedz#BaseCSVTableRead.test_simple_varied  s    5%%S"*,,/ "(**- ")++. "(**-/ 0 0 |v%%%%  sRu	%
 %
 
 
 
 
 
 
r   c           
         d}|                      |          }t          j        dt          j                    fdt          j                    fdt          j                    fdt          j                    fdt          j                    fdt          j                    fg          }|j        |k    sJ |	                                g dg d	g d
g dg dg ddk    sJ d S )NsC   a,b,c,d,e,f
1,2,,,3,N/A
nan,-5,foo,,nan,TRUE
4.5,#N/A,nan,,,false
r   r   r   r%  r3  f)r  Ng      @)r'   r  N)r   r  nanr|  )   3s   nan   )NTF)r   r   r   r%  r3  r  )
r   r6   r   r  r8   r   r   r   r  r  r  s       r   test_simple_nullsz"BaseCSVTableRead.test_simple_nulls  s   . %%S"*,,/ "(**- ")++. "')), ")++. "(**-/ 0 0 |v%%%%  !!!######((($$$%
 %
 
 
 
 
 
 
r   c                 P   t          d          }d}|                     ||          }t          j        dt          j                    fdt          j                    fg          }|j        |k    sJ |                                g dg dd	k    sJ t          d
          }|                     |||          }t          j        dt          j                    fdt          j                    fg          }|j        |k    sJ |                                g dg dd	k    sJ d S )N;r   s   a;b
1.25;2,5
NA;-3
-4;NAr2  r   r   )g      ?Ng      )z2,5z-3NAr   r3   )r   )r  r!  )z1.25r  z-4)g      @g      N)r	   r   r6   r   r  r   r  r
   )rv   r  ru   r   r   r!  s         r   test_decimal_pointz#BaseCSVTableRead.test_decimal_point  sk   $s333-MBBS"*,,/ ")++.0 1 1|v%%%%  ###$$$%
 %
 
 
 
 

 )s;;;M0?   A AS")++. "*,,/1 2 2|v%%%%  %%%"""%
 %
 
 
 
 
 
 
r   c                    d}|                      |          }t          j        dt          j                    fdt          j        d          fdt          j        d          fg          }|j        |k    sJ |                                ddgt          dd	d	          t          dd
dd	          gt          dd	d	dddd          t          dd
dd	ddd          gdk    sJ d S )Nsk   a,b,c
1970,1970-01-01 00:00:00,1970-01-01 00:00:00.123
1989,1989-07-14 01:00:00,1989-07-14 01:00:00.123456
r   r   sr   ns  i  rV   r      r   ix @ r=  )r   r6   r   r8   	timestampr  r   r  s       r   test_simple_timestampsz'BaseCSVTableRead.test_simple_timestamps  s
   I %%S"(**- ",s"3"34 ",t"4"457 8 8 |v%%%%  4A&&q"a(@(@A4Aq!Q774B1a88:%
 %
 
 
 
 
 
 
r   c                 P   d}t                      }|                     ||          }t          j        dt          j                    fdt          j        d          fg          }|j        |k    sJ |                                ddgt          dd	d	          t          dd	d
          gdk    sJ dg|_        |                     ||          }t          j        dt          j        d          fdt          j                    fg          }|j        |k    sJ |                                t          dd	d	          t          dd	d
          gddgdk    sJ dt          g|_        |                     ||          }t          j        dt          j        d          fdt          j        d          fg          }|j        |k    sJ |                                t          dd	d	          t          dd	d
          gt          dd	d	          t          dd	d
          gdk    sJ d S )Ns6   a,b
1970/01/01,1980-01-01 00
1970/01/02,1980-01-02 00
rs  r   r   r  z
1970/01/01z
1970/01/02i  rV   r'   r   z%Y/%m/%dr  z1980-01-01 00z1980-01-02 00)
r
   r   r6   r   r   r  r  r   r   r   )rv   ru   rd   r   r   s        r   test_timestamp_parsersz'BaseCSVTableRead.test_timestamp_parsers  sc   Kd;;S")++. ",s"3"346 7 7|v%%%%  -4A&&q!(<(<=%
 %
 
 
 
 

 #-d;;S",s"3"34 ")++.0 1 1|v%%%%  4A&&q!(<(<=!?3%
 %
 
 
 
 

 #-g!6d;;S",s"3"34 ",s"3"346 7 7|v%%%%  4A&&q!(<(<=4A&&q!(<(<=%
 %
 
 
 
 
 
 
r   c                 d   d}|                      |          }t          j        dt          j                    fdt          j                    fg          }|j        |k    sJ |                                t          ddd          t          ddd          gt          ddd          t          ddd          gdk    sJ t                      }t          j                    t          j                    d|_        |                      ||	          }t          j        dt          j                    fdt          j                    fg          }|j        |k    sJ |                                t          ddd          t          ddd          gt          ddd          t          ddd          gdk    sJ t                      }t          j	        d
          t          j	        d          d|_        |                      ||	          }t          j        dt          j	        d
          fdt          j	        d          fg          }|j        |k    sJ |                                t          ddd          t          ddd          gt          ddd          t          ddd          gdk    sJ d S )Ns0   a,b
1970-01-01,1970-01-02
1971-01-01,1971-01-02
r   r   r  rV   i  r'   r   rs  r  ms)r   r6   r   date32r  r   r
   date64r   r  r   )rv   ru   r   r   rd   s        r   
test_dateszBaseCSVTableRead.test_dates*  s   E%%S")++. ")++.0 1 1|v%%%%  tQ""Dq!$4$45tQ""Dq!$4$45%
 %
 
 
 
 
 "$)++BIKK@@d;;S")++. ")++.0 1 1|v%%%%  tQ""Dq!$4$45tQ""Dq!$4$45%
 %
 
 
 
 
 "$,s"3"3",t:L:LMMd;;S",s"3"34 ",t"4"457 8 8|v%%%%  4A&&q!(<(<=4A&&q!(<(<=%
 %
 
 
 
 
 
 
r   c           	         ddl m} d}|                     |          }t          j        dt          j        d          fdt          j                    fg          }|j        |k    sJ |                                 |ddd	           |d
dd          gddgdk    sJ t                      }t          j	        d          t          j        d          d|_
        |                     ||          }t          j        dt          j	        d          fdt          j        d          fg          }|j        |k    sJ |                                 |ddd	           |d
dd          g |ddd	d           |d
ddd          gdk    sJ d S )Nr   )times0   a,b
12:34:56,12:34:56.789
23:59:59,23:59:59.999
r   r  r      "   rd  r'  ;   z12:34:56.789z23:59:59.999r   usr  rs  i
 iX> )r   r  r   r6   r   time32r   r  r
   time64r   )rv   r  ru   r   r   rd   s         r   
test_timeszBaseCSVTableRead.test_timesN  s   !!!!!!E%%S")C..1 ")++.0 1 1|v%%%%  $r2r""DDR$4$45 .1%
 %
 
 
 
 
 "$)D//	$HHd;;S")D//2 ")D//24 5 5|v%%%%  $r2r""DDR$4$45$r2r6**DDRV,D,DE%
 %
 
 
 
 
 
 
r   c                    t          d          }d                                }|                     ||          }t          j        dt          j        t          j                    t          j                              fdt          j                    fg          }g dg dd	}|j        |k    sJ |	                                |k    sJ d
|_
        |                     ||          }|j        |k    sJ |	                                |k    sJ d|_
        |                     ||          }|j        t          j        dt          j                    fdt          j                    fg          k    sJ |	                                |k    sJ d|_
        d|_        d}|                     ||d          }|j        |k    sJ |d                             d          j        }t          |          d
k    sJ |d                                         dk    sJ |d                                         dk    sJ d|_        |                     ||          }t          j        dt          j        t          j                    t          j                              fdt          j                    fg          }g dg dd	}|j        |k    sJ |	                                |k    sJ d S )NT)r   u   a,b
ab,1
cdé,2
cdé,3
ab,4rs  r   r   )r      cdér  r   )rV   r'   r   r   r   r'   rV   2   Fs   a,b
ab,1
cd,2
ab,3)r!  rI  r   r      cd)   abr  r  rV   r'   r   )r
   rF   r   r6   r   
dictionaryr   r   r8   r  r   r   chunkr   as_py	as_bufferr   )rv   rd   ru   r   r   rN   dict_valuess          r   test_auto_dict_encodez&BaseCSVTableRead.test_auto_dict_encodei  s   t44407799d;;S"-

BIKK"H"HI "(**-/ 0 0 .--
 
 |v%%%%  H,,,,)*&d;;|v%%%%  H,,,, *+&d;;|ry3	*<+.

*;*=  >  > > > > >  H,,,, *,&+d.3   5 5|v%%%%Cj&&q))4;1$$$$1~##%%----1~''))Y6666 d;;S"-

BIKK"H"HI "(**-/ 0 0 +**
 
 |v%%%%  H,,,,,,r   c                    t          ddg          }d}|                     ||          }t          j        dt          j                    fdt          j                    fdt          j                    fd	t          j                    fg          }|j        |k    sJ |                                d d gdd
gddgdd gdk    sJ t          ddgd          }|                     ||          }|                                d d gd d
gddgdd gdk    sJ d|_        |                     ||          }|                                d d gdd
gddgdd gdk    sJ t          g           }d}|                     ||          }t          j        dt          j                    fdt          j                    fg          }|j        |k    sJ |                                d
gdgdk    sJ d S )NXxxZzz)r   s$   a,b,c,d
Zzz,"Xxx",1,2
Xxx,#N/A,,Zzz
rs  r   r   r   r%  z#N/A1r   r'   r$  T)r   r   Fs
   a,b
#N/A,
r   )	r
   r   r6   r   r   r   r8   r  r   rv   rd   ru   r   r   s        r   test_custom_nullsz"BaseCSVTableRead.test_custom_nulls  si   5%.999=d;;S"')), ")++. ")++. "(**-/ 0 0 |v%%%%  rT	%
 %
 
 
 
 
 5%.268 8 8d;;  rT	%
 %
 
 
 
 
 +0'd;;  rT	%
 %
 
 
 
 
 "---d;;S")++. ")++.0 1 1|v%%%%  %
 %
 
 
 
 
 
 
r   c                 d   t          ddgddg          }d}|                     ||          }t          j        dt          j                    fd	t          j                    fd
t          j                    fg          }|j        |k    sJ |                                g dg dg ddk    sJ d S )Nr   yesr   no)r   r   s>   a,b,c
True,T,t
False,F,f
True,yes,yes
False,no,no
N/A,N/A,N/A
rs  r   r   r   )TrueFalser  r  r   )TFTFN)tr  r  r  r   r=  )r
   r   r6   r   r   r  r  r  s        r   test_custom_boolsz"BaseCSVTableRead.test_custom_bools  s    3,,/;8 8 8! d;;S")++. "(**- ")++.0 1 1 |v%%%%  :::111///%
 %
 
 
 
 
 
 
r   c                    t          dddt          j        dd          dd          }d	}|                     ||
          }t          j        dt          j                    fdt          j                    fdt          j                    fdt          j                    fdt          j        dd          fg          }ddgddgddgddgt          d          t          d          gd}|j        |k    sJ |
                                |k    sJ t          t          j        dt          j                    fdt          j                    fdt          j                    fdt          j        dd          fdt          j                    fg                    }|                     ||
          }|j        |k    sJ |
                                |k    sJ d}t          j        t          j                  5 }|                     ||
           d d d            n# 1 swxY w Y   t          |j                  }d|v sJ d|v sJ d S )Nr   r   booleanr   r'   r   )r   r   r%  r3  zzr   s(   a,b,c,d,e
1,2,3,true,1.0
4,-5,6,false,0
rs  r   r   r   r%  r3  rV   r          @g      r  6TF1.00z0.00)r   r   r   r%  r3  r  s(   a,b,c,d,e
1,XXX,3,true,5
4,-5,6,false,7
zIn CSV column #1: z2CSV conversion error to float: invalid value 'XXX')r
   r6   
decimal128r   r   r8   r   r   r  r   r  r^   r_   r   rD   rg   )rv   rd   ru   r   r   rN   excerrs           r   test_column_typesz"BaseCSVTableRead.test_column_types  s   191:13r11E1E28	,: ,: ; ; ;
 >d;;S"(**- "*,,/ ")++. "(**- "-A"6"67	9 : : Qts&//76??3
 
 |v%%%%  H,,,,S"*,,$7%(")++$6%("(**$5%("-A*>*>$?%)28::$6	$8 9 9: : : d;;|v%%%%  H,,,,=]2?++ 	8sOOD$O777	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8#)nn#s****CsJJJJJJs   6HH!Hc           
      b   dt          j        t          j                    t          j                              fdt          j        t          j                    t          j                              fdt          j        t          j                    t          j        dd                    fdt          j        t          j                    t          j                              fg}t          t          |                    }d}| 	                    ||	          }t          j
        |          }g d
g dt          d          t          d          t          d          gg dd}|j
        |k    sJ |                                |k    sJ dt          j        t          j                    t          j                              f|d<   t          t          |                    }t          j        t                     5  | 	                    ||	          }d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r'   r%  r  s<   a,b,c,d
abc,123456,1.0,zz
defg,123456,0.5,xx
abc,N/A,1.0,xx
rs  )r   defgr   )r  r  Nr  z0.50)r  rw  rw  r$  r   )r6   r  r   r   r8   r  
large_utf8r
   dictr   r   r   r  int8r^   r_   r   )rv   r   rd   ru   r   r   rN   s          r   test_column_types_dictz'BaseCSVTableRead.test_column_types_dict  s)    "-

BGII667"-

BHJJ778"-

BM"a,@,@AAB"-

BMOO<<=	? 4+=+=>>>$ d;;<((''''''&//76??GFOOD###	
 
 |v%%%%  H,,,, bgii C CDQ4+=+=>>>].// 	@ 	@OOD$O??E	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@ 	@s   ?H$$H(+H(c                 x   d}t          ddg          }t          dt          j                    i          }|                     |||          }t          j        dt          j                    fdt          j                    fg          }|j        |k    sJ |                                g dg dd	k    sJ d S )
Ns   a,b
c,d
e,f
r   r   rR   r  r   )   a   c   e)r   r%  r  rm  )r   r
   r6   r   r   r   r   r  r}  s         r   #test_column_types_with_column_namesz4BaseCSVTableRead.test_column_types_with_column_names&  s     ""c
;;;(sBIKK6HIII<0?   A AS")++. ")++.0 1 1|v%%%%  ### %
 %
 
 
 
 
 
 
r   c                 |    d}|                      |          }|                                ddgddgddgdk    sJ d S )	N   a,b,c
1,2,3
4,5,6rV   r   r'   r   r   r   r=  r^  r[  s      r   test_no_ending_newlinez'BaseCSVTableRead.test_no_ending_newline6  sb    %%%  QQQ%
 %
 
 
 
 
 
 
r   c                 l    d}|                      |          }|                                dg ik    sJ d S )Ns   ,

r   r^  r[  s      r   test_trivialzBaseCSVTableRead.test_trivial@  s=    %%  RH,,,,,,r   c                    d}|                      |          }|                                ddgddgdk    sJ t          d          }|                      ||	          }|                                g d
g ddk    sJ t          d          }|                      |||          }|                                d dgd dgdk    sJ d S )Ns   a,b
1,2

3,4
rV   r   r'   r   r   F)r   r2  )NrV   Nr   )Nr'   Nr   )r   r?  )r  2)r   r  r	   r   )rv   ru   r   r  r   s        r   test_empty_linesz!BaseCSVTableRead.test_empty_linesF  s)   +%%  QQ%
 %
 
 
 
 
 %>>>MBB  ######%
 %
 
 
 
 
 #Q///M-9   ; ;  %
 %
 
 
 
 
 
 
r   c                    d}t          j        t          j        d          5  |                     |           d d d            n# 1 swxY w Y   d}t          j        t          j        d          5  |                     |           d d d            n# 1 swxY w Y   dD ]O}t          j        t          j        d          5  |                     |           d d d            n# 1 swxY w Y   Pd S )Ns   a,b,c
1,2
4,5,6
rn  r   s   a,b,c
1,2,3
4zExpected 3 columns, got 1)r   ra  rc  rb  s   

Empty CSV file)r^   r_   r6   r   r   )rv   ru   s     r   test_invalid_csvz!BaseCSVTableRead.test_invalid_csv[  s   %]2?2MNNN 	" 	"OOD!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"!]2?2MNNN 	" 	"OOD!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"9 	& 	&Dr6FGGG & &%%%& & & & & & & & & & & & & & &	& 	&s5   AA	A	2BBBC%%C)	,C)	c                     d}|                      |          }|                                dgdgdk    sJ t          d          }|                      ||          }|                                dgd	gd
k    sJ d S )N   a;b,c
de,fg;eh
defg;eha;br   r  r  r2  de,fgehr   b,c)r   r  r	   )rv   ru   r   rd   s       r   test_options_delimiterz'BaseCSVTableRead.test_options_delimiterg  s    #%%  6%
 %
 
 
 
 
 c***D99  6%
 %
 
 
 
 
 
 
r   c                    t          dd          \  }}|                     |          }|j        |j        k    sJ |                    |          sJ |                                |                                k    sJ d S Nr'   r(   r  )rO   r   r   equalsr  )rv   rK   rN   r   s       r   test_small_random_csvz&BaseCSVTableRead.test_small_random_csvu  s    'R@@@X$$|x....||H%%%%%  H$6$6$8$8888888r   c                 l   t          dd          \  }}g d}||                    d          g}|D ]}|D ]|}t          |          }|                     ||          }|j        |j        k    sJ |                    |          s,|                                |                                k    sJ }d S )Nr'     r  )r   r        r  o   rc  r   r   )rO   rstripr   r   r   r  r  )	rv   csv_baserN   block_sizescsvsrK   r   r   r   s	            r   test_stress_block_sizesz(BaseCSVTableRead.test_stress_block_sizes|  s    ,a#FFF(///(//'223 	E 	EC) E E
*jAAA,GG|x6666||H-- E ??,,0B0B0D0DDDDDE	E 	Er   c                    	 t           j        }n# t          $ r t           j         }Y nw xY wd}d t          |          D             }t	          |          } |            }t          d |d d d         D                       }|                     ||          } |            |z
  }|dk    sJ |j        |k    sJ |j        dk    sJ |j	        |k    sJ d S )	Ni'  c                 8    g | ]}d                      |          S )zK{})r  )r/   r#   s     r   r2   zGBaseCSVTableRead.test_stress_convert_options_blowup.<locals>.<listcomp>  s"    AAAU\\!__AAAr   c                 6    i | ]}|t          j                    S rW   )r6   r   )r/   ks     r   rZ   zGBaseCSVTableRead.test_stress_convert_options_blowup.<locals>.<dictcomp>  s     AAAQ!RY[[AAAr   r'   r  rs        $@r   )
r  thread_timer`   r   rS   r
   r   r   r"   rR   )	rv   clockr   rL   rK   t1r!  r   dts	            r   "test_stress_convert_options_blowupz3BaseCSVTableRead.test_stress_convert_options_blowup  s   	$EE 	 	 	IEEE	AAeK.@.@AAA	Y''UWW(AA)CCaC.AAAC C C_EEUWWr\ Tzzzz K////~""""!Y......s    ((c           	         t          j                    j        t          j                    j        k    rt	          j        d           d }d}d}d}|dk     r6|dk     r/t          d|           d	d
|z  z   }d }	 t          j                    5 }	 t          j	        |          }|
                                 t          j                    }		 |                     |           n/# t          $ r"}
|
}t          j                    |	z
  }Y d }
~
nd }
~
ww xY wt          j        |gg |gd           n# t          j        |gg |gd           w xY w	 d d d            n# 1 swxY w Y   n# t          $ r Y nw xY w||j        n|dz  }|dk     r|dk     /|t	          j        d           |dk    sJ |j        }
t#          |
t$          j                  sJ |
j        t*          j        k    sJ d S )Nz'test only works from main Python threadc                  j    t          j        d           t          j        t          j                   d S )Ng?)r  sleepsignalraise_signalSIGINTrW   r   r   signal_from_threadz>BaseCSVTableRead.test_cancellation.<locals>.signal_from_thread  s(    JsOOO.....r   g        i r   g      @r(   zworkload size:s   a,b,c
s   1,2,3
targetr  r   z0Failed to get an interruption during CSV readingr  )	threadingcurrent_threadidentmain_threadr^   r   printr   signal_wakeup_fdThreadstartr  r   KeyboardInterruptselect__context__failr}   r6   ArrowCancelledsignumr  r  )rv   r  last_durationworkload_sizeattempts	large_csvexc_infosigfdr  r  r3  s              r   test_cancellationz"BaseCSVTableRead.test_cancellation  s   $&&,%''-. .KABBB	/ 	/ 	/ c!!hmm"M222"Z-%??IH *,, BB%,4FGGG			!Y[[= OOI66660 = = ='(H,0IKK",<MMMMMM= ugrE7DAAAAugrE7DAAAAAB B B B B B B B B B B B B B B  %    # '3 *A-MM c!!hmmP KJKKK #### !R./////x6=((((((sx   9E! E<D*C! D*!
D+DD*DD*E*EE	E! EE! EE! !
E.-E.c                      t          j         fd          }|                                 |                                 d S )Nc                  .                          d          S )Ns   f64
0.1)r   r   s   r   <lambda>z=BaseCSVTableRead.test_cancellation_disabled.<locals>.<lambda>  s    4??;77 r   r  )r  r  r  rB   )rv   r  s   ` r   test_cancellation_disabledz+BaseCSVTableRead.test_cancellation_disabled  sF    
 77779 9 9					r   N)(r   r   r   r   r   rX  r\  r_  ri  ro  rq  rz  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r
  r^   rE  r  r(  r,  rW   r   r   rH  rH    s[       15     8 8 8	 	 	# # #2 2 2  1
 1
 1
f= = =2C C C:+
 +
 +
Z
 
 

 
 
 
 
 
.
 
 
2
 
 
" 
  
  
D"
 "
 "
H
 
 
6/- /- /-b,
 ,
 ,
\
 
 
*'K 'K 'KR@ @ @@
 
 
 
 
 
- - -
 
 
*
& 
& 
&
 
 
9 9 9E E E/ / /0 [@) @) @)D [    r   rH  c                   $    e Zd Zed             ZdS )TestSerialCSVTableReadc                     dS NFrW   r   s    r   r   z"TestSerialCSVTableRead.use_threads      ur   Nr   r   r   rC  r   rW   r   r   r.  r.    -          X  r   r.  c                   $    e Zd Zed             ZdS )TestThreadedCSVTableReadc                     dS NTrW   r   s    r   r   z$TestThreadedCSVTableRead.use_threads      tr   Nr2  rW   r   r   r5  r5    s-          X  r   r5  c                        e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Zd Zd Z fdZej        j         fd            Z xZS )BaseStreamingCSVReadc                 ~    |                     dt                                }| j        |_        t          |g|R i |S )z
        Reads the CSV file into memory using pyarrow's open_csv
        csv The CSV bytes
        args Positional arguments to be forwarded to pyarrow's open_csv
        kwargs Keyword arguments to be forwarded to pyarrow's open_csv
        r   )rN  r   r   r   )rv   rK   r  r   r   s        r   r   zBaseStreamingCSVRead.open_csv  sF     ((GG#'#3 -d---f---r   c                 B     | j         t          j        |          fi |S r   )r   r6   rP  r   s      r   
open_byteszBaseStreamingCSVRead.open_bytes	  rQ  r   c                 (   |j         |k    sJ t          |          }t          |          t          |          k    sJ t          ||          D ]B\  }}|                    d           |j         |k    sJ |                                |k    sJ Cd S )NTrK  )r   r>   r   zipr   r  )rv   readerexpected_schemarU  batchesbatchexpected_batchs          r   check_readerz!BaseStreamingCSVRead.check_reader  s    }////v,,7||s=111111%(-%@%@ 	7 	7!E>NNN%%%<?2222??$$66666	7 	7r   c                 B     | j         |fi |                                S r   )r=  read_allr   s      r   r   zBaseStreamingCSVRead.read_bytes  s(    tq++F++44666r   c                    d}ddgddgd}t          j        |          }|                     |          }t          j        dt          j                    fdt          j                    fg          }|                     |||g           d S )	Ns   a,b
1,2
3,4
rV   r   r'   r   r   r   r   )r<   rS  r   r6   r   r8   rE  )rv   rT  rU  rV  r@  rA  s         r   rX  z%BaseStreamingCSVRead.test_file_object  s    ! !fAq622js##)c28::%6&)28::%6%8 9 9&/M?CCCCCr   c                    d}|                      |          }t          j        dt          j                    fdt          j                    fdt          j                    fg          }|                     ||g            d S )NrZ  r   r   r   )r=  r6   r   r   rE  rv   ru   r@  rA  s       r   r\  z BaseStreamingCSVRead.test_header!  sw    &&)eRWYY%7&+RWYY%7&*BGII%6%8 9 9 	&/266666r   c                    d}t          j        dt          j                    fdt          j                    fg          }t	                      }t          |          |_        |                     ||          }|                     ||g dg ddg           t          |          dz
  |_        |                     ||          }|                     ||d	d
gddgddgdgdg           d S )N   a,b
123,456
abc,de
gh,ij
r   r   r   )123r   r   )   456   de   ijr   rV   rM  r   rN  rO  r   rP  )	r6   r   r   r   r   r   r   r=  rE  )rv   ru   rA  r   r@  s        r   test_inferencez#BaseStreamingCSVRead.test_inference)  s$   3)c29;;%7&)29;;%7%9 : : #}}"%d))LAA&/!5!5!5!;!;!;= = >	? 	? 	? #&d))a-LAA&/"'"()!46 6"&"'* *+	, 	, 	, 	, 	,r   c                    d}t                      }t          |          dz
  |_        |                     ||          }t	          j        dt	          j                    fdt	          j                    fg          }|j        |k    sJ |                                                                dgdgdk    sJ t          j
        t          d	
          5  |                                 d d d            n# 1 swxY w Y   t          j
        t                    5  |                                 d d d            d S # 1 swxY w Y   d S )NrL  r   r   r   r   {   i  r   zCSV conversion error to int64r   )r   r   r   r=  r6   r   r8   read_next_batchr  r^   r_   r   StopIterationrv   ru   r   r@  rA  s        r   test_inference_failurez+BaseStreamingCSVRead.test_inference_failure>  s   3"}}"%d))a-LAA)c28::%6&)28::%6%8 9 9}////%%''1133cU8
 8
 
 
 
 
 ]:!@B B B 	% 	%""$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% ]=)) 	% 	%""$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%s$   C55C9<C9D;;D?D?c                    d}t                      }d|_        t          j        t          j        d          5  |                     ||          }d d d            n# 1 swxY w Y   d}d|_        |                     ||          }|                                                                dgd	gd
k    sJ t          j        t          j        d          5  |                                 d d d            n# 1 swxY w Y   t          j        t                    5  |                                 d d d            d S # 1 swxY w Y   d S )Ns   a,b
1,2,3
4,5
6,7
r(   zExpected 2 columns, got 3r   r   s   a,b
1,2
3,4,5
6,7
r   rV   r'   r   )
r   r   r^   r_   r6   r   r=  rT  r  rU  )rv   ru   r   r@  s       r   r  z%BaseStreamingCSVRead.test_invalid_csvR  s   ("}}"$]2?!<> > > 	1 	1__< % 1 1F	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 )"#LAA%%''1133aSs7K7KKKKK]2?!<> > > 	% 	%""$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% ]=)) 	% 	%""$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%s5   AA #A C99C= C=D??EEc                    d}|                      |          }t          j        dt          j                    fdt          j                    fg          }|                     ||dgdgdg           t          d          }|                      ||	          }t          j        d
t          j                    fdt          j                    fg          }|                     ||dgdgdg           d S )Nr  r  r   r  r  r  r  r  r2  r   r  r  r  r  )r=  r6   r   r   rE  r	   )rv   ru   r@  rA  rd   s        r   r  z+BaseStreamingCSVRead.test_options_delimiterh  s   #&&)eRY[[%9&)29;;%7%9 : :&/$(6"), , -	. 	. 	. c***T::)c29;;%7&+RY[[%9%; < <&/")$(6+ + ,	- 	- 	- 	- 	-r   c                    d}|                      |          }t          j        dt          j                    fdt          j                    fdt          j                    fg          }|                     ||ddgddgd	d
gdg           d S )Nr  r   r   r   rV   r   r'   r   r   r   r=  )r=  r6   r   r8   rE  rJ  s       r   r  z+BaseStreamingCSVRead.test_no_ending_newliney  s    %&&)c28::%6&)28::%6&)28::%6%8 9 9 	&/"#Q"#Q"#Q) ) *	+ 	+ 	+ 	+ 	+r   c                     t          j        t          d          5  |                     d           d d d            d S # 1 swxY w Y   d S )Nr  r   r   )r^   r_   r   r=  r   s    r   test_empty_filez$BaseStreamingCSVRead.test_empty_file  s    ]:-=>>> 	! 	!OOC   	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!s   ?AAc                    d}t                      }g d|_        |                     ||          }t          j        dt          j                    fdt          j                    fdt          j                    fg          }|                     ||ddgd	d
gddgdg           t                      }ddg|_        |                     |||          }t          j        dt          j                    fdt          j                    fg          }|                     ||d	d
gddgdg           dt          j	                    i|_
        |                     |||          }t          j        dt          j                    fdt          j	                    fg          }|                     ||ddgddgdg           g d|_        t          j        t          d          5  |                     |||          }d d d            n# 1 swxY w Y   d|_        |                     |||          }t          j        dt          j                    fdt          j                    fdt          j	                    fg          }|                     ||d d gddgddgdg           t          j	                    t          j                    d|_
        |                     |||          }t          j        dt          j                    fdt          j                    fdt          j	                    fg          }|                     ||d d gddgddgdg           d S )Ns   1,2,3
4,5,6)r%  r3  r  r   r%  r3  r  rV   r   r'   r   r   r   r   )r3  r  r  5)r;  r  r3  z,Column 'g' in include_columns does not existr   Tr;  )r;  r3  r  )r3  r;  )r   rR   r=  r6   r   r8   rE  r
   r   r   r   r^   r_   ry  r   r   r  )rv   ru   r   r@  rA  r!  s         r   test_column_optionsz(BaseStreamingCSVRead.test_column_options  s#   "}}$3OO!LAA)c28::%6&)28::%6&)28::%6%8 9 9 	&/"#Q"#Q"#Q) ) *	+ 	+ 	+ )**+.*'L1@ ! B B)c28::%6&)28::%6%8 9 9&/"#Q"#Q) ) *	+ 	+ 	+
 ),RY[['9$L1@ ! B B)c28::%6&)29;;%7%9 : :&/"%s"#Q) ) *	+ 	+ 	+
 +://']DF F F 	F 	F __T5D % F FF	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 37/L1@ ! B B)c2799%5&)28::%6&)29;;%7%9 : : 	&/"&"%s"#Q) ) *	+ 	+ 	+
 .0Y[[rz||'L'L$L1@ ! B B)c2:<<%8&)28::%6&)29;;%7%9 : : 	&/"&"%s"#Q) ) *	+ 	+ 	+ 	+ 	+s   HH
Hc                    d}t                      }|                     ||          }t          j        dt          j                    fdt          j                    fg          }|                     ||dgdgdg           d|_        |                     ||          }t          j        dt          j                    fdt          j                    fg          }|                     ||dgd	gdg           d
}d|_        |                     ||          }t          j        dt          j                    fdt          j                    fg          }|                     ||dgd	gdg           d S )Ns   a,b
un,lphantr   r   r   uns   lphantr   latin1u
   éléphants    a , b 
 u n ,  l  p h a n t r   )r   r=  r6   r   r   r   rE  r   rV  s        r   test_encodingz"BaseStreamingCSVRead.test_encoding  s   ("}}LAA)c29;;%7&)29;;%7%9 : :&/"&"3!46 6 7	8 	8 	8 !)LAA)c29;;%7&)29;;%7%9 : :&/"&".1 1 2	3 	3 	3
F 'LAA)c29;;%7&)29;;%7%9 : :&/"&".1 1 2	3 	3 	3 	3 	3r   c                 *   t          dd          \  }}|                     |          }|                                }|j        |j        k    sJ |                    |          sJ |                                |                                k    sJ d S r  )rO   r=  rG  r   r  r  )rv   rK   rN   r@  r   s        r   r  z*BaseStreamingCSVRead.test_small_random_csv  s    'R@@@X%%!!|x....||H%%%%%  H$6$6$8$8888888r   c                    t          dd          \  }}g d}||                    d          g}|D ]}|D ]}|d |                             d          dk    sJ t          |          }|                     ||          }|                                }	|	j        |j        k    sJ |	                    |          s,|	                                |                                k    sJ d S )	Nr'   r  r  )      r'  r  r  r  rc  ra  r   r   )	rO   r  countr   r=  rG  r   r  r  )
rv   r  rN   r  r  rK   r   r   r@  r   s
             r   r   z,BaseStreamingCSVRead.test_stress_block_sizes  s$   ,a#FFF(///(//'223 	E 	EC) 
E 
E
;J;'--e449999*jAAAl ) 4 4))|x6666||H-- E ??,,0B0B0D0DDDDD
E	E 	Er   c                    t          j                     t          j                    }d }d}t	                      }ddg|_        d|_        |                     ||          } ||dgdgd	           t          j                    } ||d
gdgd	           t          j                    |k    sJ  ||dgdgd	           t          j                    |k    sJ  ||dgdgd	           t          j                    |k    sJ t          j	        t                    5  |                                 d d d            n# 1 swxY w Y   t          j                    |k    sJ d }t          j                    |k    sJ d S )Nc                 b    |                                  }|                                |k    sJ d S r   )rT  r  )r@  rN   rC  s      r   check_one_batchzABaseStreamingCSVRead.test_batch_lifetime.<locals>.check_one_batch  s4    **,,E??$$000000r   s   10,11
12,13
14,15
16,17
r   r   r   r   r(   r   r   r  r  r  r>     r  )gccollectr6   total_allocated_bytesr   rR   r   r=  r^   r_   rU  rT  )rv   old_allocatedrk  ru   r   r@  allocated_after_first_batchs          r   test_batch_lifetimez(BaseStreamingCSVRead.test_batch_lifetime  s   

022	1 	1 	1 /"}}%(#J!"#LAArd" 6 6777&(&>&@&@#rd" 6 6777'))-HHHHHrd" 6 6777'))-HHHHHrd" 6 6777'))-HHHHH]=)) 	% 	%""$$$	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	% 	%'))]::::'))]::::::s   D==EEc                 H   t                                                       d}t                      }d|_        ddg|_        |                     ||          }t          j        t                    5  |	                                sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   r   )
superr  r   r   rR   r=  r^   r_   rU  rT  rv   ru   rd   r@  	__class__s       r   r  z*BaseStreamingCSVRead.test_header_skip_rows  s    %%'''. }}!4LD99]=)) 	, 	,))++++++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,s   2BBBc                    t                                                       d}t                      }d|_        |                     ||          }t          j        t                    5  |                                sJ 	 d d d            n# 1 swxY w Y   d|_        |                     ||          }t          j        t                    5  |                                sJ 	 d d d            d S # 1 swxY w Y   d S )Nr   r   r   r   )	rt  r  r   r   r=  r^   r_   rU  rT  ru  s       r   r  z/BaseStreamingCSVRead.test_skip_rows_after_names+  st   **,,,. }}%&"D99]=)) 	, 	,))++++++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, &+"D99]=)) 	, 	,))++++++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,s$   )BBBC44C8;C8)r   r   r   r   r=  rE  r   rX  r\  rQ  rW  r  r  r  r\  r_  rc  r  r   rr  r  r^   rE  rF  r  __classcell__rv  s   @r   r:  r:    si       	. 	. 	.8 8 87 7 77 7 7D D D7 7 7, , ,*% % %(% % %,- - -"
+ 
+ 
+! ! !?+ ?+ ?+B3 3 3<9 9 9E E E$; ; ;8, , , , , [, , , , , , , , ,r   r:  c                   $    e Zd Zed             ZdS )TestSerialStreamingCSVReadc                     dS r0  rW   r   s    r   r   z&TestSerialStreamingCSVRead.use_threads@  r1  r   Nr2  rW   r   r   r{  r{  ?  r3  r   r{  c                   $    e Zd Zed             ZdS )TestThreadedStreamingCSVReadc                     dS r7  rW   r   s    r   r   z(TestThreadedStreamingCSVRead.use_threadsG  r8  r   Nr2  rW   r   r   r~  r~  E  s-          X  r   r~  c                   &    e Zd Zd Zd Zd Zd ZdS )BaseTestCompressedCSVReadc                 :    t          j        d          | _        d S )Nzarrow-csv-test-)prefix)tempfilemkdtemptmpdirr   s    r   setUpzBaseTestCompressedCSVRead.setUpN  s    &.?@@@r   c                 8    t          j        | j                   d S r   )shutilrmtreer  r   s    r   tearDownz"BaseTestCompressedCSVRead.tearDownQ  s    dk"""""r   c                     	 t          |          S # t          j        $ r,}t          j        t          |                     Y d }~d S d }~ww xY wr   )r   r6   ArrowNotImplementedErrorr^   r   rD   )rv   csv_pathr3  s      r   r   z"BaseTestCompressedCSVRead.read_csvT  s_    	 H%%%* 	  	  	 KA	 s    A!AAc                    t          dd          \  }}t          j                            | j        | j                  }|                     ||           |                     |          }|                    d           |j	        |j	        k    sJ |
                    |          sJ |                                |                                k    sJ d S )Nr'   r  r  TrK  )rO   ospathrB   r  csv_filename
write_filer   r   r   r  r  )rv   rK   rN   r  r   s        r   test_random_csvz)BaseTestCompressedCSVRead.test_random_csvZ  s    'SAAAX7<<T->??#&&&h''D!!!|x....||H%%%%%  H$6$6$8$8888888r   N)r   r   r   r  r  r   r  rW   r   r   r  r  L  sS        A A A# # #     9 9 9 9 9r   r  c                       e Zd ZdZd Zd ZdS )TestGZipCSVReadzcompressed.csv.gzc                     t          j        |dd          5 }|                    |           d d d            d S # 1 swxY w Y   d S )Nwbr   )gzipopenrA   rv   r  contentsr  s       r   r  zTestGZipCSVRead.write_fileh  s    YtT1%% 	GGH	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   :>>c                    t           j                            | j        | j                  }t          j        |dd          5 }|                    d           d d d            n# 1 swxY w Y   t          j        |dd          5 }|                    d           d d d            n# 1 swxY w Y   |                     |          }|	                                g dg ddk    sJ d S )	Nr  r   s   ab,cd
ef,gh
r   s   ij,kl
mn,op
rt  ru  r	  )
r  r  rB   r  r  r  r  rA   r   r  )rv   r  r  r   s       r   test_concatenatedz!TestGZipCSVRead.test_concatenatedl  sf   7<<T->??Yxq)) 	'QGG%&&&	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	'Yxq)) 	'QGG%&&&	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	'h''  $$$$$$%
 %
 
 
 
 
 
 
s$   A##A'*A'B&&B*-B*N)r   r   r   r  r  r  rW   r   r   r  r  e  s7        &L  
 
 
 
 
r   r  c                       e Zd ZdZd ZdS )TestBZ2CSVReadzcompressed.csv.bz2c                     t          j        |d          5 }|                    |           d d d            d S # 1 swxY w Y   d S )Nr   )bz2BZ2FilerA   r  s       r   r  zTestBZ2CSVRead.write_file}  s    [s## 	qGGH	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   9= =N)r   r   r   r  r  rW   r   r   r  r  z  s(        'L    r   r  c                  ^    t          j        d          } t          |            | j        rJ d S Nr  )r<   rS  r   closed)bufs    r   0test_read_csv_does_not_close_passed_file_handlesr    s1    
*+
,
,CSMMMzr   c            	         t           j                            g dg dgddg          } |                     d          d         }| |fD ]}t	          j                    }t          ||t          d	                     |                    d           | t          |          k    sJ t	          j                    }t          ||t          d
	                     |                    d           t          | j                  }| t          ||          k    sJ d d t          d	          ft          | j                  d t          d
	          fd t          d          t          dd          ft          | j                  t          d          t          d
d          ffD ])\  }}}t	          j                    }t          || j        |          5 }|                    |            d d d            n# 1 swxY w Y   |                    d           | t          |||          k    sJ t	          j                    }t          || j        |          5 }|                     d          D ]}|                    |           	 d d d            n# 1 swxY w Y   |                    d           | t          |||          k    sJ +d S )Nr  r=  c1c2r   )max_chunksizer   T)r   Fr  r   r   r  )r   r   r   write_optionsr  rV   )r6   rG   rH   rf  r<   rS  r   r   seekr   r   rR   r	   r   r   write_tablewrite_batch)	r  record_batchrT  r  r   r  r  writerrC  s	            r   test_write_read_round_tripr    s   
iii9D$<HHA<<a<003LL! = =jll$\>>>???HSMM!!!! jll$\???@@@"???HS|<<<<<<<< 
t\6667	!.	1	1	14	U	+	+	+	-	|c***	TS	9	9	9	;	!.	1	1	1		%	%	%	Ud	;	;	;	=	7 : :2m] jllsAHMBBB 	"fq!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"HS|+8: : : : : : :jllsAHMBBB 	*fA66 * *""5))))*	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	HS|+8: : : : : : : :+: :s$   G<<H 	H 	/JJ	!J	c                     t           j                            g dg dgddg          } t          j                    }t          d          dft                      dft          d	          d
ffD ]x\  }}t          || j        |          5 }|                    |            d d d            n# 1 swxY w Y   |	                                |k    sJ |
                    d           yt           j                            ddggdg          } t          j                    }t          d          dft          d          t           j        j        ffD ]\  }}t          || j        |          5 }	 |                    |            n6# t          $ r)}t          ||          sJ Y d }~d d d             d S d }~ww xY w	 d d d            n# 1 swxY w Y   |	                                |k    sJ |
                    d           d S )N)rV   r'   N)r   Nr   r  r  r   )r   s   "c1","c2"
1,a
2,
,c
s   "c1","c2"
1,"a"
2,
,"c"
r   s   "c1","c2"
"1","a"
"2",
,"c"
r  r   r3   r   r   s   "c1"
","
""""
)r6   rG   rH   r<   rS  r   r   r   r  rE   r  libr   	Exceptionr}   )r  r  r  resr  r3  s         r   test_write_quoting_styler    s   
lll,<,<,<=d|LLA
*,,C	F	+	+	+-HI	89	K	0	0	0	,	. 	 	s sAHMBBB 	"fq!!!	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	" 	"||~~$$$$ 	sDk]TF33A
*,,C	H	-	-	-/CD	F	+	+	+RV-@A  s sAHMBBB 	f""1%%%%    "!S)))))	 	 	 	 	 	 	 	 	 &	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ||~~$$$$ sN   B--B1	4B1	.G	0FG	
F9F4"G	4F99G		G	G	c                      d } t          j                    5   |             } |            J 	 d d d            d S # 1 swxY w Y   d S )Nc                  p    t          j        d          } t          |           }t          j        |          S r  )r<   rS  r   weakrefref)r  r   s     r   innerz,test_read_csv_reference_cycle.<locals>.inner  s-    j/00{5!!!r   )r   disabled_gc)r  wrs     r   test_read_csv_reference_cycler    s    " " "
 
			  UWWrtt||||                 s   <A A type_factoryc                  ,    t          j        dd          S N   rV   r6   r  rW   r   r   r+  r+        BM"a   r   c                  ,    t          j        dd          S )N&   r>  r  rW   r   r   r+  r+        BM"b!! r   c                  ,    t          j        dd          S r  r6   
decimal256rW   r   r   r+  r+    r  r   c                  ,    t          j        dd          S )NL   r(   r  rW   r   r   r+  r+    r  r   c                 `    |            }t          j        dt          j        ddg                              |          i          }t	          || dz             t          | dz            }|                    d                              |          |                    d          k    sJ d S )NcolrV   r'   zout.csv)r6   r   r7   castr   r   column)r  r  r5   r   outs        r   test_write_csv_decimalr    s     <>>DHeRXq!f--224889::EeVi'(((
6I%
&
&C::e!!$''5<<+>+>>>>>>>r   c                      d}  G d dt           j                  }t          d          D ]P}t          j        t
          j                  5  t           ||                      d d d            n# 1 swxY w Y   Qd S )Ns   a,b,cc                   (     e Zd Z fdZ fdZ xZS )-test_read_csv_gil_deadlock.<locals>.MyBytesIOc                 X    t          j        d            t                      j        | S NgMbP?)r  r  rt  readrv   r  rv  s     r   r  z2test_read_csv_gil_deadlock.<locals>.MyBytesIO.read  s&    Ju577<&&r   c                 X    t          j        d            t                      j        | S r  )r  r  rt  readintor  s     r   r  z6test_read_csv_gil_deadlock.<locals>.MyBytesIO.readinto  s'    Ju#577#T**r   )r   r   r   r  r  rx  ry  s   @r   	MyBytesIOr    sQ        	' 	' 	' 	' 	'	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+r   r  r  )r<   rS  r   r^   r_   r6   r   r   )rT  r  r#   s      r   test_read_csv_gil_deadlockr    s    
 D+ + + + +BJ + + + 2YY & &]2?++ 	& 	&YYt__%%%	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	&& &s   
A//A3	6A3	)r'   r(   r)   T)Fr   r  r   r   decimalr   rm  r  r<   r?   r  r:   r  r  r  r   r  r  r  unittestr  r^   pyarrowr6   pyarrow.csvr   r   r   r	   r
   r   r   r   r   r   pyarrow.testsr   r   r$   r&   rO   rS   rh   rp   rr   r   r   r   r   r   ABCr   rH  r.  r5  r:  r{  rE  r~  r  TestCaser  r  r  r  r  r  parametrizer  r  rW   r   r   <module>r     s  $ 


 



 # # # # # # # #       				  				     				                  4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4      ! ! !" " "     $# # #, , ,00 0 0- - - - - - - -$  &4 4 4n9 9 9xN; N; N;b  0XK XK XK XK XK#' XK XK XKv
T T T T T{ T T Tn    -       /   @, @, @, @, @,; @, @, @,F
    !5        #7   9 9 9 9 9 9 9 92
 
 
 
 
/1B 
 
 
*    .0A     ): ): ):X  B	 	 	   !!  !!	*  ? ? ?& & & & &r   