
    z-Ph-                        d dl Z d dlZd dlmZ 	 d dlmZ d dlmZm	Z	m
Z
 n# e$ r dZY nw xY w	 d dlZd dlmZ n# e$ r dxZZY nw xY we j        j        Ze j        j        d             Zd Zd Ze j        j        d             Ze j        j        d             Ze j        j        e j                            d	d ej                    g          d
                         Ze j        j        e j        j        d                         Ze j        j        e j                            d	d ej                    g          d                         Ze j        j        e j        j        d                         Ze j        j        e j        j        d                         Z e j        j        e j        j        d                         Z!e j        j        d             Z"d Z#d Z$dS )    N)fs)_read_table_test_dataframe_range_integersc                    t          d          }d|d<   t          j                            |d          }t          j                    }t          j        ||j        d          }g }t          d          D ]d}||d<   t          j                            |d          }|	                    |           |
                    |                                           e|                                 |                                }t          t          j        |                    }t!          j        |d	
          }	t%          j        |                                |	           d S Nd   r   	unique_idFpreserve_index2.6version
   Tignore_index)r   paTablefrom_pandasBufferOutputStreampqParquetWriterschemarangewrite_tableappendcopyclosegetvaluer   BufferReaderpdconcattmassert_frame_equal	to_pandas
tempdirdfarrow_tableoutwriterframesibufresultexpecteds
             i/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pyarrow/tests/parquet/test_parquet_writer.py#test_parquet_incremental_file_buildr2   ,   s2   			BB{O(&&r%&@@K


!
!Cc;#5uEEEFF2YY ! !;h**2e*DD;'''bggii    
LLNNN
,,..C--..Fyd333H&**,,h77777    c                 X   t          j        dt          j                              t          j        dt          j                              g}t          j        |          }t          j        dg          t          j        dg          g}t           j                            |ddg          }| dz  }t          j	        ||ddd	          5 }t          j        t                    5  |                    |           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )
NPOSdesc   blazsimple_validate_schema.parquetr   snappyspark)r   compressionflavor)r   fielduint32stringr   arrayr   from_arraysr   r   pytestraises
ValueErrorr   )r'   simple_fieldssimple_schemasimple_from_arraysimple_tablepathws          r1    test_validate_schema_write_tablerK   G   s    		$$
%%M
 Im,,M 1#%(9(9:8''(9E6?KKL55D		$"'&.w
@ 
@ 
@ (CD]:&& 	( 	(MM,'''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (s6   D%D;DD	DD	DD#&D#c                    t          j        t                    5  t          j        t          j        dt          j                              g          }t          j        d |           d d d            n# 1 swxY w Y   t          j        t                    5  t          j        | dz  d            d d d            d S # 1 swxY w Y   d S )Nx	some_path)	rB   rC   	TypeErrorr   r   r=   int32r   r   )r'   some_schemas     r1   test_parquet_invalid_writerrR   ]   s,   	y	!	! , ,i#rxzz!:!: ;<<
{+++, , , , , , , , , , , , , , , 
y	!	! 6 6
;.5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6s$   AA55A9<A9B??CCc                    t          d          }d|d<   t          j                            |d          }t          j                    }t          j        ||j        d          5 }g }t          d          D ]d}||d<   t          j                            |d          }|	                    |           |
                    |                                           e	 d d d            n# 1 swxY w Y   |                                }t          t          j        |                    }t          j        |d	
          }	t#          j        |                                |	           d S r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   s
             r1   test_parquet_writer_context_objrT   g   s   			BB{O(&&r%&@@K


!
!C		#{15	A	A	A %Vr 	% 	%AB{O(..r%.HHK{+++MM"''))$$$$	%% % % % % % % % % % % % % % % ,,..C--..Fyd333H&**,,h77777s   %A7C))C-0C-c                 Z   t          d          }d|d<   t          j                            |d          }t          j                    }d}	 t          j        ||j        d          5 }g }t          d	          D ]y}||d<   t          j                            |d          }|	                    |           |
                    |                                           |d
k    rt          |          z	 d d d            n# 1 swxY w Y   n,# t          $ r}t          |          |k    sJ Y d }~nd }~ww xY w|                                }	t!          t          j        |	                    }
t%          j        |d          }t)          j        |
                                |           d S )Nr	   r   r
   Fr   zArtificial Errorr   r   r      Tr   )r   r   r   r   r   r   r   r   r   r   r   r   rD   	Exceptionstrr   r   r    r!   r"   r#   r$   r%   )r'   r(   r)   r*   
error_textr+   r,   r-   er.   r/   r0   s               r1   .test_parquet_writer_context_obj_with_exceptionr[      s   			BB{O(&&r%&@@K


!
!C#J$c)0&+- - - 	106F2YY 1 1"#; h222e2LL"";///bggii(((66$Z000 1	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1  $ $ $1vv#########$ ,,..C--..Fyd333H&**,,h77777s=   D (BD5D DD D	D 
D6D11D6
filesystemc                    t          d          }t          j                            |d          }t          j                            |d          }t          | dz            }t          | dz            }t          j        ||j        |d          5 }|	                    |           d d d            n# 1 swxY w Y   t          |                                          }t          j        ||           t          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |                                          }t          j        ||           t          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |                                          }t          j        ||           t          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |                                          }t          j        ||           d S )Nr	   Fr   zdata_table.parquetzdata_batch.parquetr   r\   r   )r   r   r   r   RecordBatchrX   r   r   r   r   r   r%   r#   r$   write_batchwrite)	r'   r\   r(   tablebatch
path_table
path_batchr+   r/   s	            r1   "test_parquet_writer_write_wrappersrf      sM    
		BH  E ::EN&&r%&@@EW3344JW3344J		ELZ
 
 
 "	5!!!" " " " " " " " " " " " " " "
 $$..00F&"%%%		ELZ
 
 
 "	5!!!" " " " " " " " " " " " " " "
 $$..00F&"%%%		ELZ
 
 
 	U              
 $$..00F&"%%%		ELZ
 
 
 	U              
 $$..00F&"%%%%%sH   B55B9<B9D55D9<D9F55F9<F9H55H9<H9c                     dd fd} |dz  dz
  d            |dz  d            |dz  dz   d            |dz   dz   d            |dz  dz  d            |d d            |dz   d d           d S )Ni   i   c                 d   t           j                            t          | d          gdg          }|t	          j        |dz             nt	          j        |dz  |           t	          j        dz            }|
n|}|j        |k    sJ t          |	          }t          |dz
            D ]"}|
                    |          j        |k    sJ #| ||dz
  z  z
  }|dk    r%|
                    |dz
            j        |k    sJ d S |
                    |dz
            j        |k    sJ d S )NbrM   )namesztest.parquet)row_group_sizer7   r   )r   r   rA   r   r   r   read_metadatanum_row_groupsminr   	row_groupnum_rows)	data_size
chunk_sizeexpect_num_chunksrb   metadataexpected_chunk_sizelatched_chunk_size	chunk_idx	remainderabs_max_chunk_sizedefault_chunk_sizer'   s            r1   check_chunk_sizez8test_parquet_writer_chunk_size.<locals>.check_chunk_size   s   $$Is++&
 %   N5'N":;;;;N5'N"::VVVV#Gn$<==4>4F00J&*;;;;; !46HII01455 	P 	PI%%i009=OOOOOO!48IA8M!NO	>>%%!A%' ''/3EF F F F F F %%&7!&;<<ERRRRRRr3      r	      r7    )r'   r{   ry   rz   s   ` @@r1   test_parquet_writer_chunk_sizer      s    %)S S S S S S S. '!+-?#-EqIII'!+-?CCC'!+-?#-EqIII'#-/AC/GKKK '!+-?!-CQGGG 'q111'!+T155555r3   c                    t          d          }t          j                            |d          }t	          | dz            }t          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |          
                                }t          j        ||           d S )Nr	   Fr   zdata.parquetr   r^   )r   r   r   r   rX   r   r   r   r   r   r%   r#   r$   )r'   r\   r(   rb   rI   r+   r/   s          r1   $test_parquet_writer_filesystem_localr      s     
		BH  E ::Ew'((D		elz5
 
 
 "	5!!!" " " " " " " " " " " " " " "
 ((**F&"%%%%%s    BB	Bc                 r   t          d          }t          j                            |d          }| \  }}}t	          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          |          	                                }t          j        ||           d S )Nr	   Fr   r   r^   r   r   r   r   r   r   r   r   r   r%   r#   r$   s3_example_fsr(   rb   r   urirI   r+   r/   s           r1   !test_parquet_writer_filesystem_s3r     s     
		BH  E ::E!MBT		elr5
 
 
 "	5!!!" " " " " " " " " " " " " " "
 ''))F&"%%%%%s   A66A:=A:c                 t   t          d          }t          j                            |d          }| \  }}}t	          j        ||j        d          5 }|                    |           d d d            n# 1 swxY w Y   t          ||          	                                }t          j        ||           d S )Nr	   Fr   r   r   r\   r   r   s           r1   %test_parquet_writer_filesystem_s3_urir     s     
		BH  E ::E!MBT		#u|U	;	;	; "v5!!!" " " " " " " " " " " " " " " "---7799F&"%%%%%s   A55A9<A9c                 ~   t          d          }t          j                            |d          }| \  }}|dz   }t	          j        ||j        |d          5 }|                    |           d d d            n# 1 swxY w Y   t          ||          	                                }t          j        ||           d S )Nr	   Fr   z/test.parquetr   r^   r   r   )s3_example_s3fsr(   rb   r   	directoryrI   r+   r/   s           r1   #test_parquet_writer_filesystem_s3fsr   +  s    
		BH  E ::E#MB	&D		elr5
 
 
 "	5!!!" " " " " " " " " " " " " " "
 "---7799F&"%%%%%s   A::A>A>c                  N   t          d          } t          j                            | d          }t	          j                    }t          j        t          d          5  t          j
        t          j                    |j        |           d d d            d S # 1 swxY w Y   d S )Nr	   Fr   zspecified path is file-like)matchr   )r   r   r   r   r   LocalFileSystemrB   rC   rD   r   r   r   r   )r(   rb   r\   s      r1   ,test_parquet_writer_filesystem_buffer_raisesr   =  s    			BH  E ::E#%%J 
z)F	G	G	G 
 

!##U\j	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   .BB!Bc                     t          j        dg di          }| dz  }t          j        ||j                  5 }|                    |           d d d            n# 1 swxY w Y   t          j        |          }d|j        v sJ |j        d         sJ | dz  }t          j        ||j        d          5 }|                    |           d d d            n# 1 swxY w Y   t          j        |          }|j        J d S )Na)r7   r|   r}   ztest_with_schema.parquets   ARROW:schemaztest_without_schema.parquetF)store_schema)r   rb   r   r   r   r   rl   rt   )r'   rb   path1r+   metapath2s         r1    test_parquet_writer_store_schemar   J  s   Hc999%&&E 00E		%	.	. "&5!!!" " " " " " " " " " " " " " " E""Ddm++++=)))) 33E		%E	B	B	B "f5!!!" " " " " " " " " " " " " " " E""D=     s#   AA!A4CCCc                    t           j                            t          j        g d          gdg          }| dz  }t	          j        ||j                  5 }|                    |           |                    ddd           |                    dd	d
           d d d            n# 1 swxY w Y   t	          j	        |          }|j
        j
        }|d         dk    sJ |d         dk    sJ |d         dk    sJ d S )NrP   )typef0zmetadata.parquet1rM   )key1key223)r   key3s   key1   1s   key2   2s   key3   3)r   r   rA   r@   r   r   r   r   add_key_value_metadataParquetFilert   )r'   rb   rI   r+   readerrt   s         r1   -test_parquet_writer_append_key_value_metadatar   _  s^   H  "(2G"<"<"<!=vFFE''D		$	-	- B5!!!%%sC&@&@AAA%%sC&@&@AAAB B B B B B B B B B B B B B B ^D!!F'HG$$$$G$$$$G$$$$$$s   AB((B,/B,)%rB   pyarrowr   r   pyarrow.parquetparquetr   pyarrow.tests.parquet.commonr   r   r   ImportErrorpandasr!   pandas.testingtestingr#   mark
pytestmarkr2   rK   rR   rT   r[   parametrizer   rf   large_memoryr   r   s3r   r   r   r   r   r   r~   r3   r1   <module>r      s4  $                 ? ? ? ? ? ? ? ? ? ? ?   	BBB   NB [ 
 8 8 84( ( (,6 6 6 8 8 80 8 8 8> B(  %& %&	  
%&P &6 &6  &6R B(  & &	  
& & &  & 
& 
&  
& & &  &  	
 	
 	
! ! !*% % % % %s   ! ++
: 	AA