
    bMh                        d Z ddlZddlmZ ddlmZ ddlZddlZddlZ	ddl
Z
ddlmZmZ ddlmZ ddlmZ ddlmZmZmZmZmZ ddlZddlmZ dd	lmZ dd
lm Z m!Z!m"Z"m#Z#m$Z$ 	 ddl%Z%dZ&n# e'$ r dZ&Y nw xY w	 ddl(Z(dZ)n# e'$ r dZ)Y nw xY we
j*        +                    d          e
j*        +                    d          gZ, e
j-         e
j.        de
j*        /                    e) p edd          dk    d          e
j*        0                     e            dd          g           e
j.        de
j*        /                    e& d                    g          d             Z1e
j-        d             Z2e
j-        d             Z3e
j-        d             Z4e
j-        d             Z5e
j-        d              Z6 e
j-        ej        7                    ej8        j9                  ej        7                    ej8        j:                  ej        7                    ej8        j;                  ej        <                    d!d"          ej        <                    d#d"          ej        <                    d$d"          ej        <                    d%d"          g          d&             Z=	 	 	 	 	 	 	 	 	 d;d(Z>d) Z?d* Z@d+ ZAd, ZBd- ZCd. ZDd/ ZEd0 ZFd1 ZGd2 ZH G d3 d4          ZI G d5 d6eI          ZJ G d7 d8eI          ZK G d9 d:eI          ZLdS )<z test parquet compat     N)Decimal)BytesIO)using_copy_on_writeusing_string_dtype)_get_option)is_platform_windows)pa_version_under11p0pa_version_under13p0pa_version_under15p0pa_version_under19p0pa_version_under20p0)Version)FastParquetImplPyArrowImpl
get_engineread_parquet
to_parquetTFz2ignore:DataFrame._data is deprecated:FutureWarningz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningfastparquetmode.data_managersilentarrayz4fastparquet is not installed or ArrayManager is usedreasonTODO(infer_string) fastparquetr   strictmarkspyarrowpyarrow is not installed)paramsc                     | j         S Nparamrequests    \/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pandas/tests/io/test_parquet.pyenginer*   @   s    4 =    c                  <    t           st          j        d           dS )Nr!   r    )_HAVE_PYARROWpytestskip r+   r)   par1   ]   s      0.///9r+   c                    t           st          j        d           n)t          dd          dk    rt          j        d           t	                      r4|                     t          j                            dd	                     d
S )Nzfastparquet is not installedr   Tr   r   z.ArrayManager is not supported with fastparquetr   Fr   r   )_HAVE_FASTPARQUETr.   r/   r   r   applymarkermarkxfailr'   s    r)   fpr7   d   s     F23333	(	6	6	6'	A	ADEEE 
K%EeTT	
 	
 	
 =r+   c                  `    t          j        g dddt          j        ddg                    S )N         fooABr?   r@   columns)pd	DataFrameIndexr0   r+   r)   	df_compatrF   q   s1    <iiie44bhSz>R>RSSSSr+   c            
          t          j        t          d          t          t          dd                    t	          j        ddd          g dt          j        d	d
          d          } | S )Nabcr:            @      @float64dtypeTFT20130101r<   periods)abdef)rC   rD   listrangenparange
date_range)dfs    r)   df_cross_compatr^   v   so    	eeAqkk""39555$$$z1555
	
 
	

 
B Ir+   c                     t          j        t          d          dt          j        dgg dg dg dt          t          dd                    t          j        d	d
                              d          t          j        ddd          dt          j        dgg dt          j        dd	          t          j	        d          t           j
        t          j	        d          gd          S )NrH   rS   crS   Nr`   )   foo   bars   bazr=   barbazr:   rI   r<      u1rJ   rK   rL   rM          @      @rO   rP   rQ   20130103)stringstring_with_nanstring_with_nonebytesunicodeintuintfloatfloat_with_nanbooldatetimedatetime_with_nat)rC   rD   rX   rZ   nanrY   r[   astyper\   	TimestampNaTr0   r+   r)   df_fullr|      s    <5kk #RVS1 0 0 0---,,,a$$IaOO**400YsCy999"BFC0'''j!<<<Z((Z(("	
 	
  r+   z2019-01-04T16:41:24+0200z%Y-%m-%dT%H:%M:%S%zz2019-01-04T16:41:24+0215z2019-01-04T16:41:24-0200z2019-01-04T16:41:24-0215c                     | j         S r$   r%   r'   s    r)   timezone_aware_date_listr~      s     =r+   r;   c
                      pddipi  |r
|d<   |d<    fd}
9t          j                    5  |
|	           ddd           dS # 1 swxY w Y   dS  |
|	           dS )a  Verify parquet serializer and deserializer produce the same results.

    Performs a pandas to disk and disk to pandas round trip,
    then compares the 2 resulting DataFrames to verify equality.

    Parameters
    ----------
    df: Dataframe
    engine: str, optional
        'pyarrow' or 'fastparquet'
    path: str, optional
    write_kwargs: dict of str:str, optional
    read_kwargs: dict of str:str, optional
    expected: DataFrame, optional
        Expected deserialization result, otherwise will be equal to `df`
    check_names: list of str, optional
        Closed set of column names to be compared
    check_like: bool, optional
        If True, ignore the order of index & columns.
    repeat: int, optional
        How many times to repeat the test
    compressionNr*   c                     t          |           D ]D} j        fi 
 t          fi 	}dv r
d j        d<   t	          j        |           Ed S )Nrm   r:   rm   )check_names
check_likecheck_dtype)rY   r   r   loctmassert_frame_equal)repeat_actualr   r   r   r]   expectedpathread_kwargswrite_kwargss      r)   comparez!check_round_trip.<locals>.compare   s    v 	 	ABM$//,///!$66+66F H,,5912!'%'    	 	r+   )r   ensure_clean)r]   r*   r   r   r   r   r   r   r   r   r   s   ` ```````  r)   check_round_tripr      s   D  8M4#8L#K '!'X &H            |_ 	$GFOOO	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AA!$A!c                 n    ddl m} |                    | d          }|j        j        j        |k    sJ dS )zCheck partitions of a parquet file are as expected.

    Parameters
    ----------
    path: str
        Path of the dataset.
    expected: iterable of str
        Expected partition names.
    r   Nhive)partitioning)pyarrow.datasetdatasetr   schemanames)r   r   dsr   s       r)   check_partition_namesr      sK     !     jjFj33G&,888888r+   c                     d}t          j        t          |          5  t          | dd           d d d            d S # 1 swxY w Y   d S )Nz.engine must be one of 'pyarrow', 'fastparquet'matchr=   re   )r.   raises
ValueErrorr   )rF   msgs     r)   test_invalid_enginer     s    
:C	z	-	-	- 2 2E51112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s   =AAc                     |r&t           s| j                            d          | _        t          j        dd          5  t          |            d d d            d S # 1 swxY w Y   d S )Nstrio.parquet.enginer    )r   rB   ry   rC   option_contextr   )rF   r1   using_infer_strings      r)   test_options_pyr     s     <"6 <%-44U;;			.		:	: $ $###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $s   AA"Ac                     t          j        dd          5  t          |            d d d            d S # 1 swxY w Y   d S )Nr   r   rC   r   r   )rF   r7   s     r)   test_options_fpr     s     
	.	>	> $ $###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $   377c                     t          j        dd          5  t          |            d d d            d S # 1 swxY w Y   d S )Nr   autor   )rF   r7   r1   s      r)   test_options_autor     s     
	.	7	7 $ $###$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $r   c                 4   t          t          d          t                    sJ t          t          d          t                    sJ t	          j        dd          5  t          t          d          t                    sJ t          t          d          t                    sJ t          t          d          t                    sJ 	 d d d            n# 1 swxY w Y   t	          j        dd          5  t          t          d          t                    sJ t          t          d          t                    sJ t          t          d          t                    sJ 	 d d d            n# 1 swxY w Y   t	          j        dd          5  t          t          d          t                    sJ t          t          d          t                    sJ t          t          d          t                    sJ 	 d d d            d S # 1 swxY w Y   d S )Nr    r   r   r   )
isinstancer   r   r   rC   r   )r7   r1   s     r)   test_options_get_enginer     s   j++[99999j//AAAAA		.		:	: F F*V,,k:::::*Y//=====*]33_EEEEEEF F F F F F F F F F F F F F F
 
	.	>	> F F*V,,o>>>>>*Y//=====*]33_EEEEEEF F F F F F F F F F F F F F F
 
	.	7	7 F F*V,,k:::::*Y//=====*]33_EEEEEEF F F F F F F F F F F F F F F F F Fs9   A-CCC8A-E22E69E6A-HHHc                     ddl m}  |                     d          }|                     d          }t          sdn)t	          t
          j                  t	          |          k     }t          sdn)t	          t          j                  t	          |          k     }t          o| }t          o| }|s&|s%|rId| d}t          j
        t          |          5  t          d	           d d d            n# 1 swxY w Y   nDd
}t          j
        t          |          5  t          d	           d d d            n# 1 swxY w Y   |rKd| d}t          j
        t          |          5  t          d	           d d d            d S # 1 swxY w Y   d S d}t          j
        t          |          5  t          d	           d d d            d S # 1 swxY w Y   d S d S d S )Nr   )VERSIONSr    r   FzPandas requires version .z. or newer of .pyarrow.r   r   z%Missing optional dependency .pyarrow.z. or newer of .fastparquet.z)Missing optional dependency .fastparquet.)pandas.compat._optionalr   getr-   r   r    __version__r3   r   r.   r   ImportErrorr   )r   
pa_min_ver
fp_min_verhave_pa_bad_versionhave_fp_bad_versionhave_usable_pahave_usable_fpr   s           r)   "test_get_engine_auto_error_messager   2  s    100000 i((Jm,,J 	@W())GJ,?,??  !	D[,--
0C0CC  #>+>'>N&B/B+BN #. # 	#S
SSSE{%888 # #6"""# # # # # # # # # # # # # # # <E{%888 # #6"""# # # # # # # # # # # # # # #  	#W
WWWE{%888 # #6"""# # # # # # # # # # # # # # # # # # @E{%888 # #6"""# # # # # # # # # # # # # # # # # ### # # #sH   C00C47C4D55D9<D9#F  FF*GGGc                 @   | }t          j                    5 }|                    ||d            t          ||          }t          j        ||           t          ||ddg          }t          j        ||ddg                    d d d            d S # 1 swxY w Y   d S N)r*   r   r*   rS   rU   )r*   rB   r   r   r   r   r   r^   r1   r7   r]   r   results         r)   test_cross_engine_pa_fpr   ]  s     
B			 6d
d24888d2...
fb)))d2SzBBB
fb#sn5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6   A0BBBc                 @   | }t          j                    5 }|                    ||d            t          ||          }t          j        ||           t          ||ddg          }t          j        ||ddg                    d d d            d S # 1 swxY w Y   d S r   r   r   s         r)   test_cross_engine_fp_par   k  s    	B			 6d
d24888d2...
fb)))d2SzBBB
fb#sn5556 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6r   c                 (   t          j        dg di          }d}t          j                    5 }t          j        t
          |dd          5  |                    ||            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrS   r9   zqStarting with pandas version 3.0 all arguments of to_parquet except for the argument 'path' will be keyword-only.F)r   check_stacklevelraise_on_extra_warnings)rC   rD   r   r   assert_produces_warningFutureWarningr   )r*   r]   r   r   s       r)   !test_parquet_pos_args_deprecationr   x  s$   	sIII&	'	'B	1  
		 (d'"$)	
 
 
 	( 	( MM$'''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (s5   BA/#B/A3	3B6A3	7BBBc                       e Zd Zd Zd ZdS )Basec                     t          j                    5 }t          j        ||          5  t	          |||d            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   r   )r   r   r.   r   r   )selfr]   r*   excerr_msgr   s         r)   check_error_on_writezBase.check_error_on_write  s    _ 	?$s'222 ? ?2tV>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?s3   A#AA#A	A#A	A##A'*A'c                     t          j                    5 }t          j        |          5  t          |||d            d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   )r   r   external_error_raisedr   )r   r]   r*   r   r   s        r)   check_external_error_on_writez"Base.check_external_error_on_write  s    _ 	?$)#.. ? ?2tV>>>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	?s3   A!A	A!	A	A!A	A!!A%(A%N)__name__
__module____qualname__r   r   r0   r+   r)   r   r     s2        ? ? ?? ? ? ? ?r+   r   c                   H   e Zd Zd Zd Zej                            dg d          d             Zd Z	d Z
d Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zej                            dg d          d             Zej        j        ej        j        d                         ZdS )	TestBasicc                     t          j        g d          ddt          j        d          t          j        g d          fD ]!}d}|                     ||t          |           "d S )Nr9   r:   r=   rP   z+to_parquet only supports IO with DataFrames)rC   Seriesrz   rZ   r   r   r   )r   r*   objr   s       r)   
test_errorzTestBasic.test_error  s{    Iiii  L$$HYYY
 	D 	DC @C%%c6:sCCCC	D 	Dr+   c           	          t          j        t          d          t          t          dd                    d          }ddg|_        t          ||           d S )NrH   r:   rI   rl   rq   r=   re   rC   rD   rX   rY   rB   r   )r   r*   r]   s      r)   test_columns_dtypeszTestBasic.test_columns_dtypes  sR    \T%[[eAqkk9J9JKKLL U^
V$$$$$r+   r   )Ngzipsnappybrotlic                 ^    t          j        dg di          }t          ||d|i           d S )Nr?   r9   r   r   rC   rD   r   )r   r*   r   r]   s       r)   test_compressionzTestBasic.test_compression  s:    \3			*++V=+2NOOOOOOr+   c           	          t          j        t          d          t          t          dd                    d          }t          j        dt          d          i          }t	          |||ddgi           d S )NrH   r:   rI   r   rl   rB   r   r   rC   rD   rX   rY   r   )r   r*   r]   r   s       r)   test_read_columnszTestBasic.test_read_columns  sz    \T%[[eAqkk9J9JKKLL<4;; 788	H:7N	
 	
 	
 	
 	
 	
r+   c           
          t          j        t          t          d                    t          d          d          }t          j        dddgi          }t	          ||||ddgid	gdgd
d           d S )NrI   aabb)rq   partrq   r   r:   partition_colsr   )r   ==rS   )filtersrB   )r   r   r   r   r   r   )r   r*   tmp_pathr]   r   s        r)   test_read_filterszTestBasic.test_read_filters  s    \E!HH~~V 
 
 <A00*VH5%8$9ugNN	
 	
 	
 	
 	
 	
r+   c                    |dk    }t          j        dg di          }t          ||           g dt          j        dd          t	          d          g d	g}|D ]T}||_        t          |t           j                  r|j                            d           |_        t          |||
           Ug d|_        d|j        _	        t          ||           d S )Nr   r?   r9   )r;   r<   rI   rP   r<   rQ   rH   )r:   r<   rI   )r   )r   r:   r;   r=   )
rC   rD   r   r\   rX   indexr   DatetimeIndex
_with_freqname)r   r*   r   r]   indexesr   s         r)   test_write_indexzTestBasic.test_write_index  s    -\3			*++V$$$ IIM*a000KKII	
  	B 	BEBH%!122 58..t44R[AAAAA 99V$$$$$r+   c                     |}t          j        dg di          }t           j                            g d          }||_        t          ||           d S )Nr?   r9   )rS   r:   )rS   r;   )rT   r:   )rC   rD   
MultiIndexfrom_tuplesr   r   )r   r1   r*   r]   r   s        r)   test_write_multiindexzTestBasic.test_write_multiindex  sX    \3			*++))*H*H*HIIV$$$$$r+   c           	         |}t          j        ddd          }t          j        t          j                            d                              dt          |          z  df          t          d                    }t           j	        
                    d	d
g|gddg          }|                    d           }||fD ]8}||_        t          ||           t          ||dddgi|ddg                    9d S )Nz01-Jan-2018z01-Dec-2018MS)freqr;   r<   ABCrA   Level1Level2leveldate)r   rB   r?   r@   r   r   )rC   r\   rD   rZ   randomdefault_rngstandard_normallenrX   r  from_productcopyr   r   )r   r1   r*   datesr]   index1index2r   s           r)   test_multiindex_with_columnsz&TestBasic.test_multiindex_with_columns  s   m]FFF\I!!!$$44a#e**na5HIIKK
 
 
 ++!5)'61B , 
 
 4((f% 	 	EBHR(((FS#J(?"cSVZ.    		 	r+   c                    t          j        g dg dd          }d dd}|                    d          }t          ||||           t          j        g dg ddg d	
          }t          ||||           g dg dg}t          j        t	          t          d                    d t          d          D             d|
          }|                    d          }t          ||||           d S )Nr9   )qrs)rS   rT   F)r   r   T)dropr   r   )zyxwvutsrr   re   re   rf   rf   r=   r=   quxr%  onetwor'  r(  r'  r(  r'  r(     c                     g | ]}| S r0   r0   ).0is     r)   
<listcomp>z7TestBasic.test_write_ignoring_index.<locals>.<listcomp>  s    +A+A+A1QB+A+A+Ar+   )r'  r(  )rC   rD   reset_indexr   rX   rY   )r   r*   r]   r   r   arrayss         r)   test_write_ignoring_indexz#TestBasic.test_write_ignoring_index  s=    \			@@AA'+e<< >>t>,,V,RRRR \))///22:O:O:O
 
 
 	V,RRRR EDDDDD
 \qNN+A+Aa+A+A+ABB&
 
 
 >>t>,,V,RRRRRRr+   c                 N   t           j                            g d          }t          j        t          j                            d                              d          |          }|dk    r|                     ||t          d           d S |dk    rt          ||           d S d S )Nr  r;   )rI   r<   rA   r   Column name must be a stringr    )rC   r  r  rD   rZ   r  r  r  r   	TypeErrorr   )r   r*   
mi_columnsr]   s       r)   test_write_column_multiindexz&TestBasic.test_write_column_multiindex  s    ]../M/M/MNN
\I!!!$$44V<<j
 
 
 ]""%%FI'E     y  R((((( ! r+   c                 8   g dg dg}t          j        t          j                            d                              d          |          }ddg|j        _        |dk    r|                     ||t          d	           d S |d
k    rt          ||           d S d S )Nr$  )r:   r;   r:   r;   r:   r;   r:   r;   r;   r)  r)  rA   r  r  r   zColumn namer    )rC   rD   rZ   r  r  r  rB   r   r   r   r   r   r*   r/  r]   s       r)   &test_write_column_multiindex_nonstringz0TestBasic.test_write_column_multiindex_nonstring-  s    
 EDD$$$
 \I!!!$$44V<<f
 
 
 %h/
]""%%b&*mLLLLLy  R((((( ! r+   c                     |}g dg dg}t          j        t          j                            d                              d          |          }ddg|j        _        t          ||           d S )Nr$  r&  r;   r7  rA   	ColLevel1	ColLevel2)	rC   rD   rZ   r  r  r  rB   r   r   r   r1   r*   r/  r]   s        r)   #test_write_column_multiindex_stringz-TestBasic.test_write_column_multiindex_string>  s      EDDDDD
 \I!!!$$44V<<f
 
 
 (5
V$$$$$r+   c                     |}g d}t          j        t          j                            d                              d          |          }d|j        _        t          ||           d S )N)re   rf   r=   r%  r;   r)  rI   rA   	StringCol)	rC   rD   rZ   r  r  r  rB   r   r   r=  s        r)   test_write_column_index_stringz(TestBasic.test_write_column_index_stringO  sp      .--\I!!!$$44V<<f
 
 
 &
V$$$$$r+   c                    g d}t          j        t          j                            d                              d          |          }d|j        _        |dk    r|                     ||t          d           d S t          ||           d S )Nr:   r;   r<   rI   r;   r@  rA   NonStringColr   r2  )rC   rD   rZ   r  r  r  rB   r   r   r3  r   r8  s       r)   !test_write_column_index_nonstringz+TestBasic.test_write_column_index_nonstring]  s     \I!!!$$44V<<f
 
 
 )
]""%%FI'E     R(((((r+   c                    t          j        d          }|dk    r5t           j                            d          }|                    |           t          j        t          j        g dd          t          j        g dd          t          j        g d          t          j        g d	          t          j        g d
d          t          j        g dd          t          j        g dd          d          }t          j	                    5 }|
                    ||           t          ||          }t          ||d          }d d d            n# 1 swxY w Y   |d         j        t          j        d          k    sJ t          j        t          j        g dd          t          j        g dd          t          j        g dd          t          j        g d	d          t          j        g d
d          t          j        g dd          t          j        g dd          d          }	|dk    r.|                    dd          }|	                    dd          }	t          j        ||	           d S )Nzpyarrow.parquetr   z.Fastparquet nullable dtype support is disabledr   r:   r;   r<   Nint64uint8)rS   rT   r`   N)TFTNrD  )      ?ri   rj   Nfloat32rL   )rS   rT   r`   rU   rV   rW   gr   numpy_nullabler*   dtype_backendrS   Int64rM   UInt8rl   booleanFloat32Float64r`   r:   )axis)r.   importorskipr5   r6   r4   r    tabler   r   r   write_tabler   rN   rZ   rC   rD   r  r   )
r   r*   r(   pqr5   rX  r   result1result2r   s
             r)   test_dtype_backendzTestBasic.test_dtype_backendm  s    !233]"" ;$$G %  D %%%]???G<<]???G<<]#8#8#899]#<#<#<==]<<<99]#8#8#8)DD]#8#8#8)DD
 

 
 _ 	X$NN5$'''"4777G"4FVWWWG		X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X 	X s|!RXi%8%88888<XoooW===XoooW===X3338DDDX777yIIIXlll':::X3339EEEX3339EEE 

 

 ]"" ll3Ql//G}}Sq}11H
gx00000s   :EEErN   )	rQ  rR  rS  objectzdatetime64[ns, UTC]rs   z	period[D]rU  rl   c                     t          j        dt          j        g |          i          }d }|dk    r*t          j        dt          j        g d          i          }t          ||ddi|           d S )NvaluerM   rs   rU  rP  rN  r  )rC   rD   r   r   )r   r1   rN   r]   r   s        r)   test_read_empty_arrayzTestBasic.test_read_empty_array  s      \"E222
 
 G|RXb	::: H
 	2B Ch	
 	
 	
 	
 	
 	
r+   c                 J   |dk    rt          j        |           t           |dddd          d          5 }|                    |                                           t          |j        |	          }d d d            n# 1 swxY w Y   t          j        ||           d S )
Nr   iodataparquetzsimple.parquetrb)mode)contentr   )	r.   rW  openserve_contentreadr   urlr   r   )r   
httpserverdatapathrF   r*   rW   r]   s          r)   test_parquet_read_from_urlz$TestBasic.test_parquet_read_from_url  s     V'''((44DEEDQQQ 	=UV$$QVVXX$666jnV<<<B	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	= 	b),,,,,s   ?BB
BN)r   r   r   r   r   r.   r5   parametrizer   r   r   r  r  r  r0  r5  r9  r>  rB  rF  r]  ra  network
single_cpuro  r0   r+   r)   r   r     s       	D 	D 	D% % % [],N,N,NOOP P POP
 
 

 
 
&% % %0% % %  &S S S@) ) )) ) )"% % %"% % %) ) ) /1 /1 /1b [
	
 
	
 
	
 
 
 
& [[- -  - - -r+   r   c                      e Zd Zd Zd Zd Zd Zd Zd Zd Z	e
j                             e            d	          e
j                            e d
	          e
j                            deej        g          d                                     Zd Ze
j        j        d             Ze
j        j        d             Ze
j        j        e
j                            ddgg g          d                         Zd Zd Zd Zd Ze
j                            ded gddg          d             Zd Zd Z d Z!d Z"d  Z#d! Z$d" Z%d# Z&d$ Z'd% Z(d& Z)e
j                            d'd(d)g e
j*        d*d+ge
j                            e+e,d,-          .           e-j-        d/d)d)d(d(           e-j-        d/d)d)d)d)          gg          d0             Z.d1 Z/d2 Z0d3 Z1e
j                            e2d4	          d5             Z3d6 Z4d7S )8TestParquetPyArrowc                     |}t          j        ddd          }|                    d           }||d<   g d|d<   t          ||           d S )NrP   r<   Europe/BrusselsrR   tzdatetime_tzTNTbool_with_none)rC   r\   r   r   )r   r1   r|   r]   dtis        r)   
test_basiczTestParquetPyArrow.test_basic  sb     mJ6GHHHnnT""=111R     r+   c                 |    |}t          j        ddd          |d<   t          |||ddg         dddgi	           d S )
NrP   r<   rv  rw  ry  rl   rq   rB   r   )rC   r\   r   )r   r1   r|   r]   s       r)   test_basic_subset_columnsz,TestParquetPyArrow.test_basic_subset_columns  sf     M*aDUVVV=5)*"Xu$56		
 	
 	
 	
 	
 	
r+   c                    |                     |          }t          |t                    sJ t          |          }t	          |          }|                                }d |j        d<   t          j        ||           d S )Nr   r   )	r   r   ro   r   r   r  r   r   r   )r   r1   r|   	buf_bytes
buf_streamresr   s          r)   *test_to_bytes_without_path_or_buf_providedz=TestParquetPyArrow.test_to_bytes_without_path_or_buf_provided  s{    &&b&11	)U+++++Y''
:&&<<>>-1)*
c8,,,,,r+   c                     t          j        t          j        d                              dd          t          d                                                    }|                     ||t          d           d S )N   rI   r<   aaarA   zDuplicate column names found	rC   rD   rZ   r[   reshaperX   r  r   r   r   r1   r]   s      r)   test_duplicate_columnsz)TestParquetPyArrow.test_duplicate_columns  sa    \")B--//155tE{{KKKPPRR!!"b*6TUUUUUr+   c                 z    t          j        dt          j        dd          i          }t          ||           d S )NrS   1 dayr<   rQ   )rC   rD   timedelta_ranger   r  s      r)   test_timedeltaz!TestParquetPyArrow.test_timedelta  s<    \3 27A F F FGHHR     r+   c                 x    t          j        dg di          }|                     ||t          j                   d S )NrS   rS   r:   ri   )rC   rD   r   r    ArrowExceptionr  s      r)   test_unsupportedz#TestParquetPyArrow.test_unsupported  s=    \3.// 	**2r73IJJJJJr+   c                     t          j        ddt           j                  }t          j        |dg          }t
          r#|                     ||t          j                   d S t          ||           d S )Nr;   
   rM   fp16rd  rB   )
rZ   r[   float16rC   rD   r   r   r    r  r   )r   r1   rd  r]   s       r)   test_unsupported_float16z+TestParquetPyArrow.test_unsupported_float16   sq     yBbj111\tfX666 	%..r2w7MNNNNNR$$$$$r+   zqPyArrow does not cleanup of partial files dumps when unsupported dtypes are passed to_parquet function in windowsr   zfloat16 works on 15	path_typec                    t          j        ddt           j                  }t          j        |dg          }t          j                    5 } ||          }t          j        t          j	                  5  |
                    ||           d d d            n# 1 swxY w Y   t          j                            |          rJ 	 d d d            d S # 1 swxY w Y   d S )Nr;   r  rM   r  r  )r   r*   )rZ   r[   r  rC   rD   r   r   r   r    r  r   osr   isfile)r   r1   r  rd  r]   path_strr   s          r)    test_unsupported_float16_cleanupz3TestParquetPyArrow.test_unsupported_float16_cleanup
  sL    yBbj111\tfX666_ 	,(9X&&D)'*@AA 4 443334 4 4 4 4 4 4 4 4 4 4 4 4 4 4w~~d++++++		, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,s6   *C6BCB	C!B	"$CCCc           
         t          j        t          j        t          d                    t          j        g dt          j        g d                    t          j        g dg dd          d	          }t          ||           d S )
Nabcdef)re   r=   r=   re   Nre   rd   rM   )rS   rT   r`   rS   r`   rT   )rT   r`   rU   T)
categoriesorderedrS   rT   r`   )rC   rD   CategoricalrX   CategoricalDtyper   r  s      r)   test_categoricalz#TestParquetPyArrow.test_categorical   s    \^DNN33^===-.C.C.CDD  
 ^222.    
 
" 	R     r+   c                     t          j        d          } |j        di |}d|i}t          |||j         d||           d S )Ns3fs
filesystem/pyarrow.parquetr   r   r   r0   )r.   rW  S3FileSystemr   r   )r   rF   s3_public_bucketr1   s3sor  s3kws           r)   test_s3_roundtrip_explicit_fsz0TestParquetPyArrow.test_s3_roundtrip_explicit_fs5  sp    "6**T&&&&B$);;;	
 	
 	
 	
 	
 	
r+   c                 H    d|i}t          ||d|j         d||           d S )Nstorage_optionss3://r  r  r   r   )r   rF   r  r1   r  s        r)   test_s3_roundtripz$TestParquetPyArrow.test_s3_roundtripB  sM     "4(@).@@@	
 	
 	
 	
 	
 	
r+   partition_colr?   c                 V   t          j        d           |                                }|rX|                    t                              |t          j                            }d}||                             |          ||<   t          |||d|j	         dd|i|d |ddd	           d S )
Nr  categoryr  z/parquet_dirr  )r   r   r  Tr:   )r   r   r   r   r   r   )
r.   rW  r  ry   dictfromkeysrZ   int32r   r   )r   rF   r  r1   r  r  expected_dfpartition_col_types           r)   test_s3_roundtrip_for_dirz,TestParquetPyArrow.test_s3_roundtrip_for_dirN  s     	F###nn&&  	%,,T]]="(-S-STTK!+)4])C)J)J"* *K& 	 <).<<<*D1"/##' 
 	
 	
 	
 	
 	
 	
r+   c                    t          j        d           t                      }|                    |           t	          |          }|r&t
          s|j                            d          |_        t          j	        ||           d S )Nr    r   )
r.   rW  r   r   r   r   rB   ry   r   r   )r   rF   r   bufferdf_from_bufs        r)   test_read_file_like_obj_supportz2TestParquetPyArrow.test_read_file_like_obj_supportu  s    I&&&V$$$"6** 	@&: 	@ ) 1 8 8 ? ?I
i55555r+   c                    t          j        d           |                    dd           |                    dd           t          j        t          d          5  t          d           d d d            n# 1 swxY w Y   t          j        t          d          5  |                    d           d d d            d S # 1 swxY w Y   d S )Nr    HOMETestingUserUSERPROFILEz.*TestingUser.*r   z~/file.parquet)r.   rW  setenvr   OSErrorr   r   )r   rF   monkeypatchs      r)   test_expand_userz#TestParquetPyArrow.test_expand_user~  sH   I&&&6=111=-888]7*<=== 	+ 	+)***	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+]7*<=== 	3 	3  !1222	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3s$   A88A<?A<CCCc                     ddg}|}|                     ||d            t          ||           t          |          j        |j        k    sJ d S )Nru   rq   r   r   r   r   r   shape)r   r   r1   r|   r   r]   s         r)   test_partition_cols_supportedz0TestParquetPyArrow.test_partition_cols_supported  s\     %
h~4PPPh777H%%+rx777777r+   c                     d}|g}|}|                     ||d            t          ||           t          |          j        |j        k    sJ d S )Nru   r  r  )r   r   r1   r|   r   partition_cols_listr]   s          r)   test_partition_cols_stringz-TestParquetPyArrow.test_partition_cols_string  sb    -.
h~4PPPh(;<<<H%%+rx777777r+   c                     | S r$   r0   )xs    r)   <lambda>zTestParquetPyArrow.<lambda>  s    Q r+   rl   zpathlib.Path)idsc                     d}|g}|} ||          }|                     ||           t          |          j        |j        k    sJ d S )Nr@   )r   )r   r   r  )	r   r   r1   rF   r  r   r  r]   r   s	            r)   test_partition_cols_pathlibz.TestParquetPyArrow.test_partition_cols_pathlib  sa     -.y""
d+>???D!!'28333333r+   c                 R    t          j        g g           }t          ||           d S )N)r   rB   r   r  s      r)   test_empty_dataframez'TestParquetPyArrow.test_empty_dataframe  s,    \B///R     r+   c                     dd l }t          j        dddgi          } |j         |j        d |j                              g          }|                    t                    }t          ||d|i|           d S )Nr   r  r:   )typer   r  )	r    rC   rD   r   fieldbool_ry   ru   r   )r   r1   r    r]   r   out_dfs         r)   test_write_with_schemaz)TestParquetPyArrow.test_write_with_schema  s    \3A-((s!I!I!I JKK4Rx.@6RRRRRRr+   c                    t          j        d           t          j        t          j        g dd          t          j        g dd          t          j        g dd          d          }|r/t
          r(t          |||                    d	d
i                     nt          ||           t          j        dt          j        g dd          i          }t          ||           d S )Nr    r9   rQ  rM   UInt32ra   rl   r  r`   r   r   rS   rH  )r.   rW  rC   rD   r   r   r   ry   )r   r1   r   r]   s       r)    test_additional_extension_arraysz3TestParquetPyArrow.test_additional_extension_arrays  s     	I&&&\Yyyy888Yyyy999Y///x@@@ 
 
  	%"6 	%Rbiie.E.EFFFFFR$$$\3	/// I I IJKKR     r+   c                    t          j        d           t          j        dt          j        g dd          i          }t          j        d|          5  |rVt          r|                    d          }n|                    d| d	          }|j                            d          |_        n|                    d| d	          }t          |||
           d d d            d S # 1 swxY w Y   d S )Nr    rS   ra   string[pyarrow]rM   string_storager   zstring[]r  )
r.   rW  rC   rD   r   r   r   ry   rB   r   )r   r1   r  r   r]   r   s         r)    test_pyarrow_backed_string_arrayz3TestParquetPyArrow.test_pyarrow_backed_string_array  sF   I&&&\3	*:*:*:BS T T TUVV/@@ 		8 		8! B' F!yy//HH!yy)D>)D)D)DEEH#+#3#:#:5#A#A  99%@~%@%@%@AARh7777		8 		8 		8 		8 		8 		8 		8 		8 		8 		8 		8 		8 		8 		8 		8 		8 		8 		8s   BC''C+.C+c                 J   t          j        d           t          j        t          j                            g d          t          j        ddd          t          j                            t          j        ddd                    d          }t          ||           d S )	Nr    ))r   r:   )r:   r;   )r<   rI   z
2012-01-01r<   D)rR   r	  rI   )r`   rU   rV   )
r.   rW  rC   rD   IntervalIndexr  period_rangefrom_breaksr\   r   r  s      r)   test_additional_extension_typesz2TestParquetPyArrow.test_additional_extension_types  s     	I&&&\%112J2J2JKK_\13GGG%11M,DDD 	 	
 	
 	R     r+   c                     d}t          j        dt          j        ddd          i          }t          ||d|i           d S )	Nz2.6rS   z
2017-01-011nsr  r	  rR   versionr   )rC   rD   r\   r   )r   r1   verr]   s       r)   test_timestamp_nanosecondsz-TestParquetPyArrow.test_timestamp_nanoseconds  sO     \3lPR S S STUURy#.>??????r+   c                 h   t          j        dd           d|gz  }t          j        |d|i          }|d d          }t          r|j                            d          |_        |j        t          j	        j
        k    r	 dd l}|j        j                            |          }|                    |                                dz            }|j                            |          |_        |d         j                            |          |d<   n# t$          $ r Y nw xY wt'          ||d	|
           d S )Nr    11.0.0   index_as_colr   rd  nsr   <   F)r   r   )r.   rW  rC   rD   r	   r   as_unittzinforv   timezoneutcpytzrx  	utcoffsetFixedOffsettotal_seconds
tz_convertdtr   r   )	r   r1   r~   idxr]   r   r  offsetrx  s	            r)   test_timezone_aware_indexz,TestParquetPyArrow.test_timezone_aware_index  sG   Ix000+,,\>3*?@@@ aaa5 	:%^33D99HN#*h.?.CCCV ../GHH%%f&:&:&<&<r&ABB!)!:!:2!>!>+3N+C+F+Q+QRT+U+U((     	RUXFFFFFFs   ?D 
DDc                 b   t          j        d           t          j        dt	          t          d                    i          }t          j                    5 }|                    ||           t          ||dg          }d d d            n# 1 swxY w Y   t          |          dk    sJ d S )Nr    rS   r<   r   rS   r   r   r   r:   )r.   rW  rC   rD   rX   rY   r   r   r   r   r  )r   r1   r]   r   r   s        r)   test_filter_row_groupsz)TestParquetPyArrow.test_filter_row_groups  s    I&&&\3U1XX/00_ 	F$MM$rM***!$^4DEEEF	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 6{{as   +BBBc                    t          j        t          j                            d                              d          g d          }t          j                    5 }|                    ||           t          ||          }d d d            n# 1 swxY w Y   |r-t          |j        t           j        j        j                  sJ d S t          |j        t           j        j        j                  sJ d S )Nr;   )r  r<   )r?   r@   CrA   r   )rC   rD   rZ   r  r  r  r   r   r   r   r   _mgrcore	internalsArrayManagerBlockManager)r   r1   using_array_managerr]   r   r   s         r)   test_read_parquet_managerz,TestParquetPyArrow.test_read_parquet_manager  s   \I!!!$$44W==
 
 
 _ 	,$MM$rM***!$++F	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,  	Kfk27+<+IJJJJJJJfk27+<+IJJJJJJJs   (BBBc                 &   dd l }|}t          j        ddd          }|                    d           }||d<   g d|d<   |j                            |          }|                    t          j        	          }t          r||d
         	                    d          |d
<   |d         	                    d          |d<   |d         	                    t          j         |j
        dd                              |d<   t          ||ddi|           d S )Nr   rP   r<   rv  rw  ry  rz  r{  )types_mapperrv   ztimestamp[us][pyarrow]rw   us)unitrx  rP  r    r*   r   r   )r    rC   r\   r   Tablefrom_pandas	to_pandas
ArrowDtyper
   ry   	timestampr   )r   r1   r|   r    r]   r|  pa_tabler   s           r)   &test_read_dtype_backend_pyarrow_configz9TestParquetPyArrow.test_read_dtype_backend_pyarrow_config%  sB    mJ6GHHHnnT""=111=,,R00%%2=%AA 	#+J#7#>#>?W#X#XHZ ,45H,I,P,P(- -H() '/}&=&D&D/g/T>OPPPQQ' 'H]# 	()4		
 	
 	
 	
 	
 	
r+   c                 J   t          j        dddgit          j        ddgd          d	          }|                                }d
d l}t          |j                  t          d          k    r|j                            d          |_        t          ||ddi|           d S )NrS   r:   r;   r<   rI   testr   zint64[pyarrow])r   rN   r   r  rP  r    r  )
rC   rD   rE   r  r    r   r   r   ry   r   )r   r1   r]   r   r    s        r)   ,test_read_dtype_backend_pyarrow_config_indexz?TestParquetPyArrow.test_read_dtype_backend_pyarrow_config_indexC  s    \1a&M1a&v!>!>!>FV
 
 
 77997&'''(*;*;;;%^223CDDHN()4		
 	
 	
 	
 	
 	
r+   rB   r   r:   rb   rc   z*https://github.com/apache/arrow/pull/44171)r   r   r     c           	          t          j        t          d          t          t          dd                    d          }||_        t          ||           d S )NrH   r:   rI   r   r   )r   r1   rB   r]   s       r)   test_columns_dtypes_not_invalidz2TestParquetPyArrow.test_columns_dtypes_not_invalidS  sN    & \T%[[eAqkk9J9JKKLL
R     r+   c                 |    t          j        t          j        g dd                    }t          ||           d S )Nr  custom namer'  r#  rC   rD   rE   r   r  s      r)   test_empty_columnsz%TestParquetPyArrow.test_empty_columnsk  s=    \} M M MNNNR     r+   c                     |dz  }t          j        ddgi          }ddi|_        |                    ||           t	          ||          }|j        |j        k    sJ d S )Nztest_df_metadata.pr:   rd  test_attributer   )rC   rD   attrsr   r   )r   r   r1   r   r]   new_dfs         r)   test_df_attrs_persistencez,TestParquetPyArrow.test_df_attrs_persistencep  st    ..\A3x((($a(
d2&&&d2...|rx''''''r+   c           	         |dz  }t          j        dddgiddg          }|                    ||           t          j        dd	          5  t	          ||          }d d d            n# 1 swxY w Y   t          j        t          j        
          }t          j        dddgi|t          j        ddg|          t          j        dgt          r	|st          n|                    }t          j        ||           d S )Nztest_string_inference.prS   r  yrT   )rd  r   r   future.infer_stringTna_valuerM   )rd  rN   r   rB   )rC   rD   r   r   r   StringDtyperZ   rx   rE   r   r^  r   r   )	r   r   r1   r   r   r]   r   rN   r   s	            r)   test_string_inferencez(TestParquetPyArrow.test_string_inferencex  sY   33\c3Z0c
CCC
d2&&&4d;; 	3 	3!$r222F	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3///<Sz"(C:U333H'0Bff	  	

 

 

 	fh/////s   A,,A03A0znot supported before 11.0c           	         dd l }|dz  }t          j        dt          d          gid          }|                    ||                    d|                    d          fg                     t          |          }t          rt          j        dd	gid
          }n&t          j        dt          d          gid          }t          j
        ||           d S )Nr   z	decimal.prS   z123.00r  rM   r  )r   123zstring[python]r^  )r    rC   rD   r   r   r   
decimal128r   r   r   r   )r   r   r1   r   r]   r   r   s          r)   test_roundtrip_decimalz)TestParquetPyArrow.test_roundtrip_decimal  s     	+%\3!2!2 34<MNNN
d299sBMM!4D4D.E-F#G#GHHHd## 	P|S5'N:JKKKHH|S78+<+<*=$>hOOOH
fh/////r+   c           
      D   dd l }dd lm} |dz  }|                    d|                    g d|                                          i          }|                    ||           t          j        dd          5  t          |          }d d d            n# 1 swxY w Y   t          j
        dg dit          j        t          j                  t          j        dgt          j        t          j                            	          }t          j        ||           d S )
Nr   zlarge_string.prS   )NrT   r`   r8  Tr9  rM   )rd  rN   rB   )r    pyarrow.parquetre  rX  r   large_stringrY  rC   r   r   rD   r;  rZ   rx   rE   r   r   )r   r   r1   rZ  r   rX  r   r   s           r)   #test_infer_string_large_string_typez6TestParquetPyArrow.test_infer_string_large_string_type  s_   $$$$$$**#rxx(8(8(8"//:K:KLLMNN
ud###4d;; 	( 	(!$''F	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(<'''(."&111HcU"."&*I*I*IJJJ
 
 

 	fh/////s   :BBBN)5r   r   r   r}  r  r  r  r  r  r  r.   r5   r6   r   skipifr   rp  r   pathlibPathr  r  rr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r$  r(  r&   r   NotImplementedErrorrv   r+  r/  r5  r<  r	   r@  rD  r0   r+   r)   rt  rt    s       	! 	! 	!
 
 

- 
- 
-V V V
! ! !K K K% % % [?    [009NOO[[3*=>>, , ?> PO ,! ! !* [

 

 

 [	
 	
 	
 [[E	
 
 
  
>6 6 63 3 38 8 88 8 8 [c;;'h-G   	4 	4 	4! ! !
S S S! ! !&8 8 8! ! ! @ @ @G G GB     K K K
 
 
<
 
 
  [FFL k''(.G (     "!$1a33!!$1a33	
 $! !% $!! ! !
( ( (0 0 0* [,5PQQ0 0 RQ00 0 0 0 0r+   rt  c                   ^   e Zd Zd Zd Zd Zd Zd Zd Zd Z	e
j        j        d             Zd	 Zd
 Zd Zd Ze
j                             e            d          d             Zd Zd Zd Zd Zd Zd Zd Zd Ze
j                             e            d          d             ZdS )TestParquetFastParquetc                     |}t          j        ddd          }|                    d           }||d<   t          j        dd          |d<   t	          ||           d S )	NrP   r<   z
US/Easternrw  ry  r  rQ   	timedelta)rC   r\   r   r  r   )r   r7   r|   r]   r|  s        r)   r}  z!TestParquetFastParquet.test_basic  sg    mJlCCCnnT""=,Wa@@@;R     r+   c           	         t          j        t          d          t          t          dd                    d          }t          }d}ddg|_        |                     ||||           ddg|_        |                     ||||           t          j        d	dddd          t          j        d	dddd          g|_        |                     ||||           d S )
NrH   r:   rI   r   r2  r   rb   rc   r)  )rC   rD   rX   rY   r3  rB   r   rv   )r   r7   r]   errr   s        r)   test_columns_dtypes_invalidz2TestParquetFastParquet.test_columns_dtypes_invalid  s    \T%[[eAqkk9J9JKKLL, V
!!"b#s333 f%
!!"b#s333 dAq!Q//dAq!Q//

 	!!"b#s33333r+   c                     t          j        t          j        d                              dd          t          d                                                    }d}|                     ||t          |           d S )Nr  rI   r<   r  rA   z9Cannot create parquet dataset with duplicate column namesr  r   r7   r]   r   s       r)   r  z-TestParquetFastParquet.test_duplicate_columns  se    \")B--//155tE{{KKKPPRRI!!"b*c:::::r+   c                    dd l }t          |j                  t          d          k     r]t          t          j                  t          d          k    r3|                    t
          j                            d                     t          j	        dg di          }t          j	        ddt          j
        d	gid
          }t          |||d           d S )Nr   	2024.11.0z2.0.0z$fastparquet uses np.float_ in numpy2r   rS   )TNFrK  g        r  rM   F)r   r   )r   r   r   rZ   r4   r.   r5   r6   rC   rD   rx   r   )r   r7   r(   r   r]   r   s         r)   test_bool_with_nonez*TestParquetFastParquet.test_bool_with_none  s    ;*++gk.B.BBBwNH
 H
WH H !!B "    
 \3 3 3 3455<sBFC&8 9KKK 	R(FFFFFFr+   c                    t          j        dt          j        ddd          i          }|                     ||t          d            t          j        dg di          }d}|                     ||t          |           d S )NrS   2013Mr<   r  r  z"Can't infer object conversion type)rC   rD   r  r   r   rQ  s       r)   r  z'TestParquetFastParquet.test_unsupported  s    \3S! L L LMNN!!"b*d;;; \3.//2!!"b*c:::::r+   c                     t          j        dt          j        t          d                    i          }t	          ||           d S )NrS   rH   )rC   rD   r  rX   r   )r   r7   r]   s      r)   r  z'TestParquetFastParquet.test_categorical  s<    \3tE{{ ; ;<==R     r+   c                 B   dt          t          d                    i}t          j        |          }t	          j                    5 }|                    ||d d           t          ||dg          }d d d            n# 1 swxY w Y   t          |          dk    sJ d S )NrS   r<   r:   )r*   r   row_group_offsetsr  r  )	rX   rY   rC   rD   r   r   r   r   r  )r   r7   rU   r]   r   r   s         r)   r  z-TestParquetFastParquet.test_filter_row_groups  s    $uQxx..!\!___ 	F$MM$rtqMQQQ!$^4DEEEF	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 	F 6{{as   -A??BBc           	      J    t          ||d|j         dd|id |d           d S )Nr  z/fastparquet.parquetr  )r   r  r  r  )r   rF   r  r7   r  s        r)   r  z(TestParquetFastParquet.test_s3_roundtrip	  sP     	D).DDD*D1)-$GG	
 	
 	
 	
 	
 	
r+   c                     ddg}|}|                     |d|d            t          j                            |          sJ dd l} |j        t          |          d          j        }t          |          dk    sJ d S )Nru   rq   r   r*   r   r   r   Fr;   	r   r  r   existsr   ParquetFiler   catsr  r   r   r7   r|   r   r]   r   actual_partition_colss           r)   r  z4TestParquetFastParquet.test_partition_cols_supported  s     %
 )	 	 	
 	
 	
 w~~h''''' 7 7Hu M M R())Q......r+   c                     d}|}|                     |d|d            t          j                            |          sJ dd l} |j        t          |          d          j        }t          |          dk    sJ d S )Nru   r   r]  r   Fr:   r^  rb  s           r)   r  z1TestParquetFastParquet.test_partition_cols_string$  s    
 )	 	 	
 	
 	
 w~~h''''' 7 7Hu M M R())Q......r+   c                     ddg}|}|                     |dd |           t          j                            |          sJ dd l} |j        t          |          d          j        }t          |          dk    sJ d S )Nru   rq   r   )r*   r   partition_onr   Fr;   r^  rb  s           r)   test_partition_on_supportedz2TestParquetFastParquet.test_partition_on_supported4  s     %
 '	 	 	
 	
 	
 w~~h''''' 7 7Hu M M R())Q......r+   c                     ddg}|}d}t          j        t          |          5  |                    |dd ||           d d d            d S # 1 swxY w Y   d S )Nru   rq   zYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datar   r   )r*   r   rf  r   )r.   r   r   r   )r   r   r7   r|   r   r]   r   s          r)   3test_error_on_using_partition_cols_and_partition_onzJTestParquetFastParquet.test_error_on_using_partition_cols_and_partition_onD  s     !%  	 ]:S111 	 	MM$ +-    	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   AAAzfastparquet writes into Indexr   c                 x    t          j                    }|                                }t          |||           d S )Nr  )rC   rD   r  r   r   r7   r]   r   s       r)   r  z+TestParquetFastParquet.test_empty_dataframeW  s7     \^^7799R(333333r+   c                    dd l }t          |j                  t          d          k    rXt          |j                  t          d          k     r3|                    t          j                            d                     d|gz  }t          j        |d|i          }|	                                }d	|j
        _        t          |||
           d S )Nr   z2022.12rS  zCfastparquet bug, see https://github.com/dask/fastparquet/issues/929r   r  r  r  r   r  )r   r   r   r4   r.   r5   r6   rC   rD   r  r   r   r   )r   r7   r~   r(   r   r	  r]   r   s           r)   r  z0TestParquetFastParquet.test_timezone_aware_index^  s    ;*++gi.@.@@@W#F
 F
K  F! F! !!I "     +,,\>3*?@@@7799%R(333333r+   c                 .   t          j        dddgi          }t          j                    5 }|                    |           t          j        t          d          5  t          j        t                    5  t          |dd           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t          j        t          d          5  t          |dd	
           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NrS   r:   r;   z!not supported for the fastparquetr   r   T)r*   use_nullable_dtypesr    rO  )rC   rD   r   r   r   r.   r   r   r   r   r   )r   r7   r]   r   s       r)   &test_use_nullable_dtypes_not_supportedz=TestParquetFastParquet.test_use_nullable_dtypes_not_supportedu  sI   \3A-((_ 	R$MM$z1TUUU W W/>> W W mQUVVVVW W W W W W W W W W W W W W WW W W W W W W W W W W W W W W z1TUUU R RT-yQQQQR R R R R R R R R R R R R R R	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	R 	Rs}   1D
B-7B
B-BB-BB-!D
-B1	1D
4B1	5D
C2&D
2C6	6D
9C6	:D

DDc                    t          j        d          5 }t          j        |                              d           t          j        t          d          5  t          |d           d d d            n# 1 swxY w Y   t          j        |          	                    d           d d d            d S # 1 swxY w Y   d S )	Ntest.parquets   breakit r   r   r   F)
missing_ok)
r   r   rF  rG  write_bytesr.   r   	Exceptionr   unlink)r   r   s     r)   $test_close_file_handle_on_read_errorz;TestParquetFastParquet.test_close_file_handle_on_read_error  s1   _^,, 	8L**:666y333 9 9T-88889 9 9 9 9 9 9 9 9 9 9 9 9 9 9 L%%%777	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8s6   AB6A6*B66A:	:B6=A:	>+B66B:=B:c                    t          j        ddgddgd          }t          j        d          5 }t	          |                                d          5 }|                    |           d d d            n# 1 swxY w Y   t          ||          }d d d            n# 1 swxY w Y   t          j        ||           d S )Nr   r:   r>   r1  rq  wbr   )	rC   rD   r   r   ri  encoder   r   r   )r   r*   r]   r   rW   r   s         r)   test_bytes_file_namez+TestParquetFastParquet.test_bytes_file_name  s1   \aV1a&99:::_^,, 	7dkkmmT** !aa   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! "$v666F		7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7 	7
 	fb)))))s5   #BA6*B6A:	:B=A:	>BB"%B"c                    t          j        d           t          j        ddgddgd          }t	          j                    5 }t          j        t          d          5  |                    |dd	           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t	          j                    5 }t          j
        |                              d
           t          j        t          d          5  t          |dd	           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr   r   r:   r>   r1  zfilesystem is not implementedr   r=   r*   r  rb   )r.   rW  rC   rD   r   r   r   rH  r   rF  rG  rt  r   r   r]   r   s      r)   test_filesystem_notimplementedz5TestParquetFastParquet.test_filesystem_notimplemented  s~   M***\aV1a&99:::_ 	L$#+J   L L d=UKKKL L L L L L L L L L L L L L L	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L 	L _ 	K$L**6222#+J   K K T-EJJJJK K K K K K K K K K K K K K K	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	K 	Km   B B9BB			BB		BB #B :AD4=DD4D 	 D4#D 	$D44D8;D8c                    t          j        d           t          j        ddgddgd          }t	          j                    5 }t          j        t          d          5  |                    |dd	           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t	          j                    5 }t          j
        |                              d
           t          j        t          d          5  t          |dd	           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nr    r   r:   r>   r1  z1filesystem must be a pyarrow or fsspec FileSystemr   r=   r}  rb   )r.   rW  rC   rD   r   r   r   r   r   rF  rG  rt  r   r~  s      r)   test_invalid_filesystemz.TestParquetFastParquet.test_invalid_filesystem  s~   I&&&\aV1a&99:::_ 	H$"U   H H d9GGGH H H H H H H H H H H H H H H	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H 	H _ 	G$L**6222"U   G G T)FFFFG G G G G G G G G G G G G G G	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	G 	Gr  c           	         t          j        d          }t          j        ddgddgd          }t	          j                    5 }t          j        t          d          5  |                    |d|	                                d	d
i           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t	          j                    5 }t          j        |                              d           t          j        t          d          5  t          |d|	                                d	d
i           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz
pyarrow.fsr   r:   r>   r1  z8storage_options not supported with a pyarrow FileSystem.r   r    r=   re   )r*   r  r  rb   )r.   rW  rC   rD   r   r   r   rH  r   LocalFileSystemrF  rG  rt  r   )r   pa_fsr]   r   s       r)   .test_unsupported_pa_filesystem_storage_optionszETestParquetFastParquet.test_unsupported_pa_filesystem_storage_options  su   #L11\aV1a&99:::_ 
	$#P   	 	 $$4466%*EN	    		 	 	 	 	 	 	 	 	 	 	 	 	 	 	
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 _ 	$L**6222#P   	 	 $$4466%*EN	   		 	 	 	 	 	 	 	 	 	 	 	 	 	 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	sm   B1 .BB1B	B1!B	"B11B58B5AE(E:EE
	
EE
	EE"%E"c           	      x   d}t          j        dt          t          dd                    i          }t	          j        d          5 }|                    |           t          j        t          |          5  t          |d           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )	NzPdtype_backend numpy is invalid, only 'numpy_nullable' and 'pyarrow' are allowed.rq   r:   rI   ztmp.parquetr   numpy)rP  )rC   rD   rX   rY   r   r   r   r.   r   r   r   )r   r*   r   r]   r   s        r)   test_invalid_dtype_backendz1TestParquetFastParquet.test_invalid_dtype_backend  s2   % 	 \5$uQ{{"3"3455_]++ 	:tMM$z555 : :T9999: : : : : : : : : : : : : : :	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:s6   1B/9BB/B	B/B	B//B36B3c                     t          j        t          j        g dd                    }t          j        t          j        g dd                    }t          |||           d S )Nr  r-  r'  r#  r  r.  rk  s       r)   r/  z)TestParquetFastParquet.test_empty_columns  si     \} M M MNNN<bh]&S&S&STTTR(333333r+   N)r   r   r   r}  rO  r  rT  r  r  r  r.   r5   rr  r  r  r  rg  ri  rE  r   r  r  ro  rw  r{  r  r  r  r  r/  r0   r+   r)   rJ  rJ    s       ! ! !4 4 4*; ; ;G G G"	; 	; 	;! ! !      [
 
 
/ / / / / / / / /   & [++--6UVV4 4 WV44 4 4.	R 	R 	R8 8 8* * *K K K G G G   8	: 	: 	: [++--6UVV4 4 WV4 4 4r+   rJ  )	NNNNNTFTr;   )M__doc__rv   decimalr   rc  r   r  rF  r  rZ   r.   pandas._configr   r   pandas._config.configr   pandas.compatr   pandas.compat.pyarrowr	   r
   r   r   r   pandasrC   pandas._testing_testingr   pandas.util.versionr   pandas.io.parquetr   r   r   r   r   r    r-   r   r   r3   r5   filterwarnings
pytestmarkfixturer&   rE  r6   r*   r1   r7   rF   r^   r|   nowr  r  minmaxstrptimer~   r   r   r   r   r   r   r   r   r   r   r   r   r   rt  rJ  r0   r+   r)   <module>r     s+                  				              . - - - - - - - - - - -                        ' ' ' ' ' '             NNNMM   MMM    KSTT
KG 
 "")) P"{#6tDDDOQ #  
 !!&&((;  "  	
 	
 	
 	+$$!!*D %  	
 	
 	
!  2 3 2    	 	 	 T T T   "   . h/344h/344h/344""#=?TUU""#=?TUU""#=?TUU""#=?TUU
 
 
 
 
 	? ? ? ?D9 9 9 2 2 2$ $ $$ $ $$ $ $F F F((# (# (#V6 6 6
6 
6 
6( ( ("? ? ? ? ? ? ? ?p- p- p- p- p- p- p- p-f	c0 c0 c0 c0 c0 c0 c0 c0db4 b4 b4 b4 b4T b4 b4 b4 b4 b4s$   $A+ +A54A59B   B
	B
