
    z-Ph@                     |   d dl Z d dlZd dlZ	 d dlZn# e$ r dZY nw xY wd dlZd dlZd dl	m
Z
 	 d dlmZ d dl	mZmZ n# e$ r dZY nw xY w	 d dlZd dlmZ d dl	mZ n# e$ r dxZZY nw xY wej        j        Zej        j        d             Zej        j        d             Zej        j        d             Zej        j        d             Zej        j        d	             Zej        j        ej                            d
g d          d                         Zej        j        ej                            dddg          d                         Zej                            d
g d          d             Z d Z!ej        j        d             Z"ej        j        d             Z#ej        j        d             Z$d Z%dS )    N)_check_roundtrip)_read_table_write_table)_roundtrip_pandas_dataframec                  (   t          j        t          j        ddd          gd          } | j                            d          } | | _        t          j        | | j                            d          d| 	          }t          j	                    }t          j                            |          }t          ||           |                    d
           t          j        |          }|                                }t%          j        ||           d S )N  	      zdatetime64[us]dtypeutcz
US/Eastern)tz_aware
tz_easternindexr   )pdSeriesdatetimedttz_localizer   	DataFrame
tz_convertioBytesIOpaTablefrom_pandasr   seekpqread_pandas	to_pandastmassert_frame_equal)sdffarrow_table
table_readdf_reads         c/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pyarrow/tests/parquet/test_datetime.pytest_pandas_parquet_datetime_tzr+   4   s    
 		8$T1a0019IJJJA	AAG 
1%&T__\%B%BD D
 
 
B 	
A(&&r**Ka   FF1III""J""$$G"g&&&&&    c            	          t          j         ddddddt           j        j                  } t          j        d| gi          }t          |i            d S )Ni        -   )tzinfofoowrite_kwargs)r   timezoner   r   r   r   )valuer%   s     r*   test_datetime_timezone_tzinfor7   O   s]    dAq!R%-%6%:< < <E	uug&	'	'B444444r,   c                    ddl m}  |            }t          j        dt          j        t          j        d                              g}t          j        g dd          d d t          j        g dd          g|d<   t          j	        |          }t          j
        |          }| d	z  }t          j                            ||
          }t          ||dd           t          |          }|                                }	|                                }
t#          |
d                   D ]>\  }}t%          |t          j                  r|                    d          |
j        |df<   ?t-          j        |
|	           t1          j        t4                    5  t          ||dd           d d d            d S # 1 swxY w Y   d S )Nr   )OrderedDict
datetime64ms)z2007-07-13T01:23:34.123456789N2010-08-13T05:46:57.437699912datetime64[ms]r   )z2007-07-13T02Nr<   zpandas_roundtrip.parquetschema2.6usversioncoerce_timestampszM8[us]unknown)collectionsr9   r   fieldlist_	timestampnparrayr   r   r?   r   r   r   r   r!   copy	enumerate
isinstancendarrayastypelocr"   r#   pytestraises
ValueError)tempdirr9   arraysfieldsr%   r?   filenamer'   r(   r)   df_expectedixs                r*   test_coerce_timestampsr\   X   s8   '''''' []]Fh|xT 2 2335 5 6F 	 3 3 3 (	) 	) 	) 	
 3 3 3 (	) 	) 	)F< 
f		BYvF33H(&&r&&99KhNNNNX&&J""$$G''))K+l344 B B1a$$ 	B/0xx/A/AKOA|O,+w///	z	"	" 2 2[(E'0	2 	2 	2 	22 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s   F;;F?F?c           	      8   t          j         ddddddd          }t          j         dddddd          }t          j        dt          j        d                    g}d||gi}t	          j        |          }t          j        |          }| dz  }t          j                            ||          }t          ||d	d
d           t          |          }	|	                                }
d||gi}t	          j        |d          }t          j        ||
           dS )zc
    ARROW-2555: Test that we can truncate timestamps when coercing if
    explicitly allowed.
    r   r.   )yearmonthdayhourminutesecondmicrosecond)r^   r_   r`   ra   rb   rc   r:   rA   zpandas_truncated.parquetr>   r@   r;   T)rC   rD   allow_truncated_timestampsr=   r   N)r   r   rG   rI   r   r   r?   r   r   r   r   r!   r"   r#   )rU   dt_usdt_ms	fields_us	arrays_usdf_us	schema_usrX   table_ustable_msdf_msarrays_expectedrY   s                r*    test_coerce_timestamps_truncatedrp      s;    4qaa%&A7 7 7E4qaa%&( ( (E ,T(:(:;;<Iu~.IL##E	)$$I33Hx##E)#<<H8Ud,02 2 2 28$$H  E#eU^4O,6FGGGK+u-----r,   c           	         t          j                    }t          j        g dd          }t          j        ||          }t          j                    }|                    d          dz  }t          j        ||          }t          j        d          }t          j        d          j	        d	z  }t          j        ||d
z   |dz   gd          }	t          j        |	|          }
t          j
        d          }t          j        dd          }t          j        ||          }t          j        d          }t          j        |                    d          |          }t          j
        d          }t          j        ||          }t          j
        d          }t          j        |d	z  |          }t          j        d          }t          j        d          j	        }t          j        ||d	z   |dz   gd          }t          j        ||          }t           j                            |||
||||gg d          }t           j                            |||
||||gg d          }t          ||d           t          j        d          }t          j        dd          }t          j        ||          }t          j        d          }t          j        dd          }t          j        ||          }t          j        d          }t          j        dd          }t          j        ||          }t           j                            |||gg d          }t           j                            |||gg d          }| dz  }t!          ||d           t#          j        |          j        }t)          d          D ]"}|                    |          j        dk    sJ #t/          |          }|                    |          sJ t          j        d          } t          j        |dz  d          }!t          j        |!|           }"t          j        d          }#t          j        |d	z  d          }$t          j        |$|#          }%t           j                            |"|%|gg d          }| dz  }t!          ||dd           t#          j        |          j        }t)          d          D ]"}|                    |          j        dk    sJ #t/          |          }|                    |          sJ | dz  }t!          ||dd !           t#          j        |          j        }t)          d          D ]"}|                    |          j        dk    sJ #t/          |          }|                    |          sJ d S )"N)ikC  ilC  imC  int32r   typeint64i \&rA   z
2001-01-01  r.      r;      i4r$   ns  )date32date64ztimestamp[us]z	time32[s]z
time64[us]ztime32_from64[s]ztimestamp[ns]r@   expectedrC      )zts[ms]zts[us]zts[ns]zint64_timestamps.parquetrC   INT64i@B z!explicit_int96_timestamps.parquetTrC   use_deprecated_int96_timestampsINT96zspark_int96_timestamps.parquetspark)rC   flavor)r   r|   rJ   rK   r}   rP   rI   r   	Timestampr6   time32arangetime64r   from_arraysr   r   r   ParquetFiler?   rangecolumnphysical_typer   equals)&rU   t1data1a1t2data2a2t3startdata3a3t4data4a4t5a5t6a6ex_t6ex_a6t7data7a7tabler   t0data0a0rX   parquet_schemarZ   
read_tablet0_nsdata0_nsa0_nst1_nsdata1_nsa1_nss&                                         r*   test_date_time_typesr      s   	BH***':::E	%b	!	!	!B	BLL!!H,E	%b	!	!	!B	d		BL&&,t3EHeUQY	2'BBBE	%b	!	!	!B	4BIat$$$E	%b	!	!	!B	4B	%,,w''b	1	1	1B	3B	%b	!	!	!BIdOOEHUT\...E	d		BL&&,EHeUT\54<8"$ $ $E	%b	!	!	!BH  "b"b"b"!="3 "3 "34 4E x##RRR$C%6 %6 %67 7H UXu====	d		BIaw'''E	%b	!	!	!B	d		BIaw'''E	%b	!	!	!B	d		BIaw'''E	%b	!	!	!BH  "b"!?!?!?A AEx##RRL$B$B$BD DH 33H%0000^H--4N1XX A A$$Q''5@@@@@X&&JX&&&&&LExw777HHXE***ELExG444HHXE***Ex##UE2$6$B$B$BD DH <<H%157 7 7 7^H--4N1XX A A$$Q''5@@@@@X&&JX&&&&& 99H%! ! ! !^H--4N1XX A A$$Q''5@@@@@X&&JX&&&&&&&r,   unitr$   r;   rA   rz   c                    t          j        d          j        dz  }t          j        ||dz   dd          }|dz  }|dz  }|dz  }t          j        |t          j        d          	          }t          j        |t          j        d
          	          }t          j        |t          j        d          	          }t          j        |t          j        d          	          }	||||	d}
g d}t
          j        	                    ||||	g|          }t
          j        	                    |

                    |           gdz  |          }d| i}t          |||d           t          ||d|d           d S )N
2010-01-01 ʚ;
   r.   ru   r   rv   r$   rs   r;   rA   rz   r   )ts_sts_msts_usts_nsr   coerce_int96_timestamp_unitT)read_table_kwargsr   r@   )rC   r   r   )r   r   r6   rJ   r   r   rK   rI   r   r   getr   )r   i_sd_sd_msd_usd_nsa_sa_msa_usa_nsrV   namesr   r   r   s                  r*    test_coerce_int96_timestamp_unitr     s    ,|
$
$
*Z
7C
)Cr1G
4
4
4C:D$;D$;D
(3R\#..
/
/
/C8Dr|D11222D8Dr|D11222D8Dr|D11222Dd$d;;F///EH  #tT4!8%@@E x##VZZ%5%5$6q$8%@@H6=UH'859; ; ; ; UHe'859; ; ; ; ; ;r,   pq_reader_methodr   r   c                    d }t          j         ddd          t          j         ddd          t          j         ddd          g}t          j        d|i          }t          j        |          }|dz  }t          j        ||dd	
            || |          }t          j                    5  t          j	        ddt                     |d                                         |k    sJ 	 d d d            n# 1 swxY w Y    || |d          }|                    d          }	|d                             t                    |d<   t          j        ||	           d S )Nc                     | dk    r$t          j        |fi |                                S | dk    rt          j        |fi |S d S )Nr   r   )r   r   readr   )r   rX   kwargss      r*   	get_tablez7test_coerce_int96_timestamp_overflow.<locals>.get_table0  s\    },,>(55f55::<<<--=44V444 .-r,   rv   r.   r{     az test_round_trip_overflow.parquetT1.0)r   rC   ignorez,Discarding nonzero nanoseconds in conversionr$   )r   )timestamp_as_object)r   r   r   r   r   r   write_tablewarningscatch_warningsfilterwarningsUserWarning	to_pylistr!   rP   objectr"   r#   )
r   rU   r   oob_dtsr%   r   rX   	tab_errortab_correct
df_corrects
             r*   $test_coerce_int96_timestamp_overflowr   ,  s   5 5 5 	$1%%$1%%$1%%G
 
sGn	%	%BHRLLE;;HN5(D " " " "
 	*H55I		 	"	" 5 5 N +	- 	- 	- ~''))W44444	5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 )(  K &&4&@@JgnnV$$BsG"j)))))s   +<C44C8;C8)r;   rA   rz   c                     t          j        | d          }t          j        g d|          }t          j        |gdg          }t	          |           d S )NAmerica/New_Yorktz)r.   rw   rx   rs   f0r   )r   rI   rK   r   r   )r   tyarrts       r*   test_timestamp_restore_timezoner   U  s]     
d1	2	2	2B
(9992
&
&
&C
#tf%%%AQr,   c                  8   t          j        dd          } t          j        g d|           }t          j        |gdg          }t          j        dd          }t          j        |                    |          gdg          }t          ||d	
           d S )Nrz   r   r   )rv   r{   r   rs   r   r   rA   2.4r~   )r   rI   rK   r   castr   )r   r   r   ty_usr   s        r*   *test_timestamp_restore_timezone_nanosecondr   ^  s     
d1	2	2	2B
(%%%B
/
/
/CHcU4&)))EL"4555Ex%)$888HUXu======r,   c                      t          j        g dd          } t          j        d| j        gi          }t	          |i            d S )N)z09:00z09:30z10:00z10:30z11:00z11:30z12:00z%H:%M)formattimer3   )r   to_datetimer   r   r   )timesr%   s     r*   $test_list_of_datetime_time_roundtripr   i  s]     N . . .6=? ? ?E	v
|,	-	-B444444r,   c                     t          j        d          j        dz  } t          j        | | dz   dd          }|dz  }|dz  }|dz  }t          j        |t          j        d          	          }t          j        |t          j        d
          	          }t          j        |t          j        d          	          }t          j        |t          j        d          	          }g d}	g d}
t
          j        	                    ||||g|
          }t
          j        	                    ||||g|
          }t          ||d           t          ||d           t
          j        	                    ||||g|
          }t          ||d           t
          j        	                    ||||g|
          }|	D ]}t          ||d
|           t
          j        	                    ||||g|
          }|	D ]}t          |||d           t
          j        	                    ||||g|
          }|	D ]}t          |||d           d S )Nr   r   r   r.   ru   r   rv   r$   rs   r;   rA   rz   )r   r   r@   )zts:szts:mszts:uszts:nsr   r   r   r@   rD   rC   rB   Tr   )r   r   r6   rJ   r   r   rK   rI   r   r   r   )r   r   r   r   r   r   r   r   r   all_versionsr   r   r   vers                 r*   *test_parquet_version_timestamp_differencesr   r  sy   
,|
$
$
*Z
7C
)Cr1G
4
4
4C:D$;D$;D
(3R\#..
/
/
/C8Dr|D11222D8Dr|D11222D8Dr|D11222D(((L///EH  #tT4!8%@@E x##T4t$<eDDHUHe4444UHe4444 x##T4t$<eDDHUHe4444 x##T4t$<eDDH O OD#NNNNNx##T4t$<eDDH O O#NNNNN  x##T4t$<eDDH ? ?#9=	? 	? 	? 	? 	?? ?r,   c                 8   d}t          j        dt          |          it          j        dd|                    }t          j                            |          }| dz  }	 t          j        ||d	           n# t          $ r Y nw xY w|
                                sJ t          j        |          }|                    |          sJ | d
z  }t          j        t                    5  t          j        ||dd           d d d            d S # 1 swxY w Y   d S )Nr	   r[   z
2017-01-01rz   )freqperiodsr   zwritten.parquetr@   r   znot_written.parquetr;   r   )r   r   r   
date_ranger   r   r   r   r   	Exceptionexistsr   r   rR   rS   rT   )rU   nr%   tbrX   recovered_tables         r*   5test_noncoerced_nanoseconds_written_without_exceptionr    s    	
A	sE!HHoM,T1MMM
O 
O 
OB			b	!	!B**H
r8U33333   ??mH--O99_%%%%% ..H	z	"	" L L
r8tUKKKKL L L L L L L L L L L L L L L L L Ls$   "A: :
BB)DDDc                      d dD             } t           j                            | g d          }t          |           d S )Nc                 b    g | ],}t          j        g d t          j        |                    -S ))r   r.   rw   rx   rs   )r   rK   duration).0r   s     r*   
<listcomp>z&test_duration_type.<locals>.<listcomp>  sE     3 3 3 h|||"+d*;*;<<< 3 3 3r,   r   )zd[s]zd[ms]zd[us]zd[ns])r   r   r   r   )rV   r   s     r*   test_duration_typer    sR    3 313 3 3FH  )L)L)LMMEUr,   )&r   r   r   numpyrJ   ImportErrorrR   pyarrowr   pyarrow.tests.parquet.commonr   pyarrow.parquetparquetr   r   r   pandasr   pandas.testingtestingr"   r   mark
pytestmarkr+   r7   r\   rp   r   parametrizer   r   r   r   r   r   r  r   r,   r*   <module>r     s<  $  				    	BBB      9 9 9 9 9 9      FFFFFFFFF   	BBBHHHHHHH   NB [ 
 ' ' '4 5 5 5 '2 '2 '2T . . .: h' h' h'V !8!8!899; ; :9 ;: +m\-JKK$* $* LK $*N !3!3!344  54> > > 5 5 5 8? 8? 8?v L L L0    s-    > AAA 	A)(A)