
    bMhI                        d dl mZ d dlmZ d dlmZ d dlZd dlZd dl	Z
d dl	mZmZ d dlmZ d dlmZ ej                            d          Zej        d             Z ej        d	d
g          d             Zd Zd Zd Zd Zd Zd Zej                            dddg          d             Zd Z d Z!d Z"d Z#ej                            dddg          d             Z$ej                            dg d          d             Z%ej                            dg d          d             Z&d  Z'ej                            d!dd"g          d#             Z(ej                            d$d%d&g          d'             Z)d( Z*d) Z+d* Z,ej                            d+g d,          d-             Z-d. Z.ej                            d/d0d1g          d2             Z/d3 Z0d4 Z1d5 Z2d6 Z3dS )7    )Iterator)StringIO)PathN)	DataFrame	read_json)
JsonReaderz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningc                  \    t          g dg dd          } |                     dd          S )N                  ABTrecordslinesorient)r   to_json)dfs    c/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pandas/tests/io/json/test_readlines.pylines_json_dfr      s5    	33	4	4B::D:333    ujsonpyarrow)paramsc                 N    | j         dk    rt          j        d           | j         S )Nr   zpyarrow.json)parampytestimportorskip)requests    r   enginer&      s(    }	!!N+++=r   c                      t          t          d          d          } t          ddgddggddg          }t          j        | |           d S )	Nz!{"a": 1, "b": 2}
{"b":2, "a" :1}
Tr   r   r   abcolumns)r   r   r   tmassert_frame_equal)resultexpecteds     r   test_read_jsonlr1   #   sY    x EFFdSSSF1a&1a&)C:>>>H&(+++++r   c                     t           | dddd          d|          }t          g dg dd	          }t          j        ||           d S )
Niojsondataline_delimited.jsonTr   r&   )r   r   r   )r   r   r   r)   r*   )r   r   r-   r.   )datapathr&   r/   r0   s       r   test_read_jsonl_engine_pyarrowr:   *   sh    vv'<==  F
 yyyyyy99::H&(+++++r   c                    |dk    r;d}|                      t          j                            |t                               t          ddgddgdfgg d	
          }|                    dd          }|dk    rt          t          |          |          }nt          t          |          |          }t          g dg dgg d	
          }t          j
        ||           d S )Nr   zEPyarrow only supports a file path as an input and line delimited jsonreasonraisesr   r   
2020-03-052020-04-08T09:58:49+00:00hector)accountsdatenamer+   Tr   r   r&   )r   r?   rA   )r   r@   rA   )applymarkerr#   markxfail
ValueErrorr   r   r   r   r-   r.   )r%   r&   r=   r   	json_liner/   r0   s          r   test_read_datetimerK   4   s   XFK--VJ-OOPPP	a&<!<=x	HI,,,
 
 
B 

i
88I8I..v>>>8I..v>>>	$	$	$&P&P&PQ,,,  H &(+++++r   c                  B   d} t          |           } t          | d          }t          ddgddggddg	          }t          j        ||           d} t          t          |           d          }t          ddgddggddg	          }t          j        ||           d S )
Nu5   {"a": "foo”", "b": "bar"}
{"a": "foo", "b": "bar"}
Tr(   u   foo”barfoor)   r*   r+   )r   r   r   r-   r.   )r4   r/   r0   s      r   test_read_jsonl_unicode_charsrO   L   s    
 EDD>>Dt4(((F;.?#sTTTH&(+++ EDx~~T222F;.?#sTTTH&(+++++r   c                  &   t          ddgddggddg          } |                     dd          }d	}||k    sJ t          d
dgddggddg          } |                     dd          }d}||k    sJ t          j        t	          t          |          d          |            t          ddgddggddg          } |                     dd          }d}||k    sJ t          j        t	          t          |          d          |            d S )Nr   r   r)   r*   r+   r   Tr   r   z{"a":1,"b":2}
{"a":1,"b":2}
zfoo}rM   zfoo"z/{"a":"foo}","b":"bar"}
{"a":"foo\"","b":"bar"}
r(   zfoo\za\z4{"a\\":"foo\\","b":"bar"}
{"a\\":"foo\"","b":"bar"}
)r   r   r-   r.   r   r   )r   r/   r0   s      r   test_to_jsonlrR   ^   sE   	QFQF#c3Z	8	8	8BZZyZ55F/HX	VUOfe_5Sz	J	J	JBZZyZ55FCHX)HV$4$4DAAA2FFF 
We$vuo6	M	M	MBZZyZ55FNHX)HV$4$4DAAA2FFFFFr   c                      t          ddgddggddg          } |                     dd                              d	          }d}||k    sJ d S )
Nr   r   r)   r*   r+   r   TrQ   
)r   r   count)r   actual_new_lines_countexpected_new_lines_counts      r   test_to_jsonl_count_new_linesrX   s   sg    	QFQF#c3Z	8	8	8BZZyZEEKKDQQ !%=======r   	chunksizer   g      ?c                    |dk    r;d}|                      t          j                            |t                               t          t          |          d          }t          t          |          d||          5 }t          j        |          }d d d            n# 1 swxY w Y   t          j
        ||           d S )Nr   mPyarrow only supports a file path as an input and line delimited jsonand doesn't support chunksize parameter.r<   Tr(   r   rY   r&   )rF   r#   rG   rH   rI   r   r   pdconcatr-   r.   )r%   r   rY   r&   r=   	unchunkedreaderchunkeds           r   test_readjson_chunksrb   {   s    7 	 	FK--VJ-OOPPP(=11>>>I	ty
 
 
 $	)F##$ $ $ $ $ $ $ $ $ $ $ $ $ $ $
 '9-----s    B!!B%(B%c                     d}t          j        t          |          5  t          t	          |           dd|          5 }	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz*chunksize can only be passed if lines=TruematchFr   r\   r#   r>   rI   r   r   )r   r&   msg_s       r   &test_readjson_chunksize_requires_linesri      s    
6C	z	-	-	-  ]##5Af
 
 
 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 4   !A%AA%A	A%A	A%%A),A)c                 
   |dk    r5d}|                      t          j                            |                     t	          j        ddd          }t          |                    dd	                    }t          |dd
|          }t          |                    dd	                    }t          |dd
d|          5 }t	          j	        |          }d d d            n# 1 swxY w Y   t          j        ||           d S )Nr   r[   )r=   r   r   r   Tr   r   Series)r   typr&   )r   rm   rY   r&   )rF   r#   rG   rH   r]   rl   r   r   r   r^   r-   assert_series_equal)r%   r&   r=   sstrior_   r`   ra   s           r   test_readjson_chunks_seriesrq      sF   7 	 	FK--V-<<=== 		""##AQYYT)Y<<==E%t&IIIIQYYT)Y<<==E	Tx1V
 
 
 $	)F##$ $ $ $ $ $ $ $ $ $ $ $ $ $ $
 7I.....s   C##C'*C'c                 b   |dk    r;d}|                      t          j                            |t                               t          t          |          dd|          5 }t          |          }d d d            n# 1 swxY w Y   |d         j        dk    sJ |d	         j        d
k    sJ d S )Nr   r[   r<   Tr   r\   r   r   r   r   )r   r   )	rF   r#   rG   rH   rI   r   r   listshape)r%   r   r&   r=   r`   chunkss         r   test_readjson_each_chunkrw      s    7 	 	FK--VJ-OOPPP 
tq
 
 
 	f               !9?f$$$$!9?f$$$$$$s   "A>>BBc                 
   |dk    r;d}|                      t          j                            |t                               t          j        d          5 }t          g dg dd          }|                    |dd	
           t          |dd|          5 }t          j        |          }d d d            n# 1 swxY w Y   t          |d|          }t          j        ||           d d d            d S # 1 swxY w Y   d S )Nr   r[   r<   	test.jsonr
   r   r   Tr   r   r   r\   r7   )rF   r#   rG   rH   rI   r-   ensure_cleanr   r   r   r]   r^   r.   )r%   r&   r=   pathr   r`   ra   r_   s           r   test_readjson_chunks_from_filer|      s   7 	 	FK--VJ-OOPPP		%	% 2YYYYYY7788


4tI
666t41VDDD 	(i''G	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(d$v>>>	
i1112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s7   AC8B9-C89B=	=C8 B=	*C88C<?C<c                    t          j        d          5 }t          g dg dd          }|                    |dd           t	          |d dddddd	d d d| d d 
          }|5  |                                 d d d            n# 1 swxY w Y   |j        j        j        sJ d|              	 d d d            d S # 1 swxY w Y   d S )Nry   r
   r   r   Tr   r   frameF)r   rm   dtypeconvert_axesconvert_dateskeep_default_datesprecise_float	date_unitencodingr   rY   compressionnrowsz%didn't close stream with chunksize = )	r-   rz   r   r   r   readhandleshandleclosed)rY   r{   r   r`   s       r   test_readjson_chunks_closesr      s   		%	% ?YYYYYY7788


4tI
666#
 
 
   	 	KKMMM	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 N!(	? 	?>9>>	? 	? 	? 	?+? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?s6   AB;#B8B;B	B;B	!B;;B?B?)r   g@rN   c                     d}t          j        t          |          5  t          t	          |           d||          5 }	 d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz"'chunksize' must be an integer >=1rd   Tr\   rf   )r   rY   r&   rg   rh   s        r   test_readjson_invalid_chunksizer      s    
/C	z	-	-	-  ]##49V
 
 
 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	                 rj   )Nr   r   c                    d}t          g dg dd          }t          t          |          d|           }| .|5  t          j        |          }d d d            n# 1 swxY w Y   t          j        ||d|             d S )	NzF

    {"A":1,"B":4}



    {"A":2,"B":5}







    {"A":3,"B":6}
    r
   r   r   Tr   rY   zchunksize: )obj)r   r   r   r]   r^   r-   r.   )rY   jorigtests       r   )test_readjson_chunks_multiple_empty_linesr      s    	A  9999995566DXa[[	BBBD 	# 	#9T??D	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	# 	#$*C	*C*CDDDDDDs   AA!$A!c                    |dk    r;d}|                      t          j                            |t                               t          j        d          5 }|                    dd            t          |dd	          5 }|	                    d
           d d d            n# 1 swxY w Y   t          ||          }t          ddgi          }t          j        ||           d d d            d S # 1 swxY w Y   d S )Nr   r[   r<   ry   zlocale.getpreferredencodingc                     dS )Ncp949 )do_setlocales    r   <lambda>z'test_readjson_unicode.<locals>.<lambda>#  s    PW r   wzutf-8)r   u6   {"£©µÀÆÖÞßéöÿ":["АБВГДабвгд가"]}rE   u   £©µÀÆÖÞßéöÿu   АБВГДабвгд가)rF   r#   rG   rH   rI   r-   rz   setattropenwriter   r   r.   )r%   monkeypatchr&   r=   r{   fr/   r0   s           r   test_readjson_unicoder     s   7 	 	FK--VJ-OOPPP		%	% 09;W;WXXX$g... 	N!GGLMMM	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 	N 4///69R8STUU
fh///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s6   *C2 B"C2"B&	&C2)B&	*;C22C69C6r   r   c                     d}t          t          |          d|           }t          g dg dd          j        d |          }t	          j        ||           d S )N[{"a": 1, "b": 2}
        {"a": 3, "b": 4}
        {"a": 5, "b": 6}
        {"a": 7, "b": 8}T)r   r   r   r   r      r   r   r      r8   )r   r   r   ilocr-   r.   )r   r&   jsonlr/   r0   s        r   test_readjson_nrowsr   ,  sg    E xd%@@@F|||,,,??@@EfufMH&(+++++r   znrows,chunksizers   )r   r   c                 $   |dk    r;d}|                      t          j                            |t                               d}|dk    rNt          t          |          d|||          5 }t          j        |          }d d d            n# 1 swxY w Y   n@t          |d|||          5 }t          j        |          }d d d            n# 1 swxY w Y   t          g dg dd	          j
        d |         }t          j        ||           d S )
Nr   r[   r<   r   T)r   r   rY   r&   r   r   r8   )rF   r#   rG   rH   rI   r   r   r]   r^   r   r   r-   r.   )	r%   r   rY   r&   r=   r   r`   ra   r0   s	            r   test_readjson_nrows_chunksr   9  s    7 	 	FK--VJ-OOPPPE
 UOO4u	RX
 
 
 	(i''G	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(
 Ui
 
 
 	(i''G	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( |||,,,??@@EfufMH'8,,,,,s$   +BBB,CCCc                     d}d}t          j        t          |          5  t          |dd|            d d d            d S # 1 swxY w Y   d S )Nr   z&nrows can only be passed if lines=Truerd   Fr   )r   r   r&   )r#   r>   rI   r   )r&   r   rg   s      r   "test_readjson_nrows_requires_linesr   X  s    E 3C	z	-	-	- > >%uAf====> > > > > > > > > > > > > > > > > >s   AAAc                 .   |dk    r;d}|                      t          j                            |t                               t          ddggddgdg	          t          d
dggddgdg	          t          ddggddgdg	          g} |dddd          }t          |                                          }t          |dd|          5 }t          |          D ] \  }}	t          j        |	||                    !	 d d d            d S # 1 swxY w Y   d S )Nr   r[   r<   r   r   r)   r*   r   )r,   indexr   r   r   r   r3   r4   r5   r6   Tr\   )rF   r#   rG   rH   rI   r   r   as_urir   	enumerater-   r.   )
r%   r9   r&   r=   df_list_expectedos_pathfile_url
url_readerr   chucks
             r   "test_readjson_lines_chunks_fileurlr   d  s    7 	 	FK--VJ-OOPPP 	Aq6(S#Jqc:::Aq6(S#Jqc:::Aq6(S#Jqc:::
 htVV-BCCGG}}##%%H	841V	D	D	D B
%j11 	B 	BLE5!%)9%)@AAAA	BB B B B B B B B B B B B B B B B B Bs   1D

DDc            	          ddz  }  G d d          } ||           }t          t          t          |dd                              dk    sJ |j        d	k    sJ d S )
Nz\{"a": 1, "b": 2}
        {"a": 3, "b": 4}
        {"a": 5, "b": 6}
        {"a": 7, "b": 8}
i  c                   (    e Zd ZddZd ZdefdZdS )/test_chunksize_is_incremental.<locals>.MyReaderreturnNc                 <    d| _         t          |          | _        d S )Nr   )
read_countr   stringio)selfcontentss     r   __init__z8test_chunksize_is_incremental.<locals>.MyReader.__init__  s    DO$X..DMMMr   c                 @    | xj         dz  c_          | j        j        | S Nr   )r   r   r   )r   argss     r   r   z4test_chunksize_is_incremental.<locals>.MyReader.read  s$    OOq OO%4=%t,,r   c                 J    | xj         dz  c_         t          | j                  S r   )r   iterr   )r   s    r   __iter__z8test_chunksize_is_incremental.<locals>.MyReader.__iter__  s"    OOq OO&&&r   )r   N)__name__
__module____qualname__r   r   r   r   r   r   r   MyReaderr     sR        	/ 	/ 	/ 	/	- 	- 	-	'h 	' 	' 	' 	' 	' 	'r   r   Td   r   r   
   )lenrt   r   r   )r   r   r`   s      r   test_chunksize_is_incrementalr   {  s    	 		 
' ' ' ' ' ' ' ' Xe__FtIfDC@@@AABBQFFFFr!!!!!!r   orient_)splitr   tablec                     t          ddgddgd          }d}t          j        t          |          5  |                    d|            d d d            d S # 1 swxY w Y   d S )	Nr   r   r)   r*   col1col2Pmode='a' \(append\) is only supported when lines is True and orient is 'records'rd   )moder   r   r#   r>   rI   r   )r   r   rg   s      r   test_to_json_append_orientr     s     
QFS#J77	8	8B	0  
z	-	-	- - -


G
,,,- - - - - - - - - - - - - - - - - -s   AA Ac                      t          ddgddgd          } d}t          j        t          |          5  |                     ddd	
           d d d            d S # 1 swxY w Y   d S )Nr   r   r)   r*   r   r   rd   Fr   r   r   r   r   )r   rg   s     r   test_to_json_append_linesr     s     
QFS#J77	8	8B	0  
z	-	-	- < <


5
;;;< < < < < < < < < < < < < < < < < <s   AA!Amode_rxc                     t          ddgddgd          }d|  d}t          j        t          |          5  |                    | d	d
           d d d            d S # 1 swxY w Y   d S )Nr   r   r)   r*   r   zmode=z@ is not a valid option.Only 'w' and 'a' are currently supported.rd   Fr   r   r   )r   r   rg   s      r   test_to_json_append_moder     s     
QFS#J77	8	8B	4 	4 	4 	4  
z	-	-	- > >


U9
===> > > > > > > > > > > > > > > > > >s   AA"%A"c                     t          ddgddgd          } t          ddgdd	gd          }t          g d
g dd          }t          j        d          5 }|                     |dd           |                    |ddd           t	          |d          }t          j        ||           d d d            d S # 1 swxY w Y   d S )Nr   r   r)   r*   r   r   r   cd)r   r   r   r   )r)   r*   r   r   ry   Tr   r   r   r(   )r   r-   rz   r   r   r.   )df1df2r0   r{   r/   s        r   -test_to_json_append_output_consistent_columnsr     s(    aVc3Z88
9
9C
aVc3Z88
9
9C,,,8L8L8LMMNNH		%	% 0DY777Ds$yAAA 4t,,,
fh///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   AB<<C C c                     t          ddgddgd          } t          ddgdd	gd
          }t          g dg dt          j        t          j        dd	gd          }t          j        d          5 }|                     |ddd           |                    |ddd           t          |d          }t          j        ||           d d d            d S # 1 swxY w Y   d S )Nr   r   r)   r*   r   er   !#r   col3)r   r   NN)r)   r*   r   r   )r   r   r   ry   Tr   r   r(   )r   npnanr-   rz   r   r   r.   )r   df3r0   r{   r/   s        r   /test_to_json_append_output_inconsistent_columnsr     sG    aVc3Z88
9
9C
c3Z#s<<
=
=C&&&(((VRVS#.	
 	
 H 
	%	% 0Ds$yAAADs$yAAA 4t,,,
fh///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0s   0ACCCc                  8   t          ddgddgd          } t          ddgdd	gd          }t          d
dgddgd          }t          dddgi          }t          g ddddd	d
dt          j        t          j        gt          j        t          j        t          j        t          j        ddt          j        t          j        gg dd                              ddi          }t	          j        d          5 }|                     |ddd           |                    |ddd           |                    |ddd           |                    |ddd           t          |d          }t	          j        ||           d d d            d S # 1 swxY w Y   d S )Nr   r   r)   r*   r   r   r   r   r   r   r   r   r   r   col4TF)r   r   r   r   NNNN)NNNNNNTF)r   r   r   r   floatry   r   r   r(   	r   r   r   astyper-   rz   r   r   r.   r   r   r   df4r0   r{   r/   s          r   ,test_to_json_append_output_different_columnsr     s    aVc3Z88
9
9C
aVc3Z88
9
9C
c3Z#s<<
=
=C
VdE]+
,
,C888#sCc2626BVRVRVRVS#rvrvNEEE		
 	
  ffg  
	%	% 	0Ds$yAAADs$yAAADs$yAAADs$yAAA 4t,,,
fh///	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0   7BFFFc                  8   t          ddgddgd          } t          ddgdd	gd          }t          d
dgddgd          }t          dddgi          }t          g dt          j        t          j        d
ddd	ddgt          j        t          j        ddt          j        t          j        t          j        t          j        gg dd                              ddi          }t	          j        d          5 }|                    |ddd           |                    |ddd           |                    |ddd           |                     |ddd           t          |d          }t	          j        ||           d d d            d S # 1 swxY w Y   d S )Nr   r   r)   r*   r   r   r   r   r   r   r   r   r   r   r   TF)TFNNNNNN)NNNNr   r   r   r   )r   r   r   r   r   ry   r   r   r(   r   r   s          r   6test_to_json_append_output_different_columns_reorderedr     s    aVc3Z88
9
9C
aVc3Z88
9
9C
c3Z#s<<
=
=C
VdE]+
,
,C EEEVRVS#sCcBVRVS#rvrvrvrvN888		
 	
  ffg  
	%	% 	0Ds$yAAADs$yAAADs$yAAADs$yAAA 4t,,,
fh///	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0 	0r   )4collections.abcr   r3   r   pathlibr   numpyr   r#   pandasr]   r   r   pandas._testing_testingr-   pandas.io.json._jsonr   rG   filterwarnings
pytestmarkfixturer   r&   r1   r:   rK   rO   rR   rX   parametrizerb   ri   rq   rw   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   <module>r     s   $ $ $ $ $ $                                   + + + + + +[''C 

 4 4 4
 +,,,  -,, , ,, , ,, , ,0, , ,$G G G*> > > q#h//. . 0/.,  / / /0% % %&2 2 2$ tQi00? ? 10?6 &9&9&9::  ;: lll33E E 43E20 0 0& 1a&))	, 	, *)	, *VV,<==- - >=-<	> 	> 	>B B B." " "8 $?$?$?@@	- 	- A@	-	< 	< 	< 3*--	> 	> .-	>0 0 0$0 0 000 0 0:0 0 0 0 0r   