y-PhddlZddlmZddlmZddlZddlmZddlZddl m Z ddl Z ddl Z ddl Z ddlZ ddlZn #e$rdZYnwxYwddlZddlmZmZmZiaiaiadZdZd Zd Zd Zd Z d.d Z!dZ"dZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+d.dZ, d/dZ-dZ.d0dZ/dZ0 d1d Z1hd!Z2d"Z3d#Z4d$Z5d.d%Z6 d.d&Z7d'Z8d(Z9d)Z:d*Z;d+ZdS)2N)Sequence)futures)deepcopy) zip_longest) _pandas_api frombytesis_threading_enabledc "tstitjjdtjjdtjjdtjjdtjjdtjj dtjj dtjj dtjj d tjj d tjjd tjjd tjjd tjjdtjjdtjjdtjjdtjjdtjjdtjjditS)Nemptyboolint8int16int32int64uint8uint16uint32uint64float16float32float64datetimebytesunicode)_logical_type_mapupdatepalibType_NA Type_BOOL Type_INT8 Type_INT16 Type_INT32 Type_INT64 Type_UINT8 Type_UINT16 Type_UINT32 Type_UINT64Type_HALF_FLOAT Type_FLOAT Type_DOUBLE Type_DATE32 Type_DATE64 Type_TIME32 Type_TIME64 Type_BINARYType_FIXED_SIZE_BINARY Type_STRINGU/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/pyarrow/pandas_compat.pyget_logical_type_mapr7.sw    " FNG" F f"  F f"  F w " F w " F w "  F w"  F "  F "  F "  F "I"  F y"  F  "  F "  F " F !" " F #" $ F  F )7 F  )" "    , r5ct} ||jS#t$rt|tjjrYdSt|tjjr)dt|j cYSt|tjj r |j dndcYStj |rYdSYdSwxYw)N categoricalzlist[{}] datetimetzdatetimedecimalobject)r7idKeyError isinstancerrDictionaryTypeListTypeformatget_logical_type value_type TimestampTypetztypes is_decimal) arrow_typelogical_type_maps r6rDrDKs+--  ..     j"&"7 8 8  ==  BFO 4 4 $$%5j6K%L%LMM M M M  BF$8 9 9 #-=#<<<* L L L X  , , 99xx s" )C# AC#*C#=C#"C#ctsttjdtjdtjdtjdtjdtjdtj dtj dtj d tj d tj d d d tjdtjditS)Nr r rrrrrrrrr datetime64[D]rstringr)_numpy_logical_type_maprnpbool_r rrrrrrrrrstr_bytes_r4r5r6get_numpy_logical_type_maprT\s " && Hf GV Hg Hg Hg Hg Ix Ix Ix J J V GX Iw(    #"r5cLt} ||jjS#t$rwt |jdrYdSt |jdrt |jcYStj|}|dkrYdS|cYSwxYw)NrGr: datetime64rNr) rTdtypetyper?hasattrstr startswithr infer_dtype)pandas_collectionnumpy_logical_type_mapresults r6get_logical_type_from_numpyr`rs799 %&7&=&BCC     $*D 1 1 <<  & ' ' 2 2< @ @ 0(.// / / /():;; X  99  s!"B#;B#B#B#"B#ch|j}t|dkrKt|d|}|Jt|j|jd}t|jj}nJt|dr)dtj |j i}d}nd}t|}||fS)Ncategorycat)num_categoriesorderedrGtimezonedatetime64[ns]) rWrZgetattrlen categoriesrecodesrYrrtzinfo_to_stringrG)columnrWcatsmetadataphysical_dtypes r6get_extension_dtype_inforqs LE 5zzZvuf--!$/22|  TZ-..   $ 7 7 A AB)U 8 ##r5ct|}t|\}}|dkr|j|jd}d}|st |t rt j|sJt |ts5td |t|j t |ts$Jtt||||||dS)aConstruct the metadata for a given column Parameters ---------- column : pandas.Series or pandas.Index name : str arrow_type : pyarrow.DataType field_name : str Equivalent to `name` when `column` is a `Series`, otherwise if `column` is a pandas Index then `field_name` will not be the same as `name`. This is the name of the field in the arrow Table's schema. Returns ------- dict r<) precisionscaler=Nz6Column name must be a string. Got column {} of type {}name field_name pandas_type numpy_typero) rDrqrsrtr@floatrPisnanrZ TypeErrorrCrX__name__)rmrvrJrw logical_type string_dtypeextra_metadatas r6get_column_metadatars "$J//L#;F#C#C L.y  #-%    D%(( -/Xd^^ 4%%  D K Kd4jj)     j# & &==D,<,<(=(==== #""   r5c || d|D}td|D}t|} |d| |z } || |z d} g} t|||| D]/\} }}}t| |||}| |0g}|durSg}t|| |D]\}}}t |t r|j4t |jts||jt|t|j||}||t|dkr tj d|dtd g}t|j d |j g}t|j d |j jg}t||D]*\}}t||}||+ngx}x}}d tj||| |zdt"jdt&jddiS)aReturns a dictionary containing enough metadata to reconstruct a pandas DataFrame as an Arrow Table, including index columns. Parameters ---------- columns_to_convert : list[pd.Series] df : pandas.DataFrame column_names : list[str | None] column_field_names: list[str] index_levels : List[pd.Index] index_descriptors : List[Dict] preserve_index : bool types : List[pyarrow.DataType] Returns ------- dict Nc,g|]}t|Sr4)rZ).0rvs r6 z&construct_metadata..sAAADc$iiAAAr5c<g|]}t|t|Sr4)r@dict)rdescrs r6rz&construct_metadata..s='G'G'G.8.E.E'Gu'G'G'Gr5)rvrJrwFrz&The DataFrame has non-str index name `z@` which will be converted to string and not roundtrip correctly. stacklevellevelsnamesspandaspyarrow)libraryversion) index_columnscolumn_indexescolumnscreatorpandas_versionutf8)riziprappendr@rrvrZ_column_name_to_stringswarningswarn UserWarningrhr_get_simple_index_descriptorjsondumpsr __version__rrencode)columns_to_convertdf column_names index_levelsindex_descriptorspreserve_indexrHcolumn_field_namesnum_serialized_index_levelsntypesdf_types index_typescolumn_metadatacolrvrwrJroindex_column_metadatanon_str_index_nameslevel descriptorrrrs r6construct_metadatars*!BALAAA"%'G'G:K'G'G'G#H#HZZF:f:::;H!<<==>KO-01C\1CX.O.O)))T:z&s2<2<>>> x((((U"" -0{1B.D.D 3 3 )E:z*d++ z%jS.I.I%#**5:666*,UZ88%% H " ( ( 2 2 2 2 " # #a ' ' M09L000  + + + + X |<< Gbjo->??vu-- , ,KE43E4@@H  ! !( + + + + ,FHGG1N 4:.,&)>>$>*1      6&>>  r5ct|\}}t|}d|vrtjdtd|dkr|rJddi}|||||dS) NmixedzlThe DataFrame has column names of mixed type. They will be converted to strings and not roundtrip correctly.rrrencodingUTF-8ru)rqr`rrr)rrvrrrxs r6rr%s#;E#B#B L.-e44K+  @ A ' ' ' 'i!!!!$g."""   r5ct|tr|St|tr|dSt|tr/tt t t |St|trtd|)t|trtj |r|St|S)a!Convert a column name (or level) to either a string or a recursive collection of strings. Parameters ---------- name : str or tuple Returns ------- value : str or tuple Examples -------- >>> name = 'foo' >>> _column_name_to_strings(name) 'foo' >>> name = ('foo', 'bar') >>> _column_name_to_strings(name) "('foo', 'bar')" >>> import pandas as pd >>> name = (1, pd.Timestamp('2017-02-01 00:00:00')) >>> _column_name_to_strings(name) "('1', '2017-02-01 00:00:00')" rz%Unsupported type for MultiIndex level) r@rZrdecodetuplemaprrr|rzrPr{rvs r6rr9s2$  D% {{6""" D% 54d;;<<=== D( # #?@@@ *T511bhtnn t99r5ct|j|j|vrt|jSd|S)zReturn the name of an index level or a default name if `index.name` is None or is already a column name. Parameters ---------- index : pandas.Index i : int Returns ------- name : str Nz__index_level_{:d}__)rvrrC)indexirs r6_index_level_namer`s< z%*L"@"@&uz222%,,Q///r5ct|||}|jjs4tdt |j|t |||Sg}g}|durt|jng}g}g}|D]} || } t| } tj | r"td| | | | d| | | t| g} g} t|D]\} }t!|| |} t#|tjjr|t)|}nA| || d| }| | | ||| z}|||| | |||fS)Nz Duplicate column names found: {}F-Sparse pandas data (column {}) not supported.)_resolve_columns_of_interestr is_unique ValueErrorrClist$_get_columns_to_convert_given_schema_get_index_level_valuesrrr is_sparser|rrZ enumeraterr@pd RangeIndex_get_range_index_descriptor)rschemarrrrrrconvert_fieldsrvrrindex_column_namesr index_levelr all_namess r6_get_columns_to_convertrssH*2vw??G :   . 5 5d2:6F6F G G   3BOOOL.<5-H-H)))  N - -h&t,,   % % N?FFtLLNN N !!#&&&d###D!!!!!#d)),,,,#L11 ( (; a>> {KN$= > > ,&/ <>) ' (.vd|| 555HHH- 0   % % N?FFtLLNN N T""!!#&&&e$$$D!!!  %  % %d + + +  $ $T * * *    $ $ $11I |\3E |-? QQs$ # C$?C$4A33A.C$#C$c|}||jjvr3t|r$t|t dd}|j|S)z_ Get the index level of a DataFrame given 'name' (column name in an arrow Schema). __index_level_)rr_is_generated_index_nameintriget_level_values)rrvkeys r6rrs^ C 28>!!&>t&D&D!$s+,,R/011 8 $ $S ) ))r5cn tj||S#t$rt|cYSwxYwN)rrr|rZrs r6 _level_namersI 4 4yys 44cdt|jtj|dtj|dtj|ddS)Nrangestartstopstep)kindrvrrr)rrvrget_rangeindex_attribute)rs r6rr sSEJ''5eWEE4UFCC4UFCC   r5cxttdg}fdt|DS)Nrc:g|]}|Sr4)r)rrrs r6rz+_get_index_level_values..s' 8 8 8!E " "1 % % 8 8 8r5)rirhr)rns` r6rrs> GE8eW - -..A 8 8 8 8uQxx 8 8 88r5cr||td||j}n|fd|D}nj}|S)NzJSchema and columns arguments are mutually exclusive, pass only one of themc&g|] }|jv |Sr4)r)rcrs r6rz0_resolve_columns_of_interest.. s 999bj1r5)rrr)rrrs` r6rrsa g1<== =  ,  9999g999* Nr5c t|d||\}}}}}}} }g} | D]} | j} tj| rt j| dj} ntj| rZt| tj j r| dn | dd}t j|dj} nVt| | j d\} } tj| | } | t j| dj} | | t#| |||||| |}|| |fS)NT) from_pandasrr)rvaluesris_categoricalrarrayrXis_extension_array_dtyper@rSeriesheadget_datetimetz_typerWr_ndarray_to_arrow_typerr)rrrrrr_rrrrHrrtype_r ros r6dataframe_to_typesr'st ""dNG D DY E     %f - - ;HQD1116EE  1& 9 9 ;!+;>("*"*5AFF1III/0!u HU555:EE/FFMFEF11&%@@E}555: U!B m=N2DH eX %%r5Tc "t||||\}}}} } } } } |Ht|t|j}}||dzkr|dkrtj}nd}t sd}fd"d}|dkr"fdt | | D}ng}tj|5}t | | D]_\}}||j r | "||5| | "||` dddn #1swxYwYt|D]6\}}t|tjr|||<7d|D}|Tg}t ||D]-\}}| tj||.tj|}t%| ||| | |||}|jrt)|jn t+}||||}d}t|dkrn | dd }|d krH| dd }| dd } | dd }!tt1|| |!}n#t2$rYnwxYw|||fS)Ndrc|d}d}n|j}|j} tj||d}n_#tjtjtjf$r6}|xjd|j |j fz c_|d}~wwxYw|s@|j dkr5tdt||j |S)NT)rXrsafez0Conversion failed for column {!s} with type {!s}rz>Field {} was non-nullable but pandas column had {} null values)nullablerXrr ArrowInvalidArrowNotImplementedErrorArrowTypeErrorargsrCrvrW null_countrrZ)rrfield_nullablerr_ers r6convert_columnz+dataframe_to_arrays..convert_columncs =!NEE"^NJE Xc4dKKKFF+!#    FFIsx336 6FFG    M&"3a"7"7228&U9?9J3L3LMM M s1&B 1BB ct|tjo/|jjo#t |jjtjSr) r@rPndarrayflags contiguous issubclassrWrXinteger)arrs r6_can_definitely_zero_copyz6dataframe_to_arrays.._can_definitely_zero_copyys;3 ++7 $739>2:66 8r5c.g|]\}}||Sr4r4)rrfr s r6rz'dataframe_to_arrays..s?GGGa!.A&&GGGr5cg|] }|j Sr4rX)rxs r6rz'dataframe_to_arrays..s $ $ $QV $ $ $r5rrrrrrr)rrirr cpu_countr rrThreadPoolExecutorrrsubmitrr@Futurer_rrrrorrr with_metadatarr)#rrrnthreadsrrrrrrrrrrnrowsncolsrarraysexecutorrrr maybe_futrHfieldsrvrpandas_metadataron_rowsrrrrr s# ` @r6dataframe_to_arraysr'Js/r6>/688Y 2wwBJu 53;  5199|~~HHH  ! !,888 1}}GGGG!"4nEEGGG  ' 1 1 IX.?? I I1,,QX66IMM..A"6"67777MM(//.!Q"G"GHHHH  I I I I I I I I I I I I I I I I&f-- / /LAy)W^44 /%,,..q $ $V $ $ $E ~y%00 1 1KD% MM"(4// 0 0 0 06""(B m=N2DO-3OGx(((H OOO$$$  ! !( + +FF 6{{a $Q'/Dw)!,W5(+F3(+F3U5$5566    D  66 !!s&9A1D77D;>D;*AK KKc|jjtjkr||fSt j|r&|$|j}|j}tj ||}n|tj |j}||fSr) rWrXrPrVr is_datetimetzrGunitr timestampfrom_numpy_dtype)rrWrrGr*s r6rrsw |BM))u} ''2EM Xz T2&& #FL11 5=r5cddlmcm}|dd}|d}d|vr0tj||d|d}nd|vrtj|j \}} t||d} t j r6tj |d | d }n|}|r ||||j| } | Snnd |vrh|d }t#|dksJ||d} || } t%| dst'd| |}n|}|r|||S||fS)a Construct a pandas Block from the `item` dictionary coming from pyarrow's serialization or returned by arrow::python::ConvertTableToPandas. This function takes care of converting dictionary types to pandas categorical, Timestamp-with-timezones to the proper pandas Block, and conversion to pandas ExtensionBlock Parameters ---------- item : dict For basic types, this is a dictionary in the form of {'block': np.ndarray of values, 'placement': pandas block placement}. Additional keys are present for other types (dictionary, timezone, object). columns : Column names of the table being constructed, used for extension types extension_columns : dict Dictionary of {column_name: pandas_dtype} that includes all columns and corresponding dtypes that will be converted to a pandas ExtensionBlock. Returns ------- pandas Block rNblock placement dictionaryre)rjrerfrF)rWcopy)r/klassrWpy_arrayr__from_arrow__zGThis column does not support to be converted to a pandas ExtensionArray)r/)pandas.core.internalscore internalsgetrcategorical_type from_codesrP datetime_datarWmake_datetimetz is_ge_v21rrview make_blockDatetimeTZBlockrirYrr4)itemrextension_columns return_block_int block_arrr/rr*rrWr.rv pandas_dtypes r6_reconstruct_blockrGs8)(((((((($''I[!It*55 $|"4O6%% t  "9?33ad:&677  " " .&&w''u5'CCC  Y.2.B.3(55   t  :9~~""""y|$(. |%566 ;:;; ;))#..si888I~r5ctjrd}tj|}tj||S)NnsrG)ris_v1rrstring_to_tzinfodatetimetz_type)r*rGs r6r<r<sC   $ $B  &t 3 3 33r5Fc hg}g}|jj}|sa|_|d}|dg}|d}t||}t ||||\}} t |||||n7t j|j } t |g|||t|t|||} |j tj|||t!} t jr%ddlm} fd| D} | | | | }|Sddlm}dd lm}fd | D} | | g}|| |}t jr|||j}n ||}|S) Nrrrr)create_dataframe_from_blocksc6g|]}t|dS)F)rCrGrrArext_columns_dtypess r6rz&table_to_dataframe..-sG    l$6U L L L   r5)rr) BlockManager) DataFramec2g|]}t|Sr4rQrRs r6rz&table_to_dataframe..8s6    t\3E F F   r5)rr%r8_add_any_metadata_reconstruct_index_get_extension_dtypesrrrnum_rows'_check_data_column_metadata_consistency_deserialize_column_indexrrrtable_to_blocksrkeysis_ge_v3pandas.api.internalsrOr5rTpandasrUr= _from_mgraxes)optionstablerjignore_metadata types_mapper all_columnsrr%rrrr_rOblocksrrTrUrcmgrrrSs @@r6table_to_dataframerksNKNl2O   :%i0 (,,-=rBB+O<!%99)%1B*5|EE u2 ; gzCC))%.992 2|Wj  ,K888'{NKKG%L V # #GUJ$();)@)@)B)B$C$CEEFEEEEEE         * )&w O O O 666666$$$$$$        l64((  " " $$S#(33BB3B r5> r r rrrrr=rrrrrrc|d}|pg}i}tj|S|r(|jD] }|j}||} | | ||j<!|jD]\}|j}|j|vrJt |t jr0 |} | ||j<L#t$rYXwxYw]|D]} | d} n#t$r | d} YnwxYw| d} | |vr| tvrtj | } t | tjrt | tj jrV|s| |vr t j|j| jrn#t$rYnwxYwt%| dr| || <tjr|s|jD]}|j|vrt j|jsHt j|js$t j|jr;|j|vr2tj t.j||j<|S)a Based on the stored column pandas metadata and the extension types in the arrow schema, infer which columns should be converted to a pandas extension dtype. The 'numpy_type' field in the column metadata stores the string representation of the original pandas dtype (and, despite its name, not the 'pandas_type' field). Based on this string representation, a pandas/numpy dtype is constructed and then we can check if this dtype supports conversion from arrow. strings_to_categoricalNrwrvryr4)na_value)rextension_dtyperrXrvr@rBaseExtensionTypeto_pandas_dtypeNotImplementedErrorr?_pandas_supported_numpy_typesrFr StringDtyperH is_dictionaryrrYuses_string_dtype is_stringis_large_stringis_string_viewrPnan) recolumns_metadatargrdrjrm ext_columnsrtyprFcol_metarvrWs r6rYrYOs%%=>!rJK"*7\ 7 7E*C'<,,L'*6 EJ'77j :[ ( (ZR=Q-R-R ( 7"2244 +7 EJ'''     %55 $L)DD $ $ $F#DDD $& { " "u4Q'Q'Q'3E::L, (CDD 5lKN,FGG  .!1C1C 811%,2D2DT2J2J2OPP%$%#<)9::5(4K%$&&V/EV\ V VEz,,""5:..-8++EJ77-8**5:66-*J..*5.*D*Dbf*D*U*U EJ' s6;B B'&B'0B99C C;z:_check_data_column_metadata_consistency..sW  6d  0|q0JQvYd5Jr5)all)rhs r6r[r[sF   r5c|r d|Dfd|jD}n|j}t|dkrVtjjt ttj |d|D}n1tj ||r|ddnd}t|dkrt||}|S) Nc ni|]2}|dt|d|d3Srwrv)r8rrs r6 z-_deserialize_column_index..sI    EE, 7& B B C CQvY   r5c<g|]}||Sr4r8)rrvcolumns_name_dicts r6rz-_deserialize_column_index..s7   26  ! !$ - -   r5rcg|] }|d Srr4)r col_indexs r6rz-_deserialize_column_index..sEEE9V$EEEr5rrrvr) rrirr MultiIndex from_tuplesrrast literal_evalIndex"_reconstruct_columns_from_metadata) block_tablerhrcolumns_valuesrrs @r6r\r\s# 2          :E:R   %1 >Q.+77 S%~66 7 7EEnEEE8   .&& n!V!26!:!:RV'    >Q4WnMM Nr5cZd|D}g}g}|}|D]}t|trt|||||\}} } | 1n|ddkr_|d} tj|d|d|d| } t | t |krn(td |d| | | | tj} t |d kr| j || } nht |d kr;|d } t| | j s| | |d  } n| |j } || fS)NcHi|]}|d|d| Srrrs r6rz&_reconstruct_index..s<  lAfI&&r5rrrvrrr)rrvzUnrecognized index kind: {}rrrr)r@rZ_extract_index_levelrrrrirrCrr from_arraysrrZ) rerrhrgfield_name_to_metadata index_arrays index_names result_tablerr index_namerrs r6rXrXs LKL"'' eS ! ! 54H|U,BL5R5R 1L+z"#6]g % %vJ%.33E'N49&M9>v9C4EEK;3u::--.:$fU6]3355 5K(((:&&&& B <1 )),k)JJ \  a  Q%** 9HHUQH88E en--  r5cP||d}t||}|j|}|dkr|ddfS||}||} d| _||j|}|| |fS)Nrv)rg) _backwards_compatible_index_namerget_field_indexrm to_pandasrv remove_column) rerrwrrg logical_namerrrrs r6rrs)*5f=L1*lKKJ $$Z00ABwwT4'' ,,q//C--\-::KK--++J77L j 00r5c4||krt|rdS|S)a1Compute the name of an index column that is compatible with older versions of :mod:`pyarrow`. Parameters ---------- raw_name : str logical_name : str Returns ------- result : str Notes ----- * Part of :func:`~pyarrow.pandas_compat.table_to_blockmanager` N)r)raw_namers r6rrs($<$.{s^ E9  mS-=-=>> |T * * ,r5) fillvaluerrr:rrorfT)utcr<c6g|]}tj|Sr4)r<Decimal)rrs r6rz6_reconstruct_columns_from_metadata..s")L)L)L'/!*<*<)L)L)Lr5rZr=r)rrNrr)rWrv)rrrhroperator methodcallerrrPrSrrrrL to_datetime tz_convertr_as_unitr;rrWrastyperirrrv) rrrrlabels levels_dtypes new_levelsencoderrrF numpy_dtyperWrGs r6rr^s, BWh - - :'F Wgt , , 6F!, Nb! ! ! MJ#Hg66G,9'!'!(|[*<88 BI  IIg&&EE \ ) )((q!*-j9;;BNN5dN33>>rBBE#%% B b&6u&=&=a&@AA Y & &N(()L)Le)L)L)LMMEE K5 [H%<%<L((L.s6222"7C002W222r5rrwrvNonerrxr:rorfrIrJrr)r)rrirr8rr@rXrrrFrGrr+ArrayrrrvrrTabler)rer%modified_columnsmodified_fieldsrrn_index_levels n_columnsrr~ridxrro metadata_tz converted tz_aware_typerrr$s r6rWrWsO \F#O4M22M222M''NOI.//.@I!!;<<:: 8<< -- "'HI~~(Y7!$$X.. "99 &,66Cj!#(BF,@AA#J/&ll:66 :;#(+#=#= # I$&L+$F$F$FM$&H$8$8>K%9%M%MM,.8F3K4D4A,C,COC(,9$S) q  s5<(()) / /A$$$/2333 oa01111uQx((( el1o....x##GBIf4E4E#FFF r5ctj|}|jdj|}|S)zB Make a datetime64 Series timezone-aware for the given tz r)rrrLdt tz_localizer)seriesrGs r6 make_tz_awarersC   $ $Bi##E** 2 Mr5r)rNT)NNT)NFN)?rcollections.abcr concurrentrconcurrent.futures.threadr1rr< itertoolsrrrrrnumpyrP ImportErrorrr pyarrow.librrr rrOrr7rDrTr`rqrrrrrrrrrrrrrr'rrGr<rkrsrYr[r\rXrrrrrrrWrr4r5r6rs& $$$$$$! !!!!!!   BBBDDDDDDDDDD:"###,"$$$&---d+/\\\\~($$$N000&?Q?Q?QD;Q;Q;Q| * * *999    & & & &FIM!b"b"b"b"J   &BBBBJ444JN4444r!!!PPPfB3333n?C1111&0/// $$$&%%%.TUTUTUn:::Bs;AA