
    -PhN                       U d dl mZ d dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
mZmZ d dlmZ d dlmZmZmZmZmZ d dlmZmZmZmZmZmZmZ d d	lmZ d d
lm Z  ergd dl!m"Z" d dl#Z$d dl%m&Z& d dl'm(Z( d dl)m*Z* d dl+m,Z,m-Z-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6 d dlm7Z7m8Z8 d dl9m:Z: d dl;m<Z< d dlm=Z=m>Z> e(e	e	e	e	e	f         Z?e	Z@deAd<   	  G d de d                   ZBdS )    )annotationsN)reduce)and_)TYPE_CHECKINGAnyIteratorMappingSequenceis_native_spark_like)evaluate_exprsimport_functionsimport_native_dtypesimport_windownative_to_narwhals_dtype)Implementationfind_stacklevelgenerate_temporary_column_namenot_implementedparse_columns_to_dropparse_versionvalidate_backend_version)InvalidOperationError)CompliantLazyFrame)
ModuleType)Column)BaseDataFrameWindow)Self	TypeAliasTypeIs)CompliantDataFrameAny)SparkLikeExprSparkLikeLazyGroupBySparkLikeNamespace)Version_FullContext)	LazyFrame)DType)JoinStrategyLazyUniqueKeepStrategyr!   
Incompletec                     e Zd ZdidZed             Zed             Zedjd            Zedkd            Z	e
dld            ZdmdZdndZdodZdpdZdqd Zdrd"Zdsd$Zdtd&Zdud(Zedvd*            Zdwd0Zdxd3Zdyd6Zdyd7Zdyd8Zdzd:Zed{d<            Zd{d=Zd|dBZd}dEZd~dJZ ddOZ!ddRZ"ddUZ#ddXZ$dd_Z%dd`Z&ddeZ' e(j)        df          Z* e(            Z+ e(j)        dg          Z, e(            Z-dhS )SparkLikeLazyFramenative_dataframeSQLFrameDataFramebackend_versiontuple[int, ...]versionr)   implementationr   returnNonec                   || _         || _        || _        || _        d | _        d | _        t          | j        | j                   d S N)_native_frame_backend_version_implementation_version_cached_schema_cached_columnsr   )selfr2   r4   r6   r7   s        ^/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/narwhals/_spark_like/dataframe.py__init__zSparkLikeLazyFrame.__init__9   sP     1A /-7;15 !5t7LMMMMM    c                H    t           rddlm} |S t          | j                  S )Nr   )	functions)r   sqlframe.baserG   r   r>   )rB   rG   s     rC   _FzSparkLikeLazyFrame._FI   s3     	://////#D$8999rE   c                H    t           rddlm} |S t          | j                  S )Nr   )types)r   rH   rK   r   r>   )rB   rK   s     rC   _native_dtypesz!SparkLikeLazyFrame._native_dtypesR   s2     	>++++++L'(<===rE   type[Window]c                H    t           rddlm} |S t          | j                  S )Nr   r   )r   sqlframe.base.windowr   r   r>   )rB   r   s     rC   _WindowzSparkLikeLazyFrame._Window[   s2     	7333333M !5666rE   objSQLFrameDataFrame | AnyTypeIs[SQLFrameDataFrame]c                     t          |           S r;   r   )rQ   s    rC   
_is_nativezSparkLikeLazyFrame._is_natived   s    #C(((rE   datacontextr*   r    c              >     | ||j         |j        |j                  S Nr4   r6   r7   )r=   r?   r>   )clsrV   rW   s      rC   from_nativezSparkLikeLazyFrame.from_nativeh   s0    s#4$"2	
 
 
 	
rE   LazyFrame[SQLFrameDataFrame]c                :    | j                             | d          S )Nlazy)level)r?   	lazyframerB   s    rC   to_narwhalszSparkLikeLazyFrame.to_narwhalsq   s    }&&t6&:::rE   r   c                4    | j                                         S r;   )r>   to_native_namespacerb   s    rC   __native_namespace__z'SparkLikeLazyFrame.__native_namespace__t   s    #77999rE   r(   c                H    ddl m}  || j        | j        | j                  S )Nr   r'   rZ   )narwhals._spark_like.namespacer(   r=   r?   r>   )rB   r(   s     rC   __narwhals_namespace__z)SparkLikeLazyFrame.__narwhals_namespace__w   s?    EEEEEE!! 1M/
 
 
 	
rE   c                    | S r;    rb   s    rC   __narwhals_lazyframe__z)SparkLikeLazyFrame.__narwhals_lazyframe__   s    rE   c                R    |                      | j        | j        || j                  S rY   )	__class__nativer=   r>   )rB   r6   s     rC   _with_versionz SparkLikeLazyFrame._with_version   s1    ~~K 1/	  
 
 	
rE   dfc                R    |                      || j        | j        | j                  S rY   )rn   r=   r?   r>   )rB   rq   s     rC   _with_nativezSparkLikeLazyFrame._with_native   s1    ~~ 1M/	  
 
 	
rE   	pa.Schemac                   dd l }ddlm} g }|                                 }| j        j        }|                                D ]\  }}	  ||| j                  }|                    ||f           /# t          $ r}	||         j
        }
| j        j        }t          |
|          s(t          j        d|
 d|	t!                                 |                    | |j                    f           Y d }	~	d }	~	ww xY w |j        |          S )Nr   )narwhals_to_native_dtypezCould not convert dtype z to PyArrow dtype, )
stacklevel)pyarrownarwhals._arrow.utilsrv   collect_schemaro   schemaitemsr?   append	ExceptiondataTyperL   NullType
isinstancewarningswarnr   null)rB   parv   r{   	nw_schemanative_schemakeyvaluenative_dtypeexcnative_spark_dtype	null_types               rC   _to_arrow_schemaz#SparkLikeLazyFrame._to_arrow_schema   s\   BBBBBB02''))	*#//++ 	3 	3JC377t}MM sL12222  0 0 0%23%7%@" !/8	!"4i@@ Ma3EaaZ]aa#2#4#4    sGBGII.////////0 ry   s   A00
C9:A5C44C9pa.Tablec                   | j                                         r| j        dk     rdd l}	 |j                            | j                                                  S # t          $ r]}dt          |          v rFd | j
        D             }|                                 }|j                            ||          cY d }~S  d }~ww xY w| j                                         rV| j        dk     rKdd l}|                                 }|j                            | j                                        |          S | j                                        S )N)   r   zat least one RecordBatchc                    i | ]}|g S rk   rk   ).0ks     rC   
<dictcomp>z8SparkLikeLazyFrame._collect_to_arrow.<locals>.<dictcomp>   s    1N1N1NA!R1N1N1NrE   r{   )r>   
is_pysparkr=   rx   Tablefrom_batchesro   _collect_as_arrow
ValueErrorstrcolumnsr   from_pydictis_pyspark_connectfrom_pandastoPandastoArrow)rB   r   r   rV   	pa_schemas        rC   _collect_to_arrowz$SparkLikeLazyFrame._collect_to_arrow   sb   **,, 	)1F1M1M    
x,,T[-J-J-L-LMMM   -S99 2O1N1N1N1ND $ 5 5 7 7I8//Y/GGGGGGGG !4466 	)4;PSW;W;W    --//I8''(<(<(>(>y'QQQ;&&(((s$   0A 
C%AB=6C<B==CIterator[Column]c              #  V   K   | j         D ]}| j                            |          V  d S r;   )r   rI   col)rB   r   s     rC   _iter_columnsz SparkLikeLazyFrame._iter_columns   s>      < 	# 	#C'++c""""""	# 	#rE   	list[str]c                v    | j         ,| j        t          | j                  n| j        j        | _         | j         S r;   )rA   r@   listr{   ro   r   rb   s    rC   r   zSparkLikeLazyFrame.columns   sC    ' &2 T[!!![(  
 ##rE   backend(ModuleType | Implementation | str | Nonekwargsr   r#   c                <   |t           j        u rMdd l}ddlm}  || j                                        t           j        t          |          | j        d          S ||t           j	        u r=dd l
}ddlm}  ||                                 t          |          | j        d          S |t           j        u rSdd l}dd l
}ddlm}  ||                    |                                           t          |          | j                  S d	| }	t'          |	          )
Nr   )PandasLikeDataFrameT)r7   r4   r6   validate_column_names)ArrowDataFrame)r4   r6   r   )PolarsDataFrame)r4   r6   zUnsupported `backend` value: )r   PANDASpandasnarwhals._pandas_like.dataframer   ro   r   r   r?   PYARROWrx   narwhals._arrow.dataframer   r   POLARSpolarsnarwhals._polars.dataframer   
from_arrowr   )
rB   r   r   pdr   r   r   plr   msgs
             rC   collectzSparkLikeLazyFrame.collect   sz    n+++KKKKKK&&$$&&-4 -b 1 1&*    _>+A A A    @@@@@@!>&&(( -b 1 1&*	    ---    BBBBBB"?d446677 -b 1 1    8g77oorE   column_namesr   c                F    |                       | j        j        |           S r;   )rs   ro   select)rB   r   s     rC   simple_selectz SparkLikeLazyFrame.simple_select   s#      !3!3\!BCCCrE   exprsr$   c                x    t          | g|R  }d |D             }|                      | j        j        |           S )Nc                >    g | ]\  }}|                     |          S rk   aliasr   col_namer   s      rC   
<listcomp>z0SparkLikeLazyFrame.aggregate.<locals>.<listcomp>  s(    QQQMHcCIIh//QQQrE   )r   rs   ro   aggrB   r   new_columnsnew_columns_lists       rC   	aggregatezSparkLikeLazyFrame.aggregate  sK    $T2E222QQ[QQQ  2B!CDDDrE   c                x    t          | g|R  }d |D             }|                      | j        j        |           S )Nc                >    g | ]\  }}|                     |          S rk   r   r   s      rC   r   z-SparkLikeLazyFrame.select.<locals>.<listcomp>	  s(    SSSOXsCIIh//SSSrE   )r   rs   ro   r   r   s       rC   r   zSparkLikeLazyFrame.select  sM    $T2E222SS{SSS  !3!35E!FGGGrE   c                    t          | g|R  }|                     | j                            t	          |                              S r;   )r   rs   ro   withColumnsdict)rB   r   r   s      rC   with_columnszSparkLikeLazyFrame.with_columns  sC    $T2E222  !8!8k9J9J!K!KLLLrE   	predicatec                    |                     |           d         }| j                            |          }|                     |          S Nr   )_callro   wherers   )rB   r   	conditionspark_dfs       rC   filterzSparkLikeLazyFrame.filter  sA    OOD))!,	;$$Y//  ***rE   dict[str, DType]c                Z      j          fd j        j        D              _          j         S )Nc           	     r    i | ]3}|j         t          |j        j        j        j        j                  4S rk   )namer   r   r?   rL   ro   sparkSession)r   fieldrB   s     rC   r   z-SparkLikeLazyFrame.schema.<locals>.<dictcomp>  sR     # # #  
4NM'K,	 # # #rE   )r@   ro   r{   rb   s   `rC   r{   zSparkLikeLazyFrame.schema  sI    &# # # # "[/# # #D ""rE   c                    | j         S r;   r   rb   s    rC   rz   z!SparkLikeLazyFrame.collect_schema$  s
    {rE   r   Sequence[str]strictboolc               j    t          | ||          }|                      | j        j        |           S )N)r   )r   rs   ro   drop)rB   r   r   columns_to_drops       rC   r   zSparkLikeLazyFrame.drop'  s7    /gfMMM  !1!1?!CDDDrE   nintc                \    |                      | j                            |                    S r;   )rs   ro   limit)rB   r   s     rC   headzSparkLikeLazyFrame.head+  s&      !2!21!5!5666rE   keys'Sequence[str] | Sequence[SparkLikeExpr]drop_null_keysr&   c               *    ddl m}  || ||          S )Nr   r%   )r   )narwhals._spark_like.group_byr&   )rB   r   r   r&   s       rC   group_byzSparkLikeLazyFrame.group_by.  s/     	GFFFFF##D$~NNNNrE   by
descendingbool | Sequence[bool]
nulls_lastc               
    t          |t                    r|gt          |          z  }|r fd|D             }n fd|D             }d t          ||          D             }                       j        j        |           S )Nc              3  L   K   | ]}|rj         j        nj         j        V  d S r;   )rI   desc_nulls_lastasc_nulls_lastr   drB   s     rC   	<genexpr>z*SparkLikeLazyFrame.sort.<locals>.<genexpr>:  sK         ,-H''$'2H     rE   c              3  L   K   | ]}|rj         j        nj         j        V  d S r;   )rI   desc_nulls_firstasc_nulls_firstr   s     rC   r  z*SparkLikeLazyFrame.sort.<locals>.<genexpr>?  sK         -.J((473J     rE   c                *    g | ]\  }} ||          S rk   rk   )r   r   sort_fs      rC   r   z+SparkLikeLazyFrame.sort.<locals>.<listcomp>D  s$    HHH[S&VVC[[HHHrE   )r   r   lenziprs   ro   sort)rB   r   r   r   
sort_funcs	sort_colss   `     rC   r
  zSparkLikeLazyFrame.sort5  s    j$'' 	0$B/J 		   #  JJ
   #  J
 IHCJ4G4GHHH	  !1!19!=>>>rE   subsetSequence[str] | Nonec                    |rt          |          nd }|                     | j                            |                    S )Nr  )r   rs   ro   dropna)rB   r  s     rC   
drop_nullszSparkLikeLazyFrame.drop_nullsG  s=    !'1fT  !3!36!3!B!BCCCrE   mappingMapping[str, str]c                     fd j         D             }                      j                             fd|                                D                                 S )Nc                >    i | ]}|                     ||          S rk   )get)r   colnamer  s     rC   r   z-SparkLikeLazyFrame.rename.<locals>.<dictcomp>L  s6     
 
 
7>GW[['22
 
 
rE   c                p    g | ]2\  }}j                             |                              |          3S rk   rI   r   r   r   oldnewrB   s      rC   r   z-SparkLikeLazyFrame.rename.<locals>.<listcomp>Q  s9    TTTcS!!'',,TTTrE   )r   rs   ro   r   r|   )rB   r  rename_mappings   `` rC   renamezSparkLikeLazyFrame.renameK  s    
 
 
 
BF,
 
 
   KTTTT^=Q=Q=S=STTT 
 
 	
rE   keepr.   c                  |r|                      |          x}r||rt          |          nd }|dk    rt          d| j                  }| j                            |p| j                  }| j                            || j        	                    d          
                    |                                        | j                            |          | j                            d          k                                  | j                            |                    }|                     |          S |                     | j                            |                    S )Nnone   *   r  )_check_columns_existr   r   r   rP   partitionByro   
withColumnrI   countoverr   r   litr   rs   dropDuplicates)rB   r  r   errortmpwindowrq   s          rC   uniquezSparkLikeLazyFrame.uniqueU  s/     	 9 9& A AAu 	K!'1fT6>>0DLAAC\--f.DEEF&&sDGMM#,>,>,C,CF,K,KLLC((DGKKNN:;;dgkk#&&'' 
 $$R(((  !;!;6!;!J!JKKKrE   otherhowr-   left_onright_onsuffixc                t     j         |j         }|t          |          ng |t          |          ng }|dk    rfd|D             n|}i t          t          |                    fd|D             |j                             fd                                D                                                       }	|dv r#|	                    fd|D                        n-|dk    r'|	                    	                                           fdD             }
|dk    r0t          t           fdt          ||
          D                       n	|d	k    rd n|}|dk    rd
n|}                      j                            ||                              |	                    S )Nfullc                    g | ]}|v|	S rk   rk   )r   c	right_on_s     rC   r   z+SparkLikeLazyFrame.join.<locals>.<listcomp>x  s#    <<<1)););Q);););rE   c                *    i | ]}||v r|  n|S rk   rk   )r   r  left_columnsr5  s     rC   r   z+SparkLikeLazyFrame.join.<locals>.<dictcomp>  sE        L1H1HG-V---g  rE   c                p    g | ]2\  }}j                             |                              |          3S rk   r  r  s      rC   r   z+SparkLikeLazyFrame.join.<locals>.<listcomp>  s9    PPPXS#TW[[##C((PPPrE   >   leftcrossinnerc              3  0   K   | ]}|v|         V  d S r;   rk   )r   r  r  r:  s     rC   r  z*SparkLikeLazyFrame.join.<locals>.<genexpr>  s?        )++ w'++++ rE   c                     g | ]
}|         S rk   rk   )r   r9  r  s     rC   r   z+SparkLikeLazyFrame.join.<locals>.<listcomp>  s    BBB1^A.BBBrE   c              3  l   K   | ].\  }}t          j        |          t          |          k    V  /d S r;   )getattrro   )r   left_key	right_keyother_nativerB   s      rC   r  z*SparkLikeLazyFrame.join.<locals>.<genexpr>  sV        +) DK22glI6V6VV     rE   r?  
full_outer)onr2  )r   r   r   r	  ro   r   r|   copyextendvaluesr   r   rs   join)rB   r1  r2  r3  r4  r5  right_columnsleft_on_right_cols_to_rename	col_orderright_on_remappedon_
how_nativer<  rG  r  r:  s   `    `       @@@@rC   rM  zSparkLikeLazyFrame.joinf  sm    |191EtH~~~2	/6/Bd7mmm f}} =<<<<<<< 	
3y(++,,
    3  
 |**PPPP9M9M9O9OPPP
 
 !%%''	,,,     ,      
 F]]^2244555BBBB	BBB f}}     /28=N/O/O     g~~  	 &)F]]\\
  K\czBBII)TT
 
 	
rE   c                     j         j        }                                 }D ]*}||         }||j        k    rd| d}t	          |          + j        }t                    dk    rd}t          |           j        	                                s j        
                                r/                       j        j         fd|D                        S  j                                        rd fd
}                       j        j         fd|D                                    j                             |d                             j         fd|D                                  S d}t#          |          )Nz-`explode` operation not supported for dtype `z`, expected List typer%  zExploding on multiple columns is not supported with SparkLike backend since we cannot guarantee that the exploded columns have matching element counts.c                    g | ]h}|d          k    r-j                             |                              |          n,j                             |                              |          iS r   )rI   r   r   explode_outerr   r   r   rB   s     rC   r   z.SparkLikeLazyFrame.explode.<locals>.<listcomp>  s|        % $wqz11 H--33H===!W228<<BB8LL  rE   r   r   r8   r   c                v    j                             |           j                             |           dk    z  S r   )rI   isnull
array_size)r   rB   s    rC   null_conditionz2SparkLikeLazyFrame.explode.<locals>.null_condition  s2    w~~h//473E3Eh3O3OST3TUUrE   c                    g | ]h}|d          k    r-j                             |                              |          n,j                             |                              |          iS rW  )rI   r   r   exploderY  s     rC   r   z.SparkLikeLazyFrame.explode.<locals>.<listcomp>  sz        % $wqz11 H--33H===!W__X66<<XFF  rE   r   c                    g | ]h}|d          k    r-j                             |                              |          n,j                             d                              |          iS )r   N)rI   r   r   r+  rY  s     rC   r   z.SparkLikeLazyFrame.explode.<locals>.<listcomp>  sz        !)  (71:55 !GKK1177AAA!%T!2!2!8!8!B!B  rE   z[Unreachable code, please report an issue at https://github.com/narwhals-dev/narwhals/issues)r   r   r8   r   )r?   dtypesrz   Listr   r   r  NotImplementedErrorr>   r   r   rs   ro   r   is_sqlframeunionr   AssertionError)	rB   r   ra  r{   col_to_explodedtyper   r   r]  s	   ``       rC   r_  zSparkLikeLazyFrame.explode  s=   %$$&&% 	1 	1N>*E##)E ) ) )  ,C000 $ |w<<1^  &c*****,, *	&0D0W0W0Y0Y *	&$$""     )5	  	 	 	 !--// 	&V V V V V V $$""     )5	   %IDK&&~~gaj'A'ABBI     -9	  	 	  ( pC %%%rE   rI  indexvariable_name
value_namec                   | j                                         r/|dk    rd}t          |          |dk    rd}t          |          n	 |rt          |          nd}|At          t	          | j                                      t	          |                              nt          |          }| j                            ||||          }|
 |j	        | }| 
                    |          S )N z<`variable_name` cannot be empty string for sqlframe backend.z9`value_name` cannot be empty string for sqlframe backend.rk   )idsrL  variableColumnNamevalueColumnName)r>   rd  rc  tuplesetr   
differencero   unpivotr   rs   )	rB   rI  ri  rj  rk  r   rn  rL  unpivoted_native_frames	            rC   rt  zSparkLikeLazyFrame.unpivot  s    ++-- 		""T)#...RQ)#...   #+eElll=?ZE#dl##..s3xx88999USUYY 	 "&!4!4,&	 "5 "
 "
 =%@%;%@#%F"  !7888rE   zO`LazyFrame.gather_every` is deprecated and will be removed in a future version.zG`LazyFrame.tail` is deprecated and will be removed in a future version.N)
r2   r3   r4   r5   r6   r)   r7   r   r8   r9   )r8   rM   )rQ   rR   r8   rS   )rV   r3   rW   r*   r8   r    )r8   r]   )r8   r   )r8   r(   )r8   r    )r6   r)   r8   r    )rq   r3   r8   r    )r8   rt   )r8   r   )r8   r   )r8   r   )r   r   r   r   r8   r#   )r   r   r8   r    )r   r$   r8   r    )r   r$   r8   r    )r8   r   )r   r   r   r   r8   r    )r   r   r8   r    )r   r   r   r   r8   r&   )r   r   r   r   r   r   r8   r    )r  r  r8   r    )r  r  r8   r    )r  r  r   r.   r8   r    )r1  r    r2  r-   r3  r  r4  r  r5  r   r8   r    )r   r   r8   r    )
rI  r  ri  r  rj  r   rk  r   r8   r    ).__name__
__module____qualname__rD   propertyrI   rL   rP   staticmethodrU   classmethodr\   rc   rf   ri   rl   rp   rs   r   r   r   r   r   r   r   r   r   r   r{   rz   r   r   r   r
  r  r  r0  rM  r_  rt  r   
deprecatedgather_every	join_asoftailwith_row_indexrk   rE   rC   r1   r1   4   s~       
N N N N  : : X: > > X> 7 7 7 X7 ) ) ) \) 
 
 
 [
; ; ; ;: : : :
 
 
 
   
 
 
 

 
 
 
! ! ! !6) ) ) ).# # # # $ $ $ X$) ) ) )VD D D DE E E EH H H H
M M M M+ + + + # # # X#   E E E E7 7 7 7O O O O? ? ? ?$D D D D
 
 
 
L L L L"A
 A
 A
 A
FA& A& A& A&F9 9 9 9@ .?-Y L  !!I%?%Q D %_&&NNNrE   r1   )r$   r3   r]   )C
__future__r   r   	functoolsr   operatorr   typingr   r   r   r	   r
   narwhals._namespacer   narwhals._spark_like.utilsr   r   r   r   r   narwhals._utilsr   r   r   r   r   r   r   narwhals.exceptionsr   narwhals.typingr   rK   r   rx   r   sqlframe.base.columnr   sqlframe.base.dataframer   rO   r   typing_extensionsr    r!   r"   narwhals._compliant.typingr#   narwhals._spark_like.exprr$   r   r&   rh   r(   r)   r*   narwhals.dataframer+   narwhals.dtypesr,   r-   r.   r3   r/   __annotations__r1   rk   rE   rC   <module>r     s   " " " " " " "              B B B B B B B B B B B B B B 4 4 4 4 4 4                               6 5 5 5 5 5 . . . . . . ?      ++++++555555++++++9999999999@@@@@@777777BBBBBBAAAAAA55555555,,,,,,%%%%%%DDDDDDDD%c3S#&=>
     7_' _' _' _' _'L_' _' _' _' _'rE   