
    q-Ph                         d dl mZ d dlZd dlmZ d dlmZmZmZ d dl	m
Z d dlmZmZ d dlmZ d dlmZ erd dlZd dlmZ d d	lmZ  G d
 d          ZddZdS )    )annotationsN)partial)TYPE_CHECKINGAnyLiteral)eprintverbose)ComputeError)_scan_pyarrow_dataset_impl)Table)	LazyFramec                  b    e Zd ZdZdddd dZd!dZd"dZdddd#dZd!dZd$dZ	d%dZ
d&dZdS )'IcebergDatasetz Dataset interface for PyIceberg.N)snapshot_idiceberg_storage_propertiessourcestr | Tabler   
int | Noner   dict[str, Any] | Nonereader_override%Literal['native', 'pyiceberg'] | NonereturnNonec                   d | _         d | _        || _        || _        || _        t          |t                    r	|| _         d S || _        d S )N)_metadata_path_table_snapshot_id_iceberg_storage_properties_reader_override
isinstancestr)selfr   r   r   r   s        Y/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/polars/io/iceberg/dataset.py__init__zIcebergDataset.__init__   sX     #'+E(GV
 fc"" 	!"(D DKKK    r!   c                    dS )zName of the reader.iceberg )r"   s    r#   reader_namezIcebergDataset.reader_name0   s    yr%   	pa.schemac                l    ddl m}  ||                                                                           S )zFetch the schema of the table.r   schema_to_pyarrow)pyiceberg.io.pyarrowr-   tableschema)r"   r-   s     r#   r0   zIcebergDataset.schema4   s9    ::::::  !4!4!6!6777r%   )limit
projectionr1   r2   list[str] | Noner   c               B   ddl }|j        j                                        }|rt	          d| d|            |                                 }|dnt          |          }| j        }|)|                    |          d| }t          |          | j
        pt          j        d          }	|	r|	dvrd	|	 d
}t          |          |	dk    rdn	|	dk    rdnd}
g }|	dk    ry|                    |||          }|                                D ]L}|j        j        dk    rd|j        j         }
n*|j        rd}
n |                    |j        j                   L |
s5ddlm} |r t	          dt-          |           d            ||d          S |	dk    rd|
 }t/          |          |rt	          d|
            t1          t2          ||||          }ddlm}  ||                                          }t:          j                            ||d          }|S )zConstruct a LazyFrame scan.r   Nz*IcebergDataset: to_dataset_scan(): limit: z, projection: )*ziceberg snapshot ID not found: POLARS_ICEBERG_READER_OVERRIDE)native	pyicebergz-iceberg: unknown value for reader_override: 'z*', expected one of ('native', 'pyiceberg')r8   z"forced reader_override='pyiceberg'r7   z native scans disabled by default)r   r1   selected_fieldsPARQUETznon-parquet format: z-unimplemented: dataset contained delete files)scan_parquetz:IcebergDataset: to_dataset_scan(): native scan_parquet() (z	 sources)T)allow_missing_columnsz)iceberg reader_override='native' failed: zGIcebergDataset: to_dataset_scan(): fallback to python[pyiceberg] scan: )r   n_rowswith_columnsr,   )pyarrow) polars._utils.logging_utilsloggingr	   r   r/   tupler   snapshot_by_id
ValueErrorr   osgetenvscan
plan_filesfilefile_formatdelete_filesappend	file_pathpolars.io.parquet.functionsr;   lenr
   r   r   r.   r-   r0   plr   _scan_python_function)r"   r1   r2   polarsr	   tblr9   r   msgr   fallback_reasonsourcesrH   	file_infor;   funcr-   arrow_schemalfs                      r#   to_dataset_scanzIcebergDataset.to_dataset_scan:   s7    	%$$$-'//11 	,, ,), ,   jjll$.$6&&E*<M<M'"!!+..6EEE oo% / 
29,4
 4
  	"6MMMP#P P P  S//!
 +-- 10
 (** 43 	 k))88'uo   D "__..  	>-::Ky~/IKK $O + &UOONN9>#;<<< 	$@@@@@@ F.1'llF F F  
  <&*   
 ((OoOOCs### 	I7FI I  
 &##
 
 
 	;:::::((66\//dD/QQ	r%   c                    | j         6| j        d}t          |          |                                 j        | _         | j         S )zFetch the metadata path.N1impl error: both metadata_path and table are None)r   r   rE   r/   metadata_location)r"   rU   s     r#   metadata_pathzIcebergDataset.metadata_path   s@    &{"I oo%"&**,,"@D""r%   r   c                    | j         k| j        d}t          |          t                      rt	          d| j                   ddlm} |                    | j        | j        pi           | _         | j         S )z!Fetch the PyIceberg Table object.Nr^   z;IcebergDataset: construct table from self._metadata_path = r   )StaticTable)r_   
properties)	r   r   rE   r	   r   pyiceberg.tablerb   from_metadatar   )r"   rU   rb   s      r#   r/   zIcebergDataset.table   s    ;"*I oo%yy YWt?RWWXXX333333%33"&"5;Ar 4  DK
 {r%   dict[str, Any]c           
        |                                  | j        | j        | j        d}t	                      rH|d         }|d         }t          |d                   }|d         }t          d| d| d| d	|            |S )
N)r`   r   r   r   r`   r   r   r   #IcebergDataset: getstate(): path: '', snapshot_id: '', iceberg_storage_properties: , reader_override: )r`   r   r   r   r	   _redact_dict_valuesr   r"   state	path_reprr   	keys_reprr   s         r#   __getstate__zIcebergDataset.__getstate__   s    !//11,*.*J#4	
 
 99 	o.I.K+E2N,OPPI#$56O6#6 6!,6 6 096 6 %4	6 6   r%   rn   c           
         t                      rH|d         }|d         }t          |d                   }|d         }t          d| d| d| d|            t                              | |d         |d         |d         |d         	           d S )
Nr`   r   r   r   rh   ri   rj   rk   )r   r   r   )r	   rl   r   r   r$   rm   s         r#   __setstate__zIcebergDataset.__setstate__   s    99 	o.I.K+E2N,OPPI#$56O6#6 6!,6 6 096 6 %4	6 6   	/"m,',-I'J!"34 	  	
 	
 	
 	
 	
r%   )
r   r   r   r   r   r   r   r   r   r   )r   r!   )r   r*   )r1   r   r2   r3   r   r   )r   r   )r   rf   )rn   rf   r   r   )__name__
__module____qualname____doc__r$   r)   r0   r\   r`   r/   rq   rs   r(   r%   r#   r   r      s        ** #'<@! ! ! ! ! !4   8 8 8 8 !'+	n n n n n nh	# 	# 	# 	#   4   0
 
 
 
 
 
r%   r   objr   r   c                    t          | t                    rd |                                 D             n| dt          |           j         dndS )Nc                    i | ]}|d S )REDACTEDr(   ).0ks     r#   
<dictcomp>z'_redact_dict_values.<locals>.<dictcomp>  s    +++1J+++r%   <z object>r   )r    dictkeystypert   )rx   s    r#   rl   rl     s]     c4  	++

++++ ? .c#----r%   )rx   r   r   r   )
__future__r   rF   	functoolsr   typingr   r   r   polars._reexport	_reexportrQ   r@   r   r	   polars.exceptionsr
   polars.io.iceberg._utilsr   r?   pard   r   polars.lazyframe.framer   r   rl   r(   r%   r#   <module>r      s%   " " " " " " 				       . . . . . . . . . .       1 1 1 1 1 1 1 1 * * * * * * ? ? ? ? ? ? 1%%%%%%000000m
 m
 m
 m
 m
 m
 m
 m
`     r%   