q-Ph[2ddlmZddlZddlmZmZddlmZddlm Z ddl m Z ddl m Z ddlmZdd lmZe r8eje5dd lmZdddn #1swxYwYdd lmZmZmZeje5dd lmZdddn #1swxYwYGd dZGddZGddZd#dZd$dZGdde Zd%dZ Gdd e Z!Gd!d"e Z"dS)&) annotationsN)MappingSequence)Path) TYPE_CHECKING)col)PartitioningScheme)issue_unstable_warning)Expr)PyExpr)IOAnyCallable)PyPartitioningcBeZdZUdZd d Zded<ded<ded<d d Zd S)KeyedPartitiona0 A key-value pair for a partition. .. warning:: This functionality is currently considered **unstable**. It may be changed at any point without it being considered a breaking change. See Also -------- PartitionByKey PartitionParted KeyedPartitionContext namestr str_value raw_valuerreturnNonec0||_||_||_dSNrrr)selfrrrs S/var/www/html/test/jupyter/venv/lib/python3.11/site-packages/polars/io/partition.py__init__zKeyedPartition.__init__&s ""c$|jd|jS)zGet the `key=value`.=)rr)rs r hive_namezKeyedPartition.hive_name/s)..dn...rN)rrrrrrrr)rr)__name__ __module__ __qualname____doc__r__annotations__r"rrrrse  #### IIINNNNNN//////rrc`eZdZUdZdd Zded<ded<ded<ded<d ed<d ed <ddZdS)KeyedPartitionContexta* Callback context for a partition creation using keys. .. warning:: This functionality is currently considered **unstable**. It may be changed at any point without it being considered a breaking change. See Also -------- PartitionByKey PartitionParted file_idxintpart_idx in_part_idxkeyslist[KeyedPartition] file_pathr full_pathrrcZ||_||_||_||_||_||_dSrr+r-r.r/r1r2)rr+r-r.r/r1r2s rrzKeyedPartitionContext.__init__Bs3!   & ""rct|jdksJt|jd}|jddD]&}|t|z}'|S)z$The keys mapped to hive directories.rN)lenr/rr")rpkeys r hive_dirszKeyedPartitionContext.hive_dirs[st49~~!!!! 1'')) * *9QRR= ' 'C cmmoo&& &AArN)r+r,r-r,r.r,r/r0r1rr2rrr)rr)r#r$r%r&rr'r:r(rrr*r*4s  #### MMMMMMOOOrr*c:eZdZUdZd d Zded<ded<ded<d S) BasePartitionContexta  Callback context for a partition creation. .. warning:: This functionality is currently considered **unstable**. It may be changed at any point without it being considered a breaking change. See Also -------- PartitionMaxSize r+r,r1rr2rrc0||_||_||_dSrr+r1r2)rr+r1r2s rrzBasePartitionContext.__init__qs  ""rN)r+r,r1rr2rrr)r#r$r%r&rr'r(rrr<r<ds\  #### MMMOOOrr< file_path_cbICallable[[BasePartitionContext], Path | str | IO[bytes] | IO[str]] | NonercdSfdS)Nc t|jt|jt|jS)Nr>)r<r+rr1r2ctxr?s rz)_cast_base_file_path_cb..sD||\3=))3=))   rr(r?s`r_cast_base_file_path_cbrG}s)t    rJCallable[[KeyedPartitionContext], Path | str | IO[bytes] | IO[str]] | NonecdSfdS)Nc t|j|j|jd|jDt |jt |jS)NcPg|]#}t|j|j|j$S)r)rrrr).0kvs r z>_cast_keyed_file_path_cb....sFBLBLrr4)r*r+r-r.r/rr1r2rCs rrEz*_cast_keyed_file_path_cb..sp||\\(  3=))3=)) rr(rFs`r_cast_keyed_file_path_cbrOs)t    rc*eZdZdZddd fd ZxZS)PartitionMaxSizea Partitioning scheme to write files with a maximum size. This partitioning scheme generates files that have a given maximum size. If the size reaches the maximum size, it is closed and a new file is opened. .. warning:: This functionality is currently considered **unstable**. It may be changed at any point without it being considered a breaking change. Parameters ---------- base_path The base path for the output files. file_path A callback to register or modify the output path for each partition relative to the `base_path`. The callback provides a :class:`polars.io.partition.BasePartitionContext` that contains information about the partition. If no callback is given, it defaults to `{ctx.file_idx}.{EXT}`. max_size : int The maximum size in rows of each of the generated files. Examples -------- Split a parquet file by over smaller CSV files with 100 000 rows each: >>> pl.scan_parquet("/path/to/file.parquet").sink_csv( ... PartitionMax("./out", max_size=100_000), ... ) # doctest: +SKIP See Also -------- PartitionByKey PartitionParted polars.io.partition.BasePartitionContext N)r1 base_path str | Pathr1r@max_sizer,rrctdttj|t ||dS)N0partitioning strategies are considered unstable.)rRr?rT)r superrr new_max_sizerG)rrRr1rT __class__s rrzPartitionMaxSize.__init__s` QRRR   '#4Y??!        r)rRrSr1r@rTr,rrr#r$r%r&r __classcell__rYs@rrQrQsW%%X             rrQby6str | Expr | Sequence[str | Expr] | Mapping[str, Expr] list[PyExpr]ctd dt|trt|jg}nt|tr |jg}nit|t rfd|D}nEt|t rd|D}nd}t||S) Ni str | Exprrr cNt|trt|S|Sr) isinstancerr)ras rto_exprz_lower_by..to_exprs$ a   q66MHrc0g|]}|jSr()_pyexpr)rLeres rrNz_lower_by..s$555Qggajj(555rcHg|]\}}||j Sr()aliasrg)rLnrhs rrNz_lower_by..s)@@@TQaggajj(@@@rzinvalid `by` type)rarbrr ) rdrrrgr rritems TypeError)r] lowered_bymsgres @r _lower_byrps"c "ggo& B  j\ B ! !5555"555 B @@RXXZZ@@@ !nn rc,eZdZdZddddfdZxZS)PartitionByKeya Partitioning scheme to write files split by the values of keys. This partitioning scheme generates an arbitrary amount of files splitting the data depending on what the value is of key expressions. The amount of files that can be written is not limited. However, when writing beyond a certain amount of files, the data for the remaining partitions is buffered before writing to the file. .. warning:: This functionality is currently considered **unstable**. It may be changed at any point without it being considered a breaking change. Parameters ---------- base_path The base path for the output files. Use the `mkdir` option on the `sink_*` methods to ensure directories in the path are created. file_path A callback to register or modify the output path for each partition relative to the `base_path`. The callback provides a :class:`polars.io.partition.KeyedPartitionContext` that contains information about the partition. If no callback is given, it defaults to `{ctx.keys.hive_dirs()}/{ctx.in_part_idx}.{EXT}`. by The expressions to partition by. include_key : bool Whether to include the key columns in the output files. Examples -------- Split into a hive-partitioning style partition: >>> ( ... pl.DataFrame({"a": [1, 2, 3], "b": [5, 7, 9], "c": ["A", "B", "C"]}) ... .lazy() ... .sink_parquet( ... PartitionByKey( ... "./out", ... by=[pl.col.a, pl.col.b], ... include_key=False, ... ), ... mkdir=True, ... ) ... ) # doctest: +SKIP Split a parquet file by a column `year` into CSV files: >>> pl.scan_parquet("/path/to/file.parquet").sink_csv( ... PartitionByKey( ... "./out/", ... file_path=lambda ctx: f"year={ctx.keys[0].str_value}.csv", ... by="year", ... ), ... ) # doctest: +SKIP See Also -------- PartitionMaxSize PartitionParted polars.io.partition.KeyedPartitionContext NTr1 include_keyrRrSr1rHr]r^rtboolrrctdt|}tt j|t |||dSNrV)rRr?r]rtr rprWrr new_by_keyrOrrRr1r]rtrnrYs rrzPartitionByKey.__init__;n QRRRr]]    %#5i@@'         r rRrSr1rHr]r^rtrurrrZr\s@rrrrrs\BBR             rrrc,eZdZdZddddfdZxZS)PartitionParteda Partitioning scheme to split parted dataframes. This is a specialized version of :class:`PartitionByKey`. Where as :class:`PartitionByKey` accepts data in any order, this scheme expects the input data to be pre-grouped or pre-sorted. This scheme suffers a lot less overhead than :class:`PartitionByKey`, but may not be always applicable. Each new value of the key expressions starts a new partition, therefore repeating the same value multiple times may overwrite previous partitions. .. warning:: This functionality is currently considered **unstable**. It may be changed at any point without it being considered a breaking change. Parameters ---------- base_path The base path for the output files. Use the `mkdir` option on the `sink_*` methods to ensure directories in the path are created. file_path A callback to register or modify the output path for each partition relative to the `base_path`.The callback provides a :class:`polars.io.partition.KeyedPartitionContext` that contains information about the partition. If no callback is given, it defaults to `{ctx.keys.hive_dirs()}/{ctx.in_part_idx}.{EXT}`. by The expressions to partition by. include_key : bool Whether to include the key columns in the output files. Examples -------- Split a parquet file by a column `year` into CSV files: >>> pl.scan_parquet("/path/to/file.parquet").sink_csv( ... PartitionParted("./out", by="year"), ... mkdir=True, ... ) # doctest: +SKIP See Also -------- PartitionMaxSize PartitionByKey polars.io.partition.KeyedPartitionContext NTrsrRrSr1rHr]r^rtrurrctdt|}tt j|t |||dSrwrxrzs rrzPartitionParted.__init__r{rr|rZr\s@rr~r~QsZ11p             rr~)r?r@rr@)r?rHrrH)r]r^rr_)# __future__r contextlibcollections.abcrrpathlibrtypingrpolarsrpolars._typingr polars._utils.unstabler polars.exprr suppress ImportError polars.polarsr r rrrrr*r<rGrOrQrprrr~r(rrrs""""""-------- ------999999)  [ ) )))(((((())))))))))))))))(((((((((Z%%--,,,,,,---------------////////:--------`2    .7 7 7 7 7 )7 7 7 t2X X X X X 'X X X vG G G G G (G G G G G s$ AA #A BBB