jotUddlmZddlmZmZddlmZddlmZm Z m Z m Z ddl m Z ddlmZer^ddlZddlmZdd lmZeje5dd lmZdddn #1swxYwYdd lmZdd lmZdd lmZmZddlmZddl m!Z!GddZ"GddZ#edGddZ$edGddZ%edGddZ&ee&gdfZ'de(d<edGddZ)d%d$Z*dS)&) annotations)CallableMapping) dataclass) TYPE_CHECKINGClassVarLiteral TypeAlias)parse_into_list_of_expressions)issue_unstable_warningN)Path) DataFrame)PyExpr)Sequence)IO)StorageOptionsDictSyncOnCloseMethod)Expr)CredentialProviderBuilderceZdZUded<dS)_InternalPlPathProviderConfigz ClassVar[str]pl_path_provider_idN)__name__ __module__ __qualname____annotations__X/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/polars/io/partition.pyrrs&&&&&&rrc(eZdZdZddddddddZdS) PartitionBya Configuration for writing to multiple output files. .. warning:: This functionality is currently considered **unstable**. It may be changed at any point without it being considered a breaking change. Parameters ---------- base_path Base path to write to. file_path_provider Callable for custom file output paths. key Expressions to partition by. include_key Include the partition key expression outputs in the output files. max_rows_per_file Maximum number of rows to write for each file. Note that files may have less than this amount of rows. approximate_bytes_per_file Approximate number of bytes to write to each file. This is measured as the estimated size of the DataFrame in memory. Examples -------- Split to multiple files partitioned by year: >>> pl.LazyFrame({"year": [2026, 2027, 1970], "month": [0, 0, 0]}).sink_parquet( ... pl.PartitionBy("data/", key="year") ... ) # doctest: +SKIP Split to multiple files based on size: >>> pl.LazyFrame({"year": [2026, 2027, 1970], "month": [0, 0, 0]}).sink_parquet( ... pl.PartitionBy( ... "data/", max_rows_per_file=1000, approximate_bytes_per_file=100_000_000 ... ) ... ) # doctest: +SKIP Split to multiple files partitioned by year, with limits on individual file sizes: >>> pl.LazyFrame({"year": [2026, 2027, 1970], "month": [0, 0, 0]}).sink_parquet( ... pl.PartitionBy( ... "data/", ... key="year", ... max_rows_per_file=1000, ... approximate_bytes_per_file=100_000_000, ... ) ... ) # doctest: +SKIP Nauto)file_path_providerkey include_keymax_rows_per_fileapproximate_bytes_per_file base_path str | Pathr#eCallable[[FileProviderArgs], str | Path | IO[bytes] | IO[str]] | _InternalPlPathProviderConfig | Noner$=str | Expr | Sequence[str | Expr] | Mapping[str, Expr] | Noner% bool | Noner& int | Noner'int | Literal['auto'] | NonereturnNonecd}t||||dkrd}t|||d}t|t|}|dkr|dnd}|d}t|||t |nd||||_dS)Nz2`PartitionBy` functionality is considered unstabler"zlat least one of ('key', 'max_rows_per_file', 'approximate_bytes_per_file') must be specified for PartitionByz1cannot use 'include_key' without specifying 'key'll)r(r#r$r%r&r')r ValueErrorstr_PartitionByInner_parse_to_pyexpr_list_pl_partition_by)selfr(r#r$r%r&r'msgs r__init__zPartitionBy.__init__TsCs### K!)*f444  S// ! ;;2ECS// ! NN % / /!2!:  ' & -)6 & 11.1o%c***4#/'A ! ! ! r)r(r)r#r*r$r+r%r,r&r-r'r.r/r0)rrr__doc__r9rrrr!r!sN22xMQ#'(,CI2 2 2 2 2 2 2 2 rr!T)kw_onlyc(eZdZUdZded<ded<dS)FileProviderArgsz Holds information on the file being sinked to. .. warning:: This functionality is currently considered **unstable**. It may be changed at any point without it being considered a breaking change. intindex_in_partitionrpartition_keysNrrrr:rrrrr=r=s6rr=cPeZdZUdZded<ded<ded<ded <d ed <d ed <dS)r4zG Holds parsed partitioned sink options. For internal use. r3r(r*r#zlist[PyExpr] | Noner$r,r%r-r&r>r'NrArrrr4r4sm NNN !!!!######rr4ceZdZUdZded<dS)SinkedPathsCallbackArgszInformation on sinked paths.z list[str]pathsNrArrrrDrDs$&&rrDr SinkedPathsCallbackc`eZdZUdZded<ded<dZded<dZded <dZd ed <dZd ed <dS) _SinkOptionsz Holds sink options that are generic over file / target type. For internal use. Most of the options will parse into `UnifiedSinkArgs`. boolmkdirmaintain_orderNzSyncOnCloseMethod | None sync_on_closezStorageOptionsDict | Nonestorage_optionsz CredentialProviderBuilder | Nonecredential_providerzSinkedPathsCallback | Nonesinked_paths_callback) rrrr:rrLrMrNrOrrrrHrHs{ KKK.2M222226O5555<@@@@@8<<<<<<z)_parse_to_pyexpr_list..s)HHHtq! "HHHr) isinstanceritemsr )rPs rr5r5sH"G,,IHH/?/E/E/G/GHHHH )*: ; ;;r)rPrQr/rR)+ __future__rcollections.abcrr dataclassesrtypingrrr r polars._utils.parse.exprr polars._utils.unstabler contextlibpathlibr polarsrsuppress ImportError polars._plrrrrpolars._typingrr polars.exprr,polars.io.cloud.credential_provider._builderrrr!r=r4rDrFrrHr5rrrrlsB"""""""--------!!!!!!>>>>>>>>>>>>CCCCCC999999W       [ ) )''&&&&&&''''''''''''''')(((((DDDDDDDD      VVVVVV''''''''g g g g g g g g T 4         4$$$$$$$$& 4 "*+B*CT*I!JJJJJ 4========"<<<<<