
    j                       d dl mZ d dlZd dlmZmZmZ d dlmZmZm	Z	m
Z
 d dlmZ d dlmZmZ d dlmZ d dlmZmZ d dlmZ d dlmc mZ d d	lmZ d d
lm Z m!Z!m"Z"m#Z#m$Z$ d dlm%Z& d dlm'Z( d dlm)Z* d dl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 d dl4m5Z5m6Z6m7Z7m8Z8 d dl9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZC d dlDmEZEmFZF d dlGmHZH  ejI        eJ          5  d dlKmLZL ddd           n# 1 swxY w Y   er$d dlmMZMmNZNmOZO d dlmPZPmQZQ d dlKmRZR d dlSmTZTmUZUmVZVmWZW dZX	 d{dddddd|d(ZYdddd)d}d/ZZ	 d{dd0d~d5Z[	 	 ddd6dd:Z\ddddd;dd>Z]dd6ddAZ^	 d{ddde:ddBddGZ_edddHddK            Z`e`a                    eb          ddLddN            ZcddQZde`a                    ee          ddLddS            Zfe`a                    e          e`a                    eg          ddU                        Zhe`a                    ei          ddV            ZjddXZkddZZldd6dd[Zmdd\Zndd`ZodddZp	 d{dddde:ddeddiZqddmZr	 d{ddddddnddoZsddqZt	 d{ddddrddtZu	 d{ddddduddvZv	 	 ddd6ddwZw	 d{dddxddzZxdS )    )annotationsN)	GeneratorMappingSequence)datedatetimetime	timedelta)singledispatch)islicezip_longest)
itemgetter)TYPE_CHECKINGAny)	functions)_NUMPY_AVAILABLE_PYARROW_AVAILABLE_check_for_numpy_check_for_pandasdataclasses)numpy)pandas)pyarrow)contains_nestedget_first_non_noneis_namedtupleis_pydantic_model$is_simple_numpy_backed_pandas_seriesis_sqlalchemy_row	nt_unpacktry_get_type_hints)_is_generatorarrlenissue_warningparse_version)
N_INFER_DEFAULTCategoricalDurationEnumStringStructUnknownis_polars_dtypeparse_into_dtypetry_parse_into_dtype)DataOrientationWarning
ShapeError)thread_pool_size)PyDataFrame)CallableIterableMutableMapping)	DataFrameSeries)PySeries)OrientationPolarsDataTypeSchemaDefinition
SchemaDict  TF)schema_overridesstrictnan_to_nullallow_multithreadeddataHMapping[str, Sequence[object] | Mapping[str, Sequence[object]] | Series]schemaSchemaDefinition | Noner?   SchemaDict | Noner@   boolrA   rB   returnr3   c                   t          t                    r< r:t          fd D                       sd}t          |           fdD              t	                                                     \  }|st                     } rt          rt          fd 	                                D                       }|dk    rddl
m}	 t                      }
 |	|
	          5 }t          t          ||                    fd
t                                                               d                     ddd           n# 1 swxY w Y    srfd|D             }n/d t#                     	                                D             }t%          ||d          }t'          |          }rJ|                                t          	                                          k    rt+          ||          }|S )z7Construct a PyDataFrame from a dictionary of sequences.c              3      K   | ]}|v V  	d S N .0colrE   s     i/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/polars/_utils/construction/dataframe.py	<genexpr>zdict_to_pydf.<locals>.<genexpr>W   s'      33sC6M333333    z>the given column-schema names do not match the data dictionaryc                "    i | ]}||         S rM   rM   rO   rP   rC   s     rQ   
<dictcomp>z dict_to_pydf.<locals>.<dictcomp>Z   s    1113T#Y111rS   )lookup_namesr?   c              3     K   | ]u}t          oat          |          oRt          |t          j                  o8t          |          t          k    o o|j        t          j        t          j	        fv           V  vd S rL   )
intr   
isinstancenpndarraylen"_MIN_NUMPY_SIZE_FOR_MULTITHREADINGdtypefloat32float64)rO   valrB   rA   s     rQ   rR   zdict_to_pydf.<locals>.<genexpr>d   s       
 
  # :$S)):sBJ//: HHAA:
  : I"*bj!99 
 
 
 
 
 
rS      r   )ThreadPoolExecutor)max_workersc                    t          | d         t          j                  r#t          j        | d         | d                   n| d         S )N   r   rA   )rZ   r[   r\   plr8   )trA   s    rQ   <lambda>zdict_to_pydf.<locals>.<lambda>{   sE    #-adBJ#?#?!*	!A$!+ N N N N%&qT rS   Tr@   Nc           	     r    g | ]3}t          j        |g                     |                     j        4S )r_   r@   rA   ri   r8   get_s)rO   namerA   r?   r@   s     rQ   
<listcomp>z dict_to_pydf.<locals>.<listcomp>   s^     	
 	
 	
  I&**400'   	
 	
 	
rS   c                    g | ]	}|j         
S rM   )rq   rO   ss     rQ   rs   z dict_to_pydf.<locals>.<listcomp>   s*     
 
 
 D
 
 
rS   )r?   r@   rA   )columns	from_dictr?   r@   )rZ   r   all
ValueError_unpack_schemakeyslistr   sumvaluesconcurrent.futuresrd   r2   dictzipmapitems_expand_dict_values_handle_columns_argr3   dtypes_post_apply_columns)rC   rE   r?   r@   rA   rB   msgcolumn_namescount_numpyrd   	pool_sizepooldata_seriespydfs   ``````        rQ   dict_to_pydfr   L   s    &'"" 2t 23333d33333 	"RCS//!1111&111%3TYY[[;K& & &"L"  "Dzz "  " 
 
 
 
 
 {{}}
 
 
 
 
 ! >=====(**I##	::: d$   
 !..   $                    
$ 
	
 	
 	
 	
 	
 	
 %	
 	
 	

 
(!1'	  
 fhh
 
 
 &k<SWXXXK{##D 
DKKMMT2B2I2I2K2K-L-LLL",1A&
 
 
 Ks   -AEEE)r?   
n_expectedrW   r   
int | NonerW   Iterable[str] | Nonetuple[list[str], SchemaDict]c                  dd	 ddfd
} ||          }| s!|d t          |          D             ng }||fS t          | t                    r1t          |           }t          |                                           } nZg }t          |           D ]H\  }}t          |t                    r| o||v}	|	rd| n|}n|d         }|                    |           I|$t          |          |k    rd}
t          |
          |rd t          ||          D             nd}i }| D ]J}t          |t                    r|\  }}|  |          }|r|                    ||          n|}|||<   K|r|                    |           ||fS )z
    Unpack column names and create dtype lookup.

    Works for any (name, dtype) pairs or schema dict input,
    overriding any inferred dtypes with explicit dtypes if supplied.
    r_   r   rI   r;   c                F    t          | d          r| S t          |           S )z1Parse non-Polars data types as Polars data types.T)include_unknown)r-   r.   )r_   s    rQ   _normalize_dtypez(_unpack_schema.<locals>._normalize_dtype   s*    5$777 	+L#E***rS   Nr?   rG   dict[str, PolarsDataType]c                L    | i S fd|                                  D             S )zCParse schema overrides as a dictionary of name to Polars data type.Nc                .    i | ]\  }}| |          S rM   rM   )rO   rr   r_   r   s      rQ   rV   zC_unpack_schema.<locals>._parse_schema_overrides.<locals>.<dictcomp>   s8     
 
 
.9dED""5))
 
 
rS   )r   )r?   r   s    rQ   _parse_schema_overridesz/_unpack_schema.<locals>._parse_schema_overrides   sG     #I
 
 
 
=M=S=S=U=U
 
 
 	
rS   c                    g | ]}d | S )column_rM   )rO   is     rQ   rs   z"_unpack_schema.<locals>.<listcomp>   s    666q]q]]666rS   r   r   z)data does not match the number of columnsc                    i | ]
\  }}|||S rL   rM   )rO   rP   rr   s      rQ   rV   z"_unpack_schema.<locals>.<dictcomp>   s.     	
 	
 	
T rS   )r_   r   rI   r;   rL   )r?   rG   rI   r   )rangerZ   r   r~   r   	enumeratestrappendr]   r1   r   rp   update)rE   r?   r   rW   r   rw   r   r   rP   unnamedr   lookupcolumn_dtypesrr   r_   r   s                  @rQ   r|   r|      sM   + + + + /3	
 	
 	
 	
 	
 	
 	
 /./?@@  ):D:P66E*$5$56666VX 	 ((( &'"" %"&v,,fllnn%%'' 	% 	%FAs#s## !'Ac1A&A'.7mmmmC!f$$$$#l"3"3z"A"A9oo 	 	
 	
(|DD	
 	
 	
 	
   02M 
$ 
$c3 	e=$$U++E)/9vzz$%%%T#d  /-...&&rS   )rx   list[PySeries]rw   Sequence[str] | Nonerx   c                  || S | sd |D             S t          |           t          |          k    r2dt          |           dt          |            d}t          |          |r5d | D             t          fd|D                       rfd|D             S t          |          D ][\  }}|| |                                         k    r8| |                                         | |<   | |                             |           \| S )	z*Rename data according to columns argument.Nc                B    g | ]}t          j        |           j        S )rr   )ri   r8   rq   rO   cs     rQ   rs   z'_handle_columns_arg.<locals>.<listcomp>
  s(    666	q!!!$666rS   zdimensions of columns arg () must match data dimensions ()c                8    i | ]}|                                 |S rM   r   ru   s     rQ   rV   z'_handle_columns_arg.<locals>.<dictcomp>  s"    000aaffhh000rS   c              3      K   | ]}|v V  	d S rL   rM   rO   rP   
series_maps     rQ   rR   z&_handle_columns_arg.<locals>.<genexpr>  s(      66sz!666666rS   c                     g | ]
}|         S rM   rM   r   s     rQ   rs   z'_handle_columns_arg.<locals>.<listcomp>  s    777JsO777rS   )r]   r{   rz   r   rr   clonerename)rC   rw   rx   r   r   r   r   s         @rQ   r   r      s3     66g6666	Tc'll	"	"dCLLddX[\`XaXadddoo 8004000
6666g66666 	87777w7777'""  1Q1gmmooDGGNN1KrS   rl   r   structsdict[str, Struct] | Nonec               .   |                                  |                                 }}t          |p||          \  }}g }||k    rSt          |          t          |          k     r||dt          |                   k    r|}n|                     |           g }	t          |          D ]\  }
}|                    |          }||
         }|&|t          cxk    r|k    rJn nG|	                    t          j
        |                              t          |          j                   |t          cxk    r|k    rEn nB|	                    t          j
        |                              ||          j                   |r`|                    |          x}rI||k    rC|	                    t          j
        |                              ||          j                   ;|t          k    r||k    r|                                r`|t           k    rU|t"          k    rJt          j
        |          j                            ||          j        }|	                    |           |	                    t          j
        |                              ||          j                   |	s|rc|                                 }|	r|                    |	          }|r|                    d |D                       }|                    dd          } | S )zGApply 'columns' param *after* PyDataFrame creation (if no alternative).r?   Nrl   c                @    g | ]}t          j        |          j        S rM   )FrP   _pyexprrO   rP   s     rQ   rs   z'_post_apply_columns.<locals>.<listcomp>H  s#    !N!N!N!%**"4!N!N!NrS   z	in-memory)enginelambda_post_opt)rw   r   r|   r]   set_column_namesr   rp   r'   r   r   rP   castr   r)   r,   is_temporalr(   r*   r   strptimelazywith_columnsselectcollect)r   rw   r   r?   r@   pydf_columnspydf_dtypesr   column_subsetcolumn_castsr   rP   r_   
pydf_dtypestructtemporal_castpyldfs                    rQ   r   r     s    !%+L$		 L4D  OGV  "M,w<<#l++++<#g,,;W0W0W#MM!!'***LG$$ S S3

3 ^
=K----:-----c

F K K STTTTd((((j(((((c

f E E MNNNN 	SGKK$4$44& 	S&J:N:Nc

v F F NOOOOg%:"5"5  "" Su'8'8Z6=Q=Q !c

 7 7f 7 M M U##M2222##AE#JJOOE&O$I$I$QRRR G} G		 	5&&|44E 	PLL!N!N!N!N!NOOE}}K}FFKrS   )r?   r@   orderrA   r   dict[str, Series]c                  i | rt          d |                                 D                       rd}t          |          |pi }t          | ||          } t	          d |                                 D                       dk    rA|                                 D ]*\  }}|                    |          }	t          |t                    r|	t          k    rt          j        ||          j        dk    rdk    ryt          d j                                        D                       rIfd|                                D             }
t          j        |
                              |          }n                    |          }||<   t          |t          j                  rM||j        k    r|                    |          n|}|	r"|	|j        k    r|                    |	|          }||<   ]t+          |          t-          |          rt          j        |||	||
          |<   |@t          |t.          t0          t2          t4          t6          t8          t:          t<          f          r0t?          j         ||	d          !                    |          |<   t          j        ||gz  |	|          |<   ,nt          d |                                 D                       rI|                                 D ]3\  }}t          j        |||                    |          |          |<   4nt          d |                                 D                       rZ|                                 D ]E\  }}t          j        |t-          |          r|n|g|                    |          |          |<   F|r!tE                    |k    rfd|D             S S )zCExpand any scalar values in dict data (propagate literal as array).c              3  J   K   | ]}t          |t          j                  V  d S rL   )rZ   ri   ExprrO   rb   s     rQ   rR   z&_expand_dict_values.<locals>.<genexpr>Y  s.      AACz#rw''AAAAAArS   zpassing Expr objects to the DataFrame constructor is not supported

Hint: Try evaluating the expression first using `select`, or if you meant to create an Object column containing expressions, pass a list of Expr objects instead.rl   c              3  8   K   | ]}t          |          pd V  dS r   Nr#   r   s     rQ   rR   z&_expand_dict_values.<locals>.<genexpr>d  s-      DDs)DDDDDDrS   r   rg   c              3  @   K   | ]}|                                  V  d S rL   )	is_nestedrO   ds     rQ   rR   z&_expand_dict_values.<locals>.<genexpr>m  s-      OOaAKKMM 1OOOOOOrS   c                X    i | ]&\  }}||                              |d z
            'S )rg   )n)extend_constant)rO   nmv	array_lenvdfs      rQ   rV   z'_expand_dict_values.<locals>.<dictcomp>o  sJ     " " " %A B 7 7i!m 7 M M" " "rS   Nrr   r   r_   r@   rA   T)r_   eager)rr   r   r_   r@   c              3  <   K   | ]}t          |          d k    V  dS r   r   r   s     rQ   rR   z&_expand_dict_values.<locals>.<genexpr>  s-      ==&++"======rS   )r   r_   r@   c              3  8   K   | ]}t          |          d u V  d S rL   r   r   s     rQ   rR   z&_expand_dict_values.<locals>.<genexpr>  s-      @@3&++%@@@@@@rS   c                <    i | ]}|                     |          S rM   )pop)rO   rP   updated_datas     rQ   rV   z'_expand_dict_values.<locals>.<dictcomp>  s)    <<<s\%%c**<<<rS   )#anyr   	TypeError_expand_dict_datamaxr   rp   rZ   r   r+   ri   r7   heightrz   rE   	to_structr8   rr   r   r_   r   r#   r"   rY   floatr   rH   r   r   r	   r
   r   repeataliasr~   )rC   r?   r@   r   rA   r   r   rr   rb   r_   s_valsstrv   r   r   r   s                @@@rQ   r   r   N  s?    L FAA4;;==AAAAA 	!8  C.. !'R vf===DDdkkmmDDDDD	q==!ZZ\\ ) )	c

4((c4(( 'Uf__,s6:::C
a%MMOO3:;L;L;N;NOOOOO *" " " " ")," " "  \&11;;DAA ]]400)+L&&RY// ,0CH,<,<

4(((#A 9!'!1!1FF5F88)*L&&C[[,c0B0B,)+!"#%$/* * *L&& [J#uc4xyQ% %[ *+Ye4* * *eDkk !&& *,!3%)*;5QW* * *L&&O)V ==t{{}}===== 	!ZZ\\  	c%'YFJJt,<,<V& & &T""
 @@$++--@@@@@ 	!ZZ\\  	c%'Y#0#5#5@CCC5 **T**!	& & &T""  =l##u,,<<<<e<<<<rS   r   r=   c                   i }|                                  D ]D\  }}t          |          r+t          j        |||                    |          |          n|||<   E|S )z
    Expand any unsized generators/iterators.

    (Note that `range` is sized, and will take a fast-path on Series init).
    rl   )r   r"   ri   r8   rp   )rC   r   r@   expanded_datarr   rb   s         rQ   r   r     sw     	  ZZ\\ 
 
	c S!!BIdCD!1!1&AAAA 	d
 rS   )r?   r@   orientinfer_schema_lengthrA   Sequence[Any]r   Orientation | Noner   c          
     r    | st          i ||          S t          t          |           | ||||||          S )z(Construct a PyDataFrame from a sequence.)rE   r?   rC   rE   r?   r@   r   r   rA   )r   _sequence_to_pydf_dispatcherr   r  s          rQ   sequence_to_pydfr    sZ      RBv@PQQQQ'4  )/	 	 	 	rS   )r@   rA   first_elementr   c               ,   |||||||d}d}	t          | t                    rt          }
d |D             }|d         } d}	nt          | t          j                  rt
          }
nt          |           r"t          | t          j                  rt          }
nt          |           r9t          | t          j        t          j        t          j        f          rt          }
nt          j        |           rt"          }
ngt%          |           rt&          }
nPt)          |           rt*          }
n9t          | t,                    rt          | t.                    st          }
nt0          }
|	r(t2                              t7          |           |
           | |d<    |
di |S )Nr  Tc                ,    g | ]}t          |          S rM   )r~   )rO   rows     rQ   rs   z0_sequence_to_pydf_dispatcher.<locals>.<listcomp>  s    ***cS		***rS   r   Fr  rM   )rZ   r   _sequence_of_sequence_to_pydfri   r8   _sequence_of_series_to_pydfr   r[   r\   _sequence_of_numpy_to_pydfr   pdIndexDatetimeIndex_sequence_of_pandas_to_pydfr   is_dataclass _sequence_of_dataclasses_to_pydfr   $_sequence_of_pydantic_models_to_pydfr   _sequence_of_tuple_to_pydfr   r   _sequence_of_elements_to_pydfr  registertype)r  rC   rE   r?   r@   r   r   rA   common_paramsregister_with_singledispatchto_pydfs              rQ   r  r    s   $ ,2"% %M $( -++ 0/**T***Q',$$	M29	-	- 0-	-	(	( 0Zrz-R-R 0,	=	)	) 0j	28R-=>/ / 0 .		!-	0	0 02	=	)	) 	06	=	)	) 0,	M8	,	, 0Zs5S5S 0//# L$--d=.A.A7KKK%2M/"7##]###rS   rh   $Sequence[Any] | np.ndarray[Any, Any]c                  |b|d}n]t          |          t          |           k    ot          |          t          |          k    }|rdnd}|rt          dt                     |dk    rt          |t          |                     \  rt	                    ni }	d}
|	                                D ]m\  }}|t          t          fv rt          |	|<    |
sK|	                                t          t          fv r)t          t          | |d           j        t                    }
n|
r&d |D             }t!          j        |d d |          }nt!          j        ||	pd |          }srt'          |	          }|S |dk    rOt          |t          |                    \  fd
t)          |          D             }t!          |          S d|}t+          |          )NrP   r  zRow orientation inferred during DataFrame construction. Explicitly specify the orientation by passing `orient="row"` to silence this warning.r?   r   Fc                ,    g | ]}t          |          S rM   )r    r   s     rQ   rs   z1_sequence_of_sequence_to_pydf.<locals>.<listcomp>I  s    000aYq\\000rS   rE   r?   r@   r   rE   r   ry   c           
         g | ]B\  }}t          j        |         |                    |                              j        CS rn   ro   )rO   r   elementr   rA   r?   r@   s      rQ   rs   z1_sequence_of_sequence_to_pydf.<locals>.<listcomp>a  sj     	'
 	'
 	'
 7 IQ&**<?;;'   	'
 	'
 	'
rS   2`orient` must be one of {'col', 'row', None}, got )r]   r$   r0   r|   _include_unknownsr   r'   r)   r*   	base_typer,   r+   r   getattr	__class__r   r3   
from_dicts	from_rowsr   r   r{   )r  rC   rE   r?   r@   r   r   rA   is_row_orientedlocal_schema_overrideunpack_nestedrP   tpdictsr   r   r   r   s      ``  `         @rQ   r  r    s    ~>FF  #6{{c-.@.@@ Fs4yy(  .8UU5F m*   )7%5#mBTBT*
 *
 *
&&
  .=== 	 ,2244 	 	GCk4(((-3%c**" 7F:K(K(K /M355?! !  	004000E)!%$7  DD (,4$7  D
  	+ 	&l5Ef  D 	5)7%5#d))*
 *
 *
&&	'
 	'
 	'
 	'
 	'
 	'
 	'
 (oo	'
 	'
 	'
 ;''' PVOOoorS   r8   kwargsc                  d |D             }t          |p||t          |                    \  }}g }t          |          D ]\  }	}
|
j        s|
                    ||	                   }
|                    ||	                   }|r#||
j        k    r|
                    ||d          }
|                    |
j	                   t          ||          }t          |          S )Nc                    g | ]	}|j         
S rM   r   ru   s     rQ   rs   z/_sequence_of_series_to_pydf.<locals>.<listcomp>{  s    )))qAF)))rS   r  Fr@   wrap_numericalrw   )r|   r]   r   rr   r   rp   r_   r   r   rq   r   r3   )r  rC   rE   r?   r@   r-  series_namesr   r   r   rv   	new_dtypes               rQ   r	  r	  r  s     *)D)))L%3,)t99& & &"L"
 #%K$ ! !1v 	)Q((A$((a99	 	Gag--yFFA14    %k<HHHK{###rS   tuple[Any, ...]c          
     <   t          | j                  st          |           rc|]| j        }t	          | dd           }|rCt          |          t          |          k    r#d | j                                        D             }|d}t          | |||||||          S )N__annotations__c                6    g | ]\  }}|t          |          fS rM   )r/   )rO   rr   r+  s      rQ   rs   z._sequence_of_tuple_to_pydf.<locals>.<listcomp>  s:        b /334  rS   r  r  )	r   r%  r   _fieldsr$  r]   r7  r   r  )	r  rC   rE   r?   r@   r   r   rA   r   s	            rQ   r  r    s     ],-- 
1B=1Q1Q 
>"*F!-1BDIIK s;//3v;;>> $1$A$G$G$I$I   >F ))/	 	 	 	rS   dict[str, Any]c                   t          ||          \  }}|rt          ||pt          |                    nd }t          j        |||||          }	|	S )Nr   )r@   r   )r|   r"  r~   r3   r&  )
r  rC   rE   r?   r@   r   r-  r   dicts_schemar   s
             rQ   _sequence_of_dict_to_pydfr=    s     &4!1& & &"L"
 	*L,RDAQ<R<RSSS  !/  D KrS   c                   t          ||d          \  }}t          j        |d         ||                    |d                   |          j        g}t          ||          }t          |          S )Nrg   r  r   rl   r2  )r|   ri   r8   rp   rq   r   r3   )r  rC   rE   r?   r@   r-  r   r   s           rQ   r  r    s     &4!1a& & &"L" 		O  a11		
 	
 	

 #K &k<HHHK{###rS   np.ndarray[Any, Any]c                L    | j         dk    rt          | fi |S t          | fi |S )Nrg   )ndimr  r  )r  r-  s     rQ   r
  r
    s>     Q,]EEfEEE,]EEfEEErS   1pd.Series[Any] | pd.Index[Any] | pd.DatetimeIndexc                  |g }nt          ||d          \  }}|pi }g }t          |          D ]\  }}	|r||         n|	j        }
t          j        |
|	          }|                    |
          }|0||                                k    r|                    ||d          }|                    |           t          |          S )Nrg   r  )rr   r   Fr0  )
r|   r   rr   plcpandas_to_pyseriesrp   r_   r   r   r3   )r  rC   rE   r?   r@   r-  r   r   r   rv   rr   pyseriesr_   s                rQ   r  r    s     ~"$)7%5!*
 *
 *
&& (-2"$K$ % %1".:|AAF)tA>>> $$T**(..*:*:!:!:}}U6%}PPH8$$$${###rS   c               T   ddl mm t          | ||d          \  }}}}	|r(fd|D             }
t	          j        |
dd||          }n'fd|D             }t	          j        ||	pd|          }|	r2d	 |	                                D             }t          |||||
          }|S )z-Initialize DataFrame from Python dataclasses.r   )asdictastupleN)model_fieldsc                &    g | ]} |          S rM   rM   )rO   mdrH  s     rQ   rs   z4_sequence_of_dataclasses_to_pydf.<locals>.<listcomp>)  s!    ++++++rS   r  c                &    g | ]} |          S rM   rM   )rO   dcrI  s     rQ   rs   z4_sequence_of_dataclasses_to_pydf.<locals>.<listcomp>2  s!    ++++++rS   r  c                D    i | ]\  }}t          |t                    ||S rM   rZ   r+   rO   r   r+  s      rQ   rV   z4_sequence_of_dataclasses_to_pydf.<locals>.<dictcomp>:  -    RRRUQ:b&;Q;QR1bRRRrS   rl   )	r   rH  rI  $_establish_dataclass_or_model_schemar3   r&  r'  r   r   )r  rC   rE   r?   r   r@   r-  r*  r   	overridesr,  r   rowsr   rH  rI  s                 @@rQ   r  r    s&    ,+++++++ 	-v/d	 	 	  
++++d+++%! 3
 
 
 ,+++d+++$$ 3
 
 
  
RRioo&7&7RRR",)9&
 
 
 KrS   c               T   ddl }t          |j                  dk     }t          |r| j        n| j        j                  }	t          | |||	          \  }
}}}|
r4|rd |D             nd |D             }t          j	        |dd||          }ngt          |	          dk    r/t          |	 fd|D             }t          j        |||	          }n%d
 |D             }t          j	        ||d||          }|r2d |                                D             }t          |||||          }|S )z1Initialise DataFrame from pydantic model objects.r   N)   r   c                6    g | ]}|                                 S rM   )r   rO   rL  s     rQ   rs   z8_sequence_of_pydantic_models_to_pydf.<locals>.<listcomp>a  s     &&&2RWWYY&&&rS   c                :    g | ]}|                     d           S )python)mode)
model_dumprY  s     rQ   rs   z8_sequence_of_pydantic_models_to_pydf.<locals>.<listcomp>c  s&    >>>2"--X-..>>>rS   r  2   c                0    g | ]} |j                   S rM   __dict__)rO   rL  
get_valuess     rQ   rs   z8_sequence_of_pydantic_models_to_pydf.<locals>.<listcomp>p  s%    777B

2;''777rS   r  c                    g | ]	}|j         
S rM   r`  rY  s     rQ   rs   z8_sequence_of_pydantic_models_to_pydf.<locals>.<listcomp>v  s    ,,,,,,rS   c                D    i | ]\  }}t          |t                    ||S rM   rP  rQ  s      rQ   rV   z8_sequence_of_pydantic_models_to_pydf.<locals>.<dictcomp>  rR  rS   rl   )pydanticr%   __version__r~   
__fields__r%  rJ  rS  r3   r&  r]   r   r'  r   r   )r  rC   rE   r?   r   r@   r-  re  old_pydanticrJ  r*  r   rT  r,  r   rU  r   rb  s                    @rQ   r  r  B  s    OOO !566?L	2  $1 L 	-v/	 	   

 ?&&&&&&>>>>> 	
 %! 3
 
 
 
\		R		.
7777$777$8K
 
 

 -,t,,,%! 3
 
 
  
RRioo&7&7RRR",)9&
 
 
 KrS   rJ  list[str] | None.tuple[bool, list[str], SchemaDict, SchemaDict]c                @  
 ddl m} d}|r#t          |          \  }fd|D             }ng }fdt          | j                                                  D             }r|                               n?s;t           ||                     

fd|                                D             x}n||                                D ]v\  }}	|	t          t          fv rt          ||<    |sT|	                                t          t          fv r2t          t          | |d          rt           nt           j                  }wrQt%                    t%          |          k    r1t'          t)          |                                d	
                    }|||fS )zLShared utility code for establishing dataclasses/pydantic model cols/schema.r   )rH  Fr   c                H    i | ]}|                     |t                    S rM   rp   r,   )rO   rP   r?   s     rQ   rV   z8_establish_dataclass_or_model_schema.<locals>.<dictcomp>  s,    UUUS*..sG<<UUUrS   c                ^    i | ])\  }}r|v n|d k    |t          |          pt          *S )	__slots__)r/   r,   )rO   rP   r+  rJ  s      rQ   rV   z8_establish_dataclass_or_model_schema.<locals>.<dictcomp>  sY     
 
 
R)5
 $$$C;<N<N &r**5g<N<N<NrS   c                $    i | ]\  }}|v 	||S rM   rM   )rO   r   r+  	dc_fieldss      rQ   rV   z8_establish_dataclass_or_model_schema.<locals>.<dictcomp>  s+     , , ,!2r"	//B///rS   NTrl   )r   rH  r|   r!   r%  r   r   setr'   r)   r*   r#  r,   r+   r   r$  r   r  r]   r   r   r   )r  rE   r?   rJ  rH  r*  r   rT  rP   r+  rq  s     ``      @rQ   rS  rS    s    #"""""M ))7%5*
 *
 *
&& VUUUUUU		
 
 
 
-m.EFFLLNN
 
 
	
  	)-.... 	)FF=1122I, , , ,%.__%6%6, , , yy  )??$$  R+t$$$#IcNN 	BLLNNw6G$G$G+sD11%1O!!{7O M
  ML))S^^;;\9+;+;+=+=dKKKLL	,(8)CCrS   colsSequence[str]#MutableMapping[str, PolarsDataType]c                       fd|D             S )z7Complete partial schema dict by including Unknown type.c                V    i | ]%}|                     |t                    pt          &S rM   rm  rN   s     rQ   rV   z%_include_unknowns.<locals>.<dictcomp>  s>        	fjjg&&1'  rS   rM   )rE   rs  s   ` rQ   r"  r"    s.          rS   )r?   r@   r   
chunk_sizer   rechunkIterable[Any]rx  ry  c               p   |}g i |t          |          \  nrt          |          \  }	t          | t                    st          |           } |dk    rSrrfdt	                    D             t          j        fdt	          |           D                       j        S dfd}
d}d}|r|}nr|t                    z  }nd}d}dnt          |pd          }	 t          t          | |                    }|snp |
||          }|E|}|s&t          |j                                                  }||k    r|j        x}dk    r||z  x}}n|                    |d           |dz  }| |
g |          }|dk    r|r|                                }|j        S )z3Construct a PyDataFrame from an iterable/generator.Nr   rP   c                N    i | ]!\  }}|                     |t                    "S rM   rm  )rO   idxrP   r?   s      rQ   rV   z$iterable_to_pydf.<locals>.<dictcomp>  s@       C %))#w77  rS   c           	         i | ]=\  }}r|         nd | t          j        |                    |                    >S )r   )r_   r@   )ri   r8   rp   )rO   r}  coldatar   dtypes_by_idxr@   s      rQ   rV   z$iterable_to_pydf.<locals>.<dictcomp>  sq        !C '3Gc""#")'++C00!K K K  rS   r   	list[Any]rE   rF   rI   r7   c                8    t          j        | |d          S )Nr  )rC   rE   r@   r   r   r?   )ri   r7   )r   rE   r   r?   r@   s     rQ   to_frame_chunkz(iterable_to_pydf.<locals>.to_frame_chunk  s.    | 3-
 
 
 	
rS   r   i@B r>   T)in_placerg   )r   r  rE   rF   rI   r7   )r|   rZ   r   iterr   ri   r7   _dfr]   r   r~   r   rE   r   widthvstackry  )rC   rE   r?   r@   r   rx  r   ry  original_schema_r  n_chunksn_chunk_elemsadaptive_chunk_sizedfr   frame_chunk	n_columnsr   r  s     ``  `           @@rQ   iterable_to_pydfr    s    O L/1M)7%5*
 *
 *
&&& 
 X,VFVWWWdI&& Dzz 	, 	    ), 7 7  M
 |      %.dOO  	
 	
 		
 
 
 
 
 
 
 
 HM #(	 #+s</@/@@"B & 	$&9&ATBB 
fT:..// 	$nV_==:B" :"&ry'8'8"9"9000!#)IQ..7D	7QQJ!4IIkDI111MH  
z^B00!|||ZZ\\6MrS   pd.DataFrameinclude_indexNonec                  d | j         D             }|rd | j        j        D             nt                      }t	          |          t	          | j                   k     }|r*t	          |          t	          | j        j                  k     nd}|s|rd}t          |          ||z  }t	          |          dk    rd}t          |          dS )z:Check pandas dataframe columns can be converted to polars.c                ,    h | ]}t          |          S rM   r   r   s     rQ   	<setcomp>z(_check_pandas_columns.<locals>.<setcomp>!  s    !C!C!Cs#c((!C!C!CrS   c                ,    h | ]}t          |          S rM   r  )rO   r}  s     rQ   r  z(_check_pandas_columns.<locals>.<setcomp>#  s    ...cS...rS   Fz|Pandas dataframe contains non-unique indices and/or column names. Polars dataframes require unique string names for columns.r   z1Pandas indices and column names must not overlap.N)rw   indexnamesrr  r]   r{   )rC   r  stringified_colsstringified_indexnon_unique_colsnon_unique_indicesr   overlapping_cols_and_indicess           rQ   _check_pandas_columnsr    s    !C!Cdl!C!C!C2?J..TZ-....SUU    011C4E4EEO<IT		#dj&6"7"7	7	7u   , I 	 oo-=@Q-Q 
'((1,,Aoo -,rS   )r?   r@   ry  rA   r  c                   t           |           |ot                      }|s j        d         dk    rt          j         j        d                   S t           fd j        D                       r/t          j         fd j        D             ||||          j	        S t          sd}t          |          i }	 j        d         }
|rN j        j        D ]A}t          j         j                            |          ||
          |	t#          |          <   B                                 D ],\  }}t          j        |||
          |	t#          |          <   -t'          j        |	          }t+          |||||	          S )
z0Construct a PyDataFrame from a pandas DataFrame.)r  rg   r   c              3  B   K   | ]}t          |                   V  d S rL   )r   rU   s     rQ   rR   z!pandas_to_pydf.<locals>.<genexpr>J  s0      WW33DI>>WWWWWWrS   c                `    i | ]*}t          |          |                                         +S rM   )r   to_numpyrU   s     rQ   rV   z"pandas_to_pydf.<locals>.<dictcomp>M  s1    HHHCS49--//HHHrS   )rE   r@   r?   rA   zpyarrow is required for converting a pandas dataframe to Polars, unless each of its columns is a simple numpy-backed one (e.g. 'int64', 'bool', 'float32' - not 'Int64'))rA   length)rE   r?   r@   ry  )r  _pandas_has_default_indexshaper3   empty_with_heightrz   rw   ri   r7   r  r   ImportErrorr  r  rD  pandas_series_to_arrowget_level_valuesr   r   patablearrow_to_pydf)rC   rE   r?   r@   ry  rA   r  convert_indexr   
arrow_dictr  idxcolcol_idxcol_dataarrow_tables   `              rQ   pandas_to_pydfr  7  s    $m<<<<!I*CD*I*I&IM :a=A0A???WWWW$,WWWWW 	<HHHH4<HHH!1'     > 	
 #JZ]F j& 	 	F&)&@ 
++F33'' ' 'Js6{{## "ZZ\\ 
 
#&#=+f$
 $
 $

3w<<   (:&&K)   rS   r  c                   ddl m} | j        j        }t	          |          dk    s|dgdgfvrdS | j                             |dt	          |           d                    rdS t          | j        j                                      d	          oZt          | j        
                                t          j        t	          |                     k                                              S )
zFIdentify if the pandas frame only has a default (or equivalent) index.r   )
RangeIndexrg   N F)startstopstepTrY   )pandas.core.indexes.ranger  r  r  r]   equalsr   r_   
startswithrH   sort_valuesr[   arangerz   )r  r  
index_colss      rQ   r  r  w  s    444444J
:j$">>u	!#b''BBB	C	C 
t 28>""--e44 
X!!##ryR'9'99>>@@:
 :
 	
rS   )r?   r@   ry  pa.Table | pa.RecordBatchc                  t          |p| j        j        |          \  }}	 || j        j        k    r|                     |          } n)# t          j        $ r}d}t          |          |d}~ww xY wt          | t          j                  r| g}n8| j	        dk    rt          j        | j                  S |                                 }t          j        || j                  }	|r|	                                }	|t!          |	|||          }	|	S )z;Construct a PyDataFrame from an Arrow Table or RecordBatch.r   z4dimensions of columns arg must match data dimensionsNr   ry   )r|   rE   r  rename_columnsr  ArrowInvalidr{   rZ   RecordBatchnum_columnsr3   r  num_rows
to_batchesfrom_arrow_record_batchesry  r   )
rC   rE   r?   r@   ry  r   er   batchesr   s
             rQ   r  r    s2    &4		$4;$8H& & &"L"%4;,,,&&|44D? % % %Doo1$%
 $'' $&		Q		,T];;;//## 0$+FFD ||~~#"-	
 
 
 Ks   %A A.A))A.)r?   r   r@   rA   c                    j         t                    dk     j        j        Ed\  }}t	           j        j                  }t          |          |D ]} |         j         |s|}n dg }}dk    rdnt                    dk    rdnt                    dk    r̉d         dk    rt          j        d                   S ||d         d}n|c|at          |          }	|	d         k    r|	d         k    rd	}d         n~ j        d
         rd         d         k    rd	}|	nZd}d         nO|dk    r	d         n@|d	k    r	d         n1d|}
t          |
          dk    rd}
nd }
t          |
          |Bt          |          k    r/t          |          x}	dk    rd|	 d d}
t          |
          |	t          |          \  |r" fdt          |d          D             }ndk    r	dk    rg }nt                    dk    r?t          j        d                              d                             j        g}nK|dk    r# fdt                    D             }n" fdt                    D             }t!          |          }t          |          S )zMConstruct a PyDataFrame from a NumPy ndarray (including structured ndarrays).rW  N)TrP   F)r   r   rg   r  rP   F_CONTIGUOUSr!  rM   z3cannot create DataFrame from zero-dimensional arrayzJcannot create DataFrame from array with more than two dimensions; shape = zdimensions of `schema` (r   r   r  c           	         g | ]<\  }}t          j        ||                             |                     j        =S )r   ro   )rO   series_namerecord_namerC   rA   r?   r@   s      rQ   rs   z!numpy_to_pydf.<locals>.<listcomp>  sg     	
 	
 	
 )[ I K(&**;77'   	
 	
 	
rS   Trl   r   c           
         g | ]_}t          j        |         	rd k    rd          d k    rndd|f                             |                             j        `S )rg   Nr   ro   
rO   r   r   rC   r  rA   r?   r  r@   two_ds
     rQ   rs   z!numpy_to_pydf.<locals>.<listcomp>  s         	%a !(%.!^^a1 !!!!Q$Z*..|A??! +
 
 
   rS   c           
         g | ][}t          j        |         	rd k    rd          d k    rn|                             |                             j        \S )rg   r   ro   r  s
     rQ   rs   z!numpy_to_pydf.<locals>.<listcomp>.  s         	%a %V)q..U1X\\tTUw*..|A??! +     rS   r2  )r  r]   r_   r  r~   r3   r  flagsr{   r|   r   ri   r8   rp   rq   r   r   )rC   rE   r?   r   r@   rA   structured_arrayrecord_namesr   n_schema_colsr   r   r   r  r  r  s   ` ` ``      @@@@rQ   numpy_to_pydfr    s    JEJJ!OEz##. &DJ,--%%	 	# 	#BHNEE 	"!F */,D==IIZZ1__IIZZ1__Qx1}}"4U1X>>>~&.!!H	F$6 !$F E!H,,%(1J1J"F %aIIZ/ )E!Ha4H4H"F -II"F %aII5!!H		5!!H		WVWW oo%{{KjchjjS//!c&kkY66 [[(MQ..f]ffZcfffCS//!!	%3!1i& & &"L"
  4	
 	
 	
 	
 	
 	
 	
 -0lSW,X,X,X	
 	
 	
 
$9>>	UqI!!_&**<?;;'   
 U??           y))  KK           y))  K &k<HHHK{###rS   c                  ||t          | j        g          S | j        g}d |D             }t          |p||d          \  }}|rZt          t	          |                                                    }|| j        k    r!|d                             ||d          |d<   t          ||          }t          |          S )	z-Construct a PyDataFrame from a Polars Series.Nc                6    g | ]}|                                 S rM   r   ru   s     rQ   rs   z"series_to_pydf.<locals>.<listcomp>K  s     11116688111rS   rg   r  r   Fr0  r2  )	r3   rq   r|   nextr  r   r_   r   r   )rC   rE   r?   r@   r   r  r   r4  s           rQ   series_to_pydfr  ?  s     ~*2DG9%%%7)K11[111K%3+0@Q& & &"L"  .55778899	
""(^00& 1  KN &k<HHHK{###rS   ry   r7   c                  ||| j                                         S d | D             }t          |p| j        |          \  }}|rN| j        }|                                D ]2\  }}|||         k    r!||                             ||d          ||<   3t          t          |	                                          |          }	t          |	          S )z:Construct a PyDataFrame from an existing Polars DataFrame.Nc                (    i | ]}|j         |j        S rM   )rr   rq   r   s     rQ   rV   z%dataframe_to_pydf.<locals>.<dictcomp>e  s    ...A1614...rS   r   Fr0  r2  )r  r   r|   rw   rE   r   r   r   r~   r   r3   )
rC   rE   r?   r@   r   r   existing_schemarr   r4  series_colss
             rQ   dataframe_to_pydfr  Z  s     ~*2x~~.....K%3$,1A& & &"L"  +/5577 	 	OD)OD111$/$5$:$:fU %; % %D! &d;+=+=+?+?&@&@,WWWK{###rS   rL   )rC   rD   rE   rF   r?   rG   r@   rH   rA   rH   rB   rH   rI   r3   )
rE   rF   r?   rG   r   r   rW   r   rI   r   )rC   r   rw   r   rx   rH   rI   r   )NN)r   r3   rw   rF   r   r   r?   rG   r@   rH   rI   r3   )rC   rD   r?   rG   r@   rH   r   r   rA   rH   rI   r   )rC   rD   r   r=   r@   rH   rI   rD   )rC   r   rE   rF   r?   rG   r@   rH   r   r   r   r   rA   rH   rI   r3   )r  r   rC   r   rE   rF   r?   rG   r@   rH   r   r   r   r   rA   rH   rI   r3   )r  r  rC   r   rE   rF   r?   rG   r@   rH   r   r   r   r   rA   rH   rI   r3   )r  r8   rC   r   rE   rF   r?   rG   r@   rH   r-  r   rI   r3   )r  r5  rC   r   rE   rF   r?   rG   r@   rH   r   r   r   r   rA   rH   rI   r3   )r  r:  rC   r   rE   rF   r?   rG   r@   rH   r   r   r-  r   rI   r3   )r  r   rC   r   rE   rF   r?   rG   r@   rH   r-  r   rI   r3   )r  r?  r-  r   rI   r3   )r  rB  rC   r   rE   rF   r?   rG   r@   rH   r-  r   rI   r3   )r  r   rC   r   rE   rF   r?   rG   r   r   r@   rH   r-  r   rI   r3   )
r  r   rE   rF   r?   rG   rJ  ri  rI   rj  )rE   r=   rs  rt  rI   ru  )rC   rz  rE   rF   r?   rG   r@   rH   r   r   rx  r   r   r   ry  rH   rI   r3   )rC   r  r  rH   rI   r  )rC   r  rE   rF   r?   rG   r@   rH   ry  rH   rA   rH   r  rH   rI   r3   )r  r  rI   rH   )rC   r  rE   rF   r?   rG   r@   rH   ry  rH   rI   r3   )rC   r?  rE   rF   r?   rG   r   r   r@   rH   rA   rH   rI   r3   )
rC   r8   rE   rF   r?   rG   r@   rH   rI   r3   )
rC   r7   rE   rF   r?   rG   r@   rH   rI   r3   )y
__future__r   
contextlibcollections.abcr   r   r   r   r   r	   r
   	functoolsr   	itertoolsr   r   operatorr   typingr   r   polars._reexport	_reexportri   polars._utils.construction_utilsconstructionrD  polarsr   r   polars._dependenciesr   r   r   r   r   r   r[   r   r  r   r   polars._utils.construction.utilsr   r   r   r   r   r   r    r!   polars._utils.variousr"   r#   r$   r%   polars.datatypesr&   r'   r(   r)   r*   r+   r,   r-   r.   r/   polars.exceptionsr0   r1   polars.metar2   suppressr  polars._plrr3   r4   r5   r6   r7   r8   r9   polars._typingr:   r;   r<   r=   r^   r   r|   r   r   r   r   r  r  r  r~   r  r	  tupler  r   r=  r   r  r
  r  r  r  rS  r"  r  r  r  r  r  r  r  r  rM   rS   rQ   <module>r     s   " " " " " "     8 8 8 8 8 8 8 8 8 8 4 4 4 4 4 4 4 4 4 4 4 4 $ $ $ $ $ $ ) ) ) ) ) ) ) )             
       ( ( ( ( ( ( ( ( ( ! ! ! ! ! !              - , , , , , - - - - - - . . . . . .	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	                                   A @ @ @ @ @ @ @ ( ( ( ( ( (Z%% ( (''''''( ( ( ( ( ( ( ( ( ( ( ( ( ( (  
BBBBBBBBBB(((((((($$$$$$            &* "
 '+W +/ $W W W W W Wz +/!)-W' W' W' W' W' W'x %) 	     > )-*.	/ / / / / / /j +/"&S S S S S St 	     2 '+ +/!%&5     4   ?$ ?$ ?$ ?$ ?$ ?$D &&t,, V V V V V -,Vr$ $ $ $8 &&u-- " " " " " .-"J &&w//&&t,,   -, 0/: &&s++$ $ $ ,+$0F F F F$ $ $ $H , , , , , ,^C C C CL,D ,D ,D ,D^    '+[ +/!%!&5[ [ [ [ [ [|   4 '+= +/= = = = = =@
 
 
 
, '+) +/) ) ) ) ) )\ '+E$ +/!%E$ E$ E$ E$ E$ E$T '+*.$
 $ $ $ $ $ $: '+$ +/$ $ $ $ $ $ $ $s   CC"C