
    aj|             
       #   U d dl Z d dlZd dlZd dlT d dlZd dlmc mZ d dl	m
Z
 d dlmZ ddlmZ g Zee         ed<   i Zeeef         ed<   d	 ZddZ	 	 	 ddZddZdeddfdZdededededdf
dZd Zd Zd Z e j        ee          Z de!dz  fdZ"d Z#d Z$d Z%d Z& e ej'        j(        j)        j*        ej'        j(        j+        j*        ej'        j(        j,        j*        ej'        j(        j-        j*        ej'        j(        j.        j*        ej'        j(        j/        j*        ej'        j(        j0        j*        gd          d              Z1 e ej'        j2        j3        j*        d          d!             Z4 e ej'        j(        j5        j*        gd          dd"            Z6 e ej'        j(        j7        j*        d          d#             Z8  e ej'        j(        j7        j9        d$          e8            e ej'        j(        j:        j*        d$          d%             Z; e ej'        j(        j<        j*        d&          d'             Z= e ej'        j(        j>        j*        d(          d)             Z? e ej'        j(        j@        j*        d*          d+             ZA e ej'        j(        jB        jC        d,          d-             ZD e ej'        j(        jE        j*        d          d.             ZF e ej'        j(        jG        j*        d/          d0             ZH  e ej'        j(        jI        j*        d          e$            e ej'        j(        jJ        j*        ej'        j(        jK        j*        ej'        j(        jL        j*        ej'        j(        jM        j*        ej'        j(        jN        j*        gd          d1             ZO  e ej'        j(        jP        j*        d2          eO             e ej'        j(        jQ        j*        d3          eO             e ej'        j(        jQ        jR        d4          eO            e ej'        j(        jS        j*        d          d5             ZT e ej'        j(        jU        j*        d6          d7             ZV e ej'        j(        jW        j*        d6          d8             ZX e ej'        j(        jY        j*        d9          d:             ZZ e ej'        j(        j[        j*        d;          d<             Z\ e ej'        j(        j]        j*        d=          d>             Z^ e ej'        j(        j_        j`        d?          d@             Za e ej'        j(        j_        j*        dA          dB             Zb e ej'        j(        jc        jd        dC          dD             Ze e ej'        j(        jf        j*        dE          dF             Zg e ej'        j(        jh        j*        dG          dH             Zh e ej'        j(        ji        j*        dI          dJ             Zj e ej'        j(        jk        jl        dK          dL             Zm e ej'        j(        jn        j,        dM          dN             Zo e ej'        j(        jp        j*        dO          dP             Zq e ej'        j(        jr        j*        dQ          dR             Zs e ej'        j(        jt        j*        dS          dT             Zu e ej'        j(        jv        j*        dU          dV             Zw e ej'        j(        jx        j*        dW          dX             Zy e ej'        j(        jz        j*        dY          dZ             Z{ e ej'        j(        j|        j*        d[          d\             Z| e ej'        j(        j}        j*        d]          d^             Z} e ej'        j(        j~        j        d_          d`             Z e ej'        j(        j        j*        da          db             Z e ej'        j(        j        j*        da          dc             Z e ej'        j(        j        j*        dd          de             Zdf Z e ej'        j(        j        j*        dA          dg             Z e ej'        j(        j        j        dh          di             Z e ej'        j(        j        jl        dj          dk             Z e ej'        j(        j        j*        dl          dm             Z e ej'        j(        j        j*        ej'        j(        j        j*        gd[          dn             Z e ej'        j(        j        j*        do          dp             Z e ej'        j(        j        j*        dq          dr             Z e ej'        j(        j        jl        ds          dt             Z e ej'        j(        j        jd        du          dv             Z e ej'        j(        j        j*        dw           e ej'        j(        j        j*        dw          dx                         Z e ej'        j(        j        j*        dy          dz             Z e ej'        j(        j        j,        dh          d{             Z e ej'        j(        j        j*        dA          d|             Z e ej'        j(        j        j        d}          d~             Z e ej'        j(        j        j,        d          d             Z e ej'        j(        j        j        d}          d             Z e ej'        j(        j        j,        d          d             Z e ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        gd          d             Z e ej'        j(        j        j*        ej'        j(        j        j*        gd          d             Z e ej'        j(        j        j,        d          d             Z e ej'        j(        j        j,        d          d             Z e ej'        j(        j        j*        d}          d             Z e ej'        j(        j        j*        d}          d             Z e ej'        j(        j        j*        d}          d             Z e ej'        j(        j        j*        d}          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        dQ          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        ej'        j(        j        j*        gd          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          dd            Z e ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        gd          d             Z e ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        ej'        j(        j        j*        gd          d             Z e ej'        j(        j        j        d          d             Z e ej'        j(        j        j        d          d             Z e ej'        j(        j        jd        d          d             Z e ej'        j(        j        j*        d          d             Z e ej'        j(        j        j*        d          d             Zej                            dd          5 Z(e(                    ded           e(                    ded           e(                    ded           ddd           dS # 1 swxY w Y   dS )    N)*)normalize_function)#jagged_scaled_dot_product_attention   NestedTensor__all__JAGGED_OPS_TABLEc                     | j         r8|dk    rt          j        |           j        nt          j        |           j        S |dk    rt          d          nt          d          S )Nmaxinf-inf)is_floating_pointtorchfinfor   minfloat)dtypepadding_types     _/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/torch/nested/_internal/ops.py_get_padding_valuer      sa     H&2e&;&;EK""UASASAW	
  ,u44uU|||%--G    Fc                 r    ddl m} t          |t          t          f          rU t          |           fd|D                       } t          |          t                              |                    S |r | |          }|dk    r| k     st          d| d            |dk    rdz
  n|dz
  S )Nr   canonicalize_dimsc              3   :   K   | ]}t          |          V  d S N_outer_to_inner_dim).0dndim
ragged_dims     r   	<genexpr>z&_outer_to_inner_dim.<locals>.<genexpr>#   s0      QQ.tQ
CCQQQQQQr   zdim z out of range for ndim r   )	torch._prims_commonr   
isinstancetuplelisttypedictfromkeysAssertionError)r"   dimr#   canonicalizer   outputs   ` `   r   r   r      s    555555#t}%% 3cQQQQQSQQQQQtF||DMM&11222 +c**1HHtFCFFFFGGG ((
Qar   Tc                     ddl m}  || |          }||k    r|st          | d          |dk    r|st          | d          |rt          | ||          n|}	|r
|dk    }
|	|
fS |	S )Nr   r   z0(): not supported for NestedTensor on ragged dimz+(): not supported for NestedTensor on dim=0)r%   r   RuntimeErrorr   )r"   r-   r#   op_nameconvert_to_inner_dimallow_ragged_dimallow_batch_dimr   wrappedretoperating_on_batchs              r   _wrap_jagged_dimr9   5   s     655555c**G*%5gWWWXXX	AogRRRSSS  	D':666 
  ) %\'((Jr   c                 b   	 ddl m	 t          |t          t          f          st          dt          |                     	 fd|D             }d|v }|v }t          fd|D                       }t          t          	                     fd|D                                 }||||fS )z
    For NestedTensor operators,
    wraps dimensions to non-negative values,
    and returns metadata related to reduction dimension(s).
    r   r   z<_wrap_jagged_dims(): cannot iterate over dimensions of type c                 (    g | ]} |          S  r<   )r    r!   r   r"   s     r   
<listcomp>z%_wrap_jagged_dims.<locals>.<listcomp>_   s4       '($""  r   c              3   0   K   | ]}|d k    o|k    V  dS )r   Nr<   )r    r!   
ragged_idxs     r   r$   z$_wrap_jagged_dims.<locals>.<genexpr>e   s0      PPaqAv9!z/PPPPPPr   c              3   :   K   | ]}t          |          V  d S r   r   )r    r!   r"   r?   s     r   r$   z$_wrap_jagged_dims.<locals>.<genexpr>i   s0      UU1)$:>>UUUUUUr   )
r%   r   r&   r'   r(   r,   r)   anyr*   r+   )
r"   dimsr2   r?   wrapped_dimsoperate_on_batchoperate_on_raggedoperate_on_non_batchouter_to_inner_dimr   s
   `  `     @r   _wrap_jagged_dimsrH   R   s    655555dUDM** 
W4PT::WW
 
 	
    ,0  L L("l2PPPP<PPPPP UUUUUUUUUU  /1BDXXXr   
schema_strreturnc                    |                      d          }d |D                                 d          }t          |          |z
  }|d         dk    r|d d         }nqt          |          |k    r t          |          t          |          k    s>t          d|j         d|  d| d	t          |           d
t          |           d          d d d d d}t          |          D ]\  }}	|	                     d          \  }
}|                    d          }|r
|d d         n|}||vrt          d|           |t          |          k    r |st          d|j         d|  d|
           ||         |ffd	} |||                   s.dddddd}t          d|j         d|  d|
 d||                    d S )N, c                 8    g | ]}|                     d           S )?)endswithr    xs     r   r=   z check_schema.<locals>.<listcomp>q   s"    BBBQCBBBr   T...NestedTensor (z): expected at least z arguments and at most z arguments, but got: z
 argumentsc                 b    t          | t          j                  ot          | t                     S r   )r&   r   Tensorr   rQ   s    r   <lambda>zcheck_schema.<locals>.<lambda>   s$    z!U\22V:a;V;V7V r   c                 T    t          | t                    o| j        d u o
| j        dk    S Nr   )r&   r   _lengths_ragged_idxrX   s    r   rY   zcheck_schema.<locals>.<lambda>   s1    
1l33 J$MQ r   c                 ,    t          | t                    S r   r&   r   rX   s    r   rY   zcheck_schema.<locals>.<lambda>   s    J|
 
 r   c                     dS )NTr<   rX   s    r   rY   zcheck_schema.<locals>.<lambda>   s     r   )tjtjt_allrA   z: rN   zUnknown arg type: z) missing required argument: c                 <    |r| d u p
 |           S  |           S r   r<   )rQ   is_optional	_check_fns     r   check_fnzcheck_schema.<locals>.check_fn   s0     $Dy0IIaLL0 y||#r   tensorzoptional tensorz%contiguous jagged layout NestedTensorzjagged layout NestedTensorz
<any type>)ra   zt?rb   rc   rA   z): expected z	 to be a )splitcountlen
ValueError__name__	enumeraterO   r,   )rI   funcargskwargsnamed_arg_typesnum_optional_argsmin_argsarg_type_check_fnsinamed_arg_typenamearg_typere   normalized_arg_typerg   type_to_descrf   s                   @r   check_schemar|   o   s    &&t,,OBB/BBBHHNN?##&77H re##)#2#.D		X%%#d))s?7K7K*K*K) ) )
 ) )QY ) )),_)=)=) )t99) ) )   WV 
 
 ~	 	 '77 # #>'--d33h'',,/:Hhssmm&888 !K6I!K!KLLLD		>>  9DM 9 9J 9 9269 9   &':;	$/ 	$ 	$ 	$ 	$ 	$ 	$ xQ   	'=6# L , , ,
 , , , ,), ,  	/# #r   aa_namebb_namec           	          |j         |j                 |j         |j                 k    rt          d| j         d| d| d          d S )NrT   z: expected  and z' to have the same exact offsets tensor.)_sizer]   r1   rm   )ro   r}   r~   r   r   s        r   check_ragged_dim_samer      sk     	wq}!777)DM ) )f ) )6 ) ) )
 
 	
 87r   c                     | j         dz   }| j        d |         }|d |         }t          |          t          |          k    o&t          d t	          ||          D                       S )Nr   c              3   4   K   | ]\  }}||k    p|d k    V  dS )rR   Nr<   )r    nsss      r   r$   z%raggedness_matches.<locals>.<genexpr>   s3      HH52qB!GqBwHHHHHHr   )r]   r   rk   allzip)ntsizeend	nt_raggedsize_raggeds        r   raggedness_matchesr      sn    
.1
C#Itt*Ky>>S--- HHC	;,G,GHHHHHr   c                     |                                  dk    rO| j        d         dk    r>|                     d          } |                                  dk    r| j        d         dk    >| S Nr   r   )r-   shapesqueeze)ra   s    r   squeeze_leading_onesr      sX    & %%''A++!'!*//IIaLL %%''A++!'!*//Hr   c                 z     t          t                    sgt           t                    s g  fd}|S )Nc                 F     D ]} fd}D ]} ||          ||<    S )Nc                       fd}|S )Nc                  >    t          g| R i |  g| R i |S r   )r|   )rp   rq   aten_opro   rI   s     r   innerz@register_func.<locals>.wrapper.<locals>.get_inner.<locals>.inner   sC     TCDCCCFCCC49$999&999r   r<   )r   r   ro   rI   s   ` r   	get_innerz1register_func.<locals>.wrapper.<locals>.get_inner   s.    : : : : : : : r   r<   )ro   r   r   tableaten_opsrI   tabless   `   r   wrapperzregister_func.<locals>.wrapper   sa     
	4 
	4G        4 4!*7!3!3g4r   )r&   r(   )r   r   rI   r   s   ``` r   register_funcr      sa    h%% :fd##        Nr   c                    t                               | d           }||S t          j        j        | j        v rAddlm} |D ]'} ||          rt          d| j	         d|           (t          d |D                       }|dk    rg }| j        j        D ]]}t          |j        t          j                  r|                    |j         d            n|                    |j         d           ^|                    d	           d
                    |          }t'          || g|R i | t)          j        t,          |           S |dk    r+t'          d| g|R i | t)          j        t.          |           S d S )Nr   is_nested_intrT   z: invalid argument c              3   J   K   | ]}t          |t          j                  V  d S r   )r&   r   rW   rP   s     r   r$   z lookup_jagged.<locals>.<genexpr>
  s.      HHajEL99HHHHHHr   r   z: jt_allz: anyrS   rL      zlhs: any, rhs: any, ...)r
   getr   Tag	pointwisetags%torch.fx.experimental.symbolic_shapesr   r1   rm   sum_schema	argumentsr&   r)   
TensorTypeappendrx   joinr|   	functoolspartialjagged_unary_pointwisejagged_binary_pointwise)	ro   rp   rq   dispatch_funcr   argnum_tensor_argsschema_partscheck_schema_strs	            r   lookup_jaggedr      s   $((t44M  ydi''GGGGGG
  	 	C}S!! "KDMKKcKK   HH4HHHHHa L|- < <ch(899 < ''38(=(=(=>>>E ''38(:(:(:;;;;&&&#yy66)4A$AAA&AAA$%;TBBB!!2DJ4JJJ6JJJ$%<dCCC4r   c                 p    |                                  |                                 | j        | j        d}|S )Noffsetslengths_metadata_cacher]   r   )r   rq   s     r   extract_kwargsr      s6    ;;==;;==.	 F Mr   c                     t          d |D                       t           | fd|D             i |fi t                    S )Nc              3   D   K   | ]}t          |t                    |V  d S r   r_   r    r   s     r   r$   z)jagged_unary_pointwise.<locals>.<genexpr>,  s1      DDsjl&C&CDsDDDDDDr   c              3   2   K   | ]}|u r|j         n|V  d S r   _values)r    r   njts     r   r$   z)jagged_unary_pointwise.<locals>.<genexpr>.  s1      BBccSjjs{{cBBBBBBr   nextr   r   )ro   rp   rq   r   s      @r   r   r   *  sm    
DDdDDD
D
DCBBBBTBBBMfMM 


  r   c                 l   |d         |d         }}t          |t                    s$t          |t                    st          d          d}t          |t                    rt          |t                    rt          ||j                  r7t           | |j        |j        g|dd          R i |fi t          |          S t          |                    | j	        |j        |j                            t          |t                    }|rt          |          nt          |          }|r||fn||f\  }|
                                
                                k    rt          d          t          |          }	
                                |	
                                dz   k    r7|r	j        |	fn|	j        f\  }
}t           | |
|g|dd          R i |fi |S |
                                |
                                k    r|j        d         |j        d         k    r3t          |                    | j	        |j        |j                            ddlm} j        }j        }|j        j        j        dz
           }||                    fdt)          |j                  D                       } ||j        j        |||	          }|r|fn|f\  }
} | |
|g|dd          R i |S t          |                    | j	        |j        |j                            )
Nr   r   z4At least one of the arguments must be a NestedTensorzHcannot call binary pointwise function {} with inputs of shapes {} and {}r   z+NYI: broadcasting NT with T with larger dimnested_from_paddedc                 4    g | ]\  }}|j         k    r|nS r<   r]   )r    rv   rQ   r   padded_max_Ss      r   r=   z+jagged_binary_pointwise.<locals>.<listcomp>n  s-    WWWDAq!r~%%QQ<WWWr   r   r?   sum_S
min_seqlen
max_seqlen)r&   r   r,   r   r   r   r   r1   formatrm   r-   NotImplementedErrorr   r   nested_tensorr   _maybe_min_seqlen_maybe_max_seqlenr]   expandrn   _offsets)ro   rp   rq   r}   r   mismatch_error_msga_is_ntextracted_kwargsra   
t_squeezedlhsrhsr   r   r   total_Lt_as_ntr   r   s                    @@r   r   r   3  s   7DGqAq,'' U:a+F+F USTTT 	S  !\"" Wz!\'B'B W a)) 	QY	?DH????? CQRSCTCT   -44T]AGQWUUVVVL))G,3J~a(((9J9J )QFFAq6EBuuww!"OPPP%a((J	vvxx:>>##a'''/6TBJ
++Z<TSDDc?DH?????TTCSTTT 	uuww!%%'' 71:##"))$-!'JJ   	655555 )
)
!*"2>A#56"L HHWWWWWIagDVDVWWW
 
 %$K~!!
 
 
 %,>B=='2StC2tABBx2226222 )00QQ
R
RRr   c                 H   | t           j        j        j        u rt	          |i |S | j        dk    r' | |d         j        g|dd          R i | |d         S | j        dk    rddd}t          |||d	          \  }}|                    d
          }t          |
                                |d         |j        dd          }t          |
                                |d         |j        dd          }||k    r|S t          j        t          j        |j        ||dz                      }	g |j        d |         |	|j        |dz   d          R }
 |j        |
 S | j        dk    r|d         j        rS                                 \  }}t           j                                        5  |D ])}t+          |d           }||                                 *	 d d d            n# 1 swxY w Y   S | j        dk    rG|d         j        rdS                                 \  }}|sdS t/          fd|D                       S | j        dk    rddd}t          |||d	          \  }}|                    d
          }|                    d          }|
                                |j        z
  dz
  }t1          |          |k    rt3          d          t           j                                        5   | |i |cd d d            S # 1 swxY w Y   t5          |           )Napply_r   r   flattenrR   rJ   c                     d S r   r<   )input	start_dimend_dims      r   _flatten_sigz+jagged_torch_function.<locals>._flatten_sig      Dr   Trp   rq   normalize_to_only_use_kwargsr   r   F)r3   r   share_memory_	is_sharedc              3      K   | ]8}t          |          d uo!t          |                                          V  9d S r   )getattrr   )r    rx   r   s     r   r$   z(jagged_torch_function.<locals>.<genexpr>  sa       
 
 BT)Kgb$.?.?.I.I.K.K
 
 
 
 
 
r   rms_normc                     d S r   r<   )r   normalized_shapeweightepss       r   _rms_norm_sigz,jagged_torch_function.<locals>._rms_norm_sig  r   r   r   zNrms_norm(): Normalization over the ragged dim not supported for nested tensors)r   rR   rJ   NNN)r   _C_nnscaled_dot_product_attentionr   rm   r   r   popr9   r-   r]   r   reduceoperatormulr   reshapeis_cuda__tensor_flatten__DisableTorchFunctionSubclassr   r   r   rk   rl   r   )ro   rp   rq   r   _
new_kwargsinpr   r   product	new_shapenamesrx   	componentr   r   max_normalizabler   s                    @r   jagged_torch_functionr    sa    ux|8882DCFCCC}  T!W_2tABBx2226222Aw }	!!	 	 	 	 	 +tFQU
 
 
: nnW%% %GGII{#O!&
 
 
	 #GGIIy!O!&
 
 
 J"8<9wQR{;R1STTPci

+PWPsy17OPP	s{I&& }''!W: 	I((**qX2244 	. 	. . .#Bd33	(++---.	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.
 	 }##!W: 	5((**q 	5 
 
 
 

 
 
 
 
 	
 }
""	 	 	 	 	 +VRV
 
 
: nnW%%%>>*<== 7799s6:  #333`   X2244 	) 	)4(((	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) 	) d
#
##s$   --G''G+.G+5L

LLzself: jt_allc                 8   | t           j        j        j        j        u rdS | t           j        j        j        j        u r|d         j        S | t           j        j        j        j        u rt          |d         j                  S | t           j        j        j	        j        t           j        j        j
        j        fv r{|d         j        Ot          t          |d         j                  t          j        |d         j        dd                    z            S |d         j        
                                S | t           j        j        j        j        u r|d         j        S | t           j        j        j        j        u r|d         j                                        S d S )NFr   r   )r   opsatenis_non_overlapping_and_densedefaultsym_sizer   r-   rk   	sym_numelnumelr\   intr   mathprodr   
sym_stride_stridessym_storage_offsetstorage_offsetro   rp   rq   s      r   tensor_attr_supported_getterr    sC    uy~:BBBuuy~&...Aw}uy~!)))47=!!!	(0%).2F2NOOO7's47+,,tyaqrr9J/K/KKLLLAw$$&&&uy~(000Awuy~0888Aw--/// 98r   c                     t           j        S r   )r   jaggedr  s      r   prim_layout_defaultr"  
  s
    <r   c                 ^    | t           j        j        j        j        u rt          d          d S )NznNestedTensor does not support directly calling torch.ops.aten.size; please use `nested_tensor.size()` instead.)r   r  r  r   r  r1   r  s      r   tensor_attr_unsupported_getterr$    s8    
 uy~"***9
 
 	
 +*r   c                 "   ddl m} t          | ||d          \  }}|                    d          }|                                dS |                    dt          j                  |d<   |d         t          j        k    rdS  ||j	        fi |S )Nr   )is_contiguous_for_memory_formatTr   r   Fmemory_format)
r%   r&  r   r   r   r   r   contiguous_formatpreserve_formatr   )ro   rp   rq   r&  r  r  r  s          r   is_contiguous_generalr*    s    CCCCCC&4T  MAz ..
!
!C {{}} u",..0# #J /"e&;;;t**3;EE*EEEr   z!self: jt_all, memory_format: any?c                 F   t          | ||d          \  }}|                    d          }|                                dS |                    dt          j                  |d<   |d         t          j        k    rdS t	          j        j        j	        j
        |j        fi |S )NTr   r   Fr'  )r   r   r   r   r   r(  r)  r  r  sym_is_contiguousr  r   ro   rp   rq   r  r  r  s         r   sym_is_contiguous_generalr.  5  s     '4T  MAz ..
!
!C {{}} u",..0# #J /"e&;;;t9>+3CKNN:NNNr   z"input: jt_all, memory_format: any?c                 j   t          | ||d          \  }}|                    d          }t          |          }|j        Z|d         t          j        k    rDddlm} |j        dk    rt          d           ||
                                d           \  }}|S t           | |j        fi |fi |S )	NTr   r   r'  r   )jagged_from_listz;NJT with ragged_idx != 1 not supported for contiguous clone)r   )r   r   r   r\   r   r(  r   r0  r]   r,   unbindr   r   )	ro   rp   rq   r  r  r  new_metar0  contigs	            r   clone_defaultr4  L  s     '4T  MAz ..
!
!Cc""H
|o&%*AAA 877777 !##$Q   )(tDDDIFAMS[77J77DD8DDDr   zinput: jt, weight: t, bias: t?c                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S NTr   r   r   r   r   r   r   r-  s         r   linear_defaultr8  i  h    &4T  MAz ..
!
!CS[77J77OO>#;N;NOOOr   z6self: jt, grad_output: jt, weight: t, output_mask: anyc                 v   t          | ||d          \  }}|                    d          }|                    d          }|                    d          }|                    d          }d\  }	}
}t          | |d|d           |d	         r2t          t	          j        |j        |          fi t          |          }	|d
         r|j                            d|	                    d	                    }|j                            d|	                    d
                    }t	          j        |
                                |          }
|d         ret          t          |j        j        d
z
                      }|dk    r|j                                        }nt	          j        |j        |d          }|	|
|fS )NTr   r   grad_outputr   output_mask)NNNselfr   r   rR   r   r<   F)keepdim)r   r   r   r   r   matmulr   r   r  r   ra   r'   ranger"   cloner   )ro   rp   rq   r  r  r  r;  r   r<  dsdwdbgrad_2dinput_2dreduce_dimss                  r   linear_backward_defaultrH  t  s   
 '4T  MAz ..
!
!C..//K^^H%%F..//K!JBB$V[-HHH1~ 
L,f55
 
9G9T9T
 
 1~ 1 %--b&++a..AA;&&r6;;q>>::\'))++x001~ L E+"5":Q">??@@"$**,,BB;.UKKKBB<r   zinput: jt_all, dtype: anyc                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S r6  r7  r-  s         r   to_dtyperJ    r9  r   c                 ^   ddl m} t          | ||d          \  }}|                    d          }|                    d            | |j        fi |}|j                            |j                  }d }	|j         |j                            |j                  }	dd	l	m
}
 dd
lm}m} |j        |j        n|j        }|	|n|	}t          ||
|f          r# ||          } ||          }|j        |_        n||         ||<   t!          |          }||d<   |	|d<   t#          |fi |}|S )Nr   _tensor_symint_registryTr   r   layoutdevicer   
FakeTensorFunctionalTensormb_unwrap_functional_tensorr   r   )r   rM  r   r   r   r   torP  r\   torch._subclasses.fake_tensorrR  #torch._subclasses.functional_tensorrT  rU  r&   nested_int_memor   r   )ro   rp   rq   rM  r  r  r  
new_valuesnew_offsetsnew_lengthsrR  rT  rU  ragged_source	new_thingtgtsrc
inp_kwargsr/   s                      r   to_copy_defaultrb    s   666666&4T  MAz ..
!
!CNN8ck00Z00J,//):/;;KK
|looZ->o??888888       
 %(L$8CLLclM*2I)j*:;<< T)))44))-88!1-D]-S	*$$J'Jy'Jy*33
33FMr   z-self: jt_all, src: jt_all, non_blocking: any?c                    t          | ||d          \  }}|                    d          }|                    d          }|j        |j        k    r|                                }d |D             }|                                }	d |	D             }
||
k    rt	          d|j         d|j                   t          ||	          D ]\  }}|                    |           |j                            |j                   |S )	NTr   r   r`  c                     g | ]	}|j         
S r<   r   r    cs     r   r=   z copy_default.<locals>.<listcomp>      666q17666r   c                     g | ]	}|j         
S r<   re  rf  s     r   r=   z copy_default.<locals>.<listcomp>  rh  r   z<copy_(): expected compatible input and src shapes, but got: r   )	r   r   r   r1  r1   r   r   copy_r   )ro   rp   rq   r  r  r  r`  	inp_compsinp_comp_shapes	src_compssrc_comp_shapesinp_compsrc_comps                r   copy_defaultrq    s.    '4T  MAz ..
!
!C
..

C
yCI JJLL	66I666JJLL	66I666o--/9/ /#&9/ /   #&i";"; 	% 	%HhNN8$$$$ Kck"""Jr   c                 ~   t          | ||d          \  }}|                    d          }t          j        |d<    | |j        fi |}|j                            |j                  }d }|j         |j                            |j                  }t          |          }	d|	v r||	d<   d|	v r||	d<   |j        |j        k    rodd	l
m}
 dd
lm}m} ddlm} |j        |j        n|j        }||n|}t#          ||
|f          r# ||          } ||          }|j        |_        n||         ||<   t'          |fi |	S )NTr   r   rN  rO  r   r   r   rQ  rS  r   rL  )r   r   r   stridedr   r   rV  rP  r\   r   rW  rR  rX  rT  rU  r   rM  r&   rY  r   )ro   rp   rq   r  r  r  rZ  r[  r\  output_kwargsrR  rT  rU  rM  r]  r^  r_  r`  s                     r   like_factory_defaultru    s    '4T  MAz ..
!
!C
 !=Jxck00Z00J,//):/;;KK
|looZ->o??"3''MM!!#.i M!!#.i 
zZ&&& 	=<<<<<	
 	
 	
 	
 	
 	
 	
 	

 	;:::::(+(<#,#.#6KKK	i*.>!?@@ 	X--i88C--m<<C"%"5C1H1W#I.
44m444r   zself: jt_all, fill_value: anyzself: jt_all, high: anyz!self: jt_all, low: any, high: anyc                 |    t          | ||d          \  }}|                    d          } | |j                   |S r6  r   r   r   r-  s         r   zero__defaultrx  >  sM    &4T  MAz ..
!
!CDJr   z*self: jt_all, dim: any, half_to_float: anyc           
      (   t          | ||d          \  }}t          |d         t                    rt          d          |                    d          }t          |                                |d         fd|j                  \  |d<   }}}|rt          d          |r|j        dk    rt          d	          |r|j        t          d
          |d         d         |d<   |rt          j
        j                            t          j        j                            |j                            |j        j        d         d          |j        g|j        gt+          d                    |j                  }	t          j        j                            |	|j        g|j        j        d                   j        dg|j        j        dd          R  }
t/          |
fi t1          |          S t/           | |j        fi |fi t1          |          S )NTr   r-   zHsoftmax(): not supported for dimensions of type 'tuple' for NestedTensorr   softmaxzRsoftmax(): not supported when reducing across the batch dimension for NestedTensorr   zesoftmax(): not supported when reducing along the ragged dimension for ragged_idx > 1 for NestedTensorzksoftmax(): not supported where lengths is not None if reducing across the ragged dimension for NestedTensorr   rR   r   )max_lengthspadding_valuer-   r   )r   r&   r'   r1   r   rH   r-   r]   r\   r   nn
functionalrz  r  r  _jagged_to_padded_dense_forwardr   r  r   r   _max_seqlenr   _padded_dense_to_jagged_forwardr   r   )ro   rp   rq   r  r  r  reduce_on_batchreduce_on_ragged_reduce_on_non_batchpadded_softmax_valuessoftmax_valuess              r   _softmax_defaultr  I  s}    '4T  MAz *U#U++ 
V
 
 	
 ..
!
!C 				E			 	5  
`
 
 	
  
COa//s
 
 	
  
CL4I
 
 	

 #5)	Ju  C % 3 ; ;IN::##K%a("   _-#Fmm ;    !< 
!
 
!
 GG!\NK% H 
 
 
 "122&
 
 
 NBBnS.A.ABBBS[77J77OO>#;N;NOOOr   c                    t          | ||d          \  }}t          |d         t                    rt          d          |                    d          }t          |                                |d         fd|j                  \  |d<   }}}|rt          d          |rt          d          |d         d	         |d<   t           | |j	        fi |fi t          |          S )
NTr   r-   zLlog_softmax(): not supported for dimensions of type 'tuple' for NestedTensorr   log_softmaxzVlog_softmax(): not supported when reducing across the batch dimension for NestedTensorzVlog_softmax(): not supported when reducing along the ragged dimension for NestedTensorr   )r   r&   r'   r1   r   rH   r-   r]   r   r   r   )	ro   rp   rq   r  r  r  r  r  r  s	            r   _log_softmax_defaultr    s,    '4T  MAz *U#U++ 
Z
 
 	
 ..
!
!C 			Ju%'	 	5
  
d
 
 	
  
d
 
 	

 #5)!,JuS[77J77OO>#;N;NOOOr   z7grad_output: jt, output: jt, dim: any, input_dtype: anyc                     t          | ||d          \  }}|                    d          }|                    d          }t           | |j        |j        fi |fi t	          |          S )NTr   r;  r/   r7  )ro   rp   rq   r  r  grad_outr/   s          r   _softmax_backwardr    s    
 '4T  MAz ~~m,,H^^H%%FXv~<<<< @Nx@X@X  r   z!self: jt, float: any, train: any?c                     t          | ||d          \  }}|                    d          } | |j        fi |\  }}t          |fi t	          |          t          |fi t	          |          fS r6  r   r   r   r   r   )ro   rp   rq   r  r  r  out1out2s           r   native_dropout_defaultr    s     '4T  MAz ..
!
!Cck00Z00JD$T11^C0011T11^C0011 r   z%grad_output: jt, mask: jt, scale: anyc                     t          | ||d          \  }}|                    d          }|                    d          }t           | |j        |j        fi |fi t	          |          S )NTr   r;  maskr7  )ro   rp   rq   r  r  r;  r  s          r   native_dropout_backward_defaultr    s    
 '4T  MAz ..//K>>&!!D[ $,==*== 

%
%  r   z2self: jt_all, dim: any, keepdim: any?, dtype: any?c                 &    t          | ddg|R i |S )Nr  r   _apply_reductionr  s      r   prod_dim_intr    s&    
 D&!=d===f===r   zself: jt_all, dtype: any?c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r6  rw  r-  s         r   prod_defaultr    P    &4T  MAz ..
!
!C4**z***r   z$self: jt, split_size: any, dim: any?c                    t          | ||d          \  }}|                    d          t                                          |d         j        d          |d<   t          fd | j        fi |D                       S )NTr   r   r-   ri   c              3   N   K   | ]}t          dd |it                    V   dS )valuesNr<   r   r   r    rQ   r  s     r   r$   zsplit_tensor.<locals>.<genexpr>  sR         	55A5!4!455     r   )r   r   r9   r-   r]   r'   r   r-  s        @r   split_tensorr    s     '4T  MAz ..
!
!C(		:e$cow Ju     ck00Z00     r   z%self: jt, split_sizes: any, dim: any?c                     t          | ||d          \  }}|                    d          t                                          |d         j        d          |d<   fd | j        fi |D             S )NTr   r   r-   split_with_sizesc           	      F    g | ]}t          dd |it                    S r  r<   r  r  s     r   r=   z,split_with_sizes_default.<locals>.<listcomp>   sD        	55A5!4!455  r   )r   r   r9   r-   r]   r   r-  s        @r   split_with_sizes_defaultr    s     '4T  MAz ..
!
!C(		:e$co7I Ju   ck00Z00   r   z+self: jt, dim: any, start: any, length: anyc                 *   t          | ||d          \  }}|                    d          }t          |                                |d         |j        d          } | |j        ||d         |d                   }t          |fi t          |          S )	NTr   r   r-   narrowstartlengthr-   r  r  )r   r   r9   r-   r]   r   r   r   )ro   rp   rq   r  r  r  r-   r  s           r   r  r  &  s     '4T  MAz ..
!
!C
37799j&7(
S
SCT!(#	  F 66."5"5666r   z self: jt, chunks: any, dim: any?c                 Z   t          | ||d          \  }}|                    d          t                                          |d         j        dd          \  |d<   }|r|d         }j                                        }|                    |          }d |D             }	d	 |	D             }	fd
|	D             d |D             }
j        	                    |
          fdt          t                              D             S fd | j        fi |D             S )NTr   r   r-   chunkr5   chunksc                 :    g | ]}t          j        |d           S )r   r}  )r   cumsumrP   s     r   r=   z!chunk_default.<locals>.<listcomp>K  s'    KKKa5<q111KKKr   c                 <    g | ]}t          j        |d d          S )r   r   r   )value)FpadrP   s     r   r=   z!chunk_default.<locals>.<listcomp>L  s)    NNN15F!444NNNr   c                 $    g | ]}|j         d S ))r   r]   r   )r    per_offsetsr  s     r   r=   z!chunk_default.<locals>.<listcomp>M  s3     
 
 
 $CODD
 
 
r   c                 Z    g | ](}|                                                                 )S r<   )r   itemrP   s     r   r=   z!chunk_default.<locals>.<listcomp>S  s(    ???!quuww||~~???r   c                 D    g | ]}t          dd |         i|         S r  r   )r    rv   chunk_valuesnested_kwargss     r   r=   z!chunk_default.<locals>.<listcomp>X  sE     
 
 
 FFQFM!4DFF
 
 
r   c           	      F    g | ]}t          dd |it                    S r  r  r  s     r   r=   z!chunk_default.<locals>.<listcomp>]  sD     
 
 
 999^C%8%899
 
 
r   )r   r   r9   r-   r]   r   diffr  r   ri   r@  rk   )ro   rp   rq   r  r  r8   r  r   chunked_lengthschunked_offsetssplit_sizesr  r  r  s              @@@r   chunk_defaultr  9  s   &4T  MAz ..
!
!C,<		:e$cowPT- - -)Ju)  
H% ,##%%!--//KK?KKKNNoNNN
 
 
 
.
 
 
 @????{((55
 
 
 
 
3|,,--
 
 
 	


 
 
 
T#+4444
 
 
 	
r   zself: jt_all, dim: any?c                    
 t          | ||d          \  }}|d         }|dk    rt          d          |                    d          }|                                |                                }|                                }|j        ddt          t                   dt          t                   d z  d	d ffd
}	|K|	                                
                                
 |	
           t          j        
dz
            S dk    rt          d          |
                                
|
                                 |	
           
fdt          |j        d                   D             S )NTr   r-   r   z2unbind(): only supported for NestedTensor on dim=0r   r\   r   rJ   c                 <   d}j         dz
           }t          t          |                     D ]t}t          j        | |         dk               t          j        | |         |k               || |         z  }|)t          j        ||         | |         z   |k    d            ut          j        ||k               |[t          t          |                    D ]@}t          j        ||         dk               t          j        ||         |k               ?d S d S )Nr   r   c                      dS )NzMunbind(): nested tensor offsets and lengths do not match ragged_idx dimensionr<   r<   r   r   rY   z2unbind_int.<locals>._torch_check.<locals>.<lambda>  s    k r   )r   r@  rk   r   _check)r\   r   lengths_sumragged_dim_sizerv   r?   r  s        r   _torch_checkz unbind_int.<locals>._torch_checkt  s.     ,zA~6s8}}%% 		 		AL!)***L!78888A;&K#QK(1+-@kk   	[O34443x==)) = =Xa[A-...Xa[O;<<<<  = =r   r   r}  zAunbind(): nested tensor ragged_idx out of bounds (should be >= 1)c           	      ^    g | ])}t          j        d z
  |         |                   *S )r   r  )r   r  )r    rv   lengths_scalarsoffsets_scalarsr?   r  s     r   r=   zunbind_int.<locals>.<listcomp>  sV         	a!!$"1%		
 	
 	
  r   r   )r   r1   r   r  r   r   r]   r(   r  r  tolistr   ri   r@  r   )ro   rp   rq   r  r  r-   r  r   r   r  r  r  r?   r  s             @@@@r   
unbind_intr  c  s    '4T  MAz U
C
axxOPPP
..
!
!CZZ\\FkkmmGkkmmGJ= =tCy =DI4D =PT = = = = = = =: !,,..//11_%%%{6?aIIIIQO
 
 	
 nn&&Onn&&OL/222       w}Q'((   r   zself: jt, dim: anyc                    t          | ||d          \  }}|                    d          }|j        }t          t	          |j                  |d         |j        d          |d<   t           | |fi |fi t          |          S )NTr   r   r-   r   )	r   r   r   r9   rk   r   r]   r   r   )ro   rp   rq   r  r  r  r  s          r   squeeze_dimr    s    &4T  MAz ..
!
!C[F(CI
5)3?I Ju V22z22JJnS6I6IJJJr   zself: jt_all, dim: anyc                 f   t          | ||d          \  }}|                    d          }|j        }|d         }t          t	          |j                  dz   ||j        dd          |d<   t          |          }|d         |j        dz
  k    r|dxx         dz  cc<   t           | |fi |fi |S )	NTr   r   r-   r   	unsqueeze)r4   r]   )	r   r   r   r9   rk   r   r]   r   r   )	ro   rp   rq   r  r  r  r  r-   rt  s	            r   unsqueeze_defaultr    s    &4T  MAz ..
!
!C[F U
C(CIC+PT  Ju
 #3''M%COa///m$$$)$$$V22z22DDmDDDr   ztensors: any, dim: any?c                    t          | ||d          \  }}|                    d          }d |D             }t          |          dk    rt          d          |d         fd|D             }|d         }t	          t          j                  |j        d	          |d<   t           | d
 |D             fi |fi t          |d                   S )NTr   tensorsc                      g | ]}|j         	|S r<   )	is_nestedr    ra   s     r   r=   zcat_default.<locals>.<listcomp>  s    000AAK0a000r   r   z"At least one tensor must be nestedc                 L    g | ] }|j         r|n|                              !S r<   )r  	expand_as)r    ra   firsts     r   r=   zcat_default.<locals>.<listcomp>  s0    IIIAAK7qqQ[[%7%7IIIr   r-   catc                     g | ]	}|j         
S r<   r   r  s     r   r=   zcat_default.<locals>.<listcomp>      )))Aai)))r   )	r   r   rk   r,   r9   r   r]   r   r   )	ro   rp   rq   r  r  r  nestedr-   r  s	           @r   cat_defaultr    s   &4T  MAz nnY''G 10000F
6{{aABBB1IEIIIIIIIG U
C(EK#u0% Ju )))))88Z88 <J7ST:<V<V  r   zself: any, other: anyc                     t           ||d          \  }}|                    d          }|                    d          } fd} fd}|j        r |j        s|                                dk    rV|                                |                                k    r,|j        |                                dz
  k     r |||          S |                                d	k    rq|                                |                                k    rG|j        |                                dz
  k     r't            |j        |fi |fi t          |          S n|j        s|j        r|                                dk    rA|                                |                                k    r|j        d	k    r |||          S |                                d	k    r\|                                |                                k    r2|j        d	k    r't            ||j        fi |fi t          |          S n|j        r|j        r|                                dk    r[|                                dk    rCt          ||j	                  r.t            |j        |j                  fi t          |          S |                                dk    r|                                dk    rj|j        d	k    r_|j        dk    rT|
                    |j                  |
                    |j                  k    rt          j         |||                    S t          d
|j	         d|j                   )NTr   r   otherc                     fdt          |                                 |                                          D             S )Nc                 .    g | ]\  }} ||          S r<   r<   )r    a_compb_compro   s      r   r=   z8matmul_default.<locals>._unbind_impl.<locals>.<listcomp>  s6     
 
 
%5ffDD  
 
 
r   )r   r1  )r}   r   ro   s     r   _unbind_implz$matmul_default.<locals>._unbind_impl  sH    
 
 
 
9<QXXZZ9T9T
 
 
 	
r   c                    | j         r| }n|}ddlm} |j        }|j        }|}|j        j        |j        dz
           }||}g |j        d |j                 ||j        |j        dz   d          R }|                    d|          }	| j         r |	|          }
n | |	          }
 ||
|j	        |j        |||          S )Nr   r           output_sizer   )
r  r   r   r   r   r   r   r]   to_padded_tensorr   )r}   r   r   r   r   r   r   r   padded_shape	padded_ntpadded_tro   s              r   _padded_implz$matmul_default.<locals>._padded_impl  s"   ; 	BBB555555)
)
!*"2>A#56"L
X&&'

 Xbnq(**+
 

 '''FF	; 	*tIq))HHtAy))H!!K~!!
 
 
 	
r      r   r   z1matmul(): not supported between inputs of shapes r   )r   r   r  r-   r]   r   r   r   r   r   r   r   stackr1   r   )	ro   rp   rq   r  r  r  r  r  r  s	   `        r   matmul_defaultr    s{   &4T  MAz ..
!
!CNN7##E
 
 
 
 

!
 !
 !
 !
 !
J } 49U_ 49 GGIINN		UYY[[((#''))a-//  <U+++ IIKK1		EIIKK''#''))a-//S[%66:66 :H:M:M   ] 9u 999;;!		swwyy 8 8U=NRS=S=S<U+++
 WWYY!^^		cggii 7 7E<MQR<R<RS%-66:66 :H:O:O  
 
 95? 9 7799q==UYY[[1__1CC1U1U_S[%- @ @XXNSVDWDWXXX GGIINN		q  1$$!Q&&))UZZ8I-J-JJJ ;||C77888
YCIYYEKYY  r   zself: jt_all, mat2: anyc                 r   t          | ||d          \  }}|                    d          }|                    d          }|                                dk    rt          d          |                                dk    rt          d          t	          t
          j        j        j        j	        ||          S )NTr   r   mat2r  zbmm(): input must be 3Dzbmm(): mat2 must be 3D)
r   r   r-   rl   r  r   r  r  r?  r  ro   rp   rq   r  r  r  r  s          r   bmm_defaultr  Z  s    &4T  MAz ..
!
!CNN6""E
wwyyA~~2333yy{{a1222%)./7eDDDr   z'self: jt_all, size: any, implicit: any?c                    t          | ||d          \  }}|                    d          |d         d|v r$|                    d          rt          d          t                    st	          dj         d           fd	t          d
                                          D             }t           | j	        |          fi t                    S )NTr   r   r   implicitz implicit expand is not supportedzexpand(): cannot expand shape z -> c                 :    g | ]}|j         k    rd n|         S )rR   r   )r    r!   r  r   s     r   r=   z"expand_default.<locals>.<listcomp>{  s-    WWWaS_,,""$q'WWWr   r   )r   r   r,   r   r1   r   r@  r-   r   r   r   )ro   rp   rq   r  r  
expand_argr  r   s         @@r   expand_defaultr   k  s     '4T  MAz ..
!
!CfDZJNN:$>$>?@@@c4(( SQCIQQ4QQRRRWWWWW5CGGIICVCVWWWJS[*55MM9L9LMMMr   zself: t, other: jtc                     t          | ||d          \  }}|                    d          }|                    d          }t           | ||j                  fi t	          |          S )NTr   r   r  r7  r  s          r   expand_as_defaultr    ss    &4T  MAz ..
!
!CNN7##ES%-00JJN54I4IJJJr   zself: jt_all, size: anyc           
         t          | ||d          \  }}|                    d          }|                    d          }t          |          |                                k    rR|                    g d t          |                                t          |          z
            D             |          S t          d          )NTr   r   r   c              3      K   | ]}d V  dS )r   Nr<   )r    r  s     r   r$   zbroadcast_to.<locals>.<genexpr>  s"      EE1QEEEEEEr   zwbroadcast_to(): broadcasting to a higher-dim shape is currently not supported for nested tensors with the jagged layout)r   r   rk   r-   r   r@  rl   )ro   rp   rq   r  r  r  r   s          r   broadcast_tor    s    &4T  MAz ..
!
!C>>&!!D
4yyCGGIIzzMEEcggii#d)).C(D(DEEEMMNNN
	4  r   ztensors: anyc           
         t          | ||d          \  }}|                    d          }t          |          dk    rt          d          t          |          dk    r|d         S g }t	          j        d |D              }t          d |D                       }|D ]}	|	j        r)|                    |		                    |                     2|	
                                t          |          k     rK|                    t          |		                    |j        j                  fi t          |                     t          d	          t          |          S )
NTr   r  r   z7broadcast_tensors(): expected at least one tensor inputr   c              3   $   K   | ]}|j         V  d S r   re  r  s     r   r$   z$broadcast_tensors.<locals>.<genexpr>  s$      .H.H1qw.H.H.H.H.H.Hr   c              3   D   K   | ]}t          |t                    |V  d S r   r_   r  s     r   r$   z$broadcast_tensors.<locals>.<genexpr>  s1      AAQZ<%@%@AqAAAAAAr   zubroadcast_tensors(): broadcasting nested tensors with dense tensors of equal or higher dim is not currently supported)r   r   rk   rl   r   broadcast_shapesr   r  r   r  r-   r   r   r   r   r'   )
ro   rp   rq   r  r  r  outsbroadcast_shaper   ra   s
             r   broadcast_tensorsr    sr   &4T  MAz nnY''G
7||qRSSS
7||qqzD,.H.H.H.H.HIO
AA'AAA
A
AC  ; 
	KK778888UUWWs?++++KKQ^^CK,=>>VV.QTBUBUVV    ;  
 ;;r   z(condition: jt_all, self: any, other: anyc                 D   t          | ||d          \  }}|                    d          }|                    d          }|                    d          }t          j        |||          \  }}}t	           | |j        |j        |j        fi |fi t          |          S )NTr   	conditionr   r  )r   r   r   r  r   r   r   )ro   rp   rq   r  r  r  r  r  s           r   
where_selfr    s     '4T  MAz {++I
..
!
!CNN7##E "3IsEJJIsEYU]IIjII 

#
#  r   zself: jt, device: any?c                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S r6  r7  r-  s         r   _pin_memory_defaultr    r9  r   c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r6  rw  r-  s         r   is_pinned_defaultr    r  r   zself: jt_all, other: jt_allc                 :    |d         j         |d         j         k    S r   )r   r  s      r   is_same_size_defaultr    s     7=DGM))r   c                 ~   t          | ||d          \  }}|                    d          |d         d u p:t          |d         t          t          f          ot          |d                   dk    }|r | j        fi |}|                    dd          r[t          |t          t          f          r% t          |          fd|D                       }n|	                    j
                  }|S |d         }	t          |d         t          t          f          }
|
s|	g}	t                                          |	| j
                  \  }}}}|
s|d         }||d<   |rj        t          | d	          dd
lm} |r|r3 | j        fi |}|                    dd          r |d |          }|S |rt          | d          |                    d           |
rj
        gnj
        } |                     |          fd|i|S |rt          | d           | j        fi |}t%                    |                    dd          sVt          |d         t          t          f          r|d         n|d         g}|D ] }|j
        dz
  k     rdxx         dz  cc<   ! |fd|          S )NTr   r   r-   r   r>  Fc              3   L   K   | ]}|                     j                  V  d S r   )r  r]   )r    or  s     r   r$   z#_apply_reduction.<locals>.<genexpr>  s1      JJCO < <JJJJJJr   zf(): reducing across the ragged dimension is not supported for non-contiguous nested tensors with holes)tree_mapc                 ,    |                      d          S )Nr   )r  )r  s    r   rY   z"_apply_reduction.<locals>.<lambda>+  s    Q r   zW(): reducing along a ragged and non-batch dimension is not supported for nested tensorszg(): reducing along the batch dimension but not the ragged dimension is not supported for nested tensorsr   r]   c                     t          | fi S r   r   )r  
out_kwargss    r   rY   z"_apply_reduction.<locals>.<lambda>V  s    ,q"?"?J"?"? r   )r   r   r&   r'   r(   rk   r   r   r)   r  r]   rH   r-   r\   r1   torch.utils._pytreer  r  r   )ro   	func_nameidentity_elementrp   rq   r  r  full_reductionoutdim_to_convert
is_dimlistconverted_dimr  r  reduce_on_non_batchr  dim_to_passdimlistr!   r  r  s                      @@r   r  r    s   &4T  MAz ..
!
!C  &$. :e$udm44TZ=N9O9OST9T   d3;--*-->>)U++ 	5#t}-- 5d3iiJJJJcJJJJJmmCO44
  &NJu-t}==J *() 					 	  )%a(%Ju 
CL4 ; ; ;
 
 	

 -,,,,,  2F 	$s{11j11C~~i// >h77==J # "  : : :   NN5!!!/9N3?++sK4$$%566 <GKU    	 K K K   d3;--*--#C((
~~i// 
	3 j/%??)
5!! '( 
  3 3s***}---2--- x????EEEr   c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r6  rw  r-  s         r   sum_defaultr)  Y  r  r   z3self: jt_all, dim: any?, keepdim: any?, dtype: any?c                 &    t          | ddg|R i |S )Nr   r   r  r  s      r   sum_dim_IntListr+  d  s&    
 D%<T<<<V<<<r   z"self: jt_all, dim0: any, dim1: anyc           
      ~   t          | ||d          \  }}ddlm} |                    d          } ||                                |d         |d         f          \  }}||j        k    s||j        k    r|dk    s|dk    rt          d          ||j        k    r|}	n|}	t          |          }
|	|
d	<   t          |	                                
                    t          t          |j                  ||j                  t          t          |j                  ||j                            fi |
S t          |                                |d         |j        d
          |d<   t          |                                |d         |j        d
          |d<   t           | |j        fi |fi t          |          S )NTr   r   r   r   dim0dim1z?Transpose is not supported on the batch dimension for jagged NTr]   	transpose)r   r%   r   r   r-   r]   rl   r   r   r  r/  r   rk   r   r9   r   )ro   rp   rq   r  r  r   r  r-  r.  to_dimra  s              r   transpose_intr1  l  s    '4T  MAz 655555
..
!
!C""37799z&/A:fCU.VWWJD$
 s$#/"9"9199		Q   3?""FFF#C((
$*
=!JJLL""#C	NND#/JJ#C	NND#/JJ 
 

 
 
 	
 *		:f%s Jv *		:f%s Jv S[77J77OO>#;N;NOOOr   zself: jt_all, dims: anyc                   
 t          | ||d          \  }}|                    d          
|                    d          }t          
          }t          
j                  t          |          k    r't          d ddt          |           dz             d	d
lm}  ||          }t          |          t          t          |                    k    rt          d          
j	        t          d          |d	         d	k    rt          d          |
                    
j                  |d<   
fd|dd          D             }	|	|d<   t           | 
j        fi |fi |S )NTr   r   rB   z5permute(): number of dimensions in the tensor input (z) zAdoes not match the length of the desired ordering of dimensions ().r   r   z*permute(): duplicate dims are not allowed.zBpermute(): not supported on jagged layout nested tensor with holesz=Permute is not supported on the batch dimension for jagged NTr]   c                 <    g | ]}t          |j                  S r<   )r   r]   )r    r-   r  inp_dims     r   r=   z#permute_default.<locals>.<listcomp>  s7        	GS#/::  r   r   )r   r   r   rk   r   rl   r%   r   setr\   indexr]   r   r   )ro   rp   rq   r  r  rB   ra  r   canonicalized_dims
inner_dimsr  r5  s             @@r   permute_defaultr:    s   &4T  MAz ..
!
!C>>&!!D$$J#)nnG #d))OGOOO_RUVZR[R[___`
 
 	

 655555**7D99
#c*<&=&=">">>>EFFF
|P
 
 	
 !!!K
 
 	
 !3 8 8 I IJ}    %abb)  J $JvS[77J77FF:FFFr   c                    t          | ||d          \  }}|                    d          |                    d          j        dk    r@t          j                  t                    k    rt          dj         d d          t                    d	k     st                    st          d
j         d           fdfdt          t                    dz
            D             }t          j
                                                  5  t           | j        |          fi t                    cd d d            S # 1 swxY w Y   d S )NTr   r   r   r   zVview(): does not support ragged_idx != 1 except when inp._size == size. inp._size is (z) and size is (r3  r  zview(): cannot view shape z as c                 j    | j         dz
  k    rj                            |           S | dz            S r[   )r]   r   r   )	inner_idxr  r   s    r   get_inner_sizez$view_default.<locals>.get_inner_size  s:    !+++;##I...	A&&r   c                 &    g | ]} |          S r<   r<   )r    rv   r>  s     r   r=   z view_default.<locals>.<listcomp>  s#    BBB..##BBBr   )r   r   r]   r'   r   r1   rk   r   r@  r   inference_modeis_inferencer   r   r   )	ro   rp   rq   r  r  
inner_sizer>  r  r   s	         @@@r   view_defaultrC    s   
 '4T  MAz ..
!
!C>>&!!D
!ci 0 0E$KK ? ?@ Y@ @7;@ @ @
 
 	
 4yy1}}.sD99}M	MMtMMNNN ' ' ' ' ' ' CBBBU3t99q=-A-ABBBJ 
	c..00	1	1 R RDDj99QQ^C=P=PQQR R R R R R R R R R R R R R R R R Rs   #)EE EzHinput: jt_all, normalized_shape: any, weight: any?, bias: any?, eps: anyc                    t          | ||d          \  }}|                    d          }|                                dk    rt          d          |d         }|j        |j                 }|                                t          |          z
  }|dk    rt          d          ||v r|j        t          d	          ||v rt          j	        j
                            |j                            |j        
          |j        g|j        g          }	t          j	        j
                            t          j        |j        j        d         df|j        |j                  |j        g|j        g                              |	j                  }
|j                                                            d                              d          |	j        d         z  }t          j        |	dd          |z  }|	|z
  |
z  }t          j        t          j        |          dd          |z  }t          j        ||d         z             }||z  }t          j	        j
                            ||j        g|j        j        d                                       d|j        |j        dz   d                    }t7          |fi t9          |          ||fS  | |j        fi |\  }}}t7          |fi t9          |          ||fS )NTr   r   r   zOlayer_norm(): not supported for NestedTensor objects with 2 or fewer dimensionsr   r   zVlayer_norm(): not supported when normalizing over the batch dimension for NestedTensorzklayer_norm(): not supported where lengths is not None if operating on the ragged dimension for NestedTensorr   )r{  r   )rP  r   )r   r   )r-   r>  r   r~  rR   )r   r   r-   r1   r   r]   rk   r\   r   r  r  r  r   r   r   r  onesrP  r   r   r  r  r   squaresqrtr  	unflattenr   r   )ro   rp   rq   r  r  r  r   ragged_sizenum_dims_not_normalizedpadded_inputpadded_maskragged_lengthsmeanpadded_normalizedvariancestdpadded_layer_normjagged_layer_norm_valuesr/   s                      r   native_layer_norm_defaultrU    sm   
 '4T  MAz ..
!
!C
wwyyA~~]
 
 	
 ""45)CO,K!ggii#.>*?*?? 	 1$$d
 
 	
 &&&3<+Cy
 
 	

 	'''y~EEK/     \N) F 
 
 inDDJ)!,a039UUU\N) E 
 
 &
 
	 	 L))!,,66q99L<Nq<QQ 	
 I  
  	 D K/ 	
 I.//  
  	 jJu$5566-3#(9>#Q#Q\NK% $R $
 $
 )	#/A-//0
 
 	! 1II^C5H5HII
 	
 S[77J77FD#77>##6#677sCCr   zpgrad_out: jt, input: jt, normalized_shape: any, mean: any, rstd: any, weight: any?, bias: any?, output_mask: anyc                     t          | ||d          \  }}|                    d          }|                    d          } | |j        |j        fi |\  }}}	|d ||	fS t          |fi t	          |          ||	fS )NTr   r  r   r  )
ro   rp   rq   r  r  r  r  d_inputd_gammad_betas
             r   "native_layer_norm_backward_defaultrZ  Z  s    
 '4T  MAz ~~j))H
..
!
!C#tH$4ckPPZPPGWfgv&&88N3$7$788'6JJr   z"self: jt_all, dim: any, index: anyc                    t          | ||d          \  }}|                    d          }t          |                                |d         |j        dd          \  |d<   }|r |                                |d                  S |j        t          d          t          |          }|d         |j        d	z
  k     r|d
xx         d	z  cc<   t           | |j
        fi |fi |S )NTr   r   r-   selectr  r7  zSselect(): not yet supported on dim != 0 for non-contiguous nested tensor with holesr   r]   )r   r   r9   r-   r]   r1  r\   rl   r   r   r   )ro   rp   rq   r  r  r  r8   r  s           r   
select_intr]  k  s   &4T  MAz ..
!
!C,<		:e$coxQU- - -)Ju)  1zz||Jw/00
|a
 
 	

  $$J%3?Q...=!!!Q&!!!S[77J77FF:FFFr   z7self: jt, dim: any?, start: any?, end: any?, step: any?c                    t          | ||d          \  }}|                    d          }t          |                                |d         |j        d          |d<   t           | |j        fi |fi t          |          S )NTr   r   r-   slice)r   r   r9   r-   r]   r   r   r   r-  s         r   slice_tensorr`    s    
 '4T  MAz ..
!
!C(		:e$cow Ju S[77J77OO>#;N;NOOOr   z8input: jt_all, indices: any, values: t, accumulate: any?c                    t          | ||d          \  }}|                    d          }|                    d          }t          |          |                                k    r5t	          dt          |           d|                                 d          t          |          |j        dz   k     r|                                st          d	          dd
lm	} |j
        }|j        }	|	}
|j        j        |j        dz
           }|
|}
g |j        d |j                 |
|j        |j        dz   d          R }|                    d|          } | | ||fi ||j        |j        |||	          }| t           j        j        j        j        u r.|j                            |                                           |S |S |                                '|                                                                }n|                                }t!          j        t!          j        ||j                 |k               d           |                                |d                  ||j                 z   }|d|j                 |gz   ||j        dz   d          z   }| t           j        j        j        j        u r | |j        |fi ||_        |S t9           | |j        |fi |fi t;          |          S )NTr   r   indiceszToo many indices: got z but tensor has z dimensionsr   z[index_put(): If ragged dimension is not part of indices, this only works on contiguous NJTsr   r  r  r   z7Some indices in the ragged dimension are out of bounds!r   )r   r   rk   r-   r,   r]   is_contiguousr1   r   r   r   r   r   r   r  r   r   r  r  
index_put_r  rj  r  r   r   r  _assert_asyncr   r   r   )ro   rp   rq   r  r  r  rb  r   r   r   r   r   r  
padded_inpnew_njtr   ragged_indicesfunc_indicess                     r   rd  rd    sO    '4T  MAz #w//C nnY''G
7||cggiiYS\\YY37799YYY
 
 	
 7||co)))  "" 	m   	655555*
*
!+#COa$78"L
Y(()

 Ys*,,-
 

 ))#<)HH
$$DW33
33L!!
 
 
 59>,444Kgnn..///J
 {{}}++--$$&&++--		'#/*W455A   [[]]71:.1IIN 	CO#$
	 #/A%''
(		)  uy~(000d3;CC
CC
S[,55*55 


  r   z{input: jt, weight: t, bias: t?, stride: any, padding: any, dilation: any, transposed: any, output_padding: any, groups: anyc                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S r6  r7  r-  s         r   convolution_defaultrk    sj     '4T  MAz ..
!
!CS[77J77OO>#;N;NOOOr   c                 h   t          | ||d          \  }}|d         }t          |                                |d         d|j                  \  }}}}|r|s|rt	          d          |d         }	d|d<   t          t          j        j        j	        j
        ddfi |}
|j        |j        n|j                                        }t          |
                                d	z
            D ]}|                    d
          }|
|z  }|	s|                    |j                  }|S d}t          | d|fi |S )NTr   r   r-   rO  zTCannot reduce on both ragged and non-batch dimensions without also reducing on batchr>  r   r   rR   gzG?)r   rH   r-   r]   r,   r  r   r  r  r   dim_IntListr\   r   r  r@  r  r   )ro   rp   rq   r  r  r  r  r  r%  r>  intermediate_sumr   r!  intermediate_values                 r   mean_dimrp    s    '4T  MAz W
CBS		5	C C?Q)+>    	 f   Y' $
9+IN*FA
 
9C
 

 #&,":#,,@Q@Q@S@S'++--122 	, 	,A''++GG( 	/++co..C

 D&*<KK
KKKr   c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r6  rw  r-  s         r   mean_defaultrr  4  r  r   z&self: jt_all, dim: any?, keepdim: any?c                 &    t          | ddg|R i |S )NrA   Fr  r  s      r   any_dimsrt  ?  s$    D%@@@@@@@r   z%self: jt_all, dim: any, keepdim: any?c                     t          | ||d          \  }}|d         g|d<   t          t          j        j        j        j        fi |S NTr   r-   )r   rt  r   r  r  rA   rB   ro   rp   rq   r  r  s        r   any_dimrx  D  W    &4T  MAz
 $E*+JuEIN&+::z:::r   c                 &    t          | ddg|R i |S )Nr   Tr  r  s      r   all_dimsr{  O  s$    D%???????r   c                     t          | ||d          \  }}|d         g|d<   t          t          j        j        j        j        fi |S rv  )r   r{  r   r  r  r   rB   rw  s        r   all_dimr}  T  ry  r   c                 t    t          | ||d          \  }}|                    d          } | |j        fi |S r6  rw  r-  s         r   all_any_max_min_defaultr  _  sR     '4T  MAz ..
!
!C4**z***r   c                 x    t          | ||d          \  }}|                    d          } | |j                  S r6  rw  r-  s         r   _is_true_defaultr  r  sJ    
 '4T  MAz ..
!
!C4r   c                     t          | ||d          \  }}|d         j        }t          |d          }t          | d|g|R i |S )NTr   r   r   r   r   r   r   r  ro   rp   rq   r  r  r   	dtype_maxs          r   min_dimr    c    &4T  MAz w%E"5%00ID%DTDDDVDDDr   c                     t          | ||d          \  }}|d         j        }t          |d          }t          | d|g|R i |S )NTr   r   r   r   r  ro   rp   rq   r  r  r   	dtype_mins          r   max_dimr    r  r   c                     t          | ||d          \  }}|d         j        }t          |d          }t          | d|g|R i |S )NTr   r   r   aminr  r  s          r   amin_defaultr    e     '4T  MAz w%E"5%00ID&)EdEEEfEEEr   c                     t          | ||d          \  }}|d         j        }t          |d          }t          | d|g|R i |S )NTr   r   r   amaxr  r  s          r   amax_defaultr    r  r   c                     t          | ||d          \  }}|d         j        }t          |d          }t          | d|g|R i |S )NTr   r   r   argminr  r  s          r   argmin_defaultr    e     '4T  MAz w%E"5%00ID(IGGGGGGGr   c                     t          | ||d          \  }}|d         j        }t          |d          }t          | d|g|R i |S )NTr   r   r   argmaxr  r  s          r   argmax_defaultr    r  r   zAgrad: jt_all, dim: any, indices: jt_all, sizes: any, keepdim: anyc                 P  
 ddl m
 t          | ||d          \  }}|                    d          }|j        |d<   |                    d          }|j        |d<   t          
fdt          |d                   D                       }t          t          |d                   |d	         |d
          |d	<   |                    d          }|j        	                    |j
        dz
            ||<   |dd          }||d<   t          |          }	||	d<   t           | di |fi |	S )Nr   r   Tr   gradrb  c              3   8   K   | ]\  }} |          |V  d S r   r<   )r    rv   r   r   s      r   r$   z=value_selecting_reduction_backward_default.<locals>.<genexpr>  s7      WWDAqmmTUFVFVWaWWWWWWr   sizesr-   "value_selecting_reduction_backwardr   r]   r<   )r   r   r   r   r   r   rn   r9   rk   r   r]   r   r   )ro   rp   rq   r  r  r  rb  r?   r  rt  r   s             @r   *value_selecting_reduction_backward_defaultr    s_   
 DCCCCC&4T  MAz >>&!!DJvnnY''G#OJyWWWWIj.A$B$BWWWWWJ(Jw  5,	 Ju NN7##E,,W-@1-DEEE*!""IEJw"7++M#-M- **z**<<m<<<r   c                 `   t          | ||d          \  }}|                    d          }|D ]}t          |t                    st	          d          |                                |d                                         k    rt	          d          t          ||d         j                  st	          d          t          |d                                         dz   |d	         |d         j	        d
          |d	<   t           | d |D             fi |fi t          |d                   S )NTr   r  z+stack(): expected all nested tensors inputsr   z9stack(): expected all nested tensors to have the same dimzFstack(): expected all nested tensors to have the same nested structurer   r-   r  c                     g | ]	}|j         
S r<   r   r  s     r   r=   z!stack_default.<locals>.<listcomp>	  r  r   )r   r   r&   r   r1   r-   r   r   r9   r]   r   )ro   rp   rq   r  r  r  ra   s          r   stack_defaultr    sb   &4T  MAz
 nnY''G  !\** 	NLMMM5577gajnn&&&&K   "!WQZ%566 	X  	
 )
1j/1G Ju )))))88Z88 <J7ST:<V<V  r   zQweight: t, indices: jt, padding_idx: any?, scale_grad_by_freq: any?, sparse: any?c                     t          | ||d          \  }}|                    d          }|                    d          }t           | ||j        fi |fi t	          |          S )NTr   rb  r   r7  )ro   rp   rq   r  r  rb  r   s          r   embedding_defaultr  	  s    
 '4T  MAz
 nnY''G^^H%%FVW_33
33 7Eg7N7N  r   zYgrad_output: jt, indices: jt, num_weights: any, padding_idx: any, scale_grad_by_freq: anyc                     t          | ||d          \  }}|                    d          }|                    d          } | |j        |j        fi |S )NTr   rb  r;  rw  )ro   rp   rq   r  r  rb  r;  s          r    embedding_dense_backward_defaultr  	  sh    
 '4T  MAz nnY''G..//K4#W_CC
CCCr   c                     t          | ||d          \  }}|                    d          }|j                                        S r6  )r   r   r   detachr-  s         r   values_defaultr  ,	  sN     '4T  MAz ..
!
!C ;r   c                 x    t          | ||d          \  }}|                    d          } | |j                  S r6  rw  r-  s         r   all_defaultr  ?	  sH    &4T  MAz ..
!
!C4r   z-self: jt_all, padding: any, output_size: any?c                 v   t          | ||d          \  }}|                    d          }|j        t          d          |d         }|||j                 }n(|j        |j        n|j                            d          }|	                                }|j        dk    r|
                    |j        dz
  d          }|j        }	|                                dk    r|                    d	          }n-|                                dk    r|                    d
          }|j        t           j        u }
|
r&|j        r|                    t           j                  }t           j        j                            ||j        g|g|d                   }|
r&|j        r|                    t           j                  }t3          |	          dk    r|                    d
|	dd                    }n(t3          |	          dk    r|                    d
          }|j        dk    r|
                    |j        d          }|S )NTr   r   z?to_padded_tensor(): not supported for nested tensors with holesr  r   r   r   rE  rR   padding)r   r   r\   r1   r]   _max_seqlen_tensorr  r   r   r  r/  r   r-   r   r  r   r   boolr  rV  halfr  r  r  r   rk   rI  r   )ro   rp   rq   r  r  r  r  max_seq_lenr  values_shapeis_bool
padded_outs               r   to_padded_tensor_defaultr  J	  s>   
 '4T  MAz ..
!
!C
|M
 
 	

 ]+K!#/2 %1 OO!!!$$ 	 ZZ\\F
!!#/A"5q99<Lzz||a!,,			!!"%% lej(G '6> '5:&&??		9	 J  /:% /]]5:..
 <1))"l122.>??

	\		a		''++

))#/1==
r   zcpadded: t, offsets: t, dummy: jt, ragged_idx: any?, min_seqlen: any?, max_seqlen: any?, sum_S: any?c                    t          | ||d          \  }}|d         |d         }}|                    dd          }|dk    r|                    |d          }|j        }|                                dk    r|                    d	          }n-|                                dk     r|                    d
          }|j        t          j	        u }	|	r&|j
        r|                    t          j                  }t          j        j                            ||g|d                   }
|	r&|
j
        r|
                    t          j	                  }
t!          |          dk    r|
                    d
|dd                    }
n(t!          |          dk     r|
                    d
          }
|dk    r|
                    |dz
  d          }
|d         }|d         }i }|||d<   |||d<   t'          |
|||          S )NTr   paddedr   r?   r   r  r   rE  rR   r   r   r   r   )r]   r   )r   r   r/  r   r-   r   r  r   r   r  r  rV  r  r  r  r  rk   rI  r   r   )ro   rp   rq   r  r  r  r   r?   padded_ragged_dim1_shaper  r  r   r   metadata_caches                 r   "_nested_from_padded_tensor_defaultr  	  s   
 '4T  MAz !*Jy,AGFa00J A~~!!*a00%|zz||a!,,			!!"%% lej(G '6> '5:&&Y^;;	:g. F  '6> '5:&& #$$q((!!"&>qrr&BCC	%	&	&	*	*##A~~!!*q.!44L)JL)JN'1|$'1|$&	   r   zcvalues: t, offsets: t, dummy: jt_all, lengths: t?, ragged_idx: any?, min_seqlen: t?, max_seqlen: t?c                     t          | ||d          \  }}|d         |d         |d         }}}|d         }|d         }	|d         }
i }|	|	|d<   |
|
|d<   t          |||||	          S )
NTr   r   r   r   r?   r   r   )r   r]   r   )r   r   )ro   rp   rq   r  r  r  r   r   r?   r   r   r  s               r    _nested_view_from_jagged_defaultr  	  s    
 '4T  MAz
 	799 GF
 L)JL)JL)JN'1|$'1|$&   r   c                 f    t          | ||d          \  }}|                    d          }|j        S r6  )r   r   r   r-  s         r   _nested_get_offsetsr  	  >    &4T  MAz ..
!
!C<r   c                 f    t          | ||d          \  }}|                    d          }|j        S r6  )r   r   r\   r-  s         r   _nested_get_lengthsr  	  r  r   c                 f    t          | ||d          \  }}|                    d          }|j        S r6  )r   r   r]   r-  s         r   _nested_get_ragged_idxr  	  s>    &4T  MAz ..
!
!C?r   c                     t          | ||d          \  }}|                    d          }|j                            dd           S )NTr   r   r   r   r   r   r   r-  s         r   _nested_get_min_seqlenr   
  O    &4T  MAz ..
!
!C""<666r   c                     t          | ||d          \  }}|                    d          }|j                            dd           S )NTr   r   r   r  r-  s         r   _nested_get_max_seqlenr  

  r  r   zself: jt, mask: anyc                 <   t          | ||d          \  }}|                    d          }|                    d          }|j        dk    rt          d          |j        |j        k    rt          d|j         d|j                   |j                            |                                          }t          j	        |                                
                    d	
          d          }t          |          }||j                 |d<   t          dd|i|S )NTr   r   r  r   z3masked_select only support 2-D selections currentlyzMask with shape z& is not compatible with input's shape r   r}  r  r   r  r<   )r   r   r"   r1   r   r   masked_selectr  r  r  r  r   r   r   )	ro   rp   rq   r  r  r  r  
res_valuesmask_cumsums	            r   masked_select_defaultr  
  s(   &4T  MAz ..
!
!C>>&!!D
x!||PQQQ	dj	 	 \tz\\QTQZ\\
 
 	
 **4;;==99J%,,,33V<<K#D!#,/DO  
  r   z2grad_output: t, self: jt_all, dim: any, index: anyc                 &   t          | ||d          \  }}|                    d          }|                    d          }t          j        ||j                  }|                    |d         |d                                       |           |S )NTr   r   r;  )r   r-   r7  )r   r   r   
zeros_liker   r\  rj  )ro   rp   rq   r  r  r  r;  
grad_inputs           r   _nested_select_backward_defaultr  .
  s    
 '4T  MAz ..
!
!C..//K!#[->???Jj'G)<==CCKPPPr   zself: jt_all, s: anyc                     |d         }|d         } | |j         |            | |j        |           |j         | |j        |           d S d S r   )r   r   r\   )ro   rp   rq   r  streams        r   record_stream_defaultr  @
  sf    
q'C!WFDfDv
|S\6"""""  r   zRself: jt_all, size: any, dtype: any?, layout: any?, device: any?, pin_memory: any?c                     t          | ||d          \  }}|                    d          }t          |d                   dk    r | |j        fi |S t	          d          )NTr   r   r   r   z2new_empty() not supported for NJT with shape != ())r   r   rk   r   r1   r-  s         r   new_empty_defaultr  K
  sy     '4T  MAz ..
!
!C
:f!##tCK..:...
K
L
LLr   zself: jt_all, ...c                     t          d |D                       }t           | d |D             i |fi t          |          S )Nc              3   D   K   | ]}t          |t                    |V  d S r   r_   r   s     r   r$   z&activation_backward.<locals>.<genexpr>m
  s1      LLsjl.K.KLsLLLLLLr   c              3   R   K   | ]"}t          |t                    r|j        n|V  #d S r   )r&   r   r   r   s     r   r$   z&activation_backward.<locals>.<genexpr>p
  s7      TTZ\::CckkTTTTTTr   r   )ro   rp   rq   r;  s       r   activation_backwardr  `
  sq     LLdLLLLLKTTtTTT	
	
 	
 
 
%
%  r   zself: jt_all, value: anyc                     t          | ||d          \  }}|                    d          }t           | |j        fi |fi t	          |          S r6  r7  r-  s         r   fill_Scalarr  w
  r9  r   c                 x    t          | ||d          \  }}|                    d          } | |j        fi | |S r6  rw  r-  s         r   fill__Scalarr  
  sU    &4T  MAz ..
!
!CD##
###Jr   c                     t          | ||d          \  }}|                    d          }t          |          } | |j                  \  }}t	          |fi |t	          |fi |fS r6  )r   r   r   r   r   )	ro   rp   rq   r  r  r  rt  mantissaexponents	            r   frexp_Tensorr  
  s    &4T  MAz ..
!
!C"3''Mck**Hh22M22L5 5!5 5  r   z+grad: any, self: any, other: any, mask: anyc                    t          | ||d          \  }}|                    d          }|                    d          }|                    d          }|                    d          }|dS d }	|d         r)t          j        ||                    d	d
                    }	d }
|d         r)t          j        |                    d	d
          |          }
|	|
fS )NTr   r  r   r  r  r   r   rR   r   )r   r   r   r?  r/  )ro   rp   rq   r  r  r  r  r  grad_input_mask	grad_self
grad_others              r   matmul_backward_defaultr  
  s    
 '4T  MAz >>&!!D
..
!
!CNN7##E nnV,,O||Iq @Lur2'>'>??	Jq ?\#--B"7"7>>
z""r   z	self: anyc                 "    ddl m}  |            S )Nr   )_nt_view_dummy)$torch.nested._internal.nested_tensorr  )ro   rp   rq   r  s       r   _nested_get_jagged_dummyr  
  s"    CCCCCC>r   r  IMPLr  CPUCUDAMeta)F)TFF)r   r   )r   r  r   typingr   torch.nn.functionalr  r  r  torch.fx.operator_schemasr   torch.nested._internal.sdpar   r   r   r	   r(   Any__annotations__r
   Dictr   r   r9   rH   strr|   r   r   r   r   r   register_jagged_funcCallabler   r   r   r   r  r  r  r  r  r  r-   r  r  r  r  r  primrN  r"  r   r$  rc  r*  r'  r,  r.  rA  r4  linearr8  linear_backwardrH  rV  r   rJ  _to_copyrb  rj  rq  r  
empty_like	ones_liker  	rand_like
randn_likeru  	full_likerandint_like	low_dtypezero_rx  _softmaxr  _log_softmaxr  _softmax_backward_datar  native_dropoutr  native_dropout_backwardr  r  dim_intr  r  ri   rW   r  r  r  r  r  r  r1  r  r  r   r  r  r  r  r  r?  r  bmmr  r   r   r  r  r  r  wherer=  r  _pin_memoryr  	is_pinnedr  is_same_sizer  r  r   r)  rm  r+  r/  r1  permuter:  view_unsafe_viewrC  native_layer_normrU  native_layer_norm_backwardrZ  r\  r]  r_  r`  	index_putrd  convolutionrk  rO  rp  rr  rA   rB   rt  rx  r   r{  r}  r   r   r  _is_all_true_is_any_truer  r  r  r  r  r  r  r  r  r  r  r  r  r  r  	embeddingr  embedding_dense_backwardr  r  _nested_get_valuesr  r  r  r  _nested_from_padded_tensorr  _nested_view_from_jaggedr  r  r  r  r  r  r  r  _nested_select_backwardr  record_streamr  	new_empty	new_zerosnew_onesr  elu_backwardhardshrink_backwardhardsigmoid_backwardhardtanh_backwardsoftplus_backwardsoftshrink_backwardr  fillScalarr  fill_r  frexpr  matmul_backwardr  r  library_scoped_libraryimplr<   r   r   <module>r4     s2                        8 8 8 8 8 8 K K K K K K ' ' ' ' ' ' c   #% $sCx. % % %H H H   6    :Y Y Y Y:=S =D = = = =@

#&
+7
AD
	
 
 
 
    0  . )y(8HII &HtO & & & &R    LS LS LS^c$ c$ c$L 	3;	'	"	$	 (	!)	)1  0 0 0, ein+3^DD  ED 
Y^ ! 
 
 
	 
 ein2:NKKF F LKF(  	IN .0S   
 	IN$,.Q O O O( 	IN "F E E E4 ein+35UVVP P WVP 	IN"*< " "	 "J ein'-/JKKP P LKP ein-5~FF& & GF&R 	IN "Q   : D  UY^*2N C C  
 	!)	 (	!)	 (	!) 	 	-5 -5	 	-5` X  UY^-57V W W   U  UY^08:S T T    	IN)+N   
 ein*2NCC  DC 	IN#%Q EP EP EPP 	IN')U "P "P "PJ 	IN)1=  	  	IN!)+N 
 
 
 	IN*2+ 	 		 	 	IN8 > >	 > ein)13NOO+ + PO+ 	IN!G   " 	IN#+-T   " 	IN!#P 7 7 7  ein*24VWW&
 &
 XW&
R ein+/1JKKE E LKEP ein,02FGGK K HGK ein.68PQQE E RQE, ein(02KLL  ML2 ein+35LMMj j NMjZ ein(02KLLE E MLE  	IN!#L N N N" ein.68LMMK K NMK ein19;TUU  VU" ein6>OO  PO> 	INI   $ ein08:RSSP P TSP ein.68PQQ+ + RQ+ 	IN')F * * *gF gF gFT ein(02MNN+ + ON+ 	IN"9 = =	 = 	IN "F 'P 'P 'PT ein,46OPP%G %G QP%GP 
Y^ %)."="EF ,R ,R	 ,R^ 	IN$,N ^D ^D	 ^DB 	IN-5v 
K 
K	 
K ein+/1UVVG G WVG8 	IN= 
P 
P	 
P 	IN$>  	IN%> U U	 	 Up 	IN&G 
P P 
P 	INR &L &L &LR ein)13NOO+ + PO+ ein(-/WXXA A YXA ein(,.UVV; ; WV; ein(-/WXX@ @ YX@ ein(,.UVV; ; WV; 	"	"	"	"	  + + + 
Y^ (%).*E*MN  	  ein(,.UVVE E WVE ein(,.UVVE E WVE 	IN!I F F F 	IN!I F F F 	IN!#K H H H 	IN!#K H H H 	IN5=G = =	 =@ ein*24MNN  ON< 	IN$W  	  	IN+3_ D D	 D 	%	)1  	  	  	  ein(0.AA  BA 	IN#+3 8 8	 8v 	IN-5i 2 2	 2j 	IN+3i  	 8 ein8@.QQ  RQ ein8@.QQ  RQ ein;C^TT  UT ein;C^TT7 7 UT7 ein;C^TT7 7 UT7 ein2:<QRR  SR0 	IN*28  	  ein2:<RSS# # # TS# 	 (	 (	'
 Y 
M 
M 
M 	#+	*2	+3	(0	(0	*2 
 
	 	
 
	 ein)02LMMP P NMP ein*13MNN  ON ein*1>BB  CB 	IN"*1 # #	 #2 ein=E{SS  TS ]""6622 LdII(*BEJJJII(*BFKKKII(*BFKKKL L L L L L L L L L L L L L L L L Ls   FAAG%G%AG)G,AG)