
    Wjt                     &   U d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
mZmZ d dlmZ d dlZd dlmZ d dlmZ d dlZd dlmZmZmZ d dlmZmZmZm Z m!Z! d dl"m#Z# d dl$m%Z% d d	l&m'Z' d d
l(m)Z) d dl*m+Z+ d dlm,Z,m-Z-m.Z. d dl/m0Z1 d dl2m3Z3m4Z4 d dl5m6Z6 ddl7m8Z8m9Z9 ddlm:Z:m;Z;m<Z<  ed          Z= ed          Z>eej?        j@        ej?        jA        f         ZBe
eCd<    ejD        eE          ZFejG        jH        ZHejG        jI        ZIejG        jJ        ZJejG        jK        ZKejG        jL        ZL eg eHjM        eHjN        eHjO        eHjP        eHjQ        eHjR        eHjS        eHjT        eHjU        eHjV        eHjW        eHjX        eHjY        eHjZ        eHj[        eHj\        eHj]        eHj^        eHj_        eHj`        eHja        eHjb        eHjc        eHjd        eHje        eHjf        eHjg        eHj0        eHjh        eHji        eHjj        eHjk        eHjl        eHjm        e)eHjn        eHjo        eHjp        eHjq        jr        eHjs        jt        eJju        eKjv                  Zwi  e            ewZxeHjy        eHjz        eHj{        eHj|        j}        eHj~        eHj        eHj        eHj        eHj0        eHj        eHj        eHj        eHj        eHj        eHj        j        eHj        eHj        eHj        eHj        gZeeej?        j        ej?        jA        f                  eCd<    eexe           deeBeeB         f         deee>e=f         gee>e=f         f         fdZ eeHj        g          dej        dej        derderdedej        fd            Z eeHj0                  d ej        d!ej        d"e	ej                 d#e	ej                 d$edeej        ej        ej        f         fd%            Z eeHj        j}        g          ddd&d'ej        d(e	ej        j                 d)e	ej        j                 ddfd*            Z eeHj        g          e!	 	 dd+ej        d(e	ej        j                 d)e	ej        j                 dej        fd,                        Z eeHj        g          e d+ej        dej        fd-                        Z eeHj        g          d.eeerej        f                  d/ej        j        d0edej        fd1            Z eeHj        g          dd2d+ej        d3erd4ej        d5ej        d6ej        j        dej        fd7            Z eeHj        j}        g          d.eeerej        f                  d8eer         d0edej        fd9            Z eeHj        g          dej        d ej        d"ej        d:eer         d;eereer         f         d<eereer         f         d=eereer         f         d>ed?eer         d@erdAee         deej        ej        ej        f         fdB            Z eeHj        j        g          dd+ej        dCerdej        fdD            Z eeHj        g          e 	 ddEej        dFej        dGe	ej                 dej        fdH                        Z eeHj        g          e 	 	 	 ddEej        dIej        dJej        dGe	ej                 dKej        j        d6ej        j        dej        fdL                        Z eeHj        g          e 	 ddEej        dMej        dGe	ej                 dej        fdN                        Z eeHj        j}        g          	 ddOeej                 d3erdej        fdP            Z eeHj        g          d+ej        dej        fdQ            Z eeHj        g          dd2d+ej        dRej        d6e	ej        j                 dej        fdS            Z eeHj        g          dEej        dej        fdT            Z eeHj        eHj        g          dEej        dej        fdU            Z eeHj        eIj        g          dEej        dVej        dej        fdW            Z eeHj        eIj        g          dEej        dVej        dej        fdX            Z eeHj                  	 	 ddEej        d3e	er         dZedej        fd[            Z eeHj                  	 	 ddEej        d3e	er         dZedej        fd\            Z eeHj        g          dEej        d3erd]erd^erdej        f
d_            Z eeHj        j}        g          dEej        d.eeerej        f                  dej        fd`            Z eeHj        j        g          dEej        daej        dej        fdb            ZdEej        deej        ej        f         fdcZ eeHj                  ddddYdYej        dddEej        d/eeref         dae	ej                 dee	ej                 dfe	ej                 dgedhediej        dej        fdj            Zddej        dkdledmej        f         dEej        dae	ej                 dfe	ej                 diej        d0edej        fdnZ eeHj                  dEej        d0edej        fdo            Z eeHj                  dEej        d0edej        fdp            Z eeHj        j}                  dEej        dqerd0edej        fdr            Z eeHj        j                  dEej        dserdqerd0edej        f
dt            Z eeHj        j}                  dqerd.eeerej        f                  d0edej        fdu            Z eeJju        j}                  	 dd ej        d"ej        d#e	ej                 dej        fdv            Zu eeKjv        j}                  d ej        dwej        dxej        d"ej        dyej        dzej        d#ej        d{ej        d|ej        d}erdej        fd~            Zv eejG        jJ        j                  dej        dej        fd            Z eeHj        g          e 	 	 	 ddej        dej        derderdedej        fd                        Z eeHj        j                  	 ddEeej                 deej                 deej                 dedeej                 f
d            Z eeHj        j                  	 ddEeej                 deej                 deej                 dedeej                 f
d            Z eeHj        j                  deej                 deej                 d"ej        j        deej                 fd            Z eeHj        j                  deej                 deej                 deej        j                 deej                 fd            ZeHj        j}                            ej        j        j                   eeHj                  d ej        d"ej        d#ej	        ej                 dej	        ej                 dej	        ej                 dedededeej        ej        ej        f         fd                        Ze j        deeedmef         f         fd            Zdeeedmef         f         fdZ eeHj                  dEej        dej        dej        dej        fd            Z eeLj        j                  d ej        derderd$edaej        deej        ej        f         fd            Z eeHj                  	 ddEej        d4ej        dej        dedej        f
d            Z eeHj                  	 ddEej        d4ej        dej        dedej        f
d            Z eeHj~        j}                  e dej        dej        d3erdej        dej        f
d                        Z~ eeHj                  dddEej        d3erd4ej        dej        dededej        fd            Zd+ej        deer         d;e	eereer         f                  d<eereer         f         d=eereer         f         ded3erdeej        ej        f         fdZ eeHj                  	 	 	 	 dd+ej        deer         d;e	eereer         f                  d<eereer         f         d=eereer         f         dedeej        ej        f         fd            Z eeHj                  	 	 	 	 dd+ej        deer         d;e	eereer         f                  d<eereer         f         d=eereer         f         dedeej        ej        f         fd            Z eeHj                  d+ej        deer         deej        ej        f         fd            Z eeHj        j                  dYdYddddej        dEej        j        dedede	e         de	ej                 dej        fd            Z eeHj        j                  dYdYddEej        j        dej        dededej        f
d            Z eeHj                  	 	 	 	 ddEej        dej        dededede	ej                 deej        ej        f         fd            Z eeHj        j                  	 ddej        de	er         dej        fdĄ            Z	 	 	 	 	 dd ej        d"ej        d#e	ej                 d;eer         d<eer         d=eer         d@erdej        fdǄZdS )    N)Callable)AnyOptional	TypeAliasTypeVarUnion)	ParamSpec)core_aten_decompositionsget_decompositionsremove_decompositions)_grid_sampler_2d
_index_addembedding_dense_backwardpw_cast_for_opmath"pw_cast_for_opmath_non_tensor_args)extra_random_decomps)counters)	is_fbcode)	out_dtype)pad_listlike)elementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDtype_to_dtype)native_layer_norm)guard_or_falsestatically_known_true)
OrderedSet   )configinductor_prims)is_gpu,needs_fallback_due_to_atomic_add_limitationsuse_scatter_fallback_T_P_GenericOperatordecomps_to_excludeopsreturnc                     t          | t                    r| n| gD ]&}|t          v rt                              d|            't          j        | t                    S )Nzduplicate decomp: %s)
isinstancelistdecompositionslogwarningdecompregister_decomposition)r(   ops     b/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/torch/_inductor/decomposition.pyr1   r1      s\      T**5cc 5 5KK.444(n===    grad_outputindicesnum_weightspadding_idxscale_grad_by_freqc                 D    | j         rt          S t          | ||||          S N)is_xpuNotImplementeddecomp_embedding_dense_backward)r5   r6   r7   r8   r9   s        r3   _embedding_dense_backwardr?      s3      *Wk;8J  r4   inputnormalized_shapeweightbiasepsc                 D    | j         rt          S t          | ||||          S r;   )is_mtiar=   decomp_native_layer_norm)r@   rA   rB   rC   rD   s        r3   _native_layer_normrH      s+     } #E+;VT3OOOr4   )minmaxsymbolrI   rJ   c                    d S r;    )rK   rI   rJ   s      r3   sym_constrain_range_for_sizerN      s	     Fr4   xc                 b    ||                      |          } ||                     |          } | S r;   )	clamp_min	clamp_max)rO   rI   rJ   s      r3   clamprS      s5     KK
KKHr4   c                 Z    | d|                                                                  z   z  S )Nr   )negexp)rO   s    r3   silurW      s$     AEEGGKKMM!""r4   size
fill_valuekwargsc                     |                     d          }|2t          t          |                    |d<   t          j        | |fi |S t
          S )Ndtype)getr   typetorchfullr=   )rX   rY   rZ   r\   s       r3   r`   r`      sS     JJwE}'Z(8(899wz$
55f555r4   alphadimindextensorrb   c                    t                      s| j        t          j        k    rt          S t          | |||d|          S )NF)inplacerb   )r   r\   r_   bfloat16r=   r   )rO   rc   rd   re   rb   s        r3   	index_addri      s@     ;; M17en44!S%eLLLLr4   physical_layoutc                 \    t          |          t          t          t          |                              k    }|rt          j         fi |S dgt                     z  }t          |          D ]
\  }}|||<   t          j         fd|D             fi |                    |          S )Nr   c                      g | ]
}|         S rM   rM   ).0lrX   s     r3   
<listcomp>z"empty_permuted.<locals>.<listcomp>  s    ===DG===r4   )r,   rangelenr_   empty	enumeratepermute)rX   rj   rZ   is_identitypermprn   s   `      r3   empty_permutedrx      s     ''4c/6J6J0K0K+L+LLK W{4**6***sSYYo.. 	 	DAqDGG{====_===HHHHPPQUVVVr4   
bias_sizesstridepaddingdilation
transposedoutput_paddinggroupsoutput_maskc                 b   |
d         rt          | j        j                  st          S t                              | dgt          t          d|                                                     z             }t          	                    | |||||||||	|
d         |
d         dg          \  }}}|||fS )N   r   r   F)
r!   devicer^   r=   atensumr,   rp   rc   convolution_backward)r5   r@   rB   ry   rz   r{   r|   r}   r~   r   r   	grad_biasgrad_inpgrad_weight_s                  r3   r   r   	  s     q> (:(?!@!@ qcDq+//:K:K1L1L,M,M&MNNI#88	QQ/   Hk1 k9--r4   decimalsc                 R    d|z  }t                               | |z            d|z  z  S )Ng      $@g      ?)r   round)rO   r   ten_pow_decimalss      r3   	round_decr   *  s/    X~::a**++s5E/EFFr4   selfbatch2r   c                    t           j        r| j        j        dvr}t	          | j        d         dk              st	          |j        d         dk              rA|                     d          |                    d          z                      d          }|S | j        j        dk    rt	          |                     d          dk              rt	          |                    d          dk              rnt          d         dxx         dz  cc<   t          j        |                     d          |                    d          z  dd	
                              d          S t          S )Ncpumpsr   r   rc   r   inductordecompose_bmmTrc   keepdim)r   coordinate_descent_tuningr   r^   r   shape	unsqueezer   rX   r   r_   squeezer=   )r   r   r   outs       r3   bmmr   0  sY    ' DK,<N,R,R A!!344 	8MLOq 9
 9
 	 >>"%%(8(8(;(;;@@Q@GGCJ{5   1!233 	8MKKOOq 9
 9
 	 Z 111Q61119Q&.."4"44!T  ill r4   mat1mat2betac                 *   | j         j        dk    r|t          |                    d          dk              rt          |                    d          dk              ryt          d         dxx         dz  cc<   t          j        |                    d          |                    d          z  dd                              d          }||z  || z  z   S t          |                    d          dk              rt          |                    d          d	k              rkt          |                    d          d	k              rEt          d         dxx         dz  cc<   |j
        |z                      dd          }||z  || z  z   S t          S )
Nr   r   r   r   r   decompose_addmmTr      )r   r^   r   rX   r   r_   r   r   r   r   Tr=   )r   r   r   r   r   rb   r   s          r3   addmmr   J  s    {5   1!233 	-8MIIbMMQ9
 9
 	- Z !2333q8333)Q$,,r"2"224  ill  3;,,!$))A,,!"344	-tyy||r122	- tyy||r122	-
 Z !2333q83336D=%%!T%::C3;,,r4   input2c                    t           j        r| j        j        dvr{t	          | j        d         dk              st	          |j        d         dk              r?|                     d          |                    d          z                      d          S | j        j        dk    rt	          |                     d          dk              rt	          |                     d          dk              rt	          |                    d          dk              rj| j	        |j	        k    rZt          t          j        |           t          j        |          z   dk              r t          d	         d
xx         dz  cc<   | |z  S t	          |                     d          dk              rt	          |                    d          dk              rnt          d	         d
xx         dz  cc<   t          j        |                     d          |                    d          z  dd                              d          S t          S )Nr   r   r   r   r   r   r       r   decompose_mmTr   )r   r   r   r^   r   r   r   r   rX   r\   r   r_   numelr   r   r=   )r   r   r   s      r3   mmr   h  s#    ' HDK,<N,R,R A!!344 	H8MLOq 9
 9
 	H NN1%%(8(8(;(;;@@Q@GGG{5  !$))B--1"455	!%diillQ&677	! &fkk!nn&9::	! v|++D 1 1EK4G4G GBNOO , Z 000A5000&=  1!233 	8MKKOOq 9
 9
 	 Z 000A50009Q&.."4"44!T  ill r4   tensorsc                    dt           j        dt          ffd}t          t	          ||                     t                    dk    rYt          | dt          j        id         }d         }||j	        k    r|
                                n|                    |          S dt                    cxk     rt          |           k     r#n n t          j                                      S t                    dk    rt          fdD                       rd         }t          |j                  }dk     rt          |j                  z   n|                    t                                |                              j        |                     dz             
                                S t*          S )	NrO   r)   c                     t          | j                  dk    r t          | j        d         dk              rdS t          | j                  k     r t          | j                 dk              rdS dS )Nr   r   FT)rq   r   r   )rO   rc   s    r3   non_empty_tensorzcat.<locals>.non_empty_tensor  si    " qw<<1
a!@!@5QW.1B"C"C5tr4   r   type_promotion_kindr   r\   c              3   ,   K   | ]}|d          u V  dS )r   NrM   )rm   tfiltered_tensorss     r3   	<genexpr>zcat.<locals>.<genexpr>  s?       ) )%&a  ) ) ) ) ) )r4   )r_   Tensorboolr,   filterrq   r   r   DEFAULTr\   clonetor   catdefaultallr   insertr   expandflattenr=   )r   rc   r   promoted_dtype
filtered_tinpr   r   s    `     @r3   r   r     s   
EL T      2 F#3W==>>
!!+
 ? G
 
  &a(
 !111 ^44	

 
S!""	1	1	1	1S\\	1	1	1	1	1x 0#666 q  S ) ) ) )*:) ) ) & &  q!SY&)AggcC	NN""3S#.//000(s}}S!!(%088cAgFFLLNNN r4   c                    |                                  rWt          j        t          j        | j                  t          d          t          j        | j        | j                            S t          | t          j
                  \  }}t          j        t          j        || j                  }t          j        | dk     |d          }t          j        t          j        |           t          d          |          S )Nnan)r   r\   r   r   g        )
is_complexr_   whereisnanrealfloatatan2imagr   r   INT_TO_FLOATscalar_tensormathpir   )rO   r   r\   r   rets        r3   angler     s    ||~~ 
{Kuu{1616/J/J
 
 	
 "	;H  HAu 
	TWE!(	C	C	CB
+a!eR
%
%C;u{1~~uU||S999r4   yc                   t          j        |           o|                                 }t          j        |          o|                                }|r|st          S dt           j        dt
          fd}d}| j        dk    r|j        dk    rd}| j        dk    r|                     d          } |j        dk    r|                    d          }|}|||z  }t          j        | j	        |j	                  } ||           s ||          rt          S dt           j        dt           j        fd}	| dz   } |dz   } |	| 
                    | j        j	                            }
 |	|
                    |j        j	                            }t          j        |
|z   d	
          
                    |          }|r|d         S |S )Nre   r)   c                 X    | j         dk    rdS |                                 d         dk    S )Nr   Fr   r   )ndimrz   )re   s    r3   _requires_fallbackzadd.<locals>._requires_fallback  s,    ;!5}}r"a''r4   Fr   Tr   c                     | j         ^ }}|dz  dk    rt          d          g ||dz  dR }|                     |          }|S )zNReshape tensor from [*initial_dims, last_dim] to *initial_dims, last_dim/2, 2]r   r   zQThe size of the last dimension must be even to reshape it to [..., last_dim/2, 2])r   AssertionErrorview)re   initial_dimslast_dim	new_shapereshaped_tensors        r3   reshape_tensor_complexz#add.<locals>.reshape_tensor_complex	  sj     #),x a<1 c  
 6l5HM5155	 ++i00r4   )	start_dim)r_   	is_tensorr   r=   r   r   r   reshapepromote_typesr\   r   r   r   )rO   r   rb   x_is_complex_tensory_is_complex_tensorr   output_size_zerozcomplex_typer   
x_reshaped
z_reshapedresults                r3   addr     s     /!,,?/!,,? &9 (5< (D ( ( ( ( v{{qv{{v{{IIaLLv{{IIaLL	AAI&qw88L!  2 21 5 5 u|     $ 	
AA	AA''qv|(<(<==J''qv|(<(<==J]:
2bAAAFF|TTF ayMr4   c                 <    |                                  rt          S | S r;   )r   r=   r   s    r3   conj_physicalr   '  s     Kr4   c                     | S r;   rM   r   s    r3   liftr   .  s    Kr4   otherc                 `    t          j        t          j        |          || k    z  | |          S r;   r_   r   r   r   r   s     r3   fminr   3  )    ;u{5))UT\:D%HHHr4   c                 `    t          j        t          j        |          || k     z  | |          S r;   r   r   s     r3   fmaxr   8  r   r4   Fr   c                 h    | j         t          j        k    rt          j        | ||          S t          S Nr   )r\   r_   r   anyr=   r   rc   r   s      r3   amaxr  =  0     zUZy38888r4   c                 h    | j         t          j        k    rt          j        | ||          S t          S r  )r\   r_   r   r   r=   r  s      r3   aminr  H  r  r4   startlengthc                 l    t          j        | |||                              t           j                  S )Nmemory_format)r_   narrowr   contiguous_format)r   rc   r	  r
  s       r3   narrow_copyr  S  s7     <c5&1177- 8   r4   c                 \    t                               | |                                          S r;   )r   r   r   )r   rX   s     r3   view_copy_defaultr  a  s$    
 99T4  &&(((r4   r\   c                 P    |                                                      |          S r;   )r   r   )r   r\   s     r3   view_copy_dtyper  i  s     
 ::<<U###r4   c                      t          j                   \  }} fd|D             }dgt          |          z  }t          |          D ]
\  }}|||<   ||fS )Nc                 *    g | ]}j         |         S rM   )r   )rm   rn   r   s     r3   ro   z/_get_shape_permutation_like.<locals>.<listcomp>u  s    444qTZ]444r4   r   )utils3compute_elementwise_output_logical_to_physical_permrq   rs   )r   rj   r   r   permutationrw   rn   s   `      r3   _get_shape_permutation_liker  q  sw     RSWXXOQ4444O444E#E

"K/**  1A;r4   )r\   layoutr   
pin_memoryrequires_gradr  r  r   r  r  r  c          	         || j         n|}|| j        n|}|| j        n|}|t          j        k    r6t          j        | j        ||||||          }|                    |          S |t          j        k    sJ t          |           \  }	}
t          j        |	||||||          }|
t          t          t          |
                              k    r|S |                    |
                                          S )N)r\   r  r   r  r  r  )r\   r  r   r_   preserve_formatr`   r   r   stridedr  r,   rp   rq   rt   r   )r   rY   r\   r  r   r  r  r  r   r   r  s              r3   	full_liker!  ~  s     -DJJUE"NT[[F"NT[[F---J!'
 
 
 yy}y555 &&&&8>>{!'
 
 
 $uS%5%5667777M~~k**00222r4   )r\   r   r  rand_fn.c                   ||j         n|}||j        n|}|t          j        k    r% | |j        f||d|                    |          S t          |          \  }} | |f||d|}|t          t          t          |                              k    r|S |
                    |                                          S )Nr   r  )r\   r   r_   r  r   r   r  r,   rp   rq   rt   r   )	r"  r   r\   r   r  rZ   r   r  r   s	            r3   
_rand_liker$    s      -DJJUE"NT[[F---wJ

 
 	
 

 "="
)
)	* 5T::E;W  	 F d5[!1!1223333>>+&&,,...r4   c                 2    t          t          j        | fi |S r;   )r$  r_   randr   rZ   s     r3   	rand_liker(    s    ej$11&111r4   c                 2    t          t          j        | fi |S r;   )r$  r_   randnr'  s     r3   
randn_liker+    s    ek4226222r4   highc                 d    t          t          j        t          j        j        d|          | fi |S Nr   r$  	functoolspartialr   randintlow)r   r,  rZ   s      r3   randint_liker4    s.    i'(8!TBBDSSFSSSr4   r3  c                 d    t          t          j        t          j        j        ||          | fi |S r;   r/  )r   r3  r,  rZ   s       r3   randint_like_lowr6    s0     i'(8#tDDdUUfUUUr4   c                 4    t          j        j        d| |fi |S r.  )r   r2  r3  )r,  rX   rZ   s      r3   r2  r2    s#     <AtT44V444r4   c                     t           j        j                            |          }t           j        j                            | |||                                d                   S r.  )r_   r(   
_quantized$wrapped_fbgemm_pack_gemm_matrix_fp16!wrapped_fbgemm_linear_fp16_weightrX   )r@   rB   rC   packed_weights       r3   #linear_dynamic_fp16_unpacked_weightr=    sO     I(MMfUUM9AA}dFKKMM!$4  r4   input_scaleinput_zero_pointweight_scaleweight_zero_point	out_scaleout_zero_pointout_channelc
           	          t           j        j                            ||||          }
t           j        j                            | |||
|||	          S r;   )r_   r(   r9  _wrapped_linear_prepack#_wrapped_quantized_linear_prepacked)r@   r>  r?  rB   r@  rA  rC   rB  rC  rD  r<  s              r3   wrapped_quantized_linearrH    s\     I(@@/ M 9CC  r4   packedc                     dt           j        dt           j        fd} || dddf                   } || ddd f                   }| dd df                             t           j                  |z  |z   S )Nu8r)   c                 &     fddD             \  }}}}t           j        dk    r7||dz  z   |dz  z   |dz  z                       t          j                  d         S |dz  |dz  z   |dz  z   |z                       t          j                  d         S )Nc              3   f   K   | ]+}d |f                              t          j                  V  ,dS .N)r   r_   int32)rm   nrK  s     r3   r   zPq_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32.<locals>.<genexpr>  s9      GGQbajmmEK00GGGGGGr4   )r   r   r      little   r      rN  )sys	byteorderr   r_   float32)rK  rO   r   r   ws   `    r3   bitcast_u8_to_f32z=q_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32  s    GGGG,GGG
1a=H$$aLAG,R8>>u}MMiXX"Wb)Q!V4q8>>u}MMiXXr4   .i)r_   r   r   rW  )rI  rY  scalesoffsetss       r3   "q_embedding_bag_byte_unpack_decompr]    s    Yel Yu| Y Y Y Y vc2b5j122FsBCCx 011G#ss(u}--6@@r4   agridinterpolation_modepadding_modealign_cornersc                     | j         t          j         d          k    o%|dk    o|                     t          j                   }t	          | |||||          }|S )Nr   r   r  )r_  r`  ra  rb  _expand_grid)r   r_   is_contiguousr  decomp_grid_sampler_2d)r^  r_  r`  ra  rb  rd  outputs          r3   grid_sampler_2drh    sv     	
EL''' 	C!#	COO%*AOBBL $	-!#!  F Mr4   left_tensorsright_tensorsscalarc                     t           j                            | t           j                            ||          |          S Nra   )r   _foreach_addList_foreach_mulr   ri  rj  rk  s       r3   _foreach_addcmul_scalarrr  A  >     !!d$$\=AA "   r4   c                     t           j                            | t           j                            ||          |          S rm  )r   rn  ro  _foreach_divrq  s       r3   _foreach_addcdiv_scalarrv  M  rs  r4   start_tensorsend_tensorsc           	          t           j                            | t           j                            t           j                            ||           |                    S r;   )r   rn  ro  rp  Scalar_foreach_sub)rw  rx  rB   s      r3   _foreach_lerp_scalarr|  Y  sP     !!  "";>>	
 	
  r4   scalarsc           	          t           j                            | t           j                            t           j                            ||           |                    S r;   )r   rn  ro  rp  
ScalarListr{  )rw  rx  r}  s      r3   _foreach_lerp_scalarlistr  g  sP     !!$$"";>>	
 	
  r4   running_meanrunning_vartrainingexponential_average_factorepsilonc           
          t                               | |||||||          \  }}	}
|r||	|
fS ||                    d          |                    d          fS )Nr   )r   native_batch_norm	new_zeros)r@   rB   rC   r  r  r  r  r  r^  bcs              r3   miopen_batch_normr  u  s|     $$"	 	GAq!  1ay	 r4   c                  "    i t           t          S r;   )r-   r   rM   r4   r3   fast_random_decompsr    s    5n5 455r4   c                     t           j        rt          S t           j        r;t                              t
          j        j        j        j	        d           t          S t                      } t           j        rnt          t          j        t          j        j        t          j        j        g          dt$          dt&          ffdfd|                                 D             } | S )z"decomps can change based on configNr2   r)   c                 F    | v rdS t          | d          r	| j        v S dS )NToverloadpacketF)hasattrr  )r2   ops_to_skips    r3   should_skipz(select_decomp_table.<locals>.should_skip  s9    [  tr+,, 8(K775r4   c                 2    i | ]\  }} |          ||S rM   rM   )rm   kvr  s      r3   
<dictcomp>z'select_decomp_table.<locals>.<dictcomp>  s,    HHH41aQH!QHHHr4   )r   fallback_randomr-   "fallback_embedding_bag_byte_unpackpopr_   r(   	quantizedembedding_bag_byte_unpackr   r  emulate_precision_castsr   r   addcmul_foreach_addcmulrz  _foreach_addcdivr   r   items)r   r  r  s    @@r3   select_decomp_tabler    s     0 59.HPRVWWW ""F% I !%,%,
 
	C 	D 	 	 	 	 	 	 IHHH6<<>>HHHMr4   masksourcec                    ddl m}m}  || j        |j                  rt
                              | |g          \  } }|                    d                              d          dz
  }d | ||fD             \  }}}t
          	                    |||gd          }	t          j        ||	|                              | j                  S t          S )Nr   )BackendFeaturehas_backend_featurer   r   c              3   >   K   | ]}|                                 V  d S r;   )r   )rm   rO   s     r3   r   z!masked_scatter.<locals>.<genexpr>  s*      ,W,WQQYY[[,W,W,W,W,W,Wr4   )codegen.commonr  r  r   MASKED_SCATTER_WITH_INDEXr   broadcast_tensorsr   cumsum_unsafe_masked_indexr_   r   r   r   r=   )
r   r  r  r  r  
source_idx	self_flat	mask_flatsource_flatr   s
             r3   masked_scatterr    s     DCCCCCCC4;(PQQ J ++T4L99
d\\"%%,,Q//!3
,W,W4vBV,W,W,W)	9k**;	J<QRSS{9fi88==djIIIr4   	quant_min	quant_maxc                    t          j        |           \  }}||z
  t          ||z
            z  }t          j        |t          j        |g                    }|t          j        ||z                                t           j                  z
  }t          j        |||          }|                    t           j	                  |                    t           j
                  fS r;   )r_   aminmaxr   rJ   r   r   r   intrS   float64int64)	r@   r  r  rD   r\   min_valmax_valscale
zero_points	            r3   choose_qparams_tensorr    s     }U++GWw%	I(=">">>EIeU\3%0011EU[599<<UYGGGJZI>>J88EM""JMM%+$>$>>>r4   
accumulatec                     |                                  }t          j        ||g|                    |j                  |          }|                    | j                  S r;   )r   r_   	index_putr   r   )r   rd   r  r  	flatteneds        r3   putr    sS     IE7FNN5;77 I TZ(((r4   c                 h    t                               | |||          }|                     |          S )N)r  )r   r  copy_)r   rd   r  r  r   s        r3   put_r    s-     ((4:(
>
>C::c??r4   rg  input_dtypec                     | |z  }t          j        ||d          }t          j        | ||          }| j        |k    r|                    |          }|                                S )NTr   )r_   r   r    fmar\   r   
contiguous)r5   rg  rc   r  new_grad_outputsum_new_grad
grad_inputs          r3   _softmax_backward_datar    sm     "F*O9_#tDDDL#VG\?KKJ K'']];//
  """r4   Tinclude_selfsrcreduction_typer  c                   |dk    rt          | j                  s| j        j        p| j        j        }t	          j        |          }|r,| }t	          j        |                               |||          }	nZ|                     ||d          }t	          j        |                               |||          }	|		                    |	dk     d          }	|                    |||          }|r||	z  n||	z  S t          t          j        j        || j        |j        |j        j        d          rt           S | j        |dz   d                                          | j        d |                                         z  }
|                                g| j        |dz   d          | j        d |         R }g t'          | j        |z
  | j                  dt'          d| j        |z
            R }|                    t          j                                      |
                              |                              |          }|                     |||||          S )Nmeanr   r   Tr  )r"   r\   is_floating_pointr   r_   	ones_likeri   
index_fill
zeros_likemasked_fillr#   r   scatter_reduce_twor   r^   r=   r   r   rp   r   r   r  repeat_interleaver   rt   scatter_reduce)r   rc   rd   r  r  r  true_divisiononesr   countsrepeatsindex_shaperv   scatter_indexs                 r3   index_reducer    sG    (T
) ) 
4M
8Ms## 	7C_T**44S%FFFF//#ua00C%d++55c5$GGF''
A66FmmC,,,?sV||#-? 
	
    jq#))++dj#.>.D.D.F.FFG;;==L4:cAgii#8L4:dsd;KLLKNU49s?DI..NNE!TY_4M4MNND		7	#	#					  !    r4   kernel_size	ceil_modec                 F   |dk    rdg|z  }|dk    rdg|z  }|s|}t          ||          }t          ||          }t          ||          }t          ||          }t          j        t          j        |          }t
          j        j                            ||          s'|t          j	        t
          j
                  j        k    rt          S t                              | |||||          \  }}	t                              |	|| j        | d          |||          }
||
fS )Nr   r   )n_dim)r   r0  reduceoperatormulr_   	_inductorlowering%should_fallback_max_pool_with_indicesiinfoint8rJ   r=   prims!_low_memory_max_pool_with_offsets'_low_memory_max_pool_offsets_to_indicesr   )rO   r  rz   r{   r|   r  rc   window_sizevalsr\  r6   s              r3   _max_pool_with_indicesr  I  sC    1}}39!||#)  {C00KHc**H7C((G&#&&F"8<==K 	 FFs 	G 	
 	
 UZ00444;;	 MD' ;;	 G =r4   c           	      .    t          | |||||d          S )Nr   r   r  rO   r  rz   r{   r|   r  s         r3   max_pool2d_with_indicesr    *     "	;9!   r4   c           	      .    t          | |||||d          S )NrQ  r   r  r  s         r3   max_pool3d_with_indicesr    r  r4   output_sizec                 2   | j         ^ }}}|\  }}|dk    s|dk    r>g |||}|                     |          |                     |t          j                  fS ||z  dk    r.||z  dk    r%||z  ||z  g}t                              | |          S t          S )Nr   r   )r   	new_emptyr_   r  r   r  r=   )	rO   r  batchh_inw_inh_outw_outo_sizer  s	            r3   adaptive_max_pool2dr    s     UD$LE5zzUaZZ'5'%''{{6""AKKekK$J$JJJe|qTE\Q..u}dem4++A{;;;r4   	out_int32rightsidesortersorted_sequencer
  r  r  r  c                    t                               | t          j        |g| j                  ||||          d         S )Nr   r	  r   )r   searchsortedr_   re   r   )r  r   r
  r  r  r  s         r3   searchsorted_scalarr    sQ     dVO$:;;;    	 	r4   r
  r  
boundariesc                    t                               t          j        | g|j                  |||                              d          S )Nr  r  r   )r   	bucketizer_   re   r   r   )r   r  r
  r  s       r3   bucketize_scalarr    sL     >>dVJ$5666	   
 gajjr4         ?UUUUUU?noiselowerupper	generatorc                     |rW| dk    }t                               | |||          }t          j        || |z  |           }t          j        ||d          }	||	fS ||z   dz  }
t                               | |
          t          j                    fS )Nr   )r  r   r   )r   uniformr_   r   
leaky_relur   )r   r  r  r  r  r  not_positiverrg  	noise_outnegative_slopes              r3   rrelu_with_noise_functionalr%    s      EqyLLueyLAA\4!8T::Ka33	y  %-1,t^44elnnDDr4   repeatc                    t           j        j        rt          S |t	          |          t
          urt          S | j        j        dk    rt          S | j        t          j	        t          j
        fv sJ | j        dk    sJ |                     d          }t          j        || j                  }t          j        ||| j        t          j	        k    d          }t          j        ||                     d          dz
            S )Nr   r   r   r  Tr  )rJ   )r   tritonautotune_at_compile_timer=   r^   r  r   r\   r_   rO  r  r   r  aranger  rS   rX   )r&  r  r  posr6   s        r3   repeat_interleave_Tensorr,    s    
 }-  d;//s::}U""<EK55555;!]]1F
,{6=
9
9
9C  ;D  G ;wFKKNNQ$67777r4   r   r  c           	      |   |                                  dk    r|                                 dk    s
J d            |d         }|d         }|d         }|                     d          }|                    d          }t          j                            ||||df|df|df|          }	|	                    d          S )NrQ  z,Expect (N,C_in,L) and (C_out,C_in//groups,K)r   r   r   )rz   r{   r|   r   )rc   r   r   conv2dr   r   )
r@   rB   rC   rz   r{   r|   r   input_2d	weight_2dout_2ds
             r3   conv1d_to_conv2dr3    s     99;;!

 1 1 16 !2 11
 AYFajG{H r""H  $$I [  {!A !  F >>"r4   )NNr;   )Nr   r   )NF)r   r   F)F)Nr   r   F)r  r  FN)Nr-  r  r-  r   )r0  loggingr   r  rU  typingcollections.abcr   r   r   r   r   r   typing_extensionsr	   r_   torch._decomp_decompr0   torch._prims_common_prims_commonr  $torch.ao.quantization.fx._decomposedr
   r   r   torch._decomp.decompositionsr   rf  r   r   r>   r   r   $torch._decomp.decompositions_for_rngr   torch._dynamo.utilsr   torch._environmentr   !torch._higher_order_ops.out_dtyper   torch._inductor.utilsr   r   r   r   torch._refsr   rG   %torch.fx.experimental.symbolic_shapesr   r   torch.utils._ordered_setr    r   r    r!   r"   r#   r$   r%   _opsOperatorBaseOpOverloadPacketr&   __annotations__	getLogger__name__r.   r(   r   r  r  r9  quantized_decomposed_adaptive_avg_pool2d_backwardindex_selectaddmvr*  bitwise_and_bitwise_or_
clamp_min_distelu
empty_likeflipgeluhardtanhlcmr   linalg_vector_norm_log_softmax max_pool2d_with_indices_backward_native_batch_norm_legit#_native_batch_norm_legit_functional$_native_batch_norm_legit_no_training_batch_norm_with_update"_batch_norm_with_update_functional_batch_norm_no_updatebatch_norm_backwardr  native_group_normnll_loss2d_backwardpermute_copyrrelu_with_noise_backward_softmaxsin_sqrt__to_copytril_indicestriu_indicesunbind_copyr  upsample_bilinear2dvecr=  rH  inductor_decompositionsr-   _unsafe_indexr  #_unsafe_masked_index_put_accumulate+_scaled_dot_product_flash_attention_for_cpur   r  rR   rQ   ri   gluselect_scatterslice_scatterrW   splitr   r   r   unbindbaddbmmr'   r,   
OpOverloadr1   r   r?   	ShapeTyper   tuplerH   rN   SymInttypesNumberrS   r`   rx   r   r   r   r   r   r\   r   r   r   r   r   r   r   detach_r   r   r  r  r  	view_copyr  r  
StrideTyper  r!  r  r  r   r  r$  r(  r+  r4  	low_dtyper6  r2  r  r]  rh  r  rz  rr  r  rv  _foreach_lerpr|  r  r  r  py_impl_CDispatchKeyAutogradcachedictr  r  r  choose_qparamsre   r  r  r  r  strr  r  r  r  r  r  r  r  r%  	Generatorr  r,  r3  rM   r4   r3   <module>r     s!           



  $ $ $ $ $ $ ; ; ; ; ; ; ; ; ; ; ; ; ; ; ' ' ' ' ' '        # # # # # # + + + +         
              F E E E E E ( ( ( ( ( ( ( ( ( ( ( ( 7 7 7 7 7 7 . . . . . .         
 F E E E E E W W W W W W W W / / / / / / $ $ $ $ $ $ $ $          WT]]Yt__#	JUZ88 )    g!!y~	I	Y!
y5 ,,+*++ 	
+ 		+
 	+ 	+ 	+ 		+ 	+ 	+ 		+ 		+ 	+ 	+ 	+  	!+" 	#+$ 	-%+& 	%'+( 	0)+* 	1++, 	$-+. 	//+0 	"1+2 	 3+4 	5+6 	7+8 	9+: 	 ;+< 	=+> 	&?+@ 	A+B 		C+D 	
E+F 	G+H 	I+J 	K+L 	M+N 	O+P 	 $Q+R 	5S+T 	+U+- - \ K,,..J2IJ
 	,4<NN!NHIJLHKL'W Duz4ej6QQRS   ,  n&8 9 9 9>	&6!77	8>xB (2r6"223> > > > 6788\  	
  \   98" .//
P<
Po
P U\"
P 5<
 	
P
 

P 5<u|34
P 
P 
P 0/
P :BCDD )-(,	  L 
%+$	% 
%+$	%	
 
   ED %%# )-(,	 	|		%+$	%	 
%+$	%	 \		 	 	 $# &%	 $$#EL #U\ # # #  %$# $$	
uS%,&'
(	"	 	 \		 	 	 %$	 ()) !"M M M|M	M <M L	M ;M \M M M *)M* ,4566W
uS%,&'
(W#YW W \	W W W 76W  2344..<. L. S		.
 #tCy.!. 3S	>". CcN#. . I. . d. 5<u|34. . . 54.@ ,-..G G G GU\ G G G /.G
 
## (, 
,L $ \	    $#0 %%
 (,  ! 
,
, , $	
 +
 ; \    &%8 	"" (,   
, L  $  \	       #" N )*++ : :%,:	: \: : : ,+:z %%:U\ :el : : : &%:& 
##
 +/	@ @ @|@|@ EK&'	@
 \@ @ @ $#@F +,--     .- DL122u|     32 EJ/00Iu| IEL IU\ I I I 10I EJ/00Iu| IEL IU\ I I I 10I 	""  
,	#  \	   #" 	""  
,	#  \	   #" )*++

,
	
 
 	

 \
 
 
 ,+
 /011)
,)
uS%,&'
() \) ) ) 21) -.//$
,$;$ \$ $ $ 0/$
 
,
 
5?E,,-
  
  
  
  ''
 $(%)%)).)>)3 )3 )3
,)3c5j!)3 EK 	)3
 U\")3 U\")3 )3 )3 &)3 \)3 )3 )3 (')3` $(%)).)>/ / /c5<'(/
,/ EK 	/
 U\"/ &/ / \/ / / /@ ''2EL 2C 2EL 2 2 2 ('2 ((3U\ 3S 3U\ 3 3 3 )(3 )122Tu| T3 T# T%, T T T 32T )344V
,V V(+V7:V
\V V V 54V ,--5
5
uS%,&'
(5 5 \	5 5 5 .-5 	EMNN $( <L 5<
  \	   ON 
;CDD< l L	
 , | , | L  \   ED4 	+EFF
Au| 
A 
A 
A 
A GF
A -.//   |
,  	
  \    0/@ -455
 	 
u|
u|$ % 	
 
%,   65 -455
 	 
u|
u|$ % 	
 
%,   65 *122
%
el#
 K
 
%,	
 
 
 32
 *566
%
el#
 %+$%
 
%,	
 
 
 76
 ''(<(EFF.//<L /%,
' /%,/	
 .  !&  5<u|34   0/ GF< 6T#xS'9"9: 6 6 6 6"T#xS'9"9: " " " "J +,,
,
, L \	   -,$ ,;BCC?<?? ? 
	?
 ;? 5<%&? ? ? DC? !!
 	
) 
)
,
)<
) L
) 	
)
 \
) 
) 
) "!
) 	""
 	 
,< L 	
 \   #" 3;<<##L# 
# 	#
 \# # #  =<#( )** 2 2 2
,2	2 <2 
	2
 2 2 \2 2 2 +*2j5|5c5 U3S	>*+5 3S	>"	5
 CcN#5 5 
5 5<%&5 5 5 5p 455 /3%&&'
 
|
c
 U3S	>*+
 3S	>"	

 CcN#
 
 5<%&
 
 
 65
 455 /3%&&'
 
|
c
 U3S	>*+
 3S	>"	

 CcN#
 
 5<%&
 
 
 65
 011|"&s)
5<%&   21" )011
 %)	 	 	\	
+
	 		
 	 3-	 U\"	 \	 	 	 21	& -..
   
+
 	
  \   /. 899 %+/E E
,E<E E 	E
 E (E 5<%&E E E :9E& .566 "&8 8L8#8 \8 8 8 7684 $() )<)L) 5<
 ) #J	)
 3Z) Cj) ) \) ) ) ) ) )r4   