
    ajT                     $    d dl Z d Zd Z	 ddZdS )    Nc                    |                                  dk    r%t          d|                                   d          | j        d         dz  dk    rt          d| j        d                    | j        t          j        k    rt          d| j                   | d d dd df         dz  d	z  | d d dd df         dz  z  S )
N   weight must be 2D, got D   r   z"weight.shape[1] must be even, got weight.dtype must be int8, got       )dimAssertionErrorshapedtypetorchint8weights    n/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/torch/quantization/_quantized_conversions.pypack_int4_to_int8r      s    zz||qFvzz||FFFGGG|AaS&,q/SSTTT|uz!!Mv|MMNNNAAAqt!tG_s"q(VAAAqt!tG_s-BCC    c                 p   |                                  dk    r%t          d|                                   d          | j        t          j        k    rt          d| j                   t          j        | dz  | dz	  dz  fd                              | j        d         d| j        d	         z            S )
Nr   r   r   r   r	   r
   )r   r   r   )r   r   r   r   r   stackviewr   r   s    r   unpack_int8_to_int4r      s    zz||qFvzz||FFFGGG|uz!!Mv|MMNNN;v{c&9:BBBGGQV\!_,  r   Fc                    |                                  dk    r%t          d|                                   d          | j        t          j        k    rt          d| j                   |t          j        k    r"|t          j        k    rt          d|           | j        j        dk    rt          d| j        j                   | j        }|sg|t          j        k    r| j        }nQ|t          j        k    r>t          t          |                     t          j                            j                  }n| }|j        \  }}|t          j        k    rdnd	}||z  d
k    rt          d| d|           |d	z  d
k    rt          d|           |t          j        k    rut          j        g d|          t          j        d
|dz  |                              dd          dz                      |dz  d          z                       d          }ntt          j        g d|          t          j        d
|dz  |                              dd          dz                      |dz  d          z                       d          }|                    d||          }|t          j        k    rdnd}	d|	z  }
t          j        d
||	z  |          |dz  |	z  z                      dd                              d|dz  |	z                                d          }t          j        d
|dz  |
z  |          |	|
z  z                      dd                              d|
                              d                              |          }t          j        d
|	|          |
z                      dd                              d|dz                                d                              ||	z            }t          j        d
|
|                              |dz  |z  |
z            }||z   |z   |z   }|                    d                              t          j                  }t          j        |          }|                    d
||           |                    | j                  }|                    d          }t          j        |          }|t          j        k    r|                    t          j                  dz                       |j                  }|d
d d         |d
d d<   |dd d         |dd d<   |dd d         |dd d<   |dd d         |dd d<   n|t          j        k    r|dz  dz   dz  }|dd d         dz  |d
d d         z  }|dz	  dz  dz   dz  }|dd d         dz  |d
d d         z  }|d
d d         |d
d d<   |dd d         |dd d<   |d
d d         |dd d<   |dd d         |dd d<   |t          j        k    r
|dz  }|dz  }|                    ||                              t          j                  S )Nr   r   r   r   z%dtypeq must be int8 or quint4x2, got cudazweight must be on CUDA, got     @   r   znrows must be divisible by z, got z#ncols must be divisible by 64, got )r   r
         r      	      r      
               r	   )device   r   )r   r   r
   r    r   r!   r   r"   r   r&   r#   r'   r$   r(   r%   r	   r
      r&   r	   r   )r   r   r   r   r   quint4x2r)   typeTr   r   r   r   tensorarangereshapeexpand
index_copyrepeatint32
zeros_likescatter_
empty_liketointuint8)r   dtypeq	transposer)   outpncolsnrowsdivisorcols_permutedmagic0magic1tmp0tmp1tmp2tmp3outp_offsetstmps                    r   8quantized_weight_reorder_for_mixed_dtypes_linear_cutlassrL   "   sm    zz||qFvzz||FFFGGG|uz!!Mv|MMNNN%. 8 8MVMMNNN}V##PFM<NPPQQQ]F  UZ8DDu~%%$%8UZ9P9P%Q%Q%STTD:LE5en,,bb"Gw!Q7QQ%QQRRRrzQJ5JJKKK LFFF   |Au{6:::BB2qIIBNVVR 	 $r(( 	 LFFF   |Au{6:::BB2qIIBNVVR 	 $r(( 	 ??1mT22D EN**QQF6\F 
a&	8	8	8EQJ<O	P	b!	5A:&	'	'	b	 	 
a!v-f	=	=	=&	Q	b!	6			b	 	 
a	/	/	/&	8	b!	5A:			b		 	  	 <6&11188!e9Kv9UVVD$;%,L
))B--

U[
)
)CC  DMM!\3'''99V\""D ))B--CC  Dvvei  3&**3955AYQTT
AYQTT
AYQTT
AYQTT

	5>	!	!sa3&QTT
a41:-(c!Q&#-QTT
a41:-!$Q$ZQTT
!$Q$ZQTT
!$Q$ZQTT
!$Q$ZQTT

!99UE""''444r   )F)r   r   r   rL    r   r   <module>rN      sX    D D D   $p5 p5 p5 p5 p5 p5r   