j{SddlmZddlZddlmZddlZddlmcmZ dAd Z dBdCdZ dDdEdZ dFdGdZ dHdZ dIdJd#ZdKd%ZdLd*ZdMd/ZdNd4Z dOdPd9Z dQdRd@ZdS)S) annotationsN)Any frame_idxintcond_frame_outputsdict[int, Any]max_cond_frame_numc|dkst|kri}n|dks JditfdDd}| ||<tfdDd}| ||<|tz }tfdDfd  d|}fd |Dfd D}|fS) abSelect the closest conditioning frames to a given frame index. Args: frame_idx (int): Current frame index. cond_frame_outputs (dict[int, Any]): Dictionary of conditioning frame outputs keyed by frame indices. max_cond_frame_num (int): Maximum number of conditioning frames to select. Returns: selected_outputs (dict[int, Any]): Selected items from cond_frame_outputs. unselected_outputs (dict[int, Any]): Items not selected from cond_frame_outputs. Examples: >>> frame_idx = 5 >>> cond_frame_outputs = {1: "a", 3: "b", 7: "c", 9: "d"} >>> max_cond_frame_num = 2 >>> selected, unselected = select_closest_cond_frames(frame_idx, cond_frame_outputs, max_cond_frame_num) >>> print(selected) {3: 'b', 7: 'c'} >>> print(unselected) {1: 'a', 9: 'd'} z,we should allow using 2+ conditioning framesc3(K|] }|k|V dSN.0trs i/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/ultralytics/models/sam/modules/utils.py z-select_closest_cond_frames..*s'II1y==!====IIN)defaultc3(K|] }|k|V dSrrrs rrz-select_closest_cond_frames../s'IIq!y......IIrc3$K|] }|v|V dSrr)rrselected_outputss rrz-select_closest_cond_frames..7s. H H1a7G.G.GQ.G.G.G.G H Hrc(t|z Sr)abs)xrs rz,select_closest_cond_frames..8s#a)m,,r)keyc3,K|]}||fVdSrr)rrrs rrz-select_closest_cond_frames..:s.PPq$6q$9 :PPPPPPrc$i|] \}}|v || Srr)rrvrs r z.select_closest_cond_frames..;s*gggtq!QVfMfMfaMfMfMfr)lenmaxminsortedupdateitems) rrr unselected_outputs idx_before idx_after num_remain inds_remainrs `` @rselect_closest_cond_framesr. s,R3'9#:#:>P#P#P-!Q&&&(V&&&IIII%7IIISWXXX  !+=j+I Z (IIII$6IIISWXXX  **?*??  H H H H* H H H,,,,    :+  PPPPKPPPPPPgggg/A/G/G/I/Iggg / //r'pos_inds torch.Tensordim temperaturefloatc|dz}tj||j|j}|d|dzz|z z}|d|z }tj||gd}|S)aGenerate 1D sinusoidal positional embeddings for given positions and dimensions. Args: pos_inds (torch.Tensor): Position indices for which to generate embeddings. dim (int): Dimension of the positional embeddings. Should be an even number. temperature (float, optional): Scaling factor for the frequency of the sinusoidal functions. Returns: (torch.Tensor): Sinusoidal positional embeddings with shape (pos_inds.shape, dim). Examples: >>> pos = torch.tensor([0, 1, 2, 3]) >>> embeddings = get_1d_sine_pe(pos, 128) >>> embeddings.shape torch.Size([4, 128]) r dtypedevicer r2)torcharanger7r8 unsqueezecatsincos)r0r2r3pe_dimdim_t pos_embeds rget_1d_sine_perC@s"AXF Lx~ho N N NE A!,v5 6E""2&&.I 9==??IMMOO<"EEEI r?end_xend_yscaleoffsetctj||ztj}||z}tj||d}||z|z||z|zfS)upInitialize 1D and 2D coordinate tensors for a grid of specified dimensions. This function creates coordinate tensors for a grid with dimensions end_x × end_y. It generates a linear index tensor and corresponding x and y coordinate tensors. Args: end_x (int): Width of the grid (number of columns). end_y (int): Height of the grid (number of rows). scale (float): Scaling factor to apply to the coordinates. offset (int): Offset to add to the coordinates. Returns: t_x (torch.Tensor): X-coordinates for each position, with shape (end_x * end_y). t_y (torch.Tensor): Y-coordinates for each position, with shape (end_x * end_y). Examples: >>> t_x, t_y = init_t_xy(3, 2) >>> print(t_x) tensor([0., 1., 2., 0., 1., 2.]) >>> print(t_y) tensor([0., 0., 0., 1., 1., 1.]) )r7floor) rounding_mode)r:r;float32r4div)rErFrGrHrt_xt_ys r init_t_xyrPZsr.  UU]%-888A u9    C )AuG 4 4 4 : : < >> dim, end_x, end_y = 128, 8, 8 >>> freqs_cis = compute_axial_cis(dim, end_x, end_y) >>> freqs_cis.shape torch.Size([64, 64]) rDrN)rGr r9)r:r;r4rPouterpolar ones_liker=) r2rErFrRrSfreqs_xfreqs_yrNrO freqs_cis_x freqs_cis_ys rcompute_axial_cisr]ws,Uu|AsA66|#(|DJJLLsRSTGUu|AsA66|#(|DJJLLsRSTGY777HCk#w''Gk#w''G+eog66@@K+eog66@@K 9k;/R 8 8 88r freqs_cisrc|jdksJ|j|jd|jdfksJfdt|jD}|j|S)awReshape frequency tensor for broadcasting with input tensor. Reshapes a frequency tensor to ensure dimensional compatibility for broadcasting with an input tensor. This function is typically used in positional encoding operations. Args: freqs_cis (torch.Tensor): Frequency tensor with shape matching the last two dimensions of x. x (torch.Tensor): Input tensor to broadcast with. Returns: (torch.Tensor): Reshaped frequency tensor ready for broadcasting with the input tensor. Raises: AssertionError: If the shape of freqs_cis doesn't match the last two dimensions of x. r r c0g|]\}}|dz kr|ndS)r r)ridndims r z)reshape_for_broadcast..s- F F F41a!tax--QQQ F F Fr)reshape enumerateview)r^rrgres @rreshape_for_broadcastrjso 6D 19999 ?qwr{AGBK8 8 8 8 8 F F F F9QW3E3E F F FE 9>5 !!rFxqxkrepeat_freqs_kboolcbtj|jg|jddddR}|jddkrBtj|jg|jddddRnd}t ||}tj||zd}|/|| |j |fS|r|jd|jdzx}dkr|j j dkrZtj|}|j gdg|j dz z|ddR}tj|}n|j gdg|j dz z|dR}tj||zd}|| |j || |j fS) aApply rotary positional encoding to query and key tensors. This function applies rotary positional encoding (RoPE) to query and key tensors using complex-valued frequency components. RoPE is a technique that injects relative position information into self-attention mechanisms. Args: xq (torch.Tensor): Query tensor to encode with positional information. xk (torch.Tensor): Key tensor to encode with positional information. freqs_cis (torch.Tensor): Complex-valued frequency components for rotary encoding with shape matching the last two dimensions of xq. repeat_freqs_k (bool, optional): Whether to repeat frequency components along sequence length dimension to match key sequence length. Returns: xq_out (torch.Tensor): Query tensor with rotary positional encoding applied. xk_out (torch.Tensor): Key tensor with rotary positional encoding applied, or original xk if xk is empty. Examples: >>> import torch >>> xq = torch.randn(2, 8, 16, 64) # [batch, heads, seq_len, dim] >>> xk = torch.randn(2, 8, 16, 64) >>> freqs_cis = compute_axial_cis(64, 4, 4) # For a 4x4 spatial grid with dim=64 >>> q_encoded, k_encoded = apply_rotary_enc(xq, xk, freqs_cis) Nr r r`rrbmps)r:view_as_complexr4reshapergrj view_as_realflattentype_astor8typerepeatre contiguous) rkrlr^rmxq_xk_xq_outrxk_outs rapply_rotary_encrs+<   2 2 IBHSbSM I2 Iq I I I J JCNPhWYl^_N_N_%  2 2 IBHSbSM I2 Iq I I I J J JeiC%i55I  i 0 0 8 8 ; ;F {~~b!!$$RY//33N " 2 >>1!CC   E ) )*955I( (PA3).12D+EPP1PaPPPI-i.B.B.D.DEEII( (MA3).12D+EMM1MMMI  i 0 0 8 8 ; ;F >>"    + +V^^B-?-?-B-B29-M-M MMr window_sizec x|j\}}}}|||zz |z}|||zz |z}|dks|dkrtj|ddd|d|f}||z||z} }||||z|| |z||}|ddddddd|||} | || ffS)aePartition input tensor into non-overlapping windows with padding if needed. Args: x (torch.Tensor): Input tensor with shape (B, H, W, C). window_size (int): Size of each window. Returns: windows (torch.Tensor): Partitioned windows with shape (B * num_windows, window_size, window_size, C). padded_h_w (tuple[int, int]): Padded height and width before partition. Examples: >>> x = torch.randn(1, 16, 16, 3) >>> windows, (Hp, Wp) = window_partition(x, window_size=4) >>> print(windows.shape, Hp, Wp) torch.Size([16, 4, 4, 3]) 16 16 rrbrpr rUr )rgFpadripermuterz) rrBHWCpad_hpad_wHpWpwindowss rwindow_partitionrs"JAq!Q 1{? *k 9E 1{? *k 9E qyyEAII E!aAua/ 0 0 YE B q" #[" 2C[RSTTAii1aAq))4466;;B [Z[\\G RH rrpad_hwtuple[int, int]hwct|\}}|\}}|jd||z|z|zz}||||z||z||d} | dddddd|||d} ||ks||kr&| ddd|d|ddf} | S) a_Unpartition windowed sequences into original sequences and remove padding. This function reverses the windowing process, reconstructing the original input from windowed segments and removing any padding that was added during the windowing process. Args: windows (torch.Tensor): Input tensor of windowed sequences with shape (B * num_windows, window_size, window_size, C), where B is the batch size, num_windows is the number of windows, window_size is the size of each window, and C is the number of channels. window_size (int): Size of each window. pad_hw (tuple[int, int]): Padded height and width (Hp, Wp) of the input before windowing. hw (tuple[int, int]): Original height and width (H, W) of the input before padding and windowing. Returns: (torch.Tensor): Unpartitioned sequences with shape (B, H, W, C), where B is the batch size, H and W are the original height and width, and C is the number of channels. Examples: >>> windows = torch.rand(32, 8, 8, 64) # 32 windows of size 8x8 with 64 channels >>> pad_hw = (16, 16) # Padded height and width >>> hw = (15, 14) # Original height and width >>> x = window_unpartition(windows, window_size=8, pad_hw=pad_hw, hw=hw) >>> print(x.shape) torch.Size([1, 15, 14, 64]) rr rbrpr rUrN)rgrirrz) rrrrrrrrrrs rwindow_unpartitionrs4FB DAq aR"W 3{BCA Qk)2+>> q_size, k_size = 8, 16 >>> rel_pos = torch.randn(31, 64) # 31 = 2 * max(8, 16) - 1 >>> extracted_pos = get_rel_pos(q_size, k_size, rel_pos) >>> print(extracted_pos.shape) torch.Size([8, 16, 64]) r rbrr linear)sizemodeNrD) rr$rgr interpolatersrr:r;long)rrr max_rel_distrel_pos_resizedq_coordsk_coordsrelative_coordss r get_rel_posr$sK&q3vv...233L}Q<''- OOAw}Q/ 4 4 < >> B, C, q_h, q_w, k_h, k_w = 1, 64, 8, 8, 8, 8 >>> attn = torch.rand(B, q_h * q_w, k_h * k_w) >>> q = torch.rand(B, q_h * q_w, C) >>> rel_pos_h = torch.rand(2 * max(q_h, k_h) - 1, C) >>> rel_pos_w = torch.rand(2 * max(q_w, k_w) - 1, C) >>> q_size, k_size = (q_h, q_w), (k_h, k_w) >>> updated_attn = add_decomposed_rel_pos(attn, q, rel_pos_h, rel_pos_w, q_size, k_size) >>> print(updated_attn.shape) torch.Size([1, 64, 64]) References: https://github.com/facebookresearch/mvit/blob/main/mvit/models/attention.py bhwc,hkc->bhwkbhwc,wkc->bhwkN)rrgrsr:einsumri)rrrrrrq_hq_wk_hk_wRhRwr_r2r_qrel_hrel_ws radd_decomposed_rel_posrLsPHCHC S#y ) )B S#y ) )BIAq# ))AsC % %C L)3 3 3E L)3 3 3E IIac3 , ,uQQQ111aaa5E/F FqqqRSRSRSUVUVUVX\^_^_^_O_I` ` f f 39cCi  D Krabs_pos has_cls_tokenretain_cls_tokentilingc P|r|sJ|\}}|r|ddddf}|ddddf}|jd}ttj|} | | z|ksJ| |ks| |kr|d| | ddddd} |rT| ddgdt||f| jddDzddddd|d|f} ntj | ||fdd } |s| ddddS|sJtj || ddddd||zdgd S|s|d||dS|sJtj ||gd S) aCalculate absolute positional embeddings. If needed, resize embeddings and remove cls_token dimension for the original embeddings. Args: abs_pos (torch.Tensor): Absolute positional embeddings with shape (1, num_position, C). has_cls_token (bool): If true, has 1 embedding in abs_pos for cls token. hw (tuple[int, int]): Size of input image tokens. retain_cls_token (bool): Whether to retain the cls_token. tiling (bool): Whether to tile the embeddings, *instead* of interpolation (a la abs_win). Returns: (torch.Tensor): Absolute positional embeddings after processing with shape (1, H, W, C) if retain_cls_token is False, otherwise (1, 1+H*W, C). Nrbr rrpr c$g|] \}}||zdzS)rbr)rrys rrfzget_abs_pos..s$4o4o4oDAqQ!VaZ4o4o4orbicubicF)rr align_cornersr9) rgrmathsqrtrsrtileziprrr:r=) rrrrrhwcls_posxy_numr new_abs_poss r get_abs_posrs**} DAq!!!!RaR%.!!!QRR%. ]1 F ty  ! !D $;&  qyyDAIIooatR88@@Aq!LL  %**Aq64o4oCQRTUPVXcXijkjljlXmLnLn4o4o4o+opp111bqb"1" KK-V# K  &&q!Q22 2! =9+--aAq99AA!QUBOOP    8??1aB// / =9gw/Q777 7rkq_hwk_hwrescaler!tuple[torch.Tensor, torch.Tensor]c|\}} |\} } || kr| | ks Jd|||} ||} n"t|| |} t| | |} |j\}}}|||| |}|dz}|r || z| zdzn|}||z }tjd|| |z}tjd|| |z}tj| |j|j}tj| |j|j}|d| d|  || | | g}|dd| |  || | | g}tj ||z||gd ||| zd}tj ||| | d||gd || | zd}||fS) aConcatenate rel pos coeffs to the q & k tensors, so that qk^T is now effectively including rel pos biases. Args: q (torch.Tensor): Query tensor with shape (B, L_q, C). k (torch.Tensor): Key tensor with shape (B, L_k, C). q_hw (tuple[int, int]): Spatial size of query tensors as (height, width). k_hw (tuple[int, int]): Spatial size of key tensors as (height, width). rel_pos_h (torch.Tensor): Relative positional embeddings for the height axis. rel_pos_w (torch.Tensor): Relative positional embeddings for the width axis. rescale (bool): Whether to rescale for use with SDPA, which would scale by the wrong factor due to the concat. relative_coords (torch.Tensor | None): Precomputed relative coords index tensor. Returns: q (torch.Tensor): Query tensor padded so that qk^T accounts for relative position biases. k (torch.Tensor): Key tensor padded so that qk^T accounts for relative position biases. zonly square inputs supportedNg?rrr6rbr r9) rrgrsr:reyer7r8riexpandr=)rrrrrrrrrrrrrrrrr2r old_scale new_scale scale_ratiorreye_heye_ws rconcat_rel_posrs 4HCHC 3JJSCZZZ*HZZ ("  '  ' c9 - - c9 - -IAq# ))AsC % %CSI,3BsSS((Ii'K L)3 3 3i ?E L)3 3 3i ?E Ic : : :E Ic : : :E JJq#q# & & - -q#sC.@ A AE JJq!S# & & - -q#sC.@ A AE 3$eU3<<<AA!S3YPRSSA 166!S#r**E59rBBBGG3QT9VXYYA a4Kr)rrrrr r)r/)r0r1r2rr3r4)rDr)rErrFrrGr4rHr)rQrD) r2rrErrFrrRr4rSr4)r^r1rr1)F)rkr1rlr1r^r1rmrn)rr1rr)rr1rrrrrr)rrrrrr1rr1)rr1rr1rr1rr1rrrrrr1)FF) rr1rrnrrrrnrrnrr1)FN)rr1rr1rrrrrr1rr1rrnrr1rr) __future__rrtypingrr:torch.nn.functionalnn functionalrr.rCrPr]rjrrrrrrrrrrrs#"""""  10101010h466666:99999B""""6! /N/N/N/N/Nd<" " " " J%3%3%3%3P6666z# >8>8>8>8>8P$(:::::::r