
    WjWI              	          U d dl Z d dlZd dlmZmZ d dlmZ d dlmZm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d
dlmZmZmZmZmZmZmZ d
dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& d
dl'm(Z(m)Z) dgZ*e+e,         e-d<   de#de#de.fdZ/de+e#         de.fdZ0de+e#         de+e#         de+e#         fdZ1de
j2        defdZ3dedefdZ4dedede$fdZ5de,dede%fdZ6de,dedede%fd Z7de,de
j2        de%fd!Z8de,d"efd#Z9d$ Z:d% Z;de,d&ed'e+e         de+e"         fd(Z<d)ede#fd*Z=de,d+ede+e%         fd,Z>dedefd-Z?de
j2        de+e         fd.Z@de,d/ed0ede+e"         fd1ZAd)eBe,ef         defd2ZCd3ed4ed5ed6efd7ZDdS )8    N)bisect_rightinsort)Callable)Anycast)_get_device_module)ShardMetadata)ShardedTensor)DTensor)%compute_local_shape_and_global_offset   )BytesStorageMetadataChunkStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESTensorPropertiesTensorStorageMetadata)LoadItemTypeReadItemSavePlanTensorWriteData	WriteItemWriteItemType)"_check_shard_metadata_pair_overlap+_shards_get_overlap_region_wrt_saved_tensor create_read_items_for_chunk_list__all__plan
other_planreturnc                 J   | j         |j         k    rdS t          | j                  t          |j                  k    rdS t          | j        |j                  D ]\  }}|j        |j        k    r dS |j        }|j        }|j        |j        k    s |j        |j        k    s|j        |j        k    r dS |j        }|j        }|r|r|s|r dS |rU|rS|j	        |j	        k    r dS |j
        }|j
        }	|r|	r|s|	r dS |r%|	r#|j        |	j        k    s|j        |	j        k    r dS dS )a  
    Compare the two Save plans and return True if they are equal.

    Args:
        plan (SavePlan): First SavePlan to compare.
        other_plan (SavePlan): Second SavePlan to compare.

    Returns:
       True if the two plans are equal, False otherwise.
    FT)usablelenitemsziptypeindexfqnoffsettensor_datasizechunkoffsetssizes)
r   r    	plan_itemother_plan_itemplan_metadata_indexother_plan_metadata_indexr+   other_tensor_datar-   other_chunks
             q/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/torch/distributed/checkpoint/planner_helpers.py_compare_save_plansr7   *   s    {j'''u 4:#j.////u '*$*j6F&G&G )! )!"	?>_11155'o$3$9!  #'@'DDD")-F-MMM"(,E,KKK55  ++7 	 1 		 1	 55 	!, 	!#4#999uu  %E+1K k 5 [ uu  ! !M[%888{k&777 554    delta_plansc                 4    t          d | D                       S )z
    Check if any delta plan is usable, indicating the plan has changed.

    Args:
        delta_plans (List[SavePlan]): A list of delta plans to check.
    Returns:
        True if any delta plan is usable, False otherwise.
    c              3   (   K   | ]}|o|j         V  d S N)r#   ).0
delta_plans     r6   	<genexpr>z(_contains_usable_plan.<locals>.<genexpr>t   s*      NNJz/j/NNNNNNr8   )any)r9   s    r6   _contains_usable_planrA   k   s!     NN+NNNNNNr8   cached_plansc                     g }t          | |          D ]9\  }}|r|j        s|                    |           $|                    |           :|S )ac  
    Merge a list of delta plans into a single plan.

    Args:
        cached_plans (List[SavePlan]): A list of cached plans.
        delta_plans (List[SavePlan]): A list of delta plans to merge. It can contain empty plans

    Returns:
        A single merged plan. If a delta plan is not usable, use the cached plan. Otherwise, use the delta plan.
    )r&   r#   append)rB   r9   merged_planscached_planr>   s        r6   _merge_delta_local_plansrG   w   sm     L#&|[#A#A , ,Z 	,j/ 	,,,,,
++++r8   tensorc           	          t          t          j        dgt          |                                           z            |                                           S )Nr   r.   r/   )r   torchSizer$   r,   )rH   s    r6   _create_chunk_from_tensorrM      sG    
A3V[[]]!3!3344FKKMM   r8   shard_mdc                     t          t          j        | j                  t          j        | j                            S NrJ   )r   rK   rL   shard_offsetsshard_sizes)rN   s    r6   _chunk_for_shardrS      s9    
8122j-..   r8   sharded_tensorc                     |                                  j        }t          |j        |j        |j        |j        |j                  }t          t          |          ||                                  j
                  S )N)dtypelayoutrequires_gradmemory_format
pin_memoryr-   
propertiesr,   )metadatatensor_propertiesr   rV   rW   rX   rY   rZ   r   rS   r,   )rT   rN   shard_propertiesr\   s       r6   _sharded_tensor_metadatar`      s     &..00B!$&&4&4#.  J x(($$&&+   r8   r)   c                    t          |j        |j        |j                  \  }}t	          j        |          t	          j        |          }}t          t          | |          t          j	        t          t          ||          t          j        |                                          |                                                    S )NrJ   r[   r(   r'   r+   )r   shapedevice_mesh
placementsrK   rL   r   r   r   SHARDr   r   r   create_from_tensorto_localr,   )r)   rH   r/   r.   s       r6   _create_write_items_for_dtensorri      s    :f(&*; NE7 Z&&
7(;(;7EC)) #&   (:6??;L;LMM
 
 
   r8   c                     t          j        |j                  }t          t	          | |          t
          j        t          ||                    S )Nrb   )rK   rL   rQ   r   r   r   rf   r`   )r)   rT   rN   r.   s       r6   _create_write_item_for_shardrk      sM     j/00GC)) ,^XFF   r8   c                 n   t          j        dgt          |                                          z            }t	          t          | |          t          j        t          t          ||                                          t          j        |          |                                                    S )Nr   rJ   r[   rb   )rK   rL   r$   r,   r   r   r   TENSORr   r   r   rg   )r)   rH   r.   s      r6   _create_write_item_for_tensorrn      s    j!s6;;==11122GC))!#&wfkkmmLLL':6BB
 
 
   r8   bytesc                 R    t          t          |           t          j                  S )N)r(   r'   )r   r   r   BYTE_IO)r)   ro   s     r6   _create_write_item_for_bytesiorr      s*    C  "   r8   c           
          t          t          j        | t          j        |f          |t          j        |f          t          j        |f                    S N)r'   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   rq   rK   rL   ru   dest_offsetrw   storage_offsetlengths        r6   _create_read_item_for_byteior~      sU     !Z//#
N#455
F9%%   r8   c           
          t          t          j        | t          j        |          |t          j        |          t          j        |                    S rt   )r   r   rm   rK   rL   ru   rv   rw   rx   ry   s        r6   _create_read_item_for_tensorr      sN      Z--#
?33
7##   r8   checkpoint_mdlocal_chunksc                    ! g }|j         }|r|s|S t          |d         j                  }d!|dk    rLd}t          |          D ]:t	          fdt          j        ||          D                       }||k    r|}!;|dk    r'dgt          |          z   dgt          |          z  n!fd|D              !fd|D             t          t          t          |                     fd          }t          t          t          |                    fd	          }	g }
d}t          |          }|	D ]B}||         }|         \  }}t          |
|d
f          }|r|
d|= ||k     r?||         }||         } |         \  }}||k    rnt          |
||f           |dz  }||k     ?|
D ]\  }}||         }t          ||          sg }g }g }t          ||          D ]F\  }}}}|                    |           |                    |           |                    |           G|                    t          t          | |j        |          |t          | |j        |          ||                     ̐D|S )aW  
    Create a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    r   r   c              3   N   K   | ]}|j                  |j                 z   V   d S r<   rJ   )r=   r-   dims     r6   r?   z3create_read_items_for_chunk_list.<locals>.<genexpr>   sJ         c"U[%55     r8   )r   r   c                 `    g | ]*}|j                  |j                  |j                 z   f+S  rJ   r=   c	sweep_dims     r6   
<listcomp>z4create_read_items_for_chunk_list.<locals>.<listcomp>.  H     
 
 
 Yy!19Y#7!'):L#LM
 
 
r8   c                 `    g | ]*}|j                  |j                  |j                 z   f+S r   rJ   r   s     r6   r   z4create_read_items_for_chunk_list.<locals>.<listcomp>2  r   r8   c                      |          d         S Nr   r   )idxsaved_boundss    r6   <lambda>z2create_read_items_for_chunk_list.<locals>.<lambda>9      S)!, r8   )keyc                      |          d         S r   r   )r   local_boundss    r6   r   z2create_read_items_for_chunk_list.<locals>.<lambda>=  r   r8   N)saved_shardcurrent_shardr   )chunksr$   r.   rangemax	itertoolschainsortedr   r   r   r   rD   r   r   )"r)   r   r   
read_itemssaved_chunksnum_dimsmax_sizedim_sizesaved_sorted_indiceslocal_sorted_indicesactive_saved	saved_ptr	num_saved	local_idxlocal_chunklocal_start	local_endcutoffstorage_idxstorage_chunksaved_start	saved_end_rx   rv   ry   _dimoffset_for_saved_tensoroffset_for_current_tensorr}   r   r   r   r   s"                                 @@@@r6   r   r      s   * "$J 'L | <?*++H I!||?? 	  	 C    &_\<HH    H (""#	 1}}x#l"3"33x#l"3"33
 
 
 
!
 
 

 
 
 
!
 
 

 "c,  ,,,,   "c,  ,,,,  
 +-LI())I) 1 1	"9-!-i!8Yl[",=>> 	&WfW%)##.y9K(5M%1+%>"Ki''<)[!9:::NI )## + 	 	NA{(5M5k=QQ  OLG =)  
' 
' ')  &&'>???##$=>>>v&&&&,,S+2EyQQ!-"/]2K# # %4#  
 
 
 
)	> r8   
state_dictc                 .   g }|                                  D ]\  t          t                    r$|                    t	                               >t          t
                    r:|                    fd                                j        D                        t          t          j
                  r$|                    t                               |                    t                               t          |          S )Nc              3   :   K   | ]}t          |          V  d S r<   )rk   )r=   rN   r)   objs     r6   r?   z5_create_default_metadata_only_plan.<locals>.<genexpr>  sC         -S#x@@     r8   )r%   
isinstancer   rD   ri   r
   extendr]   shards_metadatarK   Tensorrn   rr   r   )r   requestsr)   r   s     @@r6   "_create_default_metadata_only_planr   y  s&   H$$&& F FSc7## 
	FOO;CEEFFFF]++ 	FOO      # >       U\** 	FOO9#sCCDDDDOO:3DDEEEEHr8   objectc                 6    t          d          r                               S t          t                    r! fd                                D             S t          t
          j                  rt                     gS t                     gS )N__create_write_items__c                 <    g | ]}t          |j                  S r   )rk   r]   )r=   shardr)   r   s     r6   r   z'_create_write_items.<locals>.<listcomp>  s7     
 
 
 )fenEE
 
 
r8   )	hasattrr   r   r
   local_shardsrK   r   rn   rr   )r)   r   s   ``r6   _create_write_itemsr     s    v/00 =,,S&999	FM	*	* =
 
 
 
 
,,..
 
 
 	
 
FEL	)	) =-c6::;;.sF;;<<r8   c                     t          | j        | j        | j                  \  }}t	          j        |          t	          j        |          }}t          ||          S rP   )r   rc   rd   re   rK   rL   r   )rH   r/   r.   s      r6   _create_chunk_from_dtensorr     s`    :f(&*; NE7 Z&&
7(;(;7E   r8   c                 L   t          | d          r|                                 }n~t          | t                    rd |                                 D             }nJt          | t
          j                  rt          |           g}nt          dt          |                      |S )N__create_chunk_list__c                 6    g | ]}t          |j                  S r   )rS   r]   )r=   r   s     r6   r   z&_create_chunk_list.<locals>.<listcomp>  s0     
 
 
16U^,,
 
 
r8   zMUnsupported Type, expecting one of [Tensor, DTensor, ShardedTensor] ,but got )
r   r   r   r
   r   rK   r   rM   
ValueErrorr'   )rH   r   s     r6   _create_chunk_listr     s    v.// 
3355	FM	*	* 


 
:@:M:M:O:O
 
 
 
FEL	)	) 
1&99:'V' '
 
 	

 r8   mdr   c                 @   t          |t                    s[	 t          |          }n9# t          $ r,}t          d|  ddt	          |           z             |d }~ww xY wt          | ||          S t          t          |           dt          |           dd          gS )Nz Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   rz   )r   r   r   r   r'   r   r~   r   )r)   r   r   r   exs        r6   _create_read_itemsr     s    b.// 
	-c22LL 	 	 	:3:::GT"XXGGH  	 0RFFF )(--+C00   
 	
s   ' 
A'AAc                 v    dt           fd}dt          fd}dt          j        fd}t	          | |||           dS )zP
    Initializes meta tensor if the meta tensor is DTensor or torch.Tensor.
    valuec                    t          | dd           }|t          j        d          k    rt          j                                        j        }t          t          j        t          |          	                                          }t          j
        |                                 |          }t          j        || j        | j        |                                 |                                           }|S | S )Ndevicemetar   )rd   re   rc   stride)getattrrK   r   distdistributed_c10d_get_pg_default_devicer'   r   r   current_device
empty_likerh   r   
from_localrd   re   r,   r   )r   r   device_typenew_local_tensordtensors        r6   dtensor_funcz&_init_state_dict.<locals>.dtensor_func  s    $//U\&))))/FFHHMK0==LLNN F  %/0@0@PPP ( !- +jjll||~~  G NLr8   c                     t          | dd           }|t          j        d          k    r t          dt	          |            d          | S )Nr   r   zFound unsupported type z for meta device loading.)r   rK   r   RuntimeErrorr'   )r   r   s     r6   sharded_tensor_funcz-_init_state_dict.<locals>.sharded_tensor_func  sU    $//U\&))))P$u++PPP   Lr8   c                 @   t          | dd           }|t          j        d          k    rtt          j                                        j        }t          t          j        t          |          	                                          }t          j
        | |          }|S | S )Nr   r   r   )r   rK   r   r   r   r   r'   r   r   r   r   )r   r   r   rH   s       r6   tensor_funcz%_init_state_dict.<locals>.tensor_func  s    $//U\&))))/FFHHMK0==LLNN F %eF;;;FMLr8   N)r   r   rK   r   _iterate_state_dict)r   r   r   r   s       r6   _init_state_dictr     s    
G    *3    
5< 
 
 
 
 	    r8   iter_objectr   r   r   c                 t   t          | t                    r |           S t          | t                    r |           S t          | t          j                  r |           S t          | t
          t          t          t          t          j
        f          s| | S t          | t                    r1|                                 D ]\  }}t          |          | |<   | S t          | t          t          f          r6fd| D             }t          | t                    rt          |          }|S dS )a$  
    Iterate through the state dict, applying the given functions to each tensor type
    and update the state dict in place.

    Args:
        iter_object (Any): the target state_dict.
        sharded_tensor_func (Callable): the function to apply to ShardedTensor
        dtensor_func (Callable): the function to apply to DTensor
        tensor_func (Callable): the function to apply to Tensor

    # TODO: let state_dict_util._iterate_state_dict() to support in place option
    so we don't need to have two versions of _iterate_state_dict.
    Nc                 4    g | ]}t          |          S r   )r   )r=   vr   r   r   s     r6   r   z'_iterate_state_dict.<locals>.<listcomp>+  s8     
 
 
  <1DkRR
 
 
r8   )r   r   r
   rK   r   intfloatstrro   ioBytesIOdictr%   r   listtuple)r   r   r   r   r   r   rets    ```   r6   r   r     sk   ( +w'' |K(((	K	/	/ "";///	K	.	. {;''';eS% DEE	K	&	& %++-- 	 	JC2|%8+   K 	K$	/	/ 
 
 
 
 
 
 
 
 
 k5)) 	**C
 r8   )Er   r   bisectr   r   collections.abcr   typingr   r   rK   torch.distributeddistributedr   torch._utilsr   !torch.distributed._shard.metadatar	   'torch.distributed._shard.sharded_tensorr
   torch.distributed.tensorr   torch.distributed.tensor._utilsr   r]   r   r   r   r   r   r   r   plannerr   r   r   r   r   r   
reshardingr   r   r   r   r   __annotations__boolr7   rA   rG   r   rM   rS   r`   ri   rk   rn   rr   r~   r   r   r   r   r   r   r   r   r   r   r   r8   r6   <module>r	     s   					     ' ' ' ' ' ' ' ' $ $ $ $ $ $                      + + + + + + ; ; ; ; ; ; A A A A A A , , , , , , Q Q Q Q Q Q                                        99c 9 9 9>h >H > > > > >B	OtH~ 	O$ 	O 	O 	O 	Ox.h 
(^   2el 7K    } 1E    !-:   ( g )    (	+7D   
s 
EL 
Y 
 
 
 
 C    
 
 

 
 
x	x(x +,x 
(^	x x x xv? x    "=S =# =$y/ = = = =w 3G    u| 5I0J    &
C 
] 
 
h 
 
 
 
.4c3h 4C 4 4 4 4n,,, ", 	, , , , , ,r8   