Wjb3:UddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZddlmZmZddlmZmZddlZddlZddlZddlmZddlmcm Z!ddl"m#Z#m$Z$ddl%m&Z&dd l'm(Z(dd l)m*Z+dd l,m-Z-m.Z.dd l/m0Z0dd l1m2Z2ddl3m4Z4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ddlZ>ddl?m@Z@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHddlImJZJddlKmLZLddlMmNZNdd lOm*Z*ddlPmQZQddlRmSZSmTZTmUZUmVZVddlWmXZXddlYmZZZm[Z[m\Z\ddl]m^Z^ddl_m`Z`ddlambZbmcZcmdZdmeZemfZfdd lgmhZhmiZimjZjerddlkZlddlmZme*jnZod!epd"<ejqerZsd#epd$<ejtjuZuejtjvZveGd%d&ZweGd'd(ZxeGd)d*Zydd.Zzdd1Z{dd2Z|dd4Z}Gd5d6Z~dd7Ze~Zdd:Zdd;Zdd>Z dddJZddKZddLZddMZddNZddOZddPZddQZddRZddVZdd[Zdd^Z dddhZddnZddqZddsZddtZdduZddxZddyZddzZddZ dddZddddZdddddZedZddZddZddZejddZddZddZddZddZddZddZddZddZddZ dddZddZddZd dZd dZd dZd dƄZddlmZd dʄZdd˄Z ddd̈́ZddτZdd҄ZddԄZ ddd֜dd؄Z dddZdS() annotationsN) defaultdictdeque)Callable) dataclassreplace)Any TYPE_CHECKING)countersis_node_meta_valid)(create_structured_trace_for_min_cut_info)is_with_effects)config)CustomKnapsackSolverCustomRuntimeEstimator)FakeScriptObject) is_builtin) LazyStringtrace_structured) trace_log)extract_tensor_metadata) BackwardState) is_sym_node py_sym_types) magic_methodsmethod_to_operator)find_symbol_binding_fx_nodes free_symbolsis_symbol_binding_fx_node size_hintstatically_known_falsestatically_known_true) graph_drawer) OrderedSet)CheckpointPolicy)GraphInfoProvider) dp_knapsackdp_knapsack_sliding_hirschberggreedy_knapsack ilp_knapsack)KnapsackEvaluator) AOTOutputSavedForBackwardsAOTOutput#SavedForBackwardsNoVcCheckAOTOutput)_is_functional_graph)get_aot_graph_name)_is_bwd_seed_offset_is_fwd_seed_offset _is_primal _is_tangentget_cuda_generator_meta_val) fx_graph_cseget_aten_targetraise_getitemsboolAOT_PARTITIONER_DEBUGzlogging.LoggerlogcneZdZUdZded<ded<ded<ded<ded<dd Zdd ZddZddZddZ dS)OpTypesz8Class for keeping track of different operator categorieszOrderedSet[Callable[..., Any]] fusible_opscompute_intensive_ops random_opsview_opsrecomputable_opsnodefx.Nodereturnr:c.t||jvSN)r8r?selfrDs b/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/torch/_functorch/partitioners.py is_fusiblezOpTypes.is_fusiblegst$$(888c.t||jvSrH)r8r@rIs rKis_compute_intensivezOpTypes.is_compute_intensivejst$$(BBBrMc.t||jvSrH)r8rArIs rK is_randomzOpTypes.is_randommst$$77rMc.t||jvSrH)r8rBrIs rKis_viewzOpTypes.is_viewpst$$ 55rMc.t||jvSrH)r8rCrIs rKis_recomputablezOpTypes.is_recomputablesst$$(===rMNrDrErFr:) __name__ __module__ __qualname____doc____annotations__rLrOrQrSrUrMrKr>r>]sBB////9999....,,,,44449999CCCC88886666>>>>>>rMr>ceZdZUded<ded<ded<ded<ded<ded <ded <ejdd ZddZddZddZ ddZ dS)NodeInfo list[fx.Node]inputsOrderedSet[fx.Node]_required_fw_nodesrequired_bw_nodestangents_closureunclaimed_nodesdict[fx.Node, int]fw_orderstatic_lifetime_input_nodesrFcJtdjDfdS)Nc3K|]}|VdSrHr\.0ns rK z-NodeInfo.required_fw_nodes..s" 0 01Q 0 0 0 0 0 0rMcj|SrH)rg)rmrJs rKz,NodeInfo.required_fw_nodes..s a@PrMkey)sortedrbrJs`rKrequired_fw_nodeszNodeInfo.required_fw_nodess: 0 0/ 0 0 06P6P6P6P    rMrmrEr:c||jvSrH)rbrJrms rKis_required_fwzNodeInfo.is_required_fwsD+++rMc||jvSrH)rcrws rKis_required_bwzNodeInfo.is_required_bwsD***rMc||jvSrH)rerws rK is_unclaimedzNodeInfo.is_unclaimedsD(((rMintcT||jvrtd|d|j|S)NNode z not in fw nodes!)rbAssertionErrorrgrws rK get_fw_orderzNodeInfo.get_fw_orders7 D+ + + !=!=!=!=>> >}QrMN)rFr_)rmrErFr:)rmrErFr}) rWrXrYr[ functoolscached_propertyrurxrzr|rr\rMrKr^r^ws++++****))))((((    4444    ,,,,++++))))      rMr^cBeZdZUded<ded<ded<ded<ded<dS) MinCutOptionsr:ban_if_used_far_apartban_if_long_fusible_chainsban_if_materialized_backwardban_if_not_in_allowlistban_if_reductionN)rWrXrYr[r\rMrKrrsN$$$$&&&&!!!!rMrrDrErFch|jddtjtjfvS)N recompute)metagetr%MUST_RECOMPUTEPREFER_RECOMPUTErDs rKmust_recomputers0 9==d + +')0 rMfx_gfx.GraphModulecH|jjD]}t|rdSdS)NTF)graphnodesrrrDs rKhas_recomputable_opsrs7   $   44  5rMc|jjD]F}t|r5t|jdr t jj|jjvrdSGdS)NtagsTF) rrrhasattrtargettorchTagnondeterministic_seededrrs rKhas_recomputable_rng_opsrs^   4   V,,  1T[5EEE44 5rMr}c t|jdtjtjfrdSt|jdtjs*t dt|jddS)Nvalr&z.expected node.meta['val'] to be SymFloat, got ) isinstancerrSymIntSymBoolSymFloatrtypers rK sym_node_sizersw$)E"U\5=$ABBq di& 7 7  UT$)EBR=S=S U U    1rMceZdZddZdS)InvalidNodeBaserFstrcdS)Nz Invalid Noder\rts rK__repr__zInvalidNodeBase.__repr__s~rMN)rFr)rWrXrYrr\rMrKrrs(rMrc6t|jdddkS)N namespace_c10d_functional)getattrrrs rKis_not_collectivers 4; T 2 26H HHrM getitem_nodefx.Node | Nonec:|jtjkrdS|jd}|jd}t |t jr |jdkrdSd|jvrdS|jd}||vrdS||}t |t jr|SdS)zGiven a getitem node, check if it extracts from a higher-order op that has kwargs mapping the key back to an original input. Returns the original input node if found, None otherwise. Nrr& call_functionkwargs) roperatorgetitemargsrfxNodeopr)r ho_resultrrroriginal_inputs rK_get_ho_op_original_inputrs h...t!!$I  A C i ) )Y\_-L-Lty'''t  h 'F &tC[N."'** 4rMc|jtjjjjtjjjjfvrdS|jd}t|tj sdSt|S)zCheck if node is a view/reshape of a higher-order op output that aliases an input. Returns the original input node from the higher-order op's kwargs if the pattern matches, None otherwise. Nr) rropsatenviewdefaultreshaperrrrr)rDsources rK_is_copy_node_bw_onlyrsa  {59>.6 8N8VWWWt Yq\F fbg & &t $V , ,,rMenvdict[fx.Node, Any]ct|}|'||vr#t||ts||St|}|'||vr#t||ts||SdS)aTry to find a valid input replacement for an invalid forward output. This handles cases where a forward output depends on backward nodes but semantically aliases an input. For example, a view of a getitem from a triton kernel that mutates a buffer in backward, or a direct getitem from such a higher-order op. The original input may be a primal or a valid intermediate node already present in the forward graph. N)rrrr)rDrrs rK_find_input_for_invalid_outputrs+400N" c ! !3~.@@ ">"".t44N" c ! !3~.@@ ">"" 4rMF joint_graphfx.Graphr`r_outputs outputs_descslist[AOTOutput]subgraph str | Noneignore_must_be_in_fw_bwcxtj}i|D]-}||j}|j|_||<.|jD]}|sHt |r|dkr||vr t|<)t|r|dkr||vr t|<M|vrR|j dkr t|<h|j dkr`tj |j i|j } fd| D} t| r t|<||fd|<|j dkr||fd|<|j d kr  g} t!||D]{\} } t#| tjrE| vrt'd | d t#| t(rd } | jt,jjjjurt | rt7| j d krmt#| j dtjrH| j dvr9t#| j dt(s| j d} | t9| } | | | 6t=d | d| | f| | }|tA| }||jd<|!|"|S)a Given a graph, extracts out a subgraph that takes the specified nodes as inputs and returns the specified outputs. This includes specifying non-placeholder nodes as inputs. The general strategy is to initialize all inputs with proxies as we encounter them, and trace through the graph, only keeping values which take in valid proxies. Then, all dead code is eliminated. backwardforward placeholderrczg|]7}t|tjt|t8Sr\)rrrr)rlxrs rK z6_extract_graph_with_inputs_outputs..PsIa))3q6?33rMc|SrHr\rrs rKrpz4_extract_graph_with_inputs_outputs..Y CFrMget_attrc|SrHr\rs rKrpz4_extract_graph_with_inputs_outputs..\rrMoutputrz couldn't be found in envNr&rz was invalid, but is outputdesc)#rGraphrnamerr_must_be_in_backward InvalidNode_must_be_in_forwardrpytreearg_tree_leavesrrany node_copyziprr RuntimeErrorrrrrrcopy_rlenrappendrrtupleeliminate_dead_codelint)rr`rrrr new_graphrDnew_nodeall_args output_valuesrx_desc replacementoutrs @rK"_extract_graph_with_inputs_outputsrs$ I"$C((33  D !))& $T**  **&&'D $D))  ))&&'D  3;;  W % %#CII W ' '-tyHDKHHH!H 8}} 'D !++D2B2B2B2BCCCII W " "!++D2B2B2B2BCCCII W M-00$$ 6 a ! ! $||"#G1#G#G#GHHH#a&/22 M# H  4 <<<,Q//=AF q(("16!9bg66)q S((&s16!9~GG)#&afQi.K&"@C"H"HK*!((555$%KQ%K%K%KLLL  Q ( ( ( (   # # # #   5// 0 0C$CHV !!### NN rMctjoWt|jtjjrt|j p|jtjj j kSrH) ris_non_builtin_to_includerrr_ops OpOverloadrr higher_order triton_kernel_wrapper_functionalrs rKrrsP  +  DK!6 7 7 W 4;@W@W.s$ K K$) K K K K K KrMrrr) rrr find_nodesnextiterrrr)rrrr fwd_outputs bwd_outputsfwd_outputs_descsbwd_outputs_descss rK_extract_fwd_bwd_outputsr)s$ K K 2 = = = J J K K KG* T,$//8/<< = =>>CGG TFS\\)  M *?*+K/**+K%&6&67%o&6&67  %68I IIrM saved_valuesrrNonecV|D]%}|j|kr||dS&dSrH)rremove)r*r saved_values rK_remove_by_namer/sI#  t # #    , , , EE $rMfwd_module_outputs#list[fx.Node] | tuple[fx.Node, ...]ct|}tt|dz ddD]}t||s|dz}n|S)Nr&)rranger)r0idxis rKfind_first_sym_noder7sk  ! !C 3)**Q.B 7 7-a011 a%C E  JrM@-q=rtorch.fx.Graph torch.fx.Nodemaxfloatminpositionc  ||5|tjjjj|f}tjjj|jd|jd<t|jd|jd<dddn #1swxYwY||5|tjjj j|dgdf}tjjj |jddgd|jd<t|jd|jd<dddn #1swxYwY||5|tjj j j|tj f}tjj j |jdtj |jd<t|jd|jd<dddn #1swxYwY||5|tjjj j||f}tjjj |jd||jd<t|jd|jd<dddn #1swxYwY||5|tjjjj|f} tjjj|jd| jd<t| jd| jd<dddn #1swxYwY|| 5|tjjjj| |f} tjjj| jd|| jd<t| jd| jd<dddn #1swxYwY|| 5|tjj j j| tjfd|d|j} tjj j | jdtj| jd<t| jd| jd<dddn #1swxYwY| S) Nrr tensor_metar3Tfp8_scale_pos__rr)inserting_afterrrrrabsrrramaxprimsconvert_element_typefloat64 clamp_min reciprocalmulTensorfloat32r) rrDr<r>r?abs_node amax_node amax_64_nodeclamp_min_nodereciprocal_nodemul_node scale_nodes rKcalculate_quantization_scalingrWs   t $ $UU&& IN  &'   %y~199$)E:JKK e'>x}U?S'T'T m$ UUUUUUUUUUUUUUU   x ( (WW'' IN  'RD$'(  !&  3 ; ; M% 2$! ! u)@ u@U(V(V }%WWWWWWWWWWWWWWW   y ) )   ** IO 0 8U]++  $)9?#G#O#O N5 !5=$ $ % ,C  e $, , -(                   | , ,   ,, IN $ ,$-  &+Y^%=%E%E  e $c& & E".E   &. . M*                   ~ . .   -- IN % - ".  ',in&?&G&G   &' ' U#/F   '/ / ]+                    / /UU&& IN  %!3''   %y~188   '   e(?x}U?S'T'T m$UUUUUUUUUUUUUUU   x ( ( Y Y(( IO 0 8EM*8(88TY88)  "'!E!M!M M% %-" " *AQVAW)X)X  & Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y sBB22B69B6BE44E8;E8B&II  I &BLL L$BOOO BQ>>RRB2UU #U rV quant_type torch.dtyperK clamp_maxc ||5|tjjjj|tjf}tjjj|jdtj|jd<t|jd|jd<dddn #1swxYwY||5|tjj j j ||f}tjj j |jd|jd|jd<t|jd|jd<dddn #1swxYwY||5|tjj j j||f} tjj j |jd|| jd<t| jd| jd<dddn #1swxYwY|| 5|tjj jj| |f} tjj j| jd|| jd<t| jd| jd<dddn #1swxYwY|| 5|tjjjj| |fd|d|j} tjjj| jd|| jd<t| jd| jd<dddn #1swxYwY| S)NrrrAfp8_quant_pos_rCrD)rErrrrHrIrrOrrrrMrNrKrZr) rrDrVrXrKrZr?target_node_32scaled_target_nodeclamp_min_scaled_nodeclamp_max_scaled_nodequant_activation_nodes rKperform_quantizationrb-s   z * *   ,, IO 0 8 &-  &+Y_%I%Q%Q Ie em& & E".E   &. . M*                   ~ . .   "00 IN  % *-1  */);)B)B   & (>* * &2I  #E *2 2  .                   1 2 2   % 3 3 IN $ ,$i0!4! ! -2IN,D,L,L  #E *I- - "5)5L ! &u -5 5 "=1                   4 5 5   % 3 3 IN $ ,'3!4! ! -2IN,D,L,L ! &u -y- - "5)5L ! &u -5 5 "=1                   4 5 5   % 3 3 IO 0 8'48(88TY88!4! !  IO 0 8 8%*51:   "5) 5L ! &u -5 5 "=1                 ! s^B&CC C (BFFF1BIII/BL  LL-BOOOtensor torch.Tensorcb|}|}||zdz S)z Calculate the size of a PyTorch tensor in megabytes (MB). Args: tensor (torch.Tensor): Input tensor Returns: float: Memory size in MB i)numel element_size)rc num_elementsrgs rKcalculate_tensor_sizeriss2<<>>L&&((L < 'K 88rMlist[torch.dtype]ctjjjddd}d|dD}|S)N!activation_quantization_aten_passallowed_dtypesztorch.bfloat16cjg|]0}tt|dd1S).r3)rrsplit)rldtypes rKrz&get_allowed_dtypes..s@16u{{3''+,,rM;)r _inductorrpost_grad_fusion_optionsrrp)rms rKget_allowed_dtypesrus__+D+ c ,--:H:N:Ns:S:SN rMc<t}t|r|jdj|vrdStjjjddd}t|jd}tjjjddds||kStjjjdddr't||kpt||k St||kS)NrFrl size_in_mbdskip_dynamo_guardsquantize_dynamic_shape) rur rrqrrsrrtrrir"r!)rDrmsize_thresholdrws rKshould_quantizer|s'))N d # #ty'7'=^'S'Su_+D+ c,'ty'788J ? ! :+  c &&G^++ ? ! : / #& . . G)n,J+J.,HIII J )~)EFF FrMctjjjddd}t t|ddS)NrlrXztorch.float8_e5m2ror3)rrsrrtrrrp)rXs rKget_quant_typer~sO'@+ c,+,, 5***3//3 4 44rMrqtuple[float, float]cFtj|}|j|jfS)z Calculate the range of values for a given torch.dtype. Args: dtype (torch.dtype): The input dtype. Returns: tuple: A tuple containing the minimum and maximum values. )rfinfor>r<)rqinfos rKcalculate_rangers" ;u  D 8TX rMc |dd}|jd}t}t|\}}t g}g}t |D]\}} | jddrbtj j j dddrct|| |d |} t|| | ||||} t| s|| n|| n|| 5|tjjjj| |fd |d | j } tjjj| jd || jd <t/| jd | jd<dddn #1swxYwY| |<fdt |D} t1| } ||z}|r| d| |z| | dz} |dt5| t6ddxxdz cc<dS)Nrr!rsaved_for_quantizationFrl use_scalingTr9r\rCrDrrAcBg|]\}}||Sr\)r)rlr6rDposition_to_quants rKrz*quantize_activation_fw..s;+21da&&rMinductor%activation_quantization_fwd_aten_passr&)r"rr~rdict enumeraterrrrsrrtrWrbrrrErrrHrIrrrr7 update_argrr )rrr%rXrKrZtensor_scale_nodessym_scale_nodesr?rDrV quant_nodeoutput_updated_argsr5 scale_nodesrs @rKquantize_activation_fwrs     * *1 -F+a.K!!J*:66Iy(*%'O#K00%5%5$ 9==15 9 9# 5%>3c-&& <4E8 24ZIx #:..7&--j9999#**:6666**400  !&!4!4 <D"J/DhDDDD"5""J  <DD Ie,jOE* 6M".66JOM2               +5 h '6? 6L6L 1 2 2C$6K  % 36I#$$6O O  a233444 Z@AAAQFAAAAAs6BG  G$ 'G$ c  d|jD}d}|D]O}|jddr0|jd|jd}tjjjdddr||5d|j dd z t fd |D}dddn #1swxYwY||5| tj jjj||f }tj jj|jd ||jd <t#|jd |jd <dddn #1swxYwY||5| tj jjj||f }tj jj|jd |jd |jd <t#|jd |jd <dddn #1swxYwY||5| tj jjj||f }tj jj|jd ||jd <t#|jd |jd <dddn #1swxYwYn||5| tj jjj||fdt+|j z}tj jj|jd ||jd <t#|jd |jd <dddn #1swxYwYt-|jD]$}||kr||kr|||%Qt4ddxxdz cc<dS)Nc(g|]}|jdk |S)rr!r s rKrz*quantize_activation_bw..s$JJJ$M1I1I1I1I1IrMrF dequant_typerlr fp8_scale_ fp8_quant_c32K|]}|jk |VdSrHr)rl bwd_input scale_names rKrnz)quantize_activation_bw.. s<&&%$>Z77"7777&&rMrrrAdequant_rDr%activation_quantization_bwd_aten_passr&)rrrpoprrsrrtrErrr#rrrHrIrrrdivrNrlistuserskeysreplace_input_withr ) r bw_inputsactivation_noderDrrVdivided_target_node_32 dequant_nodeuserrs @rKquantize_activation_bwrsJJ%+JJJIOH@H@ 9==15 9 9G @ IMM2 3 3 39==88L%>3c-''? **400!- 0A0A,PR0S0S!SJ!%&&&&)2&&&""J**:66  &+&9&9 <D"L1':''O  <DD Ie,l$(/ ;R',U3;;O(7               **?;;  -2-@-@ *1-z:.A..*:?9K9R9R',U3Z_U5K::*/600F0KE0RSS+/ >               **+ABB  #(#6#6 <D4lC$7$$L  <DD27> !%e, 8O$)%088L%m4               **400  #(#6#6 <D"L1'#di..8$7$$L  <DD Ie,l!%e, 8O$)%088L%m4               TZ__..// @ @<''DO,C,C++D,??? Z@AAAQFAAAAAs^0:C66C: =C: BF44F8 ;F8 BI==J J BL;;L? L? B(PP P fwd_module bwd_modulebwd_module_inputsdict[str, fx.Node]ctddfdtjtddfdtddfdjd d jd }|D]}d |jvr|t jd d|j}j|5j |j}dddn #1swxYwY|j d}|j |j d|j d<||j d<| |j |tjjjdddrt'jd }|d} t)|D]} t+| s| } njd d jd } | D]~}d|jvrsj| 5j |j} dddn #1swxYwY| j |j | } t-jtddfddS)Nartifactc dddS)N,before_activation_quantization_fwd_aten_passstringrencodingr\r\rMrKrpz5perform_fp8_activation_quantization..TB   rMc4dddSNFT print_outputinclude_strideinclude_deviceprint_readablersrKrpz5perform_fp8_activation_quantization..X#:44tD5  rM metadata_fn payload_fnc dddS)N+after_activation_quantization_fwd_aten_passrrr\r\rMrKrpz5perform_fp8_activation_quantization..aA   rMc4dddSrrrsrKrpz5perform_fp8_activation_quantization..errMc dddS)N,before_activation_quantization_bwd_aten_passrrr\r\rMrKrpz5perform_fp8_activation_quantization..lrrMc4dddSrrrsrKrpz5perform_fp8_activation_quantization..prrMrr!rrz^fp8_quant_pos_\d+_rrrTrrlrrr3rc dddS)N+after_activation_quantization_bwd_aten_passrrr\r\rMrKrpz5perform_fp8_activation_quantization..rrMc4dddSrrrsrKrpz5perform_fp8_activation_quantization..rrM)rrrr"rrresubrErrupdatereplace_all_uses_with erase_noderrsrrtrrreversedr5r) rrrquant_fwd_module_outputsfwd_noderquant_bwd_inputrquant_bwd_module_inputs bwd_input_locbw_inputscaled_fwd_module_outputsscale_bwd_inputs `` rK#perform_fp8_activation_quantizationrMs           :+,,,                   */::h:GGJOPQR, 3 3 8= ( ()-r8=AAI!11)<< S S","2">">HM">"R"R S S S S S S S S S S S S S S S$>.9L  ' ' 6 6 6=AO !9 :3?O  0  + +O < < <   ' ' 2 2 2 6+ c-0#'z'7'B'Bm'B'T'T"U"U/3  !899  Hx((  (  %/$4$?$?8$?$L$LQ$O$TUV$W!1 0 0Hx},,%55mDDWW&0&6&B&B &B&V&VOWWWWWWWWWWWWWWW$++HM::: / :+,,,           s$ !C::C> C> !I;;I? I? rhOrderedSet[fx.Node] | Nonectjdd dS|r d|Dng}d|D}tjjjdddr d|D}|jdd jd }d |jd D}d}|D]} | j |vrt| r| j |vr!t d | j Dd | j d<| j dj| j d<d || j j d<| j dj|| j j d<d }|rt|||dSdS)Nrlcg|] }|j Sr\rr s rKrz2enable_activation_quantization..s;;;t;;;rMci|] }|j| Sr\rr s rK z2enable_activation_quantization..sCCCd$)TCCCrMexclude_primalsFc0i|]}d|jv |j|S)primalsrr s rKrz2enable_activation_quantization..s/   $ 8R8RDIt8R8R8RrMrr!rci|] }|j| Sr\rr s rKrz2enable_activation_quantization..s)  4rMrz*Skipping quantization of static input %s: Trrr)inductor_configrtrrrsrrr"rrr|r<debugrrqr) r*rrrhstatic_input_namessaved_values_namesr0rshould_perform_fp8_quantrDs rKenable_activation_quantizationrs 044 /      ' ;;:;;;;  DClCCC 6+ c U##   (4   $)444AA!DI!L$.$4$?$?=$?$Q$Q %" , , 9* * *t/D/D *y... F RRR26DI. /(, %(8(>DIn %JN di ( -.F G@D %@P@V di ( -n ='+ $W+J DUVVVVVWWrM)rhsaved_sym_nodes%tuple[fx.GraphModule, fx.GraphModule]c   t||\}}}}|jd} gtt| } gtt | } gtt | } gtt| } gtt| }t|j|z| z| z||d}tj }|dD]}|j s+t|jt||j4|rIt!d|j Dr+t|jt||jt|r&t|j|st#dt%}g}g}|D]S}t'|}|r+||||>||Tt-|j}t/j|| D]c}d|jvr t5|jd|z }t7|d D]"}||vr|||#||z}d||||zg}g}g}D]}t=|jdt@r||E|jd d r||v||||ztC| tED]K\}}| kr@|jd d s%t#d |d dtCLt|j| | z|z|z|z| fdtGtCtC|ztC|zDzd}t|j|z|z| z| z|z||d}tHj%&||}tHj%&||}tO|||||fS)Nrrr!rc3K|]>}|jtjjjjuot |jdkV?dSrN)rrrr wait_tensorrrrrks rKrnz+_extract_fwd_bwd_modules..s_) )  H 2>F F "AG !) ) ) ) ) ) rMz'backward_state_inputs must not be emptyrc|jSrHr)ss rKrpz*_extract_fwd_bwd_modules.."s16rMrqsaved_tensor_with_no_vc_checkFzi=z, no_vc_check_start_idx=z, len(saved_values)=c~g|]9}|kr"|tkrt|nt|:Sr\)rr/r.)rlr6no_vc_check_start_idxr*s rKrz,_extract_fwd_bwd_modules..Zsa   )))a#l2C2C.C.C 0 2 2 2+A..   rMr)(r)rr"filterr4r5r3r2rrr distributed is_availablerr/rallrr$raddrr itertoolschainrrrsclearextendrrrrrr4r_lazy_graph_module_make_graph_moduler)!rr*rrrhr%r&r'r( placeholders primal_inputstangent_inputsfwd_seed_offset_inputsbwd_seed_offset_inputsbackward_state_inputs bwd_graphdistributed_enabledrD saved_symbolssaved_sym_nodes_bindingsaved_sym_nodes_derivedsymbolsymbol_bindings new_symbolsrsaved_values_with_vc_checksaved_values_no_vc_checksaved_opaque_objectsr6 fwd_graphrrrs! ` @rK_extract_fwd_bwd_modulesrs !OOOCK/1B %00M0BBL7fZ667M9vk<889NIv&9<HHIIv&9<HHIGf%7FFG2,&7:PP I +88::$$ $66PPz P L$) 4 4 4 OTY 7 7 7 7 ! PS) ) Z) ) ) & & P L$) 4 4 4 OTY 7 7 7 7  % % P L$) 4 4 4( P$%NOOO /9llM   11*400  1   f % % % # * *4 0 0 0 0 # * *4 0 0 0 03<3EFFO 7~VV % %  ! ! "49U#344}D  )9)9::: ? ?A'' # * *?1+= > > > >$ 25LLMMM"$!44 dimmE**,< = = 4 ' ' - - - - Y]]:E B B 4 $ + +D 1 1 1 1 & - -d 3 3 3 325MMNNN :;;\**4 % % %9==!@%HH $rrr4Irr_bco_p_prr3..l"%99OK     L!!C(<$=$==O@T@TT      I3       !  !    I&99, RRJ&99, RRJ"j*.I z !!rM)static_lifetime_input_indicesrh _joint_inputsr rlist[int] | Nonec g}d}|jjD]1}t|st|st |r|}2|t d|jjD],}t |s||||urn-td|Dt|}t|} |rRt|jd'tj dt||||St|d}t jst%|t'|t)||g}t+|||} g} g} t,jd d } d d}d fd }|jjD]}|jvr |jdkr&|jd|Dvr>|jt,jjjj t,jj!j"j t,jj!j#j t,jj!j$j t,jj!j$j%fvrtM|r| |||r|j'(dtRj*kr| |:||r3|rt d|d|j| |x| |s|jdkrt d|dfd|j+D}tYd|Dr| -|t]|s| | t_t`1| 2} t_t`1| 2} t j3rtg|j| } || j4}tk|| | ||\}}|j6tn|j6tn|r3| r"tq|||ts| \}}tu|}t j;rddl.s9$$tw(/B/B /B/B/B/B$$rMrzxTrying to unsafely apply AC to a non-functional graph with the default partitioner. Falling back to min-cut partitioner.)rrTis_default_partitionrDrErFr:cd|jvp6t|jdtjjS)NrAr)rrrr _subclasses FakeTensorrs rK is_tensorz$default_partition..is_tensors; ) Z IMM% %"3">. .  rMcntd|jDot|jdkS)Nc3@K|]}|jtjkVdSrH)rrrrlrs rKrnz=default_partition..is_multi_output..s,GGD x//GGGGGGrMr)rrrrs rKis_multi_outputz*default_partition..is_multi_outputs8 GGDJGGG G G $DJ!# rMc|do-|jdvo$ p!|jtjjjjuS)NF) impure_random)rr) is_impurerrrrrrr)rDr s rKr+z$default_partition..is_impuresW NNN / /  ('U;ei&@&L&TT rMrc3 K|] \}}|V dSrHr\)rlkvs rKrnz$default_partition..s73 3 !QA3 3 3 3 3 3 rMrz.Trying to apply AC on a graph with impure op: z, rz Expected z to be a tensorc&g|] }|jv |Sr\r)rlrmforward_node_namess rKrz%default_partition..s&UUUAFBT4T4T14T4T4TrMc34K|]}t|VdSrHrrks rKrnz$default_partition..s(77!{1~~777777rMrrrh)is_impure_noder&enable_activation_offloadingFrrV)@rrr r4r3rr5rr$rrr0warningswarn#min_cut_rematerialization_partitioncleanup_recompute_tagsr(unsafe_allow_optimization_of_collectivesforce_save_collectivesforce_save_effectful_opsforce_save_bw_mutation_srcclassify_nodesrrrrr named_modulesrrr_assert_scalarrprofiler_record_function_enter_new_record_function_enter_record_function_exit_RecordFunctionrrrr% MUST_SAVErrrrrrfromkeysr_sync_decision_cross_ranksrhrrrfunctionalize_rng_opsr#reordering_to_mimic_autograd_enginer6,_activation_offloading.activation_offloadingr9thread_graphsafe_rng_from_hopsrc)rrrrrh forward_nodes last_noderDgraph_has_recomputable_opsgraph_has_recomputable_rng_ops node_infor*rr$r(r+backward_usages fw_module bw_moduler6r r0s @@rKdefault_partitionrWysO@MI"( t $ $  4(8(8 >@@AAO (T1,2DlSS "*&/&K#3''$? IyO''7H'III O''7H'III!C ) #8iC4H4H$$ Iy8 BB *         %$    '    y))Iy))I.yeLLLI 9 &''!++29$OOO i rMg.Arfc||jzSrH)itemsize)rfrqs rK_tensor_nbytesrZUs 5> !!rMcjddd|jvr|jd}t|trdSt|ttfrt fd|DSt|t r-t fd |DSt|tj r |Std t|d ||j d ks"|j tjjjjurd Std|d)NrobjectrFr}ct|tjsdStt |d|jS)Nrfallback)rrrNrZr rfrqrs rK object_nbytesz_size_of..object_nbytesZsB!U\** 1i DAAA17KKKrMrr&c3.K|]}|VdSrHr\)rlrmrbs rKrnz_size_of..gs-55A}}Q''555555rMc34K|]\}}|VdSrHr\)rlrCrmrbs rKrnz_size_of..is1@@DAq}}Q''@@@@@@rMzUnknown metadata type z on node rrrzO didn't have `val` metadata; we should always have `val` metadata on the nodes.)rr\rFr})rrrrrsumritemsrrNrrrrrrrBr)rDrrbs @rK_size_ofrgYsVLLLL   i c< ( ( &1dE] + + &555555555 5 T " " &@@@@CIIKK@@@@@ @ U\ * * & =%% %NDIINNNNOOO w* uy~/L/T T Tq eeee  rMc .ddlm}|t}|jD]'}|jdkr||jjxxdz cc<(tdt| tj dddS)Nr)rrr&%sTrrreverse) collectionsrr}rrrrWr<rrsrfr itemgetter)rrcntrDs rK _count_opsrovs''''''%+c**C ++ 7o % %  $ % % % * % % %HHT6#))++8+>q+A+A4PPPQQQQQrM!list[torch._ops.OpOverloadPacket]cvg}ttjjD]}t tjj|}t |tjjsA|D]A}t ||}tj j |j vr| |nB|SrH) dirrrrrrrOpOverloadPacket overloadsr pointwiserr)r attr_nameopoverloadpacketoverload op_overloads rK pointwise_opsrzs-/C((   "59>9==*EJ,GHH  (2244  H!"2H==Ky"k&666 +,,,7 JrMrtuple[Any, ...] depth_maprflist[tuple[fx.Node, int]]cfd|D}t|tjddS)Ncji|]/}t|tjjj&||0Sr\)rrrrDr)rlargr|s rKrzsort_depths..sE #z#ux}?Q/R/R Ys^rMr&Trj)rsrfrrm)rr| arg_depthss ` rK sort_depthsrsY'+J *""$$(*=a*@*@$ O O OOrMgmc4 tj i |jdD]} | fd |<dt |jjD d fd }ttt|jj}d }tj }|D]"}|j D]} ||kr |}|}#||St|jjd |D]:}|j d kr-|jtjjjjur ||;t|jj |d D] }||tj| }|S) a This pass finds the first bwd node in the graph (by looking at users of tangents) and then reorders the graph by walking from this node to all the way to the end of the graph. At each op in this traversal, we insert this op in a new graph and try to bring only the relevant subgraph from the other non-bwd edges relevant for this op. This closely mimics the behavior of autograd engine. Why is this pass required in the first place? This is an artifact of how partitioners work today. The starting point of partitioner is a joint graph, which is fwd and then bwd graph. In the case of checkpointing, we keep portions of fwd graph in their original place in the joint graph, while obtaining a bwd graph. As a result, the resulting bwd graph has copies of recomputed fwd subgraphs followed by the original bwd graph. If we run this naively, this leads to bad memory footprint, because the fwd subgraphs are live for way longer duration than necessary. This pass reorders the operations such that we prioritize the ops for the original bwd graph while only realizing those ops from the fwd graph that are necessary at any given point in the graph. rr!c|SrHr\rs rKrpz5reordering_to_mimic_autograd_engine..s ArMci|]\}}|| Sr\r\rlr5rDs rKrz7reordering_to_mimic_autograd_engine..s B B B93T3 B B BrMrDrErFr+cX|g}t}t|dkrO|}||vs|vr0||||jz }t|dkOt |fd}|D]}|fd|<dS)Nrc|SrHr\)rmorders rKrpzSreordering_to_mimic_autograd_engine..insert_node_in_graph..s %(rMrqc|SrHr\rs rKrpzSreordering_to_mimic_autograd_engine..insert_node_in_graph..rrM)r$rrrall_input_nodesrsr)rD cur_nodesinsertable_nodesrrrs rKinsert_node_in_graphzAreordering_to_mimic_autograd_engine..insert_node_in_graphsF 0: )nnq  ==??D'''43;;   & & & - -I)nnq  ""28J8J8J8JKKK$ D DD!++D2B2B2B2BCCCII D DrMNrrDrErFr+)rrrr"rrrrrr5mathinfrrrrrrrr GraphModule) rrDrrfirst_node_in_bwd minimum_ordertangentrnew_gmrrrs @@@rKrLrLs. I"$C##}#55@@''.>.>.>.>??D B B "(.(A(A B B BEDDDDDDDD&&bhn==>>NHM!))M ) )DT{]** %d $(! )   RX^$$%?u->'?%?@'' 7o % %$+9M9U*U*U  & & &RX^$$U+<%=%?%?@##T""""X ! !"i 0 0F MrMrUtorch.fx.GraphModulerVfw_nodebw_nodedevice torch.device rng_countlast_fwd_inputlast_bwd_input#tuple[torch.fx.Node, torch.fx.Node]c|j}|td|j} |j} tjjj} |j|5|jd|} t|| j d<| }dddn #1swxYwY|j|5|jd|} t|| j d<| }dddn #1swxYwYt|j }| |d<|j|5| d| |jg|jR|}dddn #1swxYwY||| |t|j }| |d<| |5| d| |jg|jR|}||| |dddn #1swxYwY||fS) a% Note [CUDA Graph Safe RNG Functionalization] CUDA Graph capture doesn't work with get_rng_state and set_rng_state because these functions operate on CPU values, while CUDA Graph RNG capture uses on-device CUDA tensors. To solve this, we use graphsafe_set_state with a CUDA Generator registered to the CUDA Graph before capture begins. graphsafe_set_state updates the generator's pointer to reference a different GeneratorImpl, ensuring subsequent calls are correctly forwarded to the desired generator (and its cuda-tensor RNG state during graph capture). For each RNG operation's forward/backward pair: - We create two generators initialized with identical values - Each forward and backward call advances its respective generator equally - This keeps generators synchronized so forward and backward operations use matching RNG values When forward is called multiple times before backward (causing desynchronization): - We save the forward RNG state - We update the backward Generator's state before executing backward Before each CUDA Graph replay, replay_prologue updates captured RNG pointers with current states, ensuring backward Generator changes are reflected during replay. This function modifies both forward and backward computation graphs by: Creating RNG state placeholders for both passes Updating the forward node to use graph-safe RNG state Updating the backward node to use graph-safe RNG state For more details: https://github.com/pytorch/pytorch/issues/113541 Nzdevice_idx must not be Nonefwd_rng_state_rbwd_rng_state_ rng_staterrr)indexrrr_prims rng_primsgraphsafe_run_with_rng_staterErr6rrr create_noderrrrinserting_before)rUrVrrrrrr device_idxfw_graphbw_graphr fwd_rng_state bwd_rng_state fw_kwargsfunctional_fw_node bwd_kwargs rng_outputs rK%apply_graphsafe_rng_functionalizationrs^RJ:;;;HH#(<#9#V   ( ( 8 8''!334PY4P4PQQ $? $K$K 5!&'''''''''''''''  ( ( 8 8''!334PY4P4PQQ $? $K$K 5!& '''''''''''''''W^$$I*Ik  ( ( 1 1  %11  (.07<00 2                  !!"4555    gn%%J+J{  " "7 + +%%))  (.07<00 *  %%j111G$$$%%%%%%%%%%%%%%% > ))sI7BB!B?7DD D(E44E8;E8AH55H9<H9 num_sym_nodesc 6 'tj}dd}d d 'd!d }||}||}||} i} |jjD]} t | rtt | jd r_tjj | jj vrB| j |vs | j | vrV|| j } || j } | | j }| |d| | <tj j j}tj j j}d}|jdD]} d| j vr| }n|t#dg}t%t'|jd}t%t'|jd}t)'fd| D}|tjdt1|dk}tjj}t4jo| o|j p |jj}t?| D]V\}}|d} |d}'| }|j}|j}|r'|%|j dkrtC||| |||||\}}X|"| 5|#d|| jg| j$R| j%}|#dtLj'|dfi}|||j(d<|#dtLj'|dfi} tSj)| j(| _(| *| |+| |,|dddn #1swxYwY|"|5dt%|}!|-|!}"|||"j(d<dddn #1swxYwY|"|5|#d||"|jg|j$R|j%} |*| |+|dddn #1swxYwYX|rt%t]|jd}#|#j$d}$t1|$|z }%|$d|%t_|z|$|%dz}&|j0|&|j+|#|1|1||fS)"NgmodrrFrci}|jjD]I}|jdkr.get_rng_opsbsd+- J$ / /D?**DK00+I59III*. TY'rMrDrEtorch.device | Nonecd|jvrdS|jd}t|ts|f}|D]5}t|tjr|jjdkr |jcS6tjdS)zV Check the example value of the node outputs to find the device type. rNcudacpu)rrrrrNrr)rD candidates candidates rK get_devicez)functionalize_rng_ops..get_devicems  ! !4Yu% *e,, '$J# , ,I)U\22 ,#(F22$++++|E"""rMrrdc\ddlm}|}|td|5|H|jdkr=|t jcdddS|t jcdddS#1swxYwYdS)Nr)detect_fake_modezfake_mode must not be Noner) torch._guardsrrr from_tensorrr get_rng_state)rr fake_modes rKget_sample_rng_statez3functionalize_rng_ops..get_sample_rng_states2222222$$&&   !=>> >  @ @!fkV&;&; ,,UZ-E-E-G-GHH @ @ @ @ @ @ @ @(()<)>)>?? @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @s>B!/%B!!B%(B%r)fwdbwdrr!rzaCouldn't find tangent node in graph inputs. This is unexpected, please file a bug if you see thisc3:K|]}|dVdS)rNr\)rl node_pairrs rKrnz(functionalize_rng_ops..sC)2 9U#$$rMrr&rrrrrrrrng_state_output_r)rrrFr)rDrErFr)rrrFrd)2rcountrrrrrrrrrrrrrun_and_save_rng_staterun_with_rng_stater"rr#rr$valuesdiscardrrrsrgraphsafe_rng_functionalizationfallback_random test_configs*graphsafe_rng_func_ignores_fallback_randomrrrrrrrrrrcopyrrrrr$rr recompile)(rrUrVruidrrjoint_graph_rng_opsfw_graph_rng_opsbw_graph_rng_opsrecomputable_rng_ops_maprD base_noderrrun_and_save_rngrbw_tangent_start_nodefw_rng_state_outputsrrdevicesmulti_cuda_devices ind_config'use_rng_graphsafe_rng_functionalizationrrrrrrstater state_namebw_rng_state_nodefw_output_node fw_outputssym_node_start_idxrrs( @rKrKrKGsf2 /  C    ####$ @ @ @ @&+l33"{9--"{9--!"(SS 4  S V,, S 1T[5EEE y 000DIEU4U4U+DI6I&ty1G&ty1G:A'2R2R $Y /|-D/B **m*<<  ! !$( ! E "$ o   +-(9?#=#=#=#O#OPPQQN(9?#=#=#=#O#OPPQQN6N6U6U6W6WG OOEL''(((W)'J. " " * * R&Q ,!**B*I*I*K*K L LP-P- 9E"E"G$$?? 4G -" v%%-R . . *NNN**733! 3! 3%-%9%9#$  #>&: & &"!,,#$,a0 - %9$8$@$@ 5!%11#$*2 #')GL"9"9 --j999##G,,,$++E222C! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3! 3H**+@AA M M<c<< $,$8$8$D$D!0D0DV0L0L!&u- M M M M M M M M M M M M M M M **733 - -%11#&*! #>2   --j999##G,,, - - - - - - - - - - - - - - -& 3d9?#=#=#=#J#JKKLL#(+  __}< *** +()) *+,,- .  w'''"">222   i s8C'OO O (;P//P3 6P3 AR33R7 :R7 c|jjD]Y}t|jtjjr3|jjdkr#t|stj |j d<ZdS)z By default, the partitioner is not allowed to recompute collectives unless they come from a user-annotated AC region. See Note [Recomputing collectives in the partitioner] rrN) rrrrrrrrrr%rHr)rrDs rKr=r=+st "(@@ t{EJ$9 : : @ %);;;"4((<&6%?DIk " @@rMcdfd |jjD]:}t|r)t|st |s |;dS)a\ Force save outputs from with_effects nodes wrapping effectful ops. Effectful ops (registered via _register_effectful_op) should not be recomputed because they may have arbitrary global side effects (I/O, RNG state, collectives, etc.). We mark the tensor outputs of with_effects as MUST_SAVE to prevent recomputation of the effectful op. The with_effects node returns a tuple (token, result). We recursively find all leaf outputs extracted via getitem and mark them as MUST_SAVE. Since these are saved, the with_effects op doesn't need to be recomputed in backward. rDrErFr+c|jD]h}|jtjurS|t |jdttfstj |jd<idS)Nrr) rrrrrrrrrr%rH)rDrmark_getitem_outputss rKrz6force_save_effectful_ops..mark_getitem_outputsHsxJ H HD{h...$$T***!$)--"6"6 FFH-=-GDIk*  H HrMNr)rrrrr )rrDrs @rKr>r>:sHHHHHH"('' D ! ! '"4(( ').. ' !  & & & ''rMct}t|jjD]}|jdkr|jt jjj j u}|rmt|r | |j dt|r.|j d|vrtj|j djd<dSdS)Nrrr&r)r$rrrrrrrrrrrrrrr%rHr)rhas_mutation_in_bwrDis_copy_s rKr?r?Xs5?LL+122 7h   ;%)."6">>  +D11 5"&&ty|444*400 LTYq\EW5W5W1A1K ! !+. EE!rMc|jtjkrdS|jd}t |t jurtdt |d|jvo |j dkS)NFrz#expected parent to be fx.Node, got rAr) rrrrrrrrrr)rDparents rKis_getitem_of_multi_outputrosk {h&&&u Yq\F F||27""Q4<<QQRRR  + J?0JJrMr cX|jjD]}t|r|jD]S}t|rBd|jvr9d|jvr0|jd|jdkrt j|jd<T|jddr2td|jDst j|jd<d|jvrWtd|jDr9t|rd|j djvs|rt j|jd<|S)a If there are two consecutive checkpointed blocks with no operator in between, we would still want to stash the tensor at the boundary of checkpointed blocks. The following pass makes the last output node non-recomputable to allow for that. ac_graph_idrhas_backward_hookFc34K|]}t|VdSrHrr's rKrnz)cleanup_recompute_tags..sCEE)-t$$EEEEEErMc34K|]}t|VdSrHrr's rKrnz)cleanup_recompute_tags..s*@@TN4((@@@@@@rMr) rrrrrr%rHrrrr)rr rDrs rKr;r;xso"(+@+@ $  * @  H H"4((H%22%22 -049]3KKK-=-GDIk*y}}0%88 DEE15EEEBB D&*:)C +&  * *@@TZ@@@@@ ++400 +6CdiPQlFW5W5W$6X&6%?DIk " rMrSmin_cut_optionsdont_ban)tuple[list[fx.Node], OrderedSet[fx.Node]]c <=>?@ABCDEFGHIJKttItrZtd|jD}|tdIjDz }t d|dbd =dbd >db=>Ifd A d dlGn"#t$r}td |d}~wwxYwdcAIfd CddCIfd }dcAfd BdeBIfd }Gj Htfrd'} d}n#t j} d(}n|| j \} }|rH| t jks | tBkr-H| jd z| jdz| d)|n(H| jd z| jdz| *| j"D]5}H| jdz|jd zt jd+6dhAfd0 }j#rj$D]}fd1|j"D}fd2|j"D}tK|d kr||tM|}tO|j"D]}|rz(||kraA||rU|d?}tq|d@5}!|!9@dddn #1swxYwYn#tt$r}dA|dB}Yd}~nd}~wwxYwdC;Gj<j=>H?tmd:dD?fdE=tH}"|"r i}#d6g}$didG}%|"D]!\}&}'}(|$@|'|&d6kr5|%|'})|#A|)g@|(W|'dkr5|%|&})|#A|)g@|(|%|&|%|'kr5|%|&})|#A|)g@|(|%|&}*|%|'}+|#A|+g@dH|*#g},|#BD];\}-}.|,@dI|-dJ|.D]}/|,@dK|/dS)r_overloadpacketN)rrrrrr s rKrnz solve_min_cut..sZ& & w/))gdkCT.U.U)  + , ,))))& & rMc34K|]}t|VdSrH)rrlr6s rKrnz solve_min_cut..s94 4 CFF4 4 4 4 4 4 rMz&Ops banned from re-materialization: %sarEbrFr:c|jtjjjkrdS|jd}tjj|\}}|D]2}|j |}||urdSt|tr||vrdS3dSNFrT) rrrrauto_functionalizedr_higher_order_opsauto_functionalizeget_mutable_argsrrr)rr mutable_opmutable_arg_namesrCrrs rK!can_fuse_into_auto_functionalizedz8solve_min_cut..can_fuse_into_auto_functionalizeds 8uy-A A A5VAY   # 6 G G     &  D(4.CCxxtt#t$$ 8844urMc|jtjjjkrdS|jd}|D]/}|jd}|t d||}||urdS0dS)NFtensors_to_clonerzkwargs must not be NoneT)rrrrrrr)rrrrrrs rK.can_fuse_into_triton_kernel_wrapper_functionalzEsolve_min_cut..can_fuse_into_triton_kernel_wrapper_functionals| 8uy-N N N5H%78%  D(8,F~$%>???,CCxxtturMcHt|tjkrdS||rdS||rdS|jtjur*|jdjtjj j urdS |o |S)NTrF) r8rcatrrrrrrrrrL)rrrrop_typess rKrLz!solve_min_cut..is_fusibles 1   ) )4 , ,Q 2 2 4 9 9!Q ? ? 4 H( ( (q  y%FGG 5""1%%@(*=*=a*@*@@rMrzANeed networkx installed to perform smart recomputation heuristicsrDcz|rdSt|g}t|dkr|}|jD]P}|s||sdS|r||Qt|dkdSr)rSr$rrrrxr)rDrcurrrLrSrs rKis_materialized_backwardsz0solve_min_cut..is_materialized_backwardss   D ! ! 5v&& )nnq  --//C  ( ( //55 jjd>S>S 44##D))(MM$''' )nnq  urMrc@|jdkrdS|jtjurdS|jddt jkrdStj r |rdS|jtj j tjj fvrdSjr|sdSn?|rdS|rdSt'|rdSjr;|r0t*d |t/|jd S|jd kr|jtjkrd Sjr8t9d |jD}t=|}|dz|krdSdS)zRReturns reason string if node should be banned from recomputation, None otherwise.rNrzmarked MUST_SAVEznot in recomputable allowlistz random opzcompute intensive opznon-builtin opzmaterialized backwards: %s %szmaterialized in backwardiztoo far from backwardc3hK|]-}t|tjt|V.dSrH)rrrrgrs rKrnzBsolve_min_cut..should_ban_recomputation..AsM%% !*Q2H2H% %%%%%%rMrz reduction op)rrrrrrr%rHrrecompute_viewsrSrlift_fresh_copyr lift_freshrrUrQrOrrr<rrr dist_from_bwmax_dist_from_bwrrerrg)rDinput_tensors_size output_sizerrrs rKshould_ban_recomputationz/solve_min_cut..should_ban_recomputations 7o % %4 ;(* * *4 9==d + +/?/I I I%%  ! h&6&6t&<&< 4 ;4/79PQ Q Q4  2 (++D11 766 7!!$'' #"{,,T22 .--(.. (''  7 ..is_materialized..Ms/EE$zz$--EEEEEErM)rrr)rDrLs`rKis_materializedz&solve_min_cut..is_materializedIsA 7m # #4EEEEE$*EEEEEEErMrhratuple[float, str | None]c tjr||vrdSt|}tjr#|rt jdfSt|jdtr.t|jdtj s tdfSt|dtt|jddzz}|r|dfS|d zdfS) zReturns (weight, cannot_save_reason). cannot_save_reason is None for finite weights, or a string explaining why the node cannot be saved for infinite weights. rzview op (recompute_views=True)rz$SymFloat (non-SymInt symbolic value)g?rxr&N)r treat_parameters_as_free_to_savergrrSrrrrrrrINT_INFr}r<r>r)rDrhmem_szr%rs rKget_node_weightz&solve_min_cut..get_node_weightOs  3 3337$  ! >h&6&6t&<&< >8== = di& 5 5 Gdi. == G FFF cST%6!.ban_recomputation_if_allowedzs   D ! ! 5 8  4; (=>>@K)-?? > m u $   5 DI  *TYu-=u~"N"N 5   I X4:R0000@R    trMr_outsinkz;must be available for backward: input required for gradientr0r/z3must be computed in backward: required for gradientz+must recompute: marked by checkpoint policyz primal inputzforward RNG seedrrAznon-tensor outputz cannot save: )r1zdata dependency start_nodesr_ max_ranger}cg}|D]-}tj| ||df.t|dkrtj|\}}}|s |S|jD]l} |rU ||kr1 || ||f}||vrtj||mt|dk|S)z Finds the first unfusible node in the chain of nodes starting from `start_nodes` and returns its position. Tr)heapqheappushrrheappoprrx) r:r; sorted_nodesrmrCrDnode_is_fusiblerrrLrSs rKfind_first_unfusiblez+solve_min_cut..find_first_unfusible s4 9;  O OA N<)*@*@*C*CQ)M N N N N,!##',}\'B'B $At_" 4 --d333 : :++D11 : --d33i?? !..t44" 4..6C ,..|S999,!##rMcdg|],}||-Sr\)rxrrlrrSs rKrz!solve_min_cut.. sK++D11&&t,,rMc>g|]}||Sr\)rxrDs rKrz!solve_min_cut..$ s<I4L4LT4R4RrMz1used above/below fusible %s:(%s) -> %s -> %s:(%s)rxztoo long %s %s %s %srFTrrc dddS)Nmin_cut_failed_fx_graphrrr\r\rMrKrpzsolve_min_cut.. s5 (%%rMcSrHr\) fx_graph_strsrKrpzsolve_min_cut.. s<rMrmin_cut_failed_graphz.txtwz(failed to write: ) c dddS)Nmin_cut_failed_edge_listrrr\r\rMrKrpzsolve_min_cut.. s2$!!rMcSrHr\) edge_list_strsrKrpzsolve_min_cut.. s}rM node_namecndD]1}||r|dt| cS2|S)N)r/r7)endswithr)rRsuffixs rK get_base_namez$solve_min_cut..get_base_name sO-99F ))&119(CKK<88889  rMz depends on z :z - z -> c dddS)Nmin_cut_failed_svgrrr\r\rMrKrpzsolve_min_cut.. s 4$,))rMcSrHr\) svg_contentsrKrpzsolve_min_cut.. s{rMzFX graph dump: zMin-cut graph visualization: z[Production debugging: Use tlparse to extract debug artifacts (min_cut_failed_fx_graph, min_cut_failed_edge_list, min_cut_failed_svg)] aAOT Autograd failed to partition the joint forward-backward graph. The partitioner determines which intermediate values to save from the forward pass vs recompute in the backward pass. This error means a value is required for backward, but cannot be saved AND cannot be recomputed. This is a bug in PyTorch. Please file an issue at https://github.com/pytorch/pytorch/issues Nodes involved in the conflict: z [For PyTorch developers: one of the above constraints is wrong. Either the node should be recomputable, saveable, or not required for backward.] [Debug: min-cut path] z-Failed to compute min-cut on following graph:richdjjSNrMjoin readwriteedgelistgenerate_edgelistnxr5srKrpzsolve_min_cut.. & ","7"I"I("S"STTrMchdjjSr]r^rcsrKrpzsolve_min_cut.. rerMc3,K|]}||fVdSrHr\)rlrmr5s rKrnz solve_min_cut.. s,88Q $888888rMc3(K|] }|v|fV dSrHr\)rlr. non_reachableus rKrnz solve_min_cut.. s1AAa=.@.@q!f.@.@.@.@AArMz node_in[:-3]=z != node_out[:-4]=ci|]\}}|| Sr\r\rs rKrz!solve_min_cut.. sHHHic4cHHHrMc3(K|] }|V dSrHr\rlrD name_to_nodes rKrnz solve_min_cut..! s(22d 222222rMc|SrHr\)rnode_idxs rKrpzsolve_min_cut..! s (1+rMrq)rrErrErFr:rV)rDrErFr)rDrErhrarFr&)r)rDrEr-rrFr:)r:r_r;r}rFr})rRrrFr)Jr$get_default_op_listr;rrCr<rnetworkx ImportErrorrDiGraphrrcrdr2rrrrr4r3rxrrrrNrr=rrrrrhr*rrrurr<rrrr=r?rr> minimum_cutNetworkXUnboundedr:rhandlers owning_modulerrr_get_unique_pathopenwrite Exceptionr_r`rarb_find_infinite_capacity_pathr setdefaultrfvisualize_min_cut_graphrrrget_name_to_noderrs)LrrSrrjoint_module_ops ops_ignoreder"r,r6rD ban_reasonis_non_tensor_nodeweightcannot_save_reasonrrB used_nodeordersfw_usersfirst_unfusible_usevisited start_nodefusible start_orderrCr cut_value partition unbounded_excstructured_tracing_enabled fx_graph_filerfinf_pathnode_constraintsraw_path_nodesrV from_nodeto_noder-base from_baseto_baseconstraint_linesrR constraintscconstraints_str raw_path_strsvg_pathlocal_files_msg tlparse_msg reachablecutsetnbrs cut_nodesnode_innode_outr*r4rrrQrIrLr%rrprrrirdr5rr[rjsL ``` @@@@@@@@@@@@@@@@rK solve_min_cutrsD <<"$$H H%& & #)& & &    '4 4 $54 4 4 * *   9;GGG(    AAAAAAAA&  O           99999999vFFFFFF &$&$&$&$&$&$&$Prz||H(2 L@!VV 7h    9. . .9555!!I&!XX "!!I%!XP "  $       E!D      d   C ( (~ > > > >  & & C ( (/A B B B .-d33  # #D ) ) ;j ; ( (z : : :  " E}DI'EUty SDIe4Del)S)S%S  t   =..//F!%    $)--..@P0QRR 9%)""%8"")8i;** &F&  V6TX#5#579J9J    E! F";'9;;        di%/V1Cf  U U UJ  D    F" E!(      24,;"4 ; ;I%OF !*H6{{Q&:&:8S[[&Q&Q#!)/22;;D!0066;%22488;NNN&Jy$77O <//$O%%229==/ %22488 54T:::1#V'1||%+! V! VJ++J77 '' 33Z@2G$00<> C   **3//+2CCCG ))HH."!..s33!..z:: 10555IVVD!0066V&JsD11V! 44w1G1G1M1Mt0TUUU5g,,""8b-r~h&II 99 VVV%))*<%=%="%) #' "0  6  & ++!&tD,%%  0///     --CVLLMmS)) &Q %%% & & & & & & & & & & & & & & & 6 6 65555MMMMMM 6 ","7"I"I("S"STT  -,,,     099 ^ !68 &ZN ! ! ! ! /7  * 7F%%g...(((=11D$//b99@@HHHH&&(=33D$//b99@@HHHH"]9--w1G1GGG(=33D$//b99@@HHHH!. i 8 8I+mG44G$//<<CC1i11 +- *:*@*@*B*B : :& ; ''(9Y(9(9(9:::$::A$++LQLL9999:#ii(899O!;;~66L%9h>]J=hA$i$r5nx.DiGraph[str, dict[str, Any]]!list[tuple[str, str, str]] | Nonectdg}tdgfg}|r|\}}||D]}||vr|||}|dd}|t jks |tkrW|dd}|||f} || gz} |dkr| cS||| || f|dS)zBFS from source to sink following only infinite-capacity edges. Returns a list of (from_node, to_node, reason) tuples representing the path, or None if no such path exists. rr1rr-unknownr8N) r$rpopleft successorsrrrr*rr) r5rqueuerD edge_pathneighbor edge_datar1r-new_edgenew_paths rKrr& s ($$G>$   G '..I<<<<< = =1  '..I<<<<< = = . .' .9 . ..rMtuple[str | None, str | None]c ddl} ddl}n-#t$r tddYdSwxYw|j|}||d}| D]}|| | d}| t||tdkr|d |d }t%d d }t'|d 5} | |dddn #1swxYwY||fS)zVisualize the min-cut graph to an SVG file. Returns (path_to_svg, svg_content) tuple. Both are None if pydot is unavailable. rNzMInstall pydot to visualize the min-cut graph for debugging: pip install pydotT)exc_info)NNr1rredutf-8min_cut_failed.svgrK)rtpydotrur<rnx_pydotto_pydot to_stringgraph_from_dot_data get_edges get_sourceget_destination set_labelrr= set_color create_svgdecoder{r|r}) r5rdr dot_format dot_graphedgerr[rrs rKrrV s   [    zz %%h//99;;J))*55a8I##%%""$//++,T-A-A-C-CDZP s6{{### U5\\ ! ! NN5 ! ! !&&((//88K 0&99H h     [  s &55FFFc .gtjtjtjtjtjtjtjtjtj tj tj tj tj tjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtjtj tj!tj"tj#tj$tj%tj&tj'tj(tj)tj*tj+tj,tj-tj.tj/tj0tj1tj2tj3tj4tj5tj6tj7tj8tj9tj:tj;tj<tj=tj>tj?tj@tjAtjBtjCtjDtjEtjFtjHtjItjJtjKtjL}tjItjJtjMg}|tjNtjOtjPtjRtjStjTtjUtjVtjWg z }|}|gtjtjXtjYtjLtjZtj[tj@tj[tj\tjRtjVtj]tjNtjStjOtj^tj_tj`tjatjbtjctjdtjetjftjgtjhtjitjTtjjtjktjltjmtjntjotjpz }|tjqtjrgz }||z }|tz }|tjtgz }|dtDz }t|}ttdtftjytjztj{g}tj|tj}tj~tjtjtjtjtjtjtjtjg }||z}t|t||t||S)Nc,g|]}t|Sr\)r)rlms rKrz'get_default_op_list.. s! N N N1!3A!6!6 N N NrM.)rrrratan2rMr<r>pow remainderfmod__and____or____xor__ __lshift__ __rshift__eqnegegtleltrF bitwise_notceilfloorfracnegreluroundsilutruncr<log10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtrsqrtrLsigmoidsoftplus thresholdthreshold_backwardclampwherelerpaddcmulgelu gelu_backwardremean_grad_sum_to_size sum_to_sizerGtotype_asrrsqueeze unsqueezersub_to_copyaliasrslicetrHbroadcast_in_dimexpand as_stridedpermuteselectrprIclone full_likevarstd _unsafe_viewrbroadcast_tensors scalar_tensorones new_zerosrarangetriuvar_meanisinfrfullzerosempty empty_likeargmaxmaximumiota'_low_memory_max_pool_offsets_to_indicesrgatherrz zeros_likerr$rr native_dropout rand_like randn_likemm convolutionconvolution_backwardbmmaddmm#_scaled_dot_product_flash_attention'_scaled_dot_product_efficient_attention_flash_attention_forward_efficient_attention_forwardupsample_bilinear2d _scaled_mmr>)default_recomputable_opsrecomputable_view_opsrBrCrAr@r?s rKrsrs} ssL: L: L: L: L:  L:  L: L: L: L: L: L: L: L: L: L: !L:" #L:$ %L:& 'L:( )L:* +L:, -L:. /L:0 1L:2 3L:4 5L:6 7L:8 9L:: ;L:< =L:> ?L:@ AL:B CL:D EL:F GL:H IL:J KL:L ML:N OL:P QL:R SL:T UL:V WL:X YL:Z [L:\ ]L:^ _L:` aL:b cL:d eL:f gL:h iL:j kL:l mL:n oL:p qL:r sL:t uL:v wL:x yL:z {L:| }L:~ L:@ AL:B CL:D EL:F GL:H IL:J KL:L ML:N OL:P QL:R SL:T UL:V WL:Z"\4>4:F           %H$! $! "$! $! $!  $!  $!  $! $! $! $! $! $! $! $! $! !$!" #$!$ %$!& '$!( )$!* +$!, -$!. /$!0 1$!2 3$!4 5$!6 7$!8 9$!: ;$!< =$!> ?$!@ A$!B C$!D  E$!F 5G$!$LT[ 99( / ! N N N N NN!":;;HS#X./ dndo>J   !   0 4 % )    #Z/K ())8   rMc2i}|jD] }|||j< |SrH)rr)rrprDs rKrr% s-')L ''"& TY rMmemory list[float]runtimes max_memoryall_recomputable_banned_nodes"tuple[float, list[int], list[int]]c0tj}|dkrt|||S|dkrt|||S|dkrt |||S|dkrt |||S|dkrkt dtj ||||}t ||t| t| St|tr||||||\}} d || fStd |) Ngreedyilpdpr)dynamic_memory_budget_dpzdynamic_memory_budget_dp is an experimental solver. It does not guarantee performance improvements. Additionally, it is not guaranteed to be stable.)rrN recorded_knapsack_input_memories recorded_knapsack_input_runtimes)graph_info_provider) knapsack_algomax_mem_budgetr9z,Not aware of memory budget knapsack solver: )ractivation_memory_budget_solverr*r+r(r)r<warningr'inialize_from_graphr,get_knee_point_memory_budgetrrr) rrJrLrMrSrNSOLVERrWsaved_node_idxrecomp_node_idxs rK#_optimize_runtime_with_given_memoryra, sr 3F vx<<< 5FHj999 468Z888 3 3 3-fh KKK - - -  ?   0C#*G-3-5        $7   **))+    F0 1 1T*0& KY8U+ + '^_55R&RRSSSrM no_dispatchrr`ct|j}d fd fd|D}fd|D}|||S) Ndtorch.SymInt | intrFr}c&t|S)Nr_)r )rer`s rKrealize_symbolz8_remove_symbols_without_guarding..realize_symbolc sX....rMc&g|] }|Sr\r\rlrrhs rKrz4_remove_symbols_without_guarding..f s# . . .1^^A   . . .rMc&g|] }|Sr\r\rjs rKrz4_remove_symbols_without_guarding..g s# 4 4 4AnnQ 4 4 4rM)stride)rerfrFr})rshaperlnew_empty_strided)rr`rmrlrhs ` @rK _remove_symbols_without_guardingro` s MME////// / . . . . . .E 4 4 4 4 4 4 4F  uV  4 44rMc tj}dd}|dkrdS|dkrnt5dd lm}t j|jjf\ |  fd }|cdddS#1swxYwYdS|d krdd l m }t j|jjf\ |d 5}j i dddn #1swxYwY| }t|dSt|t r |St#d|)Nrr rFcvt|tjrAt|jdtjrt |jddSt|tjrAt|jdtjrt|jddSt|tjr't|jdtj rdSt|tjr't|jdtj rdS|S)Nrr^r_?T) rrrrrrNrorr rrras rKmaterialize_argz)estimate_runtime..materialize_argn s a ! ! j &M&M 3AF5MDQQQ Q 27 # #  16%=%,(O(O QVE]T::: : 27 # #  16%=%.(Q(Q 3 27 # #  16%=%-(P(P 4HrMtestingr&profiler) benchmarkercjiSrH)r)rrrDsrKrpz"estimate_runtime.. s;4;3O3O3OrMflops)FlopCounterModeF)displayz Not aware of runtime estimator: )rr rFr )r*activation_memory_budget_runtime_estimatorrc$torch._inductor.runtime.benchmarkingrvrtree_maprr benchmark_gputorch.utils.flop_counterryrget_total_flopsr<rrr) rD RUNTIME_MODErsrvmsrymode counted_flopsrrs ` @@rKestimate_runtimerk sDL    y  q  " " ]]   H H H H H H!??TY t_d?}|0|tb2d@|!|AdS)HNr&rzJThe valid ranges for memory budget are 0 <= m <= 1. The provided value is )rrrrrF)rrrrr*r_rFr=cLttt|dz SNeA)remaprg)r*s rKestimate_activations_sizez:choose_saved_values_set..estimate_activations_size s3x..//#55rMszc|dz z z Srr\)r max_act_size min_act_sizes rKget_normalized_sizez4choose_saved_values_set..get_normalized_size sS\L899rM activationsc,|z z z SrHr\)rrrrs rK get_mem_ratioz.choose_saved_values_set..get_mem_ratio s())+66E < '  rM)rrr)r)get_node_storagec3.K|]}|VdSrHr\)rlrDrs rKrnz*choose_saved_values_set.. s/TT4 0 0 6 6TTTTTTrMr4rac"fd|DS)Nc|g|]8}|jtdkr|vst|6|9S)r)rr}r)rlr6rinput_storagess rKrzRchoose_saved_values_set..get_recomputable_banned_nodes.. s`    S))$$Q''~==033>  >==rMr\)r4rrs rKget_recomputable_banned_nodesz>choose_saved_values_set..get_recomputable_banned_nodes s3      !    rMcdg|]-}|jddtjk+|.S)rF)rrr%rHrs rKrz+choose_saved_values_set.. sA 6::k5 ) )-=-G G G G G GrMcg|]}|v| Sr\r\)rlr6must_save_nodess rKrz+choose_saved_values_set.. s*!!!0H0H0H0H0HrMTrjc@g|]}t|Sr\rg)rlr6rs rKrz+choose_saved_values_set.. s8-.HQKK((rMc,g|]}t|Sr\)rr s rKrz+choose_saved_values_set.. s.#'rMrbrrSr^rrtuple[list[fx.Node], float]c  5t| t|d| \}}}dddn #1swxYwYt}|D].} | |#t$rY+wxYw| st dt|| |\}} tr"t| |||d D | ||fS)Nrz:dont_ban must be a subset of all_recomputable_banned_nodesc,g|]}t|Sr\rrs rKrzNchoose_saved_values_set..get_saved_values_knapsack..> s+'''$%HQKK'''rM) rrNsaved_node_idxsrecomputable_node_idxsexpected_runtimememories_banned_nodes normalized_memories_banned_nodesruntimes_banned_nodesmin_cut_saved_values) rar<r$r BaseExceptionissubsetrrr;r )rrSrrrrrr5r*rCaggressive_optionsrNrrcrs rKget_saved_values_knapsackz:choose_saved_values_set..get_saved_values_knapsack s[]]   4%%M1%%-    &               )3 )  C  :3?@@@@       !>??  L (        a !  4'.K /'=!1'')F'''2G&;%1    ---s!'?AAA99 BBrtuple[float, float, float]cb|\}}|t|z |fS)N)rSr)re)rr*rrrrrSrs rKestimate_for_budgetz4choose_saved_values_set..estimate_for_budgetI sU-F-FYK... *L*)**-== l++ rMr9rrgMbP?r(cg|] }|d S)r(r\rlitems rKrz+choose_saved_values_set..f 000DG000rMcg|] }|d Sr&r\rs rKrz+choose_saved_values_set..g rrM) )figsizeo)markerz.4fz offset points)rrcenter) textcoordsxytexthaz Memory Budgetz Runtime of Recomputed Componentsz:Pareto Frontier of Memory Budget vs. Recomputation Runtime)exist_okr_rank_memory_budget_paretorCrz%Generated Pareto frontier curve at %s)rrSr)r*r_rFr=)rr=rFr=)rr_rFr=)r4rarFr_)rr=rSr^rrrFr)rr=rFr)3rrrban_recompute_used_far_apart!ban_recompute_long_fusible_chains#ban_recompute_materialized_backwardban_recompute_not_in_allowlistban_recompute_reductionsaggressive_recomputationrr`rtorch._inductor.fx_utilsrr$rsrgrtorch.utils._mode_utilsrcvisualize_memory_budget_paretorrsortmatplotlib.pyplotpyplotfigureplotrannotatexlabelylabeltitlegridgcfshowrgetcwdmemory_budget_pareto_dirmakedirsrrris_initializedget_rankrr_r1savefigr<r[))rrSrrruntime_optimized_saved_valuesrCmore_aggressive_optionsmore_aggressive_saved_values%aggressive_recomputation_saved_valuesr4rrecomputable_banned_nodesroptionsbisectslhsrhsmidpltx_valuesy_valuesr6txtfigfig_dir rank_suffixfig_namerrNrrrrrrrrrrrcrs)`` @@@@@@@@@@@@@@rKchoose_saved_values_setr s qMA-- hYf h h   $$A#)#K%+%O & E8 O& ! "'',).$)    (5))%"A --6666-,Y-=>>L,,-KLLL|##--:::::::        &##(%*  '4Y 7''# !}122]BB++  %;HY 2;;7)<}:;;mKK44999999TTTT9CSTTTTTN       !> =l K K*O !!!!,!!!%+!x%%%! ())Q../112O+H4333331.1.1.1.1.1.1.1.1.1.f,AG          '&s++-@-@-E-EF 1:abb>WQZ^ + + GAJ/0G /";;==Sq6CF?T))NN3'''NN3'''))3q6CF?a*?@@qrr7c!""g%%NNC:...qrr7c!""g%%NNC:... /  ''''''0000000000 7 ### 8C000 ))  FAs LL hqk"*      ?### 5666 NOOO ggii  )++  * 65G K$ / / / /   ) ) + + B0A0P0P0R0R BA5#4#=#=#?#?AAK7<< TKTT:L:N:NTTT   H ;XFFF % $#yk     rMlist[torch.fx.Node]cXddlmdd}dfd }tjrwtjrXtjd kr5||r)||rt55d |Dg}d ttjD}tj ||dt|g}i}t|D]t\}} fd | D} d} | D]B} t| } | | z } |tj kr | || j<C| |d <|| utj|tjj}tj|tjjjjt-tj|}d|d|t3ddfdfd||D}dddn #1swxYwYdddn #1swxYwY|S)Nr)unset_fake_temporarilyrr:rFr:c|jD]7}t|jtjjr|jjdvrdS8dS)N>c10d_functionalrTF)rrrrrrr)rrDs rKhas_collectivesz3_sync_decision_cross_ranks..has_collectives sP%  D UZ2 +'+RRRtturMc0dd|jD}tj|d}dt tj Dt55tj |dddn #1swxYwYdddn #1swxYwYtfdDS)N/c3$K|] }|jV dSrHrrlrs rKrnzE_sync_decision_cross_ranks..has_same_nodes.. s$>>qAF>>>>>>rMrcg|]}dSrHr\rlrCs rKrzF_sync_decision_cross_ranks..has_same_nodes.. sNNNqdNNNrMc30K|]}d|kVdSrr\)rlr all_inputss rKrnzE_sync_decision_cross_ranks..has_same_nodes.. s,::!:a=A%::::::rM) r_rhashlibsha256encode hexdigestr4rrget_world_sizercall_gather_objectr)rnode_strr`rrs @rKhas_same_nodesz2_sync_decision_cross_ranks..has_same_nodes s 88>>K,=>>>>> 8 899CCEENNE%*;*J*J*L*L$M$MNNN ]] D D2244 D D   / / F C C C D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D::::z::::::s6# C2.!C C2C C2"C #C22C69C6r&cg|] }|j Sr\rrs rKrz._sync_decision_cross_ranks.. s5551555rMcg|]}gSr\r\rs rKrz._sync_decision_cross_ranks.. s%::::::rMc g|] }| Sr\r\)rlop_namerps rKrz._sync_decision_cross_ranks.. sTTT|G4TTTrMz total size)rr!zpicked_rank_idx=z, saved_nodes of current rank=rc dddS)N)aot_joint_graph_sync_decision_cross_ranksrrr\r\rMrKrpz,_sync_decision_cross_ranks.. sG (%%rMcSrHr\)sync_decision_cross_ranks_strsrKrpz,_sync_decision_cross_ranks.. s#@rMrc g|] }| Sr\r\)rlrmrps rKrz._sync_decision_cross_ranks.. s*$% QrM)rr:rFr:)torch._subclasses.fake_tensorrrrrrrrcr4rrrrgrrrrcdistributed_c10d_get_object_coll_device all_reduceReduceOpMAXr}argminrr)rr*rrobjectssaved_ops_names_all_ranks saved_sizessaved_ops_with_sizesr5saved_ops_names saved_nodes saved_sizerD size_of_nodesaved_sizes_tensorpicked_rank_idxrprrs @@@rKrJrJ sEDDDDD ; ; ; ; ; ; &&((1   , , . .1   , , . . 2 2 OK ( ( 3 N; ' ' 3]]* * 2244* * 55 5556G::!%"3"B"B"D"DEE::: %   / /0I7ST: V V V+K88L%'K35 (12K(L(L / /$_TTTTOTTT  'GGD#+D>>L,.Je/88:::::F,TY75?$\2"":....!&(9QQSS"""    ( ("u'8'I'R'V )   "%,/A"B"B"G"G"I"IJJO-E-E-EoC-E-E ) A@@@     )B?)SLQ* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * X s7 J&GJ< JJ JJ JJ#&J#modulercrd}|rdnd}tt|jd}|jdtjjjD]}t||j dj }t|tj rg}|jdD]}||jvr|j|5|j|d|} |d z }|jd | jd <| }|| d d d n #1swxYwY|r|j|5|jdtjjjg|j |Ri} || d d d d n #1swxYwY|jd} | r"| \} } g| d|DR}|| f| jd<|j||S)u Graph-safe RNG lets torch.compile use CUDA Graphs for graphs with RNG ops. For graphs without HOPs, the partitioner adds placeholder nodes fwd_rng_state_* and bw_rng_state_* to the forward and backward graphs. At runtime, the AOTDispatcher retrieves these RNG states and passes them to the compiled graphs. This works well for no-HOP graphs. With HOPs, the partitioner runs recursively: it first partitions the HOP (producing forward/backward HOP subgraphs) and then stitches them back into the outer joint graph. For HOPs that contain RNG ops, the outer joint graph now includes HOP subgraph modules with extra RNG placeholders. We must thread these placeholders through the outer module partitioned forward and backward graphs—this function does exactly that. It collects the RNG placeholder nodes from the HOPs and creates corresponding placeholders in the outer forward and backward graphs. There is a catch: for a short period, the joint graph is in a “bad” state. The HOP subgraphs expect additional inputs (because of the new placeholders), but the outer graph call sites don't yet provide them. We can't fix this in the joint graph because the joint graph's input signature is fixed (primals, tangents). As a compromise, we keep the joint graph in somewhat of a bad state for some time and, once the outer forward and backward graphs are partitioned, insert the corresponding RNG placeholders and wire up the calls. rrrrr!r)rrrCr&rNT)propagate_metaeager_input_valsc(g|]}|jdS)r)r)rlinps rKrz2thread_graphsafe_rng_from_hops..- sDDDc#(5/DDDrM)r#rrr"rrrinvoke_subgraphrrrrrrrrErrrrrrr)r"rr rng_string last_inputhop_nodernew_rng_inputsplaceholder_nodernew_hop_node_with_fixed_args eager_vals eager_args eager_kwargsnew_eager_argss rKrNrN s<I$/D_Jhv|66-6HHIIJJJL++ 59#9#I,,2,268=#3#:;; h / /( 2,.N$,N$=$=$=$O$O 9 9 !1!666 55jAA99$*L$<$<)77I77%% "Q 0@0Ee0L u-%. &--i888999999999999999 2\11(;;  39<3K3K' .>9(-9.99 440 224T3               &]../ABB  /9,J &#&DD^DDD&&N '$M056HI ''111 Ms&.AE  E E 3AGG G list[int]c t|jt|jjD]n}|jdkrd|jvr|n$t|r||vr|j ottt|jj}ttt|jj}||z}t||\}}} } } d|Dt!|j||| d} tfd| jDtfd|jjD} tfdt#|D}d }i}|jjD]}|vr |||<|d z }t%|| | ||S) Nrtangentsrc34K|]}||jdk|VdS)Nrr!)rlrs rKrnz!classify_nodes..U s;!-ADH4D4D4D4D4D4DrMrc3HK|]}|jdk |jVdSrrros rKrnz!classify_nodes..[ sC88  7h   TY    88rMc3,K|]}|v|v |VdSrHr\)rlrDrcrus rKrnz!classify_nodes..` sF66  ( ( (T9J-J-J -J-J-J-J66rMc3*K|] \}}|v |VdSrHr\)rlr6prs rKrnz!classify_nodes..e s;--a!7T2T2T2T2T2T2T--rMrr&)rrr$rrrrrrrrrr4r3r)rrrr^)rrrrDrrr`r%r&r'r(rdforward_only_graphrerhfw_cntrgrprcrus ` @@@rKr@r@8 s $L$677L-7\\"(11 7m # # dk(A(A  ! !$ ' ' ' ' !$ ' ' (  ! !$ ' ' ' $ $ $  $ $TZ 0 0 0 L,>,DEEFFM!&)P>V"W"WXX 3 3F OOOCK/1B)--//<FK1BI.88888&,888.. ,666666 &,666,,O #----- ..---##FH"( $ $ $#HTN aKF #  rMr)rcompilerc |j||j}tjrt |}||_|j}t |}t|} |rt|d}tj st|t|t||g}t|||} t| jdkrt!||||| jSt%|jjD]} | jdkrt+d| _"| | sd| _?t+d| _| jD]$} t3| j| jdz| _%tj} |jD]?} t7| jd dt<r| jd } n@t?|| | }tj rtA||}tCtEtF|}tCtEd |}tI||||| j \}}|r$| r"tK|||t|\}}tM|}tj'rdd l(m'}||||| jtS|}tS|}tU|d}tU|d}tVrtYd|D}t[d|Ddz }t\/d|t\/d|tad|jjD}tad|jjD}||z}tct*}|jjD]G} | j2|vr. s[^^);rMr3r5r7TcJg|] }t|t|f!Sr\)rgrrs rKrz7min_cut_rematerialization_partition..s)KKK SVV4KKKrMc34K|]}t|VdSrHrrs rKrnz6min_cut_rematerialization_partition..s('J'J 'J'J'J'J'J'JrMz'Theoretical Activations Stored: %.2f GBz,Theoretical Per Activation Storage Sizes: %sc3:K|]}|jdk |jVdSrNrr s rKrnz6min_cut_rematerialization_partition.. 9% % 47o;U;UDI;U;U;U;U% % rMc3:K|]}|jdk |jVdSrCrr s rKrnz6min_cut_rematerialization_partition..rDrMrz# remat/fw/bw: %d/%d/%drjzCount of Ops Rematerialized: %s):rrrrcser7rrr;r<r=r>r?r@rrcrWrhrrrr}rrxrr>activation_memory_budgetrrrr=rrJrrrrrKrLr6rMr9rNr;rsrer<rr$rrrrrrrfrrm)rrr=rrr cse_graphrrQrRrSrDrrr*rrUrVr6 sorted_sizestotal_activations_size_gbfw_module_nodesbw_module_nodes remat_nodescountsrematerialized_opss rKr:r:y sxD**,,,  Dz' && & $K!5l!C!C%=l%K%K"!X-lQVWWW  :-|,,,\***|,,,$,(*%3_I 9 &''1,,  +*G(1(M     +122RR 7h   #CD  ))$// R !D   #CD   R R$'(94;Lq;P$Q$Q!! R3M! dimmOT::E B B  Io6M E +#L (M1+|LL 6+|<<==O;;\JJKKL4''$-$I Iy" ) #8iC4H4H$$ Iy4I>>I*         %$     1    y))Iy))I.yeLLLI.ydKKKIHKKlKKKLL %('J'J\'J'J'J$J$JS$P! : >DyK''GDKAR,S,S's4;677888A=888  %         $ LLNN 3A 6 6    24FGGG i rMfx_graphTtracedfnamefigname clear_metaprogstr | list[str] | Noneparse_stack_tracedot_graph_shapec|rDtj|j}tj||}|jjD] }i|_ tj |\} } | sdtj z} t d| | tj||||} | } t#| d| dz} | | }| | |dS| ||dS)NrozWriting FX graph to file: %s%s)rWrXwrite_)rU)rdeepcopyrrrrrrrsplitextrtorch_compile_graph_formatr<rr# FxGraphDrawerget_main_dot_graphrlstrip)rQrRrSrTrUrWrXrrDrextgr write_methods rK draw_graphrd%s'M&,//  22L&  DDII  ''ID# 6F55HH -tS999"+'    A A1hC899LNSNNE | U U&&&&&&rMrV)rrrFr:)rDrErFr})rrErFr)rDrErFr)rDrErrrFr)NF)rrr`r_rr_rrrrrr:rFr)rrrr}rFr)r*r_rrrFr+)r0r1rFr})r8r9r) rr:rDr;r<r=r>r=r?r}rFr;)rr:rDr;rVr;rXrYrKr=rZr=r?r}rFr;)rcrdrFr=)rFrj)rDr;rFr:)rFrY)rqrYrFr)rr:rFr+)rrrrrrrFr+rH) r*r_rrrrrhrrFr+) rrr*r_rr_rr}rhrrFr) rrrr rr}rrrhrrFr)rfr}rqrYrFr})rrrFr+)rFrp)rr{r|rfrFr})rrrFr)rUrrVrrr;rr;rrrr}rr;rr;rFr) rrrUrrVrrr}rFr)rrrFr+)rrr r:rFr) rrrSr^rrrrrFr)r5rrFr)rrrrrFr)r5rrFr)rFr>)rrrFr)rrrJrKrLrKrMr=rSr^rNr_rFrO)rrdr`r}rFrd)rDrErFr=r)rrrSr^rr=rFr_)rr:r*rrFr)r"rrr:rFr)rrrr3rr}rFr^)r) rrrr r=rrr}rrrFr)rPTNFN)rQrrRrrSrrTr:rUrVrWr:rXrrFr+) __future__rrrrr=rloggingrrros.pathrr8rlrrcollections.abcr dataclassesrrtypingr r rtorch._inductor.inductor_primstorch.distributedtorch.fxrtorch.utils._pytreeutils_pytreertorch._dynamo.utilsr r ;torch._functorch._activation_checkpointing.ac_logging_utilsr $torch._functorch._aot_autograd.utilsrtorch._inductorrr!torch._inductor.custom_graph_passrr"torch._library.fake_class_registryrtorch._library.utilsrtorch._loggingrrtorch._logging._internalrrr%torch.fx.experimental._backward_stater"torch.fx.experimental.proxy_tensorrrtorch.fx.experimental.sym_noderr%torch.fx.experimental.symbolic_shapesrrrr r!r"torch.fx.passesr#torch.utils._ordered_setr$torch.utils.checkpointr%r-_activation_checkpointing.graph_info_providerr'"_activation_checkpointing.knapsackr(r)r*r+,_activation_checkpointing.knapsack_evaluatorr,_aot_autograd.descriptorsr-r.r/_aot_autograd.functional_utilsr0_aot_autograd.logging_utilsr1_aot_autograd.utilsr2r3r4r5r6 compile_utilsr7r8r9rtrdsympydebug_partitionerr;r[ getLoggerrWr<rrrHr>r^rrrrrrrrrrrrrrr r rrrrr)r/r7rWrbrirur|r~rrrrrrrWr}r*rZrgrocacherzrrLrrKr=r>r?rr;rrr{rrsrrarrcrorrrJrNr@r:rdr\rMrKrs """""""     ********$$$$$$********%%%%%%%% %%%%$$$$$$$$$<<<<<<<<A@@@@@555555@?????++++++77777777......AAAAAA??????HHHHHHHHLLLLLLLL)(((((//////333333LLLLLL LKKKKK A@@@@@;;;;;;IHHHHHHHHHLLL%66666'g'111111 y~  >>>>>>> >2          B       IIIIo 2 - - - -H $) lllll^XXXXCCCCBBBBJJJJKKKK    5555JJJJ$ KKKKK\C!C!C!C!L9999"GGGG45555    >G>G>G>GBMGMGMGMG`WWWW|?C ,W,W,W,W,Wj?C `"`"`"`"`"`"P7;>B V V V V V V r #c(("""":RRRR "PPPPKKKK\[*[*[*[*|a a a a H @ @ @ @''''<.KKKK5555x,0 s &s &s &s &s &lB / / / /$!$!$!$!NeeeeP-T-T-T-T`0/////5555*N*N*N*N`z z z z z zNNNNbOOOOd>>>>Hi 7; i i i i i i ^#'#"&'''''''rM