Wj@UddlmZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z ddlmZmZddlZddlZddlmZddlmZddlmcmZddlmZddlmZdd lmZd d l m!Z!m"Z"m#Z#d d l$m%Z%d d l&m'Z'm(Z(m)Z)e rddl*m+Z+m,Z,m-Z-ddl.m/Z/ddl0m1Z1edZ2edZ3ej4e5Z6dGdZ7edHdZ8e#dIdZ9 dJdKd$Z:dLd&Z;e#dMd'Z<Gd(d)ej=Z>e#dNd+Z?e#dOd,Z@dPd/ZAejBjCZCeCjDeCjEeCjFeCjGeCjHeCjIeCjJeCjKeCjLeCjMeCjNeCjOeCjPeCjQjReCjQjSeCjTeCjUeCjVeCjWeCjXeCjYeCjZhZ[ee[Z[e#dMd0Z\dQd4Z]dRd6Z^da_d7e`d8<dSd;ZadTdCZb dUdVdFZcdS)W) annotationsN)contextmanager)partial)Any TYPE_CHECKING) ParamSpecTypeVar)SymInt)get_decompositions) bind_symbols) aot_function aot_modulemake_boxed_compiler)strip_overloads)default_partition draw_graph#min_cut_rematerialization_partition)Callable GeneratorSequence)Node) IntLikeType_P_Rfx_gfx.GraphModulereturnc|jdtjjjD]}tjjj|_||S)N call_functionoptarget) graph find_nodestorchopsaten_to_copytor# recompile)rnodes _/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/torch/_functorch/compilers.py _canonicalizer./s[ %% 59>#:&((in' NN KGenerator[None, None, None]c#Ktjd} dVtj|dS#tj|wxYw)NF)r&_C_jit_set_autocast_mode)old_jit_autocast_flags r-_disable_jit_autocastr58sh"H;;EBB?  ''(=>>>>>''(=>>>>s A!A)inps Sequence[Any]torch.jit.ScriptModulect5t||jdtjjjD]V}t|j dkrzts_compile..ns2MM1:a!2!=>>MMMMMMr/N)r5rr$r%r&r'r(r)lenargskwargsr*r#nodesitemsr=devicetypelintr+jitscriptr2_jit_pass_remove_mutationfreezeevaloptimize_for_inferenceany)rr6r, new_kwargskvfs r- ts_compilerVCs  J))uy~'>*   0 0D49~~""s4;'7'71'<'<DKAWAW#in/ J$ % %DJ ))++ " "1a..A ! 1 $DKK   I  T " " **17333 I  QVVXX & & I , ,Q / /MMMMMMM  AtHH;< HsGG--G14G1T_rnamestr clear_metaboolcRt|jt||||S)N)rZ)printcoder)rrWrXrZs r-_draw_graph_compiler_ss. $)tTj1111 Kr/5Callable[[fx.GraphModule, list[Any]], fx.GraphModule]cHttt|S)NrX)rrr_rbs r-draw_graph_compilerc{s  w':FFF G GGr/c|S)z Returns the :attr:`fx_g` Fx graph module as it is. This is a no-op compiler and can be used to check accuracy. .. warning:: This API is experimental and likely to change. rrWs r-noprgs  Kr/c4eZdZddddfd Zdfd ZxZS)DebugInterpreterNT initial_envenable_io_processingrDrrkdict[Node, Any] | Nonerlr[rcft|jg|R|_tj|||dS)Nrj)r modulesymbol_mappingsuperrun)selfrkrlrD __class__s r-rrzDebugInterpreter.runsT + K     uww{ {AU    r/nrcb dfd d fd d fd d fd }t|}d|jvrtj|jd\}}tj|\}}t |t |kr.t t |dt |ttt |||D]2\ }} t| tj s!||| fd3|S)Nnirrintct|ts|Stj|jjj}|jstd|t|S)Nzexpected r to be a number, got ) r=r sympyexpandr,exprxreplacerp is_numberAssertionErrorrx)rwrrss r- subst_symintz/DebugInterpreter.run_node..subst_symintskb&))   RW\2243FGGHHA; L$%Jq%J%JKKKq66Mr/nistuple[IntLikeType, ...]tuple[int, ...]c:tfd|DS)Nc3.K|]}|VdSr<re)r@rwrs r-rBzHDebugInterpreter.run_node..subst_symint_tuple..s-88bb))888888r/)tuple)rrs r-subst_symint_tuplez5DebugInterpreter.run_node..subst_symint_tuples&8888C88888 8r/a torch.Tensorbr[c,|dkrqt|jD]\}||||kr%||dkrdS]dS)Nrr FT)numelrangendimstridesize)rridxrs r-check_significant_stridesz.check_significant_stridess|AGGII&&** ==%%C$ QXXc]]33qxx}}DD(L5599$uu4r/nvrvdescCallable[[], str]Nonec t|stdt||j|jkr)t|d|jd|j||krat|d|d|d|||}|sat|d|d|d|dS)Nz"expected desc to be callable, got z:  != z aka )callablerrIr:rr)rrr same_stridesrrs r-checkz(DebugInterpreter.run_node..checks|D>> X$%V$t**%V%VWWWx28##$%J%J"(%J%J%J%JKKK!!"'')),, 99$tvv____1C1CBGGII1N1N__TVT[T[T]T]__54R<z+DebugInterpreter.run_node..s&O&O&O$:M&O&Or/)rwrrrx)rrrr)rrrrrr[)rrrrrrrr) rqrun_nodemetapytree tree_flattenrCrziprr=r&Tensor)rsrurrn_vals_n_specr_vals_r_specrrrrrrrts` @@@@r-rzDebugInterpreter.run_nodes       9 9 9 9 9 9              GG  Q   AF??$1!&-@@OFG$1!44OFG6{{c&kk))$F %F%FV%F%FGGG s6{{!3!3VVDD Q Q 2r!"el33b"OOOOOPPPPr/)rDrrkrmrlr[rr)rurrr)__name__ __module__ __qualname__rrr __classcell__)rts@r-rirism/3%)         5555555555r/riDCallable[[DebugInterpreter, Any, dict[Node, Any] | None, bool], Any]c*t|jS)z Returns a (slow) interpreter over the FX graph module that also checks various debugging properties (e.g., that tracing strides matched real strides.) )rirrrfs r- debug_noprs D ! ! %%r/ct|tj|}tj|}|Sr<)rr&rKrLrNrO)rrWrUs r-simple_ts_compilersGD A ""A Hr/rUCallable[..., Any]c,t|tSr<)rr)rUs r-nnc_jitrs , - --r/c.t|j|Sr<)r]r^rfs r- print_compiler s $) Kr/fnCallable[_P, _R] | nn.ModulerEc ttttd}||t |t jjr t|fi|St|fi|S)a~ Wrapper function over :func:`aot_function` and :func:`aot_module` to perform memory efficient fusion. It uses the :func:`min_cut_rematerialization_partition` partitioner to perform efficient recomputation. It uses NVFuser to compile the generated forward and backward graphs. .. warning:: This API is experimental and likely to change. Args: fn (Union[Callable, nn.Module]): A Python function or a ``nn.Module`` that takes one or more arguments. Must return one or more Tensors. **kwargs: Any other overrides you want to make to the settings Returns: Returns a ``Callable`` or ``nn.Module`` that retains the eager behavior of the original :attr:`fn`, but whose forward and backward graphs have gone through recomputation optimizations, and the graphs have been compiled with nvfuser.  fw_compiler bw_compiler partition_fndecompositions) rVrdefault_decompositionsupdater=r&nnModulerr)rrEconfigs r-memory_efficient_fusionrsr6"!;0 F  MM&"eho&&*"'''''B))&)))r/Sequence[torch.Tensor]c|dtdd|Ddddlm}||t ||S)NfooaQ ############################################################## # To minimize FX graph, copy and paste the below and run it # ############################################################## import torch import torch.fx as fx from functorch.compile import minifier, check_nvfuser_subprocess, check_nvfuser_correctness_subprocess inps = c*g|]}|j|jfSre)shaper:)r@rs r- z!debug_compile..Gs!***!'17 ***r/a? inps = [torch.ones(shape, dtype=dtype, device='cuda') for (shape, dtype) in inps] from foo import FxModule mod = FxModule().cuda() with torch.jit.fuser("fuser2"): # check_nvfuser_subprocess can be replaced with check_nvfuser_correctness_subprocess minifier(fx.symbolic_trace(mod), inps, check_nvfuser_subprocess) r)FxModule) to_folderr]rrcudarV)rr6rs r- debug_compiler9s NN5   +*T***   (HHJJOOt dD ! !!r/rx graph_indexinput_data_pathlist[torch.Tensor]c 6g}t|d5}tj|}g}|D]}t|dkr|}|t j}n|\}}}} } | t jt jt jt j t jt j t thvrt j dd|| | }nt j || | }|| dddn #1swxYwY|S)zZ Return a random input for the given inputs meta generated from _save_fx_default. rbr r)r:rHN)openpickleloadrCrandomr&rxint32int64r[uint8floatrandintrandappend) rinputsrU inputs_metarrIinput_r_strider:rHs r- get_inputsr[sR"$F ot $ $"k!nn  " "D4yyA~~fmoo..6:3eWeVIKKJIK   #]1aeFSSSFF"ZU6JJJF MM& ! ! ! !' """""""""""""""". MsC.DDD current_name folder_namedump_example_inputgmtorch.fx.GraphModuleexample_inputs nn.Modulec ddlm}d fd d fd d fd }d fd }d fd }||||||tS)aO The forward, backward, and joint computation graph will be stored in {folder_name}/{current_name}/{current_name}_forward_{graph_index}, {folder_name}/{current_name}/{current_name}_backward_{graph_index}, and {folder_name}/{current_name}/{current_name}_joint_{graph_index} respectively. The input shape of the graphs will be stored in the .input files. These files can be loaded with pickle, and is a list of format (type, shape, stride, dtype, device). In the case of type = int or float, it is just (type,). For joint graph input, it is a nested list [[],[]] where the two inner lists have the same format. If dump_example_input is True, example_inputs will be stored in .pt file. Since each function might produce multiple graphs, the graph_index is used to distinguish difference graphs r)aot_module_simplifiedrDrr list[Any]cg}t|dkrEt|dtr*||dz }||dz }|S|D]}t|tust|t ur$|t|fR|t||j||j |j f|S)Nrr ) rCr=rrIrxrrrrr:rH)rD input_metaargget_input_metas r-rz(_save_fx_default..get_input_metas t99q==ZQ77= ..a11 1J ..a11 1J   CCyyC499#5#5!!499,////!!#YY 3::<<CJOr/ gm_to_saver type_namerYrc6t|jjdkr/ttjd|t dStj|}|j tj j |  |}tjdd|ddd|dt t#ddd|dt dd|dt dd5}t%j||dddn #1swxYwYr9tj|ddd|dt dd|dt d dSdS) Nrz!No nodes in graph {%s}_{%s}_{%s}./T)exist_okrWz.inputwbz.pt)rCr$rFlogloggingWARNINGrcopydeepcopy set_codegenr&fxCodeGenr+osmakedirsrrrdumpsave) rrDrrrrUrrrrs r-graph_saver_helperz,_save_fx_default..graph_saver_helpers z% & &! + + GG3     F ]: & & UX^3355666 #^D))  {33\33dCCCC  T T\ T TL T T9 T T{ T T    D D\ D DL D D9 D D{ D DUa D Ddm D Dp{ D D D    ' K A & & &  ' ' ' ' ' ' ' ' ' ' ' ' ' ' '   JEEEE EEyEE;EEYeEEhqEEtEEE       s1EEErrrc"||d|S)Nforwardrerrrs r-graph_saver_forwardz-_save_fx_default..graph_saver_forwards 2~y999 r/c6||dtdz a|S)Nbackwardr )rr s r-graph_saver_backwardz._save_fx_default..graph_saver_backwards) 2~z:::q  r/ joint_args%tuple[fx.GraphModule, fx.GraphModule]c>||dt||S)Njoint)r)rrrs r-graph_saver_jointz+_save_fx_default..graph_saver_joints* 2z7333 Z000r/r)rDrrr)rrrDrrrYrr)rrrrrr)rrrrrr)functorch.compilerr) rrrrrrr rrrrs ``` @@r-_save_fx_defaultrzs,877777      !!!!!!!!!F 111111 !  '(&-    r/F Callable[[bool, nn.Module], Any]c4datt|||S)as Dump the forward, backward, and joint computation graph. Example Usage: save_fx_func = graph_dumper_aot(current_name, folder_name, dump_example_input = False) optimize_ctx = torchdynamo.optimize( save_fx_func ) with torch.enable_grad(): with optimize_ctx: result = forward_and_backward_pass(model, example_inputs) r)rrr)rrrs r-graph_dumper_aotrsK #\;@R S SSr/)rrrr)rr0)rrr6r7rr8)T) rrrWrrXrYrZr[rr)rXrYrr`)rrrWrrr)rrrWrrr)rrrWrrr8)rUrrr)rrrErrr)rrr6rrr8)rrYrr) rrYrrYrr[rrrrrr)F)rrYrrYrr[rr)d __future__rrrrrr contextlibr functoolsrtypingrrtyping_extensionsrr rzr&torch.fxrtorch.nnrtorch.utils._pytreeutils_pytreerr torch._decompr %torch.fx.experimental.symbolic_shapesr aot_autogradrrr compile_utilsr partitionersrrrcollections.abcrrr torch.fx.noder torch.typesrrr getLoggerrrr.r5rVr_rcrg Interpreterrirrrr'r(detach gelu_backwardleaky_relu_backwardsigmoid_backwardthreshold_backwardhardtanh_backwardhardsigmoid_backwardhardswish_backward tanh_backward silu_backward elu_backwardcudnn_batch_normcudnn_batch_norm_backward masked_fillScalarrelu leaky_reluhardtanh hardswish hardsigmoid conj_physical is_same_sizerrrrr__annotations__rrrrer/r-rDs"""""""  %%%%%%%%%%%%%%00000000 $$$$$$$$$,,,,,,>>>>>>GGGGGGGGGG******(==========""""""''''''Yt__ WT]]g!! ????, , , , `AEHHHH     EEEEEr~EEEP&&&&    .... y~K"HOMN-4,+,BCC $*$*$*$*N""""> >ffffVEJTTTTTTTr/