Wj 3 UddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z m Z ddl mZddlmZddlmZmZddlmZmZddlmZmZdd lmZmZejeZej e!d <ed Gd dZ"ed GddZ#ed GddZ$GddeZ%de&e'de e"fdZ(de&e'de e"fdZ)d0de'de'de*e&e'e e"ffdZ+de'fdZ,dZ-d Z.d!Z/Gd"d#Z0Gd$d%eZ1Gd&d'eZ2Gd(d)Z3d*e4d+e'dzd,e5d-e&e%d.e6e'ddf d/Z7dS)1N)ABCabstractmethod)CallableIterable)ThreadPoolExecutor) dataclass)BaseHTTPRequestHandlerThreadingHTTPServer)parse_qsurlparse) DictLoader Environment)get_world_sizetcpstore_clientloggerT)slotsc0eZdZUeed<eed<dZdZdS)Response status_codetextcZ|jdkrtd|jd|jdS)NzHTTP z: )r RuntimeErrorrselfs f/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/torch/distributed/debug/_frontend.pyraise_for_statuszResponse.raise_for_status!s<  s " "Ft'7FF49FFGG G # "c4tj|jSN)jsonloadsrrs rr!z Response.json%sz$)$$$rN)__name__ __module__ __qualname__int__annotations__strrr!rrrrsL IIIHHH%%%%%rrc$eZdZUeed<eed<dS)NavLinkpathlabelN)r#r$r%r(r'r)rrr+r+)s" III JJJJJrr+c6eZdZUeed<edgefed<dS)Router,HTTPRequestHandlerhandlerN)r#r$r%r(r'rbytesr)rrr/r//s5 III +,e3 444444rr/ceZdZedeefdZedeefdZde e e ffdZ de dzfdZ de fdZ dS) DebugHandlerreturncdSr r)rs rrouteszDebugHandler.routes6s%(SrcdSr r)rs r nav_linkszDebugHandler.nav_links9s*-#rciSr r)rs r templateszDebugHandler.templates<s rNcdSr r)rs rdumpzDebugHandler.dump?strcNt|jSr )typer#lowerrs r dump_filenamezDebugHandler.dump_filenameBsDzz"((***r)r#r$r%rlistr/r7r+r9dictr(r;r=rAr)rrr4r45s(U (((^(-4=---^-4S>cDj+s++++++rr4urlsr5cddld}dtdtffd }t|5}|||}dddn #1swxYwY|S)Nrurlr5cb|}t|j|jSr )postrrr)rGresprequestss rgetzfetch_thread_pool..getQs)}}S!!($)444r) max_workers)rKr(rrmap)rDrMrLexecutorrespsrKs @rfetch_thread_poolrQKsOOOK55555555  4 4 4( S$''((((((((((((((( LsAAAcddldjdtdtfddttdt tffd }t j||S)NrsessionrGr5cK||4d{V}|d{V}t|j|cdddd{VS#1d{VswxYwYdSr )rIrrstatus)rSrGrJrs rfetchzfetch_aiohttp..fetch`s<<$$ / / / / / / /$$$$$$DDK.. / / / / / / / / / / / / / / / / / / / / / / / / / / / / / /s/A  A*-A*rDcK4d{Vtjfd|Dd{Vcdddd{VS#1d{VswxYwYdS)Nc(g|]}|Sr)r)).0rGrVrSs r z1fetch_aiohttp..gather..gs%)N)N)N#%%*=*=)N)N)Nr) ClientSessionasynciogather)rDrSaiohttprVs @rr]zfetch_aiohttp..gatheres.((** P P P P P P Pg )N)N)N)N)N)N)N)NOOOOOOO P P P P P P P P P P P P P P P P P P P P P P P P P P P P P Ps!A A A)r^r[r(rrBrr\run)rDr]r^rVs @@r fetch_aiohttpr`[sNNN/W2////// P49P();PPPPPPP ;vvd|| $ $$rendpointargsct}dttD}||}fd|D} t |}n#t $rt |}YnwxYw||fS)Ncg|]}d|S)rankr))rYrs rrZzfetch_all..ns 8 8 81J1JJ 8 8 8rcHg|]}|ddS)z /handler/?)decode)rYaddrrcrbs rrZzfetch_all..ps7 L L Ld 9 9 9 94 9 9 L L Lr)rranger multi_getr` ImportErrorrQ)rbrcstorekeysaddrsrPs`` r fetch_allrrls   E 8 8n&6&6 7 7 8 8 8D OOD ! !E L L L L Le L L LE)e$$ )))!%(() %<sA++BBblobcVtj|}tj|dS)N)indent)r!r"dumps)rsparseds r format_jsonryzs& Z  F :fQ ' ' ''rac {% block title %}{% endblock %} - PyTorch Distributed
{% block header %}{% endblock %} {% block content %}{% endblock %}
a {% extends "base.html" %} {% block header %}

{% block title %}{{title}}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ format_json(resp.text) }}
{% endif %} {% endfor %} {% endblock %} c NeZdZ d deedededdfdZd dZd d Z d d Z dS) PeriodicDumperN@handlers output_dirinterval_secondsr5Ncn||_||_||_tj|_d|_dSr ) _handlers _output_dir_interval_seconds threadingEvent _stop_event_thread)rr}r~rs r__init__zPeriodicDumper.__init__s6 "%!1$?,,04 rctj|jdtj|jdd|_|jdS)NT)exist_okz distributed.debug.PeriodicDumper)targetdaemonname)osmakedirsrrThread_runrstartrs rrzPeriodicDumper.startsZ D$t4444 '93    rc||j|j|jdSdSr )rsetrjoinrs rstopzPeriodicDumper.stopsA  < # L        $ #rc|jsY|jD]} |}n=#t$r0t d|YRwxYw|Ytj d}|d|d}tj |j |} t|d5}||dddn #1swxYwY#t$rt d|YwxYw|j|j|jWdSdS)NzFailed to dump %sz %Y%m%d_%H%M%S_z.txtwzFailed to write dump to %s)ris_setrr= Exceptionr exceptionrAtimestrftimerr,rropenwritewaitr)rr1content timestampfilenamer,fs rrzPeriodicDumper._run s"))++ :> I I%llnnGG $$%8':O:O:Q:QRRRH? M/:: %3355GG GGGw||D$4h??IdC)A((())))))))))))))) III$$%A4HHHHHI   ! !$"8 9 9 9#"))++ : : : : :sE;7A54A5D D5 D D D D D  %D65D6)r|r5N) r#r$r%rBr4r(floatrrrrr)rrr{r{s #' 5 5|$ 5 5 5  5 5 5 5    ::::::rr{c@eZdZUejZejed<dZe ed<dS)_IPv6HTTPServeraddress_familyirequest_queue_sizeN) r#r$r%socketAF_INET6r AddressFamilyr'rr&r)rrrr#s8+1?NF(:::""""""rrc eZdZUded<dZdZdefdZdeee effdZ defdZ d efd ed e d e de fd Zd S)r0FrontendServerfrontendcZtd|jd||zdS)Nz%s %sr)rinfoclient_address)rformatrcs r log_messagezHTTPRequestHandler.log_message+s7    " TM     rc:|j|dSr )r_handle_requestrs rdo_GETzHTTPRequestHandler.do_GET2s %%d+++++rr5c4t|jjSr )r r,rs rget_pathzHTTPRequestHandler.get_path5s ""''rcDt|Sr )r get_raw_queryrs r get_queryzHTTPRequestHandler.get_query8s**,,---rc4t|jjSr )r r,queryrs rrz HTTPRequestHandler.get_raw_query;s ""((rNrdefaultr?cd|}||vr|S|||dS)Nr)r)rrrr?rs r get_query_argz HTTPRequestHandler.get_query_arg>s;   u  NtE$KN###r)r#r$r%r'rrr(rrCrBrrobjectr?rr)rrr0r0(s   ,,,(#((((.4T#Y/....)s)))),0c$$$"($7;$ $$$$$$rr0cheZdZ d dedeedzfdZddZddZde ddfd Z d e d e de fd ZdS)rNportr}c|ddlm}|}dd|D}ttt d}|D])}||*t|}t|d|_ |j j ttt|i|_|D]-}|D]}|j|j|j<.||_t)d t*fd |i} d |f} t-| | |_t1j|jd dd |_|jdS)Nr)default_handlers c3jK|].}|D]}d|jd|jdV/dS)z z N)r9r,r-)rYr1links r z*FrontendServer.__init__..Ssq  ))++   NDI M M M M M       r)z base.htmlz raw_resp.htmlzjson_resp.htmlT)loader enable_async)zipry enumerater9r0rrar)z distributed.debug.FrontendServer)rrcrr)'torch.distributed.debug._debug_handlersrr BASE_TEMPLATERAW_RESP_TEMPLATEJSON_RESP_TEMPLATEupdater;r r _jinja_envglobalsrryr_routesr7r1r,rr?r0r_serverrr_serverr) rrr}rnav_html all_templatesr1rrouteRequestHandlerClassserver_addresss rrzFrontendServer.__init__Hs   P P P P P P''))H99  #     '.0) )   6 6G  !2!2!4!4 5 5 5 5M**%V$GGG &&# '   JL  9 9G )) 9 9+0= UZ(( 9"#  !     d&~7JKK  ';3     rr5c |jdS#t$rtdYdSwxYw)Nz got exception in frontend server)r serve_foreverrrrrs rrzFrontendServer._serves_ A L & & ( ( ( ( ( A A A   ? @ @ @ @ @ @ As$AAc8|jdSr )rrrs rrzFrontendServer.joins rreqc|}||jvr|dd|dS|j|} ||}n`#ttf$rL}t d||ddt|Yd}~dSd}~wwxYw|d| dd| |j |dS) NizHandler not found: z-Exception in frontend server when handling %siz Exception: rz Content-typez text/html) rr send_errorr SystemExitrrrepr send_response send_header end_headerswfiler)rrr,r1rJes rrzFrontendServer._handle_requests&||~~ t| # # NN3 .s:   ((**m;;;;;rz2Periodic dumper started, writing to %s every %.0fs) rsetLevelloggingINFOrrr server_portr{rrr)rrrr}rserverdumpers ` rmainrs  OOGL!!!  9 9 9F KK4fn6PQQQ$(F    '          @        KKMMMMM  6  KKMMMM s .CC5)ra)8r\r!rrrrrabcrrcollections.abcrrconcurrent.futuresr dataclassesr http.serverr r urllib.parser r jinja2r rtorch.distributed.debug._storerr getLoggerr#rLoggerr'rr+r/r4rBr(rQr`tuplerrryrrrr{rr0rr&rrrr)rrr s   ########........111111!!!!!!CCCCCCCC++++++++********JJJJJJJJ+*844444  % % % % % % % %   55555555 +++++3+++, DI (8*<     %S %hx&8%%%%"   3 d3i(AS6S0T    (c((((> @$0-:-:-:-:-:-:-:-:j#####)### $$$$$/$$$>_P_P_P_P_P_P_P_PD" "Dj""< " s8 "  """"""r