jJddlZddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z ddl m Z GddZGdd Zed kred ed eZ eZeddededddededddededdddededdddededd dd!ed"ed#d$d%ded&ed#d'd%ded(rced)ed(D]<\ZZed*ed+ed,d-d.ed/dd0ed1d2d3ed4d-d5 =n ed6ejd7a#e$red8YdSwxYwdS)9N)datetime)Path)LOGGERMACOSRANK)check_requirementsc~eZdZdZddZdZdZdZd Zd Z d Z Gd d Z Gdde j ZdS) ConsoleLoggera Console output capture with batched streaming to file, API, or custom callback. Captures stdout/stderr output and streams it with intelligent deduplication and configurable batching. Attributes: destination (str | Path | None): Target destination for streaming (URL, Path, or None for callback-only). batch_size (int): Number of lines to batch before flushing (default: 1 for immediate). flush_interval (float): Seconds between automatic flushes (default: 5.0). on_flush (callable | None): Optional callback function called with batched content on flush. active (bool): Whether console capture is currently active. Examples: File logging (immediate): >>> logger = ConsoleLogger("training.log") >>> logger.start_capture() >>> print("This will be logged") >>> logger.stop_capture() API streaming with batching: >>> logger = ConsoleLogger("https://api.example.com/logs", batch_size=10) >>> logger.start_capture() Custom callback with batching: >>> def my_handler(content, line_count, chunk_id): ... print(f"Received {line_count} lines") >>> logger = ConsoleLogger(on_flush=my_handler, batch_size=5) >>> logger.start_capture() N@c||_t|to|d|_||jst ||_t d||_||_||_ tj |_ tj |_d|_d|_g|_t%j|_d|_d|_d|_d|_d|_d|_dS)aInitialize console logger with optional batching. Args: destination (str | Path | None): API endpoint URL (http/https), local file path, or None. batch_size (int): Lines to accumulate before flush (1 = immediate, higher = batched). flush_interval (float): Max seconds between flushes when batching. on_flush (callable | None): Callback(content: str, line_count: int, chunk_id: int) for custom handling. )zhttp://zhttps://Nr Frg) destination isinstancestr startswithis_apirmax batch_sizeflush_intervalon_flushsysstdoutoriginal_stdoutstderroriginal_stderractive _log_handlerbuffer threadingLock buffer_lock flush_threadchunk_id last_line last_timelast_progress_linelast_was_progress)selfrrrrs ]/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/ultralytics/utils/logger.py__init__zConsoleLogger.__init__-s' c22f{7M7MNe7f7f  "4; "#K00D a,,,   #z"z   $>++  "$!&c&|js tdvrdSd|_||j|jt _||j|jt _ | |j|_ tj d |j n#t$rYnwxYw|jdkr;t!j|jd|_|jdSdS)zStart capturing console output and redirect stdout/stderr. Notes: In DDP training, only activates on rank 0/-1 to prevent duplicate logging. >rNT ultralyticsr )targetdaemon)rr_ConsoleCapturer _queue_logrrrr _LogHandlerrlogging getLogger addHandler Exceptionrr Thread _flush_workerr#startr)s r* start_capturezConsoleLogger.start_captureRs  ; $g-- F ))$*>PP ))$*>PP   $ 0 0 A AD   m , , 7 78I J J J J    D  ?Q   ) 08JSW X X XD    # # % % % % %  s/A B;; CCc*|jsdSd|_|jt_|jt_|jrE tjd |jn#t$rYnwxYwd|_| dS)z9Stop capturing console output and flush remaining buffer.NFr/) rrrrrrrr5r6 removeHandlerr8 _flush_bufferr<s r* stop_capturezConsoleLogger.stop_captureks{  F ) )    % !-00>>t?PQQQQ     $D  s,A(( A54A5c|jsdStj}d|vr|dd}|d}|r |ddkr||D]}|}d|vrd|vrd|v}|s'|}d}|rd |d vr7|d d dr |d }nE|d d kr't|d kr|d d |d }n|d dvr|d }|r|j|dkr|r |d|_d|_ n|s|j rd|_ d|_ ||j kr||j z dkr||_ ||_ | ds.tjd}d|d|}d} |j5|j|t|j|jkrd} dddn #1swxYwY| r|dS)z?Queue console text with deduplication and timestamp processing.N r. ru─u ━━z100%/rClassr _)ztrain:zval:z:doneTF皙?z[20z%Y-%m-%d %H:%M:%S[z] )rtimesplitpoprstripreplaceisdigitlenr'r(r%r&rrnowstrftimer"rappendrr@) r)text current_timelinesline is_completepartsseq_key timestamp should_flushs r*r3zConsoleLogger._queue_logs{  Fy{{  4<<::d##B'D 4    U2Y"__ IIKKK@ %@ %D;;==D}}D  $n # +eAh58+;+;C+D+D+L+L+N+N"'(qW,,Ua%*1X":":a":":q%777"'(t6W:K:K:KKK@18.?.?.?D+)-&& 6-2D*).&t~%%,*G#*M*M!DN)DN??5)) /$LNN334GHH .9....!L! ( ( ""4(((t{##t66#'L ( ( ( ( ( ( ( ( ( ( ( ( ( ( (  %""$$$A@ %@ %s,:H22H6 9H6 c|jr=tj|j|jr||j;dSdS)z3Background worker that flushes buffer periodically.N)rrJsleeprr@r<s r*r:zConsoleLogger._flush_workers[k % Jt* + + +{ %""$$$k % % % % %r,c|j5|js ddddS|j}|j|xjdz c_|j}dddn #1swxYwYd|}t |}|jr) ||||n#t$rYnwxYw|j | |dSdS)z4Flush buffered lines to destination and/or callback.Nr rD) r"rcopyclearr$joinrPrr8r_write_destination)r)rVr$content line_counts r*r@zConsoleLogger._flush_buffersj   % %;  % % % % % % % %K$$&&E K      MMQ MM}H  % % % % % % % % % % % % % % %))E""ZZ  =   gz8<<<<       '  # #G , , , , , ( 's) A3A A33A7:A7*C CCc |jrXddl}tj|d}|t |j|ddS|jj dd|j dd 5}| |d zddddS#1swxYwYdS#t$r$}td ||j Yd}~dSd}~wwxYw)z)Write content to file or API destination.rN)r[message)jsontimeoutT)parentsexist_okazutf-8)encodingrDzConsole logger write error: )file)rrequestsrrQ isoformatpostrrparentmkdiropenwriter8printr)r)rdrppayloadfes r*rcz ConsoleLogger._write_destinationsi Q{ ,(0 (@(@(B(BwWW c$"233'1 MMMMM '--dT-JJJ%**3*AA,QGGGdN+++,,,,,,,,,,,,,,,,,, Q Q Q 4444;O P P P P P P P P P P QsBAC!=CC7 CCC C C C?C::C?c(eZdZdZdZdZdZdZdS)ConsoleLogger._ConsoleCapturez"Lightweight stdout/stderr capture.)callbackoriginalc"||_||_dS)z^Initialize a stream wrapper that redirects writes to a callback while preserving the original.N)r~r})r)r~r}s r*r+z&ConsoleLogger._ConsoleCapture.__init__s$DM$DMMMr,cd|j|||dS)zIWrite text to the original stream and forward it to the capture callback.N)r~rvr})r)rTs r*rvz#ConsoleLogger._ConsoleCapture.writes0 M   % % % MM$     r,c8|jdS)zVFlush the wrapped stream to propagate buffered output promptly during console capture.N)r~flushr<s r*rz#ConsoleLogger._ConsoleCapture.flush s M   ! ! ! ! !r,N)__name__ __module__ __qualname____doc__ __slots__r+rvrr,r*r2r|sL00,  % % %     " " " " "r,r2c,eZdZdZdZfdZdZxZS)ConsoleLogger._LogHandlerzLightweight logging handler.)r}cVt||_dS)z\Initialize a lightweight logging.Handler that forwards log records to the provided callback.N)superr+r})r)r} __class__s r*r+z"ConsoleLogger._LogHandler.__init__s$ GG     $DMMMr,c\|||dzdS)zXFormat and forward LogRecord messages to the capture callback for unified log streaming.rDN)r}format)r)records r*emitzConsoleLogger._LogHandler.emits+ MM$++f--4 5 5 5 5 5r,)rrrrrr+r __classcell__)rs@r*r4rsR**!  % % % % %  6 6 6 6 6 6 6r,r4)Nr r N)rrrrr+r=rAr3r:r@rcr2r5Handlerr4rr,r*r r s:#'#'#'#'J&&&2(O%O%O%b%%%---0 Q Q Q""""""""& 6 6 6 6 6go 6 6 6 6 6r,r c,eZdZdZdZdZddZdZdS) SystemLoggeruLog dynamic system metrics for training monitoring. Captures real-time system metrics including CPU, RAM, disk I/O, network I/O, and NVIDIA GPU statistics for training performance monitoring and analysis. Attributes: pynvml: NVIDIA pynvml module instance if successfully imported, None otherwise. nvidia_initialized (bool): Whether NVIDIA GPU monitoring is available and initialized. net_start: Initial network I/O counters for calculating cumulative usage. disk_start: Initial disk I/O counters for calculating cumulative usage. Examples: Basic usage: >>> logger = SystemLogger() >>> metrics = logger.get_metrics() >>> print(f"CPU: {metrics['cpu']}%, RAM: {metrics['ram']}%") >>> if metrics["gpus"]: ... gpu0 = metrics["gpus"]["0"] ... print(f"GPU0: {gpu0['usage']}% usage, {gpu0['temp']}°C") Training loop integration: >>> system_logger = SystemLogger() >>> for epoch in range(epochs): ... # Training code here ... metrics = system_logger.get_metrics() ... # Log to database/file cddl}d|_||_||_||_|j|_|j|_ tj |_ dS)zInitialize the system logger.rN) psutilpynvml _init_nvidianvidia_initializednet_io_counters net_startdisk_io_counters disk_start _prev_net _prev_diskrJ _prev_time)r)rs r*r+zSystemLogger.__init__=sq  "&"3"3"5"5//11 1133/)++r,c*trdS tdtd|_|jdS#t $r?}ddl}|jrtj d|Yd}~dSd}~wwxYw)z-Initialize NVIDIA GPU monitoring with pynvml.Fznvidia-ml-py>=12.0.0rTrNzSystemLogger NVML init failed: ) rr __import__rnvmlInitr8torchcuda is_availablerwarning)r)rzrs r*rzSystemLogger._init_nvidiaKs  5  5 6 6 6$X..DK K " " "4    LLLz&&(( FDDDEEE55555  s>> logger = SystemLogger() >>> logger.get_metrics()["cpu"] # CPU percentage >>> logger.get_metrics(rates=True)["network"]["recv_mbs"] # MB/s download rate rNrE)cpuramgpusrHii@)read_mbs write_mbsused_gbdisk)recv_mbssent_mbsnetwork)read_mbwrite_mbr)recv_mbsent_mbr)rrrvirtual_memoryshutil disk_usagerJround cpu_percentpercentrr read_bytesr write_bytesused bytes_recvr bytes_sentrrrupdate_get_nvidia_metrics) r)ratesrnetrmemoryrrQmetricselapseds r* get_metricszSystemLogger.get_metrics\sX  $$&&&&((&&((&s++ ikk++--q11++  c3011  "#a$/DOY+Y^e*fip*p#q#qstuu G!;T*TY`)adk)k"l"lnopp!#a#.4>;T*TY`)adk)k"l"lnopp""GI  !$/DO4N"NSZ![]^__!4#3do6Q#QV]"^`abb G!3L"LQX!Y[\]] #.4>3L"LQX!Y[\]]""GI    " ? FO " "4#;#;#=#= > > >r,cTi}|jr|js|S |j}t|D]}|j|}|j|}|j|}|j||jj}|j |dz}t|j dt|j |j z dzd||d|t|<n#t$rYnwxYw|S)zMGet NVIDIA GPU metrics including utilization, memory, temperature, and power.ird)usagertemppower)rrnvmlDeviceGetCountrangenvmlDeviceGetHandleByIndexnvmlDeviceGetUtilizationRatesnvmlDeviceGetMemoryInfonvmlDeviceGetTemperatureNVML_TEMPERATURE_GPUnvmlDeviceGetPowerUsagergpurtotalrr8) r)r device_countihandleutilrrrs r*rz SystemLogger._get_nvidia_metricss:& dk K ;99;;L<((  ??BB{@@HH<rs  1111111111777777N6N6N6N6N6N6N6N6bqqqqqqqqh z E 2333 E "### \^^F ) ((**G E.b ) ) ) ) E0'%.0000 1 1 1 E0'%.0000 1 1 1 ED :DDDD E E E EF!<FFFF G G G ED :DDDD E E E EFwy1)<FFFF G G G EFwy1)<FFFF G G Gv 8&'''(/(=(=(?(?$FHE9998G+<D99 ( 2=99!)&!1;99#+7"389996777 DJqMMM9 < ))) '(((((()Ks3E"GG)(G)