
    /j                       d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	m	Z	m
Z
 ddlmZmZ ddlmZ ddlmZ ddlZddlZddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4m5Z5m6Z6m7Z7 ddl8m9Z9m:Z: ddl;m<Z< ddl=m>Z> ddl?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZL  G d d          ZMdS )zz
Train a model on a dataset.

Usage:
    $ yolo mode=train model=yolo26n.pt data=coco8.yaml imgsz=640 epochs=100 batch=16
    )annotationsN)copydeepcopy)datetime	timedelta)partial)Path)distributed)nnoptim)__version__)get_cfgget_save_dir)check_cls_datasetcheck_det_dataset)load_checkpoint)MuSGD)DEFAULT_CFGGIT
LOCAL_RANKLOGGERRANKTQDMYAML	callbacks	clean_urlcolorstremojis)check_train_batch_size)	check_amp
check_filecheck_imgszcheck_model_file_from_stem
print_args)ddp_cleanupgenerate_ddp_command)get_latest_run)plot_results)	TORCH_2_4EarlyStoppingModelEMAattempt_compileautocast$convert_optimizer_state_dict_to_fp16
init_seeds	one_cycleselect_devicestrip_optimizertorch_distributed_zero_firstunset_deterministicunwrap_modelc                  ,   e Zd ZdZeddfd<dZd=dZd=d	Zd=d
Zd Z	d Z
d Zd Zd Zd Zd>dZd?dZd@dAdZd Zd Zd Zd Zd Zd Zd Zd ZdBd!Zd" ZdCd%ZdDd&ZdEd'Zd( Zd) Z d* Z!d+ Z"d, Z#d- Z$d. Z%d@d/Z&d0 Z'd1 Z(d2 Z)d3 Z*d4 Z+d5 Z,dFd;Z-dS )GBaseTrainera*	  A base class for creating trainers.

    This class provides the foundation for training YOLO models, handling the training loop, validation, checkpointing,
    and various training utilities. It supports both single-GPU and multi-GPU distributed training.

    Attributes:
        args (SimpleNamespace): Configuration for the trainer.
        validator (BaseValidator): Validator instance.
        model (nn.Module): Model instance.
        callbacks (defaultdict): Dictionary of callbacks.
        save_dir (Path): Directory to save results.
        wdir (Path): Directory to save weights.
        last (Path): Path to the last checkpoint.
        best (Path): Path to the best checkpoint.
        save_period (int): Save checkpoint every x epochs (disabled if < 1).
        batch_size (int): Batch size for training.
        epochs (int): Number of epochs to train for.
        start_epoch (int): Starting epoch for training.
        device (torch.device): Device to use for training.
        amp (bool): Flag to enable AMP (Automatic Mixed Precision).
        scaler (torch.amp.GradScaler): Gradient scaler for AMP.
        data (dict): Dataset dictionary containing paths and metadata.
        ema (ModelEMA): EMA (Exponential Moving Average) of the model.
        resume (bool): Resume training from a checkpoint.
        lf (callable): Learning rate scheduling function.
        scheduler (torch.optim.lr_scheduler._LRScheduler): Learning rate scheduler.
        best_fitness (float): The best fitness value achieved.
        fitness (float): Current fitness value.
        loss (torch.Tensor): Current loss value.
        tloss (torch.Tensor): Running mean of loss items.
        loss_names (list): List of loss names.
        csv (Path): Path to results CSV file.
        metrics (dict): Dictionary of metrics.
        plots (dict): Dictionary of plots.

    Methods:
        train: Execute the training process.
        validate: Run validation on the val set.
        save_model: Save model training checkpoints.
        get_dataset: Get train and validation datasets.
        setup_model: Load, create, or download model.
        build_optimizer: Construct an optimizer for the model.

    Examples:
        Initialize a trainer and start training
        >>> trainer = BaseTrainer(cfg="config.yaml")
        >>> trainer.train()
    N
_callbacksdict | Nonec                	   |                     dd          | _        t          ||          | _        |                     |           t          | j        j                  | _        dt          | j                  v rt          j	        d          nt          | j                  | j        _        d| _
        d| _        i | _        t          | j        j        dz   t          z   | j        j                   t#          | j                  | _        | j        j        | j        _        | j        dz  | _        t          dv r| j                            d	d	
           t          | j                  | j        _        t-          | j                                                  }|                    d          d |d         D             |d<   t3          j        | j        dz  |           | j        dz  | j        dz  c| _        | _        | j        j        | _        | j        j        | _        | j        j         pd| _         d| _!        t          dk    r!tE          t-          | j                             | j        j#        dv rd| j        _$        |ptK          j&                    | _%        tO          | j        j        t                    rFtQ          | j        j                  r-tQ          | j        j        )                    d                    }nttO          | j        j        tT          tV          f          rtQ          | j        j                  }n4| j        j        dv rd}n#tX          j-        .                                rd}nd}|dk    odt          j/        v| _0        || _1        t          dv r0| j0        s)tK          j2        |            | 3                    d           ti          | j        j5                  | _5        tm          tn                    5  | 8                                | _9        ddd           n# 1 swxY w Y   d| _:        d| _;        d| _<        d| _=        d| _>        d| _?        d| _@        dg| _A        | j        dz  | _B        | jB        C                                r%| j        jD        s| jB        E                                 g d| _F        d| _G        dS )a5  Initialize the BaseTrainer class.

        Args:
            cfg (str | dict | SimpleNamespace, optional): Path to a configuration file or configuration object.
            overrides (dict, optional): Configuration overrides.
            _callbacks (dict, optional): Dictionary of callback functions.
        sessionNcudaCUDA_VISIBLE_DEVICES   )deterministicweights   r   Tparentsexist_okaugmentationsc                ,    g | ]}t          |          S  )repr).0ts     _/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/ultralytics/engine/trainer.py
<listcomp>z(BaseTrainer.__init__.<locals>.<listcomp>   s    -Z-Z-Z!d1gg-Z-Z-Z    z	args.yamlzlast.ptzbest.ptd   r   rB   >   cpumps,r   on_pretrain_routine_startLosszresults.csv)r   r>      )Hpophub_sessionr   argscheck_resumer1   devicestrosgetenv	validatormetricsplotsr/   seedr   r?   r   save_dirnamewdirmkdirvarsr   getr   savelastbestsave_periodbatch
batch_sizeepochsstart_epochr$   typeworkersr   get_default_callbacks
isinstancelensplittuplelisttorchr<   is_availableenvironddp
world_sizeadd_integration_callbacksrun_callbacksr#   modelr3   r   get_datasetdataemalf	schedulerbest_fitnessfitnesslosstloss
loss_namescsvexistsresumeunlinkplot_idxnan_recovery_attempts)selfcfg	overridesr8   	args_dictr|   s         rL   __init__zBaseTrainer.__init__u   s\    %==D99C++	)$$$#DI$455@F#dkJZJZ@Z@Z29%;<<<`cdhdo`p`p	
49>A%,DI<STTTT %TY//+	MI-	7??IOOD4O888!$T]!3!3DITY,,..I}}_--9-Z-Zy?Y-Z-Z-Z	/*Idmk19===#y94di)6K	4990)/i&-#2::tDI''' ;~-- !DI $Hy'F'H'Hdi&,, 		TY5E1F1F 		TY-33C8899JJ	(5$-88 	TY-..JJY//JJZ$$&& 	JJJ>Dl"*&D$7??48?/555:;;; 0	@@
)*55 	+ 	+((**DI	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+ 	+   !	
!(==08?? 	TY%5 	HOO!		%&"""s   >P$$P(+P(eventr[   c                F    | j         |                             |           dS )z7Append the given callback to the event's callback list.N)r   appendr   r   callbacks      rL   add_callbackzBaseTrainer.add_callback   s#    u$$X.....rN   c                    |g| j         |<   dS )zPOverride the existing callbacks with the given callback for the specified event.N)r   r   s      rL   set_callbackzBaseTrainer.set_callback   s    !)
urN   c                X    | j                             |g           D ]} ||            dS )z>Run all existing callbacks associated with a particular event.N)r   rg   r   s      rL   r~   zBaseTrainer.run_callbacks   s<    **5"55 	 	HHTNNNN	 	rN   c                n   | j         r| j        j        r t          j        d           d| j        _        | j        j        dk     r#t          d| j         d| j        dz   d          t          |           \  }}	 t          j	        t          d           d	d
                    |                      t          j        |d           n# t          $ r}|d}~ww xY w	 t          | t!          |                     dS # t          | t!          |                     w xY w|                                  dS )zcExecute the training process, using DDP subprocess for multi-GPU or direct training for single-GPU.zI'rect=True' is incompatible with Multi-GPU training, setting 'rect=False'F      ?zuAutoBatch with batch<1 not supported for Multi-GPU training, please specify a valid batch size multiple of GPU count z, i.e. batch=   .zDDP:z debug command  T)checkN)r{   rX   rectr   warningrl   
ValueErrorr|   r&   infor   join
subprocessrun	Exceptionr%   r[   	_do_train)r   cmdfilees       rL   trainzBaseTrainer.train   sm    8 	y~ 'jkkk!&	y$$ DOSD Dmqm|  @A  nAD D D   -T22IC-x//OOOOPPPs$/////    0 D#d)),,,,,D#d)),,,, NNs+   ;AC 
C? 
CCCC? ?Dc                      j         j        r&t          d j         j         j                   _        n
 fd _        t          j                             j	         j                   _
        dS )z,Initialize training learning rate scheduler.r>   c                t    t          d| j        z  z
  d          dj        j        z
  z  j        j        z   S )Nr>   r   r   )maxrn   rX   lrf)xr   s    rL   <lambda>z.BaseTrainer._setup_scheduler.<locals>.<lambda>   s6    ADK$7 ; ;sTY]?R SVZV_Vc c rN   )	lr_lambdaN)rX   cos_lrr0   r   rn   r   r   lr_schedulerLambdaLR	optimizerr   r   s   `rL   _setup_schedulerzBaseTrainer._setup_scheduler   sa    9 	d49=$+>>DGGccccDG+44T^tw4WWrN   c                4   t           j                            t                     t          j        dt                    | _        dt
          j        d<   t          j        t          j	                    rdndt          d          t          | j                   d	S )
zGInitialize and set the DistributedDataParallel parameters for training.r<   1TORCH_NCCL_BLOCKING_WAITncclglooi0*  )seconds)backendtimeoutrankr|   N)rx   r<   
set_devicer   rZ   r\   rz   distinit_process_groupis_nccl_availabler   r|   r   s    rL   
_setup_ddpzBaseTrainer._setup_ddp   s    
d###l640014
-."466BFFFe,,,		
 	
 	
 	
 	
 	
rN   c                ~   | j         t          | j        d          z  }|                     | j        d         |t
          d          | _        |                     | j                            d          p| j                            d          | j        j	        dk    r|n|dz  t
          d          | _
        t          t          | j        j        | j         z            d          | _        | j        j        | j         z  | j        z  | j        j        z  }t          j        t#          | j        j                  t          | j         | j        j                  z            | j        z  }|                     | j        | j        j        | j        j        | j        j        ||          | _        |                                  d	S )
zCBuild dataloaders, optimizer, and scheduler for current batch size.r>   r   )rm   r   modevaltestobbrU   )r   rc   lrmomentumdecay
iterationsN)rm   r   r|   get_dataloaderr   r   train_loaderrg   rX   tasktest_loaderroundnbs
accumulateweight_decaymathceilrt   datasetrn   build_optimizerr   r   lr0r   r   )r   rm   r   r   s       rL   _build_train_pipelinez!BaseTrainer._build_train_pipeline
  s   _DOQ(?(??
 //Ig:JW 0 
 
  ..IMM%  9DIMM&$9$9%)Y^u%<%<zz*q.	 / 
 
 eDIMDO$CDDaHHy-?$/QTXT]TaaYs4#4#<==DOUYU^Ub@c@ccddgkgrr
--*$y}Y'! . 
 
 	rN   c           	     
  	 |                                  }| j                            | j                  | _        |                                  t          | j        | j        | j        j                  | _        t          | j        j	        t                    r| j        j	        n9t          | j        j	        t                    rt          | j        j	                  ng }dg}d |D             |z   }|| _        | j                                        D ]r\  	}t          	fd|D                       r t!          j        d	 d           d|_        @|j        s+|j        j        rt!          j        d	 d	           d
|_        st-          j        | j        j                                      | j                  | _        | j        ret2          dv r\t4          j                                        }t-          j        t;          | j                  | j                  | _        |t4          _        t2          dk    r8| j        dk    r-t?          j         | j                                        d           tC          | j                  | _        tD          r&t,          j        #                    d| j                  n)t,          j$        j        #                    | j                  | _%        | j        dk    r2tL          j'        (                    | j        t2          gd
          | _        tS          t          tU          | j        d          r| j        j+        )                                nd          d          }tY          | j        j-        ||d          | j        _-        || _+        | j.        dk     r0t2          dk    r%| /                                x| j        _0        | _.        | 1                                 | 2                                | _3        ti          | j                  | _5        t2          dv r{| j3        j6        j7        | 8                    d          z   }ts          tu          |dgtw          |          z                      | _6        | j        j<        r| =                                 t}          | j        j?                  dc| _@        | _A        | B                    |           | jC        dz
  | jD        _E        | F                    d           dS )zYConfigure model, optimizer, dataloaders, and training utilities before the training loop.)rZ   r   z.dflc                    g | ]}d | d	S )zmodel.r   rH   rJ   r   s     rL   rM   z,BaseTrainer._setup_train.<locals>.<listcomp>6  s     AAAmqmmmAAArN   c              3      K   | ]}|v V  	d S NrH   )rJ   r   ks     rL   	<genexpr>z+BaseTrainer._setup_train.<locals>.<genexpr>:  s'      66a16666666rN   zFreezing layer ''Fz/setting 'requires_grad=True' for frozen layer 'zE'. See ultralytics.engine.trainer for customization of frozen layers.TrA   )rZ   rB   r>   r   srcr<   )enabled)
device_idsfind_unused_parametersstride    )r   floormax_dimr   )prefix)patienceon_pretrain_routine_endN)Gsetup_modelr   torZ   set_model_attributesr,   rX   compilers   freezerw   intrangefreeze_layer_namesnamed_parametersanyr   r   requires_graddtypeis_floating_pointr   rx   tensorampr   r   default_callbacksr   r    r|   r   	broadcastboolr)   
GradScalerr<   scalerr   parallelDistributedDataParallelr   hasattrr   r"   imgszrm   
auto_batchrl   r   get_validatorr^   r+   r   r_   keyslabel_loss_itemsdictziprt   r`   plot_training_labelsr*   r   stopperstopresume_trainingro   r   
last_epochr~   )
r   ckptfreeze_listalways_freeze_namesr   vcallbacks_backupgsmetric_keysr   s
            @rL   _setup_trainzBaseTrainer._setup_train$  sp   !!Z]]4;//
!!### %TZ$)J[\\\

 $)*D11DI $)*C00ty'((( 	  &hAA[AAADWW"4J//11 
	' 
	'DAq6666#566666 '3q333444"'_ ')B 'Ya Y Y Y   #' <	..11$+>>8 	;(:??AA|Idj$9$9$+NNNDH*:I'"991,,N48<<>>q1111>>>GxEI   :::UZ^MfMfosowMfMxMx 	 ?Q<<TZUYTZsw<xxDJ 
H0M0MUTZ&**,,,SUVVXZ[[%diobTUVVV	 ?Q42::040A0AADIOdo""$$$++--DJ''7??.058M8MUZ8M8[8[[KK!s;7G7G1G H HIIDLy ,))+++"/9K"L"L"LediT"""$($4q$8!455555rN   c                ,   | j         dk    r|                                  |                                  t          | j                  }| j        j        dk    r*t          t          | j        j        |z            d          nd}d}d| _	        t          j
                    | _        t          j
                    | _        |                     d           t          j        d| j        j         d| j        j         d	| j        j        | j         pdz   d
t%          d| j                   d	| j        j
        r| j        j
         dn	| j         dz              | j        j        r:| j        | j        j        z
  |z  }| j                            ||dz   |dz   g           | j        }| j                                         d| _        	 || _        |                     d           t;          j                    5  t;          j        d           | j         !                                 ddd           n# 1 swxY w Y   | "                                 tF          dk    r| j        j$        %                    |           tM          | j                  }|| j        | j        j        z
  k    r-| '                                 | j        (                                 tF          dv rIt          j        | )                                           tU          tM          | j                  |          }d| _+        |D ]\  }}|                     d           |||z  z   }	|	|k    rd|g}
t          dtY          t[          j.        |	|
d| j        j/        | j0        z  g                                                              | _1        | j        j2        D ]}t[          j.        |	|
|3                    d          dk    r| j        j4        nd|d         | 5                    |          z  g          |d<   d|v r/t[          j.        |	|
| j        j6        | j        j7        g          |d<   	 tq          | j9                  5  | :                    |          }| j        j;        rL| <                    |d                   }t{          | j<                  >                    ||          \  }| _?        n| <                    |          \  }| _?        |@                                | _>        tF          dk    r| xj>        | j         z  c_>        | j+        | j?        n| j+        |z  | j?        z   |dz   z  | _+        ddd           n# 1 swxY w Y   | jA        B                    | j>                  C                                 nN# t          jE        jF        $ r6 || j        k    s| j        dk    stF          dk    r | xj        dz  c_        | j0        }t          | j0        dz  d          x| j        _G        | _0        t          jH        d| d| j0         d | j         d!           | I                                 | J                                 | j        dz
  | j         _K        t          | j                  }| j        j        dk    r*t          t          | j        j        |z            d          nd}d}| j                                         Y  nw xY w|	|z
  | j1        k    r| L                                 |	}| j        j
        r}t          j
                    | j        z
  | j        j
        d"z  k    | _M        tF          dk    r7tF          dk    r| jM        ndg}t          jO        |d           |d         | _M        | jM        r n2tF          dv rt          | j+        jP                  r| j+        jP        d         nd}|Q                    d#d$d|z   z  z   |dz    d%| j         | R                                d&d'g|dk    r| j+        nt          jS        | j+        d          |d(         jP        d         |d         jP        d         R z             |                     d)           | j        jT        r|	| j        v r| U                    ||	           |                     d*           | jM        r n	d| _        | j        r	| jM        st          t{          | j<                  jW        d+          r+t{          | j<                  jW        X                                 d, tM          | j        j2                  D             | _Y        |                     d-           tF          dv r#| jZ        [                    | j<        g d./           |dz   | j        k    }| j        j\        s|s| j]        j^        s| jM        r7| I                    d01           | _                                \  | _`        | _a        | b                    |          rQd| _c        tF          dv r| d                    i | e                    | j+                  | j`        | jY        2           | xjM        | ]                    |dz   | ja                  p|z  c_M        | j        j
        r:| xjM        t          j
                    | j        z
  | j        j
        d"z  k    z  c_M        | j        jf        s|r)| g                                 |                     d3           t          j
                    }|| j        z
  | _	        || _        | j        j
        r|| j        z
  || j        z
  dz   z  }t          ji        | j        j
        d"z  |z            x| _        | j        _        | j                                 | j        | j         _K        | xjM        || j        k    z  c_M        |                     d4           | I                    d0           tF          dk    r7tF          dk    r| jM        ndg}t          jO        |d           |d         | _M        | jM        rn|dz  }
t          j
                    | j        z
  }t          j        d5|| j        z
  dz    d6|d"z  d7d8           | k                                 tF          dv r5| j        jT        r| l                                 |                     d9           | I                                 t                       |                     d:           dS );zbPerform the full training loop including setup, epoch iteration, validation, and final evaluation.r>   r   rO   rB   Non_train_startzImage sizes z train, z val
Using z' dataloader workers
Logging results to boldz
Starting training for z	 hours...z
 epochs...rU   Ton_train_epoch_startignorerA   )totalon_train_batch_startparam_groupbias        
initial_lrr   r   img   zCUDA out of memory with batch=z. Reducing to batch=z and retrying (z/3).i  z%11s%11sz%11.4g/z.3gGclson_batch_endon_train_batch_endupdatec                ,    i | ]\  }}d | |d         S )zlr/pgr   rH   )rJ   irr   s      rL   
<dictcomp>z)BaseTrainer._do_train.<locals>.<dictcomp>  s(    aaaQ|r||QtWaaarN   on_train_epoch_end)yamlncrX   namesr   class_weights)includeg      ?)	threshold)r_   on_model_saveon_fit_epoch_end
z epochs completed in z.3fz hours.on_train_endteardown)nr|   r   r!  rt   r   rX   warmup_epochsr   r   
epoch_timetimeepoch_time_starttrain_time_startr~   r   r   r  num_workersr   rb   rn   close_mosaicr   extendro   r   	zero_grad_oom_retriesepochwarningscatch_warningssimplefilterr   step_model_trainr   sampler	set_epoch	enumerate_close_dataloader_mosaicresetprogress_stringr   r   r   npinterpr   rm   r   param_groupsrg   warmup_bias_lrr   warmup_momentumr   r-   r  preprocess_batchr   r   r5   r   
loss_itemssumr
  scalebackwardrx   r<   OutOfMemoryErrorrl   r   _clear_memoryr   r  optimizer_stepr  r   broadcast_object_listshapeset_description_get_memory	unsqueezer`   plot_training_samplesr  	criterionr4  r   r   update_attrr   r  possible_stopvalidater_   r   _handle_nan_recoveryr   save_metricsr  rh   
save_modelr   r   r   
final_evalplot_metricsr4   )r   nbnwlast_opt_stepbase_idxrN  pbarirl   nixir   predsr   	old_batchbroadcast_listloss_lengthfinal_epochrK   mean_epoch_timer   s                        rL   r   zBaseTrainer._do_trainj  s   ?QOO"##>Bi>UXY>Y>YSty.344c:::_a $	 $	+,,,&49? & &DIO & &&2do6JK& &"*64="A"A& & & JN)wDIN)E)E)E)E`d`k]w]w]wy	
 	
 	
 9! 	Idi&<<BHM  (HqL(Q,!GHHH   """i	DJ5666(** & &%h///##%%%& & & & & & & & & & & & & & & rzz!)33E:::T.//Dty'==>>--///!'')))wD0022333Id&788CCCDJ  ]& ]&5""#9:::e^88RB&)!S2rAty}W[WfGfCg1h1h1n1n1p1p-q-q&r&rDO!^8 o o"$)<=EE-<P<PTZ<Z<Z	 8 8`c !,$''%.. @# #$ &??,.Ib"ty?XZ^ZcZl>m,n,nAjM#!$(++   $ 5 5e < <9, F$(JJuU|$<$<E4@4L4L4Q4QRWY^4_4_1D$//48JJu4E4E1D$/$(HHJJ	2:: II8II/3z/ADOO
UVY]YhHhmnqrmrGs 
                K%%di0099;;;;z2   t///43D3I3ITUWZZ%%*%% $I8;DOq<PRS8T8TTDIOdoNe e e-1_e eMQM^e e e   &&(((..000040@10DDN-T.//BJN)JadeJeJeU49#:R#?@@#FFFkmB$&MN,,...EE#$ %88'')))$&M y~ "%)Y[[43H%HTY^^bMb$c	2::;?199dii$-ON 6~qIII(6q(9DI9 "!E 7??9<TZ=M9N9N"U$*"21"5"5TUK((#h!k/&BB$qy884;88#//118888 -8!OOdjjQUQ[]^A_A_ "%L.q1	
 "%L.r2 	 	 	 &&~666y >2+>+>225"===""#78889 E %&!   |DJ//98DD <TZ((299;;;aa)DND_:`:`aaaDG3444w$$TZ9s9s9s$ttt  !)t{2Ky} = =t|/I =TY =""S"111-1]]__*dl ((// )*D&w!!*jT-B-B4:-N-N*jRVR^*jbfbi*j!kkk		T\\%!)T\BBQkQ		9> aII$)++0E"E$).[_J_!``II 9> 8[ 8OO%%%&&777 	A$"77DO$%D!y~ 2#$t'<#<IYAY\]A]"^1549>D;PSb;b1c1ccdi.%%''',0J)		Udk11		1222s### rzz/3qyy$))d!C*>1===*1-	y QJESi	V )++ 55g!11A5ggGVZNgggghhh7??y $!!###~...:&&&&&sJ   1.H++H/2H/V$C/UVU#	#V&U#	'4VE[('[(r   c                \    t          | j        | j        j        | j        | j        |          S )zJCalculate optimal batch size based on model and device memory constraints.)r   r  r  rl   max_num_obj)r   r   rX   r  r  rm   )r   r  s     rL   r  zBaseTrainer.auto_batch:  s2    %*)//#
 
 
 	
rN   Fc                   d\  }}| j         j        dk    rJt          j                                        }|r)t          d                                          j        dz  S nY| j         j        dk    rIt          j        	                                }|r)t          j        
                    | j                   j        }|r|dk    r||z  ndn|dz  S )zJGet accelerator memory utilization in GB or as a fraction of total memory.)r   r   rQ   psutilrO   rP   r   i   @)rZ   rp   rx   rQ   driver_allocated_memory
__import__virtual_memorypercentr<   memory_reservedget_device_propertiestotal_memory)r   fractionmemoryr'  s       rL   rj  zBaseTrainer._get_memoryD  s    ;u$$Y6688F K!(++::<<DsJJK[&&Z//11F S
88EER9AWEAII%11QVWrN   r>  float | Nonec                Z   |r5d|cxk    rdk    sn J d            |                      d          |k    rdS t          j                     | j        j        dk    r t
          j                                         dS | j        j        dk    rdS t
          j                                         dS )	zIClear accelerator memory by calling garbage collector and emptying cache.r   r>   z"Threshold must be between 0 and 1.T)r  NrQ   rP   )	rj  gccollectrZ   rp   rx   rQ   empty_cacher<   )r   r>  s     rL   re  zBaseTrainer._clear_memoryQ  s     		&&&&Q&&&&&(L&&&..);;

;u$$I!!#####[&&FJ""$$$$$rN   c                    ddl }	 |                    | j        d                              d          S # t          $ r i cY S w xY w)z0Read results.csv into a dictionary using polars.r   N)infer_schema_lengthF)	as_series)polarsread_csvr   to_dictr   )r   pls     rL   read_results_csvzBaseTrainer.read_results_csv_  sa    	;;txT;BBJJUZJ[[[ 	 	 	III	s   /6 AAc                   | j                                          | j                                         D ]X\  }t          t	          fd| j                            r.t          |t          j                  r|	                                 YdS )zSet model in training mode.c                    | v S r   rH   )fns    rL   r   z*BaseTrainer._model_train.<locals>.<lambda>m  s    AF rN   N)
r   r   named_modulesr   filterr   rs   r   BatchNorm2deval)r   mr  s     @rL   rS  zBaseTrainer._model_trainh  s    
J,,.. 	 	DAq6****D,CDDEE *UVXZXfJgJg 	 	rN   c                H   ddl }|                                }t          j        | j        | j        dt          t          | j        j                            	                                | j        j
        t          t          | j                                                            | j                                        t          | j                  i | j        d| j        i|                                 t)          j                                                    t.          t1          t2          j                  t2          j        t2          j        t2          j        dddd|           |                                }| j                             dd	           | j!        "                    |           | j        | j        k    r| j#        "                    |           | j$        dk    r;| j        | j$        z  dk    r*| j        d
| j         dz  "                    |           dS dS dS )z9Save model training checkpoints with additional metadata.r   Nr   )rootbranchcommitoriginz*AGPL-3.0 (https://ultralytics.com/license)zhttps://docs.ultralytics.com)rN  r   r   r   updatesr   r
  
train_argstrain_metricstrain_resultsdateversiongitlicensedocsTrC   rN  .pt)%ioBytesIOrx   rh   rN  r   r   r5   r   halfr  r.   r   
state_dictr
  rf   rX   r_   r   r  r   now	isoformatr   r[   r   r  r  r  r  getvaluerd   re   ri   write_bytesrj   rk   )r   r  bufferserialized_ckpts       rL   rs  zBaseTrainer.save_modelp  s   			 
 $ 1TX\ : :;;@@BB8+A(4>KdKdKfKfBgBghh+0022"49oo!NDL!NY4M!N!%!6!6!8!8 0022&MM!j!j!j	  H6) , /	
 	
 	
2 !//++ 		t444	o...,,I!!/222q  tzD4D'D'I'IY00000==oNNNNN ! 'I'IrN   c           	     p   	 t          | j        j                  }|                    d          s|                    d          r\d|v rXddl}ddlm} ddlm	} t          |
                     | || j        j                                                | j        _        | j        j        dk    rt          | j        j                  }nrt          | j        j                                      d	d
          d         dv s| j        j        dv r/t          | j        j                  }d|v r|d         | j        _        nL# t          $ r?}t!          t#          dt%          | j        j                   d|                     |d}~ww xY w| j        j        r t)          j        d           ddi|d<   d
|d<   |S )zGet train and validation datasets from data dictionary.

        Returns:
            (dict): A dictionary containing the training/validation/test dataset and category names.
        z.ndjsonzul://z
/datasets/r   N)convert_ndjson_to_yolo)r!   classifyr   r>   rB   >   ymlr9  >   r   posedetectsegment	yaml_filez	Dataset 'u   ' error ❌ z)Overriding class names with single class.itemr;  r:  )r[   rX   r   endswith
startswithasyncioultralytics.data.converterr  ultralytics.utils.checksr!   r   r   r   rsplitr   r   RuntimeErrorr   r   
single_clsr   r   )r   data_strr  r  r!   r   r   s          rL   r   zBaseTrainer.get_dataset  s   	f49>**H  ++ f0C0CG0L0L fQ]aiQiQiMMMMMM??????!$W[[1G1G

SWS\SaHbHb1c1c%d%d!e!e	 y~++(88TY^$$++C33B7?JJdin a O O )88$&&%)+%6DIN 	f 	f 	fv&\)DIN2K2K&\&\YZ&\&\]]^^dee	f9 	KCDDDKDMDJs   D;D> >
F:FFc                   t          | j        t          j        j                  rdS | j        d}}d}t          | j                                      d          rt          | j                  \  }}|j        }nBt          | j	        j
        t
          t          f          rt          | j	        j
                  \  }}|                     ||t          dk              | _        |S )zLoad, create, or download model for any task.

        Returns:
            (dict | None): Checkpoint to resume training from, or None if no checkpoint is loaded.
        Nr  rB   )r   r@   verbose)rs   r   rx   r   Moduler[   r  r   r9  rX   
pretrainedr	   	get_modelr   )r   r   r@   r  _s        rL   r   zBaseTrainer.setup_model  s     dj%(/22 	Fz4Wtz??##E** 	?+DJ77MGT,CC	,sDk:: 	?()=>>JGQ^^Wdbj^QQ
rN   c                   | j                             | j                   t          j        j                            | j                                        d           | j         	                    | j                   | j         
                                 | j                                         | j        r!| j        
                    | j                   dS dS )zVPerform a single step of the training optimizer with gradient clipping and EMA update.g      $@)max_normN)r
  unscale_r   rx   r   utilsclip_grad_norm_r   
parametersrR  r4  rL  r   r   s    rL   rf  zBaseTrainer.optimizer_step  s    T^,,,&&tz'<'<'>'>&NNN(((  """8 	(HOODJ'''''	( 	(rN   c                    |S )zTAllow custom preprocessing of model inputs and ground truths depending on task type.rH   )r   rl   s     rL   r_  zBaseTrainer.preprocess_batch  s    rN   c                   | j         rB| j        dk    r7| j         j                                         D ]}t          j        |d           |                     |           }|dS |                    d| j                                        	                                
                                           }| j        r| j        |k     r|| _        ||fS )aP  Run validation on val set using self.validator.

        Returns:
            (tuple): A tuple containing:
                - metrics (dict | None): Dictionary of validation metrics, or None if validation was skipped.
                - fitness (float | None): Fitness score for the validation, or None if validation was skipped.
        r>   r   r   N)NNr   )r   r|   buffersr   r  r^   rV   r   detachrP   numpyr   )r   r  r_   r   s       rL   rp  zBaseTrainer.validate  s     8 	.!++(,..00 . .v1-----..&&?:++i$)*:*:*<*<*@*@*B*B*H*H*J*J)JKK  	(D$5$?$? 'DrN   Tc                     t          d          )z>Get model and raise NotImplementedError for loading cfg files.z3This task trainer doesn't support loading cfg filesNotImplementedError)r   r   r@   r  s       rL   r  zBaseTrainer.get_model  s    !"WXXXrN   c                     t          d          )z>Raise NotImplementedError (must be implemented by subclasses).z1get_validator function not implemented in trainerr  r   s    rL   r  zBaseTrainer.get_validator      !"UVVVrN      r   c                     t          d          )zVRaise NotImplementedError (must return a `torch.utils.data.DataLoader` in subclasses).z2get_dataloader function not implemented in trainerr  )r   dataset_pathrm   r   r   s        rL   r   zBaseTrainer.get_dataloader  s    !"VWWWrN   c                     t          d          )zBuild dataset.z1build_dataset function not implemented in trainerr  )r   img_pathr   rl   s       rL   build_datasetzBaseTrainer.build_dataset  r  rN   c                    |d|indgS )zReturn a loss dict with labeled training loss items, or a list of loss names if loss_items is None.

        Notes:
            This is not needed for classification but necessary for segmentation & detection.
        Nr   rH   )r   r`  r   s      rL   r  zBaseTrainer.label_loss_items  s     (2'=
##F8KrN   c                4    | j         d         | j        _        dS )z/Set or update model parameters before training.r;  N)r   r   r;  r   s    rL   r   z BaseTrainer.set_model_attributes  s    9W-
rN   c                    dS )z-Build target tensors for training YOLO model.NrH   )r   r~  targetss      rL   build_targetszBaseTrainer.build_targets      rN   c                    dS )z-Return a string describing training progress. rH   r   s    rL   rY  zBaseTrainer.progress_string  s    rrN   c                    dS )z+Plot training samples during YOLO training.NrH   )r   rl   r|  s      rL   rl  z!BaseTrainer.plot_training_samples  r  rN   c                    dS )z$Plot training labels for YOLO model.NrH   r   s    rL   r  z BaseTrainer.plot_training_labels  r  rN   c                   t          |                                          t          |                                          }}t          |          dz   }t	          j                    | j        z
  }| j        j                            dd           | j        	                                rdn"d|z  ddg|R z  
                    d          d	z   }t          | j        d
d          5 }|                    |d|z  | j        dz   |g|R z  
                    d          z   d	z              ddd           dS # 1 swxY w Y   dS )z$Save training metrics to a CSV file.rU   TrC   r  z%s,rN  rF  rR   rA  azutf-8)encodingz%.6g,r>   N)rw   r  valuesrt   rF  rH  r   parentre   r   rstripopenwriterN  )r   r_   r  valsr  rK   sr  s           rL   rr  zBaseTrainer.save_metrics   s   ',,..))40@0@+A+AdLL1IKK$//dT:::(//##bBB%!)w6N6N6N*N)V)VWZ)[)[^b)b$(C'222 	WaGGA1
Q'AD'A'AAII#NNNQUUVVV	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	W 	Ws   &AD55D9<D9c                <    t          | j        | j                   dS )zPlot metrics from a CSV file.)r   on_plotN)r(   r   r  r   s    rL   ru  zBaseTrainer.plot_metrics*  s    $(DL999999rN   c                `    t          |          }|t          j                    d| j        |<   dS )z2Register plots (e.g. to be consumed in callbacks).)r   	timestampN)r	   rF  r`   )r   rc   r   paths       rL   r  zBaseTrainer.on_plot.  s-    Dzz$(ty{{CC
4rN   c                   | j                                         r| j         nd}t          t                    5  t          dv r\| j                                        rt          | j                  ni }|r+t          | j         d|                    d          i           ddd           n# 1 swxY w Y   |rt          j	        d| d           | j
        j        | j        j
        _        d| j        j
        _        |                     |          | _        | j                            d	d           |                     d
           dS dS )z;Perform final evaluation and validation for the YOLO model.NrA   r  )r  z
Validating z...F)r   r   r@  )rj   r   r3   r   r   ri   r2   rg   r   r   rX   r`   r^   r   r_   rV   r~   )r   r   r  s      rL   rt  zBaseTrainer.final_eval3  s   !Y--//9		T)*55 	e 	ew59Y5E5E5G5GOty111R e#DIRaIbIb7cdddd	e 	e 	e 	e 	e 	e 	e 	e 	e 	e 	e 	e 	e 	e 	e  	3K2222333(,	DN%*/DN'>>>66DLLY---122222	3 	3s   A&B))B-0B-c                H   | j         j        }|r	 t          |t          t          f          o t	          |                                          }t	          |rt          |          nt                                }t          |          d         j         }t          |d         t                    s6t	          |d                                                   s| j         j
        |d<   d}t          |          | _         t          |          x| j         _        | j         _        dD ]"}||v rt          | j         |||                    #|                    d          t          j        d|d          d           n"# t"          $ r}t%          d	          |d}~ww xY w|| _        dS )
zCCheck if resume checkpoint exists and update arguments accordingly.r   r   T)r  rl   rZ   rJ  rF   rk   rq   cacher   rF  r   r   r`   rF   Na  Custom Albumentations transforms were used in the original training run but are not being restored. To preserve custom augmentations when resuming, you need to pass the 'augmentations' parameter again to get expected results. Example: 
model.train(resume=True, augmentations=)zzResume checkpoint not found. Please pass a valid checkpoint to resume from, i.e. 'yolo train resume model=path/to/last.pt')rX   r   rs   r[   r	   r   r!   r'   r   r  r   r   r   setattrrg   r   r   r   FileNotFoundError)r   r   r   r   ri   	ckpt_argsr   r   s           rL   rY   zBaseTrainer.check_resumeD  s   ! -	,#FS$K88RT&\\=P=P=R=R&NJv...n>N>NOO ,D11!49	!)F"3T:: 74	RXHYCZCZCaCaCcCc 7(,	If%#I..	58YY>	$)"2 < <A I~~	1il;;; ==11=N` CLOB\` ` `      'E  
 s   E'E9 9
FFFc                0   |                     d           | j                            |d                    |                     d           | j                            |d                    | j        r|                     d          rtt          | j                  | _        | j        j                            |d                                                                                    |d         | j        _	        |                     dd          | _
        dS )z>Load optimizer, scaler, EMA, and best_fitness from checkpoint.r   Nr
  r   r  r   r+  )rg   r   load_state_dictr
  r   r+   r   floatr  r  r   )r   r  s     rL   _load_checkpoint_statez"BaseTrainer._load_checkpoint_statew  s    88K  ,N**4+<===88H)K''X7778 	/ 	/
++DHHL((e):):)<)<)G)G)I)IJJJ#IDH HH^S99rN   c                b   | j         duo| j                                          }| j        duot          j        | j                   }| j        o| j        dk    o
| j        dk    }t
          dv o|o|p|}|rdn|rdnd}t
          dk    r-t
          dk    r|ndg}t          j        |d           |d         }|sdS || j        k    s| j	        
                                st          j        | d	           dS | xj        d
z  c_        | j        dk    rt          d| j         d          t          j        | d| j         d           |                                  t!          | j	                  \  }}	|	d                                                                         }
t'          d |
                                D                       st          d| j	         d          t+          | j                                      |
           |                     |	           ~	~
|d
z
  | j        _        dS )zUDetect and recover from NaN/Inf loss and fitness collapse by loading last checkpoint.Nr   rA   zLoss NaN/InfzFitness NaN/InfzFitness collapserB   Fz- detected but can not recover from last.pt...r>   r.  z#Training failed: NaN persisted for z epochsz detected (attempt z/3), recovering from last.pt...r   c              3     K   | ]D}t          |t          j                  t          j        |                                          V  Ed S r   )rs   rx   Tensorisfiniteall)rJ   r  s     rL   r   z3BaseTrainer._handle_nan_recovery.<locals>.<genexpr>  sL      ffq*UVX]XdJeJef5>!$$((**ffffffrN   zCheckpoint z" is corrupted with NaN/Inf weightsT)r   r  r   rZ  r   r   r   rg  ro   ri   r   r   r   r   r  rS  r   r  r  r  r  r5   r   r  r  r   r  )r   rN  loss_nanfitness_nanfitness_collapse	corruptedreasonr  r  r  	ema_states              rL   rq  z BaseTrainer._handle_nan_recovery  s[   9D(E1C1C1E1E-El$.Pr{4<7P7P3P,\1BQ1F\4<[\K\GOVVk6UEU	#+ik1i1B1BWi2::+/199ii$?N&~q999&q)I 	5D$$$DI,<,<,>,>$NfSSSTTT5""a'""%))hTE_hhhiii&ppT5Opppqqq!$),,4K%%''2244	ffI4D4D4F4Ffffff 	\ZTYZZZ[[[TZ  00;;;##D))))$)AI!trN   c           	     B   || j         sdS |                    dd          dz   }|dk    s,J | j        j         d| j         d| j        j         d            t          j        d	| j        j         d
|dz    d| j         d           | j        |k     rCt          j        | j         d|d          d| j         d           | xj        |d         z  c_        |                     |           || _        || j        | j        j	        z
  k    r| 
                                 dS dS )z-Resume YOLO training from a given checkpoint.NrN  rB   r>   r   z training to zf epochs is finished, nothing to resume.
Start a new training without resuming, i.e. 'yolo train model=r   zResuming training z from epoch z to z total epochsz has been trained for z epochs. Fine-tuning for z more epochs.)r   rg   rX   r   rn   r   r   r  ro   rJ  rW  )r   r  ro   s      rL   r  zBaseTrainer.resume_training  sf   <t{<Fhhw++a/Qy ` `T[ ` `MQY_` ` `  	uuukTUouu[_[fuuuvvv;$$K:wwT']ww]a]hwww   KK4=(KK##D)))&$+	(>>??))+++++ @?rN   c                    t          | j        j        d          rd| j        j        _        t          | j        j        d          rHt	          j        d           | j        j                            t          | j                             dS dS )z5Update dataloaders to stop using mosaic augmentation.mosaicFrJ  zClosing dataloader mosaic)hypN)	r  r   r   r  r   r   rJ  r   rX   r   s    rL   rW  z$BaseTrainer._close_dataloader_mosaic  s    4$,h77 	5/4D%,4$,n== 	HK3444%22tDI2GGGGG	H 	HrN   autoMbP??h㈵>     j@c                   i i i i g}t          d t          j                                        D                       }|dk    rt	          j        t          d           d| j        j         d| j        j	         d           | j
                            dd          }	t          d	d
|	z   z  d          }
|dk    rdnd|
df\  }}}d| j        _        |dk    }t          |                                          D ]\  }}|                    d          D ]d\  }}|r| d| n|}|j        dk    r|r||d         |<   )d|v r||d         |<   9t%          ||          sd|v r||d         |<   Y||d         |<   e|sd |dd         D             }h d}d |D                                 |                                          }|dv rt)          ||d fd!          }nM|d"k    rt)          ||#          }n5|d$k    s|dk    rt)          ||d%&          }nt+          d'| d(| d)          t-          |d                   t-          |d                   t-          |d                   g}d*|d         i|d+di|d<   d*|d         i||d,d-|d<   d*|d         i|dd.d-|d<   d/\  }}|rt-          |d                   |d<   d*|d         i||d%d0d1|d<   ddl}|                    d2          g }|D ]}}|                    d*          }fd3|                                D             }fd4|                                D             }|                    d*|i|d5|dz  id*|i|g           ~|} t7          t8          |t;          t<          ||6                    |7          }t	          j        t          d           d8t?          |          j          d9| d:| d;|d          d<|d          d=| d>|d          d?           |S )@a  Construct an optimizer for the given model.

        Args:
            model (torch.nn.Module): The model for which to build an optimizer.
            name (str, optional): The name of the optimizer to use. If 'auto', the optimizer is selected based on the
                number of iterations.
            lr (float, optional): The learning rate for the optimizer.
            momentum (float, optional): The momentum factor for the optimizer.
            decay (float, optional): The weight decay for the optimizer.
            iterations (float, optional): The number of iterations, which determines the optimizer if name is 'auto'.

        Returns:
            (torch.optim.Optimizer): The constructed optimizer.
        c              3  (   K   | ]\  }}d |v 	|V  dS )NormNrH   )rJ   r   r  s      rL   r   z.BaseTrainer.build_optimizer.<locals>.<genexpr>  s*      DDA!1DDrN   r!  z
optimizer:z' 'optimizer=auto' found, ignoring 'lr0=z' and 'momentum=zJ' and determining best 'optimizer', 'lr0' and 'momentum' automatically... r:  
   {Gz?      i'  )r   r*  r#  AdamWr#  r+  r   F)recurser   rU   r.  r*  logit_scaler>   r   c                6    g | ]}|                                 S rH   )r  r   s     rL   rM   z/BaseTrainer.build_optimizer.<locals>.<listcomp>  s     ++++++rN   N>	   SGDAdamr!  r-  r   NAdamRAdamAdamaxRMSPropc                8    i | ]}|                                 |S rH   )lowerr   s     rL   r7  z/BaseTrainer.build_optimizer.<locals>.<dictcomp>  s"    111		1111rN   >   r2  r-  r3  r4  r5  g+?)r   betasr   r6  )r   r   r1  T)r   r   nesterovzOptimizer 'z,' not found in list of available optimizers zX. Request support for addition optimizers at https://github.com/ultralytics/ultralytics.paramsr)  weight)r   r)  bn)g?r   muon)r   use_muonr)  z(?=.*23)(?=.*cv3)|proto\.semsegc                D    g | ]\  }}                     |          |S rH   searchrJ   r   r  patterns      rL   rM   z/BaseTrainer.build_optimizer.<locals>.<listcomp>  s.    CCCDAq1B1BCaCCCrN   c                D    g | ]\  }}                     |          |S rH   rA  rC  s      rL   rM   z/BaseTrainer.build_optimizer.<locals>.<listcomp>  s.    GGGDAqW^^A5F5FGaGGGrN   r   )r>  sgd)r;  r   z(lr=z, momentum=z) with parameter groups z weight(decay=0.0), z weight(decay=z), z bias(decay=0.0))!rv   r   __dict__itemsr   r   r   rX   r   r   r   rg   r   r]  r5   r  r   ndimrs   r8  r  r  rt   rer   rV   rK  getattrr   r   r   rp   __name__)r   r   rc   r   r   r   r   gr=  r:  lr_fitr?  module_namemodule
param_nameparamfullname
optimizers
optim_args
num_paramsr>  rF  rJ  g_r   pp1p2r   rD  s                                @rL   r   zBaseTrainer.build_optimizer  su    RDD!2!2!4!4DDDDD6>>KL)) X X!%X X@D	@RX X X  
 tR((B9B/33F9Ce9K9K!5!5RY[acfQgD"h'*DI$7?#/#6#6#D#D#F#F 	+ 	+K%+%<%<U%<%K%K 
+ 
+!
E<GWk88J888Z:??x?%*AaDNNx''%*AaDNN++ +}/H/H%*AaDNN%*AaDNN
+  	,++QrrU+++Aeee
11j11155djjllCC@@@He+<3OOOJJYh777JJU]]dgoohFFFJJ%id i iPZ i i i  
 !A$iiQqTC!II6
!A$D*DmVDD!!A$]*]eT\]]]!!A$W*WcRVWWW!	c 	!IIJqMadqjq%UYjpqqqAaDIII jj!CDDGB T TEE(OOCCCCAGGIICCCGGGGAGGIIGGG		Hb<A<tR!V<<x>Qq>QRSSSSALGE4Ts)K)K)KLLTUVVV	%% y yY(@ y yb y yU] y y!}y y2<Q-y yOTy yYcdeYfy y y	
 	
 	
 rN   )r8   r9   )r   r[   )r   )Fr   )r>  r  )NNT)r  r   r   )r   N)Nr   )r!  r"  r#  r$  r%  ).rL  
__module____qualname____doc__r   r   r   r   r~   r   r   r   r   r!  r   r  rj  re  r  rS  rs  r   r   rf  r_  rp  r  r  r   r  r  r   r  rY  rl  r  rr  ru  r  rt  rY   r  rq  r  rW  r   rH   rN   rL   r7   r7   C   s       / /b '$RV W' W' W' W' W'r/ / / /+ + + +   
  6X X X

 

 

     4D6 D6 D6LN' N' N'`
 
 
 
X X X X% % % % %    'O 'O 'OR# # #J  &( ( (       (Y Y Y YW W WX X X XW W W WL L L L. . .    
    W W W: : :D D D D
3 3 3"1 1 1f
: 
: 
:  >, , ,(H H HS S S S S SrN   r7   )Nr]  
__future__r   r  r   r\   r   rF  rO  r   r   r   r   	functoolsr   pathlibr	   r  rZ  rx   r
   r   r   r   ultralyticsr   ultralytics.cfgr   r   ultralytics.data.utilsr   r   ultralytics.nn.tasksr   ultralytics.optimr   ultralytics.utilsr   r   r   r   r   r   r   r   r   r   r   ultralytics.utils.autobatchr   r  r    r!   r"   r#   r$   ultralytics.utils.distr%   r&   ultralytics.utils.filesr'   ultralytics.utils.plottingr(   ultralytics.utils.torch_utilsr)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r7   rH   rN   rL   <module>rl     s#    # " " " " " 				  				               ( ( ( ( ( ( ( (                  % % % % % %         # # # # # # 1 1 1 1 1 1 1 1 G G G G G G G G 0 0 0 0 0 0 # # # # # #                          ? > > > > > o o o o o o o o o o o o o o D D D D D D D D 2 2 2 2 2 2 3 3 3 3 3 3                             "N N N N N N N N N NrN   