
    jY|                     H   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZmZmZmZmZ d dlmZmZmZ d dlmZ d dlmZmZ defdZd	ed
eddfdZdededdfdZdededdfdZdedefdZdedefdZ defdZ!dZdede"de#e$e         e$e         e$e         f         fdZ%dede$e         ddfdZ&dedededefdZ'deded ede$e         fd!Z(d"ed#edefd$Z)d%edefd&Z*	 d[ded(ed)e$e         d ededed*ed+ed,ed-ed.ed/ed0edefd1Z+deddfd2Z,	 dZded3ed4ed5e-d6e-d7e-d8ed9ede"defd:Z.d;ededz  fd<Z/d\d8ed9ed>e-de$e         fd?Z0d@Z1dedefdAZ2dBedefdCZ3dedDede$e         fdEZ4de5dz  fdFZ6defdGZ7d\d%ed>e-de$e         fdHZ8d%ede$e         fdIZ9dJe$e         defdKZ:dedDed.edJe$e         def
dLZ;dedDede$e         fdMZ<dNedefdOZ=dNedefdPZ>dedefdQZ?dedDedefdRZ@dedDedSeddfdTZAd;edefdUZBd;edVedefdWZCd[d;edXedefdYZDdS )]    N)uuid4)BASE_DIR	MODEL_DIRTRAIN_RUNS_DIRloggerresolve_model_path)get_train_joblist_train_jobssave_train_job)submit_task)get_datasetlist_dataset_assetsreturnc                  n    dt          j        d          z   dz   t                      j        d d         z   S )Ntrain_z%Y%m%d_%H%M%S_   )timestrftimer   hex     N/home/longshao/multi-rider-rag/modules/training/services/train_task_service.py_new_train_job_idr      s/    dmO444s:UWW[!_LLr   log_pathmessagec                    t          j        t           j                            |           d           t	          | dd          5 }|                    dt          j        d           d| d	           d d d            d S # 1 swxY w Y   d S )
NTexist_okautf-8encoding[z%Y-%m-%d %H:%M:%Sz] 
)osmakedirspathdirnameopenwriter   r   )r   r   fhs      r   _append_logr-      s    K))D9999	hg	.	.	. H"
FT]#677FF7FFFGGGH H H H H H H H H H H H H H H H H Hs   /BB	Br(   contentc                     t          j        t           j                            |           d           t	          | dd          5 }|                    |           d d d            d S # 1 swxY w Y   d S )NTr   wr!   r"   )r&   r'   r(   r)   r*   r+   )r(   r.   r,   s      r   _write_textr1      s    K%%5555	dC'	*	*	* b
                 s   A))A-0A-payloadc                     t          j        t           j                            |           d           t	          | dd          5 }t          j        ||dd           d d d            d S # 1 swxY w Y   d S )	NTr   r0   r!   r"   F   )ensure_asciiindent)r&   r'   r(   r)   r*   jsondump)r(   r2   r,   s      r   _write_jsonr9   !   s    K%%5555	dC'	*	*	* =b	'2E!<<<<= = = = = = = = = = = = = = = = = =s   A,,A03A0jobc                 N    |                      |           t          |            | S N)updater   )r:   changess     r   _update_jobr?   '   s'    JJw3Jr   c                 h    t           j                            |                               dd          S )N\/)r&   r(   abspathreplace)r(   s    r   _normalize_pathrE   -   s&    7??4  ((s333r   c                  P   t           j                            t          ddd          t           j                            t           j                            t
          j                  d          g} | D ]'}|r#t           j                            |          r|c S (t          d          )Nz.venvScriptszyolo.exeuH   未找到 yolo.exe，请确认项目虚拟环境已安装 Ultralytics。)	r&   r(   joinr   r)   sys
executableisfileFileNotFoundError)
candidates	candidates     r   _resolve_yolo_executablerO   1   s    
Xw	:>>
RW__S^44jAAJ    	 		22 	
f
g
ggr   F
dataset_idconfirmed_onlyc                    t          | d          }d |D             }|rd |D             }|st          d          |                    d            |st          d          t          |          d	k    r|||fS t	          d	t          t          |          d	z
  t          t          t          |          d
z                                          }|d |         }||d          }|s|d d         }|dd          }|||fS )Ni  limitc                 <    g | ]}|                     d           |S )
is_labeledget.0items     r   
<listcomp>z)_split_labeled_assets.<locals>.<listcomp>>   s)    AAA$((<*@*@AtAAAr   c                 b    g | ],}t          |                    d           pd          dk    *|-S )review_statuspending	confirmed)strrX   rY   s     r   r\   z)_split_labeled_assets.<locals>.<listcomp>@   s=    jjjDs488O3L3L3YPY/Z/Z^i/i/i4/i/i/ir   uf   当前数据集还没有已确认样本，无法按“仅使用已确认样本”启动训练任务。c                     |                      d          p|                      d          pd                                |                      d          pdfS )Norigin_namefilename id)rX   lower)r[   s    r   <lambda>z'_split_labeled_assets.<locals>.<lambda>C   sP    DHH]$;$;$Ytxx
?S?S$YWY#`#`#b#bdhdldlmqdrdrdxvx"y r   )keyu9   当前数据集没有可用于训练的已标注图片。   g?)r   
ValueErrorsortlenmaxminintround)rP   rQ   assetslabeled	val_count
val_assetstrain_assetss          r   _split_labeled_assetsrx   <   s?    4888FAAAAAG GjjGjjj 	G  F  G  G  GLLyyLzzz VTUUU
7||qw''As3w<<!+Ss7||c7I1J1J-K-KLLMMI)$J9::&L "ss|RSS\
<++r   itemsc                 6   g }|D ]\}|                     d          pd}|rAt          j                            |          r"|                    t          |                     ]|st          d          t          | d                    |          dz              d S )N	file_pathre   u*   训练划分中没有有效图片路径。r%   )	rX   r&   r(   rK   appendrE   rl   r1   rH   )r(   ry   linesr[   r{   s        r   _write_split_filer~   S   s    E 5 5HH[))/R	 	5	22 	5LL33444 GEFFFdii&&-.....r   datasettrain_txt_pathval_txt_pathc           
      z   dt          |                     d          pd          z   dt          |          z   dt          |          z   dg}t          |                     d          pg           D ]4\  }}|                    d| d	t	          j        |d
                      5d                    |          dz   S )Nzpath: root_dirre   ztrain: zval: znames:class_namesz  z: F)r5   r%   )rE   rX   	enumerater|   r7   dumpsrH   )r   r   r   r}   indexnames         r   _build_dataset_yamlr   ^   s    ?7;;z#:#:#@bAAAON333/,///	E !]!;!;!ArBB K KtI%II4:d#G#G#GIIJJJJ99Ud""r   yolo_exedataset_yaml_pathc                     | ddd|d          d| d|d          d|d	          d
|d          dt          t                     d|d          ddddgS )Ndetecttrainzmodel=base_model_pathzdata=zepochs=epochszimgsz=imgszzbatch=
batch_sizezproject=zname=rf   zexist_ok=Truez	workers=0zcache=Falsezverbose=True)rE   r   )r   r:   r   s      r   _build_train_commandr   j   s    )&'))#!##!#h-!!W$\"$$4?>2244D	 r   run_dirartifact_dirc                     t          j        |d           t           j                            | dd          t           j                            | dd          t           j                            | d          t           j                            | d          t           j                            | d          t           j                            | d	          t           j                            | d
          t           j                            | d          t           j                            | d          t           j                            | d          t           j                            | d          t           j                            | d          t           j                            | d          t           j                            | d          d}i }|                                D ]\  }}t           j                            |          s%t           j                            |t           j                            |                    }t          j        ||           t           j        	                    |          ||<   |S )NTr   weightsbest.ptzlast.ptresults.csv	args.yamlresults.pngconfusion_matrix.pngconfusion_matrix_normalized.pngBoxF1_curve.pngBoxPR_curve.pngBoxP_curve.pngBoxR_curve.pngtrain_batch0.jpgval_batch0_labels.jpgval_batch0_pred.jpg)best_weightslast_weightsresults_csv	args_yamlresults_plotconfusion_matrixconfusion_matrix_normalizedbox_f1_curvebox_pr_curvebox_p_curvebox_r_curvetrain_batch0val_batch0_labelsval_batch0_pred)
r&   r'   r(   rH   ry   rK   basenameshutilcopy2rC   )r   r   mappingcopiedri   sourcetargets          r   _copy_artifactsr   }   s   Kt,,,,WiCCWiCCw||G];;W\\';77Wm<<GLL2HII')w||G=^'_'_W.?@@W.?@@w||G-=>>w||G-=>>W.@AAW\\'3JKK7<<1FGG G  F}} . .Vw~~f%% 	lBG,<,<V,D,DEEVV$$$goof--sMr   results_csv_pathc                 b   | rt           j                            |           si S 	 t          | ddd          5 }t	          t          j        |                    }d d d            n# 1 swxY w Y   |si S |d         }i }dD ]}||v r||         dvr||         ||<   |S # t          $ r i cY S w xY w)Nrr!   re   r#   newlinerk   )epochtrain/box_losstrain/cls_lossmetrics/precision(B)metrics/recall(B)metrics/mAP50(B)metrics/mAP50-95(B)re   Nr&   r(   rK   r*   listcsv
DictReader	Exception)r   r,   rowslastsummaryri   s         r   _read_results_summaryr      s&    27>>2B#C#C 	"C'2FFF 	,"r**++D	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	IBx Y 	) 	)Cd{{tCy
::#Cy   			s:   B "A&B &A**B -A*.B 5)B B.-B.re   request_payloadcommandrequest_pathcommand_path
notes_pathartifact_pathsr   statuserrorc                     ||||                      dd          |                      dd          |||||||                      dd          d	|	|
d}|r||d<   |S )Nr   re   r   r   )	r   r   dataset_yamltrain_split	val_splitrequest_jsoncommand_txtnotes_mdr   )r:   r   r   pathsr   r   rW   )r:   r   r   r   r   r   r   r   r   r   r   r   r   r2   s                 r   _manifest_payloadr      s      wwy"--GGNB77-)%('"
B//
 
 
 ! G$  ! Nr   c                 2   d }t           j                            | d         d          }t           j                            | d         d          }t           j                            | d         d          }t           j                            | d         d          }t           j                            | d         d          }t           j                            |d          }t           j                            |d          }i }	g }
i }	 t          t	          j                              }t          | d	|d
           t          | d         d           t          | d                   }t          | d         t          | 
                    d                              \  }}}t          j        | d         d           t          j        | d         d           t          ||           t          ||           t          | d         dt          |           dt          |                      t          |t          |||                     t          | d         d           i d| d         d| d         d| d         d| d         d| d         d| d         d| d         d| d         d| d         dt          | 
                    d                    d |
                    d d!          d"|
                    d"d!          d#|
                    d#d!          d$|
                    d$          pg d%| d%         d&t          |          t          |          t          |          z   t          |          t          |          d'}	t!          ||	           t          | d         d(           t#                      }t%          || |          }
t          |d)                    d* |
D                       d+z              t          | d         d,           d+                    d-d.d/d.d0d1d2d3d4d5d6d.d7d8| d          d9| 
                    d          rd:nd; d<t          |           d=t          |           d>d?                    |
                    d$          pg           pd@ g          }t          ||d+z              t          | dAB           t          | d         dC           t           j                                        }|                    dDdE           t,          t.          j        t.          j        ddEdF|dG}t           j        dHk    rt7          t.          dId!          |dJ<   t/          j        |
fi |}	 |j        |j                                        nd.}|r)t          | d         |                                           n|                                 nc|!                                }|d!k    rtE          dK|           tG          | d         | d                   }tI          |
                    dL          p%t           j                            | d         dM                    }t!          | dN         tK          | |	|
||||||||dO                     dP}|
                    dQ          r|dRz  }t          | dOt          t	          j                              |S           t          | d         dT           d S # tL          $ r}tO          j(        dU| 
                    d          |           t          | dVt          t	          j                              tS          |          pdWS           	 t          | d         dXtS          |          pdYz              n# tL          $ r Y nw xY w	 t!          | dN         tK          | |	p>| 
                    d          | 
                    d          | 
                    d          dZ|
|||||||i dVtS          |          [                     n# tL          $ r Y nw xY w|:|                                 ,	 |*                                 n# tL          $ r Y nw xY wY d }~d S Y d }~d S Y d }~d S Y d }~d S d }~ww xY w)\Nr   zdataset.yamlztrain_request.jsonztrain_command.txtzNEXT_STEPS.mdsplitsz	train.txtzval.txtrunningu'   正在准备训练数据与运行参数)r   start_tsr   r   zTraining job started.rP   rQ   rQ   Tr   r   zPrepared split files: train=z, val=zdataset.yaml written.job_idrf   dataset_name
base_modelr   
preset_keyr   r   r   image_countr   labeled_countreviewed_countr   
created_tssplit)total_assetslabeled_assetsrw   rv   ztrain_request.json written. c              3   .   K   | ]}d |v rd| dn|V  dS )r   "Nr   )rZ   parts     r   	<genexpr>z!_run_train_job.<locals>.<genexpr>  s7      *d*dTX#++;t;;;;4*d*d*d*d*d*dr   r%   ztrain_command.txt written.z# Train Runre   z6This run executes a real Ultralytics training command.zPrepared files:z- dataset.yamlz- splits/train.txtz- splits/val.txtz- train_request.jsonz- train_command.txtz- train.logzDataset:z- name: z- confirmed only: yesnoz- train images: z- val images: z- classes: z, --u   正在执行 Ultralytics 训练)r   z'Launching Ultralytics training process.PYTHONIOENCODINGr!   rD   )cwdstdoutstderrtextr#   errorsenvntCREATE_NO_WINDOWcreationflagsz&Ultralytics training exited with code r   r   manifest_pathdoneu   训练完成r   u   ，best.pt 已生成)r   end_tsr   zTraining finished successfully.z!failed to run training for %s: %sr   u   训练任务失败zERROR: zunknown error)r   rP   r   )r   )+r&   r(   rH   rq   r   r?   r-   r   rx   boolrX   r'   r~   rn   r1   r   r9   rO   r   environcopy
setdefaultr   
subprocessPIPESTDOUTr   getattrPopenr   readlinerstrippollwaitRuntimeErrorr   r   r   r   r   	exceptionra   kill)r:   processr   r   r   r   	split_dirr   r   r   r   r   nowr   
all_assetsrw   rv   r   notesr   popen_kwargslinereturn_coder   r   excs                             r   _run_train_jobr      sA	   GS^^DD7<<I0DEEL7<<I0CDDLc)no>>JS^X66IW\\)[99N7<<	955LOGNe$)++C	CAjkkkkC
O%<===c,/00/D(8 9 9::0
 0
 0
,
L* 	C	NT2222
C'$7777.,777,
333C
O%nCDUDU%n%n]`ak]l]l%n%nooo%':7NT`'a'abbbC
O%<===
c$i
#l+
 C/
 #l+	

 s#45
 #l+
 c(m
 S\
 #l+
 d377+;#<#<==
 7;;}a88
 W[[!<<
 gkk*:A>>
 7;;}55;
 #l+
   #J"%l"3"3c*oo"E #L 1 1!*oo	 !
. 	L/222C
O%BCCC+--&x6GHHL#((*d*d\c*d*d*d"d"dgk"klllC
O%ABBB		H! $"&%03~.00Scgg6F.G.G%QUUTSS63|#4#4662Z22SdiiM(B(B(HbIIQTSS%
 
, 	J---C!BCCCCC
O%NOOOjoo)7333 o '
 
 7d??,3J@RTU,V,VL)"7;;l;;	070J7>**,,,PRD C
OT[[]];;;;+	 llnn!UUUVVV(Y^9LMM'(:(:=(I(I(xRW\\Z]^gZhjwMxMxyy ! 	
 	
 	
$ !n-- 	.--GCs49;;/?/?QQQQC
O%FGGGGG $ $ $<cggdmmSQQQCDIKK0@0@#c((JjVjkkkk	Jc#hh6Q/)RSSSS 	 	 	D		O$!# "%''$--&)ggl&;&;(+(?(?( (
 %"   "c((#     ,  	 	 	D	7<<>>#9    #9#9#9#9#9#9I$s   W7\ bA)b6(^b
^,)b+^,,b0A<`-,b-
`:7b9`::ba)(b)
a63b5a66bbr   r   r   r   r   	owner_keyowner_ipc	                    t          |           }	|	st          d          |	                    d          sg st          d          t	          |	                    d          pd          dk    rt          d          t	          |	                    d          pd          dk    rt          d          t          |          }
t          j                            |
          st          d	          t                       t          | t          |          
           t                      }t          j                            t          |          }t          j                            |d          }t          j                            |d          }t          j                            |d          }i d|d|	d         d|	                    d          p|	d         dddddt          j                            |          d|
d|dt!          dt	          |                    dt!          dt	          |                    dt!          dt	          |                    dt          |          d|d |d!|d"|d#t	          t#          j                              d d ||d$}t%          |           t'          d%d&|i|||'           |S )(Nzdataset not foundr   u9   当前数据集还没有类别，无法创建训练任务r   r   u9   当前数据集还没有图片，无法创建训练任务r   uH   当前数据集还没有标注，至少先完成 1 张图片的框标注u   训练底模不存在r   	artifactsz	train.logztrain_manifest.jsonrf   rP   r   r   r   queuedr   u*   训练任务已创建，等待启动训练r   r   r   r   rj   r   @   r   rQ   r   r   r  r   r   )r   r  r!  r"  r   r   )r!  r"  task_id)r   LookupErrorrX   rl   rq   r   r&   r(   rK   rO   rx   r  r   rH   r   r   ro   r   r   r   )rP   r   r   r   r   r   r!  r"  rQ   r   r   r   r   r   r   r  r:   s                    r   start_train_jobr)    s    *%%G /-...KK&& V" VTUUU
7;;}%%*++q00TUUU
7;;'',1--22cddd(44O7>>/** 20111*T.5I5IJJJJ  Fgll>622G7<<55Lw||G[11HGLL*?@@Mfgdm 	F++<wt} 	(	
 	? 	bg&&z22 	? 	j 	#aV%% 	RU$$ 	c!S__-- 	$~.. 	7 	H 	  	!" 	c$)++&&#$ +  C. 3	6    Jr   r   c                      t          |           S r<   )r	   )r   s    r   get_train_job_snapshotr+    s       r      rT   c                 &    t          | ||          S )NrS   )r
   )r!  r"  rT   s      r   list_train_job_snapshotsr.    s    9he<<<<r   )
)r   zTraining Resultsr   )r   zConfusion Matrixr   )r   zConfusion Matrix (Normalized)r   )r   zBox F1 Curver   )r   zBox PR Curver   )r   zBox Precision Curver   )r   zBox Recall Curver   )r   zTrain Batch Previewr   )r   zValidation Labelsr   )r   zValidation Predictionsr   c                    | rt           j                            |           si S 	 t          | dd          5 }t	          j        |          }d d d            n# 1 swxY w Y   t          |t                    r|S n# t          $ r i cY S w xY wi S )Nr   r!   r"   )	r&   r(   rK   r*   r7   load
isinstancedictr   )r(   r,   r2   s      r   _load_json_filer3    s     rw~~d++ 	$g... 	$"immG	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$gt$$ 	N	   			Is4   A; AA; AA; A A; ;B
	B
r   c                      | D ]D}|r@t           j                            |          r!t           j                            |          c S EdS )Nre   )r&   r(   rK   rC   )r   r(   s     r   _first_existing_pathr5    sN     ) ) 	)BGNN4(( 	)7??4(((((2r   manifestc                    |                      dd          |                      dd          |                     d          pi                      d          pd|                     d          pi                      d          pdg}t                      }g }|D ]r}|st          j                            |          }||v st          j                            |          sH|                    |           |                    |           s|S )Nr   re   r   r   )rX   setr&   r(   rC   isdiraddr|   )r:   r6  rM   seendirsrN   
normalizeds          r   _collect_search_dirsr>    s   ##	2
,,w


%2	*	*>	:	:	@b
,,w


%2	*	*9	5	5	;	J 55DD    	 	W__Y//
RW]]:%>%>JKr   c                 P    | dv rd S 	 t          |           S # t          $ r Y d S w xY w)Nr   )floatr   )values    r   _safe_floatrB    sE    
tU||   tts    
%%c                 0    t          |           }|dS |dS )Nr   z.4f)rB  )rA  numerics     r   _format_metric_valuerE    s%    %  Gtr   c                    | rt           j                            |           sg S 	 t          | ddd          5 }t	          t          j        |                    }d d d            n# 1 swxY w Y   n# t          $ r g cY S w xY w|sg S |t          dt          |pd                     d          }g }|D ]}|
                    |                    d          pdt          |                    d	                    t          |                    d
                    t          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    d           |S )Nr   r!   re   r   rj   r,  r   r   r   r   r   r   r   r   r   )r   r   train_box_losstrain_cls_loss	precisionrecallmAP50mAP50_95)r&   r(   rK   r*   r   r   r   r   ro   rq   r|   rX   rE  )r   rT   r,   r   recent_rowsry   rows          r   _read_results_historyrO    s    27>>2B#C#C 	"C'2FFF 	,"r**++D	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,   			 	As5;B//000223KE 
 
))1T,SWWV__=="6sww?O7P7P"Q"Q"6sww?O7P7P"Q"Q1#'':P2Q2QRR.sww7J/K/KLL-cgg6H.I.IJJ09N1O1OPP	 		
 	
 	
 	
 Ls4   A2 "A&A2 &A**A2 -A*.A2 2B Bc                 
   | rt           j                            |           sg S 	 t          | ddd          5 }t	          t          j        |                    cd d d            S # 1 swxY w Y   d S # t          $ r g cY S w xY w)Nr   r!   re   r   r   )r   r,   s     r   _read_results_rowsrQ  5  s     27>>2B#C#C 	"C'2FFF 	,"r**++	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	, 	,   			s4   A3 !A&A3 &A**A3 -A*.A3 3BBr   c           	         d }d }| D ]3}t          |                    d                    }|'|||k    r|}|}4|si S |                    d          pdt          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    dS )Nr   r   r   r   r   r   )r   rI  rJ  rK  rL  )rB  rX   rE  )r   best_row
best_scorerN  scores        r   _best_epoch_snapshotrV  ?  s    HJ  CGG$9::;;=!3!3JH 	g&&.$)(,,7M*N*NOO&x||4G'H'HII%hll3E&F&FGG(6K)L)LMM  r   c                 f   |                     d          pi }t          |                     d          pd          }t          |                     d          pi                      d          pd          }t          |                     d          pi                      d          pd          }t          |                     d                    }t          |                     d                    }	t          |                     d	                    }
t          |                     d
                    }t          |          }d}d}d}||dk    r|
|
dk    rd}d}d}n||dk    r|
|
dk    rd}d}d}g }|dk     r|                    d           n|dk     r|                    d           |dk     r|                    d           |I|	Gt          ||	z
            dk    r1||	k    r|                    d           n|                    d            | |
|
|z
  d!k    r|                    d"           t          |                      d#          pd$          d%k    r|                    d&           |s|                    d'           d(}d)}|dk    rd}d*}n
|dk    rd+}d,}||||||||||d-
S ).Nr:   r   r   r   rw   rv   r   r   r   r   needs_improvementu   建议继续优化u]   当前模型可以继续迭代，建议先补样本并复核标注后再用于正式识别。g      ?g?goodu   效果较好uf   当前模型已经具备较好的识别能力，可优先使用 best.pt 做演示或小范围试用。gffffff?g?usableu   可以演示uh   当前模型已具备演示价值，建议优先使用 best.pt，并继续补充样本提升稳定性。   uS   当前标注样本量偏少，建议至少补到 200 张以上，再继续训练。i  uY   当前样本量可用于演示，但若要更稳定，建议继续补到 500 张左右。r,  uN   验证集样本较少，当前指标波动会偏大，建议补充验证集。g333333?ui   查准率高于查全率较多，漏检偏多，可增加复杂场景样本并适当放宽阈值测试。uZ   查全率高于查准率较多，误检偏多，建议补充负样本并复核标注框。g      ?up   mAP50 与 mAP50-95 差距较大，说明定位还不够稳定，建议继续收紧标注框并补充难样本。r   re   r  uY   发布或测试时建议优先使用 best.pt，不建议直接使用最后一轮权重。u`   当前结果较平稳，可先用 best.pt 做演示，再根据误检和漏检继续补样本。lowu   样本量偏少u   样本量较充足mediumu   样本量基本够演示)
r   titler   sample_levelsample_textsample_counttrain_countru   
best_epochrecommendations)rX   rq   rB  rV  r|   absra   )r:   r6  r   r   manifest_jobr   rb  ru   rI  rJ  map50map50_95rc  r   r^  summary_textrd  r_  r`  s                      r   _build_report_assessmentrj  T  s^   <<&&,"L((99>Q??M((117R<<^LLRQRSSKl&&w//52::<HHNQOOIGKK(>??@@I%899::F$67788E7;;'<==>>H%d++J F ErLD 0 0U5F5TX==		(d"2"2u7HUVZ]] BOstuuuu			z{{{2~~oppp!3I<N8O8OSW8W8Wv""  $O  P  P  P  P""#  A  A  A 1ux7GD6P6P   R  	S  	S  	S
3778"##v--z{{{ C   B  	C  	C  	CL#K*	#		0 $"%" *  r   c                     t          | |          }g }t          D ]8\  }}t          fd|D              }|s|                    |||d           9|S )Nc                 P    g | ]"}t           j                            |          #S r   )r&   r(   rH   )rZ   base_dirrd   s     r   r\   z*_collect_report_images.<locals>.<listcomp>  s*    ,j,j,jRZRW\\(H-M-M,j,j,jr   )ri   r^  rd   r(   )r>  _REPORT_IMAGE_SPECSr5  r|   )r:   r6  search_dirsry   ri   r^  source_pathrd   s          @r   _collect_report_imagesrq    s    &sH55KE 3 
 
UH*,j,j,j,j^i,j,j,jk 	$#	 	
 	
 	
 	
 Lr   rA  c                 @   t           j                            t          | pd                                                    }|sdS t          j        dd|                              d          }|sdS |                                                    d          s|dz  }|S )Nre   z[^A-Za-z0-9._-]+r   ._-.pt)	r&   r(   r   ra   stripresubrg   endswith)rA  rawsafes      r   _sanitize_model_namer{    s    
'

3u{++1133
4
4C r6%sC0066u==D r::<<  '' Kr   c                     t          j        ddt          | pd                                                                        d          }|pdS )Nz[^A-Za-z0-9_-]+r   re   rs  trained_model)rv  rw  ra   ru  )rA  rz  s     r   _slugify_namer~    sG    6$c3u{+;+;+A+A+C+CDDJJ5QQD"?"r   c                     t          |                     d          p|                     d          pd          }| d|                     d          pd dS )Nr   rP   r   r   rf   bestrt  )r~  rX   )r:   r   s     r   _suggest_publish_namer    sX     !8!8!^CGGL<Q<Q!^U^__L99SWWT]]4f9999r   c           
      D   |                     d          pi }t          |                     dd          t          j                            |                      dd          d          t          j                            |                      dd          dd                    S )Nr   r   re   r   r   r   r   )rX   r5  r&   r(   rH   )r:   r6  r   s      r   _resolve_best_weights_pathr    s    LL!!'RE		."%%
SWW^R00)<<
SWWY++Y	BB  r   publish_infoc                    |                      d          sd S t          |pi           }t          |                     d          pg           }|                    d|           |d d         |d<   t	          | d         |           d S )Nr  published_modelsr   
   )rX   r2  r   insertr9   )r:   r6  r  r2   r  s        r   _store_manifest_publish_infor    s    77?## 8>r""GGKK(:;;ArBBA|,,,"23B3"7GO$g.....r   c                    t          |           }|st          d          t          |                    dd                    }|                    d          pi }t	          |                    dd          t
          j                            |                    dd          d          t
          j                            |                    dd          d                    }|                    d	          pt          |          }t          ||          }t          |d
          }t          |          }t          ||||          }	|||t          |                    d                    t          |                    d                    t          |                    d                    t          |                    d                    dt          |                    d                    t          |                    d                    d||	||                    d          p|                    d          pd|                    d          p|                    d          pd|t          ||          t	          |                    dd          t
          j                            |                    dd          d          t
          j                            |                    dd          d                    |                    d          p|                    d          pddt          |          t!          |                    d          pg           dd
}
|
S )Njob not foundr  re   r   r   r   r   r   r   r,  rS   r   r   r   r   )rI  rJ  rK  rL  r   r   )box_losscls_lossr   r   r   )r   r   r   r   r   r   r  )suggested_namer  )
r:   r6  r   metricslosseshistory
assessmentimagesr   publish)r	   r(  r3  rX   r5  r&   r(   rH   r   rq  rO  rQ  rj  rE  r  r  r   )r   r:   r6  r   r   r   r  r  r   r  reports              r   build_train_job_reportr    s   


C +/***sww;;<<HLL!!'RE+		-$$
SWW^R00-@@
SWWY++];; 
 ll9%%P)>?O)P)PG#C22F#$4B???G.//D)#x$GGJ -gkk:P.Q.QRR*7;;7J+K+KLL)'++6H*I*IJJ,W[[9N-O-OPP	
 
 -W[[9I-J-JKK,W[[9I-J-JKK
 
  yy++Gswwy/A/AGR!IIn55V9P9PVTV+6sHEE-		+r**SWW^R88+FFSWWY33[AA 
 		*--J1D1DJ
 
 4C88 $X\\2D%E%E%K L L
 
;! !FD Mr   rd   c                 2   t          |           }|st          d          t          j                            t          |pd                                                    }|st          d          t          |	                    dd                    }t          ||          }|D ]b}t          j                            ||          }t          j                            |          r!t          j                            |          c S ct          d          )Nr  re   zartifact not foundr  )r	   r(  r&   r(   r   ra   ru  rL   r3  rX   r>  rH   rK   rC   )r   rd   r:   	safe_namer6  ro  rm  rN   s           r   find_train_job_artifact_pathr    s    


C +/***  X^!4!4!:!:!<!<==I 6 4555sww;;<<H&sH55K . .GLL955	7>>)$$ 	.7??9-----	.
0
1
11r   target_namec                 "   t          |           }|d         }t          |                    d          pd          dk    rt          d          |d                             d          pd}|rt          j                            |          st          d          t          |pd          }|p|d	         d
         }t          |          }|st          d          t	          j	        t          d           t          j                            t          |          }t          j                            |          t          j                            |          k    rt          j                            |          \  }	}
d}t          j                            |          rPt          j                            t          |	 d| |
           }|dz  }t          j                            |          Pt          j        ||           |                    d          |                    d          |                    d          |                    d          |pt          j                            |          ddgt%          t'          j                              ||                    d          pi t)          |                    d                    d}t          j                            |          d         dz   }t+          ||           t          j                            |          t          j                            |          |t%          t'          j                              d}t-          ||                    d          pi |           |S )Nr:   r   re   r  z training job is not finished yetr   r   zbest.pt not foundr  r  zinvalid target model nameTr   r4   r   rj   rf   rP   r   r   activeupload_inferencer   rQ   )source_job_idrP   r   r   display_name	lifecycleusagesr   source_best_weightsr   rQ   r   z
.meta.json)
model_name
model_pathmetadata_pathpublished_tsr6  )r  ra   rX   rl   r&   r(   rK   rL   r{  r'   r   rH   rC   splitextexistsr   r   r   rq   r   r  r9   r  )r   r  r  r:   rp  requested_namedesired_namer  target_path	base_nameextsuffixmetadatar  r  s                  r   publish_train_job_bestr    s   #F++F
-C
3778"##v--;<<</%%n55;K 5bgnn[99 5 3444)+*;<<N!HVI%67G%HL$\22I 64555K	D))))',,y)44K	w{##rw{'C'CCC)))44	3gnn[)) 	',,yY2N2N2N2N2NOOKaKF gnn[)) 	 Lk*** ggl++//ggl++#Drw'7'7'D'D%&$)++&&*::i((.Bsww'78899 H G$$[11!4|CMx((( g&&{33gook22&DIKK((	 L !fjj&<&<&BLQQQr   )F)re   )r,  )Er   r7   r&   rv  r   r  rI   r   uuidr   shared.config.configr   r   r   r   r   shared.db.sqliter	   r
   r   shared.task_queuer   )modules.training.services.dataset_servicer   r   ra   r   r-   r1   r2  r9   r?   rE   rO   r  tupler   rx   r~   r   r   r   r   r   r   rq   r)  r+  r.  rn  r3  r5  r>  r@  rB  rE  rO  rQ  rV  rj  rq  r{  r~  r  r  r  r  r  r  r   r   r   <module>r     s   



  				 				      



        ` ` ` ` ` ` ` ` ` ` ` ` ` ` K K K K K K K K K K ) ) ) ) ) ) V V V V V V V VM3 M M M MH# H H H H H Hc C D    =c =D =T = = = =T     4# 4# 4 4 4 4h# h h h h, ,c ,4 ,ERVW[R\^bcg^hjnosjtRtLu , , , ,./C /T
 /t / / / /	# 	#s 	## 	#RU 	# 	# 	# 	#3 T c dSVi    &S      8C D    > # #	## #Y# 	#
 # # # # # # # # # 
# # # #Lq q q q q qz !B BBB B 	B
 B B B B B 
B B B BJ!3 !4$; ! ! ! != = =s =3 =PTUYPZ = = = = 
# 
$ 
 
 
 
     d d tCy    (%$,    3     C  T$Z    : d    tDz d    *C$ C$ C CTRVZ C\` C C C CL  d    $	 	 	 	 	 	# # # # # #
:t : : : : :
D D S    /d /d /$ /SW / / / /43 44 4 4 4 4n2 2 2 2 2 2 2$3 33 3S 3$ 3 3 3 3 3 3r   