
    jH                        U d dl Z d dlZd dlZd dlZd dlZd dlmZmZmZ d dl	m	Z	 d dl
mZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZ d dlmZmZmZ d d	lm Z  d d
l!m"Z"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*Z* d dl+m,Z,m-Z-m.Z. i Z/e0e1e0f         e2d<    ej3                    Z4h dZ5de0de0fdZ6de1de0dz  fdZ7d+de1de1de8e0         fdZ9d+de1de1de1de:fdZ;de0ddfdZ<de1de:fdZ=de1ddfdZ>d,de?d e1de0fd!Z@de0de1fd"ZAd#e8eBe1e1f                  d$eCd%e?d&e?d'e1d(e1de1de1de0fd)ZDde0d#e8eBe1e1f                  d$eCd%e?d&e?d'e1d(e1ddfd*ZEdS )-    N)FIRST_COMPLETEDThreadPoolExecutorwait)datetime)OptionalSet)uuid4)Image)
BATCH_SIZECONF_THRESHIMGSZMAX_WORKERSMODEL_DEFAULT
OUTPUT_DIRlogger)get_joblist_active_jobssave_job)submit_task)add_result_bytescreate_result_storefinalize_result_store)_predict_batchdownload_image_with_status	get_model)job_matches_owner)filename_from_urlformat_timestampinfer_ext_from_bytesJOBS>   doneerrorcanceledinterruptedjobreturnc                 t    d |                                  D             }d|v rd |d         D             |d<   |S )Nc                 &    i | ]\  }}|d k    ||S )cancel .0keyvalues      H/home/longshao/multi-rider-rag/modules/detection/services/job_service.py
<dictcomp>z!_job_snapshot.<locals>.<dictcomp>4   s#    LLLzsEC8OOUOOO    	zip_partsc                 ,    g | ]}t          |          S r*   )dict)r,   parts     r/   
<listcomp>z!_job_snapshot.<locals>.<listcomp>6   s     N N Nd N N Nr1   )items)r%   snapshots     r/   _job_snapshotr9   3   sJ    LLSYY[[LLLHh N N8M N N NOr1   job_idc                     t           5  t                              |           }|t          |          cd d d            S 	 d d d            n# 1 swxY w Y   t	          |           S N)	JOBS_LOCKr    getr9   get_saved_job)r:   r%   s     r/   get_job_snapshotr@   :   s    	 & &hhv? %%& & & & & & & && & & & & & & & & & & & & & &    s   +AAA 	owner_keyowner_ipc                 @    t          | |dd          }d |D             S )N   oraclelimitjob_typec                    g | ]}|                     d           |                     d          |                     d          |                     d          |                     d          |                     dt                    dS )idstart_tstotal	processedstatus	model_key)rK   rL   rM   rN   rO   rP   )r>   r   )r,   r%   s     r/   r6   z%list_running_jobs.<locals>.<listcomp>D   s     
 
 
  ''$--
++WWW%%--ggh''m<<	
 	

 
 
r1   )list_saved_active_jobs)rB   rC   recordss      r/   list_running_jobsrS   B   s;    $YXVVVG
 
 
 
 
 
r1   c                 .   d }t           5  t                              |           }|t          |||          r|d                                          |                    d          t
          vrBd|d<   d|d<   |                    d          pt          t          j                              |d<   t          |          }d d d            n# 1 swxY w Y   |t          |           dS t          |           }|*|                    d          d	k    st          |||          sd
S |                    d          t
          v rdS d|d<   d|d<   |                    d          pt          t          j                              |d<   t          |           dS )Nr)   rO   r#   cancel requestedmessageend_tsTrI   rF   F)r=   r    r>   r   setTERMINAL_STATUSESinttimer9   r   r?   )r:   rB   rC   r8   r%   s        r/   request_cancelr\   Q   s   H	 * *hhv?0iJJ?Mwwx  (999 *H!3I # 1 1 ES5E5EH$S))H* * * * * * * * * * * * * * * t


C
{cggj))X55=NsT]_g=h=h5u
wwx---tCM'C	NGGH%%9TY[[)9)9CMSMMM4s   B5CCCr8   c                     	 t          |            d S # t          $ r4}t          j        d|                     d          |           Y d }~d S d }~ww xY w)Nzfailed to persist job %s: %srK   )r   	Exceptionr   	exceptionr>   )r8   excs     r/   _save_job_statera   o   ss    R R R R7d9K9KSQQQQQQQQQRs    
A)AAc                 N   t           5  t                              |           }|	 d d d            dS |d                                         rI|                    d          t          vrd|d<   |                    d          pd|d<   	 d d d            dS 	 d d d            n# 1 swxY w Y   t          |           }||                    d          dk    rdS t           5  t                              |           }|	 d d d            dS |d                                          |                    d          t          vrLd|d<   |                    d          pd|d<   |                    d          r|                    d          |d<   d d d            n# 1 swxY w Y   dS )	NFr)   rO   r#   rV   rU   TrW   )r=   r    r>   is_setrY   r?   rX   )r:   r%   	saved_jobs      r/   _sync_cancel_statere   v   st   	  hhv;        x=!! 	wwx  (999 *H!$!3!3!I7II       		               f%%IIMM(33zAAu	 	8 	8hhv;	8 	8 	8 	8 	8 	8 	8 	8 	H778$555&CM&]]955K9KC	N}}X&& 8 )h 7 7H	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 	8 4s0   B#AB##B'*B'!FBFF!Fc                     t          |            t          5  t                              |           }|	 d d d            d S t	          |          }d d d            n# 1 swxY w Y   t          |           d S r<   )re   r=   r    r>   r9   ra   )r:   r%   r8   s      r/   _persist_job_staterg      s    v	 & &hhv;& & & & & & & & !%%	& & & & & & & & & & & & & & &
 Hs   AAA!$A!queuedrM   rO   c                    i dddt                      j        ddddd|d|d| d	d
dd
dd
dd
dd
dt          t          j                              dd dddd dd g dddt          t
          t          d dt          ddS )NrI   rF   rK   source_namesource_typerO   rV   rM   rN   r   
downloadedkeptnotfoundfailedrL   rW   rB   rA   	zip_byteszip_path)r2   summary_text
result_dirresult_manifest_pathconf_thresh
batch_sizeimgszclassesclasses_rawrP   rC   )r	   hexrZ   r[   r   r   r   r   rM   rO   s     r/   _new_job_recordr|      s   Heggk 	x 	x	
 	& 	6 	 	Q 	a 	 	A 	! 	C	$$ 	$ 	R  	T!" 	D#$  "" "9   r1   c                    |                      d          |                      d          n't          d| d         | d         z
  | d         z
            }t          d|| d         z
            }|                      dt                    }d|                      d	t                     d
| d          d| d          d| d| d| d          d| d| d| d          d| d          dt	          |                      d                     dt	          |                      d                     g
}d                    |          dz   S )Nrl   r   rN   rn   ro   rm   ru   zmodel: rP   ztotal urls: rM   zprocessed: zdownloaded: z	kept (>= z): zdiscarded (< z404 not found: zother failed: z	started: rL   z
finished: rW   
)r>   maxr   r   r   join)r%   rl   	discarded	thresholdliness        r/   
_summarizer      sk    77<  , 	C$s:6XFGG 
 AzCK/00I{33I7#''+}5577%s7|%%(c+&((#z##/I//#f+//1	11i11+#j/++(X((;$SWWZ%8%899;;:%cggh&7&788::E 99Ud""r1   url_and_timesru   rv   rw   ry   rP   c                 z   t          ||dd          D ]'}t          |                    dd          ||           (t          t	          |           d          }	|	                    |||||||d           t          |	           t          d	|	d         | |||||d
|||	d                    t          |	          S )NrE   rF   rG   rK   rA   rh   r{   )rB   rC   rP   ru   rv   rw   ry   rV   	detection)r:   r   ru   rv   rw   ry   rP   )rB   rC   task_id)	rQ   r\   r>   r|   lenupdater   r   r9   )
r   ru   rv   rw   ry   rP   rB   rC   
active_jobr%   s
             r/   start_detection_jobr      s     -YU]^^^ F F
z~~dB//HEEEE
M 2 28
D
D
DCJJ  	 	 	 SMMM$i*&$&"	
 	
 D	    r1   c           
        +,- t          |                     d          pd                                          ++st          d          t	          |           }t          j                    |d<   t          t          |                    d          pd          t                              |d<   ||d<   ||d<   ||d	<   ||d
<   |p|                    d          pt          |d<   |                    d          pt          t          j                              |d<   t          5  |t          +<   d d d            n# 1 swxY w Y   t          +          rt          5  t          +         }|                    d          pt          t          j                              |d<   t          |          |d<   t!          |          }	d d d            n# 1 swxY w Y   t#          |	           d S t          5  t          +         }d|d<   d|d<   d d d            n# 1 swxY w Y   t%          +           d }
	 	 t'          |d                   }n.# t(          $ r!}t+          d|d          d|           |d }~ww xY wt-          +ddd          }
d }d }|d         dk    r#d |                    d          D             pd }nt1          |dd           }|r|rt3                      }t5          |t                    rd |                                D             }nd t9          |          D             }d |                    d          D             D ]}|                                r#|                    t          |                     9|                    |                                          }|"|                    t          |                     |r|}t          5  t          +         }|d         dk    r|n||d<   d d d            n# 1 swxY w Y   t%          +           dt@          f+fd,d t           dt           fd!}+,fd"}i -d#t           dtB          j"        f+-fd$}d}g }g }g } |            D ]\  }}} ,            r np	 tG          j$        tK          j&        |                    '                    d%          }nJ# t(          $ r= t          5  t          +         d&xx         d'z  cc<   d d d            n# 1 swxY w Y   Y w xY w|(                    |           |(                    ||f           |(                     ||                     t          |          |k     rtS          ||||||d         |          }t9          tU          ||                    D ]\  } \  \  }!}"}#t          5  t          +         d(xx         d'z  cc<   d d d            n# 1 swxY w Y   |#r|d'z  }|d)d*|! }$ |||                    }%|%+                    |$|"           tY          |
|$|"|!||          d+,           t          5  t          +         d-xx         d'z  cc<   d d d            n# 1 swxY w Y   |D ]'}	 |-                                 # t(          $ r Y $w xY w|.                                 |.                                 |.                                 t%          +            ,            r n|rtS          ||||||d         |          }t9          tU          ||                    D ]\  } \  \  }!}"}#t          5  t          +         d(xx         d'z  cc<   d d d            n# 1 swxY w Y   |#r|d'z  }|d)d*|! }$ |||                    }%|%+                    |$|"           tY          |
|$|"|!||          d+,           t          5  t          +         d-xx         d'z  cc<   d d d            n# 1 swxY w Y   |D ]'}	 |-                                 # t(          $ r Y $w xY w|.                                 |.                                 |.                                 t%          +           t          5  t          +         }t          t          j                              |d<   t          |          |d<   |d         }&d d d            n# 1 swxY w Y   g }'t_          -0                                          D ]l}(	 |(+                    d.|&           n# t(          $ r Y nw xY w|(j1        })	 |(-                                 n# t(          $ r Y nw xY w|'(                    |)           m|
te          |
          }*nd}*t          5  t          +         }d/ |'D             |d0<   t          |'          d'k    r|'d         nd |d1<   |
|
d2         nd|d2<   |*|d3<   |d         dk    r
d4|d<   d5|d<   t!          |          }	d d d            n# 1 swxY w Y   t#          |	           n# t(          $ r}tg          j4        d6|           t          5  t                              +          }|\d7|d<   t          |          |d<   t          t          j                              |d<   t          |          |d<   t!          |          }	nd }	d d d            n# 1 swxY w Y   |	t#          |	           Y d }~nd }~ww xY wt          5  t                              +          }|7|                    d          tj          v rt          6                    +d            d d d            d S # 1 swxY w Y   d S # t          5  t                              +          }|7|                    d          tj          v rt          6                    +d            d d d            w # 1 swxY w Y   w xY w)8NrK   rA   zmissing job idr)   rM   r   ru   rv   rw   ry   rP   rL   rW   rr   runningrO   rV   u   模型文件加载失败（u#   ），请重新部署模型文件: rF   generalc                 ^    g | ]*}|                                 |                                 +S r*   stripr,   items     r/   r6   z_run_job.<locals>.<listcomp>6  2    ^^^tQUQ[Q[Q]Q]^djjll^^^r1   ,namesc                 r    i | ]4\  }}t          |                                          t          |          5S r*   )strlowerrZ   r+   s      r/   r0   z_run_job.<locals>.<dictcomp><  s6    ]]]eE

 0 0 2 2CHH]]]r1   c                 X    i | ]'\  }}t          |                                          |(S r*   )r   r   )r,   indexr.   s      r/   r0   z_run_job.<locals>.<dictcomp>>  s0    ___leUE

 0 0 2 2E___r1   c                 ^    g | ]*}|                                 |                                 +S r*   r   r   s     r/   r6   z_run_job.<locals>.<listcomp>@  r   r1   bczjrx   r&   c                  "    t                     S r<   )re   )r:   s   r/   should_cancelz_run_job.<locals>.should_cancelO  s    %f---r1   time_strc                     	 | r*t          j        | d          }|                    d          S n# t          $ r Y nw xY wdS )Nz%Y-%m-%d %H:%M:%Sz%Y%m%dunknown)r   strptimestrftimer^   )r   dts     r/   time_bin_keyz_run_job.<locals>.time_bin_keyR  s`     1!*85HIIB;;x0001    9s   +/ 
<<c               3     K   t          t                    5 t                    } i t          dt          dz            }dt          t
          t
          f         dd ffd}t          |          D ]!}t          | d           }| n ||           "r             r	 d d d            d S t          t          
                                          t                    \  }}|D ]E}                    |          \  }}	 |                                \  }	}
}n# t          $ r d\  }	}
Y nw xY w|	Tt          5  t                    }|
d	k    r|d
xx         dz  cc<   n|dxx         dz  cc<   d d d            n# 1 swxY w Y   nt          5  t                    dxx         dz  cc<   d d d            n# 1 swxY w Y   t#          |          }t$          j                            |          \  }}|st+          |	          }||z   }||	|fV  t          | d           }| ||           Gd d d            d S # 1 swxY w Y   d S )N)max_workers      r   r&   c                 V    | \  }}                     t          |          }||f|<   d S r<   )submitr   )r   urltsfutureexecutor	in_flights       r/   
submit_onez3_run_job.<locals>.gen_downloads.<locals>.submit_onea  s4    "GC%__-GMMF),b	If%%%r1   )return_when)NNi  rn   ro   rl   )r   r   iterr   tupler   rangenextr   rX   keysr   popresultr^   r=   r    r   ospathsplitextr   )iteratormax_in_flightr   _r   r!   r   r   r   datacode_content_typecurrentnamerootextr   r   r:   r   r   s                   @@r/   gen_downloadsz_run_job.<locals>.gen_downloads[  s     #<<< /-..	 #A{Q 7 72U38_ 2 2 2 2 2 2 2 2
 }-- % %A$//D|Jt$$$$ -$} %/- /- /- /- /- /- /- /-( #3y~~'7'7#8#8oVVVGD!"& - -"+--"7"7R48>5D$( 4 4 4)3JD$$$4  <!* ; ;*.v,#'3;;$+J$7$7$71$<$7$7$7$7$+H$5$5$5$:$5$5$5; ; ; ; ; ; ; ; ; ; ; ; ; ; ; "+ @ @ $V\ : : :a ? : : :@ @ @ @ @ @ @ @ @ @ @ @ @ @ @#4S#9#9D(*(8(8(>(>ID##& 2&:4&@&@'+cz"&b.000#Hd33+&Jt,,,?   -!/- /- /- /- /- /- /- /- /- /- /- /- /- /- /- /- /- /-s   BI3AI	D"!I"D41I3D44I 5F5IFIF	IF<0I<G  IG A3IIIr-   c                                          |           }|Qt          j                            t           d|  d          }t          j        |dt
          j                  }|| <   |S )Nr   z.zipw)modecompression)r>   r   r   r   r   zipfileZipFileZIP_DEFLATED)r-   
zip_handler   r:   	zip_filess      r/   get_zip_for_keyz!_run_job.<locals>.get_zip_for_key  sh    "s++J!w||J60E0EC0E0E0EFF$_TI]^^^
!+	#r1   RGBro   r   rN   07dr   )origin_name	group_key)extrarm   zsummary.txtc                 R    g | ]$}|t           j                            |          d %S ))r   r   )r   r   basename)r,   r   s     r/   r6   z_run_job.<locals>.<listcomp>  s1    #i#i#iW[T27;K;KD;Q;Q$R$R#i#i#ir1   r2   rq   rs   rt   r!   	completedzjob failed: %sr"   )7r   r>   r   
ValueErrorr4   	threadingEventr   rZ   r   r   r[   r=   r    re   r   r9   ra   rg   r   r^   RuntimeErrorr   splitgetattrrX   
isinstancer7   	enumerateisdigitaddr   boolr   r   r
   openioBytesIOconvertappendr   zipwritestrr   closeclearlistvaluesfilenamer   r   r_   rY   r   ).r%   r   ru   rv   rw   ry   rP   runtime_jobr   r8   result_storemodelr`   allowed_classesprompt_classesr   indexesname_maptokenmappedr   r   r   sequenceimagespayloadsbinsr   image_bytesr   imagekeepsr   r   payloadkeepoutput_namezip_filerr   r2   r   r   manifest_pathr:   r   r   s.    `                                         @@@r/   _run_jobr     s    $"%%++--F +)***s))K%O--Ks;??7#;#;#@qAA3}CUCUVVK!,K *K K!,K(YKOOK,H,HYMK)ooj99MS=M=MK
	 # #"V# # # # # # # # # # # # # # # &!!  	. 	.6lG 'H 5 5 ITY[[9I9IGH&0&9&9GN#$W--H		. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.
 	!!!	 ' 'v,%&	' ' ' ' ' ' ' ' ' ' ' ' ' ' ' vLF'	k+677EE 	 	 	pk+.Fppknpp 	 +68XxPP.2+/{#y00^^{7H7H7M7M^^^fbfNNE7D11E .u .$'EEeT** `]]u{{}}]]]HH__iX]N^N^___H^^{7H7H7M7M^^^ 5 5E}} 5CJJ////!)ekkmm!<!<!-#KKF444 .&-O 	k 	k6lG4?4LPV4V4V\jGI	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	k 	6"""	.t 	. 	. 	. 	. 	. 	.	3 	3 	 	 	 	0	- 0	- 0	- 0	- 0	- 0	- 0	-d 13		 	 	 	 	 	 	 	 	 $&,.+8=?? 7	 7	'D+x} 
2:k#:#:;;CCEJJ    0 0L***a/***0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
 MM%   OOT;/000KKX..///6{{Z''"K( E 7@He@T@T6U6U 2 222+7T 3 3L---2---3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 2MH%-">">">H">">K.tE{;;H%%k7;;;$$#.6T%[QQ	    # 2 2VV,,,1,,,2 2 2 2 2 2 2 2 2 2 2 2 2 2 2    KKMMMM    DLLNNNNNJJLLLv&&&}   #	'"K( E 7@He@T@T6U6U 2 222+7T 3 3L---2---3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 2MH%-">">">H">">K.tE{;;H%%k7;;;$$#.6T%[QQ	    # 2 2VV,,,1,,,2 2 2 2 2 2 2 2 2 2 2 2 2 2 2    KKMMMM    DLLNNNNNJJLLLv&&& 	3 	36lG #DIKK 0 0GH&0&9&9GN#">2L		3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	3 	y//1122 
	# 
	#J##M<@@@@   &D  """"   T""""#1,??MMM 		. 		.6lG#i#i_h#i#i#iGK 25i..A2E2E)A,,4GJBNBZL$>$>`bGL!.;G*+x I--$*!%0	"$W--H		. 		. 		. 		. 		. 		. 		. 		. 		. 		. 		. 		. 		. 		. 		. 	!!!! & & &)3/// 		  		 hhv&&G"$+!%(XX	"$'	$4$4!*4W*=*='(11		  		  		  		  		  		  		  		  		  		  		  		  		  		  		  H%%%&  	' 	'hhv&&G"w{{8'<'<@Q'Q'Q&&&	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	'Y 	' 	'hhv&&G"w{{8'<'<@Q'Q'Q&&&	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	's  'D>>EEA'GGG5HH H7I h3 
I8I33I88E)h3 !!Ph3 Ph3 PA:h3 9S
h3 TT8TTTTTh3 TB+h3 =W%h3 %W))h3 ,W)-Ah3 	Y1%h3 1Y55h3 8Y59h3 Zh3 
Z$!h3 #Z$$B)h3 ]5)h3 5]9	9h3 <]9	=Ah3 `5h3 `	h3 `		h3 `'&h3 '
`41h3 3`44Ah3 	Ac h3  c$$h3 'c$((h3 d('h3 (
d52h3 4d55
h3  eh3 
e"h3 !e""4h3 A6hh3 hh3 h h3 2m> 3
l=lA;k l k$	$l'k$	(l<m> lm> Am11m58m5>o2Ao&o2&o**o2-o*.o2)rA   rA   )rh   )Fr   r   r   r[   r   concurrent.futuresr   r   r   r   typingr   r   uuidr	   PILr
   shared.config.configr   r   r   r   r   r   r   shared.db.sqliter   r?   r   rQ   r   shared.task_queuer   /modules.detection.services.result_store_servicer   r   r   shared.inference.infer_servicer   r   r   shared.ownership.ownershipr   shared.utils.helpersr   r   r   r    r4   r   __annotations__Lockr=   rY   r9   r@   r   rS   r   r\   ra   re   rg   rZ   r|   r   r   floatr   r  r*   r1   r/   <module>r     s   					 				       H H H H H H H H H H                                                            
 * ) ) ) ) )         
         
 9 8 8 8 8 8          d39o   IN	@@@ t     !S !TD[ ! ! ! !  S $t*     3 3 s D    <Rd Rt R R R Rs t    8s t     3  4    B#D #S # # # #.)c3h()) ) 	)
 ) ) ) ) 
) ) ) )Xp'	p'c3h(p' p' 	p'
 p' p' p' 
p' p' p' p' p' p'r1   