
    jpW              
          d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZ dZh dZh dZddhZdedefdZdee         fdZ dedefdZ!defdZ"deddfdZ#dede$fdZ%dededefdZ&dedede'e         defd Z(dIded"ede$fd#Z)dJd%e*dee$         fd&Zdede$fd'Zded(ede$fd)ZdJded%e*dee$         fd*ZdKd,ee$         d-e*dee$         fd.Z+dede$fd/Z,ded0ee$         d1ed2ede$f
d3Z-d4e$d5e$defd6Z.d4e$d5e$defd7Z/d4e$d5e$de$fd8Z0d4e$d5e$d9e$defd:Z1d4e$d5e$ddfd;Z2defd<Z3d=ede4fd>Z5d4e$d5e$de4fd?Z6d4e$d5e$de$fd@Z7d4e$de*fdAZ8d4e$de*fdBZ9d4e$de$fdCZ:ded(ede$fdDZ;ded(ede$fdEZ<ded(edFede$fdGZ=d,ee$         de$ee*f         fdHZ>dS )L    N)uuid4)Image)DATASETS_DIR)count_dataset_assets)get_dataset_asset)get_dataset)list_dataset_assets)list_datasets)save_datasetsave_dataset_asset)imageslabelssplitsexports>   .bmp.png.tif.jpeg.tiff.webp.jpg>   pendingreviewed	confirmedr   r   valuereturnc                     d                     | pd                                                                          }|st          d          t	          |          dk    rt          d          |S )N  u   数据集名称不能为空P   u   数据集名称过长)joinstripsplit
ValueErrorlen)r   names     K/home/longshao/multi-rider-rag/modules/training/services/dataset_service.py_clean_dataset_namer(      si    88U[b''))//1122D 86777
4yy2~~0111K    c                 *   t          | t                    r| }n$t          j        dt	          | pd                    }g }t                      }|D ]}d                    t	          |pd                                                                                    }|sM|                                }||v rf|	                    |           |
                    |           |st          d          t          |          dk    rt          d          |S )Nz	[,;\n\r]+r   r   u   至少填写一个类别2   u   类别数量过多)
isinstancelistrer#   strsetr!   r"   loweraddappendr$   r%   )r   	raw_itemsitemsseenraw_itemitem
normalizeds          r'   _parse_class_namesr:   $   s   % =		H\3u{+;+;<<	EUUD  xxHN++113399;;<< 	ZZ\\
T 53444
5zzB-...Lr)   c                     t          | pd                                          }t          |          dk    rt          d          |S )Nr   i  u   备注内容过长)r/   r"   r%   r$   )r   notess     r'   _clean_notesr=   =   sD    ""$$E
5zzC-...Lr)   c                  n    dt          j        d          z   dz   t                      j        d d         z   S )Nds_z%Y%m%d_%H%M%S_   )timestrftimer   hex r)   r'   _new_dataset_idrF   D   s/    4=111C7%''+bqb/IIr)   root_dirc                     t          j        | d           t          D ]6}t          j        t           j                            | |          d           7d S )NFexist_okT)osmakedirsDATASET_SUBDIRSpathr!   )rG   subdirs     r'   _ensure_dataset_dirsrP   H   s\    K5))))! C C
BGLL622TBBBBBC Cr)   
dataset_idc                 F    t          |           }|t          d          |S )Nu   数据集不存在)get_saved_datasetLookupError)rQ   datasets     r'   _require_datasetrV   N   s(    
++G.///Nr)   r&   fallbackc                     t           j                            | pd                                          p|}t	          j        dd|                              d          }|p|S )Nr   z[^A-Za-z0-9._-]+r@   z._)rK   rN   basenamer"   r.   sub)r&   rW   basecleaneds       r'   _safe_filenamer]   U   sY    7TZR..0011=XDf(#t44::4@@Ghr)   	directoryorigin_name
used_namesc                 t   t          |d          }t          j                            |          \  }}|                                }|t
          vrd}|pd}||z   }d}|                                |v s=t          j                            t          j                            | |                    ra| d| | }|dz  }|                                |v $t          j                            t          j                            | |                    a|                    |                                           |S )Nz	image.jpgr   image   r@   )	r]   rK   rN   splitextr1   
IMAGE_EXTSexistsr!   r2   )r^   r_   r`   	safe_namerootext	candidateindexs           r'   _unique_asset_filenamerl   [   s   {K88I  ++ID#
))++C
*?7Ds
IE
//

z
)
)RW^^BGLLT]<^<^-_-_
)**e*S**	
 //

z
)
)RW^^BGLLT]<^<^-_-_
) NN9??$$%%%r)   r   r<   c                 `   t          |           }t          |          }t          |          }t                      }t          j                            t          |          }t          t          j	                              }t          |           ||||ddddd|||d}	t          |	           |	S )Ndraftr   )idr&   r<   class_namesstatusimage_countlabeled_countreviewed_countversion_countrG   
created_ts
updated_ts)r(   r:   r=   rF   rK   rN   r!   r   intrB   rP   r   )
r&   rp   r<   dataset_namedataset_classesdataset_notesrQ   rG   nowrU   s
             r'   create_datasetr}   m   s    &t,,L(55O ''M ""Jw||L*55H
dikk

C""" & G Nr)   d   limitc                 "    t          |           S )Nr   )list_saved_datasetsr   s    r'   r
   r
      s    U++++r)   c                      t          |           S N)rV   )rQ   s    r'   r   r      s    J'''r)   asset_idc                 f    t          |            t          | |          }|t          d          |S )Nu   数据集图片不存在)rV   get_saved_dataset_assetrT   )rQ   r   assets      r'   r   r      s8    Z   #J99E}4555Lr)   c                 `    t          |           t          | |          }fd|D             S )Nr   c                 0    g | ]}t          |          S rE   )_attach_asset_label_status).0r8   rU   s     r'   
<listcomp>z'list_dataset_assets.<locals>.<listcomp>   s$    HHH$&w55HHHr)   )rV   list_saved_dataset_assets)rQ   r   r5   rU   s      @r'   r	   r	      s<    z**G%j>>>EHHHH%HHHHr)      r5   limit_per_datasetc                     g }| D ]@}t          |          }t          |d         |          |d<   |                    |           A|S )Nro   r   recent_assets)dictr   r3   )r5   r   outputr8   rU   s        r'   attach_recent_assetsr      sW    F  t**#<WT]Rc#d#d#d gMr)   c                 f   t          |           }|t          |dd          st          d          t          j                            |j        pd          }t          j                            |          d                                         dk    rt          d          t          j        	                    |d         d          }t          j
        |d	
           d t          j        |          D             }	 |j                            d           n# t          $ r Y nw xY wd}d}t          t!          j                              }	 t#          j        |j                  5 }	|	                                D ]}
|
                                rt          j                            |
j                  d                                         }|t*          vr|dz  }c	 |	                    |
          }n# t          $ r |dz  }Y w xY w	 t/          j        t3          j        |                    5 }|                                 |j        \  }}d d d            n# 1 swxY w Y   n# t          $ r |dz  }Y w xY wt          j                            |
j                  pd||z   dz    | }t;          |||          }t          j        	                    ||          }t1          |d          5 }|                    |           d d d            n# 1 swxY w Y   t?          tA                      j!        | ||dt          j        "                    |          ||tG          |          ||z   d
           |dz  } 	 d d d            n# 1 swxY w Y   n/# t          $ r  t          $ r}t          d|           |d }~ww xY w|dk    rt          d          tI          |           |d<   t          t!          j                              |d<   tK          |           ||||tM          | d          dS )Nfilenamer   u   请选择 ZIP 文件rc   z.zipu   仅支持 ZIP 文件rG   r   TrI   c                 6    h | ]}|                                 S rE   r1   r   entrys     r'   	<setcomp>z(import_zip_to_dataset.<locals>.<setcomp>        DDDE%++--DDDr)   r   image_wbzip)
ro   rQ   r   r_   source_type	file_pathwidthheight
size_bytesrv   u   ZIP 文件无法读取: u%   ZIP 中没有可导入的有效图片rr   rw      r   )rU   importedskippedupload_namer   )'rV   getattrr$   rK   rN   rY   r   rd   r1   r!   rL   listdirstreamseek	Exceptionrx   rB   zipfileZipFileinfolistis_dirre   readr   openioBytesIOloadsizerl   writer   r   rD   abspathr%   r   r   r   )rQ   upload_filerU   r   
images_dirr`   r   r   r|   archivememberri   payloadrb   r   r   r_   stored_name	full_pathfhexcs                        r'   import_zip_to_datasetr      s   z**G'+z2"F"F/000'"";#7#=2>>K	w$$Q'--//699/000gj18<<JK
T****DDRZ
-C-CDDDJ""""    HG
dikk

C1D_[/00 ,	G!**,, + +==?? g&&v77:@@BBj((qLG%ll622GG    qLGHBJw$7$788 3E

(-
v3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 !   qLGH !g..v??iCiHW^L^abLbCidgCiCi4ZjYYGLL[AA	)T** &bHHW%%%& & & & & & & & & & & & & & & ##ggk&0$/'2',%'W__Y%?%?!&"(&)'ll&)Hn    AW+,	 ,	 ,	 ,	 ,	 ,	 ,	 ,	 ,	 ,	 ,	 ,	 ,	 ,	 ,	Z     D D D9C99::CD 1}}@AAA1*==GM	,,GL "2:QGGG  s   =D 
D%$D%N &A8NG54N5HNHN&I(1II(I  I(#I $I('N(I:7N9I::A5N/LNLNLA"N<N NN NN O (N;;O source_itemsr   source_job_idc                 P   t          |           }t          j                            |d         d          }t          j        |d           d t          j        |          D             }d}d}t          t          j                              }	|D ]\}
t          |
	                    d          pd          
                                }|rt          j                            |          s|d	z  }`t          j                            t          |
	                    d
          pd          
                                          p}t          j                            t          |
	                    d          pd          
                                          p*t          j                            |          pd||z   d	z    d}t          |||          }t          j                            ||          }	 t          j        |          5 }|                                 |j        \  }}d d d            n# 1 swxY w Y   n# t$          $ r	 |d	z  }Y w xY wt'          j        ||           t+          t-                      j        | |||||
	                    dd          t          j                            |          ||t          j                            |          |	|z   d           |d	z  }^|dk    rt5          d          t7          |          }|||t9          | d          dS )NrG   r   TrI   c                 6    h | ]}|                                 S rE   r   r   s     r'   r   z2import_result_assets_to_dataset.<locals>.<setcomp>
  r   r)   r   rN   r   rc   r_   r&   result_r   ro   )ro   rQ   r   r_   r   r   source_asset_idr   r   r   r   rv   u'   所选结果图未能导入到数据集r   r   )rU   r   r   r   )rV   rK   rN   r!   rL   r   rx   rB   r/   getr"   isfilerY   rl   r   r   r   r   r   shutilcopy2r   r   rD   r   getsizer$   _refresh_dataset_countersr   )rQ   r   r   r   rU   r   r`   r   r   r|   r8   rN   r_   r   r   rb   r   r   s                     r'   import_result_assets_to_datasetr     sF    z**Ggj18<<JK
T****DDRZ
-C-CDDDJHG
dikk

C ) )488F##)r**0022 	27>>$// 	qLG GS-!8!8!>B??EEGGHH 6wDHHV$4$4$: ; ; A A C CDD6w%%6 6G+a/555	 	 -ZjQQGLL[99		D!! +U

 %
v+ + + + + + + + + + + + + + +  	 	 	qLGH	 	T9%%%ggk('**!.#'88D"#5#5W__Y77  gooi88!Hn 	
 	
 	
  	A1}}BCCC'00G 2:QGGG	  s6   3H>H2&H>2H6	6H>9H6	:H>>IIrU   r   c                 `   t           j                            | d         d          }t          j        |d           t           j                            |                    d          p|                    d          pd          \  }}|pd}t           j                            ||dz             S )	NrG   r   TrI   r   r_   rb   .txtrK   rN   r!   rL   rd   r   rU   r   
labels_dirstem_exts        r'   _label_pathr   H  s    gj18<<JK
T****!!%))J"7"7"^599];S;S"^W^__JD$?7D7<<
D6M222r)   c                 `   t           j                            | d         d          }t          j        |d           t           j                            |                    d          p|                    d          pd          \  }}|pd}t           j                            ||dz             S )	NrG   r   TrI   r   r_   rb   z
.meta.jsonr   r   s        r'   _label_meta_pathr   P  s    gj18<<JK
T****!!%))J"7"7"^599];S;S"^W^__JD$?7D7<<
D<$7888r)   c                 8   t          | |          }t          j                            |          si S 	 t	          |dd          5 }t          j        |          }d d d            n# 1 swxY w Y   t          |t                    r|ni S # t          $ r i cY S w xY w)Nrutf-8encoding)
r   rK   rN   r   r   jsonr   r,   r   r   )rU   r   	meta_pathr   r   s        r'   _read_annotation_metar   X  s     %00I7>>)$$ 	)S7333 	$rimmG	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$$Wd33;ww;   			s5   B
 A&B
 &A**B
 -A*.B
 
BBr   c                     t          | |          }t          |dd          5 }t          j        ||dd           d d d            n# 1 swxY w Y   |S )Nwr   r   F   )ensure_asciiindent)r   r   r   dump)rU   r   r   r   r   s        r'   _write_annotation_metar   d  s     %00I	iw	/	/	/ =2	'2E!<<<<= = = = = = = = = = = = = = =s   AAAc                     t          | |          }t          j                            |          rt          j        |           d S d S r   )r   rK   rN   r   remove)rU   r   r   s      r'   _remove_annotation_metar   k  sG     %00I	w~~i   
	) r)   c                     t          | pd                                                                          }|t          vrdS |S )Nr   r   )r/   r"   r1   REVIEW_STATUSES)r   rq   s     r'   _normalize_review_statusr   q  s@    "##%%++--F_$$yMr)   rN   c                     | rt           j                            |           sdS 	 t          | dd          5 }t	          d |D                       cd d d            S # 1 swxY w Y   d S # t
          $ r Y dS w xY w)NFr   r   r   c              3   >   K   | ]}|                                 V  d S r   r"   r   lines     r'   	<genexpr>z*_label_file_has_content.<locals>.<genexpr>}  *      33tzz||333333r)   )rK   rN   r   r   anyr   )rN   r   s     r'   _label_file_has_contentr   x  s     rw~~d++ u$g... 	4"3333333	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4 	4   uus4   A* AA* A!!A* $A!%A* *
A87A8c                 <    t          t          | |                    S r   )r   r   )rU   r   s     r'   is_asset_labeledr     s    ";w#>#>???r)   c                    t          |          }t          | |          |d<   t          | |          }g }t          |                    d          t
                    r|                    dg           ng D ]U}|                    d          }t          |t          t          f          r"|                    t          |                     Vt          |          |d<   |rt          t          |          d          nd |d<   |rt          t          |          d          nd |d<   t          |                    d          pd	          |d
<   t          |                    d                    |d<   |d         t          v |d<   |                    d          r$t          |                    d          pd          nd |d<   |S )N
is_labeledboxes
confidenceconfidence_countr   min_confidencemax_confidencesourcer   annotation_sourcereview_statusis_reviewedreviewed_tsr   )r   r   r   r,   r   r-   rx   floatr3   r%   roundminmaxr/   r   REVIEWED_STATUSES)rU   r   r8   metaconfidence_valuesboxr   s          r'   r   r     s   ;;D)'599D %00D(2488G3D3Dd(K(KStxx$$$QS 8 8WW\**
j3,// 	8$$U:%6%6777"#455D	AR\U3'8#9#91===X\D	AR\U3'8#9#91===X\D	 #DHHX$6$6$<" = =D	4TXXo5N5NOOD/3DDD?Cxx?V?V`#dhh}55:;;;\`DKr)   c                 H   t           j                            | d         d          }t           j                            |          sdS d}t          j        |          D ]}|                                                    d          s*t           j                            ||          }t           j                            |          sj	 t          |dd          5 }t          d |D                       r|d	z  }d d d            n# 1 swxY w Y   # t          $ r Y w xY w|S )
NrG   r   r   r   r   r   r   c              3   >   K   | ]}|                                 V  d S r   r   r   s     r'   r   z(_count_labeled_assets.<locals>.<genexpr>  r   r)   rc   )rK   rN   r!   isdirr   r1   endswithr   r   r   r   )rU   r   countr&   rN   r   s         r'   _count_labeled_assetsr    s`   gj18<<J7==$$ qE
:&&  zz||$$V,, 	w||J--w~~d## 		dC'222 b3333333 QJE                	 	 	H	Ls6   	DD:DD
	
DD
	D
DDc                     d}t          | d         d          D ]C}|st          | |          }t          |                    d                    t          v r|dz  }D|S )Nr   ro   i  r   r  rc   )r   r   r   r   r  )rU   r  r   r  s       r'   _count_reviewed_assetsr    sp    E*74=EEE   	$We44#DHH_$=$=>>BSSSQJELr)   c                     t          | d                   | d<   t          |           | d<   t          |           | d<   t          t	          j                              | d<   t          |            | S )Nro   rr   rs   rt   rw   )r   r  r  rx   rB   r   )rU   s    r'   r   r     sh    1'$-@@GM4W==GO 6w ? ?G	,,GLNr)   c                    t          |           }t          | |          }t          |                    d          pd          }t          |                    d          pd          }t	          ||          }t          ||          }t          |                    d          t                    r|                    dg           ng }g }	t          j	        
                    |          r|dk    r
|dk    rt          |dd          5 }
|
D ]}|                                                                }t          |          dk    r=	 t          t          |d                             }t          |d	                   |z  }t          |d
                   |z  }t          |d                   |z  }t          |d                   |z  }n# t           $ r Y w xY wt#          d||dz  z
            }t#          d||dz  z
            }t%          t          |          ||dz  z             }t%          t          |          ||dz  z             }|                    d          pg }|	                    |d|cxk    rt          |          k     rn n||         ndt)          |d
          t)          |d
          t)          |d
          t)          |d
          d d           	 d d d            n# 1 swxY w Y   t+          |          D ]q\  }}|t          |	          k    r nX|                    d          }t          |t          t          f          r&t)          t          |          d          |	|         d<   r|t-          ||          |	t          j	                            |          t          j	                            t1          ||                    t3          |	          t5          |                    d                    |                    d          r$t          |                    d          pd          nd dS )Nr   r   r   r   r   r   r      rc   r      r                  @rp   r   class_index
class_namex1y1x2y2r   r   r  r  rU   r   r   
label_pathlabel_meta_pathr   r  r  )rV   r   rx   r   r   r   r,   r-   rK   rN   r   r   r"   r#   r%   r  r   r  r
  r3   r	  	enumerater   r   r   boolr   )rQ   r   rU   r   r   r   r&  
label_meta
meta_boxesr   r   r   partsr  x_centery_center	box_width
box_heightr!  r"  r#  r$  rp   rk   meta_boxr   s                             r'   load_asset_annotationr2    sC   z**Gj(33E		'""'a((E8$$)**FWe,,J&w66J0::>>';R;RTX0Y0Ya,,,_aJE	w~~j!! eaiiFQJJ*cG444 	  

**,,u::??"%eE!Hoo"6"6K$U1X6H$U1X7H %eAh% 7I!&uQx6!9JJ    HhS899hc)99::ux)c/'ABBv:3C(CDD%kk-88>B'2BC{BeBeBeBeUXYdUeUeBeBeBeBeBek+&>&>km#Bll#Bll#Bll#Bll&* 
 
 
 
#	 	 	 	 	 	 	 	 	 	 	 	 	 	 	< %Z00 E ExCJJE\\,//
j3,// 	E).uZ/@/@!)D)DE%L& +GU;;gooj117??+;GU+K+KLL5kk1*..2Q2QRRBL..Q^B_B_is:>>-88=A>>>ei	 	 	s9   
A K0BGK0
GK0GDK00K47K4c                    t          |           }t          | |          }t          |                    d          pd          }t          |                    d          pd          }|dk    s|dk    rt	          d          t          |t                    st	          d          |                    d          pg }g }g }	g }
|D ]8}t          |t                    s	 t          |                    d                    }t          |                    d                    }t          |                    d	                    }t          |                    d
                    }t          |                    d                    }n%# t          $ r}t	          d|           |d }~ww xY w|dk     s|t          |          k    rt	          d          t          dt          ||                    }t          dt          ||                    }t          t          |          t          ||                    }t          t          |          t          ||                    }||z
  }||z
  }|dk     s|dk     r||z   dz  |z  }||z   dz  |z  }||z  }||z  }|                    | d|dd|dd|dd|d	           |	                    |||         t          |d          t          |d          t          |d          t          |d          t          |                    d          t          t          f          r0t          t          |                    d                    d          nd d           d|i}t          |                    d          t          t          f          r3t          t          |                    d                    d          |d<   |
                    |           :t          ||          }t!          ||          }|rt#          |dd          5 }|                    d                    |          dz              d d d            n# 1 swxY w Y   t)          ||i |t+          d |	D                       rdnd|
t          t-          j                              d           nCt.          j                            |          r$t/          j        |           t7          ||           t9          |          }|t;          ||          |	t.          j                            |          t.          j                            t?          ||                    tA          |	          tC          |                    d                    |                    d          r$t          |                    d          pd          nd d S )!Nr   r   r   u'   图片尺寸无效，无法保存标注zboxes must be a listrp   r  r!  r"  r#  r$  zinvalid box payload: zclass_index out of ranger  r   r  r   z.6fr   r   r  r   r   r   
c              3   D   K   | ]}|                     d           duV  dS )r   N)r   )r   r  s     r'   r   z(save_asset_annotation.<locals>.<genexpr>E  s3      'f'fc(=(=T(I'f'f'f'f'f'fr)   automanual)r  r   rw   r  r  r%  )"rV   r   rx   r   r$   r,   r-   r   r  r   r%   r  r
  r3   r	  r   r   r   r   r!   r   r   rB   rK   rN   r   r   r   r   r   r   r   r)  r   ) rQ   r   boxes_payloadrU   r   r   r   rp   linesnormalized_boxesr+  raw_boxr  r!  r"  r#  r$  r   lefttoprightbottomr/  r0  r-  r.  norm_wnorm_hr1  r&  existing_metar   s                                    r'   save_asset_annotationrC    s   z**Gj(33E		'""'a((E8$$)**FzzVq[[BCCCmT** 1/000++m,,2KE#%J  -$ -$'4(( 		Egkk-8899Kw{{4(())Bw{{4(())Bw{{4(())Bw{{4(())BB 	E 	E 	E:S::;;D	E ??kS-=-===78883B$$#s2r{{##E%LL#b"++..U6]]CBKK00DL	c\
q==JNN5LC'%/&LC'&0U"f$]]h]]]X]]]F]]]QW]]]^^^*)+6D!nnCmmE1ooFA&&gkk,77#uFFeE'++l*C*C$D$DaHHH
 
	
 	
 	
 ";/gkk,//#u>> 	P%*5\1J1J+K+KQ%O%OH\"(####We,,J)'599M 0*cG444 	.HHTYYu%%,---	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	.$''f'fUe'f'f'f$f$ft&&lt#!$)++..	  		
 		
 		
 		
 

	#	# 0
	*///'00G+GU;;!gooj117??+;GU+K+KLL+,,1-2C2CO2T2TUUEREVEVWdEeEeos=,,];;@qAAAko	 	 	s+    B*F
F-F((F-4,Q,,Q03Q0r  c                    t          |           }t          | |          }t          |          }t          t	          j                              }t          ||          }|                    dg            |                    dd           ||d<   |t          v r|nd |d<   ||d<   t          |||           t          |          }t          | |          }||d<   t          ||          |d<   ||d<   |                    d          |d<   |S )	Nr   r  r   r  r  rw   rU   r   )rV   r   r   rx   rB   r   
setdefaultr  r   r   r2  r   r   )	rQ   r   r  rU   r   rq   r|   r  r   s	            r'   update_asset_review_statusrF  [  s   z**Gj(33E%m44F
dikk

C %00DOOGR   OOHb!!!"D!'+<!<!<##$DD7E4000'00G#J99G GI1'5AAGG%GO!XXm44GMNr)   c           	         t          |           ddddd}| D ]}|dxx         t          |                    d          pd          z  cc<   |dxx         t          |                    d          pd          z  cc<   |dxx         t          |                    d          pd          z  cc<   |dxx         t          |                    d          pd          z  cc<   |S )Nr   )dataset_countrr   rs   rt   ru   rr   rs   rt   ru   )r%   rx   r   )r5   summaryr8   s      r'   summarize_datasetsrJ  q  s   U G  H H#dhh}&=&=&B"C"CC   C(A(A(FQ$G$GG    !!!S2B)C)C)Hq%I%II!!!   C(A(A(FQ$G$GG    Nr)   )r   )r~   )r   )?r   r   rK   r.   r   rB   r   uuidr   PILr   shared.config.configr   shared.db.sqliter   r   r   r   rS   r	   r   r
   r   r   r   rM   re   r   r  r/   r(   r-   r:   r=   rF   rP   r   rV   r]   r0   rl   r}   rx   r   r   r   r   r   r   r   r   r   r)  r   r   r   r  r  r   r2  rC  rF  rJ  rE   r)   r'   <module>rO     sa   				  				 				                - - - - - - 1 1 1 1 1 1 I I I I I I = = = = = = M M M M M M A A A A A A = = = = = = = = <HHH
666- s s    c    2     J J J J JC3 C4 C C C C           c  S VY    $  # t    8, , ,tDz , , , ,(C (D ( ( ( (#      I IC I Id4j I I I I T
 s 4PT:    Vc V4 V V V VrDDt*D D 	D
 
D D D DN3 3d 3s 3 3 3 39d 94 9C 9 9 9 9	4 	 	 	 	 	 	D       T $ 4    s    # $    @d @4 @D @ @ @ @ T d    &4 C    (D S    t     9c 9S 9T 9 9 9 9x[c [S [D [ [ [ [|3 # c VZ    ,d4j T#s(^      r)   