
    /j$N                        d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlZd dlZd dlmZ d dlmZmZmZmZ d d	lmZmZmZmZmZ d d
lmZ  G d de          Z dS )    )annotationsN)deepcopy)
ThreadPool)Path)Any)Dataset)FORMATS_HELP_MSGHELP_URLIMG_FORMATScheck_file_speeds)DEFAULT_CFG
LOCAL_RANKLOGGERNUM_THREADSTQDM)imreadc                       e Zd ZdZdddeddddddd	d
dfd= fd"Zd>d%Zd?d(Zd@dAd,ZdBd-Z	dCd.Z
dDdEd0ZdDdEd1ZdBd2ZdFd4ZdFd5ZdGd6ZdHd8ZdIdJd:ZdKd<Z xZS )LBaseDataseta
  Base dataset class for loading and processing image data.

    This class provides core functionality for loading images, caching, and preparing data for training and inference in
    object detection tasks.

    Attributes:
        img_path (str | list[str]): Path to the folder containing images.
        imgsz (int): Target image size for resizing.
        augment (bool): Whether to apply data augmentation.
        single_cls (bool): Whether to treat all objects as a single class.
        prefix (str): Prefix to print in log messages.
        fraction (float): Fraction of dataset to utilize.
        channels (int): Number of channels in the images (1 for grayscale, 3 for color). Color images loaded with OpenCV
            are in BGR channel order.
        cv2_flag (int): OpenCV flag for reading images.
        im_files (list[str]): List of image file paths.
        labels (list[dict]): List of label data dictionaries.
        ni (int): Number of images in the dataset.
        rect (bool): Whether to use rectangular training.
        batch_size (int): Size of batches.
        stride (int): Stride used in the model.
        pad (float): Padding value.
        buffer (list): Buffer for mosaic images.
        max_buffer_length (int): Maximum buffer size.
        ims (list): List of loaded images.
        im_hw0 (list): List of original image dimensions (h, w).
        im_hw (list): List of resized image dimensions (h, w).
        npy_files (list[Path]): List of numpy file paths.
        cache (str | None): Cache setting ('ram', 'disk', or None for no caching).
        transforms (callable): Image transformation function.
        batch_shapes (np.ndarray): Batch shapes for rectangular training.
        batch (np.ndarray): Batch index of each image.

    Methods:
        get_img_files: Read image files from the specified path.
        update_labels: Update labels to include only specified classes.
        load_image: Load an image from the dataset.
        cache_images: Cache images to memory or disk.
        cache_images_to_disk: Save an image as an *.npy file for faster loading.
        check_cache_disk: Check image caching requirements vs available disk space.
        check_cache_ram: Check image caching requirements vs available memory.
        set_rectangle: Sort images by aspect ratio and set batch shapes for rectangular training.
        get_image_and_label: Get and return label information from the dataset.
        update_labels_info: Custom label format method to be implemented by subclasses.
        build_transforms: Build transformation pipeline to be implemented by subclasses.
        get_labels: Get labels method to be implemented by subclasses.
    i  FT              ?Ng      ?   img_pathstr | list[str]imgszintcache
bool | straugmentboolhypdict[str, Any]prefixstrrect
batch_sizestridepadfloat
single_clsclasseslist[int] | Nonefractionchannelsc                   t                                                       || _        || _        || _        || _        || _        || _        || _        |dk    rt          j
        nt          j        | _        |                     | j                  | _        |                                 | _        |                     |           t%          | j                  | _        || _        || _        |	| _        |
| _        | j        r| j        J |                                  g | _        | j        rt5          | j        | j        dz  df          nd| _        dg| j        z  dg| j        z  dg| j        z  c| _        | _        | _        d | j        D             | _        tA          |tB                    r|"                                n|du rd	nd| _#        | j#        d	k    rD| $                                r0|j%        rtM          j'        d
           | (                                 n3| j#        dk    r(| )                                r| (                                 | *                    |          | _+        dS )a@  Initialize BaseDataset with given configuration and options.

        Args:
            img_path (str | list[str]): Path to the folder containing images or list of image paths.
            imgsz (int): Image size for resizing.
            cache (bool | str): Cache images to RAM or disk during training.
            augment (bool): If True, data augmentation is applied.
            hyp (dict[str, Any]): Hyperparameters to apply data augmentation.
            prefix (str): Prefix to print in log messages.
            rect (bool): If True, rectangular training is used.
            batch_size (int): Size of batches.
            stride (int): Stride used in the model.
            pad (float): Padding value.
            single_cls (bool): If True, single class training is used.
            classes (list[int], optional): List of included classes.
            fraction (float): Fraction of dataset to utilize.
            channels (int): Number of channels in the images (1 for grayscale, 3 for color). Color images loaded with
                OpenCV are in BGR channel order.
           )include_classN   i  r   c                R    g | ]$}t          |                              d           %S )z.npy)r   with_suffix).0fs     Z/home/longshao/multi-rider-rag/.venv/lib/python3.11/site-packages/ultralytics/data/base.py
<listcomp>z(BaseDataset.__init__.<locals>.<listcomp>   s,    MMM!$q''--f55MMM    Tramzcache='ram' may produce non-deterministic training results. Consider cache='disk' as a deterministic alternative if your disk space allows.disk)r"   ),super__init__r   r   r    r+   r$   r.   r/   cv2IMREAD_GRAYSCALEIMREAD_COLORcv2_flagget_img_filesim_files
get_labelslabelsupdate_labelslennir&   r'   r(   r)   set_rectanglebufferminmax_buffer_lengthimsim_hw0im_hw	npy_files
isinstancer%   lowerr   check_cache_ramdeterministicr   warningcache_imagescheck_cache_diskbuild_transforms
transforms)selfr   r   r   r    r"   r$   r&   r'   r(   r)   r+   r,   r.   r/   	__class__s                  r8   r>   zBaseDataset.__init__H   sY   H 	 
$  08A,,3CS**4=99oo''111dk""	$9 	!?...    NRl!adgt/BD%I!J!J!J`a .2FTW,<tftw>NQUPVY]Y`P`)$+tzMMt}MMM&0&<&<bU[[]]]5TX==%%^b
:4#7#7#9#9  f   Z6!!d&;&;&=&=! //C/88r:   return	list[str]c                   	 g }t          |t                    r|n|gD ]}t          |          }|                                r-|t	          j        t          |dz  dz            d          z  }S|                                rt          |d          5 }|                                	                                
                                }t          |j                  t          j        z   |fd|D             z  }ddd           n# 1 swxY w Y   t          | j         | d	          t!          d
 |D                       }|sJ | j         d| dt"                       n4# t$          $ r'}t          | j         d| dt&                     |d}~ww xY w| j        dk     r,|dt+          t-          |          | j        z                     }t/          || j                   |S )aE  Read image files from the specified path.

        Args:
            img_path (str | list[str]): Path or list of paths to image directories or files.

        Returns:
            (list[str]): List of image file paths.

        Raises:
            FileNotFoundError: If no images are found or the path doesn't exist.
        z**z*.*T)	recursivezutf-8)encodingc                j    g | ]/}|                     d           r|                    d           n|0S )z./)
startswithreplace)r6   xparents     r8   r9   z-BaseDataset.get_img_files.<locals>.<listcomp>   s?    ^^^WXd9K9KRaiif555QR^^^r:   Nz does not existc              3     K   | ]X}|                     d           d                                         t          v 6|                    dt          j                  V  YdS )./N)
rpartitionrS   r   rd   ossepr6   re   s     r8   	<genexpr>z,BaseDataset.get_img_files.<locals>.<genexpr>   s\      pp1<<PSCTCTUWCXC^C^C`C`doCoCoaiiRV44CoCoCoCoppr:   zNo images found in z. zError loading data from 
r1   )r$   )rR   listr   is_dirglobr%   is_fileopenreadstrip
splitlinesrf   rl   rm   FileNotFoundErrorr$   sortedr	   	Exceptionr
   r.   roundrH   r   )r[   r   r7   ptrD   erf   s          @r8   rC   zBaseDataset.get_img_files   s^   	kA!+Hd!;!;KXX( P PGG88:: 
P3q4x%'7#8#8DIIIIAAYY[[ Pa'222 _aFFHHNN,,7799!$QX!7^^^^\]^^^^_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ ,t{,NA,N,N,NOOOppapppppH^^^^^^L\^^^^8^ 	k 	k 	k#t{$b$bH$b$bX`$b$bccijj	k=1 F%H(E"F"F FGH(4;7777s>   BE+ A+DE+ D	E+ D	AE+ +
F5"FFr2   Nonec                   t          j        |                              dd          }t          t	          | j                            D ]}|| j        |         d         }| j        |         d         }| j        |         d         | j        |         d         }||k                        d          }||         | j        |         d<   ||         | j        |         d<   r)fdt          |          D             | j        |         d<   |||         | j        |         d<   | j        rd	| j        |         d         ddd	f<   dS )
zUpdate labels to include only specified classes.

        Args:
            include_class (list[int], optional): List of classes to include. If None, all classes are included.
        r1   ri   Nclsbboxessegments	keypointsc                *    g | ]\  }}||         S  r   )r6   siidxr   s      r8   r9   z-BaseDataset.update_labels.<locals>.<listcomp>   s'    1b1b1b72s^a1b(2,1b1b1br:   r   )	nparrayreshaperangerH   rF   any	enumerater+   )	r[   r2   include_class_arrayir   r   r   jr   s	           @r8   rG   zBaseDataset.update_labels   sM    !h}55==aDDs4;''(( 	0 	0A(k!nU+Q1;q>*5 KN;7	//44Q77(+AAu%+1!9Ax( c1b1b1b1biXYll1b1b1bDKN:.(2;A,DKN;/ 0./Au%aaad+	0 	0r:   r   	rect_mode3tuple[np.ndarray, tuple[int, int], tuple[int, int]]c                   | j         |         | j        |         | j        |         }}}||                                r	 t	          j        |          }n# t          $ rd}t          j        | j	         d| d|            t          |                              d           t          || j                  }Y d}~nd}~ww xY wt          || j                  }|t          d|           |j        dd         \  }}|r| j        t#          ||          z  }	|	d	k    rwt%          t'          j        ||	z            | j                  t%          t'          j        ||	z            | j                  }}
t+          j        ||
|ft*          j        
          }nA||cxk    r| j        k    s/n t+          j        || j        | j        ft*          j        
          }|j        dk    r|d         }| j        r|||f|j        dd         c| j         |<   | j        |<   | j        |<   | j                            |           d	t=          | j                  cxk     r| j        k    rFn nC| j                             d          }| j!        dk    rd\  | j         |<   | j        |<   | j        |<   |||f|j        dd         fS | j         |         | j        |         | j        |         fS )a  Load an image from dataset index 'i'.

        Args:
            i (int): Index of the image to load.
            rect_mode (bool): Whether to use rectangular resizing.

        Returns:
            im (np.ndarray): Loaded image as a NumPy array.
            hw_original (tuple[int, int]): Original image dimensions in (height, width) format.
            hw_resized (tuple[int, int]): Resized image dimensions in (height, width) format.

        Raises:
            FileNotFoundError: If the image file is not found.
        Nz"Removing corrupt *.npy image file z	 due to: T
missing_okflagszImage Not Found    r1   )interpolation).Nr   r;   )NNN)"rN   rD   rQ   existsr   loadr{   r   rV   r$   r   unlinkr   rB   ry   shaper   maxrL   mathceilr?   resizeINTER_LINEARndimr    rO   rP   rK   appendrH   rM   popr   )r[   r   r   imr7   fnr   h0w0rwhr   s                r8   
load_imagezBaseDataset.load_image   s    HQKq!14>!3DrA:yy{{ 48BB  8 8 8Ndk#e#eUW#e#ebc#e#efffHHOOtO444777BBBBBB8
 AT]333z'(>1(>(>???Xbqb\FB ^JR,66	"q& 1 14:>>DIbSTfDUDUW[Wa@b@bqABAc>NOOOBB,,,,$*,,,,ZTZ$<CL\]]]w!||	] | V=?"b28TVUVTV<:T[^TZ]""1%%%s4;''AAAA4+AAAAAA**AzU**EUBT[^TZ]Bx"1"--x{DKNDJqM99s    A 
CAB>>Cc                   d\  }}| j         dk    r	| j        dfn| j        df\  }}t          t                    5 }|                    |t          | j                            }t          t          |          | j        t          dk              }|D ]\  }}	| j         dk    r(|| j        |                                         j        z  }n3|	\  | j        |<   | j        |<   | j        |<   || j        |         j        z  }| j         d||z  dd	| d
|_        |                                 ddd           dS # 1 swxY w Y   dS )z3Cache images to memory or disk for faster training.r   i   @r<   DiskRAMr   )totaldisablezCaching images (.1fzGB )N)r   cache_images_to_diskr   r   r   imapr   rI   r   r   r   rQ   statst_sizerN   rO   rP   nbytesr$   descclose)
r[   bgbfcnstoragepoolresultspbarr   re   s
             r8   rW   zBaseDataset.cache_images  s   2>BjF>R>R16::Y]YhjoXpW$$ 
	iiU47^^44G	'**$':PQ>RRRD V V1:''*//1199AAAB>DHQKQA!++A#{UUAFUUU7UUU		JJLLL
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	s   C1D88D<?D<c                |   | j         |         }|                                s	 t          j        |                                t          | j        |         | j                  d           dS # t          $ rB}|	                    d           t          j        | j         d| d|            Y d}~dS d}~ww xY wdS )	z2Save an image as an *.npy file for faster loading.r   F)allow_pickleTr   u%   WARNING ⚠️ Failed to cache image z: N)rQ   r   r   saveas_posixr   rD   rB   r{   r   r   rV   r$   )r[   r   r7   r   s       r8   r   z BaseDataset.cache_images_to_disk  s    N1xxzz 	^^

fT]1-=T]&S&S&Sbghhhhhh ^ ^ ^D)))$+\\TU\\YZ\\]]]]]]]]]^	^ 	^s   AA- -
B977B44B9safety_marginc                   ddl }d\  }}t          | j        d          }t          |          D ]}t	          j        | j                  }t          |          }|-||j        z  }t          j
        t          |          j        t          j                  s&d| _        t          j        | j         d            dS || j        z  |z  d|z   z  }	|                    t          | j        d                   j                  \  }
}}|	|k    rLd| _        t          j        | j         |	|z  dd	t'          |d
z             d||z  dd|
|z  dd	           dS dS )zCheck if there's enough disk space for caching images.

        Args:
            safety_margin (float): Safety margin factor for disk space calculation.

        Returns:
            (bool): True if there's enough disk space, False otherwise.
        r   Nr      z7Skipping caching images to disk, directory not writableFr1   r   zGB disk space required, with d   % safety margin but only rj   z#GB free, not caching images to diskT)shutilrL   rI   r   randomchoicerD   r   r   rl   accessr   rf   W_OKr   r   rV   r$   
disk_usager   )r[   r   r   r   r   n_im_filer   disk_requiredr   _usedfrees                r8   rX   zBaseDataset.check_cache_disk!  s    	2q 		 		AmDM22GBzNA9T']]127;; !
$+fffggguu DGa1}+<=#..tDM!4D/E/E/LMMud4DJN; W 2P W WMC/00W W"9VW W#(2:VW W W  
 5tr:   c                V   d\  }}t          | j        d          }t          |          D ]i}t          t	          j        | j                            }|+| j        t          |j	        d         |j	        d                   z  }||j
        |dz  z  z  }j|| j        z  |z  d|z   z  }t          d                                          }	||	j        k    rVd| _        t          j        | j         ||z  dd	t%          |d
z             d|	j        |z  dd|	j        |z  dd	           dS dS )zCheck if there's enough RAM for caching images.

        Args:
            safety_margin (float): Safety margin factor for RAM calculation.

        Returns:
            (bool): True if there's enough RAM, False otherwise.
        r   r   Nr   r1   r   psutilr   z%GB RAM required to cache images with r   r   rj   z GB available, not caching imagesFT)rL   rI   r   r   r   r   rD   r   r   r   r   
__import__virtual_memory	availabler   r   rV   r$   r   r   )
r[   r   r   r   r   r   r   ratiomem_requiredmems
             r8   rT   zBaseDataset.check_cache_ramD  sg    2q 	& 	&Adm4455BzJRXa["(1+!>!>>EUAX%%AA47{Q!m*;<""1133#-''DJN; ar 1W a aMC/00a a=2%`a a,/IN`a a a  
 5tr:   c                \    t          j        t          j         j                   j        z                                t                    }|d         dz   }t          j        d  j        D                       }|dddf         |dddf         z  }|	                                } fd|D              _
         fd|D              _        ||         }ddgg|z  }t          |          D ]T}|||k             }|                                |                                }
}	|
dk     r|
dg||<   D|	dk    r
dd|	z  g||<   Ut          j        t          j        |           j        z   j        z   j        z                                 t                     j        z   _        | _        dS )zJSort images by aspect ratio and set batch shapes for rectangular training.ri   r1   c                8    g | ]}|                     d           S )r   )r   rn   s     r8   r9   z-BaseDataset.set_rectangle.<locals>.<listcomp>f  s"    :::aeeGnn:::r:   Nr   c                *    g | ]}j         |         S r   )rD   r6   r   r[   s     r8   r9   z-BaseDataset.set_rectangle.<locals>.<listcomp>i  s     999aq)999r:   c                *    g | ]}j         |         S r   )rF   r   s     r8   r9   z-BaseDataset.set_rectangle.<locals>.<listcomp>j  s    555!t{1~555r:   )r   floorarangerI   r'   astyper   r   rF   argsortrD   r   rL   r   r   r   r(   r)   batch_shapesbatch)r[   binbsarirectshapesr   ariminimaxis   `          r8   rJ   zBaseDataset.set_rectanglea  s   Xbi((4?:;;BB3GGVaZH::dk:::;;qqq!tWqAw

999959995555u555Y a&Br 	* 	*AR1W+CCGGII$Daxx!1Iq		DMq	GBHV$4$4tz$ADK$ORVRZ$Z[[bbcfggjnjuu


r:   indexc                R    |                      |                     |                    S )z5Return transformed label information for given index.)rZ   get_image_and_label)r[   r   s     r8   __getitem__zBaseDataset.__getitem__z  s"    t77>>???r:   c                   t          | j        |                   }|                    dd           |                     |          \  |d<   |d<   |d<   |d         d         |d         d         z  |d         d         |d         d         z  f|d<   | j        r| j        | j        |                  |d	<   |                     |          S )
zGet and return label information from the dataset.

        Args:
            index (int): Index of the image to retrieve.

        Returns:
            (dict[str, Any]): Label dictionary with image and metadata.
        r   Nimg	ori_shaperesized_shaper   r1   	ratio_pad
rect_shape)r   rF   r   r   r&   r   r   update_labels_info)r[   r   labels      r8   r   zBaseDataset.get_image_and_label~  s     U+,,		'4   CG??SXCYCY@eeK(%*@/"1%k(:1(==/"1%k(:1(==
k 9 	G"&"3DJu4E"FE,&&u---r:   c                *    t          | j                  S )z5Return the length of the labels list for the dataset.)rH   rF   r[   s    r8   __len__zBaseDataset.__len__  s    4;r:   r  c                    |S )z!Customize your label format here.r   )r[   r  s     r8   r  zBaseDataset.update_labels_info  s    r:   dict[str, Any] | Nonec                    t           )a  Users can customize augmentations here.

        Examples:
            >>> if self.augment:
            ...     # Training transforms
            ...     return Compose([])
            >>> else:
            ...    # Val transforms
            ...    return Compose([])
        NotImplementedError)r[   r"   s     r8   rY   zBaseDataset.build_transforms  s
     "!r:   list[dict[str, Any]]c                    t           )a   Users can customize their own format here.

        Examples:
            Ensure output is a dictionary with the following keys:
            >>> dict(
            ...     im_file=im_file,
            ...     shape=shape,  # format: (height, width)
            ...     cls=cls,
            ...     bboxes=bboxes,  # xywh
            ...     segments=segments,  # xy
            ...     keypoints=keypoints,  # xy
            ...     normalized=True,  # or False
            ...     bbox_format="xyxy",  # or xywh, ltwh
            ... )
        r	  r  s    r8   rE   zBaseDataset.get_labels  s
      "!r:   )r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r!   r'   r   r(   r   r)   r*   r+   r!   r,   r-   r.   r*   r/   r   )r   r   r]   r^   )r2   r-   r]   r   )T)r   r   r   r!   r]   r   )r]   r   )r   r   r]   r   )r   )r   r*   r]   r!   )r   r   r]   r#   )r]   r   )r  r#   r]   r#   )N)r"   r  )r]   r  )__name__
__module____qualname____doc__r   r>   rC   rG   r   rW   r   rX   rT   rJ   r   r   r  r  rY   rE   __classcell__)r\   s   @r8   r   r      s       . .f !) $(L9 L9 L9 L9 L9 L9 L9\# # # #J0 0 0 0.3: 3: 3: 3: 3:j    ^ ^ ^ ^! ! ! ! !F    :   2@ @ @ @. . . .(          " " " " "" " " " " " " "r:   r   )!
__future__r   rs   r   rl   r   copyr   multiprocessing.poolr   pathlibr   typingr   r?   numpyr   torch.utils.datar   ultralytics.data.utilsr	   r
   r   r   ultralytics.utilsr   r   r   r   r   ultralytics.utils.patchesr   r   r   r:   r8   <module>r     sJ   # " " " " "   				        + + + + + +             



     $ $ $ $ $ $ ] ] ] ] ] ] ] ] ] ] ] ] P P P P P P P P P P P P P P , , , , , ,`" `" `" `" `"' `" `" `" `" `"r:   