
    j                        d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
 dZdZdZdZdZd	Zd
ej        fdZdej        ded
ee         fdZdeeef         d
ee         fdZdej        dz  d
eeef         dz  fdZdej        dz  d
eeef         dz  fdZdej        dz  d
eeef         dz  fdZdej        dz  d
eeef         dz  fdZdej        dz  d
eeef         dz  fdZdej        dz  d
eeef         dz  fdZdAdZ deeef         d
dfdZ!ded
eeef         dz  fdZ"	 	 dBdedede#d edz  d
eeeef                  f
d!Z$d"eeef         d
dfd#Z%d$ed
eeef         dz  fd%Z&d&eeef         d
dfd'Z'd$ed
e#fd(Z(dCd$ede#d
eeeef                  fd*Z)d$ed+ed
eeef         dz  fd,Z*dCde#d
eeeef                  fd-Z+deeef         d
dfd.Z,ded
eeef         dz  fd/Z-dDdedede#d
eeeef                  fd0Z.deeef         d
dfd1Z/ded
eeef         dz  fd2Z0dDdedede#d
eeeef                  fd3Z1deeef         d
dfd4Z2ded
eeef         dz  fd5Z3d
eeef         dz  fd6Z4dDde#d
eeeef                  fd7Z5dEdedede#d
eeeef                  fd9Z6dEdedede#d
eeeef                  fd:Z7dEde#d
eeeef                  fd;Z8dFd=e#d
e#fd>Z9dGd?ee         dz  d
e#fd@Z:dS )H    N)Any)MODEL_DEFAULTSQLITE_DB_PATHlogger)job_typeidsource_namesource_typesource_pathtemp_dirframe_intervalstatusmessagetotal	processedkeptnotfoundfailed
downloadedstart_tsend_ts	owner_keyowner_ipconf_thresh
batch_sizeimgszclasses_raw	model_keyzip_paths_json
result_dirresult_manifest_pathidentity_result_pathidentity_summary_jsonsummary_textr   namenotesclass_names_jsonr   image_countlabeled_countreviewed_countversion_countroot_dir
created_ts
updated_tsr   
dataset_idfilenameorigin_namer
   source_job_idsource_asset_id	file_pathwidthheight
size_bytesr.   )r   r1   dataset_namer   r   
base_modelbase_model_path
preset_keyepochsr   r   confirmed_onlyrun_dirlog_pathmanifest_pathartifact_dirr.   r   r   r   r   )r   r1   r:   r   r   r   r   r   prompt_classesclass_mapping	overwriter   r   updatedskipped_existingno_detectionr.   r   r   r   r   r   actionr   r   stager   r   r.   r   r   errorresult_jsonlibrary_jsonreturnc                  T   t           j                            t                    } | rt          j        | d           t          j        t          d          }t
          j        |_        |	                    d           |	                    d           |	                    d           |S )NT)exist_ok   )timeoutzPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000zPRAGMA synchronous=NORMAL)
ospathdirnamer   makedirssqlite3connectRowrow_factoryexecute)parentconns     2/home/longshao/multi-rider-rag/shared/db/sqlite.py_connectra      s    W__^,,F +
FT****?>2666D{DLL*+++LL,---LL,---K    r_   
table_namec                 p    |                      d| d                                          }d |D             S )NzPRAGMA table_info()c                     h | ]
}|d          S )r&    .0rows     r`   	<setcomp>z$_existing_columns.<locals>.<setcomp>   s    (((CCK(((rb   )r]   fetchall)r_   rc   rowss      r`   _existing_columnsrn      s?    <<:Z:::;;DDFFD((4((((rb   jobc                    d| v r-t          | d         t                    rd | d         D             S d| v r-t          | d         t                    rd | d         D             S |                     d          r	| d         gS g S )N	zip_partsc                     g | ]A}t          |t                    |                    d           ,|                    d           BS )rV   )
isinstancedictget)ri   parts     r`   
<listcomp>z&_extract_zip_paths.<locals>.<listcomp>   sZ     
 
 
$%%
 +/((6*:*:
HHV
 
 
rb   	zip_pathsc                     g | ]}||S rg   rg   ri   rV   s     r`   rw   z&_extract_zip_paths.<locals>.<listcomp>   s    :::T::::rb   zip_path)rs   listru   )ro   s    r`   _extract_zip_pathsr}      s    cj[)94@@
 
K(
 
 
 	

 cj[)94@@::[!1::::
wwz !J  Irb   rj   c                      d S  fdt           D             }	 t          j        |                    d          pd          }n# t          $ r g }Y nw xY w	 t          j        |                    d          pd          }n# t          $ r i }Y nw xY w||d<   d |D             |d<   t          |          d	k    r|d
         nd |d<   ||d<   |S )Nc                 "    i | ]}||         S rg   rg   ri   columnrj   s     r`   
<dictcomp>z_row_to_job.<locals>.<dictcomp>   s    
9
9
9663v;
9
9
9rb   r   []r#   {}rx   c                 R    g | ]$}|t           j                            |          d %S ))rV   r&   )rU   rV   basenamerz   s     r`   rw   z_row_to_job.<locals>.<listcomp>   s0    ]]]4rw/?/?/E/EFF]]]rb   rq      r   r{   identity_summary)JOB_COLUMNSjsonloadsru   	Exceptionlen)rj   ro   rx   r   s   `   r`   _row_to_jobr      s   
{t
9
9
9
9[
9
9
9CJsww'788@DAA		   			:cgg.E&F&F&N$OO    !C]]S\]]]C&))nn&9&9illtC
O.CJ#   )A AA)B BBc                       d S  fdt           D             }	 t          j        |                    d          pd          }n# t          $ r g }Y nw xY w||d<   |S )Nc                 "    i | ]}||         S rg   rg   r   s     r`   r   z#_row_to_dataset.<locals>.<dictcomp>   s    AAAvvs6{AAArb   r(   r   class_names)DATASET_COLUMNSr   r   ru   r   )rj   datasetr   s   `  r`   _row_to_datasetr      s    
{tAAAAAAAGj-?!@!@!HDII    )GMNs   )A AAc                 2      d S  fdt           D             S )Nc                 "    i | ]}||         S rg   rg   r   s     r`   r   z)_row_to_dataset_asset.<locals>.<dictcomp>   s    DDDFFCKDDDrb   )DATASET_ASSET_COLUMNSrj   s   `r`   _row_to_dataset_assetr      s(    
{tDDDD.CDDDDrb   c                 2      d S  fdt           D             S )Nc                 "    i | ]}||         S rg   rg   r   s     r`   r   z%_row_to_train_job.<locals>.<dictcomp>   s    @@@FFCK@@@rb   )TRAIN_JOB_COLUMNSr   s   `r`   _row_to_train_jobr      s(    
{t@@@@.?@@@@rb   c                 2      d S  fdt           D             S )Nc                 "    i | ]}||         S rg   rg   r   s     r`   r   z-_row_to_auto_annotate_job.<locals>.<dictcomp>   s    HHHFFCKHHHrb   )AUTO_ANNOTATE_JOB_COLUMNSr   s   `r`   _row_to_auto_annotate_jobr      s(    
{tHHHH.GHHHHrb   c                      d S  fdt           D             }	 t          j        |                    d          pd          }n# t          $ r i }Y nw xY w	 t          j        |                    d          pd          }n# t          $ r i }Y nw xY w||d<   ||d<   |                    dd            |                    dd            |S )Nc                 "    i | ]}||         S rg   rg   r   s     r`   r   z,_row_to_face_library_job.<locals>.<dictcomp>   s    
F
F
F663v;
F
F
Frb   rN   r   rO   resultlibrary)FACE_LIBRARY_JOB_COLUMNSr   r   ru   r   pop)rj   ro   r   r   s   `   r`   _row_to_face_library_jobr      s    
{t
F
F
F
F-E
F
F
FCCGGM22:d;;   *SWW^44<==    CMC	NGGM4   GGND!!!Jr   c                  F   t                      5 } |                     d           t          | d          }d|vr|                     d           d|vr|                     d           d|vr|                     d           d	|vr|                     d
           d|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           |                     d           |                     d           |                     d           |                     d           |                     d           t          | d           }d!|vr|                     d"           d#|vr|                     d$           d%|vr|                     d&           d'|vr|                     d(           d)|vr|                     d*           d+|vr|                     d,           d-|vr|                     d.           d/|vr|                     d0           d1|vr|                     d2           d3|vr|                     d4           |                     d5           |                     d6           t          | d7          }d8|vr|                     d9           d	|vr|                     d:           d;|vr|                     d<           d=|vr|                     d>           d?|vr|                     d@           dA|vr|                     dB           dC|vr|                     dD           dE|vr|                     dF           d1|vr|                     dG           |                     dH           |                     dI           t          | dJ          }dK|vr|                     dL           d%|vr|                     dM           dN|vr|                     dO           dP|vr|                     dQ           dR|vr|                     dS           dT|vr|                     dU           dV|vr|                     dW           dX|vr|                     dY           dZ|vr|                     d[           d\|vr|                     d]           d^|vr|                     d_           d`|vr|                     da           db|vr|                     dc           dd|vr|                     de           d1|vr|                     df           dg|vr|                     dh           di|vr|                     dj           d|vr|                     dk           dl|vr|                     dm           |                     dn           |                     do           |                     dp           |                     dq           t          | dr          }dK|vr|                     ds           d%|vr|                     dt           dN|vr|                     du           d|vr|                     dv           dw|vr|                     dx           dX|vr|                     dy           dz|vr|                     d{           d||vr|                     d}           d~|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           d1|vr|                     d           dg|vr|                     d           di|vr|                     d           d|vr|                     d           dl|vr|                     d           |                     d           |                     d           |                     d           |                     d           t          | d          }d|vr|                     d           d%|vr|                     d           dN|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           d1|vr|                     d           dg|vr|                     d           di|vr|                     d           d|vr|                     d           d|vr|                     d           d|vr|                     d           |                     d           |                     d           |                     d           |                     d           |                     d           |                     d           |                     d           |                     d           |                     d           |                     d           |                     d           |                     d           |                     d           ddlm}  ||            |                                  d d d            d S # 1 swxY w Y   d S )Na  
            CREATE TABLE IF NOT EXISTS jobs (
                id TEXT PRIMARY KEY,
                job_type TEXT NOT NULL DEFAULT 'oracle',
                source_name TEXT,
                source_type TEXT,
                source_path TEXT,
                temp_dir TEXT,
                frame_interval INTEGER,
                status TEXT NOT NULL,
                message TEXT,
                total INTEGER NOT NULL DEFAULT 0,
                processed INTEGER NOT NULL DEFAULT 0,
                kept INTEGER NOT NULL DEFAULT 0,
                notfound INTEGER NOT NULL DEFAULT 0,
                failed INTEGER NOT NULL DEFAULT 0,
                downloaded INTEGER NOT NULL DEFAULT 0,
                start_ts INTEGER,
                end_ts INTEGER,
                owner_key TEXT,
                owner_ip TEXT,
                conf_thresh REAL,
                batch_size INTEGER,
                imgsz INTEGER,
                classes_raw TEXT,
                model_key TEXT NOT NULL DEFAULT 'general',
                zip_paths_json TEXT,
                result_dir TEXT,
                result_manifest_path TEXT,
                identity_result_path TEXT,
                identity_summary_json TEXT,
                summary_text TEXT
            )
            jobsr   zEALTER TABLE jobs ADD COLUMN model_key TEXT NOT NULL DEFAULT 'general'r   zCALTER TABLE jobs ADD COLUMN job_type TEXT NOT NULL DEFAULT 'oracle'r	   z,ALTER TABLE jobs ADD COLUMN source_name TEXTr
   z,ALTER TABLE jobs ADD COLUMN source_type TEXTr   z,ALTER TABLE jobs ADD COLUMN source_path TEXTr   z)ALTER TABLE jobs ADD COLUMN temp_dir TEXTr   z2ALTER TABLE jobs ADD COLUMN frame_interval INTEGERr   z*ALTER TABLE jobs ADD COLUMN owner_key TEXTr    z+ALTER TABLE jobs ADD COLUMN result_dir TEXTr!   z5ALTER TABLE jobs ADD COLUMN result_manifest_path TEXTr"   z5ALTER TABLE jobs ADD COLUMN identity_result_path TEXTr#   z6ALTER TABLE jobs ADD COLUMN identity_summary_json TEXTzXCREATE INDEX IF NOT EXISTS idx_jobs_owner_key_start_ts ON jobs(owner_key, start_ts DESC)zSCREATE INDEX IF NOT EXISTS idx_jobs_owner_start_ts ON jobs(owner_ip, start_ts DESC)z:CREATE INDEX IF NOT EXISTS idx_jobs_end_ts ON jobs(end_ts)z:CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)a  
            CREATE TABLE IF NOT EXISTS datasets (
                id TEXT PRIMARY KEY,
                name TEXT NOT NULL,
                notes TEXT NOT NULL DEFAULT '',
                class_names_json TEXT NOT NULL DEFAULT '[]',
                status TEXT NOT NULL DEFAULT 'draft',
                image_count INTEGER NOT NULL DEFAULT 0,
                labeled_count INTEGER NOT NULL DEFAULT 0,
                reviewed_count INTEGER NOT NULL DEFAULT 0,
                version_count INTEGER NOT NULL DEFAULT 0,
                root_dir TEXT NOT NULL,
                created_ts INTEGER NOT NULL,
                updated_ts INTEGER NOT NULL
            )
            datasetsr'   z>ALTER TABLE datasets ADD COLUMN notes TEXT NOT NULL DEFAULT ''r(   zKALTER TABLE datasets ADD COLUMN class_names_json TEXT NOT NULL DEFAULT '[]'r   zDALTER TABLE datasets ADD COLUMN status TEXT NOT NULL DEFAULT 'draft'r)   zFALTER TABLE datasets ADD COLUMN image_count INTEGER NOT NULL DEFAULT 0r*   zHALTER TABLE datasets ADD COLUMN labeled_count INTEGER NOT NULL DEFAULT 0r+   zIALTER TABLE datasets ADD COLUMN reviewed_count INTEGER NOT NULL DEFAULT 0r,   zHALTER TABLE datasets ADD COLUMN version_count INTEGER NOT NULL DEFAULT 0r-   zAALTER TABLE datasets ADD COLUMN root_dir TEXT NOT NULL DEFAULT ''r.   zEALTER TABLE datasets ADD COLUMN created_ts INTEGER NOT NULL DEFAULT 0r/   zEALTER TABLE datasets ADD COLUMN updated_ts INTEGER NOT NULL DEFAULT 0zOCREATE INDEX IF NOT EXISTS idx_datasets_updated_ts ON datasets(updated_ts DESC)a  
            CREATE TABLE IF NOT EXISTS dataset_assets (
                id TEXT PRIMARY KEY,
                dataset_id TEXT NOT NULL,
                filename TEXT NOT NULL,
                origin_name TEXT NOT NULL,
                source_type TEXT NOT NULL DEFAULT 'zip',
                source_job_id TEXT NOT NULL DEFAULT '',
                source_asset_id TEXT NOT NULL DEFAULT '',
                file_path TEXT NOT NULL,
                width INTEGER NOT NULL DEFAULT 0,
                height INTEGER NOT NULL DEFAULT 0,
                size_bytes INTEGER NOT NULL DEFAULT 0,
                created_ts INTEGER NOT NULL,
                FOREIGN KEY(dataset_id) REFERENCES datasets(id)
            )
            dataset_assetsr3   zJALTER TABLE dataset_assets ADD COLUMN origin_name TEXT NOT NULL DEFAULT ''zMALTER TABLE dataset_assets ADD COLUMN source_type TEXT NOT NULL DEFAULT 'zip'r4   zLALTER TABLE dataset_assets ADD COLUMN source_job_id TEXT NOT NULL DEFAULT ''r5   zNALTER TABLE dataset_assets ADD COLUMN source_asset_id TEXT NOT NULL DEFAULT ''r6   zHALTER TABLE dataset_assets ADD COLUMN file_path TEXT NOT NULL DEFAULT ''r7   zFALTER TABLE dataset_assets ADD COLUMN width INTEGER NOT NULL DEFAULT 0r8   zGALTER TABLE dataset_assets ADD COLUMN height INTEGER NOT NULL DEFAULT 0r9   zKALTER TABLE dataset_assets ADD COLUMN size_bytes INTEGER NOT NULL DEFAULT 0zKALTER TABLE dataset_assets ADD COLUMN created_ts INTEGER NOT NULL DEFAULT 0zoCREATE INDEX IF NOT EXISTS idx_dataset_assets_dataset_created_ts ON dataset_assets(dataset_id, created_ts DESC)a  
            CREATE TABLE IF NOT EXISTS train_jobs (
                id TEXT PRIMARY KEY,
                dataset_id TEXT NOT NULL,
                dataset_name TEXT NOT NULL,
                status TEXT NOT NULL,
                message TEXT NOT NULL DEFAULT '',
                base_model TEXT NOT NULL,
                base_model_path TEXT NOT NULL,
                preset_key TEXT NOT NULL,
                epochs INTEGER NOT NULL DEFAULT 0,
                imgsz INTEGER NOT NULL DEFAULT 0,
                batch_size INTEGER NOT NULL DEFAULT 0,
                confirmed_only INTEGER NOT NULL DEFAULT 0,
                run_dir TEXT NOT NULL,
                log_path TEXT NOT NULL,
                manifest_path TEXT NOT NULL,
                artifact_dir TEXT NOT NULL,
                created_ts INTEGER NOT NULL,
                start_ts INTEGER,
                end_ts INTEGER,
                owner_key TEXT,
                owner_ip TEXT
            )
            
train_jobsr:   zGALTER TABLE train_jobs ADD COLUMN dataset_name TEXT NOT NULL DEFAULT ''zGALTER TABLE train_jobs ADD COLUMN status TEXT NOT NULL DEFAULT 'queued'r   zBALTER TABLE train_jobs ADD COLUMN message TEXT NOT NULL DEFAULT ''r;   zEALTER TABLE train_jobs ADD COLUMN base_model TEXT NOT NULL DEFAULT ''r<   zJALTER TABLE train_jobs ADD COLUMN base_model_path TEXT NOT NULL DEFAULT ''r=   zJALTER TABLE train_jobs ADD COLUMN preset_key TEXT NOT NULL DEFAULT 'quick'r>   zCALTER TABLE train_jobs ADD COLUMN epochs INTEGER NOT NULL DEFAULT 0r   zBALTER TABLE train_jobs ADD COLUMN imgsz INTEGER NOT NULL DEFAULT 0r   zGALTER TABLE train_jobs ADD COLUMN batch_size INTEGER NOT NULL DEFAULT 0r?   zKALTER TABLE train_jobs ADD COLUMN confirmed_only INTEGER NOT NULL DEFAULT 0r@   zBALTER TABLE train_jobs ADD COLUMN run_dir TEXT NOT NULL DEFAULT ''rA   zCALTER TABLE train_jobs ADD COLUMN log_path TEXT NOT NULL DEFAULT ''rB   zHALTER TABLE train_jobs ADD COLUMN manifest_path TEXT NOT NULL DEFAULT ''rC   zGALTER TABLE train_jobs ADD COLUMN artifact_dir TEXT NOT NULL DEFAULT ''zGALTER TABLE train_jobs ADD COLUMN created_ts INTEGER NOT NULL DEFAULT 0r   z2ALTER TABLE train_jobs ADD COLUMN start_ts INTEGERr   z0ALTER TABLE train_jobs ADD COLUMN end_ts INTEGERz0ALTER TABLE train_jobs ADD COLUMN owner_key TEXTr   z/ALTER TABLE train_jobs ADD COLUMN owner_ip TEXTzhCREATE INDEX IF NOT EXISTS idx_train_jobs_owner_key_created_ts ON train_jobs(owner_key, created_ts DESC)zfCREATE INDEX IF NOT EXISTS idx_train_jobs_owner_ip_created_ts ON train_jobs(owner_ip, created_ts DESC)zFCREATE INDEX IF NOT EXISTS idx_train_jobs_status ON train_jobs(status)a%  
            CREATE TABLE IF NOT EXISTS auto_annotate_jobs (
                id TEXT PRIMARY KEY,
                dataset_id TEXT NOT NULL,
                dataset_name TEXT NOT NULL,
                status TEXT NOT NULL,
                message TEXT NOT NULL DEFAULT '',
                model_key TEXT NOT NULL,
                conf_thresh REAL NOT NULL DEFAULT 0.25,
                imgsz INTEGER NOT NULL DEFAULT 640,
                prompt_classes TEXT NOT NULL DEFAULT '',
                class_mapping TEXT NOT NULL DEFAULT '',
                overwrite INTEGER NOT NULL DEFAULT 0,
                total INTEGER NOT NULL DEFAULT 0,
                processed INTEGER NOT NULL DEFAULT 0,
                updated INTEGER NOT NULL DEFAULT 0,
                skipped_existing INTEGER NOT NULL DEFAULT 0,
                no_detection INTEGER NOT NULL DEFAULT 0,
                created_ts INTEGER NOT NULL,
                start_ts INTEGER,
                end_ts INTEGER,
                owner_key TEXT,
                owner_ip TEXT
            )
            auto_annotate_jobszOALTER TABLE auto_annotate_jobs ADD COLUMN dataset_name TEXT NOT NULL DEFAULT ''zOALTER TABLE auto_annotate_jobs ADD COLUMN status TEXT NOT NULL DEFAULT 'queued'zJALTER TABLE auto_annotate_jobs ADD COLUMN message TEXT NOT NULL DEFAULT ''zLALTER TABLE auto_annotate_jobs ADD COLUMN model_key TEXT NOT NULL DEFAULT ''r   zPALTER TABLE auto_annotate_jobs ADD COLUMN conf_thresh REAL NOT NULL DEFAULT 0.25zLALTER TABLE auto_annotate_jobs ADD COLUMN imgsz INTEGER NOT NULL DEFAULT 640rD   zQALTER TABLE auto_annotate_jobs ADD COLUMN prompt_classes TEXT NOT NULL DEFAULT ''rE   zPALTER TABLE auto_annotate_jobs ADD COLUMN class_mapping TEXT NOT NULL DEFAULT ''rF   zNALTER TABLE auto_annotate_jobs ADD COLUMN overwrite INTEGER NOT NULL DEFAULT 0r   zJALTER TABLE auto_annotate_jobs ADD COLUMN total INTEGER NOT NULL DEFAULT 0r   zNALTER TABLE auto_annotate_jobs ADD COLUMN processed INTEGER NOT NULL DEFAULT 0rG   zLALTER TABLE auto_annotate_jobs ADD COLUMN updated INTEGER NOT NULL DEFAULT 0rH   zUALTER TABLE auto_annotate_jobs ADD COLUMN skipped_existing INTEGER NOT NULL DEFAULT 0rI   zQALTER TABLE auto_annotate_jobs ADD COLUMN no_detection INTEGER NOT NULL DEFAULT 0zOALTER TABLE auto_annotate_jobs ADD COLUMN created_ts INTEGER NOT NULL DEFAULT 0z:ALTER TABLE auto_annotate_jobs ADD COLUMN start_ts INTEGERz8ALTER TABLE auto_annotate_jobs ADD COLUMN end_ts INTEGERz8ALTER TABLE auto_annotate_jobs ADD COLUMN owner_key TEXTz7ALTER TABLE auto_annotate_jobs ADD COLUMN owner_ip TEXTzxCREATE INDEX IF NOT EXISTS idx_auto_annotate_jobs_owner_key_created_ts ON auto_annotate_jobs(owner_key, created_ts DESC)zvCREATE INDEX IF NOT EXISTS idx_auto_annotate_jobs_owner_ip_created_ts ON auto_annotate_jobs(owner_ip, created_ts DESC)zVCREATE INDEX IF NOT EXISTS idx_auto_annotate_jobs_status ON auto_annotate_jobs(status)a  
            CREATE TABLE IF NOT EXISTS face_library_jobs (
                id TEXT PRIMARY KEY,
                action TEXT NOT NULL,
                status TEXT NOT NULL DEFAULT 'queued',
                message TEXT NOT NULL DEFAULT '',
                stage TEXT NOT NULL DEFAULT '',
                processed INTEGER NOT NULL DEFAULT 0,
                total INTEGER NOT NULL DEFAULT 0,
                created_ts INTEGER NOT NULL DEFAULT 0,
                start_ts INTEGER,
                end_ts INTEGER,
                error TEXT NOT NULL DEFAULT '',
                result_json TEXT NOT NULL DEFAULT '{}',
                library_json TEXT NOT NULL DEFAULT '{}'
            )
            face_library_jobsrK   zOALTER TABLE face_library_jobs ADD COLUMN action TEXT NOT NULL DEFAULT 'rebuild'zNALTER TABLE face_library_jobs ADD COLUMN status TEXT NOT NULL DEFAULT 'queued'zIALTER TABLE face_library_jobs ADD COLUMN message TEXT NOT NULL DEFAULT ''rL   zGALTER TABLE face_library_jobs ADD COLUMN stage TEXT NOT NULL DEFAULT ''zMALTER TABLE face_library_jobs ADD COLUMN processed INTEGER NOT NULL DEFAULT 0zIALTER TABLE face_library_jobs ADD COLUMN total INTEGER NOT NULL DEFAULT 0zNALTER TABLE face_library_jobs ADD COLUMN created_ts INTEGER NOT NULL DEFAULT 0z9ALTER TABLE face_library_jobs ADD COLUMN start_ts INTEGERz7ALTER TABLE face_library_jobs ADD COLUMN end_ts INTEGERrM   zGALTER TABLE face_library_jobs ADD COLUMN error TEXT NOT NULL DEFAULT ''rN   zOALTER TABLE face_library_jobs ADD COLUMN result_json TEXT NOT NULL DEFAULT '{}'rO   zPALTER TABLE face_library_jobs ADD COLUMN library_json TEXT NOT NULL DEFAULT '{}'zTCREATE INDEX IF NOT EXISTS idx_face_library_jobs_status ON face_library_jobs(status)zaCREATE INDEX IF NOT EXISTS idx_face_library_jobs_created_ts ON face_library_jobs(created_ts DESC)a  
            CREATE TABLE IF NOT EXISTS dispatch_auth_sessions (
                owner_key TEXT PRIMARY KEY,
                owner_ip TEXT,
                username TEXT NOT NULL DEFAULT '',
                access_token TEXT NOT NULL DEFAULT '',
                refresh_token TEXT NOT NULL DEFAULT '',
                token_type TEXT NOT NULL DEFAULT 'Bearer',
                expires_in INTEGER NOT NULL DEFAULT 0,
                expires_at INTEGER,
                authenticated_ts INTEGER,
                updated_ts INTEGER NOT NULL DEFAULT 0,
                status TEXT NOT NULL DEFAULT 'pending',
                is_mock INTEGER NOT NULL DEFAULT 0,
                last_error TEXT NOT NULL DEFAULT ''
            )
            zuCREATE INDEX IF NOT EXISTS idx_dispatch_auth_owner_ip_updated_ts ON dispatch_auth_sessions(owner_ip, updated_ts DESC)a"  
            CREATE TABLE IF NOT EXISTS dispatch_queue (
                id TEXT PRIMARY KEY,
                owner_key TEXT,
                owner_ip TEXT,
                source_job_id TEXT NOT NULL DEFAULT '',
                source_asset_id TEXT NOT NULL DEFAULT '',
                source_job_type TEXT NOT NULL DEFAULT '',
                source_name TEXT NOT NULL DEFAULT '',
                source_type TEXT NOT NULL DEFAULT '',
                asset_name TEXT NOT NULL DEFAULT '',
                face_index INTEGER NOT NULL DEFAULT 0,
                person_name TEXT NOT NULL DEFAULT '',
                person_id_no TEXT NOT NULL DEFAULT '',
                person_phone TEXT NOT NULL DEFAULT '',
                similarity_score REAL NOT NULL DEFAULT 0,
                illegal_type TEXT NOT NULL DEFAULT '',
                sssj_dm TEXT NOT NULL DEFAULT '',
                sssj_mc TEXT NOT NULL DEFAULT '',
                ssfj_dm TEXT NOT NULL DEFAULT '',
                ssfj_mc TEXT NOT NULL DEFAULT '',
                zbpcs_dm TEXT NOT NULL DEFAULT '',
                zbpcs_mc TEXT NOT NULL DEFAULT '',
                dzmc TEXT NOT NULL DEFAULT '',
                rwdyid TEXT NOT NULL DEFAULT '',
                sjcsly TEXT NOT NULL DEFAULT '',
                dispatch_status TEXT NOT NULL DEFAULT 'pending',
                sms_status TEXT NOT NULL DEFAULT 'pending',
                last_error TEXT NOT NULL DEFAULT '',
                draft_payload_json TEXT NOT NULL DEFAULT '',
                identity_payload_json TEXT NOT NULL DEFAULT '',
                dispatch_response_json TEXT NOT NULL DEFAULT '',
                sms_preview TEXT NOT NULL DEFAULT '',
                created_ts INTEGER NOT NULL DEFAULT 0,
                updated_ts INTEGER NOT NULL DEFAULT 0
            )
            zlCREATE INDEX IF NOT EXISTS idx_dispatch_queue_owner_created_ts ON dispatch_queue(owner_key, created_ts DESC)znCREATE INDEX IF NOT EXISTS idx_dispatch_queue_owner_ip_created_ts ON dispatch_queue(owner_ip, created_ts DESC)zCREATE UNIQUE INDEX IF NOT EXISTS idx_dispatch_queue_owner_source_face_person ON dispatch_queue(owner_key, source_job_id, source_asset_id, face_index, person_id_no)zWCREATE INDEX IF NOT EXISTS idx_dispatch_queue_status ON dispatch_queue(dispatch_status)a  
            CREATE TABLE IF NOT EXISTS dispatch_records (
                id TEXT PRIMARY KEY,
                queue_id TEXT NOT NULL,
                owner_key TEXT,
                owner_ip TEXT,
                status TEXT NOT NULL DEFAULT 'pending',
                request_payload_json TEXT NOT NULL DEFAULT '',
                response_payload_json TEXT NOT NULL DEFAULT '',
                error_message TEXT NOT NULL DEFAULT '',
                created_ts INTEGER NOT NULL DEFAULT 0
            )
            zpCREATE INDEX IF NOT EXISTS idx_dispatch_records_owner_created_ts ON dispatch_records(owner_key, created_ts DESC)am  
            CREATE TABLE IF NOT EXISTS dispatch_sms_records (
                id TEXT PRIMARY KEY,
                queue_id TEXT NOT NULL,
                owner_key TEXT,
                owner_ip TEXT,
                mobile TEXT NOT NULL DEFAULT '',
                content TEXT NOT NULL DEFAULT '',
                status TEXT NOT NULL DEFAULT 'pending',
                request_payload_json TEXT NOT NULL DEFAULT '',
                response_payload_json TEXT NOT NULL DEFAULT '',
                error_message TEXT NOT NULL DEFAULT '',
                created_ts INTEGER NOT NULL DEFAULT 0
            )
            zxCREATE INDEX IF NOT EXISTS idx_dispatch_sms_records_owner_created_ts ON dispatch_sms_records(owner_key, created_ts DESC)r   )init_task_queue_table)ra   r]   rn   shared.task_queuer   commit)r_   columnsdataset_columnsasset_columnstrain_columnsauto_columnsface_library_columnsr   s           r`   init_dbr      s   	 et!#	
 #	
 #	
J $D&11g%%LLW   W$$LL^___''LLGHHH''LLGHHH''LLGHHHW$$LLDEEE7**LLMNNNg%%LLEFFFw&&LLFGGG!00LLPQQQ!00LLPQQQ"'11LLQRRRf	
 	
 	
 	a	
 	
 	
 	QRRRQRRR	
 	
 	
& ,D*==/))LLYZZZ_44LL]   ?**LL_```//LLabbb/11LLcddd?22LLdeee/11LLcddd_,,LL\]]]..LL`aaa..LL`aaafggg	
 	
 	
( *$0@AA--LLefff--LLhiii-//LLghhhM11LLijjjm++LLcddd-''LLabbb=((LLbccc},,LLfggg},,LLfggg}	
 	
 	
 		
 	
 	
8 *$==..LLbccc=((LLbcccM))LL]^^^},,LL`aaaM11LLefff},,LLefff=((LL^___-''LL]^^^},,LLbccc=00LLfgggM))LL]^^^]**LL^___-//LLcddd..LLbccc},,LLbccc]**LLMNNN=((LLKLLLm++LLKLLL]**LLJKKKv	
 	
 	
 	t	
 	
 	
 	]^^^	
 	
 	
8 )/CDD--LLjkkk<''LLjkkkL((LLefffl**LLghhh,,LLklll,&&LLghhh<//LLlmmm,..LLkllll**LLijjj,&&LLefffl**LLijjjL((LLghhh\11LLpqqq--LLlmmm|++LLjkkk\))LLUVVV<''LLSTTTl**LLSTTT\))LLRSSS G	
 	
 	
 	 E	
 	
 	
 	mnnn	
 	
 	
(  17JKK///LLjkkk///LLijjj000LLdeee...LLbccc222LLhiii...LLdeee333LLijjj111LLTUUU///LLRSSS...LLbccc 444LLjkkk!555LLklllklllo	
 	
 	
 		
 	
 	
& 	 D	
 	
 	
 	$&	
 &	
 &	
N 	z	
 	
 	
 	|	
 	
 	
 	 s	
 	
 	
 	e	
 	
 	
 		
 	
 	
 	~	
 	
 	
 		
 	
 	
" 	 G	
 	
 	
 	<;;;;;d###Ke e e e e e e e e e e e e e e e e es   k:lllc                    t          |           }i d|                     dd          d|                     dd          d|                     dd          d|                     dd          d|                     dd          d|                     dd          d	|                     d	          d
|                     d
d          d|                     dd          dt          |                     d          pd          dt          |                     d          pd          dt          |                     d          pd          dt          |                     d          pd          dt          |                     d          pd          dt          |                     d          pd          d|                     d          d|                     d          |                     dd          |                     dd          |                     d          |                     d          |                     d          |                     dd          |                     dt                    t	          j        |d          |                     dd          |                     dd          |                     d d          t	          j        |                     d!          pi d          |                     d"d          d#}t                      5 }|                    d$|           |                                 d d d            d S # 1 swxY w Y   d S )%Nr   oracler    r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Fensure_asciir    r!   r"   r   r$   )r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   a	  
            INSERT INTO jobs (
                job_type, id, source_name, source_type, source_path, temp_dir, frame_interval,
                status, message, total, processed, kept, notfound, failed,
                downloaded, start_ts, end_ts, owner_key, owner_ip, conf_thresh, batch_size,
                imgsz, classes_raw, model_key, zip_paths_json, result_dir, result_manifest_path,
                identity_result_path, identity_summary_json, summary_text
            )
            VALUES (
                :job_type, :id, :source_name, :source_type, :source_path, :temp_dir, :frame_interval,
                :status, :message, :total, :processed, :kept, :notfound, :failed,
                :downloaded, :start_ts, :end_ts, :owner_key, :owner_ip, :conf_thresh, :batch_size,
                :imgsz, :classes_raw, :model_key, :zip_paths_json, :result_dir, :result_manifest_path,
                :identity_result_path, :identity_summary_json, :summary_text
            )
            ON CONFLICT(id) DO UPDATE SET
                job_type = excluded.job_type,
                source_name = excluded.source_name,
                source_type = excluded.source_type,
                source_path = excluded.source_path,
                temp_dir = excluded.temp_dir,
                frame_interval = excluded.frame_interval,
                status = excluded.status,
                message = excluded.message,
                total = excluded.total,
                processed = excluded.processed,
                kept = excluded.kept,
                notfound = excluded.notfound,
                failed = excluded.failed,
                downloaded = excluded.downloaded,
                start_ts = excluded.start_ts,
                end_ts = excluded.end_ts,
                owner_key = excluded.owner_key,
                owner_ip = excluded.owner_ip,
                conf_thresh = excluded.conf_thresh,
                batch_size = excluded.batch_size,
                imgsz = excluded.imgsz,
                classes_raw = excluded.classes_raw,
                model_key = excluded.model_key,
                zip_paths_json = excluded.zip_paths_json,
                result_dir = excluded.result_dir,
                result_manifest_path = excluded.result_manifest_path,
                identity_result_path = excluded.identity_result_path,
                identity_summary_json = excluded.identity_summary_json,
                summary_text = excluded.summary_text
            )	r}   ru   intr   r   dumpsra   r]   r   )ro   rx   payloadr_   s       r`   save_jobr     s   "3''ICGGJ11cggdB 	sww}b11 	sww}b11	
 	sww}b11 	CGGJ++ 	#''"233 	#''(B'' 	3779b)) 	SWWW%%*++ 	S--233 	CGGFOO(q)) 	C
++0q11 	#cggh'',1-- 	c#'',//4155  	CGGJ''!" 	#''(###$ WW["--GGJ++ww}--ggl++!!ww}b11WW[-88*YUCCCgglB// #(> C C #(> C C!%CGG4F,G,G,M2\a!b!b!b33=  GB 
 2t-\ _0	
 0	
 0	
b 	e2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s   #+MM"Mjob_idc                     t                      5 }|                    d| f                                          }d d d            n# 1 swxY w Y   t          |          S )NzSELECT * FROM jobs WHERE id = ?)ra   r]   fetchoner   r   r_   rj   s      r`   get_jobr   5  s    	 Ttll<viHHQQSST T T T T T T T T T T T T T Ts   *AA	A	   r   r   limitr   c                    | s|sg S t          dt          t          |pd          d                    }d}| |g}|r|dz  }|                    |           |dz  }|                    |           t	                      5 }|                    ||                                          }d d d            n# 1 swxY w Y   d |D             S )Nr   r      z
        SELECT *
        FROM jobs
        WHERE status IN ('queued', 'running')
          AND (
                owner_key = ?
             OR (COALESCE(owner_key, '') = '' AND owner_ip = ?)
          )
    z AND job_type = ?z=
        ORDER BY start_ts DESC, id DESC
        LIMIT ?
    c                 0    g | ]}|t          |          S Nr   rh   s     r`   rw   z$list_active_jobs.<locals>.<listcomp>Z  !    @@@Krb   )maxminr   appendra   r]   rl   )	r   r   r   r   
safe_limitqueryparamsr_   rm   s	            r`   list_active_jobsr   ;  s'     X 	QC,,c2233JE #H-F  $$h	  E MM*	 6t||E6**33556 6 6 6 6 6 6 6 6 6 6 6 6 6 6@@@@@@s   >)B33B7:B7r   c                 x   |                      dd          |                      dd          |                      dd          t          j        |                      d          pg d          |                      dd	          t          |                      d
          pd          t          |                      d          pd          t          |                      d          pd          t          |                      d          pd          |                      dd          t          |                      d          pd          t          |                      d          pd          d}t	                      5 }|                    d|           |                                 d d d            d S # 1 swxY w Y   d S )Nr   r   r&   r'   r   Fr   r   draftr)   r   r*   r+   r,   r-   r.   r/   r%   a
  
            INSERT INTO datasets (
                id, name, notes, class_names_json, status, image_count, labeled_count,
                reviewed_count, version_count, root_dir, created_ts, updated_ts
            )
            VALUES (
                :id, :name, :notes, :class_names_json, :status, :image_count, :labeled_count,
                :reviewed_count, :version_count, :root_dir, :created_ts, :updated_ts
            )
            ON CONFLICT(id) DO UPDATE SET
                name = excluded.name,
                notes = excluded.notes,
                class_names_json = excluded.class_names_json,
                status = excluded.status,
                image_count = excluded.image_count,
                labeled_count = excluded.labeled_count,
                reviewed_count = excluded.reviewed_count,
                version_count = excluded.version_count,
                root_dir = excluded.root_dir,
                created_ts = excluded.created_ts,
                updated_ts = excluded.updated_ts
            )ru   r   r   r   ra   r]   r   )r   r   r_   s      r`   save_datasetr   ]  s   kk$##FB''Wb)) Jw{{='A'A'GRV[\\\++h007;;}55:;;W[[99>Q??gkk*:;;@qAAW[[99>Q??KK
B//'++l338q99'++l338q99 G 
 t, /	
 	
 	
2 	5                 s   7+F//F36F3r1   c                     t                      5 }|                    d| f                                          }d d d            n# 1 swxY w Y   t          |          S )Nz#SELECT * FROM datasets WHERE id = ?)ra   r]   r   r   r1   r_   rj   s      r`   get_datasetr     s    	 \tll@:-PPYY[[\ \ \ \ \ \ \ \ \ \ \ \ \ \ \3r   assetc                    |                      dd          |                      dd          |                      dd          |                      dd          |                      dd          |                      dd          |                      d	d          |                      d
d          t          |                      d          pd          t          |                      d          pd          t          |                      d          pd          t          |                      d          pd          d}t                      5 }|                    d|           |                                 d d d            d S # 1 swxY w Y   d S )Nr   r   r1   r2   r3   r
   zipr4   r5   r6   r7   r   r8   r9   r.   r0   a  
            INSERT INTO dataset_assets (
                id, dataset_id, filename, origin_name, source_type, source_job_id, source_asset_id, file_path,
                width, height, size_bytes, created_ts
            )
            VALUES (
                :id, :dataset_id, :filename, :origin_name, :source_type, :source_job_id, :source_asset_id, :file_path,
                :width, :height, :size_bytes, :created_ts
            )
            ON CONFLICT(id) DO UPDATE SET
                dataset_id = excluded.dataset_id,
                filename = excluded.filename,
                origin_name = excluded.origin_name,
                source_type = excluded.source_type,
                source_job_id = excluded.source_job_id,
                source_asset_id = excluded.source_asset_id,
                file_path = excluded.file_path,
                width = excluded.width,
                height = excluded.height,
                size_bytes = excluded.size_bytes,
                created_ts = excluded.created_ts
            ru   r   ra   r]   r   )r   r   r_   s      r`   save_dataset_assetr     s   iib!!iib11IIj"--yy33yy66?B77 99%6;;YY{B//UYYw'',1--eii)).Q//%))L116Q77%))L116Q77 G 
 t, /	
 	
 	
2 	5                 s   +E>>FFc                     t                      5 }|                    d| f                                          }d d d            n# 1 swxY w Y   t          ||d         nd          S )NzASELECT COUNT(*) AS total FROM dataset_assets WHERE dataset_id = ?r   r   )ra   r]   r   r   r   s      r`   count_dataset_assetsr     s    	 tllOM
 
 (** 	              
 ss7||A666r   d   c                    t          dt          t          |pd          d                    }t                      5 }|                    d| |f                                          }d d d            n# 1 swxY w Y   d |D             S )Nr   r     z
            SELECT *
            FROM dataset_assets
            WHERE dataset_id = ?
            ORDER BY created_ts DESC, id DESC
            LIMIT ?
            c                 0    g | ]}|t          |          S r   )r   rh   s     r`   rw   z'list_dataset_assets.<locals>.<listcomp>  s"    JJJ3#/!#&&///rb   r   r   r   ra   r]   rl   )r1   r   r   r_   rm   s        r`   list_dataset_assetsr     s    QC--s3344J	 
t|| $	
 	
 (** 	
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 KJ$JJJJs   +A33A7:A7asset_idc                     t                      5 }|                    d| |f                                          }d d d            n# 1 swxY w Y   t          |          S )Nz
            SELECT *
            FROM dataset_assets
            WHERE dataset_id = ? AND id = ?
            LIMIT 1
            )ra   r]   r   r   )r1   r   r_   rj   s       r`   get_dataset_assetr     s    	 	tll "
 
 (** 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 !%%%s   +AA
A
c                    t          dt          t          | pd          d                    }t                      5 }|                    d|f                                          }d d d            n# 1 swxY w Y   d |D             S )Nr   r   r   z
            SELECT *
            FROM datasets
            ORDER BY updated_ts DESC, created_ts DESC, id DESC
            LIMIT ?
            c                 0    g | ]}|t          |          S r   )r   rh   s     r`   rw   z!list_datasets.<locals>.<listcomp>  s!    DDDSCOOC  OOOrb   r   r   r   r_   rm   s       r`   list_datasetsr     s    QC--s3344J	 	t|| M
 
 (** 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 EDDDDDD   *A22A69A6c                    i d|                      dd          d|                      dd          d|                      dd          d|                      dd          d|                      dd          d|                      dd          d	|                      d	d          d
|                      d
d          dt          |                      d          pd          dt          |                      d          pd          dt          |                      d          pd          d|                      d          rdndd|                      dd          d|                      dd          d|                      dd          d|                      dd          dt          |                      d          pd          |                      d          |                      d          |                      dd          |                      dd          d}t                      5 }|                    d|           |                                 d d d            d S # 1 swxY w Y   d S )Nr   r   r1   r:   r   queuedr   r;   r<   r=   quickr>   r   r   r   r?   r   r@   rA   rB   rC   r.   r   r   r   r   r   r   r   r   a  
            INSERT INTO train_jobs (
                id, dataset_id, dataset_name, status, message, base_model, base_model_path, preset_key,
                epochs, imgsz, batch_size, confirmed_only, run_dir, log_path, manifest_path, artifact_dir,
                created_ts, start_ts, end_ts, owner_key, owner_ip
            )
            VALUES (
                :id, :dataset_id, :dataset_name, :status, :message, :base_model, :base_model_path, :preset_key,
                :epochs, :imgsz, :batch_size, :confirmed_only, :run_dir, :log_path, :manifest_path, :artifact_dir,
                :created_ts, :start_ts, :end_ts, :owner_key, :owner_ip
            )
            ON CONFLICT(id) DO UPDATE SET
                dataset_id = excluded.dataset_id,
                dataset_name = excluded.dataset_name,
                status = excluded.status,
                message = excluded.message,
                base_model = excluded.base_model,
                base_model_path = excluded.base_model_path,
                preset_key = excluded.preset_key,
                epochs = excluded.epochs,
                imgsz = excluded.imgsz,
                batch_size = excluded.batch_size,
                confirmed_only = excluded.confirmed_only,
                run_dir = excluded.run_dir,
                log_path = excluded.log_path,
                manifest_path = excluded.manifest_path,
                artifact_dir = excluded.artifact_dir,
                created_ts = excluded.created_ts,
                start_ts = excluded.start_ts,
                end_ts = excluded.end_ts,
                owner_key = excluded.owner_key,
                owner_ip = excluded.owner_ip
            r   ro   r   r_   s      r`   save_train_jobr     s   cggdBcgglB// 	33 	#''(H--	
 	3779b)) 	cgglB// 	377#4b99 	cgglG44 	#cggh'',1-- 	SWWW%%*++ 	c#'',//4155 	sww'788?!!a 	3779b)) 	CGGJ++ 	"55  	33!" 	c#'',//4155#$ GGJ''''(##WW["--GGJ+++  G0 
 %t B E#	
 #	
 #	
H 	K% % % % % % % % % % % % % % % % % %s   (+I  I$'I$c                     t                      5 }|                    d| f                                          }d d d            n# 1 swxY w Y   t          |          S )Nz%SELECT * FROM train_jobs WHERE id = ?)ra   r]   r   r   r   s      r`   get_train_jobr   4  s    	 ZtllBVINNWWYYZ Z Z Z Z Z Z Z Z Z Z Z Z Z ZS!!!r   c                 "   | s|sg S t          dt          t          |pd          d                    }t                      5 }|                    d| ||f                                          }d d d            n# 1 swxY w Y   d |D             S )Nr   r   r   z
            SELECT *
            FROM train_jobs
            WHERE owner_key = ?
               OR (COALESCE(owner_key, '') = '' AND owner_ip = ?)
            ORDER BY created_ts DESC, id DESC
            LIMIT ?
            c                 0    g | ]}|t          |          S r   )r   rh   s     r`   rw   z#list_train_jobs.<locals>.<listcomp>K  s"    FFFscoc""ooorb   r   r   r   r   r   r_   rm   s         r`   list_train_jobsr   :  s     X 	QC,,c2233J	 t|| *-

 

 (** 	               GFdFFFF   ,A::A>A>c                 J   i d|                      dd          d|                      dd          d|                      dd          d|                      dd          d|                      dd          d|                      dd          d	t          |                      d	          pd
          dt          |                      d          pd          d|                      dd          d|                      dd          d|                      d          rdnddt          |                      d          pd          dt          |                      d          pd          dt          |                      d          pd          dt          |                      d          pd          dt          |                      d          pd          dt          |                      d          pd          |                      d          |                      d          |                      dd          |                      dd          d}t                      5 }|                    d|           |                                 d d d            d S # 1 swxY w Y   d S )Nr   r   r1   r:   r   r   r   r   r   g      ?r   i  rD   rE   rF   r   r   r   r   rG   rH   rI   r.   r   r   r   r   r   a  
            INSERT INTO auto_annotate_jobs (
                id, dataset_id, dataset_name, status, message, model_key, conf_thresh, imgsz,
                prompt_classes, class_mapping, overwrite, total, processed, updated,
                skipped_existing, no_detection, created_ts, start_ts, end_ts, owner_key, owner_ip
            )
            VALUES (
                :id, :dataset_id, :dataset_name, :status, :message, :model_key, :conf_thresh, :imgsz,
                :prompt_classes, :class_mapping, :overwrite, :total, :processed, :updated,
                :skipped_existing, :no_detection, :created_ts, :start_ts, :end_ts, :owner_key, :owner_ip
            )
            ON CONFLICT(id) DO UPDATE SET
                dataset_id = excluded.dataset_id,
                dataset_name = excluded.dataset_name,
                status = excluded.status,
                message = excluded.message,
                model_key = excluded.model_key,
                conf_thresh = excluded.conf_thresh,
                imgsz = excluded.imgsz,
                prompt_classes = excluded.prompt_classes,
                class_mapping = excluded.class_mapping,
                overwrite = excluded.overwrite,
                total = excluded.total,
                processed = excluded.processed,
                updated = excluded.updated,
                skipped_existing = excluded.skipped_existing,
                no_detection = excluded.no_detection,
                created_ts = excluded.created_ts,
                start_ts = excluded.start_ts,
                end_ts = excluded.end_ts,
                owner_key = excluded.owner_key,
                owner_ip = excluded.owner_ip
            )ru   floatr   ra   r]   r   r   s      r`   save_auto_annotate_jobr  N  s   cggdBcgglB// 	33 	#''(H--	
 	3779b)) 	SWW["-- 	uSWW]33;t<< 	SWWW%%,-- 	#''"2B77 	"55 	#''+..5QQA 	SWWW%%*++ 	S--233 	3swwy)).Q// 	C(: ; ; @qAA  	CGGN338q99!" 	c#'',//4155#$ GGJ''''(##WW["--GGJ+++  G0 
 %t B E#	
 #	
 #	
H 	K% % % % % % % % % % % % % % % % % %s    +JJJc                     t                      5 }|                    d| f                                          }d d d            n# 1 swxY w Y   t          |          S )Nz-SELECT * FROM auto_annotate_jobs WHERE id = ?)ra   r]   r   r   r   s      r`   get_auto_annotate_jobr    s    	 btllJVIVV__aab b b b b b b b b b b b b b b$S)))r   c                 "   | s|sg S t          dt          t          |pd          d                    }t                      5 }|                    d| ||f                                          }d d d            n# 1 swxY w Y   d |D             S )Nr   r   r   z
            SELECT *
            FROM auto_annotate_jobs
            WHERE owner_key = ?
               OR (COALESCE(owner_key, '') = '' AND owner_ip = ?)
            ORDER BY created_ts DESC, id DESC
            LIMIT ?
            c                 0    g | ]}|t          |          S r   )r   rh   s     r`   rw   z+list_auto_annotate_jobs.<locals>.<listcomp>  s"    NNNsco%c**ooorb   r   r   s         r`   list_auto_annotate_jobsr    s     X 	QC,,c2233J	 t|| *-

 

 (** 	               ONdNNNNr   c                    |                      dd          |                      dd          |                      dd          |                      dd          |                      dd          t          |                      d	          pd
          t          |                      d          pd
          t          |                      d          p|                      d          pd
          |                      d          |                      d          |                      dd          t          j        |                      d          pi d          t          j        |                      d          pi d          d}t	                      5 }|                    d|           |                                 d d d            d S # 1 swxY w Y   d S )Nr   r   rK   rebuildr   r   r   rL   r   r   r   r.   r   r   rM   r   Fr   r   rJ   a  
            INSERT INTO face_library_jobs (
                id, action, status, message, stage, processed, total, created_ts,
                start_ts, end_ts, error, result_json, library_json
            )
            VALUES (
                :id, :action, :status, :message, :stage, :processed, :total, :created_ts,
                :start_ts, :end_ts, :error, :result_json, :library_json
            )
            ON CONFLICT(id) DO UPDATE SET
                action = excluded.action,
                status = excluded.status,
                message = excluded.message,
                stage = excluded.stage,
                processed = excluded.processed,
                total = excluded.total,
                created_ts = excluded.created_ts,
                start_ts = excluded.start_ts,
                end_ts = excluded.end_ts,
                error = excluded.error,
                result_json = excluded.result_json,
                library_json = excluded.library_json
            )ru   r   r   r   ra   r]   r   r   s      r`   save_face_library_jobr    s   ggdB''(I..''(H--779b))"%%--233SWWW%%*++#'',//K377:3F3FK!LLGGJ''''(##"%%z#''("3"3"9rNNN
3779#5#5#;%PPP G  
 t. 1	
 	
 	
4 	7                 s   
+GG	Gc                     t                      5 }|                    d| f                                          }d d d            n# 1 swxY w Y   t          |          S )Nz,SELECT * FROM face_library_jobs WHERE id = ?ra   r]   r   r   r   s      r`   get_face_library_jobr    s    	 atllIF9UU^^``a a a a a a a a a a a a a a a#C(((r   c                      t                      5 } |                     d                                          }d d d            n# 1 swxY w Y   t          |          S )Nz
            SELECT *
            FROM face_library_jobs
            WHERE status IN ('queued', 'running')
            ORDER BY created_ts ASC, id ASC
            LIMIT 1
            r  )r_   rj   s     r`   get_active_face_library_jobr    s    	 	tll
 
 (** 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 $C(((s   (AA
Ac                    t          dt          t          | pd          d                    }t                      5 }|                    d|f                                          }d d d            n# 1 swxY w Y   d |D             S )Nr   r   r   z
            SELECT *
            FROM face_library_jobs
            ORDER BY created_ts DESC, id DESC
            LIMIT ?
            c                 0    g | ]}|t          |          S r   )r   rh   s     r`   rw   z*list_face_library_jobs.<locals>.<listcomp>  s"    MMMcS_$S))___rb   r   r   s       r`   list_face_library_jobsr    s    QC,,c2233J	 	t|| M
 
 (** 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 NMTMMMMr   2   c                 "   | s|sg S t          dt          t          |pd          d                    }t                      5 }|                    d| ||f                                          }d d d            n# 1 swxY w Y   d |D             S )Nr   r  r   z
            SELECT *
            FROM jobs
            WHERE owner_key = ?
               OR (COALESCE(owner_key, '') = '' AND owner_ip = ?)
            ORDER BY start_ts DESC
            LIMIT ?
            c                 0    g | ]}|t          |          S r   r   rh   s     r`   rw   zlist_jobs.<locals>.<listcomp>  r   rb   r   r   s         r`   	list_jobsr    s     X 	QC,,c2233J	 t|| *-

 

 (** 	               A@@@@@r   c                 &    t          | ||          S )N)r   )r  )r   r   r   s      r`   list_saved_jobsr    s    Y6666rb   c                    t          dt          t          | pd          d                    }t                      5 }|                    d|f                                          }d d d            n# 1 swxY w Y   d |D             S )Nr   r  r   zx
            SELECT *
            FROM jobs
            ORDER BY start_ts DESC, id DESC
            LIMIT ?
            c                 0    g | ]}|t          |          S r   r   rh   s     r`   rw   z!list_all_jobs.<locals>.<listcomp>  r   rb   r   r   s       r`   list_all_jobsr    s    QC,,c2233J	 	t|| M
 
 (** 		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 A@@@@@r      daysc           	         t          t          j                              t          | d          dz  dz  dz  z
  }t                      5 }|                    d|f                                          }g }|D ].}|                    |d                    	 t          j        |d         pd          }n# t          $ r g }Y nw xY w|D ]q}|rmt          j                            |          rN	 t          j        |           9# t          $ r Y Et          $ r }t          j        d||           Y d }~id }~ww xY wr|d	         }	|	rot          j                            |	          rP	 t%          j        |	d
           n8# t          $ r Y n,t          $ r }t          j        d|	|           Y d }~nd }~ww xY w|d         }
|
rot          j                            |
          rP	 t%          j        |
d
           n8# t          $ r Y n,t          $ r }t          j        d|
|           Y d }~nd }~ww xY w|d         }|rpt          j                            |          rQ	 t          j        |           # t          $ r Y t          $ r!}t          j        d||           Y d }~&d }~ww xY w0|r4|                    dd |D                        |                                 d d d            n# 1 swxY w Y   t-          |          S )Nr      <   z
            SELECT id, zip_paths_json, result_dir, source_path, temp_dir
            FROM jobs
            WHERE end_ts IS NOT NULL AND end_ts < ?
            r   r   r   z failed to remove zip file %s: %sr    F)ignore_errorsz"failed to remove result dir %s: %sr   z failed to remove temp dir %s: %sr   z#failed to remove source file %s: %szDELETE FROM jobs WHERE id = ?c                     g | ]}|fS rg   rg   )ri   r   s     r`   rw   z$cleanup_old_jobs.<locals>.<listcomp>W  s    >b>b>bVy>b>b>brb   )r   timer   ra   r]   rl   r   r   r   r   rU   rV   isfileremoveFileNotFoundErrorr   warningisdirshutilrmtreeexecutemanyr   r   )r  cutoffr_   rm   
delete_idsrj   rx   rV   excr    r   r   s               r`   cleanup_old_jobsr0  "  s   D!r 1B 6 ;;F	 4t||
 I
 
 (** 	 
 %	\ %	\Cc$i((( Js+;'<'DEE		   			! V V VBGNN400 VV	$,   $ V V V'I4QTUUUUUUUUV\*J ZbgmmJ77 ZZM*EBBBBB(   D  Z Z ZN#GUXYYYYYYYYZ:H VBGMM(33 VVM(%@@@@@(   D  V V VN#ExQTUUUUUUUUVm,K \rw~~k:: \\Ik****(   D  \ \ \N#H+WZ[[[[[[[[\  	<>b>bWa>b>b>bcccKKMMMi4 4 4 4 4 4 4 4 4 4 4 4 4 4 4l z??s  	AK)B32K)3C?K)C(K)+D ?K) 
D5	
K)	D5	D0	+K)0D5	5-K)#E:9K):
F/K)	F/F*%K)*F//,K)G32K)3
H(=K)?	H(H#K)#H((,K)I+)K)+
J"5K)8	J"JK)J"";K))K-0K-	job_typesc                    t          t          j                              }d | pg D             }d}|g}|r=d                    d |D                       }|d| dz  }|                    |           t	                      5 }|                    d| d|          }|                                 |j        cd d d            S # 1 swxY w Y   d S )	Nc                     g | ]D}t          |                                          #t          |                                          ES rg   )strstrip)ri   items     r`   rw   z1mark_running_jobs_interrupted.<locals>.<listcomp>_  s=    [[[DTIZIZ[c$iioo''[[[rb   zstatus = 'running'z, c              3      K   | ]}d V  dS )?Nrg   )ri   _s     r`   	<genexpr>z0mark_running_jobs_interrupted.<locals>.<genexpr>c  s"       = = = = = = = =rb   z AND job_type IN (re   a<  
            UPDATE jobs
            SET status = 'interrupted',
                end_ts = COALESCE(end_ts, ?),
                message = CASE
                    WHEN message IS NULL OR message = '' THEN 'service restarted before job completed'
                    ELSE message
                END
            WHERE z
            )r   r$  joinextendra   r]   r   rowcount)r1  nowsafe_job_typeswherer   placeholdersr_   cursors           r`   mark_running_jobs_interruptedrC  ]  s8   
dikk

C[[Y_"[[[N EF &yy = =n = = ===5l5555n%%%	 t	 	 	 	 
 
 	                 s   5CC
C)rP   N)r   N)r   )r   )r  )r  r   );r   rU   rY   r$  r*  typingr   shared.config.configr   r   r   r   r   r   r   r   r   
Connectionra   r4  setrn   rt   r|   r}   r[   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r  r  r  r  r0  rC  rg   rb   r`   <module>rH     s    				          F F F F F F F F F FB  0 0 "	'$ 	 	 	 	)G. )C )CH ) ) ) )
DcN tCy    W[4' DcNT,A    *t+ S#X0E    Ew{T1 Ed38nt6K E E E EA7;- A$sCx.42G A A A AI7;#5 I$sCx.4:O I I I I'+"4 c3h$9N    *f f f fRU$sCx. UT U U U UpC DcNT1     	A AAA A Dj	A
 
$sCx.A A A AD*$sCx. *T * * * *Z C  DcNT$9        *d38n * * * * *Z7S 7S 7 7 7 7K KC K Kd4S>>R K K K K &# & &c3h$9N & & & &E E EtDcN'; E E E E>S#X >4 > > > >B"# "$sCx.4"7 " " " "G Gs Gc G# GtDQTVYQYNG[ G G G G(>S#X >4 > > > >B*# *$sCx.4*? * * * *O Os Oc O# OtTXY\^aYaTbOc O O O O(,tCH~ ,$ , , , ,^) )c3h$)> ) ) ) ))T#s(^d%: ) ) ) )N N# NtDcN/C N N N NA A A AC Ad3PS8nAU A A A A(7 7s 7c 7# 7tDQTVYQYNG[ 7 7 7 7A A Ad4S>&: A A A A8 83 8s 8 8 8 8v T#Y-=       rb   