yjc'dZddlmZddlZddlZddlZddlZddlmZddl m Z m Z m Z m Z mZddlmZddlmZedz d z Zd Zd ZejZejjZddZd?dZ ddd@dZ!dAd Z"dBd#Z#d=d$Z$dCd)Z%dDd+Z&ddd,dEd/Z'dFd0Z(dd1dd2dGd6Z)dHd7Z*dId;Z+dS)Ja Profile-local durable audit ledger for cron execution attempts. The ledger records what is known about each attempt; it is not a retry queue. Interrupted attempts become ``unknown`` only after their exact owner process is proved gone. Terminal states are immutable. ) annotationsN)contextmanager)AnyDictIteratorListOptional)get_hermes_home)nowcronz executions.dbi) completedfailedunknownreturnsqlite3.ConnectioncztjddtjtdS)NT)parentsexist_ok)timeout)EXECUTIONS_FILEparentmkdirsqlite3connect5/home/ragecks/.hermes/hermes-agent/cron/executions.py_connectrs3   === ??A 6 6 66rconnNonec ddlm}tj|_|d||d|d|d|d|d dS) Nr)apply_wal_with_fallbackzPRAGMA busy_timeout=5000zcron/executions.db)db_labelzPRAGMA synchronous=FULLaCREATE TABLE IF NOT EXISTS executions ( id TEXT PRIMARY KEY, job_id TEXT NOT NULL, source TEXT NOT NULL, process_id TEXT NOT NULL, pid INTEGER NOT NULL, process_started_at INTEGER, status TEXT NOT NULL CHECK(status IN ('claimed','running','completed','failed','unknown')), claimed_at TEXT NOT NULL, started_at TEXT, finished_at TEXT, error TEXT )zeCREATE INDEX IF NOT EXISTS idx_executions_job_claimed ON executions(job_id, claimed_at DESC, id DESC)zhCREATE INDEX IF NOT EXISTS idx_executions_status_claimed ON executions(status, claimed_at DESC, id DESC)) hermes_stater#rRow row_factoryexecute)r r#s r_initialize_schemar) s444444{DLL+,,,D+?@@@@LL*+++LL   LL : LL :rIterator[sqlite3.Connection]c# Kt5t} t||5|Vdddn #1swxYwY|n#|wxYw ddddS#1swxYwYdS)aOpen a connection, commit/rollback on exit, always close. ``sqlite3.Connection.__enter__``/``__exit__`` only commit or roll back the transaction; it does not close the connection. Relying on that alone leaks a connection (and its WAL/SHM file descriptors) on every call, since closing then depends on the garbage collector. Schema init runs inside the ``try`` too, so a PRAGMA/DDL failure after a successful ``connect()`` still closes the connection instead of leaking it. N)_lockrr)close)r s r _transactionr.As zz  t $ $ $                   JJLLLLDJJLLLLLsHBA< AA AA ABA22BB BrowOptional[sqlite3.Row]Optional[Dict[str, Any]]c(|t|ndS)Ndict)r/s r_recordr5Vs4999T1rdelivery_outcomerecordr7 Optional[str]cP ddlm}|||dS#t$rYdSwxYw)zFProject durable state to monitoring without affecting ledger behavior.r)emit_execution_stater6N)agent.monitoring.cron_healthr; Exception)r8r7r;s r_emit_execution_stater>ZsX EEEEEEV6FGGGGGG      s  %%pidint Optional[int]cH ddlm}||S#t$rYdSwxYw)Nr)get_process_start_time)gateway.statusrCr=)r?rCs r_process_start_timerEfsL999999%%c*** tts  !! started_atboolc ddlm}||sdSn#t$rYdSwxYw||tjkSt |}|duo||kS)Nr) _pid_existsFT)rDrIr=osgetpidrE)r?rFrIcurrents r_owner_is_liverMns......{3 5  ttbikk!!!#&&G $  87j#88s  $$cxtdtt}|d|fdS)NrzDELETE FROM executions WHERE id IN ( SELECT id FROM executions WHERE status IN ('completed','failed','unknown') ORDER BY claimed_at DESC, id DESC LIMIT -1 OFFSET ? ))maxr@MAX_TERMINAL_EXECUTIONSr()r limits r_prune_unlockedrR{sF 3.// 0 0ELL   rjob_idstrsourceDict[str, Any]c t}tjj}t j}t5}|d|t|t|t|t||f|d|f }dddn #1swxYwYt|}t||S)z)rSrUr execution_idr?r r/r8s rcreate_executionras -- ! ! # #C:<<#L )++C  4  :3v;;F [#  % %s ,    ll 1L?  (**                S\\F&!!! MsA3CC Cr`c~t}t5}|d||f}|jdkr ddddSt |d|f}dddn #1swxYwYt||S)z7Transition one claimed attempt to running exactly once.zcUPDATE executions SET status='running', started_at=? WHERE id=? AND status='claimed'NrX)rYrZr.r(rowcountr5r_r>)r`r r curr8s rmark_execution_runningrfs -- ! ! # #C  4ll 2 ,    <1           1L?  (**               &!!! Ms%B#!6B##B'*B')errorr7successrgct}|rdnd}|rdn|rt|nd}t5}|d||||f}|jdkr ddddSt |t|d|f} dddn #1swxYwYt| || S) zDWrite a terminal result once; terminal attempts cannot be rewritten.r rNzunknown failureztUPDATE executions SET status=?, finished_at=?, error=? WHERE id=? AND status IN ('claimed','running')rcrXr6) rYrZrTr.r(rdrRr5r_r>) r`rhrgr7r statusdetailr rer8s rfinish_executionrlsZ -- ! ! # #C# 1[[F NTT#M3u:::|d|d|df}||jz }|jrSt|d |df }|| ||rt|d d d n #1swxYwY|D]}t||S) zDMark provably abandoned attempts unknown without scheduling retries.rzsSELECT id, process_id, pid, process_started_at FROM executions WHERE status IN ('claimed','running') process_idr?process_started_atzUPDATE executions SET status='unknown', finished_at=?, error=? WHERE id=? AND status IN ('claimed','running')z}Scheduler restarted after this execution's owner exited before a durable terminal state; whether side effects ran is unknown.idrXN)rYrZr.r(fetchallr^rMr@rdr5r_appendrRr>)r changed recoveredr rowsr/rer8s rrecover_interrupted_executionsrvs -- ! ! # #CG&(I "4|| 8   (**  - -C< K//c#e*oos3G/HII ,,EHTC s| #G| - 9CI<""(**%$$V,,,  " D ! ! !5"""""""""""""""6&&f%%%% NsC=D<<EE2)rSrQbefore_claimed_atrQrxList[Dict[str, Any]]c bg}g}|7|d|t||7|d|t||rdd|znd}|tdt t |dt 5}|d |zd z|}dddn #1swxYwYd |DS) zFReturn indexed, newest-first execution history with cursor pagination.Nzjob_id=?zclaimed_at < ?z WHERE z AND rcizSELECT * FROM executionsz* ORDER BY claimed_at DESC, id DESC LIMIT ?c,g|]}t|Srr3.0r/s r z#list_executions..s & & &#DII & & &r) rrrTjoinrOminr@r.r(rq)rSrQrxclausesparamswherer rus rlist_executionsrsd GF z""" c&kk"""$'((( c+,,---18 @I W-- - -bE MM#aSZZ--../// 4|| & .: ;    (**  ' & & & &&s/DD!Dc<t|d}|r|dndS)Nrc)rSrQr)r)rSrus rlatest_executionrs' & 2 2 2D $477$rjob_ids List[str]Dict[str, Dict[str, Any]]cLdt|D}|siSdd|D}t5}|d|d|}dddn #1swxYwYd|DS)z9Load latest execution for many jobs in one indexed query.c0g|]}|t|Sr)rT)r~rSs rrz%latest_executions.. s# H H HV HS[[ H H Hr,c3K|]}dVdS)?Nr)r~_s r z$latest_executions..s"//AC//////rz@SELECT e.* FROM executions e WHERE e.job_id IN (z) AND e.id=(SELECT e2.id FROM executions e2 WHERE e2.job_id=e.job_id ORDER BY e2.claimed_at DESC, e2.id DESC LIMIT 1)Nc:i|]}|dt|S)rSr3r}s r z%latest_executions..s$ 5 5 5CM499 5 5 5r)r4fromkeysrr.r(rq)rclean placeholdersr rus rlatest_executionsr s H Ht}}W'='= H H HE  88///////L 4|| P$0 P P P     (**  6 5 5 5 55s-BBB)rr)r rrr!)rr*)r/r0rr1)r8r1r7r9rr!)r?r@rrA)r?r@rFrArrG)rSrTrUrTrrV)r`rTrr1) r`rTrhrGrgr9r7r9rr1)rr@)rSr9rQr@rxr9rry)rSrTrr1)rrrr),__doc__ __future__rrJr threadingr[ contextlibrtypingrrrrr hermes_constantsr hermes_timer rYresolverrP_TERMINAL_STATESRLockr,r\r]r^rr)r.r5r>rErMrRrarfrlrvrrrrrrrsu#"""""  %%%%%%66666666666666,,,,,,******!/##++--6H5 djll 7777 B(2222 LP        9 9 9 9    ,&AE&*0""""L $"'+''''''0%%%% 666666r