فهرست منبع

Requeue crash-orphaned sessions when an idle saddle asks for work

A daemon that asks /api/saddle/next-task is provably idle (its ask
loop parks while any session runs), so an open session row on an
assigned/working task can only be a crash orphan: both sides
restart, nobody closes the row, and the open session IS the busy
record - the saddle asks, the claim rolls back, forever, while the
UI shows it busy with the interrupted task.

The ask now requeues every orphan older than a two-minute grace
window (requeue_task closes the session, frees the busy record,
clears suspension stamps, adds a system comment), so the very ask
that found the orphan can win the task straight back and continue
it. The grace window protects a manual /start whose session row
lands while an ask is in flight.
Billy Barrow 1 ماه پیش
والد
کامیت
3d8a8b1897
2فایلهای تغییر یافته به همراه55 افزوده شده و 0 حذف شده
  1. 11 0
      src/Api/SaddleApi.vala
  2. 44 0
      src/Services/TaskService.vala

+ 11 - 0
src/Api/SaddleApi.vala

@@ -746,6 +746,7 @@ namespace ClankerSaddle.Api {
         private SaddleService saddles = inject<SaddleService>();
         private AllocationService allocation = inject<AllocationService>();
         private ProjectService projects = inject<ProjectService>();
+        private TaskService tasks = inject<TaskService>();
         private SaddlePollState poll_state = inject<SaddlePollState>();
         private InvercargillSql.Orm.OrmSession db = inject<InvercargillSql.Orm.OrmSession>();
         private StatePulse pulse = inject<StatePulse>();
@@ -768,6 +769,16 @@ namespace ClankerSaddle.Api {
                 print(@"[ClankerSaddle] Saddle '$(((!)saddle).name)' asked while offline; marking idle\n");
             }
 
+            // Crash recovery: an asking daemon is provably idle, so any
+            // open session on an assigned/working task is a crash orphan
+            // wedging the busy record. Requeue it — the very ask that
+            // found it can win the task straight back.
+            var recovered = yield tasks.requeue_interrupted_sessions((!)saddle);
+            if (recovered > 0) {
+                print(@"[ClankerSaddle] Recovered $(recovered) interrupted session(s) on saddle '$(((!)saddle).name)'\n");
+                yield pulse.bump(0, "crash-orphaned session(s) requeued");
+            }
+
             // The ask's arrival answers any pending wake (the lost-wake
             // guard must not fire for a saddle that just asked); the
             // outcome is recorded once the allocation resolves.

+ 44 - 0
src/Services/TaskService.vala

@@ -1472,6 +1472,50 @@ namespace ClankerSaddle.Services {
             yield pulse.bump(task_id, "task requeued");
         }
 
+        /**
+         * Crash recovery at the ask: a daemon that asks for work is
+         * provably idle — its own state machine parks the ask loop
+         * while any session runs — so an open session on an
+         * assigned/working task can only be a crash orphan (both
+         * sides restart, nobody closes the row, and the busy record
+         * then wedges the fleet: the saddle asks, the claim rolls
+         * back, forever, while the UI shows it busy). Every orphan
+         * older than the grace window requeues — the very ask that
+         * found it can win the task straight back. The grace window
+         * protects a manual /start whose session row lands while an
+         * ask is in flight.
+         */
+        public const int64 ASK_RECOVERY_GRACE_US = 120 * 1000000L;
+
+        public async int requeue_interrupted_sessions(Saddle saddle) throws Error {
+            var requeued = 0;
+            foreach (var session in yield db.query<Session>()
+                    .where(expr("saddle_id == $0", elem<int64?>(saddle.id)))
+                    .order_by(expr("id"))
+                    .materialise_async()) {
+                if (session.ended_at != null) {
+                    continue;
+                }
+                var started = session.started_at ?? new DateTime.now_local();
+                if (new DateTime.now_local().difference(started)
+                        < ASK_RECOVERY_GRACE_US) {
+                    continue;
+                }
+                var task = yield db.query<TaskEntity>()
+                    .where(expr("id == $0", elem<int64?>(session.task_id)))
+                    .first_async();
+                if (task == null || (((!)task).status != STATUS_ASSIGNED
+                        && ((!)task).status != STATUS_WORKING)) {
+                    continue;
+                }
+                yield requeue_task(((!)task).id);
+                yield add_system_comment(((!)task).id,
+                    @"Session interrupted (saddle '$(saddle.name)' restarted mid-run); requeued");
+                requeued++;
+            }
+            return requeued;
+        }
+
         /**
          * Returns a failed task to the queue for another attempt.
          * Plan 16 S4: the failure summary is the terminal cache of the