|
@@ -1472,6 +1472,50 @@ namespace ClankerSaddle.Services {
|
|
|
yield pulse.bump(task_id, "task requeued");
|
|
yield pulse.bump(task_id, "task requeued");
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ /**
|
|
|
|
|
+ * Crash recovery at the ask: a daemon that asks for work is
|
|
|
|
|
+ * provably idle — its own state machine parks the ask loop
|
|
|
|
|
+ * while any session runs — so an open session on an
|
|
|
|
|
+ * assigned/working task can only be a crash orphan (both
|
|
|
|
|
+ * sides restart, nobody closes the row, and the busy record
|
|
|
|
|
+ * then wedges the fleet: the saddle asks, the claim rolls
|
|
|
|
|
+ * back, forever, while the UI shows it busy). Every orphan
|
|
|
|
|
+ * older than the grace window requeues — the very ask that
|
|
|
|
|
+ * found it can win the task straight back. The grace window
|
|
|
|
|
+ * protects a manual /start whose session row lands while an
|
|
|
|
|
+ * ask is in flight.
|
|
|
|
|
+ */
|
|
|
|
|
+ public const int64 ASK_RECOVERY_GRACE_US = 120 * 1000000L;
|
|
|
|
|
+
|
|
|
|
|
+ public async int requeue_interrupted_sessions(Saddle saddle) throws Error {
|
|
|
|
|
+ var requeued = 0;
|
|
|
|
|
+ foreach (var session in yield db.query<Session>()
|
|
|
|
|
+ .where(expr("saddle_id == $0", elem<int64?>(saddle.id)))
|
|
|
|
|
+ .order_by(expr("id"))
|
|
|
|
|
+ .materialise_async()) {
|
|
|
|
|
+ if (session.ended_at != null) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+ var started = session.started_at ?? new DateTime.now_local();
|
|
|
|
|
+ if (new DateTime.now_local().difference(started)
|
|
|
|
|
+ < ASK_RECOVERY_GRACE_US) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+ var task = yield db.query<TaskEntity>()
|
|
|
|
|
+ .where(expr("id == $0", elem<int64?>(session.task_id)))
|
|
|
|
|
+ .first_async();
|
|
|
|
|
+ if (task == null || (((!)task).status != STATUS_ASSIGNED
|
|
|
|
|
+ && ((!)task).status != STATUS_WORKING)) {
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+ yield requeue_task(((!)task).id);
|
|
|
|
|
+ yield add_system_comment(((!)task).id,
|
|
|
|
|
+ @"Session interrupted (saddle '$(saddle.name)' restarted mid-run); requeued");
|
|
|
|
|
+ requeued++;
|
|
|
|
|
+ }
|
|
|
|
|
+ return requeued;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
/**
|
|
/**
|
|
|
* Returns a failed task to the queue for another attempt.
|
|
* Returns a failed task to the queue for another attempt.
|
|
|
* Plan 16 S4: the failure summary is the terminal cache of the
|
|
* Plan 16 S4: the failure summary is the terminal cache of the
|