|
|
@@ -323,6 +323,75 @@ namespace ClankerSaddle.Services {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /**
|
|
|
+ * Recovers stranded assignments: a task stuck `assigned`/`working`
|
|
|
+ * whose session never collected a single entry (its push hung or
|
|
|
+ * the daemon never picked the workspace up), older than the grace
|
|
|
+ * period, on a daemon that is demonstrably not working — is
|
|
|
+ * requeued with a bot comment so the timeline shows why it
|
|
|
+ * restarted. Extends the stale-busy healing; daemons that answer
|
|
|
+ * `working` are left alone (a silent-but-running task is not
|
|
|
+ * stranded by this definition).
|
|
|
+ */
|
|
|
+ public async void recover_stranded_assignments(TaskService tasks) throws Error {
|
|
|
+ var now = new DateTime.now_local();
|
|
|
+ foreach (var status in new string[] { STATUS_ASSIGNED, STATUS_WORKING }) {
|
|
|
+ foreach (var task in yield db.query<TaskEntity>()
|
|
|
+ .where(expr("status == $0", elem(status)))
|
|
|
+ .materialise_async()) {
|
|
|
+ var session = yield tasks.open_session_for_task(task.id);
|
|
|
+ if (session == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ int64 age_seconds = now.difference(((!)session).started_at)
|
|
|
+ / TimeSpan.SECOND;
|
|
|
+ if (age_seconds < strand_grace_seconds()) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ var entries = yield db.query<SessionCommand>()
|
|
|
+ .where(expr("session_id == $0", elem<int64?>(((!)session).id)))
|
|
|
+ .materialise_async();
|
|
|
+ if (entries.length > 0) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ var saddle = yield get_saddle(((!)session).saddle_id);
|
|
|
+ if (saddle == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ Json.Object status_body;
|
|
|
+ try {
|
|
|
+ status_body = yield request_saddle_json(((!)saddle).host,
|
|
|
+ "GET", "/status", null);
|
|
|
+ } catch (Error e) {
|
|
|
+ // Unreachable: the three-strike / offline paths own it.
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ string state = status_body.has_member("state")
|
|
|
+ ? status_body.get_string_member("state") : "";
|
|
|
+ // Only a demonstrably IDLE daemon strands an
|
|
|
+ // assignment: `working` may be silently running it,
|
|
|
+ // `done`/`failed` results belong to the poll loop
|
|
|
+ // (racing them here would requeue completing tasks),
|
|
|
+ // and parked states mean the daemon still holds the
|
|
|
+ // workspace.
|
|
|
+ if (state != "idle") {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ print(@"[ClankerSaddle] Task $(task.id) stranded on an idle daemon; requeued\n");
|
|
|
+ yield tasks.requeue_task(task.id);
|
|
|
+ yield tasks.add_comment(task.id, "bot",
|
|
|
+ "task stranded with no daemon output; task requeued");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /** SADDLE_STRAND_GRACE_S (default 120s). */
|
|
|
+ private static uint strand_grace_seconds() {
|
|
|
+ var raw = Environment.get_variable("SADDLE_STRAND_GRACE_S") ?? "";
|
|
|
+ int64 parsed = int64.parse(raw);
|
|
|
+ return parsed > 0 ? (uint) parsed : 120;
|
|
|
+ }
|
|
|
+
|
|
|
/**
|
|
|
* Requeues tasks caught running on saddles that have since gone
|
|
|
* offline (a manual status refresh, a restart) — without this
|