Przeglądaj źródła

feat: stranded-assignment recovery — assigned/working tasks with an open session, zero stored output past the grace period (SADDLE_STRAND_GRACE_S, default 120s), on a demonstrably idle daemon requeue with a bot comment; extends the stale-busy healing

clanker 1 miesiąc temu
rodzic
commit
5e08dc201b

+ 1 - 0
src/Services/Dispatcher.vala

@@ -114,6 +114,7 @@ namespace ClankerSaddle.Services {
                 // offline saddles requeue right after.
                 yield saddles.heal_stale_busy_saddles(tasks);
                 yield saddles.requeue_tasks_on_offline_saddles(tasks);
+                yield saddles.recover_stranded_assignments(tasks);
                 var ready = yield tasks.get_ready_tasks();
                 foreach (var task in ready) {
                     var saddle = yield pick_idle_capable_saddle(saddles, capabilities, task);

+ 69 - 0
src/Services/SaddleService.vala

@@ -323,6 +323,75 @@ namespace ClankerSaddle.Services {
             }
         }
 
+        /**
+         * Recovers stranded assignments: a task stuck `assigned`/`working`
+         * whose session never collected a single entry (its push hung or
+         * the daemon never picked the workspace up), older than the grace
+         * period, on a daemon that is demonstrably not working — is
+         * requeued with a bot comment so the timeline shows why it
+         * restarted. Extends the stale-busy healing; daemons that answer
+         * `working` are left alone (a silent-but-running task is not
+         * stranded by this definition).
+         */
+        public async void recover_stranded_assignments(TaskService tasks) throws Error {
+            var now = new DateTime.now_local();
+            foreach (var status in new string[] { STATUS_ASSIGNED, STATUS_WORKING }) {
+                foreach (var task in yield db.query<TaskEntity>()
+                        .where(expr("status == $0", elem(status)))
+                        .materialise_async()) {
+                    var session = yield tasks.open_session_for_task(task.id);
+                    if (session == null) {
+                        continue;
+                    }
+                    int64 age_seconds = now.difference(((!)session).started_at)
+                        / TimeSpan.SECOND;
+                    if (age_seconds < strand_grace_seconds()) {
+                        continue;
+                    }
+                    var entries = yield db.query<SessionCommand>()
+                        .where(expr("session_id == $0", elem<int64?>(((!)session).id)))
+                        .materialise_async();
+                    if (entries.length > 0) {
+                        continue;
+                    }
+                    var saddle = yield get_saddle(((!)session).saddle_id);
+                    if (saddle == null) {
+                        continue;
+                    }
+                    Json.Object status_body;
+                    try {
+                        status_body = yield request_saddle_json(((!)saddle).host,
+                            "GET", "/status", null);
+                    } catch (Error e) {
+                        // Unreachable: the three-strike / offline paths own it.
+                        continue;
+                    }
+                    string state = status_body.has_member("state")
+                        ? status_body.get_string_member("state") : "";
+                    // Only a demonstrably IDLE daemon strands an
+                    // assignment: `working` may be silently running it,
+                    // `done`/`failed` results belong to the poll loop
+                    // (racing them here would requeue completing tasks),
+                    // and parked states mean the daemon still holds the
+                    // workspace.
+                    if (state != "idle") {
+                        continue;
+                    }
+                    print(@"[ClankerSaddle] Task $(task.id) stranded on an idle daemon; requeued\n");
+                    yield tasks.requeue_task(task.id);
+                    yield tasks.add_comment(task.id, "bot",
+                        "task stranded with no daemon output; task requeued");
+                }
+            }
+        }
+
+        /** SADDLE_STRAND_GRACE_S (default 120s). */
+        private static uint strand_grace_seconds() {
+            var raw = Environment.get_variable("SADDLE_STRAND_GRACE_S") ?? "";
+            int64 parsed = int64.parse(raw);
+            return parsed > 0 ? (uint) parsed : 120;
+        }
+
         /**
          * Requeues tasks caught running on saddles that have since gone
          * offline (a manual status refresh, a restart) — without this

+ 25 - 0
test/web_stack_test.py

@@ -389,6 +389,7 @@ def start_app():
         "SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
         "SADDLE_HTTP_TIMEOUT_S": "5",
         "SADDLE_RAW_PROBE_INTERVAL_S": "5",
+        "SADDLE_STRAND_GRACE_S": "10",
         # Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
         # resolves the ancient /lib64 copies): empty makes the loader use
         # the binary's RUNPATH, which is the dev stack it was built
@@ -1873,6 +1874,30 @@ def main():
         check(wait_task_done(auth, raw_retry, timeout=60) is not None,
               "raw retry task completes")
 
+
+        print("== R6 F4: stranded assigned recovery ==")
+        # The live incident's shape: task assigned, session open, zero
+        # output ever, daemon idle. Grace period 10s in the harness env.
+        fake.state = "idle"
+        conn = db()
+        conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
+                     " values (1, 'Stranded assigned', 'assigned', 2, strftime('%s','now'))")
+        stranded = conn.execute("select max(id) from tasks").fetchone()[0]
+        conn.execute("insert into sessions (task_id, saddle_id, started_at)"
+                     " values (?, ?, strftime('%s','now','-180 seconds'))", (stranded, fake_id))
+        # Saddle row idle (the busy@task shape is the stale-busy heal's
+        # case); this exercises the strand pass specifically.
+        conn.execute("update saddles set status = 'idle', current_task_id = 0,"
+                     " allocated_at = null where name = 'fake-saddle'")
+        conn.commit()
+        conn.close()
+        check(wait_for(lambda: any(c["body"] == "task stranded with no daemon output; task requeued"
+                                   for c in task_state(auth, stranded).get("comments", []))
+                       or False, timeout=40, what="stranded requeue comment") is not None,
+              "stranded assignment requeues with a bot comment")
+        check(wait_task_done(auth, stranded, timeout=90) is not None,
+              "recovered task dispatches and completes")
+
     finally:
         # Exit-code diagnostic: distinguishes a crashed app (negative
         # returncode, e.g. -11 SIGSEGV) from a clean shutdown. When