feat: dispatch-failure observability — saddle abandons carry the reason into a bot comment (push failure / repeated poll failures), and tasks caught on offline saddles requeue with a bot comment each tick (extends stale-busy healing)

src/Services/Dispatcher.vala

@@ -96,8 +96,10 @@ namespace ClankerSaddle.Services {
             } else {
                 warned_no_llm = false;
                 // Stale-busy healing comes first so a freed saddle is
-                // dispatchable within the same tick.
+                // dispatchable within the same tick; tasks stranded on
+                // offline saddles requeue right after.
                 yield saddles.heal_stale_busy_saddles(tasks);
+                yield saddles.requeue_tasks_on_offline_saddles(tasks);
                 var ready = yield tasks.get_ready_tasks();
                 foreach (var task in ready) {
                     var saddle = yield pick_idle_capable_saddle(saddles, capabilities, task);
@@ -141,7 +143,8 @@ namespace ClankerSaddle.Services {
                         print(@"[ClankerSaddle] Dispatched task $(task.id) to saddle '$(((!)saddle).name)' (session $(session.id))\n");
                     } catch (Error e) {
                         warning(@"[ClankerSaddle] Pushing task $(task.id) to saddle $(((!)saddle).id) failed: $(e.message)");
-                        yield abandon_saddle(((!)saddle).id, saddles, tasks);
+                        yield abandon_saddle(((!)saddle).id, saddles, tasks,
+                            @"assignment push failed: $(e.message)");
                     }
                 }
             }
@@ -160,7 +163,8 @@ namespace ClankerSaddle.Services {
                     record_strike(saddle.id, count);
                     warning(@"[ClankerSaddle] Polling saddle $(saddle.id) failed (attempt $(count)/3): $(e.message)");
                     if (count >= 3) {
-                        yield abandon_saddle(saddle.id, saddles, tasks);
+                        yield abandon_saddle(saddle.id, saddles, tasks,
+                            @"daemon unreachable after $(count) failed polls: $(e.message)");
                     }
                 }
             }
@@ -242,9 +246,9 @@ namespace ClankerSaddle.Services {
          * its session closes.
          */
         private async void abandon_saddle(int64 saddle_id, SaddleService saddles,
-                TaskService tasks) throws Error {
+                TaskService tasks, string? reason = null) throws Error {
             record_strike(saddle_id, 0);
-            yield saddles.mark_saddle_unreachable(saddle_id, tasks);
+            yield saddles.mark_saddle_unreachable(saddle_id, tasks, reason);
         }
 
         /**

src/Services/SaddleService.vala

@@ -305,6 +305,36 @@ namespace ClankerSaddle.Services {
             }
         }
 
+        /**
+         * Requeues tasks caught running on saddles that have since gone
+         * offline (a manual status refresh, a restart) — without this
+         * pass they would wait forever for a daemon that will not answer.
+         * Only `assigned`/`working` tasks move: parked tasks
+         * (blocked/review/suspended) keep their sessions, which the
+         * resume machinery needs. Each move lands a bot comment so the
+         * timeline shows why the task restarted.
+         */
+        public async void requeue_tasks_on_offline_saddles(TaskService tasks) throws Error {
+            foreach (var status in new string[] { STATUS_ASSIGNED, STATUS_WORKING }) {
+                foreach (var task in yield db.query<TaskEntity>()
+                        .where(expr("status == $0", elem(status)))
+                        .materialise_async()) {
+                    var session = yield tasks.open_session_for_task(task.id);
+                    if (session == null) {
+                        continue;
+                    }
+                    var saddle = yield get_saddle(((!)session).saddle_id);
+                    if (saddle == null || ((!)saddle).status != "offline") {
+                        continue;
+                    }
+                    print(@"[ClankerSaddle] Task $(task.id) requeued: its saddle went offline\n");
+                    yield tasks.requeue_task(task.id);
+                    yield tasks.add_comment(task.id, "bot",
+                        "saddle went offline; task requeued");
+                }
+            }
+        }
+
         /**
          * Updates a saddle's status and current task. A null
          * `current_task_id` clears the assignment (and `allocated_at`).
@@ -350,7 +380,8 @@ namespace ClankerSaddle.Services {
          * crash-resume, the next dispatch restores the workspace and
          * reports `resumed` on /status.
          */
-        public async void mark_saddle_unreachable(int64 id, TaskService tasks) throws Error {
+        public async void mark_saddle_unreachable(int64 id, TaskService tasks,
+                string? reason = null) throws Error {
             var saddle = yield get_saddle(id);
             if (saddle == null) {
                 return;
@@ -365,8 +396,10 @@ namespace ClankerSaddle.Services {
                         || ((!)task).status == STATUS_REVIEW_REQUESTED)) {
                     ((!)task).status = STATUS_QUEUED;
                     yield db.update_async<TaskEntity>((!)task);
-                    yield tasks.add_comment(task_id, "bot",
-                        "Saddle became unreachable; task requeued");
+                    string unreachable_text = reason != null && reason.length > 0
+                        ? @"Saddle became unreachable ($(reason)); task requeued"
+                        : "Saddle became unreachable; task requeued";
+                    yield tasks.add_comment(task_id, "bot", unreachable_text);
                     tasks.note_requeued_subtask(((!)task));
                 }
 

test/web_stack_test.py

@@ -1608,6 +1608,69 @@ def main():
               and "ONE final review" in notice and "saddle review" in notice,
               "resume notice ends with the final-review instruction")
 
+
+        print("== R4 B3: dispatch-failure observability ==")
+        # A dead saddle that alone carries the required tag attracts the
+        # dispatch; the push fails and the task learns why via a bot comment.
+        saddles_state = slot_public(auth, "/saddles", "saddles")
+        act(auth, saddles_state["register"], {"name": "dead-saddle",
+                                              "host": "127.0.0.1:1"})
+        dead_row = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
+                    if s["name"] == "dead-saddle"][0]
+        act(auth, dead_row["attach_capability"], {"capability_id": str(gpu_option)})
+        act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
+            "project_id": "1",
+            "title": "Dead saddle task",
+            "capability_id": [str(gpu_option)],
+        })
+        dead_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+                         if t["title"] == "Dead saddle task"][0])
+        check(wait_for(lambda: any("assignment push failed" in c["body"]
+                                   for c in task_state(auth, dead_task).get("comments", []))
+                       or False, timeout=60, what="push-failure comment") is not None,
+              "failed push writes the reason on the task")
+        check(task_state(auth, dead_task)["status"] == "queued",
+              "push-failed task returns to the queue")
+        dead_rows = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
+                     if s["name"] == "dead-saddle"]
+        check(dead_rows[0]["status"] == "offline",
+              "the failing saddle goes offline")
+        act(auth, dead_rows[0]["unregister"], {})
+        conn = db()
+        conn.execute("delete from task_capabilities where capability_id = ?"
+                     " and task_id = ?", (gpu_option, dead_task))
+        conn.commit()
+        conn.close()
+        check(wait_task_done(auth, dead_task, timeout=60) is not None,
+              "the task recovers on the healthy saddle")
+
+        # Tasks caught working on a saddle that went offline requeue with a
+        # bot comment each tick.
+        fake.suspend_after_start = False
+        act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
+            "project_id": "1",
+            "title": "Offline saddle task",
+            "priority": "2",
+        })
+        offline_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+                            if t["title"] == "Offline saddle task"][0])
+        check(wait_for(lambda: task_state(auth, offline_task).get("status") == "working"
+                       or False, timeout=60, what="offline task starts working") is not None,
+              "task reaches working before its saddle disappears")
+        conn = db()
+        conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
+        conn.commit()
+        conn.close()
+        # The requeue frees the saddle in the same tick, so 'queued' is
+        # transient; the durable evidence is the bot comment (and the task
+        # completing again on the healthy daemon).
+        check(wait_for(lambda: any(c["body"] == "saddle went offline; task requeued"
+                                   for c in task_state(auth, offline_task).get("comments", []))
+                       or False, timeout=30, what="offline requeue comment") is not None,
+              "offline saddle requeues the task")
+        check(wait_task_done(auth, offline_task, timeout=60) is not None,
+              "requeued task completes again on the healthy daemon")
+
     finally:
         # Exit-code diagnostic: distinguishes a crashed app (negative
         # returncode, e.g. -11 SIGSEGV) from a clean shutdown. When