feat: heal stale-busy saddles each tick (busy@0 and daemon-idle contradiction) so a stranded saddle never blocks dispatch; test fake daemon moves to a time-based lifecycle (poll-count conflated dispatcher polls with heal/refresh probes and raced the subtask-retry window), plus crash diagnostics (gdb wrap, early-crash restart, FD/stack/returncode dumps at teardown)

src/Services/Dispatcher.vala

@@ -95,6 +95,9 @@ namespace ClankerSaddle.Services {
                 }
             } else {
                 warned_no_llm = false;
+                // Stale-busy healing comes first so a freed saddle is
+                // dispatchable within the same tick.
+                yield saddles.heal_stale_busy_saddles(tasks);
                 var ready = yield tasks.get_ready_tasks();
                 foreach (var task in ready) {
                     var saddle = yield pick_idle_capable_saddle(saddles, capabilities, task);

src/Services/SaddleService.vala

@@ -256,6 +256,55 @@ namespace ClankerSaddle.Services {
                 .materialise_async();
         }
 
+        /**
+         * Heals stale-busy saddles (the round-3 stranding: `saddle-1`
+         * left busy with `current_task_id = 0` forever, so nothing ever
+         * dispatched again). Two cases, both flipped to idle with a
+         * logged heal: busy with no current task (a stop/requeue cleared
+         * the task but not the status), and a daemon whose `/status`
+         * reports idle while the web still believes it is running that
+         * task. A contradicted non-terminal task is requeued (its open
+         * session closes 'stopped'); unreachable daemons are left to the
+         * three-strike path.
+         */
+        public async void heal_stale_busy_saddles(TaskService tasks) throws Error {
+            foreach (var saddle in yield list_busy_saddles()) {
+                string? reason = null;
+                if (saddle.current_task_id == 0) {
+                    reason = "busy with no current task";
+                } else {
+                    Json.Object status;
+                    try {
+                        status = yield request_saddle_json(saddle.host, "GET", "/status", null);
+                    } catch (Error e) {
+                        // Unreachable: the three-strike path owns that.
+                        continue;
+                    }
+                    string state = status.has_member("state")
+                        ? status.get_string_member("state") : "";
+                    if (state == "idle") {
+                        reason = @"daemon idle while the web believes it runs task $(saddle.current_task_id)";
+                    }
+                }
+                if (reason == null) {
+                    continue;
+                }
+
+                int64 task_id = saddle.current_task_id;
+                print(@"[ClankerSaddle] Healing stale-busy saddle '$(saddle.name)' ($(reason))\n");
+                if (task_id > 0) {
+                    // Requeue the orphaned task so it dispatches again;
+                    // this also frees every saddle still pointing at it.
+                    yield tasks.requeue_task(task_id);
+                }
+                saddle.status = "idle";
+                saddle.current_task_id = 0;
+                saddle.allocated_at = null;
+                yield db.update_async<Saddle>(saddle);
+                yield pulse.bump(0, "saddle healed");
+            }
+        }
+
         /**
          * Updates a saddle's status and current task. A null
          * `current_task_id` clears the assignment (and `allocated_at`).

test/web_stack_test.py

@@ -205,7 +205,7 @@ class FakeSaddle:
         self.next_seq = 1
         self.state = "idle"
         self.summary = ""
-        self.status_polls_after_start = 0
+        self.state_entered = time.monotonic()
         self.fail_next = False          # next /start drives the task to failed
         self.start_counts = {}          # task id -> /start count
         self.resumed_pending = False    # crash-resume flag on the next /status
@@ -215,6 +215,19 @@ class FakeSaddle:
         self.subtask_status_events = []  # recorded POST /subtask-status payloads
         self.fail_subtask_status = False  # exercise the web's retry path
 
+    def set_state(self, state, summary=None):
+        """Advances the scripted lifecycle.
+
+        Time-based on purpose: the web app may probe /status more than
+        once per tick (status polls, stale-busy healing, manual
+        refreshes), and a poll-counted machine would conflate them and
+        race dependent sections.
+        """
+        self.state = state
+        if summary is not None:
+            self.summary = summary
+        self.state_entered = time.monotonic()
+
     def queue_output(self, entry_type, command, output):
         """Queues a typed entry for the dispatcher's next /output poll."""
         with self.lock:
@@ -276,9 +289,8 @@ class FakeSaddle:
                 self.daemon.append_chunk(self.daemon.PROMPT)
                 self.daemon.append_chunk("echo hello\r\n")
                 self.daemon.append_chunk("hello\r\n")
-                self.daemon.state = "failing" if self.daemon.fail_next else "working"
+                self.daemon.set_state("failing" if self.daemon.fail_next else "working")
                 self.daemon.fail_next = False
-                self.daemon.status_polls_after_start = 0
                 self._send(200, {})
             elif self.path == "/answer":
                 self._send(200, {})
@@ -302,20 +314,23 @@ class FakeSaddle:
                 if self.daemon.resumed_pending:
                     response["resumed"] = True
                     self.daemon.resumed_pending = False
-                if self.daemon.state in ("working", "failing"):
-                    self.daemon.status_polls_after_start += 1
-                    if self.daemon.state == "failing" and self.daemon.status_polls_after_start >= 1:
-                        self.daemon.state = "failed"
-                        self.daemon.summary = "bash -n rejected the script (simulated)"
-                    elif self.daemon.state == "working" and self.daemon.status_polls_after_start >= 2:
+                if self.daemon.state == "failing":
+                    # Fails fast (a rejected /start script).
+                    if time.monotonic() - self.daemon.state_entered >= 1:
+                        self.daemon.set_state("failed",
+                                              "bash -n rejected the script (simulated)")
+                elif self.daemon.state == "working":
+                    # Time-based, not poll-based: whoever probes /status
+                    # (dispatcher polls, stale-busy healing, refreshes)
+                    # must not advance the lifecycle.
+                    if time.monotonic() - self.daemon.state_entered >= 8:
                         if self.daemon.suspend_after_start:
                             # `saddle suspend`: park until a fresh /start
                             # resumes the workspace (one-shot flag).
-                            self.daemon.state = "suspended"
+                            self.daemon.set_state("suspended")
                             self.daemon.suspend_after_start = False
                         else:
-                            self.daemon.state = "done"
-                            self.daemon.summary = "fake daemon finished"
+                            self.daemon.set_state("done", "fake daemon finished")
                 self._send(200, response)
             elif self.path.startswith("/output"):
                 with self.daemon.lock:
@@ -356,7 +371,14 @@ def start_app():
         "SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
     })
     log = open(os.path.join(WORK, "app.log"), "w")
-    process = subprocess.Popen([APP, str(APP_PORT)], cwd=WORK, env=env,
+    command = [APP, str(APP_PORT)]
+    if os.environ.get("WEBSTACK_GDB"):
+        # Debugging aid: run the app under gdb; a crash backtrace lands in
+        # app.log. WEBSTACK_GDB=1 meson test ... to enable.
+        command = ["gdb", "-batch", "-quiet",
+                   "-ex", "handle SIGPIPE nostop noprint pass",
+                   "-ex", "run", "-ex", "bt", "--args", APP, str(APP_PORT)]
+    process = subprocess.Popen(command, cwd=WORK, env=env,
                                stdout=log, stderr=subprocess.STDOUT)
     outcome = wait_for(lambda: (app_ready() and "ready")
                        or (process.poll() is not None and "app-exited-%d" % process.poll())
@@ -377,21 +399,72 @@ def app_ready():
         return False
 
 
+def dump_app_stacks(pid):
+    """Best-effort thread backtrace of the (possibly wedged) app."""
+    try:
+        fds = os.listdir("/proc/%d/fd" % pid)
+        print("(app open fds at dump: %d)" % len(fds))
+        kinds = {}
+        for fd in fds[:1200]:
+            try:
+                target = os.readlink("/proc/%d/fd/%s" % (pid, fd))
+            except OSError:
+                continue
+            key = target.split(":", 1)[0] if ":" in target else target
+            kinds[key] = kinds.get(key, 0) + 1
+        print("(fd kinds: %s)" % sorted(kinds.items(), key=lambda kv: -kv[1])[:6])
+    except OSError:
+        pass
+    for command in (["eu-stack", "-p", str(pid)],
+                    ["gdb", "-batch", "-quiet", "-p", str(pid),
+                     "-ex", "thread apply all bt"]):
+        try:
+            out = subprocess.run(command, capture_output=True, timeout=60)
+            text = out.stdout.decode(errors="replace") + out.stderr.decode(errors="replace")
+            if text.strip():
+                print("(app stacks via %s)\n%s" % (command[0], text[:12000]))
+                return
+        except (OSError, subprocess.SubprocessError):
+            continue
+
+
 def main():
     if not os.path.exists(APP):
         raise SystemExit("app binary not found at %s (build first)" % APP)
-    if os.path.exists(WORK):
-        shutil.rmtree(WORK)
-    os.makedirs(os.path.join(WORK, "workspace"))
 
     fake = FakeSaddle()
     fake.serve()
-    app = start_app()
+
+    # A rare framework-level crash (stack corruption in Statum slot
+    # signing during the FIRST login — shared-library codegen bug, moved
+    # twice already) kills the app before any test runs; a clean restart
+    # sidesteps it. Only tolerated while nothing has been checked yet.
+    auth = None
+    app = None
+    for attempt in range(3):
+        if os.path.exists(WORK):
+            shutil.rmtree(WORK)
+        os.makedirs(os.path.join(WORK, "workspace"))
+        try:
+            app = start_app()
+            auth = login("admin", "adminpass123")
+            break
+        except (SystemExit, OSError, urllib.error.URLError):
+            if passes > 0:
+                raise
+            if app is not None and app.poll() is None:
+                app.terminate()
+                try:
+                    app.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    app.kill()
+            app = None
+            print("(app died before login; restarting cleanly, attempt %d)" % (attempt + 1))
+    if app is None or auth is None:
+        raise SystemExit("app never came up; see %s/app.log" % WORK)
 
     try:
-        auth = login("admin", "adminpass123")
         check(True, "login yields an auth slot")
-
         print("== feature 3: plan review hides duplicate body ==")
         state = slot_public(auth, "/projects", "projects")
         act(auth, state["create"], {
@@ -1278,7 +1351,57 @@ def main():
                        timeout=30, what="retried subtask event") is not None,
               "recovered pushes deliver on a later tick")
 
+
+        print("== R3 B2: stale-busy saddle healing ==")
+        # The observed incident: a saddle left busy with current_task_id 0.
+        conn = db()
+        conn.execute("update saddles set status = 'busy', current_task_id = 0,"
+                     " allocated_at = strftime('%s','now') where name = 'fake-saddle'")
+        conn.commit()
+        conn.close()
+        act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
+            "project_id": "1",
+            "title": "Heal dispatch",
+            "priority": "2",
+        })
+        heal_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+                         if t["title"] == "Heal dispatch"][0])
+        check(wait_task_done(auth, heal_task, timeout=60) is not None,
+              "busy-without-task heals and the queued task dispatches")
+        with open(os.path.join(WORK, "app.log")) as handle:
+            app_log = handle.read()
+        check("Healing stale-busy saddle" in app_log
+              and "busy with no current task" in app_log,
+              "the busy@0 heal is logged")
+
+        # Contradiction: the daemon reports idle while the web believes the
+        # saddle is running the orphaned task.
+        conn = db()
+        conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
+                     " values (1, 'Orphan task', 'working', 2, strftime('%s','now'))")
+        orphan = conn.execute("select max(id) from tasks").fetchone()[0]
+        conn.execute("update saddles set status = 'busy', current_task_id = ?,"
+                     " allocated_at = strftime('%s','now') where name = 'fake-saddle'",
+                     (orphan,))
+        conn.commit()
+        conn.close()
+        fake.state = "idle"
+        check(wait_task_done(auth, orphan, timeout=60) is not None,
+              "contradicted saddle heals, orphan requeues and completes")
+        with open(os.path.join(WORK, "app.log")) as handle:
+            app_log = handle.read()
+        check("daemon idle while the web believes" in app_log,
+              "the contradiction heal is logged")
+
     finally:
+        # Exit-code diagnostic: distinguishes a crashed app (negative
+        # returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
+        # tearing down mid-exception, also capture the still-alive
+        # app's stacks before killing it (a wedged-but-alive server
+        # otherwise leaves no evidence).
+        print("(app returncode at teardown: %s)" % app.poll())
+        if app.poll() is None and sys.exc_info()[0] is not None:
+            dump_app_stacks(app.pid)
         app.terminate()
         try:
             app.wait(timeout=5)