Procházet zdrojové kódy

feat: dead/wedged daemon reconciliation — vanishing daemons drive offline+requeue with narrated comments (3-strike verified against connection-refused), stale last_activity wedges fence+requeue, saddles page health panel (state/uptime/last-contact/badge), task rows mark the unreachable window, raw-log polling verified from the first push; harness timeout 1800s

clanker před 1 měsícem
rodič
revize
b55f9a8719

+ 1 - 1
meson.build

@@ -188,5 +188,5 @@ clanker_saddle_exe = executable('clanker-saddle',
 python_test = import('python').find_installation('python3')
 test('web-stack', python_test,
     args: [files('test/web_stack_test.py'), clanker_saddle_exe],
-    timeout: 900,
+    timeout: 1800,
 )

+ 33 - 0
src/Services/SaddlePollState.vala

@@ -17,6 +17,10 @@ namespace ClankerSaddle.Services {
      */
     public class SaddlePollState : Object {
 
+        private Dictionary<string, string> probe_state = new Dictionary<string, string>();
+        private Dictionary<string, string> probe_uptime = new Dictionary<string, string>();
+        private Dictionary<string, string> probe_activity = new Dictionary<string, string>();
+        private Dictionary<string, string> probe_contact = new Dictionary<string, string>();
         private Dictionary<string, string> last_stored_output_id = new Dictionary<string, string>();
         private Dictionary<string, string> last_stored_raw_seq = new Dictionary<string, string>();
         private Dictionary<string, string> last_raw_probe_ms = new Dictionary<string, string>();
@@ -82,6 +86,35 @@ namespace ClankerSaddle.Services {
             state_lock.unlock();
         }
 
+        /**
+         * Records a daemon /status observation for the saddles-page
+         * health panel: reported state, uptime, last-activity epoch and
+         * the web-side contact time (monotonic ms).
+         */
+        public void record_probe(int64 saddle_id, string state, int64 uptime_seconds,
+                int64 last_activity_epoch) {
+            state_lock.lock();
+            string key = saddle_id.to_string();
+            probe_state.set(key, state);
+            probe_uptime.set(key, uptime_seconds.to_string());
+            probe_activity.set(key, last_activity_epoch.to_string());
+            probe_contact.set(key, (get_monotonic_time() / 1000).to_string());
+            state_lock.unlock();
+        }
+
+        /** The last probe snapshot for `saddle_id` ("" when never probed). */
+        public void health_for(int64 saddle_id, out string state, out int64 uptime,
+                out int64 activity, out int64 contact_ms) {
+            state_lock.lock();
+            string key = saddle_id.to_string();
+            string stored = "";
+            state = probe_state.try_get(key, out stored) ? stored : "";
+            uptime = int64.parse(probe_uptime.get_or_default(key) ?? "0");
+            activity = int64.parse(probe_activity.get_or_default(key) ?? "0");
+            contact_ms = int64.parse(probe_contact.get_or_default(key) ?? "0");
+            state_lock.unlock();
+        }
+
         /**
          * True when a fresh raw-log capability probe for `session_id` is
          * due: never probed, or the last probe is older than

+ 63 - 0
src/Services/SaddleService.vala

@@ -277,6 +277,22 @@ namespace ClankerSaddle.Services {
                 .materialise_async();
         }
 
+        /**
+         * Records a /status observation into the health snapshot,
+         * tolerating older daemons without the uptime/activity fields.
+         */
+        private void record_status_probe(int64 saddle_id, Json.Object status) {
+            string state = status.has_member("state")
+                ? status.get_string_member_with_default("state", "") : "";
+            int64 uptime = status.has_member("uptime_seconds")
+                && status.get_member("uptime_seconds").get_value_type() == typeof(int64)
+                ? status.get_int_member("uptime_seconds") : 0;
+            int64 activity = status.has_member("last_activity_epoch")
+                && status.get_member("last_activity_epoch").get_value_type() == typeof(int64)
+                ? status.get_int_member("last_activity_epoch") : 0;
+            poll_state.record_probe(saddle_id, state, uptime, activity);
+        }
+
         /**
          * {@link push_task_to_saddle} with 409-aware recovery: a busy
          * daemon answering `/start` with a conflict is either running a
@@ -458,6 +474,7 @@ namespace ClankerSaddle.Services {
                     var status = yield request_saddle_json(saddle.host, "GET", "/status", null);
                     string state = status.has_member("state")
                         ? status.get_string_member("state") : "";
+                    record_status_probe(saddle.id, status);
                     if (state == "idle") {
                         saddle.status = "idle";
                         saddle.current_task_id = 0;
@@ -512,10 +529,28 @@ namespace ClankerSaddle.Services {
                         // Unreachable: the three-strike path owns that.
                         continue;
                     }
+                    record_status_probe(saddle.id, status);
                     string state = status.has_member("state")
                         ? status.get_string_member("state") : "";
                     if (state == "idle") {
                         reason = @"daemon idle while the web believes it runs task $(saddle.current_task_id)";
+                    } else if (state == "working" && wedged(status)) {
+                        // Alive-but-wedged: no loop round or command
+                        // completion within the threshold. The task cannot
+                        // progress on a frozen daemon — fence the saddle
+                        // and requeue with a comment.
+                        int64 stale_for = (int64) (get_real_time() / 1000000)
+                            - activity_epoch_of(status);
+                        print(@"[ClankerSaddle] Saddle '$(saddle.name)' wedged (no activity for $(stale_for)s); fencing and requeueing task $(saddle.current_task_id)\n");
+                        yield tasks.requeue_task(saddle.current_task_id);
+                        yield tasks.add_comment(saddle.current_task_id, "bot",
+                            @"saddle wedged (no daemon activity for $(stale_for)s); task requeued");
+                        saddle.status = "recovering";
+                        saddle.current_task_id = 0;
+                        saddle.allocated_at = null;
+                        yield db.update_async<Saddle>(saddle);
+                        yield pulse.bump(0, "saddle wedged");
+                        continue;
                     }
                 }
                 if (reason == null) {
@@ -599,6 +634,33 @@ namespace ClankerSaddle.Services {
             }
         }
 
+        /** SADDLE_WEDGE_THRESHOLD_S (default 600s). */
+        public static uint wedge_threshold_seconds() {
+            var raw = Environment.get_variable("SADDLE_WEDGE_THRESHOLD_S") ?? "";
+            int64 parsed = int64.parse(raw);
+            return parsed > 0 ? (uint) parsed : 600;
+        }
+
+        private static int64 activity_epoch_of(Json.Object status) {
+            return status.has_member("last_activity_epoch")
+                && status.get_member("last_activity_epoch").get_value_type() == typeof(int64)
+                ? status.get_int_member("last_activity_epoch") : 0;
+        }
+
+        /**
+         * True when the daemon reports `working` but its last activity
+         * stamp is older than the wedge threshold (absent on older
+         * daemons — never wedged without evidence).
+         */
+        private static bool wedged(Json.Object status) {
+            int64 activity = activity_epoch_of(status);
+            if (activity <= 0) {
+                return false;
+            }
+            int64 stale_for = (int64) (get_real_time() / 1000000) - activity;
+            return stale_for >= (int64) wedge_threshold_seconds();
+        }
+
         /** SADDLE_STRAND_GRACE_S (default 120s). */
         private static uint strand_grace_seconds() {
             var raw = Environment.get_variable("SADDLE_STRAND_GRACE_S") ?? "";
@@ -1146,6 +1208,7 @@ namespace ClankerSaddle.Services {
             try {
                 var status = yield request_saddle_json(((!)saddle).host, "GET", "/status", null);
                 string state = status.has_member("state") ? status.get_string_member_with_default("state", "") : "";
+                record_status_probe(id, status);
                 if (state == "working") {
                     // Reconcile with the web's own records: busy only when
                     // a live session truly holds this saddle.

+ 27 - 0
src/Services/TaskService.vala

@@ -679,6 +679,33 @@ namespace ClankerSaddle.Services {
             yield db.update_async<Session>((!)row);
         }
 
+        /**
+         * Task ids currently `assigned`/`working` whose open session sits
+         * on an offline or fenced saddle — the queue marks them
+         * "saddle unreachable — requeueing" while recovery runs.
+         */
+        public async Invercargill.DataStructures.HashSet<string> unreachable_task_ids() throws Error {
+            var ids = new Invercargill.DataStructures.HashSet<string>();
+            foreach (var status in new string[] { STATUS_ASSIGNED, STATUS_WORKING }) {
+                foreach (var task in yield db.query<TaskEntity>()
+                        .where(expr("status == $0", elem(status)))
+                        .materialise_async()) {
+                    var session = yield open_session_for_task(task.id);
+                    if (session == null) {
+                        continue;
+                    }
+                    var saddle = yield db.query<Saddle>()
+                        .where(expr("id == $0", elem<int64?>(((!)session).saddle_id)))
+                        .first_async();
+                    if (saddle != null && (((!)saddle).status == "offline"
+                            || ((!)saddle).status == "recovering")) {
+                        ids.add(task.id.to_string());
+                    }
+                }
+            }
+            return ids;
+        }
+
         /** Persists the auto-approve toggle for `task_id`. */
         public async void save_auto_approve(int64 task_id, bool enabled) throws Error {
             var task = yield db.query<TaskEntity>()

+ 2 - 1
src/actions/TaskActions.vala

@@ -525,6 +525,7 @@ namespace ClankerSaddle {
         private SaddleService saddle_service = inject<SaddleService>();
         private RuleService rule_service = inject<RuleService>();
         private CapabilityService capability_service = inject<CapabilityService>();
+        private SaddlePollState poll_state = inject<SaddlePollState>();
         private UserService users = inject<UserService>();
 
         public override async DirectiveBuilder handle() throws GLib.Error {
@@ -540,7 +541,7 @@ namespace ClankerSaddle {
 
             var state = yield TaskDetailState.build(state_service, task_service,
                 project_service, saddle_service, rule_service, capability_service,
-                action_registry, task_id);
+                action_registry, poll_state, task_id);
             return directives().set(state);
         }
     }

+ 43 - 0
src/entrypoints/SaddlesEntrypoint.vala

@@ -9,6 +9,18 @@ using ClankerSaddle.Services;
 
 namespace ClankerSaddle {
 
+    /** "3m ago"-style compact duration for the health panel. */
+    private static string relative_time(int64 delta_ms) {
+        int64 seconds = delta_ms.clamp(0, int64.MAX) / 1000;
+        if (seconds < 60) {
+            return @"$(seconds)s";
+        }
+        if (seconds < 3600) {
+            return @"$(seconds / 60)m";
+        }
+        return @"$(seconds / 3600)h";
+    }
+
     /** A capability chip on a saddle card, with its remove action. */
     public class SaddleCapabilityView : Object {
         public int64 link_id { get; set; }
@@ -22,6 +34,10 @@ namespace ClankerSaddle {
         public string name { get; set; default = ""; }
         public string host { get; set; default = ""; }
         public string status { get; set; default = ""; }
+        public string health { get; set; default = "unknown"; }
+        public string daemon_state { get; set; default = ""; }
+        public string uptime_display { get; set; default = ""; }
+        public string last_contact_display { get; set; default = ""; }
         public int64 current_task_id { get; set; }
         public string current_task_title { get; set; default = ""; }
         public string allocated_display { get; set; default = ""; }
@@ -61,6 +77,7 @@ namespace ClankerSaddle {
     public class SaddlesEntrypoint : StatumEntrypoint {
 
         private SaddleService saddle_service = Inversion.inject<SaddleService>();
+        private SaddlePollState poll_state = Inversion.inject<SaddlePollState>();
         private CapabilityService capability_service = Inversion.inject<CapabilityService>();
         private InvercargillSql.Orm.OrmSession db = Inversion.inject<InvercargillSql.Orm.OrmSession>();
         private UserService users = Inversion.inject<UserService>();
@@ -110,13 +127,39 @@ namespace ClankerSaddle {
             }
 
             var rows = new Vector<SaddleRowView>();
+            int64 now_ms = get_monotonic_time() / 1000;
             foreach (var saddle in yield saddle_service.list_saddles()) {
+                string daemon_state = "";
+                int64 uptime = 0;
+                int64 activity = 0;
+                int64 contact_ms = 0;
+                poll_state.health_for(saddle.id, out daemon_state, out uptime,
+                    out activity, out contact_ms);
+                string health = "unknown";
+                if (saddle.status == "offline") {
+                    health = "offline";
+                } else if (saddle.status == "recovering") {
+                    health = "fenced";
+                } else if (daemon_state.length > 0) {
+                    health = "healthy";
+                    if (daemon_state == "working" && activity > 0
+                            && now_ms / 1000 - activity
+                                >= SaddleService.wedge_threshold_seconds()) {
+                        health = "stale";
+                    }
+                }
+                string last_contact = contact_ms > 0
+                    ? relative_time(now_ms - contact_ms) : "never";
                 var sealed = new SaddleActionPrivate() { saddle_id = saddle.id };
                 var view = new SaddleRowView() {
                     id = saddle.id,
                     name = saddle.name,
                     host = saddle.host,
                     status = saddle.status,
+                    health = health,
+                    daemon_state = daemon_state,
+                    uptime_display = uptime > 0 ? relative_time(uptime * 1000) : "",
+                    last_contact_display = last_contact,
                     current_task_id = saddle.current_task_id,
                     allocated_display = saddle.allocated_at != null
                         ? ((!)saddle.allocated_at).format("%Y-%m-%d %H:%M") : "",

+ 15 - 2
src/entrypoints/TaskDetailEntrypoint.vala

@@ -111,6 +111,7 @@ namespace ClankerSaddle {
         public string review_summary { get; set; default = ""; }
         public string review_diff_url { get; set; default = ""; }
         public bool resume_available { get; set; }
+        public string saddle_health { get; set; default = ""; }
         public bool auto_approve { get; set; }
         public Statum.Model.ActionDto? toggle_auto_approve { get; set; }
         public string display { get; set; default = ""; }
@@ -156,7 +157,7 @@ namespace ClankerSaddle {
         public static async Slot build(StateService state_service, TaskService tasks,
                 ProjectService projects, SaddleService saddles, RuleService rules,
                 CapabilityService capabilities, ActionRegistry actions,
-                int64 task_id) throws GLib.Error {
+                SaddlePollState polls, int64 task_id) throws GLib.Error {
             var detail = yield tasks.get_task(task_id);
             if (detail == null) {
                 return slot_for(state_service, new TaskDetailPublic() { exists = false },
@@ -355,6 +356,17 @@ namespace ClankerSaddle {
                     outcome = session.outcome ?? "",
                     open = session.ended_at == null
                 });
+                if (session.ended_at == null && saddle != null) {
+                    string daemon_state = "";
+                    int64 uptime = 0;
+                    int64 activity = 0;
+                    int64 contact_ms = 0;
+                    polls.health_for(session.saddle_id, out daemon_state,
+                        out uptime, out activity, out contact_ms);
+                    if (daemon_state.length > 0) {
+                        pub.saddle_health = @"saddle '$(((!)saddle).name)' — daemon $(daemon_state)";
+                    }
+                }
                 // A task stuck working/blocked whose open session sits on
                 // an offline saddle offers the manual requeue-for-resume
                 // path (the automatic three-strike abandon only fires
@@ -450,6 +462,7 @@ namespace ClankerSaddle {
         private SaddleService saddle_service = Inversion.inject<SaddleService>();
         private RuleService rule_service = Inversion.inject<RuleService>();
         private CapabilityService capability_service = Inversion.inject<CapabilityService>();
+        private SaddlePollState poll_state = Inversion.inject<SaddlePollState>();
         private UserService users = Inversion.inject<UserService>();
 
         public override async DirectiveBuilder handle() throws GLib.Error {
@@ -461,7 +474,7 @@ namespace ClankerSaddle {
             int64 task_id = int64.parse(request.route("task_id") ?? "0");
             var state = yield TaskDetailState.build(state_service, task_service,
                 project_service, saddle_service, rule_service, capability_service,
-                action_registry, task_id);
+                action_registry, poll_state, task_id);
             return directives().set(state).subscribe_global(StatePulse.SLOT_TYPE);
         }
     }

+ 3 - 0
src/entrypoints/TasksEntrypoint.vala

@@ -25,6 +25,7 @@ namespace ClankerSaddle {
         public bool ready { get; set; }
         public string[] capabilities { get; set; }
         public bool waiting_for_saddle { get; set; }
+        public bool unreachable { get; set; }
     }
 
     /**
@@ -119,6 +120,7 @@ namespace ClankerSaddle {
                 }
                 ((!)names).add(name);
             }
+            var unreachable_ids = yield task_service.unreachable_task_ids();
             var idle_capability_sets = new Vector<Invercargill.DataStructures.HashSet<string>>();
             foreach (var saddle in yield saddle_service.list_idle_saddles()) {
                 var carried = new Invercargill.DataStructures.HashSet<string>();
@@ -183,6 +185,7 @@ namespace ClankerSaddle {
                     }
                     view.waiting_for_saddle = !covered && idle_capability_sets.length > 0;
                 }
+                view.unreachable = unreachable_ids.has(task.id.to_string());
                 rows.add(view);
             }
 

+ 22 - 0
src/pages/app.css

@@ -150,6 +150,28 @@ summary { cursor: pointer; font-weight: 600; }
     vertical-align: middle;
 }
 
+/* Health badges on the saddles page */
+.badge-health-healthy {
+    background: #14532d;
+    color: #bbf7d0;
+}
+.badge-health-stale {
+    background: #713f12;
+    color: #fde68a;
+}
+.badge-health-fenced {
+    background: #7c2d12;
+    color: #fed7aa;
+}
+.badge-health-offline {
+    background: #7f1d1d;
+    color: #fecaca;
+}
+.badge-health-unknown {
+    background: #374151;
+    color: #d1d5db;
+}
+
 /* The × inside a capability chip (badge) */
 .chip-remove {
     background: none;

+ 5 - 0
src/pages/saddles.html

@@ -30,6 +30,11 @@
                         <a stm-attribute.href="'/tasks/' + saddle.current_task_id" stm-text="saddle.current_task_title">task</a>
                         <span class="muted" stm-if="saddle.allocated_display.length > 0" stm-text="'since ' + saddle.allocated_display">since</span></p>
                     <p stm-if="saddle.current_task_id == 0" class="muted">No current task</p>
+                    <p><span stm-attribute.class="'badge badge-health-' + saddle.health"
+                              stm-text="saddle.health">healthy</span>
+                        <span class="muted" stm-if="saddle.daemon_state.length > 0"
+                              stm-text="'daemon ' + saddle.daemon_state + (saddle.uptime_display.length > 0 ? ' · up ' + saddle.uptime_display : '')">daemon</span>
+                        <span class="muted" stm-text="'contacted ' + saddle.last_contact_display + ' ago'">contacted</span></p>
                     <p stm-if="saddle.token.length > 0" class="muted">API token:
                         <code stm-text="saddle.token">token</code></p>
                     <p stm-if="saddle.capabilities.length > 0">Capabilities:

+ 2 - 0
src/pages/task_detail.html

@@ -508,6 +508,8 @@
             </form>
 
             <h3>Sessions</h3>
+            <p stm-if="task.saddle_health.length > 0" class="muted"
+               stm-text="task.saddle_health">saddle health</p>
             <table>
                 <thead><tr><th>#</th><th>Saddle</th><th>Started</th><th>Ended</th><th>Outcome</th><th></th></tr></thead>
                 <tbody>

+ 1 - 0
src/pages/tasks.html

@@ -84,6 +84,7 @@
                                     <span class="badge" stm-text="capability">tag</span>
                                 </span>
                                 <span class="muted" stm-if="task.waiting_for_saddle"> · waiting for a capable saddle</span>
+                            <span class="muted" stm-if="task.unreachable"> · saddle unreachable — requeueing</span>
                             </span></td>
                         <td stm-text="task.project_name">project</td>
                         <td><span stm-attribute.class="'badge badge-' + task.status" stm-text="task.display">status</span></td>

+ 141 - 4
test/web_stack_test.py

@@ -212,6 +212,7 @@ class FakeSaddle:
         self.suspend_after_start = False  # next task parks as suspended
         self.pending_output = []        # entries drained by the next /output poll
         self.output_seq = 100
+        self.last_activity_epoch = int(time.time())
         self.subtask_status_events = []  # recorded POST /subtask-status payloads
         self.fail_subtask_status = False  # exercise the web's retry path
         self.hang_next = {}              # path suffix -> hang count (never responds)
@@ -222,6 +223,8 @@ class FakeSaddle:
         self.zombie_409 = 0              # /start 409s once per unit (then decrements)
         self.conflict_running = 0        # while > 0, /start persistently 409s with this id
         self.hold_state = False          # freeze /status lifecycle transitions
+        self.started_mono = time.monotonic()
+        self.freeze_activity = False     # hold last_activity_epoch still
 
     def set_state(self, state, summary=None):
         """Advances the scripted lifecycle.
@@ -249,6 +252,10 @@ class FakeSaddle:
             })
             self.output_seq += 1
 
+    def touch_activity(self):
+        if not self.freeze_activity:
+            self.last_activity_epoch = int(time.time())
+
     def append_chunk(self, text):
         with self.lock:
             self.chunks.append({"seq": self.next_seq, "text": text})
@@ -287,6 +294,7 @@ class FakeSaddle:
             self.wfile.write(body)
 
         def do_POST(self):
+            self.daemon.touch_activity()
             length = int(self.headers.get("Content-Length") or 0)
             raw = self.rfile.read(length).decode() if length else "{}"
             if self.path == "/start":
@@ -356,8 +364,14 @@ class FakeSaddle:
             if self.maybe_hang(self.path):
                 time.sleep(600)
                 return
+            if self.path != "/status":
+                # /status itself is not daemon activity: the wedge signal
+                # must reflect loop rounds / command completions.
+                self.daemon.touch_activity()
             if self.path == "/status":
-                response = {"state": self.daemon.state, "summary": self.daemon.summary}
+                response = {"state": self.daemon.state, "summary": self.daemon.summary,
+                            "uptime_seconds": int(time.monotonic() - self.daemon.started_mono),
+                            "last_activity_epoch": self.daemon.last_activity_epoch}
                 if self.daemon.resumed_pending:
                     response["resumed"] = True
                     self.daemon.resumed_pending = False
@@ -405,9 +419,17 @@ class FakeSaddle:
 
     def serve(self):
         handler = type("BoundHandler", (FakeSaddle.Handler,), {"daemon": self})
-        server = ThreadingHTTPServer(("127.0.0.1", DAEMON_PORT), handler)
-        threading.Thread(target=server.serve_forever, daemon=True).start()
-        return server
+        self.server = ThreadingHTTPServer(("127.0.0.1", DAEMON_PORT), handler)
+        threading.Thread(target=self.server.serve_forever, daemon=True).start()
+        return self.server
+
+    def kill(self):
+        """Stops listening: subsequent connections are refused — a daemon
+        that died mid-task."""
+        if getattr(self, "server", None) is not None:
+            self.server.shutdown()
+            self.server.server_close()
+            self.server = None
 
 
 def start_app():
@@ -426,6 +448,7 @@ def start_app():
         "SADDLE_HTTP_TIMEOUT_S": "5",
         "SADDLE_RAW_PROBE_INTERVAL_S": "5",
         "SADDLE_STRAND_GRACE_S": "10",
+        "SADDLE_WEDGE_THRESHOLD_S": "8",
         # Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
         # resolves the ancient /lib64 copies): empty makes the loader use
         # the binary's RUNPATH, which is the dev stack it was built
@@ -2085,6 +2108,120 @@ def main():
         check(wait_task_done(auth, conflict_new, timeout=90) is not None,
               "the refused task dispatches after the fence clears")
 
+
+        print("== R7 B3: wedged daemon, health panel, raw log, vanishing daemon ==")
+        # Raw-log polling starts from the FIRST push (no probe failure):
+        # a normally dispatched task accumulates raw rows immediately.
+        act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
+            "project_id": "1",
+            "title": "Raw from first push",
+            "priority": "2",
+        })
+        raw_first = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+                         if t["title"] == "Raw from first push"][0])
+
+        def raw_count(task_id):
+            conn = db()
+            try:
+                return conn.execute(
+                    "select count(*) from session_raw_log where session_id in"
+                    " (select id from sessions where task_id = ?)", (task_id,)).fetchone()[0]
+            finally:
+                conn.close()
+
+        check(wait_for(lambda: raw_count(raw_first) > 0 or False,
+                       timeout=60, what="raw rows from first push") is not None,
+              "raw-log polling runs from the first push (session-4 gap)")
+        check(wait_task_done(auth, raw_first, timeout=60) is not None,
+              "raw-first task completes")
+
+        # Health panel: after the polls above the snapshot knows the fake.
+        card = saddle_card("fake-saddle")
+        check(card.get("health") in ("healthy", "stale")
+              and card.get("daemon_state") in ("working", "idle", "done")
+              and card.get("uptime_display", "") != ""
+              and card.get("last_contact_display", "") not in ("", "never"),
+              "saddles page shows daemon state, uptime and last contact")
+
+        # Alive-but-wedged: a working daemon whose last activity is stale
+        # (threshold 3s in the harness) fences the saddle and requeues the
+        # task with a bot comment.
+        act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
+            "project_id": "1",
+            "title": "Wedge victim",
+            "priority": "2",
+        })
+        wedge = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+                     if t["title"] == "Wedge victim"][0])
+        check(wait_for(lambda: task_state(auth, wedge).get("status") == "working"
+                       or False, timeout=60, what="wedge victim working") is not None,
+              "wedge victim reaches working")
+        fake.hold_state = True
+        fake.freeze_activity = True
+        check(wait_for(lambda: any("saddle wedged" in c["body"]
+                                   for c in task_state(auth, wedge).get("comments", []))
+                       or False, timeout=40, what="wedge comment") is not None,
+              "stale activity wedges the saddle with a bot comment")
+        check(saddle_card("fake-saddle")["status"] == "recovering",
+              "wedged saddle is fenced")
+        card = saddle_card("fake-saddle")
+        check(card.get("health") == "fenced",
+              "health badge reports fenced")
+        fake.hold_state = False
+        fake.freeze_activity = False
+        # The fake's scripted lifecycle parks on 'done' when released
+        # mid-working; a real daemon lands idle after its round.
+        fake.state = "idle"
+        check(wait_for(lambda: saddle_card("fake-saddle")["health"] in ("healthy", "stale")
+                       or False, timeout=40, what="health recovers") is not None,
+              "health badge recovers with the fence")
+        check(wait_task_done(auth, wedge, timeout=90) is not None,
+              "wedged task requeues and completes after recovery")
+
+        # The daemon vanishing mid-task: three refused polls → offline,
+        # task requeued with a bot comment, UI reflects.
+        act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
+            "project_id": "1",
+            "title": "Vanish victim",
+            "priority": "2",
+        })
+        vanish = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+                      if t["title"] == "Vanish victim"][0])
+        check(wait_for(lambda: task_state(auth, vanish).get("status") == "working"
+                       or False, timeout=60, what="vanish victim working") is not None,
+              "vanish victim reaches working")
+        fake.kill()
+        check(wait_for(lambda: saddle_card("fake-saddle")["status"] == "offline" or False,
+                       timeout=60, what="daemon death → offline") is not None,
+              "vanishing daemon drives the saddle offline")
+        check(wait_for(lambda: any("unreachable" in c["body"]
+                                   for c in task_state(auth, vanish).get("comments", []))
+                       or False, timeout=30, what="unreachable comment") is not None,
+              "the affected task gets an unreachable bot comment")
+        check(task_state(auth, vanish)["status"] == "queued",
+              "the task requeues after the daemon death")
+        card = saddle_card("fake-saddle")
+        check(card.get("health") == "offline",
+              "health badge reports offline")
+
+        # The unreachable row flag marks the detection window: a task the
+        # web still sees working while its saddle is offline/fenced, before
+        # the requeue pass moves it. Controlled window (the real
+        # transitions above are atomic with their requeues).
+        conn = db()
+        conn.execute("insert into tasks (project_id, title, status, priority,"
+                     " created_at) values (1, 'Unreachable row', 'working', 5,"
+                     " strftime('%s','now'))")
+        unreachable_row = conn.execute("select max(id) from tasks").fetchone()[0]
+        conn.execute("insert into sessions (task_id, saddle_id, started_at)"
+                     " values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
+        conn.commit()
+        conn.close()
+        row = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
+               if int(t["id"]) == unreachable_row][0]
+        check(row.get("unreachable") is True,
+              "queue row shows the unreachable state in the window")
+
     finally:
         # Exit-code diagnostic: distinguishes a crashed app (negative
         # returncode, e.g. -11 SIGSEGV) from a clean shutdown. When