|
|
@@ -212,6 +212,7 @@ class FakeSaddle:
|
|
|
self.suspend_after_start = False # next task parks as suspended
|
|
|
self.pending_output = [] # entries drained by the next /output poll
|
|
|
self.output_seq = 100
|
|
|
+ self.last_activity_epoch = int(time.time())
|
|
|
self.subtask_status_events = [] # recorded POST /subtask-status payloads
|
|
|
self.fail_subtask_status = False # exercise the web's retry path
|
|
|
self.hang_next = {} # path suffix -> hang count (never responds)
|
|
|
@@ -222,6 +223,8 @@ class FakeSaddle:
|
|
|
self.zombie_409 = 0 # /start 409s once per unit (then decrements)
|
|
|
self.conflict_running = 0 # while > 0, /start persistently 409s with this id
|
|
|
self.hold_state = False # freeze /status lifecycle transitions
|
|
|
+ self.started_mono = time.monotonic()
|
|
|
+ self.freeze_activity = False # hold last_activity_epoch still
|
|
|
|
|
|
def set_state(self, state, summary=None):
|
|
|
"""Advances the scripted lifecycle.
|
|
|
@@ -249,6 +252,10 @@ class FakeSaddle:
|
|
|
})
|
|
|
self.output_seq += 1
|
|
|
|
|
|
+ def touch_activity(self):
|
|
|
+ if not self.freeze_activity:
|
|
|
+ self.last_activity_epoch = int(time.time())
|
|
|
+
|
|
|
def append_chunk(self, text):
|
|
|
with self.lock:
|
|
|
self.chunks.append({"seq": self.next_seq, "text": text})
|
|
|
@@ -287,6 +294,7 @@ class FakeSaddle:
|
|
|
self.wfile.write(body)
|
|
|
|
|
|
def do_POST(self):
|
|
|
+ self.daemon.touch_activity()
|
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
|
raw = self.rfile.read(length).decode() if length else "{}"
|
|
|
if self.path == "/start":
|
|
|
@@ -356,8 +364,14 @@ class FakeSaddle:
|
|
|
if self.maybe_hang(self.path):
|
|
|
time.sleep(600)
|
|
|
return
|
|
|
+ if self.path != "/status":
|
|
|
+ # /status itself is not daemon activity: the wedge signal
|
|
|
+ # must reflect loop rounds / command completions.
|
|
|
+ self.daemon.touch_activity()
|
|
|
if self.path == "/status":
|
|
|
- response = {"state": self.daemon.state, "summary": self.daemon.summary}
|
|
|
+ response = {"state": self.daemon.state, "summary": self.daemon.summary,
|
|
|
+ "uptime_seconds": int(time.monotonic() - self.daemon.started_mono),
|
|
|
+ "last_activity_epoch": self.daemon.last_activity_epoch}
|
|
|
if self.daemon.resumed_pending:
|
|
|
response["resumed"] = True
|
|
|
self.daemon.resumed_pending = False
|
|
|
@@ -405,9 +419,17 @@ class FakeSaddle:
|
|
|
|
|
|
def serve(self):
|
|
|
handler = type("BoundHandler", (FakeSaddle.Handler,), {"daemon": self})
|
|
|
- server = ThreadingHTTPServer(("127.0.0.1", DAEMON_PORT), handler)
|
|
|
- threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
|
- return server
|
|
|
+ self.server = ThreadingHTTPServer(("127.0.0.1", DAEMON_PORT), handler)
|
|
|
+ threading.Thread(target=self.server.serve_forever, daemon=True).start()
|
|
|
+ return self.server
|
|
|
+
|
|
|
+ def kill(self):
|
|
|
+ """Stops listening: subsequent connections are refused — a daemon
|
|
|
+ that died mid-task."""
|
|
|
+ if getattr(self, "server", None) is not None:
|
|
|
+ self.server.shutdown()
|
|
|
+ self.server.server_close()
|
|
|
+ self.server = None
|
|
|
|
|
|
|
|
|
def start_app():
|
|
|
@@ -426,6 +448,7 @@ def start_app():
|
|
|
"SADDLE_HTTP_TIMEOUT_S": "5",
|
|
|
"SADDLE_RAW_PROBE_INTERVAL_S": "5",
|
|
|
"SADDLE_STRAND_GRACE_S": "10",
|
|
|
+ "SADDLE_WEDGE_THRESHOLD_S": "8",
|
|
|
# Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
|
|
|
# resolves the ancient /lib64 copies): empty makes the loader use
|
|
|
# the binary's RUNPATH, which is the dev stack it was built
|
|
|
@@ -2085,6 +2108,120 @@ def main():
|
|
|
check(wait_task_done(auth, conflict_new, timeout=90) is not None,
|
|
|
"the refused task dispatches after the fence clears")
|
|
|
|
|
|
+
|
|
|
+ print("== R7 B3: wedged daemon, health panel, raw log, vanishing daemon ==")
|
|
|
+ # Raw-log polling starts from the FIRST push (no probe failure):
|
|
|
+ # a normally dispatched task accumulates raw rows immediately.
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Raw from first push",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ raw_first = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Raw from first push"][0])
|
|
|
+
|
|
|
+ def raw_count(task_id):
|
|
|
+ conn = db()
|
|
|
+ try:
|
|
|
+ return conn.execute(
|
|
|
+ "select count(*) from session_raw_log where session_id in"
|
|
|
+ " (select id from sessions where task_id = ?)", (task_id,)).fetchone()[0]
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ check(wait_for(lambda: raw_count(raw_first) > 0 or False,
|
|
|
+ timeout=60, what="raw rows from first push") is not None,
|
|
|
+ "raw-log polling runs from the first push (session-4 gap)")
|
|
|
+ check(wait_task_done(auth, raw_first, timeout=60) is not None,
|
|
|
+ "raw-first task completes")
|
|
|
+
|
|
|
+ # Health panel: after the polls above the snapshot knows the fake.
|
|
|
+ card = saddle_card("fake-saddle")
|
|
|
+ check(card.get("health") in ("healthy", "stale")
|
|
|
+ and card.get("daemon_state") in ("working", "idle", "done")
|
|
|
+ and card.get("uptime_display", "") != ""
|
|
|
+ and card.get("last_contact_display", "") not in ("", "never"),
|
|
|
+ "saddles page shows daemon state, uptime and last contact")
|
|
|
+
|
|
|
+ # Alive-but-wedged: a working daemon whose last activity is stale
|
|
|
+ # (threshold 3s in the harness) fences the saddle and requeues the
|
|
|
+ # task with a bot comment.
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Wedge victim",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ wedge = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Wedge victim"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, wedge).get("status") == "working"
|
|
|
+ or False, timeout=60, what="wedge victim working") is not None,
|
|
|
+ "wedge victim reaches working")
|
|
|
+ fake.hold_state = True
|
|
|
+ fake.freeze_activity = True
|
|
|
+ check(wait_for(lambda: any("saddle wedged" in c["body"]
|
|
|
+ for c in task_state(auth, wedge).get("comments", []))
|
|
|
+ or False, timeout=40, what="wedge comment") is not None,
|
|
|
+ "stale activity wedges the saddle with a bot comment")
|
|
|
+ check(saddle_card("fake-saddle")["status"] == "recovering",
|
|
|
+ "wedged saddle is fenced")
|
|
|
+ card = saddle_card("fake-saddle")
|
|
|
+ check(card.get("health") == "fenced",
|
|
|
+ "health badge reports fenced")
|
|
|
+ fake.hold_state = False
|
|
|
+ fake.freeze_activity = False
|
|
|
+ # The fake's scripted lifecycle parks on 'done' when released
|
|
|
+ # mid-working; a real daemon lands idle after its round.
|
|
|
+ fake.state = "idle"
|
|
|
+ check(wait_for(lambda: saddle_card("fake-saddle")["health"] in ("healthy", "stale")
|
|
|
+ or False, timeout=40, what="health recovers") is not None,
|
|
|
+ "health badge recovers with the fence")
|
|
|
+ check(wait_task_done(auth, wedge, timeout=90) is not None,
|
|
|
+ "wedged task requeues and completes after recovery")
|
|
|
+
|
|
|
+ # The daemon vanishing mid-task: three refused polls → offline,
|
|
|
+ # task requeued with a bot comment, UI reflects.
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Vanish victim",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ vanish = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Vanish victim"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, vanish).get("status") == "working"
|
|
|
+ or False, timeout=60, what="vanish victim working") is not None,
|
|
|
+ "vanish victim reaches working")
|
|
|
+ fake.kill()
|
|
|
+ check(wait_for(lambda: saddle_card("fake-saddle")["status"] == "offline" or False,
|
|
|
+ timeout=60, what="daemon death → offline") is not None,
|
|
|
+ "vanishing daemon drives the saddle offline")
|
|
|
+ check(wait_for(lambda: any("unreachable" in c["body"]
|
|
|
+ for c in task_state(auth, vanish).get("comments", []))
|
|
|
+ or False, timeout=30, what="unreachable comment") is not None,
|
|
|
+ "the affected task gets an unreachable bot comment")
|
|
|
+ check(task_state(auth, vanish)["status"] == "queued",
|
|
|
+ "the task requeues after the daemon death")
|
|
|
+ card = saddle_card("fake-saddle")
|
|
|
+ check(card.get("health") == "offline",
|
|
|
+ "health badge reports offline")
|
|
|
+
|
|
|
+ # The unreachable row flag marks the detection window: a task the
|
|
|
+ # web still sees working while its saddle is offline/fenced, before
|
|
|
+ # the requeue pass moves it. Controlled window (the real
|
|
|
+ # transitions above are atomic with their requeues).
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
+ " created_at) values (1, 'Unreachable row', 'working', 5,"
|
|
|
+ " strftime('%s','now'))")
|
|
|
+ unreachable_row = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at)"
|
|
|
+ " values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ row = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if int(t["id"]) == unreachable_row][0]
|
|
|
+ check(row.get("unreachable") is True,
|
|
|
+ "queue row shows the unreachable state in the window")
|
|
|
+
|
|
|
finally:
|
|
|
# Exit-code diagnostic: distinguishes a crashed app (negative
|
|
|
# returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
|