|
|
@@ -46,6 +46,13 @@ Plan 10 phase 2 (telemetry flip):
|
|
|
idempotent on the daemon entry ids), blocked raises the block row +
|
|
|
parks + bumps the pulse (sealed refresh shows it), done/failed
|
|
|
complete/fail the task, suspended parks for the affinity resume
|
|
|
+- answer resumes, both daemon shapes: a SLEEPING affine saddle (its
|
|
|
+ post-park ask returned 204) is woken, its ask allocating the resume
|
|
|
+ with the verdict riding the assignment; a daemon parked MID-ROUND
|
|
|
+ (its round holds the block's answer wait open, so it can never ask)
|
|
|
+ gets the verdict delivered into that wait via POST /answer — same
|
|
|
+ session, no wake. A daemon dying with a parked-with-answered task
|
|
|
+ never requeues it: the verdict and session survive for the resume
|
|
|
- staleness: a silent runner gets ONE exceptional GET /status probe —
|
|
|
alive keeps it running, dead requeues the task with a bot comment and
|
|
|
marks the saddle offline; a recovered daemon's ask flips offline rows
|
|
|
@@ -269,11 +276,14 @@ class FakeSaddle:
|
|
|
to, then PUSHES its telemetry home: raw PTY chunks via POST
|
|
|
/api/saddle/chunks and round results with state transitions via
|
|
|
POST /api/saddle/results (working at round start, terminal states
|
|
|
- when the scripted lifecycle ends them). /output, /log and /start are
|
|
|
- gone (404) like a post-phase-2 daemon; /task-update serves the
|
|
|
- cancel/comment/subtask-status kinds. Every trigger can be disabled
|
|
|
- per test for determinism; ask_now() performs a synchronous manual
|
|
|
- ask.
|
|
|
+ when the scripted lifecycle ends them). /output, /log and /start
|
|
|
+ are gone (404) like a post-phase-2 daemon; /task-update serves the
|
|
|
+ cancel/comment/subtask-status kinds; POST /answer receives a
|
|
|
+ delivered verdict when the round is parked in a block's answer
|
|
|
+ wait (409 when nothing waits — the real daemon's contract; the
|
|
|
+ resumed round continues after the response). Every trigger can be
|
|
|
+ disabled per test for determinism; ask_now() performs a
|
|
|
+ synchronous manual ask.
|
|
|
"""
|
|
|
|
|
|
PROMPT = "bbarrow@fedora:/workspace/task-1$ "
|
|
|
@@ -320,6 +330,10 @@ class FakeSaddle:
|
|
|
self.serve_register = True # False: 404 /register (pre-flip daemon)
|
|
|
self.last_ask_trigger = 0.0
|
|
|
self.port = None
|
|
|
+ # --- answer delivery (the parked-session resume) ---
|
|
|
+ self.pending_block = None # the block payload this daemon waits on (state blocked)
|
|
|
+ self.answer_log = [] # recorded POST /answer bodies
|
|
|
+ self.answer_push_delay = 0.2 # the resumed round's first push, after the response
|
|
|
|
|
|
def set_state(self, state, summary=None):
|
|
|
"""Advances the scripted lifecycle."""
|
|
|
@@ -391,6 +405,10 @@ class FakeSaddle:
|
|
|
payload["entries"] = entries
|
|
|
if block is not None:
|
|
|
payload["block"] = block
|
|
|
+ if state == "blocked":
|
|
|
+ # The daemon parks its round on this block's answer wait
|
|
|
+ # (the real daemon's wait_for_answer).
|
|
|
+ self.pending_block = dict(block)
|
|
|
if summary is not None:
|
|
|
payload["summary"] = summary
|
|
|
try:
|
|
|
@@ -460,6 +478,9 @@ class FakeSaddle:
|
|
|
self.running_task = int(task_id or 0)
|
|
|
self.running_title = task.get("title") or ""
|
|
|
self.session_id = payload.get("session_id")
|
|
|
+ # The assignment's answers resolve any block this daemon was
|
|
|
+ # parked on (resume-carries-answer).
|
|
|
+ self.pending_block = None
|
|
|
count = self.start_counts.get(task_id, 0) + 1
|
|
|
self.start_counts[task_id] = count
|
|
|
if count > 1:
|
|
|
@@ -539,6 +560,43 @@ class FakeSaddle:
|
|
|
if self.daemon.ask_on_wake:
|
|
|
self.daemon.maybe_trigger_ask("wake")
|
|
|
return
|
|
|
+ if self.path == "/answer":
|
|
|
+ # The web delivers a human verdict into the daemon's
|
|
|
+ # live answer wait (the pre-flip channel, and the only
|
|
|
+ # resume for a session parked mid-round — a blocked
|
|
|
+ # daemon cannot ask). 409 mirrors the real daemon when
|
|
|
+ # no block is pending. The resumed round continues
|
|
|
+ # AFTER the response goes out (the real daemon resumes
|
|
|
+ # its parked wait_for_answer via an idle callback).
|
|
|
+ try:
|
|
|
+ payload = json.loads(raw)
|
|
|
+ except ValueError:
|
|
|
+ self._send(400, {"error": "bad json"})
|
|
|
+ return
|
|
|
+ parked = self.daemon.pending_block
|
|
|
+ if self.daemon.state != "blocked" or parked is None:
|
|
|
+ self._send(409, {"error": "no block is waiting for an answer"})
|
|
|
+ return
|
|
|
+ self.daemon.pending_block = None
|
|
|
+ self.daemon.answer_log.append({
|
|
|
+ "answer": payload.get("answer", ""),
|
|
|
+ "branch_advanced": payload.get("branch_advanced"),
|
|
|
+ "block": dict(parked)})
|
|
|
+ self.daemon.set_state("working")
|
|
|
+ self._send(200, {"ok": True})
|
|
|
+ daemon_ref = self.daemon
|
|
|
+
|
|
|
+ def continue_round():
|
|
|
+ time.sleep(daemon_ref.answer_push_delay)
|
|
|
+ if daemon_ref.dead or daemon_ref.silent:
|
|
|
+ return
|
|
|
+ daemon_ref.push_results("working")
|
|
|
+ daemon_ref.set_state("done", "resumed after the verdict")
|
|
|
+ daemon_ref.push_results("done", summary="resumed after the verdict")
|
|
|
+ if daemon_ref.ask_on_terminal:
|
|
|
+ daemon_ref.maybe_trigger_ask("terminal")
|
|
|
+ threading.Thread(target=continue_round, daemon=True).start()
|
|
|
+ return
|
|
|
if self.path == "/task-update":
|
|
|
# Plan 10: cancel (abort machinery), comment (injected
|
|
|
# turn) and subtask-status (live subtask set) — delivered
|
|
|
@@ -3083,6 +3141,217 @@ def main():
|
|
|
check(wait_task_done(auth, live_task, timeout=60) is not None,
|
|
|
"live-block task completes after answering")
|
|
|
|
|
|
+ print("== phase 2: answer resume — wake -> ask -> affinity claim ==")
|
|
|
+ # The full live sequence against a daemon that asks after its
|
|
|
+ # park: ask -> assignment -> blocked push (park) -> ask -> 204
|
|
|
+ # -> sleeps (row idle) -> human answers -> the dispatcher WAKES
|
|
|
+ # the sleeping affine saddle -> its ask allocates the resume,
|
|
|
+ # the verdict riding the assignment -> the session continues.
|
|
|
+ # A private capability tag keeps the fleet's other sleeping
|
|
|
+ # saddles from claiming these tasks.
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "answerflow"})
|
|
|
+ answerflow_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
+ if c["name"] == "answerflow"][0])
|
|
|
+
|
|
|
+ def answerflow_daemon(**kwargs):
|
|
|
+ daemon = FakeSaddle()
|
|
|
+ daemon.ask_on_register = False # manual, deterministic asks
|
|
|
+ daemon.hold_state = True # the tests drive the lifecycle
|
|
|
+ for key, value in kwargs.items():
|
|
|
+ setattr(daemon, key, value)
|
|
|
+ daemon.serve()
|
|
|
+ return daemon
|
|
|
+
|
|
|
+ def register_answerflow(daemon, name):
|
|
|
+ act(auth, slot_public(auth, "/saddles", "saddles")["register"], {
|
|
|
+ "name": name, "host": "127.0.0.1:%d" % daemon.port})
|
|
|
+ act(auth, saddle_card(name)["attach_capability"],
|
|
|
+ {"capability_id": str(answerflow_cap)})
|
|
|
+
|
|
|
+ def create_answerflow_task(title):
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1", "title": title, "priority": "2",
|
|
|
+ "capability_id": [str(answerflow_cap)]})
|
|
|
+ return int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == title][0])
|
|
|
+
|
|
|
+ def retire_daemon(daemon, name):
|
|
|
+ # deterministic teardown: a dead daemon and an offline row
|
|
|
+ # never interfere with the sections that follow
|
|
|
+ daemon.kill()
|
|
|
+ act(auth, saddle_card(name)["refresh"], {})
|
|
|
+
|
|
|
+ sleeper = answerflow_daemon(ask_on_wake=True)
|
|
|
+ register_answerflow(sleeper, "resume-sleeper")
|
|
|
+ sleeping_task = create_answerflow_task("Sleeping resume task")
|
|
|
+ ask_status, assignment = sleeper.ask_now()
|
|
|
+ check(ask_status == 200 and int(assignment["task"]["id"]) == sleeping_task,
|
|
|
+ "the daemon's ask claims the queued task")
|
|
|
+ sleeping_session = assignment.get("session_id")
|
|
|
+ sleeper.state = "blocked"
|
|
|
+ check(sleeper.push_results("blocked", block={
|
|
|
+ "question": "Pick a side", "type": "question",
|
|
|
+ "options": ["Port", "Starboard"]}) == 200,
|
|
|
+ "the blocked push parks the session")
|
|
|
+ check(wait_for(lambda: task_state(auth, sleeping_task).get("status") == "blocked"
|
|
|
+ or False, timeout=30, what="sleeping task parked") is not None,
|
|
|
+ "the task parks as blocked")
|
|
|
+ check(saddle_card("resume-sleeper")["status"] == "idle",
|
|
|
+ "the parked session's saddle row reads idle")
|
|
|
+ ask_status, _ = sleeper.ask_now()
|
|
|
+ check(ask_status == 204,
|
|
|
+ "the parked daemon's ask returns 204 — it sleeps until woken")
|
|
|
+ time.sleep(3) # several dispatch ticks pass with the daemon asleep
|
|
|
+ check(len(sleeper.wake_log) == 0,
|
|
|
+ "the sleeping daemon is never woken while the block waits")
|
|
|
+ detail = task_state(auth, sleeping_task)
|
|
|
+ side_block = [b for b in detail["blocks"] if b["question"] == "Pick a side"][0]
|
|
|
+ act(auth, side_block["answer_action"], {"option": "Starboard"})
|
|
|
+ check(wait_for(lambda: len(sleeper.wake_log) > 0 or False,
|
|
|
+ timeout=30, what="wake after the answer") is not None,
|
|
|
+ "answering the block wakes the sleeping affine saddle")
|
|
|
+ check(wait_for(lambda: any(
|
|
|
+ int(p.get("task", {}).get("id") or 0) == sleeping_task
|
|
|
+ and p.get("session_id") == sleeping_session
|
|
|
+ and any(a.get("answer") == "Starboard"
|
|
|
+ for a in p.get("task", {}).get("answers", []))
|
|
|
+ for p in sleeper.start_payloads[1:]) or False,
|
|
|
+ timeout=30, what="resume allocation carries the verdict") is not None,
|
|
|
+ "the woken daemon's ask allocates the resume — same session, verdict carried")
|
|
|
+ # The session continues: the resumed round runs to completion on
|
|
|
+ # the SAME session id.
|
|
|
+ sleeper.push_results("done", summary="sleep-resumed round complete")
|
|
|
+ check(wait_task_done(auth, sleeping_task, timeout=30) is not None,
|
|
|
+ "the resumed session continues to completion")
|
|
|
+ conn = db()
|
|
|
+ session_rows = conn.execute(
|
|
|
+ "select count(*) from sessions where task_id = ?", (sleeping_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(session_rows == 1,
|
|
|
+ "the sleeping resume kept one session throughout")
|
|
|
+ retire_daemon(sleeper, "resume-sleeper")
|
|
|
+
|
|
|
+ print("== phase 2: answer resume — verdict delivered to the parked session ==")
|
|
|
+ # The live regression's daemon shape (clanker-test.db): after the
|
|
|
+ # blocked push the daemon CANNOT ask again — its round is parked
|
|
|
+ # in the block's answer wait, so the saddle's last ask stays the
|
|
|
+ # one that claimed the task (never sleeping, never woken). The
|
|
|
+ # verdict must be DELIVERED into that answer wait.
|
|
|
+ parked = answerflow_daemon(ask_on_wake=False, # a blocked daemon cannot ask on wake
|
|
|
+ ask_on_terminal=False, # and not after the round either
|
|
|
+ ask_on_idle_status=False)
|
|
|
+ register_answerflow(parked, "resume-parked")
|
|
|
+ parked_task = create_answerflow_task("Parked resume task")
|
|
|
+ ask_status, assignment = parked.ask_now()
|
|
|
+ check(ask_status == 200 and int(assignment["task"]["id"]) == parked_task,
|
|
|
+ "the daemon's ask claims the task (its last ask)")
|
|
|
+ parked_session = assignment.get("session_id")
|
|
|
+ parked.state = "blocked"
|
|
|
+ check(parked.push_results("blocked", block={
|
|
|
+ "question": "Which database?", "type": "question",
|
|
|
+ "options": ["SQLite", "Postgres"]}) == 200,
|
|
|
+ "the blocked push parks the session mid-round")
|
|
|
+ check(wait_for(lambda: task_state(auth, parked_task).get("status") == "blocked"
|
|
|
+ or False, timeout=30, what="parked task blocked") is not None,
|
|
|
+ "the task parks as blocked")
|
|
|
+ check(saddle_card("resume-parked")["status"] == "idle",
|
|
|
+ "the parked session's saddle row reads idle")
|
|
|
+ time.sleep(3) # dispatch ticks pass: no wake, no ask ever reaches it
|
|
|
+ asks_at_answer = len(parked.asks)
|
|
|
+ check(len(parked.wake_log) == 0 and asks_at_answer == 1,
|
|
|
+ "the parked daemon is never woken and never asks again (the live signature)")
|
|
|
+ detail = task_state(auth, parked_task)
|
|
|
+ db_block = [b for b in detail["blocks"] if b["question"] == "Which database?"][0]
|
|
|
+ act(auth, db_block["answer_action"], {"option": "SQLite"})
|
|
|
+ check(wait_for(lambda: len(parked.answer_log) > 0 or False,
|
|
|
+ timeout=30, what="verdict delivered") is not None,
|
|
|
+ "the verdict is delivered into the parked daemon's answer wait")
|
|
|
+ check(parked.answer_log[0]["answer"] == "SQLite",
|
|
|
+ "the delivered verdict is the human's choice")
|
|
|
+ check(parked.answer_log[0]["branch_advanced"] is False,
|
|
|
+ "the delivery carries branch_advanced=false with no siblings")
|
|
|
+ check(len(parked.wake_log) == 0 and len(parked.asks) == asks_at_answer,
|
|
|
+ "no wake, no ask — the delivery itself resumed the session")
|
|
|
+ check(wait_task_done(auth, parked_task, timeout=30) is not None,
|
|
|
+ "the parked session continues to completion")
|
|
|
+ conn = db()
|
|
|
+ row = conn.execute("select outcome from sessions where id = ?",
|
|
|
+ (parked_session,)).fetchone()
|
|
|
+ session_rows = conn.execute(
|
|
|
+ "select count(*) from sessions where task_id = ?", (parked_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(row is not None and row[0] == "done" and session_rows == 1,
|
|
|
+ "the delivered verdict resumed the SAME session to done")
|
|
|
+ retire_daemon(parked, "resume-parked")
|
|
|
+
|
|
|
+ print("== phase 2: offline requeue keeps parked-with-answered tasks ==")
|
|
|
+ # A daemon dying while its parked task carries an answered
|
|
|
+ # verdict: going offline must not requeue the task or close its
|
|
|
+ # session — the resume machinery (delivery once the daemon
|
|
|
+ # returns, or the affinity allocation on its next ask) needs
|
|
|
+ # both. The question park leaves the row idle; the review park
|
|
|
+ # keeps it busy@task (`saddle review` blocks the round without
|
|
|
+ # a parking state push), exercising the unreachable path's
|
|
|
+ # requeue decision both ways.
|
|
|
+ stranded = answerflow_daemon()
|
|
|
+ register_answerflow(stranded, "resume-stranded")
|
|
|
+ stranded_task = create_answerflow_task("Stranded answered task")
|
|
|
+ ask_status, _ = stranded.ask_now()
|
|
|
+ check(ask_status == 200, "the stranded daemon claims its task")
|
|
|
+ stranded.state = "blocked"
|
|
|
+ stranded.push_results("blocked", block={
|
|
|
+ "question": "Hold or fold?", "type": "question", "options": ["Hold", "Fold"]})
|
|
|
+ check(wait_for(lambda: task_state(auth, stranded_task).get("status") == "blocked"
|
|
|
+ or False, timeout=30, what="stranded task parked") is not None,
|
|
|
+ "the stranded task parks as blocked")
|
|
|
+ detail = task_state(auth, stranded_task)
|
|
|
+ hold_block = [b for b in detail["blocks"] if b["question"] == "Hold or fold?"][0]
|
|
|
+ act(auth, hold_block["answer_action"], {"option": "Hold"})
|
|
|
+ stranded.kill() # the daemon dies before the verdict is delivered
|
|
|
+ act(auth, saddle_card("resume-stranded")["refresh"], {})
|
|
|
+ check(wait_for(lambda: saddle_card("resume-stranded")["status"] == "offline"
|
|
|
+ or False, timeout=30, what="stranded saddle offline") is not None,
|
|
|
+ "the dead daemon's saddle row goes offline")
|
|
|
+ time.sleep(7) # dispatch ticks run their offline passes
|
|
|
+ check(task_state(auth, stranded_task)["status"] == "blocked",
|
|
|
+ "the parked-with-answered task is not requeued by the offline pass")
|
|
|
+ conn = db()
|
|
|
+ open_session = conn.execute(
|
|
|
+ "select count(*) from sessions where task_id = ? and ended_at is null",
|
|
|
+ (stranded_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(open_session == 1,
|
|
|
+ "the parked-with-answered task keeps its open session for the resume")
|
|
|
+
|
|
|
+ reviewer = answerflow_daemon(ask_on_terminal=False)
|
|
|
+ register_answerflow(reviewer, "resume-reviewer")
|
|
|
+ review_task = create_answerflow_task("Review park task")
|
|
|
+ ask_status, _ = reviewer.ask_now()
|
|
|
+ check(ask_status == 200, "the reviewer claims its task")
|
|
|
+ reviewer.queue_output("review", "saddle review", "Ready for review: wired and tested.")
|
|
|
+ check(wait_for(lambda: task_state(auth, review_task).get("status") == "review_requested"
|
|
|
+ or False, timeout=30, what="review park raised") is not None,
|
|
|
+ "the review entry parks the task as review_requested (row stays busy)")
|
|
|
+ reviewer.kill() # the daemon dies before the verdict can be delivered
|
|
|
+ detail = task_state(auth, review_task)
|
|
|
+ verdict_block = [b for b in detail["blocks"]
|
|
|
+ if b["block_type"] == "review"][0]
|
|
|
+ act(auth, verdict_block["answer_action"], {"answer": "Human approved the review"})
|
|
|
+ check(wait_for(lambda: saddle_card("resume-reviewer")["status"] == "offline"
|
|
|
+ or False, timeout=30, what="reviewer offline") is not None,
|
|
|
+ "the failed verdict delivery drives the dead reviewer offline")
|
|
|
+ time.sleep(7) # dispatch ticks run their offline passes
|
|
|
+ check(task_state(auth, review_task)["status"] == "review_requested",
|
|
|
+ "the review-parked task with an answered verdict is never requeued")
|
|
|
+ conn = db()
|
|
|
+ open_session = conn.execute(
|
|
|
+ "select count(*) from sessions where task_id = ? and ended_at is null",
|
|
|
+ (review_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(open_session == 1,
|
|
|
+ "the review-parked task keeps its open session for the resume")
|
|
|
+
|
|
|
print("== phase 2: staleness — one probe decides ==")
|
|
|
# A silent-but-alive runner (long think, nothing to flush): after
|
|
|
# SADDLE_STALE_S (3s here) of silence it owes ONE exceptional
|