"""The dispatch bundle (moved verbatim from the pre-split web_stack_test.py; see test/harness.py for the shared machinery and the runner).""" from harness import * # noqa: F401,F403 — the shared wire helpers # ============================================================================= # Bundle: dispatch — the task pipeline end to end: registration tokens, # rules, project notes, pushed chunks/results ingestion, capability # routing, the create form, and retry/manual resume. # ============================================================================= def run(ctx): auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token app = ctx.app print("== registration: token minting and delivery ==") registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"] if s["name"] == "fake-saddle"] check(len(registered) == 1 and registered[0]["token"].startswith("saddle-"), "fake saddle registered with an API token") # Plan 10 phase 1: registration delivers the per-saddle token to # the daemon via POST /register {saddle_token, web_base_url}. check(wait_for(lambda: len(fake.register_log) > 0 or False, timeout=15, what="/register delivery") is not None, "registration POSTs /register to the daemon") register_body = fake.register_log[0] check(register_body.get("saddle_token") == fake_token, "/register carries the minted per-saddle token") check(register_body.get("web_base_url") == WEB, "/register carries the web base URL") check(fake.token == fake_token, "daemon persisted the token (its asks use it)") token_conn = db() row_token = token_conn.execute("select token from saddles where id = ?", (fake_id,)).fetchone()[0] token_conn.close() check(row_token == fake_token, "per-saddle token lives on the saddle row") # The register-time ask (idle-awake daemon): 204, nothing # offerable yet — the saddle is asleep and wakeable. check(wait_for(lambda: any(a["status"] == 204 for a in fake.asks) or False, timeout=15, what="register-time ask") is not None, "the daemon's idle ask returns 204 (nothing offerable)") print("== feature 5: rules ==") rules_state = slot_public(auth, "/rules", "rules") for name, body in [ ("Use Invercargill", "Never use libgee; use Invercargill enumerables."), ("Write Vala docs", "Every public class carries a doc comment."), ("Prefer meson", "Builds go through meson; no makefiles."), ("Co-dev rule", "Keep the wire contract exactly."), ("Reference rule", "Must never ride along."), ]: act(auth, rules_state["create"], {"name": name, "body": body}) listed = {r["name"]: r for r in slot_public(auth, "/rules", "rules")["rules"]} check(len(listed) == 5, "five rules created and listed on /rules") def rule_id(name): rows = [r for r in slot_public(auth, "/rules", "rules")["rules"] if r["name"] == name] return int(rows[0]["id"]) state = slot_public(auth, "/projects", "projects") act(auth, state["create"], {"name": "co", "git_url": "https://example.com/co.git"}) act(auth, state["create"], {"name": "ref", "git_url": "https://example.com/ref.git"}) co_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"] if p["name"] == "co"][0]) ref_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"] if p["name"] == "ref"][0]) project = slot_public(auth, "/projects/1", "project") act(auth, project["add_link"], {"linked_project_id": str(co_id), "relationship": "co-developed"}) act(auth, project["add_link"], {"linked_project_id": str(ref_id), "relationship": "reference"}) act(auth, project["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))}) act(auth, project["attach_rule"], {"rule_id": str(rule_id("Prefer meson"))}) project = slot_public(auth, "/projects/1", "project") check(sorted(r["name"] for r in project["rules"]) == ["Prefer meson", "Use Invercargill"], "project page lists its attached rules") co_project = slot_public(auth, "/projects/%d" % co_id, "project") act(auth, co_project["attach_rule"], {"rule_id": str(rule_id("Co-dev rule"))}) ref_project = slot_public(auth, "/projects/%d" % ref_id, "project") act(auth, ref_project["attach_rule"], {"rule_id": str(rule_id("Reference rule"))}) tasks_state = slot_public(auth, "/tasks", "tasks") # Draft-first (deterministic rule attachment): the dispatcher claims # a queued task on its 1s tick, which under load could beat the # attach_rule calls below and ship an assignment without the task's # own rules — attach while draft, enqueue last. act(auth, tasks_state["create"], { "project_id": "1", "title": "Rules dispatch", "description": "Effective rule set rides the /start payload.", "priority": "2", }) rule_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"] if t["title"] == "Rules dispatch"][0]) detail = task_state(auth, rule_task) act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Write Vala docs"))}) act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))}) enqueue_task(auth, rule_task) detail = task_state(auth, rule_task) check(sorted(r["name"] for r in detail["rules"]) == ["Use Invercargill", "Write Vala docs"], "task page lists its attached rules") check(wait_for(lambda: len(fake.start_payloads) > 0 or False, timeout=30, what="/start push") is not None, "dispatcher pushes the task to the fake saddle") payload = fake.start_payloads[-1] rule_names = [r["name"] for r in payload.get("rules", [])] check(rule_names == ["Write Vala docs", "Use Invercargill", "Prefer meson", "Co-dev rule"], "effective rules in /start: task, primary, co-developed order (%s)" % rule_names) check(all(r.get("body") for r in payload.get("rules", [])), "rules carry their bodies") check("Reference rule" not in rule_names, "reference project rules excluded") detail = task_state(auth, rule_task) act(auth, detail["rules"][0]["detach"], {}) check([r["name"] for r in task_state(auth, rule_task)["rules"]] == ["Use Invercargill"], "detached task rule disappears") rules_rows = slot_public(auth, "/rules", "rules")["rules"] docs_row = [r for r in rules_rows if r["name"] == "Write Vala docs"][0] act(auth, docs_row["update"], {"name": "Write Vala docs", "body": "Doc comments or bust."}) check(any(r["body"] == "Doc comments or bust." for r in slot_public(auth, "/rules", "rules")["rules"]), "rule edit saves the body") check(wait_task_done(auth, rule_task, timeout=60) is not None, "dispatched task completes against the fake daemon") print("== feature 6: project notes + callback ==") # Plan 10 phase 1: the shared saddle-callback token is gone — # the per-saddle token authorises the callback endpoints. callback_token = fake_token check(callback_token.startswith("saddle-") and "saddle-callback-" not in callback_token, "note callback authorised by the per-saddle token") status, _ = saddle_api("POST", "/api/saddle/note", None, {"project": "demo", "body": "x"}) check(status == 401, "note endpoint rejects a missing token") status, _ = saddle_api("POST", "/api/saddle/note", "saddle-callback-not-real", {"project": "demo", "body": "x"}) check(status == 401, "note endpoint rejects a wrong (legacy-style) token") status, body = saddle_api("POST", "/api/saddle/note", callback_token, {"task_id": rule_task, "project": "demo", "body": "style.css tokens live in :root"}) check(status == 200 and body.get("ok") is True, "note accepted with the callback token") status, _ = saddle_api("POST", "/api/saddle/note", callback_token, {"project": "no-such-project", "body": "x"}) check(status == 400, "unknown project rejected") status, _ = saddle_api("POST", "/api/saddle/note", callback_token, {"project": "demo", "body": " "}) check(status == 400, "empty note body rejected") project = slot_public(auth, "/projects/1", "project") check(any("style.css tokens" in n["body"] and int(n["source_task_id"]) == rule_task for n in project["notes"]), "project page lists the note with its source task") act(auth, [n for n in project["notes"] if "style.css tokens" in n["body"]][0]["delete"], {}) check(all("style.css tokens" not in n["body"] for n in slot_public(auth, "/projects/1", "project")["notes"]), "note deleted from the project page") # /start payload: notes of primary + co-developed (not reference), # oldest first; callback carries the configured base URL + token. saddle_api("POST", "/api/saddle/note", callback_token, {"project": "demo", "body": "note-one"}) saddle_api("POST", "/api/saddle/note", callback_token, {"project": "demo", "body": "note-two"}) saddle_api("POST", "/api/saddle/note", callback_token, {"project": "co", "body": "co-note"}) conn = db() conn.execute("insert into project_notes (project_id, body, created_at)" " values (?, 'ref-note-must-not-ship', strftime('%s','now'))", (ref_id,)) conn.commit() conn.close() task_id_tmp = create_queued(auth, { "project_id": "1", "title": "Notes dispatch", "description": "Project notes ride the /start payload.", "priority": "2", }) check(wait_for(lambda: len(fake.start_payloads) >= 2 or False, timeout=30, what="second /start push") is not None, "dispatcher pushes the notes task") payload = fake.start_payloads[-1] notes = payload.get("project_notes", []) check([n["body"] for n in notes] == ["note-one", "note-two", "co-note"], "project_notes: primary + co-developed, oldest first (%s)" % [n["body"] for n in notes]) check(all(n.get("project") for n in notes) and notes[2]["project"] == "co", "project_notes carry the project name") check("ref-note-must-not-ship" not in [n["body"] for n in notes], "reference project notes excluded") callback = payload.get("callback", {}) check(callback.get("base_url") == WEB and callback.get("token") == callback_token, "callback carries the configured base URL and the minted token") # entry cap: 30 of 33 candidates conn = db() for i in range(33): conn.execute("insert into project_notes (project_id, body, created_at)" " values (?, ?, strftime('%s','now'))", (1, "bulk-%02d" % i)) conn.commit() conn.close() create_queued(auth, { "project_id": "1", "title": "Entry cap dispatch", "priority": "2", }) check(wait_for(lambda: len(fake.start_payloads) >= 3 or False, timeout=30, what="third /start push") is not None, "dispatcher pushes the entry-cap task") notes = fake.start_payloads[-1].get("project_notes", []) check(len(notes) == 30 and notes[0]["body"] == "note-one" and notes[1]["body"] == "note-two", "project_notes capped at 30 entries, oldest kept (%d)" % len(notes)) # char cap: the giant note is truncated so the total stays <= 20000 conn = db() conn.execute("delete from project_notes where body like 'bulk-%' or body = 'co-note'") conn.execute("insert into project_notes (project_id, body, created_at)" " values (?, ?, strftime('%s','now'))", (1, "giant!" + "x" * 25000,)) conn.commit() conn.close() create_queued(auth, { "project_id": "1", "title": "Char cap dispatch", "priority": "2", }) check(wait_for(lambda: len(fake.start_payloads) >= 4 or False, timeout=30, what="fourth /start push") is not None, "dispatcher pushes the char-cap task") notes = fake.start_payloads[-1].get("project_notes", []) total = sum(len(n["body"]) for n in notes) check(len(notes) == 3 and total == 20000 and notes[2]["body"].startswith("giant!"), "project_notes char cap: head kept, giant truncated (entries=%d, chars=%d)" % (len(notes), total)) print("== phase 2: pushed chunks -> raw log + terminal view ==") # Wait for the last dispatch to finish so its pushes landed. check(wait_task_done(auth, task_id_of("Char cap dispatch"), timeout=60) is not None, "char-cap task completes") conn = db() raw_session = conn.execute( "select s.id from sessions s join tasks t on t.id = s.task_id" " where t.title = 'Char cap dispatch' order by s.id desc limit 1").fetchone()[0] rows = conn.execute("select seq, chunk from session_raw_log where session_id = ?" " order by seq", (raw_session,)).fetchall() conn.close() check(len(rows) >= 3, "session_raw_log captured the pushed chunks (%d)" % len(rows)) seqs = [r[0] for r in rows] check(seqs == sorted(seqs) and len(set(seqs)) == len(seqs), "pushed chunks stored once, seq strictly increasing") text = "".join(r[1] for r in rows) check(fake.PROMPT in text and "echo hello" in text and "hello" in text, "pushed raw log keeps the prompt and echoed output byte-exact") check("working-heartbeat" in text, "the flush-timer heartbeat chunks keep arriving mid-round") terminal = slot_public(auth, "/tasks/%d/terminal" % task_id_of("Char cap dispatch"), "terminal") check(terminal.get("has_raw_log") is True and int(terminal["session_id"]) == raw_session, "terminal view selects raw rendering for the raw session") # Old sessions (typed entries, no raw rows) keep the fallback view # (a dedicated fixture task: its only session is the legacy one, # so the terminal view cannot prefer a dispatched raw session). conn = db() conn.execute("insert into tasks (project_id, title, status, priority, created_at)" " values (1, 'Legacy terminal task', 'working', 5, strftime('%s','now'))") legacy_task = conn.execute("select max(id) from tasks").fetchone()[0] conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)" " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')", (legacy_task, fake_id)) conn.commit() legacy_session = conn.execute("select max(id) from sessions").fetchone()[0] conn.execute("insert into session_commands (session_id, command_text, output_text," " type, started_at) values (?, 'ls', 'old-output', 'command'," " strftime('%s','now'))", (legacy_session,)) conn.commit() conn.close() legacy = slot_public(auth, "/tasks/%d/terminal" % legacy_task, "terminal") check(legacy.get("has_raw_log") is False, "legacy session falls back to the entry-type terminal view") request = urllib.request.Request(WEB + "/terminal-stream/%d" % raw_session) with urllib.request.urlopen(request, timeout=15) as stream: raw_events = [] deadline = time.monotonic() + 8 while time.monotonic() < deadline and len(raw_events) < 3: line = stream.readline().decode().strip() if not line.startswith("data: ") or line == "data: ping": continue event = json.loads(line[6:]) if event.get("type") == "raw": raw_events.append(event) check(len(raw_events) >= 3 and raw_events[0]["text"] == fake.PROMPT and raw_events[0]["seq"] < raw_events[1]["seq"], "terminal stream serves pushed chunks in seq order (prompt first)") print("== phase 2: chunks/results ingestion — auth, idempotency ==") # Token auth on both push channels (plan 10: the per-saddle # bearer authorises every daemon->web call). status, _ = saddle_api("POST", "/api/saddle/chunks", None, {"session_id": raw_session, "chunks": []}) check(status == 401, "chunks endpoint rejects a missing token") status, _ = saddle_api("POST", "/api/saddle/results", "saddle-bogus-token", {"session_id": raw_session, "state": "working"}) check(status == 401, "results endpoint rejects a wrong token") status, _ = saddle_api("POST", "/api/saddle/chunks", fake_token, {"session_id": 99999, "chunks": []}) check(status == 400, "chunks for an unknown session rejected") status, _ = saddle_api("POST", "/api/saddle/results", fake_token, {"session_id": raw_session, "state": "done"}) check(status == 200, "the owning saddle's session accepts its results") # Pushed chunk idempotency: re-pushing a batch (the outbox's # at-least-once redelivery after a lost response) consumes the # duplicates and stores only the tail. conn = db() before = conn.execute("select count(*) from session_raw_log where session_id = ?", (raw_session,)).fetchone()[0] conn.close() redelivery = [{"seq": r[0], "text": r[1]} for r in rows[:3]] status, body = saddle_api("POST", "/api/saddle/chunks", fake_token, {"session_id": raw_session, "chunks": redelivery}) check(status == 200 and body.get("ok") is True and int(body.get("stored") or 0) == 0, "a fully-duplicate batch stores nothing (duplicates consumed)") fresh = [{"seq": 9000, "text": "late-flush\r\n"}, {"seq": 9001, "text": ""}] # truly empty text: no row status, body = saddle_api("POST", "/api/saddle/chunks", fake_token, {"session_id": raw_session, "chunks": fresh}) check(status == 200 and int(body.get("stored") or 0) == 1, "a partial-redelivery batch stores only the new chunks") conn = db() after = conn.execute("select count(*) from session_raw_log where session_id = ?", (raw_session,)).fetchone()[0] empty_row = conn.execute( "select count(*) from session_raw_log where session_id = ? and seq = 9001", (raw_session,)).fetchone()[0] conn.close() check(after == before + 1 and empty_row == 0, "redelivered chunks never duplicate rows; empty text stores nothing") # Pushed entries: today's OutputEntry shape with the daemon's # global entry ids — a redelivered results push stores its # entries exactly once. def command_rows(session): conn = db() try: return conn.execute( "select command_text, type from session_commands" " where session_id = ? order by id", (session,)).fetchall() finally: conn.close() entries = [{"id": 8101, "command": "sed -n 1p README.md", "output": "# demo", "exit_code": 0, "type": "command", "timestamp": "2026-09-06T00:00:00Z"}] status, _ = saddle_api("POST", "/api/saddle/results", fake_token, {"session_id": raw_session, "state": "working", "entries": entries}) check(status == 200, "entries push accepted") status, _ = saddle_api("POST", "/api/saddle/results", fake_token, {"session_id": raw_session, "state": "working", "entries": entries}) check(status == 200, "redelivered entries push accepted (at-least-once)") stored = command_rows(raw_session) check(len([r for r in stored if r[0] == "sed -n 1p README.md"]) == 1, "redelivered entries store exactly once (daemon-global dedup)") # The retired daemon routes 404 (phase 2 deleted /output, /log # and /start serving) — and the web app never calls them. for method, path in (("GET", "/output?since=0"), ("GET", "/log?since=0"), ("POST", "/start")): request = urllib.request.Request( "http://127.0.0.1:%d%s" % (fake.port, path), data=b"{}" if method == "POST" else None, method=method) try: urllib.request.urlopen(request, timeout=5) gone = False except urllib.error.HTTPError as error: gone = error.code == 404 check(gone, "the fake daemon's %s %s is deleted (404)" % (method, path)) with open(os.path.join(WORK, "app.log")) as handle: app_log = handle.read() check("Saddle output request" not in app_log and "Saddle log request" not in app_log, "the web never polls /output or /log anymore") print("== capabilities: tag-based routing ==") saddles_state = slot_public(auth, "/saddles", "saddles") act(auth, saddles_state["create_capability"], {"name": "gpu"}) act(auth, saddles_state["create_capability"], {"name": "bigmem"}) saddles_state = slot_public(auth, "/saddles", "saddles") caps = {c["name"]: c for c in saddles_state["capabilities"]} check(len(caps) == 2 and all(int(c["saddle_count"]) == 0 and int(c["task_count"]) == 0 for c in caps.values()), "capabilities created and listed with zero usage") fake_card = saddle_card(auth, "fake-saddle") check(fake_card["capabilities"] == [], "saddle card starts with no capability chips") # A task requiring 'gpu' waits while the only idle saddle lacks it; # a later task without requirements still dispatches (skip-and-continue). tasks_state = slot_public(auth, "/tasks", "tasks") task_id_tmp = create_queued(auth, { "project_id": "1", "title": "GPU task", "priority": "2", }) gpu_task = task_id_tmp detail = task_state(auth, gpu_task) act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))}) check(wait_for(lambda: [c["name"] for c in task_state(auth, gpu_task)["capabilities"]] == ["gpu"] or False, timeout=90, what="capability attached") is not None, "task page lists the required capability") detail = task_state(auth, gpu_task) # The only idle saddle lacks the tag, so the queue flags the wait # (read before any other task claims the idle saddle). The state is # stable until the any-saddle task below dispatches — poll for it # rather than sampling once (a single read under load once sampled # the row before the attach's pulse settled). check(wait_for(lambda: (lambda row: row["status"] == "queued" and row["capabilities"] == ["gpu"] and row.get("waiting_for_saddle") is True)( [t for t in slot_public(auth, "/tasks", "tasks")["tasks"] if int(t["id"]) == gpu_task][0]) or False, timeout=20, what="waiting flag") is not None, "queue row shows the waiting task with its tag and wait flag") task_id_tmp = create_queued(auth, { "project_id": "1", "title": "Any-saddle task", "priority": "3", }) any_task = task_id_tmp check(wait_for(lambda: task_state(auth, any_task).get("status") != "queued" or False, timeout=30, what="uncapable task dispatches") is not None, "task without requirements dispatches past the waiting task") def capability_wait_logged(): with open(os.path.join(WORK, "app.log")) as handle: app_log = handle.read() return "waits for a capable saddle" in app_log and "requires: gpu" in app_log check(wait_for(capability_wait_logged, timeout=90, what="capability wait log") is not None, "dispatcher logs the capability wait") # Attaching the tag to the saddle releases the task. fake_card = saddle_card(auth, "fake-saddle") act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))}) fake_card = saddle_card(auth, "fake-saddle") check([c["name"] for c in fake_card["capabilities"]] == ["gpu"], "saddle card shows the attached chip") check(wait_task_done(auth, gpu_task, timeout=60) is not None, "task dispatches once the saddle carries the required tag") queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"] if int(t["id"]) == gpu_task][0] check(queue.get("waiting_for_saddle") is not True, "wait flag clears once dispatched") # Multi-tag: gpu + bigmem required, only gpu carried → waits; adding # bigmem completes the superset. Both requirements ride the create # form (atomic — attaching them one at a time would race a tick # into dispatching on the partial set). task_id_tmp = create_queued(auth, { "project_id": "1", "title": "Big task", "priority": "2", "capability_id": [str(int(caps["gpu"]["id"])), str(int(caps["bigmem"]["id"]))], }) big_task = task_id_tmp check(wait_for(lambda: ([t for t in slot_public(auth, "/tasks", "tasks")["tasks"] if int(t["id"]) == big_task][0].get("waiting_for_saddle") is True) or False, timeout=30, what="big task waits") is not None, "task waits when the saddle's tags are not a superset") fake_card = saddle_card(auth, "fake-saddle") act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["bigmem"]["id"]))}) check(wait_task_done(auth, big_task, timeout=60) is not None, "task dispatches once the superset is complete") # Detach a requirement (on a draft — capabilities lock once a # worker has the job); delete a capability everywhere. act(auth, slot_public(auth, "/tasks", "tasks")["create"], { "project_id": "1", "title": "Detach probe task", "priority": "4", "capability_id": [str(int(caps["gpu"]["id"])), str(int(caps["bigmem"]["id"]))], }) detach_probe = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"] if t["title"] == "Detach probe task"][0]) act(auth, [c for c in task_state(auth, detach_probe)["capabilities"] if c["name"] == "bigmem"][0]["detach"], {}) check([c["name"] for c in task_state(auth, detach_probe)["capabilities"]] == ["gpu"], "detached requirement disappears from the task") bigmem_row = [c for c in slot_public(auth, "/saddles", "saddles")["capabilities"] if c["name"] == "bigmem"][0] act(auth, bigmem_row["delete"], {}) saddles_state = slot_public(auth, "/saddles", "saddles") check(all(c["name"] != "bigmem" for c in saddles_state["capabilities"]), "deleted capability gone from the catalogue") check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"], "deleting a capability drops its task requirements") check(all(c["name"] != "bigmem" for c in saddle_card(auth, "fake-saddle")["capabilities"]), "deleting a capability drops it from saddles") print("== B4: required capabilities on the create form ==") tasks_state = slot_public(auth, "/tasks", "tasks") gpu_option = [o for o in tasks_state.get("capability_options", []) if o["name"] == "gpu"][0] check(len(tasks_state.get("capability_options", [])) >= 1, "queue page offers capability options in the create form") conn = db() before = conn.execute("select count(*) from tasks").fetchone()[0] conn.close() act(auth, tasks_state["create"], { "project_id": "1", "title": "Bad cap task", "capability_id": [gpu_option["id"], "999"], }) conn = db() after = conn.execute("select count(*) from tasks").fetchone()[0] conn.close() check(after == before, "unknown capability id rejects the create") task_id_tmp = create_queued(auth, { "project_id": "1", "title": "Cap-on-create task", "capability_id": [gpu_option["id"]], }) cap_create = task_id_tmp detail = task_state(auth, cap_create) check([c["name"] for c in detail["capabilities"]] == ["gpu"], "capabilities from the create form land on the task") queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"] if int(t["id"]) == cap_create][0] check(queue["capabilities"] == ["gpu"], "queue row shows the created task's requirements") # The waiting flag must hold it back until the capable saddle frees # up (the fake saddle still carries gpu from the routing checks). check(queue.get("waiting_for_saddle") is not True, "capable saddle available: no wait flag on the new task") # Drain barrier: the cap task must be CLAIMED before the flaky # arc arms fail_next (under parallel-shard load the dispatch tick # can lag task creation; an unclaimed predecessor would consume # the one-shot failure flag instead of the flaky task). check(wait_for(lambda: task_state(auth, cap_create).get("status") in ("working", "done", "failed") or False, timeout=60, what="cap task claimed") is not None, "the capable task is claimed before the flaky arc starts") print("== retry + resume ==") # A daemon-side failure fails the task with its explanation. fake.fail_next = True task_id_tmp = create_queued(auth, { "project_id": "1", "title": "Flaky task", "priority": "2", }) flaky = task_id_tmp check(wait_for(lambda: task_state(auth, flaky).get("status") == "failed" or False, timeout=90, what="flaky task fails") is not None, "daemon failure fails the task") detail = task_state(auth, flaky) check("bash -n rejected" in (detail.get("summary") or ""), "failure summary shows the daemon's explanation") # Retry: the terminal summary dies with its attempt (plan 16 S4 — # it is NOT copied to the thread), the task requeues, history # stays, and the re-dispatch restores the workspace. comments_before = len(detail.get("comments", [])) progress_before = len(detail.get("progress", [])) act(auth, detail["retry"], {}) detail = task_state(auth, flaky) check(detail["status"] == "queued" and (detail.get("summary") or "") == "", "retry requeues the task and clears the stale summary") check(not any("bash -n rejected" in c["body"] for c in detail["comments"]), "no summary→comment copy joins the thread (the terminal cache dies with its attempt, S4)") check(any(c["body"] == "Task retried by user" and c["author"] == "system" and c.get("is_system") for c in detail["comments"]), "retry lands its audit comment as a system row (never an agent row)") check(len(detail["comments"]) >= comments_before, "retry keeps the comment history") pushes_before = len([p for p in fake.start_payloads if int(p.get("task", {}).get("id") or 0) == flaky]) check(wait_for(lambda: len([p for p in fake.start_payloads if int(p.get("task", {}).get("id") or 0) == flaky]) > pushes_before or False, timeout=90, what="flaky task re-dispatched") is not None, "retried task dispatches again") # The resumed arc surfaces on the next /status read — in the # push world that is the operator's manual refresh. act(auth, saddle_card(auth, "fake-saddle")["refresh"], {}) check(wait_task_done(auth, flaky, timeout=60) is not None, "retried task completes on the restored workspace") detail = task_state(auth, flaky) check(all("restored the session context" not in c["body"] for c in detail.get("comments", [])), "the session-restore stays OUT of the user-facing thread (U2: raw terminal only)") check(all("restored the session context" not in p["summary"] for p in detail.get("progress", [])), "the resume note stays OUT of the progress feed (F8: status is not progress)") # Plan 16 S4: terminal text has ONE representation — the failed # attempt's summary lived in tasks.summary alone (asserted above: # no agent comment), and a HUMAN comment lands as exactly one # user-authored row (the web is the single writer; the daemon's # terminal echo is raw-log only). act(auth, task_state(auth, flaky)["comment"], {"body": "The retry landed — nice recovery"}) flaky_comments = task_state(auth, flaky)["comments"] check(len([c for c in flaky_comments if c["body"] == "The retry landed — nice recovery"]) == 1 and all(c["author"] != "agent" for c in flaky_comments if c["body"] == "The retry landed — nice recovery"), "a human comment lands as exactly ONE user row (single writer, S4)") # Manual resume: a task stuck working on an offline saddle. task_id_tmp = create_queued(auth, { "project_id": "1", "title": "Stuck task", "priority": "2", }) stuck = task_id_tmp check(wait_for(lambda: task_state(auth, stuck).get("status") == "working" or False, timeout=60, what="stuck task starts working") is not None, "stuck task reaches working") # Deterministic claim for the resume leg below: the round's end, # the refresh probe AND the dispatcher's wake for the requeued task # all trigger self-asks that could steal it — freeze every trigger # until the manual ask below has made the claim. fake.ask_on_terminal = False fake.ask_on_idle_status = False fake.ask_on_wake = False fake.settle_asks() conn = db() conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'") conn.commit() conn.close() detail = task_state(auth, stuck) check(detail.get("resume_available") is True, "task page offers resume while its saddle is offline") # The resume POSTs task-update cancel to the daemon; the abort # handler fires an ungated re-ask that would race the manual claim # below — slow it past the manual ask (the knob the slow-ask # regression uses). fake.ask_delay = 5.0 act(auth, detail["resume"], {}) detail = task_state(auth, stuck) check(detail["status"] == "queued", "manual resume requeues the stuck task") check(any(c["body"] == "Saddle became unreachable; task requeued" and c["author"] == "system" and c.get("is_system") for c in detail["comments"]), "manual resume lands the interruption comment as a system row") check(all(s["status"] == "offline" for s in slot_public(auth, "/saddles", "saddles")["saddles"]), "saddle stays offline after the manual requeue") # Bringing the saddle back lets the requeued task resume and finish. fake.state = "idle" fake_card = saddle_card(auth, "fake-saddle") act(auth, fake_card["refresh"], {}) check(any(s["status"] == "idle" for s in slot_public(auth, "/saddles", "saddles")["saddles"] if s["name"] == "fake-saddle"), "refreshed saddle returns idle") # The refreshed daemon is idle-awake: it asks (the real loop's # own ask; driven directly for determinism — every self-ask trigger # is frozen and the abort re-ask is slowed past it; joining that # delayed thread here would fire its ask first, so it is left to # sleep out the rest of the bundle). status, payload = fake.ask_now() fake.ask_on_idle_status = True fake.ask_on_terminal = True fake.ask_on_wake = True fake.ask_delay = 0.0 check(status == 200, "the refreshed daemon's ask claims the requeued task") check(wait_task_done(auth, stuck, timeout=60) is not None, "requeued task resumes on the refreshed saddle and completes")