|
|
@@ -79,9 +79,34 @@ Plan 10 phase 3 (capability awareness + UI):
|
|
|
path (offline + re-wake of the next capable) stays green
|
|
|
|
|
|
Prints PASS/FAIL lines Statum-style; exits non-zero on any failure.
|
|
|
+
|
|
|
+Speed pass (plan 11, task 0):
|
|
|
+
|
|
|
+- SHARDING: the scenario groups are partitioned into seven independent
|
|
|
+ bundles (pages, dispatch, subtasks, lineage, allocation, lifecycle,
|
|
|
+ answers), each running in its own process against its own app
|
|
|
+ instance, fake daemons and SQLite database. The default invocation
|
|
|
+ runs all bundles in parallel and aggregates the verdicts and check
|
|
|
+ counts into one exit code — meson sees a single green/red test.
|
|
|
+ `--bundle NAME` runs one shard for debugging; WEBSTACK_SEQUENTIAL=1
|
|
|
+ runs them in-process, one after another.
|
|
|
+- Within a bundle, scenario groups SHARE the app+daemon instance and
|
|
|
+ carry per-group state as database rows (task/session/block
|
|
|
+ fixtures), never process restarts; the one deliberate restart (the
|
|
|
+ startup token migration) lives in the answers bundle.
|
|
|
+- EVENT-DRIVEN WAITS: wait_for polls at 50ms; the fixed sleeps were
|
|
|
+ converted either to observable-condition waits (daemon-side
|
|
|
+ counters, e.g. failed-delivery hits) or to quiet_window() — a
|
|
|
+ bounded negative-assertion poll that FAILS LOUDLY the moment the
|
|
|
+ event it guards against happens (a sleep could silently pass).
|
|
|
+- ENV KNOBS: SADDLE_DISPATCH_TICK_S=1 (dispatcher cadence; default 5
|
|
|
+ in production), SADDLE_HTTP_TIMEOUT_S=2, SADDLE_LOST_WAKE_S=2,
|
|
|
+ SADDLE_STALE_S=3, plus the fake daemon's scripted timings — the
|
|
|
+ wire-traffic shapes stay realistic, just faster.
|
|
|
"""
|
|
|
import json
|
|
|
import os
|
|
|
+import re
|
|
|
import shutil
|
|
|
import socket
|
|
|
import sqlite3
|
|
|
@@ -96,6 +121,25 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
|
|
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
APP = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(REPO, "builddir", "clanker-saddle")
|
|
|
+
|
|
|
+# --- speed-pass tuning -----------------------------------------------------
|
|
|
+# The fake daemon's scripted timings and the dispatch-tick cadence the
|
|
|
+# quiet windows are sized against. All overridable for debugging.
|
|
|
+TICK_S = float(os.environ.get("SADDLE_DISPATCH_TICK_S", "1") or 1)
|
|
|
+FAKE_WORK_S = float(os.environ.get("WEBSTACK_FAKE_WORK_S", "1.2") or 1.2)
|
|
|
+FAKE_FAIL_S = float(os.environ.get("WEBSTACK_FAKE_FAIL_S", "0.5") or 0.5)
|
|
|
+FAKE_HEARTBEAT_S = float(os.environ.get("WEBSTACK_FAKE_HEARTBEAT_S", "0.4") or 0.4)
|
|
|
+FAKE_TRIGGER_S = float(os.environ.get("WEBSTACK_FAKE_TRIGGER_S", "0.05") or 0.05)
|
|
|
+SHARD_TIMEOUT_S = float(os.environ.get("WEBSTACK_SHARD_TIMEOUT_S", "600") or 600)
|
|
|
+
|
|
|
+
|
|
|
+def quiet_seconds():
|
|
|
+ """A negative-assertion window spanning several dispatch ticks (the
|
|
|
+ converted fixed sleeps were 3-7s at the old 5s tick; the bound never
|
|
|
+ exceeds the old timeout)."""
|
|
|
+ return min(2 * TICK_S + 0.8, 7.0)
|
|
|
+
|
|
|
+
|
|
|
WORK = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
|
|
|
DB_PATH = os.path.join(WORK, "web.sqlite")
|
|
|
|
|
|
@@ -124,6 +168,19 @@ DAEMON_PORT = free_port()
|
|
|
WEB = "http://127.0.0.1:%d" % APP_PORT
|
|
|
|
|
|
|
|
|
+def bundle_env(name):
|
|
|
+ """Point the module's runtime globals at one shard's scratch dir,
|
|
|
+ database and ports (each bundle is a process with its own app,
|
|
|
+ daemons and SQLite file)."""
|
|
|
+ global WORK, DB_PATH, APP_PORT, DAEMON_PORT, WEB
|
|
|
+ base = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
|
|
|
+ WORK = "%s-%s" % (base, name)
|
|
|
+ DB_PATH = os.path.join(WORK, "web.sqlite")
|
|
|
+ APP_PORT = free_port()
|
|
|
+ DAEMON_PORT = free_port()
|
|
|
+ WEB = "http://127.0.0.1:%d" % APP_PORT
|
|
|
+
|
|
|
+
|
|
|
def http(method, url, data=None, headers=None, timeout=30):
|
|
|
body = None
|
|
|
if data is not None:
|
|
|
@@ -202,7 +259,7 @@ def login(username, password):
|
|
|
raise SystemExit("login response carried no auth slot: " + body[:300])
|
|
|
|
|
|
|
|
|
-def wait_for(predicate, timeout=60, what="condition", interval=0.5):
|
|
|
+def wait_for(predicate, timeout=60, what="condition", interval=0.05):
|
|
|
deadline = time.monotonic() + timeout
|
|
|
last = None
|
|
|
while time.monotonic() < deadline:
|
|
|
@@ -215,6 +272,24 @@ def wait_for(predicate, timeout=60, what="condition", interval=0.5):
|
|
|
return None
|
|
|
|
|
|
|
|
|
+def quiet_window(seconds, predicate, what):
|
|
|
+ """A converted fixed sleep: asserts a NEGATIVE condition (nothing may
|
|
|
+ happen) across a window spanning several dispatch ticks, polling at
|
|
|
+ 50ms. The moment the condition breaks the window fails loudly
|
|
|
+ (diagnostics + thread dump) instead of silently passing a sleep."""
|
|
|
+ deadline = time.monotonic() + seconds
|
|
|
+ while time.monotonic() < deadline:
|
|
|
+ value = predicate()
|
|
|
+ if value is False or value is None:
|
|
|
+ remaining = deadline - time.monotonic()
|
|
|
+ print(" (quiet window '%s' broken after %.2fs of %.2fs — the"
|
|
|
+ " guarded event DID happen)" % (what, seconds - remaining, seconds))
|
|
|
+ dump_threads()
|
|
|
+ return False
|
|
|
+ time.sleep(0.05)
|
|
|
+ return True
|
|
|
+
|
|
|
+
|
|
|
def dump_threads():
|
|
|
"""Stacks of suite threads not parked in sleep — pinpoints a wedged
|
|
|
fake lifecycle when a wait times out."""
|
|
|
@@ -266,6 +341,31 @@ def db():
|
|
|
return sqlite3.connect(DB_PATH)
|
|
|
|
|
|
|
|
|
+def saddle_card(auth, name):
|
|
|
+ rows = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == name]
|
|
|
+ return rows[0]
|
|
|
+
|
|
|
+
|
|
|
+def queued_count():
|
|
|
+ conn = db()
|
|
|
+ try:
|
|
|
+ return conn.execute(
|
|
|
+ "select count(*) from tasks where status = 'queued'").fetchone()[0]
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+
|
|
|
+def command_rows(session):
|
|
|
+ conn = db()
|
|
|
+ try:
|
|
|
+ return conn.execute(
|
|
|
+ "select command_text, type from session_commands"
|
|
|
+ " where session_id = ? order by id", (session,)).fetchall()
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+
|
|
|
class FakeSaddle:
|
|
|
"""In-process saddle daemon implementing the wire contract.
|
|
|
|
|
|
@@ -301,6 +401,7 @@ class FakeSaddle:
|
|
|
self.output_seq = 100 # daemon-global entry ids for pushed results
|
|
|
self.last_activity_epoch = int(time.time())
|
|
|
self.subtask_status_events = [] # recorded /task-update subtask-status payloads
|
|
|
+ self.subtask_fail_hits = 0 # delivery attempts that 500ed (fail_subtask_status)
|
|
|
self.fail_subtask_status = False # exercise the web's retry path
|
|
|
self.hang_next = {} # path suffix -> hang count (never responds)
|
|
|
self.abort_failures = 0 # cancel task-updates failing (500) while > 0
|
|
|
@@ -330,6 +431,7 @@ class FakeSaddle:
|
|
|
self.serve_register = True # False: 404 /register (pre-flip daemon)
|
|
|
self.last_ask_trigger = 0.0
|
|
|
self.port = None
|
|
|
+ self.ask_threads = [] # scheduled ask-trigger threads (settle_asks joins them)
|
|
|
# --- answer delivery (the parked-session resume) ---
|
|
|
self.pending_block = None # the block payload this daemon waits on (state blocked)
|
|
|
self.answer_log = [] # recorded POST /answer bodies
|
|
|
@@ -457,7 +559,28 @@ class FakeSaddle:
|
|
|
except Exception as error: # app restarting / blips: retry
|
|
|
print(" (ask %s failed: %r, attempt %d)" % (reason, error, attempt + 1))
|
|
|
time.sleep(1.0)
|
|
|
- threading.Thread(target=lambda: (time.sleep(0.2), run()), daemon=True).start()
|
|
|
+
|
|
|
+ def starter():
|
|
|
+ time.sleep(FAKE_TRIGGER_S)
|
|
|
+ run()
|
|
|
+
|
|
|
+ thread = threading.Thread(target=starter, daemon=True)
|
|
|
+ with self.lock:
|
|
|
+ self.ask_threads.append(thread)
|
|
|
+ thread.start()
|
|
|
+
|
|
|
+ def settle_asks(self, timeout=15):
|
|
|
+ """Joins every ask-trigger thread scheduled so far — the
|
|
|
+ event-driven replacement for the fixed sleeps that waited for
|
|
|
+ in-flight asks to land after triggers were frozen."""
|
|
|
+ with self.lock:
|
|
|
+ threads = [t for t in self.ask_threads if t.is_alive()]
|
|
|
+ deadline = time.monotonic() + timeout
|
|
|
+ for thread in threads:
|
|
|
+ thread.join(timeout=max(0.1, deadline - time.monotonic()))
|
|
|
+ with self.lock:
|
|
|
+ self.ask_threads = [t for t in self.ask_threads if t.is_alive()]
|
|
|
+ return all(not thread.is_alive() for thread in threads)
|
|
|
|
|
|
def maybe_trigger_ask(self, reason, throttle=False):
|
|
|
"""Schedules an idle-awake ask. Protocol events (wake, abort,
|
|
|
@@ -626,6 +749,7 @@ class FakeSaddle:
|
|
|
self._send(200, {"ok": True})
|
|
|
elif kind == "subtask-status":
|
|
|
if self.daemon.fail_subtask_status:
|
|
|
+ self.daemon.subtask_fail_hits += 1
|
|
|
self._send(500, {"error": "simulated failure"})
|
|
|
return
|
|
|
self.daemon.subtask_status_events.append(payload)
|
|
|
@@ -666,26 +790,27 @@ class FakeSaddle:
|
|
|
|
|
|
def run_lifecycle(self):
|
|
|
"""Applies the scripted lifecycle on a timer (like a real daemon
|
|
|
- whose task ends on its own): working heartbeats chunks (the 10s
|
|
|
+ whose task ends on its own): working heartbeats chunks (the
|
|
|
flush timer's stand-in, keeping the staleness clock fed), works
|
|
|
- for ~8s, then pushes done/suspended; failing pushes failed after
|
|
|
- ~1s — and a task END makes the idle-awake daemon ask again."""
|
|
|
+ for FAKE_WORK_S, then pushes done/suspended; failing pushes
|
|
|
+ failed after FAKE_FAIL_S — and a task END makes the idle-awake
|
|
|
+ daemon ask again."""
|
|
|
while True:
|
|
|
- time.sleep(0.5)
|
|
|
+ time.sleep(0.1)
|
|
|
if self.hold_state or self.silent:
|
|
|
# hold_state freezes the script; silent models a daemon
|
|
|
# mid-round with nothing to flush — its round has not
|
|
|
# ended, so no terminal push fires while quiet.
|
|
|
continue
|
|
|
if self.state == "failing":
|
|
|
- if time.monotonic() - self.state_entered >= 1:
|
|
|
+ if time.monotonic() - self.state_entered >= FAKE_FAIL_S:
|
|
|
self.set_state("failed", "bash -n rejected the script (simulated)")
|
|
|
self.push_results("failed",
|
|
|
summary="bash -n rejected the script (simulated)")
|
|
|
if self.ask_on_terminal:
|
|
|
self.maybe_trigger_ask("terminal")
|
|
|
elif self.state == "working":
|
|
|
- if time.monotonic() - self.state_entered >= 8:
|
|
|
+ if time.monotonic() - self.state_entered >= FAKE_WORK_S:
|
|
|
if self.suspend_on_title and self.suspend_on_title == self.running_title:
|
|
|
# `saddle suspend`: park until a fresh assignment
|
|
|
# resumes the workspace (title-targeted one-shot,
|
|
|
@@ -702,7 +827,7 @@ class FakeSaddle:
|
|
|
self.push_results("done", summary="fake daemon finished")
|
|
|
if self.ask_on_terminal:
|
|
|
self.maybe_trigger_ask("terminal")
|
|
|
- elif time.monotonic() - getattr(self, "last_heartbeat", 0) >= 2:
|
|
|
+ elif time.monotonic() - getattr(self, "last_heartbeat", 0) >= FAKE_HEARTBEAT_S:
|
|
|
# The flush timer: output keeps flowing, so chunks
|
|
|
# keep arriving (the staleness window never opens on
|
|
|
# a healthy runner).
|
|
|
@@ -753,9 +878,10 @@ def start_app():
|
|
|
"SPRY_SEED_ADMIN_PASSWORD": "adminpass123",
|
|
|
"SADDLE_LLM_BASE_URL": "http://127.0.0.1:%d" % DAEMON_PORT,
|
|
|
"SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
|
|
|
- "SADDLE_HTTP_TIMEOUT_S": "5",
|
|
|
+ "SADDLE_HTTP_TIMEOUT_S": "2",
|
|
|
"SADDLE_LOST_WAKE_S": "2",
|
|
|
"SADDLE_STALE_S": "3",
|
|
|
+ "SADDLE_DISPATCH_TICK_S": "1",
|
|
|
# Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
|
|
|
# resolves the ancient /lib64 copies): empty makes the loader use
|
|
|
# the binary's RUNPATH, which is the dev stack it was built
|
|
|
@@ -823,20 +949,37 @@ def dump_app_stacks(pid):
|
|
|
continue
|
|
|
|
|
|
|
|
|
-def main():
|
|
|
- if not os.path.exists(APP):
|
|
|
- raise SystemExit("app binary not found at %s (build first)" % APP)
|
|
|
+# =============================================================================
|
|
|
+# Shard orchestration
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+class Ctx:
|
|
|
+ """Per-bundle shared fixtures: the app process, the admin auth slot
|
|
|
+ and the primary fake saddle (registered + token delivered)."""
|
|
|
+
|
|
|
|
|
|
+def start_bundle(name):
|
|
|
+ """Boots one shard's environment: scratch dir, database, ports, the
|
|
|
+ primary fake daemon and the app, then the shared fixtures every
|
|
|
+ bundle builds on (the demo project + the registered fake saddle).
|
|
|
+ Scenario groups inside the bundle share this app+daemon instance
|
|
|
+ and sequence their state as DB rows, never process restarts."""
|
|
|
+ global APP_PORT, WEB, DAEMON_PORT
|
|
|
+ bundle_env(name)
|
|
|
fake = FakeSaddle()
|
|
|
fake.serve()
|
|
|
+ DAEMON_PORT = fake.port # the actually-bound port (fallback-safe)
|
|
|
|
|
|
# A rare framework-level crash (stack corruption in Statum slot
|
|
|
# signing during the FIRST login — shared-library codegen bug, moved
|
|
|
# twice already) kills the app before any test runs; a clean restart
|
|
|
- # sidesteps it. Only tolerated while nothing has been checked yet.
|
|
|
+ # (with fresh ports) sidesteps it. Only tolerated while nothing has
|
|
|
+ # been checked yet.
|
|
|
auth = None
|
|
|
app = None
|
|
|
for attempt in range(3):
|
|
|
+ APP_PORT = free_port()
|
|
|
+ WEB = "http://127.0.0.1:%d" % APP_PORT
|
|
|
if os.path.exists(WORK):
|
|
|
shutil.rmtree(WORK)
|
|
|
os.makedirs(os.path.join(WORK, "workspace"))
|
|
|
@@ -858,2673 +1001,2995 @@ def main():
|
|
|
if app is None or auth is None:
|
|
|
raise SystemExit("app never came up; see %s/app.log" % WORK)
|
|
|
|
|
|
- try:
|
|
|
- check(True, "login yields an auth slot")
|
|
|
- print("== feature 3: plan review hides duplicate body ==")
|
|
|
+ state = slot_public(auth, "/projects", "projects")
|
|
|
+ act(auth, state["create"], {
|
|
|
+ "name": "demo",
|
|
|
+ "git_url": "https://example.com/demo.git",
|
|
|
+ "default_branch": "main",
|
|
|
+ })
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["register"], {"name": "fake-saddle",
|
|
|
+ "host": "127.0.0.1:%d" % fake.port})
|
|
|
+ registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "fake-saddle"]
|
|
|
+ if wait_for(lambda: len(fake.register_log) > 0 or False,
|
|
|
+ timeout=15, what="/register delivery") is None:
|
|
|
+ raise SystemExit("fake saddle never received /register")
|
|
|
+
|
|
|
+ ctx = Ctx()
|
|
|
+ ctx.name = name
|
|
|
+ ctx.app = app
|
|
|
+ ctx.auth = auth
|
|
|
+ ctx.fake = fake
|
|
|
+ ctx.fake_id = int(registered[0]["id"])
|
|
|
+ ctx.fake_token = registered[0]["token"]
|
|
|
+ return ctx
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# Bundle: pages — block presentation, diff URLs, computed display state,
|
|
|
+# terminal rendering (rejected entries, ANSI SGR), task-page layout and
|
|
|
+# the /poll verdict channel. Fixture-driven: tasks and sessions are
|
|
|
+# inserted as rows; only the rejected-ride scenario dispatches.
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+def bundle_pages(ctx):
|
|
|
+ auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
|
|
|
+ app = ctx.app
|
|
|
+
|
|
|
+ check(True, "login yields an auth slot")
|
|
|
+ print("== feature 3: plan review hides duplicate body ==")
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
+ " is_design_task, plan, created_at)"
|
|
|
+ " values (1, 'Plan review task', 'working', 5, 1, '# The plan\n\n1. Do it',"
|
|
|
+ " strftime('%s','now'))")
|
|
|
+ plan_task = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
|
|
|
+ " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
|
|
|
+ (plan_task, fake_id))
|
|
|
+ conn.commit()
|
|
|
+ session_id = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "block", "session_id": session_id, "data": {
|
|
|
+ "question": "Plan submitted for review",
|
|
|
+ "type": "rfc",
|
|
|
+ "body": "# The plan\n\n1. Do it"}})
|
|
|
+ check(status == 200 and body.get("block_id"), "plan-review block raised")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "block", "session_id": session_id, "data": {
|
|
|
+ "question": "Which database?",
|
|
|
+ "type": "question",
|
|
|
+ "options": ["SQLite", "Postgres"],
|
|
|
+ "body": "# Storage choice"}})
|
|
|
+ check(status == 200, "question block raised")
|
|
|
+
|
|
|
+ detail = task_state(auth, plan_task)
|
|
|
+ blocks = {b["block_type"]: b for b in detail["blocks"]}
|
|
|
+ check(blocks["rfc"].get("is_plan_review") is True,
|
|
|
+ "rfc block flagged as the plan review")
|
|
|
+ check(blocks["rfc"].get("show_body") is False,
|
|
|
+ "pending plan review hides the duplicate body preview")
|
|
|
+ check(blocks["question"].get("show_body") is True,
|
|
|
+ "question block keeps its body preview")
|
|
|
+
|
|
|
+ act(auth, blocks["rfc"]["answer_action"], {"answer": "Plan approved. Proceed."})
|
|
|
+ detail = task_state(auth, plan_task)
|
|
|
+ rfc = [b for b in detail["blocks"] if b["block_type"] == "rfc"][0]
|
|
|
+ check(rfc.get("show_body") is True,
|
|
|
+ "answered plan review shows the body again (review section gone)")
|
|
|
+
|
|
|
+ print("== B1: highlight-to-comment quote dialog ==")
|
|
|
+ status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
|
|
|
+ check(status == 200 and "quote-comment-dialog" in page and "quote-text" in page,
|
|
|
+ "task page carries the quote-comment dialog the popover opens")
|
|
|
+ check('name="quote"' in page and 'name="return_to"' in page,
|
|
|
+ "quote dialog posts the quote and return-to fields")
|
|
|
+ status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
|
|
|
+ check(status == 200 and "quote-comment-dialog" in page,
|
|
|
+ "terminal page keeps its quote dialog")
|
|
|
+
|
|
|
+ print("== B3: review section hides the duplicate body ==")
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "block", "session_id": session_id, "data": {
|
|
|
+ "question": "Review the diff",
|
|
|
+ "type": "review",
|
|
|
+ "body": "# The change\n\nReworks the flange module."}})
|
|
|
+ check(status == 200 and body.get("block_id"), "review block raised")
|
|
|
+ detail = task_state(auth, plan_task)
|
|
|
+ review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
|
|
|
+ check(detail.get("review_pending") is True,
|
|
|
+ "review section shows while the review is unanswered")
|
|
|
+ check(review_block.get("show_body") is False,
|
|
|
+ "pending review block hides its duplicate body preview")
|
|
|
+ check(review_block.get("body", "").startswith("# The change"),
|
|
|
+ "review body itself still arrives in state (B2 counterpart)")
|
|
|
+ act(auth, review_block["answer_action"], {"answer": "Human approved the review"})
|
|
|
+ detail = task_state(auth, plan_task)
|
|
|
+ review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
|
|
|
+ check(review_block.get("show_body") is True,
|
|
|
+ "answered review shows the body again (banner gone)")
|
|
|
+
|
|
|
+ print("== B2: question block body above options ==")
|
|
|
+ # Contract side (A2 will restore bodies end-to-end): a question
|
|
|
+ # block raised with a heredoc body carries it; the body preview
|
|
|
+ # rides above the options (template order) and survives answering
|
|
|
+ # alongside the selected answer.
|
|
|
+ detail = task_state(auth, plan_task)
|
|
|
+ question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
|
|
|
+ check(question.get("body") == "# Storage choice"
|
|
|
+ and question.get("show_body") is True,
|
|
|
+ "question block keeps its body markdown with the preview on")
|
|
|
+ act(auth, question["answer_action"], {"option": "SQLite"})
|
|
|
+ detail = task_state(auth, plan_task)
|
|
|
+ question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
|
|
|
+ check(question.get("answered") is True and question.get("answer") == "SQLite"
|
|
|
+ and question.get("body") == "# Storage choice"
|
|
|
+ and question.get("show_body") is True,
|
|
|
+ "answered question shows body + selected answer")
|
|
|
+
|
|
|
+ print("== B5: compare-changes SSH -> HTTPS ==")
|
|
|
+ def diff_url_for_project(git_url, suffix):
|
|
|
state = slot_public(auth, "/projects", "projects")
|
|
|
- act(auth, state["create"], {
|
|
|
- "name": "demo",
|
|
|
- "git_url": "https://example.com/demo.git",
|
|
|
- "default_branch": "main",
|
|
|
- })
|
|
|
+ act(auth, state["create"], {"name": "diff-%s" % suffix, "git_url": git_url})
|
|
|
+ project_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
|
|
|
+ if p["name"] == "diff-%s" % suffix][0])
|
|
|
conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
- " is_design_task, plan, created_at)"
|
|
|
- " values (1, 'Plan review task', 'working', 5, 1, '# The plan\n\n1. Do it',"
|
|
|
- " strftime('%s','now'))")
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, task_branch,"
|
|
|
+ " branch_from, created_at) values (?, ?, 'working', 'feature/x',"
|
|
|
+ " 'main', strftime('%s','now'))", (project_id, "Diff %s" % suffix))
|
|
|
conn.commit()
|
|
|
- plan_task = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
-
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- act(auth, saddles_state["register"], {"name": "fake-saddle",
|
|
|
- "host": "127.0.0.1:%d" % DAEMON_PORT})
|
|
|
- registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "fake-saddle"]
|
|
|
- check(len(registered) == 1 and registered[0]["token"].startswith("saddle-"),
|
|
|
- "fake saddle registered with an API token")
|
|
|
- fake_id = int(registered[0]["id"])
|
|
|
- fake_token = registered[0]["token"]
|
|
|
-
|
|
|
- # Plan 10 phase 1: registration delivers the per-saddle token to
|
|
|
- # the daemon via POST /register {saddle_token, web_base_url}.
|
|
|
- check(wait_for(lambda: len(fake.register_log) > 0 or False,
|
|
|
- timeout=15, what="/register delivery") is not None,
|
|
|
- "registration POSTs /register to the daemon")
|
|
|
- register_body = fake.register_log[0]
|
|
|
- check(register_body.get("saddle_token") == fake_token,
|
|
|
- "/register carries the minted per-saddle token")
|
|
|
- check(register_body.get("web_base_url") == WEB,
|
|
|
- "/register carries the web base URL")
|
|
|
- check(fake.token == fake_token,
|
|
|
- "daemon persisted the token (its asks use it)")
|
|
|
- token_conn = db()
|
|
|
- row_token = token_conn.execute("select token from saddles where id = ?",
|
|
|
- (fake_id,)).fetchone()[0]
|
|
|
- token_conn.close()
|
|
|
- check(row_token == fake_token,
|
|
|
- "per-saddle token lives on the saddle row")
|
|
|
- # The register-time ask (idle-awake daemon): 204, nothing
|
|
|
- # offerable yet — the saddle is asleep and wakeable.
|
|
|
- check(wait_for(lambda: any(a["status"] == 204 for a in fake.asks) or False,
|
|
|
- timeout=15, what="register-time ask") is not None,
|
|
|
- "the daemon's idle ask returns 204 (nothing offerable)")
|
|
|
-
|
|
|
- conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
|
|
|
- " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
|
|
|
- (plan_task, fake_id))
|
|
|
+ task_id = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
|
|
|
+ " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
|
|
|
+ " 'done')", (task_id, fake_id))
|
|
|
conn.commit()
|
|
|
- session_id = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
+ session = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
conn.close()
|
|
|
-
|
|
|
- status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "block", "session_id": session_id, "data": {
|
|
|
- "question": "Plan submitted for review",
|
|
|
- "type": "rfc",
|
|
|
- "body": "# The plan\n\n1. Do it"}})
|
|
|
- check(status == 200 and body.get("block_id"), "plan-review block raised")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "block", "session_id": session_id, "data": {
|
|
|
- "question": "Which database?",
|
|
|
- "type": "question",
|
|
|
- "options": ["SQLite", "Postgres"],
|
|
|
- "body": "# Storage choice"}})
|
|
|
- check(status == 200, "question block raised")
|
|
|
-
|
|
|
- detail = task_state(auth, plan_task)
|
|
|
- blocks = {b["block_type"]: b for b in detail["blocks"]}
|
|
|
- check(blocks["rfc"].get("is_plan_review") is True,
|
|
|
- "rfc block flagged as the plan review")
|
|
|
- check(blocks["rfc"].get("show_body") is False,
|
|
|
- "pending plan review hides the duplicate body preview")
|
|
|
- check(blocks["question"].get("show_body") is True,
|
|
|
- "question block keeps its body preview")
|
|
|
-
|
|
|
- act(auth, blocks["rfc"]["answer_action"], {"answer": "Plan approved. Proceed."})
|
|
|
- detail = task_state(auth, plan_task)
|
|
|
- rfc = [b for b in detail["blocks"] if b["block_type"] == "rfc"][0]
|
|
|
- check(rfc.get("show_body") is True,
|
|
|
- "answered plan review shows the body again (review section gone)")
|
|
|
-
|
|
|
- print("== B1: highlight-to-comment quote dialog ==")
|
|
|
- status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
|
|
|
- check(status == 200 and "quote-comment-dialog" in page and "quote-text" in page,
|
|
|
- "task page carries the quote-comment dialog the popover opens")
|
|
|
- check('name="quote"' in page and 'name="return_to"' in page,
|
|
|
- "quote dialog posts the quote and return-to fields")
|
|
|
- status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
|
|
|
- check(status == 200 and "quote-comment-dialog" in page,
|
|
|
- "terminal page keeps its quote dialog")
|
|
|
-
|
|
|
- print("== B3: review section hides the duplicate body ==")
|
|
|
- status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "block", "session_id": session_id, "data": {
|
|
|
- "question": "Review the diff",
|
|
|
- "type": "review",
|
|
|
- "body": "# The change\n\nReworks the flange module."}})
|
|
|
- check(status == 200 and body.get("block_id"), "review block raised")
|
|
|
- detail = task_state(auth, plan_task)
|
|
|
- review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
|
|
|
- check(detail.get("review_pending") is True,
|
|
|
- "review section shows while the review is unanswered")
|
|
|
- check(review_block.get("show_body") is False,
|
|
|
- "pending review block hides its duplicate body preview")
|
|
|
- check(review_block.get("body", "").startswith("# The change"),
|
|
|
- "review body itself still arrives in state (B2 counterpart)")
|
|
|
- act(auth, review_block["answer_action"], {"answer": "Human approved the review"})
|
|
|
- detail = task_state(auth, plan_task)
|
|
|
- review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
|
|
|
- check(review_block.get("show_body") is True,
|
|
|
- "answered review shows the body again (banner gone)")
|
|
|
-
|
|
|
- print("== B2: question block body above options ==")
|
|
|
- # Contract side (A2 will restore bodies end-to-end): a question
|
|
|
- # block raised with a heredoc body carries it; the body preview
|
|
|
- # rides above the options (template order) and survives answering
|
|
|
- # alongside the selected answer.
|
|
|
- detail = task_state(auth, plan_task)
|
|
|
- question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
|
|
|
- check(question.get("body") == "# Storage choice"
|
|
|
- and question.get("show_body") is True,
|
|
|
- "question block keeps its body markdown with the preview on")
|
|
|
- act(auth, question["answer_action"], {"option": "SQLite"})
|
|
|
- detail = task_state(auth, plan_task)
|
|
|
- question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
|
|
|
- check(question.get("answered") is True and question.get("answer") == "SQLite"
|
|
|
- and question.get("body") == "# Storage choice"
|
|
|
- and question.get("show_body") is True,
|
|
|
- "answered question shows body + selected answer")
|
|
|
-
|
|
|
- print("== B5: compare-changes SSH -> HTTPS ==")
|
|
|
- def diff_url_for_project(git_url, suffix):
|
|
|
- state = slot_public(auth, "/projects", "projects")
|
|
|
- act(auth, state["create"], {"name": "diff-%s" % suffix, "git_url": git_url})
|
|
|
- project_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
|
|
|
- if p["name"] == "diff-%s" % suffix][0])
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, task_branch,"
|
|
|
- " branch_from, created_at) values (?, ?, 'working', 'feature/x',"
|
|
|
- " 'main', strftime('%s','now'))", (project_id, "Diff %s" % suffix))
|
|
|
- conn.commit()
|
|
|
- task_id = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
- conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
|
|
|
- " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
|
|
|
- " 'done')", (task_id, fake_id))
|
|
|
- conn.commit()
|
|
|
- session = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "block", "session_id": session, "data": {
|
|
|
+ "question": "Review the diff", "type": "review"}})
|
|
|
+ return task_state(auth, task_id).get("review_diff_url") or ""
|
|
|
+
|
|
|
+ check(diff_url_for_project("ssh://fabrica.unitatem.net:222/clanker/clanker-saddle.git",
|
|
|
+ "fabrica")
|
|
|
+ == "https://fabrica.unitatem.net/clanker/clanker-saddle/compare/main...feature/x",
|
|
|
+ "ssh://git@host:port/owner/repo.git translates (port dropped)")
|
|
|
+ check(diff_url_for_project("git@github.com:acme/widgets.git", "scp")
|
|
|
+ == "https://github.com/acme/widgets/compare/main...feature/x",
|
|
|
+ "git@host:owner/repo.git translates to the browse host")
|
|
|
+ check(diff_url_for_project("https://example.com/plain/repo", "plain")
|
|
|
+ == "https://example.com/plain/repo/compare/main...feature/x",
|
|
|
+ "HTTPS URLs pass through unchanged")
|
|
|
+
|
|
|
+ print("== B6: computed display state ==")
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
+ " values (1, 'Parent', 'done', 5, strftime('%s','now'))")
|
|
|
+ parent6 = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ # The post-mortem case: stored done WITH an unanswered review block.
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
|
|
|
+ " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
|
|
|
+ (parent6, fake_id))
|
|
|
+ session6 = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "block", "session_id": session6, "data": {
|
|
|
+ "question": "Review the final diff", "type": "review"}})
|
|
|
+
|
|
|
+ detail = task_state(auth, parent6)
|
|
|
+ check(detail.get("display") == "review requested" and detail.get("terminal") is True,
|
|
|
+ "unanswered review overrides the stored done status (terminal flag keeps stored truth)")
|
|
|
+ queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if int(t["id"]) == parent6][0]
|
|
|
+ check(queue["display"] == "review requested",
|
|
|
+ "queue row shows the computed review-requested display")
|
|
|
+
|
|
|
+ # Delegation: with the review answered, open subtasks delegate.
|
|
|
+ conn = db()
|
|
|
+ conn.execute("update task_blocks set answered_at = strftime('%s','now'),"
|
|
|
+ " answer = 'Human approved the review' where task_id = ?", (parent6,))
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
+ " created_at) values (1, 'Sub A', 'working', 5, ?, strftime('%s','now'))",
|
|
|
+ (parent6,))
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
+ " created_at) values (1, 'Sub B', 'done', 5, ?, strftime('%s','now'))",
|
|
|
+ (parent6,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ detail = task_state(auth, parent6)
|
|
|
+ check(detail.get("display") == "delegating 1/2" and detail.get("subtask_open") == 1,
|
|
|
+ "open subtasks show the delegating breakdown (%s)" % detail.get("display"))
|
|
|
+
|
|
|
+ # Flags track the stored status once the display falls through
|
|
|
+ # (subtasks terminal first — delegating leads while any are open).
|
|
|
+ conn = db()
|
|
|
+ conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
|
|
|
+ " where id = ?", (parent6,))
|
|
|
+ conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
|
|
|
+ " where title = 'Sub A'")
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ detail = task_state(auth, parent6)
|
|
|
+ check(detail.get("display") == "failed" and detail.get("failed") is True
|
|
|
+ and detail.get("terminal") is True,
|
|
|
+ "failed task with answered review and terminal subtasks shows failed + terminal flags")
|
|
|
+
|
|
|
+ print("== rejected entries (non-fatal) ==")
|
|
|
+ # Ingest API: a rejected entry stores on the session and streams
|
|
|
+ # through SSE; the task is untouched.
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "output", "session_id": session_id, "data": {
|
|
|
+ "command": "bash -n script.sh",
|
|
|
+ "output": "syntax error near line 3 (recoverable)",
|
|
|
+ "exit_code": 1,
|
|
|
+ "type": "rejected"}})
|
|
|
+ check(status == 200, "rejected entry accepted by the ingest API")
|
|
|
+ request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
|
|
|
+ with urllib.request.urlopen(request, timeout=15) as stream:
|
|
|
+ rejected_event = None
|
|
|
+ deadline = time.monotonic() + 8
|
|
|
+ while time.monotonic() < deadline and rejected_event is None:
|
|
|
+ line = stream.readline().decode().strip()
|
|
|
+ if not line.startswith("data: ") or line == "data: ping":
|
|
|
+ continue
|
|
|
+ event = json.loads(line[6:])
|
|
|
+ if event.get("type") == "rejected":
|
|
|
+ rejected_event = event
|
|
|
+ check(rejected_event is not None
|
|
|
+ and "syntax error near line 3" in (rejected_event.get("output") or ""),
|
|
|
+ "terminal stream carries rejected entries with their reason")
|
|
|
+
|
|
|
+ # The page template renders them with the distinct non-fatal style.
|
|
|
+ status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
|
|
|
+ check(status == 200 and "term-rejected" in page,
|
|
|
+ "terminal page renders rejected entries in their own style")
|
|
|
+
|
|
|
+ print("== B8: ANSI SGR in the terminal view ==")
|
|
|
+ # Byte-exact SGR delivery: the daemon's coloured chunks flow through
|
|
|
+ # the store + SSE untouched (the client renderer styles them).
|
|
|
+ coloured = "\x1b[1;32mbold green\x1b[0m then \x1b[38;5;196m256-red\x1b[0m\n"
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into session_raw_log (session_id, seq, chunk)"
|
|
|
+ " values (?, 900, ?)", (session_id, coloured))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
|
|
|
+ with urllib.request.urlopen(request, timeout=15) as stream:
|
|
|
+ coloured_event = None
|
|
|
+ deadline = time.monotonic() + 8
|
|
|
+ while time.monotonic() < deadline and coloured_event is None:
|
|
|
+ line = stream.readline().decode().strip()
|
|
|
+ if not line.startswith("data: ") or line == "data: ping":
|
|
|
+ continue
|
|
|
+ event = json.loads(line[6:])
|
|
|
+ if event.get("type") == "raw" and "\x1b[1;32m" in (event.get("text") or ""):
|
|
|
+ coloured_event = event
|
|
|
+ check(coloured_event is not None
|
|
|
+ and "\x1b[38;5;196m" in coloured_event["text"],
|
|
|
+ "SGR sequences stream byte-exact to the client renderer")
|
|
|
+ status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
|
|
|
+ check(status == 200 and "makeAnsiRenderer" in page and "colour256" in page,
|
|
|
+ "terminal page embeds the ANSI renderer (SGR + 256-colour palette)")
|
|
|
+
|
|
|
+ # Dispatch path: a rejected entry in a round's results push flows
|
|
|
+ # into the audit trail without failing (or finishing) the task.
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Rejected ride",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ rejected_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Rejected ride"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, rejected_task).get("status") == "working"
|
|
|
+ or False, timeout=60, what="rejected-ride working") is not None,
|
|
|
+ "rejected-ride task reaches working")
|
|
|
+ fake.queue_output("rejected", "echo 'too long' | saddle",
|
|
|
+ "max_tokens exceeded (recoverable)")
|
|
|
+ check(wait_task_done(auth, rejected_task, timeout=60) is not None,
|
|
|
+ "task completes normally past rejected entries")
|
|
|
+ conn = db()
|
|
|
+ stored = conn.execute(
|
|
|
+ "select count(*) from session_commands sc join sessions s on s.id = sc.session_id"
|
|
|
+ " where s.task_id = ? and sc.type = 'rejected'", (rejected_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(stored >= 1,
|
|
|
+ "rejected entries stored in the audit trail (%d rows)" % stored)
|
|
|
+ check(task_state(auth, rejected_task)["status"] == "done"
|
|
|
+ and "fake daemon finished" in (task_state(auth, rejected_task).get("summary") or ""),
|
|
|
+ "rejected entries never fail the task")
|
|
|
+
|
|
|
+ print("== phase 3: task page — status on top, blocks newest-first ==")
|
|
|
+ detail = task_state(auth, plan_task)
|
|
|
+ block_ids = [int(b["id"]) for b in detail.get("blocks", [])]
|
|
|
+ check(len(block_ids) >= 3 and block_ids == sorted(block_ids, reverse=True),
|
|
|
+ "the blocks section orders newest-first via the sealed GET state (%s)"
|
|
|
+ % block_ids)
|
|
|
+ status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
|
|
|
+ check(status == 200 and "status-head" in page
|
|
|
+ and page.index("status-head") < page.index("<h3>Description</h3>")
|
|
|
+ and page.index("status-head") < page.index("<h3>Plan</h3>"),
|
|
|
+ "the status block sits at the very top of the task page")
|
|
|
+ # The agent's progress feed renders DIRECTLY below the status
|
|
|
+ # head (above the description cards), newest line first.
|
|
|
+ check(page.index("status-head") < page.index("<h3>Progress</h3>")
|
|
|
+ < page.index("<h3>Description</h3>"),
|
|
|
+ "the progress feed sits directly below the status head")
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
+ " created_at) values (1, 'Progress layout task', 'working', 5,"
|
|
|
+ " strftime('%s','now','-120 seconds'))")
|
|
|
+ layout_task = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ # backdated so the dashboard-timeline fixtures below stay the
|
|
|
+ # newest events
|
|
|
+ conn.execute("insert into task_progress (task_id, summary, created_at)"
|
|
|
+ " values (?, 'first update', strftime('%s','now','-90 seconds'))",
|
|
|
+ (layout_task,))
|
|
|
+ conn.execute("insert into task_progress (task_id, summary, created_at)"
|
|
|
+ " values (?, 'second update', strftime('%s','now','-89 seconds'))",
|
|
|
+ (layout_task,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ status, history_page = http("GET", "/tasks/%d" % layout_task, timeout=15)
|
|
|
+ check(status == 200
|
|
|
+ and history_page.index("progress-latest") < history_page.index("progress-history"),
|
|
|
+ "the progress timeline renders newest-first (latest above the history)")
|
|
|
+
|
|
|
+ print("== R10: branch_advanced on the /poll verdict channel ==")
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
+ " created_at) values (1, 'Poll grandparent', 'working', 5,"
|
|
|
+ " strftime('%s','now','-300 seconds'))")
|
|
|
+ poll_grand = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
+ " parent_task_id, created_at) values (1, 'Poll answer parent',"
|
|
|
+ " 'blocked', 5, ?, strftime('%s','now','-120 seconds'))",
|
|
|
+ (poll_grand,))
|
|
|
+ poll_parent = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at)"
|
|
|
+ " values (?, ?, strftime('%s','now','-120 seconds'))",
|
|
|
+ (poll_parent, fake_id))
|
|
|
+ conn.execute("insert into task_blocks (task_id, question, type, answer,"
|
|
|
+ " answered_at, created_at) values (?, 'Which DB?', 'question',"
|
|
|
+ " 'SQLite', strftime('%s','now','-60 seconds'),"
|
|
|
+ " strftime('%s','now','-100 seconds'))", (poll_parent,))
|
|
|
+ conn.execute("update saddles set status = 'busy', current_task_id = ?,"
|
|
|
+ " allocated_at = strftime('%s','now') where name = 'fake-saddle'",
|
|
|
+ (poll_parent,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/%d/poll" % fake_id, fake_token)
|
|
|
+ check(status == 200 and len(body.get("answers", [])) == 1
|
|
|
+ and body["answers"][0].get("branch_advanced") is False,
|
|
|
+ "/poll verdicts carry branch_advanced=false with no siblings")
|
|
|
+
|
|
|
+ # A sibling under the same parent completed after the park marker.
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
+ " parent_task_id, summary, completed_at, created_at)"
|
|
|
+ " values (1, 'Poll sibling', 'done', 5, ?, 'sibling finished',"
|
|
|
+ " strftime('%s','now'), strftime('%s','now','-200 seconds'))",
|
|
|
+ (poll_grand,))
|
|
|
+ conn.execute("insert into task_blocks (task_id, question, type, answer,"
|
|
|
+ " answered_at, created_at) values (?, 'Second question?',"
|
|
|
+ " 'question', 'Yes', strftime('%s','now'),"
|
|
|
+ " strftime('%s','now','-90 seconds'))", (poll_parent,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/%d/poll" % fake_id, fake_token)
|
|
|
+ check(status == 200 and len(body.get("answers", [])) == 1
|
|
|
+ and body["answers"][0].get("branch_advanced") is True,
|
|
|
+ "sibling completion after park flips branch_advanced to true")
|
|
|
+
|
|
|
+ # Release the staged fixture: cancel it so its busy@task row does
|
|
|
+ # not linger into the sections below.
|
|
|
+ act(auth, task_state(auth, poll_parent)["cancel"], {})
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# Bundle: dispatch — the task pipeline end to end: registration tokens,
|
|
|
+# rules, project notes, pushed chunks/results ingestion, capability
|
|
|
+# routing, the create form, and retry/manual resume.
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+def bundle_dispatch(ctx):
|
|
|
+ auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
|
|
|
+ app = ctx.app
|
|
|
+
|
|
|
+ print("== registration: token minting and delivery ==")
|
|
|
+ registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "fake-saddle"]
|
|
|
+ check(len(registered) == 1 and registered[0]["token"].startswith("saddle-"),
|
|
|
+ "fake saddle registered with an API token")
|
|
|
+
|
|
|
+ # Plan 10 phase 1: registration delivers the per-saddle token to
|
|
|
+ # the daemon via POST /register {saddle_token, web_base_url}.
|
|
|
+ check(wait_for(lambda: len(fake.register_log) > 0 or False,
|
|
|
+ timeout=15, what="/register delivery") is not None,
|
|
|
+ "registration POSTs /register to the daemon")
|
|
|
+ register_body = fake.register_log[0]
|
|
|
+ check(register_body.get("saddle_token") == fake_token,
|
|
|
+ "/register carries the minted per-saddle token")
|
|
|
+ check(register_body.get("web_base_url") == WEB,
|
|
|
+ "/register carries the web base URL")
|
|
|
+ check(fake.token == fake_token,
|
|
|
+ "daemon persisted the token (its asks use it)")
|
|
|
+ token_conn = db()
|
|
|
+ row_token = token_conn.execute("select token from saddles where id = ?",
|
|
|
+ (fake_id,)).fetchone()[0]
|
|
|
+ token_conn.close()
|
|
|
+ check(row_token == fake_token,
|
|
|
+ "per-saddle token lives on the saddle row")
|
|
|
+ # The register-time ask (idle-awake daemon): 204, nothing
|
|
|
+ # offerable yet — the saddle is asleep and wakeable.
|
|
|
+ check(wait_for(lambda: any(a["status"] == 204 for a in fake.asks) or False,
|
|
|
+ timeout=15, what="register-time ask") is not None,
|
|
|
+ "the daemon's idle ask returns 204 (nothing offerable)")
|
|
|
+
|
|
|
+ print("== feature 5: rules ==")
|
|
|
+ rules_state = slot_public(auth, "/rules", "rules")
|
|
|
+ for name, body in [
|
|
|
+ ("Use Invercargill", "Never use libgee; use Invercargill enumerables."),
|
|
|
+ ("Write Vala docs", "Every public class carries a doc comment."),
|
|
|
+ ("Prefer meson", "Builds go through meson; no makefiles."),
|
|
|
+ ("Co-dev rule", "Keep the wire contract exactly."),
|
|
|
+ ("Reference rule", "Must never ride along."),
|
|
|
+ ]:
|
|
|
+ act(auth, rules_state["create"], {"name": name, "body": body})
|
|
|
+ listed = {r["name"]: r for r in slot_public(auth, "/rules", "rules")["rules"]}
|
|
|
+ check(len(listed) == 5, "five rules created and listed on /rules")
|
|
|
+
|
|
|
+ def rule_id(name):
|
|
|
+ rows = [r for r in slot_public(auth, "/rules", "rules")["rules"] if r["name"] == name]
|
|
|
+ return int(rows[0]["id"])
|
|
|
+
|
|
|
+ state = slot_public(auth, "/projects", "projects")
|
|
|
+ act(auth, state["create"], {"name": "co", "git_url": "https://example.com/co.git"})
|
|
|
+ act(auth, state["create"], {"name": "ref", "git_url": "https://example.com/ref.git"})
|
|
|
+ co_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
|
|
|
+ if p["name"] == "co"][0])
|
|
|
+ ref_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
|
|
|
+ if p["name"] == "ref"][0])
|
|
|
+
|
|
|
+ project = slot_public(auth, "/projects/1", "project")
|
|
|
+ act(auth, project["add_link"], {"linked_project_id": str(co_id),
|
|
|
+ "relationship": "co-developed"})
|
|
|
+ act(auth, project["add_link"], {"linked_project_id": str(ref_id),
|
|
|
+ "relationship": "reference"})
|
|
|
+ act(auth, project["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
|
|
|
+ act(auth, project["attach_rule"], {"rule_id": str(rule_id("Prefer meson"))})
|
|
|
+ project = slot_public(auth, "/projects/1", "project")
|
|
|
+ check(sorted(r["name"] for r in project["rules"]) == ["Prefer meson", "Use Invercargill"],
|
|
|
+ "project page lists its attached rules")
|
|
|
+
|
|
|
+ co_project = slot_public(auth, "/projects/%d" % co_id, "project")
|
|
|
+ act(auth, co_project["attach_rule"], {"rule_id": str(rule_id("Co-dev rule"))})
|
|
|
+ ref_project = slot_public(auth, "/projects/%d" % ref_id, "project")
|
|
|
+ act(auth, ref_project["attach_rule"], {"rule_id": str(rule_id("Reference rule"))})
|
|
|
+
|
|
|
+ tasks_state = slot_public(auth, "/tasks", "tasks")
|
|
|
+ act(auth, tasks_state["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Rules dispatch",
|
|
|
+ "description": "Effective rule set rides the /start payload.",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ rule_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Rules dispatch"][0])
|
|
|
+ detail = task_state(auth, rule_task)
|
|
|
+ act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Write Vala docs"))})
|
|
|
+ act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
|
|
|
+ detail = task_state(auth, rule_task)
|
|
|
+ check(sorted(r["name"] for r in detail["rules"]) == ["Use Invercargill", "Write Vala docs"],
|
|
|
+ "task page lists its attached rules")
|
|
|
+
|
|
|
+ check(wait_for(lambda: len(fake.start_payloads) > 0 or False,
|
|
|
+ timeout=30, what="/start push") is not None,
|
|
|
+ "dispatcher pushes the task to the fake saddle")
|
|
|
+ payload = fake.start_payloads[-1]
|
|
|
+ rule_names = [r["name"] for r in payload.get("rules", [])]
|
|
|
+ check(rule_names == ["Write Vala docs", "Use Invercargill", "Prefer meson", "Co-dev rule"],
|
|
|
+ "effective rules in /start: task, primary, co-developed order (%s)" % rule_names)
|
|
|
+ check(all(r.get("body") for r in payload.get("rules", [])),
|
|
|
+ "rules carry their bodies")
|
|
|
+ check("Reference rule" not in rule_names,
|
|
|
+ "reference project rules excluded")
|
|
|
+
|
|
|
+ detail = task_state(auth, rule_task)
|
|
|
+ act(auth, detail["rules"][0]["detach"], {})
|
|
|
+ check([r["name"] for r in task_state(auth, rule_task)["rules"]] == ["Use Invercargill"],
|
|
|
+ "detached task rule disappears")
|
|
|
+
|
|
|
+ rules_rows = slot_public(auth, "/rules", "rules")["rules"]
|
|
|
+ docs_row = [r for r in rules_rows if r["name"] == "Write Vala docs"][0]
|
|
|
+ act(auth, docs_row["update"], {"name": "Write Vala docs", "body": "Doc comments or bust."})
|
|
|
+ check(any(r["body"] == "Doc comments or bust."
|
|
|
+ for r in slot_public(auth, "/rules", "rules")["rules"]),
|
|
|
+ "rule edit saves the body")
|
|
|
+
|
|
|
+ check(wait_task_done(auth, rule_task, timeout=60) is not None,
|
|
|
+ "dispatched task completes against the fake daemon")
|
|
|
+
|
|
|
+ print("== feature 6: project notes + callback ==")
|
|
|
+ # Plan 10 phase 1: the shared saddle-callback token is gone —
|
|
|
+ # the per-saddle token authorises the callback endpoints.
|
|
|
+ callback_token = fake_token
|
|
|
+ check(callback_token.startswith("saddle-") and "saddle-callback-" not in callback_token,
|
|
|
+ "note callback authorised by the per-saddle token")
|
|
|
+
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/note", None,
|
|
|
+ {"project": "demo", "body": "x"})
|
|
|
+ check(status == 401, "note endpoint rejects a missing token")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/note", "saddle-callback-not-real",
|
|
|
+ {"project": "demo", "body": "x"})
|
|
|
+ check(status == 401, "note endpoint rejects a wrong (legacy-style) token")
|
|
|
+
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
+ {"task_id": rule_task, "project": "demo",
|
|
|
+ "body": "style.css tokens live in :root"})
|
|
|
+ check(status == 200 and body.get("ok") is True, "note accepted with the callback token")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
+ {"project": "no-such-project", "body": "x"})
|
|
|
+ check(status == 400, "unknown project rejected")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
+ {"project": "demo", "body": " "})
|
|
|
+ check(status == 400, "empty note body rejected")
|
|
|
+
|
|
|
+ project = slot_public(auth, "/projects/1", "project")
|
|
|
+ check(any("style.css tokens" in n["body"] and int(n["source_task_id"]) == rule_task
|
|
|
+ for n in project["notes"]),
|
|
|
+ "project page lists the note with its source task")
|
|
|
+ act(auth, [n for n in project["notes"] if "style.css tokens" in n["body"]][0]["delete"], {})
|
|
|
+ check(all("style.css tokens" not in n["body"]
|
|
|
+ for n in slot_public(auth, "/projects/1", "project")["notes"]),
|
|
|
+ "note deleted from the project page")
|
|
|
+
|
|
|
+ # /start payload: notes of primary + co-developed (not reference),
|
|
|
+ # oldest first; callback carries the configured base URL + token.
|
|
|
+ saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
+ {"project": "demo", "body": "note-one"})
|
|
|
+ saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
+ {"project": "demo", "body": "note-two"})
|
|
|
+ saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
+ {"project": "co", "body": "co-note"})
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into project_notes (project_id, body, created_at)"
|
|
|
+ " values (?, 'ref-note-must-not-ship', strftime('%s','now'))", (ref_id,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Notes dispatch",
|
|
|
+ "description": "Project notes ride the /start payload.",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ check(wait_for(lambda: len(fake.start_payloads) >= 2 or False,
|
|
|
+ timeout=30, what="second /start push") is not None,
|
|
|
+ "dispatcher pushes the notes task")
|
|
|
+ payload = fake.start_payloads[-1]
|
|
|
+ notes = payload.get("project_notes", [])
|
|
|
+ check([n["body"] for n in notes] == ["note-one", "note-two", "co-note"],
|
|
|
+ "project_notes: primary + co-developed, oldest first (%s)" % [n["body"] for n in notes])
|
|
|
+ check(all(n.get("project") for n in notes)
|
|
|
+ and notes[2]["project"] == "co",
|
|
|
+ "project_notes carry the project name")
|
|
|
+ check("ref-note-must-not-ship" not in [n["body"] for n in notes],
|
|
|
+ "reference project notes excluded")
|
|
|
+ callback = payload.get("callback", {})
|
|
|
+ check(callback.get("base_url") == WEB and callback.get("token") == callback_token,
|
|
|
+ "callback carries the configured base URL and the minted token")
|
|
|
+
|
|
|
+ # entry cap: 30 of 33 candidates
|
|
|
+ conn = db()
|
|
|
+ for i in range(33):
|
|
|
+ conn.execute("insert into project_notes (project_id, body, created_at)"
|
|
|
+ " values (?, ?, strftime('%s','now'))", (1, "bulk-%02d" % i))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Entry cap dispatch",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ check(wait_for(lambda: len(fake.start_payloads) >= 3 or False,
|
|
|
+ timeout=30, what="third /start push") is not None,
|
|
|
+ "dispatcher pushes the entry-cap task")
|
|
|
+ notes = fake.start_payloads[-1].get("project_notes", [])
|
|
|
+ check(len(notes) == 30 and notes[0]["body"] == "note-one" and notes[1]["body"] == "note-two",
|
|
|
+ "project_notes capped at 30 entries, oldest kept (%d)" % len(notes))
|
|
|
+
|
|
|
+ # char cap: the giant note is truncated so the total stays <= 20000
|
|
|
+ conn = db()
|
|
|
+ conn.execute("delete from project_notes where body like 'bulk-%' or body = 'co-note'")
|
|
|
+ conn.execute("insert into project_notes (project_id, body, created_at)"
|
|
|
+ " values (?, ?, strftime('%s','now'))", (1, "giant!" + "x" * 25000,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Char cap dispatch",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ check(wait_for(lambda: len(fake.start_payloads) >= 4 or False,
|
|
|
+ timeout=30, what="fourth /start push") is not None,
|
|
|
+ "dispatcher pushes the char-cap task")
|
|
|
+ notes = fake.start_payloads[-1].get("project_notes", [])
|
|
|
+ total = sum(len(n["body"]) for n in notes)
|
|
|
+ check(len(notes) == 3 and total == 20000 and notes[2]["body"].startswith("giant!"),
|
|
|
+ "project_notes char cap: head kept, giant truncated (entries=%d, chars=%d)"
|
|
|
+ % (len(notes), total))
|
|
|
+
|
|
|
+ print("== phase 2: pushed chunks -> raw log + terminal view ==")
|
|
|
+ # Wait for the last dispatch to finish so its pushes landed.
|
|
|
+ check(wait_task_done(auth, task_id_of("Char cap dispatch"), timeout=60) is not None,
|
|
|
+ "char-cap task completes")
|
|
|
+ conn = db()
|
|
|
+ raw_session = conn.execute(
|
|
|
+ "select s.id from sessions s join tasks t on t.id = s.task_id"
|
|
|
+ " where t.title = 'Char cap dispatch' order by s.id desc limit 1").fetchone()[0]
|
|
|
+ rows = conn.execute("select seq, chunk from session_raw_log where session_id = ?"
|
|
|
+ " order by seq", (raw_session,)).fetchall()
|
|
|
+ conn.close()
|
|
|
+ check(len(rows) >= 3, "session_raw_log captured the pushed chunks (%d)" % len(rows))
|
|
|
+ seqs = [r[0] for r in rows]
|
|
|
+ check(seqs == sorted(seqs) and len(set(seqs)) == len(seqs),
|
|
|
+ "pushed chunks stored once, seq strictly increasing")
|
|
|
+ text = "".join(r[1] for r in rows)
|
|
|
+ check(fake.PROMPT in text and "echo hello" in text and "hello" in text,
|
|
|
+ "pushed raw log keeps the prompt and echoed output byte-exact")
|
|
|
+ check("working-heartbeat" in text,
|
|
|
+ "the flush-timer heartbeat chunks keep arriving mid-round")
|
|
|
+
|
|
|
+ terminal = slot_public(auth, "/tasks/%d/terminal" % task_id_of("Char cap dispatch"),
|
|
|
+ "terminal")
|
|
|
+ check(terminal.get("has_raw_log") is True and int(terminal["session_id"]) == raw_session,
|
|
|
+ "terminal view selects raw rendering for the raw session")
|
|
|
+
|
|
|
+ # Old sessions (typed entries, no raw rows) keep the fallback view
|
|
|
+ # (a dedicated fixture task: its only session is the legacy one,
|
|
|
+ # so the terminal view cannot prefer a dispatched raw session).
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
+ " values (1, 'Legacy terminal task', 'working', 5, strftime('%s','now'))")
|
|
|
+ legacy_task = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
|
|
|
+ " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
|
|
|
+ (legacy_task, fake_id))
|
|
|
+ conn.commit()
|
|
|
+ legacy_session = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
+ conn.execute("insert into session_commands (session_id, command_text, output_text,"
|
|
|
+ " type, started_at) values (?, 'ls', 'old-output', 'command',"
|
|
|
+ " strftime('%s','now'))", (legacy_session,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ legacy = slot_public(auth, "/tasks/%d/terminal" % legacy_task, "terminal")
|
|
|
+ check(legacy.get("has_raw_log") is False,
|
|
|
+ "legacy session falls back to the entry-type terminal view")
|
|
|
+
|
|
|
+ request = urllib.request.Request(WEB + "/terminal-stream/%d" % raw_session)
|
|
|
+ with urllib.request.urlopen(request, timeout=15) as stream:
|
|
|
+ raw_events = []
|
|
|
+ deadline = time.monotonic() + 8
|
|
|
+ while time.monotonic() < deadline and len(raw_events) < 3:
|
|
|
+ line = stream.readline().decode().strip()
|
|
|
+ if not line.startswith("data: ") or line == "data: ping":
|
|
|
+ continue
|
|
|
+ event = json.loads(line[6:])
|
|
|
+ if event.get("type") == "raw":
|
|
|
+ raw_events.append(event)
|
|
|
+ check(len(raw_events) >= 3 and raw_events[0]["text"] == fake.PROMPT
|
|
|
+ and raw_events[0]["seq"] < raw_events[1]["seq"],
|
|
|
+ "terminal stream serves pushed chunks in seq order (prompt first)")
|
|
|
+
|
|
|
+ print("== phase 2: chunks/results ingestion — auth, idempotency ==")
|
|
|
+ # Token auth on both push channels (plan 10: the per-saddle
|
|
|
+ # bearer authorises every daemon->web call).
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/chunks", None,
|
|
|
+ {"session_id": raw_session, "chunks": []})
|
|
|
+ check(status == 401, "chunks endpoint rejects a missing token")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/results", "saddle-bogus-token",
|
|
|
+ {"session_id": raw_session, "state": "working"})
|
|
|
+ check(status == 401, "results endpoint rejects a wrong token")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/chunks", fake_token,
|
|
|
+ {"session_id": 99999, "chunks": []})
|
|
|
+ check(status == 400, "chunks for an unknown session rejected")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
+ {"session_id": raw_session, "state": "done"})
|
|
|
+ check(status == 200, "the owning saddle's session accepts its results")
|
|
|
+
|
|
|
+ # Pushed chunk idempotency: re-pushing a batch (the outbox's
|
|
|
+ # at-least-once redelivery after a lost response) consumes the
|
|
|
+ # duplicates and stores only the tail.
|
|
|
+ conn = db()
|
|
|
+ before = conn.execute("select count(*) from session_raw_log where session_id = ?",
|
|
|
+ (raw_session,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ redelivery = [{"seq": r[0], "text": r[1]} for r in rows[:3]]
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
|
|
|
+ {"session_id": raw_session, "chunks": redelivery})
|
|
|
+ check(status == 200 and body.get("ok") is True and int(body.get("stored") or 0) == 0,
|
|
|
+ "a fully-duplicate batch stores nothing (duplicates consumed)")
|
|
|
+ fresh = [{"seq": 9000, "text": "late-flush\r\n"},
|
|
|
+ {"seq": 9001, "text": ""}] # truly empty text: no row
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
|
|
|
+ {"session_id": raw_session, "chunks": fresh})
|
|
|
+ check(status == 200 and int(body.get("stored") or 0) == 1,
|
|
|
+ "a partial-redelivery batch stores only the new chunks")
|
|
|
+ conn = db()
|
|
|
+ after = conn.execute("select count(*) from session_raw_log where session_id = ?",
|
|
|
+ (raw_session,)).fetchone()[0]
|
|
|
+ empty_row = conn.execute(
|
|
|
+ "select count(*) from session_raw_log where session_id = ? and seq = 9001",
|
|
|
+ (raw_session,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(after == before + 1 and empty_row == 0,
|
|
|
+ "redelivered chunks never duplicate rows; empty text stores nothing")
|
|
|
+
|
|
|
+ # Pushed entries: today's OutputEntry shape with the daemon's
|
|
|
+ # global entry ids — a redelivered results push stores its
|
|
|
+ # entries exactly once.
|
|
|
+ def command_rows(session):
|
|
|
+ conn = db()
|
|
|
+ try:
|
|
|
+ return conn.execute(
|
|
|
+ "select command_text, type from session_commands"
|
|
|
+ " where session_id = ? order by id", (session,)).fetchall()
|
|
|
+ finally:
|
|
|
conn.close()
|
|
|
- saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "block", "session_id": session, "data": {
|
|
|
- "question": "Review the diff", "type": "review"}})
|
|
|
- return task_state(auth, task_id).get("review_diff_url") or ""
|
|
|
-
|
|
|
- check(diff_url_for_project("ssh://fabrica.unitatem.net:222/clanker/clanker-saddle.git",
|
|
|
- "fabrica")
|
|
|
- == "https://fabrica.unitatem.net/clanker/clanker-saddle/compare/main...feature/x",
|
|
|
- "ssh://git@host:port/owner/repo.git translates (port dropped)")
|
|
|
- check(diff_url_for_project("git@github.com:acme/widgets.git", "scp")
|
|
|
- == "https://github.com/acme/widgets/compare/main...feature/x",
|
|
|
- "git@host:owner/repo.git translates to the browse host")
|
|
|
- check(diff_url_for_project("https://example.com/plain/repo", "plain")
|
|
|
- == "https://example.com/plain/repo/compare/main...feature/x",
|
|
|
- "HTTPS URLs pass through unchanged")
|
|
|
-
|
|
|
- print("== feature 5: rules ==")
|
|
|
- rules_state = slot_public(auth, "/rules", "rules")
|
|
|
- for name, body in [
|
|
|
- ("Use Invercargill", "Never use libgee; use Invercargill enumerables."),
|
|
|
- ("Write Vala docs", "Every public class carries a doc comment."),
|
|
|
- ("Prefer meson", "Builds go through meson; no makefiles."),
|
|
|
- ("Co-dev rule", "Keep the wire contract exactly."),
|
|
|
- ("Reference rule", "Must never ride along."),
|
|
|
- ]:
|
|
|
- act(auth, rules_state["create"], {"name": name, "body": body})
|
|
|
- listed = {r["name"]: r for r in slot_public(auth, "/rules", "rules")["rules"]}
|
|
|
- check(len(listed) == 5, "five rules created and listed on /rules")
|
|
|
-
|
|
|
- def rule_id(name):
|
|
|
- rows = [r for r in slot_public(auth, "/rules", "rules")["rules"] if r["name"] == name]
|
|
|
- return int(rows[0]["id"])
|
|
|
|
|
|
- state = slot_public(auth, "/projects", "projects")
|
|
|
- act(auth, state["create"], {"name": "co", "git_url": "https://example.com/co.git"})
|
|
|
- act(auth, state["create"], {"name": "ref", "git_url": "https://example.com/ref.git"})
|
|
|
- co_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
|
|
|
- if p["name"] == "co"][0])
|
|
|
- ref_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
|
|
|
- if p["name"] == "ref"][0])
|
|
|
-
|
|
|
- project = slot_public(auth, "/projects/1", "project")
|
|
|
- act(auth, project["add_link"], {"linked_project_id": str(co_id),
|
|
|
- "relationship": "co-developed"})
|
|
|
- act(auth, project["add_link"], {"linked_project_id": str(ref_id),
|
|
|
- "relationship": "reference"})
|
|
|
- act(auth, project["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
|
|
|
- act(auth, project["attach_rule"], {"rule_id": str(rule_id("Prefer meson"))})
|
|
|
- project = slot_public(auth, "/projects/1", "project")
|
|
|
- check(sorted(r["name"] for r in project["rules"]) == ["Prefer meson", "Use Invercargill"],
|
|
|
- "project page lists its attached rules")
|
|
|
-
|
|
|
- co_project = slot_public(auth, "/projects/%d" % co_id, "project")
|
|
|
- act(auth, co_project["attach_rule"], {"rule_id": str(rule_id("Co-dev rule"))})
|
|
|
- ref_project = slot_public(auth, "/projects/%d" % ref_id, "project")
|
|
|
- act(auth, ref_project["attach_rule"], {"rule_id": str(rule_id("Reference rule"))})
|
|
|
-
|
|
|
- tasks_state = slot_public(auth, "/tasks", "tasks")
|
|
|
- act(auth, tasks_state["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Rules dispatch",
|
|
|
- "description": "Effective rule set rides the /start payload.",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- rule_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Rules dispatch"][0])
|
|
|
- detail = task_state(auth, rule_task)
|
|
|
- act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Write Vala docs"))})
|
|
|
- act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
|
|
|
- detail = task_state(auth, rule_task)
|
|
|
- check(sorted(r["name"] for r in detail["rules"]) == ["Use Invercargill", "Write Vala docs"],
|
|
|
- "task page lists its attached rules")
|
|
|
-
|
|
|
- check(wait_for(lambda: len(fake.start_payloads) > 0 or False,
|
|
|
- timeout=30, what="/start push") is not None,
|
|
|
- "dispatcher pushes the task to the fake saddle")
|
|
|
- payload = fake.start_payloads[-1]
|
|
|
- rule_names = [r["name"] for r in payload.get("rules", [])]
|
|
|
- check(rule_names == ["Write Vala docs", "Use Invercargill", "Prefer meson", "Co-dev rule"],
|
|
|
- "effective rules in /start: task, primary, co-developed order (%s)" % rule_names)
|
|
|
- check(all(r.get("body") for r in payload.get("rules", [])),
|
|
|
- "rules carry their bodies")
|
|
|
- check("Reference rule" not in rule_names,
|
|
|
- "reference project rules excluded")
|
|
|
-
|
|
|
- detail = task_state(auth, rule_task)
|
|
|
- act(auth, detail["rules"][0]["detach"], {})
|
|
|
- check([r["name"] for r in task_state(auth, rule_task)["rules"]] == ["Use Invercargill"],
|
|
|
- "detached task rule disappears")
|
|
|
-
|
|
|
- rules_rows = slot_public(auth, "/rules", "rules")["rules"]
|
|
|
- docs_row = [r for r in rules_rows if r["name"] == "Write Vala docs"][0]
|
|
|
- act(auth, docs_row["update"], {"name": "Write Vala docs", "body": "Doc comments or bust."})
|
|
|
- check(any(r["body"] == "Doc comments or bust."
|
|
|
- for r in slot_public(auth, "/rules", "rules")["rules"]),
|
|
|
- "rule edit saves the body")
|
|
|
-
|
|
|
- check(wait_task_done(auth, rule_task, timeout=60) is not None,
|
|
|
- "dispatched task completes against the fake daemon")
|
|
|
-
|
|
|
- print("== feature 6: project notes + callback ==")
|
|
|
- # Plan 10 phase 1: the shared saddle-callback token is gone —
|
|
|
- # the per-saddle token authorises the callback endpoints.
|
|
|
- callback_token = fake_token
|
|
|
- check(callback_token.startswith("saddle-") and "saddle-callback-" not in callback_token,
|
|
|
- "note callback authorised by the per-saddle token")
|
|
|
-
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/note", None,
|
|
|
- {"project": "demo", "body": "x"})
|
|
|
- check(status == 401, "note endpoint rejects a missing token")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/note", "saddle-callback-not-real",
|
|
|
- {"project": "demo", "body": "x"})
|
|
|
- check(status == 401, "note endpoint rejects a wrong (legacy-style) token")
|
|
|
-
|
|
|
- status, body = saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
- {"task_id": rule_task, "project": "demo",
|
|
|
- "body": "style.css tokens live in :root"})
|
|
|
- check(status == 200 and body.get("ok") is True, "note accepted with the callback token")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
- {"project": "no-such-project", "body": "x"})
|
|
|
- check(status == 400, "unknown project rejected")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
- {"project": "demo", "body": " "})
|
|
|
- check(status == 400, "empty note body rejected")
|
|
|
-
|
|
|
- project = slot_public(auth, "/projects/1", "project")
|
|
|
- check(any("style.css tokens" in n["body"] and int(n["source_task_id"]) == rule_task
|
|
|
- for n in project["notes"]),
|
|
|
- "project page lists the note with its source task")
|
|
|
- act(auth, [n for n in project["notes"] if "style.css tokens" in n["body"]][0]["delete"], {})
|
|
|
- check(all("style.css tokens" not in n["body"]
|
|
|
- for n in slot_public(auth, "/projects/1", "project")["notes"]),
|
|
|
- "note deleted from the project page")
|
|
|
-
|
|
|
- # /start payload: notes of primary + co-developed (not reference),
|
|
|
- # oldest first; callback carries the configured base URL + token.
|
|
|
- saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
- {"project": "demo", "body": "note-one"})
|
|
|
- saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
- {"project": "demo", "body": "note-two"})
|
|
|
- saddle_api("POST", "/api/saddle/note", callback_token,
|
|
|
- {"project": "co", "body": "co-note"})
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into project_notes (project_id, body, created_at)"
|
|
|
- " values (?, 'ref-note-must-not-ship', strftime('%s','now'))", (ref_id,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
+ entries = [{"id": 8101, "command": "sed -n 1p README.md", "output": "# demo",
|
|
|
+ "exit_code": 0, "type": "command", "timestamp": "2026-09-06T00:00:00Z"}]
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
+ {"session_id": raw_session, "state": "working",
|
|
|
+ "entries": entries})
|
|
|
+ check(status == 200, "entries push accepted")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
+ {"session_id": raw_session, "state": "working",
|
|
|
+ "entries": entries})
|
|
|
+ check(status == 200, "redelivered entries push accepted (at-least-once)")
|
|
|
+ stored = command_rows(raw_session)
|
|
|
+ check(len([r for r in stored if r[0] == "sed -n 1p README.md"]) == 1,
|
|
|
+ "redelivered entries store exactly once (entry-id cursor)")
|
|
|
+
|
|
|
+ # The retired daemon routes 404 (phase 2 deleted /output, /log
|
|
|
+ # and /start serving) — and the web app never calls them.
|
|
|
+ for method, path in (("GET", "/output?since=0"), ("GET", "/log?since=0"),
|
|
|
+ ("POST", "/start")):
|
|
|
+ request = urllib.request.Request(
|
|
|
+ "http://127.0.0.1:%d%s" % (fake.port, path),
|
|
|
+ data=b"{}" if method == "POST" else None, method=method)
|
|
|
+ try:
|
|
|
+ urllib.request.urlopen(request, timeout=5)
|
|
|
+ gone = False
|
|
|
+ except urllib.error.HTTPError as error:
|
|
|
+ gone = error.code == 404
|
|
|
+ check(gone, "the fake daemon's %s %s is deleted (404)" % (method, path))
|
|
|
+ with open(os.path.join(WORK, "app.log")) as handle:
|
|
|
+ app_log = handle.read()
|
|
|
+ check("Saddle output request" not in app_log and "Saddle log request" not in app_log,
|
|
|
+ "the web never polls /output or /log anymore")
|
|
|
+
|
|
|
+ print("== capabilities: tag-based routing ==")
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "gpu"})
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "bigmem"})
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ caps = {c["name"]: c for c in saddles_state["capabilities"]}
|
|
|
+ check(len(caps) == 2 and all(int(c["saddle_count"]) == 0 and int(c["task_count"]) == 0
|
|
|
+ for c in caps.values()),
|
|
|
+ "capabilities created and listed with zero usage")
|
|
|
+
|
|
|
+
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ check(fake_card["capabilities"] == [],
|
|
|
+ "saddle card starts with no capability chips")
|
|
|
+
|
|
|
+ # A task requiring 'gpu' waits while the only idle saddle lacks it;
|
|
|
+ # a later task without requirements still dispatches (skip-and-continue).
|
|
|
+ tasks_state = slot_public(auth, "/tasks", "tasks")
|
|
|
+ act(auth, tasks_state["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "GPU task",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ gpu_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "GPU task"][0])
|
|
|
+ detail = task_state(auth, gpu_task)
|
|
|
+ act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
|
|
|
+ detail = task_state(auth, gpu_task)
|
|
|
+ check([c["name"] for c in detail["capabilities"]] == ["gpu"],
|
|
|
+ "task page lists the required capability")
|
|
|
+ # The only idle saddle lacks the tag, so the queue flags the wait
|
|
|
+ # (read before any other task claims the idle saddle).
|
|
|
+ queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if int(t["id"]) == gpu_task][0]
|
|
|
+ check(queue["status"] == "queued" and queue["capabilities"] == ["gpu"]
|
|
|
+ and queue.get("waiting_for_saddle") is True,
|
|
|
+ "queue row shows the waiting task with its tag and wait flag")
|
|
|
+
|
|
|
+ act(auth, tasks_state["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Any-saddle task",
|
|
|
+ "priority": "3",
|
|
|
+ })
|
|
|
+ any_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Any-saddle task"][0])
|
|
|
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Notes dispatch",
|
|
|
- "description": "Project notes ride the /start payload.",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- check(wait_for(lambda: len(fake.start_payloads) >= 2 or False,
|
|
|
- timeout=30, what="second /start push") is not None,
|
|
|
- "dispatcher pushes the notes task")
|
|
|
- payload = fake.start_payloads[-1]
|
|
|
- notes = payload.get("project_notes", [])
|
|
|
- check([n["body"] for n in notes] == ["note-one", "note-two", "co-note"],
|
|
|
- "project_notes: primary + co-developed, oldest first (%s)" % [n["body"] for n in notes])
|
|
|
- check(all(n.get("project") for n in notes)
|
|
|
- and notes[2]["project"] == "co",
|
|
|
- "project_notes carry the project name")
|
|
|
- check("ref-note-must-not-ship" not in [n["body"] for n in notes],
|
|
|
- "reference project notes excluded")
|
|
|
- callback = payload.get("callback", {})
|
|
|
- check(callback.get("base_url") == WEB and callback.get("token") == callback_token,
|
|
|
- "callback carries the configured base URL and the minted token")
|
|
|
-
|
|
|
- # entry cap: 30 of 33 candidates
|
|
|
- conn = db()
|
|
|
- for i in range(33):
|
|
|
- conn.execute("insert into project_notes (project_id, body, created_at)"
|
|
|
- " values (?, ?, strftime('%s','now'))", (1, "bulk-%02d" % i))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Entry cap dispatch",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- check(wait_for(lambda: len(fake.start_payloads) >= 3 or False,
|
|
|
- timeout=30, what="third /start push") is not None,
|
|
|
- "dispatcher pushes the entry-cap task")
|
|
|
- notes = fake.start_payloads[-1].get("project_notes", [])
|
|
|
- check(len(notes) == 30 and notes[0]["body"] == "note-one" and notes[1]["body"] == "note-two",
|
|
|
- "project_notes capped at 30 entries, oldest kept (%d)" % len(notes))
|
|
|
-
|
|
|
- # char cap: the giant note is truncated so the total stays <= 20000
|
|
|
- conn = db()
|
|
|
- conn.execute("delete from project_notes where body like 'bulk-%' or body = 'co-note'")
|
|
|
- conn.execute("insert into project_notes (project_id, body, created_at)"
|
|
|
- " values (?, ?, strftime('%s','now'))", (1, "giant!" + "x" * 25000,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Char cap dispatch",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- check(wait_for(lambda: len(fake.start_payloads) >= 4 or False,
|
|
|
- timeout=30, what="fourth /start push") is not None,
|
|
|
- "dispatcher pushes the char-cap task")
|
|
|
- notes = fake.start_payloads[-1].get("project_notes", [])
|
|
|
- total = sum(len(n["body"]) for n in notes)
|
|
|
- check(len(notes) == 3 and total == 20000 and notes[2]["body"].startswith("giant!"),
|
|
|
- "project_notes char cap: head kept, giant truncated (entries=%d, chars=%d)"
|
|
|
- % (len(notes), total))
|
|
|
-
|
|
|
- print("== phase 2: pushed chunks -> raw log + terminal view ==")
|
|
|
- # Wait for the last dispatch to finish so its pushes landed.
|
|
|
- check(wait_task_done(auth, task_id_of("Char cap dispatch"), timeout=60) is not None,
|
|
|
- "char-cap task completes")
|
|
|
- conn = db()
|
|
|
- raw_session = conn.execute(
|
|
|
- "select s.id from sessions s join tasks t on t.id = s.task_id"
|
|
|
- " where t.title = 'Char cap dispatch' order by s.id desc limit 1").fetchone()[0]
|
|
|
- rows = conn.execute("select seq, chunk from session_raw_log where session_id = ?"
|
|
|
- " order by seq", (raw_session,)).fetchall()
|
|
|
- conn.close()
|
|
|
- check(len(rows) >= 3, "session_raw_log captured the pushed chunks (%d)" % len(rows))
|
|
|
- seqs = [r[0] for r in rows]
|
|
|
- check(seqs == sorted(seqs) and len(set(seqs)) == len(seqs),
|
|
|
- "pushed chunks stored once, seq strictly increasing")
|
|
|
- text = "".join(r[1] for r in rows)
|
|
|
- check(fake.PROMPT in text and "echo hello" in text and "hello" in text,
|
|
|
- "pushed raw log keeps the prompt and echoed output byte-exact")
|
|
|
- check("working-heartbeat" in text,
|
|
|
- "the flush-timer heartbeat chunks keep arriving mid-round")
|
|
|
-
|
|
|
- terminal = slot_public(auth, "/tasks/%d/terminal" % task_id_of("Char cap dispatch"),
|
|
|
- "terminal")
|
|
|
- check(terminal.get("has_raw_log") is True and int(terminal["session_id"]) == raw_session,
|
|
|
- "terminal view selects raw rendering for the raw session")
|
|
|
-
|
|
|
- # Old sessions (typed entries, no raw rows) keep the fallback view.
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into session_commands (session_id, command_text, output_text,"
|
|
|
- " type, started_at) values (?, 'ls', 'old-output', 'command',"
|
|
|
- " strftime('%s','now'))", (session_id,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- legacy = slot_public(auth, "/tasks/%d/terminal" % plan_task, "terminal")
|
|
|
- check(legacy.get("has_raw_log") is False,
|
|
|
- "legacy session falls back to the entry-type terminal view")
|
|
|
-
|
|
|
- request = urllib.request.Request(WEB + "/terminal-stream/%d" % raw_session)
|
|
|
- with urllib.request.urlopen(request, timeout=15) as stream:
|
|
|
- raw_events = []
|
|
|
- deadline = time.monotonic() + 8
|
|
|
- while time.monotonic() < deadline and len(raw_events) < 3:
|
|
|
- line = stream.readline().decode().strip()
|
|
|
- if not line.startswith("data: ") or line == "data: ping":
|
|
|
- continue
|
|
|
- event = json.loads(line[6:])
|
|
|
- if event.get("type") == "raw":
|
|
|
- raw_events.append(event)
|
|
|
- check(len(raw_events) >= 3 and raw_events[0]["text"] == fake.PROMPT
|
|
|
- and raw_events[0]["seq"] < raw_events[1]["seq"],
|
|
|
- "terminal stream serves pushed chunks in seq order (prompt first)")
|
|
|
-
|
|
|
- print("== phase 2: chunks/results ingestion — auth, idempotency ==")
|
|
|
- # Token auth on both push channels (plan 10: the per-saddle
|
|
|
- # bearer authorises every daemon->web call).
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/chunks", None,
|
|
|
- {"session_id": raw_session, "chunks": []})
|
|
|
- check(status == 401, "chunks endpoint rejects a missing token")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/results", "saddle-bogus-token",
|
|
|
- {"session_id": raw_session, "state": "working"})
|
|
|
- check(status == 401, "results endpoint rejects a wrong token")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/chunks", fake_token,
|
|
|
- {"session_id": 99999, "chunks": []})
|
|
|
- check(status == 400, "chunks for an unknown session rejected")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
- {"session_id": raw_session, "state": "done"})
|
|
|
- check(status == 200, "the owning saddle's session accepts its results")
|
|
|
-
|
|
|
- # Pushed chunk idempotency: re-pushing a batch (the outbox's
|
|
|
- # at-least-once redelivery after a lost response) consumes the
|
|
|
- # duplicates and stores only the tail.
|
|
|
- conn = db()
|
|
|
- before = conn.execute("select count(*) from session_raw_log where session_id = ?",
|
|
|
- (raw_session,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- redelivery = [{"seq": r[0], "text": r[1]} for r in rows[:3]]
|
|
|
- status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
|
|
|
- {"session_id": raw_session, "chunks": redelivery})
|
|
|
- check(status == 200 and body.get("ok") is True and int(body.get("stored") or 0) == 0,
|
|
|
- "a fully-duplicate batch stores nothing (duplicates consumed)")
|
|
|
- fresh = [{"seq": 9000, "text": "late-flush\r\n"},
|
|
|
- {"seq": 9001, "text": ""}] # truly empty text: no row
|
|
|
- status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
|
|
|
- {"session_id": raw_session, "chunks": fresh})
|
|
|
- check(status == 200 and int(body.get("stored") or 0) == 1,
|
|
|
- "a partial-redelivery batch stores only the new chunks")
|
|
|
- conn = db()
|
|
|
- after = conn.execute("select count(*) from session_raw_log where session_id = ?",
|
|
|
- (raw_session,)).fetchone()[0]
|
|
|
- empty_row = conn.execute(
|
|
|
- "select count(*) from session_raw_log where session_id = ? and seq = 9001",
|
|
|
- (raw_session,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(after == before + 1 and empty_row == 0,
|
|
|
- "redelivered chunks never duplicate rows; empty text stores nothing")
|
|
|
-
|
|
|
- # Pushed entries: today's OutputEntry shape with the daemon's
|
|
|
- # global entry ids — a redelivered results push stores its
|
|
|
- # entries exactly once.
|
|
|
- def command_rows(session):
|
|
|
- conn = db()
|
|
|
- try:
|
|
|
- return conn.execute(
|
|
|
- "select command_text, type from session_commands"
|
|
|
- " where session_id = ? order by id", (session,)).fetchall()
|
|
|
- finally:
|
|
|
- conn.close()
|
|
|
-
|
|
|
- entries = [{"id": 8101, "command": "sed -n 1p README.md", "output": "# demo",
|
|
|
- "exit_code": 0, "type": "command", "timestamp": "2026-09-06T00:00:00Z"}]
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
- {"session_id": raw_session, "state": "working",
|
|
|
- "entries": entries})
|
|
|
- check(status == 200, "entries push accepted")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
|
|
|
- {"session_id": raw_session, "state": "working",
|
|
|
- "entries": entries})
|
|
|
- check(status == 200, "redelivered entries push accepted (at-least-once)")
|
|
|
- stored = command_rows(raw_session)
|
|
|
- check(len([r for r in stored if r[0] == "sed -n 1p README.md"]) == 1,
|
|
|
- "redelivered entries store exactly once (entry-id cursor)")
|
|
|
-
|
|
|
- # The retired daemon routes 404 (phase 2 deleted /output, /log
|
|
|
- # and /start serving) — and the web app never calls them.
|
|
|
- for method, path in (("GET", "/output?since=0"), ("GET", "/log?since=0"),
|
|
|
- ("POST", "/start")):
|
|
|
- request = urllib.request.Request(
|
|
|
- "http://127.0.0.1:%d%s" % (fake.port, path),
|
|
|
- data=b"{}" if method == "POST" else None, method=method)
|
|
|
- try:
|
|
|
- urllib.request.urlopen(request, timeout=5)
|
|
|
- gone = False
|
|
|
- except urllib.error.HTTPError as error:
|
|
|
- gone = error.code == 404
|
|
|
- check(gone, "the fake daemon's %s %s is deleted (404)" % (method, path))
|
|
|
+ check(wait_for(lambda: task_state(auth, any_task).get("status") != "queued" or False,
|
|
|
+ timeout=30, what="uncapable task dispatches") is not None,
|
|
|
+ "task without requirements dispatches past the waiting task")
|
|
|
+ def capability_wait_logged():
|
|
|
with open(os.path.join(WORK, "app.log")) as handle:
|
|
|
app_log = handle.read()
|
|
|
- check("Saddle output request" not in app_log and "Saddle log request" not in app_log,
|
|
|
- "the web never polls /output or /log anymore")
|
|
|
-
|
|
|
- print("== capabilities: tag-based routing ==")
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- act(auth, saddles_state["create_capability"], {"name": "gpu"})
|
|
|
- act(auth, saddles_state["create_capability"], {"name": "bigmem"})
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- caps = {c["name"]: c for c in saddles_state["capabilities"]}
|
|
|
- check(len(caps) == 2 and all(int(c["saddle_count"]) == 0 and int(c["task_count"]) == 0
|
|
|
- for c in caps.values()),
|
|
|
- "capabilities created and listed with zero usage")
|
|
|
-
|
|
|
- def saddle_card(name):
|
|
|
- rows = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == name]
|
|
|
- return rows[0]
|
|
|
-
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- check(fake_card["capabilities"] == [],
|
|
|
- "saddle card starts with no capability chips")
|
|
|
-
|
|
|
- # A task requiring 'gpu' waits while the only idle saddle lacks it;
|
|
|
- # a later task without requirements still dispatches (skip-and-continue).
|
|
|
- tasks_state = slot_public(auth, "/tasks", "tasks")
|
|
|
- act(auth, tasks_state["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "GPU task",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- gpu_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "GPU task"][0])
|
|
|
- detail = task_state(auth, gpu_task)
|
|
|
- act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
|
|
|
- detail = task_state(auth, gpu_task)
|
|
|
- check([c["name"] for c in detail["capabilities"]] == ["gpu"],
|
|
|
- "task page lists the required capability")
|
|
|
- # The only idle saddle lacks the tag, so the queue flags the wait
|
|
|
- # (read before any other task claims the idle saddle).
|
|
|
- queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if int(t["id"]) == gpu_task][0]
|
|
|
- check(queue["status"] == "queued" and queue["capabilities"] == ["gpu"]
|
|
|
- and queue.get("waiting_for_saddle") is True,
|
|
|
- "queue row shows the waiting task with its tag and wait flag")
|
|
|
-
|
|
|
- act(auth, tasks_state["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Any-saddle task",
|
|
|
- "priority": "3",
|
|
|
- })
|
|
|
- any_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Any-saddle task"][0])
|
|
|
-
|
|
|
- check(wait_for(lambda: task_state(auth, any_task).get("status") != "queued" or False,
|
|
|
- timeout=30, what="uncapable task dispatches") is not None,
|
|
|
- "task without requirements dispatches past the waiting task")
|
|
|
- def capability_wait_logged():
|
|
|
- with open(os.path.join(WORK, "app.log")) as handle:
|
|
|
- app_log = handle.read()
|
|
|
- return "waits for a capable saddle" in app_log and "requires: gpu" in app_log
|
|
|
- check(wait_for(capability_wait_logged, timeout=20,
|
|
|
- what="capability wait log") is not None,
|
|
|
- "dispatcher logs the capability wait")
|
|
|
-
|
|
|
- # Attaching the tag to the saddle releases the task.
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- check([c["name"] for c in fake_card["capabilities"]] == ["gpu"],
|
|
|
- "saddle card shows the attached chip")
|
|
|
- check(wait_task_done(auth, gpu_task, timeout=60) is not None,
|
|
|
- "task dispatches once the saddle carries the required tag")
|
|
|
- queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if int(t["id"]) == gpu_task][0]
|
|
|
- check(queue.get("waiting_for_saddle") is not True,
|
|
|
- "wait flag clears once dispatched")
|
|
|
-
|
|
|
- # Multi-tag: gpu + bigmem required, only gpu carried → waits; adding
|
|
|
- # bigmem completes the superset. Both requirements ride the create
|
|
|
- # form (atomic — attaching them one at a time would race a tick
|
|
|
- # into dispatching on the partial set).
|
|
|
- act(auth, tasks_state["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Big task",
|
|
|
- "priority": "2",
|
|
|
- "capability_id": [str(int(caps["gpu"]["id"])), str(int(caps["bigmem"]["id"]))],
|
|
|
- })
|
|
|
- big_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Big task"][0])
|
|
|
- check(wait_for(lambda: ([t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if int(t["id"]) == big_task][0].get("waiting_for_saddle")
|
|
|
- is True) or False, timeout=30, what="big task waits") is not None,
|
|
|
- "task waits when the saddle's tags are not a superset")
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["bigmem"]["id"]))})
|
|
|
- check(wait_task_done(auth, big_task, timeout=60) is not None,
|
|
|
- "task dispatches once the superset is complete")
|
|
|
-
|
|
|
- # Detach a requirement; delete a capability everywhere.
|
|
|
- detail = task_state(auth, big_task)
|
|
|
- act(auth, [c for c in detail["capabilities"] if c["name"] == "bigmem"][0]["detach"], {})
|
|
|
- check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
|
|
|
- "detached requirement disappears from the task")
|
|
|
- bigmem_row = [c for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
- if c["name"] == "bigmem"][0]
|
|
|
- act(auth, bigmem_row["delete"], {})
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- check(all(c["name"] != "bigmem" for c in saddles_state["capabilities"]),
|
|
|
- "deleted capability gone from the catalogue")
|
|
|
- check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
|
|
|
- "deleting a capability drops its task requirements")
|
|
|
- check(all(c["name"] != "bigmem"
|
|
|
- for c in saddle_card("fake-saddle")["capabilities"]),
|
|
|
- "deleting a capability drops it from saddles")
|
|
|
-
|
|
|
- print("== B4: required capabilities on the create form ==")
|
|
|
- tasks_state = slot_public(auth, "/tasks", "tasks")
|
|
|
- gpu_option = [o for o in tasks_state.get("capability_options", [])
|
|
|
- if o["name"] == "gpu"][0]
|
|
|
- check(len(tasks_state.get("capability_options", [])) >= 1,
|
|
|
- "queue page offers capability options in the create form")
|
|
|
-
|
|
|
- conn = db()
|
|
|
- before = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
- conn.close()
|
|
|
- act(auth, tasks_state["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Bad cap task",
|
|
|
- "capability_id": [gpu_option["id"], "999"],
|
|
|
- })
|
|
|
- conn = db()
|
|
|
- after = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(after == before, "unknown capability id rejects the create")
|
|
|
-
|
|
|
- act(auth, tasks_state["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Cap-on-create task",
|
|
|
- "capability_id": [gpu_option["id"]],
|
|
|
- })
|
|
|
- cap_create = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Cap-on-create task"][0])
|
|
|
- detail = task_state(auth, cap_create)
|
|
|
- check([c["name"] for c in detail["capabilities"]] == ["gpu"],
|
|
|
- "capabilities from the create form land on the task")
|
|
|
- queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if int(t["id"]) == cap_create][0]
|
|
|
- check(queue["capabilities"] == ["gpu"],
|
|
|
- "queue row shows the created task's requirements")
|
|
|
- # The waiting flag must hold it back until the capable saddle frees
|
|
|
- # up (the fake saddle still carries gpu from the routing checks).
|
|
|
- check(queue.get("waiting_for_saddle") is not True,
|
|
|
- "capable saddle available: no wait flag on the new task")
|
|
|
-
|
|
|
- print("== B6: computed display state ==")
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
- " values (1, 'Parent', 'done', 5, strftime('%s','now'))")
|
|
|
- parent6 = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
- # The post-mortem case: stored done WITH an unanswered review block.
|
|
|
- conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
|
|
|
- " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
|
|
|
- (parent6, fake_id))
|
|
|
- session6 = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "block", "session_id": session6, "data": {
|
|
|
- "question": "Review the final diff", "type": "review"}})
|
|
|
-
|
|
|
- detail = task_state(auth, parent6)
|
|
|
- check(detail.get("display") == "review requested" and detail.get("terminal") is True,
|
|
|
- "unanswered review overrides the stored done status (terminal flag keeps stored truth)")
|
|
|
- queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if int(t["id"]) == parent6][0]
|
|
|
- check(queue["display"] == "review requested",
|
|
|
- "queue row shows the computed review-requested display")
|
|
|
-
|
|
|
- # Delegation: with the review answered, open subtasks delegate.
|
|
|
- conn = db()
|
|
|
- conn.execute("update task_blocks set answered_at = strftime('%s','now'),"
|
|
|
- " answer = 'Human approved the review' where task_id = ?", (parent6,))
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
- " created_at) values (1, 'Sub A', 'working', 5, ?, strftime('%s','now'))",
|
|
|
- (parent6,))
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
- " created_at) values (1, 'Sub B', 'done', 5, ?, strftime('%s','now'))",
|
|
|
- (parent6,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- detail = task_state(auth, parent6)
|
|
|
- check(detail.get("display") == "delegating 1/2" and detail.get("subtask_open") == 1,
|
|
|
- "open subtasks show the delegating breakdown (%s)" % detail.get("display"))
|
|
|
-
|
|
|
- # Flags track the stored status once the display falls through
|
|
|
- # (subtasks terminal first — delegating leads while any are open).
|
|
|
- conn = db()
|
|
|
- conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
|
|
|
- " where id = ?", (parent6,))
|
|
|
- conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
|
|
|
- " where title = 'Sub A'")
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- detail = task_state(auth, parent6)
|
|
|
- check(detail.get("display") == "failed" and detail.get("failed") is True
|
|
|
- and detail.get("terminal") is True,
|
|
|
- "failed task with answered review and terminal subtasks shows failed + terminal flags")
|
|
|
-
|
|
|
- print("== retry + resume ==")
|
|
|
- # A daemon-side failure fails the task with its explanation.
|
|
|
- fake.fail_next = True
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Flaky task",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- flaky = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Flaky task"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, flaky).get("status") == "failed" or False,
|
|
|
- timeout=60, what="flaky task fails") is not None,
|
|
|
- "daemon failure fails the task")
|
|
|
- detail = task_state(auth, flaky)
|
|
|
- check("bash -n rejected" in (detail.get("summary") or ""),
|
|
|
- "failure summary shows the daemon's explanation")
|
|
|
-
|
|
|
- # Retry: summary moves into a comment and clears, task requeues,
|
|
|
- # history stays, and the re-dispatch restores the workspace.
|
|
|
- comments_before = len(detail.get("comments", []))
|
|
|
- progress_before = len(detail.get("progress", []))
|
|
|
- act(auth, detail["retry"], {})
|
|
|
- detail = task_state(auth, flaky)
|
|
|
- check(detail["status"] == "queued" and (detail.get("summary") or "") == "",
|
|
|
- "retry requeues the task and clears the stale summary")
|
|
|
- check(any("bash -n rejected" in c["body"] for c in detail["comments"]),
|
|
|
- "failure explanation kept as a bot comment")
|
|
|
- check(any(c["body"] == "Task retried by user" for c in detail["comments"]),
|
|
|
- "retry lands its audit comment")
|
|
|
- check(len(detail["comments"]) >= comments_before + 1,
|
|
|
- "retry keeps the comment history")
|
|
|
- pushes_before = len([p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == flaky])
|
|
|
- check(wait_for(lambda: len([p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == flaky])
|
|
|
- > pushes_before or False, timeout=60,
|
|
|
- what="flaky task re-dispatched") is not None,
|
|
|
- "retried task dispatches again")
|
|
|
- # The resumed arc surfaces on the next /status read — in the
|
|
|
- # push world that is the operator's manual refresh.
|
|
|
- act(auth, saddle_card("fake-saddle")["refresh"], {})
|
|
|
- check(wait_task_done(auth, flaky, timeout=60) is not None,
|
|
|
- "retried task completes on the restored workspace")
|
|
|
- detail = task_state(auth, flaky)
|
|
|
- check(any("restored the session context" in p["summary"]
|
|
|
- for p in detail.get("progress", [])),
|
|
|
- "daemon resume reported on the first status after the restore")
|
|
|
- check(len(detail["progress"]) > progress_before,
|
|
|
- "resume note appended to the progress timeline")
|
|
|
-
|
|
|
- # Manual resume: a task stuck working on an offline saddle.
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Stuck task",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- stuck = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Stuck task"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, stuck).get("status") == "working" or False,
|
|
|
- timeout=60, what="stuck task starts working") is not None,
|
|
|
- "stuck task reaches working")
|
|
|
- conn = db()
|
|
|
- conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- detail = task_state(auth, stuck)
|
|
|
- check(detail.get("resume_available") is True,
|
|
|
- "task page offers resume while its saddle is offline")
|
|
|
- act(auth, detail["resume"], {})
|
|
|
- detail = task_state(auth, stuck)
|
|
|
- check(detail["status"] == "queued",
|
|
|
- "manual resume requeues the stuck task")
|
|
|
- check(any(c["body"] == "Saddle became unreachable; task requeued"
|
|
|
- for c in detail["comments"]),
|
|
|
- "manual resume lands the interruption comment")
|
|
|
- check(all(s["status"] == "offline"
|
|
|
- for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
|
|
|
- "saddle stays offline after the manual requeue")
|
|
|
-
|
|
|
- # Bringing the saddle back lets the requeued task resume and finish.
|
|
|
- fake.state = "idle"
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- act(auth, fake_card["refresh"], {})
|
|
|
- check(any(s["status"] == "idle"
|
|
|
- for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "fake-saddle"),
|
|
|
- "refreshed saddle returns idle")
|
|
|
- # The refreshed daemon is idle-awake: it asks (the real loop's
|
|
|
- # own ask; driven directly for determinism).
|
|
|
- status, _ = fake.ask_now()
|
|
|
- check(status == 200, "the refreshed daemon's ask claims the requeued task")
|
|
|
- check(wait_task_done(auth, stuck, timeout=60) is not None,
|
|
|
- "requeued task resumes on the refreshed saddle and completes")
|
|
|
-
|
|
|
- print("== rejected entries (non-fatal) ==")
|
|
|
- # Ingest API: a rejected entry stores on the session and streams
|
|
|
- # through SSE; the task is untouched.
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "output", "session_id": session_id, "data": {
|
|
|
- "command": "bash -n script.sh",
|
|
|
- "output": "syntax error near line 3 (recoverable)",
|
|
|
- "exit_code": 1,
|
|
|
- "type": "rejected"}})
|
|
|
- check(status == 200, "rejected entry accepted by the ingest API")
|
|
|
- request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
|
|
|
- with urllib.request.urlopen(request, timeout=15) as stream:
|
|
|
- rejected_event = None
|
|
|
- deadline = time.monotonic() + 8
|
|
|
- while time.monotonic() < deadline and rejected_event is None:
|
|
|
- line = stream.readline().decode().strip()
|
|
|
- if not line.startswith("data: ") or line == "data: ping":
|
|
|
- continue
|
|
|
- event = json.loads(line[6:])
|
|
|
- if event.get("type") == "rejected":
|
|
|
- rejected_event = event
|
|
|
- check(rejected_event is not None
|
|
|
- and "syntax error near line 3" in (rejected_event.get("output") or ""),
|
|
|
- "terminal stream carries rejected entries with their reason")
|
|
|
-
|
|
|
- # The page template renders them with the distinct non-fatal style.
|
|
|
- status, page = http("GET", "/tasks/%d/terminal" % stuck, timeout=15)
|
|
|
- check(status == 200 and "term-rejected" in page,
|
|
|
- "terminal page renders rejected entries in their own style")
|
|
|
-
|
|
|
- print("== B8: ANSI SGR in the terminal view ==")
|
|
|
- # Byte-exact SGR delivery: the daemon's coloured chunks flow through
|
|
|
- # the store + SSE untouched (the client renderer styles them).
|
|
|
- coloured = "\x1b[1;32mbold green\x1b[0m then \x1b[38;5;196m256-red\x1b[0m\n"
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into session_raw_log (session_id, seq, chunk)"
|
|
|
- " values (?, 900, ?)", (session_id, coloured))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
|
|
|
- with urllib.request.urlopen(request, timeout=15) as stream:
|
|
|
- coloured_event = None
|
|
|
- deadline = time.monotonic() + 8
|
|
|
- while time.monotonic() < deadline and coloured_event is None:
|
|
|
- line = stream.readline().decode().strip()
|
|
|
- if not line.startswith("data: ") or line == "data: ping":
|
|
|
- continue
|
|
|
- event = json.loads(line[6:])
|
|
|
- if event.get("type") == "raw" and "\x1b[1;32m" in (event.get("text") or ""):
|
|
|
- coloured_event = event
|
|
|
- check(coloured_event is not None
|
|
|
- and "\x1b[38;5;196m" in coloured_event["text"],
|
|
|
- "SGR sequences stream byte-exact to the client renderer")
|
|
|
- status, page = http("GET", "/tasks/%d/terminal" % stuck, timeout=15)
|
|
|
- check(status == 200 and "makeAnsiRenderer" in page and "colour256" in page,
|
|
|
- "terminal page embeds the ANSI renderer (SGR + 256-colour palette)")
|
|
|
-
|
|
|
- # Dispatch path: a rejected entry in a round's results push flows
|
|
|
- # into the audit trail without failing (or finishing) the task.
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Rejected ride",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- rejected_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Rejected ride"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, rejected_task).get("status") == "working"
|
|
|
- or False, timeout=60, what="rejected-ride working") is not None,
|
|
|
- "rejected-ride task reaches working")
|
|
|
- fake.queue_output("rejected", "echo 'too long' | saddle",
|
|
|
- "max_tokens exceeded (recoverable)")
|
|
|
- check(wait_task_done(auth, rejected_task, timeout=60) is not None,
|
|
|
- "task completes normally past rejected entries")
|
|
|
- conn = db()
|
|
|
- stored = conn.execute(
|
|
|
- "select count(*) from session_commands sc join sessions s on s.id = sc.session_id"
|
|
|
- " where s.task_id = ? and sc.type = 'rejected'", (rejected_task,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(stored >= 1,
|
|
|
- "rejected entries stored in the audit trail (%d rows)" % stored)
|
|
|
- check(task_state(auth, rejected_task)["status"] == "done"
|
|
|
- and "fake daemon finished" in (task_state(auth, rejected_task).get("summary") or ""),
|
|
|
- "rejected entries never fail the task")
|
|
|
- print("== B7: suspended state + per-subtask resume ==")
|
|
|
- fake.suspend_on_title = "Suspend parent"
|
|
|
- pushes_before = len(fake.start_payloads)
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Suspend parent",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- parent7 = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Suspend parent"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, parent7).get("status") == "suspended"
|
|
|
- or False, timeout=60, what="parent parks as suspended") is not None,
|
|
|
- "daemon 'suspended' state parks the task")
|
|
|
- detail = task_state(auth, parent7)
|
|
|
- check(detail.get("display") == "waiting on subtasks",
|
|
|
- "suspended task displays as waiting on subtasks")
|
|
|
- check(any(s["status"] == "idle"
|
|
|
- for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
|
|
|
- "suspension frees the saddle")
|
|
|
- first_push = fake.start_payloads[pushes_before]
|
|
|
- check(first_push.get("subtasks") == [],
|
|
|
- "the /start payload carries the subtasks array (empty here)")
|
|
|
-
|
|
|
- # Two subtask events land while parked (staged, not dispatched):
|
|
|
- # both must coalesce into one resume with a full report.
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
- " summary, completed_at, created_at) values (1, 'Child one', 'done', 5, ?,"
|
|
|
- " 'built and verified the flange', strftime('%s','now','+1 second'),"
|
|
|
- " strftime('%s','now'))", (parent7,))
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
- " summary, completed_at, created_at) values (1, 'Child two', 'failed', 5, ?,"
|
|
|
- " 'the harness refused to start', strftime('%s','now','+1 second'),"
|
|
|
- " strftime('%s','now'))", (parent7,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
-
|
|
|
- def parent_push_count():
|
|
|
- return len([p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == parent7])
|
|
|
-
|
|
|
- check(wait_for(lambda: parent_push_count() >= 2 or False,
|
|
|
- timeout=45, what="suspended parent resumes") is not None,
|
|
|
- "subtask events resume the suspended parent")
|
|
|
- parent_pushes = [p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == parent7]
|
|
|
- check(len(parent_pushes) >= 2,
|
|
|
- "the parent's resume is a fresh /start for the same task")
|
|
|
- resume_push = parent_pushes[-1]
|
|
|
- notice = resume_push.get("resume_notice") or ""
|
|
|
- check("Child one" in notice and "completed" in notice
|
|
|
- and "built and verified the flange" in notice,
|
|
|
- "resume notice reports the finished subtask with its summary")
|
|
|
- check("Child two" in notice and "failed" in notice
|
|
|
- and "the harness refused to start" in notice,
|
|
|
- "coalesced: the second event rides the same resume notice")
|
|
|
- subtask_entries = {(s["title"], s["status"]) for s in resume_push.get("subtasks", [])}
|
|
|
- check(("Child one", "done") in subtask_entries
|
|
|
- and ("Child two", "failed") in subtask_entries,
|
|
|
- "subtasks array carries the children's states at dispatch time")
|
|
|
- # The resumed arc surfaces on a /status read while the resumed
|
|
|
- # round still runs — the operator's manual refresh in the push
|
|
|
- # world.
|
|
|
- act(auth, saddle_card("fake-saddle")["refresh"], {})
|
|
|
- check(wait_task_done(auth, parent7, timeout=60) is not None,
|
|
|
- "resumed parent completes")
|
|
|
- detail = task_state(auth, parent7)
|
|
|
- check(any("restored the session context" in p["summary"]
|
|
|
- for p in detail.get("progress", [])),
|
|
|
- "resumed flag reported on the first status after the restore")
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- print("== R3 B1: subtask status push ==")
|
|
|
- # A parked (suspended) parent gives delivery a stable open session.
|
|
|
- fake.suspend_on_title = "Push parent"
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Push parent",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- push_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Push parent"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, push_parent).get("status") == "suspended"
|
|
|
- or False, timeout=60, what="push parent parks") is not None,
|
|
|
- "push parent parks with an open session")
|
|
|
-
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Push child",
|
|
|
- "parent_task_id": str(push_parent),
|
|
|
- })
|
|
|
- push_child = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Push child"][0])
|
|
|
-
|
|
|
- def events_for(subtask_id):
|
|
|
- return [e for e in fake.subtask_status_events
|
|
|
- if int(e.get("subtask_id") or 0) == subtask_id]
|
|
|
-
|
|
|
- check(wait_for(lambda: len(events_for(push_child)) > 0 or False,
|
|
|
- timeout=30, what="queued subtask event") is not None,
|
|
|
- "subtask creation pushes a queued status event")
|
|
|
- first_event = events_for(push_child)[0]
|
|
|
- check(int(first_event.get("parent_task_id") or 0) == push_parent
|
|
|
- and first_event.get("title") == "Push child"
|
|
|
- and first_event.get("status") in ("queued", "working"),
|
|
|
- "status event carries parent id, title and a wire status (%s)"
|
|
|
- % first_event.get("status"))
|
|
|
-
|
|
|
- # Real lifecycle: the child dispatches (working) and completes (done
|
|
|
- # with summary) — the parent's daemon hears every transition.
|
|
|
- check(wait_for(lambda: any(e.get("status") == "done"
|
|
|
- for e in events_for(push_child)) or False,
|
|
|
- timeout=90, what="done subtask event") is not None,
|
|
|
- "subtask completion pushes the done transition")
|
|
|
- check(any(e.get("status") == "working" for e in events_for(push_child)),
|
|
|
- "dispatch pushes a working status event")
|
|
|
-
|
|
|
- # Best-effort retry: while the daemon 500s, events stay queued and
|
|
|
- # land once it recovers. The parent's resumed run re-parks (the
|
|
|
- # ask-flow resume races this section otherwise: its completion
|
|
|
- # would close the parked session the events are delivered to).
|
|
|
- fake.fail_subtask_status = True
|
|
|
- fake.suspend_on_title = "Push parent"
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Push child two",
|
|
|
- "parent_task_id": str(push_parent),
|
|
|
- })
|
|
|
- push_child_two = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Push child two"][0])
|
|
|
- time.sleep(7)
|
|
|
- check(len(events_for(push_child_two)) == 0,
|
|
|
- "failed pushes stay queued (no event delivered)")
|
|
|
- fake.fail_subtask_status = False
|
|
|
- check(wait_for(lambda: len(events_for(push_child_two)) > 0 or False,
|
|
|
- timeout=30, what="retried subtask event") is not None,
|
|
|
- "recovered pushes deliver on a later tick")
|
|
|
-
|
|
|
-
|
|
|
- print("== R4 B1: synchronous subtask callback ==")
|
|
|
- # Plan 10: the subtask callback is per-saddle-token authenticated.
|
|
|
- r4_token = fake_token
|
|
|
-
|
|
|
- # Parent with a live parked session (suspended) hosts the callback.
|
|
|
- fake.suspend_on_title = "Callback parent"
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Callback parent",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- cb_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Callback parent"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, cb_parent).get("status") == "suspended"
|
|
|
- or False, timeout=60, what="callback parent parks") is not None,
|
|
|
- "callback parent parks")
|
|
|
-
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/subtask", None,
|
|
|
- {"parent_task_id": cb_parent, "title": "no token"})
|
|
|
- check(status == 401, "subtask callback rejects a missing token")
|
|
|
-
|
|
|
- anchor = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Rules dispatch"][0])
|
|
|
- status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
- "parent_task_id": cb_parent,
|
|
|
- "title": "Wire the flange",
|
|
|
- "description": "torque to spec",
|
|
|
- "depends_on": [anchor],
|
|
|
- "session_id": 1,
|
|
|
- })
|
|
|
- check(status == 200 and body.get("ok") is True and int(body.get("task_id") or 0) > 0,
|
|
|
- "callback creates the subtask synchronously")
|
|
|
- subtask_id = int(body.get("task_id"))
|
|
|
- detail = task_state(auth, subtask_id)
|
|
|
- check(detail["title"] == "Wire the flange" and int(detail["project_id"]) == 1
|
|
|
- and int(detail["parent_task_id"]) == cb_parent,
|
|
|
- "subtask row inherits the parent's project and linkage")
|
|
|
- check(detail.get("task_branch") == task_state(auth, cb_parent).get("task_branch"),
|
|
|
- "subtask shares the parent's task branch")
|
|
|
- conn = db()
|
|
|
- dep_rows = conn.execute(
|
|
|
- "select depends_on_task_id from task_dependencies where task_id = ?",
|
|
|
- (subtask_id,)).fetchall()
|
|
|
- conn.close()
|
|
|
- check([r[0] for r in dep_rows] == [anchor],
|
|
|
- "depends_on wired as task_dependencies")
|
|
|
-
|
|
|
- # Idempotency + entry-poll fallback dedup.
|
|
|
- status, body2 = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
- "parent_task_id": cb_parent, "title": "Wire the flange"})
|
|
|
- check(status == 200 and int(body2.get("task_id") or 0) == subtask_id,
|
|
|
- "retried callback returns the same task id")
|
|
|
- conn = db()
|
|
|
- before = conn.execute("select count(*) from tasks where parent_task_id = ?",
|
|
|
- (cb_parent,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- conn = db()
|
|
|
- cb_session = conn.execute(
|
|
|
- "select id from sessions where task_id = ? and ended_at is null",
|
|
|
- (cb_parent,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "subtask", "session_id": cb_session, "data": {
|
|
|
- "title": "Wire the flange", "description": "again"}})
|
|
|
- conn = db()
|
|
|
- after = conn.execute("select count(*) from tasks where parent_task_id = ?",
|
|
|
- (cb_parent,)).fetchone()[0]
|
|
|
- timeline = conn.execute(
|
|
|
- "select count(*) from session_commands where session_id = ?"
|
|
|
- " and type = 'subtask'", (cb_session,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(after == before,
|
|
|
- "entry-poll fallback dedupes against the callback's row")
|
|
|
- check(timeline >= 1, "timeline entry recorded on the parent's session")
|
|
|
-
|
|
|
- # Validation: unknown parent, unknown depends_on, ancestor cycle.
|
|
|
- status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
- {"parent_task_id": 99999, "title": "ghost"})
|
|
|
- check(status == 400, "unknown parent rejected")
|
|
|
- status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
- {"parent_task_id": cb_parent, "title": "bad dep",
|
|
|
- "depends_on": [99999]})
|
|
|
- check(status == 400 and "unknown task" in body.get("error", ""),
|
|
|
- "unknown depends_on id rejected with a clear error")
|
|
|
- status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
- {"parent_task_id": cb_parent, "title": "cycle dep",
|
|
|
- "depends_on": [cb_parent]})
|
|
|
- check(status == 400 and "cycle" in body.get("error", ""),
|
|
|
- "depending on an ancestor rejected as a cycle")
|
|
|
-
|
|
|
-
|
|
|
- print("== R4 B2: subtask auto-approve ==")
|
|
|
- # Settle earlier stragglers first: the round-3/B1 suspended parents
|
|
|
- # resume (a fresh /start) once their last subtask completes — those
|
|
|
- # pushes would consume the one-shot suspend flag below.
|
|
|
- check(wait_task_done(auth, push_parent, timeout=90) is not None
|
|
|
- and wait_task_done(auth, push_child_two, timeout=90) is not None
|
|
|
- and wait_task_done(auth, cb_parent, timeout=90) is not None,
|
|
|
- "earlier suspended parents settle before the auto-approve run")
|
|
|
-
|
|
|
- # Parent parks; its toggle then covers all descendants. Children are
|
|
|
- # created with an unmet 'gpu' requirement so they stay queued (and
|
|
|
- # untouched) until the requirement is lifted for the final-resume leg.
|
|
|
- fake.suspend_on_title = "Auto parent"
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Auto parent",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- auto_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Auto parent"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, auto_parent).get("status") == "suspended"
|
|
|
- or False, timeout=60, what="auto parent parks") is not None,
|
|
|
- "auto parent parks")
|
|
|
-
|
|
|
- detail = task_state(auth, auto_parent)
|
|
|
- check(detail.get("auto_approve") is False,
|
|
|
- "auto-approve defaults to off")
|
|
|
- act(auth, detail["toggle_auto_approve"], {})
|
|
|
- check(task_state(auth, auto_parent).get("auto_approve") is True,
|
|
|
- "toggle turns auto-approve on")
|
|
|
-
|
|
|
- gpu_option = int([o["id"] for o in slot_public(auth, "/tasks", "tasks")["capability_options"]
|
|
|
- if o["name"] == "gpu"][0])
|
|
|
- status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
- {"parent_task_id": auto_parent,
|
|
|
- "title": "Auto child", "depends_on": []})
|
|
|
- auto_child = int(body["task_id"])
|
|
|
- status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
- {"parent_task_id": auto_child, "title": "Auto grandchild"})
|
|
|
- auto_grand = int(body["task_id"])
|
|
|
- act(auth, task_state(auth, auto_child)["attach_capability"],
|
|
|
- {"capability_id": str(gpu_option)})
|
|
|
- act(auth, task_state(auth, auto_grand)["attach_capability"],
|
|
|
- {"capability_id": str(gpu_option)})
|
|
|
-
|
|
|
- # Transitive coverage: the GRANDCHILD's review is auto-answered
|
|
|
- # (any ancestor set), answered-by 'auto', timeline entry recorded.
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into sessions (task_id, saddle_id, started_at)"
|
|
|
- " values (?, ?, strftime('%s','now'))", (auto_grand, fake_id))
|
|
|
- conn.commit()
|
|
|
- grand_session = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
- conn.close()
|
|
|
- saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "review", "session_id": grand_session, "data": {
|
|
|
- "summary": "the flange is wired"}})
|
|
|
- grand_detail = task_state(auth, auto_grand)
|
|
|
- grand_review = [b for b in grand_detail["blocks"] if b["block_type"] == "review"][0]
|
|
|
- check(grand_review.get("answered") is True
|
|
|
- and grand_review.get("answer") == "Auto-approved (subtask auto-approve is on)"
|
|
|
- and grand_review.get("answered_by") == "auto",
|
|
|
- "covered grandchild review auto-answered as 'auto'")
|
|
|
- conn = db()
|
|
|
- timeline_auto = conn.execute(
|
|
|
- "select count(*) from session_commands where session_id = ?"
|
|
|
- " and type = 'auto-approve'", (grand_session,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(timeline_auto >= 1, "auto action visible in the timeline")
|
|
|
-
|
|
|
- # Questions and RFCs still block for a human.
|
|
|
- saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "block", "session_id": grand_session, "data": {
|
|
|
- "question": "Which torque spec?", "type": "question"}})
|
|
|
- saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "block", "session_id": grand_session, "data": {
|
|
|
- "question": "RFC: approve the design", "type": "rfc",
|
|
|
- "body": "# Design"}})
|
|
|
- grand_detail = task_state(auth, auto_grand)
|
|
|
- check(all(b.get("answered") is not True for b in grand_detail["blocks"]
|
|
|
- if b["block_type"] in ("question", "rfc")),
|
|
|
- "questions and RFCs are never auto-answered")
|
|
|
-
|
|
|
- # The toggle covers DESCENDANTS only: the parent's own review waits.
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
|
|
|
- " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
|
|
|
- " 'done')", (auto_parent, fake_id))
|
|
|
- conn.commit()
|
|
|
- parent_session = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
- conn.close()
|
|
|
- saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "review", "session_id": parent_session, "data": {
|
|
|
- "summary": "the whole change"}})
|
|
|
- parent_detail = task_state(auth, auto_parent)
|
|
|
- parent_review = [b for b in parent_detail["blocks"] if b["block_type"] == "review"][0]
|
|
|
- check(parent_review.get("answered") is not True,
|
|
|
- "the top-level parent's own review is never auto-approved")
|
|
|
-
|
|
|
- # All descendants terminal → the suspended parent resumes with the
|
|
|
- # final-review instruction (lift the capability requirement first).
|
|
|
- act(auth, task_state(auth, auto_child)["capabilities"][0]["detach"], {})
|
|
|
- act(auth, task_state(auth, auto_grand)["capabilities"][0]["detach"], {})
|
|
|
-
|
|
|
- def pushes_for(task_id):
|
|
|
- return len([p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == task_id])
|
|
|
-
|
|
|
- before_pushes = pushes_for(auto_parent)
|
|
|
- check(wait_for(lambda: pushes_for(auto_parent) > before_pushes or False,
|
|
|
- timeout=90, what="auto parent resumes") is not None,
|
|
|
- "terminal subtasks resume the suspended parent")
|
|
|
- notice = ""
|
|
|
- for p in fake.start_payloads:
|
|
|
- if int(p.get("task", {}).get("id") or 0) == auto_parent:
|
|
|
- notice = p.get("resume_notice") or ""
|
|
|
- check("All subtasks are complete" in notice
|
|
|
- and "ONE final review" in notice and "saddle review" in notice,
|
|
|
- "resume notice ends with the final-review instruction")
|
|
|
-
|
|
|
-
|
|
|
- print("== phase 1: ask/allocation — token auth + concurrent race ==")
|
|
|
- # Plan 10 phase 1. Registration mints per-saddle tokens; the
|
|
|
- # ask endpoint authenticates by them; concurrent asks race to
|
|
|
- # exactly one winner through the allocation transaction.
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/next-task", None)
|
|
|
- check(status == 401, "next-task rejects a missing token")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-bogus-token")
|
|
|
- check(status == 401, "next-task rejects a wrong token")
|
|
|
- status, body = saddle_api("POST", "/api/saddle/next-task", fake_token)
|
|
|
- check(status == 204, "next-task answers 204 (nothing offerable — daemon sleeps)")
|
|
|
-
|
|
|
- # Two more racers: real daemons on their own ports; fake2 reports
|
|
|
- # its ssh key at /register so the key lands on the row directly.
|
|
|
- fake2 = FakeSaddle()
|
|
|
- fake2.ssh_public_key = "ssh-ed25519 AAAAFake2RacerKey saddle-2@fedora"
|
|
|
- fake2.serve(port=free_port())
|
|
|
- fake3 = FakeSaddle()
|
|
|
- fake3.serve(port=free_port())
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- act(auth, saddles_state["register"], {"name": "racer-two",
|
|
|
- "host": "127.0.0.1:%d" % fake2.port})
|
|
|
- act(auth, saddles_state["register"], {"name": "racer-three",
|
|
|
- "host": "127.0.0.1:%d" % fake3.port})
|
|
|
- check(wait_for(lambda: len(fake2.register_log) > 0 and len(fake3.register_log) > 0
|
|
|
- or False, timeout=15, what="racers registered") is not None,
|
|
|
- "both racers received /register")
|
|
|
- check(wait_for(lambda: "AAAAFake2RacerKey" in
|
|
|
- [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "racer-two"][0].get("ssh_public_key", "") or False,
|
|
|
- timeout=15, what="register-reported key") is not None,
|
|
|
- "the ssh key reported at /register lands on the saddle row")
|
|
|
-
|
|
|
- racer_cards = {}
|
|
|
- for name, daemon in (("fake-saddle", fake), ("racer-two", fake2), ("racer-three", fake3)):
|
|
|
- card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == name][0]
|
|
|
- racer_cards[daemon] = {"id": int(card["id"]), "token": card["token"]}
|
|
|
- # Revocation: a registered saddle's token works until the saddle
|
|
|
- # is removed, then nothing authorises its calls.
|
|
|
- fake4 = FakeSaddle()
|
|
|
- fake4.ask_on_register = False # a pre-flip daemon: never asks
|
|
|
- fake4.serve(port=free_port())
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- act(auth, saddles_state["register"], {"name": "old-daemon",
|
|
|
- "host": "127.0.0.1:%d" % fake4.port})
|
|
|
- old_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "old-daemon"][0]
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
|
|
|
- check(status in (200, 204),
|
|
|
- "the old daemon's fresh token authenticates next-task")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
|
|
|
- {"project": "demo", "body": "from the old daemon"})
|
|
|
- check(status == 200, "note accepts any registered saddle's token")
|
|
|
- act(auth, old_card["unregister"], {})
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
|
|
|
- check(status == 401, "removing the saddle revokes its token")
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
|
|
|
- {"project": "demo", "body": "should fail"})
|
|
|
- check(status == 401, "revocation covers the note endpoint too")
|
|
|
-
|
|
|
- # Deterministic race: settle the queue first (the auto-approve
|
|
|
- # stragglers must finish — their parked resumes stay offerable
|
|
|
- # otherwise and would steal the race's asks), freeze every
|
|
|
- # daemon's self-ask triggers, put all three to sleep (204),
|
|
|
- # then fire one task worth of concurrent asks.
|
|
|
- check(wait_task_done(auth, auto_child, timeout=90) is not None
|
|
|
- and wait_task_done(auth, auto_grand, timeout=90) is not None
|
|
|
- and wait_task_done(auth, auto_parent, timeout=90) is not None,
|
|
|
- "the auto-approve stragglers settle before the race")
|
|
|
- for daemon in (fake, fake2, fake3):
|
|
|
- daemon.ask_on_register = False
|
|
|
- daemon.ask_on_wake = False
|
|
|
- daemon.ask_on_terminal = False
|
|
|
- daemon.ask_on_idle_status = False
|
|
|
- time.sleep(1.5) # let any in-flight triggered asks land
|
|
|
- def queued_count():
|
|
|
- conn = db()
|
|
|
- try:
|
|
|
- return conn.execute(
|
|
|
- "select count(*) from tasks where status = 'queued'").fetchone()[0]
|
|
|
- finally:
|
|
|
- conn.close()
|
|
|
+ return "waits for a capable saddle" in app_log and "requires: gpu" in app_log
|
|
|
+ check(wait_for(capability_wait_logged, timeout=20,
|
|
|
+ what="capability wait log") is not None,
|
|
|
+ "dispatcher logs the capability wait")
|
|
|
+
|
|
|
+ # Attaching the tag to the saddle releases the task.
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ check([c["name"] for c in fake_card["capabilities"]] == ["gpu"],
|
|
|
+ "saddle card shows the attached chip")
|
|
|
+ check(wait_task_done(auth, gpu_task, timeout=60) is not None,
|
|
|
+ "task dispatches once the saddle carries the required tag")
|
|
|
+ queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if int(t["id"]) == gpu_task][0]
|
|
|
+ check(queue.get("waiting_for_saddle") is not True,
|
|
|
+ "wait flag clears once dispatched")
|
|
|
+
|
|
|
+ # Multi-tag: gpu + bigmem required, only gpu carried → waits; adding
|
|
|
+ # bigmem completes the superset. Both requirements ride the create
|
|
|
+ # form (atomic — attaching them one at a time would race a tick
|
|
|
+ # into dispatching on the partial set).
|
|
|
+ act(auth, tasks_state["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Big task",
|
|
|
+ "priority": "2",
|
|
|
+ "capability_id": [str(int(caps["gpu"]["id"])), str(int(caps["bigmem"]["id"]))],
|
|
|
+ })
|
|
|
+ big_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Big task"][0])
|
|
|
+ check(wait_for(lambda: ([t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if int(t["id"]) == big_task][0].get("waiting_for_saddle")
|
|
|
+ is True) or False, timeout=30, what="big task waits") is not None,
|
|
|
+ "task waits when the saddle's tags are not a superset")
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["bigmem"]["id"]))})
|
|
|
+ check(wait_task_done(auth, big_task, timeout=60) is not None,
|
|
|
+ "task dispatches once the superset is complete")
|
|
|
+
|
|
|
+ # Detach a requirement; delete a capability everywhere.
|
|
|
+ detail = task_state(auth, big_task)
|
|
|
+ act(auth, [c for c in detail["capabilities"] if c["name"] == "bigmem"][0]["detach"], {})
|
|
|
+ check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
|
|
|
+ "detached requirement disappears from the task")
|
|
|
+ bigmem_row = [c for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
+ if c["name"] == "bigmem"][0]
|
|
|
+ act(auth, bigmem_row["delete"], {})
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ check(all(c["name"] != "bigmem" for c in saddles_state["capabilities"]),
|
|
|
+ "deleted capability gone from the catalogue")
|
|
|
+ check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
|
|
|
+ "deleting a capability drops its task requirements")
|
|
|
+ check(all(c["name"] != "bigmem"
|
|
|
+ for c in saddle_card(auth, "fake-saddle")["capabilities"]),
|
|
|
+ "deleting a capability drops it from saddles")
|
|
|
+
|
|
|
+ print("== B4: required capabilities on the create form ==")
|
|
|
+ tasks_state = slot_public(auth, "/tasks", "tasks")
|
|
|
+ gpu_option = [o for o in tasks_state.get("capability_options", [])
|
|
|
+ if o["name"] == "gpu"][0]
|
|
|
+ check(len(tasks_state.get("capability_options", [])) >= 1,
|
|
|
+ "queue page offers capability options in the create form")
|
|
|
|
|
|
- check(wait_for(lambda: queued_count() == 0 or False,
|
|
|
- timeout=60, what="queue settles before the race") is not None,
|
|
|
- "the queue settles before the race")
|
|
|
+ conn = db()
|
|
|
+ before = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ act(auth, tasks_state["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Bad cap task",
|
|
|
+ "capability_id": [gpu_option["id"], "999"],
|
|
|
+ })
|
|
|
+ conn = db()
|
|
|
+ after = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(after == before, "unknown capability id rejects the create")
|
|
|
+
|
|
|
+ act(auth, tasks_state["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Cap-on-create task",
|
|
|
+ "capability_id": [gpu_option["id"]],
|
|
|
+ })
|
|
|
+ cap_create = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Cap-on-create task"][0])
|
|
|
+ detail = task_state(auth, cap_create)
|
|
|
+ check([c["name"] for c in detail["capabilities"]] == ["gpu"],
|
|
|
+ "capabilities from the create form land on the task")
|
|
|
+ queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if int(t["id"]) == cap_create][0]
|
|
|
+ check(queue["capabilities"] == ["gpu"],
|
|
|
+ "queue row shows the created task's requirements")
|
|
|
+ # The waiting flag must hold it back until the capable saddle frees
|
|
|
+ # up (the fake saddle still carries gpu from the routing checks).
|
|
|
+ check(queue.get("waiting_for_saddle") is not True,
|
|
|
+ "capable saddle available: no wait flag on the new task")
|
|
|
+
|
|
|
+ print("== retry + resume ==")
|
|
|
+ # A daemon-side failure fails the task with its explanation.
|
|
|
+ fake.fail_next = True
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Flaky task",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ flaky = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Flaky task"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, flaky).get("status") == "failed" or False,
|
|
|
+ timeout=60, what="flaky task fails") is not None,
|
|
|
+ "daemon failure fails the task")
|
|
|
+ detail = task_state(auth, flaky)
|
|
|
+ check("bash -n rejected" in (detail.get("summary") or ""),
|
|
|
+ "failure summary shows the daemon's explanation")
|
|
|
+
|
|
|
+ # Retry: summary moves into a comment and clears, task requeues,
|
|
|
+ # history stays, and the re-dispatch restores the workspace.
|
|
|
+ comments_before = len(detail.get("comments", []))
|
|
|
+ progress_before = len(detail.get("progress", []))
|
|
|
+ act(auth, detail["retry"], {})
|
|
|
+ detail = task_state(auth, flaky)
|
|
|
+ check(detail["status"] == "queued" and (detail.get("summary") or "") == "",
|
|
|
+ "retry requeues the task and clears the stale summary")
|
|
|
+ check(any("bash -n rejected" in c["body"] for c in detail["comments"]),
|
|
|
+ "failure explanation kept as a bot comment")
|
|
|
+ check(any(c["body"] == "Task retried by user" for c in detail["comments"]),
|
|
|
+ "retry lands its audit comment")
|
|
|
+ check(len(detail["comments"]) >= comments_before + 1,
|
|
|
+ "retry keeps the comment history")
|
|
|
+ pushes_before = len([p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == flaky])
|
|
|
+ check(wait_for(lambda: len([p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == flaky])
|
|
|
+ > pushes_before or False, timeout=60,
|
|
|
+ what="flaky task re-dispatched") is not None,
|
|
|
+ "retried task dispatches again")
|
|
|
+ # The resumed arc surfaces on the next /status read — in the
|
|
|
+ # push world that is the operator's manual refresh.
|
|
|
+ act(auth, saddle_card(auth, "fake-saddle")["refresh"], {})
|
|
|
+ check(wait_task_done(auth, flaky, timeout=60) is not None,
|
|
|
+ "retried task completes on the restored workspace")
|
|
|
+ detail = task_state(auth, flaky)
|
|
|
+ check(any("restored the session context" in p["summary"]
|
|
|
+ for p in detail.get("progress", [])),
|
|
|
+ "daemon resume reported on the first status after the restore")
|
|
|
+ check(len(detail["progress"]) > progress_before,
|
|
|
+ "resume note appended to the progress timeline")
|
|
|
+
|
|
|
+ # Manual resume: a task stuck working on an offline saddle.
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Stuck task",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ stuck = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Stuck task"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, stuck).get("status") == "working" or False,
|
|
|
+ timeout=60, what="stuck task starts working") is not None,
|
|
|
+ "stuck task reaches working")
|
|
|
+ # Deterministic claim for the resume leg below: the round's end,
|
|
|
+ # the refresh probe AND the dispatcher's wake for the requeued task
|
|
|
+ # all trigger self-asks that could steal it — freeze every trigger
|
|
|
+ # until the manual ask below has made the claim.
|
|
|
+ fake.ask_on_terminal = False
|
|
|
+ fake.ask_on_idle_status = False
|
|
|
+ fake.ask_on_wake = False
|
|
|
+ fake.settle_asks()
|
|
|
+ conn = db()
|
|
|
+ conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ detail = task_state(auth, stuck)
|
|
|
+ check(detail.get("resume_available") is True,
|
|
|
+ "task page offers resume while its saddle is offline")
|
|
|
+ # The resume POSTs task-update cancel to the daemon; the abort
|
|
|
+ # handler fires an ungated re-ask that would race the manual claim
|
|
|
+ # below — slow it past the manual ask (the knob the slow-ask
|
|
|
+ # regression uses).
|
|
|
+ fake.ask_delay = 5.0
|
|
|
+ act(auth, detail["resume"], {})
|
|
|
+ detail = task_state(auth, stuck)
|
|
|
+ check(detail["status"] == "queued",
|
|
|
+ "manual resume requeues the stuck task")
|
|
|
+ check(any(c["body"] == "Saddle became unreachable; task requeued"
|
|
|
+ for c in detail["comments"]),
|
|
|
+ "manual resume lands the interruption comment")
|
|
|
+ check(all(s["status"] == "offline"
|
|
|
+ for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
|
|
|
+ "saddle stays offline after the manual requeue")
|
|
|
+
|
|
|
+ # Bringing the saddle back lets the requeued task resume and finish.
|
|
|
+ fake.state = "idle"
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ act(auth, fake_card["refresh"], {})
|
|
|
+ check(any(s["status"] == "idle"
|
|
|
+ for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "fake-saddle"),
|
|
|
+ "refreshed saddle returns idle")
|
|
|
+ # The refreshed daemon is idle-awake: it asks (the real loop's
|
|
|
+ # own ask; driven directly for determinism — every self-ask trigger
|
|
|
+ # is frozen and the abort re-ask is slowed past it; joining that
|
|
|
+ # delayed thread here would fire its ask first, so it is left to
|
|
|
+ # sleep out the rest of the bundle).
|
|
|
+ status, payload = fake.ask_now()
|
|
|
+ fake.ask_on_idle_status = True
|
|
|
+ fake.ask_on_terminal = True
|
|
|
+ fake.ask_on_wake = True
|
|
|
+ fake.ask_delay = 0.0
|
|
|
+ check(status == 200, "the refreshed daemon's ask claims the requeued task")
|
|
|
+ check(wait_task_done(auth, stuck, timeout=60) is not None,
|
|
|
+ "requeued task resumes on the refreshed saddle and completes")
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# Bundle: subtasks — suspension parking, the subtask status push (with
|
|
|
+# its best-effort retry), the synchronous subtask callback, and the
|
|
|
+# settling waits for the parked parents this shard created.
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+def bundle_subtasks(ctx):
|
|
|
+ auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
|
|
|
+ app = ctx.app
|
|
|
+
|
|
|
+ # A hold tag nothing carries: R3's second child stays queued (a
|
|
|
+ # stable parked session for the failed-delivery leg) until released.
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "gpu"})
|
|
|
+
|
|
|
+ print("== B7: suspended state + per-subtask resume ==")
|
|
|
+ fake.suspend_on_title = "Suspend parent"
|
|
|
+ pushes_before = len(fake.start_payloads)
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Suspend parent",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ parent7 = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Suspend parent"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, parent7).get("status") == "suspended"
|
|
|
+ or False, timeout=60, what="parent parks as suspended") is not None,
|
|
|
+ "daemon 'suspended' state parks the task")
|
|
|
+ detail = task_state(auth, parent7)
|
|
|
+ check(detail.get("display") == "waiting on subtasks",
|
|
|
+ "suspended task displays as waiting on subtasks")
|
|
|
+ check(any(s["status"] == "idle"
|
|
|
+ for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
|
|
|
+ "suspension frees the saddle")
|
|
|
+ first_push = fake.start_payloads[pushes_before]
|
|
|
+ check(first_push.get("subtasks") == [],
|
|
|
+ "the /start payload carries the subtasks array (empty here)")
|
|
|
+
|
|
|
+ # Two subtask events land while parked (staged, not dispatched):
|
|
|
+ # both must coalesce into one resume with a full report.
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
+ " summary, completed_at, created_at) values (1, 'Child one', 'done', 5, ?,"
|
|
|
+ " 'built and verified the flange', strftime('%s','now','+1 second'),"
|
|
|
+ " strftime('%s','now'))", (parent7,))
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
+ " summary, completed_at, created_at) values (1, 'Child two', 'failed', 5, ?,"
|
|
|
+ " 'the harness refused to start', strftime('%s','now','+1 second'),"
|
|
|
+ " strftime('%s','now'))", (parent7,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ def parent_push_count():
|
|
|
+ return len([p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == parent7])
|
|
|
+
|
|
|
+ check(wait_for(lambda: parent_push_count() >= 2 or False,
|
|
|
+ timeout=45, what="suspended parent resumes") is not None,
|
|
|
+ "subtask events resume the suspended parent")
|
|
|
+ parent_pushes = [p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == parent7]
|
|
|
+ check(len(parent_pushes) >= 2,
|
|
|
+ "the parent's resume is a fresh /start for the same task")
|
|
|
+ resume_push = parent_pushes[-1]
|
|
|
+ notice = resume_push.get("resume_notice") or ""
|
|
|
+ check("Child one" in notice and "completed" in notice
|
|
|
+ and "built and verified the flange" in notice,
|
|
|
+ "resume notice reports the finished subtask with its summary")
|
|
|
+ check("Child two" in notice and "failed" in notice
|
|
|
+ and "the harness refused to start" in notice,
|
|
|
+ "coalesced: the second event rides the same resume notice")
|
|
|
+ subtask_entries = {(s["title"], s["status"]) for s in resume_push.get("subtasks", [])}
|
|
|
+ check(("Child one", "done") in subtask_entries
|
|
|
+ and ("Child two", "failed") in subtask_entries,
|
|
|
+ "subtasks array carries the children's states at dispatch time")
|
|
|
+ # The resumed arc surfaces on a /status read while the resumed
|
|
|
+ # round still runs — the operator's manual refresh in the push
|
|
|
+ # world.
|
|
|
+ act(auth, saddle_card(auth, "fake-saddle")["refresh"], {})
|
|
|
+ check(wait_task_done(auth, parent7, timeout=60) is not None,
|
|
|
+ "resumed parent completes")
|
|
|
+ detail = task_state(auth, parent7)
|
|
|
+ check(any("restored the session context" in p["summary"]
|
|
|
+ for p in detail.get("progress", [])),
|
|
|
+ "resumed flag reported on the first status after the restore")
|
|
|
+
|
|
|
+ print("== R3 B1: subtask status push ==")
|
|
|
+ # A parked (suspended) parent gives delivery a stable open session.
|
|
|
+ fake.suspend_on_title = "Push parent"
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Push parent",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ push_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Push parent"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, push_parent).get("status") == "suspended"
|
|
|
+ or False, timeout=60, what="push parent parks") is not None,
|
|
|
+ "push parent parks with an open session")
|
|
|
+
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Push child",
|
|
|
+ "parent_task_id": str(push_parent),
|
|
|
+ })
|
|
|
+ push_child = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Push child"][0])
|
|
|
+
|
|
|
+ def events_for(subtask_id):
|
|
|
+ return [e for e in fake.subtask_status_events
|
|
|
+ if int(e.get("subtask_id") or 0) == subtask_id]
|
|
|
+
|
|
|
+ check(wait_for(lambda: len(events_for(push_child)) > 0 or False,
|
|
|
+ timeout=30, what="queued subtask event") is not None,
|
|
|
+ "subtask creation pushes a queued status event")
|
|
|
+ first_event = events_for(push_child)[0]
|
|
|
+ check(int(first_event.get("parent_task_id") or 0) == push_parent
|
|
|
+ and first_event.get("title") == "Push child"
|
|
|
+ and first_event.get("status") in ("queued", "working"),
|
|
|
+ "status event carries parent id, title and a wire status (%s)"
|
|
|
+ % first_event.get("status"))
|
|
|
+
|
|
|
+ # Real lifecycle: the child dispatches (working) and completes (done
|
|
|
+ # with summary) — the parent's daemon hears every transition.
|
|
|
+ check(wait_for(lambda: any(e.get("status") == "done"
|
|
|
+ for e in events_for(push_child)) or False,
|
|
|
+ timeout=90, what="done subtask event") is not None,
|
|
|
+ "subtask completion pushes the done transition")
|
|
|
+ check(any(e.get("status") == "working" for e in events_for(push_child)),
|
|
|
+ "dispatch pushes a working status event")
|
|
|
+
|
|
|
+ # Best-effort retry: while the daemon 500s, events stay queued and
|
|
|
+ # land once it recovers. The parent's resumed run re-parks (the
|
|
|
+ # ask-flow resume races this section otherwise: its completion
|
|
|
+ # would close the parked session the events are delivered to).
|
|
|
+ fake.fail_subtask_status = True
|
|
|
+ fake.suspend_on_title = "Push parent"
|
|
|
+ # The child carries the hold tag (nothing can satisfy it) so it
|
|
|
+ # stays queued: its status events keep targeting the parent's
|
|
|
+ # stable parked session until the hold is released below.
|
|
|
+ gpu_cap_id = [c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
+ if c["name"] == "gpu"][0]
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Push child two",
|
|
|
+ "parent_task_id": str(push_parent),
|
|
|
+ "capability_id": [str(gpu_cap_id)],
|
|
|
+ })
|
|
|
+ push_child_two = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Push child two"][0])
|
|
|
+ # Converted sleep(7): wait until the dispatcher has actually
|
|
|
+ # attempted (and failed) the delivery twice — two ticks' worth of
|
|
|
+ # evidence the events stay queued — instead of sleeping blindly.
|
|
|
+ check(wait_for(lambda: fake.subtask_fail_hits >= 2 or False,
|
|
|
+ timeout=15, what="failed subtask-event deliveries") is not None
|
|
|
+ and len(events_for(push_child_two)) == 0,
|
|
|
+ "failed pushes stay queued (no event delivered)")
|
|
|
+ fake.fail_subtask_status = False
|
|
|
+ check(wait_for(lambda: len(events_for(push_child_two)) > 0 or False,
|
|
|
+ timeout=30, what="retried subtask event") is not None,
|
|
|
+ "recovered pushes deliver on a later tick")
|
|
|
+ # Release the hold: the child dispatches and completes; the settle
|
|
|
+ # waits at the end of this bundle cover the parent's final resume.
|
|
|
+ act(auth, task_state(auth, push_child_two)["capabilities"][0]["detach"], {})
|
|
|
+
|
|
|
+ print("== R4 B1: synchronous subtask callback ==")
|
|
|
+ # Plan 10: the subtask callback is per-saddle-token authenticated.
|
|
|
+ r4_token = fake_token
|
|
|
+
|
|
|
+ # Parent with a live parked session (suspended) hosts the callback.
|
|
|
+ fake.suspend_on_title = "Callback parent"
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Callback parent",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ cb_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Callback parent"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, cb_parent).get("status") == "suspended"
|
|
|
+ or False, timeout=60, what="callback parent parks") is not None,
|
|
|
+ "callback parent parks")
|
|
|
|
|
|
- sleep_statuses = [daemon.ask_now()[0] for daemon in (fake, fake2, fake3)]
|
|
|
- check(sleep_statuses == [204, 204, 204],
|
|
|
- "all three racers sleep (204) before the task exists")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/subtask", None,
|
|
|
+ {"parent_task_id": cb_parent, "title": "no token"})
|
|
|
+ check(status == 401, "subtask callback rejects a missing token")
|
|
|
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Race task",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- race_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Race task"][0])
|
|
|
-
|
|
|
- race_results = []
|
|
|
- def race_ask(daemon):
|
|
|
- # A real racing daemon accepts the assignment it wins — the
|
|
|
- # manual ask path does exactly that on a 200.
|
|
|
- race_results.append(daemon.ask_now())
|
|
|
- threads = [threading.Thread(target=race_ask, args=(daemon,))
|
|
|
- for daemon in (fake, fake2, fake3)]
|
|
|
- for thread in threads:
|
|
|
- thread.start()
|
|
|
- for thread in threads:
|
|
|
- thread.join()
|
|
|
- race_statuses = sorted(r[0] for r in race_results)
|
|
|
- check(race_statuses == [200, 204, 204],
|
|
|
- "concurrent asks race to exactly one winner (%s)" % race_statuses)
|
|
|
- winner = [r for r in race_results if r[0] == 200][0][1]
|
|
|
- check(int(winner["task"]["id"]) == race_task, "the winner's payload carries the task")
|
|
|
- check(int(winner["session_id"]) > 0, "the assignment carries the session id")
|
|
|
- conn = db()
|
|
|
- race_sessions = conn.execute(
|
|
|
- "select count(*) from sessions where task_id = ?", (race_task,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(race_sessions == 1,
|
|
|
- "exactly one session row — the transaction serialised the claims (%d)"
|
|
|
- % race_sessions)
|
|
|
- check(wait_task_done(auth, race_task, timeout=60) is not None,
|
|
|
- "the raced task completes on the winning daemon")
|
|
|
- for daemon in (fake, fake2, fake3):
|
|
|
- daemon.ask_on_register = True
|
|
|
- daemon.ask_on_wake = True
|
|
|
- daemon.ask_on_terminal = True
|
|
|
- daemon.ask_on_idle_status = True
|
|
|
- # Re-establish sleeping explicitly: the winner's post-race
|
|
|
- # re-ask timing is not deterministic, and the affinity section
|
|
|
- # below needs every racer provably asleep. Any stray offerable
|
|
|
- # drains into these asks first.
|
|
|
- def all_asleep():
|
|
|
- return all(daemon.ask_now()[0] == 204 for daemon in (fake, fake2, fake3))
|
|
|
- check(wait_for(lambda: all_asleep() or False, timeout=30,
|
|
|
- what="racers re-sleep after the race") is not None,
|
|
|
- "all racers re-sleep after the race")
|
|
|
-
|
|
|
- print("== phase 1: workspace affinity pins parked tasks ==")
|
|
|
- # A suspended parent parked on racer-two may be resumed ONLY by
|
|
|
- # racer-two: a tag-capable non-affine saddle asking gets 204, and
|
|
|
- # the wake targets the affine saddle. Only racer-two carries the
|
|
|
- # tag at dispatch time (the race's winner is nondeterministic, so
|
|
|
- # the park must not depend on which racers are free); racer-three
|
|
|
- # gains it afterwards to prove the pinning is by workspace, not
|
|
|
- # tags.
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- act(auth, saddles_state["create_capability"], {"name": "affinity-tag"})
|
|
|
- affinity_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
- if c["name"] == "affinity-tag"][0])
|
|
|
- racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "racer-two"][0]
|
|
|
- racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "racer-three"][0]
|
|
|
- act(auth, racer2_card["attach_capability"], {"capability_id": str(affinity_cap)})
|
|
|
-
|
|
|
- fake2.suspend_on_title = "Affinity parent"
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Affinity parent",
|
|
|
- "priority": "2",
|
|
|
- "capability_id": [str(affinity_cap)],
|
|
|
- })
|
|
|
- affinity_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Affinity parent"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, affinity_parent).get("status") == "suspended"
|
|
|
- or False, timeout=60, what="affinity parent parks") is not None,
|
|
|
- "affinity parent parks (suspended) on the tag-carrying racer")
|
|
|
- conn = db()
|
|
|
- affinity_session = conn.execute(
|
|
|
- "select saddle_id from sessions where task_id = ? and ended_at is null",
|
|
|
- (affinity_parent,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(affinity_session == int(racer2_card["id"]),
|
|
|
- "the parked workspace sits on racer-two (the only tag-carrier)")
|
|
|
-
|
|
|
- # Now racer-three carries the tag too — the pinning is by
|
|
|
- # workspace, not tags.
|
|
|
-
|
|
|
- # A terminal subtask makes the parked task resume-offerable —
|
|
|
- # affine to racer-two only. The park is observed the moment the
|
|
|
- # suspended push lands (no poller lag to compensate), so the
|
|
|
- # child is inserted only after racer-two's ask triggers are
|
|
|
- # frozen and the wake measurement armed — its completion stamped
|
|
|
- # one second ahead so completed_at > suspended_at holds.
|
|
|
- fake2.ask_on_wake = False
|
|
|
- fake2.ask_on_idle_status = False
|
|
|
- wakes_before_2 = len(fake2.wake_log)
|
|
|
- time.sleep(1.2) # any in-flight park-time trigger ask lands (204)
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
+ " values (1, 'Anchor task', 'done', 5, strftime('%s','now'))")
|
|
|
+ anchor = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
+ "parent_task_id": cb_parent,
|
|
|
+ "title": "Wire the flange",
|
|
|
+ "description": "torque to spec",
|
|
|
+ "depends_on": [anchor],
|
|
|
+ "session_id": 1,
|
|
|
+ })
|
|
|
+ check(status == 200 and body.get("ok") is True and int(body.get("task_id") or 0) > 0,
|
|
|
+ "callback creates the subtask synchronously")
|
|
|
+ subtask_id = int(body.get("task_id"))
|
|
|
+ detail = task_state(auth, subtask_id)
|
|
|
+ check(detail["title"] == "Wire the flange" and int(detail["project_id"]) == 1
|
|
|
+ and int(detail["parent_task_id"]) == cb_parent,
|
|
|
+ "subtask row inherits the parent's project and linkage")
|
|
|
+ check(detail.get("task_branch") == task_state(auth, cb_parent).get("task_branch"),
|
|
|
+ "subtask shares the parent's task branch")
|
|
|
+ conn = db()
|
|
|
+ dep_rows = conn.execute(
|
|
|
+ "select depends_on_task_id from task_dependencies where task_id = ?",
|
|
|
+ (subtask_id,)).fetchall()
|
|
|
+ conn.close()
|
|
|
+ check([r[0] for r in dep_rows] == [anchor],
|
|
|
+ "depends_on wired as task_dependencies")
|
|
|
+
|
|
|
+ # Idempotency + entry-poll fallback dedup.
|
|
|
+ status, body2 = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
+ "parent_task_id": cb_parent, "title": "Wire the flange"})
|
|
|
+ check(status == 200 and int(body2.get("task_id") or 0) == subtask_id,
|
|
|
+ "retried callback returns the same task id")
|
|
|
+ conn = db()
|
|
|
+ before = conn.execute("select count(*) from tasks where parent_task_id = ?",
|
|
|
+ (cb_parent,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ conn = db()
|
|
|
+ cb_session = conn.execute(
|
|
|
+ "select id from sessions where task_id = ? and ended_at is null",
|
|
|
+ (cb_parent,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "subtask", "session_id": cb_session, "data": {
|
|
|
+ "title": "Wire the flange", "description": "again"}})
|
|
|
+ conn = db()
|
|
|
+ after = conn.execute("select count(*) from tasks where parent_task_id = ?",
|
|
|
+ (cb_parent,)).fetchone()[0]
|
|
|
+ timeline = conn.execute(
|
|
|
+ "select count(*) from session_commands where session_id = ?"
|
|
|
+ " and type = 'subtask'", (cb_session,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(after == before,
|
|
|
+ "entry-poll fallback dedupes against the callback's row")
|
|
|
+ check(timeline >= 1, "timeline entry recorded on the parent's session")
|
|
|
+
|
|
|
+ # Validation: unknown parent, unknown depends_on, ancestor cycle.
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
+ {"parent_task_id": 99999, "title": "ghost"})
|
|
|
+ check(status == 400, "unknown parent rejected")
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
+ {"parent_task_id": cb_parent, "title": "bad dep",
|
|
|
+ "depends_on": [99999]})
|
|
|
+ check(status == 400 and "unknown task" in body.get("error", ""),
|
|
|
+ "unknown depends_on id rejected with a clear error")
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
+ {"parent_task_id": cb_parent, "title": "cycle dep",
|
|
|
+ "depends_on": [cb_parent]})
|
|
|
+ check(status == 400 and "cycle" in body.get("error", ""),
|
|
|
+ "depending on an ancestor rejected as a cycle")
|
|
|
+
|
|
|
+ print("== settle: this shard's parked parents resume and finish ==")
|
|
|
+ # Settle earlier stragglers first: the round-3/B1 suspended parents
|
|
|
+ # resume (a fresh /start) once their last subtask completes — those
|
|
|
+ # pushes would consume the one-shot suspend flag below.
|
|
|
+ check(wait_task_done(auth, push_parent, timeout=90) is not None
|
|
|
+ and wait_task_done(auth, push_child_two, timeout=90) is not None
|
|
|
+ and wait_task_done(auth, cb_parent, timeout=90) is not None,
|
|
|
+ "earlier suspended parents settle before the auto-approve run")
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# Bundle: lineage — subtask auto-approve (transitive coverage, the
|
|
|
+# toggle's descendant-only scope, the final resume), /start parent
|
|
|
+# context and the daemon task-reading API.
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+def bundle_lineage(ctx):
|
|
|
+ auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
|
|
|
+ app = ctx.app
|
|
|
+ r4_token = fake_token
|
|
|
+ # gpu exists only as a hold (no saddle carries it): the auto-approve
|
|
|
+ # children stay queued until the final-resume leg lifts it.
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "gpu"})
|
|
|
+
|
|
|
+ # Parent parks; its toggle then covers all descendants. Children are
|
|
|
+ # created with an unmet 'gpu' requirement so they stay queued (and
|
|
|
+ # untouched) until the requirement is lifted for the final-resume leg.
|
|
|
+ fake.suspend_on_title = "Auto parent"
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Auto parent",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ auto_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Auto parent"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, auto_parent).get("status") == "suspended"
|
|
|
+ or False, timeout=60, what="auto parent parks") is not None,
|
|
|
+ "auto parent parks")
|
|
|
+
|
|
|
+ detail = task_state(auth, auto_parent)
|
|
|
+ check(detail.get("auto_approve") is False,
|
|
|
+ "auto-approve defaults to off")
|
|
|
+ act(auth, detail["toggle_auto_approve"], {})
|
|
|
+ check(task_state(auth, auto_parent).get("auto_approve") is True,
|
|
|
+ "toggle turns auto-approve on")
|
|
|
+
|
|
|
+ gpu_option = int([o["id"] for o in slot_public(auth, "/tasks", "tasks")["capability_options"]
|
|
|
+ if o["name"] == "gpu"][0])
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
+ {"parent_task_id": auto_parent,
|
|
|
+ "title": "Auto child", "depends_on": []})
|
|
|
+ auto_child = int(body["task_id"])
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
|
|
|
+ {"parent_task_id": auto_child, "title": "Auto grandchild"})
|
|
|
+ auto_grand = int(body["task_id"])
|
|
|
+ # Fixture: one more held subtask whose title the task-reading API's
|
|
|
+ # q= search matches below (this bundle's "Wire the flange").
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
+ "parent_task_id": auto_parent, "title": "Wire the flange",
|
|
|
+ "description": "torque to spec"})
|
|
|
+ flange_task = int(body["task_id"])
|
|
|
+ act(auth, task_state(auth, auto_child)["attach_capability"],
|
|
|
+ {"capability_id": str(gpu_option)})
|
|
|
+ act(auth, task_state(auth, auto_grand)["attach_capability"],
|
|
|
+ {"capability_id": str(gpu_option)})
|
|
|
+ act(auth, task_state(auth, flange_task)["attach_capability"],
|
|
|
+ {"capability_id": str(gpu_option)})
|
|
|
+
|
|
|
+ # Transitive coverage: the GRANDCHILD's review is auto-answered
|
|
|
+ # (any ancestor set), answered-by 'auto', timeline entry recorded.
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at)"
|
|
|
+ " values (?, ?, strftime('%s','now'))", (auto_grand, fake_id))
|
|
|
+ conn.commit()
|
|
|
+ grand_session = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "review", "session_id": grand_session, "data": {
|
|
|
+ "summary": "the flange is wired"}})
|
|
|
+ grand_detail = task_state(auth, auto_grand)
|
|
|
+ grand_review = [b for b in grand_detail["blocks"] if b["block_type"] == "review"][0]
|
|
|
+ check(grand_review.get("answered") is True
|
|
|
+ and grand_review.get("answer") == "Auto-approved (subtask auto-approve is on)"
|
|
|
+ and grand_review.get("answered_by") == "auto",
|
|
|
+ "covered grandchild review auto-answered as 'auto'")
|
|
|
+ conn = db()
|
|
|
+ timeline_auto = conn.execute(
|
|
|
+ "select count(*) from session_commands where session_id = ?"
|
|
|
+ " and type = 'auto-approve'", (grand_session,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(timeline_auto >= 1, "auto action visible in the timeline")
|
|
|
+
|
|
|
+ # Questions and RFCs still block for a human.
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "block", "session_id": grand_session, "data": {
|
|
|
+ "question": "Which torque spec?", "type": "question"}})
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "block", "session_id": grand_session, "data": {
|
|
|
+ "question": "RFC: approve the design", "type": "rfc",
|
|
|
+ "body": "# Design"}})
|
|
|
+ grand_detail = task_state(auth, auto_grand)
|
|
|
+ check(all(b.get("answered") is not True for b in grand_detail["blocks"]
|
|
|
+ if b["block_type"] in ("question", "rfc")),
|
|
|
+ "questions and RFCs are never auto-answered")
|
|
|
+
|
|
|
+ # The toggle covers DESCENDANTS only: the parent's own review waits.
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
|
|
|
+ " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
|
|
|
+ " 'done')", (auto_parent, fake_id))
|
|
|
+ conn.commit()
|
|
|
+ parent_session = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "review", "session_id": parent_session, "data": {
|
|
|
+ "summary": "the whole change"}})
|
|
|
+ parent_detail = task_state(auth, auto_parent)
|
|
|
+ parent_review = [b for b in parent_detail["blocks"] if b["block_type"] == "review"][0]
|
|
|
+ check(parent_review.get("answered") is not True,
|
|
|
+ "the top-level parent's own review is never auto-approved")
|
|
|
+
|
|
|
+ # All descendants terminal → the suspended parent resumes with the
|
|
|
+ # final-review instruction. The resume fires per completion (ANY
|
|
|
+ # terminal child makes the parent offerable again), so release the
|
|
|
+ # held descendants one at a time and re-park the parent between
|
|
|
+ # releases: only the last resume sees every descendant terminal,
|
|
|
+ # and its notice then carries the final-review instruction.
|
|
|
+ for step, held_task in enumerate((auto_child, auto_grand, flange_task)):
|
|
|
+ if step < 2:
|
|
|
+ fake.suspend_on_title = "Auto parent" # interim resumes re-park
|
|
|
+ act(auth, task_state(auth, held_task)["capabilities"][0]["detach"], {})
|
|
|
+ check(wait_task_done(auth, held_task, timeout=90) is not None,
|
|
|
+ "released descendant settles before the next release (step %d)" % step)
|
|
|
+
|
|
|
+ def pushes_for(task_id):
|
|
|
+ return len([p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == task_id])
|
|
|
+
|
|
|
+ before_pushes = pushes_for(auto_parent)
|
|
|
+ check(wait_for(lambda: pushes_for(auto_parent) > before_pushes or False,
|
|
|
+ timeout=90, what="auto parent resumes") is not None,
|
|
|
+ "terminal subtasks resume the suspended parent")
|
|
|
+ notice = ""
|
|
|
+ for p in fake.start_payloads:
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == auto_parent:
|
|
|
+ notice = p.get("resume_notice") or ""
|
|
|
+ check("All subtasks are complete" in notice
|
|
|
+ and "ONE final review" in notice and "saddle review" in notice,
|
|
|
+ "resume notice ends with the final-review instruction")
|
|
|
+
|
|
|
+ print("== R5 B1: /start parent context ==")
|
|
|
+ # A fresh subtask dispatch carries the parent object (plan clamped).
|
|
|
+ conn = db()
|
|
|
+ conn.execute("update tasks set plan = ? where id = ?",
|
|
|
+ ("# Parent plan\n\n" + "step. " * 1200, auto_parent))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ # Barrier: the dispatch reads the parent row through the app's own
|
|
|
+ # connection — wait until the app serves the externally written
|
|
|
+ # plan before creating the dispatchable task (a stale snapshot
|
|
|
+ # would otherwise ship an empty parent plan).
|
|
|
+ check(wait_for(lambda: "Parent plan" in
|
|
|
+ (((saddle_api("GET", "/api/saddle/task/%d" % auto_parent,
|
|
|
+ r4_token)[1].get("task")) or {}).get("plan") or "")
|
|
|
+ or False, timeout=15, what="app serves the parent plan") is not None,
|
|
|
+ "the task API serves the parent plan before the lineage dispatch")
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Lineage subtask",
|
|
|
+ "priority": "3",
|
|
|
+ })
|
|
|
+ lineage_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Lineage subtask"][0])
|
|
|
+ conn = db()
|
|
|
+ conn.execute("update tasks set parent_task_id = ?, description = 'torque to spec'"
|
|
|
+ " where id = ?", (auto_parent, lineage_task))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == lineage_task
|
|
|
+ for p in fake.start_payloads) or False,
|
|
|
+ timeout=60, what="lineage subtask dispatches") is not None,
|
|
|
+ "lineage subtask dispatches")
|
|
|
+ lineage_push = [p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == lineage_task][-1]
|
|
|
+ parent_obj = lineage_push.get("parent")
|
|
|
+ check(parent_obj is not None
|
|
|
+ and int(parent_obj.get("id") or 0) == auto_parent
|
|
|
+ and parent_obj.get("title") == "Auto parent"
|
|
|
+ and parent_obj.get("status") in ("suspended", "working", "done"),
|
|
|
+ "subtask /start carries the parent object")
|
|
|
+ check(parent_obj.get("description") == "torque to spec"
|
|
|
+ or "Parent plan" in (parent_obj.get("plan") or ""),
|
|
|
+ "parent description/plan content flows through")
|
|
|
+ check(len(parent_obj.get("plan") or "") <= 4000,
|
|
|
+ "parent plan clamped to ~4000 chars (%d)"
|
|
|
+ % len(parent_obj.get("plan") or ""))
|
|
|
+ root_pushes = [p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == auto_parent]
|
|
|
+ check(all("parent" not in p for p in root_pushes),
|
|
|
+ "root task /start carries no parent object")
|
|
|
+
|
|
|
+ print("== R5 B2: task-reading API ==")
|
|
|
+ status, _ = saddle_api("GET", "/api/saddle/tasks", None)
|
|
|
+ check(status == 401, "tasks endpoint rejects a missing token")
|
|
|
+ status, _ = saddle_api("GET", "/api/saddle/task/%d" % auto_child, None)
|
|
|
+ check(status == 401, "task detail endpoint rejects a missing token")
|
|
|
+
|
|
|
+ # Fixture: a second project whose tasks all carry the "Diff " title
|
|
|
+ # prefix (this bundle's own copy of the pages bundle's diff
|
|
|
+ # projects) for the project_id filter checks — created first so the
|
|
|
+ # unfiltered list has the population the count check expects.
|
|
|
+ state = slot_public(auth, "/projects", "projects")
|
|
|
+ act(auth, state["create"], {"name": "diff-fabrica",
|
|
|
+ "git_url": "ssh://fabrica.unitatem.net:222/clanker/clanker-saddle.git"})
|
|
|
+ conn = db()
|
|
|
+ other_project = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
|
|
|
+ if p["name"] == "diff-fabrica"][0])
|
|
|
+ for suffix in ("fabrica", "scp"):
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
+ " values (?, ?, 'done', 5, strftime('%s','now'))",
|
|
|
+ (other_project, "Diff %s" % suffix))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/tasks", r4_token)
|
|
|
+ check(status == 200 and len(body.get("tasks", [])) > 5,
|
|
|
+ "tasks endpoint lists tasks without filters")
|
|
|
+ row = [t for t in body["tasks"] if int(t["id"]) == auto_child][0]
|
|
|
+ check(row["title"] == "Auto child" and int(row["parent_task_id"]) == auto_parent
|
|
|
+ and row["status"] in ("done", "queued", "working"),
|
|
|
+ "compact rows carry id/title/status/parent_task_id")
|
|
|
+
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/tasks?project_id=1", r4_token)
|
|
|
+ check(all(True for _ in [1]) # project 1 dominates the data set
|
|
|
+ and not any(int(t["id"]) == 0 for t in body["tasks"]),
|
|
|
+ "project_id filter accepted")
|
|
|
+ status, body = saddle_api("GET",
|
|
|
+ "/api/saddle/tasks?project_id=%d" % other_project, r4_token)
|
|
|
+ check(all(t["title"].startswith("Diff") for t in body["tasks"]),
|
|
|
+ "other project lists only its own tasks")
|
|
|
+
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/tasks?q=FLANGE", r4_token)
|
|
|
+ titles = [t["title"] for t in body["tasks"]]
|
|
|
+ check("Wire the flange" in titles,
|
|
|
+ "q matches titles case-insensitively")
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/tasks?q=torque%20to%20spec", r4_token)
|
|
|
+ check(any(t["title"] == "Lineage subtask" for t in body["tasks"]),
|
|
|
+ "q matches description text")
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/tasks?q=parent%20plan", r4_token)
|
|
|
+ check(any(t["title"] == "Auto parent" for t in body["tasks"]),
|
|
|
+ "q matches plan text")
|
|
|
+
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/task/%d" % auto_child, r4_token)
|
|
|
+ task_view = body.get("task", {})
|
|
|
+ check(status == 200 and int(task_view.get("id") or 0) == auto_child
|
|
|
+ and task_view.get("parent_title") == "Auto parent"
|
|
|
+ and int(task_view.get("parent_task_id") or 0) == auto_parent,
|
|
|
+ "single-task view joins the parent title")
|
|
|
+ check("status" in task_view and "description" in task_view
|
|
|
+ and "plan" in task_view and "summary" in task_view,
|
|
|
+ "single-task view carries the full detail fields")
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/task/99999", r4_token)
|
|
|
+ check(status == 404, "unknown task id returns 404")
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# Bundle: allocation — the ask/wake/allocate machinery: concurrent ask
|
|
|
+# race, token revocation, workspace affinity, the lost-wake guard, the
|
|
|
+# slow-ask regression, assignment capability context + the tag
|
|
|
+# catalogue, and the pinned-blocks/activity dashboard.
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+def bundle_allocation(ctx):
|
|
|
+ auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
|
|
|
+ app = ctx.app
|
|
|
+ r4_token = fake_token
|
|
|
+ # gpu rides the primary fake from the start (the routing bundle's
|
|
|
+ # early attach, replayed): its attach row precedes lostwake's, so
|
|
|
+ # the assignment's saddle_tags order matches the assertion in the
|
|
|
+ # capability-context section below.
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "gpu"})
|
|
|
+ act(auth, saddle_card(auth, "fake-saddle")["attach_capability"],
|
|
|
+ {"capability_id": str(int([c["id"] for c in
|
|
|
+ slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
+ if c["name"] == "gpu"][0]))})
|
|
|
+
|
|
|
+ print("== phase 1: ask/allocation — token auth + concurrent race ==")
|
|
|
+ # Plan 10 phase 1. Registration mints per-saddle tokens; the
|
|
|
+ # ask endpoint authenticates by them; concurrent asks race to
|
|
|
+ # exactly one winner through the allocation transaction.
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/next-task", None)
|
|
|
+ check(status == 401, "next-task rejects a missing token")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-bogus-token")
|
|
|
+ check(status == 401, "next-task rejects a wrong token")
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/next-task", fake_token)
|
|
|
+ check(status == 204, "next-task answers 204 (nothing offerable — daemon sleeps)")
|
|
|
+
|
|
|
+ # Two more racers: real daemons on their own ports; fake2 reports
|
|
|
+ # its ssh key at /register so the key lands on the row directly.
|
|
|
+ fake2 = FakeSaddle()
|
|
|
+ fake2.ssh_public_key = "ssh-ed25519 AAAAFake2RacerKey saddle-2@fedora"
|
|
|
+ fake2.serve(port=free_port())
|
|
|
+ fake3 = FakeSaddle()
|
|
|
+ fake3.serve(port=free_port())
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["register"], {"name": "racer-two",
|
|
|
+ "host": "127.0.0.1:%d" % fake2.port})
|
|
|
+ act(auth, saddles_state["register"], {"name": "racer-three",
|
|
|
+ "host": "127.0.0.1:%d" % fake3.port})
|
|
|
+ check(wait_for(lambda: len(fake2.register_log) > 0 and len(fake3.register_log) > 0
|
|
|
+ or False, timeout=15, what="racers registered") is not None,
|
|
|
+ "both racers received /register")
|
|
|
+ check(wait_for(lambda: "AAAAFake2RacerKey" in
|
|
|
+ [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "racer-two"][0].get("ssh_public_key", "") or False,
|
|
|
+ timeout=15, what="register-reported key") is not None,
|
|
|
+ "the ssh key reported at /register lands on the saddle row")
|
|
|
+
|
|
|
+ racer_cards = {}
|
|
|
+ for name, daemon in (("fake-saddle", fake), ("racer-two", fake2), ("racer-three", fake3)):
|
|
|
+ card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == name][0]
|
|
|
+ racer_cards[daemon] = {"id": int(card["id"]), "token": card["token"]}
|
|
|
+ # Revocation: a registered saddle's token works until the saddle
|
|
|
+ # is removed, then nothing authorises its calls.
|
|
|
+ fake4 = FakeSaddle()
|
|
|
+ fake4.ask_on_register = False # a pre-flip daemon: never asks
|
|
|
+ fake4.serve(port=free_port())
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["register"], {"name": "old-daemon",
|
|
|
+ "host": "127.0.0.1:%d" % fake4.port})
|
|
|
+ old_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "old-daemon"][0]
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
|
|
|
+ check(status in (200, 204),
|
|
|
+ "the old daemon's fresh token authenticates next-task")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
|
|
|
+ {"project": "demo", "body": "from the old daemon"})
|
|
|
+ check(status == 200, "note accepts any registered saddle's token")
|
|
|
+ act(auth, old_card["unregister"], {})
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
|
|
|
+ check(status == 401, "removing the saddle revokes its token")
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
|
|
|
+ {"project": "demo", "body": "should fail"})
|
|
|
+ check(status == 401, "revocation covers the note endpoint too")
|
|
|
+
|
|
|
+ # Deterministic race: settle the queue first (this shard runs the
|
|
|
+ # race before any parked work exists, so only in-flight asks need
|
|
|
+ # draining), freeze every daemon's self-ask triggers, put all three
|
|
|
+ # to sleep (204), then fire one task worth of concurrent asks.
|
|
|
+ for daemon in (fake, fake2, fake3):
|
|
|
+ daemon.ask_on_register = False
|
|
|
+ daemon.ask_on_wake = False
|
|
|
+ daemon.ask_on_terminal = False
|
|
|
+ daemon.ask_on_idle_status = False
|
|
|
+ daemon.settle_asks() # converted sleep: in-flight asks land (204)
|
|
|
+ def queued_count():
|
|
|
conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
- " summary, completed_at, created_at) values (1, 'Affinity child', 'done',"
|
|
|
- " 5, ?, 'affinity child finished', strftime('%s','now','+1 second'),"
|
|
|
- " strftime('%s','now'))", (affinity_parent,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- act(auth, racer3_card["attach_capability"], {"capability_id": str(affinity_cap)})
|
|
|
- status, _ = fake3.ask_now()
|
|
|
- check(status == 204,
|
|
|
- "a tag-capable non-affine saddle is refused the parked task (204)")
|
|
|
-
|
|
|
- check(wait_for(lambda: len(fake2.wake_log) > wakes_before_2 or False,
|
|
|
- timeout=30, what="affine wake") is not None,
|
|
|
- "the dispatcher's wake targets the affine saddle")
|
|
|
- status, resume_payload = fake2.ask_now()
|
|
|
- fake2.ask_on_wake = True # restored whatever the claim returned
|
|
|
- fake2.ask_on_idle_status = True
|
|
|
- check(status == 200 and int(resume_payload["task"]["id"]) == affinity_parent,
|
|
|
- "the affine saddle's ask claims the parked task")
|
|
|
- check("Affinity child" in (resume_payload.get("resume_notice") or "")
|
|
|
- and "affinity child finished" in (resume_payload.get("resume_notice") or ""),
|
|
|
- "the resume assignment carries the subtask report notice")
|
|
|
- check(wait_task_done(auth, affinity_parent, timeout=60) is not None,
|
|
|
- "the resumed parent completes")
|
|
|
-
|
|
|
- print("== phase 1: lost wake -> offline + re-wake of the next capable ==")
|
|
|
- # racer-three sleeps through wakes (a daemon that died after its
|
|
|
- # last 204): the guard window (SADDLE_LOST_WAKE_S=2) passes with
|
|
|
- # no ask, the saddle goes offline, and the work moves to the
|
|
|
- # next capable sleeping saddle.
|
|
|
- fake3.ask_on_wake = False
|
|
|
- fake3.ask_on_idle_status = False # sleeps through everything
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- act(auth, saddles_state["create_capability"], {"name": "lostwake"})
|
|
|
- lostwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
- if c["name"] == "lostwake"][0])
|
|
|
- racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "racer-three"][0]
|
|
|
- act(auth, racer3_card["attach_capability"], {"capability_id": str(lostwake_cap)})
|
|
|
- # Racer-three must be asleep to be woken: park it with a manual
|
|
|
- # 204 ask (nothing offerable for it — the task below is not
|
|
|
- # created yet).
|
|
|
- status, _ = fake3.ask_now()
|
|
|
- check(status == 204, "racer-three sleeps before the lost-wake run")
|
|
|
+ try:
|
|
|
+ return conn.execute(
|
|
|
+ "select count(*) from tasks where status = 'queued'").fetchone()[0]
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Lost wake task",
|
|
|
- "priority": "2",
|
|
|
- "capability_id": [str(lostwake_cap)],
|
|
|
- })
|
|
|
- lost_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Lost wake task"][0])
|
|
|
- check(wait_for(lambda: len(fake3.wake_log) > 0 or False,
|
|
|
- timeout=30, what="wake sent to racer-three") is not None,
|
|
|
- "the only capable sleeping saddle is woken")
|
|
|
- check(wait_for(lambda: [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "racer-three"][0]["status"] == "offline"
|
|
|
- or False, timeout=30, what="lost-wake -> offline") is not None,
|
|
|
- "no ask after the wake: the saddle goes offline (lost-wake guard)")
|
|
|
- check(task_state(auth, lost_task)["status"] == "queued",
|
|
|
- "the task is untouched while its wake was pending")
|
|
|
-
|
|
|
- # Re-wake the next capable: the healthy saddle gains the tag and
|
|
|
- # picks the work up via its own wake -> ask -> allocate roundtrip.
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- act(auth, fake_card["attach_capability"], {"capability_id": str(lostwake_cap)})
|
|
|
- wakes_before = len(fake.wake_log)
|
|
|
- check(wait_for(lambda: len(fake.wake_log) > wakes_before or False,
|
|
|
- timeout=30, what="re-wake of the next capable") is not None,
|
|
|
- "the next capable sleeping saddle is woken for the task")
|
|
|
- check(wait_task_done(auth, lost_task, timeout=60) is not None,
|
|
|
- "wake -> ask -> allocate: the task completes on the healthy saddle")
|
|
|
- check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == lost_task
|
|
|
- for p in fake.start_payloads) or False,
|
|
|
- timeout=10, what="lost task assignment captured") is not None,
|
|
|
- "the re-woken saddle's ask produced the assignment")
|
|
|
- fake3.ask_on_wake = True
|
|
|
- fake3.ask_on_idle_status = True
|
|
|
-
|
|
|
- print("== phase 3: capability context, catalogue, subtask tags ==")
|
|
|
- # Assignment capability context (plan 10 phase 3): every
|
|
|
- # assignment carries {saddle_tags: the allocated saddle's tags,
|
|
|
- # fleet_tags: every tag name known across the fleet} — the
|
|
|
- # daemon renders it into its system prompt. fake-saddle carries
|
|
|
- # gpu + lostwake; the catalogue is affinity-tag, gpu, lostwake.
|
|
|
- fake2.ask_on_register = False
|
|
|
- fake2.ask_on_wake = False
|
|
|
- fake2.ask_on_terminal = False
|
|
|
- fake2.ask_on_idle_status = False
|
|
|
- time.sleep(1.2) # any in-flight trigger asks land
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Capabilities context",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- cap_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Capabilities context"][0])
|
|
|
- check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == cap_task
|
|
|
- for p in fake.start_payloads) or False,
|
|
|
- timeout=60, what="capabilities-context task dispatched") is not None,
|
|
|
- "the capabilities-context task dispatches")
|
|
|
- cap_payload = [p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == cap_task][-1]
|
|
|
- context = cap_payload.get("capabilities") or {}
|
|
|
- check(context.get("saddle_tags") == ["gpu", "lostwake"],
|
|
|
- "assignment saddle_tags = the allocated saddle's tags (%s)"
|
|
|
- % context.get("saddle_tags"))
|
|
|
- check(context.get("fleet_tags") == ["affinity-tag", "gpu", "lostwake"],
|
|
|
- "assignment fleet_tags = every tag name known across the fleet (%s)"
|
|
|
- % context.get("fleet_tags"))
|
|
|
- check(wait_task_done(auth, cap_task, timeout=60) is not None,
|
|
|
- "capabilities-context task completes")
|
|
|
-
|
|
|
- # GET /api/saddle/capabilities feeds the daemon's `saddle
|
|
|
- # capabilities` CLI: per-saddle-token auth, tag order by name,
|
|
|
- # each tag with the saddles carrying it (name order).
|
|
|
- status, _ = saddle_api("GET", "/api/saddle/capabilities", None)
|
|
|
- check(status == 401, "capabilities endpoint rejects a missing token")
|
|
|
- status, body = saddle_api("GET", "/api/saddle/capabilities", "saddle-bogus")
|
|
|
- check(status == 401, "capabilities endpoint rejects a wrong token")
|
|
|
- status, body = saddle_api("GET", "/api/saddle/capabilities", fake_token)
|
|
|
- catalogue = {t["name"]: t["saddles"] for t in body.get("tags", [])}
|
|
|
- check(status == 200 and body.get("ok") is True
|
|
|
- and [t["name"] for t in body.get("tags", [])]
|
|
|
- == ["affinity-tag", "gpu", "lostwake"],
|
|
|
- "catalogue lists every tag by name")
|
|
|
- check(catalogue.get("gpu") == ["fake-saddle"]
|
|
|
- and catalogue.get("affinity-tag") == ["racer-three", "racer-two"]
|
|
|
- and catalogue.get("lostwake") == ["fake-saddle", "racer-three"],
|
|
|
- "each tag names its carrying saddles in name order (%s)" % catalogue)
|
|
|
-
|
|
|
- # Subtask capabilities (`saddle subtask --capability`): explicit
|
|
|
- # only — never inherited — and validated against the operator
|
|
|
- # tag set; unknown tags reject with the valid list in the error
|
|
|
- # (the agent reads it verbatim in the CLI output).
|
|
|
- conn = db()
|
|
|
- tasks_before = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
- conn.close()
|
|
|
- status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
- "parent_task_id": auto_parent, "title": "Bad tag subtask",
|
|
|
- "capabilities": ["gpu", "warp-drive"]})
|
|
|
- conn = db()
|
|
|
- tasks_after = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(status == 400 and tasks_after == tasks_before,
|
|
|
- "an unknown capability tag rejects the subtask call")
|
|
|
- error_text = body.get("error", "")
|
|
|
- check("warp-drive" in error_text and "valid tags" in error_text
|
|
|
- and "gpu" in error_text and "lostwake" in error_text,
|
|
|
- "the rejection names the unknown tag and the valid list (%s)" % error_text)
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
- "parent_task_id": auto_parent, "title": "Bad shape subtask",
|
|
|
- "capabilities": "gpu"})
|
|
|
- check(status == 400, "a non-array capabilities member rejects")
|
|
|
-
|
|
|
- # Explicit-only: a parent requiring 'lostwake' (the lost-wake
|
|
|
- # task) gets a subtask tagged just 'gpu' — nothing inherited.
|
|
|
- status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
- "parent_task_id": lost_task, "title": "Explicit tag subtask",
|
|
|
- "capabilities": ["gpu"]})
|
|
|
- check(status == 200 and body.get("ok") is True,
|
|
|
- "a valid tag creates the subtask synchronously")
|
|
|
- tagged_subtask = int(body["task_id"])
|
|
|
- detail = task_state(auth, tagged_subtask)
|
|
|
- check([c["name"] for c in detail.get("capabilities", [])] == ["gpu"],
|
|
|
- "the subtask carries exactly the explicit tag (no inheritance)")
|
|
|
- check([c["name"] for c in task_state(auth, lost_task).get("capabilities", [])]
|
|
|
- == ["lostwake"],
|
|
|
- "the parent keeps its own requirement")
|
|
|
- conn = db()
|
|
|
- tag_rows = conn.execute(
|
|
|
- "select c.name from task_capabilities tc"
|
|
|
- " join capabilities c on c.id = tc.capability_id"
|
|
|
- " where tc.task_id = ?", (tagged_subtask,)).fetchall()
|
|
|
- conn.close()
|
|
|
- check([r[0] for r in tag_rows] == ["gpu"],
|
|
|
- "the task_capabilities rows record the explicit tag")
|
|
|
- # The queued subtask dispatches (gpu-carrier) and settles before
|
|
|
- # the slow-wake run needs an empty queue.
|
|
|
- check(wait_task_done(auth, tagged_subtask, timeout=90) is not None,
|
|
|
- "the tagged subtask dispatches and completes")
|
|
|
-
|
|
|
- print("== phase 3: task page — status on top, blocks newest-first ==")
|
|
|
- detail = task_state(auth, plan_task)
|
|
|
- block_ids = [int(b["id"]) for b in detail.get("blocks", [])]
|
|
|
- check(len(block_ids) >= 3 and block_ids == sorted(block_ids, reverse=True),
|
|
|
- "the blocks section orders newest-first via the sealed GET state (%s)"
|
|
|
- % block_ids)
|
|
|
- status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
|
|
|
- check(status == 200 and "status-head" in page
|
|
|
- and page.index("status-head") < page.index("<h3>Description</h3>")
|
|
|
- and page.index("status-head") < page.index("<h3>Plan</h3>"),
|
|
|
- "the status block sits at the very top of the task page")
|
|
|
- # The agent's progress feed renders DIRECTLY below the status
|
|
|
- # head (above the description cards), newest line first.
|
|
|
- check(page.index("status-head") < page.index("<h3>Progress</h3>")
|
|
|
- < page.index("<h3>Description</h3>"),
|
|
|
- "the progress feed sits directly below the status head")
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
- " created_at) values (1, 'Progress layout task', 'working', 5,"
|
|
|
- " strftime('%s','now','-120 seconds'))")
|
|
|
- layout_task = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
- # backdated so the dashboard-timeline fixtures below stay the
|
|
|
- # newest events
|
|
|
- conn.execute("insert into task_progress (task_id, summary, created_at)"
|
|
|
- " values (?, 'first update', strftime('%s','now','-90 seconds'))",
|
|
|
- (layout_task,))
|
|
|
- conn.execute("insert into task_progress (task_id, summary, created_at)"
|
|
|
- " values (?, 'second update', strftime('%s','now','-89 seconds'))",
|
|
|
- (layout_task,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- status, history_page = http("GET", "/tasks/%d" % layout_task, timeout=15)
|
|
|
- check(status == 200
|
|
|
- and history_page.index("progress-latest") < history_page.index("progress-history"),
|
|
|
- "the progress timeline renders newest-first (latest above the history)")
|
|
|
-
|
|
|
- print("== phase 3: dashboard — pinned blocks + activity timeline ==")
|
|
|
- # Pinned OPEN blocking items as link-through cards, newest
|
|
|
- # first; no inline answer forms (the card links to the task
|
|
|
- # page). At this point: the auto-approve grandchild's question
|
|
|
- # and RFC plus the parent's own review are all unanswered.
|
|
|
- dash = slot_public(auth, "/", "dashboard")
|
|
|
- pinned = dash.get("pinned", [])
|
|
|
- pinned_ids = [int(b["id"]) for b in pinned]
|
|
|
- check(len(pinned_ids) >= 3 and pinned_ids == sorted(pinned_ids, reverse=True),
|
|
|
- "pinned open blocks order newest-first (%s)" % pinned_ids)
|
|
|
- check({b["block_type"] for b in pinned} >= {"question", "rfc", "review"},
|
|
|
- "pinned cards cover questions, RFCs and reviews")
|
|
|
- check(all(b["question"] and b["task_title"] and int(b["task_id"]) > 0
|
|
|
- for b in pinned)
|
|
|
- and all("answer_action" not in b for b in pinned),
|
|
|
- "pinned cards are link-through only (task title, kind badge, question)")
|
|
|
-
|
|
|
- # The activity timeline: last <= 50 events, newest first, fed
|
|
|
- # from the existing tables. Fixture rows a few seconds old
|
|
|
- # guarantee the progress kind and a human-authored comment ride
|
|
|
- # along; the probe block below proves same-source ordering.
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into task_comments (task_id, author, body, created_at)"
|
|
|
- " values (?, 'user:admin', 'timeline fixture comment by a human',"
|
|
|
- " strftime('%s','now','-5 seconds'))", (cap_task,))
|
|
|
- conn.execute("insert into task_progress (task_id, summary, created_at)"
|
|
|
- " values (?, 'timeline fixture progress',"
|
|
|
- " strftime('%s','now','-5 seconds'))",
|
|
|
- (cap_task,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- dash = slot_public(auth, "/", "dashboard")
|
|
|
- activity = dash.get("activity", [])
|
|
|
- epochs = [int(e["at_epoch"]) for e in activity]
|
|
|
- check(0 < len(activity) <= 50 and epochs == sorted(epochs, reverse=True),
|
|
|
- "the timeline shows the last <=50 events newest-first (%d events)"
|
|
|
- % len(activity))
|
|
|
- kinds = {e["kind"] for e in activity}
|
|
|
- check({"created", "started", "dispatch", "comment", "progress", "completed"} <= kinds,
|
|
|
- "the timeline covers status changes, dispatches, comments, progress and"
|
|
|
- " completions (%s)" % sorted(kinds))
|
|
|
- check(any(e["kind"] == "comment" and e["text"].startswith("user:admin:")
|
|
|
- and "fixture comment" in e["text"] for e in activity),
|
|
|
- "comment events carry author + snippet")
|
|
|
-
|
|
|
- # A freshly raised block pins immediately (newest card) and
|
|
|
- # lands in the timeline as block-raised; answering it records
|
|
|
- # block-answered.
|
|
|
- saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
- {"type": "block", "session_id": grand_session, "data": {
|
|
|
- "question": "Timeline probe: approve the dashboard flow",
|
|
|
- "type": "rfc", "body": "# Probe"}})
|
|
|
- dash = slot_public(auth, "/", "dashboard")
|
|
|
- check(dash["pinned"][0]["question"] == "Timeline probe: approve the dashboard flow"
|
|
|
- and int(dash["pinned"][0]["task_id"]) == auto_grand,
|
|
|
- "the freshest open block pins as the first card")
|
|
|
- check(dash["activity"][0]["kind"] == "block-raised"
|
|
|
- and "Timeline probe" in dash["activity"][0]["text"],
|
|
|
- "the raised block lands atop the timeline")
|
|
|
- probe_block = [b for b in task_state(auth, auto_grand)["blocks"]
|
|
|
- if b["question"] == "Timeline probe: approve the dashboard flow"][0]
|
|
|
- act(auth, probe_block["answer_action"], {"answer": "Ship the dashboard"})
|
|
|
- dash = slot_public(auth, "/", "dashboard")
|
|
|
- activity = dash.get("activity", [])
|
|
|
- raised_at = next((i for i, e in enumerate(activity)
|
|
|
- if e["kind"] == "block-raised" and "Timeline probe" in e["text"]), -1)
|
|
|
- answered_at = next((i for i, e in enumerate(activity)
|
|
|
- if e["kind"] == "block-answered"
|
|
|
- and "Ship the dashboard" in e["text"]), -1)
|
|
|
- check(answered_at >= 0 and raised_at >= 0 and answered_at < raised_at,
|
|
|
- "the answered event sorts above its raised event (newest-first, same source)")
|
|
|
- check(all(b["question"] != "Timeline probe: approve the dashboard flow"
|
|
|
- for b in dash["pinned"]),
|
|
|
- "the answered block leaves the pinned cards")
|
|
|
-
|
|
|
- print("== phase 3 regression: a slow ask after a wake is never dropped ==")
|
|
|
- # The daemon worker's lost-wake-guard regression: a saddle whose
|
|
|
- # ask arrives SLOWLY after the wake (busy winding down when the
|
|
|
- # wake lands) must not lose the wake — the delayed ask (inside
|
|
|
- # the guard window) claims the task, no re-wake storm fires
|
|
|
- # while it is pending, and the saddle stays healthy. The
|
|
|
- # never-asks path (guard -> offline + re-wake of the next
|
|
|
- # capable) is proven by the racer-three section above.
|
|
|
- check(wait_for(lambda: queued_count() == 0 or False,
|
|
|
- timeout=60, what="queue settles before the slow-wake run") is not None,
|
|
|
- "the queue settles before the slow-wake run")
|
|
|
- status, _ = fake2.ask_now()
|
|
|
- check(status == 204, "the slow-ask racer sleeps before its wake")
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- act(auth, saddles_state["create_capability"], {"name": "slowwake"})
|
|
|
- slowwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")
|
|
|
- ["capabilities"] if c["name"] == "slowwake"][0])
|
|
|
- racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "racer-two"][0]
|
|
|
- act(auth, racer2_card["attach_capability"], {"capability_id": str(slowwake_cap)})
|
|
|
- fake2.ask_delay = 1.2 # slow: asks well after the wake, inside the 2s guard
|
|
|
- fake2.ask_on_wake = True
|
|
|
- wakes_before_slow = len(fake2.wake_log)
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Slow wake task",
|
|
|
- "priority": "2",
|
|
|
- "capability_id": [str(slowwake_cap)],
|
|
|
- })
|
|
|
- slow_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Slow wake task"][0])
|
|
|
- check(wait_for(lambda: len(fake2.wake_log) > wakes_before_slow or False,
|
|
|
- timeout=30, what="the slow-ask racer woken") is not None,
|
|
|
- "the only capable sleeping saddle is woken")
|
|
|
- check(wait_task_done(auth, slow_task, timeout=120) is not None,
|
|
|
- "the delayed ask claims the task — the wake was not dropped")
|
|
|
- check(any(a.get("status") == 200
|
|
|
- and int((a.get("payload") or {}).get("task", {}).get("id") or 0)
|
|
|
- == slow_task for a in fake2.asks),
|
|
|
- "the slow ask's 200 carries the task")
|
|
|
- check(len(fake2.wake_log) == wakes_before_slow + 1,
|
|
|
- "the pending-wake skip re-wakes nobody while the slow ask is in flight")
|
|
|
- racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
- if s["name"] == "racer-two"][0]
|
|
|
- check(racer2_card["status"] in ("idle", "busy"),
|
|
|
- "the slow asker never went offline (guard did not fire)")
|
|
|
- fake2.ask_delay = 0.0
|
|
|
- fake2.ask_on_wake = False # stays out of the pool below
|
|
|
-
|
|
|
- # The racers leave the dispatch pool: the remaining sections
|
|
|
- # (rollback resume, R5-R11) predate multi-daemon routing and
|
|
|
- # assert against the primary fake's payloads. racer-three is
|
|
|
- # already offline (the lost-wake guard); racer-two joins it.
|
|
|
- conn = db()
|
|
|
- conn.execute("update saddles set status = 'offline' where name = 'racer-two'")
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
+ check(wait_for(lambda: queued_count() == 0 or False,
|
|
|
+ timeout=60, what="queue settles before the race") is not None,
|
|
|
+ "the queue settles before the race")
|
|
|
|
|
|
- # Tasks caught working on a saddle that went offline requeue with a
|
|
|
- # bot comment each tick.
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Offline saddle task",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- offline_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Offline saddle task"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, offline_task).get("status") == "working"
|
|
|
- or False, timeout=60, what="offline task starts working") is not None,
|
|
|
- "task reaches working before its saddle disappears")
|
|
|
- conn = db()
|
|
|
- conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- # The requeue frees the saddle in the same tick, so 'queued' is
|
|
|
- # transient; the durable evidence is the bot comment (and the task
|
|
|
- # completing again on the healthy daemon).
|
|
|
- check(wait_for(lambda: any(c["body"] == "saddle went offline; task requeued"
|
|
|
- for c in task_state(auth, offline_task).get("comments", []))
|
|
|
- or False, timeout=30, what="offline requeue comment") is not None,
|
|
|
- "offline saddle requeues the task")
|
|
|
- check(wait_task_done(auth, offline_task, timeout=60) is not None,
|
|
|
- "requeued task completes again on the healthy daemon")
|
|
|
-
|
|
|
-
|
|
|
- print("== R5 B1: /start parent context ==")
|
|
|
- # A fresh subtask dispatch carries the parent object (plan clamped).
|
|
|
- conn = db()
|
|
|
- conn.execute("update tasks set plan = ? where id = ?",
|
|
|
- ("# Parent plan\n\n" + "step. " * 1200, auto_parent))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Lineage subtask",
|
|
|
- "priority": "3",
|
|
|
- })
|
|
|
- lineage_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Lineage subtask"][0])
|
|
|
- conn = db()
|
|
|
- conn.execute("update tasks set parent_task_id = ?, description = 'torque to spec'"
|
|
|
- " where id = ?", (auto_parent, lineage_task))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == lineage_task
|
|
|
- for p in fake.start_payloads) or False,
|
|
|
- timeout=60, what="lineage subtask dispatches") is not None,
|
|
|
- "lineage subtask dispatches")
|
|
|
- lineage_push = [p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == lineage_task][-1]
|
|
|
- parent_obj = lineage_push.get("parent")
|
|
|
- check(parent_obj is not None
|
|
|
- and int(parent_obj.get("id") or 0) == auto_parent
|
|
|
- and parent_obj.get("title") == "Auto parent"
|
|
|
- and parent_obj.get("status") in ("suspended", "working", "done"),
|
|
|
- "subtask /start carries the parent object")
|
|
|
- check(parent_obj.get("description") == "torque to spec"
|
|
|
- or "Parent plan" in (parent_obj.get("plan") or ""),
|
|
|
- "parent description/plan content flows through")
|
|
|
- check(len(parent_obj.get("plan") or "") <= 4000,
|
|
|
- "parent plan clamped to ~4000 chars (%d)"
|
|
|
- % len(parent_obj.get("plan") or ""))
|
|
|
- root_pushes = [p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == offline_task]
|
|
|
- check(all("parent" not in p for p in root_pushes),
|
|
|
- "root task /start carries no parent object")
|
|
|
-
|
|
|
-
|
|
|
- print("== R5 B2: task-reading API ==")
|
|
|
- status, _ = saddle_api("GET", "/api/saddle/tasks", None)
|
|
|
- check(status == 401, "tasks endpoint rejects a missing token")
|
|
|
- status, _ = saddle_api("GET", "/api/saddle/task/%d" % auto_child, None)
|
|
|
- check(status == 401, "task detail endpoint rejects a missing token")
|
|
|
-
|
|
|
- status, body = saddle_api("GET", "/api/saddle/tasks", r4_token)
|
|
|
- check(status == 200 and len(body.get("tasks", [])) > 5,
|
|
|
- "tasks endpoint lists tasks without filters")
|
|
|
- row = [t for t in body["tasks"] if int(t["id"]) == auto_child][0]
|
|
|
- check(row["title"] == "Auto child" and int(row["parent_task_id"]) == auto_parent
|
|
|
- and row["status"] in ("done", "queued", "working"),
|
|
|
- "compact rows carry id/title/status/parent_task_id")
|
|
|
-
|
|
|
- status, body = saddle_api("GET", "/api/saddle/tasks?project_id=1", r4_token)
|
|
|
- other_project = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
|
|
|
- if p["name"] == "diff-fabrica"][0])
|
|
|
- check(all(True for _ in [1]) # project 1 dominates the data set
|
|
|
- and not any(int(t["id"]) == 0 for t in body["tasks"]),
|
|
|
- "project_id filter accepted")
|
|
|
- status, body = saddle_api("GET",
|
|
|
- "/api/saddle/tasks?project_id=%d" % other_project, r4_token)
|
|
|
- check(all(t["title"].startswith("Diff") for t in body["tasks"]),
|
|
|
- "other project lists only its own tasks")
|
|
|
-
|
|
|
- status, body = saddle_api("GET", "/api/saddle/tasks?q=FLANGE", r4_token)
|
|
|
- titles = [t["title"] for t in body["tasks"]]
|
|
|
- check("Wire the flange" in titles,
|
|
|
- "q matches titles case-insensitively")
|
|
|
- status, body = saddle_api("GET", "/api/saddle/tasks?q=torque%20to%20spec", r4_token)
|
|
|
- check(any(t["title"] == "Lineage subtask" for t in body["tasks"]),
|
|
|
- "q matches description text")
|
|
|
- status, body = saddle_api("GET", "/api/saddle/tasks?q=parent%20plan", r4_token)
|
|
|
- check(any(t["title"] == "Auto parent" for t in body["tasks"]),
|
|
|
- "q matches plan text")
|
|
|
-
|
|
|
- status, body = saddle_api("GET", "/api/saddle/task/%d" % auto_child, r4_token)
|
|
|
- task_view = body.get("task", {})
|
|
|
- check(status == 200 and int(task_view.get("id") or 0) == auto_child
|
|
|
- and task_view.get("parent_title") == "Auto parent"
|
|
|
- and int(task_view.get("parent_task_id") or 0) == auto_parent,
|
|
|
- "single-task view joins the parent title")
|
|
|
- check("status" in task_view and "description" in task_view
|
|
|
- and "plan" in task_view and "summary" in task_view,
|
|
|
- "single-task view carries the full detail fields")
|
|
|
- status, body = saddle_api("GET", "/api/saddle/task/99999", r4_token)
|
|
|
- check(status == 404, "unknown task id returns 404")
|
|
|
-
|
|
|
-
|
|
|
- print("== R6 F1: a hanging request cannot freeze the tick ==")
|
|
|
- # One wedged /task-update (never responds) blocks the comment/
|
|
|
- # subtask-event delivery pass; the Soup timeout must fail it and
|
|
|
- # later ticks keep dispatching.
|
|
|
- task_state(auth, lineage_task)
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
- " values (1, 'Hang survivor', 'queued', 2, strftime('%s','now'))")
|
|
|
- hang_task = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
- conn.close()
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Hang runner",
|
|
|
- "priority": "3",
|
|
|
- })
|
|
|
- hang_runner = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Hang runner"][0])
|
|
|
- # A human comment on the running task gives the tick a
|
|
|
- # /task-update delivery to make; the next one hangs.
|
|
|
- check(wait_for(lambda: task_state(auth, hang_runner).get("status") == "working"
|
|
|
- or False, timeout=60, what="hang runner working") is not None,
|
|
|
- "hang runner reaches working")
|
|
|
- act(auth, task_state(auth, hang_runner)["comment"],
|
|
|
- {"body": "operator ping while wedged"})
|
|
|
- fake.hang_next["/task-update"] = 1
|
|
|
- check(wait_task_done(auth, hang_runner, timeout=90) is not None,
|
|
|
- "dispatching task survives a hung daemon request")
|
|
|
- check(wait_for(lambda: any(c.get("text") == "operator ping while wedged"
|
|
|
- or c.get("body") == "operator ping while wedged"
|
|
|
- for c in fake.comment_log) or False,
|
|
|
- timeout=30, what="wedged comment eventually delivered") is not None,
|
|
|
- "the comment delivery retries after the hang")
|
|
|
- # The hung delivery itself fails after the timeout; the system
|
|
|
- # kept moving either way.
|
|
|
- with open(os.path.join(WORK, "app.log")) as handle:
|
|
|
- app_log = handle.read()
|
|
|
- check("tick exceeded" not in app_log or "Dispatch tick failed" in app_log,
|
|
|
- "a hung request never freezes the tick latch")
|
|
|
+ sleep_statuses = [daemon.ask_now()[0] for daemon in (fake, fake2, fake3)]
|
|
|
+ check(sleep_statuses == [204, 204, 204],
|
|
|
+ "all three racers sleep (204) before the task exists")
|
|
|
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Race task",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ race_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Race task"][0])
|
|
|
+
|
|
|
+ race_results = []
|
|
|
+ def race_ask(daemon):
|
|
|
+ # A real racing daemon accepts the assignment it wins — the
|
|
|
+ # manual ask path does exactly that on a 200.
|
|
|
+ race_results.append(daemon.ask_now())
|
|
|
+ threads = [threading.Thread(target=race_ask, args=(daemon,))
|
|
|
+ for daemon in (fake, fake2, fake3)]
|
|
|
+ for thread in threads:
|
|
|
+ thread.start()
|
|
|
+ for thread in threads:
|
|
|
+ thread.join()
|
|
|
+ race_statuses = sorted(r[0] for r in race_results)
|
|
|
+ check(race_statuses == [200, 204, 204],
|
|
|
+ "concurrent asks race to exactly one winner (%s)" % race_statuses)
|
|
|
+ winner = [r for r in race_results if r[0] == 200][0][1]
|
|
|
+ check(int(winner["task"]["id"]) == race_task, "the winner's payload carries the task")
|
|
|
+ check(int(winner["session_id"]) > 0, "the assignment carries the session id")
|
|
|
+ conn = db()
|
|
|
+ race_sessions = conn.execute(
|
|
|
+ "select count(*) from sessions where task_id = ?", (race_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(race_sessions == 1,
|
|
|
+ "exactly one session row — the transaction serialised the claims (%d)"
|
|
|
+ % race_sessions)
|
|
|
+ check(wait_task_done(auth, race_task, timeout=60) is not None,
|
|
|
+ "the raced task completes on the winning daemon")
|
|
|
+ for daemon in (fake, fake2, fake3):
|
|
|
+ daemon.ask_on_register = True
|
|
|
+ daemon.ask_on_wake = True
|
|
|
+ daemon.ask_on_terminal = True
|
|
|
+ daemon.ask_on_idle_status = True
|
|
|
+ # Re-establish sleeping explicitly: the winner's post-race
|
|
|
+ # re-ask timing is not deterministic, and the affinity section
|
|
|
+ # below needs every racer provably asleep. Any stray offerable
|
|
|
+ # drains into these asks first.
|
|
|
+ def all_asleep():
|
|
|
+ return all(daemon.ask_now()[0] == 204 for daemon in (fake, fake2, fake3))
|
|
|
+ check(wait_for(lambda: all_asleep() or False, timeout=30,
|
|
|
+ what="racers re-sleep after the race") is not None,
|
|
|
+ "all racers re-sleep after the race")
|
|
|
+
|
|
|
+ print("== phase 1: workspace affinity pins parked tasks ==")
|
|
|
+ # A suspended parent parked on racer-two may be resumed ONLY by
|
|
|
+ # racer-two: a tag-capable non-affine saddle asking gets 204, and
|
|
|
+ # the wake targets the affine saddle. Only racer-two carries the
|
|
|
+ # tag at dispatch time (the race's winner is nondeterministic, so
|
|
|
+ # the park must not depend on which racers are free); racer-three
|
|
|
+ # gains it afterwards to prove the pinning is by workspace, not
|
|
|
+ # tags.
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "affinity-tag"})
|
|
|
+ affinity_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
+ if c["name"] == "affinity-tag"][0])
|
|
|
+ racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "racer-two"][0]
|
|
|
+ racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "racer-three"][0]
|
|
|
+ act(auth, racer2_card["attach_capability"], {"capability_id": str(affinity_cap)})
|
|
|
+
|
|
|
+ fake2.suspend_on_title = "Affinity parent"
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Affinity parent",
|
|
|
+ "priority": "2",
|
|
|
+ "capability_id": [str(affinity_cap)],
|
|
|
+ })
|
|
|
+ affinity_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Affinity parent"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, affinity_parent).get("status") == "suspended"
|
|
|
+ or False, timeout=60, what="affinity parent parks") is not None,
|
|
|
+ "affinity parent parks (suspended) on the tag-carrying racer")
|
|
|
+ conn = db()
|
|
|
+ affinity_session = conn.execute(
|
|
|
+ "select saddle_id from sessions where task_id = ? and ended_at is null",
|
|
|
+ (affinity_parent,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(affinity_session == int(racer2_card["id"]),
|
|
|
+ "the parked workspace sits on racer-two (the only tag-carrier)")
|
|
|
+
|
|
|
+ # Now racer-three carries the tag too — the pinning is by
|
|
|
+ # workspace, not tags.
|
|
|
+
|
|
|
+ # A terminal subtask makes the parked task resume-offerable —
|
|
|
+ # affine to racer-two only. The park is observed the moment the
|
|
|
+ # suspended push lands (no poller lag to compensate), so the
|
|
|
+ # child is inserted only after racer-two's ask triggers are
|
|
|
+ # frozen and the wake measurement armed — its completion stamped
|
|
|
+ # one second ahead so completed_at > suspended_at holds.
|
|
|
+ fake2.ask_on_wake = False
|
|
|
+ fake2.ask_on_idle_status = False
|
|
|
+ wakes_before_2 = len(fake2.wake_log)
|
|
|
+ fake2.settle_asks() # converted sleep: park-time ask lands (204)
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
+ " summary, completed_at, created_at) values (1, 'Affinity child', 'done',"
|
|
|
+ " 5, ?, 'affinity child finished', strftime('%s','now','+1 second'),"
|
|
|
+ " strftime('%s','now'))", (affinity_parent,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ act(auth, racer3_card["attach_capability"], {"capability_id": str(affinity_cap)})
|
|
|
+ status, _ = fake3.ask_now()
|
|
|
+ check(status == 204,
|
|
|
+ "a tag-capable non-affine saddle is refused the parked task (204)")
|
|
|
+
|
|
|
+ check(wait_for(lambda: len(fake2.wake_log) > wakes_before_2 or False,
|
|
|
+ timeout=30, what="affine wake") is not None,
|
|
|
+ "the dispatcher's wake targets the affine saddle")
|
|
|
+ status, resume_payload = fake2.ask_now()
|
|
|
+ fake2.ask_on_wake = True # restored whatever the claim returned
|
|
|
+ fake2.ask_on_idle_status = True
|
|
|
+ check(status == 200 and int(resume_payload["task"]["id"]) == affinity_parent,
|
|
|
+ "the affine saddle's ask claims the parked task")
|
|
|
+ check("Affinity child" in (resume_payload.get("resume_notice") or "")
|
|
|
+ and "affinity child finished" in (resume_payload.get("resume_notice") or ""),
|
|
|
+ "the resume assignment carries the subtask report notice")
|
|
|
+ check(wait_task_done(auth, affinity_parent, timeout=60) is not None,
|
|
|
+ "the resumed parent completes")
|
|
|
+
|
|
|
+ print("== phase 1: lost wake -> offline + re-wake of the next capable ==")
|
|
|
+ # racer-three sleeps through wakes (a daemon that died after its
|
|
|
+ # last 204): the guard window (SADDLE_LOST_WAKE_S=2) passes with
|
|
|
+ # no ask, the saddle goes offline, and the work moves to the
|
|
|
+ # next capable sleeping saddle.
|
|
|
+ fake3.ask_on_wake = False
|
|
|
+ fake3.ask_on_idle_status = False # sleeps through everything
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "lostwake"})
|
|
|
+ lostwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
+ if c["name"] == "lostwake"][0])
|
|
|
+ racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "racer-three"][0]
|
|
|
+ act(auth, racer3_card["attach_capability"], {"capability_id": str(lostwake_cap)})
|
|
|
+ # Racer-three must be asleep to be woken: park it with a manual
|
|
|
+ # 204 ask (nothing offerable for it — the task below is not
|
|
|
+ # created yet).
|
|
|
+ status, _ = fake3.ask_now()
|
|
|
+ check(status == 204, "racer-three sleeps before the lost-wake run")
|
|
|
+
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Lost wake task",
|
|
|
+ "priority": "2",
|
|
|
+ "capability_id": [str(lostwake_cap)],
|
|
|
+ })
|
|
|
+ lost_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Lost wake task"][0])
|
|
|
+ check(wait_for(lambda: len(fake3.wake_log) > 0 or False,
|
|
|
+ timeout=30, what="wake sent to racer-three") is not None,
|
|
|
+ "the only capable sleeping saddle is woken")
|
|
|
+ check(wait_for(lambda: [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "racer-three"][0]["status"] == "offline"
|
|
|
+ or False, timeout=30, what="lost-wake -> offline") is not None,
|
|
|
+ "no ask after the wake: the saddle goes offline (lost-wake guard)")
|
|
|
+ check(task_state(auth, lost_task)["status"] == "queued",
|
|
|
+ "the task is untouched while its wake was pending")
|
|
|
+
|
|
|
+ # Re-wake the next capable: the healthy saddle gains the tag and
|
|
|
+ # picks the work up via its own wake -> ask -> allocate roundtrip.
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ act(auth, fake_card["attach_capability"], {"capability_id": str(lostwake_cap)})
|
|
|
+ wakes_before = len(fake.wake_log)
|
|
|
+ check(wait_for(lambda: len(fake.wake_log) > wakes_before or False,
|
|
|
+ timeout=30, what="re-wake of the next capable") is not None,
|
|
|
+ "the next capable sleeping saddle is woken for the task")
|
|
|
+ check(wait_task_done(auth, lost_task, timeout=60) is not None,
|
|
|
+ "wake -> ask -> allocate: the task completes on the healthy saddle")
|
|
|
+ check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == lost_task
|
|
|
+ for p in fake.start_payloads) or False,
|
|
|
+ timeout=10, what="lost task assignment captured") is not None,
|
|
|
+ "the re-woken saddle's ask produced the assignment")
|
|
|
+ fake3.ask_on_wake = True
|
|
|
+ fake3.ask_on_idle_status = True
|
|
|
+
|
|
|
+ print("== phase 3: capability context, catalogue, subtask tags ==")
|
|
|
+ # Assignment capability context (plan 10 phase 3): every
|
|
|
+ # assignment carries {saddle_tags: the allocated saddle's tags,
|
|
|
+ # fleet_tags: every tag name known across the fleet} — the
|
|
|
+ # daemon renders it into its system prompt. fake-saddle carries
|
|
|
+ # gpu + lostwake; the catalogue is affinity-tag, gpu, lostwake.
|
|
|
+ fake2.ask_on_register = False
|
|
|
+ fake2.ask_on_wake = False
|
|
|
+ fake2.ask_on_terminal = False
|
|
|
+ fake2.ask_on_idle_status = False
|
|
|
+ fake2.settle_asks() # converted sleep: in-flight trigger asks land
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Capabilities context",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ cap_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Capabilities context"][0])
|
|
|
+ check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == cap_task
|
|
|
+ for p in fake.start_payloads) or False,
|
|
|
+ timeout=60, what="capabilities-context task dispatched") is not None,
|
|
|
+ "the capabilities-context task dispatches")
|
|
|
+ cap_payload = [p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == cap_task][-1]
|
|
|
+ context = cap_payload.get("capabilities") or {}
|
|
|
+ check(context.get("saddle_tags") == ["gpu", "lostwake"],
|
|
|
+ "assignment saddle_tags = the allocated saddle's tags (%s)"
|
|
|
+ % context.get("saddle_tags"))
|
|
|
+ check(context.get("fleet_tags") == ["affinity-tag", "gpu", "lostwake"],
|
|
|
+ "assignment fleet_tags = every tag name known across the fleet (%s)"
|
|
|
+ % context.get("fleet_tags"))
|
|
|
+ check(wait_task_done(auth, cap_task, timeout=60) is not None,
|
|
|
+ "capabilities-context task completes")
|
|
|
+
|
|
|
+ # GET /api/saddle/capabilities feeds the daemon's `saddle
|
|
|
+ # capabilities` CLI: per-saddle-token auth, tag order by name,
|
|
|
+ # each tag with the saddles carrying it (name order).
|
|
|
+ status, _ = saddle_api("GET", "/api/saddle/capabilities", None)
|
|
|
+ check(status == 401, "capabilities endpoint rejects a missing token")
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/capabilities", "saddle-bogus")
|
|
|
+ check(status == 401, "capabilities endpoint rejects a wrong token")
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/capabilities", fake_token)
|
|
|
+ catalogue = {t["name"]: t["saddles"] for t in body.get("tags", [])}
|
|
|
+ check(status == 200 and body.get("ok") is True
|
|
|
+ and [t["name"] for t in body.get("tags", [])]
|
|
|
+ == ["affinity-tag", "gpu", "lostwake"],
|
|
|
+ "catalogue lists every tag by name")
|
|
|
+ check(catalogue.get("gpu") == ["fake-saddle"]
|
|
|
+ and catalogue.get("affinity-tag") == ["racer-three", "racer-two"]
|
|
|
+ and catalogue.get("lostwake") == ["fake-saddle", "racer-three"],
|
|
|
+ "each tag names its carrying saddles in name order (%s)" % catalogue)
|
|
|
+
|
|
|
+ # Subtask capabilities (`saddle subtask --capability`): explicit
|
|
|
+ # only — never inherited — and validated against the operator
|
|
|
+ # tag set; unknown tags reject with the valid list in the error
|
|
|
+ # (the agent reads it verbatim in the CLI output).
|
|
|
+ conn = db()
|
|
|
+ tasks_before = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
+ "parent_task_id": cap_task, "title": "Bad tag subtask",
|
|
|
+ "capabilities": ["gpu", "warp-drive"]})
|
|
|
+ conn = db()
|
|
|
+ tasks_after = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(status == 400 and tasks_after == tasks_before,
|
|
|
+ "an unknown capability tag rejects the subtask call")
|
|
|
+ error_text = body.get("error", "")
|
|
|
+ check("warp-drive" in error_text and "valid tags" in error_text
|
|
|
+ and "gpu" in error_text and "lostwake" in error_text,
|
|
|
+ "the rejection names the unknown tag and the valid list (%s)" % error_text)
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
+ "parent_task_id": cap_task, "title": "Bad shape subtask",
|
|
|
+ "capabilities": "gpu"})
|
|
|
+ check(status == 400, "a non-array capabilities member rejects")
|
|
|
+
|
|
|
+ # Explicit-only: a parent requiring 'lostwake' (the lost-wake
|
|
|
+ # task) gets a subtask tagged just 'gpu' — nothing inherited.
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
+ "parent_task_id": lost_task, "title": "Explicit tag subtask",
|
|
|
+ "capabilities": ["gpu"]})
|
|
|
+ check(status == 200 and body.get("ok") is True,
|
|
|
+ "a valid tag creates the subtask synchronously")
|
|
|
+ tagged_subtask = int(body["task_id"])
|
|
|
+ detail = task_state(auth, tagged_subtask)
|
|
|
+ check([c["name"] for c in detail.get("capabilities", [])] == ["gpu"],
|
|
|
+ "the subtask carries exactly the explicit tag (no inheritance)")
|
|
|
+ check([c["name"] for c in task_state(auth, lost_task).get("capabilities", [])]
|
|
|
+ == ["lostwake"],
|
|
|
+ "the parent keeps its own requirement")
|
|
|
+ conn = db()
|
|
|
+ tag_rows = conn.execute(
|
|
|
+ "select c.name from task_capabilities tc"
|
|
|
+ " join capabilities c on c.id = tc.capability_id"
|
|
|
+ " where tc.task_id = ?", (tagged_subtask,)).fetchall()
|
|
|
+ conn.close()
|
|
|
+ check([r[0] for r in tag_rows] == ["gpu"],
|
|
|
+ "the task_capabilities rows record the explicit tag")
|
|
|
+ # The queued subtask dispatches (gpu-carrier) and settles before
|
|
|
+ # the slow-wake run needs an empty queue.
|
|
|
+ check(wait_task_done(auth, tagged_subtask, timeout=90) is not None,
|
|
|
+ "the tagged subtask dispatches and completes")
|
|
|
+
|
|
|
+ print("== phase 3: dashboard — pinned blocks + activity timeline ==")
|
|
|
+ # Pinned OPEN blocking items as link-through cards, newest
|
|
|
+ # first; no inline answer forms (the card links to the task
|
|
|
+ # page). Fixture: one open question + one open review raised on a
|
|
|
+ # local task (stand-ins for the auto-approve grandchild's
|
|
|
+ # leftovers) join the probe RFC raised below.
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
+ " values (1, 'Dash fixture task', 'working', 5, strftime('%s','now'))")
|
|
|
+ dash_task = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at)"
|
|
|
+ " values (?, ?, strftime('%s','now'))", (dash_task, fake_id))
|
|
|
+ dash_session = conn.execute("select max(id) from sessions").fetchone()[0]
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "block", "session_id": dash_session, "data": {
|
|
|
+ "question": "Dash fixture question?", "type": "question"}})
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "review", "session_id": dash_session, "data": {
|
|
|
+ "summary": "dash fixture review"}})
|
|
|
+
|
|
|
+ # The activity timeline: last <= 50 events, newest first, fed
|
|
|
+ # from the existing tables. Fixture rows a few seconds old
|
|
|
+ # guarantee the progress kind and a human-authored comment ride
|
|
|
+ # along; the probe block below proves same-source ordering.
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into task_comments (task_id, author, body, created_at)"
|
|
|
+ " values (?, 'user:admin', 'timeline fixture comment by a human',"
|
|
|
+ " strftime('%s','now','-5 seconds'))", (cap_task,))
|
|
|
+ conn.execute("insert into task_progress (task_id, summary, created_at)"
|
|
|
+ " values (?, 'timeline fixture progress',"
|
|
|
+ " strftime('%s','now','-5 seconds'))",
|
|
|
+ (cap_task,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ # The probe RFC raised last: the newest pinned card and the newest
|
|
|
+ # timeline event (its raise+answer below prove same-source ordering
|
|
|
+ # both ways). The timeline sorts same-second events by source
|
|
|
+ # materialisation order, so the probe must land in a LATER clock
|
|
|
+ # second than the fixtures — wait for the boundary (bounded).
|
|
|
+ fixtures_epoch = int(time.time())
|
|
|
+ wait_for(lambda: int(time.time()) > fixtures_epoch or False,
|
|
|
+ timeout=3, what="next clock second before the probe raise")
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
+ {"type": "block", "session_id": dash_session, "data": {
|
|
|
+ "question": "Timeline probe: approve the dashboard flow",
|
|
|
+ "type": "rfc", "body": "# Probe"}})
|
|
|
+ dash = slot_public(auth, "/", "dashboard")
|
|
|
+ pinned = dash.get("pinned", [])
|
|
|
+ pinned_ids = [int(b["id"]) for b in pinned]
|
|
|
+ check(len(pinned_ids) >= 3 and pinned_ids == sorted(pinned_ids, reverse=True),
|
|
|
+ "pinned open blocks order newest-first (%s)" % pinned_ids)
|
|
|
+ check({b["block_type"] for b in pinned} >= {"question", "rfc", "review"},
|
|
|
+ "pinned cards cover questions, RFCs and reviews")
|
|
|
+ check(all(b["question"] and b["task_title"] and int(b["task_id"]) > 0
|
|
|
+ for b in pinned)
|
|
|
+ and all("answer_action" not in b for b in pinned),
|
|
|
+ "pinned cards are link-through only (task title, kind badge, question)")
|
|
|
+ activity = dash.get("activity", [])
|
|
|
+ epochs = [int(e["at_epoch"]) for e in activity]
|
|
|
+ check(0 < len(activity) <= 50 and epochs == sorted(epochs, reverse=True),
|
|
|
+ "the timeline shows the last <=50 events newest-first (%d events)"
|
|
|
+ % len(activity))
|
|
|
+ kinds = {e["kind"] for e in activity}
|
|
|
+ check({"created", "started", "dispatch", "comment", "progress", "completed"} <= kinds,
|
|
|
+ "the timeline covers status changes, dispatches, comments, progress and"
|
|
|
+ " completions (%s)" % sorted(kinds))
|
|
|
+ check(any(e["kind"] == "comment" and e["text"].startswith("user:admin:")
|
|
|
+ and "fixture comment" in e["text"] for e in activity),
|
|
|
+ "comment events carry author + snippet")
|
|
|
+
|
|
|
+ # The freshly raised probe pinned as the newest card and landed
|
|
|
+ # atop the timeline as block-raised; answering it records
|
|
|
+ # block-answered.
|
|
|
+ check(dash["pinned"][0]["question"] == "Timeline probe: approve the dashboard flow"
|
|
|
+ and int(dash["pinned"][0]["task_id"]) == dash_task,
|
|
|
+ "the freshest open block pins as the first card")
|
|
|
+ check(dash["activity"][0]["kind"] == "block-raised"
|
|
|
+ and "Timeline probe" in dash["activity"][0]["text"],
|
|
|
+ "the raised block lands atop the timeline")
|
|
|
+ probe_block = [b for b in task_state(auth, dash_task)["blocks"]
|
|
|
+ if b["question"] == "Timeline probe: approve the dashboard flow"][0]
|
|
|
+ act(auth, probe_block["answer_action"], {"answer": "Ship the dashboard"})
|
|
|
+ dash = slot_public(auth, "/", "dashboard")
|
|
|
+ activity = dash.get("activity", [])
|
|
|
+ raised_at = next((i for i, e in enumerate(activity)
|
|
|
+ if e["kind"] == "block-raised" and "Timeline probe" in e["text"]), -1)
|
|
|
+ answered_at = next((i for i, e in enumerate(activity)
|
|
|
+ if e["kind"] == "block-answered"
|
|
|
+ and "Ship the dashboard" in e["text"]), -1)
|
|
|
+ check(answered_at >= 0 and raised_at >= 0 and answered_at < raised_at,
|
|
|
+ "the answered event sorts above its raised event (newest-first, same source)")
|
|
|
+ check(all(b["question"] != "Timeline probe: approve the dashboard flow"
|
|
|
+ for b in dash["pinned"]),
|
|
|
+ "the answered block leaves the pinned cards")
|
|
|
+
|
|
|
+ print("== phase 3 regression: a slow ask after a wake is never dropped ==")
|
|
|
+ # The daemon worker's lost-wake-guard regression: a saddle whose
|
|
|
+ # ask arrives SLOWLY after the wake (busy winding down when the
|
|
|
+ # wake lands) must not lose the wake — the delayed ask (inside
|
|
|
+ # the guard window) claims the task, no re-wake storm fires
|
|
|
+ # while it is pending, and the saddle stays healthy. The
|
|
|
+ # never-asks path (guard -> offline + re-wake of the next
|
|
|
+ # capable) is proven by the racer-three section above.
|
|
|
+ check(wait_for(lambda: queued_count() == 0 or False,
|
|
|
+ timeout=60, what="queue settles before the slow-wake run") is not None,
|
|
|
+ "the queue settles before the slow-wake run")
|
|
|
+ status, _ = fake2.ask_now()
|
|
|
+ check(status == 204, "the slow-ask racer sleeps before its wake")
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "slowwake"})
|
|
|
+ slowwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")
|
|
|
+ ["capabilities"] if c["name"] == "slowwake"][0])
|
|
|
+ racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "racer-two"][0]
|
|
|
+ act(auth, racer2_card["attach_capability"], {"capability_id": str(slowwake_cap)})
|
|
|
+ fake2.ask_delay = 1.2 # slow: asks well after the wake, inside the 2s guard
|
|
|
+ fake2.ask_on_wake = True
|
|
|
+ wakes_before_slow = len(fake2.wake_log)
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Slow wake task",
|
|
|
+ "priority": "2",
|
|
|
+ "capability_id": [str(slowwake_cap)],
|
|
|
+ })
|
|
|
+ slow_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Slow wake task"][0])
|
|
|
+ check(wait_for(lambda: len(fake2.wake_log) > wakes_before_slow or False,
|
|
|
+ timeout=30, what="the slow-ask racer woken") is not None,
|
|
|
+ "the only capable sleeping saddle is woken")
|
|
|
+ check(wait_task_done(auth, slow_task, timeout=120) is not None,
|
|
|
+ "the delayed ask claims the task — the wake was not dropped")
|
|
|
+ check(any(a.get("status") == 200
|
|
|
+ and int((a.get("payload") or {}).get("task", {}).get("id") or 0)
|
|
|
+ == slow_task for a in fake2.asks),
|
|
|
+ "the slow ask's 200 carries the task")
|
|
|
+ check(len(fake2.wake_log) == wakes_before_slow + 1,
|
|
|
+ "the pending-wake skip re-wakes nobody while the slow ask is in flight")
|
|
|
+ racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "racer-two"][0]
|
|
|
+ check(racer2_card["status"] in ("idle", "busy"),
|
|
|
+ "the slow asker never went offline (guard did not fire)")
|
|
|
+ fake2.ask_delay = 0.0
|
|
|
+ fake2.ask_on_wake = False # stays out of the pool below
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# Bundle: lifecycle — offline requeues, the hung-request tick survival,
|
|
|
+# parked-refresh bookkeeping, cancellation, the health panel, raw-log
|
|
|
+# losslessness, git identity and the live blocked-push roundtrip.
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+def bundle_lifecycle(ctx):
|
|
|
+ auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
|
|
|
+ app = ctx.app
|
|
|
+
|
|
|
+ # Tasks caught working on a saddle that went offline requeue with a
|
|
|
+ # bot comment each tick.
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Offline saddle task",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ offline_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Offline saddle task"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, offline_task).get("status") == "working"
|
|
|
+ or False, timeout=60, what="offline task starts working") is not None,
|
|
|
+ "task reaches working before its saddle disappears")
|
|
|
+ # Hold the daemon mid-round across the requeue window (the
|
|
|
+ # original relied on its 8s work period outlasting the 5s tick):
|
|
|
+ # the task completing on its own would race the offline pass and
|
|
|
+ # skip the requeue the check exists to observe.
|
|
|
+ fake.hold_state = True
|
|
|
+ conn = db()
|
|
|
+ conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ # The requeue frees the saddle in the same tick, so 'queued' is
|
|
|
+ # transient; the durable evidence is the bot comment (and the task
|
|
|
+ # completing again on the healthy daemon).
|
|
|
+ check(wait_for(lambda: any(c["body"] == "saddle went offline; task requeued"
|
|
|
+ for c in task_state(auth, offline_task).get("comments", []))
|
|
|
+ or False, timeout=30, what="offline requeue comment") is not None,
|
|
|
+ "offline saddle requeues the task")
|
|
|
+ fake.hold_state = False # release: the round ends, the daemon re-asks
|
|
|
+ check(wait_task_done(auth, offline_task, timeout=60) is not None,
|
|
|
+ "requeued task completes again on the healthy daemon")
|
|
|
+
|
|
|
+ print("== R6 F1: a hanging request cannot freeze the tick ==")
|
|
|
+ # One wedged /task-update (never responds) blocks the comment/
|
|
|
+ # subtask-event delivery pass; the Soup timeout must fail it and
|
|
|
+ # later ticks keep dispatching.
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
|
|
|
+ " values (1, 'Hang survivor', 'queued', 2, strftime('%s','now'))")
|
|
|
+ hang_task = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Hang runner",
|
|
|
+ "priority": "3",
|
|
|
+ })
|
|
|
+ hang_runner = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Hang runner"][0])
|
|
|
+ # A human comment on the running task gives the tick a
|
|
|
+ # /task-update delivery to make; the next one hangs.
|
|
|
+ check(wait_for(lambda: task_state(auth, hang_runner).get("status") == "working"
|
|
|
+ or False, timeout=60, what="hang runner working") is not None,
|
|
|
+ "hang runner reaches working")
|
|
|
+ act(auth, task_state(auth, hang_runner)["comment"],
|
|
|
+ {"body": "operator ping while wedged"})
|
|
|
+ fake.hang_next["/task-update"] = 1
|
|
|
+ check(wait_task_done(auth, hang_runner, timeout=90) is not None,
|
|
|
+ "dispatching task survives a hung daemon request")
|
|
|
+ check(wait_for(lambda: any(c.get("text") == "operator ping while wedged"
|
|
|
+ or c.get("body") == "operator ping while wedged"
|
|
|
+ for c in fake.comment_log) or False,
|
|
|
+ timeout=30, what="wedged comment eventually delivered") is not None,
|
|
|
+ "the comment delivery retries after the hang")
|
|
|
+ # The hung delivery itself fails after the timeout; the system
|
|
|
+ # kept moving either way.
|
|
|
+ with open(os.path.join(WORK, "app.log")) as handle:
|
|
|
+ app_log = handle.read()
|
|
|
+ check("tick exceeded" not in app_log or "Dispatch tick failed" in app_log,
|
|
|
+ "a hung request never freezes the tick latch")
|
|
|
+
|
|
|
+ print("== R6 F2: refresh must not corrupt parked bookkeeping ==")
|
|
|
+ fake.suspend_on_title = "Parked refresh parent"
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Parked refresh parent",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ parked = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Parked refresh parent"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, parked).get("status") == "suspended"
|
|
|
+ or False, timeout=60, what="parked parent suspends") is not None,
|
|
|
+ "parent parks before the refresh")
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ act(auth, fake_card["refresh"], {})
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ check(fake_card["status"] == "idle" and int(fake_card["current_task_id"] or 0) == 0,
|
|
|
+ "refresh on a parked (suspended) daemon leaves the saddle idle")
|
|
|
+ # The parked task keeps its open session and resumes normally.
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
+ " summary, completed_at, created_at) values (1, 'Parked child', 'done', 5,"
|
|
|
+ " ?, 'child finished', strftime('%s','now','+1 second'),"
|
|
|
+ " strftime('%s','now'))", (parked,))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ check(wait_task_done(auth, parked, timeout=90) is not None,
|
|
|
+ "parked parent still resumes after the refresh")
|
|
|
+
|
|
|
+ print("== R7 B1: cancel reaches the daemon ==")
|
|
|
+ # Plan 10 phase 2: cancellation is POST /task-update {kind: cancel}
|
|
|
+ # to the RUNNING daemon (the abort machinery, workspace kept).
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Cancel live",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ cancel_live = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Cancel live"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, cancel_live).get("status") == "working"
|
|
|
+ or False, timeout=60, what="cancel-live working") is not None,
|
|
|
+ "task reaches working before the cancel")
|
|
|
+ act(auth, task_state(auth, cancel_live)["cancel"], {})
|
|
|
+ check(wait_for(lambda: cancel_live in fake.abort_log or False,
|
|
|
+ timeout=30, what="daemon abort recorded") is not None,
|
|
|
+ "cancelling POSTs task-update cancel to the daemon")
|
|
|
+ check(any(c["body"] == "cancelled; daemon aborted"
|
|
|
+ for c in task_state(auth, cancel_live).get("comments", [])),
|
|
|
+ "the abort outcome is narrated")
|
|
|
+ check(task_state(auth, cancel_live)["status"] == "failed",
|
|
|
+ "cancelled task lands failed as before")
|
|
|
+
|
|
|
+ # A 500 on the cancel marks the saddle offline (no fence layer
|
|
|
+ # anymore); nothing dispatches to it while offline, and the
|
|
|
+ # daemon's own ask flips the row back to idle (proof of life).
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Fence runner",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ fence_runner = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Fence runner"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, fence_runner).get("status") == "working"
|
|
|
+ or False, timeout=60, what="fence runner working") is not None,
|
|
|
+ "second task reaches working")
|
|
|
+ fake.abort_failures = 1
|
|
|
+ act(auth, task_state(auth, fence_runner)["cancel"], {})
|
|
|
+ check(any(c["body"] == "cancelled; daemon unreachable — saddle marked offline"
|
|
|
+ for c in task_state(auth, fence_runner).get("comments", [])),
|
|
|
+ "failed cancel narrates the offline marking")
|
|
|
+ check(saddle_card(auth, "fake-saddle")["status"] == "offline",
|
|
|
+ "saddle marked offline after the failed cancel")
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "During fence",
|
|
|
+ "priority": "3",
|
|
|
+ })
|
|
|
+ during_fence = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "During fence"][0])
|
|
|
+ # Converted sleep(7): a quiet window spanning several dispatch
|
|
|
+ # ticks in which the task MUST stay queued — if the offline
|
|
|
+ # saddle ever dispatches it, this fails loudly. The daemon is held
|
|
|
+ # mid-round for the window (the original relied on its 8s work
|
|
|
+ # period outlasting the sleep): a finishing round would ask, prove
|
|
|
+ # liveness and flip the offline row — only after the window may it.
|
|
|
+ fake.hold_state = True
|
|
|
+ check(quiet_window(quiet_seconds(),
|
|
|
+ lambda: task_state(auth, during_fence)["status"] == "queued",
|
|
|
+ "offline saddle does not dispatch during_fence"),
|
|
|
+ "offline saddle is not dispatchable")
|
|
|
+ fake.hold_state = False
|
|
|
+ # The (unaborted) daemon finishes its round, goes idle-awake and
|
|
|
+ # asks — the ask is the proof of life that flips the row back.
|
|
|
+ check(wait_for(lambda: saddle_card(auth, "fake-saddle")["status"] == "idle" or False,
|
|
|
+ timeout=30, what="ask flips offline to idle") is not None,
|
|
|
+ "the daemon's ask flips the offline row back to idle")
|
|
|
+ check(wait_task_done(auth, during_fence, timeout=60) is not None,
|
|
|
+ "queued work dispatches once the saddle proves it is alive")
|
|
|
+
|
|
|
+ print("== R7 B3: health panel + pushed raw log from the first round ==")
|
|
|
+ # A normally dispatched task pushes its chunks immediately: raw
|
|
|
+ # rows exist from the first round (the session-4 gap stays
|
|
|
+ # fixed without any probe machinery).
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Raw from first push",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ raw_first = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Raw from first push"][0])
|
|
|
|
|
|
- print("== R6 F2: refresh must not corrupt parked bookkeeping ==")
|
|
|
- fake.suspend_on_title = "Parked refresh parent"
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Parked refresh parent",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- parked = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Parked refresh parent"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, parked).get("status") == "suspended"
|
|
|
- or False, timeout=60, what="parked parent suspends") is not None,
|
|
|
- "parent parks before the refresh")
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- act(auth, fake_card["refresh"], {})
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- check(fake_card["status"] == "idle" and int(fake_card["current_task_id"] or 0) == 0,
|
|
|
- "refresh on a parked (suspended) daemon leaves the saddle idle")
|
|
|
- # The parked task keeps its open session and resumes normally.
|
|
|
+ def raw_count(task_id):
|
|
|
conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
|
|
|
- " summary, completed_at, created_at) values (1, 'Parked child', 'done', 5,"
|
|
|
- " ?, 'child finished', strftime('%s','now','+1 second'),"
|
|
|
- " strftime('%s','now'))", (parked,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- check(wait_task_done(auth, parked, timeout=90) is not None,
|
|
|
- "parked parent still resumes after the refresh")
|
|
|
+ try:
|
|
|
+ return conn.execute(
|
|
|
+ "select count(*) from session_raw_log where session_id in"
|
|
|
+ " (select id from sessions where task_id = ?)", (task_id,)).fetchone()[0]
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
|
|
|
+ check(wait_for(lambda: raw_count(raw_first) > 0 or False,
|
|
|
+ timeout=60, what="raw rows from first push") is not None,
|
|
|
+ "pushed chunks land from the first round (session-4 gap)")
|
|
|
+ check(wait_task_done(auth, raw_first, timeout=60) is not None,
|
|
|
+ "raw-first task completes")
|
|
|
+
|
|
|
+ # Health panel: a manual refresh populates the snapshot.
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ act(auth, fake_card["refresh"], {})
|
|
|
+ card = saddle_card(auth, "fake-saddle")
|
|
|
+ check(card.get("health") in ("healthy", "stale")
|
|
|
+ and card.get("daemon_state") in ("working", "idle", "done")
|
|
|
+ and card.get("uptime_display", "") != ""
|
|
|
+ and card.get("last_contact_display", "") not in ("", "never"),
|
|
|
+ "saddles page shows daemon state, uptime and last contact")
|
|
|
+
|
|
|
+ print("== R8: pushed chunk ingestion is lossless ==")
|
|
|
+ # The daemon's opening flush: a coloured prompt run, plain text,
|
|
|
+ # a bare CRLF between prompts and the tail — every non-empty
|
|
|
+ # chunk must land, exactly once, in seq order. A mid-flight
|
|
|
+ # redelivery of the whole batch (the outbox's at-least-once
|
|
|
+ # retry) consumes the duplicates.
|
|
|
+ corpus = [
|
|
|
+ {"seq": 1, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
|
|
|
+ {"seq": 2, "text": "echo hello\r\n"},
|
|
|
+ {"seq": 3, "text": "\r\n"},
|
|
|
+ {"seq": 4, "text": "hello\r\n"},
|
|
|
+ {"seq": 5, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
|
|
|
+ {"seq": 6, "text": "done\r\n"},
|
|
|
+ ]
|
|
|
+ fake.scripted_corpus = corpus
|
|
|
+ fake.hold_state = True
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Lossless log task",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ lossless = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Lossless log task"][0])
|
|
|
|
|
|
- print("== R7 B1: cancel reaches the daemon ==")
|
|
|
- # Plan 10 phase 2: cancellation is POST /task-update {kind: cancel}
|
|
|
- # to the RUNNING daemon (the abort machinery, workspace kept).
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Cancel live",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- cancel_live = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Cancel live"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, cancel_live).get("status") == "working"
|
|
|
- or False, timeout=60, what="cancel-live working") is not None,
|
|
|
- "task reaches working before the cancel")
|
|
|
- act(auth, task_state(auth, cancel_live)["cancel"], {})
|
|
|
- check(wait_for(lambda: cancel_live in fake.abort_log or False,
|
|
|
- timeout=30, what="daemon abort recorded") is not None,
|
|
|
- "cancelling POSTs task-update cancel to the daemon")
|
|
|
- check(any(c["body"] == "cancelled; daemon aborted"
|
|
|
- for c in task_state(auth, cancel_live).get("comments", [])),
|
|
|
- "the abort outcome is narrated")
|
|
|
- check(task_state(auth, cancel_live)["status"] == "failed",
|
|
|
- "cancelled task lands failed as before")
|
|
|
-
|
|
|
- # A 500 on the cancel marks the saddle offline (no fence layer
|
|
|
- # anymore); nothing dispatches to it while offline, and the
|
|
|
- # daemon's own ask flips the row back to idle (proof of life).
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Fence runner",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- fence_runner = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Fence runner"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, fence_runner).get("status") == "working"
|
|
|
- or False, timeout=60, what="fence runner working") is not None,
|
|
|
- "second task reaches working")
|
|
|
- fake.abort_failures = 1
|
|
|
- act(auth, task_state(auth, fence_runner)["cancel"], {})
|
|
|
- check(any(c["body"] == "cancelled; daemon unreachable — saddle marked offline"
|
|
|
- for c in task_state(auth, fence_runner).get("comments", [])),
|
|
|
- "failed cancel narrates the offline marking")
|
|
|
- check(saddle_card("fake-saddle")["status"] == "offline",
|
|
|
- "saddle marked offline after the failed cancel")
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "During fence",
|
|
|
- "priority": "3",
|
|
|
- })
|
|
|
- during_fence = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "During fence"][0])
|
|
|
- time.sleep(7)
|
|
|
- check(task_state(auth, during_fence)["status"] == "queued",
|
|
|
- "offline saddle is not dispatchable")
|
|
|
- # The (unaborted) daemon finishes its round, goes idle-awake and
|
|
|
- # asks — the ask is the proof of life that flips the row back.
|
|
|
- check(wait_for(lambda: saddle_card("fake-saddle")["status"] == "idle" or False,
|
|
|
- timeout=30, what="ask flips offline to idle") is not None,
|
|
|
- "the daemon's ask flips the offline row back to idle")
|
|
|
- check(wait_task_done(auth, during_fence, timeout=60) is not None,
|
|
|
- "queued work dispatches once the saddle proves it is alive")
|
|
|
-
|
|
|
-
|
|
|
- print("== R7 B3: health panel + pushed raw log from the first round ==")
|
|
|
- # A normally dispatched task pushes its chunks immediately: raw
|
|
|
- # rows exist from the first round (the session-4 gap stays
|
|
|
- # fixed without any probe machinery).
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Raw from first push",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- raw_first = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Raw from first push"][0])
|
|
|
-
|
|
|
- def raw_count(task_id):
|
|
|
- conn = db()
|
|
|
- try:
|
|
|
- return conn.execute(
|
|
|
- "select count(*) from session_raw_log where session_id in"
|
|
|
- " (select id from sessions where task_id = ?)", (task_id,)).fetchone()[0]
|
|
|
- finally:
|
|
|
- conn.close()
|
|
|
-
|
|
|
- check(wait_for(lambda: raw_count(raw_first) > 0 or False,
|
|
|
- timeout=60, what="raw rows from first push") is not None,
|
|
|
- "pushed chunks land from the first round (session-4 gap)")
|
|
|
- check(wait_task_done(auth, raw_first, timeout=60) is not None,
|
|
|
- "raw-first task completes")
|
|
|
-
|
|
|
- # Health panel: a manual refresh populates the snapshot.
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- act(auth, fake_card["refresh"], {})
|
|
|
- card = saddle_card("fake-saddle")
|
|
|
- check(card.get("health") in ("healthy", "stale")
|
|
|
- and card.get("daemon_state") in ("working", "idle", "done")
|
|
|
- and card.get("uptime_display", "") != ""
|
|
|
- and card.get("last_contact_display", "") not in ("", "never"),
|
|
|
- "saddles page shows daemon state, uptime and last contact")
|
|
|
-
|
|
|
- print("== R8: pushed chunk ingestion is lossless ==")
|
|
|
- # The daemon's opening flush: a coloured prompt run, plain text,
|
|
|
- # a bare CRLF between prompts and the tail — every non-empty
|
|
|
- # chunk must land, exactly once, in seq order. A mid-flight
|
|
|
- # redelivery of the whole batch (the outbox's at-least-once
|
|
|
- # retry) consumes the duplicates.
|
|
|
- corpus = [
|
|
|
- {"seq": 1, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
|
|
|
- {"seq": 2, "text": "echo hello\r\n"},
|
|
|
- {"seq": 3, "text": "\r\n"},
|
|
|
- {"seq": 4, "text": "hello\r\n"},
|
|
|
- {"seq": 5, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
|
|
|
- {"seq": 6, "text": "done\r\n"},
|
|
|
- ]
|
|
|
- fake.scripted_corpus = corpus
|
|
|
- fake.hold_state = True
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Lossless log task",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- lossless = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Lossless log task"][0])
|
|
|
-
|
|
|
- def stored_seqs(task_id):
|
|
|
- conn = db()
|
|
|
- try:
|
|
|
- return [r[0] for r in conn.execute(
|
|
|
- "select seq from session_raw_log where session_id in"
|
|
|
- " (select id from sessions where task_id = ?)"
|
|
|
- " order by seq", (task_id,))]
|
|
|
- finally:
|
|
|
- conn.close()
|
|
|
-
|
|
|
- check(wait_for(lambda: task_state(auth, lossless).get("status") == "working"
|
|
|
- or False, timeout=60, what="lossless task working") is not None,
|
|
|
- "lossless task dispatches and pushes its opening flush")
|
|
|
- check(wait_for(lambda: stored_seqs(lossless) == [1, 2, 3, 4, 5, 6] or False,
|
|
|
- timeout=30, what="all six chunks stored") is not None,
|
|
|
- "every pushed chunk lands in session_raw_log, CRLF-only included (%s)"
|
|
|
- % stored_seqs(lossless))
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/chunks", fake.token,
|
|
|
- {"session_id": fake.session_id,
|
|
|
- "chunks": [dict(chunk) for chunk in corpus]})
|
|
|
- check(status == 200 and stored_seqs(lossless) == [1, 2, 3, 4, 5, 6],
|
|
|
- "a full-batch redelivery stores nothing new (exactly-once)")
|
|
|
- request = urllib.request.Request(
|
|
|
- WEB + "/terminal-stream/%d" % [r[0] for r in db().execute(
|
|
|
- "select id from sessions where task_id = ?", (lossless,))][0])
|
|
|
- with urllib.request.urlopen(request, timeout=15) as stream:
|
|
|
- texts = []
|
|
|
- deadline = time.monotonic() + 8
|
|
|
- while time.monotonic() < deadline and len(texts) < 6:
|
|
|
- line = stream.readline().decode().strip()
|
|
|
- if not line.startswith("data: ") or line == "data: ping":
|
|
|
- continue
|
|
|
- event = json.loads(line[6:])
|
|
|
- if event.get("type") == "raw":
|
|
|
- texts.append(event.get("text") or "")
|
|
|
- check("\r\n" in texts and len(texts) == 6,
|
|
|
- "SSE serves the whitespace chunk unfiltered")
|
|
|
- fake.hold_state = False
|
|
|
- fake.scripted_corpus = None
|
|
|
- check(wait_task_done(auth, lossless, timeout=60) is not None,
|
|
|
- "lossless task completes")
|
|
|
-
|
|
|
- print("== R9 B1/B2: git identity + ssh key ==")
|
|
|
- # The key travels on /status into the saddle row and the page view
|
|
|
- # (the manual refresh's exceptional probe).
|
|
|
- fake.ssh_public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1teStKey saddle-1@fedora"
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- check(fake_card.get("ssh_public_key", "") == "",
|
|
|
- "no key shown before the daemon reports one")
|
|
|
- act(auth, fake_card["refresh"], {})
|
|
|
- check(wait_for(lambda: "AAAAC3NzaC1lZDI1teStKey" in
|
|
|
- saddle_card("fake-saddle").get("ssh_public_key", "") or False,
|
|
|
- timeout=30, what="key lands on the row") is not None,
|
|
|
- "reported ssh_public_key persists onto the saddle row")
|
|
|
+ def stored_seqs(task_id):
|
|
|
conn = db()
|
|
|
- row_key = conn.execute("select ssh_public_key from saddles where name = 'fake-saddle'"
|
|
|
- ).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(row_key == fake.ssh_public_key,
|
|
|
- "the key is stored verbatim in the database")
|
|
|
-
|
|
|
- # Identity edit flows through the UI into the assignment payload.
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- act(auth, fake_card["update_identity"],
|
|
|
- {"git_name": "clanker-bot", "git_email": "bot@unitatem.net"})
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- check(fake_card.get("git_name") == "clanker-bot"
|
|
|
- and fake_card.get("git_email") == "bot@unitatem.net",
|
|
|
- "inline edit saves the git identity")
|
|
|
+ try:
|
|
|
+ return [r[0] for r in conn.execute(
|
|
|
+ "select seq from session_raw_log where session_id in"
|
|
|
+ " (select id from sessions where task_id = ?)"
|
|
|
+ " order by seq", (task_id,))]
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
|
|
|
+ check(wait_for(lambda: task_state(auth, lossless).get("status") == "working"
|
|
|
+ or False, timeout=60, what="lossless task working") is not None,
|
|
|
+ "lossless task dispatches and pushes its opening flush")
|
|
|
+ check(wait_for(lambda: stored_seqs(lossless) == [1, 2, 3, 4, 5, 6] or False,
|
|
|
+ timeout=30, what="all six chunks stored") is not None,
|
|
|
+ "every pushed chunk lands in session_raw_log, CRLF-only included (%s)"
|
|
|
+ % stored_seqs(lossless))
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/chunks", fake.token,
|
|
|
+ {"session_id": fake.session_id,
|
|
|
+ "chunks": [dict(chunk) for chunk in corpus]})
|
|
|
+ check(status == 200 and stored_seqs(lossless) == [1, 2, 3, 4, 5, 6],
|
|
|
+ "a full-batch redelivery stores nothing new (exactly-once)")
|
|
|
+ request = urllib.request.Request(
|
|
|
+ WEB + "/terminal-stream/%d" % [r[0] for r in db().execute(
|
|
|
+ "select id from sessions where task_id = ?", (lossless,))][0])
|
|
|
+ with urllib.request.urlopen(request, timeout=15) as stream:
|
|
|
+ texts = []
|
|
|
+ deadline = time.monotonic() + 8
|
|
|
+ while time.monotonic() < deadline and len(texts) < 6:
|
|
|
+ line = stream.readline().decode().strip()
|
|
|
+ if not line.startswith("data: ") or line == "data: ping":
|
|
|
+ continue
|
|
|
+ event = json.loads(line[6:])
|
|
|
+ if event.get("type") == "raw":
|
|
|
+ texts.append(event.get("text") or "")
|
|
|
+ check("\r\n" in texts and len(texts) == 6,
|
|
|
+ "SSE serves the whitespace chunk unfiltered")
|
|
|
+ fake.hold_state = False
|
|
|
+ fake.scripted_corpus = None
|
|
|
+ check(wait_task_done(auth, lossless, timeout=60) is not None,
|
|
|
+ "lossless task completes")
|
|
|
+
|
|
|
+ print("== R9 B1/B2: git identity + ssh key ==")
|
|
|
+ # The key travels on /status into the saddle row and the page view
|
|
|
+ # (the manual refresh's exceptional probe).
|
|
|
+ fake.ssh_public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1teStKey saddle-1@fedora"
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ check(fake_card.get("ssh_public_key", "") == "",
|
|
|
+ "no key shown before the daemon reports one")
|
|
|
+ act(auth, fake_card["refresh"], {})
|
|
|
+ check(wait_for(lambda: "AAAAC3NzaC1lZDI1teStKey" in
|
|
|
+ saddle_card(auth, "fake-saddle").get("ssh_public_key", "") or False,
|
|
|
+ timeout=30, what="key lands on the row") is not None,
|
|
|
+ "reported ssh_public_key persists onto the saddle row")
|
|
|
+ conn = db()
|
|
|
+ row_key = conn.execute("select ssh_public_key from saddles where name = 'fake-saddle'"
|
|
|
+ ).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(row_key == fake.ssh_public_key,
|
|
|
+ "the key is stored verbatim in the database")
|
|
|
+
|
|
|
+ # Identity edit flows through the UI into the assignment payload.
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ act(auth, fake_card["update_identity"],
|
|
|
+ {"git_name": "clanker-bot", "git_email": "bot@unitatem.net"})
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ check(fake_card.get("git_name") == "clanker-bot"
|
|
|
+ and fake_card.get("git_email") == "bot@unitatem.net",
|
|
|
+ "inline edit saves the git identity")
|
|
|
+
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Identity task",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ identity_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Identity task"][0])
|
|
|
+ check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == identity_task
|
|
|
+ for p in fake.start_payloads) or False,
|
|
|
+ timeout=60, what="identity task dispatched") is not None,
|
|
|
+ "identity task dispatches")
|
|
|
+ payload = [p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == identity_task][-1]
|
|
|
+ identity = payload.get("git_identity") or {}
|
|
|
+ check(identity.get("name") == "clanker-bot"
|
|
|
+ and identity.get("email") == "bot@unitatem.net",
|
|
|
+ "the assignment carries the saddle's git_identity")
|
|
|
+
|
|
|
+ # Earlier assignments (identity unset) carry no git_identity member.
|
|
|
+ check(all("git_identity" not in p for p in fake.start_payloads[:-1]),
|
|
|
+ "unset identity omits the member (daemon fallback applies)")
|
|
|
+
|
|
|
+ # Clearing the identity omits it from later assignments.
|
|
|
+ fake_card = saddle_card(auth, "fake-saddle")
|
|
|
+ act(auth, fake_card["update_identity"], {"git_name": "", "git_email": ""})
|
|
|
+ check(saddle_card(auth, "fake-saddle").get("git_name", "") == "",
|
|
|
+ "clearing the identity empties the row")
|
|
|
+ check(wait_task_done(auth, identity_task, timeout=60) is not None,
|
|
|
+ "identity task completes")
|
|
|
+
|
|
|
+ print("== phase 2: blocked push -> block row + pulse ==")
|
|
|
+ # The browser flow on each pulse: statum.act(task.refresh) — the
|
|
|
+ # sealed GET action must rehydrate the task slot with the new
|
|
|
+ # block the daemon just PUSHED (state blocked + block payload).
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Live block task",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ live_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Live block task"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, live_task).get("status") == "working"
|
|
|
+ or False, timeout=60, what="live task working") is not None,
|
|
|
+ "live-block task reaches working")
|
|
|
+ fake.hold_state = True
|
|
|
+ fake.state = "blocked" # the daemon parks on its question
|
|
|
+ status = fake.push_results("blocked", block={
|
|
|
+ "question": "Pick one", "type": "question", "options": ["A", "B"]})
|
|
|
+ check(status == 200, "blocked state push accepted")
|
|
|
+ # The parked daemon is idle-awake: it asks (and sleeps on 204).
|
|
|
+ fake.maybe_trigger_ask("parked")
|
|
|
+ check(wait_for(lambda: any(b.get("question") == "Pick one"
|
|
|
+ for b in task_state(auth, live_task).get("blocks", []))
|
|
|
+ or False, timeout=30, what="block lands in state") is not None,
|
|
|
+ "the pushed block reaches the sealed refresh without a reload")
|
|
|
+ detail = task_state(auth, live_task)
|
|
|
+ check(detail.get("status") == "blocked",
|
|
|
+ "the blocked push flips the task to blocked")
|
|
|
+ pushed_block = [b for b in detail["blocks"] if b.get("question") == "Pick one"][0]
|
|
|
+ conn = db()
|
|
|
+ options_json = conn.execute(
|
|
|
+ "select options from task_blocks where id = ?", (int(pushed_block["id"]),)
|
|
|
+ ).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(json.loads(options_json) == ["A", "B"],
|
|
|
+ "the block row carries its options payload")
|
|
|
+ check(any(s["status"] == "idle"
|
|
|
+ for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
|
|
|
+ "the blocked push parks the saddle")
|
|
|
+ detail = task_state(auth, live_task)
|
|
|
+ check(detail.get("refresh") is not None,
|
|
|
+ "the sealed GET refresh action is still authored")
|
|
|
+ status, _ = http("GET", detail["refresh"]["uri"], timeout=15)
|
|
|
+ check(status in (200, 401),
|
|
|
+ "the refresh URI resolves as a GET (client roundtrip shape)")
|
|
|
+ act(auth, pushed_block["answer_action"], {"answer": "A"})
|
|
|
+ # The answer makes the parked task resume-offerable: the affine
|
|
|
+ # saddle is woken, asks, and its assignment carries the answer.
|
|
|
+ pushes_before = len([p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == live_task])
|
|
|
+ check(wait_for(lambda: len([p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == live_task])
|
|
|
+ > pushes_before or False, timeout=60,
|
|
|
+ what="blocked task resumes") is not None,
|
|
|
+ "answering the pushed block resumes the parked workspace")
|
|
|
+ fake.hold_state = False # the resumed round runs to completion
|
|
|
+ resume_payload = [p for p in fake.start_payloads
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == live_task][-1]
|
|
|
+ answers = resume_payload.get("task", {}).get("answers", [])
|
|
|
+ check(any(a.get("answer") == "A" for a in answers),
|
|
|
+ "the resume assignment carries the human's answer")
|
|
|
+ check(wait_task_done(auth, live_task, timeout=60) is not None,
|
|
|
+ "live-block task completes after answering")
|
|
|
+
|
|
|
+
|
|
|
+# =============================================================================
|
|
|
+# Bundle: answers — the answer-resume machinery in both daemon shapes
|
|
|
+# (sleeping affine wake, mid-round verdict delivery), offline requeue
|
|
|
+# keeping parked-with-answered tasks, the staleness probe, and the
|
|
|
+# startup token migration (this shard's one deliberate app restart).
|
|
|
+# =============================================================================
|
|
|
+
|
|
|
+def bundle_answers(ctx):
|
|
|
+ auth, fake, fake_id, fake_token = ctx.auth, ctx.fake, ctx.fake_id, ctx.fake_token
|
|
|
+ app = ctx.app
|
|
|
+
|
|
|
+ print("== phase 2: answer resume — wake -> ask -> affinity claim ==")
|
|
|
+ # The full live sequence against a daemon that asks after its
|
|
|
+ # park: ask -> assignment -> blocked push (park) -> ask -> 204
|
|
|
+ # -> sleeps (row idle) -> human answers -> the dispatcher WAKES
|
|
|
+ # the sleeping affine saddle -> its ask allocates the resume,
|
|
|
+ # the verdict riding the assignment -> the session continues.
|
|
|
+ # A private capability tag keeps the fleet's other sleeping
|
|
|
+ # saddles from claiming these tasks.
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "answerflow"})
|
|
|
+ answerflow_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
+ if c["name"] == "answerflow"][0])
|
|
|
+
|
|
|
+ def answerflow_daemon(**kwargs):
|
|
|
+ daemon = FakeSaddle()
|
|
|
+ daemon.ask_on_register = False # manual, deterministic asks
|
|
|
+ daemon.hold_state = True # the tests drive the lifecycle
|
|
|
+ for key, value in kwargs.items():
|
|
|
+ setattr(daemon, key, value)
|
|
|
+ daemon.serve()
|
|
|
+ return daemon
|
|
|
+
|
|
|
+ def register_answerflow(daemon, name):
|
|
|
+ act(auth, slot_public(auth, "/saddles", "saddles")["register"], {
|
|
|
+ "name": name, "host": "127.0.0.1:%d" % daemon.port})
|
|
|
+ act(auth, saddle_card(auth, name)["attach_capability"],
|
|
|
+ {"capability_id": str(answerflow_cap)})
|
|
|
+
|
|
|
+ def create_answerflow_task(title):
|
|
|
act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Identity task",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- identity_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Identity task"][0])
|
|
|
- check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == identity_task
|
|
|
- for p in fake.start_payloads) or False,
|
|
|
- timeout=60, what="identity task dispatched") is not None,
|
|
|
- "identity task dispatches")
|
|
|
- payload = [p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == identity_task][-1]
|
|
|
- identity = payload.get("git_identity") or {}
|
|
|
- check(identity.get("name") == "clanker-bot"
|
|
|
- and identity.get("email") == "bot@unitatem.net",
|
|
|
- "the assignment carries the saddle's git_identity")
|
|
|
-
|
|
|
- # Earlier assignments (identity unset) carry no git_identity member.
|
|
|
- check(all("git_identity" not in p for p in fake.start_payloads[:-1]),
|
|
|
- "unset identity omits the member (daemon fallback applies)")
|
|
|
-
|
|
|
- # Clearing the identity omits it from later assignments.
|
|
|
- fake_card = saddle_card("fake-saddle")
|
|
|
- act(auth, fake_card["update_identity"], {"git_name": "", "git_email": ""})
|
|
|
- check(saddle_card("fake-saddle").get("git_name", "") == "",
|
|
|
- "clearing the identity empties the row")
|
|
|
- check(wait_task_done(auth, identity_task, timeout=60) is not None,
|
|
|
- "identity task completes")
|
|
|
-
|
|
|
- print("== R10: branch_advanced on the /poll verdict channel ==")
|
|
|
+ "project_id": "1", "title": title, "priority": "2",
|
|
|
+ "capability_id": [str(answerflow_cap)]})
|
|
|
+ return int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == title][0])
|
|
|
+
|
|
|
+ def retire_daemon(daemon, name):
|
|
|
+ # deterministic teardown: a dead daemon and an offline row
|
|
|
+ # never interfere with the sections that follow
|
|
|
+ daemon.kill()
|
|
|
+ act(auth, saddle_card(auth, name)["refresh"], {})
|
|
|
+
|
|
|
+ sleeper = answerflow_daemon(ask_on_wake=True)
|
|
|
+ register_answerflow(sleeper, "resume-sleeper")
|
|
|
+ sleeping_task = create_answerflow_task("Sleeping resume task")
|
|
|
+ ask_status, assignment = sleeper.ask_now()
|
|
|
+ check(ask_status == 200 and int(assignment["task"]["id"]) == sleeping_task,
|
|
|
+ "the daemon's ask claims the queued task")
|
|
|
+ sleeping_session = assignment.get("session_id")
|
|
|
+ sleeper.state = "blocked"
|
|
|
+ check(sleeper.push_results("blocked", block={
|
|
|
+ "question": "Pick a side", "type": "question",
|
|
|
+ "options": ["Port", "Starboard"]}) == 200,
|
|
|
+ "the blocked push parks the session")
|
|
|
+ check(wait_for(lambda: task_state(auth, sleeping_task).get("status") == "blocked"
|
|
|
+ or False, timeout=30, what="sleeping task parked") is not None,
|
|
|
+ "the task parks as blocked")
|
|
|
+ check(saddle_card(auth, "resume-sleeper")["status"] == "idle",
|
|
|
+ "the parked session's saddle row reads idle")
|
|
|
+ ask_status, _ = sleeper.ask_now()
|
|
|
+ check(ask_status == 204,
|
|
|
+ "the parked daemon's ask returns 204 — it sleeps until woken")
|
|
|
+ # Converted sleep(3): ticks pass with the daemon asleep and no
|
|
|
+ # wake may arrive — a wake during the window fails loudly.
|
|
|
+ check(quiet_window(quiet_seconds(),
|
|
|
+ lambda: len(sleeper.wake_log) == 0,
|
|
|
+ "sleeping daemon never woken"),
|
|
|
+ "the sleeping daemon is never woken while the block waits")
|
|
|
+ detail = task_state(auth, sleeping_task)
|
|
|
+ side_block = [b for b in detail["blocks"] if b["question"] == "Pick a side"][0]
|
|
|
+ act(auth, side_block["answer_action"], {"option": "Starboard"})
|
|
|
+ check(wait_for(lambda: len(sleeper.wake_log) > 0 or False,
|
|
|
+ timeout=30, what="wake after the answer") is not None,
|
|
|
+ "answering the block wakes the sleeping affine saddle")
|
|
|
+ check(wait_for(lambda: any(
|
|
|
+ int(p.get("task", {}).get("id") or 0) == sleeping_task
|
|
|
+ and p.get("session_id") == sleeping_session
|
|
|
+ and any(a.get("answer") == "Starboard"
|
|
|
+ for a in p.get("task", {}).get("answers", []))
|
|
|
+ for p in sleeper.start_payloads[1:]) or False,
|
|
|
+ timeout=30, what="resume allocation carries the verdict") is not None,
|
|
|
+ "the woken daemon's ask allocates the resume — same session, verdict carried")
|
|
|
+ # Barrier: the resume's own working push must commit before the
|
|
|
+ # manual done push (a late working write would otherwise race the
|
|
|
+ # completion's full-row write).
|
|
|
+ check(wait_for(lambda: task_state(auth, sleeping_task).get("status") == "working"
|
|
|
+ or False, timeout=15, what="resumed round working") is not None,
|
|
|
+ "the resumed round reports working before its completion push")
|
|
|
+ # The session continues: the resumed round runs to completion on
|
|
|
+ # the SAME session id.
|
|
|
+ sleeper.push_results("done", summary="sleep-resumed round complete")
|
|
|
+ check(wait_task_done(auth, sleeping_task, timeout=30) is not None,
|
|
|
+ "the resumed session continues to completion")
|
|
|
+ conn = db()
|
|
|
+ session_rows = conn.execute(
|
|
|
+ "select count(*) from sessions where task_id = ?", (sleeping_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(session_rows == 1,
|
|
|
+ "the sleeping resume kept one session throughout")
|
|
|
+ retire_daemon(sleeper, "resume-sleeper")
|
|
|
+
|
|
|
+ print("== phase 2: answer resume — verdict delivered to the parked session ==")
|
|
|
+ # The live regression's daemon shape (clanker-test.db): after the
|
|
|
+ # blocked push the daemon CANNOT ask again — its round is parked
|
|
|
+ # in the block's answer wait, so the saddle's last ask stays the
|
|
|
+ # one that claimed the task (never sleeping, never woken). The
|
|
|
+ # verdict must be DELIVERED into that answer wait.
|
|
|
+ parked = answerflow_daemon(ask_on_wake=False, # a blocked daemon cannot ask on wake
|
|
|
+ ask_on_terminal=False, # and not after the round either
|
|
|
+ ask_on_idle_status=False)
|
|
|
+ register_answerflow(parked, "resume-parked")
|
|
|
+ parked_task = create_answerflow_task("Parked resume task")
|
|
|
+ ask_status, assignment = parked.ask_now()
|
|
|
+ check(ask_status == 200 and int(assignment["task"]["id"]) == parked_task,
|
|
|
+ "the daemon's ask claims the task (its last ask)")
|
|
|
+ parked_session = assignment.get("session_id")
|
|
|
+ parked.state = "blocked"
|
|
|
+ check(parked.push_results("blocked", block={
|
|
|
+ "question": "Which database?", "type": "question",
|
|
|
+ "options": ["SQLite", "Postgres"]}) == 200,
|
|
|
+ "the blocked push parks the session mid-round")
|
|
|
+ check(wait_for(lambda: task_state(auth, parked_task).get("status") == "blocked"
|
|
|
+ or False, timeout=30, what="parked task blocked") is not None,
|
|
|
+ "the task parks as blocked")
|
|
|
+ check(saddle_card(auth, "resume-parked")["status"] == "idle",
|
|
|
+ "the parked session's saddle row reads idle")
|
|
|
+ # Converted sleep(3): ticks pass with no wake and no ask —
|
|
|
+ # either arrival during the window fails loudly.
|
|
|
+ asks_at_answer = len(parked.asks)
|
|
|
+ check(quiet_window(quiet_seconds(),
|
|
|
+ lambda: len(parked.wake_log) == 0
|
|
|
+ and len(parked.asks) == asks_at_answer,
|
|
|
+ "parked daemon never woken, never asks"),
|
|
|
+ "the parked daemon is never woken and never asks again (the live signature)")
|
|
|
+ detail = task_state(auth, parked_task)
|
|
|
+ db_block = [b for b in detail["blocks"] if b["question"] == "Which database?"][0]
|
|
|
+ act(auth, db_block["answer_action"], {"option": "SQLite"})
|
|
|
+ check(wait_for(lambda: len(parked.answer_log) > 0 or False,
|
|
|
+ timeout=30, what="verdict delivered") is not None,
|
|
|
+ "the verdict is delivered into the parked daemon's answer wait")
|
|
|
+ check(parked.answer_log[0]["answer"] == "SQLite",
|
|
|
+ "the delivered verdict is the human's choice")
|
|
|
+ check(parked.answer_log[0]["branch_advanced"] is False,
|
|
|
+ "the delivery carries branch_advanced=false with no siblings")
|
|
|
+ check(len(parked.wake_log) == 0 and len(parked.asks) == asks_at_answer,
|
|
|
+ "no wake, no ask — the delivery itself resumed the session")
|
|
|
+ check(wait_task_done(auth, parked_task, timeout=30) is not None,
|
|
|
+ "the parked session continues to completion")
|
|
|
+ conn = db()
|
|
|
+ row = conn.execute("select outcome from sessions where id = ?",
|
|
|
+ (parked_session,)).fetchone()
|
|
|
+ session_rows = conn.execute(
|
|
|
+ "select count(*) from sessions where task_id = ?", (parked_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(row is not None and row[0] == "done" and session_rows == 1,
|
|
|
+ "the delivered verdict resumed the SAME session to done")
|
|
|
+ retire_daemon(parked, "resume-parked")
|
|
|
+
|
|
|
+ print("== phase 2: offline requeue keeps parked-with-answered tasks ==")
|
|
|
+ # A daemon dying while its parked task carries an answered
|
|
|
+ # verdict: going offline must not requeue the task or close its
|
|
|
+ # session — the resume machinery (delivery once the daemon
|
|
|
+ # returns, or the affinity allocation on its next ask) needs
|
|
|
+ # both. The question park leaves the row idle; the review park
|
|
|
+ # keeps it busy@task (`saddle review` blocks the round without
|
|
|
+ # a parking state push), exercising the unreachable path's
|
|
|
+ # requeue decision both ways.
|
|
|
+ stranded = answerflow_daemon()
|
|
|
+ register_answerflow(stranded, "resume-stranded")
|
|
|
+ stranded_task = create_answerflow_task("Stranded answered task")
|
|
|
+ ask_status, _ = stranded.ask_now()
|
|
|
+ check(ask_status == 200, "the stranded daemon claims its task")
|
|
|
+ stranded.state = "blocked"
|
|
|
+ stranded.push_results("blocked", block={
|
|
|
+ "question": "Hold or fold?", "type": "question", "options": ["Hold", "Fold"]})
|
|
|
+ check(wait_for(lambda: task_state(auth, stranded_task).get("status") == "blocked"
|
|
|
+ or False, timeout=30, what="stranded task parked") is not None,
|
|
|
+ "the stranded task parks as blocked")
|
|
|
+ detail = task_state(auth, stranded_task)
|
|
|
+ hold_block = [b for b in detail["blocks"] if b["question"] == "Hold or fold?"][0]
|
|
|
+ act(auth, hold_block["answer_action"], {"option": "Hold"})
|
|
|
+ stranded.kill() # the daemon dies before the verdict is delivered
|
|
|
+ act(auth, saddle_card(auth, "resume-stranded")["refresh"], {})
|
|
|
+ check(wait_for(lambda: saddle_card(auth, "resume-stranded")["status"] == "offline"
|
|
|
+ or False, timeout=30, what="stranded saddle offline") is not None,
|
|
|
+ "the dead daemon's saddle row goes offline")
|
|
|
+ # Converted sleep(7): offline ticks run while the task MUST stay
|
|
|
+ # parked — a requeue during the window fails loudly.
|
|
|
+ check(quiet_window(quiet_seconds(),
|
|
|
+ lambda: task_state(auth, stranded_task)["status"] == "blocked",
|
|
|
+ "stranded task not requeued"),
|
|
|
+ "the parked-with-answered task is not requeued by the offline pass")
|
|
|
+ conn = db()
|
|
|
+ open_session = conn.execute(
|
|
|
+ "select count(*) from sessions where task_id = ? and ended_at is null",
|
|
|
+ (stranded_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(open_session == 1,
|
|
|
+ "the parked-with-answered task keeps its open session for the resume")
|
|
|
+
|
|
|
+ reviewer = answerflow_daemon(ask_on_terminal=False)
|
|
|
+ register_answerflow(reviewer, "resume-reviewer")
|
|
|
+ review_task = create_answerflow_task("Review park task")
|
|
|
+ ask_status, _ = reviewer.ask_now()
|
|
|
+ check(ask_status == 200, "the reviewer claims its task")
|
|
|
+ reviewer.queue_output("review", "saddle review", "Ready for review: wired and tested.")
|
|
|
+ check(wait_for(lambda: task_state(auth, review_task).get("status") == "review_requested"
|
|
|
+ or False, timeout=30, what="review park raised") is not None,
|
|
|
+ "the review entry parks the task as review_requested (row stays busy)")
|
|
|
+ reviewer.kill() # the daemon dies before the verdict can be delivered
|
|
|
+ detail = task_state(auth, review_task)
|
|
|
+ verdict_block = [b for b in detail["blocks"]
|
|
|
+ if b["block_type"] == "review"][0]
|
|
|
+ act(auth, verdict_block["answer_action"], {"answer": "Human approved the review"})
|
|
|
+ check(wait_for(lambda: saddle_card(auth, "resume-reviewer")["status"] == "offline"
|
|
|
+ or False, timeout=30, what="reviewer offline") is not None,
|
|
|
+ "the failed verdict delivery drives the dead reviewer offline")
|
|
|
+ # Converted sleep(7): offline ticks run while the task MUST stay
|
|
|
+ # review-parked — a requeue during the window fails loudly.
|
|
|
+ check(quiet_window(quiet_seconds(),
|
|
|
+ lambda: task_state(auth, review_task)["status"] == "review_requested",
|
|
|
+ "review-parked task not requeued"),
|
|
|
+ "the review-parked task with an answered verdict is never requeued")
|
|
|
+ conn = db()
|
|
|
+ open_session = conn.execute(
|
|
|
+ "select count(*) from sessions where task_id = ? and ended_at is null",
|
|
|
+ (review_task,)).fetchone()[0]
|
|
|
+ conn.close()
|
|
|
+ check(open_session == 1,
|
|
|
+ "the review-parked task keeps its open session for the resume")
|
|
|
+
|
|
|
+ print("== phase 2: staleness — one probe decides ==")
|
|
|
+ # A silent-but-alive runner (long think, nothing to flush): after
|
|
|
+ # SADDLE_STALE_S (3s here) of silence it owes ONE exceptional
|
|
|
+ # GET /status; the answer keeps it running and the silence
|
|
|
+ # episode gets no second probe.
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Silent thinker",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ thinker = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Silent thinker"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, thinker).get("status") == "working"
|
|
|
+ or False, timeout=60, what="thinker working") is not None,
|
|
|
+ "silent thinker reaches working")
|
|
|
+ fake.silent = True # the daemon goes quiet mid-round (alive)
|
|
|
+ probes_at_start = fake.status_hits
|
|
|
+ check(wait_for(lambda: fake.status_hits > probes_at_start or False,
|
|
|
+ timeout=30, what="exceptional probe arrives") is not None,
|
|
|
+ "a silent runner gets its exceptional GET /status probe")
|
|
|
+ # Converted sleep(8)+sleep(6): a quiet window across several
|
|
|
+ # ticks in which the probe count MUST NOT move — a second probe
|
|
|
+ # (the poll drumbeat this guards against) fails loudly.
|
|
|
+ second_window = fake.status_hits
|
|
|
+ check(quiet_window(quiet_seconds(),
|
|
|
+ lambda: fake.status_hits == second_window,
|
|
|
+ "no second staleness probe"),
|
|
|
+ "the silence episode is probed exactly once (no poll drumbeat)")
|
|
|
+ check(task_state(auth, thinker).get("status") == "working"
|
|
|
+ and saddle_card(auth, "fake-saddle")["status"] == "busy",
|
|
|
+ "the answered probe keeps the task running (no requeue)")
|
|
|
+ # Output flows again: the episode ends, the clock resets.
|
|
|
+ fake.silent = False
|
|
|
+ fake.push_chunks(["resumed output\r\n"])
|
|
|
+ check(wait_task_done(auth, thinker, timeout=60) is not None,
|
|
|
+ "the silent thinker completes once output flows again")
|
|
|
+
|
|
|
+ # The daemon vanishing mid-task: silence for the window, the
|
|
|
+ # probe fails, the task requeues with a bot comment and the
|
|
|
+ # saddle goes offline.
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
+ "project_id": "1",
|
|
|
+ "title": "Vanish victim",
|
|
|
+ "priority": "2",
|
|
|
+ })
|
|
|
+ vanish = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if t["title"] == "Vanish victim"][0])
|
|
|
+ check(wait_for(lambda: task_state(auth, vanish).get("status") == "working"
|
|
|
+ or False, timeout=60, what="vanish victim working") is not None,
|
|
|
+ "vanish victim reaches working")
|
|
|
+ fake.kill()
|
|
|
+ check(wait_for(lambda: saddle_card(auth, "fake-saddle")["status"] == "offline" or False,
|
|
|
+ timeout=60, what="daemon death -> offline") is not None,
|
|
|
+ "vanishing daemon drives the saddle offline (failed staleness probe)")
|
|
|
+ check(wait_for(lambda: any("unreachable" in c["body"]
|
|
|
+ for c in task_state(auth, vanish).get("comments", []))
|
|
|
+ or False, timeout=30, what="unreachable comment") is not None,
|
|
|
+ "the affected task gets an unreachable bot comment")
|
|
|
+ check(task_state(auth, vanish)["status"] == "queued",
|
|
|
+ "the task requeues after the daemon death")
|
|
|
+ card = saddle_card(auth, "fake-saddle")
|
|
|
+ check(card.get("health") == "offline",
|
|
|
+ "health badge reports offline")
|
|
|
+
|
|
|
+ # The unreachable row flag marks the detection window: a task the
|
|
|
+ # web still sees working while its saddle is offline/fenced, before
|
|
|
+ # the requeue pass moves it. Controlled window (the real
|
|
|
+ # transitions above are atomic with their requeues).
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
+ " created_at) values (1, 'Unreachable row', 'working', 5,"
|
|
|
+ " strftime('%s','now'))")
|
|
|
+ unreachable_row = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
+ conn.execute("insert into sessions (task_id, saddle_id, started_at)"
|
|
|
+ " values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+ row = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
+ if int(t["id"]) == unreachable_row][0]
|
|
|
+ check(row.get("unreachable") is True,
|
|
|
+ "queue row shows the unreachable state in the window")
|
|
|
+
|
|
|
+ print("== phase 1: startup migrates the per-saddle token ==")
|
|
|
+ # Saddles registered before the flip hold legacy api_tokens rows.
|
|
|
+ # On startup every saddle row gets its canonical token: the
|
|
|
+ # legacy value it already holds (daemons keep their credential)
|
|
|
+ # or a freshly minted one. Fixture: this shard's own racer.
|
|
|
+ fake2 = FakeSaddle()
|
|
|
+ fake2.ask_on_register = False
|
|
|
+ fake2.serve(port=free_port())
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
+ act(auth, saddles_state["register"], {"name": "racer-two",
|
|
|
+ "host": "127.0.0.1:%d" % fake2.port})
|
|
|
+ racer_two_id = int([s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
+ if s["name"] == "racer-two"][0]["id"])
|
|
|
+ conn = db()
|
|
|
+ conn.execute("insert into api_tokens (saddle_id, token, created_at)"
|
|
|
+ " values (?, 'saddle-legacy-copied-token', strftime('%s','now'))",
|
|
|
+ (racer_two_id,))
|
|
|
+ conn.execute("update saddles set token = null where name = 'racer-two'")
|
|
|
+ conn.execute("insert into saddles (name, host, status, current_task_id)"
|
|
|
+ " values ('tokenless-orphan', '127.0.0.1:1', 'offline', 0)")
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
+
|
|
|
+ app.terminate()
|
|
|
+ app.wait(timeout=10)
|
|
|
+ app = start_app()
|
|
|
+ auth = login("admin", "adminpass123")
|
|
|
+ ctx.app = app # teardown tracks the restarted process
|
|
|
+ ctx.auth = auth
|
|
|
+
|
|
|
+ def saddle_row(name):
|
|
|
conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
- " created_at) values (1, 'Poll grandparent', 'working', 5,"
|
|
|
- " strftime('%s','now','-300 seconds'))")
|
|
|
- poll_grand = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
- " parent_task_id, created_at) values (1, 'Poll answer parent',"
|
|
|
- " 'blocked', 5, ?, strftime('%s','now','-120 seconds'))",
|
|
|
- (poll_grand,))
|
|
|
- poll_parent = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
- conn.execute("insert into sessions (task_id, saddle_id, started_at)"
|
|
|
- " values (?, ?, strftime('%s','now','-120 seconds'))",
|
|
|
- (poll_parent, fake_id))
|
|
|
- conn.execute("insert into task_blocks (task_id, question, type, answer,"
|
|
|
- " answered_at, created_at) values (?, 'Which DB?', 'question',"
|
|
|
- " 'SQLite', strftime('%s','now','-60 seconds'),"
|
|
|
- " strftime('%s','now','-100 seconds'))", (poll_parent,))
|
|
|
- conn.execute("update saddles set status = 'busy', current_task_id = ?,"
|
|
|
- " allocated_at = strftime('%s','now') where name = 'fake-saddle'",
|
|
|
- (poll_parent,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- status, body = saddle_api("GET", "/api/saddle/%d/poll" % fake_id, fake_token)
|
|
|
- check(status == 200 and len(body.get("answers", [])) == 1
|
|
|
- and body["answers"][0].get("branch_advanced") is False,
|
|
|
- "/poll verdicts carry branch_advanced=false with no siblings")
|
|
|
+ try:
|
|
|
+ return conn.execute("select id, token from saddles where name = ?",
|
|
|
+ (name,)).fetchone()
|
|
|
+ finally:
|
|
|
+ conn.close()
|
|
|
|
|
|
- # A sibling under the same parent completed after the park marker.
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
- " parent_task_id, summary, completed_at, created_at)"
|
|
|
- " values (1, 'Poll sibling', 'done', 5, ?, 'sibling finished',"
|
|
|
- " strftime('%s','now'), strftime('%s','now','-200 seconds'))",
|
|
|
- (poll_grand,))
|
|
|
- conn.execute("insert into task_blocks (task_id, question, type, answer,"
|
|
|
- " answered_at, created_at) values (?, 'Second question?',"
|
|
|
- " 'question', 'Yes', strftime('%s','now'),"
|
|
|
- " strftime('%s','now','-90 seconds'))", (poll_parent,))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- status, body = saddle_api("GET", "/api/saddle/%d/poll" % fake_id, fake_token)
|
|
|
- check(status == 200 and len(body.get("answers", [])) == 1
|
|
|
- and body["answers"][0].get("branch_advanced") is True,
|
|
|
- "sibling completion after park flips branch_advanced to true")
|
|
|
-
|
|
|
- # Release the staged fixture: cancel it so its busy@task row does
|
|
|
- # not linger into the sections below.
|
|
|
- act(auth, task_state(auth, poll_parent)["cancel"], {})
|
|
|
-
|
|
|
- print("== phase 2: blocked push -> block row + pulse ==")
|
|
|
- # The browser flow on each pulse: statum.act(task.refresh) — the
|
|
|
- # sealed GET action must rehydrate the task slot with the new
|
|
|
- # block the daemon just PUSHED (state blocked + block payload).
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Live block task",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- live_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Live block task"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, live_task).get("status") == "working"
|
|
|
- or False, timeout=60, what="live task working") is not None,
|
|
|
- "live-block task reaches working")
|
|
|
- fake.hold_state = True
|
|
|
- fake.state = "blocked" # the daemon parks on its question
|
|
|
- status = fake.push_results("blocked", block={
|
|
|
- "question": "Pick one", "type": "question", "options": ["A", "B"]})
|
|
|
- check(status == 200, "blocked state push accepted")
|
|
|
- # The parked daemon is idle-awake: it asks (and sleeps on 204).
|
|
|
- fake.maybe_trigger_ask("parked")
|
|
|
- check(wait_for(lambda: any(b.get("question") == "Pick one"
|
|
|
- for b in task_state(auth, live_task).get("blocks", []))
|
|
|
- or False, timeout=30, what="block lands in state") is not None,
|
|
|
- "the pushed block reaches the sealed refresh without a reload")
|
|
|
- detail = task_state(auth, live_task)
|
|
|
- check(detail.get("status") == "blocked",
|
|
|
- "the blocked push flips the task to blocked")
|
|
|
- pushed_block = [b for b in detail["blocks"] if b.get("question") == "Pick one"][0]
|
|
|
- conn = db()
|
|
|
- options_json = conn.execute(
|
|
|
- "select options from task_blocks where id = ?", (int(pushed_block["id"]),)
|
|
|
- ).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(json.loads(options_json) == ["A", "B"],
|
|
|
- "the block row carries its options payload")
|
|
|
- check(any(s["status"] == "idle"
|
|
|
- for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
|
|
|
- "the blocked push parks the saddle")
|
|
|
- detail = task_state(auth, live_task)
|
|
|
- check(detail.get("refresh") is not None,
|
|
|
- "the sealed GET refresh action is still authored")
|
|
|
- status, _ = http("GET", detail["refresh"]["uri"], timeout=15)
|
|
|
- check(status in (200, 401),
|
|
|
- "the refresh URI resolves as a GET (client roundtrip shape)")
|
|
|
- act(auth, pushed_block["answer_action"], {"answer": "A"})
|
|
|
- # The answer makes the parked task resume-offerable: the affine
|
|
|
- # saddle is woken, asks, and its assignment carries the answer.
|
|
|
- pushes_before = len([p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == live_task])
|
|
|
- check(wait_for(lambda: len([p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == live_task])
|
|
|
- > pushes_before or False, timeout=60,
|
|
|
- what="blocked task resumes") is not None,
|
|
|
- "answering the pushed block resumes the parked workspace")
|
|
|
- fake.hold_state = False # the resumed round runs to completion
|
|
|
- resume_payload = [p for p in fake.start_payloads
|
|
|
- if int(p.get("task", {}).get("id") or 0) == live_task][-1]
|
|
|
- answers = resume_payload.get("task", {}).get("answers", [])
|
|
|
- check(any(a.get("answer") == "A" for a in answers),
|
|
|
- "the resume assignment carries the human's answer")
|
|
|
- check(wait_task_done(auth, live_task, timeout=60) is not None,
|
|
|
- "live-block task completes after answering")
|
|
|
-
|
|
|
- print("== phase 2: answer resume — wake -> ask -> affinity claim ==")
|
|
|
- # The full live sequence against a daemon that asks after its
|
|
|
- # park: ask -> assignment -> blocked push (park) -> ask -> 204
|
|
|
- # -> sleeps (row idle) -> human answers -> the dispatcher WAKES
|
|
|
- # the sleeping affine saddle -> its ask allocates the resume,
|
|
|
- # the verdict riding the assignment -> the session continues.
|
|
|
- # A private capability tag keeps the fleet's other sleeping
|
|
|
- # saddles from claiming these tasks.
|
|
|
- saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
- act(auth, saddles_state["create_capability"], {"name": "answerflow"})
|
|
|
- answerflow_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
|
|
|
- if c["name"] == "answerflow"][0])
|
|
|
-
|
|
|
- def answerflow_daemon(**kwargs):
|
|
|
- daemon = FakeSaddle()
|
|
|
- daemon.ask_on_register = False # manual, deterministic asks
|
|
|
- daemon.hold_state = True # the tests drive the lifecycle
|
|
|
- for key, value in kwargs.items():
|
|
|
- setattr(daemon, key, value)
|
|
|
- daemon.serve()
|
|
|
- return daemon
|
|
|
-
|
|
|
- def register_answerflow(daemon, name):
|
|
|
- act(auth, slot_public(auth, "/saddles", "saddles")["register"], {
|
|
|
- "name": name, "host": "127.0.0.1:%d" % daemon.port})
|
|
|
- act(auth, saddle_card(name)["attach_capability"],
|
|
|
- {"capability_id": str(answerflow_cap)})
|
|
|
-
|
|
|
- def create_answerflow_task(title):
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1", "title": title, "priority": "2",
|
|
|
- "capability_id": [str(answerflow_cap)]})
|
|
|
- return int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == title][0])
|
|
|
-
|
|
|
- def retire_daemon(daemon, name):
|
|
|
- # deterministic teardown: a dead daemon and an offline row
|
|
|
- # never interfere with the sections that follow
|
|
|
- daemon.kill()
|
|
|
- act(auth, saddle_card(name)["refresh"], {})
|
|
|
-
|
|
|
- sleeper = answerflow_daemon(ask_on_wake=True)
|
|
|
- register_answerflow(sleeper, "resume-sleeper")
|
|
|
- sleeping_task = create_answerflow_task("Sleeping resume task")
|
|
|
- ask_status, assignment = sleeper.ask_now()
|
|
|
- check(ask_status == 200 and int(assignment["task"]["id"]) == sleeping_task,
|
|
|
- "the daemon's ask claims the queued task")
|
|
|
- sleeping_session = assignment.get("session_id")
|
|
|
- sleeper.state = "blocked"
|
|
|
- check(sleeper.push_results("blocked", block={
|
|
|
- "question": "Pick a side", "type": "question",
|
|
|
- "options": ["Port", "Starboard"]}) == 200,
|
|
|
- "the blocked push parks the session")
|
|
|
- check(wait_for(lambda: task_state(auth, sleeping_task).get("status") == "blocked"
|
|
|
- or False, timeout=30, what="sleeping task parked") is not None,
|
|
|
- "the task parks as blocked")
|
|
|
- check(saddle_card("resume-sleeper")["status"] == "idle",
|
|
|
- "the parked session's saddle row reads idle")
|
|
|
- ask_status, _ = sleeper.ask_now()
|
|
|
- check(ask_status == 204,
|
|
|
- "the parked daemon's ask returns 204 — it sleeps until woken")
|
|
|
- time.sleep(3) # several dispatch ticks pass with the daemon asleep
|
|
|
- check(len(sleeper.wake_log) == 0,
|
|
|
- "the sleeping daemon is never woken while the block waits")
|
|
|
- detail = task_state(auth, sleeping_task)
|
|
|
- side_block = [b for b in detail["blocks"] if b["question"] == "Pick a side"][0]
|
|
|
- act(auth, side_block["answer_action"], {"option": "Starboard"})
|
|
|
- check(wait_for(lambda: len(sleeper.wake_log) > 0 or False,
|
|
|
- timeout=30, what="wake after the answer") is not None,
|
|
|
- "answering the block wakes the sleeping affine saddle")
|
|
|
- check(wait_for(lambda: any(
|
|
|
- int(p.get("task", {}).get("id") or 0) == sleeping_task
|
|
|
- and p.get("session_id") == sleeping_session
|
|
|
- and any(a.get("answer") == "Starboard"
|
|
|
- for a in p.get("task", {}).get("answers", []))
|
|
|
- for p in sleeper.start_payloads[1:]) or False,
|
|
|
- timeout=30, what="resume allocation carries the verdict") is not None,
|
|
|
- "the woken daemon's ask allocates the resume — same session, verdict carried")
|
|
|
- # The session continues: the resumed round runs to completion on
|
|
|
- # the SAME session id.
|
|
|
- sleeper.push_results("done", summary="sleep-resumed round complete")
|
|
|
- check(wait_task_done(auth, sleeping_task, timeout=30) is not None,
|
|
|
- "the resumed session continues to completion")
|
|
|
- conn = db()
|
|
|
- session_rows = conn.execute(
|
|
|
- "select count(*) from sessions where task_id = ?", (sleeping_task,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(session_rows == 1,
|
|
|
- "the sleeping resume kept one session throughout")
|
|
|
- retire_daemon(sleeper, "resume-sleeper")
|
|
|
-
|
|
|
- print("== phase 2: answer resume — verdict delivered to the parked session ==")
|
|
|
- # The live regression's daemon shape (clanker-test.db): after the
|
|
|
- # blocked push the daemon CANNOT ask again — its round is parked
|
|
|
- # in the block's answer wait, so the saddle's last ask stays the
|
|
|
- # one that claimed the task (never sleeping, never woken). The
|
|
|
- # verdict must be DELIVERED into that answer wait.
|
|
|
- parked = answerflow_daemon(ask_on_wake=False, # a blocked daemon cannot ask on wake
|
|
|
- ask_on_terminal=False, # and not after the round either
|
|
|
- ask_on_idle_status=False)
|
|
|
- register_answerflow(parked, "resume-parked")
|
|
|
- parked_task = create_answerflow_task("Parked resume task")
|
|
|
- ask_status, assignment = parked.ask_now()
|
|
|
- check(ask_status == 200 and int(assignment["task"]["id"]) == parked_task,
|
|
|
- "the daemon's ask claims the task (its last ask)")
|
|
|
- parked_session = assignment.get("session_id")
|
|
|
- parked.state = "blocked"
|
|
|
- check(parked.push_results("blocked", block={
|
|
|
- "question": "Which database?", "type": "question",
|
|
|
- "options": ["SQLite", "Postgres"]}) == 200,
|
|
|
- "the blocked push parks the session mid-round")
|
|
|
- check(wait_for(lambda: task_state(auth, parked_task).get("status") == "blocked"
|
|
|
- or False, timeout=30, what="parked task blocked") is not None,
|
|
|
- "the task parks as blocked")
|
|
|
- check(saddle_card("resume-parked")["status"] == "idle",
|
|
|
- "the parked session's saddle row reads idle")
|
|
|
- time.sleep(3) # dispatch ticks pass: no wake, no ask ever reaches it
|
|
|
- asks_at_answer = len(parked.asks)
|
|
|
- check(len(parked.wake_log) == 0 and asks_at_answer == 1,
|
|
|
- "the parked daemon is never woken and never asks again (the live signature)")
|
|
|
- detail = task_state(auth, parked_task)
|
|
|
- db_block = [b for b in detail["blocks"] if b["question"] == "Which database?"][0]
|
|
|
- act(auth, db_block["answer_action"], {"option": "SQLite"})
|
|
|
- check(wait_for(lambda: len(parked.answer_log) > 0 or False,
|
|
|
- timeout=30, what="verdict delivered") is not None,
|
|
|
- "the verdict is delivered into the parked daemon's answer wait")
|
|
|
- check(parked.answer_log[0]["answer"] == "SQLite",
|
|
|
- "the delivered verdict is the human's choice")
|
|
|
- check(parked.answer_log[0]["branch_advanced"] is False,
|
|
|
- "the delivery carries branch_advanced=false with no siblings")
|
|
|
- check(len(parked.wake_log) == 0 and len(parked.asks) == asks_at_answer,
|
|
|
- "no wake, no ask — the delivery itself resumed the session")
|
|
|
- check(wait_task_done(auth, parked_task, timeout=30) is not None,
|
|
|
- "the parked session continues to completion")
|
|
|
- conn = db()
|
|
|
- row = conn.execute("select outcome from sessions where id = ?",
|
|
|
- (parked_session,)).fetchone()
|
|
|
- session_rows = conn.execute(
|
|
|
- "select count(*) from sessions where task_id = ?", (parked_task,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(row is not None and row[0] == "done" and session_rows == 1,
|
|
|
- "the delivered verdict resumed the SAME session to done")
|
|
|
- retire_daemon(parked, "resume-parked")
|
|
|
-
|
|
|
- print("== phase 2: offline requeue keeps parked-with-answered tasks ==")
|
|
|
- # A daemon dying while its parked task carries an answered
|
|
|
- # verdict: going offline must not requeue the task or close its
|
|
|
- # session — the resume machinery (delivery once the daemon
|
|
|
- # returns, or the affinity allocation on its next ask) needs
|
|
|
- # both. The question park leaves the row idle; the review park
|
|
|
- # keeps it busy@task (`saddle review` blocks the round without
|
|
|
- # a parking state push), exercising the unreachable path's
|
|
|
- # requeue decision both ways.
|
|
|
- stranded = answerflow_daemon()
|
|
|
- register_answerflow(stranded, "resume-stranded")
|
|
|
- stranded_task = create_answerflow_task("Stranded answered task")
|
|
|
- ask_status, _ = stranded.ask_now()
|
|
|
- check(ask_status == 200, "the stranded daemon claims its task")
|
|
|
- stranded.state = "blocked"
|
|
|
- stranded.push_results("blocked", block={
|
|
|
- "question": "Hold or fold?", "type": "question", "options": ["Hold", "Fold"]})
|
|
|
- check(wait_for(lambda: task_state(auth, stranded_task).get("status") == "blocked"
|
|
|
- or False, timeout=30, what="stranded task parked") is not None,
|
|
|
- "the stranded task parks as blocked")
|
|
|
- detail = task_state(auth, stranded_task)
|
|
|
- hold_block = [b for b in detail["blocks"] if b["question"] == "Hold or fold?"][0]
|
|
|
- act(auth, hold_block["answer_action"], {"option": "Hold"})
|
|
|
- stranded.kill() # the daemon dies before the verdict is delivered
|
|
|
- act(auth, saddle_card("resume-stranded")["refresh"], {})
|
|
|
- check(wait_for(lambda: saddle_card("resume-stranded")["status"] == "offline"
|
|
|
- or False, timeout=30, what="stranded saddle offline") is not None,
|
|
|
- "the dead daemon's saddle row goes offline")
|
|
|
- time.sleep(7) # dispatch ticks run their offline passes
|
|
|
- check(task_state(auth, stranded_task)["status"] == "blocked",
|
|
|
- "the parked-with-answered task is not requeued by the offline pass")
|
|
|
- conn = db()
|
|
|
- open_session = conn.execute(
|
|
|
- "select count(*) from sessions where task_id = ? and ended_at is null",
|
|
|
- (stranded_task,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(open_session == 1,
|
|
|
- "the parked-with-answered task keeps its open session for the resume")
|
|
|
-
|
|
|
- reviewer = answerflow_daemon(ask_on_terminal=False)
|
|
|
- register_answerflow(reviewer, "resume-reviewer")
|
|
|
- review_task = create_answerflow_task("Review park task")
|
|
|
- ask_status, _ = reviewer.ask_now()
|
|
|
- check(ask_status == 200, "the reviewer claims its task")
|
|
|
- reviewer.queue_output("review", "saddle review", "Ready for review: wired and tested.")
|
|
|
- check(wait_for(lambda: task_state(auth, review_task).get("status") == "review_requested"
|
|
|
- or False, timeout=30, what="review park raised") is not None,
|
|
|
- "the review entry parks the task as review_requested (row stays busy)")
|
|
|
- reviewer.kill() # the daemon dies before the verdict can be delivered
|
|
|
- detail = task_state(auth, review_task)
|
|
|
- verdict_block = [b for b in detail["blocks"]
|
|
|
- if b["block_type"] == "review"][0]
|
|
|
- act(auth, verdict_block["answer_action"], {"answer": "Human approved the review"})
|
|
|
- check(wait_for(lambda: saddle_card("resume-reviewer")["status"] == "offline"
|
|
|
- or False, timeout=30, what="reviewer offline") is not None,
|
|
|
- "the failed verdict delivery drives the dead reviewer offline")
|
|
|
- time.sleep(7) # dispatch ticks run their offline passes
|
|
|
- check(task_state(auth, review_task)["status"] == "review_requested",
|
|
|
- "the review-parked task with an answered verdict is never requeued")
|
|
|
- conn = db()
|
|
|
- open_session = conn.execute(
|
|
|
- "select count(*) from sessions where task_id = ? and ended_at is null",
|
|
|
- (review_task,)).fetchone()[0]
|
|
|
- conn.close()
|
|
|
- check(open_session == 1,
|
|
|
- "the review-parked task keeps its open session for the resume")
|
|
|
-
|
|
|
- print("== phase 2: staleness — one probe decides ==")
|
|
|
- # A silent-but-alive runner (long think, nothing to flush): after
|
|
|
- # SADDLE_STALE_S (3s here) of silence it owes ONE exceptional
|
|
|
- # GET /status; the answer keeps it running and the silence
|
|
|
- # episode gets no second probe.
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Silent thinker",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- thinker = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Silent thinker"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, thinker).get("status") == "working"
|
|
|
- or False, timeout=60, what="thinker working") is not None,
|
|
|
- "silent thinker reaches working")
|
|
|
- fake.silent = True # the daemon goes quiet mid-round (alive)
|
|
|
- probes_at_start = fake.status_hits
|
|
|
- check(wait_for(lambda: fake.status_hits > probes_at_start or False,
|
|
|
- timeout=30, what="exceptional probe arrives") is not None,
|
|
|
- "a silent runner gets its exceptional GET /status probe")
|
|
|
- time.sleep(8)
|
|
|
- second_window = fake.status_hits
|
|
|
- time.sleep(6)
|
|
|
- check(fake.status_hits == second_window,
|
|
|
- "the silence episode is probed exactly once (no poll drumbeat)")
|
|
|
- check(task_state(auth, thinker).get("status") == "working"
|
|
|
- and saddle_card("fake-saddle")["status"] == "busy",
|
|
|
- "the answered probe keeps the task running (no requeue)")
|
|
|
- # Output flows again: the episode ends, the clock resets.
|
|
|
- fake.silent = False
|
|
|
- fake.push_chunks(["resumed output\r\n"])
|
|
|
- check(wait_task_done(auth, thinker, timeout=60) is not None,
|
|
|
- "the silent thinker completes once output flows again")
|
|
|
-
|
|
|
- # The daemon vanishing mid-task: silence for the window, the
|
|
|
- # probe fails, the task requeues with a bot comment and the
|
|
|
- # saddle goes offline.
|
|
|
- act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
- "project_id": "1",
|
|
|
- "title": "Vanish victim",
|
|
|
- "priority": "2",
|
|
|
- })
|
|
|
- vanish = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if t["title"] == "Vanish victim"][0])
|
|
|
- check(wait_for(lambda: task_state(auth, vanish).get("status") == "working"
|
|
|
- or False, timeout=60, what="vanish victim working") is not None,
|
|
|
- "vanish victim reaches working")
|
|
|
- fake.kill()
|
|
|
- check(wait_for(lambda: saddle_card("fake-saddle")["status"] == "offline" or False,
|
|
|
- timeout=60, what="daemon death -> offline") is not None,
|
|
|
- "vanishing daemon drives the saddle offline (failed staleness probe)")
|
|
|
- check(wait_for(lambda: any("unreachable" in c["body"]
|
|
|
- for c in task_state(auth, vanish).get("comments", []))
|
|
|
- or False, timeout=30, what="unreachable comment") is not None,
|
|
|
- "the affected task gets an unreachable bot comment")
|
|
|
- check(task_state(auth, vanish)["status"] == "queued",
|
|
|
- "the task requeues after the daemon death")
|
|
|
- card = saddle_card("fake-saddle")
|
|
|
- check(card.get("health") == "offline",
|
|
|
- "health badge reports offline")
|
|
|
-
|
|
|
- # The unreachable row flag marks the detection window: a task the
|
|
|
- # web still sees working while its saddle is offline/fenced, before
|
|
|
- # the requeue pass moves it. Controlled window (the real
|
|
|
- # transitions above are atomic with their requeues).
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into tasks (project_id, title, status, priority,"
|
|
|
- " created_at) values (1, 'Unreachable row', 'working', 5,"
|
|
|
- " strftime('%s','now'))")
|
|
|
- unreachable_row = conn.execute("select max(id) from tasks").fetchone()[0]
|
|
|
- conn.execute("insert into sessions (task_id, saddle_id, started_at)"
|
|
|
- " values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
- row = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
- if int(t["id"]) == unreachable_row][0]
|
|
|
- check(row.get("unreachable") is True,
|
|
|
- "queue row shows the unreachable state in the window")
|
|
|
-
|
|
|
- print("== phase 1: startup migrates the per-saddle token ==")
|
|
|
- # Saddles registered before the flip hold legacy api_tokens rows.
|
|
|
- # On startup every saddle row gets its canonical token: the
|
|
|
- # legacy value it already holds (daemons keep their credential)
|
|
|
- # or a freshly minted one.
|
|
|
- conn = db()
|
|
|
- conn.execute("insert into api_tokens (saddle_id, token, created_at)"
|
|
|
- " values (?, 'saddle-legacy-copied-token', strftime('%s','now'))",
|
|
|
- (int(racer_cards[fake2]["id"]),))
|
|
|
- conn.execute("update saddles set token = null where name = 'racer-two'")
|
|
|
- conn.execute("insert into saddles (name, host, status, current_task_id)"
|
|
|
- " values ('tokenless-orphan', '127.0.0.1:1', 'offline', 0)")
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
+ check(wait_for(lambda: saddle_row("tokenless-orphan") is not None
|
|
|
+ and (saddle_row("tokenless-orphan")[1] or "").startswith("saddle-")
|
|
|
+ or False, timeout=20, what="tokenless row minted") is not None,
|
|
|
+ "startup mints a token for a saddle row without one")
|
|
|
+ check(wait_for(lambda: saddle_row("racer-two") is not None
|
|
|
+ and saddle_row("racer-two")[1] == "saddle-legacy-copied-token"
|
|
|
+ or False, timeout=20, what="legacy token copied") is not None,
|
|
|
+ "startup copies the legacy api_tokens value onto the row")
|
|
|
+ fake2.token = "saddle-legacy-copied-token" # the daemon asks with it
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-legacy-copied-token")
|
|
|
+ check(status in (200, 204),
|
|
|
+ "the legacy token still authenticates (copied, not replaced)")
|
|
|
+ conn = db()
|
|
|
+ conn.execute("delete from saddles where name = 'tokenless-orphan'")
|
|
|
+ conn.commit()
|
|
|
+ conn.close()
|
|
|
|
|
|
- app.terminate()
|
|
|
- app.wait(timeout=10)
|
|
|
- app = start_app()
|
|
|
- auth = login("admin", "adminpass123")
|
|
|
+ # The restart wiped the in-memory ask bookkeeping; the fleet's
|
|
|
+ # daemons re-establish sleeping with a fresh ask each (the real
|
|
|
+ # daemons' idle-awake ask / 15-minute safety self-ask).
|
|
|
+ for daemon in (fake2,):
|
|
|
+ daemon.ask_now()
|
|
|
+ check(True, "fleet re-asks after the restart (sleeping re-established)")
|
|
|
|
|
|
- def saddle_row(name):
|
|
|
- conn = db()
|
|
|
- try:
|
|
|
- return conn.execute("select id, token from saddles where name = ?",
|
|
|
- (name,)).fetchone()
|
|
|
- finally:
|
|
|
- conn.close()
|
|
|
-
|
|
|
- check(wait_for(lambda: saddle_row("tokenless-orphan") is not None
|
|
|
- and (saddle_row("tokenless-orphan")[1] or "").startswith("saddle-")
|
|
|
- or False, timeout=20, what="tokenless row minted") is not None,
|
|
|
- "startup mints a token for a saddle row without one")
|
|
|
- check(wait_for(lambda: saddle_row("racer-two") is not None
|
|
|
- and saddle_row("racer-two")[1] == "saddle-legacy-copied-token"
|
|
|
- or False, timeout=20, what="legacy token copied") is not None,
|
|
|
- "startup copies the legacy api_tokens value onto the row")
|
|
|
- fake2.token = "saddle-legacy-copied-token" # the daemon asks with it
|
|
|
- status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-legacy-copied-token")
|
|
|
- check(status in (200, 204),
|
|
|
- "the legacy token still authenticates (copied, not replaced)")
|
|
|
- conn = db()
|
|
|
- conn.execute("delete from saddles where name = 'tokenless-orphan'")
|
|
|
- conn.commit()
|
|
|
- conn.close()
|
|
|
|
|
|
- # The restart wiped the in-memory ask bookkeeping; the fleet's
|
|
|
- # daemons re-establish sleeping with a fresh ask each (the real
|
|
|
- # daemons' idle-awake ask / 15-minute safety self-ask).
|
|
|
- for daemon in (fake2, fake3):
|
|
|
- daemon.ask_now()
|
|
|
- check(True, "fleet re-asks after the restart (sleeping re-established)")
|
|
|
+BUNDLES_ORDER = ("pages", "dispatch", "subtasks", "lineage",
|
|
|
+ "allocation", "lifecycle", "answers")
|
|
|
|
|
|
|
|
|
+def run_bundle(name):
|
|
|
+ """Runs one shard to completion; returns a failure count."""
|
|
|
+ started = time.monotonic()
|
|
|
+ ctx = start_bundle(name)
|
|
|
+ runner = globals()["bundle_" + name]
|
|
|
+ try:
|
|
|
+ runner(ctx)
|
|
|
finally:
|
|
|
# Exit-code diagnostic: distinguishes a crashed app (negative
|
|
|
# returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
|
|
|
# tearing down mid-exception, also capture the still-alive
|
|
|
# app's stacks before killing it (a wedged-but-alive server
|
|
|
# otherwise leaves no evidence).
|
|
|
- print("(app returncode at teardown: %s)" % app.poll())
|
|
|
- if app.poll() is None and sys.exc_info()[0] is not None:
|
|
|
- dump_app_stacks(app.pid)
|
|
|
- app.terminate()
|
|
|
+ print("(app returncode at teardown: %s)" % ctx.app.poll())
|
|
|
+ if ctx.app.poll() is None and sys.exc_info()[0] is not None:
|
|
|
+ dump_app_stacks(ctx.app.pid)
|
|
|
+ ctx.fake.kill()
|
|
|
try:
|
|
|
- app.wait(timeout=5)
|
|
|
+ ctx.app.terminate()
|
|
|
+ try:
|
|
|
+ ctx.app.wait(timeout=5)
|
|
|
+ except subprocess.TimeoutExpired:
|
|
|
+ ctx.app.kill()
|
|
|
+ check(ctx.app.poll() in (0, -15),
|
|
|
+ "bundle %s app shut down cleanly (exit %s)"
|
|
|
+ % (name, ctx.app.poll()))
|
|
|
+ except OSError:
|
|
|
+ pass
|
|
|
+ print("%d passed, %d failed" % (passes, failures))
|
|
|
+ print("[bundle %s] %.1fs" % (name, time.monotonic() - started))
|
|
|
+ return failures
|
|
|
+
|
|
|
+
|
|
|
+def run_all_parallel():
|
|
|
+ """The default meson entry: one subprocess per bundle (each with its
|
|
|
+ own app, daemons and database), verdicts aggregated into one exit
|
|
|
+ code and one combined check-count summary."""
|
|
|
+ base = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
|
|
|
+ log_dir = os.path.join(os.path.dirname(base) or "/tmp/kilo",
|
|
|
+ "webstack-shards")
|
|
|
+ os.makedirs(log_dir, exist_ok=True)
|
|
|
+ procs = {}
|
|
|
+ started = time.monotonic()
|
|
|
+ for name in BUNDLES_ORDER:
|
|
|
+ out_path = os.path.join(log_dir, "bundle-%s.log" % name)
|
|
|
+ handle = open(out_path, "w")
|
|
|
+ procs[name] = (subprocess.Popen(
|
|
|
+ [sys.executable, "-u", os.path.abspath(__file__), APP,
|
|
|
+ "--bundle", name],
|
|
|
+ stdout=handle, stderr=subprocess.STDOUT), handle, out_path)
|
|
|
+ total_pass = total_fail = 0
|
|
|
+ broken = []
|
|
|
+ for name, (proc, handle, out_path) in procs.items():
|
|
|
+ try:
|
|
|
+ proc.wait(timeout=SHARD_TIMEOUT_S)
|
|
|
except subprocess.TimeoutExpired:
|
|
|
- app.kill()
|
|
|
+ proc.kill()
|
|
|
+ proc.wait()
|
|
|
+ handle.close()
|
|
|
+ text = open(out_path).read()
|
|
|
+ match = re.search(r"(\d+) passed, (\d+) failed", text)
|
|
|
+ took = re.search(r"\[bundle %s\] ([0-9.]+)s" % name, text)
|
|
|
+ bundle_pass = int(match.group(1)) if match else 0
|
|
|
+ bundle_fail = int(match.group(2)) if match else -1
|
|
|
+ total_pass += max(bundle_pass, 0)
|
|
|
+ total_fail += max(bundle_fail, 0)
|
|
|
+ status = "ok"
|
|
|
+ if proc.returncode != 0 or bundle_fail != 0 or match is None:
|
|
|
+ status = "BROKEN (exit %s)" % proc.returncode
|
|
|
+ broken.append(name)
|
|
|
+ print("[shard %-10s] %6ss %3d passed %3d failed %s"
|
|
|
+ % (name, took.group(1) if took else "?",
|
|
|
+ bundle_pass, bundle_fail, status))
|
|
|
+ if status != "ok":
|
|
|
+ tail = "\n".join(text.rstrip().split("\n")[-60:])
|
|
|
+ print("--- web-stack/%s tail (full log: %s) ---" % (name, out_path))
|
|
|
+ print(tail)
|
|
|
+ print("--- end web-stack/%s tail ---" % name)
|
|
|
+ print("shards wall: %.1fs (logs under %s)" % (time.monotonic() - started, log_dir))
|
|
|
+ print("ALL SHARDS: %d passed, %d failed%s"
|
|
|
+ % (total_pass, total_fail,
|
|
|
+ "" if not broken else " BROKEN: " + ", ".join(broken)))
|
|
|
+ return 1 if broken or total_fail else 0
|
|
|
|
|
|
- print("%d passed, %d failed" % (passes, failures))
|
|
|
- return 1 if failures else 0
|
|
|
+
|
|
|
+def main():
|
|
|
+ if not os.path.exists(APP):
|
|
|
+ raise SystemExit("app binary not found at %s (build first)" % APP)
|
|
|
+ argv = sys.argv[2:]
|
|
|
+ if "--bundle" in argv:
|
|
|
+ sys.exit(1 if run_bundle(argv[argv.index("--bundle") + 1]) else 0)
|
|
|
+ if os.environ.get("WEBSTACK_SEQUENTIAL"):
|
|
|
+ # Debug path: every bundle in-process, one after another.
|
|
|
+ for name in BUNDLES_ORDER:
|
|
|
+ if run_bundle(name):
|
|
|
+ return 1
|
|
|
+ return 0
|
|
|
+ return run_all_parallel()
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|