|
@@ -53,6 +53,24 @@ Plan 10 phase 2 (telemetry flip):
|
|
|
- the deleted layers are gone: /output, /log and /start 404 on the
|
|
- the deleted layers are gone: /output, /log and /start 404 on the
|
|
|
daemon side, and nothing on the web side calls them
|
|
daemon side, and nothing on the web side calls them
|
|
|
|
|
|
|
|
|
|
+Plan 10 phase 3 (capability awareness + UI):
|
|
|
|
|
+
|
|
|
|
|
+- assignment capability context: {saddle_tags, fleet_tags} on every
|
|
|
|
|
+ payload, GET /api/saddle/capabilities (per-saddle token) feeding
|
|
|
|
|
+ the daemon's `saddle capabilities` CLI
|
|
|
|
|
+- subtask callback capabilities: explicit-only attachment validated
|
|
|
|
|
+ against the operator tag set, unknown tags rejected with the valid
|
|
|
|
|
+ list in the error
|
|
|
|
|
+- task page: status block pinned to the top, blocking sections
|
|
|
|
|
+ newest-first (sealed GET state)
|
|
|
|
|
+- home dashboard: pinned OPEN blocks as newest-first link-through
|
|
|
|
|
+ cards (no inline answer forms) above the cross-task activity
|
|
|
|
|
+ timeline (last ~50 events newest-first)
|
|
|
|
|
+- lost-wake-guard regression: a saddle that asks SLOWLY after its
|
|
|
|
|
+ wake (busy winding down) still claims the task — the wake is never
|
|
|
|
|
+ silently dropped, no re-wake storm fires, and the never-asks guard
|
|
|
|
|
+ path (offline + re-wake of the next capable) stays green
|
|
|
|
|
+
|
|
|
Prints PASS/FAIL lines Statum-style; exits non-zero on any failure.
|
|
Prints PASS/FAIL lines Statum-style; exits non-zero on any failure.
|
|
|
"""
|
|
"""
|
|
|
import json
|
|
import json
|
|
@@ -298,6 +316,7 @@ class FakeSaddle:
|
|
|
self.ask_on_wake = True
|
|
self.ask_on_wake = True
|
|
|
self.ask_on_terminal = True
|
|
self.ask_on_terminal = True
|
|
|
self.ask_on_idle_status = True
|
|
self.ask_on_idle_status = True
|
|
|
|
|
+ self.ask_delay = 0.0 # seconds; a slow daemon delays its wake-triggered ask
|
|
|
self.serve_register = True # False: 404 /register (pre-flip daemon)
|
|
self.serve_register = True # False: 404 /register (pre-flip daemon)
|
|
|
self.last_ask_trigger = 0.0
|
|
self.last_ask_trigger = 0.0
|
|
|
self.port = None
|
|
self.port = None
|
|
@@ -402,10 +421,15 @@ class FakeSaddle:
|
|
|
return status, payload
|
|
return status, payload
|
|
|
|
|
|
|
|
def _trigger_ask(self, reason):
|
|
def _trigger_ask(self, reason):
|
|
|
- """Schedules a delayed ask (the daemon woke / went idle-awake)."""
|
|
|
|
|
|
|
+ """Schedules a delayed ask (the daemon woke / went idle-awake).
|
|
|
|
|
+ ask_delay models a SLOW asker — a daemon still winding down its
|
|
|
|
|
+ round when the wake lands: it honours the wake, but its ask
|
|
|
|
|
+ arrives late (still inside the web's lost-wake guard window)."""
|
|
|
def run():
|
|
def run():
|
|
|
if self.state in ("working", "failing"):
|
|
if self.state in ("working", "failing"):
|
|
|
return # a running daemon does not ask
|
|
return # a running daemon does not ask
|
|
|
|
|
+ if self.ask_delay > 0:
|
|
|
|
|
+ time.sleep(self.ask_delay)
|
|
|
for attempt in range(3):
|
|
for attempt in range(3):
|
|
|
try:
|
|
try:
|
|
|
status, _ = self.ask_now()
|
|
status, _ = self.ask_now()
|
|
@@ -2286,6 +2310,245 @@ def main():
|
|
|
fake3.ask_on_wake = True
|
|
fake3.ask_on_wake = True
|
|
|
fake3.ask_on_idle_status = True
|
|
fake3.ask_on_idle_status = True
|
|
|
|
|
|
|
|
|
|
+ print("== phase 3: capability context, catalogue, subtask tags ==")
|
|
|
|
|
+ # Assignment capability context (plan 10 phase 3): every
|
|
|
|
|
+ # assignment carries {saddle_tags: the allocated saddle's tags,
|
|
|
|
|
+ # fleet_tags: every tag name known across the fleet} — the
|
|
|
|
|
+ # daemon renders it into its system prompt. fake-saddle carries
|
|
|
|
|
+ # gpu + lostwake; the catalogue is affinity-tag, gpu, lostwake.
|
|
|
|
|
+ fake2.ask_on_register = False
|
|
|
|
|
+ fake2.ask_on_wake = False
|
|
|
|
|
+ fake2.ask_on_terminal = False
|
|
|
|
|
+ fake2.ask_on_idle_status = False
|
|
|
|
|
+ time.sleep(1.2) # any in-flight trigger asks land
|
|
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
|
|
+ "project_id": "1",
|
|
|
|
|
+ "title": "Capabilities context",
|
|
|
|
|
+ "priority": "2",
|
|
|
|
|
+ })
|
|
|
|
|
+ cap_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
|
|
+ if t["title"] == "Capabilities context"][0])
|
|
|
|
|
+ check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == cap_task
|
|
|
|
|
+ for p in fake.start_payloads) or False,
|
|
|
|
|
+ timeout=60, what="capabilities-context task dispatched") is not None,
|
|
|
|
|
+ "the capabilities-context task dispatches")
|
|
|
|
|
+ cap_payload = [p for p in fake.start_payloads
|
|
|
|
|
+ if int(p.get("task", {}).get("id") or 0) == cap_task][-1]
|
|
|
|
|
+ context = cap_payload.get("capabilities") or {}
|
|
|
|
|
+ check(context.get("saddle_tags") == ["gpu", "lostwake"],
|
|
|
|
|
+ "assignment saddle_tags = the allocated saddle's tags (%s)"
|
|
|
|
|
+ % context.get("saddle_tags"))
|
|
|
|
|
+ check(context.get("fleet_tags") == ["affinity-tag", "gpu", "lostwake"],
|
|
|
|
|
+ "assignment fleet_tags = every tag name known across the fleet (%s)"
|
|
|
|
|
+ % context.get("fleet_tags"))
|
|
|
|
|
+ check(wait_task_done(auth, cap_task, timeout=60) is not None,
|
|
|
|
|
+ "capabilities-context task completes")
|
|
|
|
|
+
|
|
|
|
|
+ # GET /api/saddle/capabilities feeds the daemon's `saddle
|
|
|
|
|
+ # capabilities` CLI: per-saddle-token auth, tag order by name,
|
|
|
|
|
+ # each tag with the saddles carrying it (name order).
|
|
|
|
|
+ status, _ = saddle_api("GET", "/api/saddle/capabilities", None)
|
|
|
|
|
+ check(status == 401, "capabilities endpoint rejects a missing token")
|
|
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/capabilities", "saddle-bogus")
|
|
|
|
|
+ check(status == 401, "capabilities endpoint rejects a wrong token")
|
|
|
|
|
+ status, body = saddle_api("GET", "/api/saddle/capabilities", fake_token)
|
|
|
|
|
+ catalogue = {t["name"]: t["saddles"] for t in body.get("tags", [])}
|
|
|
|
|
+ check(status == 200 and body.get("ok") is True
|
|
|
|
|
+ and [t["name"] for t in body.get("tags", [])]
|
|
|
|
|
+ == ["affinity-tag", "gpu", "lostwake"],
|
|
|
|
|
+ "catalogue lists every tag by name")
|
|
|
|
|
+ check(catalogue.get("gpu") == ["fake-saddle"]
|
|
|
|
|
+ and catalogue.get("affinity-tag") == ["racer-three", "racer-two"]
|
|
|
|
|
+ and catalogue.get("lostwake") == ["fake-saddle", "racer-three"],
|
|
|
|
|
+ "each tag names its carrying saddles in name order (%s)" % catalogue)
|
|
|
|
|
+
|
|
|
|
|
+ # Subtask capabilities (`saddle subtask --capability`): explicit
|
|
|
|
|
+ # only — never inherited — and validated against the operator
|
|
|
|
|
+ # tag set; unknown tags reject with the valid list in the error
|
|
|
|
|
+ # (the agent reads it verbatim in the CLI output).
|
|
|
|
|
+ conn = db()
|
|
|
|
|
+ tasks_before = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
|
|
+ conn.close()
|
|
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
|
|
+ "parent_task_id": auto_parent, "title": "Bad tag subtask",
|
|
|
|
|
+ "capabilities": ["gpu", "warp-drive"]})
|
|
|
|
|
+ conn = db()
|
|
|
|
|
+ tasks_after = conn.execute("select count(*) from tasks").fetchone()[0]
|
|
|
|
|
+ conn.close()
|
|
|
|
|
+ check(status == 400 and tasks_after == tasks_before,
|
|
|
|
|
+ "an unknown capability tag rejects the subtask call")
|
|
|
|
|
+ error_text = body.get("error", "")
|
|
|
|
|
+ check("warp-drive" in error_text and "valid tags" in error_text
|
|
|
|
|
+ and "gpu" in error_text and "lostwake" in error_text,
|
|
|
|
|
+ "the rejection names the unknown tag and the valid list (%s)" % error_text)
|
|
|
|
|
+ status, _ = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
|
|
+ "parent_task_id": auto_parent, "title": "Bad shape subtask",
|
|
|
|
|
+ "capabilities": "gpu"})
|
|
|
|
|
+ check(status == 400, "a non-array capabilities member rejects")
|
|
|
|
|
+
|
|
|
|
|
+ # Explicit-only: a parent requiring 'lostwake' (the lost-wake
|
|
|
|
|
+ # task) gets a subtask tagged just 'gpu' — nothing inherited.
|
|
|
|
|
+ status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
|
|
|
|
|
+ "parent_task_id": lost_task, "title": "Explicit tag subtask",
|
|
|
|
|
+ "capabilities": ["gpu"]})
|
|
|
|
|
+ check(status == 200 and body.get("ok") is True,
|
|
|
|
|
+ "a valid tag creates the subtask synchronously")
|
|
|
|
|
+ tagged_subtask = int(body["task_id"])
|
|
|
|
|
+ detail = task_state(auth, tagged_subtask)
|
|
|
|
|
+ check([c["name"] for c in detail.get("capabilities", [])] == ["gpu"],
|
|
|
|
|
+ "the subtask carries exactly the explicit tag (no inheritance)")
|
|
|
|
|
+ check([c["name"] for c in task_state(auth, lost_task).get("capabilities", [])]
|
|
|
|
|
+ == ["lostwake"],
|
|
|
|
|
+ "the parent keeps its own requirement")
|
|
|
|
|
+ conn = db()
|
|
|
|
|
+ tag_rows = conn.execute(
|
|
|
|
|
+ "select c.name from task_capabilities tc"
|
|
|
|
|
+ " join capabilities c on c.id = tc.capability_id"
|
|
|
|
|
+ " where tc.task_id = ?", (tagged_subtask,)).fetchall()
|
|
|
|
|
+ conn.close()
|
|
|
|
|
+ check([r[0] for r in tag_rows] == ["gpu"],
|
|
|
|
|
+ "the task_capabilities rows record the explicit tag")
|
|
|
|
|
+ # The queued subtask dispatches (gpu-carrier) and settles before
|
|
|
|
|
+ # the slow-wake run needs an empty queue.
|
|
|
|
|
+ check(wait_task_done(auth, tagged_subtask, timeout=90) is not None,
|
|
|
|
|
+ "the tagged subtask dispatches and completes")
|
|
|
|
|
+
|
|
|
|
|
+ print("== phase 3: task page — status on top, blocks newest-first ==")
|
|
|
|
|
+ detail = task_state(auth, plan_task)
|
|
|
|
|
+ block_ids = [int(b["id"]) for b in detail.get("blocks", [])]
|
|
|
|
|
+ check(len(block_ids) >= 3 and block_ids == sorted(block_ids, reverse=True),
|
|
|
|
|
+ "the blocks section orders newest-first via the sealed GET state (%s)"
|
|
|
|
|
+ % block_ids)
|
|
|
|
|
+ status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
|
|
|
|
|
+ check(status == 200 and "status-head" in page
|
|
|
|
|
+ and page.index("status-head") < page.index("<h3>Description</h3>")
|
|
|
|
|
+ and page.index("status-head") < page.index("<h3>Plan</h3>"),
|
|
|
|
|
+ "the status block sits at the very top of the task page")
|
|
|
|
|
+
|
|
|
|
|
+ print("== phase 3: dashboard — pinned blocks + activity timeline ==")
|
|
|
|
|
+ # Pinned OPEN blocking items as link-through cards, newest
|
|
|
|
|
+ # first; no inline answer forms (the card links to the task
|
|
|
|
|
+ # page). At this point: the auto-approve grandchild's question
|
|
|
|
|
+ # and RFC plus the parent's own review are all unanswered.
|
|
|
|
|
+ dash = slot_public(auth, "/", "dashboard")
|
|
|
|
|
+ pinned = dash.get("pinned", [])
|
|
|
|
|
+ pinned_ids = [int(b["id"]) for b in pinned]
|
|
|
|
|
+ check(len(pinned_ids) >= 3 and pinned_ids == sorted(pinned_ids, reverse=True),
|
|
|
|
|
+ "pinned open blocks order newest-first (%s)" % pinned_ids)
|
|
|
|
|
+ check({b["block_type"] for b in pinned} >= {"question", "rfc", "review"},
|
|
|
|
|
+ "pinned cards cover questions, RFCs and reviews")
|
|
|
|
|
+ check(all(b["question"] and b["task_title"] and int(b["task_id"]) > 0
|
|
|
|
|
+ for b in pinned)
|
|
|
|
|
+ and all("answer_action" not in b for b in pinned),
|
|
|
|
|
+ "pinned cards are link-through only (task title, kind badge, question)")
|
|
|
|
|
+
|
|
|
|
|
+ # The activity timeline: last <= 50 events, newest first, fed
|
|
|
|
|
+ # from the existing tables. Fixture rows a few seconds old
|
|
|
|
|
+ # guarantee the progress kind and a human-authored comment ride
|
|
|
|
|
+ # along; the probe block below proves same-source ordering.
|
|
|
|
|
+ conn = db()
|
|
|
|
|
+ conn.execute("insert into task_comments (task_id, author, body, created_at)"
|
|
|
|
|
+ " values (?, 'user:admin', 'timeline fixture comment by a human',"
|
|
|
|
|
+ " strftime('%s','now','-5 seconds'))", (cap_task,))
|
|
|
|
|
+ conn.execute("insert into task_progress (task_id, summary, created_at)"
|
|
|
|
|
+ " values (?, 'timeline fixture progress',"
|
|
|
|
|
+ " strftime('%s','now','-5 seconds'))",
|
|
|
|
|
+ (cap_task,))
|
|
|
|
|
+ conn.commit()
|
|
|
|
|
+ conn.close()
|
|
|
|
|
+ dash = slot_public(auth, "/", "dashboard")
|
|
|
|
|
+ activity = dash.get("activity", [])
|
|
|
|
|
+ epochs = [int(e["at_epoch"]) for e in activity]
|
|
|
|
|
+ check(0 < len(activity) <= 50 and epochs == sorted(epochs, reverse=True),
|
|
|
|
|
+ "the timeline shows the last <=50 events newest-first (%d events)"
|
|
|
|
|
+ % len(activity))
|
|
|
|
|
+ kinds = {e["kind"] for e in activity}
|
|
|
|
|
+ check({"created", "started", "dispatch", "comment", "progress", "completed"} <= kinds,
|
|
|
|
|
+ "the timeline covers status changes, dispatches, comments, progress and"
|
|
|
|
|
+ " completions (%s)" % sorted(kinds))
|
|
|
|
|
+ check(any(e["kind"] == "comment" and e["text"].startswith("user:admin:")
|
|
|
|
|
+ and "fixture comment" in e["text"] for e in activity),
|
|
|
|
|
+ "comment events carry author + snippet")
|
|
|
|
|
+
|
|
|
|
|
+ # A freshly raised block pins immediately (newest card) and
|
|
|
|
|
+ # lands in the timeline as block-raised; answering it records
|
|
|
|
|
+ # block-answered.
|
|
|
|
|
+ saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
|
|
|
|
|
+ {"type": "block", "session_id": grand_session, "data": {
|
|
|
|
|
+ "question": "Timeline probe: approve the dashboard flow",
|
|
|
|
|
+ "type": "rfc", "body": "# Probe"}})
|
|
|
|
|
+ dash = slot_public(auth, "/", "dashboard")
|
|
|
|
|
+ check(dash["pinned"][0]["question"] == "Timeline probe: approve the dashboard flow"
|
|
|
|
|
+ and int(dash["pinned"][0]["task_id"]) == auto_grand,
|
|
|
|
|
+ "the freshest open block pins as the first card")
|
|
|
|
|
+ check(dash["activity"][0]["kind"] == "block-raised"
|
|
|
|
|
+ and "Timeline probe" in dash["activity"][0]["text"],
|
|
|
|
|
+ "the raised block lands atop the timeline")
|
|
|
|
|
+ probe_block = [b for b in task_state(auth, auto_grand)["blocks"]
|
|
|
|
|
+ if b["question"] == "Timeline probe: approve the dashboard flow"][0]
|
|
|
|
|
+ act(auth, probe_block["answer_action"], {"answer": "Ship the dashboard"})
|
|
|
|
|
+ dash = slot_public(auth, "/", "dashboard")
|
|
|
|
|
+ activity = dash.get("activity", [])
|
|
|
|
|
+ raised_at = next((i for i, e in enumerate(activity)
|
|
|
|
|
+ if e["kind"] == "block-raised" and "Timeline probe" in e["text"]), -1)
|
|
|
|
|
+ answered_at = next((i for i, e in enumerate(activity)
|
|
|
|
|
+ if e["kind"] == "block-answered"
|
|
|
|
|
+ and "Ship the dashboard" in e["text"]), -1)
|
|
|
|
|
+ check(answered_at >= 0 and raised_at >= 0 and answered_at < raised_at,
|
|
|
|
|
+ "the answered event sorts above its raised event (newest-first, same source)")
|
|
|
|
|
+ check(all(b["question"] != "Timeline probe: approve the dashboard flow"
|
|
|
|
|
+ for b in dash["pinned"]),
|
|
|
|
|
+ "the answered block leaves the pinned cards")
|
|
|
|
|
+
|
|
|
|
|
+ print("== phase 3 regression: a slow ask after a wake is never dropped ==")
|
|
|
|
|
+ # The daemon worker's lost-wake-guard regression: a saddle whose
|
|
|
|
|
+ # ask arrives SLOWLY after the wake (busy winding down when the
|
|
|
|
|
+ # wake lands) must not lose the wake — the delayed ask (inside
|
|
|
|
|
+ # the guard window) claims the task, no re-wake storm fires
|
|
|
|
|
+ # while it is pending, and the saddle stays healthy. The
|
|
|
|
|
+ # never-asks path (guard -> offline + re-wake of the next
|
|
|
|
|
+ # capable) is proven by the racer-three section above.
|
|
|
|
|
+ check(wait_for(lambda: queued_count() == 0 or False,
|
|
|
|
|
+ timeout=60, what="queue settles before the slow-wake run") is not None,
|
|
|
|
|
+ "the queue settles before the slow-wake run")
|
|
|
|
|
+ status, _ = fake2.ask_now()
|
|
|
|
|
+ check(status == 204, "the slow-ask racer sleeps before its wake")
|
|
|
|
|
+ saddles_state = slot_public(auth, "/saddles", "saddles")
|
|
|
|
|
+ act(auth, saddles_state["create_capability"], {"name": "slowwake"})
|
|
|
|
|
+ slowwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")
|
|
|
|
|
+ ["capabilities"] if c["name"] == "slowwake"][0])
|
|
|
|
|
+ racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
|
|
+ if s["name"] == "racer-two"][0]
|
|
|
|
|
+ act(auth, racer2_card["attach_capability"], {"capability_id": str(slowwake_cap)})
|
|
|
|
|
+ fake2.ask_delay = 1.2 # slow: asks well after the wake, inside the 2s guard
|
|
|
|
|
+ fake2.ask_on_wake = True
|
|
|
|
|
+ wakes_before_slow = len(fake2.wake_log)
|
|
|
|
|
+ act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
|
|
|
|
|
+ "project_id": "1",
|
|
|
|
|
+ "title": "Slow wake task",
|
|
|
|
|
+ "priority": "2",
|
|
|
|
|
+ "capability_id": [str(slowwake_cap)],
|
|
|
|
|
+ })
|
|
|
|
|
+ slow_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
|
|
|
|
|
+ if t["title"] == "Slow wake task"][0])
|
|
|
|
|
+ check(wait_for(lambda: len(fake2.wake_log) > wakes_before_slow or False,
|
|
|
|
|
+ timeout=30, what="the slow-ask racer woken") is not None,
|
|
|
|
|
+ "the only capable sleeping saddle is woken")
|
|
|
|
|
+ check(wait_task_done(auth, slow_task, timeout=120) is not None,
|
|
|
|
|
+ "the delayed ask claims the task — the wake was not dropped")
|
|
|
|
|
+ check(any(a.get("status") == 200
|
|
|
|
|
+ and int((a.get("payload") or {}).get("task", {}).get("id") or 0)
|
|
|
|
|
+ == slow_task for a in fake2.asks),
|
|
|
|
|
+ "the slow ask's 200 carries the task")
|
|
|
|
|
+ check(len(fake2.wake_log) == wakes_before_slow + 1,
|
|
|
|
|
+ "the pending-wake skip re-wakes nobody while the slow ask is in flight")
|
|
|
|
|
+ racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
|
|
|
|
|
+ if s["name"] == "racer-two"][0]
|
|
|
|
|
+ check(racer2_card["status"] in ("idle", "busy"),
|
|
|
|
|
+ "the slow asker never went offline (guard did not fire)")
|
|
|
|
|
+ fake2.ask_delay = 0.0
|
|
|
|
|
+ fake2.ask_on_wake = False # stays out of the pool below
|
|
|
|
|
+
|
|
|
# The racers leave the dispatch pool: the remaining sections
|
|
# The racers leave the dispatch pool: the remaining sections
|
|
|
# (rollback resume, R5-R11) predate multi-daemon routing and
|
|
# (rollback resume, R5-R11) predate multi-daemon routing and
|
|
|
# assert against the primary fake's payloads. racer-three is
|
|
# assert against the primary fake's payloads. racer-three is
|