integration_test.py 59 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070
  1. #!/usr/bin/env python3
  2. """End-to-end integration test: ClankerSaddle web app + saddle daemon.
  3. Drives the web app the way statum.js does (entrypoint state loads plus
  4. action POSTs carrying the auth slot and sealed private blobs) while a
  5. REAL saddle daemon runs a REAL goal against a real git checkout under
  6. WORKSPACE, powered by saddle/test/mock_llm.py with a deterministic
  7. tool-call script — both binaries, the true cross-binary contract proof
  8. (the per-repo fake-peer suits live in web_stack_test.py / saddle e2e).
  9. The protocol-native goal-era arc (PLAN-17; every agent turn is a tool
  10. call, questions ride `saddle block` through run_command):
  11. 1. goal create -> start: the goal-run assignment reaches the daemon
  12. over the ask/allocate channel with the full plan-12 payload (kind/
  13. mode/creator/goal context/ephemeral branches for workers)
  14. 2. a narrated probe round (turn_text + tool_call/tool_result events in
  15. the ledger) precedes the pre-plan question block -> answered on the
  16. goal page -> the ONE answer resolves the goal-run task's single
  17. task_blocks row and the parked goal agent resumes with the verdict
  18. 3. submit_plan -> reject-with-feedback -> revision -> approval (the
  19. agreed verdict words ride the resume)
  20. 4. task fan-out through the create_task/enqueue_task tools (the real
  21. /api/saddle/task/create + enqueue callbacks); the worker's repeated
  22. identical run_command probes trip the loop warning (never fatal),
  23. then the clean run works in its ephemeral worktree branch cut from
  24. the goal clone, commits, finishes — worktree auto-pruned, branch
  25. kept
  26. 5. the terminal transition (the goals row's last_terminal_watermark)
  27. coalesces (~SADDLE_GOAL_RESUME_S) into the goal agent's batched
  28. resume; it merges, deletes the task branch, pushes the goal branch
  29. (push-before-review) and submit_report enters completion review
  30. 6. completion review accept — the live per-repo push flags gate it:
  31. the accept REFUSES while origin is behind, passes once the branch
  32. is re-pushed and the probe re-cached -> goal complete, run task done
  33. 7. the audit trail: the DERIVED goal narrative, the unified events
  34. ledger (turn cards with tool_call/tool_result, usage rows, the §7.2
  35. kind set, web-originated-only comment kinds), the event log page +
  36. SSE stream, the /status goal_branches cache sampled by the manual
  37. saddle refresh
  38. 8. the push wire surface (401s on the id-less routes, the §7.2 chunks
  39. ingestion with (session_id, seq) exactly-once, block raises riding
  40. POST /api/saddle/results, the retired results-entries array
  41. consumed silently, the deleted poll-era endpoints 404ing) and the
  42. stm-markdown renderer
  43. Prerequisites (see README.md "Integration testing"): the web app
  44. running with SPRY_SEED_ADMIN_USERNAME=admin,
  45. SPRY_SEED_ADMIN_PASSWORD=adminpass123, SADDLE_LLM_BASE_URL pointing
  46. at the mock, SADDLE_PROJECTS_ROOT=$WORKSPACE and a FRESH database
  47. (the harness asserts an empty tasks/goals state); a saddle daemon
  48. with SADDLE_WORKSPACE_ROOT=$WORKSPACE (the shared-volume deployment
  49. assumption — the goal clone anchors the worker worktrees); a git
  50. repo with at least two commits at $WORKSPACE/demo whose origin is
  51. pushable (the harness prepares demo + demo-origin.git when WORKSPACE
  52. is empty).
  53. Environment:
  54. WEB web app base URL (default http://localhost:8180 —
  55. 8080 is the production port, never the e2e default)
  56. SADDLE daemon control URL (default http://localhost:7100)
  57. SADDLE_HOST host:port to register (default localhost:7100)
  58. SADDLE_NAME registration name (default saddle-e2e-1)
  59. WORKSPACE demo repo parent dir (default /tmp/clanker-e2e/workspace)
  60. DB_PATH web app SQLite file (for audit assertions)
  61. MOCK mock_llm.py path (default <repo>/../saddle/test/mock_llm.py)
  62. MOCK_PORT local mock port (default 9999)
  63. MOCK_CMD run instead of the local mock (e.g. a podman exec line
  64. that starts the mock inside the saddle container); the
  65. script skips spawning the mock when set (the mock then
  66. needs its own --capture for the receipt assertions)
  67. Prints PASS/FAIL lines Statum-style; exits non-zero on any failure.
  68. """
  69. import json
  70. import os
  71. import socket
  72. import sqlite3
  73. import subprocess
  74. import sys
  75. import time
  76. import urllib.error
  77. import urllib.parse
  78. import urllib.request
  79. REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  80. WEB = os.environ.get("WEB", "http://localhost:8180")
  81. SADDLE = os.environ.get("SADDLE", "http://localhost:7100")
  82. SADDLE_HOST = os.environ.get("SADDLE_HOST", "localhost:7100")
  83. SADDLE_NAME = os.environ.get("SADDLE_NAME", "saddle-e2e-1")
  84. WORKSPACE = os.environ.get("WORKSPACE", "/tmp/clanker-e2e/workspace")
  85. DB_PATH = os.environ.get("DB_PATH", os.path.join(WORKSPACE, "..", "web", "clanker-saddle.sqlite"))
  86. MOCK = os.environ.get("MOCK", os.path.join(os.path.dirname(REPO), "saddle", "test", "mock_llm.py"))
  87. MOCK_PORT = os.environ.get("MOCK_PORT", "9999")
  88. MOCK_CMD = os.environ.get("MOCK_CMD", "")
  89. # The mock's request capture (one JSON line per arrival carrying the
  90. # full message list) — the receipt forensics the D-side assertions read.
  91. CAPTURE_PATH = os.environ.get("CAPTURE_PATH",
  92. os.path.join(WORKSPACE, "..", "mock-capture.jsonl"))
  93. DEMO = os.path.join(WORKSPACE, "demo")
  94. HOSTNAME = socket.gethostname().split(".")[0]
  95. # The verdict wording is SERVER-OWNED (plan 16 S2/R4): the harness
  96. # posts `verdict` markers / `feedback` text and never the constants.
  97. passes = 0
  98. failures = 0
  99. def check(condition, label):
  100. global passes, failures
  101. if condition:
  102. passes += 1
  103. print("PASS %s" % label)
  104. else:
  105. failures += 1
  106. print("FAIL %s" % label)
  107. def http(method, url, data=None, headers=None, timeout=30):
  108. body = None
  109. if data is not None:
  110. body = "&".join("%s=%s" % (k, urllib.parse.quote(str(v))) for k, v in data.items()).encode()
  111. request = urllib.request.Request(WEB + url if url.startswith("/") else url,
  112. data=body, method=method)
  113. if data is not None:
  114. request.add_header("Content-Type", "application/x-www-form-urlencoded")
  115. for key, value in (headers or {}).items():
  116. request.add_header(key, value)
  117. try:
  118. with urllib.request.urlopen(request, timeout=timeout) as response:
  119. return response.status, response.read().decode()
  120. except urllib.error.HTTPError as error:
  121. return error.code, error.read().decode()
  122. def directives(uri, auth=None):
  123. headers = {}
  124. if auth:
  125. headers["X-Statum-Slot"] = "key=%s; as_at=%s" % (auth["key"], auth["as_at"])
  126. status, body = http("GET", "/_statum/entrypoint?uri=" + urllib.parse.quote(uri.split("?")[0]),
  127. headers=headers)
  128. if status != 200:
  129. raise SystemExit("entrypoint %s -> HTTP %d: %s" % (uri, status, body[:300]))
  130. return json.loads(body)
  131. def slot_public(auth, uri, type_name):
  132. for directive in directives(uri, auth):
  133. if directive["type"] == "set":
  134. content = json.loads(directive["snapshot_frame"]["content"])
  135. if content["slot"]["type"] == type_name:
  136. return content["public"]
  137. raise SystemExit("no %s slot in entrypoint state for %s" % (type_name, uri))
  138. def act(auth, action, fields):
  139. headers = {"X-Statum-Slot": "key=%s; as_at=%s" % (auth["key"], auth["as_at"])}
  140. if action.get("private"):
  141. headers["X-Statum-Private"] = action["private"]
  142. status, body = http(action.get("method", "POST"), action["uri"], fields, headers)
  143. if status != 200:
  144. raise SystemExit("action %s -> HTTP %d: %s" % (action["uri"], status, body[:300]))
  145. return json.loads(body)
  146. def login(username, password):
  147. frames = directives("/login")
  148. content = json.loads(frames[0]["snapshot_frame"]["content"])
  149. action = content["public"]["login"]
  150. status, body = http(action.get("method", "POST"), action["uri"],
  151. {"username": username, "password": password})
  152. if status != 200:
  153. raise SystemExit("login -> HTTP %d: %s" % (status, body[:300]))
  154. for directive in json.loads(body):
  155. if directive["type"] == "set":
  156. frame = json.loads(directive["snapshot_frame"]["content"])
  157. if frame["slot"]["type"] == "auth":
  158. return {"key": frame["slot"]["key"], "as_at": frame["as_at"]}
  159. raise SystemExit("login response carried no auth slot: " + body[:300])
  160. def saddle_status():
  161. try:
  162. status, body = http("GET", SADDLE + "/status", timeout=8)
  163. return json.loads(body) if status == 200 else None
  164. except (urllib.error.URLError, OSError):
  165. return None
  166. def wait_for(predicate, timeout=60, what="condition", interval=1.0):
  167. deadline = time.monotonic() + timeout
  168. last = None
  169. while time.monotonic() < deadline:
  170. last = predicate()
  171. if last is not False and last is not None:
  172. return last
  173. time.sleep(interval)
  174. print(" (timeout waiting for %s; last: %s)" % (what, last))
  175. return None
  176. def task_state(auth, task_id):
  177. return slot_public(auth, "/tasks/%d" % task_id, "task")
  178. def wait_task_status(auth, task_id, status, timeout=90):
  179. return wait_for(lambda: task_state(auth, task_id).get("status") == status or False,
  180. timeout=timeout, what="task %d -> %s" % (task_id, status))
  181. def goal_state(auth, goal_id):
  182. return slot_public(auth, "/goals/%d" % goal_id, "goal")
  183. def wait_goal_status(auth, goal_id, status, timeout=90):
  184. return wait_for(lambda: goal_state(auth, goal_id).get("status") == status or False,
  185. timeout=timeout, what="goal %d -> %s" % (goal_id, status))
  186. def saddle_api(method, path, token, payload=None):
  187. """Raw call against the saddle ingest API; returns (status, json)."""
  188. request = urllib.request.Request(WEB + path,
  189. data=json.dumps(payload or {}).encode(), method=method)
  190. request.add_header("Content-Type", "application/json")
  191. if token is not None:
  192. request.add_header("Authorization", "Bearer " + token)
  193. try:
  194. with urllib.request.urlopen(request, timeout=15) as response:
  195. return response.status, json.loads(response.read().decode() or "{}")
  196. except urllib.error.HTTPError as error:
  197. body = error.read().decode()
  198. try:
  199. return error.code, json.loads(body)
  200. except ValueError:
  201. return error.code, {"raw": body}
  202. def db_query(sql, params=()):
  203. db = sqlite3.connect(DB_PATH, timeout=10)
  204. try:
  205. return db.execute(sql, params).fetchall()
  206. finally:
  207. db.close()
  208. def capture_messages():
  209. """Every message the mock received, oldest first (the mock's
  210. --capture file: one JSON line per request). Tool results ride back
  211. to the model as role=tool turns; harness notices (loop warnings,
  212. verdicts) as role=user turns."""
  213. messages = []
  214. if os.path.exists(CAPTURE_PATH):
  215. with open(CAPTURE_PATH) as handle:
  216. for line in handle:
  217. try:
  218. arrival = json.loads(line)
  219. except ValueError:
  220. continue
  221. for message in arrival.get("messages", []):
  222. if message.get("role") in ("user", "tool"):
  223. messages.append((message.get("role"),
  224. message.get("content") or ""))
  225. return messages
  226. def prepare_workspace():
  227. """Ensures $WORKSPACE/demo exists with a pushable origin (the goal
  228. branch push needs a remote that accepts it). Reuses an existing
  229. demo checkout; the DATABASE must still be fresh (checked in main)."""
  230. if os.path.isdir(os.path.join(DEMO, ".git")):
  231. return
  232. os.makedirs(WORKSPACE, exist_ok=True)
  233. origin = os.path.join(WORKSPACE, "demo-origin.git")
  234. subprocess.run(["git", "init", "--bare", "-b", "main", origin], check=True,
  235. capture_output=True)
  236. subprocess.run(["git", "clone", origin, DEMO], check=True, capture_output=True)
  237. subprocess.run(["git", "-C", DEMO, "config", "user.email", "e2e@localhost"], check=True)
  238. subprocess.run(["git", "-C", DEMO, "config", "user.name", "e2e"], check=True)
  239. with open(os.path.join(DEMO, "hello.py"), "w") as handle:
  240. handle.write("print('demo')\n")
  241. subprocess.run(["git", "-C", DEMO, "add", "-A"], check=True, capture_output=True)
  242. subprocess.run(["git", "-C", DEMO, "commit", "-m", "one"], check=True, capture_output=True)
  243. with open(os.path.join(DEMO, "notes.md"), "w") as handle:
  244. handle.write("# notes\n")
  245. subprocess.run(["git", "-C", DEMO, "add", "-A"], check=True, capture_output=True)
  246. subprocess.run(["git", "-C", DEMO, "commit", "-m", "two"], check=True, capture_output=True)
  247. subprocess.run(["git", "-C", DEMO, "push", "origin", "main"], check=True, capture_output=True)
  248. def tool_call(name, arguments):
  249. return "tool:%s:%s" % (name, json.dumps(arguments))
  250. def parse_call(spec):
  251. """One mock call spec back into the {name, arguments} shape the
  252. JSON array and content+calls grammars expect."""
  253. name, arguments = spec[len("tool:"):].split(":", 1)
  254. return {"name": name, "arguments": json.loads(arguments)}
  255. def mock_argv():
  256. """The protocol-native arc, one scripted assistant turn each,
  257. consumed in strict arrival order (single daemon, sequential
  258. sessions: goal run parks before the worker starts, the worker
  259. finishes before the goal agent's terminal-event resume)."""
  260. question = ("cat << 'SADDLE_EOF' | saddle block question \"Demo only\" \"Whole workspace\"\n"
  261. "Which scope should the greeting cover? The demo repo is small; confirm the blast radius.\n"
  262. "SADDLE_EOF")
  263. responses = [
  264. # G1 (fresh goal agent): narrated probe — turn_text plus one
  265. # tool call, the protocol-native shape of the old commentary
  266. # round
  267. json.dumps({"content": "Reading the brief before touching anything — the demo repo is"
  268. " tiny; the greeting belongs in greeting.txt.",
  269. "calls": [parse_call(tool_call("run_command", {"command": "ls -la"}))]}),
  270. # G2: investigate, then block on a scope question (the saddle
  271. # CLI stays the question surface; run_command feeds the PTY)
  272. json.dumps({"content": "Two commits and one file — the blast radius is greeting.txt"
  273. " alone; confirming the scope with the human.",
  274. "calls": [parse_call(tool_call("run_command", {"command": "git log --oneline | head -3"}))]}),
  275. tool_call("run_command", {"command": question}),
  276. # G3 (answer resume): submit the plan -> plan review park
  277. tool_call("submit_plan", {"markdown": "# Plan v1\n\n1. Add greeting.txt to the demo repo."}),
  278. # G4 (feedback resume): the revision incorporating the feedback
  279. tool_call("submit_plan", {"markdown": "# Plan v2\n\n1. Add greeting.txt to the demo repo.\n"
  280. "2. Export it for reuse."}),
  281. # G5/G6 (approval resume): fan out through the real callbacks,
  282. # then park suspended (enqueue + suspend tolerate sharing one
  283. # message — the harness executes them in order)
  284. tool_call("create_task", {"title": "Write the greeting", "mode": "implement",
  285. "brief": "Create greeting.txt containing hello-from-the-goal."}),
  286. json.dumps([parse_call(tool_call("enqueue_task", {"task_id": 2})),
  287. parse_call(tool_call("suspend", {}))]),
  288. # W1..W5 (the worker): five IDENTICAL probes — the loop
  289. # detector injects its warning user turn after the fifth,
  290. # never refusing execution
  291. ] + [tool_call("run_command", {"command": "echo probing-the-workspace"})] * 5 + [
  292. # W6: post a durable remark (§7.2 comment kind — the body is
  293. # the record; the web renders it as a narrative footnote)
  294. tool_call("add_comment", {"body": "Probing done — the mandate starts on the"
  295. " ephemeral branch."}),
  296. # W7: the clean mandate — verify the ephemeral branch, commit
  297. json.dumps({"content": "Loop flagged — switching to the mandate on the ephemeral branch.",
  298. "calls": [parse_call(tool_call("run_command", {"command": "git branch --show-current"}))]}),
  299. tool_call("run_command", {"command": "echo hello-from-the-goal > greeting.txt && git add -A"
  300. " && git commit -m 'greeting from the goal'"}),
  301. tool_call("finish", {"status": "done",
  302. "summary": "# Report\n\n- changed: greeting.txt\n"
  303. "- verified: committed on the ephemeral branch"}),
  304. # G7 (the coalesced terminal-event resume): hold the working
  305. # window open a moment (the /status branch probe samples it),
  306. # then merge, prune, push
  307. tool_call("run_command", {"command": "sleep 2"}),
  308. tool_call("run_command", {"command": "git merge --no-edit $(git for-each-ref"
  309. " --format='%(refname:short)' refs/heads/task/*)"
  310. " && git branch -D $(git for-each-ref"
  311. " --format='%(refname:short)' refs/heads/task/*)"
  312. " && git push -u origin $(git branch --show-current)"}),
  313. # G8: submit_report -> completion review park
  314. tool_call("submit_report", {"markdown": "# Goal completion report\n\n"
  315. "- achieved: greeting.txt added via one worker task\n"
  316. "- verified: the goal branch is pushed to origin"}),
  317. ]
  318. argv = []
  319. for response in responses:
  320. argv += ["-r", response]
  321. argv += [
  322. "--on-query", "The hostname is %s." % HOSTNAME,
  323. # Every reply lingers half a second: the local mock answers so
  324. # instantly that a worker's whole run (its DERIVED-busy open
  325. # window) can close inside one sampling interval otherwise.
  326. "--delay", "0.5",
  327. # receipt forensics: every prompt the mock received, so the
  328. # harness can assert the tool results reached the model
  329. "--capture", CAPTURE_PATH,
  330. ]
  331. return argv
  332. def start_mock():
  333. if MOCK_CMD:
  334. # e.g. MOCK_CMD='podman exec -d clanker-saddle python3
  335. # /usr/libexec/saddle/mock_llm.py' — the same scripted argv is
  336. # appended, so the in-container mock behaves identically.
  337. argv = MOCK_CMD.split() + mock_argv()
  338. else:
  339. argv = [sys.executable, MOCK, "--port", MOCK_PORT] + mock_argv()
  340. return subprocess.Popen(argv, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
  341. def markdown_renderer_checks():
  342. """Exercise the stm-markdown renderer shipped inside statum.js via node."""
  343. status, body = http("GET", "/_statum/resource/statum.js", timeout=15)
  344. if status != 200:
  345. check(False, "statum.js served to the browser")
  346. return
  347. check("stm-markdown" in body, "statum.js ships the stm-markdown directive")
  348. def extract(name, source):
  349. start = source.index("function %s(" % name)
  350. depth = 0
  351. index = source.index("{", start)
  352. end = index
  353. while True:
  354. if source[end] == "{":
  355. depth += 1
  356. elif source[end] == "}":
  357. depth -= 1
  358. if depth == 0:
  359. break
  360. end += 1
  361. return source[start:end + 1]
  362. try:
  363. parts = [extract(name, body)
  364. for name in ("safeMarkdownUrl", "mdLinks", "mdInline", "renderMarkdown")]
  365. except ValueError:
  366. check(False, "statum.js carries the markdown renderer functions")
  367. return
  368. harness = (
  369. "%s\n%s\n%s\n%s\n"
  370. "console.log(renderMarkdown('# Title\\n\\n- **bold** item\\n- `code` item\\n"
  371. "\\n> quoted text\\n\\n```text\\nfenced\\n```'));\n"
  372. "console.log('@@' + renderMarkdown('<script>alert(1)</script> [bad](javascript:alert(2))"
  373. " [ok](https://example.com) <img src=x onerror=alert(3)>'));\n" % tuple(parts)
  374. )
  375. try:
  376. result = subprocess.run(["node", "-e", harness], capture_output=True, text=True, timeout=20)
  377. except (OSError, subprocess.SubprocessError):
  378. check(True, "stm-markdown renderer present (node unavailable, deep check skipped)")
  379. return
  380. rendered, _, xss = result.stdout.partition("@@")
  381. check("<h1>" in rendered and "<li>" in rendered and "<code>" in rendered
  382. and "<blockquote>" in rendered and "<strong>" in rendered,
  383. "stm-markdown renders headings, lists, code, quotes, bold")
  384. check("<script" not in xss and "<img" not in xss
  385. and 'href="javascript:' not in xss and 'href="#"' in xss
  386. and '<a href="https://example.com">ok</a>' in xss,
  387. "stm-markdown escapes raw tags, defangs javascript: links, keeps safe ones")
  388. def main():
  389. prepare_workspace()
  390. if os.path.exists(CAPTURE_PATH):
  391. os.remove(CAPTURE_PATH)
  392. if db_query("select count(*) from tasks")[0][0] or db_query("select count(*) from goals")[0][0]:
  393. raise SystemExit("the web database is not fresh (tasks/goals exist) — "
  394. "wipe the DB and restart the app+daemon for a clean goal-era run")
  395. mock = start_mock()
  396. time.sleep(2.0)
  397. try:
  398. # A previous suite run (or a crashed orchestrator) can leave the
  399. # daemon parked on a block; /stop releases it so dispatch works.
  400. try:
  401. http("POST", SADDLE + "/stop", timeout=5)
  402. except (urllib.error.URLError, OSError):
  403. pass
  404. auth = login("admin", "adminpass123")
  405. check(True, "login yields an auth slot")
  406. state = slot_public(auth, "/projects", "projects")
  407. act(auth, state["create"], {
  408. "name": "demo",
  409. "git_url": DEMO,
  410. "default_branch": "main",
  411. "summary": "Integration test repository",
  412. })
  413. check(any(p["name"] == "demo" for p in slot_public(auth, "/projects", "projects")["projects"]),
  414. "project demo created (local git_url, pushable origin)")
  415. state = slot_public(auth, "/saddles", "saddles")
  416. act(auth, state["register"], {"name": SADDLE_NAME, "host": SADDLE_HOST})
  417. registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  418. if s["name"] == SADDLE_NAME]
  419. check(len(registered) == 1 and registered[0]["token"].startswith("saddle-"),
  420. "saddle registered at %s with an API token" % SADDLE_HOST)
  421. print("== goal create -> start ==")
  422. goals_state = slot_public(auth, "/goals", "goals")
  423. act(auth, goals_state["create"], {
  424. "project_id": "1",
  425. "title": "Greet the demo",
  426. "brief": "Add a greeting file to the demo repo end to end.",
  427. "branch_name": "goal/demo-run",
  428. "priority": "3",
  429. })
  430. goal1 = int([g["id"] for g in slot_public(auth, "/goals", "goals")["goals"]
  431. if g["title"] == "Greet the demo"][0])
  432. check(goal_state(auth, goal1)["status"] == "draft", "the goal lands in draft")
  433. act(auth, goal_state(auth, goal1)["start"], {})
  434. check(wait_goal_status(auth, goal1, "planning", timeout=30) is not None,
  435. "starting the goal moves it to planning (agent dispatched)")
  436. run_task = db_query(
  437. "select id from tasks where goal_id = ? and kind = 'goal_run'", (goal1,))[0][0]
  438. check(wait_for(lambda: task_state(auth, run_task)["status"]
  439. in ("assigned", "working", "blocked", "suspended") or False,
  440. timeout=30, what="goal run dispatched") is not None,
  441. "the goal-run task reaches the daemon (assigned/working/blocked)")
  442. goal_clone = os.path.join(WORKSPACE, "goal-%d" % goal1, "demo")
  443. check(wait_for(lambda: os.path.isdir(os.path.join(goal_clone, ".git")) or False,
  444. timeout=30, what="the goal clone") is not None,
  445. "the goal clone appears under the shared workspace root (goal-<id>)")
  446. branch = subprocess.run(["git", "-C", goal_clone, "branch", "--show-current"],
  447. capture_output=True, text=True).stdout.strip()
  448. check(branch == "goal/demo-run", "the goal clone sits on the user-named goal branch")
  449. print("== pre-plan question -> one answer, one row ==")
  450. check(wait_for(lambda: any(b["block_type"] == "question" and not b.get("answered")
  451. for b in goal_state(auth, goal1).get("blocks", [])) or False,
  452. timeout=90, what="the agent's question block") is not None,
  453. "the pre-plan question surfaces as a goal block")
  454. check(wait_task_status(auth, run_task, "blocked", timeout=30) is not None,
  455. "the goal-run task parks blocked on its own block row")
  456. question = [b for b in goal_state(auth, goal1)["blocks"]
  457. if b["block_type"] == "question" and not b.get("answered")][0]
  458. check(question["options"] == ["Demo only", "Whole workspace"],
  459. "the question carries its options")
  460. check("Which scope should the greeting cover" in question["question"],
  461. "the question title derives from the body's first line")
  462. check("confirm the blast radius" in question["body"], "the question carries the markdown body")
  463. dupes = db_query(
  464. "select count(*) from task_blocks tb join tasks t on t.id = tb.task_id"
  465. " where t.goal_id = ? and t.kind = 'goal_run' and tb.type = 'question'",
  466. (goal1,))[0][0]
  467. check(dupes == 1, "one question row despite the park-unwinding round-end push")
  468. dash = slot_public(auth, "/", "dashboard")
  469. check(any(int(r.get("goal_id") or 0) == goal1 for r in dash["pinned"]),
  470. "the dashboard pile pins the goal block")
  471. act(auth, question["answer_action"], {"option": "Demo only"})
  472. answered_rows = db_query(
  473. "select answer from task_blocks tb join tasks t on t.id = tb.task_id"
  474. " where t.goal_id = ? and t.kind = 'goal_run' and tb.type = 'question'",
  475. (goal1,))
  476. check(answered_rows and answered_rows[0][0] == "Demo only",
  477. "the ONE answer lands on the goal-run task's single row")
  478. print("== plan review: reject -> revise -> approve ==")
  479. check(wait_goal_status(auth, goal1, "plan_review", timeout=90) is not None,
  480. "the plan submission moves the goal to plan_review")
  481. check(wait_for(lambda: goal_state(auth, goal1).get("plan_pending_block") not in
  482. (None, False) or False, timeout=30, what="pending plan block") is not None,
  483. "the pending plan block renders on the goal page")
  484. pending = goal_state(auth, goal1)["plan_pending_block"]
  485. check("Plan v1" in pending["body"], "the plan block carries the submitted markdown")
  486. # Plan 16 S2/R4: the goal page posts a `feedback` reply / a
  487. # `verdict` marker — the server owns the stable wording the
  488. # daemon's resume matching reads.
  489. act(auth, pending["answer_action"], {"feedback": "also export the greeting"})
  490. check(wait_goal_status(auth, goal1, "planning", timeout=30) is not None,
  491. "rejecting with feedback returns the goal to planning")
  492. check(wait_for(lambda: goal_state(auth, goal1).get("plan_pending_block") not in
  493. (None, False) and "Plan v2" in
  494. goal_state(auth, goal1)["plan_pending_block"].get("body", "") or False,
  495. timeout=90, what="revised plan") is not None,
  496. "the agent revises and re-submits incorporating the feedback")
  497. revised = goal_state(auth, goal1)
  498. check(len([r for r in revised["plan_revisions"] if r.get("answered")]) >= 1,
  499. "the rejected revision stays discoverable behind the expander")
  500. act(auth, revised["plan_pending_block"]["answer_action"], {"verdict": "accept"})
  501. check(wait_goal_status(auth, goal1, "active", timeout=30) is not None,
  502. "approving with the agreed wording activates the goal")
  503. print("== task fan-out via the real callbacks ==")
  504. worker_row = wait_for(lambda: db_query(
  505. "select id from tasks where goal_id = ? and kind = 'task'", (goal1,)) or False,
  506. timeout=90, what="the agent-created task")
  507. worker_task = int(worker_row[0][0])
  508. check(bool(worker_row), "the goal agent created its worker task through the callback")
  509. creator = db_query("select creator, mode from tasks where id = ?", (worker_task,))[0]
  510. check(creator == ("goal", "implement"),
  511. "the created task carries creator=goal and the requested mode")
  512. check(wait_task_status(auth, worker_task, "queued", timeout=30) is not None
  513. or task_state(auth, worker_task)["status"] in ("working", "done"),
  514. "the agent enqueued its task (creator-only rule honoured by construction)")
  515. ephemeral = "task/%d-%d" % (goal1, worker_task)
  516. check(wait_for(lambda: task_state(auth, worker_task)["status"]
  517. in ("assigned", "working", "done", "failed") or False,
  518. timeout=60, what="worker dispatched") is not None,
  519. "the dispatcher hands the worker to the freed saddle")
  520. check(wait_for(lambda: db_query(
  521. "select suspended_at is not null from tasks where id = ?", (run_task,))[0][0] or False,
  522. timeout=30, what="the goal agent parks") is not None,
  523. "the goal agent parked suspended after fanning out (suspended_at stamped)")
  524. # Plan 16 item 23: busy DERIVES from the open session (the
  525. # schema carries no work cache) — while the worker runs, the
  526. # card reads busy with the worker as its current task. The
  527. # open window can be sub-second (the local mock answers
  528. # instantly), so the card read shares the predicate iteration
  529. # with the open-session probe.
  530. check(wait_for(lambda: bool(db_query(
  531. "select 1 from sessions where task_id = ? and ended_at is null",
  532. (worker_task,))[0:1]) and (lambda card: card is not None
  533. and card["status"] == "busy"
  534. and int(card["current_task_id"] or 0) == worker_task)(
  535. next((s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  536. if s["name"] == SADDLE_NAME), None)) or False,
  537. timeout=90, interval=0.2, what="derived busy while the worker runs") is not None,
  538. "the saddle card derives busy from the worker's open session (no cached row)")
  539. print("== the worker's ephemeral worktree ==")
  540. # The branch probe rides the ledger: the tool_call event carries
  541. # the git command, its tool_result event the branch on stdout.
  542. branch_rows = wait_for(lambda: db_query(
  543. "select payload from events where scope_type = 'task' and scope_id = ?"
  544. " and kind = 'tool_result' and payload like ?",
  545. (worker_task, "%" + ephemeral + "%")) or False,
  546. timeout=90, what="the worker's branch probe")
  547. check(branch_rows and ephemeral in (branch_rows[0][0] or ""),
  548. "the worker runs inside its ephemeral branch %s (tool_result stdout)" % ephemeral)
  549. branch_call = db_query(
  550. "select count(*) from events where scope_type = 'task' and scope_id = ?"
  551. " and kind = 'tool_call' and payload like '%git branch --show-current%'",
  552. (worker_task,))[0][0]
  553. check(branch_call == 1, "the branch probe's tool_call event sits in the ledger")
  554. worktree_dir = os.path.join(WORKSPACE, "task-%d" % worker_task)
  555. check(wait_for(lambda: task_state(auth, worker_task)["status"] == "done" or False,
  556. timeout=120, what="worker done") is not None,
  557. "the worker completes with its report")
  558. detail = task_state(auth, worker_task)
  559. check("greeting.txt" in (detail.get("summary") or ""),
  560. "the worker summary carries its completion report")
  561. check(not os.path.exists(worktree_dir), "the worker's worktree auto-pruned on done")
  562. kept = subprocess.run(["git", "-C", goal_clone, "rev-parse", "--verify", ephemeral],
  563. capture_output=True).returncode == 0
  564. check(kept, "the ephemeral branch survives in the goal clone for the merge")
  565. print("== coalesced resume -> merge -> push -> completion review ==")
  566. check(wait_for(lambda: any(
  567. e["kind"] == "task-done" and int(e["task_id"]) == worker_task
  568. for e in goal_state(auth, goal1).get("events", [])) or False,
  569. timeout=30, what="the task-done narrative row") is not None,
  570. "the worker's terminal transition lands in the derived goal narrative")
  571. # the goal agent's /status reports the branch push state while it
  572. # works; sample the moment the resumed agent is mid-round, then
  573. # cache it through the manual saddle refresh
  574. check(wait_for(lambda: (saddle_status() or {}).get("goal_branches") or False,
  575. timeout=90, what="the resumed agent exposes goal_branches") is not None,
  576. "the daemon's /status exposes goal_branches while the goal agent works")
  577. saddles_state = slot_public(auth, "/saddles", "saddles")
  578. card = [s for s in saddles_state["saddles"] if s["name"] == SADDLE_NAME][0]
  579. act(auth, card["refresh"], {})
  580. check(wait_for(lambda: any(
  581. "demo" in (b.get("branch") or "") and isinstance(b.get("pushed"), bool)
  582. for b in goal_state(auth, goal1).get("push_branches", []))
  583. or False, timeout=60, what="cached push state") is not None,
  584. "the refresh probe caches the branch push state on the goal (live chips)")
  585. check(wait_goal_status(auth, goal1, "completion_review", timeout=120) is not None,
  586. "the resumed agent merges, pushes and enters completion review")
  587. state = goal_state(auth, goal1)
  588. completion = state["completion_pending_block"]
  589. # Plan 16 S1 (canonical-on-accept): pending, the report lives
  590. # ONLY in the block — goals.completion_report stays empty
  591. # until the accept verdict writes the winning body.
  592. check("greeting.txt added via one worker task" in (completion or {}).get("body", ""),
  593. "the completion report renders on the goal page (the pending block's body)")
  594. check(not (state.get("completion_report") or ""),
  595. "goals.completion_report stays clean before the verdict (canonical-on-accept)")
  596. merged = subprocess.run(
  597. ["git", "-C", goal_clone, "show", "goal/demo-run:greeting.txt"],
  598. capture_output=True, text=True).stdout
  599. check("hello-from-the-goal" in merged, "the merged task branch landed on the goal branch")
  600. deleted = subprocess.run(["git", "-C", goal_clone, "rev-parse", "--verify", ephemeral],
  601. capture_output=True).returncode != 0
  602. check(deleted, "the goal agent deleted the merged task branch")
  603. pushed = subprocess.run(["git", "-C", DEMO, "rev-parse", "--verify", "goal/demo-run"],
  604. capture_output=True).returncode == 0
  605. check(pushed, "the goal branch was pushed to origin (push-before-review)")
  606. print("== accept refuses while origin is behind (the live push gate) ==")
  607. # Rewind origin's goal branch to the pre-merge base and refresh
  608. # the daemon probe: the live per-repo flags flip to unpushed and
  609. # the completion accept must REFUSE (plan 15 C4 / F9) until the
  610. # branch is pushed again — enforcing push-after-each-merge at
  611. # the human gate, read from probe state (never a snapshot).
  612. completion = state["completion_pending_block"]
  613. subprocess.run(["git", "-C", goal_clone, "push", "-f", "origin",
  614. "main:refs/heads/goal/demo-run"], check=True, capture_output=True)
  615. subprocess.run(["git", "-C", goal_clone, "fetch", "origin"],
  616. check=True, capture_output=True)
  617. act(auth, card["refresh"], {})
  618. check(wait_for(lambda: any(
  619. "demo" in (b.get("branch") or "") and b.get("pushed") is False
  620. for b in goal_state(auth, goal1).get("push_branches", [])) or False,
  621. timeout=60, what="unpushed chip") is not None,
  622. "the rewound origin flips the live push chip to unpushed")
  623. refusal = act(auth, completion["answer_action"], {"verdict": "accept"})
  624. check(any(d.get("type") == "notify" and d.get("kind") == "error"
  625. and "Origin is behind" in (d.get("message") or "")
  626. for d in refusal),
  627. "the accept refuses with a clear message while origin is behind")
  628. check(goal_state(auth, goal1)["status"] == "completion_review",
  629. "the refused accept leaves the goal in completion review")
  630. check(goal_state(auth, goal1)["completion_pending_block"]
  631. not in (None, False),
  632. "the refused accept leaves the completion block unanswered")
  633. subprocess.run(["git", "-C", goal_clone, "push", "origin", "goal/demo-run"],
  634. check=True, capture_output=True)
  635. act(auth, card["refresh"], {})
  636. check(wait_for(lambda: any(
  637. "demo" in (b.get("branch") or "") and b.get("pushed") is True
  638. for b in goal_state(auth, goal1).get("push_branches", [])) or False,
  639. timeout=60, what="pushed chip restored") is not None,
  640. "re-pushing the branch restores the live push chip (after push)")
  641. completion = goal_state(auth, goal1)["completion_pending_block"]
  642. print("== completion accept ==")
  643. check("greeting.txt added via one worker task" in (completion or {}).get("body", ""),
  644. "the run task's completion block carries the report")
  645. winning_body = (completion or {}).get("body", "")
  646. accept = act(auth, completion["answer_action"], {"verdict": "accept"})
  647. check(not any(d.get("type") == "notify" and d.get("kind") == "error" for d in accept),
  648. "the accept passes once the push flags read pushed")
  649. check(wait_goal_status(auth, goal1, "complete", timeout=30) is not None,
  650. "accepting the verdict marker completes the goal (server-owned wording)")
  651. check(wait_task_status(auth, run_task, "done", timeout=30) is not None,
  652. "the goal-run task closes done with the goal")
  653. final = goal_state(auth, goal1)
  654. check(final.get("completion_report") == winning_body,
  655. "the accept canonises the winning block's body as goals.completion_report (S1)")
  656. print("== audit trail (the derived narrative + the unified ledger) ==")
  657. narrative = goal_state(auth, goal1)["events"]
  658. kinds = [e["kind"] for e in narrative]
  659. for expected in ("started", "block-raised", "plan-submitted", "plan-rejected",
  660. "plan-approved", "task-created", "task-done",
  661. "completion-submitted", "goal-approved"):
  662. check(expected in kinds, "the derived narrative records %s" % expected)
  663. check("system" not in kinds,
  664. "mechanical system rows never reach the user-facing narrative (U2)")
  665. check(all(k != "task-enqueued" for k in kinds),
  666. "timestamp-less bookkeeping never returns (task-enqueued is gone)")
  667. check(db_query(
  668. "select count(*) from sqlite_master where type = 'table'"
  669. " and name = 'goal_events'")[0][0] == 0,
  670. "the goal_events table is gone (nothing is written on ingest)")
  671. check(db_query(
  672. "select count(*) from sqlite_master where type = 'table'"
  673. " and name = 'session_commands'")[0][0] == 0,
  674. "the session_commands table is gone (the ledger is the record)")
  675. run_kinds = {row[0] for row in db_query(
  676. "select distinct kind from events where scope_type = 'task' and scope_id = ?",
  677. (run_task,))}
  678. check({"turn_text", "tool_call", "tool_result", "usage", "lifecycle"} <= run_kinds,
  679. "the goal agent's ledger carries the protocol kinds (%s)" % sorted(run_kinds))
  680. worker_kinds = {row[0] for row in db_query(
  681. "select distinct kind from events where scope_type = 'task' and scope_id = ?",
  682. (worker_task,))}
  683. check({"turn_text", "tool_call", "tool_result", "usage"} <= worker_kinds,
  684. "the worker's ledger carries the protocol kinds (%s)" % sorted(worker_kinds))
  685. daemon_kinds = {row[0] for row in db_query(
  686. "select distinct kind from events where session_id is not null")}
  687. check(daemon_kinds <= {"turn_text", "tool_call", "tool_result", "progress",
  688. "comment", "lifecycle", "usage", "state_change"},
  689. "every daemon-pushed kind is a §7.2 kind (%s)" % sorted(daemon_kinds))
  690. remark_rows = db_query(
  691. "select actor, payload from events where kind = 'comment'"
  692. " and session_id is not null and scope_id = ?", (worker_task,))
  693. check(len(remark_rows) == 1 and remark_rows[0][0] == "model"
  694. and "the mandate starts on the ephemeral branch" in (remark_rows[0][1] or ""),
  695. "the add_comment remark rides the ledger as a model-actor comment event")
  696. narrative_remarks = [e["text"] for e in goal_state(auth, goal1)["events"]
  697. if e["kind"] == "comment"]
  698. check(any(text.startswith("Probing done") for text in narrative_remarks),
  699. "the goal narrative renders the worker's remark as a comment row")
  700. plan_call = db_query(
  701. "select count(*) from events where scope_type = 'task' and scope_id = ?"
  702. " and kind = 'tool_call' and payload like '%submit_plan%'", (run_task,))[0][0]
  703. check(plan_call >= 2, "both plan submissions sit as tool_call events")
  704. usage_rows = db_query(
  705. "select payload from events where kind = 'usage' and scope_type = 'task'"
  706. " and scope_id = ? limit 1", (worker_task,))
  707. check(bool(usage_rows) and "prompt_tokens" in (usage_rows[0][0] or "")
  708. and "elapsed_ms" in (usage_rows[0][0] or ""),
  709. "usage events carry prompt/completion tokens and elapsed time")
  710. print("== the terminal watermark (C3) drives the resume ==")
  711. # The fresh-DB schema diff (the joint-QA gate): the removed
  712. # surfaces stay removed — the saddle work cache (item 23), the
  713. # /update-era columns (arch 19) and the legacy token table
  714. # (arch 24).
  715. saddle_cols = {row[1] for row in db_query("pragma table_info('saddles')")}
  716. check(not ({"current_task_id", "allocated_at"} & saddle_cols),
  717. "the saddles work-cache columns are gone (sessions are truth)")
  718. task_cols = {row[1] for row in db_query("pragma table_info('tasks')")}
  719. check("working_branch" not in task_cols,
  720. "tasks.working_branch is gone (only the deleted /update done wrote it)")
  721. check(db_query(
  722. "select count(*) from sqlite_master where type = 'table'"
  723. " and name = 'api_tokens'")[0][0] == 0,
  724. "the api_tokens table is gone (saddles.token is the single credential store)")
  725. watermark_row = db_query(
  726. "select last_terminal_watermark from goals where id = ?", (goal1,))
  727. worker_done_at = db_query(
  728. "select completed_at from tasks where id = ?", (worker_task,))[0][0]
  729. check(watermark_row[0][0] is not None and worker_done_at is not None
  730. and watermark_row[0][0] >= worker_done_at,
  731. "the goals row's terminal watermark advanced past the worker's transition"
  732. " (the resume consumed it)")
  733. print("== the harness bookkeeping stays ledger-internal (C2 + U2) ==")
  734. # Plan 16 S5 (U2): the mechanical rows (restores, park notices)
  735. # author NO user-facing comment rows — they persist as
  736. # system-authored comment events (the unified ledger) and never
  737. # render on a user-facing timeline.
  738. mechanical = db_query(
  739. "select count(*) from events where kind = 'comment' and actor = 'system'"
  740. " and (payload like '%session context restored%'"
  741. " or payload like '%parked until%')")[0][0]
  742. check(mechanical >= 1,
  743. "the harness bookkeeping (restore, park) persists as system comment events")
  744. thread = goal_state(auth, goal1)["comments"]
  745. check(not any("session context restored" in c["body"] or "parked until" in c["body"]
  746. for c in thread),
  747. "mechanical rows never render on the goal thread (U2)")
  748. run_thread = task_state(auth, run_task)["comments"]
  749. check(not any("session context restored" in c["body"] or "parked until" in c["body"]
  750. for c in run_thread),
  751. "the task page reads the same filtered thread (no mechanical rows)")
  752. check(not any(c.get("from_agent") and "[harness]" in c["body"]
  753. for c in thread),
  754. "no harness-authored row renders as agent content")
  755. print("== parks close their sessions (D4) ==")
  756. outcomes = db_query(
  757. "select outcome, ended_at is not null from sessions where task_id in (?, ?)",
  758. (run_task, worker_task))
  759. check(bool(outcomes)
  760. and all(row[0] in ("parked", "done") and row[1] for row in outcomes),
  761. "every run/worker session closed exactly once — parks stamp outcome 'parked',"
  762. " none dangle open (%s)" % sorted({row[0] for row in outcomes}))
  763. print("== D-side through both binaries: tool results + loop warning ==")
  764. messages = capture_messages()
  765. tool_turns = [text for role, text in messages if role == "tool"]
  766. user_turns = [text for role, text in messages if role == "user"]
  767. check(any("probing-the-workspace" in turn for turn in tool_turns),
  768. "the probes' tool results reached the model as role=tool turns")
  769. check(any(ephemeral in turn for turn in tool_turns),
  770. "the branch probe's result (the ephemeral branch name) reached the model")
  771. check(any("you ran the exact same command" in turn and "you are in a loop" in turn
  772. for turn in user_turns),
  773. "the loop-warning notice reached the model as its next user turn")
  774. check(any("Human selected" in turn for turn in user_turns),
  775. "the question's verdict wording reached the model on resume")
  776. print("== event log page + SSE ==")
  777. # The §7.4 JSON page: newest-first window of the run task's
  778. # ledger (id, session_id, seq, kind, actor, payload).
  779. status, page = saddle_api("GET", "/tasks/%d/events?limit=200" % run_task, None)
  780. page_events = page.get("events", []) if status == 200 else []
  781. check(status == 200 and len(page_events) >= 10,
  782. "the event log page serves the run's ledger (%d events)" % len(page_events))
  783. ids = [e["id"] for e in page_events]
  784. check(ids == sorted(ids, reverse=True), "the page is newest-first")
  785. check(any(e["kind"] == "tool_call" and e["payload"].get("tool") == "submit_plan"
  786. for e in page_events),
  787. "turn cards render: submit_plan tool_call events in the page")
  788. check(any(e["kind"] == "tool_result" and e["payload"].get("tool") == "run_command"
  789. and "stdout" in e["payload"].get("result", "") for e in page_events),
  790. "tool_result events carry the §7.1 result shape (stdout embedded)")
  791. check(any(e["kind"] == "turn_text" and "Reading the brief" in e["payload"].get("text", "")
  792. for e in page_events),
  793. "the narrated turn renders as turn_text")
  794. check(all(e.get("seq") is not None and e["seq"] > 0 for e in page_events
  795. if e["kind"] in ("turn_text", "tool_call", "tool_result", "usage")),
  796. "daemon events carry their per-session seq")
  797. state = goal_state(auth, goal1)
  798. check(any(r.get("terminal_url", "").endswith("/tasks/%d/log" % run_task)
  799. for r in state.get("runs", [])),
  800. "the goal page's run list deep-links each session's event log")
  801. request = urllib.request.Request(WEB + "/tasks/%d/events/stream" % run_task)
  802. with urllib.request.urlopen(request, timeout=15) as stream:
  803. check(stream.headers.get("Content-Type", "").startswith("text/event-stream"),
  804. "event stream is text/event-stream")
  805. streamed = []
  806. deadline = time.monotonic() + 10
  807. while time.monotonic() < deadline and len(streamed) < 120:
  808. line = stream.readline().decode().strip()
  809. if not line.startswith("data: ") or line == "data: ping":
  810. continue
  811. try:
  812. streamed.append(json.loads(line[6:]))
  813. except ValueError:
  814. continue
  815. if len(streamed) >= len(page_events):
  816. break
  817. check(len(streamed) >= 10,
  818. "the stream replays the initial most-recent-30 window (%d events)" % len(streamed))
  819. check(any(e.get("kind") == "tool_call" for e in streamed)
  820. and any(e.get("kind") == "tool_result" for e in streamed),
  821. "the SSE replay carries tool_call/tool_result events from the daemon")
  822. pulse = directives("/goals/%d" % goal1, auth)
  823. check(any(d.get("type") == "subscribe" for d in pulse),
  824. "goal page subscribes to the pulse channel for live refresh")
  825. print("== push wire surface (§7.2 chunks + results states) ==")
  826. saddles_state = slot_public(auth, "/saddles", "saddles")
  827. act(auth, saddles_state["register"], {"name": "saddle-e2e-api", "host": "localhost:7199"})
  828. api_rows = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  829. if s["name"] == "saddle-e2e-api"]
  830. check(len(api_rows) == 1 and api_rows[0]["token"].startswith("saddle-"),
  831. "registration returns an API token")
  832. api_saddle = int(api_rows[0]["id"])
  833. api_token = api_rows[0]["token"]
  834. tasks_state = slot_public(auth, "/tasks", "tasks")
  835. act(auth, tasks_state["create"], {
  836. "project_id": "1",
  837. "title": "API surface probe",
  838. "description": "Chunks ingestion, exactly-once pushes",
  839. "priority": "6",
  840. "task_branch": "task/api-surface",
  841. })
  842. api_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  843. if t["title"] == "API surface probe"][0])
  844. db = sqlite3.connect(DB_PATH)
  845. db.execute("insert into sessions (task_id, saddle_id, started_at)"
  846. " values (?, ?, strftime('%s','now'))", (api_task, api_saddle))
  847. db.execute("update tasks set status = 'working' where id = ?", (api_task,))
  848. db.commit()
  849. api_session = db.execute("select max(id) from sessions").fetchone()[0]
  850. db.close()
  851. def push_state(state="working", block=None, summary=None, entries=None):
  852. """One state push in the daemon's /results shape."""
  853. payload = {"session_id": api_session, "state": state}
  854. if block is not None:
  855. payload["block"] = block
  856. if summary is not None:
  857. payload["summary"] = summary
  858. if entries is not None:
  859. payload["entries"] = entries
  860. return saddle_api("POST", "/api/saddle/results", api_token, payload)
  861. def push_events(events):
  862. """One §7.2 chunks batch ({task_id, session_id, events})."""
  863. return saddle_api("POST", "/api/saddle/chunks", api_token,
  864. {"task_id": api_task, "session_id": api_session,
  865. "events": events})
  866. status, _ = saddle_api("POST", "/api/saddle/results", None,
  867. {"session_id": api_session, "state": "working"})
  868. check(status == 401, "results push without a token is rejected")
  869. status, _ = saddle_api("POST", "/api/saddle/results", "saddle-not-a-token",
  870. {"session_id": api_session, "state": "working"})
  871. check(status == 401, "results push with an invalid token is rejected")
  872. status, _ = saddle_api("POST", "/api/saddle/chunks", None,
  873. {"session_id": api_session, "events": []})
  874. check(status == 401, "chunks push without a token is rejected")
  875. status, _ = saddle_api("POST", "/api/saddle/chunks", api_token,
  876. {"session_id": api_session, "events": [
  877. {"kind": "progress", "actor": "model", "ts": "now"}]})
  878. check(status == 400, "an event without a positive seq is refused (seq is mandatory)")
  879. # The poll-era ingest pair is GONE (arch 19): both routes 404
  880. # for any caller, authenticated or not — the subtask/plan
  881. # refusal arms (plan 12 removal) died with the channel.
  882. status, _ = saddle_api("POST", "/api/saddle/%d/update" % api_saddle, api_token,
  883. {"type": "status", "session_id": api_session,
  884. "data": {"state": "working"}})
  885. check(status == 404, "POST /api/saddle/{id}/update is gone (404)")
  886. status, _ = saddle_api("GET", "/api/saddle/%d/poll" % api_saddle, api_token)
  887. check(status == 404, "GET /api/saddle/{id}/poll is gone (404)")
  888. status, _ = saddle_api("POST", "/api/saddle/%d/update" % api_saddle, api_token,
  889. {"type": "subtask", "session_id": api_session,
  890. "data": {"title": "Ghost child"}})
  891. check(status == 404,
  892. "the /update subtask arm is gone with the channel (404, plan 12 removal)")
  893. status, _ = saddle_api("POST", "/api/saddle/%d/update" % api_saddle, api_token,
  894. {"type": "plan", "session_id": api_session,
  895. "data": {"plan": "# nope"}})
  896. check(status == 404,
  897. "the /update plan arm is gone with the channel (404, plans are goal-only)")
  898. # The retired results-entries array is consumed silently: no
  899. # comment row, no projection — round entries live as ledger
  900. # events on the chunks channel now.
  901. status, body = push_state(entries=[{"id": 9001, "command": "saddle comment",
  902. "output": "Surveyed the module", "exit_code": 0,
  903. "type": "comment",
  904. "timestamp": "2026-09-13T00:00:00Z"}])
  905. check(status == 200 and body.get("ok"),
  906. "a results push carrying entries is accepted (consumed silently)")
  907. check(not any(c["body"] == "Surveyed the module"
  908. for c in task_state(auth, api_task)["comments"]),
  909. "the retired entries array projects nothing (no comment row)")
  910. # §7.2 ingestion: a progress event lands in the ledger and the
  911. # task's progress view; the (session_id, seq) unique key makes
  912. # redelivery exactly-once.
  913. status, body = push_events([{"seq": 9000, "kind": "progress", "actor": "model",
  914. "ts": "2026-09-13T00:00:00Z",
  915. "payload": {"text": "Migrated the schemas"}}])
  916. check(status == 200 and body.get("stored") == 1,
  917. "the chunks push stores the progress event")
  918. status, body = push_events([{"seq": 9000, "kind": "progress", "actor": "model",
  919. "ts": "2026-09-13T00:00:01Z",
  920. "payload": {"text": "Migrated the schemas AGAIN"}}])
  921. check(status == 200 and body.get("stored") == 0,
  922. "the redelivered seq is consumed silently (UNIQUE(session_id, seq))")
  923. progress_view = task_state(auth, api_task)["progress"]
  924. check(len([p for p in progress_view
  925. if p["summary"] in ("Migrated the schemas", "Migrated the schemas AGAIN")]) == 1
  926. and progress_view[0]["summary"] == "Migrated the schemas",
  927. "progress events project exactly once per seq")
  928. status, body = push_state(state="blocked", block={
  929. "question": "Which database?", "type": "question",
  930. "options": ["SQLite", "Postgres"], "body": "# Pick one"})
  931. check(status == 200 and body.get("ok"),
  932. "the blocked push raises the question through the results channel")
  933. parked = db_query("select outcome from sessions where id = ?", (api_session,))[0][0]
  934. check(parked == "parked",
  935. "the raise parked the round's session (outcome 'parked', D4)")
  936. status, body = push_state(state="blocked", block={
  937. "question": "Review the migration", "type": "review",
  938. "body": "Migration **ready** for review"})
  939. check(status == 200, "the review-type block push is accepted")
  940. detail = task_state(auth, api_task)
  941. check(detail["status"] == "review_requested", "the review block flips the task status")
  942. check([b["options"] for b in detail["blocks"]
  943. if b["block_type"] == "question"] == [["SQLite", "Postgres"]],
  944. "the pushed block carries its options")
  945. check(detail["progress"][0]["summary"] == "Migrated the schemas",
  946. "progress history newest-first")
  947. status, _ = push_state(state="working")
  948. check(status == 200 and task_state(auth, api_task)["status"] == "review_requested",
  949. "a park-unwinding working push never resurrects the parked round")
  950. question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
  951. act(auth, question["answer_action"], {"option": "SQLite"})
  952. check([b["answer"] for b in task_state(auth, api_task)["blocks"]
  953. if b["block_type"] == "question"] == ["SQLite"],
  954. "option answer stored verbatim (the resuming assignment formats it)")
  955. act(auth, detail["comment"],
  956. {"body": "Check the migration output", "quote": "ERROR: table demos exists"})
  957. check(any(c["body"] == "Check the migration output" and "ERROR" in (c.get("quote") or "")
  958. and not c.get("from_agent")
  959. for c in task_state(auth, api_task)["comments"]),
  960. "the quoted user comment lands on the task thread (delivery rides the"
  961. " assignment carrier / dispatcher pass — unit-pinned)")
  962. act(auth, detail["cancel"], {"task_id": str(api_task)})
  963. print("== stm-markdown ==")
  964. markdown_renderer_checks()
  965. print("== quiesce ==")
  966. check(wait_for(
  967. lambda: (lambda rows: all(int(s["current_task_id"] or 0) == 0
  968. and s["status"] == "idle"
  969. for s in rows))(
  970. [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  971. if s["name"] == SADDLE_NAME]) or False,
  972. timeout=30, what="saddle idle with no current task") is not None,
  973. "saddle derived idle with no running work once the goal finished")
  974. finally:
  975. if mock is not None:
  976. mock.terminate()
  977. print("%d passed, %d failed" % (passes, failures))
  978. return 1 if failures else 0
  979. if __name__ == "__main__":
  980. sys.exit(main())