web_stack_test.py 61 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216
  1. #!/usr/bin/env python3
  2. """Self-contained web-stack feature tests: app + fake saddle daemon.
  3. Unlike integration_test.py (which needs a real saddle daemon and a mock
  4. LLM), this suite spawns everything it needs:
  5. - the clanker-saddle app binary (path from argv[1], or builddir default)
  6. in a scratch directory with its own web-config.json (statum keys copied
  7. from the repo, plus `saddle_callback_base_url`) and SQLite database
  8. - an in-process fake saddle daemon speaking the wire contract: POST
  9. /start (captures the payload), GET /status (scripted state machine),
  10. GET /output (empty), GET /log?since= (raw PTY chunks, one new chunk
  11. per request so since-paging is exercised)
  12. It then drives the app the way statum.js does (entrypoint loads + action
  13. POSTs) and asserts the Workstream B features against the wire contract:
  14. 3. plan-review blocks hide their duplicate body preview
  15. 5. rules CRUD/attach + the effective `rules` array in /start
  16. 6. project notes, POST /api/saddle/note, `project_notes` + `callback`
  17. in /start
  18. plus the raw-log side: session_raw_log polling + terminal raw view
  19. Prints PASS/FAIL lines Statum-style; exits non-zero on any failure.
  20. """
  21. import json
  22. import os
  23. import shutil
  24. import socket
  25. import sqlite3
  26. import subprocess
  27. import sys
  28. import threading
  29. import time
  30. import urllib.error
  31. import urllib.parse
  32. import urllib.request
  33. from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
  34. REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  35. APP = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(REPO, "builddir", "clanker-saddle")
  36. WORK = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
  37. DB_PATH = os.path.join(WORK, "web.sqlite")
  38. passes = 0
  39. failures = 0
  40. def check(condition, label):
  41. global passes, failures
  42. if condition:
  43. passes += 1
  44. print("PASS %s" % label)
  45. else:
  46. failures += 1
  47. print("FAIL %s" % label)
  48. def free_port():
  49. with socket.socket() as sock:
  50. sock.bind(("127.0.0.1", 0))
  51. return sock.getsockname()[1]
  52. APP_PORT = free_port()
  53. DAEMON_PORT = free_port()
  54. WEB = "http://127.0.0.1:%d" % APP_PORT
  55. def http(method, url, data=None, headers=None, timeout=30):
  56. body = None
  57. if data is not None:
  58. pairs = []
  59. for k, v in data.items():
  60. for item in (v if isinstance(v, (list, tuple)) else [v]):
  61. pairs.append((k, item))
  62. body = "&".join("%s=%s" % (k, urllib.parse.quote(str(item))) for k, item in pairs).encode()
  63. request = urllib.request.Request(WEB + url if url.startswith("/") else url,
  64. data=body, method=method)
  65. if data is not None:
  66. request.add_header("Content-Type", "application/x-www-form-urlencoded")
  67. for key, value in (headers or {}).items():
  68. request.add_header(key, value)
  69. # Retry dropped/refused connections: the freshly started app can
  70. # close a very early keep-alive connection once while warming up,
  71. # and its first seconds can wobble under load.
  72. for attempt in range(3):
  73. try:
  74. with urllib.request.urlopen(request, timeout=timeout) as response:
  75. return response.status, response.read().decode()
  76. except urllib.error.HTTPError as error:
  77. return error.code, error.read().decode()
  78. except (urllib.error.URLError, OSError):
  79. if attempt >= 2:
  80. raise
  81. time.sleep(0.75)
  82. def directives(uri, auth=None):
  83. headers = {}
  84. if auth:
  85. headers["X-Statum-Slot"] = "key=%s; as_at=%s" % (auth["key"], auth["as_at"])
  86. status, body = http("GET", "/_statum/entrypoint?uri=" + urllib.parse.quote(uri.split("?")[0]),
  87. headers=headers)
  88. if status != 200:
  89. raise SystemExit("entrypoint %s -> HTTP %d: %s" % (uri, status, body[:300]))
  90. return json.loads(body)
  91. def slot_public(auth, uri, type_name):
  92. for directive in directives(uri, auth):
  93. if directive["type"] == "set":
  94. content = json.loads(directive["snapshot_frame"]["content"])
  95. if content["slot"]["type"] == type_name:
  96. return content["public"]
  97. raise SystemExit("no %s slot in entrypoint state for %s" % (type_name, uri))
  98. def act(auth, action, fields):
  99. headers = {"X-Statum-Slot": "key=%s; as_at=%s" % (auth["key"], auth["as_at"])}
  100. if action.get("private"):
  101. headers["X-Statum-Private"] = action["private"]
  102. status, body = http(action.get("method", "POST"), action["uri"], fields, headers)
  103. if status != 200:
  104. raise SystemExit("action %s -> HTTP %d: %s" % (action["uri"], status, body[:300]))
  105. return json.loads(body)
  106. def login(username, password):
  107. frames = directives("/login")
  108. content = json.loads(frames[0]["snapshot_frame"]["content"])
  109. action = content["public"]["login"]
  110. status, body = http(action.get("method", "POST"), action["uri"],
  111. {"username": username, "password": password})
  112. if status != 200:
  113. raise SystemExit("login -> HTTP %d: %s" % (status, body[:300]))
  114. for directive in json.loads(body):
  115. if directive["type"] == "set":
  116. frame = json.loads(directive["snapshot_frame"]["content"])
  117. if frame["slot"]["type"] == "auth":
  118. return {"key": frame["slot"]["key"], "as_at": frame["as_at"]}
  119. raise SystemExit("login response carried no auth slot: " + body[:300])
  120. def wait_for(predicate, timeout=60, what="condition", interval=0.5):
  121. deadline = time.monotonic() + timeout
  122. last = None
  123. while time.monotonic() < deadline:
  124. last = predicate()
  125. if last is not False and last is not None:
  126. return last
  127. time.sleep(interval)
  128. print(" (timeout waiting for %s; last: %s)" % (what, last))
  129. return None
  130. def task_state(auth, task_id):
  131. return slot_public(auth, "/tasks/%d" % task_id, "task")
  132. def wait_task_done(auth, task_id, timeout=90):
  133. return wait_for(lambda: task_state(auth, task_id).get("status") == "done" or False,
  134. timeout=timeout, what="task %d -> done" % task_id)
  135. def task_id_of(title):
  136. conn = db()
  137. try:
  138. return conn.execute("select id from tasks where title = ?", (title,)).fetchone()[0]
  139. finally:
  140. conn.close()
  141. def saddle_api(method, path, token, payload=None):
  142. """Raw call against the saddle ingest API; returns (status, json)."""
  143. request = urllib.request.Request(WEB + path,
  144. data=json.dumps(payload or {}).encode(), method=method)
  145. request.add_header("Content-Type", "application/json")
  146. if token is not None:
  147. request.add_header("Authorization", "Bearer " + token)
  148. try:
  149. with urllib.request.urlopen(request, timeout=15) as response:
  150. return response.status, json.loads(response.read().decode() or "{}")
  151. except urllib.error.HTTPError as error:
  152. body = error.read().decode()
  153. try:
  154. return error.code, json.loads(body)
  155. except ValueError:
  156. return error.code, {"raw": body}
  157. def db():
  158. return sqlite3.connect(DB_PATH)
  159. class FakeSaddle:
  160. """In-process saddle daemon implementing the wire contract."""
  161. PROMPT = "bbarrow@fedora:/workspace/task-1$ "
  162. def __init__(self):
  163. self.lock = threading.Lock()
  164. self.start_payloads = [] # captured POST /start bodies
  165. self.chunks = [] # [{seq, text}] — the raw log ring
  166. self.next_seq = 1
  167. self.state = "idle"
  168. self.summary = ""
  169. self.status_polls_after_start = 0
  170. self.fail_next = False # next /start drives the task to failed
  171. self.start_counts = {} # task id -> /start count
  172. self.resumed_pending = False # crash-resume flag on the next /status
  173. self.suspend_after_start = False # next task parks as suspended
  174. self.pending_output = [] # entries drained by the next /output poll
  175. self.output_seq = 100
  176. def queue_output(self, entry_type, command, output):
  177. """Queues a typed entry for the dispatcher's next /output poll."""
  178. with self.lock:
  179. self.pending_output.append({
  180. "id": self.output_seq,
  181. "command": command,
  182. "output": output,
  183. "exit_code": 1,
  184. "type": entry_type,
  185. "timestamp": "2026-09-06T00:00:00Z",
  186. })
  187. self.output_seq += 1
  188. def append_chunk(self, text):
  189. with self.lock:
  190. self.chunks.append({"seq": self.next_seq, "text": text})
  191. self.next_seq += 1
  192. def log_since(self, since):
  193. with self.lock:
  194. # `since` is the inclusive resume point (wire contract: a
  195. # response ending at seq 12 carries next=13, and polling
  196. # with since=13 must still deliver the chunk numbered 13).
  197. chunks = [c for c in self.chunks if c["seq"] >= since]
  198. return chunks, self.next_seq
  199. class Handler(BaseHTTPRequestHandler):
  200. daemon = None # set on the class before serving
  201. protocol_version = "HTTP/1.1"
  202. def log_message(self, fmt, *args):
  203. pass
  204. def _send(self, status, obj):
  205. body = json.dumps(obj).encode()
  206. self.send_response(status)
  207. self.send_header("Content-Type", "application/json")
  208. self.send_header("Content-Length", str(len(body)))
  209. self.end_headers()
  210. self.wfile.write(body)
  211. def do_POST(self):
  212. length = int(self.headers.get("Content-Length") or 0)
  213. raw = self.rfile.read(length).decode() if length else "{}"
  214. if self.path == "/start":
  215. try:
  216. payload = json.loads(raw)
  217. except ValueError:
  218. payload = {"raw": raw}
  219. self.daemon.start_payloads.append(payload)
  220. task_id = (payload.get("task") or {}).get("id")
  221. count = self.daemon.start_counts.get(task_id, 0) + 1
  222. self.daemon.start_counts[task_id] = count
  223. if count > 1:
  224. # Crash-resume contract: re-pushing a task whose
  225. # workspace still exists restores it; the first
  226. # /status after the restore reports "resumed": true.
  227. self.daemon.resumed_pending = True
  228. self.daemon.append_chunk(self.daemon.PROMPT)
  229. self.daemon.append_chunk("echo hello\r\n")
  230. self.daemon.append_chunk("hello\r\n")
  231. self.daemon.state = "failing" if self.daemon.fail_next else "working"
  232. self.daemon.fail_next = False
  233. self.daemon.status_polls_after_start = 0
  234. self._send(200, {})
  235. elif self.path == "/answer":
  236. self._send(200, {})
  237. else:
  238. self._send(404, {"error": "no such route"})
  239. def do_GET(self):
  240. if self.path == "/status":
  241. response = {"state": self.daemon.state, "summary": self.daemon.summary}
  242. if self.daemon.resumed_pending:
  243. response["resumed"] = True
  244. self.daemon.resumed_pending = False
  245. if self.daemon.state in ("working", "failing"):
  246. self.daemon.status_polls_after_start += 1
  247. if self.daemon.state == "failing" and self.daemon.status_polls_after_start >= 1:
  248. self.daemon.state = "failed"
  249. self.daemon.summary = "bash -n rejected the script (simulated)"
  250. elif self.daemon.state == "working" and self.daemon.status_polls_after_start >= 2:
  251. if self.daemon.suspend_after_start:
  252. # `saddle suspend`: park until a fresh /start
  253. # resumes the workspace (one-shot flag).
  254. self.daemon.state = "suspended"
  255. self.daemon.suspend_after_start = False
  256. else:
  257. self.daemon.state = "done"
  258. self.daemon.summary = "fake daemon finished"
  259. self._send(200, response)
  260. elif self.path.startswith("/output"):
  261. with self.daemon.lock:
  262. entries = self.daemon.pending_output
  263. self.daemon.pending_output = []
  264. self._send(200, entries)
  265. elif self.path.startswith("/log"):
  266. query = urllib.parse.urlparse(self.path).query
  267. since = int(urllib.parse.parse_qs(query).get("since", ["0"])[0])
  268. # Simulate the live PTY: each poll produces one more chunk
  269. # (before slicing, so the poller actually sees it).
  270. self.daemon.append_chunk("stream-line-%d\r\n" % self.daemon.next_seq)
  271. chunks, nxt = self.daemon.log_since(since)
  272. self._send(200, {"chunks": chunks, "next": nxt})
  273. else:
  274. self._send(404, {"error": "no such route"})
  275. self.close_connection = True
  276. def serve(self):
  277. handler = type("BoundHandler", (FakeSaddle.Handler,), {"daemon": self})
  278. server = ThreadingHTTPServer(("127.0.0.1", DAEMON_PORT), handler)
  279. threading.Thread(target=server.serve_forever, daemon=True).start()
  280. return server
  281. def start_app():
  282. config = json.load(open(os.path.join(REPO, "web-config.json")))
  283. config["saddle_callback_base_url"] = WEB
  284. with open(os.path.join(WORK, "web-config.json"), "w") as handle:
  285. json.dump(config, handle)
  286. env = dict(os.environ)
  287. env.update({
  288. "SPRY_DB_PATH": DB_PATH,
  289. "SPRY_SEED_ADMIN_USERNAME": "admin",
  290. "SPRY_SEED_ADMIN_PASSWORD": "adminpass123",
  291. "SADDLE_LLM_BASE_URL": "http://127.0.0.1:%d" % DAEMON_PORT,
  292. "SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
  293. })
  294. log = open(os.path.join(WORK, "app.log"), "w")
  295. process = subprocess.Popen([APP, str(APP_PORT)], cwd=WORK, env=env,
  296. stdout=log, stderr=subprocess.STDOUT)
  297. outcome = wait_for(lambda: (app_ready() and "ready")
  298. or (process.poll() is not None and "app-exited-%d" % process.poll())
  299. or False, timeout=30, what="app ready")
  300. if outcome is None:
  301. raise SystemExit("app did not come up; see %s/app.log" % WORK)
  302. if str(outcome).startswith("app-exited-"):
  303. raise SystemExit("app exited during startup (%s); see %s/app.log"
  304. % outcome, WORK)
  305. return process
  306. def app_ready():
  307. try:
  308. status, _ = http("GET", "/login", timeout=3)
  309. return status == 200
  310. except (urllib.error.URLError, OSError):
  311. return False
  312. def main():
  313. if not os.path.exists(APP):
  314. raise SystemExit("app binary not found at %s (build first)" % APP)
  315. if os.path.exists(WORK):
  316. shutil.rmtree(WORK)
  317. os.makedirs(os.path.join(WORK, "workspace"))
  318. fake = FakeSaddle()
  319. fake.serve()
  320. app = start_app()
  321. try:
  322. auth = login("admin", "adminpass123")
  323. check(True, "login yields an auth slot")
  324. print("== feature 3: plan review hides duplicate body ==")
  325. state = slot_public(auth, "/projects", "projects")
  326. act(auth, state["create"], {
  327. "name": "demo",
  328. "git_url": "https://example.com/demo.git",
  329. "default_branch": "main",
  330. })
  331. conn = db()
  332. conn.execute("insert into tasks (project_id, title, status, priority,"
  333. " is_design_task, plan, created_at)"
  334. " values (1, 'Plan review task', 'working', 5, 1, '# The plan\n\n1. Do it',"
  335. " strftime('%s','now'))")
  336. conn.commit()
  337. plan_task = conn.execute("select max(id) from tasks").fetchone()[0]
  338. saddles_state = slot_public(auth, "/saddles", "saddles")
  339. act(auth, saddles_state["register"], {"name": "fake-saddle",
  340. "host": "127.0.0.1:%d" % DAEMON_PORT})
  341. registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  342. if s["name"] == "fake-saddle"]
  343. check(len(registered) == 1 and registered[0]["token"].startswith("saddle-"),
  344. "fake saddle registered with an API token")
  345. fake_id = int(registered[0]["id"])
  346. fake_token = registered[0]["token"]
  347. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
  348. " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
  349. (plan_task, fake_id))
  350. conn.commit()
  351. session_id = conn.execute("select max(id) from sessions").fetchone()[0]
  352. conn.close()
  353. status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  354. {"type": "block", "session_id": session_id, "data": {
  355. "question": "Plan submitted for review",
  356. "type": "rfc",
  357. "body": "# The plan\n\n1. Do it"}})
  358. check(status == 200 and body.get("block_id"), "plan-review block raised")
  359. status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  360. {"type": "block", "session_id": session_id, "data": {
  361. "question": "Which database?",
  362. "type": "question",
  363. "options": ["SQLite", "Postgres"],
  364. "body": "# Storage choice"}})
  365. check(status == 200, "question block raised")
  366. detail = task_state(auth, plan_task)
  367. blocks = {b["block_type"]: b for b in detail["blocks"]}
  368. check(blocks["rfc"].get("is_plan_review") is True,
  369. "rfc block flagged as the plan review")
  370. check(blocks["rfc"].get("show_body") is False,
  371. "pending plan review hides the duplicate body preview")
  372. check(blocks["question"].get("show_body") is True,
  373. "question block keeps its body preview")
  374. act(auth, blocks["rfc"]["answer_action"], {"answer": "Plan approved. Proceed."})
  375. detail = task_state(auth, plan_task)
  376. rfc = [b for b in detail["blocks"] if b["block_type"] == "rfc"][0]
  377. check(rfc.get("show_body") is True,
  378. "answered plan review shows the body again (review section gone)")
  379. print("== B1: highlight-to-comment quote dialog ==")
  380. status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
  381. check(status == 200 and "quote-comment-dialog" in page and "quote-text" in page,
  382. "task page carries the quote-comment dialog the popover opens")
  383. check('name="quote"' in page and 'name="return_to"' in page,
  384. "quote dialog posts the quote and return-to fields")
  385. status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
  386. check(status == 200 and "quote-comment-dialog" in page,
  387. "terminal page keeps its quote dialog")
  388. print("== B3: review section hides the duplicate body ==")
  389. status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  390. {"type": "block", "session_id": session_id, "data": {
  391. "question": "Review the diff",
  392. "type": "review",
  393. "body": "# The change\n\nReworks the flange module."}})
  394. check(status == 200 and body.get("block_id"), "review block raised")
  395. detail = task_state(auth, plan_task)
  396. review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
  397. check(detail.get("review_pending") is True,
  398. "review section shows while the review is unanswered")
  399. check(review_block.get("show_body") is False,
  400. "pending review block hides its duplicate body preview")
  401. check(review_block.get("body", "").startswith("# The change"),
  402. "review body itself still arrives in state (B2 counterpart)")
  403. act(auth, review_block["answer_action"], {"answer": "Human approved the review"})
  404. detail = task_state(auth, plan_task)
  405. review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
  406. check(review_block.get("show_body") is True,
  407. "answered review shows the body again (banner gone)")
  408. print("== B2: question block body above options ==")
  409. # Contract side (A2 will restore bodies end-to-end): a question
  410. # block raised with a heredoc body carries it; the body preview
  411. # rides above the options (template order) and survives answering
  412. # alongside the selected answer.
  413. detail = task_state(auth, plan_task)
  414. question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
  415. check(question.get("body") == "# Storage choice"
  416. and question.get("show_body") is True,
  417. "question block keeps its body markdown with the preview on")
  418. act(auth, question["answer_action"], {"option": "SQLite"})
  419. detail = task_state(auth, plan_task)
  420. question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
  421. check(question.get("answered") is True and question.get("answer") == "SQLite"
  422. and question.get("body") == "# Storage choice"
  423. and question.get("show_body") is True,
  424. "answered question shows body + selected answer")
  425. print("== B5: compare-changes SSH -> HTTPS ==")
  426. def diff_url_for_project(git_url, suffix):
  427. state = slot_public(auth, "/projects", "projects")
  428. act(auth, state["create"], {"name": "diff-%s" % suffix, "git_url": git_url})
  429. project_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  430. if p["name"] == "diff-%s" % suffix][0])
  431. conn = db()
  432. conn.execute("insert into tasks (project_id, title, status, task_branch,"
  433. " branch_from, created_at) values (?, ?, 'working', 'feature/x',"
  434. " 'main', strftime('%s','now'))", (project_id, "Diff %s" % suffix))
  435. conn.commit()
  436. task_id = conn.execute("select max(id) from tasks").fetchone()[0]
  437. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
  438. " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
  439. " 'done')", (task_id, fake_id))
  440. conn.commit()
  441. session = conn.execute("select max(id) from sessions").fetchone()[0]
  442. conn.close()
  443. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  444. {"type": "block", "session_id": session, "data": {
  445. "question": "Review the diff", "type": "review"}})
  446. return task_state(auth, task_id).get("review_diff_url") or ""
  447. check(diff_url_for_project("ssh://fabrica.unitatem.net:222/clanker/clanker-saddle.git",
  448. "fabrica")
  449. == "https://fabrica.unitatem.net/clanker/clanker-saddle/compare/main...feature/x",
  450. "ssh://git@host:port/owner/repo.git translates (port dropped)")
  451. check(diff_url_for_project("git@github.com:acme/widgets.git", "scp")
  452. == "https://github.com/acme/widgets/compare/main...feature/x",
  453. "git@host:owner/repo.git translates to the browse host")
  454. check(diff_url_for_project("https://example.com/plain/repo", "plain")
  455. == "https://example.com/plain/repo/compare/main...feature/x",
  456. "HTTPS URLs pass through unchanged")
  457. print("== feature 5: rules ==")
  458. rules_state = slot_public(auth, "/rules", "rules")
  459. for name, body in [
  460. ("Use Invercargill", "Never use libgee; use Invercargill enumerables."),
  461. ("Write Vala docs", "Every public class carries a doc comment."),
  462. ("Prefer meson", "Builds go through meson; no makefiles."),
  463. ("Co-dev rule", "Keep the wire contract exactly."),
  464. ("Reference rule", "Must never ride along."),
  465. ]:
  466. act(auth, rules_state["create"], {"name": name, "body": body})
  467. listed = {r["name"]: r for r in slot_public(auth, "/rules", "rules")["rules"]}
  468. check(len(listed) == 5, "five rules created and listed on /rules")
  469. def rule_id(name):
  470. rows = [r for r in slot_public(auth, "/rules", "rules")["rules"] if r["name"] == name]
  471. return int(rows[0]["id"])
  472. state = slot_public(auth, "/projects", "projects")
  473. act(auth, state["create"], {"name": "co", "git_url": "https://example.com/co.git"})
  474. act(auth, state["create"], {"name": "ref", "git_url": "https://example.com/ref.git"})
  475. co_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  476. if p["name"] == "co"][0])
  477. ref_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  478. if p["name"] == "ref"][0])
  479. project = slot_public(auth, "/projects/1", "project")
  480. act(auth, project["add_link"], {"linked_project_id": str(co_id),
  481. "relationship": "co-developed"})
  482. act(auth, project["add_link"], {"linked_project_id": str(ref_id),
  483. "relationship": "reference"})
  484. act(auth, project["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
  485. act(auth, project["attach_rule"], {"rule_id": str(rule_id("Prefer meson"))})
  486. project = slot_public(auth, "/projects/1", "project")
  487. check(sorted(r["name"] for r in project["rules"]) == ["Prefer meson", "Use Invercargill"],
  488. "project page lists its attached rules")
  489. co_project = slot_public(auth, "/projects/%d" % co_id, "project")
  490. act(auth, co_project["attach_rule"], {"rule_id": str(rule_id("Co-dev rule"))})
  491. ref_project = slot_public(auth, "/projects/%d" % ref_id, "project")
  492. act(auth, ref_project["attach_rule"], {"rule_id": str(rule_id("Reference rule"))})
  493. tasks_state = slot_public(auth, "/tasks", "tasks")
  494. act(auth, tasks_state["create"], {
  495. "project_id": "1",
  496. "title": "Rules dispatch",
  497. "description": "Effective rule set rides the /start payload.",
  498. "priority": "2",
  499. })
  500. rule_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  501. if t["title"] == "Rules dispatch"][0])
  502. detail = task_state(auth, rule_task)
  503. act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Write Vala docs"))})
  504. act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
  505. detail = task_state(auth, rule_task)
  506. check(sorted(r["name"] for r in detail["rules"]) == ["Use Invercargill", "Write Vala docs"],
  507. "task page lists its attached rules")
  508. check(wait_for(lambda: len(fake.start_payloads) > 0 or False,
  509. timeout=30, what="/start push") is not None,
  510. "dispatcher pushes the task to the fake saddle")
  511. payload = fake.start_payloads[-1]
  512. rule_names = [r["name"] for r in payload.get("rules", [])]
  513. check(rule_names == ["Write Vala docs", "Use Invercargill", "Prefer meson", "Co-dev rule"],
  514. "effective rules in /start: task, primary, co-developed order (%s)" % rule_names)
  515. check(all(r.get("body") for r in payload.get("rules", [])),
  516. "rules carry their bodies")
  517. check("Reference rule" not in rule_names,
  518. "reference project rules excluded")
  519. detail = task_state(auth, rule_task)
  520. act(auth, detail["rules"][0]["detach"], {})
  521. check([r["name"] for r in task_state(auth, rule_task)["rules"]] == ["Use Invercargill"],
  522. "detached task rule disappears")
  523. rules_rows = slot_public(auth, "/rules", "rules")["rules"]
  524. docs_row = [r for r in rules_rows if r["name"] == "Write Vala docs"][0]
  525. act(auth, docs_row["update"], {"name": "Write Vala docs", "body": "Doc comments or bust."})
  526. check(any(r["body"] == "Doc comments or bust."
  527. for r in slot_public(auth, "/rules", "rules")["rules"]),
  528. "rule edit saves the body")
  529. check(wait_task_done(auth, rule_task, timeout=60) is not None,
  530. "dispatched task completes against the fake daemon")
  531. print("== feature 6: project notes + callback ==")
  532. conn = db()
  533. callback_rows = conn.execute(
  534. "select token from api_tokens where saddle_id = 0").fetchall()
  535. conn.close()
  536. check(len(callback_rows) == 1 and callback_rows[0][0].startswith("saddle-callback-"),
  537. "dedicated saddle-callback token minted at startup")
  538. callback_token = callback_rows[0][0]
  539. status, _ = saddle_api("POST", "/api/saddle/note", None,
  540. {"project": "demo", "body": "x"})
  541. check(status == 401, "note endpoint rejects a missing token")
  542. status, _ = saddle_api("POST", "/api/saddle/note", "saddle-callback-not-real",
  543. {"project": "demo", "body": "x"})
  544. check(status == 401, "note endpoint rejects a wrong token")
  545. status, body = saddle_api("POST", "/api/saddle/note", callback_token,
  546. {"task_id": rule_task, "project": "demo",
  547. "body": "style.css tokens live in :root"})
  548. check(status == 200 and body.get("ok") is True, "note accepted with the callback token")
  549. status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
  550. {"project": "no-such-project", "body": "x"})
  551. check(status == 400, "unknown project rejected")
  552. status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
  553. {"project": "demo", "body": " "})
  554. check(status == 400, "empty note body rejected")
  555. project = slot_public(auth, "/projects/1", "project")
  556. check(any("style.css tokens" in n["body"] and int(n["source_task_id"]) == rule_task
  557. for n in project["notes"]),
  558. "project page lists the note with its source task")
  559. act(auth, [n for n in project["notes"] if "style.css tokens" in n["body"]][0]["delete"], {})
  560. check(all("style.css tokens" not in n["body"]
  561. for n in slot_public(auth, "/projects/1", "project")["notes"]),
  562. "note deleted from the project page")
  563. # /start payload: notes of primary + co-developed (not reference),
  564. # oldest first; callback carries the configured base URL + token.
  565. saddle_api("POST", "/api/saddle/note", callback_token,
  566. {"project": "demo", "body": "note-one"})
  567. saddle_api("POST", "/api/saddle/note", callback_token,
  568. {"project": "demo", "body": "note-two"})
  569. saddle_api("POST", "/api/saddle/note", callback_token,
  570. {"project": "co", "body": "co-note"})
  571. conn = db()
  572. conn.execute("insert into project_notes (project_id, body, created_at)"
  573. " values (?, 'ref-note-must-not-ship', strftime('%s','now'))", (ref_id,))
  574. conn.commit()
  575. conn.close()
  576. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  577. "project_id": "1",
  578. "title": "Notes dispatch",
  579. "description": "Project notes ride the /start payload.",
  580. "priority": "2",
  581. })
  582. check(wait_for(lambda: len(fake.start_payloads) >= 2 or False,
  583. timeout=30, what="second /start push") is not None,
  584. "dispatcher pushes the notes task")
  585. payload = fake.start_payloads[-1]
  586. notes = payload.get("project_notes", [])
  587. check([n["body"] for n in notes] == ["note-one", "note-two", "co-note"],
  588. "project_notes: primary + co-developed, oldest first (%s)" % [n["body"] for n in notes])
  589. check(all(n.get("project") for n in notes)
  590. and notes[2]["project"] == "co",
  591. "project_notes carry the project name")
  592. check("ref-note-must-not-ship" not in [n["body"] for n in notes],
  593. "reference project notes excluded")
  594. callback = payload.get("callback", {})
  595. check(callback.get("base_url") == WEB and callback.get("token") == callback_token,
  596. "callback carries the configured base URL and the minted token")
  597. # entry cap: 30 of 33 candidates
  598. conn = db()
  599. for i in range(33):
  600. conn.execute("insert into project_notes (project_id, body, created_at)"
  601. " values (?, ?, strftime('%s','now'))", (1, "bulk-%02d" % i))
  602. conn.commit()
  603. conn.close()
  604. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  605. "project_id": "1",
  606. "title": "Entry cap dispatch",
  607. "priority": "2",
  608. })
  609. check(wait_for(lambda: len(fake.start_payloads) >= 3 or False,
  610. timeout=30, what="third /start push") is not None,
  611. "dispatcher pushes the entry-cap task")
  612. notes = fake.start_payloads[-1].get("project_notes", [])
  613. check(len(notes) == 30 and notes[0]["body"] == "note-one" and notes[1]["body"] == "note-two",
  614. "project_notes capped at 30 entries, oldest kept (%d)" % len(notes))
  615. # char cap: the giant note is truncated so the total stays <= 20000
  616. conn = db()
  617. conn.execute("delete from project_notes where body like 'bulk-%' or body = 'co-note'")
  618. conn.execute("insert into project_notes (project_id, body, created_at)"
  619. " values (?, ?, strftime('%s','now'))", (1, "giant!" + "x" * 25000,))
  620. conn.commit()
  621. conn.close()
  622. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  623. "project_id": "1",
  624. "title": "Char cap dispatch",
  625. "priority": "2",
  626. })
  627. check(wait_for(lambda: len(fake.start_payloads) >= 4 or False,
  628. timeout=30, what="fourth /start push") is not None,
  629. "dispatcher pushes the char-cap task")
  630. notes = fake.start_payloads[-1].get("project_notes", [])
  631. total = sum(len(n["body"]) for n in notes)
  632. check(len(notes) == 3 and total == 20000 and notes[2]["body"].startswith("giant!"),
  633. "project_notes char cap: head kept, giant truncated (entries=%d, chars=%d)"
  634. % (len(notes), total))
  635. print("== raw log polling + terminal raw view ==")
  636. # Wait for the last dispatch to finish so its final polls ran.
  637. check(wait_task_done(auth, task_id_of("Char cap dispatch"), timeout=60) is not None,
  638. "char-cap task completes")
  639. conn = db()
  640. raw_session = conn.execute(
  641. "select s.id from sessions s join tasks t on t.id = s.task_id"
  642. " where t.title = 'Char cap dispatch' order by s.id desc limit 1").fetchone()[0]
  643. rows = conn.execute("select seq, chunk from session_raw_log where session_id = ?"
  644. " order by seq", (raw_session,)).fetchall()
  645. conn.close()
  646. check(len(rows) >= 3, "session_raw_log captured the daemon's chunks (%d)" % len(rows))
  647. seqs = [r[0] for r in rows]
  648. check(seqs == sorted(seqs) and len(set(seqs)) == len(seqs),
  649. "raw chunks stored once, seq strictly increasing")
  650. text = "".join(r[1] for r in rows)
  651. check(fake.PROMPT in text and "echo hello" in text and "hello" in text,
  652. "raw log keeps the prompt and echoed output byte-exact")
  653. check("stream-line-" in text, "incremental /log polls append later chunks")
  654. terminal = slot_public(auth, "/tasks/%d/terminal" % task_id_of("Char cap dispatch"),
  655. "terminal")
  656. check(terminal.get("has_raw_log") is True and int(terminal["session_id"]) == raw_session,
  657. "terminal view selects raw rendering for the raw session")
  658. # Old sessions (typed entries, no raw rows) keep the fallback view.
  659. conn = db()
  660. conn.execute("insert into session_commands (session_id, command_text, output_text,"
  661. " type, started_at) values (?, 'ls', 'old-output', 'command',"
  662. " strftime('%s','now'))", (session_id,))
  663. conn.commit()
  664. conn.close()
  665. legacy = slot_public(auth, "/tasks/%d/terminal" % plan_task, "terminal")
  666. check(legacy.get("has_raw_log") is False,
  667. "legacy session falls back to the entry-type terminal view")
  668. request = urllib.request.Request(WEB + "/terminal-stream/%d" % raw_session)
  669. with urllib.request.urlopen(request, timeout=15) as stream:
  670. raw_events = []
  671. deadline = time.monotonic() + 8
  672. while time.monotonic() < deadline and len(raw_events) < 3:
  673. line = stream.readline().decode().strip()
  674. if not line.startswith("data: ") or line == "data: ping":
  675. continue
  676. event = json.loads(line[6:])
  677. if event.get("type") == "raw":
  678. raw_events.append(event)
  679. check(len(raw_events) >= 3 and raw_events[0]["text"] == fake.PROMPT
  680. and raw_events[0]["seq"] < raw_events[1]["seq"],
  681. "terminal stream serves raw chunks in seq order (prompt first)")
  682. print("== capabilities: tag-based routing ==")
  683. saddles_state = slot_public(auth, "/saddles", "saddles")
  684. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  685. act(auth, saddles_state["create_capability"], {"name": "bigmem"})
  686. saddles_state = slot_public(auth, "/saddles", "saddles")
  687. caps = {c["name"]: c for c in saddles_state["capabilities"]}
  688. check(len(caps) == 2 and all(int(c["saddle_count"]) == 0 and int(c["task_count"]) == 0
  689. for c in caps.values()),
  690. "capabilities created and listed with zero usage")
  691. def saddle_card(name):
  692. rows = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  693. if s["name"] == name]
  694. return rows[0]
  695. fake_card = saddle_card("fake-saddle")
  696. check(fake_card["capabilities"] == [],
  697. "saddle card starts with no capability chips")
  698. # A task requiring 'gpu' waits while the only idle saddle lacks it;
  699. # a later task without requirements still dispatches (skip-and-continue).
  700. tasks_state = slot_public(auth, "/tasks", "tasks")
  701. act(auth, tasks_state["create"], {
  702. "project_id": "1",
  703. "title": "GPU task",
  704. "priority": "2",
  705. })
  706. gpu_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  707. if t["title"] == "GPU task"][0])
  708. detail = task_state(auth, gpu_task)
  709. act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
  710. detail = task_state(auth, gpu_task)
  711. check([c["name"] for c in detail["capabilities"]] == ["gpu"],
  712. "task page lists the required capability")
  713. # The only idle saddle lacks the tag, so the queue flags the wait
  714. # (read before any other task claims the idle saddle).
  715. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  716. if int(t["id"]) == gpu_task][0]
  717. check(queue["status"] == "queued" and queue["capabilities"] == ["gpu"]
  718. and queue.get("waiting_for_saddle") is True,
  719. "queue row shows the waiting task with its tag and wait flag")
  720. act(auth, tasks_state["create"], {
  721. "project_id": "1",
  722. "title": "Any-saddle task",
  723. "priority": "3",
  724. })
  725. any_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  726. if t["title"] == "Any-saddle task"][0])
  727. check(wait_for(lambda: task_state(auth, any_task).get("status") != "queued" or False,
  728. timeout=30, what="uncapable task dispatches") is not None,
  729. "task without requirements dispatches past the waiting task")
  730. with open(os.path.join(WORK, "app.log")) as handle:
  731. app_log = handle.read()
  732. check("waits for a capable saddle" in app_log and "requires: gpu" in app_log,
  733. "dispatcher logs the capability wait")
  734. # Attaching the tag to the saddle releases the task.
  735. fake_card = saddle_card("fake-saddle")
  736. act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
  737. fake_card = saddle_card("fake-saddle")
  738. check([c["name"] for c in fake_card["capabilities"]] == ["gpu"],
  739. "saddle card shows the attached chip")
  740. check(wait_task_done(auth, gpu_task, timeout=60) is not None,
  741. "task dispatches once the saddle carries the required tag")
  742. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  743. if int(t["id"]) == gpu_task][0]
  744. check(queue.get("waiting_for_saddle") is not True,
  745. "wait flag clears once dispatched")
  746. # Multi-tag: gpu + bigmem required, only gpu carried → waits; adding
  747. # bigmem completes the superset.
  748. act(auth, tasks_state["create"], {
  749. "project_id": "1",
  750. "title": "Big task",
  751. "priority": "2",
  752. })
  753. big_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  754. if t["title"] == "Big task"][0])
  755. detail = task_state(auth, big_task)
  756. act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
  757. act(auth, detail["attach_capability"], {"capability_id": str(int(caps["bigmem"]["id"]))})
  758. check(wait_for(lambda: ([t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  759. if int(t["id"]) == big_task][0].get("waiting_for_saddle")
  760. is True) or False, timeout=30, what="big task waits") is not None,
  761. "task waits when the saddle's tags are not a superset")
  762. fake_card = saddle_card("fake-saddle")
  763. act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["bigmem"]["id"]))})
  764. check(wait_task_done(auth, big_task, timeout=60) is not None,
  765. "task dispatches once the superset is complete")
  766. # Detach a requirement; delete a capability everywhere.
  767. detail = task_state(auth, big_task)
  768. act(auth, [c for c in detail["capabilities"] if c["name"] == "bigmem"][0]["detach"], {})
  769. check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
  770. "detached requirement disappears from the task")
  771. bigmem_row = [c for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  772. if c["name"] == "bigmem"][0]
  773. act(auth, bigmem_row["delete"], {})
  774. saddles_state = slot_public(auth, "/saddles", "saddles")
  775. check(all(c["name"] != "bigmem" for c in saddles_state["capabilities"]),
  776. "deleted capability gone from the catalogue")
  777. check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
  778. "deleting a capability drops its task requirements")
  779. check(all(c["name"] != "bigmem"
  780. for c in saddle_card("fake-saddle")["capabilities"]),
  781. "deleting a capability drops it from saddles")
  782. print("== B4: required capabilities on the create form ==")
  783. tasks_state = slot_public(auth, "/tasks", "tasks")
  784. gpu_option = [o for o in tasks_state.get("capability_options", [])
  785. if o["name"] == "gpu"][0]
  786. check(len(tasks_state.get("capability_options", [])) >= 1,
  787. "queue page offers capability options in the create form")
  788. conn = db()
  789. before = conn.execute("select count(*) from tasks").fetchone()[0]
  790. conn.close()
  791. act(auth, tasks_state["create"], {
  792. "project_id": "1",
  793. "title": "Bad cap task",
  794. "capability_id": [gpu_option["id"], "999"],
  795. })
  796. conn = db()
  797. after = conn.execute("select count(*) from tasks").fetchone()[0]
  798. conn.close()
  799. check(after == before, "unknown capability id rejects the create")
  800. act(auth, tasks_state["create"], {
  801. "project_id": "1",
  802. "title": "Cap-on-create task",
  803. "capability_id": [gpu_option["id"]],
  804. })
  805. cap_create = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  806. if t["title"] == "Cap-on-create task"][0])
  807. detail = task_state(auth, cap_create)
  808. check([c["name"] for c in detail["capabilities"]] == ["gpu"],
  809. "capabilities from the create form land on the task")
  810. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  811. if int(t["id"]) == cap_create][0]
  812. check(queue["capabilities"] == ["gpu"],
  813. "queue row shows the created task's requirements")
  814. # The waiting flag must hold it back until the capable saddle frees
  815. # up (the fake saddle still carries gpu from the routing checks).
  816. check(queue.get("waiting_for_saddle") is not True,
  817. "capable saddle available: no wait flag on the new task")
  818. print("== B6: computed display state ==")
  819. conn = db()
  820. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  821. " values (1, 'Parent', 'done', 5, strftime('%s','now'))")
  822. parent6 = conn.execute("select max(id) from tasks").fetchone()[0]
  823. # The post-mortem case: stored done WITH an unanswered review block.
  824. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
  825. " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
  826. (parent6, fake_id))
  827. session6 = conn.execute("select max(id) from sessions").fetchone()[0]
  828. conn.commit()
  829. conn.close()
  830. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  831. {"type": "block", "session_id": session6, "data": {
  832. "question": "Review the final diff", "type": "review"}})
  833. detail = task_state(auth, parent6)
  834. check(detail.get("display") == "review requested" and detail.get("terminal") is True,
  835. "unanswered review overrides the stored done status (terminal flag keeps stored truth)")
  836. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  837. if int(t["id"]) == parent6][0]
  838. check(queue["display"] == "review requested",
  839. "queue row shows the computed review-requested display")
  840. # Delegation: with the review answered, open subtasks delegate.
  841. conn = db()
  842. conn.execute("update task_blocks set answered_at = strftime('%s','now'),"
  843. " answer = 'Human approved the review' where task_id = ?", (parent6,))
  844. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  845. " created_at) values (1, 'Sub A', 'working', 5, ?, strftime('%s','now'))",
  846. (parent6,))
  847. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  848. " created_at) values (1, 'Sub B', 'done', 5, ?, strftime('%s','now'))",
  849. (parent6,))
  850. conn.commit()
  851. conn.close()
  852. detail = task_state(auth, parent6)
  853. check(detail.get("display") == "delegating 1/2" and detail.get("subtask_open") == 1,
  854. "open subtasks show the delegating breakdown (%s)" % detail.get("display"))
  855. # Flags track the stored status once the display falls through
  856. # (subtasks terminal first — delegating leads while any are open).
  857. conn = db()
  858. conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
  859. " where id = ?", (parent6,))
  860. conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
  861. " where title = 'Sub A'")
  862. conn.commit()
  863. conn.close()
  864. detail = task_state(auth, parent6)
  865. check(detail.get("display") == "failed" and detail.get("failed") is True
  866. and detail.get("terminal") is True,
  867. "failed task with answered review and terminal subtasks shows failed + terminal flags")
  868. print("== retry + resume ==")
  869. # A daemon-side failure fails the task with its explanation.
  870. fake.fail_next = True
  871. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  872. "project_id": "1",
  873. "title": "Flaky task",
  874. "priority": "2",
  875. })
  876. flaky = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  877. if t["title"] == "Flaky task"][0])
  878. check(wait_for(lambda: task_state(auth, flaky).get("status") == "failed" or False,
  879. timeout=60, what="flaky task fails") is not None,
  880. "daemon failure fails the task")
  881. detail = task_state(auth, flaky)
  882. check("bash -n rejected" in (detail.get("summary") or ""),
  883. "failure summary shows the daemon's explanation")
  884. # Retry: summary moves into a comment and clears, task requeues,
  885. # history stays, and the re-dispatch restores the workspace.
  886. comments_before = len(detail.get("comments", []))
  887. progress_before = len(detail.get("progress", []))
  888. act(auth, detail["retry"], {})
  889. detail = task_state(auth, flaky)
  890. check(detail["status"] == "queued" and (detail.get("summary") or "") == "",
  891. "retry requeues the task and clears the stale summary")
  892. check(any("bash -n rejected" in c["body"] for c in detail["comments"]),
  893. "failure explanation kept as a bot comment")
  894. check(any(c["body"] == "Task retried by user" for c in detail["comments"]),
  895. "retry lands its audit comment")
  896. check(len(detail["comments"]) >= comments_before + 1,
  897. "retry keeps the comment history")
  898. check(wait_task_done(auth, flaky, timeout=60) is not None,
  899. "retried task dispatches again and completes")
  900. detail = task_state(auth, flaky)
  901. check(any("restored the session context" in p["summary"]
  902. for p in detail.get("progress", [])),
  903. "daemon resume reported on the first status after the restore")
  904. check(len(detail["progress"]) > progress_before,
  905. "resume note appended to the progress timeline")
  906. # Manual resume: a task stuck working on an offline saddle.
  907. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  908. "project_id": "1",
  909. "title": "Stuck task",
  910. "priority": "2",
  911. })
  912. stuck = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  913. if t["title"] == "Stuck task"][0])
  914. check(wait_for(lambda: task_state(auth, stuck).get("status") == "working" or False,
  915. timeout=60, what="stuck task starts working") is not None,
  916. "stuck task reaches working")
  917. conn = db()
  918. conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
  919. conn.commit()
  920. conn.close()
  921. detail = task_state(auth, stuck)
  922. check(detail.get("resume_available") is True,
  923. "task page offers resume while its saddle is offline")
  924. act(auth, detail["resume"], {})
  925. detail = task_state(auth, stuck)
  926. check(detail["status"] == "queued",
  927. "manual resume requeues the stuck task")
  928. check(any(c["body"] == "Saddle became unreachable; task requeued"
  929. for c in detail["comments"]),
  930. "manual resume lands the interruption comment")
  931. check(all(s["status"] == "offline"
  932. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  933. "saddle stays offline after the manual requeue")
  934. # Bringing the saddle back lets the requeued task resume and finish.
  935. fake.state = "idle"
  936. fake_card = saddle_card("fake-saddle")
  937. act(auth, fake_card["refresh"], {})
  938. check(any(s["status"] == "idle"
  939. for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  940. if s["name"] == "fake-saddle"),
  941. "refreshed saddle returns idle")
  942. check(wait_task_done(auth, stuck, timeout=60) is not None,
  943. "requeued task resumes on the refreshed saddle and completes")
  944. print("== rejected entries (non-fatal) ==")
  945. # Ingest API: a rejected entry stores on the session and streams
  946. # through SSE; the task is untouched.
  947. status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  948. {"type": "output", "session_id": session_id, "data": {
  949. "command": "bash -n script.sh",
  950. "output": "syntax error near line 3 (recoverable)",
  951. "exit_code": 1,
  952. "type": "rejected"}})
  953. check(status == 200, "rejected entry accepted by the ingest API")
  954. request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
  955. with urllib.request.urlopen(request, timeout=15) as stream:
  956. rejected_event = None
  957. deadline = time.monotonic() + 8
  958. while time.monotonic() < deadline and rejected_event is None:
  959. line = stream.readline().decode().strip()
  960. if not line.startswith("data: ") or line == "data: ping":
  961. continue
  962. event = json.loads(line[6:])
  963. if event.get("type") == "rejected":
  964. rejected_event = event
  965. check(rejected_event is not None
  966. and "syntax error near line 3" in (rejected_event.get("output") or ""),
  967. "terminal stream carries rejected entries with their reason")
  968. # The page template renders them with the distinct non-fatal style.
  969. status, page = http("GET", "/tasks/%d/terminal" % stuck, timeout=15)
  970. check(status == 200 and "term-rejected" in page,
  971. "terminal page renders rejected entries in their own style")
  972. print("== B8: ANSI SGR in the terminal view ==")
  973. # Byte-exact SGR delivery: the daemon's coloured chunks flow through
  974. # the store + SSE untouched (the client renderer styles them).
  975. coloured = "\x1b[1;32mbold green\x1b[0m then \x1b[38;5;196m256-red\x1b[0m\n"
  976. conn = db()
  977. conn.execute("insert into session_raw_log (session_id, seq, chunk)"
  978. " values (?, 900, ?)", (session_id, coloured))
  979. conn.commit()
  980. conn.close()
  981. request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
  982. with urllib.request.urlopen(request, timeout=15) as stream:
  983. coloured_event = None
  984. deadline = time.monotonic() + 8
  985. while time.monotonic() < deadline and coloured_event is None:
  986. line = stream.readline().decode().strip()
  987. if not line.startswith("data: ") or line == "data: ping":
  988. continue
  989. event = json.loads(line[6:])
  990. if event.get("type") == "raw" and "\x1b[1;32m" in (event.get("text") or ""):
  991. coloured_event = event
  992. check(coloured_event is not None
  993. and "\x1b[38;5;196m" in coloured_event["text"],
  994. "SGR sequences stream byte-exact to the client renderer")
  995. status, page = http("GET", "/tasks/%d/terminal" % stuck, timeout=15)
  996. check(status == 200 and "makeAnsiRenderer" in page and "colour256" in page,
  997. "terminal page embeds the ANSI renderer (SGR + 256-colour palette)")
  998. # Dispatch path: a rejected entry from /output flows into the audit
  999. # trail without failing (or finishing) the task.
  1000. fake.queue_output("rejected", "echo 'too long' | saddle",
  1001. "max_tokens exceeded (recoverable)")
  1002. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1003. "project_id": "1",
  1004. "title": "Rejected ride",
  1005. "priority": "2",
  1006. })
  1007. rejected_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1008. if t["title"] == "Rejected ride"][0])
  1009. check(wait_task_done(auth, rejected_task, timeout=60) is not None,
  1010. "task completes normally past rejected entries")
  1011. conn = db()
  1012. stored = conn.execute(
  1013. "select count(*) from session_commands sc join sessions s on s.id = sc.session_id"
  1014. " where s.task_id = ? and sc.type = 'rejected'", (rejected_task,)).fetchone()[0]
  1015. conn.close()
  1016. check(stored >= 1,
  1017. "rejected entries stored in the audit trail (%d rows)" % stored)
  1018. check(task_state(auth, rejected_task)["status"] == "done"
  1019. and "fake daemon finished" in (task_state(auth, rejected_task).get("summary") or ""),
  1020. "rejected entries never fail the task")
  1021. print("== B7: suspended state + per-subtask resume ==")
  1022. fake.suspend_after_start = True
  1023. pushes_before = len(fake.start_payloads)
  1024. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1025. "project_id": "1",
  1026. "title": "Suspend parent",
  1027. "priority": "2",
  1028. })
  1029. parent7 = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1030. if t["title"] == "Suspend parent"][0])
  1031. check(wait_for(lambda: task_state(auth, parent7).get("status") == "suspended"
  1032. or False, timeout=60, what="parent parks as suspended") is not None,
  1033. "daemon 'suspended' state parks the task")
  1034. detail = task_state(auth, parent7)
  1035. check(detail.get("display") == "waiting on subtasks",
  1036. "suspended task displays as waiting on subtasks")
  1037. check(any(s["status"] == "idle"
  1038. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  1039. "suspension frees the saddle")
  1040. first_push = fake.start_payloads[pushes_before]
  1041. check(first_push.get("subtasks") == [],
  1042. "the /start payload carries the subtasks array (empty here)")
  1043. # Two subtask events land while parked (staged, not dispatched):
  1044. # both must coalesce into one resume with a full report.
  1045. conn = db()
  1046. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  1047. " summary, completed_at, created_at) values (1, 'Child one', 'done', 5, ?,"
  1048. " 'built and verified the flange', strftime('%s','now','+1 second'),"
  1049. " strftime('%s','now'))", (parent7,))
  1050. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  1051. " summary, completed_at, created_at) values (1, 'Child two', 'failed', 5, ?,"
  1052. " 'the harness refused to start', strftime('%s','now','+1 second'),"
  1053. " strftime('%s','now'))", (parent7,))
  1054. conn.commit()
  1055. conn.close()
  1056. def parent_push_count():
  1057. return len([p for p in fake.start_payloads
  1058. if int(p.get("task", {}).get("id") or 0) == parent7])
  1059. check(wait_for(lambda: parent_push_count() >= 2 or False,
  1060. timeout=45, what="suspended parent resumes") is not None,
  1061. "subtask events resume the suspended parent")
  1062. parent_pushes = [p for p in fake.start_payloads
  1063. if int(p.get("task", {}).get("id") or 0) == parent7]
  1064. check(len(parent_pushes) >= 2,
  1065. "the parent's resume is a fresh /start for the same task")
  1066. resume_push = parent_pushes[-1]
  1067. notice = resume_push.get("resume_notice") or ""
  1068. check("Child one" in notice and "completed" in notice
  1069. and "built and verified the flange" in notice,
  1070. "resume notice reports the finished subtask with its summary")
  1071. check("Child two" in notice and "failed" in notice
  1072. and "the harness refused to start" in notice,
  1073. "coalesced: the second event rides the same resume notice")
  1074. subtask_entries = {(s["title"], s["status"]) for s in resume_push.get("subtasks", [])}
  1075. check(("Child one", "done") in subtask_entries
  1076. and ("Child two", "failed") in subtask_entries,
  1077. "subtasks array carries the children's states at dispatch time")
  1078. check(wait_task_done(auth, parent7, timeout=60) is not None,
  1079. "resumed parent completes")
  1080. detail = task_state(auth, parent7)
  1081. check(any("restored the session context" in p["summary"]
  1082. for p in detail.get("progress", [])),
  1083. "resumed flag reported on the first status after the restore")
  1084. finally:
  1085. app.terminate()
  1086. try:
  1087. app.wait(timeout=5)
  1088. except subprocess.TimeoutExpired:
  1089. app.kill()
  1090. print("%d passed, %d failed" % (passes, failures))
  1091. return 1 if failures else 0
  1092. if __name__ == "__main__":
  1093. sys.exit(main())