web_stack_test.py 186 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507
  1. #!/usr/bin/env python3
  2. """Self-contained web-stack feature tests: app + fake saddle daemon.
  3. Unlike integration_test.py (which needs a real saddle daemon and a mock
  4. LLM), this suite spawns everything it needs:
  5. - the clanker-saddle app binary (path from argv[1], or builddir default)
  6. in a scratch directory with its own web-config.json (statum keys copied
  7. from the repo, plus `saddle_callback_base_url`) and SQLite database
  8. - in-process fake saddle daemons speaking the wire contract: POST
  9. /register (plan 10: receives {saddle_token, web_base_url}, answers
  10. {ok, ssh_public_key, version}), POST /wake (records the wake; the
  11. daemon then asks POST /api/saddle/next-task), POST /task-update
  12. (cancel/comment/subtask-status), GET /status (scripted reflection —
  13. exceptional use only), and the phase-2 push channels home:
  14. POST /api/saddle/chunks (raw PTY batches) and POST /api/saddle/results
  15. (round entries + state transitions). The retired /start, /output and
  16. /log routes 404 like a post-phase-2 daemon.
  17. It then drives the app the way statum.js does (entrypoint loads + action
  18. POSTs) and asserts the Workstream B features against the wire contract:
  19. 3. plan-review blocks hide their duplicate body preview
  20. 5. rules CRUD/attach + the effective `rules` array in assignments
  21. 6. project notes, POST /api/saddle/note, `project_notes` + `callback`
  22. in assignments
  23. plus the raw-log side: pushed chunks -> session_raw_log + terminal
  24. raw view
  25. Plan 10 phase 1 (work acquisition flip):
  26. - per-saddle tokens: registration mints + /register-delivers them,
  27. legacy tokens migrate onto the row at startup, removal revokes
  28. - token auth on every /api/saddle/* endpoint
  29. - next-task allocation: concurrent asks race exactly one winner,
  30. affinity pinning for parked tasks, resume_notice on suspended resumes
  31. - dispatcher: wake -> ask -> allocate roundtrip, the lost-wake guard
  32. (no ask after a wake -> offline, re-wake the next capable), and the
  33. pre-flip /answer + /start resume paths kept as the rollback route
  34. Plan 10 phase 2 (telemetry flip):
  35. - pushed chunks: idempotent on (session_id, seq), ordered, landing in
  36. session_raw_log so the SSE terminal keeps working
  37. - pushed results: entries store like the /output path did (redelivery
  38. idempotent on the daemon entry ids), blocked raises the block row +
  39. parks + bumps the pulse (sealed refresh shows it), done/failed
  40. complete/fail the task, suspended parks for the affinity resume
  41. - answer resumes, both daemon shapes: a SLEEPING affine saddle (its
  42. post-park ask returned 204) is woken, its ask allocating the resume
  43. with the verdict riding the assignment; a daemon parked MID-ROUND
  44. (its round holds the block's answer wait open, so it can never ask)
  45. gets the verdict delivered into that wait via POST /answer — same
  46. session, no wake. A daemon dying with a parked-with-answered task
  47. never requeues it: the verdict and session survive for the resume
  48. - staleness: a silent runner gets ONE exceptional GET /status probe —
  49. alive keeps it running, dead requeues the task with a bot comment and
  50. marks the saddle offline; a recovered daemon's ask flips offline rows
  51. back to idle
  52. - the deleted layers are gone: /output, /log and /start 404 on the
  53. daemon side, and nothing on the web side calls them
  54. Plan 10 phase 3 (capability awareness + UI):
  55. - assignment capability context: {saddle_tags, fleet_tags} on every
  56. payload, GET /api/saddle/capabilities (per-saddle token) feeding
  57. the daemon's `saddle capabilities` CLI
  58. - subtask callback capabilities: explicit-only attachment validated
  59. against the operator tag set, unknown tags rejected with the valid
  60. list in the error
  61. - task page: status block pinned to the top, blocking sections
  62. newest-first (sealed GET state)
  63. - home dashboard: pinned OPEN blocks as newest-first link-through
  64. cards (no inline answer forms) above the cross-task activity
  65. timeline (last ~50 events newest-first)
  66. - lost-wake-guard regression: a saddle that asks SLOWLY after its
  67. wake (busy winding down) still claims the task — the wake is never
  68. silently dropped, no re-wake storm fires, and the never-asks guard
  69. path (offline + re-wake of the next capable) stays green
  70. Prints PASS/FAIL lines Statum-style; exits non-zero on any failure.
  71. """
  72. import json
  73. import os
  74. import shutil
  75. import socket
  76. import sqlite3
  77. import subprocess
  78. import sys
  79. import threading
  80. import time
  81. import urllib.error
  82. import urllib.parse
  83. import urllib.request
  84. from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
  85. REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  86. APP = os.path.abspath(sys.argv[1]) if len(sys.argv) > 1 else os.path.join(REPO, "builddir", "clanker-saddle")
  87. WORK = os.environ.get("WEBSTACK_WORK", "/tmp/kilo/webstack-e2e")
  88. DB_PATH = os.path.join(WORK, "web.sqlite")
  89. passes = 0
  90. failures = 0
  91. def check(condition, label):
  92. global passes, failures
  93. if condition:
  94. passes += 1
  95. print("PASS %s" % label)
  96. else:
  97. failures += 1
  98. print("FAIL %s" % label)
  99. def free_port():
  100. with socket.socket() as sock:
  101. sock.bind(("127.0.0.1", 0))
  102. return sock.getsockname()[1]
  103. APP_PORT = free_port()
  104. DAEMON_PORT = free_port()
  105. WEB = "http://127.0.0.1:%d" % APP_PORT
  106. def http(method, url, data=None, headers=None, timeout=30):
  107. body = None
  108. if data is not None:
  109. pairs = []
  110. for k, v in data.items():
  111. for item in (v if isinstance(v, (list, tuple)) else [v]):
  112. pairs.append((k, item))
  113. body = "&".join("%s=%s" % (k, urllib.parse.quote(str(item))) for k, item in pairs).encode()
  114. request = urllib.request.Request(WEB + url if url.startswith("/") else url,
  115. data=body, method=method)
  116. if data is not None:
  117. request.add_header("Content-Type", "application/x-www-form-urlencoded")
  118. for key, value in (headers or {}).items():
  119. request.add_header(key, value)
  120. # Retry dropped/refused connections: the freshly started app can
  121. # close a very early keep-alive connection once while warming up,
  122. # and its first seconds can wobble under load.
  123. for attempt in range(3):
  124. try:
  125. with urllib.request.urlopen(request, timeout=timeout) as response:
  126. return response.status, response.read().decode()
  127. except urllib.error.HTTPError as error:
  128. return error.code, error.read().decode()
  129. except (urllib.error.URLError, OSError):
  130. if attempt >= 2:
  131. raise
  132. time.sleep(0.75)
  133. def directives(uri, auth=None):
  134. headers = {}
  135. if auth:
  136. headers["X-Statum-Slot"] = "key=%s; as_at=%s" % (auth["key"], auth["as_at"])
  137. status, body = http("GET", "/_statum/entrypoint?uri=" + urllib.parse.quote(uri.split("?")[0]),
  138. headers=headers)
  139. if status != 200:
  140. raise SystemExit("entrypoint %s -> HTTP %d: %s" % (uri, status, body[:300]))
  141. return json.loads(body)
  142. def slot_public(auth, uri, type_name):
  143. for directive in directives(uri, auth):
  144. if directive["type"] == "set":
  145. content = directive["snapshot_frame"].get("content")
  146. if not content:
  147. continue
  148. content = json.loads(content)
  149. if content["slot"]["type"] == type_name:
  150. return content["public"]
  151. raise SystemExit("no %s slot in entrypoint state for %s" % (type_name, uri))
  152. def act(auth, action, fields):
  153. headers = {"X-Statum-Slot": "key=%s; as_at=%s" % (auth["key"], auth["as_at"])}
  154. if action.get("private"):
  155. headers["X-Statum-Private"] = action["private"]
  156. status, body = http(action.get("method", "POST"), action["uri"], fields, headers)
  157. if status != 200:
  158. raise SystemExit("action %s -> HTTP %d: %s" % (action["uri"], status, body[:300]))
  159. return json.loads(body)
  160. def login(username, password):
  161. frames = directives("/login")
  162. content = json.loads(frames[0]["snapshot_frame"]["content"])
  163. action = content["public"]["login"]
  164. status, body = http(action.get("method", "POST"), action["uri"],
  165. {"username": username, "password": password})
  166. if status != 200:
  167. raise SystemExit("login -> HTTP %d: %s" % (status, body[:300]))
  168. for directive in json.loads(body):
  169. if directive["type"] == "set":
  170. frame = json.loads(directive["snapshot_frame"]["content"])
  171. if frame["slot"]["type"] == "auth":
  172. return {"key": frame["slot"]["key"], "as_at": frame["as_at"]}
  173. raise SystemExit("login response carried no auth slot: " + body[:300])
  174. def wait_for(predicate, timeout=60, what="condition", interval=0.5):
  175. deadline = time.monotonic() + timeout
  176. last = None
  177. while time.monotonic() < deadline:
  178. last = predicate()
  179. if last is not False and last is not None:
  180. return last
  181. time.sleep(interval)
  182. print(" (timeout waiting for %s; last: %s)" % (what, last))
  183. dump_threads()
  184. return None
  185. def dump_threads():
  186. """Stacks of suite threads not parked in sleep — pinpoints a wedged
  187. fake lifecycle when a wait times out."""
  188. import traceback
  189. shown = 0
  190. for thread_id, frame in sys._current_frames().items():
  191. stack = "".join(traceback.format_stack(frame)[-4:])
  192. if ("web_stack_test.py" in stack) and ("sleep" not in stack) and shown < 6:
  193. shown += 1
  194. print(" --- thread %s ---\n%s" % (thread_id, stack))
  195. def task_state(auth, task_id):
  196. return slot_public(auth, "/tasks/%d" % task_id, "task")
  197. def wait_task_done(auth, task_id, timeout=90):
  198. return wait_for(lambda: task_state(auth, task_id).get("status") == "done" or False,
  199. timeout=timeout, what="task %d -> done" % task_id)
  200. def task_id_of(title):
  201. conn = db()
  202. try:
  203. return conn.execute("select id from tasks where title = ?", (title,)).fetchone()[0]
  204. finally:
  205. conn.close()
  206. def saddle_api(method, path, token, payload=None):
  207. """Raw call against the saddle ingest API; returns (status, json)."""
  208. request = urllib.request.Request(WEB + path,
  209. data=json.dumps(payload or {}).encode(), method=method)
  210. request.add_header("Content-Type", "application/json")
  211. if token is not None:
  212. request.add_header("Authorization", "Bearer " + token)
  213. try:
  214. with urllib.request.urlopen(request, timeout=15) as response:
  215. return response.status, json.loads(response.read().decode() or "{}")
  216. except urllib.error.HTTPError as error:
  217. body = error.read().decode()
  218. try:
  219. return error.code, json.loads(body)
  220. except ValueError:
  221. return error.code, {"raw": body}
  222. def db():
  223. return sqlite3.connect(DB_PATH)
  224. class FakeSaddle:
  225. """In-process saddle daemon implementing the wire contract.
  226. Phase-2 push behaviour: the daemon asks POST /api/saddle/next-task
  227. whenever idle-awake (at registration, on POST /wake, after its task
  228. reaches a terminal state, and when a /status probe observes it
  229. parked idle — throttled), accepts assignments like POST /start used
  230. to, then PUSHES its telemetry home: raw PTY chunks via POST
  231. /api/saddle/chunks and round results with state transitions via
  232. POST /api/saddle/results (working at round start, terminal states
  233. when the scripted lifecycle ends them). /output, /log and /start
  234. are gone (404) like a post-phase-2 daemon; /task-update serves the
  235. cancel/comment/subtask-status kinds; POST /answer receives a
  236. delivered verdict when the round is parked in a block's answer
  237. wait (409 when nothing waits — the real daemon's contract; the
  238. resumed round continues after the response). Every trigger can be
  239. disabled per test for determinism; ask_now() performs a
  240. synchronous manual ask.
  241. """
  242. PROMPT = "bbarrow@fedora:/workspace/task-1$ "
  243. def __init__(self):
  244. self.lock = threading.Lock()
  245. self.start_payloads = [] # captured assignment payloads (/start + next-task 200s)
  246. self.state = "idle"
  247. self.summary = ""
  248. self.state_entered = time.monotonic()
  249. self.fail_next = False # next task drives the task to failed
  250. self.start_counts = {} # task id -> assignment count
  251. self.resumed_pending = False # crash-resume flag on the next /status
  252. self.suspend_on_title = None # this task (by title) parks as suspended
  253. self.output_seq = 100 # daemon-global entry ids for pushed results
  254. self.last_activity_epoch = int(time.time())
  255. self.subtask_status_events = [] # recorded /task-update subtask-status payloads
  256. self.fail_subtask_status = False # exercise the web's retry path
  257. self.hang_next = {} # path suffix -> hang count (never responds)
  258. self.abort_failures = 0 # cancel task-updates failing (500) while > 0
  259. self.abort_log = [] # task ids aborted via task-update cancel
  260. self.running_task = 0 # task id the daemon believes it runs
  261. self.running_title = "" # its title (suspend targeting)
  262. self.session_id = None # the web-side run row of the running task
  263. self.hold_state = False # freeze lifecycle + heartbeat pushes
  264. self.ssh_public_key = "" # reported on /status + /register
  265. self.started_mono = time.monotonic()
  266. self.freeze_activity = False # hold last_activity_epoch still
  267. self.silent = False # suppress pushes (staleness tests)
  268. self.dead = False # kill(): no serving AND no calls home
  269. self.status_hits = 0 # GET /status arrivals (probe accounting)
  270. self.comment_log = [] # recorded /task-update comment payloads
  271. self.scripted_corpus = None # chunk batch pushed when a task starts
  272. # --- plan 10 phase 1: ask/wake/register ---
  273. self.token = None # per-saddle token from POST /register
  274. self.register_log = [] # recorded /register bodies
  275. self.wake_log = [] # recorded POST /wake arrivals
  276. self.asks = [] # [{"status": int, "payload": dict|None}]
  277. self.ask_on_register = True
  278. self.ask_on_wake = True
  279. self.ask_on_terminal = True
  280. self.ask_on_idle_status = True
  281. self.ask_delay = 0.0 # seconds; a slow daemon delays its wake-triggered ask
  282. self.serve_register = True # False: 404 /register (pre-flip daemon)
  283. self.last_ask_trigger = 0.0
  284. self.port = None
  285. # --- answer delivery (the parked-session resume) ---
  286. self.pending_block = None # the block payload this daemon waits on (state blocked)
  287. self.answer_log = [] # recorded POST /answer bodies
  288. self.answer_push_delay = 0.2 # the resumed round's first push, after the response
  289. def set_state(self, state, summary=None):
  290. """Advances the scripted lifecycle."""
  291. self.state = state
  292. if summary is not None:
  293. self.summary = summary
  294. self.state_entered = time.monotonic()
  295. def touch_activity(self):
  296. if not self.freeze_activity:
  297. self.last_activity_epoch = int(time.time())
  298. def queue_output(self, entry_type, command, output):
  299. """Pushes a typed entry as a round-results push (the daemon's
  300. entry log riding the results channel)."""
  301. with self.lock:
  302. entry = {
  303. "id": self.output_seq,
  304. "command": command,
  305. "output": output,
  306. "exit_code": 1,
  307. "type": entry_type,
  308. "timestamp": "2026-09-06T00:00:00Z",
  309. }
  310. self.output_seq += 1
  311. self.push_results(self.state if self.state in ("working",) else "working",
  312. entries=[entry])
  313. # --- plan 10 phase 2: the push channels ---
  314. def push_chunks(self, chunks):
  315. """Pushes raw PTY chunks home: elements are texts (assigned
  316. fresh seqs) or {seq, text} dicts (a redelivery or explicit
  317. corpus). Pushes are suppressed while silent/hold_state (the
  318. staleness tests' silent runner)."""
  319. if self.dead or self.silent or self.token is None or self.session_id is None:
  320. return
  321. batch = []
  322. with self.lock:
  323. for chunk in chunks:
  324. if isinstance(chunk, dict):
  325. batch.append({"seq": int(chunk["seq"]), "text": chunk["text"]})
  326. else:
  327. batch.append({"seq": self.next_push_seq(), "text": chunk})
  328. try:
  329. saddle_api("POST", "/api/saddle/chunks", self.token,
  330. {"session_id": self.session_id, "chunks": batch})
  331. except (OSError, ValueError):
  332. pass # app restarting / shutting down mid-test
  333. def next_push_seq(self):
  334. # The daemon-side chunk counter for ad-hoc pushes.
  335. self._push_seq = getattr(self, "_push_seq", 0) + 1
  336. return self._push_seq
  337. def push_results(self, state, entries=None, block=None, summary=None, session_id=None):
  338. """Pushes one round-results payload home (state transition plus
  339. its entries). Returns the HTTP status when it went out, None
  340. when suppressed."""
  341. if self.dead or self.silent:
  342. return None
  343. if self.token is None:
  344. return None
  345. target = session_id if session_id is not None else self.session_id
  346. if target is None:
  347. return None
  348. payload = {"session_id": target, "state": state}
  349. if entries is not None:
  350. payload["entries"] = entries
  351. if block is not None:
  352. payload["block"] = block
  353. if state == "blocked":
  354. # The daemon parks its round on this block's answer wait
  355. # (the real daemon's wait_for_answer).
  356. self.pending_block = dict(block)
  357. if summary is not None:
  358. payload["summary"] = summary
  359. try:
  360. status, _ = saddle_api("POST", "/api/saddle/results", self.token, payload)
  361. if status != 200:
  362. print(" [fake %s] push %s -> HTTP %s (task=%s)"
  363. % (self.port, state, status, self.running_task))
  364. return status
  365. except (OSError, ValueError) as error:
  366. print(" [fake %s] push %s failed: %r (task=%s)"
  367. % (self.port, state, error, self.running_task))
  368. return None
  369. # --- plan 10 phase 1: the ask path ---
  370. def ask_now(self):
  371. """Synchronously asks the web for work (a phase-1 daemon's idle
  372. ask). A 200 assignment is accepted exactly like POST /start;
  373. returns (status, payload|None)."""
  374. token = self.token
  375. if token is None:
  376. raise SystemExit("fake daemon asked before receiving its token")
  377. status, payload = saddle_api("POST", "/api/saddle/next-task", token)
  378. with self.lock:
  379. self.asks.append({"status": status, "payload": payload})
  380. if status == 200:
  381. self.accept_assignment(payload)
  382. return status, payload
  383. def _trigger_ask(self, reason):
  384. """Schedules a delayed ask (the daemon woke / went idle-awake).
  385. ask_delay models a SLOW asker — a daemon still winding down its
  386. round when the wake lands: it honours the wake, but its ask
  387. arrives late (still inside the web's lost-wake guard window)."""
  388. def run():
  389. if self.state in ("working", "failing"):
  390. return # a running daemon does not ask
  391. if self.ask_delay > 0:
  392. time.sleep(self.ask_delay)
  393. for attempt in range(3):
  394. try:
  395. status, _ = self.ask_now()
  396. if status in (200, 204):
  397. return
  398. print(" (ask %s -> HTTP %s, attempt %d)" % (reason, status, attempt + 1))
  399. except Exception as error: # app restarting / blips: retry
  400. print(" (ask %s failed: %r, attempt %d)" % (reason, error, attempt + 1))
  401. time.sleep(1.0)
  402. threading.Thread(target=lambda: (time.sleep(0.2), run()), daemon=True).start()
  403. def maybe_trigger_ask(self, reason, throttle=False):
  404. """Schedules an idle-awake ask. Protocol events (wake, abort,
  405. round end, park) ask singularly and immediately; only the
  406. /status-observation path throttles (a burst of probes must not
  407. hammer the ask endpoint)."""
  408. now = time.monotonic()
  409. if throttle and now - self.last_ask_trigger < 1.0:
  410. return
  411. self.last_ask_trigger = now
  412. self._trigger_ask(reason)
  413. def accept_assignment(self, payload):
  414. """A 200 next-task assignment: adopted exactly like POST /start."""
  415. self.start_payloads.append(payload)
  416. task = payload.get("task") or {}
  417. task_id = task.get("id")
  418. self.running_task = int(task_id or 0)
  419. self.running_title = task.get("title") or ""
  420. self.session_id = payload.get("session_id")
  421. # The assignment's answers resolve any block this daemon was
  422. # parked on (resume-carries-answer).
  423. self.pending_block = None
  424. count = self.start_counts.get(task_id, 0) + 1
  425. self.start_counts[task_id] = count
  426. if count > 1:
  427. # Crash-resume contract: re-pushing a task whose
  428. # workspace still exists restores it; the first
  429. # /status after the restore reports "resumed": true.
  430. self.resumed_pending = True
  431. self.set_state("failing" if self.fail_next else "working")
  432. self.fail_next = False
  433. # The round's opening telemetry: the flushed-so-far raw PTY
  434. # batch plus the working state push (>= 2 flushes per round
  435. # and the 10s timer are the real daemon's contract; the fake
  436. # pushes its batch up front and heartbeats while working).
  437. if self.scripted_corpus is not None:
  438. self.push_chunks([dict(chunk) for chunk in self.scripted_corpus])
  439. else:
  440. self.push_chunks([self.PROMPT, "echo hello\r\n", "hello\r\n"])
  441. self.push_results("working")
  442. class Handler(BaseHTTPRequestHandler):
  443. daemon = None # set on the class before serving
  444. protocol_version = "HTTP/1.1"
  445. def log_message(self, fmt, *args):
  446. pass
  447. def maybe_hang(self, path):
  448. """True when this request should hang forever (never respond),
  449. consuming one unit of the path's hang budget."""
  450. for suffix, count in list(self.daemon.hang_next.items()):
  451. if count > 0 and path.startswith(suffix):
  452. self.daemon.hang_next[suffix] = count - 1
  453. return True
  454. return False
  455. def _send(self, status, obj):
  456. body = json.dumps(obj).encode()
  457. self.send_response(status)
  458. self.send_header("Content-Type", "application/json")
  459. self.send_header("Content-Length", str(len(body)))
  460. self.end_headers()
  461. self.wfile.write(body)
  462. def do_POST(self):
  463. if self.maybe_hang(self.path):
  464. time.sleep(600)
  465. return
  466. self.daemon.touch_activity()
  467. length = int(self.headers.get("Content-Length") or 0)
  468. raw = self.rfile.read(length).decode() if length else "{}"
  469. if self.path == "/register":
  470. # Plan 10: the web delivers the per-saddle token + its
  471. # own base URL; the daemon persists the token for its
  472. # calls home and reports its ssh key + version.
  473. if not self.daemon.serve_register:
  474. self._send(404, {"error": "no such route"})
  475. return
  476. try:
  477. payload = json.loads(raw)
  478. except ValueError:
  479. payload = {"raw": raw}
  480. self.daemon.register_log.append(payload)
  481. if payload.get("saddle_token"):
  482. self.daemon.token = payload["saddle_token"]
  483. self._send(200, {"ok": True,
  484. "ssh_public_key": self.daemon.ssh_public_key,
  485. "version": "0.1-fake"})
  486. if self.daemon.ask_on_register:
  487. self.daemon.maybe_trigger_ask("register")
  488. return
  489. if self.path == "/wake":
  490. # Plan 10: a capable task is ready; a sleeping daemon
  491. # asks for work (immediately — a wake is a direct
  492. # command, never throttled).
  493. self.daemon.wake_log.append(time.monotonic())
  494. self._send(200, {"ok": True})
  495. if self.daemon.ask_on_wake:
  496. self.daemon.maybe_trigger_ask("wake")
  497. return
  498. if self.path == "/answer":
  499. # The web delivers a human verdict into the daemon's
  500. # live answer wait (the pre-flip channel, and the only
  501. # resume for a session parked mid-round — a blocked
  502. # daemon cannot ask). 409 mirrors the real daemon when
  503. # no block is pending. The resumed round continues
  504. # AFTER the response goes out (the real daemon resumes
  505. # its parked wait_for_answer via an idle callback).
  506. try:
  507. payload = json.loads(raw)
  508. except ValueError:
  509. self._send(400, {"error": "bad json"})
  510. return
  511. parked = self.daemon.pending_block
  512. if self.daemon.state != "blocked" or parked is None:
  513. self._send(409, {"error": "no block is waiting for an answer"})
  514. return
  515. self.daemon.pending_block = None
  516. self.daemon.answer_log.append({
  517. "answer": payload.get("answer", ""),
  518. "branch_advanced": payload.get("branch_advanced"),
  519. "block": dict(parked)})
  520. self.daemon.set_state("working")
  521. self._send(200, {"ok": True})
  522. daemon_ref = self.daemon
  523. def continue_round():
  524. time.sleep(daemon_ref.answer_push_delay)
  525. if daemon_ref.dead or daemon_ref.silent:
  526. return
  527. daemon_ref.push_results("working")
  528. daemon_ref.set_state("done", "resumed after the verdict")
  529. daemon_ref.push_results("done", summary="resumed after the verdict")
  530. if daemon_ref.ask_on_terminal:
  531. daemon_ref.maybe_trigger_ask("terminal")
  532. threading.Thread(target=continue_round, daemon=True).start()
  533. return
  534. if self.path == "/task-update":
  535. # Plan 10: cancel (abort machinery), comment (injected
  536. # turn) and subtask-status (live subtask set) — delivered
  537. # directly to a RUNNING daemon, no auth.
  538. try:
  539. payload = json.loads(raw)
  540. except ValueError:
  541. self._send(400, {"error": "bad json"})
  542. return
  543. kind = payload.get("kind")
  544. if kind == "cancel":
  545. if self.daemon.abort_failures > 0:
  546. self.daemon.abort_failures -= 1
  547. self._send(500, {"error": "cancel failed (simulated)"})
  548. return
  549. aborted = self.daemon.running_task
  550. self.daemon.set_state("idle")
  551. self.daemon.summary = ""
  552. self.daemon.running_task = 0
  553. self.daemon.session_id = None
  554. self.daemon.abort_log.append(aborted)
  555. self._send(200, {"ok": True, "aborted_task_id": aborted})
  556. # An aborted daemon returns to its idle loop: it asks.
  557. self.daemon.maybe_trigger_ask("aborted")
  558. elif kind == "comment":
  559. self.daemon.comment_log.append(payload)
  560. self._send(200, {"ok": True})
  561. elif kind == "subtask-status":
  562. if self.daemon.fail_subtask_status:
  563. self._send(500, {"error": "simulated failure"})
  564. return
  565. self.daemon.subtask_status_events.append(payload)
  566. self._send(200, {"ok": True})
  567. else:
  568. self._send(400, {"error": "unknown task-update kind"})
  569. return
  570. self._send(404, {"error": "no such route"})
  571. self.close_connection = True
  572. def do_GET(self):
  573. if self.maybe_hang(self.path):
  574. time.sleep(600)
  575. return
  576. if self.path != "/status":
  577. # /status itself is not daemon activity: the liveness
  578. # signal must reflect loop rounds / command completions.
  579. self.daemon.touch_activity()
  580. if self.path == "/status":
  581. self.daemon.status_hits += 1
  582. response = {"state": self.daemon.state, "summary": self.daemon.summary,
  583. "uptime_seconds": int(time.monotonic() - self.daemon.started_mono),
  584. "last_activity_epoch": self.daemon.last_activity_epoch,
  585. "ssh_public_key": self.daemon.ssh_public_key}
  586. if self.daemon.resumed_pending:
  587. response["resumed"] = True
  588. self.daemon.resumed_pending = False
  589. # A parked/terminal daemon is idle-awake: a phase-1
  590. # daemon asks (and sleeps on 204). Throttled so status
  591. # poll bursts do not hammer the ask endpoint.
  592. if self.daemon.ask_on_idle_status and self.daemon.state in (
  593. "idle", "done", "failed", "suspended"):
  594. self.daemon.maybe_trigger_ask("idle-status", throttle=True)
  595. self._send(200, response)
  596. else:
  597. self._send(404, {"error": "no such route"})
  598. self.close_connection = True
  599. def run_lifecycle(self):
  600. """Applies the scripted lifecycle on a timer (like a real daemon
  601. whose task ends on its own): working heartbeats chunks (the 10s
  602. flush timer's stand-in, keeping the staleness clock fed), works
  603. for ~8s, then pushes done/suspended; failing pushes failed after
  604. ~1s — and a task END makes the idle-awake daemon ask again."""
  605. while True:
  606. time.sleep(0.5)
  607. if self.hold_state or self.silent:
  608. # hold_state freezes the script; silent models a daemon
  609. # mid-round with nothing to flush — its round has not
  610. # ended, so no terminal push fires while quiet.
  611. continue
  612. if self.state == "failing":
  613. if time.monotonic() - self.state_entered >= 1:
  614. self.set_state("failed", "bash -n rejected the script (simulated)")
  615. self.push_results("failed",
  616. summary="bash -n rejected the script (simulated)")
  617. if self.ask_on_terminal:
  618. self.maybe_trigger_ask("terminal")
  619. elif self.state == "working":
  620. if time.monotonic() - self.state_entered >= 8:
  621. if self.suspend_on_title and self.suspend_on_title == self.running_title:
  622. # `saddle suspend`: park until a fresh assignment
  623. # resumes the workspace (title-targeted one-shot,
  624. # so an interleaved dispatch cannot eat the flag).
  625. self.set_state("suspended")
  626. self.suspend_on_title = None
  627. self.push_results("suspended")
  628. # A parked daemon is idle-awake: it asks (and
  629. # sleeps on 204) so a later wake can reach it.
  630. if self.ask_on_idle_status:
  631. self.maybe_trigger_ask("parked")
  632. else:
  633. self.set_state("done", "fake daemon finished")
  634. self.push_results("done", summary="fake daemon finished")
  635. if self.ask_on_terminal:
  636. self.maybe_trigger_ask("terminal")
  637. elif time.monotonic() - getattr(self, "last_heartbeat", 0) >= 2:
  638. # The flush timer: output keeps flowing, so chunks
  639. # keep arriving (the staleness window never opens on
  640. # a healthy runner).
  641. self.last_heartbeat = time.monotonic()
  642. self.push_chunks(["working-heartbeat\r\n"])
  643. def serve(self, port=None):
  644. """Binds the daemon's HTTP server. A requested port that is
  645. already taken (free_port() hands out recyclable ports — two
  646. calls can collide) falls back to an OS-assigned one instead of
  647. silently dying and impersonating whatever owns that port."""
  648. if port is None:
  649. port = DAEMON_PORT
  650. handler = type("BoundHandler", (FakeSaddle.Handler,), {"daemon": self})
  651. self.server = None
  652. try:
  653. self.server = ThreadingHTTPServer(("127.0.0.1", port), handler)
  654. except OSError:
  655. print("(fake daemon: port %d taken, falling back)" % port)
  656. self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
  657. self.port = self.server.server_address[1]
  658. threading.Thread(target=self.server.serve_forever, daemon=True).start()
  659. threading.Thread(target=self.run_lifecycle, daemon=True).start()
  660. return self.server
  661. def kill(self):
  662. """The daemon dies: it stops listening (connections refused) AND
  663. stops calling home — no pushes, no asks (a dead process has no
  664. outbound anything; in-flight handler threads may linger but the
  665. wire contract's traffic is one-directional now)."""
  666. self.dead = True
  667. if getattr(self, "server", None) is not None:
  668. self.server.shutdown()
  669. self.server.server_close()
  670. self.server = None
  671. def start_app():
  672. config = json.load(open(os.path.join(REPO, "web-config.json")))
  673. config["saddle_callback_base_url"] = WEB
  674. with open(os.path.join(WORK, "web-config.json"), "w") as handle:
  675. json.dump(config, handle)
  676. env = dict(os.environ)
  677. env.update({
  678. "SPRY_DB_PATH": DB_PATH,
  679. "SPRY_SEED_ADMIN_USERNAME": "admin",
  680. "SPRY_SEED_ADMIN_PASSWORD": "adminpass123",
  681. "SADDLE_LLM_BASE_URL": "http://127.0.0.1:%d" % DAEMON_PORT,
  682. "SADDLE_PROJECTS_ROOT": os.path.join(WORK, "workspace"),
  683. "SADDLE_HTTP_TIMEOUT_S": "5",
  684. "SADDLE_LOST_WAKE_S": "2",
  685. "SADDLE_STALE_S": "3",
  686. # Never inherit a caller-poisoned LD_LIBRARY_PATH (e.g. one that
  687. # resolves the ancient /lib64 copies): empty makes the loader use
  688. # the binary's RUNPATH, which is the dev stack it was built
  689. # against.
  690. "LD_LIBRARY_PATH": "",
  691. })
  692. log_previous = os.path.join(WORK, "app.log.1")
  693. if os.path.exists(os.path.join(WORK, "app.log")):
  694. shutil.move(os.path.join(WORK, "app.log"), log_previous)
  695. log = open(os.path.join(WORK, "app.log"), "w")
  696. command = [APP, str(APP_PORT)]
  697. if os.environ.get("WEBSTACK_GDB"):
  698. # Debugging aid: run the app under gdb; a crash backtrace lands in
  699. # app.log. WEBSTACK_GDB=1 meson test ... to enable.
  700. command = ["gdb", "-batch", "-quiet",
  701. "-ex", "handle SIGPIPE nostop noprint pass",
  702. "-ex", "run", "-ex", "bt", "--args", APP, str(APP_PORT)]
  703. process = subprocess.Popen(command, cwd=WORK, env=env,
  704. stdout=log, stderr=subprocess.STDOUT)
  705. outcome = wait_for(lambda: (app_ready() and "ready")
  706. or (process.poll() is not None and "app-exited-%d" % process.poll())
  707. or False, timeout=30, what="app ready")
  708. if outcome is None:
  709. raise SystemExit("app did not come up; see %s/app.log" % WORK)
  710. if str(outcome).startswith("app-exited-"):
  711. raise SystemExit("app exited during startup (%s); see %s/app.log"
  712. % outcome, WORK)
  713. return process
  714. def app_ready():
  715. try:
  716. status, _ = http("GET", "/login", timeout=3)
  717. return status == 200
  718. except (urllib.error.URLError, OSError):
  719. return False
  720. def dump_app_stacks(pid):
  721. """Best-effort thread backtrace of the (possibly wedged) app."""
  722. try:
  723. fds = os.listdir("/proc/%d/fd" % pid)
  724. print("(app open fds at dump: %d)" % len(fds))
  725. kinds = {}
  726. for fd in fds[:1200]:
  727. try:
  728. target = os.readlink("/proc/%d/fd/%s" % (pid, fd))
  729. except OSError:
  730. continue
  731. key = target.split(":", 1)[0] if ":" in target else target
  732. kinds[key] = kinds.get(key, 0) + 1
  733. print("(fd kinds: %s)" % sorted(kinds.items(), key=lambda kv: -kv[1])[:6])
  734. except OSError:
  735. pass
  736. for command in (["eu-stack", "-p", str(pid)],
  737. ["gdb", "-batch", "-quiet", "-p", str(pid),
  738. "-ex", "thread apply all bt"]):
  739. try:
  740. out = subprocess.run(command, capture_output=True, timeout=60)
  741. text = out.stdout.decode(errors="replace") + out.stderr.decode(errors="replace")
  742. if text.strip():
  743. print("(app stacks via %s)\n%s" % (command[0], text[:12000]))
  744. return
  745. except (OSError, subprocess.SubprocessError):
  746. continue
  747. def main():
  748. if not os.path.exists(APP):
  749. raise SystemExit("app binary not found at %s (build first)" % APP)
  750. fake = FakeSaddle()
  751. fake.serve()
  752. # A rare framework-level crash (stack corruption in Statum slot
  753. # signing during the FIRST login — shared-library codegen bug, moved
  754. # twice already) kills the app before any test runs; a clean restart
  755. # sidesteps it. Only tolerated while nothing has been checked yet.
  756. auth = None
  757. app = None
  758. for attempt in range(3):
  759. if os.path.exists(WORK):
  760. shutil.rmtree(WORK)
  761. os.makedirs(os.path.join(WORK, "workspace"))
  762. try:
  763. app = start_app()
  764. auth = login("admin", "adminpass123")
  765. break
  766. except (SystemExit, OSError, urllib.error.URLError):
  767. if passes > 0:
  768. raise
  769. if app is not None and app.poll() is None:
  770. app.terminate()
  771. try:
  772. app.wait(timeout=5)
  773. except subprocess.TimeoutExpired:
  774. app.kill()
  775. app = None
  776. print("(app died before login; restarting cleanly, attempt %d)" % (attempt + 1))
  777. if app is None or auth is None:
  778. raise SystemExit("app never came up; see %s/app.log" % WORK)
  779. try:
  780. check(True, "login yields an auth slot")
  781. print("== feature 3: plan review hides duplicate body ==")
  782. state = slot_public(auth, "/projects", "projects")
  783. act(auth, state["create"], {
  784. "name": "demo",
  785. "git_url": "https://example.com/demo.git",
  786. "default_branch": "main",
  787. })
  788. conn = db()
  789. conn.execute("insert into tasks (project_id, title, status, priority,"
  790. " is_design_task, plan, created_at)"
  791. " values (1, 'Plan review task', 'working', 5, 1, '# The plan\n\n1. Do it',"
  792. " strftime('%s','now'))")
  793. conn.commit()
  794. plan_task = conn.execute("select max(id) from tasks").fetchone()[0]
  795. saddles_state = slot_public(auth, "/saddles", "saddles")
  796. act(auth, saddles_state["register"], {"name": "fake-saddle",
  797. "host": "127.0.0.1:%d" % DAEMON_PORT})
  798. registered = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  799. if s["name"] == "fake-saddle"]
  800. check(len(registered) == 1 and registered[0]["token"].startswith("saddle-"),
  801. "fake saddle registered with an API token")
  802. fake_id = int(registered[0]["id"])
  803. fake_token = registered[0]["token"]
  804. # Plan 10 phase 1: registration delivers the per-saddle token to
  805. # the daemon via POST /register {saddle_token, web_base_url}.
  806. check(wait_for(lambda: len(fake.register_log) > 0 or False,
  807. timeout=15, what="/register delivery") is not None,
  808. "registration POSTs /register to the daemon")
  809. register_body = fake.register_log[0]
  810. check(register_body.get("saddle_token") == fake_token,
  811. "/register carries the minted per-saddle token")
  812. check(register_body.get("web_base_url") == WEB,
  813. "/register carries the web base URL")
  814. check(fake.token == fake_token,
  815. "daemon persisted the token (its asks use it)")
  816. token_conn = db()
  817. row_token = token_conn.execute("select token from saddles where id = ?",
  818. (fake_id,)).fetchone()[0]
  819. token_conn.close()
  820. check(row_token == fake_token,
  821. "per-saddle token lives on the saddle row")
  822. # The register-time ask (idle-awake daemon): 204, nothing
  823. # offerable yet — the saddle is asleep and wakeable.
  824. check(wait_for(lambda: any(a["status"] == 204 for a in fake.asks) or False,
  825. timeout=15, what="register-time ask") is not None,
  826. "the daemon's idle ask returns 204 (nothing offerable)")
  827. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
  828. " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
  829. (plan_task, fake_id))
  830. conn.commit()
  831. session_id = conn.execute("select max(id) from sessions").fetchone()[0]
  832. conn.close()
  833. status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  834. {"type": "block", "session_id": session_id, "data": {
  835. "question": "Plan submitted for review",
  836. "type": "rfc",
  837. "body": "# The plan\n\n1. Do it"}})
  838. check(status == 200 and body.get("block_id"), "plan-review block raised")
  839. status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  840. {"type": "block", "session_id": session_id, "data": {
  841. "question": "Which database?",
  842. "type": "question",
  843. "options": ["SQLite", "Postgres"],
  844. "body": "# Storage choice"}})
  845. check(status == 200, "question block raised")
  846. detail = task_state(auth, plan_task)
  847. blocks = {b["block_type"]: b for b in detail["blocks"]}
  848. check(blocks["rfc"].get("is_plan_review") is True,
  849. "rfc block flagged as the plan review")
  850. check(blocks["rfc"].get("show_body") is False,
  851. "pending plan review hides the duplicate body preview")
  852. check(blocks["question"].get("show_body") is True,
  853. "question block keeps its body preview")
  854. act(auth, blocks["rfc"]["answer_action"], {"answer": "Plan approved. Proceed."})
  855. detail = task_state(auth, plan_task)
  856. rfc = [b for b in detail["blocks"] if b["block_type"] == "rfc"][0]
  857. check(rfc.get("show_body") is True,
  858. "answered plan review shows the body again (review section gone)")
  859. print("== B1: highlight-to-comment quote dialog ==")
  860. status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
  861. check(status == 200 and "quote-comment-dialog" in page and "quote-text" in page,
  862. "task page carries the quote-comment dialog the popover opens")
  863. check('name="quote"' in page and 'name="return_to"' in page,
  864. "quote dialog posts the quote and return-to fields")
  865. status, page = http("GET", "/tasks/%d/terminal" % plan_task, timeout=15)
  866. check(status == 200 and "quote-comment-dialog" in page,
  867. "terminal page keeps its quote dialog")
  868. print("== B3: review section hides the duplicate body ==")
  869. status, body = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  870. {"type": "block", "session_id": session_id, "data": {
  871. "question": "Review the diff",
  872. "type": "review",
  873. "body": "# The change\n\nReworks the flange module."}})
  874. check(status == 200 and body.get("block_id"), "review block raised")
  875. detail = task_state(auth, plan_task)
  876. review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
  877. check(detail.get("review_pending") is True,
  878. "review section shows while the review is unanswered")
  879. check(review_block.get("show_body") is False,
  880. "pending review block hides its duplicate body preview")
  881. check(review_block.get("body", "").startswith("# The change"),
  882. "review body itself still arrives in state (B2 counterpart)")
  883. act(auth, review_block["answer_action"], {"answer": "Human approved the review"})
  884. detail = task_state(auth, plan_task)
  885. review_block = [b for b in detail["blocks"] if b["block_type"] == "review"][0]
  886. check(review_block.get("show_body") is True,
  887. "answered review shows the body again (banner gone)")
  888. print("== B2: question block body above options ==")
  889. # Contract side (A2 will restore bodies end-to-end): a question
  890. # block raised with a heredoc body carries it; the body preview
  891. # rides above the options (template order) and survives answering
  892. # alongside the selected answer.
  893. detail = task_state(auth, plan_task)
  894. question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
  895. check(question.get("body") == "# Storage choice"
  896. and question.get("show_body") is True,
  897. "question block keeps its body markdown with the preview on")
  898. act(auth, question["answer_action"], {"option": "SQLite"})
  899. detail = task_state(auth, plan_task)
  900. question = [b for b in detail["blocks"] if b["block_type"] == "question"][0]
  901. check(question.get("answered") is True and question.get("answer") == "SQLite"
  902. and question.get("body") == "# Storage choice"
  903. and question.get("show_body") is True,
  904. "answered question shows body + selected answer")
  905. print("== B5: compare-changes SSH -> HTTPS ==")
  906. def diff_url_for_project(git_url, suffix):
  907. state = slot_public(auth, "/projects", "projects")
  908. act(auth, state["create"], {"name": "diff-%s" % suffix, "git_url": git_url})
  909. project_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  910. if p["name"] == "diff-%s" % suffix][0])
  911. conn = db()
  912. conn.execute("insert into tasks (project_id, title, status, task_branch,"
  913. " branch_from, created_at) values (?, ?, 'working', 'feature/x',"
  914. " 'main', strftime('%s','now'))", (project_id, "Diff %s" % suffix))
  915. conn.commit()
  916. task_id = conn.execute("select max(id) from tasks").fetchone()[0]
  917. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
  918. " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
  919. " 'done')", (task_id, fake_id))
  920. conn.commit()
  921. session = conn.execute("select max(id) from sessions").fetchone()[0]
  922. conn.close()
  923. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  924. {"type": "block", "session_id": session, "data": {
  925. "question": "Review the diff", "type": "review"}})
  926. return task_state(auth, task_id).get("review_diff_url") or ""
  927. check(diff_url_for_project("ssh://fabrica.unitatem.net:222/clanker/clanker-saddle.git",
  928. "fabrica")
  929. == "https://fabrica.unitatem.net/clanker/clanker-saddle/compare/main...feature/x",
  930. "ssh://git@host:port/owner/repo.git translates (port dropped)")
  931. check(diff_url_for_project("git@github.com:acme/widgets.git", "scp")
  932. == "https://github.com/acme/widgets/compare/main...feature/x",
  933. "git@host:owner/repo.git translates to the browse host")
  934. check(diff_url_for_project("https://example.com/plain/repo", "plain")
  935. == "https://example.com/plain/repo/compare/main...feature/x",
  936. "HTTPS URLs pass through unchanged")
  937. print("== feature 5: rules ==")
  938. rules_state = slot_public(auth, "/rules", "rules")
  939. for name, body in [
  940. ("Use Invercargill", "Never use libgee; use Invercargill enumerables."),
  941. ("Write Vala docs", "Every public class carries a doc comment."),
  942. ("Prefer meson", "Builds go through meson; no makefiles."),
  943. ("Co-dev rule", "Keep the wire contract exactly."),
  944. ("Reference rule", "Must never ride along."),
  945. ]:
  946. act(auth, rules_state["create"], {"name": name, "body": body})
  947. listed = {r["name"]: r for r in slot_public(auth, "/rules", "rules")["rules"]}
  948. check(len(listed) == 5, "five rules created and listed on /rules")
  949. def rule_id(name):
  950. rows = [r for r in slot_public(auth, "/rules", "rules")["rules"] if r["name"] == name]
  951. return int(rows[0]["id"])
  952. state = slot_public(auth, "/projects", "projects")
  953. act(auth, state["create"], {"name": "co", "git_url": "https://example.com/co.git"})
  954. act(auth, state["create"], {"name": "ref", "git_url": "https://example.com/ref.git"})
  955. co_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  956. if p["name"] == "co"][0])
  957. ref_id = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  958. if p["name"] == "ref"][0])
  959. project = slot_public(auth, "/projects/1", "project")
  960. act(auth, project["add_link"], {"linked_project_id": str(co_id),
  961. "relationship": "co-developed"})
  962. act(auth, project["add_link"], {"linked_project_id": str(ref_id),
  963. "relationship": "reference"})
  964. act(auth, project["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
  965. act(auth, project["attach_rule"], {"rule_id": str(rule_id("Prefer meson"))})
  966. project = slot_public(auth, "/projects/1", "project")
  967. check(sorted(r["name"] for r in project["rules"]) == ["Prefer meson", "Use Invercargill"],
  968. "project page lists its attached rules")
  969. co_project = slot_public(auth, "/projects/%d" % co_id, "project")
  970. act(auth, co_project["attach_rule"], {"rule_id": str(rule_id("Co-dev rule"))})
  971. ref_project = slot_public(auth, "/projects/%d" % ref_id, "project")
  972. act(auth, ref_project["attach_rule"], {"rule_id": str(rule_id("Reference rule"))})
  973. tasks_state = slot_public(auth, "/tasks", "tasks")
  974. act(auth, tasks_state["create"], {
  975. "project_id": "1",
  976. "title": "Rules dispatch",
  977. "description": "Effective rule set rides the /start payload.",
  978. "priority": "2",
  979. })
  980. rule_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  981. if t["title"] == "Rules dispatch"][0])
  982. detail = task_state(auth, rule_task)
  983. act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Write Vala docs"))})
  984. act(auth, detail["attach_rule"], {"rule_id": str(rule_id("Use Invercargill"))})
  985. detail = task_state(auth, rule_task)
  986. check(sorted(r["name"] for r in detail["rules"]) == ["Use Invercargill", "Write Vala docs"],
  987. "task page lists its attached rules")
  988. check(wait_for(lambda: len(fake.start_payloads) > 0 or False,
  989. timeout=30, what="/start push") is not None,
  990. "dispatcher pushes the task to the fake saddle")
  991. payload = fake.start_payloads[-1]
  992. rule_names = [r["name"] for r in payload.get("rules", [])]
  993. check(rule_names == ["Write Vala docs", "Use Invercargill", "Prefer meson", "Co-dev rule"],
  994. "effective rules in /start: task, primary, co-developed order (%s)" % rule_names)
  995. check(all(r.get("body") for r in payload.get("rules", [])),
  996. "rules carry their bodies")
  997. check("Reference rule" not in rule_names,
  998. "reference project rules excluded")
  999. detail = task_state(auth, rule_task)
  1000. act(auth, detail["rules"][0]["detach"], {})
  1001. check([r["name"] for r in task_state(auth, rule_task)["rules"]] == ["Use Invercargill"],
  1002. "detached task rule disappears")
  1003. rules_rows = slot_public(auth, "/rules", "rules")["rules"]
  1004. docs_row = [r for r in rules_rows if r["name"] == "Write Vala docs"][0]
  1005. act(auth, docs_row["update"], {"name": "Write Vala docs", "body": "Doc comments or bust."})
  1006. check(any(r["body"] == "Doc comments or bust."
  1007. for r in slot_public(auth, "/rules", "rules")["rules"]),
  1008. "rule edit saves the body")
  1009. check(wait_task_done(auth, rule_task, timeout=60) is not None,
  1010. "dispatched task completes against the fake daemon")
  1011. print("== feature 6: project notes + callback ==")
  1012. # Plan 10 phase 1: the shared saddle-callback token is gone —
  1013. # the per-saddle token authorises the callback endpoints.
  1014. callback_token = fake_token
  1015. check(callback_token.startswith("saddle-") and "saddle-callback-" not in callback_token,
  1016. "note callback authorised by the per-saddle token")
  1017. status, _ = saddle_api("POST", "/api/saddle/note", None,
  1018. {"project": "demo", "body": "x"})
  1019. check(status == 401, "note endpoint rejects a missing token")
  1020. status, _ = saddle_api("POST", "/api/saddle/note", "saddle-callback-not-real",
  1021. {"project": "demo", "body": "x"})
  1022. check(status == 401, "note endpoint rejects a wrong (legacy-style) token")
  1023. status, body = saddle_api("POST", "/api/saddle/note", callback_token,
  1024. {"task_id": rule_task, "project": "demo",
  1025. "body": "style.css tokens live in :root"})
  1026. check(status == 200 and body.get("ok") is True, "note accepted with the callback token")
  1027. status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
  1028. {"project": "no-such-project", "body": "x"})
  1029. check(status == 400, "unknown project rejected")
  1030. status, _ = saddle_api("POST", "/api/saddle/note", callback_token,
  1031. {"project": "demo", "body": " "})
  1032. check(status == 400, "empty note body rejected")
  1033. project = slot_public(auth, "/projects/1", "project")
  1034. check(any("style.css tokens" in n["body"] and int(n["source_task_id"]) == rule_task
  1035. for n in project["notes"]),
  1036. "project page lists the note with its source task")
  1037. act(auth, [n for n in project["notes"] if "style.css tokens" in n["body"]][0]["delete"], {})
  1038. check(all("style.css tokens" not in n["body"]
  1039. for n in slot_public(auth, "/projects/1", "project")["notes"]),
  1040. "note deleted from the project page")
  1041. # /start payload: notes of primary + co-developed (not reference),
  1042. # oldest first; callback carries the configured base URL + token.
  1043. saddle_api("POST", "/api/saddle/note", callback_token,
  1044. {"project": "demo", "body": "note-one"})
  1045. saddle_api("POST", "/api/saddle/note", callback_token,
  1046. {"project": "demo", "body": "note-two"})
  1047. saddle_api("POST", "/api/saddle/note", callback_token,
  1048. {"project": "co", "body": "co-note"})
  1049. conn = db()
  1050. conn.execute("insert into project_notes (project_id, body, created_at)"
  1051. " values (?, 'ref-note-must-not-ship', strftime('%s','now'))", (ref_id,))
  1052. conn.commit()
  1053. conn.close()
  1054. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1055. "project_id": "1",
  1056. "title": "Notes dispatch",
  1057. "description": "Project notes ride the /start payload.",
  1058. "priority": "2",
  1059. })
  1060. check(wait_for(lambda: len(fake.start_payloads) >= 2 or False,
  1061. timeout=30, what="second /start push") is not None,
  1062. "dispatcher pushes the notes task")
  1063. payload = fake.start_payloads[-1]
  1064. notes = payload.get("project_notes", [])
  1065. check([n["body"] for n in notes] == ["note-one", "note-two", "co-note"],
  1066. "project_notes: primary + co-developed, oldest first (%s)" % [n["body"] for n in notes])
  1067. check(all(n.get("project") for n in notes)
  1068. and notes[2]["project"] == "co",
  1069. "project_notes carry the project name")
  1070. check("ref-note-must-not-ship" not in [n["body"] for n in notes],
  1071. "reference project notes excluded")
  1072. callback = payload.get("callback", {})
  1073. check(callback.get("base_url") == WEB and callback.get("token") == callback_token,
  1074. "callback carries the configured base URL and the minted token")
  1075. # entry cap: 30 of 33 candidates
  1076. conn = db()
  1077. for i in range(33):
  1078. conn.execute("insert into project_notes (project_id, body, created_at)"
  1079. " values (?, ?, strftime('%s','now'))", (1, "bulk-%02d" % i))
  1080. conn.commit()
  1081. conn.close()
  1082. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1083. "project_id": "1",
  1084. "title": "Entry cap dispatch",
  1085. "priority": "2",
  1086. })
  1087. check(wait_for(lambda: len(fake.start_payloads) >= 3 or False,
  1088. timeout=30, what="third /start push") is not None,
  1089. "dispatcher pushes the entry-cap task")
  1090. notes = fake.start_payloads[-1].get("project_notes", [])
  1091. check(len(notes) == 30 and notes[0]["body"] == "note-one" and notes[1]["body"] == "note-two",
  1092. "project_notes capped at 30 entries, oldest kept (%d)" % len(notes))
  1093. # char cap: the giant note is truncated so the total stays <= 20000
  1094. conn = db()
  1095. conn.execute("delete from project_notes where body like 'bulk-%' or body = 'co-note'")
  1096. conn.execute("insert into project_notes (project_id, body, created_at)"
  1097. " values (?, ?, strftime('%s','now'))", (1, "giant!" + "x" * 25000,))
  1098. conn.commit()
  1099. conn.close()
  1100. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1101. "project_id": "1",
  1102. "title": "Char cap dispatch",
  1103. "priority": "2",
  1104. })
  1105. check(wait_for(lambda: len(fake.start_payloads) >= 4 or False,
  1106. timeout=30, what="fourth /start push") is not None,
  1107. "dispatcher pushes the char-cap task")
  1108. notes = fake.start_payloads[-1].get("project_notes", [])
  1109. total = sum(len(n["body"]) for n in notes)
  1110. check(len(notes) == 3 and total == 20000 and notes[2]["body"].startswith("giant!"),
  1111. "project_notes char cap: head kept, giant truncated (entries=%d, chars=%d)"
  1112. % (len(notes), total))
  1113. print("== phase 2: pushed chunks -> raw log + terminal view ==")
  1114. # Wait for the last dispatch to finish so its pushes landed.
  1115. check(wait_task_done(auth, task_id_of("Char cap dispatch"), timeout=60) is not None,
  1116. "char-cap task completes")
  1117. conn = db()
  1118. raw_session = conn.execute(
  1119. "select s.id from sessions s join tasks t on t.id = s.task_id"
  1120. " where t.title = 'Char cap dispatch' order by s.id desc limit 1").fetchone()[0]
  1121. rows = conn.execute("select seq, chunk from session_raw_log where session_id = ?"
  1122. " order by seq", (raw_session,)).fetchall()
  1123. conn.close()
  1124. check(len(rows) >= 3, "session_raw_log captured the pushed chunks (%d)" % len(rows))
  1125. seqs = [r[0] for r in rows]
  1126. check(seqs == sorted(seqs) and len(set(seqs)) == len(seqs),
  1127. "pushed chunks stored once, seq strictly increasing")
  1128. text = "".join(r[1] for r in rows)
  1129. check(fake.PROMPT in text and "echo hello" in text and "hello" in text,
  1130. "pushed raw log keeps the prompt and echoed output byte-exact")
  1131. check("working-heartbeat" in text,
  1132. "the flush-timer heartbeat chunks keep arriving mid-round")
  1133. terminal = slot_public(auth, "/tasks/%d/terminal" % task_id_of("Char cap dispatch"),
  1134. "terminal")
  1135. check(terminal.get("has_raw_log") is True and int(terminal["session_id"]) == raw_session,
  1136. "terminal view selects raw rendering for the raw session")
  1137. # Old sessions (typed entries, no raw rows) keep the fallback view.
  1138. conn = db()
  1139. conn.execute("insert into session_commands (session_id, command_text, output_text,"
  1140. " type, started_at) values (?, 'ls', 'old-output', 'command',"
  1141. " strftime('%s','now'))", (session_id,))
  1142. conn.commit()
  1143. conn.close()
  1144. legacy = slot_public(auth, "/tasks/%d/terminal" % plan_task, "terminal")
  1145. check(legacy.get("has_raw_log") is False,
  1146. "legacy session falls back to the entry-type terminal view")
  1147. request = urllib.request.Request(WEB + "/terminal-stream/%d" % raw_session)
  1148. with urllib.request.urlopen(request, timeout=15) as stream:
  1149. raw_events = []
  1150. deadline = time.monotonic() + 8
  1151. while time.monotonic() < deadline and len(raw_events) < 3:
  1152. line = stream.readline().decode().strip()
  1153. if not line.startswith("data: ") or line == "data: ping":
  1154. continue
  1155. event = json.loads(line[6:])
  1156. if event.get("type") == "raw":
  1157. raw_events.append(event)
  1158. check(len(raw_events) >= 3 and raw_events[0]["text"] == fake.PROMPT
  1159. and raw_events[0]["seq"] < raw_events[1]["seq"],
  1160. "terminal stream serves pushed chunks in seq order (prompt first)")
  1161. print("== phase 2: chunks/results ingestion — auth, idempotency ==")
  1162. # Token auth on both push channels (plan 10: the per-saddle
  1163. # bearer authorises every daemon->web call).
  1164. status, _ = saddle_api("POST", "/api/saddle/chunks", None,
  1165. {"session_id": raw_session, "chunks": []})
  1166. check(status == 401, "chunks endpoint rejects a missing token")
  1167. status, _ = saddle_api("POST", "/api/saddle/results", "saddle-bogus-token",
  1168. {"session_id": raw_session, "state": "working"})
  1169. check(status == 401, "results endpoint rejects a wrong token")
  1170. status, _ = saddle_api("POST", "/api/saddle/chunks", fake_token,
  1171. {"session_id": 99999, "chunks": []})
  1172. check(status == 400, "chunks for an unknown session rejected")
  1173. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  1174. {"session_id": raw_session, "state": "done"})
  1175. check(status == 200, "the owning saddle's session accepts its results")
  1176. # Pushed chunk idempotency: re-pushing a batch (the outbox's
  1177. # at-least-once redelivery after a lost response) consumes the
  1178. # duplicates and stores only the tail.
  1179. conn = db()
  1180. before = conn.execute("select count(*) from session_raw_log where session_id = ?",
  1181. (raw_session,)).fetchone()[0]
  1182. conn.close()
  1183. redelivery = [{"seq": r[0], "text": r[1]} for r in rows[:3]]
  1184. status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
  1185. {"session_id": raw_session, "chunks": redelivery})
  1186. check(status == 200 and body.get("ok") is True and int(body.get("stored") or 0) == 0,
  1187. "a fully-duplicate batch stores nothing (duplicates consumed)")
  1188. fresh = [{"seq": 9000, "text": "late-flush\r\n"},
  1189. {"seq": 9001, "text": ""}] # truly empty text: no row
  1190. status, body = saddle_api("POST", "/api/saddle/chunks", fake_token,
  1191. {"session_id": raw_session, "chunks": fresh})
  1192. check(status == 200 and int(body.get("stored") or 0) == 1,
  1193. "a partial-redelivery batch stores only the new chunks")
  1194. conn = db()
  1195. after = conn.execute("select count(*) from session_raw_log where session_id = ?",
  1196. (raw_session,)).fetchone()[0]
  1197. empty_row = conn.execute(
  1198. "select count(*) from session_raw_log where session_id = ? and seq = 9001",
  1199. (raw_session,)).fetchone()[0]
  1200. conn.close()
  1201. check(after == before + 1 and empty_row == 0,
  1202. "redelivered chunks never duplicate rows; empty text stores nothing")
  1203. # Pushed entries: today's OutputEntry shape with the daemon's
  1204. # global entry ids — a redelivered results push stores its
  1205. # entries exactly once.
  1206. def command_rows(session):
  1207. conn = db()
  1208. try:
  1209. return conn.execute(
  1210. "select command_text, type from session_commands"
  1211. " where session_id = ? order by id", (session,)).fetchall()
  1212. finally:
  1213. conn.close()
  1214. entries = [{"id": 8101, "command": "sed -n 1p README.md", "output": "# demo",
  1215. "exit_code": 0, "type": "command", "timestamp": "2026-09-06T00:00:00Z"}]
  1216. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  1217. {"session_id": raw_session, "state": "working",
  1218. "entries": entries})
  1219. check(status == 200, "entries push accepted")
  1220. status, _ = saddle_api("POST", "/api/saddle/results", fake_token,
  1221. {"session_id": raw_session, "state": "working",
  1222. "entries": entries})
  1223. check(status == 200, "redelivered entries push accepted (at-least-once)")
  1224. stored = command_rows(raw_session)
  1225. check(len([r for r in stored if r[0] == "sed -n 1p README.md"]) == 1,
  1226. "redelivered entries store exactly once (entry-id cursor)")
  1227. # The retired daemon routes 404 (phase 2 deleted /output, /log
  1228. # and /start serving) — and the web app never calls them.
  1229. for method, path in (("GET", "/output?since=0"), ("GET", "/log?since=0"),
  1230. ("POST", "/start")):
  1231. request = urllib.request.Request(
  1232. "http://127.0.0.1:%d%s" % (fake.port, path),
  1233. data=b"{}" if method == "POST" else None, method=method)
  1234. try:
  1235. urllib.request.urlopen(request, timeout=5)
  1236. gone = False
  1237. except urllib.error.HTTPError as error:
  1238. gone = error.code == 404
  1239. check(gone, "the fake daemon's %s %s is deleted (404)" % (method, path))
  1240. with open(os.path.join(WORK, "app.log")) as handle:
  1241. app_log = handle.read()
  1242. check("Saddle output request" not in app_log and "Saddle log request" not in app_log,
  1243. "the web never polls /output or /log anymore")
  1244. print("== capabilities: tag-based routing ==")
  1245. saddles_state = slot_public(auth, "/saddles", "saddles")
  1246. act(auth, saddles_state["create_capability"], {"name": "gpu"})
  1247. act(auth, saddles_state["create_capability"], {"name": "bigmem"})
  1248. saddles_state = slot_public(auth, "/saddles", "saddles")
  1249. caps = {c["name"]: c for c in saddles_state["capabilities"]}
  1250. check(len(caps) == 2 and all(int(c["saddle_count"]) == 0 and int(c["task_count"]) == 0
  1251. for c in caps.values()),
  1252. "capabilities created and listed with zero usage")
  1253. def saddle_card(name):
  1254. rows = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  1255. if s["name"] == name]
  1256. return rows[0]
  1257. fake_card = saddle_card("fake-saddle")
  1258. check(fake_card["capabilities"] == [],
  1259. "saddle card starts with no capability chips")
  1260. # A task requiring 'gpu' waits while the only idle saddle lacks it;
  1261. # a later task without requirements still dispatches (skip-and-continue).
  1262. tasks_state = slot_public(auth, "/tasks", "tasks")
  1263. act(auth, tasks_state["create"], {
  1264. "project_id": "1",
  1265. "title": "GPU task",
  1266. "priority": "2",
  1267. })
  1268. gpu_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1269. if t["title"] == "GPU task"][0])
  1270. detail = task_state(auth, gpu_task)
  1271. act(auth, detail["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
  1272. detail = task_state(auth, gpu_task)
  1273. check([c["name"] for c in detail["capabilities"]] == ["gpu"],
  1274. "task page lists the required capability")
  1275. # The only idle saddle lacks the tag, so the queue flags the wait
  1276. # (read before any other task claims the idle saddle).
  1277. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1278. if int(t["id"]) == gpu_task][0]
  1279. check(queue["status"] == "queued" and queue["capabilities"] == ["gpu"]
  1280. and queue.get("waiting_for_saddle") is True,
  1281. "queue row shows the waiting task with its tag and wait flag")
  1282. act(auth, tasks_state["create"], {
  1283. "project_id": "1",
  1284. "title": "Any-saddle task",
  1285. "priority": "3",
  1286. })
  1287. any_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1288. if t["title"] == "Any-saddle task"][0])
  1289. check(wait_for(lambda: task_state(auth, any_task).get("status") != "queued" or False,
  1290. timeout=30, what="uncapable task dispatches") is not None,
  1291. "task without requirements dispatches past the waiting task")
  1292. def capability_wait_logged():
  1293. with open(os.path.join(WORK, "app.log")) as handle:
  1294. app_log = handle.read()
  1295. return "waits for a capable saddle" in app_log and "requires: gpu" in app_log
  1296. check(wait_for(capability_wait_logged, timeout=20,
  1297. what="capability wait log") is not None,
  1298. "dispatcher logs the capability wait")
  1299. # Attaching the tag to the saddle releases the task.
  1300. fake_card = saddle_card("fake-saddle")
  1301. act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["gpu"]["id"]))})
  1302. fake_card = saddle_card("fake-saddle")
  1303. check([c["name"] for c in fake_card["capabilities"]] == ["gpu"],
  1304. "saddle card shows the attached chip")
  1305. check(wait_task_done(auth, gpu_task, timeout=60) is not None,
  1306. "task dispatches once the saddle carries the required tag")
  1307. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1308. if int(t["id"]) == gpu_task][0]
  1309. check(queue.get("waiting_for_saddle") is not True,
  1310. "wait flag clears once dispatched")
  1311. # Multi-tag: gpu + bigmem required, only gpu carried → waits; adding
  1312. # bigmem completes the superset. Both requirements ride the create
  1313. # form (atomic — attaching them one at a time would race a tick
  1314. # into dispatching on the partial set).
  1315. act(auth, tasks_state["create"], {
  1316. "project_id": "1",
  1317. "title": "Big task",
  1318. "priority": "2",
  1319. "capability_id": [str(int(caps["gpu"]["id"])), str(int(caps["bigmem"]["id"]))],
  1320. })
  1321. big_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1322. if t["title"] == "Big task"][0])
  1323. check(wait_for(lambda: ([t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1324. if int(t["id"]) == big_task][0].get("waiting_for_saddle")
  1325. is True) or False, timeout=30, what="big task waits") is not None,
  1326. "task waits when the saddle's tags are not a superset")
  1327. fake_card = saddle_card("fake-saddle")
  1328. act(auth, fake_card["attach_capability"], {"capability_id": str(int(caps["bigmem"]["id"]))})
  1329. check(wait_task_done(auth, big_task, timeout=60) is not None,
  1330. "task dispatches once the superset is complete")
  1331. # Detach a requirement; delete a capability everywhere.
  1332. detail = task_state(auth, big_task)
  1333. act(auth, [c for c in detail["capabilities"] if c["name"] == "bigmem"][0]["detach"], {})
  1334. check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
  1335. "detached requirement disappears from the task")
  1336. bigmem_row = [c for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  1337. if c["name"] == "bigmem"][0]
  1338. act(auth, bigmem_row["delete"], {})
  1339. saddles_state = slot_public(auth, "/saddles", "saddles")
  1340. check(all(c["name"] != "bigmem" for c in saddles_state["capabilities"]),
  1341. "deleted capability gone from the catalogue")
  1342. check([c["name"] for c in task_state(auth, big_task)["capabilities"]] == ["gpu"],
  1343. "deleting a capability drops its task requirements")
  1344. check(all(c["name"] != "bigmem"
  1345. for c in saddle_card("fake-saddle")["capabilities"]),
  1346. "deleting a capability drops it from saddles")
  1347. print("== B4: required capabilities on the create form ==")
  1348. tasks_state = slot_public(auth, "/tasks", "tasks")
  1349. gpu_option = [o for o in tasks_state.get("capability_options", [])
  1350. if o["name"] == "gpu"][0]
  1351. check(len(tasks_state.get("capability_options", [])) >= 1,
  1352. "queue page offers capability options in the create form")
  1353. conn = db()
  1354. before = conn.execute("select count(*) from tasks").fetchone()[0]
  1355. conn.close()
  1356. act(auth, tasks_state["create"], {
  1357. "project_id": "1",
  1358. "title": "Bad cap task",
  1359. "capability_id": [gpu_option["id"], "999"],
  1360. })
  1361. conn = db()
  1362. after = conn.execute("select count(*) from tasks").fetchone()[0]
  1363. conn.close()
  1364. check(after == before, "unknown capability id rejects the create")
  1365. act(auth, tasks_state["create"], {
  1366. "project_id": "1",
  1367. "title": "Cap-on-create task",
  1368. "capability_id": [gpu_option["id"]],
  1369. })
  1370. cap_create = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1371. if t["title"] == "Cap-on-create task"][0])
  1372. detail = task_state(auth, cap_create)
  1373. check([c["name"] for c in detail["capabilities"]] == ["gpu"],
  1374. "capabilities from the create form land on the task")
  1375. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1376. if int(t["id"]) == cap_create][0]
  1377. check(queue["capabilities"] == ["gpu"],
  1378. "queue row shows the created task's requirements")
  1379. # The waiting flag must hold it back until the capable saddle frees
  1380. # up (the fake saddle still carries gpu from the routing checks).
  1381. check(queue.get("waiting_for_saddle") is not True,
  1382. "capable saddle available: no wait flag on the new task")
  1383. print("== B6: computed display state ==")
  1384. conn = db()
  1385. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  1386. " values (1, 'Parent', 'done', 5, strftime('%s','now'))")
  1387. parent6 = conn.execute("select max(id) from tasks").fetchone()[0]
  1388. # The post-mortem case: stored done WITH an unanswered review block.
  1389. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at, outcome)"
  1390. " values (?, ?, strftime('%s','now'), strftime('%s','now'), 'done')",
  1391. (parent6, fake_id))
  1392. session6 = conn.execute("select max(id) from sessions").fetchone()[0]
  1393. conn.commit()
  1394. conn.close()
  1395. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1396. {"type": "block", "session_id": session6, "data": {
  1397. "question": "Review the final diff", "type": "review"}})
  1398. detail = task_state(auth, parent6)
  1399. check(detail.get("display") == "review requested" and detail.get("terminal") is True,
  1400. "unanswered review overrides the stored done status (terminal flag keeps stored truth)")
  1401. queue = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1402. if int(t["id"]) == parent6][0]
  1403. check(queue["display"] == "review requested",
  1404. "queue row shows the computed review-requested display")
  1405. # Delegation: with the review answered, open subtasks delegate.
  1406. conn = db()
  1407. conn.execute("update task_blocks set answered_at = strftime('%s','now'),"
  1408. " answer = 'Human approved the review' where task_id = ?", (parent6,))
  1409. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  1410. " created_at) values (1, 'Sub A', 'working', 5, ?, strftime('%s','now'))",
  1411. (parent6,))
  1412. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  1413. " created_at) values (1, 'Sub B', 'done', 5, ?, strftime('%s','now'))",
  1414. (parent6,))
  1415. conn.commit()
  1416. conn.close()
  1417. detail = task_state(auth, parent6)
  1418. check(detail.get("display") == "delegating 1/2" and detail.get("subtask_open") == 1,
  1419. "open subtasks show the delegating breakdown (%s)" % detail.get("display"))
  1420. # Flags track the stored status once the display falls through
  1421. # (subtasks terminal first — delegating leads while any are open).
  1422. conn = db()
  1423. conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
  1424. " where id = ?", (parent6,))
  1425. conn.execute("update tasks set status = 'failed', completed_at = strftime('%s','now')"
  1426. " where title = 'Sub A'")
  1427. conn.commit()
  1428. conn.close()
  1429. detail = task_state(auth, parent6)
  1430. check(detail.get("display") == "failed" and detail.get("failed") is True
  1431. and detail.get("terminal") is True,
  1432. "failed task with answered review and terminal subtasks shows failed + terminal flags")
  1433. print("== retry + resume ==")
  1434. # A daemon-side failure fails the task with its explanation.
  1435. fake.fail_next = True
  1436. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1437. "project_id": "1",
  1438. "title": "Flaky task",
  1439. "priority": "2",
  1440. })
  1441. flaky = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1442. if t["title"] == "Flaky task"][0])
  1443. check(wait_for(lambda: task_state(auth, flaky).get("status") == "failed" or False,
  1444. timeout=60, what="flaky task fails") is not None,
  1445. "daemon failure fails the task")
  1446. detail = task_state(auth, flaky)
  1447. check("bash -n rejected" in (detail.get("summary") or ""),
  1448. "failure summary shows the daemon's explanation")
  1449. # Retry: summary moves into a comment and clears, task requeues,
  1450. # history stays, and the re-dispatch restores the workspace.
  1451. comments_before = len(detail.get("comments", []))
  1452. progress_before = len(detail.get("progress", []))
  1453. act(auth, detail["retry"], {})
  1454. detail = task_state(auth, flaky)
  1455. check(detail["status"] == "queued" and (detail.get("summary") or "") == "",
  1456. "retry requeues the task and clears the stale summary")
  1457. check(any("bash -n rejected" in c["body"] for c in detail["comments"]),
  1458. "failure explanation kept as a bot comment")
  1459. check(any(c["body"] == "Task retried by user" for c in detail["comments"]),
  1460. "retry lands its audit comment")
  1461. check(len(detail["comments"]) >= comments_before + 1,
  1462. "retry keeps the comment history")
  1463. pushes_before = len([p for p in fake.start_payloads
  1464. if int(p.get("task", {}).get("id") or 0) == flaky])
  1465. check(wait_for(lambda: len([p for p in fake.start_payloads
  1466. if int(p.get("task", {}).get("id") or 0) == flaky])
  1467. > pushes_before or False, timeout=60,
  1468. what="flaky task re-dispatched") is not None,
  1469. "retried task dispatches again")
  1470. # The resumed arc surfaces on the next /status read — in the
  1471. # push world that is the operator's manual refresh.
  1472. act(auth, saddle_card("fake-saddle")["refresh"], {})
  1473. check(wait_task_done(auth, flaky, timeout=60) is not None,
  1474. "retried task completes on the restored workspace")
  1475. detail = task_state(auth, flaky)
  1476. check(any("restored the session context" in p["summary"]
  1477. for p in detail.get("progress", [])),
  1478. "daemon resume reported on the first status after the restore")
  1479. check(len(detail["progress"]) > progress_before,
  1480. "resume note appended to the progress timeline")
  1481. # Manual resume: a task stuck working on an offline saddle.
  1482. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1483. "project_id": "1",
  1484. "title": "Stuck task",
  1485. "priority": "2",
  1486. })
  1487. stuck = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1488. if t["title"] == "Stuck task"][0])
  1489. check(wait_for(lambda: task_state(auth, stuck).get("status") == "working" or False,
  1490. timeout=60, what="stuck task starts working") is not None,
  1491. "stuck task reaches working")
  1492. conn = db()
  1493. conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
  1494. conn.commit()
  1495. conn.close()
  1496. detail = task_state(auth, stuck)
  1497. check(detail.get("resume_available") is True,
  1498. "task page offers resume while its saddle is offline")
  1499. act(auth, detail["resume"], {})
  1500. detail = task_state(auth, stuck)
  1501. check(detail["status"] == "queued",
  1502. "manual resume requeues the stuck task")
  1503. check(any(c["body"] == "Saddle became unreachable; task requeued"
  1504. for c in detail["comments"]),
  1505. "manual resume lands the interruption comment")
  1506. check(all(s["status"] == "offline"
  1507. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  1508. "saddle stays offline after the manual requeue")
  1509. # Bringing the saddle back lets the requeued task resume and finish.
  1510. fake.state = "idle"
  1511. fake_card = saddle_card("fake-saddle")
  1512. act(auth, fake_card["refresh"], {})
  1513. check(any(s["status"] == "idle"
  1514. for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  1515. if s["name"] == "fake-saddle"),
  1516. "refreshed saddle returns idle")
  1517. # The refreshed daemon is idle-awake: it asks (the real loop's
  1518. # own ask; driven directly for determinism).
  1519. status, _ = fake.ask_now()
  1520. check(status == 200, "the refreshed daemon's ask claims the requeued task")
  1521. check(wait_task_done(auth, stuck, timeout=60) is not None,
  1522. "requeued task resumes on the refreshed saddle and completes")
  1523. print("== rejected entries (non-fatal) ==")
  1524. # Ingest API: a rejected entry stores on the session and streams
  1525. # through SSE; the task is untouched.
  1526. status, _ = saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1527. {"type": "output", "session_id": session_id, "data": {
  1528. "command": "bash -n script.sh",
  1529. "output": "syntax error near line 3 (recoverable)",
  1530. "exit_code": 1,
  1531. "type": "rejected"}})
  1532. check(status == 200, "rejected entry accepted by the ingest API")
  1533. request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
  1534. with urllib.request.urlopen(request, timeout=15) as stream:
  1535. rejected_event = None
  1536. deadline = time.monotonic() + 8
  1537. while time.monotonic() < deadline and rejected_event is None:
  1538. line = stream.readline().decode().strip()
  1539. if not line.startswith("data: ") or line == "data: ping":
  1540. continue
  1541. event = json.loads(line[6:])
  1542. if event.get("type") == "rejected":
  1543. rejected_event = event
  1544. check(rejected_event is not None
  1545. and "syntax error near line 3" in (rejected_event.get("output") or ""),
  1546. "terminal stream carries rejected entries with their reason")
  1547. # The page template renders them with the distinct non-fatal style.
  1548. status, page = http("GET", "/tasks/%d/terminal" % stuck, timeout=15)
  1549. check(status == 200 and "term-rejected" in page,
  1550. "terminal page renders rejected entries in their own style")
  1551. print("== B8: ANSI SGR in the terminal view ==")
  1552. # Byte-exact SGR delivery: the daemon's coloured chunks flow through
  1553. # the store + SSE untouched (the client renderer styles them).
  1554. coloured = "\x1b[1;32mbold green\x1b[0m then \x1b[38;5;196m256-red\x1b[0m\n"
  1555. conn = db()
  1556. conn.execute("insert into session_raw_log (session_id, seq, chunk)"
  1557. " values (?, 900, ?)", (session_id, coloured))
  1558. conn.commit()
  1559. conn.close()
  1560. request = urllib.request.Request(WEB + "/terminal-stream/%d" % session_id)
  1561. with urllib.request.urlopen(request, timeout=15) as stream:
  1562. coloured_event = None
  1563. deadline = time.monotonic() + 8
  1564. while time.monotonic() < deadline and coloured_event is None:
  1565. line = stream.readline().decode().strip()
  1566. if not line.startswith("data: ") or line == "data: ping":
  1567. continue
  1568. event = json.loads(line[6:])
  1569. if event.get("type") == "raw" and "\x1b[1;32m" in (event.get("text") or ""):
  1570. coloured_event = event
  1571. check(coloured_event is not None
  1572. and "\x1b[38;5;196m" in coloured_event["text"],
  1573. "SGR sequences stream byte-exact to the client renderer")
  1574. status, page = http("GET", "/tasks/%d/terminal" % stuck, timeout=15)
  1575. check(status == 200 and "makeAnsiRenderer" in page and "colour256" in page,
  1576. "terminal page embeds the ANSI renderer (SGR + 256-colour palette)")
  1577. # Dispatch path: a rejected entry in a round's results push flows
  1578. # into the audit trail without failing (or finishing) the task.
  1579. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1580. "project_id": "1",
  1581. "title": "Rejected ride",
  1582. "priority": "2",
  1583. })
  1584. rejected_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1585. if t["title"] == "Rejected ride"][0])
  1586. check(wait_for(lambda: task_state(auth, rejected_task).get("status") == "working"
  1587. or False, timeout=60, what="rejected-ride working") is not None,
  1588. "rejected-ride task reaches working")
  1589. fake.queue_output("rejected", "echo 'too long' | saddle",
  1590. "max_tokens exceeded (recoverable)")
  1591. check(wait_task_done(auth, rejected_task, timeout=60) is not None,
  1592. "task completes normally past rejected entries")
  1593. conn = db()
  1594. stored = conn.execute(
  1595. "select count(*) from session_commands sc join sessions s on s.id = sc.session_id"
  1596. " where s.task_id = ? and sc.type = 'rejected'", (rejected_task,)).fetchone()[0]
  1597. conn.close()
  1598. check(stored >= 1,
  1599. "rejected entries stored in the audit trail (%d rows)" % stored)
  1600. check(task_state(auth, rejected_task)["status"] == "done"
  1601. and "fake daemon finished" in (task_state(auth, rejected_task).get("summary") or ""),
  1602. "rejected entries never fail the task")
  1603. print("== B7: suspended state + per-subtask resume ==")
  1604. fake.suspend_on_title = "Suspend parent"
  1605. pushes_before = len(fake.start_payloads)
  1606. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1607. "project_id": "1",
  1608. "title": "Suspend parent",
  1609. "priority": "2",
  1610. })
  1611. parent7 = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1612. if t["title"] == "Suspend parent"][0])
  1613. check(wait_for(lambda: task_state(auth, parent7).get("status") == "suspended"
  1614. or False, timeout=60, what="parent parks as suspended") is not None,
  1615. "daemon 'suspended' state parks the task")
  1616. detail = task_state(auth, parent7)
  1617. check(detail.get("display") == "waiting on subtasks",
  1618. "suspended task displays as waiting on subtasks")
  1619. check(any(s["status"] == "idle"
  1620. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  1621. "suspension frees the saddle")
  1622. first_push = fake.start_payloads[pushes_before]
  1623. check(first_push.get("subtasks") == [],
  1624. "the /start payload carries the subtasks array (empty here)")
  1625. # Two subtask events land while parked (staged, not dispatched):
  1626. # both must coalesce into one resume with a full report.
  1627. conn = db()
  1628. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  1629. " summary, completed_at, created_at) values (1, 'Child one', 'done', 5, ?,"
  1630. " 'built and verified the flange', strftime('%s','now','+1 second'),"
  1631. " strftime('%s','now'))", (parent7,))
  1632. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  1633. " summary, completed_at, created_at) values (1, 'Child two', 'failed', 5, ?,"
  1634. " 'the harness refused to start', strftime('%s','now','+1 second'),"
  1635. " strftime('%s','now'))", (parent7,))
  1636. conn.commit()
  1637. conn.close()
  1638. def parent_push_count():
  1639. return len([p for p in fake.start_payloads
  1640. if int(p.get("task", {}).get("id") or 0) == parent7])
  1641. check(wait_for(lambda: parent_push_count() >= 2 or False,
  1642. timeout=45, what="suspended parent resumes") is not None,
  1643. "subtask events resume the suspended parent")
  1644. parent_pushes = [p for p in fake.start_payloads
  1645. if int(p.get("task", {}).get("id") or 0) == parent7]
  1646. check(len(parent_pushes) >= 2,
  1647. "the parent's resume is a fresh /start for the same task")
  1648. resume_push = parent_pushes[-1]
  1649. notice = resume_push.get("resume_notice") or ""
  1650. check("Child one" in notice and "completed" in notice
  1651. and "built and verified the flange" in notice,
  1652. "resume notice reports the finished subtask with its summary")
  1653. check("Child two" in notice and "failed" in notice
  1654. and "the harness refused to start" in notice,
  1655. "coalesced: the second event rides the same resume notice")
  1656. subtask_entries = {(s["title"], s["status"]) for s in resume_push.get("subtasks", [])}
  1657. check(("Child one", "done") in subtask_entries
  1658. and ("Child two", "failed") in subtask_entries,
  1659. "subtasks array carries the children's states at dispatch time")
  1660. # The resumed arc surfaces on a /status read while the resumed
  1661. # round still runs — the operator's manual refresh in the push
  1662. # world.
  1663. act(auth, saddle_card("fake-saddle")["refresh"], {})
  1664. check(wait_task_done(auth, parent7, timeout=60) is not None,
  1665. "resumed parent completes")
  1666. detail = task_state(auth, parent7)
  1667. check(any("restored the session context" in p["summary"]
  1668. for p in detail.get("progress", [])),
  1669. "resumed flag reported on the first status after the restore")
  1670. print("== R3 B1: subtask status push ==")
  1671. # A parked (suspended) parent gives delivery a stable open session.
  1672. fake.suspend_on_title = "Push parent"
  1673. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1674. "project_id": "1",
  1675. "title": "Push parent",
  1676. "priority": "2",
  1677. })
  1678. push_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1679. if t["title"] == "Push parent"][0])
  1680. check(wait_for(lambda: task_state(auth, push_parent).get("status") == "suspended"
  1681. or False, timeout=60, what="push parent parks") is not None,
  1682. "push parent parks with an open session")
  1683. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1684. "project_id": "1",
  1685. "title": "Push child",
  1686. "parent_task_id": str(push_parent),
  1687. })
  1688. push_child = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1689. if t["title"] == "Push child"][0])
  1690. def events_for(subtask_id):
  1691. return [e for e in fake.subtask_status_events
  1692. if int(e.get("subtask_id") or 0) == subtask_id]
  1693. check(wait_for(lambda: len(events_for(push_child)) > 0 or False,
  1694. timeout=30, what="queued subtask event") is not None,
  1695. "subtask creation pushes a queued status event")
  1696. first_event = events_for(push_child)[0]
  1697. check(int(first_event.get("parent_task_id") or 0) == push_parent
  1698. and first_event.get("title") == "Push child"
  1699. and first_event.get("status") in ("queued", "working"),
  1700. "status event carries parent id, title and a wire status (%s)"
  1701. % first_event.get("status"))
  1702. # Real lifecycle: the child dispatches (working) and completes (done
  1703. # with summary) — the parent's daemon hears every transition.
  1704. check(wait_for(lambda: any(e.get("status") == "done"
  1705. for e in events_for(push_child)) or False,
  1706. timeout=90, what="done subtask event") is not None,
  1707. "subtask completion pushes the done transition")
  1708. check(any(e.get("status") == "working" for e in events_for(push_child)),
  1709. "dispatch pushes a working status event")
  1710. # Best-effort retry: while the daemon 500s, events stay queued and
  1711. # land once it recovers. The parent's resumed run re-parks (the
  1712. # ask-flow resume races this section otherwise: its completion
  1713. # would close the parked session the events are delivered to).
  1714. fake.fail_subtask_status = True
  1715. fake.suspend_on_title = "Push parent"
  1716. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1717. "project_id": "1",
  1718. "title": "Push child two",
  1719. "parent_task_id": str(push_parent),
  1720. })
  1721. push_child_two = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1722. if t["title"] == "Push child two"][0])
  1723. time.sleep(7)
  1724. check(len(events_for(push_child_two)) == 0,
  1725. "failed pushes stay queued (no event delivered)")
  1726. fake.fail_subtask_status = False
  1727. check(wait_for(lambda: len(events_for(push_child_two)) > 0 or False,
  1728. timeout=30, what="retried subtask event") is not None,
  1729. "recovered pushes deliver on a later tick")
  1730. print("== R4 B1: synchronous subtask callback ==")
  1731. # Plan 10: the subtask callback is per-saddle-token authenticated.
  1732. r4_token = fake_token
  1733. # Parent with a live parked session (suspended) hosts the callback.
  1734. fake.suspend_on_title = "Callback parent"
  1735. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1736. "project_id": "1",
  1737. "title": "Callback parent",
  1738. "priority": "2",
  1739. })
  1740. cb_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1741. if t["title"] == "Callback parent"][0])
  1742. check(wait_for(lambda: task_state(auth, cb_parent).get("status") == "suspended"
  1743. or False, timeout=60, what="callback parent parks") is not None,
  1744. "callback parent parks")
  1745. status, _ = saddle_api("POST", "/api/saddle/subtask", None,
  1746. {"parent_task_id": cb_parent, "title": "no token"})
  1747. check(status == 401, "subtask callback rejects a missing token")
  1748. anchor = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1749. if t["title"] == "Rules dispatch"][0])
  1750. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  1751. "parent_task_id": cb_parent,
  1752. "title": "Wire the flange",
  1753. "description": "torque to spec",
  1754. "depends_on": [anchor],
  1755. "session_id": 1,
  1756. })
  1757. check(status == 200 and body.get("ok") is True and int(body.get("task_id") or 0) > 0,
  1758. "callback creates the subtask synchronously")
  1759. subtask_id = int(body.get("task_id"))
  1760. detail = task_state(auth, subtask_id)
  1761. check(detail["title"] == "Wire the flange" and int(detail["project_id"]) == 1
  1762. and int(detail["parent_task_id"]) == cb_parent,
  1763. "subtask row inherits the parent's project and linkage")
  1764. check(detail.get("task_branch") == task_state(auth, cb_parent).get("task_branch"),
  1765. "subtask shares the parent's task branch")
  1766. conn = db()
  1767. dep_rows = conn.execute(
  1768. "select depends_on_task_id from task_dependencies where task_id = ?",
  1769. (subtask_id,)).fetchall()
  1770. conn.close()
  1771. check([r[0] for r in dep_rows] == [anchor],
  1772. "depends_on wired as task_dependencies")
  1773. # Idempotency + entry-poll fallback dedup.
  1774. status, body2 = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  1775. "parent_task_id": cb_parent, "title": "Wire the flange"})
  1776. check(status == 200 and int(body2.get("task_id") or 0) == subtask_id,
  1777. "retried callback returns the same task id")
  1778. conn = db()
  1779. before = conn.execute("select count(*) from tasks where parent_task_id = ?",
  1780. (cb_parent,)).fetchone()[0]
  1781. conn.close()
  1782. conn = db()
  1783. cb_session = conn.execute(
  1784. "select id from sessions where task_id = ? and ended_at is null",
  1785. (cb_parent,)).fetchone()[0]
  1786. conn.close()
  1787. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1788. {"type": "subtask", "session_id": cb_session, "data": {
  1789. "title": "Wire the flange", "description": "again"}})
  1790. conn = db()
  1791. after = conn.execute("select count(*) from tasks where parent_task_id = ?",
  1792. (cb_parent,)).fetchone()[0]
  1793. timeline = conn.execute(
  1794. "select count(*) from session_commands where session_id = ?"
  1795. " and type = 'subtask'", (cb_session,)).fetchone()[0]
  1796. conn.close()
  1797. check(after == before,
  1798. "entry-poll fallback dedupes against the callback's row")
  1799. check(timeline >= 1, "timeline entry recorded on the parent's session")
  1800. # Validation: unknown parent, unknown depends_on, ancestor cycle.
  1801. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  1802. {"parent_task_id": 99999, "title": "ghost"})
  1803. check(status == 400, "unknown parent rejected")
  1804. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  1805. {"parent_task_id": cb_parent, "title": "bad dep",
  1806. "depends_on": [99999]})
  1807. check(status == 400 and "unknown task" in body.get("error", ""),
  1808. "unknown depends_on id rejected with a clear error")
  1809. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  1810. {"parent_task_id": cb_parent, "title": "cycle dep",
  1811. "depends_on": [cb_parent]})
  1812. check(status == 400 and "cycle" in body.get("error", ""),
  1813. "depending on an ancestor rejected as a cycle")
  1814. print("== R4 B2: subtask auto-approve ==")
  1815. # Settle earlier stragglers first: the round-3/B1 suspended parents
  1816. # resume (a fresh /start) once their last subtask completes — those
  1817. # pushes would consume the one-shot suspend flag below.
  1818. check(wait_task_done(auth, push_parent, timeout=90) is not None
  1819. and wait_task_done(auth, push_child_two, timeout=90) is not None
  1820. and wait_task_done(auth, cb_parent, timeout=90) is not None,
  1821. "earlier suspended parents settle before the auto-approve run")
  1822. # Parent parks; its toggle then covers all descendants. Children are
  1823. # created with an unmet 'gpu' requirement so they stay queued (and
  1824. # untouched) until the requirement is lifted for the final-resume leg.
  1825. fake.suspend_on_title = "Auto parent"
  1826. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  1827. "project_id": "1",
  1828. "title": "Auto parent",
  1829. "priority": "2",
  1830. })
  1831. auto_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  1832. if t["title"] == "Auto parent"][0])
  1833. check(wait_for(lambda: task_state(auth, auto_parent).get("status") == "suspended"
  1834. or False, timeout=60, what="auto parent parks") is not None,
  1835. "auto parent parks")
  1836. detail = task_state(auth, auto_parent)
  1837. check(detail.get("auto_approve") is False,
  1838. "auto-approve defaults to off")
  1839. act(auth, detail["toggle_auto_approve"], {})
  1840. check(task_state(auth, auto_parent).get("auto_approve") is True,
  1841. "toggle turns auto-approve on")
  1842. gpu_option = int([o["id"] for o in slot_public(auth, "/tasks", "tasks")["capability_options"]
  1843. if o["name"] == "gpu"][0])
  1844. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  1845. {"parent_task_id": auto_parent,
  1846. "title": "Auto child", "depends_on": []})
  1847. auto_child = int(body["task_id"])
  1848. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token,
  1849. {"parent_task_id": auto_child, "title": "Auto grandchild"})
  1850. auto_grand = int(body["task_id"])
  1851. act(auth, task_state(auth, auto_child)["attach_capability"],
  1852. {"capability_id": str(gpu_option)})
  1853. act(auth, task_state(auth, auto_grand)["attach_capability"],
  1854. {"capability_id": str(gpu_option)})
  1855. # Transitive coverage: the GRANDCHILD's review is auto-answered
  1856. # (any ancestor set), answered-by 'auto', timeline entry recorded.
  1857. conn = db()
  1858. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  1859. " values (?, ?, strftime('%s','now'))", (auto_grand, fake_id))
  1860. conn.commit()
  1861. grand_session = conn.execute("select max(id) from sessions").fetchone()[0]
  1862. conn.close()
  1863. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1864. {"type": "review", "session_id": grand_session, "data": {
  1865. "summary": "the flange is wired"}})
  1866. grand_detail = task_state(auth, auto_grand)
  1867. grand_review = [b for b in grand_detail["blocks"] if b["block_type"] == "review"][0]
  1868. check(grand_review.get("answered") is True
  1869. and grand_review.get("answer") == "Auto-approved (subtask auto-approve is on)"
  1870. and grand_review.get("answered_by") == "auto",
  1871. "covered grandchild review auto-answered as 'auto'")
  1872. conn = db()
  1873. timeline_auto = conn.execute(
  1874. "select count(*) from session_commands where session_id = ?"
  1875. " and type = 'auto-approve'", (grand_session,)).fetchone()[0]
  1876. conn.close()
  1877. check(timeline_auto >= 1, "auto action visible in the timeline")
  1878. # Questions and RFCs still block for a human.
  1879. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1880. {"type": "block", "session_id": grand_session, "data": {
  1881. "question": "Which torque spec?", "type": "question"}})
  1882. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1883. {"type": "block", "session_id": grand_session, "data": {
  1884. "question": "RFC: approve the design", "type": "rfc",
  1885. "body": "# Design"}})
  1886. grand_detail = task_state(auth, auto_grand)
  1887. check(all(b.get("answered") is not True for b in grand_detail["blocks"]
  1888. if b["block_type"] in ("question", "rfc")),
  1889. "questions and RFCs are never auto-answered")
  1890. # The toggle covers DESCENDANTS only: the parent's own review waits.
  1891. conn = db()
  1892. conn.execute("insert into sessions (task_id, saddle_id, started_at, ended_at,"
  1893. " outcome) values (?, ?, strftime('%s','now'), strftime('%s','now'),"
  1894. " 'done')", (auto_parent, fake_id))
  1895. conn.commit()
  1896. parent_session = conn.execute("select max(id) from sessions").fetchone()[0]
  1897. conn.close()
  1898. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  1899. {"type": "review", "session_id": parent_session, "data": {
  1900. "summary": "the whole change"}})
  1901. parent_detail = task_state(auth, auto_parent)
  1902. parent_review = [b for b in parent_detail["blocks"] if b["block_type"] == "review"][0]
  1903. check(parent_review.get("answered") is not True,
  1904. "the top-level parent's own review is never auto-approved")
  1905. # All descendants terminal → the suspended parent resumes with the
  1906. # final-review instruction (lift the capability requirement first).
  1907. act(auth, task_state(auth, auto_child)["capabilities"][0]["detach"], {})
  1908. act(auth, task_state(auth, auto_grand)["capabilities"][0]["detach"], {})
  1909. def pushes_for(task_id):
  1910. return len([p for p in fake.start_payloads
  1911. if int(p.get("task", {}).get("id") or 0) == task_id])
  1912. before_pushes = pushes_for(auto_parent)
  1913. check(wait_for(lambda: pushes_for(auto_parent) > before_pushes or False,
  1914. timeout=90, what="auto parent resumes") is not None,
  1915. "terminal subtasks resume the suspended parent")
  1916. notice = ""
  1917. for p in fake.start_payloads:
  1918. if int(p.get("task", {}).get("id") or 0) == auto_parent:
  1919. notice = p.get("resume_notice") or ""
  1920. check("All subtasks are complete" in notice
  1921. and "ONE final review" in notice and "saddle review" in notice,
  1922. "resume notice ends with the final-review instruction")
  1923. print("== phase 1: ask/allocation — token auth + concurrent race ==")
  1924. # Plan 10 phase 1. Registration mints per-saddle tokens; the
  1925. # ask endpoint authenticates by them; concurrent asks race to
  1926. # exactly one winner through the allocation transaction.
  1927. status, _ = saddle_api("POST", "/api/saddle/next-task", None)
  1928. check(status == 401, "next-task rejects a missing token")
  1929. status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-bogus-token")
  1930. check(status == 401, "next-task rejects a wrong token")
  1931. status, body = saddle_api("POST", "/api/saddle/next-task", fake_token)
  1932. check(status == 204, "next-task answers 204 (nothing offerable — daemon sleeps)")
  1933. # Two more racers: real daemons on their own ports; fake2 reports
  1934. # its ssh key at /register so the key lands on the row directly.
  1935. fake2 = FakeSaddle()
  1936. fake2.ssh_public_key = "ssh-ed25519 AAAAFake2RacerKey saddle-2@fedora"
  1937. fake2.serve(port=free_port())
  1938. fake3 = FakeSaddle()
  1939. fake3.serve(port=free_port())
  1940. saddles_state = slot_public(auth, "/saddles", "saddles")
  1941. act(auth, saddles_state["register"], {"name": "racer-two",
  1942. "host": "127.0.0.1:%d" % fake2.port})
  1943. act(auth, saddles_state["register"], {"name": "racer-three",
  1944. "host": "127.0.0.1:%d" % fake3.port})
  1945. check(wait_for(lambda: len(fake2.register_log) > 0 and len(fake3.register_log) > 0
  1946. or False, timeout=15, what="racers registered") is not None,
  1947. "both racers received /register")
  1948. check(wait_for(lambda: "AAAAFake2RacerKey" in
  1949. [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  1950. if s["name"] == "racer-two"][0].get("ssh_public_key", "") or False,
  1951. timeout=15, what="register-reported key") is not None,
  1952. "the ssh key reported at /register lands on the saddle row")
  1953. racer_cards = {}
  1954. for name, daemon in (("fake-saddle", fake), ("racer-two", fake2), ("racer-three", fake3)):
  1955. card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  1956. if s["name"] == name][0]
  1957. racer_cards[daemon] = {"id": int(card["id"]), "token": card["token"]}
  1958. # Revocation: a registered saddle's token works until the saddle
  1959. # is removed, then nothing authorises its calls.
  1960. fake4 = FakeSaddle()
  1961. fake4.ask_on_register = False # a pre-flip daemon: never asks
  1962. fake4.serve(port=free_port())
  1963. saddles_state = slot_public(auth, "/saddles", "saddles")
  1964. act(auth, saddles_state["register"], {"name": "old-daemon",
  1965. "host": "127.0.0.1:%d" % fake4.port})
  1966. old_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  1967. if s["name"] == "old-daemon"][0]
  1968. status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
  1969. check(status in (200, 204),
  1970. "the old daemon's fresh token authenticates next-task")
  1971. status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
  1972. {"project": "demo", "body": "from the old daemon"})
  1973. check(status == 200, "note accepts any registered saddle's token")
  1974. act(auth, old_card["unregister"], {})
  1975. status, _ = saddle_api("POST", "/api/saddle/next-task", old_card["token"])
  1976. check(status == 401, "removing the saddle revokes its token")
  1977. status, _ = saddle_api("POST", "/api/saddle/note", old_card["token"],
  1978. {"project": "demo", "body": "should fail"})
  1979. check(status == 401, "revocation covers the note endpoint too")
  1980. # Deterministic race: settle the queue first (the auto-approve
  1981. # stragglers must finish — their parked resumes stay offerable
  1982. # otherwise and would steal the race's asks), freeze every
  1983. # daemon's self-ask triggers, put all three to sleep (204),
  1984. # then fire one task worth of concurrent asks.
  1985. check(wait_task_done(auth, auto_child, timeout=90) is not None
  1986. and wait_task_done(auth, auto_grand, timeout=90) is not None
  1987. and wait_task_done(auth, auto_parent, timeout=90) is not None,
  1988. "the auto-approve stragglers settle before the race")
  1989. for daemon in (fake, fake2, fake3):
  1990. daemon.ask_on_register = False
  1991. daemon.ask_on_wake = False
  1992. daemon.ask_on_terminal = False
  1993. daemon.ask_on_idle_status = False
  1994. time.sleep(1.5) # let any in-flight triggered asks land
  1995. def queued_count():
  1996. conn = db()
  1997. try:
  1998. return conn.execute(
  1999. "select count(*) from tasks where status = 'queued'").fetchone()[0]
  2000. finally:
  2001. conn.close()
  2002. check(wait_for(lambda: queued_count() == 0 or False,
  2003. timeout=60, what="queue settles before the race") is not None,
  2004. "the queue settles before the race")
  2005. sleep_statuses = [daemon.ask_now()[0] for daemon in (fake, fake2, fake3)]
  2006. check(sleep_statuses == [204, 204, 204],
  2007. "all three racers sleep (204) before the task exists")
  2008. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2009. "project_id": "1",
  2010. "title": "Race task",
  2011. "priority": "2",
  2012. })
  2013. race_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2014. if t["title"] == "Race task"][0])
  2015. race_results = []
  2016. def race_ask(daemon):
  2017. # A real racing daemon accepts the assignment it wins — the
  2018. # manual ask path does exactly that on a 200.
  2019. race_results.append(daemon.ask_now())
  2020. threads = [threading.Thread(target=race_ask, args=(daemon,))
  2021. for daemon in (fake, fake2, fake3)]
  2022. for thread in threads:
  2023. thread.start()
  2024. for thread in threads:
  2025. thread.join()
  2026. race_statuses = sorted(r[0] for r in race_results)
  2027. check(race_statuses == [200, 204, 204],
  2028. "concurrent asks race to exactly one winner (%s)" % race_statuses)
  2029. winner = [r for r in race_results if r[0] == 200][0][1]
  2030. check(int(winner["task"]["id"]) == race_task, "the winner's payload carries the task")
  2031. check(int(winner["session_id"]) > 0, "the assignment carries the session id")
  2032. conn = db()
  2033. race_sessions = conn.execute(
  2034. "select count(*) from sessions where task_id = ?", (race_task,)).fetchone()[0]
  2035. conn.close()
  2036. check(race_sessions == 1,
  2037. "exactly one session row — the transaction serialised the claims (%d)"
  2038. % race_sessions)
  2039. check(wait_task_done(auth, race_task, timeout=60) is not None,
  2040. "the raced task completes on the winning daemon")
  2041. for daemon in (fake, fake2, fake3):
  2042. daemon.ask_on_register = True
  2043. daemon.ask_on_wake = True
  2044. daemon.ask_on_terminal = True
  2045. daemon.ask_on_idle_status = True
  2046. # Re-establish sleeping explicitly: the winner's post-race
  2047. # re-ask timing is not deterministic, and the affinity section
  2048. # below needs every racer provably asleep. Any stray offerable
  2049. # drains into these asks first.
  2050. def all_asleep():
  2051. return all(daemon.ask_now()[0] == 204 for daemon in (fake, fake2, fake3))
  2052. check(wait_for(lambda: all_asleep() or False, timeout=30,
  2053. what="racers re-sleep after the race") is not None,
  2054. "all racers re-sleep after the race")
  2055. print("== phase 1: workspace affinity pins parked tasks ==")
  2056. # A suspended parent parked on racer-two may be resumed ONLY by
  2057. # racer-two: a tag-capable non-affine saddle asking gets 204, and
  2058. # the wake targets the affine saddle. Only racer-two carries the
  2059. # tag at dispatch time (the race's winner is nondeterministic, so
  2060. # the park must not depend on which racers are free); racer-three
  2061. # gains it afterwards to prove the pinning is by workspace, not
  2062. # tags.
  2063. saddles_state = slot_public(auth, "/saddles", "saddles")
  2064. act(auth, saddles_state["create_capability"], {"name": "affinity-tag"})
  2065. affinity_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  2066. if c["name"] == "affinity-tag"][0])
  2067. racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2068. if s["name"] == "racer-two"][0]
  2069. racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2070. if s["name"] == "racer-three"][0]
  2071. act(auth, racer2_card["attach_capability"], {"capability_id": str(affinity_cap)})
  2072. fake2.suspend_on_title = "Affinity parent"
  2073. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2074. "project_id": "1",
  2075. "title": "Affinity parent",
  2076. "priority": "2",
  2077. "capability_id": [str(affinity_cap)],
  2078. })
  2079. affinity_parent = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2080. if t["title"] == "Affinity parent"][0])
  2081. check(wait_for(lambda: task_state(auth, affinity_parent).get("status") == "suspended"
  2082. or False, timeout=60, what="affinity parent parks") is not None,
  2083. "affinity parent parks (suspended) on the tag-carrying racer")
  2084. conn = db()
  2085. affinity_session = conn.execute(
  2086. "select saddle_id from sessions where task_id = ? and ended_at is null",
  2087. (affinity_parent,)).fetchone()[0]
  2088. conn.close()
  2089. check(affinity_session == int(racer2_card["id"]),
  2090. "the parked workspace sits on racer-two (the only tag-carrier)")
  2091. # Now racer-three carries the tag too — the pinning is by
  2092. # workspace, not tags.
  2093. # A terminal subtask makes the parked task resume-offerable —
  2094. # affine to racer-two only. The park is observed the moment the
  2095. # suspended push lands (no poller lag to compensate), so the
  2096. # child is inserted only after racer-two's ask triggers are
  2097. # frozen and the wake measurement armed — its completion stamped
  2098. # one second ahead so completed_at > suspended_at holds.
  2099. fake2.ask_on_wake = False
  2100. fake2.ask_on_idle_status = False
  2101. wakes_before_2 = len(fake2.wake_log)
  2102. time.sleep(1.2) # any in-flight park-time trigger ask lands (204)
  2103. conn = db()
  2104. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  2105. " summary, completed_at, created_at) values (1, 'Affinity child', 'done',"
  2106. " 5, ?, 'affinity child finished', strftime('%s','now','+1 second'),"
  2107. " strftime('%s','now'))", (affinity_parent,))
  2108. conn.commit()
  2109. conn.close()
  2110. act(auth, racer3_card["attach_capability"], {"capability_id": str(affinity_cap)})
  2111. status, _ = fake3.ask_now()
  2112. check(status == 204,
  2113. "a tag-capable non-affine saddle is refused the parked task (204)")
  2114. check(wait_for(lambda: len(fake2.wake_log) > wakes_before_2 or False,
  2115. timeout=30, what="affine wake") is not None,
  2116. "the dispatcher's wake targets the affine saddle")
  2117. status, resume_payload = fake2.ask_now()
  2118. fake2.ask_on_wake = True # restored whatever the claim returned
  2119. fake2.ask_on_idle_status = True
  2120. check(status == 200 and int(resume_payload["task"]["id"]) == affinity_parent,
  2121. "the affine saddle's ask claims the parked task")
  2122. check("Affinity child" in (resume_payload.get("resume_notice") or "")
  2123. and "affinity child finished" in (resume_payload.get("resume_notice") or ""),
  2124. "the resume assignment carries the subtask report notice")
  2125. check(wait_task_done(auth, affinity_parent, timeout=60) is not None,
  2126. "the resumed parent completes")
  2127. print("== phase 1: lost wake -> offline + re-wake of the next capable ==")
  2128. # racer-three sleeps through wakes (a daemon that died after its
  2129. # last 204): the guard window (SADDLE_LOST_WAKE_S=2) passes with
  2130. # no ask, the saddle goes offline, and the work moves to the
  2131. # next capable sleeping saddle.
  2132. fake3.ask_on_wake = False
  2133. fake3.ask_on_idle_status = False # sleeps through everything
  2134. saddles_state = slot_public(auth, "/saddles", "saddles")
  2135. act(auth, saddles_state["create_capability"], {"name": "lostwake"})
  2136. lostwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  2137. if c["name"] == "lostwake"][0])
  2138. racer3_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2139. if s["name"] == "racer-three"][0]
  2140. act(auth, racer3_card["attach_capability"], {"capability_id": str(lostwake_cap)})
  2141. # Racer-three must be asleep to be woken: park it with a manual
  2142. # 204 ask (nothing offerable for it — the task below is not
  2143. # created yet).
  2144. status, _ = fake3.ask_now()
  2145. check(status == 204, "racer-three sleeps before the lost-wake run")
  2146. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2147. "project_id": "1",
  2148. "title": "Lost wake task",
  2149. "priority": "2",
  2150. "capability_id": [str(lostwake_cap)],
  2151. })
  2152. lost_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2153. if t["title"] == "Lost wake task"][0])
  2154. check(wait_for(lambda: len(fake3.wake_log) > 0 or False,
  2155. timeout=30, what="wake sent to racer-three") is not None,
  2156. "the only capable sleeping saddle is woken")
  2157. check(wait_for(lambda: [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2158. if s["name"] == "racer-three"][0]["status"] == "offline"
  2159. or False, timeout=30, what="lost-wake -> offline") is not None,
  2160. "no ask after the wake: the saddle goes offline (lost-wake guard)")
  2161. check(task_state(auth, lost_task)["status"] == "queued",
  2162. "the task is untouched while its wake was pending")
  2163. # Re-wake the next capable: the healthy saddle gains the tag and
  2164. # picks the work up via its own wake -> ask -> allocate roundtrip.
  2165. fake_card = saddle_card("fake-saddle")
  2166. act(auth, fake_card["attach_capability"], {"capability_id": str(lostwake_cap)})
  2167. wakes_before = len(fake.wake_log)
  2168. check(wait_for(lambda: len(fake.wake_log) > wakes_before or False,
  2169. timeout=30, what="re-wake of the next capable") is not None,
  2170. "the next capable sleeping saddle is woken for the task")
  2171. check(wait_task_done(auth, lost_task, timeout=60) is not None,
  2172. "wake -> ask -> allocate: the task completes on the healthy saddle")
  2173. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == lost_task
  2174. for p in fake.start_payloads) or False,
  2175. timeout=10, what="lost task assignment captured") is not None,
  2176. "the re-woken saddle's ask produced the assignment")
  2177. fake3.ask_on_wake = True
  2178. fake3.ask_on_idle_status = True
  2179. print("== phase 3: capability context, catalogue, subtask tags ==")
  2180. # Assignment capability context (plan 10 phase 3): every
  2181. # assignment carries {saddle_tags: the allocated saddle's tags,
  2182. # fleet_tags: every tag name known across the fleet} — the
  2183. # daemon renders it into its system prompt. fake-saddle carries
  2184. # gpu + lostwake; the catalogue is affinity-tag, gpu, lostwake.
  2185. fake2.ask_on_register = False
  2186. fake2.ask_on_wake = False
  2187. fake2.ask_on_terminal = False
  2188. fake2.ask_on_idle_status = False
  2189. time.sleep(1.2) # any in-flight trigger asks land
  2190. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2191. "project_id": "1",
  2192. "title": "Capabilities context",
  2193. "priority": "2",
  2194. })
  2195. cap_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2196. if t["title"] == "Capabilities context"][0])
  2197. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == cap_task
  2198. for p in fake.start_payloads) or False,
  2199. timeout=60, what="capabilities-context task dispatched") is not None,
  2200. "the capabilities-context task dispatches")
  2201. cap_payload = [p for p in fake.start_payloads
  2202. if int(p.get("task", {}).get("id") or 0) == cap_task][-1]
  2203. context = cap_payload.get("capabilities") or {}
  2204. check(context.get("saddle_tags") == ["gpu", "lostwake"],
  2205. "assignment saddle_tags = the allocated saddle's tags (%s)"
  2206. % context.get("saddle_tags"))
  2207. check(context.get("fleet_tags") == ["affinity-tag", "gpu", "lostwake"],
  2208. "assignment fleet_tags = every tag name known across the fleet (%s)"
  2209. % context.get("fleet_tags"))
  2210. check(wait_task_done(auth, cap_task, timeout=60) is not None,
  2211. "capabilities-context task completes")
  2212. # GET /api/saddle/capabilities feeds the daemon's `saddle
  2213. # capabilities` CLI: per-saddle-token auth, tag order by name,
  2214. # each tag with the saddles carrying it (name order).
  2215. status, _ = saddle_api("GET", "/api/saddle/capabilities", None)
  2216. check(status == 401, "capabilities endpoint rejects a missing token")
  2217. status, body = saddle_api("GET", "/api/saddle/capabilities", "saddle-bogus")
  2218. check(status == 401, "capabilities endpoint rejects a wrong token")
  2219. status, body = saddle_api("GET", "/api/saddle/capabilities", fake_token)
  2220. catalogue = {t["name"]: t["saddles"] for t in body.get("tags", [])}
  2221. check(status == 200 and body.get("ok") is True
  2222. and [t["name"] for t in body.get("tags", [])]
  2223. == ["affinity-tag", "gpu", "lostwake"],
  2224. "catalogue lists every tag by name")
  2225. check(catalogue.get("gpu") == ["fake-saddle"]
  2226. and catalogue.get("affinity-tag") == ["racer-three", "racer-two"]
  2227. and catalogue.get("lostwake") == ["fake-saddle", "racer-three"],
  2228. "each tag names its carrying saddles in name order (%s)" % catalogue)
  2229. # Subtask capabilities (`saddle subtask --capability`): explicit
  2230. # only — never inherited — and validated against the operator
  2231. # tag set; unknown tags reject with the valid list in the error
  2232. # (the agent reads it verbatim in the CLI output).
  2233. conn = db()
  2234. tasks_before = conn.execute("select count(*) from tasks").fetchone()[0]
  2235. conn.close()
  2236. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  2237. "parent_task_id": auto_parent, "title": "Bad tag subtask",
  2238. "capabilities": ["gpu", "warp-drive"]})
  2239. conn = db()
  2240. tasks_after = conn.execute("select count(*) from tasks").fetchone()[0]
  2241. conn.close()
  2242. check(status == 400 and tasks_after == tasks_before,
  2243. "an unknown capability tag rejects the subtask call")
  2244. error_text = body.get("error", "")
  2245. check("warp-drive" in error_text and "valid tags" in error_text
  2246. and "gpu" in error_text and "lostwake" in error_text,
  2247. "the rejection names the unknown tag and the valid list (%s)" % error_text)
  2248. status, _ = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  2249. "parent_task_id": auto_parent, "title": "Bad shape subtask",
  2250. "capabilities": "gpu"})
  2251. check(status == 400, "a non-array capabilities member rejects")
  2252. # Explicit-only: a parent requiring 'lostwake' (the lost-wake
  2253. # task) gets a subtask tagged just 'gpu' — nothing inherited.
  2254. status, body = saddle_api("POST", "/api/saddle/subtask", r4_token, {
  2255. "parent_task_id": lost_task, "title": "Explicit tag subtask",
  2256. "capabilities": ["gpu"]})
  2257. check(status == 200 and body.get("ok") is True,
  2258. "a valid tag creates the subtask synchronously")
  2259. tagged_subtask = int(body["task_id"])
  2260. detail = task_state(auth, tagged_subtask)
  2261. check([c["name"] for c in detail.get("capabilities", [])] == ["gpu"],
  2262. "the subtask carries exactly the explicit tag (no inheritance)")
  2263. check([c["name"] for c in task_state(auth, lost_task).get("capabilities", [])]
  2264. == ["lostwake"],
  2265. "the parent keeps its own requirement")
  2266. conn = db()
  2267. tag_rows = conn.execute(
  2268. "select c.name from task_capabilities tc"
  2269. " join capabilities c on c.id = tc.capability_id"
  2270. " where tc.task_id = ?", (tagged_subtask,)).fetchall()
  2271. conn.close()
  2272. check([r[0] for r in tag_rows] == ["gpu"],
  2273. "the task_capabilities rows record the explicit tag")
  2274. # The queued subtask dispatches (gpu-carrier) and settles before
  2275. # the slow-wake run needs an empty queue.
  2276. check(wait_task_done(auth, tagged_subtask, timeout=90) is not None,
  2277. "the tagged subtask dispatches and completes")
  2278. print("== phase 3: task page — status on top, blocks newest-first ==")
  2279. detail = task_state(auth, plan_task)
  2280. block_ids = [int(b["id"]) for b in detail.get("blocks", [])]
  2281. check(len(block_ids) >= 3 and block_ids == sorted(block_ids, reverse=True),
  2282. "the blocks section orders newest-first via the sealed GET state (%s)"
  2283. % block_ids)
  2284. status, page = http("GET", "/tasks/%d" % plan_task, timeout=15)
  2285. check(status == 200 and "status-head" in page
  2286. and page.index("status-head") < page.index("<h3>Description</h3>")
  2287. and page.index("status-head") < page.index("<h3>Plan</h3>"),
  2288. "the status block sits at the very top of the task page")
  2289. print("== phase 3: dashboard — pinned blocks + activity timeline ==")
  2290. # Pinned OPEN blocking items as link-through cards, newest
  2291. # first; no inline answer forms (the card links to the task
  2292. # page). At this point: the auto-approve grandchild's question
  2293. # and RFC plus the parent's own review are all unanswered.
  2294. dash = slot_public(auth, "/", "dashboard")
  2295. pinned = dash.get("pinned", [])
  2296. pinned_ids = [int(b["id"]) for b in pinned]
  2297. check(len(pinned_ids) >= 3 and pinned_ids == sorted(pinned_ids, reverse=True),
  2298. "pinned open blocks order newest-first (%s)" % pinned_ids)
  2299. check({b["block_type"] for b in pinned} >= {"question", "rfc", "review"},
  2300. "pinned cards cover questions, RFCs and reviews")
  2301. check(all(b["question"] and b["task_title"] and int(b["task_id"]) > 0
  2302. for b in pinned)
  2303. and all("answer_action" not in b for b in pinned),
  2304. "pinned cards are link-through only (task title, kind badge, question)")
  2305. # The activity timeline: last <= 50 events, newest first, fed
  2306. # from the existing tables. Fixture rows a few seconds old
  2307. # guarantee the progress kind and a human-authored comment ride
  2308. # along; the probe block below proves same-source ordering.
  2309. conn = db()
  2310. conn.execute("insert into task_comments (task_id, author, body, created_at)"
  2311. " values (?, 'user:admin', 'timeline fixture comment by a human',"
  2312. " strftime('%s','now','-5 seconds'))", (cap_task,))
  2313. conn.execute("insert into task_progress (task_id, summary, created_at)"
  2314. " values (?, 'timeline fixture progress',"
  2315. " strftime('%s','now','-5 seconds'))",
  2316. (cap_task,))
  2317. conn.commit()
  2318. conn.close()
  2319. dash = slot_public(auth, "/", "dashboard")
  2320. activity = dash.get("activity", [])
  2321. epochs = [int(e["at_epoch"]) for e in activity]
  2322. check(0 < len(activity) <= 50 and epochs == sorted(epochs, reverse=True),
  2323. "the timeline shows the last <=50 events newest-first (%d events)"
  2324. % len(activity))
  2325. kinds = {e["kind"] for e in activity}
  2326. check({"created", "started", "dispatch", "comment", "progress", "completed"} <= kinds,
  2327. "the timeline covers status changes, dispatches, comments, progress and"
  2328. " completions (%s)" % sorted(kinds))
  2329. check(any(e["kind"] == "comment" and e["text"].startswith("user:admin:")
  2330. and "fixture comment" in e["text"] for e in activity),
  2331. "comment events carry author + snippet")
  2332. # A freshly raised block pins immediately (newest card) and
  2333. # lands in the timeline as block-raised; answering it records
  2334. # block-answered.
  2335. saddle_api("POST", "/api/saddle/%d/update" % fake_id, fake_token,
  2336. {"type": "block", "session_id": grand_session, "data": {
  2337. "question": "Timeline probe: approve the dashboard flow",
  2338. "type": "rfc", "body": "# Probe"}})
  2339. dash = slot_public(auth, "/", "dashboard")
  2340. check(dash["pinned"][0]["question"] == "Timeline probe: approve the dashboard flow"
  2341. and int(dash["pinned"][0]["task_id"]) == auto_grand,
  2342. "the freshest open block pins as the first card")
  2343. check(dash["activity"][0]["kind"] == "block-raised"
  2344. and "Timeline probe" in dash["activity"][0]["text"],
  2345. "the raised block lands atop the timeline")
  2346. probe_block = [b for b in task_state(auth, auto_grand)["blocks"]
  2347. if b["question"] == "Timeline probe: approve the dashboard flow"][0]
  2348. act(auth, probe_block["answer_action"], {"answer": "Ship the dashboard"})
  2349. dash = slot_public(auth, "/", "dashboard")
  2350. activity = dash.get("activity", [])
  2351. raised_at = next((i for i, e in enumerate(activity)
  2352. if e["kind"] == "block-raised" and "Timeline probe" in e["text"]), -1)
  2353. answered_at = next((i for i, e in enumerate(activity)
  2354. if e["kind"] == "block-answered"
  2355. and "Ship the dashboard" in e["text"]), -1)
  2356. check(answered_at >= 0 and raised_at >= 0 and answered_at < raised_at,
  2357. "the answered event sorts above its raised event (newest-first, same source)")
  2358. check(all(b["question"] != "Timeline probe: approve the dashboard flow"
  2359. for b in dash["pinned"]),
  2360. "the answered block leaves the pinned cards")
  2361. print("== phase 3 regression: a slow ask after a wake is never dropped ==")
  2362. # The daemon worker's lost-wake-guard regression: a saddle whose
  2363. # ask arrives SLOWLY after the wake (busy winding down when the
  2364. # wake lands) must not lose the wake — the delayed ask (inside
  2365. # the guard window) claims the task, no re-wake storm fires
  2366. # while it is pending, and the saddle stays healthy. The
  2367. # never-asks path (guard -> offline + re-wake of the next
  2368. # capable) is proven by the racer-three section above.
  2369. check(wait_for(lambda: queued_count() == 0 or False,
  2370. timeout=60, what="queue settles before the slow-wake run") is not None,
  2371. "the queue settles before the slow-wake run")
  2372. status, _ = fake2.ask_now()
  2373. check(status == 204, "the slow-ask racer sleeps before its wake")
  2374. saddles_state = slot_public(auth, "/saddles", "saddles")
  2375. act(auth, saddles_state["create_capability"], {"name": "slowwake"})
  2376. slowwake_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")
  2377. ["capabilities"] if c["name"] == "slowwake"][0])
  2378. racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2379. if s["name"] == "racer-two"][0]
  2380. act(auth, racer2_card["attach_capability"], {"capability_id": str(slowwake_cap)})
  2381. fake2.ask_delay = 1.2 # slow: asks well after the wake, inside the 2s guard
  2382. fake2.ask_on_wake = True
  2383. wakes_before_slow = len(fake2.wake_log)
  2384. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2385. "project_id": "1",
  2386. "title": "Slow wake task",
  2387. "priority": "2",
  2388. "capability_id": [str(slowwake_cap)],
  2389. })
  2390. slow_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2391. if t["title"] == "Slow wake task"][0])
  2392. check(wait_for(lambda: len(fake2.wake_log) > wakes_before_slow or False,
  2393. timeout=30, what="the slow-ask racer woken") is not None,
  2394. "the only capable sleeping saddle is woken")
  2395. check(wait_task_done(auth, slow_task, timeout=120) is not None,
  2396. "the delayed ask claims the task — the wake was not dropped")
  2397. check(any(a.get("status") == 200
  2398. and int((a.get("payload") or {}).get("task", {}).get("id") or 0)
  2399. == slow_task for a in fake2.asks),
  2400. "the slow ask's 200 carries the task")
  2401. check(len(fake2.wake_log) == wakes_before_slow + 1,
  2402. "the pending-wake skip re-wakes nobody while the slow ask is in flight")
  2403. racer2_card = [s for s in slot_public(auth, "/saddles", "saddles")["saddles"]
  2404. if s["name"] == "racer-two"][0]
  2405. check(racer2_card["status"] in ("idle", "busy"),
  2406. "the slow asker never went offline (guard did not fire)")
  2407. fake2.ask_delay = 0.0
  2408. fake2.ask_on_wake = False # stays out of the pool below
  2409. # The racers leave the dispatch pool: the remaining sections
  2410. # (rollback resume, R5-R11) predate multi-daemon routing and
  2411. # assert against the primary fake's payloads. racer-three is
  2412. # already offline (the lost-wake guard); racer-two joins it.
  2413. conn = db()
  2414. conn.execute("update saddles set status = 'offline' where name = 'racer-two'")
  2415. conn.commit()
  2416. conn.close()
  2417. # Tasks caught working on a saddle that went offline requeue with a
  2418. # bot comment each tick.
  2419. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2420. "project_id": "1",
  2421. "title": "Offline saddle task",
  2422. "priority": "2",
  2423. })
  2424. offline_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2425. if t["title"] == "Offline saddle task"][0])
  2426. check(wait_for(lambda: task_state(auth, offline_task).get("status") == "working"
  2427. or False, timeout=60, what="offline task starts working") is not None,
  2428. "task reaches working before its saddle disappears")
  2429. conn = db()
  2430. conn.execute("update saddles set status = 'offline' where name = 'fake-saddle'")
  2431. conn.commit()
  2432. conn.close()
  2433. # The requeue frees the saddle in the same tick, so 'queued' is
  2434. # transient; the durable evidence is the bot comment (and the task
  2435. # completing again on the healthy daemon).
  2436. check(wait_for(lambda: any(c["body"] == "saddle went offline; task requeued"
  2437. for c in task_state(auth, offline_task).get("comments", []))
  2438. or False, timeout=30, what="offline requeue comment") is not None,
  2439. "offline saddle requeues the task")
  2440. check(wait_task_done(auth, offline_task, timeout=60) is not None,
  2441. "requeued task completes again on the healthy daemon")
  2442. print("== R5 B1: /start parent context ==")
  2443. # A fresh subtask dispatch carries the parent object (plan clamped).
  2444. conn = db()
  2445. conn.execute("update tasks set plan = ? where id = ?",
  2446. ("# Parent plan\n\n" + "step. " * 1200, auto_parent))
  2447. conn.commit()
  2448. conn.close()
  2449. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2450. "project_id": "1",
  2451. "title": "Lineage subtask",
  2452. "priority": "3",
  2453. })
  2454. lineage_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2455. if t["title"] == "Lineage subtask"][0])
  2456. conn = db()
  2457. conn.execute("update tasks set parent_task_id = ?, description = 'torque to spec'"
  2458. " where id = ?", (auto_parent, lineage_task))
  2459. conn.commit()
  2460. conn.close()
  2461. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == lineage_task
  2462. for p in fake.start_payloads) or False,
  2463. timeout=60, what="lineage subtask dispatches") is not None,
  2464. "lineage subtask dispatches")
  2465. lineage_push = [p for p in fake.start_payloads
  2466. if int(p.get("task", {}).get("id") or 0) == lineage_task][-1]
  2467. parent_obj = lineage_push.get("parent")
  2468. check(parent_obj is not None
  2469. and int(parent_obj.get("id") or 0) == auto_parent
  2470. and parent_obj.get("title") == "Auto parent"
  2471. and parent_obj.get("status") in ("suspended", "working", "done"),
  2472. "subtask /start carries the parent object")
  2473. check(parent_obj.get("description") == "torque to spec"
  2474. or "Parent plan" in (parent_obj.get("plan") or ""),
  2475. "parent description/plan content flows through")
  2476. check(len(parent_obj.get("plan") or "") <= 4000,
  2477. "parent plan clamped to ~4000 chars (%d)"
  2478. % len(parent_obj.get("plan") or ""))
  2479. root_pushes = [p for p in fake.start_payloads
  2480. if int(p.get("task", {}).get("id") or 0) == offline_task]
  2481. check(all("parent" not in p for p in root_pushes),
  2482. "root task /start carries no parent object")
  2483. print("== R5 B2: task-reading API ==")
  2484. status, _ = saddle_api("GET", "/api/saddle/tasks", None)
  2485. check(status == 401, "tasks endpoint rejects a missing token")
  2486. status, _ = saddle_api("GET", "/api/saddle/task/%d" % auto_child, None)
  2487. check(status == 401, "task detail endpoint rejects a missing token")
  2488. status, body = saddle_api("GET", "/api/saddle/tasks", r4_token)
  2489. check(status == 200 and len(body.get("tasks", [])) > 5,
  2490. "tasks endpoint lists tasks without filters")
  2491. row = [t for t in body["tasks"] if int(t["id"]) == auto_child][0]
  2492. check(row["title"] == "Auto child" and int(row["parent_task_id"]) == auto_parent
  2493. and row["status"] in ("done", "queued", "working"),
  2494. "compact rows carry id/title/status/parent_task_id")
  2495. status, body = saddle_api("GET", "/api/saddle/tasks?project_id=1", r4_token)
  2496. other_project = int([p["id"] for p in slot_public(auth, "/projects", "projects")["projects"]
  2497. if p["name"] == "diff-fabrica"][0])
  2498. check(all(True for _ in [1]) # project 1 dominates the data set
  2499. and not any(int(t["id"]) == 0 for t in body["tasks"]),
  2500. "project_id filter accepted")
  2501. status, body = saddle_api("GET",
  2502. "/api/saddle/tasks?project_id=%d" % other_project, r4_token)
  2503. check(all(t["title"].startswith("Diff") for t in body["tasks"]),
  2504. "other project lists only its own tasks")
  2505. status, body = saddle_api("GET", "/api/saddle/tasks?q=FLANGE", r4_token)
  2506. titles = [t["title"] for t in body["tasks"]]
  2507. check("Wire the flange" in titles,
  2508. "q matches titles case-insensitively")
  2509. status, body = saddle_api("GET", "/api/saddle/tasks?q=torque%20to%20spec", r4_token)
  2510. check(any(t["title"] == "Lineage subtask" for t in body["tasks"]),
  2511. "q matches description text")
  2512. status, body = saddle_api("GET", "/api/saddle/tasks?q=parent%20plan", r4_token)
  2513. check(any(t["title"] == "Auto parent" for t in body["tasks"]),
  2514. "q matches plan text")
  2515. status, body = saddle_api("GET", "/api/saddle/task/%d" % auto_child, r4_token)
  2516. task_view = body.get("task", {})
  2517. check(status == 200 and int(task_view.get("id") or 0) == auto_child
  2518. and task_view.get("parent_title") == "Auto parent"
  2519. and int(task_view.get("parent_task_id") or 0) == auto_parent,
  2520. "single-task view joins the parent title")
  2521. check("status" in task_view and "description" in task_view
  2522. and "plan" in task_view and "summary" in task_view,
  2523. "single-task view carries the full detail fields")
  2524. status, body = saddle_api("GET", "/api/saddle/task/99999", r4_token)
  2525. check(status == 404, "unknown task id returns 404")
  2526. print("== R6 F1: a hanging request cannot freeze the tick ==")
  2527. # One wedged /task-update (never responds) blocks the comment/
  2528. # subtask-event delivery pass; the Soup timeout must fail it and
  2529. # later ticks keep dispatching.
  2530. task_state(auth, lineage_task)
  2531. conn = db()
  2532. conn.execute("insert into tasks (project_id, title, status, priority, created_at)"
  2533. " values (1, 'Hang survivor', 'queued', 2, strftime('%s','now'))")
  2534. hang_task = conn.execute("select max(id) from tasks").fetchone()[0]
  2535. conn.close()
  2536. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2537. "project_id": "1",
  2538. "title": "Hang runner",
  2539. "priority": "3",
  2540. })
  2541. hang_runner = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2542. if t["title"] == "Hang runner"][0])
  2543. # A human comment on the running task gives the tick a
  2544. # /task-update delivery to make; the next one hangs.
  2545. check(wait_for(lambda: task_state(auth, hang_runner).get("status") == "working"
  2546. or False, timeout=60, what="hang runner working") is not None,
  2547. "hang runner reaches working")
  2548. act(auth, task_state(auth, hang_runner)["comment"],
  2549. {"body": "operator ping while wedged"})
  2550. fake.hang_next["/task-update"] = 1
  2551. check(wait_task_done(auth, hang_runner, timeout=90) is not None,
  2552. "dispatching task survives a hung daemon request")
  2553. check(wait_for(lambda: any(c.get("text") == "operator ping while wedged"
  2554. or c.get("body") == "operator ping while wedged"
  2555. for c in fake.comment_log) or False,
  2556. timeout=30, what="wedged comment eventually delivered") is not None,
  2557. "the comment delivery retries after the hang")
  2558. # The hung delivery itself fails after the timeout; the system
  2559. # kept moving either way.
  2560. with open(os.path.join(WORK, "app.log")) as handle:
  2561. app_log = handle.read()
  2562. check("tick exceeded" not in app_log or "Dispatch tick failed" in app_log,
  2563. "a hung request never freezes the tick latch")
  2564. print("== R6 F2: refresh must not corrupt parked bookkeeping ==")
  2565. fake.suspend_on_title = "Parked refresh parent"
  2566. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2567. "project_id": "1",
  2568. "title": "Parked refresh parent",
  2569. "priority": "2",
  2570. })
  2571. parked = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2572. if t["title"] == "Parked refresh parent"][0])
  2573. check(wait_for(lambda: task_state(auth, parked).get("status") == "suspended"
  2574. or False, timeout=60, what="parked parent suspends") is not None,
  2575. "parent parks before the refresh")
  2576. fake_card = saddle_card("fake-saddle")
  2577. act(auth, fake_card["refresh"], {})
  2578. fake_card = saddle_card("fake-saddle")
  2579. check(fake_card["status"] == "idle" and int(fake_card["current_task_id"] or 0) == 0,
  2580. "refresh on a parked (suspended) daemon leaves the saddle idle")
  2581. # The parked task keeps its open session and resumes normally.
  2582. conn = db()
  2583. conn.execute("insert into tasks (project_id, title, status, priority, parent_task_id,"
  2584. " summary, completed_at, created_at) values (1, 'Parked child', 'done', 5,"
  2585. " ?, 'child finished', strftime('%s','now','+1 second'),"
  2586. " strftime('%s','now'))", (parked,))
  2587. conn.commit()
  2588. conn.close()
  2589. check(wait_task_done(auth, parked, timeout=90) is not None,
  2590. "parked parent still resumes after the refresh")
  2591. print("== R7 B1: cancel reaches the daemon ==")
  2592. # Plan 10 phase 2: cancellation is POST /task-update {kind: cancel}
  2593. # to the RUNNING daemon (the abort machinery, workspace kept).
  2594. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2595. "project_id": "1",
  2596. "title": "Cancel live",
  2597. "priority": "2",
  2598. })
  2599. cancel_live = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2600. if t["title"] == "Cancel live"][0])
  2601. check(wait_for(lambda: task_state(auth, cancel_live).get("status") == "working"
  2602. or False, timeout=60, what="cancel-live working") is not None,
  2603. "task reaches working before the cancel")
  2604. act(auth, task_state(auth, cancel_live)["cancel"], {})
  2605. check(wait_for(lambda: cancel_live in fake.abort_log or False,
  2606. timeout=30, what="daemon abort recorded") is not None,
  2607. "cancelling POSTs task-update cancel to the daemon")
  2608. check(any(c["body"] == "cancelled; daemon aborted"
  2609. for c in task_state(auth, cancel_live).get("comments", [])),
  2610. "the abort outcome is narrated")
  2611. check(task_state(auth, cancel_live)["status"] == "failed",
  2612. "cancelled task lands failed as before")
  2613. # A 500 on the cancel marks the saddle offline (no fence layer
  2614. # anymore); nothing dispatches to it while offline, and the
  2615. # daemon's own ask flips the row back to idle (proof of life).
  2616. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2617. "project_id": "1",
  2618. "title": "Fence runner",
  2619. "priority": "2",
  2620. })
  2621. fence_runner = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2622. if t["title"] == "Fence runner"][0])
  2623. check(wait_for(lambda: task_state(auth, fence_runner).get("status") == "working"
  2624. or False, timeout=60, what="fence runner working") is not None,
  2625. "second task reaches working")
  2626. fake.abort_failures = 1
  2627. act(auth, task_state(auth, fence_runner)["cancel"], {})
  2628. check(any(c["body"] == "cancelled; daemon unreachable — saddle marked offline"
  2629. for c in task_state(auth, fence_runner).get("comments", [])),
  2630. "failed cancel narrates the offline marking")
  2631. check(saddle_card("fake-saddle")["status"] == "offline",
  2632. "saddle marked offline after the failed cancel")
  2633. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2634. "project_id": "1",
  2635. "title": "During fence",
  2636. "priority": "3",
  2637. })
  2638. during_fence = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2639. if t["title"] == "During fence"][0])
  2640. time.sleep(7)
  2641. check(task_state(auth, during_fence)["status"] == "queued",
  2642. "offline saddle is not dispatchable")
  2643. # The (unaborted) daemon finishes its round, goes idle-awake and
  2644. # asks — the ask is the proof of life that flips the row back.
  2645. check(wait_for(lambda: saddle_card("fake-saddle")["status"] == "idle" or False,
  2646. timeout=30, what="ask flips offline to idle") is not None,
  2647. "the daemon's ask flips the offline row back to idle")
  2648. check(wait_task_done(auth, during_fence, timeout=60) is not None,
  2649. "queued work dispatches once the saddle proves it is alive")
  2650. print("== R7 B3: health panel + pushed raw log from the first round ==")
  2651. # A normally dispatched task pushes its chunks immediately: raw
  2652. # rows exist from the first round (the session-4 gap stays
  2653. # fixed without any probe machinery).
  2654. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2655. "project_id": "1",
  2656. "title": "Raw from first push",
  2657. "priority": "2",
  2658. })
  2659. raw_first = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2660. if t["title"] == "Raw from first push"][0])
  2661. def raw_count(task_id):
  2662. conn = db()
  2663. try:
  2664. return conn.execute(
  2665. "select count(*) from session_raw_log where session_id in"
  2666. " (select id from sessions where task_id = ?)", (task_id,)).fetchone()[0]
  2667. finally:
  2668. conn.close()
  2669. check(wait_for(lambda: raw_count(raw_first) > 0 or False,
  2670. timeout=60, what="raw rows from first push") is not None,
  2671. "pushed chunks land from the first round (session-4 gap)")
  2672. check(wait_task_done(auth, raw_first, timeout=60) is not None,
  2673. "raw-first task completes")
  2674. # Health panel: a manual refresh populates the snapshot.
  2675. fake_card = saddle_card("fake-saddle")
  2676. act(auth, fake_card["refresh"], {})
  2677. card = saddle_card("fake-saddle")
  2678. check(card.get("health") in ("healthy", "stale")
  2679. and card.get("daemon_state") in ("working", "idle", "done")
  2680. and card.get("uptime_display", "") != ""
  2681. and card.get("last_contact_display", "") not in ("", "never"),
  2682. "saddles page shows daemon state, uptime and last contact")
  2683. print("== R8: pushed chunk ingestion is lossless ==")
  2684. # The daemon's opening flush: a coloured prompt run, plain text,
  2685. # a bare CRLF between prompts and the tail — every non-empty
  2686. # chunk must land, exactly once, in seq order. A mid-flight
  2687. # redelivery of the whole batch (the outbox's at-least-once
  2688. # retry) consumes the duplicates.
  2689. corpus = [
  2690. {"seq": 1, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
  2691. {"seq": 2, "text": "echo hello\r\n"},
  2692. {"seq": 3, "text": "\r\n"},
  2693. {"seq": 4, "text": "hello\r\n"},
  2694. {"seq": 5, "text": "\x1b[01;32muser@host\x1b[00m:\x1b[01;34m~/task-1\x1b[00m$ "},
  2695. {"seq": 6, "text": "done\r\n"},
  2696. ]
  2697. fake.scripted_corpus = corpus
  2698. fake.hold_state = True
  2699. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2700. "project_id": "1",
  2701. "title": "Lossless log task",
  2702. "priority": "2",
  2703. })
  2704. lossless = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2705. if t["title"] == "Lossless log task"][0])
  2706. def stored_seqs(task_id):
  2707. conn = db()
  2708. try:
  2709. return [r[0] for r in conn.execute(
  2710. "select seq from session_raw_log where session_id in"
  2711. " (select id from sessions where task_id = ?)"
  2712. " order by seq", (task_id,))]
  2713. finally:
  2714. conn.close()
  2715. check(wait_for(lambda: task_state(auth, lossless).get("status") == "working"
  2716. or False, timeout=60, what="lossless task working") is not None,
  2717. "lossless task dispatches and pushes its opening flush")
  2718. check(wait_for(lambda: stored_seqs(lossless) == [1, 2, 3, 4, 5, 6] or False,
  2719. timeout=30, what="all six chunks stored") is not None,
  2720. "every pushed chunk lands in session_raw_log, CRLF-only included (%s)"
  2721. % stored_seqs(lossless))
  2722. status, _ = saddle_api("POST", "/api/saddle/chunks", fake.token,
  2723. {"session_id": fake.session_id,
  2724. "chunks": [dict(chunk) for chunk in corpus]})
  2725. check(status == 200 and stored_seqs(lossless) == [1, 2, 3, 4, 5, 6],
  2726. "a full-batch redelivery stores nothing new (exactly-once)")
  2727. request = urllib.request.Request(
  2728. WEB + "/terminal-stream/%d" % [r[0] for r in db().execute(
  2729. "select id from sessions where task_id = ?", (lossless,))][0])
  2730. with urllib.request.urlopen(request, timeout=15) as stream:
  2731. texts = []
  2732. deadline = time.monotonic() + 8
  2733. while time.monotonic() < deadline and len(texts) < 6:
  2734. line = stream.readline().decode().strip()
  2735. if not line.startswith("data: ") or line == "data: ping":
  2736. continue
  2737. event = json.loads(line[6:])
  2738. if event.get("type") == "raw":
  2739. texts.append(event.get("text") or "")
  2740. check("\r\n" in texts and len(texts) == 6,
  2741. "SSE serves the whitespace chunk unfiltered")
  2742. fake.hold_state = False
  2743. fake.scripted_corpus = None
  2744. check(wait_task_done(auth, lossless, timeout=60) is not None,
  2745. "lossless task completes")
  2746. print("== R9 B1/B2: git identity + ssh key ==")
  2747. # The key travels on /status into the saddle row and the page view
  2748. # (the manual refresh's exceptional probe).
  2749. fake.ssh_public_key = "ssh-ed25519 AAAAC3NzaC1lZDI1teStKey saddle-1@fedora"
  2750. fake_card = saddle_card("fake-saddle")
  2751. check(fake_card.get("ssh_public_key", "") == "",
  2752. "no key shown before the daemon reports one")
  2753. act(auth, fake_card["refresh"], {})
  2754. check(wait_for(lambda: "AAAAC3NzaC1lZDI1teStKey" in
  2755. saddle_card("fake-saddle").get("ssh_public_key", "") or False,
  2756. timeout=30, what="key lands on the row") is not None,
  2757. "reported ssh_public_key persists onto the saddle row")
  2758. conn = db()
  2759. row_key = conn.execute("select ssh_public_key from saddles where name = 'fake-saddle'"
  2760. ).fetchone()[0]
  2761. conn.close()
  2762. check(row_key == fake.ssh_public_key,
  2763. "the key is stored verbatim in the database")
  2764. # Identity edit flows through the UI into the assignment payload.
  2765. fake_card = saddle_card("fake-saddle")
  2766. act(auth, fake_card["update_identity"],
  2767. {"git_name": "clanker-bot", "git_email": "bot@unitatem.net"})
  2768. fake_card = saddle_card("fake-saddle")
  2769. check(fake_card.get("git_name") == "clanker-bot"
  2770. and fake_card.get("git_email") == "bot@unitatem.net",
  2771. "inline edit saves the git identity")
  2772. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2773. "project_id": "1",
  2774. "title": "Identity task",
  2775. "priority": "2",
  2776. })
  2777. identity_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2778. if t["title"] == "Identity task"][0])
  2779. check(wait_for(lambda: any(int(p.get("task", {}).get("id") or 0) == identity_task
  2780. for p in fake.start_payloads) or False,
  2781. timeout=60, what="identity task dispatched") is not None,
  2782. "identity task dispatches")
  2783. payload = [p for p in fake.start_payloads
  2784. if int(p.get("task", {}).get("id") or 0) == identity_task][-1]
  2785. identity = payload.get("git_identity") or {}
  2786. check(identity.get("name") == "clanker-bot"
  2787. and identity.get("email") == "bot@unitatem.net",
  2788. "the assignment carries the saddle's git_identity")
  2789. # Earlier assignments (identity unset) carry no git_identity member.
  2790. check(all("git_identity" not in p for p in fake.start_payloads[:-1]),
  2791. "unset identity omits the member (daemon fallback applies)")
  2792. # Clearing the identity omits it from later assignments.
  2793. fake_card = saddle_card("fake-saddle")
  2794. act(auth, fake_card["update_identity"], {"git_name": "", "git_email": ""})
  2795. check(saddle_card("fake-saddle").get("git_name", "") == "",
  2796. "clearing the identity empties the row")
  2797. check(wait_task_done(auth, identity_task, timeout=60) is not None,
  2798. "identity task completes")
  2799. print("== R10: branch_advanced on the /poll verdict channel ==")
  2800. conn = db()
  2801. conn.execute("insert into tasks (project_id, title, status, priority,"
  2802. " created_at) values (1, 'Poll grandparent', 'working', 5,"
  2803. " strftime('%s','now','-300 seconds'))")
  2804. poll_grand = conn.execute("select max(id) from tasks").fetchone()[0]
  2805. conn.execute("insert into tasks (project_id, title, status, priority,"
  2806. " parent_task_id, created_at) values (1, 'Poll answer parent',"
  2807. " 'blocked', 5, ?, strftime('%s','now','-120 seconds'))",
  2808. (poll_grand,))
  2809. poll_parent = conn.execute("select max(id) from tasks").fetchone()[0]
  2810. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  2811. " values (?, ?, strftime('%s','now','-120 seconds'))",
  2812. (poll_parent, fake_id))
  2813. conn.execute("insert into task_blocks (task_id, question, type, answer,"
  2814. " answered_at, created_at) values (?, 'Which DB?', 'question',"
  2815. " 'SQLite', strftime('%s','now','-60 seconds'),"
  2816. " strftime('%s','now','-100 seconds'))", (poll_parent,))
  2817. conn.execute("update saddles set status = 'busy', current_task_id = ?,"
  2818. " allocated_at = strftime('%s','now') where name = 'fake-saddle'",
  2819. (poll_parent,))
  2820. conn.commit()
  2821. conn.close()
  2822. status, body = saddle_api("GET", "/api/saddle/%d/poll" % fake_id, fake_token)
  2823. check(status == 200 and len(body.get("answers", [])) == 1
  2824. and body["answers"][0].get("branch_advanced") is False,
  2825. "/poll verdicts carry branch_advanced=false with no siblings")
  2826. # A sibling under the same parent completed after the park marker.
  2827. conn = db()
  2828. conn.execute("insert into tasks (project_id, title, status, priority,"
  2829. " parent_task_id, summary, completed_at, created_at)"
  2830. " values (1, 'Poll sibling', 'done', 5, ?, 'sibling finished',"
  2831. " strftime('%s','now'), strftime('%s','now','-200 seconds'))",
  2832. (poll_grand,))
  2833. conn.execute("insert into task_blocks (task_id, question, type, answer,"
  2834. " answered_at, created_at) values (?, 'Second question?',"
  2835. " 'question', 'Yes', strftime('%s','now'),"
  2836. " strftime('%s','now','-90 seconds'))", (poll_parent,))
  2837. conn.commit()
  2838. conn.close()
  2839. status, body = saddle_api("GET", "/api/saddle/%d/poll" % fake_id, fake_token)
  2840. check(status == 200 and len(body.get("answers", [])) == 1
  2841. and body["answers"][0].get("branch_advanced") is True,
  2842. "sibling completion after park flips branch_advanced to true")
  2843. # Release the staged fixture: cancel it so its busy@task row does
  2844. # not linger into the sections below.
  2845. act(auth, task_state(auth, poll_parent)["cancel"], {})
  2846. print("== phase 2: blocked push -> block row + pulse ==")
  2847. # The browser flow on each pulse: statum.act(task.refresh) — the
  2848. # sealed GET action must rehydrate the task slot with the new
  2849. # block the daemon just PUSHED (state blocked + block payload).
  2850. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2851. "project_id": "1",
  2852. "title": "Live block task",
  2853. "priority": "2",
  2854. })
  2855. live_task = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2856. if t["title"] == "Live block task"][0])
  2857. check(wait_for(lambda: task_state(auth, live_task).get("status") == "working"
  2858. or False, timeout=60, what="live task working") is not None,
  2859. "live-block task reaches working")
  2860. fake.hold_state = True
  2861. fake.state = "blocked" # the daemon parks on its question
  2862. status = fake.push_results("blocked", block={
  2863. "question": "Pick one", "type": "question", "options": ["A", "B"]})
  2864. check(status == 200, "blocked state push accepted")
  2865. # The parked daemon is idle-awake: it asks (and sleeps on 204).
  2866. fake.maybe_trigger_ask("parked")
  2867. check(wait_for(lambda: any(b.get("question") == "Pick one"
  2868. for b in task_state(auth, live_task).get("blocks", []))
  2869. or False, timeout=30, what="block lands in state") is not None,
  2870. "the pushed block reaches the sealed refresh without a reload")
  2871. detail = task_state(auth, live_task)
  2872. check(detail.get("status") == "blocked",
  2873. "the blocked push flips the task to blocked")
  2874. pushed_block = [b for b in detail["blocks"] if b.get("question") == "Pick one"][0]
  2875. conn = db()
  2876. options_json = conn.execute(
  2877. "select options from task_blocks where id = ?", (int(pushed_block["id"]),)
  2878. ).fetchone()[0]
  2879. conn.close()
  2880. check(json.loads(options_json) == ["A", "B"],
  2881. "the block row carries its options payload")
  2882. check(any(s["status"] == "idle"
  2883. for s in slot_public(auth, "/saddles", "saddles")["saddles"]),
  2884. "the blocked push parks the saddle")
  2885. detail = task_state(auth, live_task)
  2886. check(detail.get("refresh") is not None,
  2887. "the sealed GET refresh action is still authored")
  2888. status, _ = http("GET", detail["refresh"]["uri"], timeout=15)
  2889. check(status in (200, 401),
  2890. "the refresh URI resolves as a GET (client roundtrip shape)")
  2891. act(auth, pushed_block["answer_action"], {"answer": "A"})
  2892. # The answer makes the parked task resume-offerable: the affine
  2893. # saddle is woken, asks, and its assignment carries the answer.
  2894. pushes_before = len([p for p in fake.start_payloads
  2895. if int(p.get("task", {}).get("id") or 0) == live_task])
  2896. check(wait_for(lambda: len([p for p in fake.start_payloads
  2897. if int(p.get("task", {}).get("id") or 0) == live_task])
  2898. > pushes_before or False, timeout=60,
  2899. what="blocked task resumes") is not None,
  2900. "answering the pushed block resumes the parked workspace")
  2901. fake.hold_state = False # the resumed round runs to completion
  2902. resume_payload = [p for p in fake.start_payloads
  2903. if int(p.get("task", {}).get("id") or 0) == live_task][-1]
  2904. answers = resume_payload.get("task", {}).get("answers", [])
  2905. check(any(a.get("answer") == "A" for a in answers),
  2906. "the resume assignment carries the human's answer")
  2907. check(wait_task_done(auth, live_task, timeout=60) is not None,
  2908. "live-block task completes after answering")
  2909. print("== phase 2: answer resume — wake -> ask -> affinity claim ==")
  2910. # The full live sequence against a daemon that asks after its
  2911. # park: ask -> assignment -> blocked push (park) -> ask -> 204
  2912. # -> sleeps (row idle) -> human answers -> the dispatcher WAKES
  2913. # the sleeping affine saddle -> its ask allocates the resume,
  2914. # the verdict riding the assignment -> the session continues.
  2915. # A private capability tag keeps the fleet's other sleeping
  2916. # saddles from claiming these tasks.
  2917. saddles_state = slot_public(auth, "/saddles", "saddles")
  2918. act(auth, saddles_state["create_capability"], {"name": "answerflow"})
  2919. answerflow_cap = int([c["id"] for c in slot_public(auth, "/saddles", "saddles")["capabilities"]
  2920. if c["name"] == "answerflow"][0])
  2921. def answerflow_daemon(**kwargs):
  2922. daemon = FakeSaddle()
  2923. daemon.ask_on_register = False # manual, deterministic asks
  2924. daemon.hold_state = True # the tests drive the lifecycle
  2925. for key, value in kwargs.items():
  2926. setattr(daemon, key, value)
  2927. daemon.serve()
  2928. return daemon
  2929. def register_answerflow(daemon, name):
  2930. act(auth, slot_public(auth, "/saddles", "saddles")["register"], {
  2931. "name": name, "host": "127.0.0.1:%d" % daemon.port})
  2932. act(auth, saddle_card(name)["attach_capability"],
  2933. {"capability_id": str(answerflow_cap)})
  2934. def create_answerflow_task(title):
  2935. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  2936. "project_id": "1", "title": title, "priority": "2",
  2937. "capability_id": [str(answerflow_cap)]})
  2938. return int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  2939. if t["title"] == title][0])
  2940. def retire_daemon(daemon, name):
  2941. # deterministic teardown: a dead daemon and an offline row
  2942. # never interfere with the sections that follow
  2943. daemon.kill()
  2944. act(auth, saddle_card(name)["refresh"], {})
  2945. sleeper = answerflow_daemon(ask_on_wake=True)
  2946. register_answerflow(sleeper, "resume-sleeper")
  2947. sleeping_task = create_answerflow_task("Sleeping resume task")
  2948. ask_status, assignment = sleeper.ask_now()
  2949. check(ask_status == 200 and int(assignment["task"]["id"]) == sleeping_task,
  2950. "the daemon's ask claims the queued task")
  2951. sleeping_session = assignment.get("session_id")
  2952. sleeper.state = "blocked"
  2953. check(sleeper.push_results("blocked", block={
  2954. "question": "Pick a side", "type": "question",
  2955. "options": ["Port", "Starboard"]}) == 200,
  2956. "the blocked push parks the session")
  2957. check(wait_for(lambda: task_state(auth, sleeping_task).get("status") == "blocked"
  2958. or False, timeout=30, what="sleeping task parked") is not None,
  2959. "the task parks as blocked")
  2960. check(saddle_card("resume-sleeper")["status"] == "idle",
  2961. "the parked session's saddle row reads idle")
  2962. ask_status, _ = sleeper.ask_now()
  2963. check(ask_status == 204,
  2964. "the parked daemon's ask returns 204 — it sleeps until woken")
  2965. time.sleep(3) # several dispatch ticks pass with the daemon asleep
  2966. check(len(sleeper.wake_log) == 0,
  2967. "the sleeping daemon is never woken while the block waits")
  2968. detail = task_state(auth, sleeping_task)
  2969. side_block = [b for b in detail["blocks"] if b["question"] == "Pick a side"][0]
  2970. act(auth, side_block["answer_action"], {"option": "Starboard"})
  2971. check(wait_for(lambda: len(sleeper.wake_log) > 0 or False,
  2972. timeout=30, what="wake after the answer") is not None,
  2973. "answering the block wakes the sleeping affine saddle")
  2974. check(wait_for(lambda: any(
  2975. int(p.get("task", {}).get("id") or 0) == sleeping_task
  2976. and p.get("session_id") == sleeping_session
  2977. and any(a.get("answer") == "Starboard"
  2978. for a in p.get("task", {}).get("answers", []))
  2979. for p in sleeper.start_payloads[1:]) or False,
  2980. timeout=30, what="resume allocation carries the verdict") is not None,
  2981. "the woken daemon's ask allocates the resume — same session, verdict carried")
  2982. # The session continues: the resumed round runs to completion on
  2983. # the SAME session id.
  2984. sleeper.push_results("done", summary="sleep-resumed round complete")
  2985. check(wait_task_done(auth, sleeping_task, timeout=30) is not None,
  2986. "the resumed session continues to completion")
  2987. conn = db()
  2988. session_rows = conn.execute(
  2989. "select count(*) from sessions where task_id = ?", (sleeping_task,)).fetchone()[0]
  2990. conn.close()
  2991. check(session_rows == 1,
  2992. "the sleeping resume kept one session throughout")
  2993. retire_daemon(sleeper, "resume-sleeper")
  2994. print("== phase 2: answer resume — verdict delivered to the parked session ==")
  2995. # The live regression's daemon shape (clanker-test.db): after the
  2996. # blocked push the daemon CANNOT ask again — its round is parked
  2997. # in the block's answer wait, so the saddle's last ask stays the
  2998. # one that claimed the task (never sleeping, never woken). The
  2999. # verdict must be DELIVERED into that answer wait.
  3000. parked = answerflow_daemon(ask_on_wake=False, # a blocked daemon cannot ask on wake
  3001. ask_on_terminal=False, # and not after the round either
  3002. ask_on_idle_status=False)
  3003. register_answerflow(parked, "resume-parked")
  3004. parked_task = create_answerflow_task("Parked resume task")
  3005. ask_status, assignment = parked.ask_now()
  3006. check(ask_status == 200 and int(assignment["task"]["id"]) == parked_task,
  3007. "the daemon's ask claims the task (its last ask)")
  3008. parked_session = assignment.get("session_id")
  3009. parked.state = "blocked"
  3010. check(parked.push_results("blocked", block={
  3011. "question": "Which database?", "type": "question",
  3012. "options": ["SQLite", "Postgres"]}) == 200,
  3013. "the blocked push parks the session mid-round")
  3014. check(wait_for(lambda: task_state(auth, parked_task).get("status") == "blocked"
  3015. or False, timeout=30, what="parked task blocked") is not None,
  3016. "the task parks as blocked")
  3017. check(saddle_card("resume-parked")["status"] == "idle",
  3018. "the parked session's saddle row reads idle")
  3019. time.sleep(3) # dispatch ticks pass: no wake, no ask ever reaches it
  3020. asks_at_answer = len(parked.asks)
  3021. check(len(parked.wake_log) == 0 and asks_at_answer == 1,
  3022. "the parked daemon is never woken and never asks again (the live signature)")
  3023. detail = task_state(auth, parked_task)
  3024. db_block = [b for b in detail["blocks"] if b["question"] == "Which database?"][0]
  3025. act(auth, db_block["answer_action"], {"option": "SQLite"})
  3026. check(wait_for(lambda: len(parked.answer_log) > 0 or False,
  3027. timeout=30, what="verdict delivered") is not None,
  3028. "the verdict is delivered into the parked daemon's answer wait")
  3029. check(parked.answer_log[0]["answer"] == "SQLite",
  3030. "the delivered verdict is the human's choice")
  3031. check(parked.answer_log[0]["branch_advanced"] is False,
  3032. "the delivery carries branch_advanced=false with no siblings")
  3033. check(len(parked.wake_log) == 0 and len(parked.asks) == asks_at_answer,
  3034. "no wake, no ask — the delivery itself resumed the session")
  3035. check(wait_task_done(auth, parked_task, timeout=30) is not None,
  3036. "the parked session continues to completion")
  3037. conn = db()
  3038. row = conn.execute("select outcome from sessions where id = ?",
  3039. (parked_session,)).fetchone()
  3040. session_rows = conn.execute(
  3041. "select count(*) from sessions where task_id = ?", (parked_task,)).fetchone()[0]
  3042. conn.close()
  3043. check(row is not None and row[0] == "done" and session_rows == 1,
  3044. "the delivered verdict resumed the SAME session to done")
  3045. retire_daemon(parked, "resume-parked")
  3046. print("== phase 2: offline requeue keeps parked-with-answered tasks ==")
  3047. # A daemon dying while its parked task carries an answered
  3048. # verdict: going offline must not requeue the task or close its
  3049. # session — the resume machinery (delivery once the daemon
  3050. # returns, or the affinity allocation on its next ask) needs
  3051. # both. The question park leaves the row idle; the review park
  3052. # keeps it busy@task (`saddle review` blocks the round without
  3053. # a parking state push), exercising the unreachable path's
  3054. # requeue decision both ways.
  3055. stranded = answerflow_daemon()
  3056. register_answerflow(stranded, "resume-stranded")
  3057. stranded_task = create_answerflow_task("Stranded answered task")
  3058. ask_status, _ = stranded.ask_now()
  3059. check(ask_status == 200, "the stranded daemon claims its task")
  3060. stranded.state = "blocked"
  3061. stranded.push_results("blocked", block={
  3062. "question": "Hold or fold?", "type": "question", "options": ["Hold", "Fold"]})
  3063. check(wait_for(lambda: task_state(auth, stranded_task).get("status") == "blocked"
  3064. or False, timeout=30, what="stranded task parked") is not None,
  3065. "the stranded task parks as blocked")
  3066. detail = task_state(auth, stranded_task)
  3067. hold_block = [b for b in detail["blocks"] if b["question"] == "Hold or fold?"][0]
  3068. act(auth, hold_block["answer_action"], {"option": "Hold"})
  3069. stranded.kill() # the daemon dies before the verdict is delivered
  3070. act(auth, saddle_card("resume-stranded")["refresh"], {})
  3071. check(wait_for(lambda: saddle_card("resume-stranded")["status"] == "offline"
  3072. or False, timeout=30, what="stranded saddle offline") is not None,
  3073. "the dead daemon's saddle row goes offline")
  3074. time.sleep(7) # dispatch ticks run their offline passes
  3075. check(task_state(auth, stranded_task)["status"] == "blocked",
  3076. "the parked-with-answered task is not requeued by the offline pass")
  3077. conn = db()
  3078. open_session = conn.execute(
  3079. "select count(*) from sessions where task_id = ? and ended_at is null",
  3080. (stranded_task,)).fetchone()[0]
  3081. conn.close()
  3082. check(open_session == 1,
  3083. "the parked-with-answered task keeps its open session for the resume")
  3084. reviewer = answerflow_daemon(ask_on_terminal=False)
  3085. register_answerflow(reviewer, "resume-reviewer")
  3086. review_task = create_answerflow_task("Review park task")
  3087. ask_status, _ = reviewer.ask_now()
  3088. check(ask_status == 200, "the reviewer claims its task")
  3089. reviewer.queue_output("review", "saddle review", "Ready for review: wired and tested.")
  3090. check(wait_for(lambda: task_state(auth, review_task).get("status") == "review_requested"
  3091. or False, timeout=30, what="review park raised") is not None,
  3092. "the review entry parks the task as review_requested (row stays busy)")
  3093. reviewer.kill() # the daemon dies before the verdict can be delivered
  3094. detail = task_state(auth, review_task)
  3095. verdict_block = [b for b in detail["blocks"]
  3096. if b["block_type"] == "review"][0]
  3097. act(auth, verdict_block["answer_action"], {"answer": "Human approved the review"})
  3098. check(wait_for(lambda: saddle_card("resume-reviewer")["status"] == "offline"
  3099. or False, timeout=30, what="reviewer offline") is not None,
  3100. "the failed verdict delivery drives the dead reviewer offline")
  3101. time.sleep(7) # dispatch ticks run their offline passes
  3102. check(task_state(auth, review_task)["status"] == "review_requested",
  3103. "the review-parked task with an answered verdict is never requeued")
  3104. conn = db()
  3105. open_session = conn.execute(
  3106. "select count(*) from sessions where task_id = ? and ended_at is null",
  3107. (review_task,)).fetchone()[0]
  3108. conn.close()
  3109. check(open_session == 1,
  3110. "the review-parked task keeps its open session for the resume")
  3111. print("== phase 2: staleness — one probe decides ==")
  3112. # A silent-but-alive runner (long think, nothing to flush): after
  3113. # SADDLE_STALE_S (3s here) of silence it owes ONE exceptional
  3114. # GET /status; the answer keeps it running and the silence
  3115. # episode gets no second probe.
  3116. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3117. "project_id": "1",
  3118. "title": "Silent thinker",
  3119. "priority": "2",
  3120. })
  3121. thinker = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3122. if t["title"] == "Silent thinker"][0])
  3123. check(wait_for(lambda: task_state(auth, thinker).get("status") == "working"
  3124. or False, timeout=60, what="thinker working") is not None,
  3125. "silent thinker reaches working")
  3126. fake.silent = True # the daemon goes quiet mid-round (alive)
  3127. probes_at_start = fake.status_hits
  3128. check(wait_for(lambda: fake.status_hits > probes_at_start or False,
  3129. timeout=30, what="exceptional probe arrives") is not None,
  3130. "a silent runner gets its exceptional GET /status probe")
  3131. time.sleep(8)
  3132. second_window = fake.status_hits
  3133. time.sleep(6)
  3134. check(fake.status_hits == second_window,
  3135. "the silence episode is probed exactly once (no poll drumbeat)")
  3136. check(task_state(auth, thinker).get("status") == "working"
  3137. and saddle_card("fake-saddle")["status"] == "busy",
  3138. "the answered probe keeps the task running (no requeue)")
  3139. # Output flows again: the episode ends, the clock resets.
  3140. fake.silent = False
  3141. fake.push_chunks(["resumed output\r\n"])
  3142. check(wait_task_done(auth, thinker, timeout=60) is not None,
  3143. "the silent thinker completes once output flows again")
  3144. # The daemon vanishing mid-task: silence for the window, the
  3145. # probe fails, the task requeues with a bot comment and the
  3146. # saddle goes offline.
  3147. act(auth, slot_public(auth, "/tasks", "tasks")["create"], {
  3148. "project_id": "1",
  3149. "title": "Vanish victim",
  3150. "priority": "2",
  3151. })
  3152. vanish = int([t["id"] for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3153. if t["title"] == "Vanish victim"][0])
  3154. check(wait_for(lambda: task_state(auth, vanish).get("status") == "working"
  3155. or False, timeout=60, what="vanish victim working") is not None,
  3156. "vanish victim reaches working")
  3157. fake.kill()
  3158. check(wait_for(lambda: saddle_card("fake-saddle")["status"] == "offline" or False,
  3159. timeout=60, what="daemon death -> offline") is not None,
  3160. "vanishing daemon drives the saddle offline (failed staleness probe)")
  3161. check(wait_for(lambda: any("unreachable" in c["body"]
  3162. for c in task_state(auth, vanish).get("comments", []))
  3163. or False, timeout=30, what="unreachable comment") is not None,
  3164. "the affected task gets an unreachable bot comment")
  3165. check(task_state(auth, vanish)["status"] == "queued",
  3166. "the task requeues after the daemon death")
  3167. card = saddle_card("fake-saddle")
  3168. check(card.get("health") == "offline",
  3169. "health badge reports offline")
  3170. # The unreachable row flag marks the detection window: a task the
  3171. # web still sees working while its saddle is offline/fenced, before
  3172. # the requeue pass moves it. Controlled window (the real
  3173. # transitions above are atomic with their requeues).
  3174. conn = db()
  3175. conn.execute("insert into tasks (project_id, title, status, priority,"
  3176. " created_at) values (1, 'Unreachable row', 'working', 5,"
  3177. " strftime('%s','now'))")
  3178. unreachable_row = conn.execute("select max(id) from tasks").fetchone()[0]
  3179. conn.execute("insert into sessions (task_id, saddle_id, started_at)"
  3180. " values (?, ?, strftime('%s','now'))", (unreachable_row, fake_id))
  3181. conn.commit()
  3182. conn.close()
  3183. row = [t for t in slot_public(auth, "/tasks", "tasks")["tasks"]
  3184. if int(t["id"]) == unreachable_row][0]
  3185. check(row.get("unreachable") is True,
  3186. "queue row shows the unreachable state in the window")
  3187. print("== phase 1: startup migrates the per-saddle token ==")
  3188. # Saddles registered before the flip hold legacy api_tokens rows.
  3189. # On startup every saddle row gets its canonical token: the
  3190. # legacy value it already holds (daemons keep their credential)
  3191. # or a freshly minted one.
  3192. conn = db()
  3193. conn.execute("insert into api_tokens (saddle_id, token, created_at)"
  3194. " values (?, 'saddle-legacy-copied-token', strftime('%s','now'))",
  3195. (int(racer_cards[fake2]["id"]),))
  3196. conn.execute("update saddles set token = null where name = 'racer-two'")
  3197. conn.execute("insert into saddles (name, host, status, current_task_id)"
  3198. " values ('tokenless-orphan', '127.0.0.1:1', 'offline', 0)")
  3199. conn.commit()
  3200. conn.close()
  3201. app.terminate()
  3202. app.wait(timeout=10)
  3203. app = start_app()
  3204. auth = login("admin", "adminpass123")
  3205. def saddle_row(name):
  3206. conn = db()
  3207. try:
  3208. return conn.execute("select id, token from saddles where name = ?",
  3209. (name,)).fetchone()
  3210. finally:
  3211. conn.close()
  3212. check(wait_for(lambda: saddle_row("tokenless-orphan") is not None
  3213. and (saddle_row("tokenless-orphan")[1] or "").startswith("saddle-")
  3214. or False, timeout=20, what="tokenless row minted") is not None,
  3215. "startup mints a token for a saddle row without one")
  3216. check(wait_for(lambda: saddle_row("racer-two") is not None
  3217. and saddle_row("racer-two")[1] == "saddle-legacy-copied-token"
  3218. or False, timeout=20, what="legacy token copied") is not None,
  3219. "startup copies the legacy api_tokens value onto the row")
  3220. fake2.token = "saddle-legacy-copied-token" # the daemon asks with it
  3221. status, _ = saddle_api("POST", "/api/saddle/next-task", "saddle-legacy-copied-token")
  3222. check(status in (200, 204),
  3223. "the legacy token still authenticates (copied, not replaced)")
  3224. conn = db()
  3225. conn.execute("delete from saddles where name = 'tokenless-orphan'")
  3226. conn.commit()
  3227. conn.close()
  3228. # The restart wiped the in-memory ask bookkeeping; the fleet's
  3229. # daemons re-establish sleeping with a fresh ask each (the real
  3230. # daemons' idle-awake ask / 15-minute safety self-ask).
  3231. for daemon in (fake2, fake3):
  3232. daemon.ask_now()
  3233. check(True, "fleet re-asks after the restart (sleeping re-established)")
  3234. finally:
  3235. # Exit-code diagnostic: distinguishes a crashed app (negative
  3236. # returncode, e.g. -11 SIGSEGV) from a clean shutdown. When
  3237. # tearing down mid-exception, also capture the still-alive
  3238. # app's stacks before killing it (a wedged-but-alive server
  3239. # otherwise leaves no evidence).
  3240. print("(app returncode at teardown: %s)" % app.poll())
  3241. if app.poll() is None and sys.exc_info()[0] is not None:
  3242. dump_app_stacks(app.pid)
  3243. app.terminate()
  3244. try:
  3245. app.wait(timeout=5)
  3246. except subprocess.TimeoutExpired:
  3247. app.kill()
  3248. print("%d passed, %d failed" % (passes, failures))
  3249. return 1 if failures else 0
  3250. if __name__ == "__main__":
  3251. sys.exit(main())